diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx
index 0ccb28dbd1..78681d399d 100644
--- a/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx
+++ b/docs/cookbook/autoregressive/Qwen/Qwen3.5.mdx
@@ -777,6 +777,37 @@ Tool Call: None
Arguments: }
```
+#### 4.3.3 Semantic Decisions
+
+Qwen3.5-35B-A3B can answer typed questions about an input with a probability for every option through `/v1/decisions`, without generating text. The server turns thinking off for these requests, so the answer is read after a closed think block, and the same server keeps serving chat traffic. No launch flag is needed. Until a release contains `/v1/decisions`, install a [nightly build](/docs/get-started/install#nightly-builds). With a server started by `sglang serve --model-path Qwen/Qwen3.5-35B-A3B --host 127.0.0.1 --port 30000`:
+
+```python Example
+import requests
+
+response = requests.post(
+ "http://127.0.0.1:30000/v1/decisions",
+ json={
+ "input": "I've been trying to connect my Stripe account for 3 days and the integration keeps failing.",
+ "questions": [
+ {
+ "id": "team",
+ "type": "choice",
+ "question": "Which team should handle this ticket?",
+ "options": [{"name": "billing"}, {"name": "technical"}, {"name": "sales"}],
+ },
+ {"id": "urgent", "type": "yes_no", "question": "The customer needs an answer today."},
+ ],
+ },
+ timeout=60,
+)
+response.raise_for_status()
+answers = response.json()["answers"]
+print(answers["team"]["choice"], answers["team"]["probabilities"])
+print(answers["urgent"]["probabilities"]["yes"])
+```
+
+See [Decision models](/docs/supported-models/decision_models) for the request reference, how the probabilities are computed, and the limits.
+
## 5. Benchmark
### 5.1 Accuracy Benchmark
diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx
index 30cd5eac5e..e99d6986f3 100644
--- a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx
+++ b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx
@@ -541,3 +541,34 @@ providers:
```
+
+## 4. Semantic Decisions
+
+Qwen3.8-27B can answer typed questions about an input with a probability for every option through `/v1/decisions`, without generating text. The server turns thinking off for these requests, so the answer is read after a closed think block, and the same server keeps serving chat traffic. No launch flag is needed. Until a release contains `/v1/decisions`, install a [nightly build](/docs/get-started/install#nightly-builds). With a server started by `sglang serve --model-path Qwen/Qwen3.8-27B --host 127.0.0.1 --port 30000`:
+
+```python Example
+import requests
+
+response = requests.post(
+ "http://127.0.0.1:30000/v1/decisions",
+ json={
+ "input": "I've been trying to connect my Stripe account for 3 days and the integration keeps failing.",
+ "questions": [
+ {
+ "id": "team",
+ "type": "choice",
+ "question": "Which team should handle this ticket?",
+ "options": [{"name": "billing"}, {"name": "technical"}, {"name": "sales"}],
+ },
+ {"id": "urgent", "type": "yes_no", "question": "The customer needs an answer today."},
+ ],
+ },
+ timeout=60,
+)
+response.raise_for_status()
+answers = response.json()["answers"]
+print(answers["team"]["choice"], answers["team"]["probabilities"])
+print(answers["urgent"]["probabilities"]["yes"])
+```
+
+See [Decision models](/docs/supported-models/decision_models) for the request reference, how the probabilities are computed, and the limits.
diff --git a/docs/docs.json b/docs/docs.json
index 0cf88a03b1..fc942d1065 100644
--- a/docs/docs.json
+++ b/docs/docs.json
@@ -998,7 +998,8 @@
{
"group": "Specialized Models",
"pages": [
- "docs/supported-models/reward_models"
+ "docs/supported-models/reward_models",
+ "docs/supported-models/decision_models"
]
},
{
diff --git a/docs/docs/basic_usage/native_api.mdx b/docs/docs/basic_usage/native_api.mdx
index eff2293166..1673efc5b5 100644
--- a/docs/docs/basic_usage/native_api.mdx
+++ b/docs/docs/basic_usage/native_api.mdx
@@ -15,6 +15,7 @@ Apart from the OpenAI compatible APIs, the SGLang Runtime also provides its nati
- `/encode`(embedding model)
- `/v1/rerank`(cross encoder rerank model)
- `/v1/score`(decoder-only scoring)
+- `/v1/decisions`(typed decisions on a chat model)
- `/classify`(reward model)
- `/start_expert_distribution_record`
- `/stop_expert_distribution_record`
@@ -256,6 +257,8 @@ Parameters:
- `item_first`: Whether items come first in concatenation order (default: False)
- `model`: Model name
+`query` and `items` also accept token ids, with `query` set to `[]` when each item is a complete prompt. `label_token_ids` can be one list per item. `temperature` divides the label logits before the softmax and needs `apply_softmax`, and `return_token_logprobs` adds `token_logprobs`, the full-vocabulary log-probabilities of the labels.
+
The response contains `scores` - a list of probability lists, one per item, each in the order of `label_token_ids`.
```python Example
@@ -293,6 +296,20 @@ for item, scores in zip(items, response_json["scores"]):
terminate_process(score_process)
```
+## v1/decisions (typed decisions on a chat model)
+
+Answer typed questions about an input with per-option probabilities, without generating. The server renders each question with the model's chat template and thinking turned off, assigns one-token answer labels, and scores them through the same path as `v1/score`.
+
+Parameters:
+- `input`: Text, object, or array the questions are about
+- `questions`: List of questions, each with a unique `id` and a `type` of `choice` (with `options`), `score` (with `levels`), or `yes_no`
+- `temperature`: Divides the label logits before the softmax (default: 1)
+- `chat_template_kwargs`: Extra chat template arguments
+- `prompt_format_version`: Optional pin of the server-owned prompt wording
+- `return_prompt_token_ids`: Return the scored prompt and label token ids for replay through `v1/score` (default: False)
+
+The response contains `answers` keyed by question id, each with `type`, `probabilities`, and `label_mass`, plus `choice` for a choice question or `score` for a score question. A yes or no answer is read from `probabilities["yes"]`. See [Decision models](/docs/supported-models/decision_models) for examples, the full reference, and error cases.
+
## Classify (reward model)
SGLang Runtime also supports reward models. Here we use a reward model to classify the quality of pairwise generations.
diff --git a/docs/docs/supported-models.mdx b/docs/docs/supported-models.mdx
index bcfcf89a75..33ae19614b 100644
--- a/docs/docs/supported-models.mdx
+++ b/docs/docs/supported-models.mdx
@@ -84,4 +84,13 @@ SGLang supports model families across text generation, retrieval, and reward wor
>
RLHF and reward scoring pipelines optimized for production latency.
+
+ Typed choice, score, and yes or no answers with a probability for every option.
+
diff --git a/docs/docs/supported-models/decision_models.mdx b/docs/docs/supported-models/decision_models.mdx
new file mode 100644
index 0000000000..60200987c1
--- /dev/null
+++ b/docs/docs/supported-models/decision_models.mdx
@@ -0,0 +1,274 @@
+---
+title: Decision models
+description: Answer typed choice, score, and yes or no questions with a probability for every option from a chat model, without generating text.
+---
+
+`/v1/decisions` turns a chat model into a decision model. You send an input and a list of typed questions, and each answer comes back with the probability of every option, read from the model's next-token scores at the answer position. No text is generated and no output is parsed. It needs no special checkpoint: any generation model served with a Jinja chat template can answer, as long as the checks below pass for it.
+
+
+ `/v1/decisions` is an SGLang extension under `/v1`, like `/v1/score` and `/v1/rerank`. It is not part of the OpenAI API, so call it over HTTP rather than through an OpenAI SDK method. Until a release contains it, install a [nightly build](/docs/get-started/install#nightly-builds), which is built from the main branch.
+
+
+## Supported models
+
+| Model | Example HuggingFace identifier | Notes |
+| --- | --- | --- |
+| **Qwen3.8-27B** | `Qwen/Qwen3.8-27B` | Validated on one H200 in BF16, with and without NEXTN speculative decoding. See the [model page](/cookbook/autoregressive/Qwen/Qwen3.8-27B). |
+| **Qwen3.5-35B-A3B** | `Qwen/Qwen3.5-35B-A3B` | Validated on one H200 in BF16. See the [model page](/cookbook/autoregressive/Qwen/Qwen3.5). |
+
+Other chat models are served when the answer labels are single tokens at the answer position and the answer does not start inside a reasoning block. The server checks these per request and returns a 400 that names the reason when one fails, see [Errors](#errors). It does not check how the chat template renders the question or where the template itself would put an answer, so inspect `prompt_token_ids` from `return_prompt_token_ids` before relying on a new model.
+
+## Launch command
+
+```bash Command
+python -m sglang.launch_server \
+ --model-path Qwen/Qwen3.8-27B \
+ --host 127.0.0.1 --port 30000
+```
+
+No extra flag is needed. The same server keeps serving `/v1/chat/completions` and `/generate`, and decisions can run alongside that traffic. A server launched with `--enable-mis` or `--dllm-algorithm`, or one that uses a built-in conversation template instead of the tokenizer's Jinja template, refuses decisions.
+
+## Example request
+
+```python Example
+import requests
+
+URL = "http://127.0.0.1:30000"
+
+response = requests.post(
+ f"{URL}/v1/decisions",
+ json={
+ "input": "I've been trying to connect my Stripe account for 3 days and the integration keeps failing. I'm losing sales.",
+ "questions": [
+ {
+ "id": "team",
+ "type": "choice",
+ "question": "Which team should handle this ticket?",
+ "options": [
+ {"name": "billing", "description": "Payment or subscription issues"},
+ {"name": "technical", "description": "Bugs or integration problems"},
+ {"name": "sales", "description": "Pricing or account questions"},
+ ],
+ },
+ {
+ "id": "frustration",
+ "type": "score",
+ "question": "How frustrated is the customer?",
+ "levels": ["Calm", "Frustrated but civil", "Very angry"],
+ },
+ {
+ "id": "urgent",
+ "type": "yes_no",
+ "question": "The customer needs an answer today.",
+ },
+ ],
+ },
+ timeout=60,
+)
+response.raise_for_status()
+for question_id, answer in response.json()["answers"].items():
+ print(question_id, answer)
+```
+
+Each answer contains:
+
+- `type`: the question type, `choice`, `score`, or `yes_no`.
+- `probabilities`: one value per option name, level index, or `yes` and `no`, summing to 1. For a yes or no question, `probabilities["yes"]` is the answer.
+- `choice` for a choice question (the most probable option) or `score` for a score question (the probability-weighted mean level index).
+- `label_mass`: the full-vocabulary probability of the answer labels at the answer position. A low value means the model puts most of its probability outside the offered answers.
+
+Questions are answered independently and scored together in one batch. To route the ticket in this example, send it to the chosen team, and hand it to a person when the top probability falls below a threshold you validate on your own labeled tickets.
+
+## How answers are computed
+
+The server renders each question as one user message with the model's chat template and thinking turned off, and labels the answers `A` to `Z` for options, `0` to `9` for levels, and `yes` and `no`. It checks that each label is one distinct token at the answer position, then runs one prefill pass per question through the scoring path of `/v1/score` and reads the next-token log-probabilities of the labels over the full vocabulary. For a yes or no question with log-probabilities `lp_yes` and `lp_no` and request temperature `T`:
+
+- `probabilities["yes"] = exp(lp_yes / T) / (exp(lp_yes / T) + exp(lp_no / T))`, which equals a softmax of the two label logits divided by `T`, because the vocabulary normalizer cancels.
+- `label_mass = exp(lp_yes) + exp(lp_no)`, which does not depend on `T`.
+
+Choice and score questions work the same way over their labels. None of these values is a calibrated probability that the decision is correct. Validate any threshold on labeled data from your workload.
+
+## Request and response reference
+
+Request fields:
+
+| Field | Type | Meaning |
+| --- | --- | --- |
+| `input` | string, object, or array | What the questions are about. Objects and arrays are rendered as compact JSON. Must not be blank. |
+| `questions` | list of questions | At least one. Each question has a unique `id`, which keys its answer. |
+| `temperature` | number above 0, default 1 | Divides the label logits before the softmax over the labels. It does not change `label_mass`. |
+| `chat_template_kwargs` | object, default empty | Extra chat template arguments, applied after `--default-chat-template-kwargs`. The thinking toggle that the chat template or its reasoning parser names stays off in either case. |
+| `prompt_format_version` | integer, optional | Pins the prompt wording. A server that serves another version returns 400. |
+| `return_prompt_token_ids` | boolean, default false | Adds `prompt_token_ids` and `label_token_ids` to each answer. |
+| `model` | string, optional | Echoed in the response, as `default` when omitted. It does not select a LoRA adapter, and the `base:adapter` form returns 400. |
+
+Question fields:
+
+| Type | Fields | Labels |
+| --- | --- | --- |
+| `choice` | `id`, `question`, and `options`: 2 to 26 objects with a `name` and an optional `description` | `A` to `Z` in list order |
+| `score` | `id`, `question`, and `levels`: 2 to 10 level descriptions, lowest first | `0` to `9` |
+| `yes_no` | `id`, `question`, and optional `yes` and `no` descriptions | `yes` and `no` |
+
+`question`, descriptions, and levels can also be JSON objects or arrays. Each question becomes one user message: the input, a blank line, the question line, one line per option, level, or described yes or no answer, and a closing instruction to answer with one label only. An option renders as `A: billing` or `A: billing - `, a level as `0: `, and a described yes or no answer as `yes: `.
+
+The response has `object` set to `decisions`, `model`, `prompt_format_version`, `answers` keyed by question id, and `usage` with `prompt_tokens` for all questions and `completion_tokens` 0.
+
+## Errors
+
+The request fails with HTTP 400 for:
+
+- an unknown question type or an unknown field anywhere in the request
+- option or level counts outside the ranges above
+- a blank input, question, question id, or level
+- a repeated question id, or option names that are blank, contain control or line break characters, or repeat another name after trimming and case folding
+- a label that is not one distinct token at the answer position for the served tokenizer
+- a prompt that does not fit the model's context length
+- a chat template that always thinks before answering or starts every answer with a reasoning block, `chat_template_kwargs` that set the thinking toggle to anything but `false`, a rendered prompt that leaves a reasoning block open, or a model whose reasoning parser expects answers to start inside a reasoning block when the rendered prompt does not close one
+- a `prompt_format_version` other than the served one, or a `model` that names a LoRA adapter
+- a server launched with `--enable-mis` or `--dllm-algorithm`, or one that uses a built-in conversation template, whether named with `--chat-template`, loaded from a JSON template file, or inferred from the model path
+- a model whose chat route uses a built-in encoder instead of a chat template, a tokenizer that does not encode the rendered chat text back to the same ids, or a model that is not a generation model
+- a chat template that raises an error for the question message, or a server launched with `--skip-tokenizer-init`
+
+Errors about one question name its id or its position in `questions`.
+
+## Pin and replay the prompt
+
+The server owns the prompt wording of `/v1/decisions` and versions it. Every response carries `prompt_format_version`, and a change to the wording ships as a new version. Send `prompt_format_version` to fail loudly instead of receiving answers from different wording after a server upgrade.
+
+To keep an answer reproducible independently of the server's wording, ask for the scored ids and replay them through `/v1/score`, which only scores the ids you send:
+
+```python Example
+import requests
+
+URL = "http://127.0.0.1:30000"
+
+body = {
+ "input": "The integration keeps failing.",
+ "questions": [{"id": "urgent", "type": "yes_no", "question": "The customer needs an answer today."}],
+ "prompt_format_version": 1,
+ "return_prompt_token_ids": True,
+}
+answer = requests.post(f"{URL}/v1/decisions", json=body, timeout=60).json()["answers"]["urgent"]
+replay = requests.post(
+ f"{URL}/v1/score",
+ json={
+ "query": [],
+ "items": [answer["prompt_token_ids"]],
+ "label_token_ids": [answer["label_token_ids"]],
+ "apply_softmax": True,
+ "return_token_logprobs": True,
+ },
+ timeout=60,
+).json()
+print(answer["probabilities"], replay["scores"][0])
+```
+
+The replayed scores equal the answer's probabilities when both requests use the same `temperature` and run in the same cache state. If the decision set `temperature`, send the same value in the `/v1/score` body. A replay right after the decision reuses its cached prefix, so it can differ slightly, as described in [Reproducibility](#reproducibility).
+
+## Thinking models
+
+Qwen3.8-27B and Qwen3.5-35B-A3B think by default. `/v1/decisions` turns thinking off for every question, refuses a request that turns it back on, and refuses a rendered prompt that leaves a reasoning block open, so the answer is never read inside the reasoning. Chat requests to the same server keep the model's default, and launching with `--reasoning-parser qwen3` affects chat only.
+
+## Reproducibility
+
+These models mix full and linear attention. Answer probabilities can move by up to several hundredths (0.07 in our checks), and `label_mass` by up to about 0.14, between cold and prefix-cached requests and across batch compositions, while the chosen option stayed the same in our checks. On Qwen3.5-35B-A3B, `--disable-radix-cache` gave identical values across sequential repeats, at the cost of prefix reuse.
+
+## System One compatible API
+
+`POST /v1/systemone` serves the same decisions in the request and response shape of the System One API, with a `state`, a map of `noul`, `choice`, and `score` questions keyed by your ids, and one answer per id. Clients written for that API, including the official TypeSafe SDKs, get decisions by pointing their base URL at the server, with the exceptions listed below. The route uses the same rendering, label checks, and scoring as `/v1/decisions`, and refuses the same servers, chat templates, and prompts, listed in [Errors](#errors). Its request fields are checked as described below.
+
+```bash Command
+pip install typesafe-sdk
+```
+
+```python Example
+from typesafe_sdk import Choice, Noul, Score, TypeSafeClient
+
+# A server started without --api-key accepts any key, so a placeholder works.
+client = TypeSafeClient(base_url="http://127.0.0.1:30000", api_key="unused")
+result = client.system_one(
+ "I've been trying to connect my Stripe account for 3 days and the integration keeps failing.",
+ {
+ "team": Choice(
+ instructions="Which team should handle this ticket?",
+ criteria={"billing": None, "technical": "Bugs or integration problems", "sales": None},
+ ),
+ "urgent": Noul(instructions="The customer needs an answer today."),
+ "frustration": Score(
+ instructions="How frustrated is the customer?",
+ criteria=["Calm", "Frustrated but civil", "Very angry"],
+ ),
+ },
+)
+for name, answer in result.answers.items():
+ print(name, answer)
+```
+
+What to expect from this route:
+
+- Any `model` name is accepted, including the SDK default `jev-latest`, unless it names a LoRA adapter after a colon, as in `base:adapter`. The response `model` is always the served model.
+- A `noul` answer is the probability of yes. A `choice` answer has the most probable option and a `score` answer the probability-weighted mean level, each with `probabilities` and `confidence`, and a score answer echoes its levels in `legend`.
+- `confidence` follows the formulas TypeSafe publishes for its adapter. It is a statistic of the returned probabilities, which are renormalized over the answer labels and ignore probability outside them. It is not calibrated, it can read 0 for a score split between distant levels, and thresholds tuned on another provider may not transfer.
+- Each answer also carries `x_label_mass`, the full-vocabulary probability of its labels, which `/v1/decisions` returns as `label_mass`. The Python SDK drops fields it does not define, so read it from `result.raw_http_response.json()`.
+- A choice takes 1 to 255 options and a score 1 to 10 levels. Option names that are blank, contain control or line break characters, or repeat another name after trimming and case folding are refused, and so are blank or null levels, as on `/v1/decisions`.
+- Beyond 26 options, every option gets a two-letter label (`AA`, `AB`, and so on) checked to be one token for the served tokenizer. These labels include common words and have unequal priors, so answers above 26 options can depend on option order. More than 26 options also need a Hugging Face tokenizer that reports its added tokens, and a chat template with an added token between the message and the answer position whose following text tokenizes the same on its own, as the Qwen models do with ``. Without them, a choice is limited to 26 options.
+- `instructions` are optional. A choice or score question without them drops its question line, and a `noul` question is answered from its `true` and `false` descriptions. A `noul` question with neither, such as the SDK's bare `Noul()`, is refused.
+- Unknown top-level fields are ignored. Unknown keys inside a question or inside `noul` criteria are refused, and so are the `/v1/decisions` fields `temperature`, `prompt_format_version`, and `return_prompt_token_ids` with any value other than null. `chat_template_kwargs` is accepted as an SGLang extension.
+- Unlike `/v1/decisions`, an empty `state` and empty or blank question ids are accepted, as the System One schema allows.
+- Invalid requests return 422 with the location of the error, other refusals return 400, and neither is retried by the SDKs.
+- `usage.input_tokens` counts the prompt of every question, so the state is counted once per question, and `usage.output_tokens` is 0.
+- The SDKs' `models.list()` and response `request_id` are not supported, because `GET /v1/models` keeps its OpenAI format and no request id header is sent.
+- The SDKs time out after 10 seconds and retry, so raise their timeout for requests with many questions over a long state. A request whose client gives up still finishes its encoding and part of its prefill, so each retry repeats at least the encoding.
+
+## Build decisions without `/v1/decisions`
+
+When you need your own prompt or labels, build the prompt on the client and score it through `/v1/score`, which takes token ids and one label list per item. This needs `transformers` and `jinja2` on the client. Every label must be exactly one token at the answer position:
+
+```python Example
+import requests
+from transformers import AutoTokenizer
+
+MODEL = "Qwen/Qwen3.8-27B"
+URL = "http://127.0.0.1:30000"
+tokenizer = AutoTokenizer.from_pretrained(MODEL)
+
+options = ["billing", "technical", "sales"]
+letters = ["A", "B", "C"]
+prompt = (
+ "I've been trying to connect my Stripe account for 3 days and the integration keeps failing.\n\n"
+ "Question: Which team should handle this ticket?\n"
+ + "\n".join(f"{letter}: {name}" for letter, name in zip(letters, options))
+ + "\nAnswer with the letter of one option only."
+)
+text = tokenizer.apply_chat_template(
+ [{"role": "user", "content": prompt}], add_generation_prompt=True, enable_thinking=False, tokenize=False
+)
+input_ids = tokenizer.encode(text, add_special_tokens=False)
+label_ids = []
+for letter in letters:
+ ids = tokenizer.encode(text + letter, add_special_tokens=False)
+ if ids[: len(input_ids)] != input_ids or len(ids) != len(input_ids) + 1:
+ raise ValueError(f"Label {letter} is not one token at the answer position")
+ label_ids.append(ids[-1])
+
+scores = requests.post(
+ f"{URL}/v1/score",
+ json={"query": [], "items": [input_ids], "label_token_ids": [label_ids], "apply_softmax": True},
+ timeout=60,
+).json()["scores"][0]
+print(dict(zip(options, scores)))
+```
+
+`enable_thinking=False` renders a closed think block, so the answer token follows it directly. Do not score a prompt that leaves the think block open, because the scored position then falls inside the reasoning.
+
+Structured output gives the same decision through generation. A one-token regex such as `(A|B|C)` with `temperature` 0 and logprobs returns probabilities renormalized over the labels, equal to the `/v1/score` values for one-character labels in the same cache state. A JSON schema with `enum` fields fills several fields in one response, but gives no clean per-option probability, because an enum value can span several tokens. See [Structured outputs](/docs/advanced_features/structured_outputs) for the request formats.
+
+## Limitations
+
+- Labels must be one token at the answer position. `/v1/decisions` assigns one-token labels for you, and a client-built prompt must do the same.
+- For `yes_no`, `label_mass` counts only the lowercase `yes` and `no` tokens. The model also puts probability on `Yes` and `No`, so this value reads lower than for choice and score questions even on clear cases, while `probabilities["yes"]` is unaffected.
+- The server tells from the chat template and its reasoning parser whether answers could start inside a reasoning block, using the parser's reasoning tags in the generation prompt and in the template's own rendering of a finished answer. Reasoning that shows neither is not detected, so check `prompt_token_ids` for such a model. Some templates of models that do not reason still contain reasoning tags without ever closing a block in the generation prompt, and those models are refused.
+- A default temperature in `--preferred-sampling-params` can reach scoring requests and scale both the probabilities and `label_mass` when a decision shares a batch with generation or runs under speculative decoding. Leave it unset for decisions.
+- The server tokenizes each question's full prompt once on the HTTP event loop, as `/v1/score` does for its items, and lets other requests run between questions. Tokenizers whose label check falls back to the full prompt encode it once more per label.
+- A decision takes a single `input`, not a chat history. To decide about a conversation, pass the history as the input text or as a JSON array.
diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py
index 46a6855e9f..3258cfb9e9 100644
--- a/python/sglang/srt/entrypoints/http_server.py
+++ b/python/sglang/srt/entrypoints/http_server.py
@@ -58,6 +58,7 @@ from fastapi import (
UploadFile,
WebSocket,
)
+from fastapi.encoders import jsonable_encoder
from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import ORJSONResponse, Response, StreamingResponse
@@ -88,6 +89,7 @@ from sglang.srt.entrypoints.openai.protocol import (
ChatCompletionRequest,
ClassifyRequest,
CompletionRequest,
+ DecisionRequest,
DetokenizeRequest,
EmbeddingRequest,
ErrorResponse,
@@ -100,6 +102,7 @@ from sglang.srt.entrypoints.openai.protocol import (
)
from sglang.srt.entrypoints.openai.serving_classify import OpenAIServingClassify
from sglang.srt.entrypoints.openai.serving_completions import OpenAIServingCompletion
+from sglang.srt.entrypoints.openai.serving_decisions import OpenAIServingDecisions
from sglang.srt.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
from sglang.srt.entrypoints.openai.serving_rerank import OpenAIServingRerank
from sglang.srt.entrypoints.openai.serving_score import OpenAIServingScore
@@ -111,6 +114,8 @@ from sglang.srt.entrypoints.openai.serving_transcription import (
OpenAIServingTranscription,
)
from sglang.srt.entrypoints.request_headers import apply_header_overrides
+from sglang.srt.entrypoints.systemone.protocol import SystemOneRequest
+from sglang.srt.entrypoints.systemone.serving import SystemOneServing
from sglang.srt.entrypoints.warmup import execute_warmups
from sglang.srt.environ import envs
from sglang.srt.function_call.function_call_parser import FunctionCallParser
@@ -323,6 +328,9 @@ async def lifespan(fast_api_app: FastAPI):
fast_api_app.state.openai_serving_score = OpenAIServingScore(
_global_state.tokenizer_manager
)
+ fast_api_app.state.openai_serving_decisions = OpenAIServingDecisions(
+ fast_api_app.state.openai_serving_chat
+ )
fast_api_app.state.openai_serving_rerank = OpenAIServingRerank(
_global_state.tokenizer_manager, _global_state.template_manager
)
@@ -344,6 +352,11 @@ async def lifespan(fast_api_app: FastAPI):
fast_api_app.state.openai_serving_chat
)
+ # Initialize System One compatible decision handler
+ fast_api_app.state.systemone_serving = SystemOneServing(
+ fast_api_app.state.openai_serving_chat
+ )
+
# Launch tool server
tool_server = None
if get_serving().tool_server == "demo":
@@ -603,8 +616,9 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
For /v1/messages, emit Anthropic-style envelope and scrub the message so
file paths or Python internals from the default ``str(exc)`` representation
- never reach the client. For /v1/responses, keep OpenAI-style. Otherwise
- use the legacy ErrorResponse shape.
+ never reach the client. For /v1/responses, keep OpenAI-style. For
+ /v1/systemone, keep the 422 detail list that the System One API documents.
+ Otherwise use the legacy ErrorResponse shape.
"""
if request.url.path.startswith("/v1/messages"):
return _anthropic_error_response(
@@ -613,6 +627,19 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE
message=_anthropic_validation_message(exc.errors()),
)
+ route_path = request.url.path.removeprefix(request.scope.get("root_path", ""))
+ if route_path == "/v1/systemone":
+ # The System One API documents FastAPI's default 422 detail list. The
+ # optional input echo is left out, since it can be any client value.
+ detail = [
+ {key: value for key, value in error.items() if key != "input"}
+ for error in exc.errors()
+ ]
+ return ORJSONResponse(
+ status_code=HTTPStatus.UNPROCESSABLE_ENTITY.value,
+ content={"detail": jsonable_encoder(detail)},
+ )
+
exc_str = str(exc)
errors_str = str(exc.errors())
@@ -1933,6 +1960,14 @@ async def v1_score_request(request: ScoringRequest, raw_request: Request):
)
+@app.post("/v1/decisions", dependencies=[Depends(validate_json_request)])
+async def v1_decisions_request(request: DecisionRequest, raw_request: Request):
+ """Answer typed choice, score, and yes or no questions about an input by scoring single-token answer labels through the scoring API, without generation."""
+ return await raw_request.app.state.openai_serving_decisions.handle_request(
+ request, raw_request
+ )
+
+
@app.post("/v1/responses", dependencies=[Depends(validate_json_request)])
async def v1_responses_request(request: ResponsesRequest, raw_request: Request):
"""Endpoint for the responses API with reasoning support."""
@@ -2047,6 +2082,15 @@ async def anthropic_v1_count_tokens(
)
+## System One compatible decision API
+@app.post("/v1/systemone", dependencies=[Depends(validate_json_request)])
+async def systemone_decisions(request: SystemOneRequest, raw_request: Request):
+ """System One compatible decisions, answered by candidate scoring without generation."""
+ return await raw_request.app.state.systemone_serving.handle_request(
+ request, raw_request
+ )
+
+
## SageMaker API
@app.get("/ping")
async def sagemaker_health() -> Response:
diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py
index 6215400b07..5e503c0b29 100644
--- a/python/sglang/srt/entrypoints/openai/protocol.py
+++ b/python/sglang/srt/entrypoints/openai/protocol.py
@@ -17,6 +17,7 @@ from __future__ import annotations
import logging
import time
+import unicodedata
import uuid
from dataclasses import dataclass
from typing import (
@@ -1438,6 +1439,137 @@ class ScoringResponse(BaseModel):
object: str = "scoring"
+def is_blank_decision_text(value) -> bool:
+ return not (value.strip() if isinstance(value, str) else value)
+
+
+def _nonblank_decision_text(value):
+ if is_blank_decision_text(value):
+ raise ValueError("must not be blank")
+ return value
+
+
+def check_option_names(names) -> None:
+ """Refuse option names that would make the rendered option lines ambiguous."""
+ seen = set()
+ for name in names:
+ key = name.strip().casefold()
+ if not key:
+ raise ValueError("option names must be nonempty")
+ # Each option is rendered as one prompt line.
+ if any(unicodedata.category(c) in ("Cc", "Zl", "Zp") for c in name):
+ raise ValueError(
+ f"option name {name!r} must not contain control or line break "
+ "characters"
+ )
+ if key in seen:
+ raise ValueError(f"option name {name!r} repeats another option")
+ seen.add(key)
+
+
+# Objects and arrays are rendered into the prompt as compact JSON.
+DecisionText = Union[str, Dict[str, Any], List[Any]]
+RequiredDecisionText = Annotated[DecisionText, AfterValidator(_nonblank_decision_text)]
+
+
+class DecisionOption(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ name: str
+ description: Optional[DecisionText] = None
+
+
+class DecisionChoiceQuestion(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ id: Annotated[str, AfterValidator(_nonblank_decision_text)]
+ type: Literal["choice"]
+ question: RequiredDecisionText
+ # Options are labeled A to Z in order, so at most 26.
+ options: List[DecisionOption] = Field(min_length=2, max_length=26)
+
+ @model_validator(mode="after")
+ def _option_names_distinct(self):
+ try:
+ check_option_names(option.name for option in self.options)
+ except ValueError as e:
+ raise ValueError(f"question {self.id!r}: {e}") from None
+ return self
+
+
+class DecisionScoreQuestion(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ id: Annotated[str, AfterValidator(_nonblank_decision_text)]
+ type: Literal["score"]
+ question: RequiredDecisionText
+ # Levels are labeled 0 to 9 in order, so at most 10.
+ levels: List[RequiredDecisionText] = Field(min_length=2, max_length=10)
+
+
+class DecisionYesNoQuestion(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ id: Annotated[str, AfterValidator(_nonblank_decision_text)]
+ type: Literal["yes_no"]
+ question: RequiredDecisionText
+ yes: Optional[DecisionText] = None
+ no: Optional[DecisionText] = None
+
+
+DecisionQuestion = Annotated[
+ Union[DecisionChoiceQuestion, DecisionScoreQuestion, DecisionYesNoQuestion],
+ Field(discriminator="type"),
+]
+
+
+class DecisionRequest(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ input: RequiredDecisionText
+ questions: List[DecisionQuestion] = Field(min_length=1)
+ # Scales option probabilities only, not label_mass.
+ temperature: float = Field(default=1.0, gt=0, allow_inf_nan=False)
+ # Applied over the server defaults, with the template reasoning toggle off.
+ chat_template_kwargs: Dict[str, Any] = Field(default_factory=dict)
+ # Pins the server-owned prompt wording. A different served version is refused.
+ prompt_format_version: Optional[int] = None
+ return_prompt_token_ids: bool = False
+ model: str = DEFAULT_MODEL_NAME
+
+ @field_validator("questions")
+ @classmethod
+ def _question_ids_distinct(cls, questions):
+ seen = set()
+ for question in questions:
+ if question.id in seen:
+ raise ValueError(
+ f"question id {question.id!r} repeats another question"
+ )
+ seen.add(question.id)
+ return questions
+
+
+class DecisionAnswer(BaseModel):
+ type: Literal["choice", "score", "yes_no"]
+ probabilities: Dict[str, float]
+ # Full-vocabulary probability of all answer labels at the answer position.
+ label_mass: float
+ choice: Optional[str] = None
+ score: Optional[float] = None
+ # The exact /v1/score inputs, with return_prompt_token_ids.
+ prompt_token_ids: Optional[List[int]] = None
+ label_token_ids: Optional[List[int]] = None
+
+
+class DecisionResponse(BaseModel):
+ object: str = "decisions"
+ model: str
+ prompt_format_version: int
+ answers: Dict[str, DecisionAnswer]
+ usage: UsageInfo
+
+
class V1RerankReqInput(BaseModel):
query: RerankContent = Field(
...,
@@ -1563,6 +1695,7 @@ OpenAIServingRequest = Union[
EmbeddingRequest,
ClassifyRequest,
ScoringRequest,
+ DecisionRequest,
V1RerankReqInput,
TokenizeRequest,
DetokenizeRequest,
diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py
index 92842acff2..38fd20470e 100644
--- a/python/sglang/srt/entrypoints/openai/serving_chat.py
+++ b/python/sglang/srt/entrypoints/openai/serving_chat.py
@@ -2445,7 +2445,7 @@ class OpenAIServingChat(OpenAIServingBase):
output_token_logprobs = ret_item["meta_info"]["output_token_logprobs"]
output_top_logprobs = ret_item["meta_info"].get("output_top_logprobs", None)
token_logprobs = self._build_token_logprobs_from_raw(
- output_token_logprobs, output_top_logprobs, use_token_index=True
+ output_token_logprobs, output_top_logprobs
)
return ChoiceLogprobs(content=token_logprobs)
@@ -2453,7 +2453,6 @@ class OpenAIServingChat(OpenAIServingBase):
self,
output_token_logprobs: list[Any],
output_top_logprobs: list[Any] | None,
- use_token_index: bool = False,
) -> list[ChatCompletionTokenLogprob]:
"""Build OpenAI ChatCompletionTokenLogprob from the engine's raw
``(logprob, token_id, token_text)`` triples.
@@ -2487,13 +2486,9 @@ class OpenAIServingChat(OpenAIServingBase):
top_logprobs: list[TopLogprob] = []
if output_top_logprobs:
- # - Non-streaming (use_token_index=True): output_top_logprobs is
- # the full per-position list; take the row for this token.
- # - Streaming (use_token_index=False): rows are pre-sliced so the
- # current chunk holds exactly one row at index 0.
- top_row_idx = token_idx if use_token_index else 0
- if top_row_idx < len(output_top_logprobs):
- top_row = output_top_logprobs[top_row_idx]
+ # Both callers pass rows aligned with output_token_logprobs.
+ if token_idx < len(output_top_logprobs):
+ top_row = output_top_logprobs[token_idx]
if top_row is not None:
for top_logprob, top_id, top_text in top_row:
if is_byte_level:
@@ -2686,7 +2681,7 @@ class OpenAIServingChat(OpenAIServingBase):
n_prev_token:total_output_logprobs
]
token_logprobs = self._build_token_logprobs_from_raw(
- output_token_logprobs, output_top_logprobs, use_token_index=False
+ output_token_logprobs, output_top_logprobs
)
return ChoiceLogprobs(content=token_logprobs)
diff --git a/python/sglang/srt/entrypoints/openai/serving_decisions.py b/python/sglang/srt/entrypoints/openai/serving_decisions.py
new file mode 100644
index 0000000000..b6329b48fc
--- /dev/null
+++ b/python/sglang/srt/entrypoints/openai/serving_decisions.py
@@ -0,0 +1,535 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import math
+import string
+from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple
+
+import msgspec
+from fastapi import Request
+from fastapi.responses import ORJSONResponse
+from transformers import PreTrainedTokenizerBase
+
+from sglang.srt.entrypoints.openai.protocol import (
+ DecisionAnswer,
+ DecisionChoiceQuestion,
+ DecisionQuestion,
+ DecisionRequest,
+ DecisionResponse,
+ DecisionScoreQuestion,
+ DecisionText,
+ UsageInfo,
+ is_blank_decision_text,
+)
+from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase
+from sglang.srt.entrypoints.openai.serving_chat import _CHAT_TEMPLATE_CLIENT_ERRORS
+from sglang.srt.parser.reasoning_parser import ReasoningParser
+from sglang.srt.runtime_context import get_exec
+
+if TYPE_CHECKING:
+ from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
+
+logger = logging.getLogger(__name__)
+
+# Version of the server-owned prompt wording and answer labels.
+# Any change to a rendering /v1/decisions can produce needs a new version.
+PROMPT_FORMAT_VERSION = 1
+
+# Parser defaults that name the chat template kwarg toggling reasoning.
+_PARSER_TOGGLE_MODES = (
+ "thinking",
+ "enable_thinking",
+ "explicit_thinking",
+ "explicit_enable_thinking",
+)
+# Answer text of a finished reply, rendered only to see what precedes an answer.
+_REPLY_SENTINEL = "DECISION_ANSWER"
+
+
+class QuestionView(msgspec.Struct, frozen=True):
+ """A question as the renderer and scorer see it, shared by the decision routes."""
+
+ # choice, score, or yes_no
+ kind: str
+ # None or blank when the question has no text of its own
+ question: Any
+ # Option names, level indices, or yes and no, in candidate order
+ names: List[str]
+ # Option descriptions, levels, or the yes and no descriptions
+ details: List[Any]
+
+
+class OpenAIServingDecisions(OpenAIServingBase):
+ """Handler for /v1/decisions requests, answered by candidate scoring without generation"""
+
+ # Named in setup refusals, so each decision route reports itself.
+ route = "/v1/decisions"
+
+ def __init__(self, chat_serving: OpenAIServingChat):
+ super().__init__(chat_serving.tokenizer_manager)
+ # Render the way the chat route does, and refuse where it renders differently.
+ self.template_manager = chat_serving.template_manager
+ self.default_chat_template_kwargs = chat_serving.default_chat_template_kwargs
+ self.chat_encoding_spec = chat_serving.chat_encoding_spec
+ self.prompt_text_is_lossy = chat_serving._prompt_text_round_trip_is_lossy
+ tokenizer = self.tokenizer_manager.tokenizer
+ # Other tokenizers skip the shortcut in _encode_labels and check the full prompt.
+ self.added_tokens = (
+ {i: token for token, i in tokenizer.get_added_vocab().items()}
+ if isinstance(tokenizer, PreTrainedTokenizerBase)
+ else {}
+ )
+ # The configured reasoning parser, or the one the chat template suggests,
+ # tells where reasoning blocks start and end and whether answers open one.
+ parser = (
+ chat_serving.reasoning_parser
+ or self.template_manager.suggested_reasoning_parser
+ )
+ self.reasoning_markers = None
+ self.answers_open_reasoning = False
+ # The kwarg that turns reasoning on and off: the template's, else, when
+ # detection finds no config, the one the configured or suggested parser names.
+ config = self.template_manager.reasoning_config
+ self.reasoning_toggle = config.toggle_param if config is not None else None
+ if parser is not None:
+ try:
+ detector = ReasoningParser(
+ model_type=parser, tokenizer=tokenizer
+ ).detector
+ except ValueError as e:
+ logger.warning(
+ "No reasoning block check for %s with parser '%s': %s",
+ self.route,
+ parser,
+ e,
+ )
+ else:
+ mode = detector.reasoning_default
+ if config is None and mode in _PARSER_TOGGLE_MODES:
+ self.reasoning_toggle = mode.removeprefix("explicit_")
+ if detector.think_start_token and detector.think_end_token:
+ self.reasoning_markers = (
+ detector.think_start_token,
+ detector.think_end_token,
+ )
+ self.answers_open_reasoning = detector.reasoning_default == "always"
+
+ def _request_id_prefix(self) -> str:
+ return "decision-"
+
+ def _validate_request(self, request: DecisionRequest) -> Optional[str]:
+ error = self._validate_server(request.model)
+ if error is not None:
+ return error
+ version = request.prompt_format_version
+ if version is not None and version != PROMPT_FORMAT_VERSION:
+ return (
+ f"prompt_format_version {version} is not served, this server "
+ f"uses version {PROMPT_FORMAT_VERSION}"
+ )
+ return self._validate_reasoning(request.chat_template_kwargs)
+
+ def _validate_server(self, model: str) -> Optional[str]:
+ """Refuse servers this route cannot render faithfully."""
+ route = self.route
+ if not self.tokenizer_manager.is_generation:
+ return f"{route} requires a generation model"
+ if self.tokenizer_manager.tokenizer is None:
+ return f"{route} requires the server tokenizer"
+ if self.chat_encoding_spec is not None:
+ return (
+ f"{route} requires a chat template, but this model's chat "
+ f"route uses the {self.chat_encoding_spec!r} encoder"
+ )
+ if self.prompt_text_is_lossy:
+ return (
+ f"{route} places answer labels on the rendered chat text, "
+ "which this tokenizer does not encode back to the same ids"
+ )
+ if self.template_manager.chat_template_name is not None:
+ return (
+ f"{route} renders the tokenizer's Jinja chat template, but "
+ "this server uses the built-in chat template "
+ f"{self.template_manager.chat_template_name!r}"
+ )
+ if get_exec().features.enable_mis:
+ return f"{route} does not support --enable-mis"
+ if get_exec().dllm.dllm_algorithm is not None:
+ return (
+ f"{route} does not support diffusion language models "
+ "served with --dllm-algorithm"
+ )
+ _, adapter = self._parse_model_parameter(model)
+ if adapter is not None:
+ return (
+ f"model names the LoRA adapter {adapter!r}, which {route} "
+ "does not support"
+ )
+ return None
+
+ def _validate_reasoning(
+ self, chat_template_kwargs: Dict[str, Any]
+ ) -> Optional[str]:
+ """The answer position must follow the reasoning block, not sit inside it."""
+ config = self.template_manager.reasoning_config
+ if config is not None and config.always_on:
+ return (
+ f"{self.route} does not support chat templates that always "
+ "reason before answering"
+ )
+ toggle = self.reasoning_toggle
+ if toggle in chat_template_kwargs and chat_template_kwargs[toggle] is not False:
+ return (
+ f"chat_template_kwargs sets {toggle!r} to "
+ f"{chat_template_kwargs[toggle]!r}, but decisions need it false or unset"
+ )
+ return None
+
+ def _chat_template_kwargs(self, request_kwargs: Dict[str, Any]) -> Dict[str, Any]:
+ """Reasoning off, then the server defaults, then the request kwargs."""
+ kwargs = {}
+ if self.reasoning_toggle is not None:
+ kwargs[self.reasoning_toggle] = False
+ for key, value in self.default_chat_template_kwargs.items():
+ kwargs.setdefault(key, value)
+ kwargs.update(request_kwargs)
+ return kwargs
+
+ def _convert_to_internal_request(
+ self,
+ request: DecisionRequest,
+ raw_request: Request = None,
+ ) -> Tuple[Iterator[Tuple[List[int], List[int]]], DecisionRequest]:
+ # Lazy, so the handler can let other requests run between questions.
+ return self._encoded_questions(request), request
+
+ def _encoded_questions(
+ self, request: DecisionRequest
+ ) -> Iterator[Tuple[List[int], List[int]]]:
+ """Prompt and label ids for each question, in request order."""
+ text = render_text(request.input)
+ chat_template_kwargs = self._chat_template_kwargs(request.chat_template_kwargs)
+ for question in request.questions:
+ view = _decision_view(question)
+ try:
+ encoded = self._encode_question(
+ text=text,
+ view=view,
+ labels=default_labels(view),
+ chat_template_kwargs=chat_template_kwargs,
+ )
+ except ValueError as e:
+ raise ValueError(f"question {question.id!r}: {e}") from e
+ yield encoded
+
+ def _encode_question(
+ self,
+ text: str,
+ view: QuestionView,
+ labels: List[str],
+ chat_template_kwargs: Dict[str, Any],
+ ) -> Tuple[List[int], List[int]]:
+ tokenizer = self.tokenizer_manager.tokenizer
+ content = _render_question(text=text, view=view, labels=labels)
+ prompt = self._apply_chat_template(content, chat_template_kwargs)
+ if self.reasoning_markers is not None:
+ # Look only after the message, whose last line is fixed text.
+ closing = content.rsplit("\n", 1)[-1]
+ cut = prompt.rfind(closing)
+ generation_prompt = prompt if cut < 0 else prompt[cut + len(closing) :]
+ start, end = self.reasoning_markers
+ opened = generation_prompt.rfind(start)
+ closed = generation_prompt.rfind(end)
+ if opened > closed:
+ raise ValueError(
+ "the chat template leaves a reasoning block open at the "
+ "answer position, so this model is not supported with these "
+ "chat_template_kwargs"
+ )
+ if self.answers_open_reasoning and closed < 0:
+ raise ValueError(
+ "the reasoning parser for this model expects answers to start "
+ "with a reasoning block, and the chat template does not close "
+ "one. Send chat_template_kwargs that turn thinking off, if the "
+ "template supports it"
+ )
+ # The template's own finished reply shows whether answers start with
+ # a reasoning block that the generation prompt leaves out.
+ reply = self._render_reply(closing, chat_template_kwargs)
+ begin = reply.rfind(closing) if reply is not None else -1
+ answer = reply.find(_REPLY_SENTINEL, begin) if begin >= 0 else -1
+ if answer >= 0 and reply[begin:answer].count(start) > (
+ generation_prompt.count(start)
+ ):
+ raise ValueError(
+ "the chat template starts every answer with a reasoning "
+ "block, so this model is not supported"
+ )
+ prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
+ # Refuse here because --allow-auto-truncate would cut off the answer position.
+ context_len = self.tokenizer_manager.context_len
+ if len(prompt_ids) + self.tokenizer_manager.num_reserved_tokens >= context_len:
+ raise ValueError(
+ f"the prompt has {len(prompt_ids)} tokens, which does not fit "
+ f"the context length of {context_len} tokens"
+ )
+ label_ids = _encode_labels(
+ tokenizer=tokenizer,
+ prompt=prompt,
+ prompt_ids=prompt_ids,
+ labels=labels,
+ added_tokens=self.added_tokens,
+ )
+ return prompt_ids, label_ids
+
+ def _apply_chat_template(
+ self, content: str, chat_template_kwargs: Dict[str, Any]
+ ) -> str:
+ try:
+ return self.tokenizer_manager.tokenizer.apply_chat_template(
+ [{"role": "user", "content": content}],
+ tokenize=False,
+ add_generation_prompt=True,
+ **chat_template_kwargs,
+ )
+ except _CHAT_TEMPLATE_CLIENT_ERRORS as e:
+ raise ValueError(f"the chat template failed: {e}") from e
+
+ def _render_reply(
+ self, message: str, chat_template_kwargs: Dict[str, Any]
+ ) -> Optional[str]:
+ try:
+ return self.tokenizer_manager.tokenizer.apply_chat_template(
+ [
+ {"role": "user", "content": message},
+ {"role": "assistant", "content": _REPLY_SENTINEL},
+ ],
+ tokenize=False,
+ **chat_template_kwargs,
+ )
+ except _CHAT_TEMPLATE_CLIENT_ERRORS:
+ # The generation prompt checks above still apply.
+ return None
+
+ async def _score(
+ self,
+ adapted_request: Iterator[Tuple[List[int], List[int]]],
+ raw_request: Request,
+ temperature: float = 1.0,
+ ):
+ """Encode every question, then score them all in one call."""
+ prompts, label_token_ids = await _encode_all(adapted_request)
+ result = await self.tokenizer_manager.score_prompts(
+ prompts=prompts,
+ label_token_ids=label_token_ids,
+ apply_softmax=True,
+ request=raw_request,
+ temperature=temperature,
+ return_token_logprobs=True,
+ )
+ return prompts, label_token_ids, result
+
+ async def _handle_non_streaming_request(
+ self,
+ adapted_request: Iterator[Tuple[List[int], List[int]]],
+ request: DecisionRequest,
+ raw_request: Request,
+ ) -> ORJSONResponse:
+ prompts, label_token_ids, result = await self._score(
+ adapted_request=adapted_request,
+ raw_request=raw_request,
+ temperature=request.temperature,
+ )
+ answers = {}
+ for i, question in enumerate(request.questions):
+ answer = _build_answer(
+ question=question,
+ view=_decision_view(question),
+ probabilities=result.scores[i],
+ token_logprobs=result.token_logprobs[i],
+ )
+ if request.return_prompt_token_ids:
+ answer.prompt_token_ids = prompts[i]
+ answer.label_token_ids = label_token_ids[i]
+ answers[question.id] = answer
+ response = DecisionResponse(
+ model=request.model,
+ prompt_format_version=PROMPT_FORMAT_VERSION,
+ answers=answers,
+ usage=UsageInfo(
+ prompt_tokens=result.prompt_tokens,
+ total_tokens=result.prompt_tokens,
+ ),
+ )
+ return ORJSONResponse(content=response.model_dump(exclude_none=True))
+
+
+def render_text(value: Optional[DecisionText]) -> str:
+ if value is None:
+ return ""
+ if isinstance(value, str):
+ return value
+ return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+
+
+async def _encode_all(
+ encoded: Iterator[Tuple[List[int], List[int]]],
+) -> Tuple[List[List[int]], List[List[int]]]:
+ """Collect prompt and label ids, letting other requests run between questions."""
+ prompts, label_token_ids = [], []
+ for prompt_ids, label_ids in encoded:
+ prompts.append(prompt_ids)
+ label_token_ids.append(label_ids)
+ # Each question renders and tokenizes the whole input on the event loop.
+ await asyncio.sleep(0)
+ return prompts, label_token_ids
+
+
+def _decision_view(question: DecisionQuestion) -> QuestionView:
+ if isinstance(question, DecisionChoiceQuestion):
+ return QuestionView(
+ kind="choice",
+ question=question.question,
+ names=[option.name for option in question.options],
+ details=[option.description for option in question.options],
+ )
+ if isinstance(question, DecisionScoreQuestion):
+ return QuestionView(
+ kind="score",
+ question=question.question,
+ names=[str(level) for level in range(len(question.levels))],
+ details=list(question.levels),
+ )
+ return QuestionView(
+ kind="yes_no",
+ question=question.question,
+ names=["yes", "no"],
+ details=[question.yes, question.no],
+ )
+
+
+def default_labels(view: QuestionView) -> List[str]:
+ """Single-token labels in candidate order: A to Z, level indices, or yes and no."""
+ if view.kind == "choice":
+ return list(string.ascii_uppercase[: len(view.names)])
+ return list(view.names)
+
+
+def _render_question(text: str, view: QuestionView, labels: List[str]) -> str:
+ """Prompt wording of PROMPT_FORMAT_VERSION."""
+ # Every /v1/decisions question has text. A question without its own text
+ # drops the question line, and a yes or no question keeps its lead in.
+ question_text = (
+ "" if is_blank_decision_text(view.question) else render_text(view.question)
+ )
+ if view.kind == "choice":
+ lines = [f"Question: {question_text}"] if question_text else []
+ for label, name, description in zip(labels, view.names, view.details):
+ detail = render_text(description)
+ lines.append(
+ f"{label}: {name} - {detail}" if detail else f"{label}: {name}"
+ )
+ lines.append("Answer with the letter of one option only.")
+ elif view.kind == "score":
+ lines = [f"Question: {question_text}"] if question_text else []
+ lines += [
+ f"{label}: {render_text(level)}"
+ for label, level in zip(labels, view.details)
+ ]
+ lines.append("Answer with the number of one level only.")
+ else:
+ lines = [
+ f"Is the following true? {question_text}"
+ if question_text
+ else "Is the following true?"
+ ]
+ for label, description in zip(labels, view.details):
+ detail = render_text(description)
+ if detail:
+ lines.append(f"{label}: {detail}")
+ lines.append("Answer with yes or no only.")
+ return "\n".join([text, "", *lines])
+
+
+def label_context(
+ tokenizer: Any,
+ prompt: str,
+ prompt_ids: List[int],
+ added_tokens: Dict[int, str],
+) -> Tuple[str, List[int], bool]:
+ """Text and ids after which labels are checked, and whether the shortcut applies."""
+ # Added tokens are split off before tokenization.
+ # The text after the last one tokenizes on its own,
+ # so the check does not grow with the input.
+ # When the prompt ends with an added token, each label starts a new segment,
+ # which is how the model continues after that token.
+ last = next(
+ (i for i in reversed(range(len(prompt_ids))) if prompt_ids[i] in added_tokens),
+ None,
+ )
+ if last is not None:
+ token = added_tokens[prompt_ids[last]]
+ start = prompt.rfind(token)
+ suffix = prompt[start + len(token) :]
+ if start >= 0 and (
+ tokenizer.encode(suffix, add_special_tokens=False) == prompt_ids[last + 1 :]
+ ):
+ return suffix, prompt_ids[last + 1 :], True
+ return prompt, prompt_ids, False
+
+
+def label_token_id(
+ tokenizer: Any, text: str, text_ids: List[int], label: str
+) -> Optional[int]:
+ """The token a label adds after the text, or None when it is not exactly one."""
+ ids = tokenizer.encode(text + label, add_special_tokens=False)
+ if len(ids) != len(text_ids) + 1 or ids[:-1] != text_ids:
+ return None
+ return ids[-1]
+
+
+def _encode_labels(
+ tokenizer: Any,
+ prompt: str,
+ prompt_ids: List[int],
+ labels: List[str],
+ added_tokens: Dict[int, str],
+) -> List[int]:
+ """Check that each label adds exactly one distinct token after the prompt."""
+ text, text_ids, _ = label_context(tokenizer, prompt, prompt_ids, added_tokens)
+ label_ids = []
+ for label in labels:
+ token_id = label_token_id(tokenizer, text, text_ids, label)
+ if token_id is None or token_id in label_ids:
+ raise ValueError(
+ f"the answer label {label!r} is not one distinct token after the "
+ "chat prompt for this tokenizer, so this model is not supported"
+ )
+ label_ids.append(token_id)
+ return label_ids
+
+
+def label_mass(token_logprobs: List[float]) -> float:
+ return math.fsum(math.exp(logprob) for logprob in token_logprobs)
+
+
+def _build_answer(
+ question: DecisionQuestion,
+ view: QuestionView,
+ probabilities: List[float],
+ token_logprobs: List[float],
+) -> DecisionAnswer:
+ names = view.names
+ value = {}
+ if view.kind == "choice":
+ value["choice"] = names[probabilities.index(max(probabilities))]
+ elif view.kind == "score":
+ value["score"] = math.fsum(i * p for i, p in enumerate(probabilities))
+ return DecisionAnswer(
+ type=question.type,
+ probabilities=dict(zip(names, probabilities)),
+ label_mass=label_mass(token_logprobs),
+ **value,
+ )
diff --git a/python/sglang/srt/entrypoints/systemone/__init__.py b/python/sglang/srt/entrypoints/systemone/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/python/sglang/srt/entrypoints/systemone/protocol.py b/python/sglang/srt/entrypoints/systemone/protocol.py
new file mode 100644
index 0000000000..3412917d6e
--- /dev/null
+++ b/python/sglang/srt/entrypoints/systemone/protocol.py
@@ -0,0 +1,150 @@
+"""Pydantic models for the System One decision API.
+
+Follows the published System One OpenAPI 0.2.0 request and response shapes:
+a state, a map of noul, choice, and score questions keyed by caller ids, and
+one answer per question id.
+"""
+
+from typing import Annotated, Any, Dict, List, Literal, Optional, Union
+
+from pydantic import (
+ BaseModel,
+ ConfigDict,
+ Field,
+ NonNegativeInt,
+ ValidationInfo,
+ field_validator,
+ model_validator,
+)
+from pydantic.json_schema import SkipJsonSchema
+
+from sglang.srt.entrypoints.openai.protocol import (
+ DecisionText,
+ RequiredDecisionText,
+ check_option_names,
+ is_blank_decision_text,
+)
+
+# The documented maximum. Options beyond 26 get two-letter labels, checked per request.
+MAX_CHOICE_OPTIONS = 255
+# Levels are labeled 0 to 9, so at most 10.
+MAX_SCORE_LEVELS = 10
+
+
+class _Question(BaseModel):
+ # Misspelled keys inside a question would otherwise answer a different question.
+ model_config = ConfigDict(extra="forbid")
+
+ instructions: Optional[DecisionText] = None
+
+
+class SystemOneNoulCriteria(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ true: Optional[DecisionText] = None
+ false: Optional[DecisionText] = None
+
+
+class SystemOneNoulQuestion(_Question):
+ type: Literal["noul"]
+ criteria: Optional[SystemOneNoulCriteria] = None
+
+ @model_validator(mode="after")
+ def _asks_something(self):
+ criteria = self.criteria or SystemOneNoulCriteria()
+ if all(
+ is_blank_decision_text(value)
+ for value in (self.instructions, criteria.true, criteria.false)
+ ):
+ raise ValueError(
+ "a noul question needs instructions or a true or false "
+ "description to decide on"
+ )
+ return self
+
+
+class SystemOneChoiceQuestion(_Question):
+ type: Literal["choice"]
+ criteria: Dict[str, Optional[DecisionText]] = Field(
+ min_length=1, max_length=MAX_CHOICE_OPTIONS
+ )
+
+ @field_validator("criteria")
+ @classmethod
+ def _option_names_distinct(cls, criteria):
+ check_option_names(criteria)
+ return criteria
+
+
+class SystemOneScoreQuestion(_Question):
+ type: Literal["score"]
+ criteria: List[RequiredDecisionText] = Field(
+ min_length=1, max_length=MAX_SCORE_LEVELS
+ )
+
+
+SystemOneQuestion = Annotated[
+ Union[SystemOneNoulQuestion, SystemOneChoiceQuestion, SystemOneScoreQuestion],
+ Field(discriminator="type"),
+]
+
+
+class SystemOneRequest(BaseModel):
+ # Unknown top-level fields are ignored, as the published schema allows.
+ state: DecisionText
+ model: str
+ questions: Dict[str, SystemOneQuestion] = Field(min_length=1)
+ # SGLang extension, for chat templates whose reasoning toggle needs a kwarg.
+ chat_template_kwargs: Dict[str, Any] = Field(default_factory=dict)
+
+ # /v1/decisions fields, which would change the answers if honored or ignored.
+ # Declared only to refuse them by name, and hidden from the schema.
+ temperature: SkipJsonSchema[Any] = None
+ prompt_format_version: SkipJsonSchema[Any] = None
+ return_prompt_token_ids: SkipJsonSchema[Any] = None
+
+ @field_validator("temperature", "prompt_format_version", "return_prompt_token_ids")
+ @classmethod
+ def _refuse_decisions_fields(cls, value, info: ValidationInfo):
+ if value is not None:
+ raise ValueError(
+ f"{info.field_name} is not part of this API, use /v1/decisions for it"
+ )
+ return value
+
+
+class SystemOneNoulAnswer(BaseModel):
+ type: Literal["noul"] = "noul"
+ noul: float
+ # Full-vocabulary probability of the answer labels, an SGLang extension.
+ x_label_mass: float
+
+
+class SystemOneChoiceAnswer(BaseModel):
+ type: Literal["choice"] = "choice"
+ choice: str
+ confidence: float
+ probabilities: Dict[str, float]
+ x_label_mass: float
+
+
+class SystemOneScoreAnswer(BaseModel):
+ type: Literal["score"] = "score"
+ score: float
+ confidence: float
+ legend: Dict[str, Any]
+ probabilities: Dict[str, float]
+ x_label_mass: float
+
+
+class SystemOneUsage(BaseModel):
+ input_tokens: NonNegativeInt
+ output_tokens: NonNegativeInt = 0
+
+
+class SystemOneResponse(BaseModel):
+ model: str
+ answers: Dict[
+ str, Union[SystemOneNoulAnswer, SystemOneChoiceAnswer, SystemOneScoreAnswer]
+ ]
+ usage: SystemOneUsage
diff --git a/python/sglang/srt/entrypoints/systemone/serving.py b/python/sglang/srt/entrypoints/systemone/serving.py
new file mode 100644
index 0000000000..6039d1a188
--- /dev/null
+++ b/python/sglang/srt/entrypoints/systemone/serving.py
@@ -0,0 +1,258 @@
+"""Handler for the System One compatible decision API, on the /v1/decisions scoring path."""
+
+from __future__ import annotations
+
+import math
+import string
+from typing import Any, Dict, Iterator, List, Optional, Tuple
+
+import orjson
+from fastapi import Request
+from fastapi.responses import ORJSONResponse
+
+from sglang.srt.entrypoints.openai.serving_decisions import (
+ OpenAIServingDecisions,
+ QuestionView,
+ default_labels,
+ label_context,
+ label_mass,
+ label_token_id,
+ render_text,
+)
+from sglang.srt.entrypoints.systemone.protocol import (
+ SystemOneChoiceAnswer,
+ SystemOneChoiceQuestion,
+ SystemOneNoulAnswer,
+ SystemOneNoulQuestion,
+ SystemOneQuestion,
+ SystemOneRequest,
+ SystemOneResponse,
+ SystemOneScoreAnswer,
+ SystemOneUsage,
+)
+
+# Beyond A to Z, every option gets a two-letter label, in this fixed order.
+_PAIR_LABELS = [a + b for a in string.ascii_uppercase for b in string.ascii_uppercase]
+
+
+class SystemOneServing(OpenAIServingDecisions):
+ """Answers System One questions with the rendering, label checks, and scoring of /v1/decisions."""
+
+ route = "/v1/systemone"
+
+ def _request_id_prefix(self) -> str:
+ return "systemone-"
+
+ def _validate_request(self, request: SystemOneRequest) -> Optional[str]:
+ return self._validate_server(request.model) or self._validate_reasoning(
+ request.chat_template_kwargs
+ )
+
+ def _convert_to_internal_request(
+ self,
+ request: SystemOneRequest,
+ raw_request: Request = None,
+ ) -> Tuple[
+ Iterator[Tuple[List[int], List[int]]],
+ Tuple[SystemOneRequest, List[QuestionView]],
+ ]:
+ views = [_view(question) for question in request.questions.values()]
+ # Lazy, so the async handler can yield to other requests between questions.
+ return self._encoded_systemone_questions(request, views), (request, views)
+
+ def _encoded_systemone_questions(
+ self, request: SystemOneRequest, views: List[QuestionView]
+ ) -> Iterator[Tuple[List[int], List[int]]]:
+ """Prompt and label ids for each question, in request order."""
+ text = render_text(request.state)
+ chat_template_kwargs = self._chat_template_kwargs(request.chat_template_kwargs)
+ pair_labels = None
+ for question_id, view in zip(request.questions, views):
+ try:
+ labels = default_labels(view)
+ if view.kind == "choice" and len(view.names) > len(labels):
+ if pair_labels is None:
+ pair_labels = self._pair_labels(chat_template_kwargs)
+ if len(view.names) > len(pair_labels):
+ raise ValueError(
+ f"it has {len(view.names)} options, but the served "
+ "tokenizer and chat template can label at most "
+ f"{max(len(pair_labels), len(string.ascii_uppercase))}"
+ )
+ labels = pair_labels[: len(view.names)]
+ if view.kind == "score":
+ _check_legend(question_id, view)
+ encoded = self._encode_question(
+ text=text,
+ view=view,
+ labels=labels,
+ chat_template_kwargs=chat_template_kwargs,
+ )
+ except ValueError as e:
+ raise ValueError(f"question {question_id!r}: {e}") from e
+ yield encoded
+
+ def _pair_labels(self, chat_template_kwargs: Dict[str, Any]) -> List[str]:
+ """Two-letter labels that are distinct single tokens at the answer position."""
+ if not self.added_tokens:
+ raise ValueError(
+ "more than 26 options needs added tokens the server can read "
+ "from the tokenizer, and the served tokenizer reports none"
+ )
+ tokenizer = self.tokenizer_manager.tokenizer
+ contexts = []
+ for message in ("x", "y"):
+ prompt = self._apply_chat_template(message, chat_template_kwargs)
+ prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
+ contexts.append(
+ label_context(tokenizer, prompt, prompt_ids, self.added_tokens)
+ )
+ (text, text_ids, shortcut), other = contexts
+ # Labels must be checked on text that excludes the message. Otherwise
+ # every label re-encodes the whole input on the event loop, which is
+ # unbounded work for hundreds of options.
+ if not (shortcut and other[2] and other[0] == text):
+ raise ValueError(
+ "more than 26 options needs an added token between the message "
+ "and the answer position, after which the text tokenizes the same on "
+ "its own, which this tokenizer and chat template do not provide"
+ )
+ labels, label_ids = [], set()
+ for label in _PAIR_LABELS:
+ token_id = label_token_id(tokenizer, text, text_ids, label)
+ if token_id is not None and token_id not in label_ids:
+ labels.append(label)
+ label_ids.add(token_id)
+ # Each final prompt is still checked as a whole in _encode_question.
+ return labels
+
+ async def _handle_non_streaming_request(
+ self,
+ adapted_request: Iterator[Tuple[List[int], List[int]]],
+ processed: Tuple[SystemOneRequest, List[QuestionView]],
+ raw_request: Request,
+ ) -> ORJSONResponse:
+ request, views = processed
+ _, _, result = await self._score(
+ adapted_request=adapted_request, raw_request=raw_request
+ )
+ answers = {}
+ for i, question_id in enumerate(request.questions):
+ answers[question_id] = _answer(
+ view=views[i],
+ probabilities=result.scores[i],
+ mass=label_mass(result.token_logprobs[i]),
+ question_id=question_id,
+ )
+ response = SystemOneResponse(
+ # The served model answered, whatever name the request used.
+ model=self.tokenizer_manager.served_model_name,
+ answers=answers,
+ usage=SystemOneUsage(input_tokens=result.prompt_tokens),
+ )
+ return ORJSONResponse(content=response.model_dump())
+
+
+def _view(question: SystemOneQuestion) -> QuestionView:
+ if isinstance(question, SystemOneChoiceQuestion):
+ return QuestionView(
+ kind="choice",
+ question=question.instructions,
+ names=list(question.criteria),
+ details=list(question.criteria.values()),
+ )
+ if isinstance(question, SystemOneNoulQuestion):
+ criteria = question.criteria
+ return QuestionView(
+ kind="yes_no",
+ question=question.instructions,
+ names=["yes", "no"],
+ details=[
+ criteria.true if criteria else None,
+ criteria.false if criteria else None,
+ ],
+ )
+ return QuestionView(
+ kind="score",
+ question=question.instructions,
+ names=[str(level) for level in range(len(question.criteria))],
+ details=list(question.criteria),
+ )
+
+
+def _answer(
+ view: QuestionView,
+ probabilities: List[float],
+ mass: float,
+ question_id: str,
+):
+ if not all(math.isfinite(value) for value in [*probabilities, mass]):
+ # A server fault, reported as 500 rather than as a client error.
+ raise RuntimeError(f"question {question_id!r} scored non-finite values")
+ # Reported as scored, like /v1/decisions, and normalized only for confidence.
+ if view.kind == "yes_no":
+ return SystemOneNoulAnswer(noul=probabilities[0], x_label_mass=mass)
+ probabilities_by_name = dict(zip(view.names, probabilities))
+ if view.kind == "choice":
+ return SystemOneChoiceAnswer(
+ choice=view.names[probabilities.index(max(probabilities))],
+ confidence=_choice_confidence(_normalized(probabilities)),
+ probabilities=probabilities_by_name,
+ x_label_mass=mass,
+ )
+ return SystemOneScoreAnswer(
+ score=math.fsum(i * p for i, p in enumerate(probabilities)),
+ confidence=_score_confidence(_normalized(probabilities)),
+ legend=_legend(view),
+ probabilities=probabilities_by_name,
+ x_label_mass=mass,
+ )
+
+
+def _legend(view: QuestionView) -> Dict[str, Any]:
+ return dict(zip(view.names, view.details))
+
+
+def _check_legend(question_id: str, view: QuestionView) -> None:
+ """Refuse, before scoring, levels that the response cannot echo in its legend."""
+ answer = SystemOneScoreAnswer(
+ score=0.0,
+ confidence=0.0,
+ legend=_legend(view),
+ probabilities={},
+ x_label_mass=0.0,
+ )
+ response = SystemOneResponse(
+ model="", answers={question_id: answer}, usage=SystemOneUsage(input_tokens=0)
+ )
+ try:
+ # The same encoder and options as the real response.
+ ORJSONResponse(content=response.model_dump())
+ except orjson.JSONEncodeError as e:
+ raise ValueError(f"a level cannot be returned in the legend: {e}") from e
+
+
+def _normalized(probabilities: List[float]) -> List[float]:
+ total = math.fsum(probabilities)
+ if total <= 0:
+ return [1.0 / len(probabilities)] * len(probabilities)
+ return [p / total for p in probabilities]
+
+
+def _choice_confidence(q: List[float]) -> float:
+ """How far the top option stands above a uniform guess, from 0 to 1."""
+ n = len(q)
+ if n == 1:
+ return 1.0
+ return min(1.0, max(0.0, (n * max(q) - 1) / (n - 1)))
+
+
+def _score_confidence(q: List[float]) -> float:
+ """One minus the spread around the top level relative to a uniform spread, floored at 0."""
+ n = len(q)
+ if n == 1:
+ return 1.0
+ top = q.index(max(q))
+ spread = math.fsum(p * abs(i - top) for i, p in enumerate(q))
+ uniform_spread = math.fsum(abs(i - (n - 1) / 2) for i in range(n)) / n
+ return max(0.0, 1 - spread / uniform_spread)
diff --git a/python/sglang/srt/layers/logprob_processor.py b/python/sglang/srt/layers/logprob_processor.py
index d336895f8d..9d5690a7ca 100644
--- a/python/sglang/srt/layers/logprob_processor.py
+++ b/python/sglang/srt/layers/logprob_processor.py
@@ -155,15 +155,17 @@ def get_token_ids_logprobs_raw(
if stage == LogprobStage.DECODE:
for i, token_ids in enumerate(token_ids_logprobs_list):
if token_ids is None:
- vals.append([])
- idxs.append([])
+ # The CPU copy expects a tensor in every entry under no_copy_to_cpu.
+ # A new empty one does not keep the full logprobs alive like a view.
+ row = logprobs.new_empty((0,))
+ token_ids = []
else:
token_ids_tensor = torch.tensor(
token_ids, dtype=torch.long, pin_memory=pin_memory
).to(logprobs.device, non_blocking=True)
row = logprobs[i, token_ids_tensor]
- vals.append(row if no_copy_to_cpu else row.tolist())
- idxs.append(token_ids)
+ vals.append(row if no_copy_to_cpu else row.tolist())
+ idxs.append(token_ids)
else: # prefill
pt = 0
for i, (token_ids, pruned_len) in enumerate(
diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py
index e3d04a9e06..dff813f3b8 100644
--- a/python/sglang/srt/managers/tokenizer_manager.py
+++ b/python/sglang/srt/managers/tokenizer_manager.py
@@ -2853,7 +2853,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
)
if top_logprobs_num > 0:
- if len(recv_obj.input_top_logprobs_val) > 0:
+ if (
+ recv_obj.input_top_logprobs_val is not None
+ and len(recv_obj.input_top_logprobs_val) > 0
+ ):
state.input_top_logprobs_val.extend(
recv_obj.input_top_logprobs_val[recv_obj_index]
)
@@ -2869,27 +2872,32 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
recv_obj.input_top_logprobs_idx_flat[recv_obj_index],
recv_obj.input_top_logprobs_flat_null_prefix[recv_obj_index],
)
- state.output_top_logprobs_val.extend(
- recv_obj.output_top_logprobs_val[recv_obj_index]
- )
- state.output_top_logprobs_idx.extend(
- recv_obj.output_top_logprobs_idx[recv_obj_index]
- )
+ if recv_obj.output_top_logprobs_val is not None:
+ state.output_top_logprobs_val.extend(
+ recv_obj.output_top_logprobs_val[recv_obj_index]
+ )
+ state.output_top_logprobs_idx.extend(
+ recv_obj.output_top_logprobs_idx[recv_obj_index]
+ )
if token_ids_logprob is not None:
- if len(recv_obj.input_token_ids_logprobs_val) > 0:
+ if (
+ recv_obj.input_token_ids_logprobs_val is not None
+ and len(recv_obj.input_token_ids_logprobs_val) > 0
+ ):
state.input_token_ids_logprobs_val.extend(
recv_obj.input_token_ids_logprobs_val[recv_obj_index]
)
state.input_token_ids_logprobs_idx.extend(
recv_obj.input_token_ids_logprobs_idx[recv_obj_index]
)
- state.output_token_ids_logprobs_val.extend(
- recv_obj.output_token_ids_logprobs_val[recv_obj_index]
- )
- state.output_token_ids_logprobs_idx.extend(
- recv_obj.output_token_ids_logprobs_idx[recv_obj_index]
- )
+ if recv_obj.output_token_ids_logprobs_val is not None:
+ state.output_token_ids_logprobs_val.extend(
+ recv_obj.output_token_ids_logprobs_val[recv_obj_index]
+ )
+ state.output_token_ids_logprobs_idx.extend(
+ recv_obj.output_token_ids_logprobs_idx[recv_obj_index]
+ )
self.add_logprob_to_meta_info(
meta_info,
diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py
index 42115543cc..17a4cc47f9 100644
--- a/test/registered/unit/entrypoints/openai/test_serving_chat.py
+++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py
@@ -2956,6 +2956,31 @@ class ServingChatTestCase(unittest.TestCase):
chunks.append(chunk)
return chunks
+ def test_streaming_top_logprobs_follow_each_token_in_chunk(self):
+ """Each token of a multi-token streaming chunk keeps its own alternatives."""
+ content = {
+ "meta_info": {
+ "output_token_logprobs": [
+ (-0.1, 1, "a"),
+ (-0.2, 2, "b"),
+ (-0.3, 3, "c"),
+ ],
+ "output_top_logprobs": [
+ [(-0.1, 1, "a"), (-2.0, 9, "x")],
+ [(-0.2, 2, "b"), (-3.0, 8, "y")],
+ [(-0.3, 3, "c"), (-4.0, 7, "z")],
+ ],
+ },
+ }
+ choice_logprobs = self.chat._process_streaming_logprobs(content, 1, 3)
+ tokens = [entry.token for entry in choice_logprobs.content]
+ alternatives = [
+ [top.token for top in entry.top_logprobs]
+ for entry in choice_logprobs.content
+ ]
+ self.assertEqual(tokens, ["b", "c"])
+ self.assertEqual(alternatives, [["b", "y"], ["c", "z"]])
+
def test_streaming_logprobs_attached_with_reasoning_parser(self):
"""Logprobs must ride on the reasoning chunk when a reasoning parser is active."""
self.chat.reasoning_parser = "qwen3"
diff --git a/test/registered/unit/entrypoints/openai/test_serving_decisions.py b/test/registered/unit/entrypoints/openai/test_serving_decisions.py
new file mode 100644
index 0000000000..ecd049865d
--- /dev/null
+++ b/test/registered/unit/entrypoints/openai/test_serving_decisions.py
@@ -0,0 +1,1187 @@
+"""Unit tests for /v1/decisions and /v1/systemone: request limits, prompt text, answer labels, and scoring."""
+
+import asyncio
+import json
+import math
+import string
+import unittest
+from types import SimpleNamespace
+
+import torch
+from pydantic import ValidationError
+from transformers import AddedToken, AutoTokenizer
+
+from sglang.srt.entrypoints.openai import chat_encoding
+from sglang.srt.entrypoints.openai.protocol import DecisionRequest
+from sglang.srt.entrypoints.openai.serving_decisions import (
+ PROMPT_FORMAT_VERSION,
+ OpenAIServingDecisions,
+ _decision_view,
+ _encode_labels,
+ _render_question,
+)
+from sglang.srt.entrypoints.systemone.protocol import SystemOneRequest
+from sglang.srt.entrypoints.systemone.serving import (
+ SystemOneServing,
+ _answer,
+ _choice_confidence,
+ _score_confidence,
+ _view,
+)
+from sglang.srt.managers.tokenizer_manager_score_mixin import TokenizerManagerScoreMixin
+from sglang.srt.parser.template_detection import (
+ ReasoningToggleConfig,
+ detect_reasoning_parser,
+ detect_reasoning_pattern,
+)
+from sglang.srt.runtime_context import publish, restore_context, snapshot_context
+from sglang.srt.server_args import ServerArgs
+from sglang.test.ci.ci_register import register_cpu_ci
+
+register_cpu_ci(est_time=25, suite="base-a-test-cpu")
+
+# Tokenizer files only, of a chat template that thinks by default.
+TOKENIZER = "Qwen/Qwen3.5-35B-A3B"
+
+
+def _question(kind, entries=None, question="Q"):
+ """Options as a name to description map, levels as a list, or yes and no text."""
+ body = {"type": kind, "question": question}
+ if kind == "choice" and entries is not None:
+ body["options"] = [
+ {"name": name} if text is None else {"name": name, "description": text}
+ for name, text in entries.items()
+ ]
+ elif kind == "score" and entries is not None:
+ body["levels"] = entries
+ elif entries is not None:
+ body.update(entries)
+ return body
+
+
+def _request(input_, questions, **kwargs):
+ """Questions as an id to question map, in order."""
+ return DecisionRequest(
+ input=input_,
+ questions=[{"id": qid, **question} for qid, question in questions.items()],
+ **kwargs,
+ )
+
+
+def _by_id(request, question_id):
+ return next(q for q in request.questions if q.id == question_id)
+
+
+def _encoded(handler, request):
+ """Prompt and label ids for every question, as the handler scores them."""
+ encoded, _ = handler._convert_to_internal_request(request)
+ prompts, label_ids = zip(*encoded)
+ return list(prompts), list(label_ids)
+
+
+def _handler(
+ manager,
+ reasoning_config=None,
+ reasoning_parser=None,
+ lossy=False,
+ serving_class=OpenAIServingDecisions,
+):
+ """Build the handler over the chat serving state the server builds at startup."""
+ template = manager.tokenizer.chat_template
+ force_reasoning, detected = detect_reasoning_pattern(template)
+ if reasoning_config is None:
+ reasoning_config = detected
+ template_manager = SimpleNamespace(
+ chat_template_name=None,
+ reasoning_config=reasoning_config,
+ suggested_reasoning_parser=detect_reasoning_parser(
+ template, manager.tokenizer, detected, force_reasoning
+ ),
+ )
+ chat_serving = SimpleNamespace(
+ tokenizer_manager=manager,
+ template_manager=template_manager,
+ default_chat_template_kwargs={},
+ chat_encoding_spec=chat_encoding.resolve_chat_encoding_spec(
+ hf_config=manager.model_config.hf_config,
+ tokenizer=manager.tokenizer,
+ tool_call_parser=None,
+ ),
+ _prompt_text_round_trip_is_lossy=lossy,
+ reasoning_parser=reasoning_parser,
+ )
+ return serving_class(chat_serving)
+
+
+class UnknownTokenizer:
+ """Encodes every character as the same unknown token."""
+
+ def encode(self, text, add_special_tokens=False):
+ return [0] * len(text)
+
+
+class PlainTokenizer:
+ """A tokenizer outside transformers, without added-token metadata."""
+
+ def __init__(self, tokenizer):
+ self.chat_template = tokenizer.chat_template
+ self.encode = tokenizer.encode
+ self.apply_chat_template = tokenizer.apply_chat_template
+ self.vocab_size = len(tokenizer)
+
+ def __len__(self):
+ return self.vocab_size
+
+
+class ScoringManager(TokenizerManagerScoreMixin):
+ """Replace only model execution with fixed full-vocabulary logprobs."""
+
+ def __init__(
+ self,
+ tokenizer,
+ is_generation=True,
+ context_len=4096,
+ architecture=None,
+ **server_args,
+ ):
+ self.server_args = ServerArgs(model_path="dummy", **server_args)
+ publish(self.server_args, role="test")
+ self.tokenizer = tokenizer
+ self.model_config = SimpleNamespace(
+ hf_config=SimpleNamespace(
+ architectures=[architecture or "Qwen3_5MoeForConditionalGeneration"],
+ model_type="qwen3_5_moe",
+ )
+ )
+ self.is_generation = is_generation
+ self.context_len = context_len
+ self.num_reserved_tokens = 0
+ self.request_logger = SimpleNamespace(log_requests=False)
+ generator = torch.Generator().manual_seed(0)
+ logits = torch.randn(len(tokenizer), generator=generator, dtype=torch.float64)
+ self.logprobs = torch.log_softmax(logits * 4, dim=0)
+ self.requests = []
+ self.served_model_name = "served-model"
+
+ def config_value(self, name):
+ return None
+
+ async def generate_request(self, request, raw_request):
+ self.requests.append(request)
+ request.normalize_batch_and_arguments()
+ results = []
+ for ids, labels in zip(request.input_ids, request.token_ids_logprob):
+ logprobs = [(self.logprobs[token].item(), token, None) for token in labels]
+ meta = {"prompt_tokens": len(ids), "output_token_ids_logprobs": [logprobs]}
+ results.append({"meta_info": meta})
+ yield results
+
+
+# Rendered lines of test_prompt_text by prompt format version.
+PROMPT_FIXTURES = {
+ 1: {
+ "choice": [
+ 'Question: {"question":"Which team?"}',
+ "A: billing - Payments",
+ "B: sales",
+ 'C: other - {"k":1}',
+ "Answer with the letter of one option only.",
+ ],
+ "score": [
+ "Question: Mood?",
+ "0: Calm",
+ "1: Angry",
+ "Answer with the number of one level only.",
+ ],
+ "yes_no": [
+ "Is the following true? Urgent",
+ "no: Can wait",
+ "Answer with yes or no only.",
+ ],
+ },
+}
+
+
+class TestDecisions(unittest.IsolatedAsyncioTestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
+
+ def setUp(self):
+ self.addCleanup(restore_context, snapshot_context())
+
+ def test_request_limits_follow_the_label_alphabets(self):
+ options = {name: None for name in string.ascii_lowercase}
+ rejected = {
+ "unknown type": {"type": "rank", "question": "Q"},
+ "one option": _question("choice", {"a": None}),
+ "27 options": _question("choice", {**options, "extra": None}),
+ "blank option name": _question("choice", {"a": None, " ": None}),
+ "repeated option name": _question("choice", {"a": None, " A": None}),
+ "option name with a line break": _question(
+ "choice", {"a\nB: b": None, "c": None}
+ ),
+ "one level": _question("score", ["low"]),
+ "11 levels": _question("score", [str(i) for i in range(11)]),
+ "blank level": _question("score", ["low", " "]),
+ "no question text": {"type": "yes_no"},
+ "blank question text": _question("yes_no", question=" "),
+ "empty object question text": _question("yes_no", question={}),
+ "misspelled yes_no description": _question("yes_no", {"true": "Urgent"}),
+ }
+ for case, question in rejected.items():
+ with self.subTest(case), self.assertRaises(ValidationError) as caught:
+ _request("s", {"q1": question})
+ self.assertIn("questions.0", str(caught.exception))
+ question = [{"id": "q1", **_question("yes_no")}]
+ for payload in (
+ {"input": "s", "questions": []},
+ {"input": "s", "questions": question, "temperature": 0},
+ {"input": " ", "questions": question},
+ {"input": [], "questions": question},
+ {"input": "s", "questions": question, "top_p": 0.5},
+ {"input": "s", "questions": [{"id": " ", **_question("yes_no")}]},
+ {"input": "s", "questions": question * 2},
+ ):
+ with self.subTest(payload), self.assertRaises(ValidationError):
+ DecisionRequest(**payload)
+ request = _request(
+ "s",
+ {
+ "choice": _question("choice", options),
+ "score": _question("score", [str(i) for i in range(10)]),
+ "yes_no": _question("yes_no"),
+ },
+ )
+ self.assertEqual(request.chat_template_kwargs, {})
+
+ def test_prompt_text(self):
+ request = _request(
+ {"ticket": "Refund please", "tags": ["billing"]},
+ {
+ "choice": _question(
+ "choice",
+ {"billing": "Payments", "sales": None, "other": {"k": 1}},
+ question={"question": "Which team?"},
+ ),
+ "score": _question("score", ["Calm", "Angry"], question="Mood?"),
+ "yes_no": _question("yes_no", {"no": "Can wait"}, question="Urgent"),
+ },
+ )
+ text = '{"ticket":"Refund please","tags":["billing"]}'
+ # A wording change needs a new PROMPT_FORMAT_VERSION and its own fixture.
+ self.assertIn(PROMPT_FORMAT_VERSION, PROMPT_FIXTURES)
+ labels = {
+ "choice": ["A", "B", "C"],
+ "score": ["0", "1"],
+ "yes_no": ["yes", "no"],
+ }
+ for question_id, lines in PROMPT_FIXTURES[PROMPT_FORMAT_VERSION].items():
+ rendered = _render_question(
+ text=text,
+ view=_decision_view(_by_id(request, question_id)),
+ labels=labels[question_id],
+ )
+ self.assertEqual(rendered, "\n".join([text, "", *lines]))
+ # Renderings only /v1/systemone reaches: no question text, criteria-only
+ # yes or no questions, and two-letter labels beyond 26 options.
+ systemone = SystemOneRequest(
+ state="s",
+ model="m",
+ questions={
+ "choice": {
+ "type": "choice",
+ "criteria": {"billing": "Payments", "sales": None},
+ },
+ "score": {"type": "score", "criteria": ["Calm", "Angry"]},
+ "noul": {
+ "type": "noul",
+ "criteria": {"true": "Reply today", "false": "Can wait"},
+ },
+ },
+ )
+ cases = {
+ "choice": (
+ ["AA", "AB"],
+ [
+ "AA: billing - Payments",
+ "AB: sales",
+ "Answer with the letter of one option only.",
+ ],
+ ),
+ "score": (
+ ["0", "1"],
+ ["0: Calm", "1: Angry", "Answer with the number of one level only."],
+ ),
+ "noul": (
+ ["yes", "no"],
+ [
+ "Is the following true?",
+ "yes: Reply today",
+ "no: Can wait",
+ "Answer with yes or no only.",
+ ],
+ ),
+ }
+ for question_id, (labels, lines) in cases.items():
+ rendered = _render_question(
+ text="s", view=_view(systemone.questions[question_id]), labels=labels
+ )
+ self.assertEqual(rendered, "\n".join(["s", "", *lines]))
+
+ def test_labels_are_vocabulary_tokens_after_the_non_thinking_prompt(self):
+ handler = _handler(ScoringManager(self.tokenizer))
+ questions = {
+ "choice": _question("choice", {c: None for c in string.ascii_lowercase}),
+ "score": _question("score", [str(i) for i in range(10)]),
+ "yes_no": _question("yes_no"),
+ }
+ labels = [
+ list(string.ascii_uppercase),
+ [str(i) for i in range(10)],
+ ["yes", "no"],
+ ]
+ label_ids = [self.tokenizer.convert_tokens_to_ids(t) for t in labels]
+ # Other kwargs and a server default that thinks keep reasoning off.
+ for kwargs, server_kwargs in (
+ ({}, {}),
+ ({"chat_template_kwargs": {}}, {}),
+ ({"chat_template_kwargs": {"custom_flag": 1}}, {}),
+ ({}, {"enable_thinking": True}),
+ ):
+ with self.subTest(kwargs=kwargs, server_kwargs=server_kwargs):
+ handler.default_chat_template_kwargs = server_kwargs
+ request = _request("s", questions, **kwargs)
+ prompts, ids = _encoded(handler, request)
+ self.assertEqual(ids, label_ids)
+ for prompt in prompts:
+ self.assertTrue(
+ self.tokenizer.decode(prompt).endswith("\n\n")
+ )
+ # The suffix check agrees with the whole-prompt check.
+ text = self.tokenizer.decode(prompt)
+ for question_labels, question_ids in zip(labels, label_ids):
+ self.assertEqual(
+ _encode_labels(
+ tokenizer=self.tokenizer,
+ prompt=text,
+ prompt_ids=prompt,
+ labels=question_labels,
+ added_tokens={},
+ ),
+ question_ids,
+ )
+
+ def test_request_kwargs_override_the_server_defaults(self):
+ handler = _handler(ScoringManager(self.tokenizer))
+ handler.default_chat_template_kwargs = {"note": "server"}
+ original_template = self.tokenizer.chat_template
+ self.addCleanup(setattr, self.tokenizer, "chat_template", original_template)
+ self.tokenizer.chat_template = "{{ messages[0]['content'] }} {{ note }}\n\n"
+ question = {"q": _question("choice", {"a": None, "b": None})}
+ for kwargs, note in (({}, "server"), ({"note": "request"}, "request")):
+ with self.subTest(kwargs):
+ request = _request("s", question, chat_template_kwargs=kwargs)
+ prompts, _ = _encoded(handler, request)
+ self.assertTrue(
+ self.tokenizer.decode(prompts[0]).endswith(f" {note}\n")
+ )
+
+ def test_label_check_skips_the_text_before_the_last_added_token(self):
+ encoded = []
+ tokenizer = SimpleNamespace(
+ encode=lambda text, **kwargs: (
+ encoded.append(text) or self.tokenizer.encode(text, **kwargs)
+ )
+ )
+ added_tokens = {i: t for t, i in self.tokenizer.get_added_vocab().items()}
+ prompt = self.tokenizer.apply_chat_template(
+ [{"role": "user", "content": "state " * 2000}],
+ tokenize=False,
+ add_generation_prompt=True,
+ enable_thinking=False,
+ )
+ prompt_ids = self.tokenizer.encode(prompt, add_special_tokens=False)
+ label_ids = _encode_labels(
+ tokenizer=tokenizer,
+ prompt=prompt,
+ prompt_ids=prompt_ids,
+ labels=["A", "B"],
+ added_tokens=added_tokens,
+ )
+ self.assertEqual(label_ids, self.tokenizer.convert_tokens_to_ids(["A", "B"]))
+ self.assertLess(max(len(text) for text in encoded), 16)
+
+ def test_label_check_falls_back_when_the_suffix_does_not_split(self):
+ # An added token that absorbs the following space tokenizes differently
+ # from the suffix on its own, so the whole prompt is checked.
+ tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
+ tokenizer.add_tokens([AddedToken("", rstrip=True)])
+ prompt = "state "
+ prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
+ added_tokens = {i: t for t, i in tokenizer.get_added_vocab().items()}
+ self.assertEqual(
+ _encode_labels(
+ tokenizer=tokenizer,
+ prompt=prompt,
+ prompt_ids=prompt_ids,
+ labels=["A"],
+ added_tokens=added_tokens,
+ ),
+ tokenizer.encode(prompt + "A", add_special_tokens=False)[-1:],
+ )
+
+ def test_labels_must_be_distinct_tokens(self):
+ with self.assertRaisesRegex(ValueError, "label 'B' is not one distinct"):
+ _encode_labels(
+ tokenizer=UnknownTokenizer(),
+ prompt="p",
+ prompt_ids=[0],
+ labels=["A", "B"],
+ added_tokens={},
+ )
+
+ async def test_questions_yield_to_other_requests(self):
+ noul = {"type": "noul", "instructions": "x"}
+ for route, handler, request in (
+ (
+ "decisions",
+ _handler(ScoringManager(self.tokenizer)),
+ _request("s", {q: _question("yes_no") for q in "abc"}),
+ ),
+ (
+ "systemone",
+ _handler(
+ ScoringManager(self.tokenizer), serving_class=SystemOneServing
+ ),
+ _systemone_request({q: noul for q in "abc"}),
+ ),
+ ):
+ with self.subTest(route):
+ events = []
+ encode = handler._encode_question
+
+ def recorded(**kwargs):
+ events.append("encode")
+ return encode(**kwargs)
+
+ async def other_request():
+ for _ in range(3):
+ events.append("other")
+ await asyncio.sleep(0)
+
+ handler._encode_question = recorded
+ other = asyncio.create_task(other_request())
+ response = await handler.handle_request(request, None)
+ await other
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(events, ["encode", "other"] * 3)
+
+ async def test_all_questions_are_scored_in_one_call(self):
+ manager = ScoringManager(self.tokenizer)
+ request = _request(
+ "The integration keeps failing and I am losing sales.",
+ {
+ "department": _question(
+ "choice", {"billing": None, "technical": None, "sales": None}
+ ),
+ "frustration": _question("score", ["Calm", "Civil", "Angry"]),
+ "is_urgent": _question("yes_no"),
+ },
+ temperature=2.0,
+ prompt_format_version=PROMPT_FORMAT_VERSION,
+ )
+ response = await _handler(manager).handle_request(request, None)
+ body = json.loads(response.body)
+ self.assertEqual(len(manager.requests), 1)
+ self.assertEqual(body["object"], "decisions")
+ self.assertEqual(body["prompt_format_version"], PROMPT_FORMAT_VERSION)
+ prompt_tokens = sum(len(ids) for ids in manager.requests[0].input_ids)
+ self.assertEqual(body["usage"]["prompt_tokens"], prompt_tokens)
+ self.assertEqual(body["usage"]["total_tokens"], prompt_tokens)
+ self.assertEqual(body["usage"]["completion_tokens"], 0)
+ cases = {
+ "department": (["billing", "technical", "sales"], ["A", "B", "C"]),
+ "frustration": (["0", "1", "2"], ["0", "1", "2"]),
+ "is_urgent": (["yes", "no"], ["yes", "no"]),
+ }
+ self.assertEqual(list(body["answers"]), list(cases))
+ for question_id, (names, tokens) in cases.items():
+ answer = body["answers"][question_id]
+ logprobs = manager.logprobs[self.tokenizer.convert_tokens_to_ids(tokens)]
+ probabilities = torch.softmax(logprobs / 2.0, dim=0)
+ self.assertEqual(list(answer["probabilities"]), names)
+ torch.testing.assert_close(
+ torch.tensor(
+ list(answer["probabilities"].values()), dtype=torch.float64
+ ),
+ probabilities,
+ )
+ # Relative only, label mass under these logits is far below 1e-7.
+ torch.testing.assert_close(
+ torch.tensor(answer["label_mass"], dtype=torch.float64),
+ logprobs.exp().sum(),
+ rtol=1e-9,
+ atol=0,
+ )
+ self.assertEqual(answer["type"], _by_id(request, question_id).type)
+ self.assertNotIn("prompt_token_ids", answer)
+ department = body["answers"]["department"]
+ self.assertEqual(
+ department["choice"],
+ max(department["probabilities"], key=department["probabilities"].get),
+ )
+ self.assertNotIn("score", department)
+ frustration = body["answers"]["frustration"]["probabilities"]
+ self.assertAlmostEqual(
+ body["answers"]["frustration"]["score"],
+ sum(int(level) * p for level, p in frustration.items()),
+ )
+ self.assertEqual(
+ set(body["answers"]["is_urgent"]),
+ {"type", "probabilities", "label_mass"},
+ )
+
+ async def test_returned_ids_are_the_scored_ids(self):
+ manager = ScoringManager(self.tokenizer)
+ request = _request(
+ "s",
+ {
+ "team": _question("choice", {"a": None, "b": None}),
+ "urgent": _question("yes_no"),
+ },
+ return_prompt_token_ids=True,
+ )
+ response = await _handler(manager).handle_request(request, None)
+ answers = json.loads(response.body)["answers"]
+ scored = manager.requests[0]
+ self.assertEqual(
+ [answers[q]["prompt_token_ids"] for q in ("team", "urgent")],
+ scored.input_ids,
+ )
+ self.assertEqual(
+ [answers[q]["label_token_ids"] for q in ("team", "urgent")],
+ scored.token_ids_logprob,
+ )
+
+ async def test_refusals_name_the_question_and_skip_scoring(self):
+ request = _request("s", {"first": _question("choice", {"a": None, "b": None})})
+ prompt_len = len(
+ _encoded(_handler(ScoringManager(self.tokenizer)), request)[0][0]
+ )
+ at_limit = ScoringManager(self.tokenizer, context_len=prompt_len + 3)
+ at_limit.num_reserved_tokens = 3
+ # The label merges with the trailing space, or follows a split double space.
+ trailing_space = "{{ messages[0]['content'] }}\nAnswer: "
+ double_space = "{{ messages[0]['content'] }}\nAnswer: "
+ cases = {
+ "generation model": (
+ ScoringManager(self.tokenizer, is_generation=False),
+ None,
+ ),
+ "context length": (at_limit, None),
+ "label 'A' is not one distinct token": (
+ ScoringManager(self.tokenizer),
+ trailing_space,
+ ),
+ "label 'A' is not one distinct token after": (
+ ScoringManager(self.tokenizer),
+ double_space,
+ ),
+ }
+ original_template = self.tokenizer.chat_template
+ self.addCleanup(setattr, self.tokenizer, "chat_template", original_template)
+ for message, (manager, template) in cases.items():
+ with self.subTest(message):
+ self.tokenizer.chat_template = template or original_template
+ response = await _handler(manager).handle_request(request, None)
+ self.assertEqual(response.status_code, 400)
+ error = json.loads(response.body)["message"]
+ self.assertIn(message, error)
+ if message != "generation model":
+ self.assertIn("question 'first'", error)
+ self.assertEqual(manager.requests, [])
+ self.tokenizer.chat_template = original_template
+ below_limit = ScoringManager(self.tokenizer, context_len=prompt_len + 4)
+ below_limit.num_reserved_tokens = 3
+ response = await _handler(below_limit).handle_request(request, None)
+ self.assertEqual(response.status_code, 200)
+
+ async def test_refusals_keep_the_answer_outside_reasoning(self):
+ question = {"first": _question("yes_no")}
+ original_template = self.tokenizer.chat_template
+ self.addCleanup(setattr, self.tokenizer, "chat_template", original_template)
+ # Reasoning that the detected toggle does not control.
+ open_block = "{{ messages[0]['content'] }}\nassistant\n\n"
+ cases = {
+ "always reason": (None, ReasoningToggleConfig(special_case="always"), {}),
+ "sets 'enable_thinking' to True": (None, None, {"enable_thinking": True}),
+ "sets 'enable_thinking' to None": (None, None, {"enable_thinking": None}),
+ "sets 'enable_thinking' to 0": (None, None, {"enable_thinking": 0}),
+ "leaves a reasoning block open": (open_block, None, {}),
+ }
+ for message, (template, config, kwargs) in cases.items():
+ with self.subTest(message):
+ manager = ScoringManager(self.tokenizer)
+ handler = _handler(manager, config)
+ self.tokenizer.chat_template = template or original_template
+ request = _request("s", question, chat_template_kwargs=kwargs)
+ response = await handler.handle_request(request, None)
+ self.tokenizer.chat_template = original_template
+ self.assertEqual(response.status_code, 400)
+ self.assertIn(message, json.loads(response.body)["message"])
+ self.assertEqual(manager.requests, [])
+ # A parser whose answers start inside reasoning needs a closed block, and
+ # its advice comes first when the template's own replies also open one.
+ no_block = (
+ "{% for m in messages %}{% if m['role'] == 'user' %}"
+ "{{ m['content'] }}\nassistant:\n\n{% else %}"
+ "{{ m['content'] }}{% endif %}{% endfor %}"
+ )
+ closed_block = (
+ "{{ messages[0]['content'] }}\nassistant:\n\n\n\n\n"
+ )
+ for template, status in ((no_block, 400), (closed_block, 200)):
+ with self.subTest(template=template[-24:]):
+ manager = ScoringManager(self.tokenizer)
+ handler = _handler(manager, reasoning_parser="deepseek-r1")
+ self.tokenizer.chat_template = template
+ request = _request(
+ "s", {"first": _question("choice", {"a": None, "b": None})}
+ )
+ response = await handler.handle_request(request, None)
+ self.tokenizer.chat_template = original_template
+ self.assertEqual(response.status_code, status)
+ if status == 400:
+ self.assertIn(
+ "expects answers to start with a reasoning block",
+ json.loads(response.body)["message"],
+ )
+ # A toggle that only the parser names, in its plain or explicit form, is
+ # turned off and checked like a detected one.
+ choice = {"first": _question("choice", {"a": None, "b": None})}
+ for parser, toggle in (
+ ("qwen3", "enable_thinking"),
+ ("deepseek-v3", "thinking"),
+ ):
+ with self.subTest(parser=parser):
+ self.tokenizer.chat_template = (
+ f"{{% if {toggle} is defined and {toggle} %}}/think\n{{% endif %}}"
+ "{{ messages[0]['content'] }}\nassistant:\n\n"
+ )
+ handler = _handler(
+ ScoringManager(self.tokenizer), reasoning_parser=parser
+ )
+ handler.default_chat_template_kwargs = {toggle: True}
+ prompts, _ = _encoded(handler, _request("s", choice))
+ thinking = _request("s", choice, chat_template_kwargs={toggle: True})
+ response = await handler.handle_request(thinking, None)
+ self.tokenizer.chat_template = original_template
+ self.assertNotIn("/think", self.tokenizer.decode(prompts[0]))
+ self.assertEqual(response.status_code, 400)
+ self.assertIn(
+ f"sets '{toggle}' to True", json.loads(response.body)["message"]
+ )
+ # A template that opens reasoning before every answer is refused.
+ for reply, status in (("", 400), ("", 200)):
+ with self.subTest(reply=reply):
+ self.tokenizer.chat_template = (
+ "{% for m in messages %}{% if m['role'] == 'user' %}"
+ "{{ m['content'] }}\nassistant:\n\n{% else %}"
+ + reply
+ + "{{ m['content'] }}\n{% endif %}{% endfor %}"
+ )
+ manager = ScoringManager(self.tokenizer)
+ handler = _handler(manager, reasoning_parser="qwen3")
+ response = await handler.handle_request(_request("s", choice), None)
+ self.tokenizer.chat_template = original_template
+ self.assertEqual(response.status_code, status)
+ if status == 400:
+ self.assertIn(
+ "starts every answer with a reasoning block",
+ json.loads(response.body)["message"],
+ )
+ self.assertEqual(manager.requests, [])
+ # A reasoning tag inside the input does not count, with or without a
+ # reasoning block after the message.
+ request = _request(
+ " draft", {"first": _question("choice", {"a": None, "b": None})}
+ )
+ no_block = "{{ messages[0]['content'] }}\nassistant:\n\n"
+ for template in (original_template, no_block):
+ with self.subTest(template=template[-20:]):
+ handler = _handler(ScoringManager(self.tokenizer))
+ self.tokenizer.chat_template = template
+ response = await handler.handle_request(request, None)
+ self.tokenizer.chat_template = original_template
+ self.assertEqual(response.status_code, 200)
+
+ async def test_refusals_for_unsupported_serving_setups(self):
+ def named_template():
+ handler = _handler(ScoringManager(self.tokenizer))
+ handler.template_manager.chat_template_name = "chatml"
+ return handler
+
+ # Each handler is built in its own case, since ScoringManager publishes
+ # its server args globally.
+ cases = {
+ "model names the LoRA adapter 'adapter'": (
+ lambda: _handler(ScoringManager(self.tokenizer)),
+ {"model": "base:adapter"},
+ ),
+ # The version is checked before the reasoning settings.
+ "uses version 1": (
+ lambda: _handler(ScoringManager(self.tokenizer)),
+ {
+ "prompt_format_version": 2,
+ "chat_template_kwargs": {"enable_thinking": True},
+ },
+ ),
+ "'dsv4' encoder": (
+ lambda: _handler(
+ ScoringManager(self.tokenizer, architecture="DeepseekV4ForCausalLM")
+ ),
+ {},
+ ),
+ "built-in chat template 'chatml'": (named_template, {}),
+ "does not encode back to the same ids": (
+ lambda: _handler(ScoringManager(self.tokenizer), lossy=True),
+ {},
+ ),
+ "--enable-mis": (
+ lambda: _handler(ScoringManager(self.tokenizer, enable_mis=True)),
+ {},
+ ),
+ "--dllm-algorithm": (
+ lambda: _handler(
+ ScoringManager(self.tokenizer, dllm_algorithm="LowConfidence")
+ ),
+ {},
+ ),
+ }
+ for message, (build, kwargs) in cases.items():
+ with self.subTest(message):
+ handler = build()
+ request = _request("s", {"q": _question("yes_no")}, **kwargs)
+ response = await handler.handle_request(request, None)
+ self.assertEqual(response.status_code, 400)
+ self.assertIn(message, json.loads(response.body)["message"])
+ self.assertEqual(handler.tokenizer_manager.requests, [])
+
+ async def test_tokenizers_without_added_tokens_use_the_full_prompt(self):
+ request = _request("s", {"q": _question("choice", {"a": None, "b": None})})
+ plain = _handler(ScoringManager(PlainTokenizer(self.tokenizer)))
+ self.assertEqual(plain.added_tokens, {})
+ _, plain_ids = _encoded(plain, request)
+ _, ids = _encoded(_handler(ScoringManager(self.tokenizer)), request)
+ self.assertEqual(plain_ids, ids)
+
+ def test_label_after_a_final_added_token_starts_a_new_segment(self):
+ added_tokens = {i: t for t, i in self.tokenizer.get_added_vocab().items()}
+ prompt = "state"
+ prompt_ids = self.tokenizer.encode(prompt, add_special_tokens=False)
+ self.assertIn(prompt_ids[-1], added_tokens)
+ self.assertEqual(
+ _encode_labels(
+ tokenizer=self.tokenizer,
+ prompt=prompt,
+ prompt_ids=prompt_ids,
+ labels=["A"],
+ added_tokens=added_tokens,
+ ),
+ self.tokenizer.encode("A", add_special_tokens=False),
+ )
+
+
+def _systemone_request(questions, **kwargs):
+ return SystemOneRequest(
+ state="The integration keeps failing.",
+ model="jev-latest",
+ questions=questions,
+ **kwargs,
+ )
+
+
+class TestSystemOne(unittest.IsolatedAsyncioTestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
+
+ def setUp(self):
+ self.addCleanup(restore_context, snapshot_context())
+
+ def _serving(self, manager=None, **kwargs):
+ return _handler(
+ manager or ScoringManager(self.tokenizer),
+ serving_class=SystemOneServing,
+ **kwargs,
+ )
+
+ async def test_answers_follow_the_published_shapes(self):
+ manager = ScoringManager(self.tokenizer)
+ legend_level = {"label": "Angry", "examples": ["caps"]}
+ request = _systemone_request(
+ {
+ "urgent": {"type": "noul", "instructions": "Needs an answer today"},
+ "team": {
+ "type": "choice",
+ "criteria": {"billing": None, "technical": "Bugs", "sales": None},
+ },
+ "mood": {"type": "score", "criteria": ["Calm", "Civil", legend_level]},
+ }
+ )
+ response = await self._serving(manager).handle_request(request, None)
+ self.assertEqual(response.status_code, 200)
+ body = json.loads(response.body)
+ self.assertEqual(set(body), {"model", "answers", "usage"})
+ # The served model answers, never the requested alias.
+ self.assertEqual(body["model"], "served-model")
+ self.assertEqual(list(body["answers"]), ["urgent", "team", "mood"])
+ prompt_tokens = sum(len(ids) for ids in manager.requests[0].input_ids)
+ self.assertEqual(
+ body["usage"], {"input_tokens": prompt_tokens, "output_tokens": 0}
+ )
+
+ urgent = body["answers"]["urgent"]
+ self.assertEqual(set(urgent), {"type", "noul", "x_label_mass"})
+ yes_no = manager.logprobs[self.tokenizer.convert_tokens_to_ids(["yes", "no"])]
+ torch.testing.assert_close(
+ torch.tensor(urgent["noul"], dtype=torch.float64),
+ torch.softmax(yes_no, dim=0)[0],
+ )
+
+ team = body["answers"]["team"]
+ self.assertEqual(
+ set(team), {"type", "choice", "confidence", "probabilities", "x_label_mass"}
+ )
+ self.assertEqual(list(team["probabilities"]), ["billing", "technical", "sales"])
+ self.assertEqual(
+ team["choice"], max(team["probabilities"], key=team["probabilities"].get)
+ )
+ self.assertAlmostEqual(
+ team["confidence"], _choice_confidence(list(team["probabilities"].values()))
+ )
+
+ mood = body["answers"]["mood"]
+ self.assertEqual(
+ set(mood),
+ {"type", "score", "confidence", "legend", "probabilities", "x_label_mass"},
+ )
+ self.assertEqual(mood["legend"], {"0": "Calm", "1": "Civil", "2": legend_level})
+ self.assertEqual(list(mood["probabilities"]), ["0", "1", "2"])
+ self.assertAlmostEqual(
+ mood["score"], sum(int(k) * p for k, p in mood["probabilities"].items())
+ )
+
+ async def test_answers_equal_v1_decisions_bit_for_bit(self):
+ legend_level = {"label": "Angry"}
+ systemone = _systemone_request(
+ {
+ "urgent": {"type": "noul", "instructions": "Needs an answer today"},
+ "team": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {"billing": None, "technical": "Bugs", "sales": None},
+ },
+ "mood": {
+ "type": "score",
+ "instructions": "Mood?",
+ "criteria": ["Calm", "Civil", legend_level],
+ },
+ }
+ )
+ decisions = DecisionRequest(
+ input=systemone.state,
+ questions=[
+ {"id": "urgent", "type": "yes_no", "question": "Needs an answer today"},
+ _question(
+ "choice",
+ {"billing": None, "technical": "Bugs", "sales": None},
+ "Which team?",
+ )
+ | {"id": "team"},
+ _question("score", ["Calm", "Civil", legend_level], "Mood?")
+ | {"id": "mood"},
+ ],
+ )
+ managers = [ScoringManager(self.tokenizer) for _ in range(2)]
+ bodies = []
+ for manager, handler, request in (
+ (managers[0], self._serving(managers[0]), systemone),
+ (managers[1], _handler(managers[1]), decisions),
+ ):
+ response = await handler.handle_request(request, None)
+ self.assertEqual(response.status_code, 200)
+ bodies.append(json.loads(response.body)["answers"])
+ self.assertEqual(
+ managers[0].requests[0].input_ids, managers[1].requests[0].input_ids
+ )
+ self.assertEqual(
+ managers[0].requests[0].token_ids_logprob,
+ managers[1].requests[0].token_ids_logprob,
+ )
+ systemone_answers, decision_answers = bodies
+ for question_id, answer in systemone_answers.items():
+ with self.subTest(question_id):
+ expected = decision_answers[question_id]
+ self.assertEqual(answer["x_label_mass"], expected["label_mass"])
+ if question_id == "urgent":
+ self.assertEqual(answer["noul"], expected["probabilities"]["yes"])
+ continue
+ self.assertEqual(answer["probabilities"], expected["probabilities"])
+ key = "choice" if question_id == "team" else "score"
+ self.assertEqual(answer[key], expected[key])
+
+ def test_answers_report_probabilities_as_scored(self):
+ # A softmax can miss a sum of 1 by a few ulp. Only confidence normalizes.
+ questions = _systemone_request(
+ {
+ "urgent": {"type": "noul", "instructions": "x"},
+ "team": {
+ "type": "choice",
+ "criteria": {"a": None, "b": None, "c": None},
+ },
+ "mood": {"type": "score", "criteria": ["low", "mid", "high"]},
+ }
+ ).questions
+ answers = {
+ question_id: _answer(
+ view=_view(question),
+ probabilities=[0.3] * (2 if question_id == "urgent" else 3),
+ mass=0.9,
+ question_id=question_id,
+ )
+ for question_id, question in questions.items()
+ }
+ self.assertEqual(answers["urgent"].noul, 0.3)
+ for question_id in ("team", "mood"):
+ self.assertEqual(
+ list(answers[question_id].probabilities.values()), [0.3] * 3
+ )
+ self.assertEqual(answers["mood"].score, math.fsum([0, 0.3, 0.6]))
+ self.assertAlmostEqual(answers["team"].confidence, 0.0)
+ self.assertAlmostEqual(
+ answers["mood"].confidence, _score_confidence([1 / 3] * 3)
+ )
+
+ def test_confidence_matches_the_published_formulas(self):
+ # Documented examples, a uniform choice, a split score, and one candidate.
+ self.assertAlmostEqual(_score_confidence([0, 0.14, 0.86, 0, 0]), 1 - 0.14 / 1.2)
+ self.assertAlmostEqual(_score_confidence([0, 0, 0.48, 0.52]), 0.52)
+ self.assertEqual(_score_confidence([0.5, 0, 0, 0, 0.5]), 0.0)
+ self.assertAlmostEqual(_choice_confidence([0.25] * 4), 0.0)
+ self.assertAlmostEqual(_choice_confidence([0.1, 0.9]), 0.8)
+ self.assertEqual(_choice_confidence([1.0]), 1.0)
+ self.assertEqual(_score_confidence([1.0]), 1.0)
+
+ async def test_options_beyond_26_get_two_letter_labels(self):
+ manager = ScoringManager(self.tokenizer)
+ criteria = {f"option {i}": None for i in range(30)}
+ request = _systemone_request(
+ {"q": {"type": "choice", "instructions": "Pick one", "criteria": criteria}}
+ )
+ encoded, _ = self._serving(manager)._convert_to_internal_request(request)
+ prompts, label_ids = zip(*encoded)
+ text = self.tokenizer.decode(prompts[0])
+ self.assertIn("AA: option 0", text)
+ self.assertNotIn("\nA: option 0", text)
+ self.assertEqual(len(set(label_ids[0])), 30)
+ # A tokenizer that reports no added tokens stays at 26 options.
+ plain = self._serving(ScoringManager(PlainTokenizer(self.tokenizer)))
+ response = await plain.handle_request(request, None)
+ self.assertEqual(response.status_code, 400)
+ self.assertIn(
+ "more than 26 options needs added tokens the server can read",
+ json.loads(response.body)["message"],
+ )
+ # So does a template whose only added token comes before the message.
+ original_template = self.tokenizer.chat_template
+ self.addCleanup(setattr, self.tokenizer, "chat_template", original_template)
+ self.tokenizer.chat_template = (
+ "<|im_start|>{{ messages[0]['content'] }}\nassistant\n\n"
+ )
+ leading = self._serving()
+ response = await leading.handle_request(request, None)
+ self.assertEqual(response.status_code, 400)
+ self.assertIn(
+ "more than 26 options needs an added token between the message",
+ json.loads(response.body)["message"],
+ )
+ self.assertEqual(leading.tokenizer_manager.requests, [])
+ few = _systemone_request(
+ {"q": {"type": "choice", "criteria": {"a": None, "b": None}}}
+ )
+ response = await leading.handle_request(few, None)
+ self.assertEqual(response.status_code, 200)
+
+ def test_request_validation(self):
+ levels = [str(i) for i in range(11)]
+ refused = {
+ "noul that asks nothing": {"q": {"type": "noul"}},
+ "noul with blank criteria": {
+ "q": {"type": "noul", "instructions": None, "criteria": {"true": " "}}
+ },
+ "unknown key in a question": {
+ "q": {"type": "choice", "instruction": "x", "criteria": {"a": None}}
+ },
+ "unknown key in noul criteria": {
+ "q": {"type": "noul", "criteria": {"yes": "x"}}
+ },
+ "no options": {"q": {"type": "choice", "criteria": {}}},
+ "256 options": {
+ "q": {"type": "choice", "criteria": {str(i): None for i in range(256)}}
+ },
+ "11 levels": {"q": {"type": "score", "criteria": levels}},
+ "null level": {"q": {"type": "score", "criteria": ["low", None]}},
+ "blank level": {"q": {"type": "score", "criteria": ["low", " "]}},
+ "repeated option name": {
+ "q": {"type": "choice", "criteria": {"a": None, "A ": None}}
+ },
+ "unknown type": {"q": {"type": "yes_no", "instructions": "x"}},
+ "no questions": {},
+ }
+ for case, questions in refused.items():
+ with self.subTest(case), self.assertRaises(ValidationError):
+ _systemone_request(questions)
+ for field, value in (
+ ("temperature", 0.5),
+ ("temperature", 1),
+ ("prompt_format_version", 1),
+ ("return_prompt_token_ids", True),
+ ("return_prompt_token_ids", False),
+ ):
+ with (
+ self.subTest(field=field, value=value),
+ self.assertRaisesRegex(ValidationError, "/v1/decisions") as caught,
+ ):
+ _systemone_request(
+ {"q": {"type": "noul", "instructions": "x"}}, **{field: value}
+ )
+ self.assertEqual(caught.exception.errors()[0]["loc"], (field,))
+ self.assertNotIn(
+ "temperature", SystemOneRequest.model_json_schema()["properties"]
+ )
+ accepted = SystemOneRequest(
+ state="",
+ model="jev-latest",
+ questions={
+ "": {"type": "choice", "criteria": {"only": None}},
+ " ": {"type": "score", "criteria": ["one"]},
+ },
+ temperature=None,
+ future_field=True,
+ )
+ self.assertEqual(list(accepted.questions), ["", " "])
+
+ async def test_validation_errors_are_422_on_this_route_only(self):
+ from fastapi.exceptions import RequestValidationError
+
+ from sglang.srt.entrypoints.http_server import validation_exception_handler
+
+ # The input echo is left out, since JSON encoding can fail on it.
+ nested = []
+ for _ in range(300):
+ nested = [nested]
+ error = RequestValidationError(
+ errors=[
+ {
+ "loc": ("body", "questions"),
+ "msg": "Field required",
+ "type": "missing",
+ "input": {"state": nested},
+ }
+ ]
+ )
+
+ def request(root_path, path):
+ # Behind --fastapi-root-path the URL path starts with the root path.
+ return SimpleNamespace(
+ url=SimpleNamespace(path=root_path + path),
+ scope={"root_path": root_path},
+ )
+
+ for root_path in ["", "/prefix"]:
+ with self.subTest(root_path=root_path):
+ systemone = await validation_exception_handler(
+ request(root_path, "/v1/systemone"), error
+ )
+ self.assertEqual(systemone.status_code, 422)
+ self.assertEqual(
+ json.loads(systemone.body)["detail"],
+ [
+ {
+ "loc": ["body", "questions"],
+ "msg": "Field required",
+ "type": "missing",
+ }
+ ],
+ )
+ decisions = await validation_exception_handler(
+ request(root_path, "/v1/decisions"), error
+ )
+ self.assertEqual(decisions.status_code, 400)
+
+ async def test_levels_the_legend_cannot_return_are_refused_before_scoring(self):
+ def nested(depth):
+ level = "Calm"
+ for _ in range(depth):
+ level = [level]
+ return level
+
+ for case, level, status in (
+ ("nested 200 deep", nested(200), 200),
+ ("nested 300 deep", nested(300), 400),
+ ):
+ with self.subTest(case):
+ manager = ScoringManager(self.tokenizer)
+ request = _systemone_request(
+ {"mood": {"type": "score", "criteria": ["Fine", level]}}
+ )
+ response = await self._serving(manager).handle_request(request, None)
+ self.assertEqual(response.status_code, status)
+ if status == 400:
+ self.assertIn(
+ "question 'mood': a level cannot be returned in the legend",
+ json.loads(response.body)["message"],
+ )
+ self.assertEqual(manager.requests, [])
+
+ async def test_reasoning_refusals_apply(self):
+ request = _systemone_request(
+ {"q": {"type": "noul", "instructions": "x"}},
+ chat_template_kwargs={"enable_thinking": True},
+ )
+ plain = _systemone_request({"q": {"type": "noul", "instructions": "x"}})
+ always = ReasoningToggleConfig(special_case="always")
+ for message, serving, body in (
+ ("sets 'enable_thinking' to True", self._serving(), request),
+ ("always reason", self._serving(reasoning_config=always), plain),
+ ):
+ with self.subTest(message):
+ response = await serving.handle_request(body, None)
+ self.assertEqual(response.status_code, 400)
+ self.assertIn(message, json.loads(response.body)["message"])
+ self.assertEqual(serving.tokenizer_manager.requests, [])
+ # The server checks of /v1/decisions apply too.
+ serving = self._serving(
+ ScoringManager(self.tokenizer, dllm_algorithm="LowConfidence")
+ )
+ response = await serving.handle_request(plain, None)
+ self.assertEqual(response.status_code, 400)
+ self.assertIn(
+ "/v1/systemone does not support diffusion",
+ json.loads(response.body)["message"],
+ )
+
+ async def test_non_finite_scores_are_a_server_error(self):
+ manager = ScoringManager(self.tokenizer)
+ manager.logprobs[self.tokenizer.convert_tokens_to_ids(["yes", "no"])] = float(
+ "nan"
+ )
+ request = _systemone_request({"q": {"type": "noul", "instructions": "x"}})
+ response = await self._serving(manager).handle_request(request, None)
+ self.assertEqual(response.status_code, 500)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/layers/test_logprob_chunk_stitching.py b/test/registered/unit/layers/test_logprob_chunk_stitching.py
index 51e5cafd98..c4bdaa7b8d 100644
--- a/test/registered/unit/layers/test_logprob_chunk_stitching.py
+++ b/test/registered/unit/layers/test_logprob_chunk_stitching.py
@@ -3,7 +3,8 @@
Regression for the cross-chunk stitching accounting: zero-logprob-row
sequences (logprob opt-outs in mixed batches, mid-chunked-prefill segments)
were skipped or double-emitted, drifting the per-request entry counts that
-the scheduler asserts on.
+the scheduler asserts on. Next-token token-ids logprobs of mixed batches
+must also copy to the CPU like the eager path.
"""
import unittest
@@ -13,7 +14,13 @@ from unittest.mock import Mock
import torch
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
-from sglang.srt.layers.logprob_processor import InputLogprobProcessor
+from sglang.srt.layers.logprob_processor import (
+ InputLogprobProcessor,
+ get_token_ids_logprobs,
+)
+from sglang.srt.managers.scheduler_components.batch_result_processor import (
+ SchedulerBatchResultProcessor,
+)
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.logprob_test_utils import coverage_cases
from sglang.test.test_utils import CustomTestCase
@@ -158,6 +165,34 @@ class TestLogprobChunkStitching(CustomTestCase):
output.finalize_input_logprobs()
self.assertIs(output.input_token_ids_logprobs_val, rows)
+ def test_mixed_batch_token_ids_copy_to_cpu_like_the_eager_path(self):
+ # One request with token ids, one without, and one with an empty probe set.
+ token_ids = [[1, 3], None, []]
+ logprobs = torch.log_softmax(torch.randn(len(token_ids), 5), dim=-1)
+ vals, idxs = get_token_ids_logprobs(logprobs, token_ids, no_copy_to_cpu=True)
+ output = LogitsProcessorOutput(
+ next_token_logits=None,
+ next_token_token_ids_logprobs_val=vals,
+ next_token_token_ids_logprobs_idx=idxs,
+ )
+ SchedulerBatchResultProcessor.move_logprobs_to_cpu(
+ None,
+ batch=SimpleNamespace(return_logprob=True),
+ logits_output=output,
+ )
+ # The empty entry must not be a view that keeps the full logprobs alive.
+ self.assertIsNone(vals[1]._base)
+ expected = get_token_ids_logprobs(logprobs, token_ids, no_copy_to_cpu=False)
+ self.assertEqual(
+ (
+ output.next_token_token_ids_logprobs_val,
+ output.next_token_token_ids_logprobs_idx,
+ ),
+ expected,
+ )
+ self.assertEqual(expected[0][1:], [[], []])
+ self.assertEqual(expected[1], [[1, 3], [], []])
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/registered/unit/managers/test_flat_raw_top_logprobs.py b/test/registered/unit/managers/test_flat_raw_top_logprobs.py
index f7e4e09e64..646ccb9d28 100644
--- a/test/registered/unit/managers/test_flat_raw_top_logprobs.py
+++ b/test/registered/unit/managers/test_flat_raw_top_logprobs.py
@@ -569,6 +569,41 @@ class TestTokenizerManagerLogprobs(CustomTestCase):
self.assertEqual(meta_info["output_token_logprobs"], [(-0.25, 42, None)])
self.assertEqual(meta_info["output_token_logprobs_length"], 1)
+ def test_top_and_token_ids_logprobs_without_logprob_lists(self):
+ """An output without logprob lists, as sent for a request aborted before
+ prefill, must not crash a request that asked for top or token id logprobs."""
+ state = _make_state(
+ return_logprob=True, top_logprobs_num=2, token_ids_logprob=[42]
+ )
+ recv_obj = SimpleNamespace(
+ input_token_logprobs_val=None,
+ output_token_logprobs_val=None,
+ input_top_logprobs_val=None,
+ input_top_logprobs_val_flat=None,
+ output_top_logprobs_val=None,
+ input_token_ids_logprobs_val=None,
+ output_token_ids_logprobs_val=None,
+ )
+ meta_info = {}
+
+ _TokenizerManagerStub().convert_logprob_style(
+ meta_info,
+ state,
+ top_logprobs_num=2,
+ token_ids_logprob=[42],
+ return_text_in_logprobs=False,
+ recv_obj=recv_obj,
+ recv_obj_index=0,
+ )
+
+ for key in (
+ "input_top_logprobs",
+ "output_top_logprobs",
+ "input_token_ids_logprobs",
+ "output_token_ids_logprobs",
+ ):
+ self.assertEqual(meta_info[key], [])
+
def _make_batch_token_id_output(**overrides) -> BatchTokenIDOutput:
"""A two-request BatchTokenIDOutput with the required fields stubbed."""