[model] support Clef and Clef-Flash decision models on /v1/systemone (#42721)

Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com>
This commit is contained in:
areneau-cf
2026-10-09 14:25:52 +08:00
committed by GitHub
co-authored by Jimmy Shong
parent 51f7ea84c7
commit bd2d73daa5
20 changed files with 1763 additions and 2 deletions
@@ -977,6 +977,37 @@ def handle_model_capability_adjustments(server_args: Any):
"prefill; using breakable CUDA graph for CUDA prefill."
)
# A Clef checkpoint's joint schema head reads the final hidden state of
# every prompt token in one pass, so each request is one complete prefill
# with no decode: embedding mode, without prefix reuse or chunking. This
# also puts the FA backend on its raw K/V path, which skips the KV pool.
if model_config.joint_head_config is not None:
if cfg.tp_size != 1 or cfg.pp_size != 1:
raise ValueError(
"Clef checkpoints are served with --tp-size 1 and --pp-size 1, "
"since the joint schema head reads full LM head rows"
)
for key, value in (
("is_embedding", True),
("disable_radix_cache", True),
("chunked_prefill_size", -1),
):
declare_resolution(
server_args, "_handle_model_capability_adjustments", **{key: value}
)
for phase in (Phase.DECODE, Phase.PREFILL):
declare_resolution(
server_args,
"_handle_model_capability_adjustments",
cuda_graph_config=with_phase(
cfg.cuda_graph_config, phase, backend=Backend.DISABLED
),
)
logger.info(
"Clef joint schema head detected: serving /v1/systemone decisions in "
"embedding mode without radix cache, chunked prefill, or CUDA graphs."
)
if (
model_config.is_multimodal
and not model_config.is_multimodal_chunked_prefill_supported
+20
View File
@@ -291,6 +291,23 @@ def load_decision_config(model_path: str, revision: Optional[str]) -> Optional[d
return config
@lru_cache
def load_joint_head_config(model_path: str, revision: Optional[str]) -> Optional[dict]:
"""joint_head_config.json of a Clef checkpoint, whose joint schema head
answers /v1/systemone decisions from the backbone's hidden states."""
path = cached_file(
model_path,
"joint_head_config.json",
revision=revision,
_raise_exceptions_for_missing_entries=False,
_raise_exceptions_for_connection_errors=False,
)
if path is None:
return None
with open(path) as f:
return json.load(f)
def is_qwen3_5(config) -> bool:
return _hf_arch(config) in (
"Qwen3_5ForConditionalGeneration",
@@ -544,6 +561,9 @@ class ModelConfig:
):
# Full attention sees the whole prompt, while linear attention stays causal.
self.hf_text_config.is_causal = False
self.joint_head_config = None
if self.hf_config.architectures == ["Qwen3_5ForConditionalGeneration"]:
self.joint_head_config = load_joint_head_config(self.model_path, revision)
self.requires_mm_token_modalities = requires_mm_token_modalities(
self.hf_config.architectures, self.hf_text_config
)
@@ -66,6 +66,7 @@ from fastapi.routing import APIRoute
from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.configs.embedding_model_spec import resolved_embedding_plan
from sglang.srt.configs.model_config import load_joint_head_config
from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX
from sglang.srt.disaggregation.utils import FAKE_BOOTSTRAP_HOST, DisaggregationMode
from sglang.srt.entrypoints.anthropic.protocol import (
@@ -972,6 +973,7 @@ async def serve_generate_request(obj: GenerateReqInput, request: Request):
async def encode_request(obj: EmbeddingReqInput, request: Request):
"""Handle an embedding request."""
try:
_refuse_decision_layout(obj)
ret = await _global_state.tokenizer_manager.generate_request(
obj, request
).__anext__()
@@ -984,6 +986,7 @@ async def encode_request(obj: EmbeddingReqInput, request: Request):
async def classify_request(obj: EmbeddingReqInput, request: Request):
"""Handle a reward model request. Now the arguments and return values are the same as embedding models."""
try:
_refuse_decision_layout(obj)
ret = await _global_state.tokenizer_manager.generate_request(
obj, request
).__anext__()
@@ -992,6 +995,12 @@ async def classify_request(obj: EmbeddingReqInput, request: Request):
return _create_error_response(e)
def _refuse_decision_layout(obj: EmbeddingReqInput) -> None:
# Only /v1/systemone builds a decision layout, from the prompt it compiles.
if obj.decision_layout is not None:
raise ValueError("decision_layout is set only by /v1/systemone")
@app.api_route("/flush_cache", methods=["GET", "POST"])
@auth_level(AuthLevel.ADMIN_OPTIONAL)
async def flush_cache(timeout: float = Query(0.0, ge=0.0)):
@@ -2379,6 +2388,22 @@ def _execute_server_warmup(server_args: ServerArgs):
).tolist()
json_data["sampling_params"]["max_new_tokens"] = 0
if (
not model_info["is_generation"]
and model_info.get("architectures") == ["Qwen3_5ForConditionalGeneration"]
and load_joint_head_config(get_model().model_path, get_model().revision)
is not None
):
# A Clef checkpoint answers /v1/systemone decisions only.
request_name = "/v1/systemone"
json_data = {
"model": "warmup",
"state": "The capital city of France is Paris.",
"questions": {
"warmup": {"type": "noul", "instructions": "Is the state true?"}
},
}
# Send a warmup request
warmup_timeout = envs.SGLANG_WARMUP_TIMEOUT.get()
try:
@@ -83,6 +83,7 @@ class OpenAIServingDecisions(OpenAIServingBase):
self.chat_encoding_spec = chat_serving.chat_encoding_spec
self.prompt_text_is_lossy = chat_serving._prompt_text_round_trip_is_lossy
self.decision_config = self.tokenizer_manager.model_config.decision_config
self.joint_head_config = self.tokenizer_manager.model_config.joint_head_config
tokenizer = self.tokenizer_manager.tokenizer
# Other tokenizers skip the shortcut in _encode_labels and check the full prompt.
self.added_tokens = (
@@ -0,0 +1,137 @@
"""The prompt that a Clef checkpoint's joint schema head reads.
Clef answers /v1/systemone questions from the backbone's hidden states instead
of from generated text. One prompt holds the state and every question with its
allowed options, and the head scores the span of each option. This follows
encode_record in the checkpoint's joint_schema_model.py, piece by piece, since
each piece is tokenized on its own.
"""
from __future__ import annotations
import json
from typing import Any, List, Sequence, Tuple
from sglang.srt.entrypoints.systemone.protocol import (
SystemOneChoiceQuestion,
SystemOneNoulQuestion,
SystemOneQuestion,
SystemOneRequest,
)
from sglang.srt.layers.joint_schema_head import (
QUESTION_TYPES,
LayoutQuestion,
pack_decision_layout,
)
_SYSTEM_PROMPT = (
"Read the complete state and schema. Decide every field jointly. Each answer "
"must be exactly one of that field's allowed options."
)
_PREFIX = f"<|im_start|>system\n{_SYSTEM_PROMPT}<|im_end|>\n<|im_start|>user\nSTATE:\n"
_IMAGE = "<|vision_start|><|image_pad|><|vision_end|>"
_SUFFIX = (
"\n<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
"JOINT SCHEMA DECISIONS:"
)
_NOUL_CRITERIA = {
"true": "The proposition is true or the answer is yes.",
"false": "The proposition is false or the answer is no.",
}
# The default max_length of encode_record and systemone in joint_schema_model.py.
REFERENCE_MAX_LENGTH = 16384
def _render(value: Any) -> str:
if isinstance(value, str):
return value
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
def joint_schema_options(question: SystemOneQuestion) -> List[Tuple[str, Any]]:
"""Each option's name and description, in the order the head scores them."""
if isinstance(question, SystemOneNoulQuestion):
criteria = dict(_NOUL_CRITERIA)
if question.criteria is not None:
# A description sent as null drops the default, as in encode_record.
criteria.update(question.criteria.model_dump(exclude_unset=True))
return [(name, criteria[name]) for name in ("true", "false")]
if isinstance(question, SystemOneChoiceQuestion):
return sorted(question.criteria.items(), key=lambda item: item[0])
return [(str(level), detail) for level, detail in enumerate(question.criteria)]
def encode_joint_schema(
tokenizer: Any,
request: SystemOneRequest,
max_length: int,
image_token_counts: Sequence[int],
) -> Tuple[List[int], List[int]]:
"""The prompt ids and their decision layout.
Each image's placeholder expands to its count in ``image_token_counts``, and
the state is truncated so that the expanded prompt has at most
``max_length`` tokens, as encode_record budgets the expanded media.
"""
def tokens(text: str) -> List[int]:
return tokenizer.encode(text, add_special_tokens=False)
schema = tokens("\n\nSCHEMA FIELDS:\n")
questions = []
for index, (question_id, question) in enumerate(request.questions.items()):
schema += tokens(
f"\nFIELD {index + 1}\nID: {question_id}\nTYPE: {question.type}\n"
"INSTRUCTION: "
)
instructions = question.instructions
if instructions is None or instructions == "":
instructions = question_id
question_start = len(schema)
schema += tokens(_render(instructions))
question_span = (question_start, len(schema))
schema += tokens("\nALLOWED OPTIONS:\n")
option_spans = []
for option_index, (name, description) in enumerate(
joint_schema_options(question)
):
schema += tokens(f"OPTION {option_index + 1}: ")
option_start = len(schema)
semantics = {"option_id": name}
if description is not None:
semantics["description"] = description
schema += tokens(_render(semantics))
option_spans.append((option_start, len(schema)))
schema += tokens("\n")
schema += tokens("END FIELD\n")
questions.append(
(QUESTION_TYPES.index(question.type), question_span, option_spans)
)
prefix = tokens(_PREFIX)
if request.images:
prefix += tokens(_IMAGE * len(request.images) + "\n")
suffix = tokens(_SUFFIX)
# Each image's single <|image_pad|> becomes that image's tokens.
expansion = sum(count - 1 for count in image_token_counts)
fixed_length = len(prefix) + expansion + len(schema) + len(suffix)
if fixed_length > max_length:
raise ValueError(
f"the questions need {fixed_length} prompt tokens before the state, "
f"but the prompt holds at most {max_length}"
)
state = tokens(_render(request.state))[: max_length - fixed_length]
offset = len(prefix) + len(state)
input_ids = prefix + state + schema + suffix
layout = pack_decision_layout(
len(input_ids),
[
LayoutQuestion(
question_type,
(question_span[0] + offset, question_span[1] + offset),
tuple((start + offset, end + offset) for start, end in option_spans),
)
for question_type, question_span, option_spans in questions
],
)
return input_ids, layout
@@ -2,14 +2,20 @@
from __future__ import annotations
import asyncio
import dataclasses
import json
import math
import string
from io import BytesIO
from typing import Any, Dict, Iterator, List, Optional, Tuple
from urllib.parse import unquote, urlparse
import orjson
import pybase64
from fastapi import Request
from fastapi.responses import ORJSONResponse
from PIL import Image
from sglang.srt.entrypoints.openai.serving_decisions import (
EncodedQuestion,
@@ -22,6 +28,11 @@ from sglang.srt.entrypoints.openai.serving_decisions import (
render_question,
render_text,
)
from sglang.srt.entrypoints.systemone.joint_schema import (
REFERENCE_MAX_LENGTH,
encode_joint_schema,
joint_schema_options,
)
from sglang.srt.entrypoints.systemone.protocol import (
SystemOneChoiceAnswer,
SystemOneChoiceQuestion,
@@ -33,6 +44,10 @@ from sglang.srt.entrypoints.systemone.protocol import (
SystemOneScoreAnswer,
SystemOneUsage,
)
from sglang.srt.layers.joint_schema_head import max_joint_prompt_tokens
from sglang.srt.managers.io_struct import EmbeddingReqInput
from sglang.srt.runtime_context import get_schedule
from sglang.srt.utils import CLIENT_MEDIA_EXCEPTIONS, ImageData, get_image_bytes
# Beyond A to Z, every option gets a two-letter label, in this fixed order.
_PAIR_LABELS = [a + b for a in string.ascii_uppercase for b in string.ascii_uppercase]
@@ -44,9 +59,16 @@ _DECIDER_SYSTEM = (
"Reply with only the selected option code."
)
# Image sources whose content can change between reads, as load_image reads them.
_MUTABLE_SOURCES = ("http://", "https://", "file://", "/")
# Base64 characters decoded to size an inline image, 48 KiB of image data.
# Arbitrary; an image whose header runs longer is decoded in full.
_INLINE_HEADER_CHARS = 1 << 16
class SystemOneServing(OpenAIServingDecisions):
"""Answers System One questions with the rendering, label checks, and scoring of /v1/decisions."""
"""Answers System One questions with the rendering, label checks, and scoring of
/v1/decisions, or, for a Clef checkpoint, with its joint schema head."""
route = "/v1/systemone"
@@ -54,12 +76,36 @@ class SystemOneServing(OpenAIServingDecisions):
return "systemone-"
def _validate_request(self, request: SystemOneRequest) -> Optional[str]:
if self.joint_head_config is not None:
return self._validate_joint_schema_request(request)
return (
self._validate_server(request.model)
or self._validate_images(request.images)
or self._validate_reasoning(request.chat_template_kwargs)
)
def _validate_joint_schema_request(
self, request: SystemOneRequest
) -> Optional[str]:
"""A Clef checkpoint renders its own prompt, without the chat template."""
route = self.route
if self.tokenizer_manager.tokenizer is None:
return f"{route} requires the server tokenizer"
_, adapter = self._parse_model_parameter(request.model)
if adapter is not None:
return (
f"model names the LoRA adapter {adapter!r}, which {route} "
"does not support"
)
if request.images and self.tokenizer_manager.mm_processor is None:
return f"{route} images require a model served with multimodal input"
if request.chat_template_kwargs:
return (
f"{route} renders the prompt this checkpoint's joint schema head "
"was trained with, which takes no chat_template_kwargs"
)
return None
def _convert_to_internal_request(
self,
request: SystemOneRequest,
@@ -69,6 +115,8 @@ class SystemOneServing(OpenAIServingDecisions):
Tuple[SystemOneRequest, List[QuestionView]],
]:
views = [_view(question) for question in request.questions.values()]
if self.joint_head_config is not None:
return self._joint_schema_request(request, views), (request, views)
encode = (
self._encoded_systemone_questions
if self.decision_config is None
@@ -77,6 +125,118 @@ class SystemOneServing(OpenAIServingDecisions):
# Lazy, so the async handler can yield to other requests between questions.
return encode(request, views), (request, views)
def _joint_schema_request(
self, request: SystemOneRequest, views: List[QuestionView]
) -> EmbeddingReqInput:
"""All questions in one prompt, which the joint schema head scores in one prefill.
The handler encodes the prompt once it has read the image sizes."""
for question_id, view in zip(request.questions, views):
if view.kind == "score":
_check_legend(question_id, view)
return EmbeddingReqInput(
image_data=[
ImageData(url=image.url, detail=image.detail or "auto")
for image in request.images
]
or None,
)
async def _encode_joint_schema_prompt(
self, adapted_request: EmbeddingReqInput, request: SystemOneRequest
) -> None:
tokenizer_manager = self.tokenizer_manager
max_length = min(
REFERENCE_MAX_LENGTH,
max_joint_prompt_tokens(
context_len=tokenizer_manager.context_len,
num_reserved_tokens=tokenizer_manager.num_reserved_tokens,
max_req_input_len=tokenizer_manager.max_req_input_len,
max_prefill_tokens=get_schedule().max_prefill_tokens,
),
)
image_token_counts = []
if adapted_request.image_data:
adapted_request.image_data, image_token_counts = await self._read_images(
adapted_request.image_data
)
input_ids, layout = encode_joint_schema(
tokenizer_manager.tokenizer,
request,
max_length=max_length,
image_token_counts=image_token_counts,
)
adapted_request.input_ids = input_ids
adapted_request.decision_layout = layout
async def _read_images(
self, images: List[ImageData]
) -> Tuple[List[ImageData], List[int]]:
"""Each image read once, and the tokens it expands to at the size the
processor resizes it to."""
read = await asyncio.gather(
*(
asyncio.to_thread(_read_image, index, image)
for index, image in enumerate(images)
)
)
counts = self.tokenizer_manager.mm_processor.resolve_image_token_counts(
[header for _, header in read]
)
return [image for image, _ in read], counts
async def _answer_joint_schema(
self,
adapted_request: EmbeddingReqInput,
request: SystemOneRequest,
views: List[QuestionView],
raw_request: Request,
) -> ORJSONResponse:
await self._encode_joint_schema_prompt(adapted_request, request)
ret = await self.tokenizer_manager.generate_request(
adapted_request, raw_request
).__anext__()
logits = ret["embedding"]
names = [
[name for name, _ in joint_schema_options(question)]
for question in request.questions.values()
]
if len(logits) != sum(map(len, names)):
raise RuntimeError("the joint schema head did not score this request")
answers = {}
offset = 0
for question_id, view, question_names in zip(request.questions, views, names):
scores = logits[offset : offset + len(question_names)]
offset += len(question_names)
top = max(scores)
weights = [math.exp(score - top) for score in scores]
total = math.fsum(weights)
by_name = {
name: weight / total for name, weight in zip(question_names, weights)
}
# The head scores true then false, which the view names yes and no.
answers[question_id] = _answer(
view=view,
probabilities=(
[by_name["true"], by_name["false"]]
if view.kind == "yes_no"
else [by_name[name] for name in view.names]
),
mass=1.0,
question_id=question_id,
)
response = SystemOneResponse(
model=self.tokenizer_manager.served_model_name,
answers=answers,
usage=SystemOneUsage(input_tokens=ret["meta_info"]["prompt_tokens"]),
)
# The head scores only the allowed options, so there is no label mass.
return ORJSONResponse(
content=response.model_dump(
exclude={"answers": {"__all__": {"x_label_mass"}}}
)
)
def _encoded_systemone_questions(
self, request: SystemOneRequest, views: List[QuestionView]
) -> Iterator[EncodedQuestion]:
@@ -179,6 +339,10 @@ class SystemOneServing(OpenAIServingDecisions):
raw_request: Request,
) -> ORJSONResponse:
request, views = processed
if self.joint_head_config is not None:
return await self._answer_joint_schema(
adapted_request, request, views, raw_request
)
_, _, result = await self._score(
adapted_request=adapted_request,
raw_request=raw_request,
@@ -344,3 +508,38 @@ def _score_confidence(q: List[float]) -> float:
spread = math.fsum(p * abs(i - top) for i, p in enumerate(q))
uniform_spread = math.fsum(abs(i - (n - 1) / 2) for i in range(n)) / n
return max(0.0, 1 - spread / uniform_spread)
def _read_image(index: int, image: ImageData) -> Tuple[ImageData, Image.Image]:
"""The image to forward, and its header opened only as far as its size, read
once from the sources load_image reads."""
url = image.url
if not url.startswith(_MUTABLE_SOURCES):
# An inline image is forwarded as sent, and the processor decodes all of it.
header = _inline_header(url)
if header is not None:
return image, header
source = unquote(urlparse(url).path) if url.startswith("file://") else url
try:
data = get_image_bytes(source)
header = Image.open(BytesIO(data))
except CLIENT_MEDIA_EXCEPTIONS as e:
raise ValueError(f"image {index} could not be read: {e}") from e
if url.startswith(_MUTABLE_SOURCES):
# A source that can change between reads is forwarded as the bytes counted.
mime = Image.MIME.get(header.format, "application/octet-stream")
encoded = pybase64.b64encode(data).decode()
image = dataclasses.replace(image, url=f"data:{mime};base64,{encoded}")
return image, header
def _inline_header(url: str) -> Optional[Image.Image]:
"""An inline image's header from a prefix of its base64, or None when the
header does not open from that prefix."""
start = url.find(",") + 1 if url.startswith("data:") else 0
prefix = url[start : start + _INLINE_HEADER_CHARS]
try:
return Image.open(BytesIO(pybase64.b64decode(prefix, validate=True)))
except Exception:
# A header past the prefix, or malformed data that the full read reports.
return None
@@ -0,0 +1,539 @@
"""Joint schema head of Clef decision checkpoints.
A Clef checkpoint is a Qwen3.5 backbone saved with ``joint_head_config.json``
and ``joint_head.safetensors``. The head reads the backbone's final hidden
state at every prompt token and scores every allowed option of every question
together: each option gathers evidence from the prompt, the questions attend
to each other and to the prompt, and a lexical prior built from the LM head
rows of each option's tokens keeps the option's meaning.
One head pass serves a whole prefill batch. Token-wise steps run on the flat
batch, attention is variable-length per request, and span means come from
per-request prefix sums over the schema region only, so a long state costs no
more than its memory projection.
"""
from __future__ import annotations
import math
from typing import Callable, List, Optional, Sequence, Tuple
import msgspec
import torch
import torch.nn.functional as F
from torch import nn
from sglang.srt.layers.pooler import EmbeddingPoolerOutput
# Question types in the order of the head's type embedding.
QUESTION_TYPES = ("noul", "choice", "score")
# Bounds of one head pass, so a large prefill batch is scored in a few passes.
_MAX_GROUP_TOKENS = 65536
_MAX_GROUP_SLOTS = 1 << 18
class LayoutQuestion(msgspec.Struct, frozen=True):
question_type: int
question_span: Tuple[int, int]
option_spans: Tuple[Tuple[int, int], ...]
def max_joint_prompt_tokens(
*,
context_len: int,
num_reserved_tokens: int,
max_req_input_len: int,
max_prefill_tokens: int,
) -> int:
"""The longest prompt, images expanded, that the server runs as one prefill.
The tokenizer manager refuses a prompt of context_len - num_reserved_tokens
tokens, the scheduler one of max_req_input_len, and without chunked prefill
activation memory is reserved for max_prefill_tokens.
"""
return min(
context_len - num_reserved_tokens - 1,
max_req_input_len - 1,
max_prefill_tokens,
)
def pack_decision_layout(
prompt_length: int, questions: Sequence[LayoutQuestion]
) -> List[int]:
"""Flatten a prompt's questions into the ints a request carries.
The layout is ``[prompt_length, num_questions]`` followed, per question,
by ``type, question_start, question_end, num_options`` and one
``option_start, option_end`` pair per option. Positions index the prompt
before its image placeholders expand.
"""
layout = [prompt_length, len(questions)]
for question in questions:
layout += [
question.question_type,
*question.question_span,
len(question.option_spans),
]
for start, end in question.option_spans:
layout += [start, end]
return layout
def parse_decision_layout(
layout: Sequence[int], num_tokens: int
) -> List[LayoutQuestion]:
"""The questions of a layout, placed on the sequence the model reads.
Every span follows the prompt's images, so expanding their placeholders
shifts all spans by the number of added tokens. Raises ValueError when the
layout does not describe the sequence.
"""
if len(layout) < 2 or any(type(value) is not int for value in layout):
raise ValueError("decision_layout must be a list of ints")
prompt_length, num_questions = layout[0], layout[1]
shift = num_tokens - prompt_length
if prompt_length < 1 or shift < 0 or num_questions < 1:
raise ValueError("decision_layout does not match the prompt length")
questions = []
cursor = 2
for _ in range(num_questions):
if cursor + 4 > len(layout):
raise ValueError("decision_layout is truncated")
question_type, start, end, num_options = layout[cursor : cursor + 4]
cursor += 4
if (
not 0 <= question_type < len(QUESTION_TYPES)
or num_options < 1
or not 0 <= start < end <= prompt_length
or cursor + 2 * num_options > len(layout)
):
raise ValueError("decision_layout has an invalid question")
spans = []
for _ in range(num_options):
option_start, option_end = layout[cursor : cursor + 2]
cursor += 2
if not 0 <= option_start < option_end <= prompt_length:
raise ValueError("decision_layout has an invalid option span")
spans.append((option_start + shift, option_end + shift))
questions.append(
LayoutQuestion(question_type, (start + shift, end + shift), tuple(spans))
)
if cursor != len(layout):
raise ValueError("decision_layout has trailing values")
return questions
class EvidenceRoutingLayer(nn.Module):
def __init__(self, width: int, heads: int, feedforward: int) -> None:
super().__init__()
self.query_norm = nn.LayerNorm(width)
self.memory_norm = nn.LayerNorm(width)
self.attention = nn.MultiheadAttention(width, heads, batch_first=True)
self.feedforward_norm = nn.LayerNorm(width)
# Identity slots keep the checkpoint's Sequential indices (dropout at 2 and 4).
self.feedforward = nn.Sequential(
nn.Linear(width, feedforward),
nn.GELU(),
nn.Identity(),
nn.Linear(feedforward, width),
nn.Identity(),
)
class JointSchemaHead(nn.Module):
"""Parameters named as in the checkpoint's ``joint_head.safetensors``."""
def __init__(
self,
hidden_size: int,
width: int,
routing_layers: int,
layers: int,
heads: int,
feedforward: int,
) -> None:
super().__init__()
self.hidden_norm = nn.LayerNorm(hidden_size)
self.memory_projection = nn.Linear(hidden_size, width, bias=False)
self.question_projection = nn.Linear(hidden_size, width, bias=False)
self.option_question_projection = nn.Linear(hidden_size, width, bias=False)
self.global_projection = nn.Linear(hidden_size, width, bias=False)
self.option_context_projection = nn.Linear(hidden_size, width, bias=False)
self.option_lexical_projection = nn.Linear(hidden_size, width, bias=False)
self.type_embedding = nn.Embedding(len(QUESTION_TYPES), width)
self.evidence_layers = nn.ModuleList(
EvidenceRoutingLayer(width, heads, feedforward)
for _ in range(routing_layers)
)
self.option_summary_norm = nn.LayerNorm(width)
self.layers = nn.ModuleList(
nn.TransformerDecoderLayer(
d_model=width,
nhead=heads,
dim_feedforward=feedforward,
dropout=0.0,
activation="gelu",
batch_first=True,
norm_first=True,
)
for _ in range(layers)
)
self.field_norm = nn.LayerNorm(width)
self.option_norm = nn.LayerNorm(width)
self.residual_scorer = nn.Sequential(
nn.Linear(width * 4, width), nn.GELU(), nn.Identity(), nn.Linear(width, 1)
)
self.prior_logit_scale = nn.Parameter(torch.zeros(()))
self.joint_logit_scale = nn.Parameter(torch.zeros(()))
self.residual_gate = nn.Parameter(torch.zeros(()))
@torch.no_grad()
def forward_batch(
self,
hidden: torch.Tensor,
input_ids: torch.Tensor,
lengths: List[int],
layouts: List[List[LayoutQuestion]],
output_embedding_weight: torch.Tensor,
) -> List[torch.Tensor]:
"""One FP32 logit vector per request, its questions' options in layout order.
``hidden`` holds the final hidden states of the requests back to back.
"""
outputs: List[torch.Tensor] = []
start = 0
for group in _groups(lengths, layouts):
group_lengths = [lengths[i] for i in group]
end = start + sum(group_lengths)
outputs += self._forward_group(
hidden[start:end],
input_ids[start:end],
group_lengths,
[layouts[i] for i in group],
output_embedding_weight,
)
start = end
return outputs
def _forward_group(
self,
hidden: torch.Tensor,
input_ids: torch.Tensor,
lengths: List[int],
layouts: List[List[LayoutQuestion]],
output_embedding_weight: torch.Tensor,
) -> List[torch.Tensor]:
device, dtype = hidden.device, hidden.dtype
batch = len(layouts)
question_bounds, option_bounds = [], []
type_ids, owner, slot, question_request, last_tokens = [], [], [], [], []
regions, region_offsets = [], [0]
token_cu, question_cu, option_cu, option_counts = [0], [0], [0], []
for request, (length, questions) in enumerate(zip(lengths, layouts)):
offset = token_cu[-1]
low = min(min(q.question_span[0], q.option_spans[0][0]) for q in questions)
high = max(
max(q.question_span[1], *(e for _, e in q.option_spans))
for q in questions
)
# Prefix row of span position p in this request's schema region.
base = region_offsets[-1] + request - low
regions.append((offset + low, offset + high))
request_options = 0
for question in questions:
count = len(question.option_spans)
question_bounds += (
base + question.question_span[0],
base + question.question_span[1],
)
for option_slot, (option_start, option_end) in enumerate(
question.option_spans
):
option_bounds += (base + option_start, base + option_end)
owner.append(len(type_ids))
slot.append(option_slot)
type_ids.append(question.question_type)
question_request.append(request)
request_options += count
last_tokens.append(offset + length - 1)
token_cu.append(offset + length)
region_offsets.append(region_offsets[-1] + high - low)
question_cu.append(len(type_ids))
option_cu.append(option_cu[-1] + request_options)
option_counts.append(request_options)
num_questions = len(type_ids)
widest = max(len(q.option_spans) for questions in layouts for q in questions)
sizes = [
2 * num_questions,
len(option_bounds),
num_questions,
len(owner),
len(slot),
num_questions,
batch,
3 * (batch + 1),
]
packed = torch.tensor(
question_bounds
+ option_bounds
+ type_ids
+ owner
+ slot
+ question_request
+ last_tokens
+ token_cu
+ question_cu
+ option_cu,
dtype=torch.long,
).to(device, non_blocking=True)
(
question_bounds_t,
option_bounds_t,
type_ids_t,
owner_t,
slot_t,
question_request_t,
last_tokens_t,
cu_t,
) = torch.split(packed, sizes)
question_bounds_t = question_bounds_t.view(-1, 2)
option_bounds_t = option_bounds_t.view(-1, 2)
token_cu_t, question_cu_t, option_cu_t = (
cu_t.to(torch.int32).view(3, batch + 1).unbind(0)
)
token_bounds = (token_cu_t, max(lengths))
question_bounds_cu = (question_cu_t, max(len(q) for q in layouts))
option_bounds_cu = (option_cu_t, max(option_counts))
normalized = self.hidden_norm(hidden)
# Running sums over each schema region, after one zero row per request.
region_total = region_offsets[-1] + batch
hidden_prefix = torch.zeros(
region_total, normalized.shape[-1], dtype=torch.float32, device=device
)
lexical_prefix = torch.zeros(
region_total,
output_embedding_weight.shape[-1],
dtype=torch.float32,
device=device,
)
for request, (low, high) in enumerate(regions):
row = region_offsets[request] + request + 1
torch.cumsum(
normalized[low:high].float(),
dim=0,
out=hidden_prefix[row : row + high - low],
)
torch.cumsum(
output_embedding_weight[input_ids[low:high]].float(),
dim=0,
out=lexical_prefix[row : row + high - low],
)
def span_means(prefix: torch.Tensor, bounds: torch.Tensor) -> torch.Tensor:
sums = prefix[bounds[:, 1]] - prefix[bounds[:, 0]]
return (sums / (bounds[:, 1] - bounds[:, 0]).unsqueeze(-1)).to(dtype)
memory = self.memory_projection(normalized)
global_vectors = normalized[last_tokens_t]
question_vectors = span_means(hidden_prefix, question_bounds_t)
option_contexts = span_means(hidden_prefix, option_bounds_t)
lexical = span_means(lexical_prefix, option_bounds_t)
routed = (
self.option_context_projection(option_contexts)
+ self.option_lexical_projection(lexical)
+ self.option_question_projection(question_vectors)[owner_t]
)
for layer in self.evidence_layers:
routed = routed + _attention(
layer.attention,
layer.query_norm(routed),
layer.memory_norm(memory),
option_bounds_cu,
token_bounds,
)
routed = routed + layer.feedforward(layer.feedforward_norm(routed))
base_fields = self.question_projection(question_vectors)
scores = torch.matmul(
routed.unsqueeze(1), base_fields[owner_t].unsqueeze(2)
).view(-1) / math.sqrt(routed.shape[-1])
padded_scores = torch.full(
(num_questions, widest), float("-inf"), dtype=torch.float32, device=device
)
padded_scores[owner_t, slot_t] = scores.float()
weights = torch.softmax(padded_scores, dim=1).to(dtype)
padded_options = routed.new_zeros(num_questions, widest, routed.shape[-1])
padded_options[owner_t, slot_t] = routed
summaries = torch.sum(weights.unsqueeze(-1) * padded_options, dim=1)
fields = (
base_fields
+ self.option_summary_norm(summaries)
+ self.global_projection(global_vectors)[question_request_t]
+ self.type_embedding(type_ids_t)
)
for layer in self.layers:
fields = fields + _attention(
layer.self_attn,
layer.norm1(fields),
None,
question_bounds_cu,
question_bounds_cu,
)
fields = fields + _attention(
layer.multihead_attn,
layer.norm2(fields),
memory,
question_bounds_cu,
token_bounds,
)
fields = fields + layer.linear2(
layer.activation(layer.linear1(layer.norm3(fields)))
)
fields = self.field_norm(fields)
prior_scale = self.prior_logit_scale.clamp(max=math.log(100.0)).exp()
joint_scale = self.joint_logit_scale.clamp(max=math.log(100.0)).exp()
anchor = F.normalize(
question_vectors + global_vectors[question_request_t], dim=-1
)
prior = prior_scale * torch.matmul(
F.normalize(lexical, dim=-1).unsqueeze(1), anchor[owner_t].unsqueeze(2)
).view(-1)
options = self.option_norm(routed)
repeated = fields[owner_t]
cosine = F.cosine_similarity(repeated, options, dim=-1)
features = torch.cat(
[repeated, options, repeated * options, torch.abs(repeated - options)],
dim=-1,
)
residual = self.residual_scorer(features).squeeze(-1)
logits = prior + torch.sigmoid(self.residual_gate) * (
joint_scale * cosine + residual
)
return list(torch.split(logits.float(), option_counts))
def _attention(
module: nn.MultiheadAttention,
queries: torch.Tensor,
keys: Optional[torch.Tensor],
query_bounds: Tuple[torch.Tensor, int],
key_bounds: Tuple[torch.Tensor, int],
) -> torch.Tensor:
"""nn.MultiheadAttention over packed sequences, keys and values from ``keys``
(or from ``queries`` when None), attending within each request."""
width = queries.shape[-1]
heads = module.num_heads
if keys is None:
query, key, value = F.linear(
queries, module.in_proj_weight, module.in_proj_bias
).chunk(3, dim=-1)
else:
query_weight, key_value_weight = module.in_proj_weight.split([width, 2 * width])
query_bias, key_value_bias = module.in_proj_bias.split([width, 2 * width])
query = F.linear(queries, query_weight, query_bias)
key, value = F.linear(keys, key_value_weight, key_value_bias).chunk(2, dim=-1)
head_dim = width // heads
output = torch.ops.aten._flash_attention_forward(
query.reshape(-1, heads, head_dim),
key.reshape(-1, heads, head_dim),
value.reshape(-1, heads, head_dim),
query_bounds[0],
key_bounds[0],
query_bounds[1],
key_bounds[1],
0.0,
False,
False,
)[0]
return module.out_proj(output.reshape(-1, width))
def _groups(lengths: List[int], layouts: List[List[LayoutQuestion]]) -> List[range]:
"""Consecutive requests per head pass, bounded in tokens and padded option slots."""
groups, start, tokens, questions, widest = [], 0, 0, 0, 0
for index, (length, layout) in enumerate(zip(lengths, layouts)):
layout_widest = max(len(question.option_spans) for question in layout)
grown = (questions + len(layout)) * max(widest, layout_widest)
if index > start and (
tokens + length > _MAX_GROUP_TOKENS or grown > _MAX_GROUP_SLOTS
):
groups.append(range(start, index))
start, tokens, questions, widest = index, 0, 0, 0
tokens += length
questions += len(layout)
widest = max(widest, layout_widest)
groups.append(range(start, len(lengths)))
return groups
class JointSchemaPooler(nn.Module):
"""Pooler of a Clef checkpoint: the joint schema head's option logits per request."""
def __init__(
self, head: JointSchemaHead, output_embedding: Callable[[], torch.Tensor]
) -> None:
super().__init__()
self.head = head
# A callable, so the LM head is not registered twice as a parameter.
self._output_embedding = output_embedding
def forward(
self, hidden_states: torch.Tensor, forward_batch
) -> EmbeddingPoolerOutput:
if (
forward_batch.decision_layouts is None
or forward_batch.extend_seq_lens_cpu is None
):
return EmbeddingPoolerOutput(
embeddings=[
hidden_states.new_empty(0, dtype=torch.float32)
for _ in range(forward_batch.batch_size)
]
)
lengths = list(forward_batch.extend_seq_lens_cpu)
prefix_lengths = forward_batch.extend_prefix_lens_cpu or [0] * len(lengths)
raw_layouts = forward_batch.decision_layouts
embeddings: List[torch.Tensor] = [
hidden_states.new_empty(0, dtype=torch.float32) for _ in lengths
]
# The tokenizer manager checks every layout against its prompt, and the
# server runs each prompt in one uncached prefill. A layout that no longer
# fits its prompt, as after a truncation, gets no scores, which the
# decision route reports as a server error.
layouts = {}
for i, (layout, prefix, length) in enumerate(
zip(raw_layouts, prefix_lengths, lengths)
):
if layout is None or prefix != 0:
continue
try:
layouts[i] = parse_decision_layout(layout, length)
except ValueError:
continue
offsets = [0]
for length in lengths:
offsets.append(offsets[-1] + length)
runs: List[List[int]] = []
for i in layouts:
if runs and runs[-1][-1] == i - 1:
runs[-1].append(i)
else:
runs.append([i])
for run in runs:
start, end = offsets[run[0]], offsets[run[-1] + 1]
logits = self.head.forward_batch(
hidden_states[start:end],
forward_batch.input_ids[start:end],
[lengths[i] for i in run],
[layouts[i] for i in run],
self._output_embedding(),
)
for i, values in zip(run, logits):
embeddings[i] = values
return EmbeddingPoolerOutput(embeddings=embeddings)
+12
View File
@@ -1253,6 +1253,11 @@ class EmbeddingReqInput:
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
token_indices_to_pool: Optional[Union[List[List[int]], List[int]]] = None
# Question and option spans that a Clef checkpoint's joint schema head
# scores, as packed by layers.joint_schema_head.pack_decision_layout.
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
decision_layout: Optional[Union[List[List[int]], List[int]]] = None
def regenerate_rid(self):
"""Generate a new request ID and return it."""
if isinstance(self.rid, list):
@@ -1469,6 +1474,11 @@ class EmbeddingReqInput:
if self.token_indices_to_pool is not None
else None
),
decision_layout=(
self.decision_layout[i]
if self.decision_layout is not None
else None
),
)
cache[i] = sub
return sub
@@ -1500,6 +1510,8 @@ class TokenizedEmbeddingReqInput(BaseReq, kw_only=True):
multi_item_delimiter_indices: Optional[List[int]] = None
# Token positions for setwise pooling readout
token_indices_to_pool: Optional[List[int]] = None
# Question and option spans for a joint schema head
decision_layout: Optional[List[int]] = None
# For observability
# Pickled Optional[Union[APIServerReqTimeStats, DPControllerReqTimeStats]]
time_stats: Optional[PickleWrapper] = None
@@ -1050,6 +1050,7 @@ class Req(ReqDllmMixin):
return_pooled_hidden_states: bool = False,
multi_item_delimiter_indices: Optional[List[int]] = None,
token_indices_to_pool: Optional[List[int]] = None,
decision_layout: Optional[List[int]] = None,
session_id: Optional[str] = None,
cache_salt: Optional[str] = None,
):
@@ -1083,6 +1084,7 @@ class Req(ReqDllmMixin):
self.positional_embed_overrides = positional_embed_overrides
self.multi_item_delimiter_indices = multi_item_delimiter_indices
self.token_indices_to_pool = token_indices_to_pool
self.decision_layout = decision_layout
# For req-level memory management
self.kv = ReqKvInfo()
+1
View File
@@ -3465,6 +3465,7 @@ class Scheduler(
return_pooled_hidden_states=recv_req.return_pooled_hidden_states,
multi_item_delimiter_indices=recv_req.multi_item_delimiter_indices,
token_indices_to_pool=recv_req.token_indices_to_pool,
decision_layout=recv_req.decision_layout,
)
req.tokenizer = self.tokenizer
self._maybe_namespace_elastic_radix_cache(req)
@@ -64,6 +64,10 @@ from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX
from sglang.srt.disaggregation.encoder.receiver import create_mm_receiver
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.environ import envs
from sglang.srt.layers.joint_schema_head import (
max_joint_prompt_tokens,
parse_decision_layout,
)
from sglang.srt.lora.lora_registry import LoRARef, LoRARegistry
from sglang.srt.managers.async_dynamic_batch_tokenizer import AsyncDynamicbatchTokenizer
from sglang.srt.managers.disagg_service import start_disagg_service
@@ -1295,6 +1299,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
raise ValueError(
"encoder SWA replay cannot return cached prompt logprobs"
)
# A decision layout indexes the whole prompt, so it is checked before any
# truncation below could cut the prompt.
self._validate_joint_schema_request(obj, input_ids)
_max_req_len = self.context_len
input_token_num = len(input_ids) if input_ids is not None else 0
input_token_num += self.num_reserved_tokens
@@ -1399,6 +1406,36 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
"return_sampling_mask=true."
)
def _validate_joint_schema_request(
self, obj: Union[GenerateReqInput, EmbeddingReqInput], input_ids: List[int]
) -> None:
"""A Clef checkpoint scores the spans of a decision layout, and nothing else."""
if self.model_config.joint_head_config is None:
return
if isinstance(obj, GenerateReqInput) or obj.decision_layout is None:
# Health checks only need a response, which is an empty embedding.
if (
isinstance(obj, EmbeddingReqInput)
and isinstance(obj.rid, str)
and obj.rid.startswith(HEALTH_CHECK_RID_PREFIX)
):
return
raise ValueError(
"This checkpoint answers schema decisions, send them to /v1/systemone"
)
limit = max_joint_prompt_tokens(
context_len=self.context_len,
num_reserved_tokens=self.num_reserved_tokens,
max_req_input_len=self.max_req_input_len,
max_prefill_tokens=get_schedule().max_prefill_tokens,
)
if len(input_ids) > limit:
raise ValueError(
f"The prompt has {len(input_ids)} tokens, but this server scores "
f"at most {limit} in one prefill"
)
parse_decision_layout(obj.decision_layout, len(input_ids))
def _validate_mm_limits(
self, obj: Union[GenerateReqInput, EmbeddingReqInput]
) -> None:
@@ -1575,6 +1612,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
return_pooled_hidden_states=obj.return_pooled_hidden_states,
multi_item_delimiter_indices=obj.multi_item_delimiter_indices,
token_indices_to_pool=obj.token_indices_to_pool,
decision_layout=obj.decision_layout,
)
tokenized_obj.time_stats = self.rid_to_state[obj.rid].time_stats
@@ -614,6 +614,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# Setwise pooling readout positions (CPU tensors, one per request)
token_indices_to_pool: Optional[List[torch.Tensor]] = None
# Joint schema head decision layouts (one per request, None if absent)
decision_layouts: Optional[List[Optional[List[int]]]] = None
# === Borrowed from ScheduleBatch: compound (carry their own device tensors) ===
# Sampling info
sampling_info: SamplingBatchInfo = None
@@ -1264,6 +1267,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
for r in batch.reqs
]
if any(r.decision_layout is not None for r in batch.reqs):
self.decision_layouts = [r.decision_layout for r in batch.reqs]
token_type_ids = [
r.token_type_ids for r in batch.reqs if r.token_type_ids is not None
]
+30 -1
View File
@@ -34,7 +34,10 @@ from sglang.kernels.ops.attention.triton_gdn_fused_proj import (
from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul
# Configs
from sglang.srt.configs.model_config import load_decision_config
from sglang.srt.configs.model_config import (
load_decision_config,
load_joint_head_config,
)
from sglang.srt.configs.qwen3_5 import (
Qwen3_5Config,
Qwen3_5MoeConfig,
@@ -54,6 +57,7 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
from sglang.srt.layers.joint_schema_head import JointSchemaHead, JointSchemaPooler
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
@@ -2330,6 +2334,16 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration):
self.visual.deepstack_visual_indexes if self.visual is not None else []
)
if config.architectures == ["Qwen3_5ForConditionalGeneration"]:
model = get_model()
joint_head_config = load_joint_head_config(model.model_path, model.revision)
if joint_head_config is not None:
# A Clef checkpoint, whose joint schema head scores every option
# of every question from the final hidden states and LM head rows.
self.pooler = JointSchemaPooler(
JointSchemaHead(**joint_head_config), lambda: self.lm_head.weight
)
def get_hidden_dim(self, module_name: str, layer_idx: int):
return self.model.get_hidden_dim(module_name, layer_idx)
@@ -2470,8 +2484,23 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration):
if bare_backbone and self.pp_group.is_last_rank:
self._load_decision_readout()
loaded_params.add("lm_head.weight")
if isinstance(self.pooler, JointSchemaPooler):
self._load_joint_head()
loaded_params.update(
f"pooler.head.{name}" for name, _ in self.pooler.head.named_parameters()
)
return loaded_params
def _load_joint_head(self) -> None:
"""Load joint_head.safetensors, which the checkpoint index leaves out."""
model = get_model()
state_dict = load_file(
cached_file(
model.model_path, "joint_head.safetensors", revision=model.revision
)
)
self.pooler.head.load_state_dict(state_dict, strict=True)
def _load_decision_readout(self) -> None:
"""Place the checkpoint's decision readout in the LM head rows of its codes."""
model = get_model()
@@ -169,6 +169,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
args = ServerArgs(model_path="dummy")
args._model_config = SimpleNamespace(
is_embedding_gemma=True,
joint_head_config=None,
is_multimodal=False,
context_len=2048,
hf_config=SimpleNamespace(architectures=["Gemma3TextModel"]),
@@ -205,6 +206,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
is_embedding_requested=False,
is_embedding_gemma=False,
),
joint_head_config=None,
is_multimodal=False,
hf_config=SimpleNamespace(architectures=["BertModel"]),
)
@@ -159,6 +159,7 @@ class ScoringManager(TokenizerManagerScoreMixin):
model_type="qwen3_5_moe",
),
decision_config=None,
joint_head_config=None,
)
pad, video = (
tokenizer.encode(token, add_special_tokens=False)[0]
@@ -0,0 +1,474 @@
"""Unit tests for /v1/systemone on Clef checkpoints: the joint schema prompt and its
decision layout, and answers built from the joint schema head's option logits."""
import asyncio
import base64
import json
import math
import random
import unittest
from io import BytesIO
from types import SimpleNamespace
from unittest import mock
import pybase64
from PIL import Image
from transformers import AutoTokenizer
from sglang.srt.entrypoints import http_server
from sglang.srt.entrypoints.openai.protocol import DecisionRequest
from sglang.srt.entrypoints.openai.serving_decisions import OpenAIServingDecisions
from sglang.srt.entrypoints.systemone.joint_schema import encode_joint_schema
from sglang.srt.entrypoints.systemone.protocol import SystemOneRequest
from sglang.srt.entrypoints.systemone.serving import SystemOneServing, _read_image
from sglang.srt.layers.joint_schema_head import (
LayoutQuestion,
pack_decision_layout,
parse_decision_layout,
)
from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput
from sglang.srt.managers.tokenizer_manager import TokenizerManager
from sglang.srt.runtime_context import (
get_context,
get_schedule,
publish,
restore_context,
snapshot_context,
)
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import ImageData
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
# Tokenizer files only, with the Qwen3.5 vocabulary of Clef checkpoints.
TOKENIZER = "Qwen/Qwen3.5-35B-A3B"
REQUEST = {
"model": "clef",
"state": {"ticket": "Refund for order 1182", "amount": 42},
"questions": {
"team": {
"type": "choice",
"instructions": "Which team handles it?",
"criteria": {
"support": "Customer support",
"billing": "Payments and refunds",
"legal": None,
},
},
"urgent": {
"type": "noul",
"instructions": "Is it urgent?",
"criteria": {"false": None},
},
"severity": {"type": "score", "criteria": ["none", "low", "high"]},
},
}
class PatchCounter:
"""Stands in for the multimodal processor: one image token per 28x28 patch."""
@staticmethod
def resolve_image_token_counts(images):
return [(image.height // 28) * (image.width // 28) for image in images]
def _png_bytes(width, height):
buffer = BytesIO()
Image.new("RGB", (width, height), "red").save(buffer, format="PNG")
return buffer.getvalue()
def _png(width, height):
return (
"data:image/png;base64," + base64.b64encode(_png_bytes(width, height)).decode()
)
def _expanded_length(request):
images = [
Image.open(BytesIO(base64.b64decode(image.url.split(",", 1)[1])))
for image in request.image_data or []
]
counts = PatchCounter.resolve_image_token_counts(images)
return len(request.input_ids) + sum(count - 1 for count in counts)
class HeadManager:
"""Replace model execution with option logits 0, 1, 2, ... per question, in head order,
after expanding images and refusing prompts that the tokenizer manager, the
scheduler, or the memory reserved for one prefill cannot take."""
def __init__(self, tokenizer):
self.server_args = ServerArgs(model_path="dummy")
publish(self.server_args, role="test")
self.tokenizer = tokenizer
self.model_config = SimpleNamespace(
is_multimodal=True,
hf_config=SimpleNamespace(
architectures=["Qwen3_5ForConditionalGeneration"], model_type="qwen3_5"
),
decision_config=None,
joint_head_config={"hidden_size": 4096},
)
self.is_generation = False
self.allow_auto_truncate = False
self.context_len = 4096
self.num_reserved_tokens = 0
# min(context_len - 1, kv_capacity - 1) - 5, as the scheduler reports it.
self.max_req_input_len = 4090
self.mm_processor = PatchCounter()
self.request_logger = SimpleNamespace(log_requests=False)
self.served_model_name = "served-model"
self.requests = []
def config_value(self, name):
return None
async def generate_request(self, request, raw_request):
self.requests.append(request)
num_tokens = _expanded_length(request)
if num_tokens + self.num_reserved_tokens >= self.context_len:
raise ValueError(f"The input ({num_tokens} tokens) is too long")
if num_tokens >= self.max_req_input_len:
raise ValueError(f"The scheduler refuses a prompt of {num_tokens} tokens")
if num_tokens > get_schedule().max_prefill_tokens:
raise ValueError(f"A prefill of {num_tokens} tokens outgrows its memory")
questions = parse_decision_layout(request.decision_layout, num_tokens)
logits = [float(i) for q in questions for i in range(len(q.option_spans))]
yield {
"embedding": logits,
"meta_info": {"prompt_tokens": num_tokens},
}
def _handler(manager, serving_class=SystemOneServing):
template_manager = SimpleNamespace(
chat_template_name=None,
jinja_template_content_format="openai",
reasoning_config=None,
suggested_reasoning_parser=None,
)
chat_serving = SimpleNamespace(
tokenizer_manager=manager,
template_manager=template_manager,
default_chat_template_kwargs={},
chat_encoding_spec=None,
_prompt_text_round_trip_is_lossy=False,
reasoning_parser=None,
)
return serving_class(chat_serving)
def _softmax(values):
weights = [math.exp(value - max(values)) for value in values]
return [weight / sum(weights) for weight in weights]
class TestJointSchemaPrompt(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
def _encode(self, request=REQUEST, max_length=4096):
input_ids, layout = encode_joint_schema(
self.tokenizer,
SystemOneRequest(**request),
max_length=max_length,
image_token_counts=[],
)
return input_ids, parse_decision_layout(layout, len(input_ids))
def _text(self, input_ids, span):
return self.tokenizer.decode(input_ids[span[0] : span[1]])
def test_prompt_is_the_trained_joint_schema_prompt(self):
input_ids, _ = self._encode()
prompt = self.tokenizer.decode(input_ids)
self.assertTrue(
prompt.startswith(
"<|im_start|>system\nRead the complete state and schema. Decide every "
"field jointly. Each answer must be exactly one of that field's allowed "
"options.<|im_end|>\n<|im_start|>user\nSTATE:\n"
'{"amount":42,"ticket":"Refund for order 1182"}\n\nSCHEMA FIELDS:\n'
"\nFIELD 1\nID: team\nTYPE: choice\nINSTRUCTION: Which team handles it?"
"\nALLOWED OPTIONS:\n"
'OPTION 1: {"description":"Payments and refunds","option_id":"billing"}\n'
)
)
self.assertTrue(
prompt.endswith(
"END FIELD\n\n<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
"JOINT SCHEMA DECISIONS:"
)
)
def test_layout_spans_hold_questions_and_options_in_head_order(self):
input_ids, questions = self._encode()
self.assertEqual([q.question_type for q in questions], [1, 0, 2])
self.assertEqual(
[self._text(input_ids, q.question_span) for q in questions],
["Which team handles it?", "Is it urgent?", "severity"],
)
options = [
[json.loads(self._text(input_ids, span)) for span in q.option_spans]
for q in questions
]
self.assertEqual(
options[0],
[
{"description": "Payments and refunds", "option_id": "billing"},
{"option_id": "legal"},
{"description": "Customer support", "option_id": "support"},
],
)
self.assertEqual(
options[1],
[
{
"description": "The proposition is true or the answer is yes.",
"option_id": "true",
},
{"option_id": "false"},
],
)
self.assertEqual(
options[2],
[
{"description": level, "option_id": str(index)}
for index, level in enumerate(["none", "low", "high"])
],
)
def test_state_is_truncated_to_fit(self):
input_ids, questions = self._encode()
shorter_ids, shorter = self._encode(max_length=len(input_ids) - 3)
self.assertEqual(len(shorter_ids), len(input_ids) - 3)
self.assertEqual(
[self._text(shorter_ids, q.question_span) for q in shorter],
[self._text(input_ids, q.question_span) for q in questions],
)
with self.assertRaisesRegex(ValueError, "before the state"):
self._encode(max_length=100)
def test_image_placeholders_precede_the_state(self):
request = {**REQUEST, "images": ["data:image/png;base64,AAAA"]}
input_ids, layout = encode_joint_schema(
self.tokenizer,
SystemOneRequest(**request),
max_length=4096,
image_token_counts=[100],
)
self.assertIn(
"STATE:\n<|vision_start|><|image_pad|><|vision_end|>\n{",
self.tokenizer.decode(input_ids),
)
unexpanded = parse_decision_layout(layout, len(input_ids))
expanded = parse_decision_layout(layout, len(input_ids) + 99)
self.assertEqual(
[q.option_spans[0][0] + 99 for q in unexpanded],
[q.option_spans[0][0] for q in expanded],
)
class TestJointSchemaAnswers(unittest.IsolatedAsyncioTestCase):
@classmethod
def setUpClass(cls):
cls.tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
def setUp(self):
self.addCleanup(restore_context, snapshot_context())
async def test_answers_map_head_order_back_to_the_request(self):
manager = HeadManager(self.tokenizer)
response = await _handler(manager).handle_request(
SystemOneRequest(**REQUEST), None
)
self.assertEqual(response.status_code, 200)
body = json.loads(response.body)
input_ids, layout = encode_joint_schema(
self.tokenizer,
SystemOneRequest(**REQUEST),
max_length=4089,
image_token_counts=[],
)
(sent,) = manager.requests
self.assertEqual((sent.input_ids, sent.decision_layout), (input_ids, layout))
self.assertIsNone(sent.image_data)
self.assertEqual(body["model"], "served-model")
self.assertEqual(body["usage"]["input_tokens"], len(input_ids))
three, two = _softmax([0.0, 1.0, 2.0]), _softmax([0.0, 1.0])
team, urgent, severity = (body["answers"][q] for q in REQUEST["questions"])
# The head scores billing, legal, support; answers keep the request order.
self.assertEqual(team["choice"], "support")
self.assertEqual(list(team["probabilities"]), ["support", "billing", "legal"])
for name, expected in zip(["billing", "legal", "support"], three):
self.assertAlmostEqual(team["probabilities"][name], expected)
self.assertAlmostEqual(urgent["noul"], two[0])
self.assertAlmostEqual(severity["score"], three[1] + 2 * three[2])
self.assertEqual(severity["legend"], {"0": "none", "1": "low", "2": "high"})
for answer in (team, urgent, severity):
self.assertNotIn("x_label_mass", answer)
async def test_url_images_are_read_once_and_forwarded_as_counted(self):
"""An image URL whose content changes between reads must be scored as the
image that was counted against the prompt budget."""
manager = HeadManager(self.tokenizer)
url = "http://images.example/chart.png"
data = _png_bytes(224, 224)
image = {"url": url, "detail": "high"}
request = {**REQUEST, "state": "word " * 5000, "images": [image]}
target = "sglang.srt.entrypoints.systemone.serving.get_image_bytes"
with mock.patch(target, return_value=data) as fetch:
response = await _handler(manager).handle_request(
SystemOneRequest(**request), None
)
fetch.assert_called_once_with(url)
(forwarded,) = manager.requests[0].image_data
self.assertTrue(forwarded.url.startswith("data:image/png;base64,"))
self.assertEqual(base64.b64decode(forwarded.url.split(",", 1)[1]), data)
self.assertEqual(forwarded.detail, "high")
self.assertEqual(response.status_code, 200)
self.assertEqual(json.loads(response.body)["usage"]["input_tokens"], 4089)
async def test_refusals(self):
manager = HeadManager(self.tokenizer)
cases = {
"takes no chat_template_kwargs": (
SystemOneServing,
SystemOneRequest(**REQUEST, chat_template_kwargs={"x": 1}),
),
"LoRA adapter": (
SystemOneServing,
SystemOneRequest(**{**REQUEST, "model": "clef:a"}),
),
# Its 5329 image tokens leave no room for the questions.
"before the state": (
SystemOneServing,
SystemOneRequest(**{**REQUEST, "images": [_png(2048, 2048)]}),
),
"requires a generation model": (
OpenAIServingDecisions,
DecisionRequest(
model="m",
input="x",
questions=[{"id": "q", "type": "yes_no", "question": "Q?"}],
),
),
}
for message, (serving_class, request) in cases.items():
with self.subTest(message):
response = await _handler(manager, serving_class).handle_request(
request, None
)
self.assertEqual(response.status_code, 400)
self.assertIn(message, json.loads(response.body)["message"])
self.assertEqual(manager.requests, [])
def _admission_manager(joint_head_config):
manager = TokenizerManager.__new__(TokenizerManager)
manager.model_config = SimpleNamespace(joint_head_config=joint_head_config)
manager.is_generation = joint_head_config is None
manager.context_len = 4096
manager.num_reserved_tokens = 0
manager.max_req_input_len = 4090
# A layout must be refused before auto-truncation could cut its prompt.
manager.allow_auto_truncate = True
manager.validate_total_tokens = False
manager._validate_token_ids_logprob = mock.Mock()
return manager
class TestJointSchemaAdmission(CustomTestCase):
def setUp(self):
override = get_context().override_server_args()
override.install()
self.addCleanup(override.restore)
def test_tokenizer_refuses_generation_and_prompts_past_one_prefill(self):
clef = _admission_manager({"hidden_size": 4096})
layout = [LayoutQuestion(0, (10, 12), ((13, 14), (15, 16)))]
def embedding(length):
return EmbeddingReqInput(
input_ids=[1] * length,
decision_layout=pack_decision_layout(length, layout),
sampling_params={},
)
generation = GenerateReqInput(input_ids=[1, 2, 3], sampling_params={})
refused = {
"generation": (generation, "/v1/systemone"),
# 4095 tokens pass the context check, but the scheduler takes 4089.
"past one prefill": (embedding(4095), "at most 4089"),
}
for name, (obj, message) in refused.items():
with self.subTest(name), self.assertRaisesRegex(ValueError, message):
clef._validate_one_request(obj, obj.input_ids)
clef._validate_one_request(embedding(4089), [1] * 4089)
# A server without a joint schema head keeps generating.
_admission_manager(None)._validate_one_request(generation, [1, 2, 3])
def test_client_decision_layouts_are_refused_at_encode_and_classify(self):
manager = SimpleNamespace(requests=[])
async def generate_request(obj, request):
manager.requests.append(obj)
yield {"embedding": [0.0]}
manager.generate_request = generate_request
self.addCleanup(
setattr, http_server, "_global_state", http_server.get_global_state()
)
http_server.set_global_state(SimpleNamespace(tokenizer_manager=manager))
layout = pack_decision_layout(16, [LayoutQuestion(0, (0, 1), ((1, 2),))])
responses = [
asyncio.run(
handler(
EmbeddingReqInput(
input_ids=list(range(16)), decision_layout=layout
),
None,
)
)
for handler in (http_server.encode_request, http_server.classify_request)
]
self.assertEqual(manager.requests, [])
self.assertEqual([response.status_code for response in responses], [400, 400])
def test_inline_images_are_sized_from_a_header_prefix(self):
"""Sizing an inline image must not decode its whole payload, and an image
whose header runs past the decoded prefix must still be sized."""
noise = BytesIO()
pixels = random.Random(0).randbytes(256 * 192 * 3)
Image.frombytes("RGB", (256, 192), pixels).save(noise, format="PNG")
late = BytesIO()
# A 60 KB EXIF segment puts the JPEG frame header far into the payload.
exif = b"Exif\x00\x00" + bytes(60000)
Image.new("RGB", (64, 48)).save(late, format="JPEG", exif=exif)
cases = {
"png": ("image/png", noise.getvalue(), (256, 192)),
"late jpeg header": ("image/jpeg", late.getvalue(), (64, 48)),
}
for name, (mime, data, size) in cases.items():
with self.subTest(name):
payload = base64.b64encode(data).decode()
image = ImageData(url=f"data:{mime};base64,{payload}")
with mock.patch("pybase64.b64decode", wraps=pybase64.b64decode) as read:
forwarded, header = _read_image(0, image)
self.assertIs(forwarded, image)
self.assertEqual(header.size, size)
decoded = [len(call.args[0]) for call in read.call_args_list]
if name == "png":
self.assertLess(max(decoded), len(payload))
else:
self.assertEqual(decoded[-1], len(payload))
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,240 @@
"""Unit tests for the joint schema head of Clef checkpoints: decision layouts, and
the batched forward against a per-request loop over the stock torch modules."""
import math
import unittest
from types import SimpleNamespace
import torch
import torch.nn.functional as F
from sglang.srt.layers.joint_schema_head import (
JointSchemaHead,
JointSchemaPooler,
LayoutQuestion,
pack_decision_layout,
parse_decision_layout,
)
from sglang.test.ci.ci_register import register_cpu_ci, register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small")
def _questions(offset=0):
return [
LayoutQuestion(
1,
(10 + offset, 14 + offset),
(
(16 + offset, 17 + offset),
(17 + offset, 19 + offset),
(19 + offset, 21 + offset),
),
),
LayoutQuestion(
0,
(22 + offset, 25 + offset),
((26 + offset, 28 + offset), (29 + offset, 31 + offset)),
),
]
class TestDecisionLayout(CustomTestCase):
def test_round_trip(self):
layout = pack_decision_layout(40, _questions())
self.assertEqual(parse_decision_layout(layout, 40), _questions())
def test_expanded_images_shift_every_span(self):
layout = pack_decision_layout(40, _questions())
self.assertEqual(parse_decision_layout(layout, 140), _questions(offset=100))
def test_rejects_layouts_that_do_not_fit_the_prompt(self):
layout = pack_decision_layout(40, _questions())
invalid = {
"shorter prompt": (layout, 39),
"not ints": ([float(value) for value in layout], 40),
"truncated": (layout[:-1], 40),
"trailing values": (layout + [0], 40),
"no questions": ([40, 0], 40),
"empty span": (
pack_decision_layout(40, [LayoutQuestion(0, (5, 5), ((6, 7),))]),
40,
),
"past the prompt": (
pack_decision_layout(40, [LayoutQuestion(0, (5, 6), ((6, 41),))]),
40,
),
"unknown type": (
pack_decision_layout(40, [LayoutQuestion(3, (5, 6), ((6, 7),))]),
40,
),
}
for name, (values, num_tokens) in invalid.items():
with self.subTest(name), self.assertRaises(ValueError):
parse_decision_layout(values, num_tokens)
class _RecordingHead:
"""Stands in for the head, scoring each option with its request's length."""
def __init__(self):
self.calls = []
def forward_batch(self, hidden, input_ids, lengths, layouts, embedding_weight):
self.calls.append(lengths)
return [
torch.full((sum(len(q.option_spans) for q in layout),), float(length))
for length, layout in zip(lengths, layouts)
]
class TestJointSchemaPooler(CustomTestCase):
def test_a_layout_cut_from_its_prompt_is_skipped_and_neighbors_are_scored(self):
"""A prompt shorter than its layout, as after a truncation, must not stop the
scheduler or the other requests of its batch."""
head = _RecordingHead()
pooler = JointSchemaPooler(head, lambda: torch.zeros(1))
layout = pack_decision_layout(40, _questions())
lengths = [40, 35, 40]
forward_batch = SimpleNamespace(
decision_layouts=[layout, layout, layout],
extend_seq_lens_cpu=lengths,
extend_prefix_lens_cpu=[0, 0, 0],
batch_size=3,
input_ids=torch.zeros(sum(lengths), dtype=torch.long),
)
output = pooler(torch.zeros(sum(lengths), 8), forward_batch)
self.assertEqual([len(scores) for scores in output.embeddings], [5, 0, 5])
self.assertEqual(head.calls, [[40], [40]])
def _reference_logits(head, hidden, input_ids, questions, lm_head):
"""One request through the modules' own forward methods, as the checkpoint's loop does."""
normalized = head.hidden_norm(hidden)
memory = head.memory_projection(normalized).unsqueeze(0)
global_vector = normalized[-1]
def mean(values, span):
return values[span[0] : span[1]].mean(dim=0)
question_vectors = torch.stack(
[mean(normalized, q.question_span) for q in questions]
)
queries, lexicals = [], []
for index, question in enumerate(questions):
context = torch.stack(
[mean(normalized, span) for span in question.option_spans]
)
lexical = torch.stack(
[
lm_head[input_ids[start:end]].mean(dim=0)
for start, end in question.option_spans
]
)
queries.append(
head.option_context_projection(context)
+ head.option_lexical_projection(lexical)
+ head.option_question_projection(question_vectors[index]).unsqueeze(0)
)
lexicals.append(lexical)
routed = torch.cat(queries).unsqueeze(0)
for layer in head.evidence_layers:
attended, _ = layer.attention(
layer.query_norm(routed),
layer.memory_norm(memory),
layer.memory_norm(memory),
need_weights=False,
)
routed = routed + attended
routed = routed + layer.feedforward(layer.feedforward_norm(routed))
options = list(torch.split(routed[0], [len(q.option_spans) for q in questions]))
base_fields = head.question_projection(question_vectors)
summaries = []
for field, routed_options in zip(base_fields, options):
weights = torch.softmax(
routed_options @ field / math.sqrt(field.shape[-1]), dim=0
)
summaries.append((weights.unsqueeze(-1) * routed_options).sum(dim=0))
fields = (
base_fields
+ head.option_summary_norm(torch.stack(summaries))
+ head.global_projection(global_vector).unsqueeze(0)
+ head.type_embedding(
torch.tensor([q.question_type for q in questions], device=hidden.device)
)
).unsqueeze(0)
for layer in head.layers:
fields = layer(fields, memory)
fields = head.field_norm(fields[0])
prior_scale = head.prior_logit_scale.clamp(max=math.log(100.0)).exp()
joint_scale = head.joint_logit_scale.clamp(max=math.log(100.0)).exp()
logits = []
for index, (field, lexical, routed_options) in enumerate(
zip(fields, lexicals, options)
):
anchor = F.normalize(question_vectors[index] + global_vector, dim=-1)
prior = prior_scale * (F.normalize(lexical, dim=-1) @ anchor)
normed = head.option_norm(routed_options)
repeated = field.unsqueeze(0).expand_as(normed)
features = torch.cat(
[repeated, normed, repeated * normed, torch.abs(repeated - normed)], dim=-1
)
residual = head.residual_scorer(features).squeeze(-1)
cosine = F.cosine_similarity(repeated, normed, dim=-1)
logits.append(
prior
+ torch.sigmoid(head.residual_gate) * (joint_scale * cosine + residual)
)
return torch.cat(logits).float()
@unittest.skipUnless(torch.cuda.is_available(), "the batched head uses FlashAttention")
class TestJointSchemaHeadBatch(CustomTestCase):
def test_batched_forward_matches_each_request_alone(self):
torch.manual_seed(0)
hidden_size, vocab = 256, 512
head = JointSchemaHead(
hidden_size, width=128, routing_layers=2, layers=2, heads=4, feedforward=256
)
for parameter in (
head.prior_logit_scale,
head.joint_logit_scale,
head.residual_gate,
):
parameter.data.fill_(0.5)
head = head.to("cuda", torch.bfloat16).eval()
lm_head = torch.randn(vocab, hidden_size, device="cuda", dtype=torch.bfloat16)
lengths = [40, 140, 64]
layouts = [
_questions(),
_questions(offset=100),
[
LayoutQuestion(
2, (3, 9), tuple((10 + 2 * i, 12 + 2 * i) for i in range(20))
)
],
]
hidden = torch.randn(
sum(lengths), hidden_size, device="cuda", dtype=torch.bfloat16
)
input_ids = torch.randint(0, vocab, (sum(lengths),), device="cuda")
with torch.no_grad():
batched = head.forward_batch(hidden, input_ids, lengths, layouts, lm_head)
start = 0
for length, questions, logits in zip(lengths, layouts, batched):
end = start + length
expected = _reference_logits(
head, hidden[start:end], input_ids[start:end], questions, lm_head
)
self.assertEqual(logits.shape, expected.shape)
torch.testing.assert_close(logits, expected, atol=5e-2, rtol=5e-2)
torch.testing.assert_close(
logits.softmax(-1), expected.softmax(-1), atol=1e-2, rtol=0
)
start = end
if __name__ == "__main__":
unittest.main()
@@ -26,6 +26,7 @@ class TestHiddenStateServerMode(CustomTestCase):
manager.allow_auto_truncate = False
manager.validate_total_tokens = False
manager.is_generation = True
manager.model_config = SimpleNamespace(joint_head_config=None)
manager.server_args = SimpleNamespace(enable_custom_logit_processor=False)
manager._validate_token_ids_logprob = Mock()
return manager
@@ -1,4 +1,5 @@
import unittest
from types import SimpleNamespace
from unittest.mock import patch
from sglang.srt.managers.io_struct import GenerateReqInput
@@ -28,6 +29,7 @@ class TestSamplingMaskValidation(CustomTestCase):
self.manager.allow_auto_truncate = False
self.manager.validate_total_tokens = False
self.manager.is_generation = True
self.manager.model_config = SimpleNamespace(joint_head_config=None)
def _validate(
self,
@@ -987,6 +987,7 @@ class TestSchedulerMmTransportBoundary(CustomTestCase):
return_pooled_hidden_states=False,
multi_item_delimiter_indices=None,
token_indices_to_pool=None,
decision_layout=None,
mm_inputs=object(),
)