diff --git a/python/sglang/srt/arg_groups/model_hook.py b/python/sglang/srt/arg_groups/model_hook.py
index dc1a9be316..485addec92 100644
--- a/python/sglang/srt/arg_groups/model_hook.py
+++ b/python/sglang/srt/arg_groups/model_hook.py
@@ -977,6 +977,37 @@ def handle_model_capability_adjustments(server_args: Any):
"prefill; using breakable CUDA graph for CUDA prefill."
)
+ # A Clef checkpoint's joint schema head reads the final hidden state of
+ # every prompt token in one pass, so each request is one complete prefill
+ # with no decode: embedding mode, without prefix reuse or chunking. This
+ # also puts the FA backend on its raw K/V path, which skips the KV pool.
+ if model_config.joint_head_config is not None:
+ if cfg.tp_size != 1 or cfg.pp_size != 1:
+ raise ValueError(
+ "Clef checkpoints are served with --tp-size 1 and --pp-size 1, "
+ "since the joint schema head reads full LM head rows"
+ )
+ for key, value in (
+ ("is_embedding", True),
+ ("disable_radix_cache", True),
+ ("chunked_prefill_size", -1),
+ ):
+ declare_resolution(
+ server_args, "_handle_model_capability_adjustments", **{key: value}
+ )
+ for phase in (Phase.DECODE, Phase.PREFILL):
+ declare_resolution(
+ server_args,
+ "_handle_model_capability_adjustments",
+ cuda_graph_config=with_phase(
+ cfg.cuda_graph_config, phase, backend=Backend.DISABLED
+ ),
+ )
+ logger.info(
+ "Clef joint schema head detected: serving /v1/systemone decisions in "
+ "embedding mode without radix cache, chunked prefill, or CUDA graphs."
+ )
+
if (
model_config.is_multimodal
and not model_config.is_multimodal_chunked_prefill_supported
diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py
index 37cbbe45ff..2f5da7fc36 100644
--- a/python/sglang/srt/configs/model_config.py
+++ b/python/sglang/srt/configs/model_config.py
@@ -291,6 +291,23 @@ def load_decision_config(model_path: str, revision: Optional[str]) -> Optional[d
return config
+@lru_cache
+def load_joint_head_config(model_path: str, revision: Optional[str]) -> Optional[dict]:
+ """joint_head_config.json of a Clef checkpoint, whose joint schema head
+ answers /v1/systemone decisions from the backbone's hidden states."""
+ path = cached_file(
+ model_path,
+ "joint_head_config.json",
+ revision=revision,
+ _raise_exceptions_for_missing_entries=False,
+ _raise_exceptions_for_connection_errors=False,
+ )
+ if path is None:
+ return None
+ with open(path) as f:
+ return json.load(f)
+
+
def is_qwen3_5(config) -> bool:
return _hf_arch(config) in (
"Qwen3_5ForConditionalGeneration",
@@ -544,6 +561,9 @@ class ModelConfig:
):
# Full attention sees the whole prompt, while linear attention stays causal.
self.hf_text_config.is_causal = False
+ self.joint_head_config = None
+ if self.hf_config.architectures == ["Qwen3_5ForConditionalGeneration"]:
+ self.joint_head_config = load_joint_head_config(self.model_path, revision)
self.requires_mm_token_modalities = requires_mm_token_modalities(
self.hf_config.architectures, self.hf_text_config
)
diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py
index ea9addb0ba..c662f6a2f5 100644
--- a/python/sglang/srt/entrypoints/http_server.py
+++ b/python/sglang/srt/entrypoints/http_server.py
@@ -66,6 +66,7 @@ from fastapi.routing import APIRoute
from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.configs.embedding_model_spec import resolved_embedding_plan
+from sglang.srt.configs.model_config import load_joint_head_config
from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX
from sglang.srt.disaggregation.utils import FAKE_BOOTSTRAP_HOST, DisaggregationMode
from sglang.srt.entrypoints.anthropic.protocol import (
@@ -972,6 +973,7 @@ async def serve_generate_request(obj: GenerateReqInput, request: Request):
async def encode_request(obj: EmbeddingReqInput, request: Request):
"""Handle an embedding request."""
try:
+ _refuse_decision_layout(obj)
ret = await _global_state.tokenizer_manager.generate_request(
obj, request
).__anext__()
@@ -984,6 +986,7 @@ async def encode_request(obj: EmbeddingReqInput, request: Request):
async def classify_request(obj: EmbeddingReqInput, request: Request):
"""Handle a reward model request. Now the arguments and return values are the same as embedding models."""
try:
+ _refuse_decision_layout(obj)
ret = await _global_state.tokenizer_manager.generate_request(
obj, request
).__anext__()
@@ -992,6 +995,12 @@ async def classify_request(obj: EmbeddingReqInput, request: Request):
return _create_error_response(e)
+def _refuse_decision_layout(obj: EmbeddingReqInput) -> None:
+ # Only /v1/systemone builds a decision layout, from the prompt it compiles.
+ if obj.decision_layout is not None:
+ raise ValueError("decision_layout is set only by /v1/systemone")
+
+
@app.api_route("/flush_cache", methods=["GET", "POST"])
@auth_level(AuthLevel.ADMIN_OPTIONAL)
async def flush_cache(timeout: float = Query(0.0, ge=0.0)):
@@ -2379,6 +2388,22 @@ def _execute_server_warmup(server_args: ServerArgs):
).tolist()
json_data["sampling_params"]["max_new_tokens"] = 0
+ if (
+ not model_info["is_generation"]
+ and model_info.get("architectures") == ["Qwen3_5ForConditionalGeneration"]
+ and load_joint_head_config(get_model().model_path, get_model().revision)
+ is not None
+ ):
+ # A Clef checkpoint answers /v1/systemone decisions only.
+ request_name = "/v1/systemone"
+ json_data = {
+ "model": "warmup",
+ "state": "The capital city of France is Paris.",
+ "questions": {
+ "warmup": {"type": "noul", "instructions": "Is the state true?"}
+ },
+ }
+
# Send a warmup request
warmup_timeout = envs.SGLANG_WARMUP_TIMEOUT.get()
try:
diff --git a/python/sglang/srt/entrypoints/openai/serving_decisions.py b/python/sglang/srt/entrypoints/openai/serving_decisions.py
index 960219d5ac..edf7698786 100644
--- a/python/sglang/srt/entrypoints/openai/serving_decisions.py
+++ b/python/sglang/srt/entrypoints/openai/serving_decisions.py
@@ -83,6 +83,7 @@ class OpenAIServingDecisions(OpenAIServingBase):
self.chat_encoding_spec = chat_serving.chat_encoding_spec
self.prompt_text_is_lossy = chat_serving._prompt_text_round_trip_is_lossy
self.decision_config = self.tokenizer_manager.model_config.decision_config
+ self.joint_head_config = self.tokenizer_manager.model_config.joint_head_config
tokenizer = self.tokenizer_manager.tokenizer
# Other tokenizers skip the shortcut in _encode_labels and check the full prompt.
self.added_tokens = (
diff --git a/python/sglang/srt/entrypoints/systemone/joint_schema.py b/python/sglang/srt/entrypoints/systemone/joint_schema.py
new file mode 100644
index 0000000000..66d9568fbf
--- /dev/null
+++ b/python/sglang/srt/entrypoints/systemone/joint_schema.py
@@ -0,0 +1,137 @@
+"""The prompt that a Clef checkpoint's joint schema head reads.
+
+Clef answers /v1/systemone questions from the backbone's hidden states instead
+of from generated text. One prompt holds the state and every question with its
+allowed options, and the head scores the span of each option. This follows
+encode_record in the checkpoint's joint_schema_model.py, piece by piece, since
+each piece is tokenized on its own.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, List, Sequence, Tuple
+
+from sglang.srt.entrypoints.systemone.protocol import (
+ SystemOneChoiceQuestion,
+ SystemOneNoulQuestion,
+ SystemOneQuestion,
+ SystemOneRequest,
+)
+from sglang.srt.layers.joint_schema_head import (
+ QUESTION_TYPES,
+ LayoutQuestion,
+ pack_decision_layout,
+)
+
+_SYSTEM_PROMPT = (
+ "Read the complete state and schema. Decide every field jointly. Each answer "
+ "must be exactly one of that field's allowed options."
+)
+_PREFIX = f"<|im_start|>system\n{_SYSTEM_PROMPT}<|im_end|>\n<|im_start|>user\nSTATE:\n"
+_IMAGE = "<|vision_start|><|image_pad|><|vision_end|>"
+_SUFFIX = (
+ "\n<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"
+ "JOINT SCHEMA DECISIONS:"
+)
+_NOUL_CRITERIA = {
+ "true": "The proposition is true or the answer is yes.",
+ "false": "The proposition is false or the answer is no.",
+}
+# The default max_length of encode_record and systemone in joint_schema_model.py.
+REFERENCE_MAX_LENGTH = 16384
+
+
+def _render(value: Any) -> str:
+ if isinstance(value, str):
+ return value
+ return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
+
+
+def joint_schema_options(question: SystemOneQuestion) -> List[Tuple[str, Any]]:
+ """Each option's name and description, in the order the head scores them."""
+ if isinstance(question, SystemOneNoulQuestion):
+ criteria = dict(_NOUL_CRITERIA)
+ if question.criteria is not None:
+ # A description sent as null drops the default, as in encode_record.
+ criteria.update(question.criteria.model_dump(exclude_unset=True))
+ return [(name, criteria[name]) for name in ("true", "false")]
+ if isinstance(question, SystemOneChoiceQuestion):
+ return sorted(question.criteria.items(), key=lambda item: item[0])
+ return [(str(level), detail) for level, detail in enumerate(question.criteria)]
+
+
+def encode_joint_schema(
+ tokenizer: Any,
+ request: SystemOneRequest,
+ max_length: int,
+ image_token_counts: Sequence[int],
+) -> Tuple[List[int], List[int]]:
+ """The prompt ids and their decision layout.
+
+ Each image's placeholder expands to its count in ``image_token_counts``, and
+ the state is truncated so that the expanded prompt has at most
+ ``max_length`` tokens, as encode_record budgets the expanded media.
+ """
+
+ def tokens(text: str) -> List[int]:
+ return tokenizer.encode(text, add_special_tokens=False)
+
+ schema = tokens("\n\nSCHEMA FIELDS:\n")
+ questions = []
+ for index, (question_id, question) in enumerate(request.questions.items()):
+ schema += tokens(
+ f"\nFIELD {index + 1}\nID: {question_id}\nTYPE: {question.type}\n"
+ "INSTRUCTION: "
+ )
+ instructions = question.instructions
+ if instructions is None or instructions == "":
+ instructions = question_id
+ question_start = len(schema)
+ schema += tokens(_render(instructions))
+ question_span = (question_start, len(schema))
+ schema += tokens("\nALLOWED OPTIONS:\n")
+ option_spans = []
+ for option_index, (name, description) in enumerate(
+ joint_schema_options(question)
+ ):
+ schema += tokens(f"OPTION {option_index + 1}: ")
+ option_start = len(schema)
+ semantics = {"option_id": name}
+ if description is not None:
+ semantics["description"] = description
+ schema += tokens(_render(semantics))
+ option_spans.append((option_start, len(schema)))
+ schema += tokens("\n")
+ schema += tokens("END FIELD\n")
+ questions.append(
+ (QUESTION_TYPES.index(question.type), question_span, option_spans)
+ )
+
+ prefix = tokens(_PREFIX)
+ if request.images:
+ prefix += tokens(_IMAGE * len(request.images) + "\n")
+ suffix = tokens(_SUFFIX)
+ # Each image's single <|image_pad|> becomes that image's tokens.
+ expansion = sum(count - 1 for count in image_token_counts)
+ fixed_length = len(prefix) + expansion + len(schema) + len(suffix)
+ if fixed_length > max_length:
+ raise ValueError(
+ f"the questions need {fixed_length} prompt tokens before the state, "
+ f"but the prompt holds at most {max_length}"
+ )
+ state = tokens(_render(request.state))[: max_length - fixed_length]
+ offset = len(prefix) + len(state)
+ input_ids = prefix + state + schema + suffix
+ layout = pack_decision_layout(
+ len(input_ids),
+ [
+ LayoutQuestion(
+ question_type,
+ (question_span[0] + offset, question_span[1] + offset),
+ tuple((start + offset, end + offset) for start, end in option_spans),
+ )
+ for question_type, question_span, option_spans in questions
+ ],
+ )
+ return input_ids, layout
diff --git a/python/sglang/srt/entrypoints/systemone/serving.py b/python/sglang/srt/entrypoints/systemone/serving.py
index 4496d50933..c966e3b670 100644
--- a/python/sglang/srt/entrypoints/systemone/serving.py
+++ b/python/sglang/srt/entrypoints/systemone/serving.py
@@ -2,14 +2,20 @@
from __future__ import annotations
+import asyncio
+import dataclasses
import json
import math
import string
+from io import BytesIO
from typing import Any, Dict, Iterator, List, Optional, Tuple
+from urllib.parse import unquote, urlparse
import orjson
+import pybase64
from fastapi import Request
from fastapi.responses import ORJSONResponse
+from PIL import Image
from sglang.srt.entrypoints.openai.serving_decisions import (
EncodedQuestion,
@@ -22,6 +28,11 @@ from sglang.srt.entrypoints.openai.serving_decisions import (
render_question,
render_text,
)
+from sglang.srt.entrypoints.systemone.joint_schema import (
+ REFERENCE_MAX_LENGTH,
+ encode_joint_schema,
+ joint_schema_options,
+)
from sglang.srt.entrypoints.systemone.protocol import (
SystemOneChoiceAnswer,
SystemOneChoiceQuestion,
@@ -33,6 +44,10 @@ from sglang.srt.entrypoints.systemone.protocol import (
SystemOneScoreAnswer,
SystemOneUsage,
)
+from sglang.srt.layers.joint_schema_head import max_joint_prompt_tokens
+from sglang.srt.managers.io_struct import EmbeddingReqInput
+from sglang.srt.runtime_context import get_schedule
+from sglang.srt.utils import CLIENT_MEDIA_EXCEPTIONS, ImageData, get_image_bytes
# Beyond A to Z, every option gets a two-letter label, in this fixed order.
_PAIR_LABELS = [a + b for a in string.ascii_uppercase for b in string.ascii_uppercase]
@@ -44,9 +59,16 @@ _DECIDER_SYSTEM = (
"Reply with only the selected option code."
)
+# Image sources whose content can change between reads, as load_image reads them.
+_MUTABLE_SOURCES = ("http://", "https://", "file://", "/")
+# Base64 characters decoded to size an inline image, 48 KiB of image data.
+# Arbitrary; an image whose header runs longer is decoded in full.
+_INLINE_HEADER_CHARS = 1 << 16
+
class SystemOneServing(OpenAIServingDecisions):
- """Answers System One questions with the rendering, label checks, and scoring of /v1/decisions."""
+ """Answers System One questions with the rendering, label checks, and scoring of
+ /v1/decisions, or, for a Clef checkpoint, with its joint schema head."""
route = "/v1/systemone"
@@ -54,12 +76,36 @@ class SystemOneServing(OpenAIServingDecisions):
return "systemone-"
def _validate_request(self, request: SystemOneRequest) -> Optional[str]:
+ if self.joint_head_config is not None:
+ return self._validate_joint_schema_request(request)
return (
self._validate_server(request.model)
or self._validate_images(request.images)
or self._validate_reasoning(request.chat_template_kwargs)
)
+ def _validate_joint_schema_request(
+ self, request: SystemOneRequest
+ ) -> Optional[str]:
+ """A Clef checkpoint renders its own prompt, without the chat template."""
+ route = self.route
+ if self.tokenizer_manager.tokenizer is None:
+ return f"{route} requires the server tokenizer"
+ _, adapter = self._parse_model_parameter(request.model)
+ if adapter is not None:
+ return (
+ f"model names the LoRA adapter {adapter!r}, which {route} "
+ "does not support"
+ )
+ if request.images and self.tokenizer_manager.mm_processor is None:
+ return f"{route} images require a model served with multimodal input"
+ if request.chat_template_kwargs:
+ return (
+ f"{route} renders the prompt this checkpoint's joint schema head "
+ "was trained with, which takes no chat_template_kwargs"
+ )
+ return None
+
def _convert_to_internal_request(
self,
request: SystemOneRequest,
@@ -69,6 +115,8 @@ class SystemOneServing(OpenAIServingDecisions):
Tuple[SystemOneRequest, List[QuestionView]],
]:
views = [_view(question) for question in request.questions.values()]
+ if self.joint_head_config is not None:
+ return self._joint_schema_request(request, views), (request, views)
encode = (
self._encoded_systemone_questions
if self.decision_config is None
@@ -77,6 +125,118 @@ class SystemOneServing(OpenAIServingDecisions):
# Lazy, so the async handler can yield to other requests between questions.
return encode(request, views), (request, views)
+ def _joint_schema_request(
+ self, request: SystemOneRequest, views: List[QuestionView]
+ ) -> EmbeddingReqInput:
+ """All questions in one prompt, which the joint schema head scores in one prefill.
+
+ The handler encodes the prompt once it has read the image sizes."""
+ for question_id, view in zip(request.questions, views):
+ if view.kind == "score":
+ _check_legend(question_id, view)
+ return EmbeddingReqInput(
+ image_data=[
+ ImageData(url=image.url, detail=image.detail or "auto")
+ for image in request.images
+ ]
+ or None,
+ )
+
+ async def _encode_joint_schema_prompt(
+ self, adapted_request: EmbeddingReqInput, request: SystemOneRequest
+ ) -> None:
+ tokenizer_manager = self.tokenizer_manager
+ max_length = min(
+ REFERENCE_MAX_LENGTH,
+ max_joint_prompt_tokens(
+ context_len=tokenizer_manager.context_len,
+ num_reserved_tokens=tokenizer_manager.num_reserved_tokens,
+ max_req_input_len=tokenizer_manager.max_req_input_len,
+ max_prefill_tokens=get_schedule().max_prefill_tokens,
+ ),
+ )
+ image_token_counts = []
+ if adapted_request.image_data:
+ adapted_request.image_data, image_token_counts = await self._read_images(
+ adapted_request.image_data
+ )
+ input_ids, layout = encode_joint_schema(
+ tokenizer_manager.tokenizer,
+ request,
+ max_length=max_length,
+ image_token_counts=image_token_counts,
+ )
+ adapted_request.input_ids = input_ids
+ adapted_request.decision_layout = layout
+
+ async def _read_images(
+ self, images: List[ImageData]
+ ) -> Tuple[List[ImageData], List[int]]:
+ """Each image read once, and the tokens it expands to at the size the
+ processor resizes it to."""
+ read = await asyncio.gather(
+ *(
+ asyncio.to_thread(_read_image, index, image)
+ for index, image in enumerate(images)
+ )
+ )
+ counts = self.tokenizer_manager.mm_processor.resolve_image_token_counts(
+ [header for _, header in read]
+ )
+ return [image for image, _ in read], counts
+
+ async def _answer_joint_schema(
+ self,
+ adapted_request: EmbeddingReqInput,
+ request: SystemOneRequest,
+ views: List[QuestionView],
+ raw_request: Request,
+ ) -> ORJSONResponse:
+ await self._encode_joint_schema_prompt(adapted_request, request)
+ ret = await self.tokenizer_manager.generate_request(
+ adapted_request, raw_request
+ ).__anext__()
+ logits = ret["embedding"]
+ names = [
+ [name for name, _ in joint_schema_options(question)]
+ for question in request.questions.values()
+ ]
+ if len(logits) != sum(map(len, names)):
+ raise RuntimeError("the joint schema head did not score this request")
+ answers = {}
+ offset = 0
+ for question_id, view, question_names in zip(request.questions, views, names):
+ scores = logits[offset : offset + len(question_names)]
+ offset += len(question_names)
+ top = max(scores)
+ weights = [math.exp(score - top) for score in scores]
+ total = math.fsum(weights)
+ by_name = {
+ name: weight / total for name, weight in zip(question_names, weights)
+ }
+ # The head scores true then false, which the view names yes and no.
+ answers[question_id] = _answer(
+ view=view,
+ probabilities=(
+ [by_name["true"], by_name["false"]]
+ if view.kind == "yes_no"
+ else [by_name[name] for name in view.names]
+ ),
+ mass=1.0,
+ question_id=question_id,
+ )
+ response = SystemOneResponse(
+ model=self.tokenizer_manager.served_model_name,
+ answers=answers,
+ usage=SystemOneUsage(input_tokens=ret["meta_info"]["prompt_tokens"]),
+ )
+ # The head scores only the allowed options, so there is no label mass.
+ return ORJSONResponse(
+ content=response.model_dump(
+ exclude={"answers": {"__all__": {"x_label_mass"}}}
+ )
+ )
+
def _encoded_systemone_questions(
self, request: SystemOneRequest, views: List[QuestionView]
) -> Iterator[EncodedQuestion]:
@@ -179,6 +339,10 @@ class SystemOneServing(OpenAIServingDecisions):
raw_request: Request,
) -> ORJSONResponse:
request, views = processed
+ if self.joint_head_config is not None:
+ return await self._answer_joint_schema(
+ adapted_request, request, views, raw_request
+ )
_, _, result = await self._score(
adapted_request=adapted_request,
raw_request=raw_request,
@@ -344,3 +508,38 @@ def _score_confidence(q: List[float]) -> float:
spread = math.fsum(p * abs(i - top) for i, p in enumerate(q))
uniform_spread = math.fsum(abs(i - (n - 1) / 2) for i in range(n)) / n
return max(0.0, 1 - spread / uniform_spread)
+
+
+def _read_image(index: int, image: ImageData) -> Tuple[ImageData, Image.Image]:
+ """The image to forward, and its header opened only as far as its size, read
+ once from the sources load_image reads."""
+ url = image.url
+ if not url.startswith(_MUTABLE_SOURCES):
+ # An inline image is forwarded as sent, and the processor decodes all of it.
+ header = _inline_header(url)
+ if header is not None:
+ return image, header
+ source = unquote(urlparse(url).path) if url.startswith("file://") else url
+ try:
+ data = get_image_bytes(source)
+ header = Image.open(BytesIO(data))
+ except CLIENT_MEDIA_EXCEPTIONS as e:
+ raise ValueError(f"image {index} could not be read: {e}") from e
+ if url.startswith(_MUTABLE_SOURCES):
+ # A source that can change between reads is forwarded as the bytes counted.
+ mime = Image.MIME.get(header.format, "application/octet-stream")
+ encoded = pybase64.b64encode(data).decode()
+ image = dataclasses.replace(image, url=f"data:{mime};base64,{encoded}")
+ return image, header
+
+
+def _inline_header(url: str) -> Optional[Image.Image]:
+ """An inline image's header from a prefix of its base64, or None when the
+ header does not open from that prefix."""
+ start = url.find(",") + 1 if url.startswith("data:") else 0
+ prefix = url[start : start + _INLINE_HEADER_CHARS]
+ try:
+ return Image.open(BytesIO(pybase64.b64decode(prefix, validate=True)))
+ except Exception:
+ # A header past the prefix, or malformed data that the full read reports.
+ return None
diff --git a/python/sglang/srt/layers/joint_schema_head.py b/python/sglang/srt/layers/joint_schema_head.py
new file mode 100644
index 0000000000..0e939bdee1
--- /dev/null
+++ b/python/sglang/srt/layers/joint_schema_head.py
@@ -0,0 +1,539 @@
+"""Joint schema head of Clef decision checkpoints.
+
+A Clef checkpoint is a Qwen3.5 backbone saved with ``joint_head_config.json``
+and ``joint_head.safetensors``. The head reads the backbone's final hidden
+state at every prompt token and scores every allowed option of every question
+together: each option gathers evidence from the prompt, the questions attend
+to each other and to the prompt, and a lexical prior built from the LM head
+rows of each option's tokens keeps the option's meaning.
+
+One head pass serves a whole prefill batch. Token-wise steps run on the flat
+batch, attention is variable-length per request, and span means come from
+per-request prefix sums over the schema region only, so a long state costs no
+more than its memory projection.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Callable, List, Optional, Sequence, Tuple
+
+import msgspec
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from sglang.srt.layers.pooler import EmbeddingPoolerOutput
+
+# Question types in the order of the head's type embedding.
+QUESTION_TYPES = ("noul", "choice", "score")
+
+# Bounds of one head pass, so a large prefill batch is scored in a few passes.
+_MAX_GROUP_TOKENS = 65536
+_MAX_GROUP_SLOTS = 1 << 18
+
+
+class LayoutQuestion(msgspec.Struct, frozen=True):
+ question_type: int
+ question_span: Tuple[int, int]
+ option_spans: Tuple[Tuple[int, int], ...]
+
+
+def max_joint_prompt_tokens(
+ *,
+ context_len: int,
+ num_reserved_tokens: int,
+ max_req_input_len: int,
+ max_prefill_tokens: int,
+) -> int:
+ """The longest prompt, images expanded, that the server runs as one prefill.
+
+ The tokenizer manager refuses a prompt of context_len - num_reserved_tokens
+ tokens, the scheduler one of max_req_input_len, and without chunked prefill
+ activation memory is reserved for max_prefill_tokens.
+ """
+ return min(
+ context_len - num_reserved_tokens - 1,
+ max_req_input_len - 1,
+ max_prefill_tokens,
+ )
+
+
+def pack_decision_layout(
+ prompt_length: int, questions: Sequence[LayoutQuestion]
+) -> List[int]:
+ """Flatten a prompt's questions into the ints a request carries.
+
+ The layout is ``[prompt_length, num_questions]`` followed, per question,
+ by ``type, question_start, question_end, num_options`` and one
+ ``option_start, option_end`` pair per option. Positions index the prompt
+ before its image placeholders expand.
+ """
+ layout = [prompt_length, len(questions)]
+ for question in questions:
+ layout += [
+ question.question_type,
+ *question.question_span,
+ len(question.option_spans),
+ ]
+ for start, end in question.option_spans:
+ layout += [start, end]
+ return layout
+
+
+def parse_decision_layout(
+ layout: Sequence[int], num_tokens: int
+) -> List[LayoutQuestion]:
+ """The questions of a layout, placed on the sequence the model reads.
+
+ Every span follows the prompt's images, so expanding their placeholders
+ shifts all spans by the number of added tokens. Raises ValueError when the
+ layout does not describe the sequence.
+ """
+ if len(layout) < 2 or any(type(value) is not int for value in layout):
+ raise ValueError("decision_layout must be a list of ints")
+ prompt_length, num_questions = layout[0], layout[1]
+ shift = num_tokens - prompt_length
+ if prompt_length < 1 or shift < 0 or num_questions < 1:
+ raise ValueError("decision_layout does not match the prompt length")
+ questions = []
+ cursor = 2
+ for _ in range(num_questions):
+ if cursor + 4 > len(layout):
+ raise ValueError("decision_layout is truncated")
+ question_type, start, end, num_options = layout[cursor : cursor + 4]
+ cursor += 4
+ if (
+ not 0 <= question_type < len(QUESTION_TYPES)
+ or num_options < 1
+ or not 0 <= start < end <= prompt_length
+ or cursor + 2 * num_options > len(layout)
+ ):
+ raise ValueError("decision_layout has an invalid question")
+ spans = []
+ for _ in range(num_options):
+ option_start, option_end = layout[cursor : cursor + 2]
+ cursor += 2
+ if not 0 <= option_start < option_end <= prompt_length:
+ raise ValueError("decision_layout has an invalid option span")
+ spans.append((option_start + shift, option_end + shift))
+ questions.append(
+ LayoutQuestion(question_type, (start + shift, end + shift), tuple(spans))
+ )
+ if cursor != len(layout):
+ raise ValueError("decision_layout has trailing values")
+ return questions
+
+
+class EvidenceRoutingLayer(nn.Module):
+ def __init__(self, width: int, heads: int, feedforward: int) -> None:
+ super().__init__()
+ self.query_norm = nn.LayerNorm(width)
+ self.memory_norm = nn.LayerNorm(width)
+ self.attention = nn.MultiheadAttention(width, heads, batch_first=True)
+ self.feedforward_norm = nn.LayerNorm(width)
+ # Identity slots keep the checkpoint's Sequential indices (dropout at 2 and 4).
+ self.feedforward = nn.Sequential(
+ nn.Linear(width, feedforward),
+ nn.GELU(),
+ nn.Identity(),
+ nn.Linear(feedforward, width),
+ nn.Identity(),
+ )
+
+
+class JointSchemaHead(nn.Module):
+ """Parameters named as in the checkpoint's ``joint_head.safetensors``."""
+
+ def __init__(
+ self,
+ hidden_size: int,
+ width: int,
+ routing_layers: int,
+ layers: int,
+ heads: int,
+ feedforward: int,
+ ) -> None:
+ super().__init__()
+ self.hidden_norm = nn.LayerNorm(hidden_size)
+ self.memory_projection = nn.Linear(hidden_size, width, bias=False)
+ self.question_projection = nn.Linear(hidden_size, width, bias=False)
+ self.option_question_projection = nn.Linear(hidden_size, width, bias=False)
+ self.global_projection = nn.Linear(hidden_size, width, bias=False)
+ self.option_context_projection = nn.Linear(hidden_size, width, bias=False)
+ self.option_lexical_projection = nn.Linear(hidden_size, width, bias=False)
+ self.type_embedding = nn.Embedding(len(QUESTION_TYPES), width)
+ self.evidence_layers = nn.ModuleList(
+ EvidenceRoutingLayer(width, heads, feedforward)
+ for _ in range(routing_layers)
+ )
+ self.option_summary_norm = nn.LayerNorm(width)
+ self.layers = nn.ModuleList(
+ nn.TransformerDecoderLayer(
+ d_model=width,
+ nhead=heads,
+ dim_feedforward=feedforward,
+ dropout=0.0,
+ activation="gelu",
+ batch_first=True,
+ norm_first=True,
+ )
+ for _ in range(layers)
+ )
+ self.field_norm = nn.LayerNorm(width)
+ self.option_norm = nn.LayerNorm(width)
+ self.residual_scorer = nn.Sequential(
+ nn.Linear(width * 4, width), nn.GELU(), nn.Identity(), nn.Linear(width, 1)
+ )
+ self.prior_logit_scale = nn.Parameter(torch.zeros(()))
+ self.joint_logit_scale = nn.Parameter(torch.zeros(()))
+ self.residual_gate = nn.Parameter(torch.zeros(()))
+
+ @torch.no_grad()
+ def forward_batch(
+ self,
+ hidden: torch.Tensor,
+ input_ids: torch.Tensor,
+ lengths: List[int],
+ layouts: List[List[LayoutQuestion]],
+ output_embedding_weight: torch.Tensor,
+ ) -> List[torch.Tensor]:
+ """One FP32 logit vector per request, its questions' options in layout order.
+
+ ``hidden`` holds the final hidden states of the requests back to back.
+ """
+ outputs: List[torch.Tensor] = []
+ start = 0
+ for group in _groups(lengths, layouts):
+ group_lengths = [lengths[i] for i in group]
+ end = start + sum(group_lengths)
+ outputs += self._forward_group(
+ hidden[start:end],
+ input_ids[start:end],
+ group_lengths,
+ [layouts[i] for i in group],
+ output_embedding_weight,
+ )
+ start = end
+ return outputs
+
+ def _forward_group(
+ self,
+ hidden: torch.Tensor,
+ input_ids: torch.Tensor,
+ lengths: List[int],
+ layouts: List[List[LayoutQuestion]],
+ output_embedding_weight: torch.Tensor,
+ ) -> List[torch.Tensor]:
+ device, dtype = hidden.device, hidden.dtype
+ batch = len(layouts)
+ question_bounds, option_bounds = [], []
+ type_ids, owner, slot, question_request, last_tokens = [], [], [], [], []
+ regions, region_offsets = [], [0]
+ token_cu, question_cu, option_cu, option_counts = [0], [0], [0], []
+ for request, (length, questions) in enumerate(zip(lengths, layouts)):
+ offset = token_cu[-1]
+ low = min(min(q.question_span[0], q.option_spans[0][0]) for q in questions)
+ high = max(
+ max(q.question_span[1], *(e for _, e in q.option_spans))
+ for q in questions
+ )
+ # Prefix row of span position p in this request's schema region.
+ base = region_offsets[-1] + request - low
+ regions.append((offset + low, offset + high))
+ request_options = 0
+ for question in questions:
+ count = len(question.option_spans)
+ question_bounds += (
+ base + question.question_span[0],
+ base + question.question_span[1],
+ )
+ for option_slot, (option_start, option_end) in enumerate(
+ question.option_spans
+ ):
+ option_bounds += (base + option_start, base + option_end)
+ owner.append(len(type_ids))
+ slot.append(option_slot)
+ type_ids.append(question.question_type)
+ question_request.append(request)
+ request_options += count
+ last_tokens.append(offset + length - 1)
+ token_cu.append(offset + length)
+ region_offsets.append(region_offsets[-1] + high - low)
+ question_cu.append(len(type_ids))
+ option_cu.append(option_cu[-1] + request_options)
+ option_counts.append(request_options)
+
+ num_questions = len(type_ids)
+ widest = max(len(q.option_spans) for questions in layouts for q in questions)
+ sizes = [
+ 2 * num_questions,
+ len(option_bounds),
+ num_questions,
+ len(owner),
+ len(slot),
+ num_questions,
+ batch,
+ 3 * (batch + 1),
+ ]
+ packed = torch.tensor(
+ question_bounds
+ + option_bounds
+ + type_ids
+ + owner
+ + slot
+ + question_request
+ + last_tokens
+ + token_cu
+ + question_cu
+ + option_cu,
+ dtype=torch.long,
+ ).to(device, non_blocking=True)
+ (
+ question_bounds_t,
+ option_bounds_t,
+ type_ids_t,
+ owner_t,
+ slot_t,
+ question_request_t,
+ last_tokens_t,
+ cu_t,
+ ) = torch.split(packed, sizes)
+ question_bounds_t = question_bounds_t.view(-1, 2)
+ option_bounds_t = option_bounds_t.view(-1, 2)
+ token_cu_t, question_cu_t, option_cu_t = (
+ cu_t.to(torch.int32).view(3, batch + 1).unbind(0)
+ )
+ token_bounds = (token_cu_t, max(lengths))
+ question_bounds_cu = (question_cu_t, max(len(q) for q in layouts))
+ option_bounds_cu = (option_cu_t, max(option_counts))
+
+ normalized = self.hidden_norm(hidden)
+ # Running sums over each schema region, after one zero row per request.
+ region_total = region_offsets[-1] + batch
+ hidden_prefix = torch.zeros(
+ region_total, normalized.shape[-1], dtype=torch.float32, device=device
+ )
+ lexical_prefix = torch.zeros(
+ region_total,
+ output_embedding_weight.shape[-1],
+ dtype=torch.float32,
+ device=device,
+ )
+ for request, (low, high) in enumerate(regions):
+ row = region_offsets[request] + request + 1
+ torch.cumsum(
+ normalized[low:high].float(),
+ dim=0,
+ out=hidden_prefix[row : row + high - low],
+ )
+ torch.cumsum(
+ output_embedding_weight[input_ids[low:high]].float(),
+ dim=0,
+ out=lexical_prefix[row : row + high - low],
+ )
+
+ def span_means(prefix: torch.Tensor, bounds: torch.Tensor) -> torch.Tensor:
+ sums = prefix[bounds[:, 1]] - prefix[bounds[:, 0]]
+ return (sums / (bounds[:, 1] - bounds[:, 0]).unsqueeze(-1)).to(dtype)
+
+ memory = self.memory_projection(normalized)
+ global_vectors = normalized[last_tokens_t]
+ question_vectors = span_means(hidden_prefix, question_bounds_t)
+ option_contexts = span_means(hidden_prefix, option_bounds_t)
+ lexical = span_means(lexical_prefix, option_bounds_t)
+
+ routed = (
+ self.option_context_projection(option_contexts)
+ + self.option_lexical_projection(lexical)
+ + self.option_question_projection(question_vectors)[owner_t]
+ )
+ for layer in self.evidence_layers:
+ routed = routed + _attention(
+ layer.attention,
+ layer.query_norm(routed),
+ layer.memory_norm(memory),
+ option_bounds_cu,
+ token_bounds,
+ )
+ routed = routed + layer.feedforward(layer.feedforward_norm(routed))
+
+ base_fields = self.question_projection(question_vectors)
+ scores = torch.matmul(
+ routed.unsqueeze(1), base_fields[owner_t].unsqueeze(2)
+ ).view(-1) / math.sqrt(routed.shape[-1])
+ padded_scores = torch.full(
+ (num_questions, widest), float("-inf"), dtype=torch.float32, device=device
+ )
+ padded_scores[owner_t, slot_t] = scores.float()
+ weights = torch.softmax(padded_scores, dim=1).to(dtype)
+ padded_options = routed.new_zeros(num_questions, widest, routed.shape[-1])
+ padded_options[owner_t, slot_t] = routed
+ summaries = torch.sum(weights.unsqueeze(-1) * padded_options, dim=1)
+ fields = (
+ base_fields
+ + self.option_summary_norm(summaries)
+ + self.global_projection(global_vectors)[question_request_t]
+ + self.type_embedding(type_ids_t)
+ )
+ for layer in self.layers:
+ fields = fields + _attention(
+ layer.self_attn,
+ layer.norm1(fields),
+ None,
+ question_bounds_cu,
+ question_bounds_cu,
+ )
+ fields = fields + _attention(
+ layer.multihead_attn,
+ layer.norm2(fields),
+ memory,
+ question_bounds_cu,
+ token_bounds,
+ )
+ fields = fields + layer.linear2(
+ layer.activation(layer.linear1(layer.norm3(fields)))
+ )
+ fields = self.field_norm(fields)
+
+ prior_scale = self.prior_logit_scale.clamp(max=math.log(100.0)).exp()
+ joint_scale = self.joint_logit_scale.clamp(max=math.log(100.0)).exp()
+ anchor = F.normalize(
+ question_vectors + global_vectors[question_request_t], dim=-1
+ )
+ prior = prior_scale * torch.matmul(
+ F.normalize(lexical, dim=-1).unsqueeze(1), anchor[owner_t].unsqueeze(2)
+ ).view(-1)
+ options = self.option_norm(routed)
+ repeated = fields[owner_t]
+ cosine = F.cosine_similarity(repeated, options, dim=-1)
+ features = torch.cat(
+ [repeated, options, repeated * options, torch.abs(repeated - options)],
+ dim=-1,
+ )
+ residual = self.residual_scorer(features).squeeze(-1)
+ logits = prior + torch.sigmoid(self.residual_gate) * (
+ joint_scale * cosine + residual
+ )
+ return list(torch.split(logits.float(), option_counts))
+
+
+def _attention(
+ module: nn.MultiheadAttention,
+ queries: torch.Tensor,
+ keys: Optional[torch.Tensor],
+ query_bounds: Tuple[torch.Tensor, int],
+ key_bounds: Tuple[torch.Tensor, int],
+) -> torch.Tensor:
+ """nn.MultiheadAttention over packed sequences, keys and values from ``keys``
+ (or from ``queries`` when None), attending within each request."""
+ width = queries.shape[-1]
+ heads = module.num_heads
+ if keys is None:
+ query, key, value = F.linear(
+ queries, module.in_proj_weight, module.in_proj_bias
+ ).chunk(3, dim=-1)
+ else:
+ query_weight, key_value_weight = module.in_proj_weight.split([width, 2 * width])
+ query_bias, key_value_bias = module.in_proj_bias.split([width, 2 * width])
+ query = F.linear(queries, query_weight, query_bias)
+ key, value = F.linear(keys, key_value_weight, key_value_bias).chunk(2, dim=-1)
+ head_dim = width // heads
+ output = torch.ops.aten._flash_attention_forward(
+ query.reshape(-1, heads, head_dim),
+ key.reshape(-1, heads, head_dim),
+ value.reshape(-1, heads, head_dim),
+ query_bounds[0],
+ key_bounds[0],
+ query_bounds[1],
+ key_bounds[1],
+ 0.0,
+ False,
+ False,
+ )[0]
+ return module.out_proj(output.reshape(-1, width))
+
+
+def _groups(lengths: List[int], layouts: List[List[LayoutQuestion]]) -> List[range]:
+ """Consecutive requests per head pass, bounded in tokens and padded option slots."""
+ groups, start, tokens, questions, widest = [], 0, 0, 0, 0
+ for index, (length, layout) in enumerate(zip(lengths, layouts)):
+ layout_widest = max(len(question.option_spans) for question in layout)
+ grown = (questions + len(layout)) * max(widest, layout_widest)
+ if index > start and (
+ tokens + length > _MAX_GROUP_TOKENS or grown > _MAX_GROUP_SLOTS
+ ):
+ groups.append(range(start, index))
+ start, tokens, questions, widest = index, 0, 0, 0
+ tokens += length
+ questions += len(layout)
+ widest = max(widest, layout_widest)
+ groups.append(range(start, len(lengths)))
+ return groups
+
+
+class JointSchemaPooler(nn.Module):
+ """Pooler of a Clef checkpoint: the joint schema head's option logits per request."""
+
+ def __init__(
+ self, head: JointSchemaHead, output_embedding: Callable[[], torch.Tensor]
+ ) -> None:
+ super().__init__()
+ self.head = head
+ # A callable, so the LM head is not registered twice as a parameter.
+ self._output_embedding = output_embedding
+
+ def forward(
+ self, hidden_states: torch.Tensor, forward_batch
+ ) -> EmbeddingPoolerOutput:
+ if (
+ forward_batch.decision_layouts is None
+ or forward_batch.extend_seq_lens_cpu is None
+ ):
+ return EmbeddingPoolerOutput(
+ embeddings=[
+ hidden_states.new_empty(0, dtype=torch.float32)
+ for _ in range(forward_batch.batch_size)
+ ]
+ )
+ lengths = list(forward_batch.extend_seq_lens_cpu)
+ prefix_lengths = forward_batch.extend_prefix_lens_cpu or [0] * len(lengths)
+ raw_layouts = forward_batch.decision_layouts
+ embeddings: List[torch.Tensor] = [
+ hidden_states.new_empty(0, dtype=torch.float32) for _ in lengths
+ ]
+ # The tokenizer manager checks every layout against its prompt, and the
+ # server runs each prompt in one uncached prefill. A layout that no longer
+ # fits its prompt, as after a truncation, gets no scores, which the
+ # decision route reports as a server error.
+ layouts = {}
+ for i, (layout, prefix, length) in enumerate(
+ zip(raw_layouts, prefix_lengths, lengths)
+ ):
+ if layout is None or prefix != 0:
+ continue
+ try:
+ layouts[i] = parse_decision_layout(layout, length)
+ except ValueError:
+ continue
+ offsets = [0]
+ for length in lengths:
+ offsets.append(offsets[-1] + length)
+ runs: List[List[int]] = []
+ for i in layouts:
+ if runs and runs[-1][-1] == i - 1:
+ runs[-1].append(i)
+ else:
+ runs.append([i])
+ for run in runs:
+ start, end = offsets[run[0]], offsets[run[-1] + 1]
+ logits = self.head.forward_batch(
+ hidden_states[start:end],
+ forward_batch.input_ids[start:end],
+ [lengths[i] for i in run],
+ [layouts[i] for i in run],
+ self._output_embedding(),
+ )
+ for i, values in zip(run, logits):
+ embeddings[i] = values
+ return EmbeddingPoolerOutput(embeddings=embeddings)
diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py
index 3ba6045ff6..028226a683 100644
--- a/python/sglang/srt/managers/io_struct.py
+++ b/python/sglang/srt/managers/io_struct.py
@@ -1253,6 +1253,11 @@ class EmbeddingReqInput:
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
token_indices_to_pool: Optional[Union[List[List[int]], List[int]]] = None
+ # Question and option spans that a Clef checkpoint's joint schema head
+ # scores, as packed by layers.joint_schema_head.pack_decision_layout.
+ # Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
+ decision_layout: Optional[Union[List[List[int]], List[int]]] = None
+
def regenerate_rid(self):
"""Generate a new request ID and return it."""
if isinstance(self.rid, list):
@@ -1469,6 +1474,11 @@ class EmbeddingReqInput:
if self.token_indices_to_pool is not None
else None
),
+ decision_layout=(
+ self.decision_layout[i]
+ if self.decision_layout is not None
+ else None
+ ),
)
cache[i] = sub
return sub
@@ -1500,6 +1510,8 @@ class TokenizedEmbeddingReqInput(BaseReq, kw_only=True):
multi_item_delimiter_indices: Optional[List[int]] = None
# Token positions for setwise pooling readout
token_indices_to_pool: Optional[List[int]] = None
+ # Question and option spans for a joint schema head
+ decision_layout: Optional[List[int]] = None
# For observability
# Pickled Optional[Union[APIServerReqTimeStats, DPControllerReqTimeStats]]
time_stats: Optional[PickleWrapper] = None
diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py
index 8346c93c3d..a964f40505 100755
--- a/python/sglang/srt/managers/schedule_batch.py
+++ b/python/sglang/srt/managers/schedule_batch.py
@@ -1050,6 +1050,7 @@ class Req(ReqDllmMixin):
return_pooled_hidden_states: bool = False,
multi_item_delimiter_indices: Optional[List[int]] = None,
token_indices_to_pool: Optional[List[int]] = None,
+ decision_layout: Optional[List[int]] = None,
session_id: Optional[str] = None,
cache_salt: Optional[str] = None,
):
@@ -1083,6 +1084,7 @@ class Req(ReqDllmMixin):
self.positional_embed_overrides = positional_embed_overrides
self.multi_item_delimiter_indices = multi_item_delimiter_indices
self.token_indices_to_pool = token_indices_to_pool
+ self.decision_layout = decision_layout
# For req-level memory management
self.kv = ReqKvInfo()
diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py
index 4622e32faf..1a3f60b857 100644
--- a/python/sglang/srt/managers/scheduler.py
+++ b/python/sglang/srt/managers/scheduler.py
@@ -3465,6 +3465,7 @@ class Scheduler(
return_pooled_hidden_states=recv_req.return_pooled_hidden_states,
multi_item_delimiter_indices=recv_req.multi_item_delimiter_indices,
token_indices_to_pool=recv_req.token_indices_to_pool,
+ decision_layout=recv_req.decision_layout,
)
req.tokenizer = self.tokenizer
self._maybe_namespace_elastic_radix_cache(req)
diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py
index b3975bf3f8..aebe9ea6b8 100644
--- a/python/sglang/srt/managers/tokenizer_manager.py
+++ b/python/sglang/srt/managers/tokenizer_manager.py
@@ -64,6 +64,10 @@ from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX
from sglang.srt.disaggregation.encoder.receiver import create_mm_receiver
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.environ import envs
+from sglang.srt.layers.joint_schema_head import (
+ max_joint_prompt_tokens,
+ parse_decision_layout,
+)
from sglang.srt.lora.lora_registry import LoRARef, LoRARegistry
from sglang.srt.managers.async_dynamic_batch_tokenizer import AsyncDynamicbatchTokenizer
from sglang.srt.managers.disagg_service import start_disagg_service
@@ -1295,6 +1299,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
raise ValueError(
"encoder SWA replay cannot return cached prompt logprobs"
)
+ # A decision layout indexes the whole prompt, so it is checked before any
+ # truncation below could cut the prompt.
+ self._validate_joint_schema_request(obj, input_ids)
_max_req_len = self.context_len
input_token_num = len(input_ids) if input_ids is not None else 0
input_token_num += self.num_reserved_tokens
@@ -1399,6 +1406,36 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
"return_sampling_mask=true."
)
+ def _validate_joint_schema_request(
+ self, obj: Union[GenerateReqInput, EmbeddingReqInput], input_ids: List[int]
+ ) -> None:
+ """A Clef checkpoint scores the spans of a decision layout, and nothing else."""
+ if self.model_config.joint_head_config is None:
+ return
+ if isinstance(obj, GenerateReqInput) or obj.decision_layout is None:
+ # Health checks only need a response, which is an empty embedding.
+ if (
+ isinstance(obj, EmbeddingReqInput)
+ and isinstance(obj.rid, str)
+ and obj.rid.startswith(HEALTH_CHECK_RID_PREFIX)
+ ):
+ return
+ raise ValueError(
+ "This checkpoint answers schema decisions, send them to /v1/systemone"
+ )
+ limit = max_joint_prompt_tokens(
+ context_len=self.context_len,
+ num_reserved_tokens=self.num_reserved_tokens,
+ max_req_input_len=self.max_req_input_len,
+ max_prefill_tokens=get_schedule().max_prefill_tokens,
+ )
+ if len(input_ids) > limit:
+ raise ValueError(
+ f"The prompt has {len(input_ids)} tokens, but this server scores "
+ f"at most {limit} in one prefill"
+ )
+ parse_decision_layout(obj.decision_layout, len(input_ids))
+
def _validate_mm_limits(
self, obj: Union[GenerateReqInput, EmbeddingReqInput]
) -> None:
@@ -1575,6 +1612,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
return_pooled_hidden_states=obj.return_pooled_hidden_states,
multi_item_delimiter_indices=obj.multi_item_delimiter_indices,
token_indices_to_pool=obj.token_indices_to_pool,
+ decision_layout=obj.decision_layout,
)
tokenized_obj.time_stats = self.rid_to_state[obj.rid].time_stats
diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py
index a7cbd93559..1efa971f40 100644
--- a/python/sglang/srt/model_executor/forward_batch_info.py
+++ b/python/sglang/srt/model_executor/forward_batch_info.py
@@ -614,6 +614,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# Setwise pooling readout positions (CPU tensors, one per request)
token_indices_to_pool: Optional[List[torch.Tensor]] = None
+ # Joint schema head decision layouts (one per request, None if absent)
+ decision_layouts: Optional[List[Optional[List[int]]]] = None
+
# === Borrowed from ScheduleBatch: compound (carry their own device tensors) ===
# Sampling info
sampling_info: SamplingBatchInfo = None
@@ -1264,6 +1267,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
for r in batch.reqs
]
+ if any(r.decision_layout is not None for r in batch.reqs):
+ self.decision_layouts = [r.decision_layout for r in batch.reqs]
+
token_type_ids = [
r.token_type_ids for r in batch.reqs if r.token_type_ids is not None
]
diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py
index 81bdfbb56f..1f1e30dd52 100644
--- a/python/sglang/srt/models/qwen3_5.py
+++ b/python/sglang/srt/models/qwen3_5.py
@@ -34,7 +34,10 @@ from sglang.kernels.ops.attention.triton_gdn_fused_proj import (
from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul
# Configs
-from sglang.srt.configs.model_config import load_decision_config
+from sglang.srt.configs.model_config import (
+ load_decision_config,
+ load_joint_head_config,
+)
from sglang.srt.configs.qwen3_5 import (
Qwen3_5Config,
Qwen3_5MoeConfig,
@@ -54,6 +57,7 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
+from sglang.srt.layers.joint_schema_head import JointSchemaHead, JointSchemaPooler
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
@@ -2330,6 +2334,16 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration):
self.visual.deepstack_visual_indexes if self.visual is not None else []
)
+ if config.architectures == ["Qwen3_5ForConditionalGeneration"]:
+ model = get_model()
+ joint_head_config = load_joint_head_config(model.model_path, model.revision)
+ if joint_head_config is not None:
+ # A Clef checkpoint, whose joint schema head scores every option
+ # of every question from the final hidden states and LM head rows.
+ self.pooler = JointSchemaPooler(
+ JointSchemaHead(**joint_head_config), lambda: self.lm_head.weight
+ )
+
def get_hidden_dim(self, module_name: str, layer_idx: int):
return self.model.get_hidden_dim(module_name, layer_idx)
@@ -2470,8 +2484,23 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration):
if bare_backbone and self.pp_group.is_last_rank:
self._load_decision_readout()
loaded_params.add("lm_head.weight")
+ if isinstance(self.pooler, JointSchemaPooler):
+ self._load_joint_head()
+ loaded_params.update(
+ f"pooler.head.{name}" for name, _ in self.pooler.head.named_parameters()
+ )
return loaded_params
+ def _load_joint_head(self) -> None:
+ """Load joint_head.safetensors, which the checkpoint index leaves out."""
+ model = get_model()
+ state_dict = load_file(
+ cached_file(
+ model.model_path, "joint_head.safetensors", revision=model.revision
+ )
+ )
+ self.pooler.head.load_state_dict(state_dict, strict=True)
+
def _load_decision_readout(self) -> None:
"""Place the checkpoint's decision readout in the LM head rows of its codes."""
model = get_model()
diff --git a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py
index 035cd667a0..5e9c92d103 100644
--- a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py
+++ b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py
@@ -169,6 +169,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
args = ServerArgs(model_path="dummy")
args._model_config = SimpleNamespace(
is_embedding_gemma=True,
+ joint_head_config=None,
is_multimodal=False,
context_len=2048,
hf_config=SimpleNamespace(architectures=["Gemma3TextModel"]),
@@ -205,6 +206,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
is_embedding_requested=False,
is_embedding_gemma=False,
),
+ joint_head_config=None,
is_multimodal=False,
hf_config=SimpleNamespace(architectures=["BertModel"]),
)
diff --git a/test/registered/unit/entrypoints/openai/test_serving_decisions.py b/test/registered/unit/entrypoints/openai/test_serving_decisions.py
index 9b6dcf8bad..a561a04431 100644
--- a/test/registered/unit/entrypoints/openai/test_serving_decisions.py
+++ b/test/registered/unit/entrypoints/openai/test_serving_decisions.py
@@ -159,6 +159,7 @@ class ScoringManager(TokenizerManagerScoreMixin):
model_type="qwen3_5_moe",
),
decision_config=None,
+ joint_head_config=None,
)
pad, video = (
tokenizer.encode(token, add_special_tokens=False)[0]
diff --git a/test/registered/unit/entrypoints/systemone/test_joint_schema.py b/test/registered/unit/entrypoints/systemone/test_joint_schema.py
new file mode 100644
index 0000000000..3d03aada1d
--- /dev/null
+++ b/test/registered/unit/entrypoints/systemone/test_joint_schema.py
@@ -0,0 +1,474 @@
+"""Unit tests for /v1/systemone on Clef checkpoints: the joint schema prompt and its
+decision layout, and answers built from the joint schema head's option logits."""
+
+import asyncio
+import base64
+import json
+import math
+import random
+import unittest
+from io import BytesIO
+from types import SimpleNamespace
+from unittest import mock
+
+import pybase64
+from PIL import Image
+from transformers import AutoTokenizer
+
+from sglang.srt.entrypoints import http_server
+from sglang.srt.entrypoints.openai.protocol import DecisionRequest
+from sglang.srt.entrypoints.openai.serving_decisions import OpenAIServingDecisions
+from sglang.srt.entrypoints.systemone.joint_schema import encode_joint_schema
+from sglang.srt.entrypoints.systemone.protocol import SystemOneRequest
+from sglang.srt.entrypoints.systemone.serving import SystemOneServing, _read_image
+from sglang.srt.layers.joint_schema_head import (
+ LayoutQuestion,
+ pack_decision_layout,
+ parse_decision_layout,
+)
+from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput
+from sglang.srt.managers.tokenizer_manager import TokenizerManager
+from sglang.srt.runtime_context import (
+ get_context,
+ get_schedule,
+ publish,
+ restore_context,
+ snapshot_context,
+)
+from sglang.srt.server_args import ServerArgs
+from sglang.srt.utils import ImageData
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=15, suite="base-a-test-cpu")
+
+# Tokenizer files only, with the Qwen3.5 vocabulary of Clef checkpoints.
+TOKENIZER = "Qwen/Qwen3.5-35B-A3B"
+
+REQUEST = {
+ "model": "clef",
+ "state": {"ticket": "Refund for order 1182", "amount": 42},
+ "questions": {
+ "team": {
+ "type": "choice",
+ "instructions": "Which team handles it?",
+ "criteria": {
+ "support": "Customer support",
+ "billing": "Payments and refunds",
+ "legal": None,
+ },
+ },
+ "urgent": {
+ "type": "noul",
+ "instructions": "Is it urgent?",
+ "criteria": {"false": None},
+ },
+ "severity": {"type": "score", "criteria": ["none", "low", "high"]},
+ },
+}
+
+
+class PatchCounter:
+ """Stands in for the multimodal processor: one image token per 28x28 patch."""
+
+ @staticmethod
+ def resolve_image_token_counts(images):
+ return [(image.height // 28) * (image.width // 28) for image in images]
+
+
+def _png_bytes(width, height):
+ buffer = BytesIO()
+ Image.new("RGB", (width, height), "red").save(buffer, format="PNG")
+ return buffer.getvalue()
+
+
+def _png(width, height):
+ return (
+ "data:image/png;base64," + base64.b64encode(_png_bytes(width, height)).decode()
+ )
+
+
+def _expanded_length(request):
+ images = [
+ Image.open(BytesIO(base64.b64decode(image.url.split(",", 1)[1])))
+ for image in request.image_data or []
+ ]
+ counts = PatchCounter.resolve_image_token_counts(images)
+ return len(request.input_ids) + sum(count - 1 for count in counts)
+
+
+class HeadManager:
+ """Replace model execution with option logits 0, 1, 2, ... per question, in head order,
+ after expanding images and refusing prompts that the tokenizer manager, the
+ scheduler, or the memory reserved for one prefill cannot take."""
+
+ def __init__(self, tokenizer):
+ self.server_args = ServerArgs(model_path="dummy")
+ publish(self.server_args, role="test")
+ self.tokenizer = tokenizer
+ self.model_config = SimpleNamespace(
+ is_multimodal=True,
+ hf_config=SimpleNamespace(
+ architectures=["Qwen3_5ForConditionalGeneration"], model_type="qwen3_5"
+ ),
+ decision_config=None,
+ joint_head_config={"hidden_size": 4096},
+ )
+ self.is_generation = False
+ self.allow_auto_truncate = False
+ self.context_len = 4096
+ self.num_reserved_tokens = 0
+ # min(context_len - 1, kv_capacity - 1) - 5, as the scheduler reports it.
+ self.max_req_input_len = 4090
+ self.mm_processor = PatchCounter()
+ self.request_logger = SimpleNamespace(log_requests=False)
+ self.served_model_name = "served-model"
+ self.requests = []
+
+ def config_value(self, name):
+ return None
+
+ async def generate_request(self, request, raw_request):
+ self.requests.append(request)
+ num_tokens = _expanded_length(request)
+ if num_tokens + self.num_reserved_tokens >= self.context_len:
+ raise ValueError(f"The input ({num_tokens} tokens) is too long")
+ if num_tokens >= self.max_req_input_len:
+ raise ValueError(f"The scheduler refuses a prompt of {num_tokens} tokens")
+ if num_tokens > get_schedule().max_prefill_tokens:
+ raise ValueError(f"A prefill of {num_tokens} tokens outgrows its memory")
+ questions = parse_decision_layout(request.decision_layout, num_tokens)
+ logits = [float(i) for q in questions for i in range(len(q.option_spans))]
+ yield {
+ "embedding": logits,
+ "meta_info": {"prompt_tokens": num_tokens},
+ }
+
+
+def _handler(manager, serving_class=SystemOneServing):
+ template_manager = SimpleNamespace(
+ chat_template_name=None,
+ jinja_template_content_format="openai",
+ reasoning_config=None,
+ suggested_reasoning_parser=None,
+ )
+ chat_serving = SimpleNamespace(
+ tokenizer_manager=manager,
+ template_manager=template_manager,
+ default_chat_template_kwargs={},
+ chat_encoding_spec=None,
+ _prompt_text_round_trip_is_lossy=False,
+ reasoning_parser=None,
+ )
+ return serving_class(chat_serving)
+
+
+def _softmax(values):
+ weights = [math.exp(value - max(values)) for value in values]
+ return [weight / sum(weights) for weight in weights]
+
+
+class TestJointSchemaPrompt(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
+
+ def _encode(self, request=REQUEST, max_length=4096):
+ input_ids, layout = encode_joint_schema(
+ self.tokenizer,
+ SystemOneRequest(**request),
+ max_length=max_length,
+ image_token_counts=[],
+ )
+ return input_ids, parse_decision_layout(layout, len(input_ids))
+
+ def _text(self, input_ids, span):
+ return self.tokenizer.decode(input_ids[span[0] : span[1]])
+
+ def test_prompt_is_the_trained_joint_schema_prompt(self):
+ input_ids, _ = self._encode()
+ prompt = self.tokenizer.decode(input_ids)
+ self.assertTrue(
+ prompt.startswith(
+ "<|im_start|>system\nRead the complete state and schema. Decide every "
+ "field jointly. Each answer must be exactly one of that field's allowed "
+ "options.<|im_end|>\n<|im_start|>user\nSTATE:\n"
+ '{"amount":42,"ticket":"Refund for order 1182"}\n\nSCHEMA FIELDS:\n'
+ "\nFIELD 1\nID: team\nTYPE: choice\nINSTRUCTION: Which team handles it?"
+ "\nALLOWED OPTIONS:\n"
+ 'OPTION 1: {"description":"Payments and refunds","option_id":"billing"}\n'
+ )
+ )
+ self.assertTrue(
+ prompt.endswith(
+ "END FIELD\n\n<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"
+ "JOINT SCHEMA DECISIONS:"
+ )
+ )
+
+ def test_layout_spans_hold_questions_and_options_in_head_order(self):
+ input_ids, questions = self._encode()
+ self.assertEqual([q.question_type for q in questions], [1, 0, 2])
+ self.assertEqual(
+ [self._text(input_ids, q.question_span) for q in questions],
+ ["Which team handles it?", "Is it urgent?", "severity"],
+ )
+ options = [
+ [json.loads(self._text(input_ids, span)) for span in q.option_spans]
+ for q in questions
+ ]
+ self.assertEqual(
+ options[0],
+ [
+ {"description": "Payments and refunds", "option_id": "billing"},
+ {"option_id": "legal"},
+ {"description": "Customer support", "option_id": "support"},
+ ],
+ )
+ self.assertEqual(
+ options[1],
+ [
+ {
+ "description": "The proposition is true or the answer is yes.",
+ "option_id": "true",
+ },
+ {"option_id": "false"},
+ ],
+ )
+ self.assertEqual(
+ options[2],
+ [
+ {"description": level, "option_id": str(index)}
+ for index, level in enumerate(["none", "low", "high"])
+ ],
+ )
+
+ def test_state_is_truncated_to_fit(self):
+ input_ids, questions = self._encode()
+ shorter_ids, shorter = self._encode(max_length=len(input_ids) - 3)
+ self.assertEqual(len(shorter_ids), len(input_ids) - 3)
+ self.assertEqual(
+ [self._text(shorter_ids, q.question_span) for q in shorter],
+ [self._text(input_ids, q.question_span) for q in questions],
+ )
+ with self.assertRaisesRegex(ValueError, "before the state"):
+ self._encode(max_length=100)
+
+ def test_image_placeholders_precede_the_state(self):
+ request = {**REQUEST, "images": ["data:image/png;base64,AAAA"]}
+ input_ids, layout = encode_joint_schema(
+ self.tokenizer,
+ SystemOneRequest(**request),
+ max_length=4096,
+ image_token_counts=[100],
+ )
+ self.assertIn(
+ "STATE:\n<|vision_start|><|image_pad|><|vision_end|>\n{",
+ self.tokenizer.decode(input_ids),
+ )
+ unexpanded = parse_decision_layout(layout, len(input_ids))
+ expanded = parse_decision_layout(layout, len(input_ids) + 99)
+ self.assertEqual(
+ [q.option_spans[0][0] + 99 for q in unexpanded],
+ [q.option_spans[0][0] for q in expanded],
+ )
+
+
+class TestJointSchemaAnswers(unittest.IsolatedAsyncioTestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
+
+ def setUp(self):
+ self.addCleanup(restore_context, snapshot_context())
+
+ async def test_answers_map_head_order_back_to_the_request(self):
+ manager = HeadManager(self.tokenizer)
+ response = await _handler(manager).handle_request(
+ SystemOneRequest(**REQUEST), None
+ )
+ self.assertEqual(response.status_code, 200)
+ body = json.loads(response.body)
+ input_ids, layout = encode_joint_schema(
+ self.tokenizer,
+ SystemOneRequest(**REQUEST),
+ max_length=4089,
+ image_token_counts=[],
+ )
+ (sent,) = manager.requests
+ self.assertEqual((sent.input_ids, sent.decision_layout), (input_ids, layout))
+ self.assertIsNone(sent.image_data)
+ self.assertEqual(body["model"], "served-model")
+ self.assertEqual(body["usage"]["input_tokens"], len(input_ids))
+
+ three, two = _softmax([0.0, 1.0, 2.0]), _softmax([0.0, 1.0])
+ team, urgent, severity = (body["answers"][q] for q in REQUEST["questions"])
+ # The head scores billing, legal, support; answers keep the request order.
+ self.assertEqual(team["choice"], "support")
+ self.assertEqual(list(team["probabilities"]), ["support", "billing", "legal"])
+ for name, expected in zip(["billing", "legal", "support"], three):
+ self.assertAlmostEqual(team["probabilities"][name], expected)
+ self.assertAlmostEqual(urgent["noul"], two[0])
+ self.assertAlmostEqual(severity["score"], three[1] + 2 * three[2])
+ self.assertEqual(severity["legend"], {"0": "none", "1": "low", "2": "high"})
+ for answer in (team, urgent, severity):
+ self.assertNotIn("x_label_mass", answer)
+
+ async def test_url_images_are_read_once_and_forwarded_as_counted(self):
+ """An image URL whose content changes between reads must be scored as the
+ image that was counted against the prompt budget."""
+ manager = HeadManager(self.tokenizer)
+ url = "http://images.example/chart.png"
+ data = _png_bytes(224, 224)
+ image = {"url": url, "detail": "high"}
+ request = {**REQUEST, "state": "word " * 5000, "images": [image]}
+ target = "sglang.srt.entrypoints.systemone.serving.get_image_bytes"
+ with mock.patch(target, return_value=data) as fetch:
+ response = await _handler(manager).handle_request(
+ SystemOneRequest(**request), None
+ )
+ fetch.assert_called_once_with(url)
+ (forwarded,) = manager.requests[0].image_data
+ self.assertTrue(forwarded.url.startswith("data:image/png;base64,"))
+ self.assertEqual(base64.b64decode(forwarded.url.split(",", 1)[1]), data)
+ self.assertEqual(forwarded.detail, "high")
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(json.loads(response.body)["usage"]["input_tokens"], 4089)
+
+ async def test_refusals(self):
+ manager = HeadManager(self.tokenizer)
+ cases = {
+ "takes no chat_template_kwargs": (
+ SystemOneServing,
+ SystemOneRequest(**REQUEST, chat_template_kwargs={"x": 1}),
+ ),
+ "LoRA adapter": (
+ SystemOneServing,
+ SystemOneRequest(**{**REQUEST, "model": "clef:a"}),
+ ),
+ # Its 5329 image tokens leave no room for the questions.
+ "before the state": (
+ SystemOneServing,
+ SystemOneRequest(**{**REQUEST, "images": [_png(2048, 2048)]}),
+ ),
+ "requires a generation model": (
+ OpenAIServingDecisions,
+ DecisionRequest(
+ model="m",
+ input="x",
+ questions=[{"id": "q", "type": "yes_no", "question": "Q?"}],
+ ),
+ ),
+ }
+ for message, (serving_class, request) in cases.items():
+ with self.subTest(message):
+ response = await _handler(manager, serving_class).handle_request(
+ request, None
+ )
+ self.assertEqual(response.status_code, 400)
+ self.assertIn(message, json.loads(response.body)["message"])
+ self.assertEqual(manager.requests, [])
+
+
+def _admission_manager(joint_head_config):
+ manager = TokenizerManager.__new__(TokenizerManager)
+ manager.model_config = SimpleNamespace(joint_head_config=joint_head_config)
+ manager.is_generation = joint_head_config is None
+ manager.context_len = 4096
+ manager.num_reserved_tokens = 0
+ manager.max_req_input_len = 4090
+ # A layout must be refused before auto-truncation could cut its prompt.
+ manager.allow_auto_truncate = True
+ manager.validate_total_tokens = False
+ manager._validate_token_ids_logprob = mock.Mock()
+ return manager
+
+
+class TestJointSchemaAdmission(CustomTestCase):
+ def setUp(self):
+ override = get_context().override_server_args()
+ override.install()
+ self.addCleanup(override.restore)
+
+ def test_tokenizer_refuses_generation_and_prompts_past_one_prefill(self):
+ clef = _admission_manager({"hidden_size": 4096})
+ layout = [LayoutQuestion(0, (10, 12), ((13, 14), (15, 16)))]
+
+ def embedding(length):
+ return EmbeddingReqInput(
+ input_ids=[1] * length,
+ decision_layout=pack_decision_layout(length, layout),
+ sampling_params={},
+ )
+
+ generation = GenerateReqInput(input_ids=[1, 2, 3], sampling_params={})
+ refused = {
+ "generation": (generation, "/v1/systemone"),
+ # 4095 tokens pass the context check, but the scheduler takes 4089.
+ "past one prefill": (embedding(4095), "at most 4089"),
+ }
+ for name, (obj, message) in refused.items():
+ with self.subTest(name), self.assertRaisesRegex(ValueError, message):
+ clef._validate_one_request(obj, obj.input_ids)
+ clef._validate_one_request(embedding(4089), [1] * 4089)
+ # A server without a joint schema head keeps generating.
+ _admission_manager(None)._validate_one_request(generation, [1, 2, 3])
+
+ def test_client_decision_layouts_are_refused_at_encode_and_classify(self):
+ manager = SimpleNamespace(requests=[])
+
+ async def generate_request(obj, request):
+ manager.requests.append(obj)
+ yield {"embedding": [0.0]}
+
+ manager.generate_request = generate_request
+ self.addCleanup(
+ setattr, http_server, "_global_state", http_server.get_global_state()
+ )
+ http_server.set_global_state(SimpleNamespace(tokenizer_manager=manager))
+ layout = pack_decision_layout(16, [LayoutQuestion(0, (0, 1), ((1, 2),))])
+ responses = [
+ asyncio.run(
+ handler(
+ EmbeddingReqInput(
+ input_ids=list(range(16)), decision_layout=layout
+ ),
+ None,
+ )
+ )
+ for handler in (http_server.encode_request, http_server.classify_request)
+ ]
+ self.assertEqual(manager.requests, [])
+ self.assertEqual([response.status_code for response in responses], [400, 400])
+
+ def test_inline_images_are_sized_from_a_header_prefix(self):
+ """Sizing an inline image must not decode its whole payload, and an image
+ whose header runs past the decoded prefix must still be sized."""
+ noise = BytesIO()
+ pixels = random.Random(0).randbytes(256 * 192 * 3)
+ Image.frombytes("RGB", (256, 192), pixels).save(noise, format="PNG")
+ late = BytesIO()
+ # A 60 KB EXIF segment puts the JPEG frame header far into the payload.
+ exif = b"Exif\x00\x00" + bytes(60000)
+ Image.new("RGB", (64, 48)).save(late, format="JPEG", exif=exif)
+ cases = {
+ "png": ("image/png", noise.getvalue(), (256, 192)),
+ "late jpeg header": ("image/jpeg", late.getvalue(), (64, 48)),
+ }
+ for name, (mime, data, size) in cases.items():
+ with self.subTest(name):
+ payload = base64.b64encode(data).decode()
+ image = ImageData(url=f"data:{mime};base64,{payload}")
+ with mock.patch("pybase64.b64decode", wraps=pybase64.b64decode) as read:
+ forwarded, header = _read_image(0, image)
+ self.assertIs(forwarded, image)
+ self.assertEqual(header.size, size)
+ decoded = [len(call.args[0]) for call in read.call_args_list]
+ if name == "png":
+ self.assertLess(max(decoded), len(payload))
+ else:
+ self.assertEqual(decoded[-1], len(payload))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/layers/test_joint_schema_head.py b/test/registered/unit/layers/test_joint_schema_head.py
new file mode 100644
index 0000000000..a6fc11a28d
--- /dev/null
+++ b/test/registered/unit/layers/test_joint_schema_head.py
@@ -0,0 +1,240 @@
+"""Unit tests for the joint schema head of Clef checkpoints: decision layouts, and
+the batched forward against a per-request loop over the stock torch modules."""
+
+import math
+import unittest
+from types import SimpleNamespace
+
+import torch
+import torch.nn.functional as F
+
+from sglang.srt.layers.joint_schema_head import (
+ JointSchemaHead,
+ JointSchemaPooler,
+ LayoutQuestion,
+ pack_decision_layout,
+ parse_decision_layout,
+)
+from sglang.test.ci.ci_register import register_cpu_ci, register_cuda_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=5, suite="base-a-test-cpu")
+register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small")
+
+
+def _questions(offset=0):
+ return [
+ LayoutQuestion(
+ 1,
+ (10 + offset, 14 + offset),
+ (
+ (16 + offset, 17 + offset),
+ (17 + offset, 19 + offset),
+ (19 + offset, 21 + offset),
+ ),
+ ),
+ LayoutQuestion(
+ 0,
+ (22 + offset, 25 + offset),
+ ((26 + offset, 28 + offset), (29 + offset, 31 + offset)),
+ ),
+ ]
+
+
+class TestDecisionLayout(CustomTestCase):
+ def test_round_trip(self):
+ layout = pack_decision_layout(40, _questions())
+ self.assertEqual(parse_decision_layout(layout, 40), _questions())
+
+ def test_expanded_images_shift_every_span(self):
+ layout = pack_decision_layout(40, _questions())
+ self.assertEqual(parse_decision_layout(layout, 140), _questions(offset=100))
+
+ def test_rejects_layouts_that_do_not_fit_the_prompt(self):
+ layout = pack_decision_layout(40, _questions())
+ invalid = {
+ "shorter prompt": (layout, 39),
+ "not ints": ([float(value) for value in layout], 40),
+ "truncated": (layout[:-1], 40),
+ "trailing values": (layout + [0], 40),
+ "no questions": ([40, 0], 40),
+ "empty span": (
+ pack_decision_layout(40, [LayoutQuestion(0, (5, 5), ((6, 7),))]),
+ 40,
+ ),
+ "past the prompt": (
+ pack_decision_layout(40, [LayoutQuestion(0, (5, 6), ((6, 41),))]),
+ 40,
+ ),
+ "unknown type": (
+ pack_decision_layout(40, [LayoutQuestion(3, (5, 6), ((6, 7),))]),
+ 40,
+ ),
+ }
+ for name, (values, num_tokens) in invalid.items():
+ with self.subTest(name), self.assertRaises(ValueError):
+ parse_decision_layout(values, num_tokens)
+
+
+class _RecordingHead:
+ """Stands in for the head, scoring each option with its request's length."""
+
+ def __init__(self):
+ self.calls = []
+
+ def forward_batch(self, hidden, input_ids, lengths, layouts, embedding_weight):
+ self.calls.append(lengths)
+ return [
+ torch.full((sum(len(q.option_spans) for q in layout),), float(length))
+ for length, layout in zip(lengths, layouts)
+ ]
+
+
+class TestJointSchemaPooler(CustomTestCase):
+ def test_a_layout_cut_from_its_prompt_is_skipped_and_neighbors_are_scored(self):
+ """A prompt shorter than its layout, as after a truncation, must not stop the
+ scheduler or the other requests of its batch."""
+ head = _RecordingHead()
+ pooler = JointSchemaPooler(head, lambda: torch.zeros(1))
+ layout = pack_decision_layout(40, _questions())
+ lengths = [40, 35, 40]
+ forward_batch = SimpleNamespace(
+ decision_layouts=[layout, layout, layout],
+ extend_seq_lens_cpu=lengths,
+ extend_prefix_lens_cpu=[0, 0, 0],
+ batch_size=3,
+ input_ids=torch.zeros(sum(lengths), dtype=torch.long),
+ )
+ output = pooler(torch.zeros(sum(lengths), 8), forward_batch)
+ self.assertEqual([len(scores) for scores in output.embeddings], [5, 0, 5])
+ self.assertEqual(head.calls, [[40], [40]])
+
+
+def _reference_logits(head, hidden, input_ids, questions, lm_head):
+ """One request through the modules' own forward methods, as the checkpoint's loop does."""
+ normalized = head.hidden_norm(hidden)
+ memory = head.memory_projection(normalized).unsqueeze(0)
+ global_vector = normalized[-1]
+
+ def mean(values, span):
+ return values[span[0] : span[1]].mean(dim=0)
+
+ question_vectors = torch.stack(
+ [mean(normalized, q.question_span) for q in questions]
+ )
+ queries, lexicals = [], []
+ for index, question in enumerate(questions):
+ context = torch.stack(
+ [mean(normalized, span) for span in question.option_spans]
+ )
+ lexical = torch.stack(
+ [
+ lm_head[input_ids[start:end]].mean(dim=0)
+ for start, end in question.option_spans
+ ]
+ )
+ queries.append(
+ head.option_context_projection(context)
+ + head.option_lexical_projection(lexical)
+ + head.option_question_projection(question_vectors[index]).unsqueeze(0)
+ )
+ lexicals.append(lexical)
+ routed = torch.cat(queries).unsqueeze(0)
+ for layer in head.evidence_layers:
+ attended, _ = layer.attention(
+ layer.query_norm(routed),
+ layer.memory_norm(memory),
+ layer.memory_norm(memory),
+ need_weights=False,
+ )
+ routed = routed + attended
+ routed = routed + layer.feedforward(layer.feedforward_norm(routed))
+ options = list(torch.split(routed[0], [len(q.option_spans) for q in questions]))
+ base_fields = head.question_projection(question_vectors)
+ summaries = []
+ for field, routed_options in zip(base_fields, options):
+ weights = torch.softmax(
+ routed_options @ field / math.sqrt(field.shape[-1]), dim=0
+ )
+ summaries.append((weights.unsqueeze(-1) * routed_options).sum(dim=0))
+ fields = (
+ base_fields
+ + head.option_summary_norm(torch.stack(summaries))
+ + head.global_projection(global_vector).unsqueeze(0)
+ + head.type_embedding(
+ torch.tensor([q.question_type for q in questions], device=hidden.device)
+ )
+ ).unsqueeze(0)
+ for layer in head.layers:
+ fields = layer(fields, memory)
+ fields = head.field_norm(fields[0])
+ prior_scale = head.prior_logit_scale.clamp(max=math.log(100.0)).exp()
+ joint_scale = head.joint_logit_scale.clamp(max=math.log(100.0)).exp()
+ logits = []
+ for index, (field, lexical, routed_options) in enumerate(
+ zip(fields, lexicals, options)
+ ):
+ anchor = F.normalize(question_vectors[index] + global_vector, dim=-1)
+ prior = prior_scale * (F.normalize(lexical, dim=-1) @ anchor)
+ normed = head.option_norm(routed_options)
+ repeated = field.unsqueeze(0).expand_as(normed)
+ features = torch.cat(
+ [repeated, normed, repeated * normed, torch.abs(repeated - normed)], dim=-1
+ )
+ residual = head.residual_scorer(features).squeeze(-1)
+ cosine = F.cosine_similarity(repeated, normed, dim=-1)
+ logits.append(
+ prior
+ + torch.sigmoid(head.residual_gate) * (joint_scale * cosine + residual)
+ )
+ return torch.cat(logits).float()
+
+
+@unittest.skipUnless(torch.cuda.is_available(), "the batched head uses FlashAttention")
+class TestJointSchemaHeadBatch(CustomTestCase):
+ def test_batched_forward_matches_each_request_alone(self):
+ torch.manual_seed(0)
+ hidden_size, vocab = 256, 512
+ head = JointSchemaHead(
+ hidden_size, width=128, routing_layers=2, layers=2, heads=4, feedforward=256
+ )
+ for parameter in (
+ head.prior_logit_scale,
+ head.joint_logit_scale,
+ head.residual_gate,
+ ):
+ parameter.data.fill_(0.5)
+ head = head.to("cuda", torch.bfloat16).eval()
+ lm_head = torch.randn(vocab, hidden_size, device="cuda", dtype=torch.bfloat16)
+ lengths = [40, 140, 64]
+ layouts = [
+ _questions(),
+ _questions(offset=100),
+ [
+ LayoutQuestion(
+ 2, (3, 9), tuple((10 + 2 * i, 12 + 2 * i) for i in range(20))
+ )
+ ],
+ ]
+ hidden = torch.randn(
+ sum(lengths), hidden_size, device="cuda", dtype=torch.bfloat16
+ )
+ input_ids = torch.randint(0, vocab, (sum(lengths),), device="cuda")
+ with torch.no_grad():
+ batched = head.forward_batch(hidden, input_ids, lengths, layouts, lm_head)
+ start = 0
+ for length, questions, logits in zip(lengths, layouts, batched):
+ end = start + length
+ expected = _reference_logits(
+ head, hidden[start:end], input_ids[start:end], questions, lm_head
+ )
+ self.assertEqual(logits.shape, expected.shape)
+ torch.testing.assert_close(logits, expected, atol=5e-2, rtol=5e-2)
+ torch.testing.assert_close(
+ logits.softmax(-1), expected.softmax(-1), atol=1e-2, rtol=0
+ )
+ start = end
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/managers/test_hidden_state_server_mode.py b/test/registered/unit/managers/test_hidden_state_server_mode.py
index 5aa30c20a8..47a8c617f5 100644
--- a/test/registered/unit/managers/test_hidden_state_server_mode.py
+++ b/test/registered/unit/managers/test_hidden_state_server_mode.py
@@ -26,6 +26,7 @@ class TestHiddenStateServerMode(CustomTestCase):
manager.allow_auto_truncate = False
manager.validate_total_tokens = False
manager.is_generation = True
+ manager.model_config = SimpleNamespace(joint_head_config=None)
manager.server_args = SimpleNamespace(enable_custom_logit_processor=False)
manager._validate_token_ids_logprob = Mock()
return manager
diff --git a/test/registered/unit/managers/test_sampling_mask_validation.py b/test/registered/unit/managers/test_sampling_mask_validation.py
index 963bd94d23..8b7bbb85a5 100644
--- a/test/registered/unit/managers/test_sampling_mask_validation.py
+++ b/test/registered/unit/managers/test_sampling_mask_validation.py
@@ -1,4 +1,5 @@
import unittest
+from types import SimpleNamespace
from unittest.mock import patch
from sglang.srt.managers.io_struct import GenerateReqInput
@@ -28,6 +29,7 @@ class TestSamplingMaskValidation(CustomTestCase):
self.manager.allow_auto_truncate = False
self.manager.validate_total_tokens = False
self.manager.is_generation = True
+ self.manager.model_config = SimpleNamespace(joint_head_config=None)
def _validate(
self,
diff --git a/test/registered/unit/multimodal/test_gpu_feature_transport.py b/test/registered/unit/multimodal/test_gpu_feature_transport.py
index 8ad893112e..afa13fb6e6 100644
--- a/test/registered/unit/multimodal/test_gpu_feature_transport.py
+++ b/test/registered/unit/multimodal/test_gpu_feature_transport.py
@@ -987,6 +987,7 @@ class TestSchedulerMmTransportBoundary(CustomTestCase):
return_pooled_hidden_states=False,
multi_item_delimiter_indices=None,
token_indices_to_pool=None,
+ decision_layout=None,
mm_inputs=object(),
)