[model] support Clef and Clef-Flash decision models on /v1/systemone (#42721)

Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com>
This commit is contained in:
areneau-cf
2026-10-09 14:25:52 +08:00
committed by GitHub
co-authored by Jimmy Shong
parent 51f7ea84c7
commit bd2d73daa5
20 changed files with 1763 additions and 2 deletions
@@ -977,6 +977,37 @@ def handle_model_capability_adjustments(server_args: Any):
"prefill; using breakable CUDA graph for CUDA prefill."
)
# A Clef checkpoint's joint schema head reads the final hidden state of
# every prompt token in one pass, so each request is one complete prefill
# with no decode: embedding mode, without prefix reuse or chunking. This
# also puts the FA backend on its raw K/V path, which skips the KV pool.
if model_config.joint_head_config is not None:
if cfg.tp_size != 1 or cfg.pp_size != 1:
raise ValueError(
"Clef checkpoints are served with --tp-size 1 and --pp-size 1, "
"since the joint schema head reads full LM head rows"
)
for key, value in (
("is_embedding", True),
("disable_radix_cache", True),
("chunked_prefill_size", -1),
):
declare_resolution(
server_args, "_handle_model_capability_adjustments", **{key: value}
)
for phase in (Phase.DECODE, Phase.PREFILL):
declare_resolution(
server_args,
"_handle_model_capability_adjustments",
cuda_graph_config=with_phase(
cfg.cuda_graph_config, phase, backend=Backend.DISABLED
),
)
logger.info(
"Clef joint schema head detected: serving /v1/systemone decisions in "
"embedding mode without radix cache, chunked prefill, or CUDA graphs."
)
if (
model_config.is_multimodal
and not model_config.is_multimodal_chunked_prefill_supported
+20
View File
@@ -291,6 +291,23 @@ def load_decision_config(model_path: str, revision: Optional[str]) -> Optional[d
return config
@lru_cache
def load_joint_head_config(model_path: str, revision: Optional[str]) -> Optional[dict]:
"""joint_head_config.json of a Clef checkpoint, whose joint schema head
answers /v1/systemone decisions from the backbone's hidden states."""
path = cached_file(
model_path,
"joint_head_config.json",
revision=revision,
_raise_exceptions_for_missing_entries=False,
_raise_exceptions_for_connection_errors=False,
)
if path is None:
return None
with open(path) as f:
return json.load(f)
def is_qwen3_5(config) -> bool:
return _hf_arch(config) in (
"Qwen3_5ForConditionalGeneration",
@@ -544,6 +561,9 @@ class ModelConfig:
):
# Full attention sees the whole prompt, while linear attention stays causal.
self.hf_text_config.is_causal = False
self.joint_head_config = None
if self.hf_config.architectures == ["Qwen3_5ForConditionalGeneration"]:
self.joint_head_config = load_joint_head_config(self.model_path, revision)
self.requires_mm_token_modalities = requires_mm_token_modalities(
self.hf_config.architectures, self.hf_text_config
)
@@ -66,6 +66,7 @@ from fastapi.routing import APIRoute
from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.configs.embedding_model_spec import resolved_embedding_plan
from sglang.srt.configs.model_config import load_joint_head_config
from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX
from sglang.srt.disaggregation.utils import FAKE_BOOTSTRAP_HOST, DisaggregationMode
from sglang.srt.entrypoints.anthropic.protocol import (
@@ -972,6 +973,7 @@ async def serve_generate_request(obj: GenerateReqInput, request: Request):
async def encode_request(obj: EmbeddingReqInput, request: Request):
"""Handle an embedding request."""
try:
_refuse_decision_layout(obj)
ret = await _global_state.tokenizer_manager.generate_request(
obj, request
).__anext__()
@@ -984,6 +986,7 @@ async def encode_request(obj: EmbeddingReqInput, request: Request):
async def classify_request(obj: EmbeddingReqInput, request: Request):
"""Handle a reward model request. Now the arguments and return values are the same as embedding models."""
try:
_refuse_decision_layout(obj)
ret = await _global_state.tokenizer_manager.generate_request(
obj, request
).__anext__()
@@ -992,6 +995,12 @@ async def classify_request(obj: EmbeddingReqInput, request: Request):
return _create_error_response(e)
def _refuse_decision_layout(obj: EmbeddingReqInput) -> None:
# Only /v1/systemone builds a decision layout, from the prompt it compiles.
if obj.decision_layout is not None:
raise ValueError("decision_layout is set only by /v1/systemone")
@app.api_route("/flush_cache", methods=["GET", "POST"])
@auth_level(AuthLevel.ADMIN_OPTIONAL)
async def flush_cache(timeout: float = Query(0.0, ge=0.0)):
@@ -2379,6 +2388,22 @@ def _execute_server_warmup(server_args: ServerArgs):
).tolist()
json_data["sampling_params"]["max_new_tokens"] = 0
if (
not model_info["is_generation"]
and model_info.get("architectures") == ["Qwen3_5ForConditionalGeneration"]
and load_joint_head_config(get_model().model_path, get_model().revision)
is not None
):
# A Clef checkpoint answers /v1/systemone decisions only.
request_name = "/v1/systemone"
json_data = {
"model": "warmup",
"state": "The capital city of France is Paris.",
"questions": {
"warmup": {"type": "noul", "instructions": "Is the state true?"}
},
}
# Send a warmup request
warmup_timeout = envs.SGLANG_WARMUP_TIMEOUT.get()
try:
@@ -83,6 +83,7 @@ class OpenAIServingDecisions(OpenAIServingBase):
self.chat_encoding_spec = chat_serving.chat_encoding_spec
self.prompt_text_is_lossy = chat_serving._prompt_text_round_trip_is_lossy
self.decision_config = self.tokenizer_manager.model_config.decision_config
self.joint_head_config = self.tokenizer_manager.model_config.joint_head_config
tokenizer = self.tokenizer_manager.tokenizer
# Other tokenizers skip the shortcut in _encode_labels and check the full prompt.
self.added_tokens = (
@@ -0,0 +1,137 @@
"""The prompt that a Clef checkpoint's joint schema head reads.
Clef answers /v1/systemone questions from the backbone's hidden states instead
of from generated text. One prompt holds the state and every question with its
allowed options, and the head scores the span of each option. This follows
encode_record in the checkpoint's joint_schema_model.py, piece by piece, since
each piece is tokenized on its own.
"""
from __future__ import annotations
import json
from typing import Any, List, Sequence, Tuple
from sglang.srt.entrypoints.systemone.protocol import (
SystemOneChoiceQuestion,
SystemOneNoulQuestion,
SystemOneQuestion,
SystemOneRequest,
)
from sglang.srt.layers.joint_schema_head import (
QUESTION_TYPES,
LayoutQuestion,
pack_decision_layout,
)
_SYSTEM_PROMPT = (
"Read the complete state and schema. Decide every field jointly. Each answer "
"must be exactly one of that field's allowed options."
)
_PREFIX = f"<|im_start|>system\n{_SYSTEM_PROMPT}<|im_end|>\n<|im_start|>user\nSTATE:\n"
_IMAGE = "<|vision_start|><|image_pad|><|vision_end|>"
_SUFFIX = (
"\n<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
"JOINT SCHEMA DECISIONS:"
)
_NOUL_CRITERIA = {
"true": "The proposition is true or the answer is yes.",
"false": "The proposition is false or the answer is no.",
}
# The default max_length of encode_record and systemone in joint_schema_model.py.
REFERENCE_MAX_LENGTH = 16384
def _render(value: Any) -> str:
if isinstance(value, str):
return value
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
def joint_schema_options(question: SystemOneQuestion) -> List[Tuple[str, Any]]:
"""Each option's name and description, in the order the head scores them."""
if isinstance(question, SystemOneNoulQuestion):
criteria = dict(_NOUL_CRITERIA)
if question.criteria is not None:
# A description sent as null drops the default, as in encode_record.
criteria.update(question.criteria.model_dump(exclude_unset=True))
return [(name, criteria[name]) for name in ("true", "false")]
if isinstance(question, SystemOneChoiceQuestion):
return sorted(question.criteria.items(), key=lambda item: item[0])
return [(str(level), detail) for level, detail in enumerate(question.criteria)]
def encode_joint_schema(
tokenizer: Any,
request: SystemOneRequest,
max_length: int,
image_token_counts: Sequence[int],
) -> Tuple[List[int], List[int]]:
"""The prompt ids and their decision layout.
Each image's placeholder expands to its count in ``image_token_counts``, and
the state is truncated so that the expanded prompt has at most
``max_length`` tokens, as encode_record budgets the expanded media.
"""
def tokens(text: str) -> List[int]:
return tokenizer.encode(text, add_special_tokens=False)
schema = tokens("\n\nSCHEMA FIELDS:\n")
questions = []
for index, (question_id, question) in enumerate(request.questions.items()):
schema += tokens(
f"\nFIELD {index + 1}\nID: {question_id}\nTYPE: {question.type}\n"
"INSTRUCTION: "
)
instructions = question.instructions
if instructions is None or instructions == "":
instructions = question_id
question_start = len(schema)
schema += tokens(_render(instructions))
question_span = (question_start, len(schema))
schema += tokens("\nALLOWED OPTIONS:\n")
option_spans = []
for option_index, (name, description) in enumerate(
joint_schema_options(question)
):
schema += tokens(f"OPTION {option_index + 1}: ")
option_start = len(schema)
semantics = {"option_id": name}
if description is not None:
semantics["description"] = description
schema += tokens(_render(semantics))
option_spans.append((option_start, len(schema)))
schema += tokens("\n")
schema += tokens("END FIELD\n")
questions.append(
(QUESTION_TYPES.index(question.type), question_span, option_spans)
)
prefix = tokens(_PREFIX)
if request.images:
prefix += tokens(_IMAGE * len(request.images) + "\n")
suffix = tokens(_SUFFIX)
# Each image's single <|image_pad|> becomes that image's tokens.
expansion = sum(count - 1 for count in image_token_counts)
fixed_length = len(prefix) + expansion + len(schema) + len(suffix)
if fixed_length > max_length:
raise ValueError(
f"the questions need {fixed_length} prompt tokens before the state, "
f"but the prompt holds at most {max_length}"
)
state = tokens(_render(request.state))[: max_length - fixed_length]
offset = len(prefix) + len(state)
input_ids = prefix + state + schema + suffix
layout = pack_decision_layout(
len(input_ids),
[
LayoutQuestion(
question_type,
(question_span[0] + offset, question_span[1] + offset),
tuple((start + offset, end + offset) for start, end in option_spans),
)
for question_type, question_span, option_spans in questions
],
)
return input_ids, layout
@@ -2,14 +2,20 @@
from __future__ import annotations
import asyncio
import dataclasses
import json
import math
import string
from io import BytesIO
from typing import Any, Dict, Iterator, List, Optional, Tuple
from urllib.parse import unquote, urlparse
import orjson
import pybase64
from fastapi import Request
from fastapi.responses import ORJSONResponse
from PIL import Image
from sglang.srt.entrypoints.openai.serving_decisions import (
EncodedQuestion,
@@ -22,6 +28,11 @@ from sglang.srt.entrypoints.openai.serving_decisions import (
render_question,
render_text,
)
from sglang.srt.entrypoints.systemone.joint_schema import (
REFERENCE_MAX_LENGTH,
encode_joint_schema,
joint_schema_options,
)
from sglang.srt.entrypoints.systemone.protocol import (
SystemOneChoiceAnswer,
SystemOneChoiceQuestion,
@@ -33,6 +44,10 @@ from sglang.srt.entrypoints.systemone.protocol import (
SystemOneScoreAnswer,
SystemOneUsage,
)
from sglang.srt.layers.joint_schema_head import max_joint_prompt_tokens
from sglang.srt.managers.io_struct import EmbeddingReqInput
from sglang.srt.runtime_context import get_schedule
from sglang.srt.utils import CLIENT_MEDIA_EXCEPTIONS, ImageData, get_image_bytes
# Beyond A to Z, every option gets a two-letter label, in this fixed order.
_PAIR_LABELS = [a + b for a in string.ascii_uppercase for b in string.ascii_uppercase]
@@ -44,9 +59,16 @@ _DECIDER_SYSTEM = (
"Reply with only the selected option code."
)
# Image sources whose content can change between reads, as load_image reads them.
_MUTABLE_SOURCES = ("http://", "https://", "file://", "/")
# Base64 characters decoded to size an inline image, 48 KiB of image data.
# Arbitrary; an image whose header runs longer is decoded in full.
_INLINE_HEADER_CHARS = 1 << 16
class SystemOneServing(OpenAIServingDecisions):
"""Answers System One questions with the rendering, label checks, and scoring of /v1/decisions."""
"""Answers System One questions with the rendering, label checks, and scoring of
/v1/decisions, or, for a Clef checkpoint, with its joint schema head."""
route = "/v1/systemone"
@@ -54,12 +76,36 @@ class SystemOneServing(OpenAIServingDecisions):
return "systemone-"
def _validate_request(self, request: SystemOneRequest) -> Optional[str]:
if self.joint_head_config is not None:
return self._validate_joint_schema_request(request)
return (
self._validate_server(request.model)
or self._validate_images(request.images)
or self._validate_reasoning(request.chat_template_kwargs)
)
def _validate_joint_schema_request(
self, request: SystemOneRequest
) -> Optional[str]:
"""A Clef checkpoint renders its own prompt, without the chat template."""
route = self.route
if self.tokenizer_manager.tokenizer is None:
return f"{route} requires the server tokenizer"
_, adapter = self._parse_model_parameter(request.model)
if adapter is not None:
return (
f"model names the LoRA adapter {adapter!r}, which {route} "
"does not support"
)
if request.images and self.tokenizer_manager.mm_processor is None:
return f"{route} images require a model served with multimodal input"
if request.chat_template_kwargs:
return (
f"{route} renders the prompt this checkpoint's joint schema head "
"was trained with, which takes no chat_template_kwargs"
)
return None
def _convert_to_internal_request(
self,
request: SystemOneRequest,
@@ -69,6 +115,8 @@ class SystemOneServing(OpenAIServingDecisions):
Tuple[SystemOneRequest, List[QuestionView]],
]:
views = [_view(question) for question in request.questions.values()]
if self.joint_head_config is not None:
return self._joint_schema_request(request, views), (request, views)
encode = (
self._encoded_systemone_questions
if self.decision_config is None
@@ -77,6 +125,118 @@ class SystemOneServing(OpenAIServingDecisions):
# Lazy, so the async handler can yield to other requests between questions.
return encode(request, views), (request, views)
def _joint_schema_request(
self, request: SystemOneRequest, views: List[QuestionView]
) -> EmbeddingReqInput:
"""All questions in one prompt, which the joint schema head scores in one prefill.
The handler encodes the prompt once it has read the image sizes."""
for question_id, view in zip(request.questions, views):
if view.kind == "score":
_check_legend(question_id, view)
return EmbeddingReqInput(
image_data=[
ImageData(url=image.url, detail=image.detail or "auto")
for image in request.images
]
or None,
)
async def _encode_joint_schema_prompt(
self, adapted_request: EmbeddingReqInput, request: SystemOneRequest
) -> None:
tokenizer_manager = self.tokenizer_manager
max_length = min(
REFERENCE_MAX_LENGTH,
max_joint_prompt_tokens(
context_len=tokenizer_manager.context_len,
num_reserved_tokens=tokenizer_manager.num_reserved_tokens,
max_req_input_len=tokenizer_manager.max_req_input_len,
max_prefill_tokens=get_schedule().max_prefill_tokens,
),
)
image_token_counts = []
if adapted_request.image_data:
adapted_request.image_data, image_token_counts = await self._read_images(
adapted_request.image_data
)
input_ids, layout = encode_joint_schema(
tokenizer_manager.tokenizer,
request,
max_length=max_length,
image_token_counts=image_token_counts,
)
adapted_request.input_ids = input_ids
adapted_request.decision_layout = layout
async def _read_images(
self, images: List[ImageData]
) -> Tuple[List[ImageData], List[int]]:
"""Each image read once, and the tokens it expands to at the size the
processor resizes it to."""
read = await asyncio.gather(
*(
asyncio.to_thread(_read_image, index, image)
for index, image in enumerate(images)
)
)
counts = self.tokenizer_manager.mm_processor.resolve_image_token_counts(
[header for _, header in read]
)
return [image for image, _ in read], counts
async def _answer_joint_schema(
self,
adapted_request: EmbeddingReqInput,
request: SystemOneRequest,
views: List[QuestionView],
raw_request: Request,
) -> ORJSONResponse:
await self._encode_joint_schema_prompt(adapted_request, request)
ret = await self.tokenizer_manager.generate_request(
adapted_request, raw_request
).__anext__()
logits = ret["embedding"]
names = [
[name for name, _ in joint_schema_options(question)]
for question in request.questions.values()
]
if len(logits) != sum(map(len, names)):
raise RuntimeError("the joint schema head did not score this request")
answers = {}
offset = 0
for question_id, view, question_names in zip(request.questions, views, names):
scores = logits[offset : offset + len(question_names)]
offset += len(question_names)
top = max(scores)
weights = [math.exp(score - top) for score in scores]
total = math.fsum(weights)
by_name = {
name: weight / total for name, weight in zip(question_names, weights)
}
# The head scores true then false, which the view names yes and no.
answers[question_id] = _answer(
view=view,
probabilities=(
[by_name["true"], by_name["false"]]
if view.kind == "yes_no"
else [by_name[name] for name in view.names]
),
mass=1.0,
question_id=question_id,
)
response = SystemOneResponse(
model=self.tokenizer_manager.served_model_name,
answers=answers,
usage=SystemOneUsage(input_tokens=ret["meta_info"]["prompt_tokens"]),
)
# The head scores only the allowed options, so there is no label mass.
return ORJSONResponse(
content=response.model_dump(
exclude={"answers": {"__all__": {"x_label_mass"}}}
)
)
def _encoded_systemone_questions(
self, request: SystemOneRequest, views: List[QuestionView]
) -> Iterator[EncodedQuestion]:
@@ -179,6 +339,10 @@ class SystemOneServing(OpenAIServingDecisions):
raw_request: Request,
) -> ORJSONResponse:
request, views = processed
if self.joint_head_config is not None:
return await self._answer_joint_schema(
adapted_request, request, views, raw_request
)
_, _, result = await self._score(
adapted_request=adapted_request,
raw_request=raw_request,
@@ -344,3 +508,38 @@ def _score_confidence(q: List[float]) -> float:
spread = math.fsum(p * abs(i - top) for i, p in enumerate(q))
uniform_spread = math.fsum(abs(i - (n - 1) / 2) for i in range(n)) / n
return max(0.0, 1 - spread / uniform_spread)
def _read_image(index: int, image: ImageData) -> Tuple[ImageData, Image.Image]:
"""The image to forward, and its header opened only as far as its size, read
once from the sources load_image reads."""
url = image.url
if not url.startswith(_MUTABLE_SOURCES):
# An inline image is forwarded as sent, and the processor decodes all of it.
header = _inline_header(url)
if header is not None:
return image, header
source = unquote(urlparse(url).path) if url.startswith("file://") else url
try:
data = get_image_bytes(source)
header = Image.open(BytesIO(data))
except CLIENT_MEDIA_EXCEPTIONS as e:
raise ValueError(f"image {index} could not be read: {e}") from e
if url.startswith(_MUTABLE_SOURCES):
# A source that can change between reads is forwarded as the bytes counted.
mime = Image.MIME.get(header.format, "application/octet-stream")
encoded = pybase64.b64encode(data).decode()
image = dataclasses.replace(image, url=f"data:{mime};base64,{encoded}")
return image, header
def _inline_header(url: str) -> Optional[Image.Image]:
"""An inline image's header from a prefix of its base64, or None when the
header does not open from that prefix."""
start = url.find(",") + 1 if url.startswith("data:") else 0
prefix = url[start : start + _INLINE_HEADER_CHARS]
try:
return Image.open(BytesIO(pybase64.b64decode(prefix, validate=True)))
except Exception:
# A header past the prefix, or malformed data that the full read reports.
return None
@@ -0,0 +1,539 @@
"""Joint schema head of Clef decision checkpoints.
A Clef checkpoint is a Qwen3.5 backbone saved with ``joint_head_config.json``
and ``joint_head.safetensors``. The head reads the backbone's final hidden
state at every prompt token and scores every allowed option of every question
together: each option gathers evidence from the prompt, the questions attend
to each other and to the prompt, and a lexical prior built from the LM head
rows of each option's tokens keeps the option's meaning.
One head pass serves a whole prefill batch. Token-wise steps run on the flat
batch, attention is variable-length per request, and span means come from
per-request prefix sums over the schema region only, so a long state costs no
more than its memory projection.
"""
from __future__ import annotations
import math
from typing import Callable, List, Optional, Sequence, Tuple
import msgspec
import torch
import torch.nn.functional as F
from torch import nn
from sglang.srt.layers.pooler import EmbeddingPoolerOutput
# Question types in the order of the head's type embedding.
QUESTION_TYPES = ("noul", "choice", "score")
# Bounds of one head pass, so a large prefill batch is scored in a few passes.
_MAX_GROUP_TOKENS = 65536
_MAX_GROUP_SLOTS = 1 << 18
class LayoutQuestion(msgspec.Struct, frozen=True):
question_type: int
question_span: Tuple[int, int]
option_spans: Tuple[Tuple[int, int], ...]
def max_joint_prompt_tokens(
*,
context_len: int,
num_reserved_tokens: int,
max_req_input_len: int,
max_prefill_tokens: int,
) -> int:
"""The longest prompt, images expanded, that the server runs as one prefill.
The tokenizer manager refuses a prompt of context_len - num_reserved_tokens
tokens, the scheduler one of max_req_input_len, and without chunked prefill
activation memory is reserved for max_prefill_tokens.
"""
return min(
context_len - num_reserved_tokens - 1,
max_req_input_len - 1,
max_prefill_tokens,
)
def pack_decision_layout(
prompt_length: int, questions: Sequence[LayoutQuestion]
) -> List[int]:
"""Flatten a prompt's questions into the ints a request carries.
The layout is ``[prompt_length, num_questions]`` followed, per question,
by ``type, question_start, question_end, num_options`` and one
``option_start, option_end`` pair per option. Positions index the prompt
before its image placeholders expand.
"""
layout = [prompt_length, len(questions)]
for question in questions:
layout += [
question.question_type,
*question.question_span,
len(question.option_spans),
]
for start, end in question.option_spans:
layout += [start, end]
return layout
def parse_decision_layout(
layout: Sequence[int], num_tokens: int
) -> List[LayoutQuestion]:
"""The questions of a layout, placed on the sequence the model reads.
Every span follows the prompt's images, so expanding their placeholders
shifts all spans by the number of added tokens. Raises ValueError when the
layout does not describe the sequence.
"""
if len(layout) < 2 or any(type(value) is not int for value in layout):
raise ValueError("decision_layout must be a list of ints")
prompt_length, num_questions = layout[0], layout[1]
shift = num_tokens - prompt_length
if prompt_length < 1 or shift < 0 or num_questions < 1:
raise ValueError("decision_layout does not match the prompt length")
questions = []
cursor = 2
for _ in range(num_questions):
if cursor + 4 > len(layout):
raise ValueError("decision_layout is truncated")
question_type, start, end, num_options = layout[cursor : cursor + 4]
cursor += 4
if (
not 0 <= question_type < len(QUESTION_TYPES)
or num_options < 1
or not 0 <= start < end <= prompt_length
or cursor + 2 * num_options > len(layout)
):
raise ValueError("decision_layout has an invalid question")
spans = []
for _ in range(num_options):
option_start, option_end = layout[cursor : cursor + 2]
cursor += 2
if not 0 <= option_start < option_end <= prompt_length:
raise ValueError("decision_layout has an invalid option span")
spans.append((option_start + shift, option_end + shift))
questions.append(
LayoutQuestion(question_type, (start + shift, end + shift), tuple(spans))
)
if cursor != len(layout):
raise ValueError("decision_layout has trailing values")
return questions
class EvidenceRoutingLayer(nn.Module):
def __init__(self, width: int, heads: int, feedforward: int) -> None:
super().__init__()
self.query_norm = nn.LayerNorm(width)
self.memory_norm = nn.LayerNorm(width)
self.attention = nn.MultiheadAttention(width, heads, batch_first=True)
self.feedforward_norm = nn.LayerNorm(width)
# Identity slots keep the checkpoint's Sequential indices (dropout at 2 and 4).
self.feedforward = nn.Sequential(
nn.Linear(width, feedforward),
nn.GELU(),
nn.Identity(),
nn.Linear(feedforward, width),
nn.Identity(),
)
class JointSchemaHead(nn.Module):
"""Parameters named as in the checkpoint's ``joint_head.safetensors``."""
def __init__(
self,
hidden_size: int,
width: int,
routing_layers: int,
layers: int,
heads: int,
feedforward: int,
) -> None:
super().__init__()
self.hidden_norm = nn.LayerNorm(hidden_size)
self.memory_projection = nn.Linear(hidden_size, width, bias=False)
self.question_projection = nn.Linear(hidden_size, width, bias=False)
self.option_question_projection = nn.Linear(hidden_size, width, bias=False)
self.global_projection = nn.Linear(hidden_size, width, bias=False)
self.option_context_projection = nn.Linear(hidden_size, width, bias=False)
self.option_lexical_projection = nn.Linear(hidden_size, width, bias=False)
self.type_embedding = nn.Embedding(len(QUESTION_TYPES), width)
self.evidence_layers = nn.ModuleList(
EvidenceRoutingLayer(width, heads, feedforward)
for _ in range(routing_layers)
)
self.option_summary_norm = nn.LayerNorm(width)
self.layers = nn.ModuleList(
nn.TransformerDecoderLayer(
d_model=width,
nhead=heads,
dim_feedforward=feedforward,
dropout=0.0,
activation="gelu",
batch_first=True,
norm_first=True,
)
for _ in range(layers)
)
self.field_norm = nn.LayerNorm(width)
self.option_norm = nn.LayerNorm(width)
self.residual_scorer = nn.Sequential(
nn.Linear(width * 4, width), nn.GELU(), nn.Identity(), nn.Linear(width, 1)
)
self.prior_logit_scale = nn.Parameter(torch.zeros(()))
self.joint_logit_scale = nn.Parameter(torch.zeros(()))
self.residual_gate = nn.Parameter(torch.zeros(()))
@torch.no_grad()
def forward_batch(
self,
hidden: torch.Tensor,
input_ids: torch.Tensor,
lengths: List[int],
layouts: List[List[LayoutQuestion]],
output_embedding_weight: torch.Tensor,
) -> List[torch.Tensor]:
"""One FP32 logit vector per request, its questions' options in layout order.
``hidden`` holds the final hidden states of the requests back to back.
"""
outputs: List[torch.Tensor] = []
start = 0
for group in _groups(lengths, layouts):
group_lengths = [lengths[i] for i in group]
end = start + sum(group_lengths)
outputs += self._forward_group(
hidden[start:end],
input_ids[start:end],
group_lengths,
[layouts[i] for i in group],
output_embedding_weight,
)
start = end
return outputs
def _forward_group(
self,
hidden: torch.Tensor,
input_ids: torch.Tensor,
lengths: List[int],
layouts: List[List[LayoutQuestion]],
output_embedding_weight: torch.Tensor,
) -> List[torch.Tensor]:
device, dtype = hidden.device, hidden.dtype
batch = len(layouts)
question_bounds, option_bounds = [], []
type_ids, owner, slot, question_request, last_tokens = [], [], [], [], []
regions, region_offsets = [], [0]
token_cu, question_cu, option_cu, option_counts = [0], [0], [0], []
for request, (length, questions) in enumerate(zip(lengths, layouts)):
offset = token_cu[-1]
low = min(min(q.question_span[0], q.option_spans[0][0]) for q in questions)
high = max(
max(q.question_span[1], *(e for _, e in q.option_spans))
for q in questions
)
# Prefix row of span position p in this request's schema region.
base = region_offsets[-1] + request - low
regions.append((offset + low, offset + high))
request_options = 0
for question in questions:
count = len(question.option_spans)
question_bounds += (
base + question.question_span[0],
base + question.question_span[1],
)
for option_slot, (option_start, option_end) in enumerate(
question.option_spans
):
option_bounds += (base + option_start, base + option_end)
owner.append(len(type_ids))
slot.append(option_slot)
type_ids.append(question.question_type)
question_request.append(request)
request_options += count
last_tokens.append(offset + length - 1)
token_cu.append(offset + length)
region_offsets.append(region_offsets[-1] + high - low)
question_cu.append(len(type_ids))
option_cu.append(option_cu[-1] + request_options)
option_counts.append(request_options)
num_questions = len(type_ids)
widest = max(len(q.option_spans) for questions in layouts for q in questions)
sizes = [
2 * num_questions,
len(option_bounds),
num_questions,
len(owner),
len(slot),
num_questions,
batch,
3 * (batch + 1),
]
packed = torch.tensor(
question_bounds
+ option_bounds
+ type_ids
+ owner
+ slot
+ question_request
+ last_tokens
+ token_cu
+ question_cu
+ option_cu,
dtype=torch.long,
).to(device, non_blocking=True)
(
question_bounds_t,
option_bounds_t,
type_ids_t,
owner_t,
slot_t,
question_request_t,
last_tokens_t,
cu_t,
) = torch.split(packed, sizes)
question_bounds_t = question_bounds_t.view(-1, 2)
option_bounds_t = option_bounds_t.view(-1, 2)
token_cu_t, question_cu_t, option_cu_t = (
cu_t.to(torch.int32).view(3, batch + 1).unbind(0)
)
token_bounds = (token_cu_t, max(lengths))
question_bounds_cu = (question_cu_t, max(len(q) for q in layouts))
option_bounds_cu = (option_cu_t, max(option_counts))
normalized = self.hidden_norm(hidden)
# Running sums over each schema region, after one zero row per request.
region_total = region_offsets[-1] + batch
hidden_prefix = torch.zeros(
region_total, normalized.shape[-1], dtype=torch.float32, device=device
)
lexical_prefix = torch.zeros(
region_total,
output_embedding_weight.shape[-1],
dtype=torch.float32,
device=device,
)
for request, (low, high) in enumerate(regions):
row = region_offsets[request] + request + 1
torch.cumsum(
normalized[low:high].float(),
dim=0,
out=hidden_prefix[row : row + high - low],
)
torch.cumsum(
output_embedding_weight[input_ids[low:high]].float(),
dim=0,
out=lexical_prefix[row : row + high - low],
)
def span_means(prefix: torch.Tensor, bounds: torch.Tensor) -> torch.Tensor:
sums = prefix[bounds[:, 1]] - prefix[bounds[:, 0]]
return (sums / (bounds[:, 1] - bounds[:, 0]).unsqueeze(-1)).to(dtype)
memory = self.memory_projection(normalized)
global_vectors = normalized[last_tokens_t]
question_vectors = span_means(hidden_prefix, question_bounds_t)
option_contexts = span_means(hidden_prefix, option_bounds_t)
lexical = span_means(lexical_prefix, option_bounds_t)
routed = (
self.option_context_projection(option_contexts)
+ self.option_lexical_projection(lexical)
+ self.option_question_projection(question_vectors)[owner_t]
)
for layer in self.evidence_layers:
routed = routed + _attention(
layer.attention,
layer.query_norm(routed),
layer.memory_norm(memory),
option_bounds_cu,
token_bounds,
)
routed = routed + layer.feedforward(layer.feedforward_norm(routed))
base_fields = self.question_projection(question_vectors)
scores = torch.matmul(
routed.unsqueeze(1), base_fields[owner_t].unsqueeze(2)
).view(-1) / math.sqrt(routed.shape[-1])
padded_scores = torch.full(
(num_questions, widest), float("-inf"), dtype=torch.float32, device=device
)
padded_scores[owner_t, slot_t] = scores.float()
weights = torch.softmax(padded_scores, dim=1).to(dtype)
padded_options = routed.new_zeros(num_questions, widest, routed.shape[-1])
padded_options[owner_t, slot_t] = routed
summaries = torch.sum(weights.unsqueeze(-1) * padded_options, dim=1)
fields = (
base_fields
+ self.option_summary_norm(summaries)
+ self.global_projection(global_vectors)[question_request_t]
+ self.type_embedding(type_ids_t)
)
for layer in self.layers:
fields = fields + _attention(
layer.self_attn,
layer.norm1(fields),
None,
question_bounds_cu,
question_bounds_cu,
)
fields = fields + _attention(
layer.multihead_attn,
layer.norm2(fields),
memory,
question_bounds_cu,
token_bounds,
)
fields = fields + layer.linear2(
layer.activation(layer.linear1(layer.norm3(fields)))
)
fields = self.field_norm(fields)
prior_scale = self.prior_logit_scale.clamp(max=math.log(100.0)).exp()
joint_scale = self.joint_logit_scale.clamp(max=math.log(100.0)).exp()
anchor = F.normalize(
question_vectors + global_vectors[question_request_t], dim=-1
)
prior = prior_scale * torch.matmul(
F.normalize(lexical, dim=-1).unsqueeze(1), anchor[owner_t].unsqueeze(2)
).view(-1)
options = self.option_norm(routed)
repeated = fields[owner_t]
cosine = F.cosine_similarity(repeated, options, dim=-1)
features = torch.cat(
[repeated, options, repeated * options, torch.abs(repeated - options)],
dim=-1,
)
residual = self.residual_scorer(features).squeeze(-1)
logits = prior + torch.sigmoid(self.residual_gate) * (
joint_scale * cosine + residual
)
return list(torch.split(logits.float(), option_counts))
def _attention(
module: nn.MultiheadAttention,
queries: torch.Tensor,
keys: Optional[torch.Tensor],
query_bounds: Tuple[torch.Tensor, int],
key_bounds: Tuple[torch.Tensor, int],
) -> torch.Tensor:
"""nn.MultiheadAttention over packed sequences, keys and values from ``keys``
(or from ``queries`` when None), attending within each request."""
width = queries.shape[-1]
heads = module.num_heads
if keys is None:
query, key, value = F.linear(
queries, module.in_proj_weight, module.in_proj_bias
).chunk(3, dim=-1)
else:
query_weight, key_value_weight = module.in_proj_weight.split([width, 2 * width])
query_bias, key_value_bias = module.in_proj_bias.split([width, 2 * width])
query = F.linear(queries, query_weight, query_bias)
key, value = F.linear(keys, key_value_weight, key_value_bias).chunk(2, dim=-1)
head_dim = width // heads
output = torch.ops.aten._flash_attention_forward(
query.reshape(-1, heads, head_dim),
key.reshape(-1, heads, head_dim),
value.reshape(-1, heads, head_dim),
query_bounds[0],
key_bounds[0],
query_bounds[1],
key_bounds[1],
0.0,
False,
False,
)[0]
return module.out_proj(output.reshape(-1, width))
def _groups(lengths: List[int], layouts: List[List[LayoutQuestion]]) -> List[range]:
"""Consecutive requests per head pass, bounded in tokens and padded option slots."""
groups, start, tokens, questions, widest = [], 0, 0, 0, 0
for index, (length, layout) in enumerate(zip(lengths, layouts)):
layout_widest = max(len(question.option_spans) for question in layout)
grown = (questions + len(layout)) * max(widest, layout_widest)
if index > start and (
tokens + length > _MAX_GROUP_TOKENS or grown > _MAX_GROUP_SLOTS
):
groups.append(range(start, index))
start, tokens, questions, widest = index, 0, 0, 0
tokens += length
questions += len(layout)
widest = max(widest, layout_widest)
groups.append(range(start, len(lengths)))
return groups
class JointSchemaPooler(nn.Module):
"""Pooler of a Clef checkpoint: the joint schema head's option logits per request."""
def __init__(
self, head: JointSchemaHead, output_embedding: Callable[[], torch.Tensor]
) -> None:
super().__init__()
self.head = head
# A callable, so the LM head is not registered twice as a parameter.
self._output_embedding = output_embedding
def forward(
self, hidden_states: torch.Tensor, forward_batch
) -> EmbeddingPoolerOutput:
if (
forward_batch.decision_layouts is None
or forward_batch.extend_seq_lens_cpu is None
):
return EmbeddingPoolerOutput(
embeddings=[
hidden_states.new_empty(0, dtype=torch.float32)
for _ in range(forward_batch.batch_size)
]
)
lengths = list(forward_batch.extend_seq_lens_cpu)
prefix_lengths = forward_batch.extend_prefix_lens_cpu or [0] * len(lengths)
raw_layouts = forward_batch.decision_layouts
embeddings: List[torch.Tensor] = [
hidden_states.new_empty(0, dtype=torch.float32) for _ in lengths
]
# The tokenizer manager checks every layout against its prompt, and the
# server runs each prompt in one uncached prefill. A layout that no longer
# fits its prompt, as after a truncation, gets no scores, which the
# decision route reports as a server error.
layouts = {}
for i, (layout, prefix, length) in enumerate(
zip(raw_layouts, prefix_lengths, lengths)
):
if layout is None or prefix != 0:
continue
try:
layouts[i] = parse_decision_layout(layout, length)
except ValueError:
continue
offsets = [0]
for length in lengths:
offsets.append(offsets[-1] + length)
runs: List[List[int]] = []
for i in layouts:
if runs and runs[-1][-1] == i - 1:
runs[-1].append(i)
else:
runs.append([i])
for run in runs:
start, end = offsets[run[0]], offsets[run[-1] + 1]
logits = self.head.forward_batch(
hidden_states[start:end],
forward_batch.input_ids[start:end],
[lengths[i] for i in run],
[layouts[i] for i in run],
self._output_embedding(),
)
for i, values in zip(run, logits):
embeddings[i] = values
return EmbeddingPoolerOutput(embeddings=embeddings)
+12
View File
@@ -1253,6 +1253,11 @@ class EmbeddingReqInput:
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
token_indices_to_pool: Optional[Union[List[List[int]], List[int]]] = None
# Question and option spans that a Clef checkpoint's joint schema head
# scores, as packed by layers.joint_schema_head.pack_decision_layout.
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
decision_layout: Optional[Union[List[List[int]], List[int]]] = None
def regenerate_rid(self):
"""Generate a new request ID and return it."""
if isinstance(self.rid, list):
@@ -1469,6 +1474,11 @@ class EmbeddingReqInput:
if self.token_indices_to_pool is not None
else None
),
decision_layout=(
self.decision_layout[i]
if self.decision_layout is not None
else None
),
)
cache[i] = sub
return sub
@@ -1500,6 +1510,8 @@ class TokenizedEmbeddingReqInput(BaseReq, kw_only=True):
multi_item_delimiter_indices: Optional[List[int]] = None
# Token positions for setwise pooling readout
token_indices_to_pool: Optional[List[int]] = None
# Question and option spans for a joint schema head
decision_layout: Optional[List[int]] = None
# For observability
# Pickled Optional[Union[APIServerReqTimeStats, DPControllerReqTimeStats]]
time_stats: Optional[PickleWrapper] = None
@@ -1050,6 +1050,7 @@ class Req(ReqDllmMixin):
return_pooled_hidden_states: bool = False,
multi_item_delimiter_indices: Optional[List[int]] = None,
token_indices_to_pool: Optional[List[int]] = None,
decision_layout: Optional[List[int]] = None,
session_id: Optional[str] = None,
cache_salt: Optional[str] = None,
):
@@ -1083,6 +1084,7 @@ class Req(ReqDllmMixin):
self.positional_embed_overrides = positional_embed_overrides
self.multi_item_delimiter_indices = multi_item_delimiter_indices
self.token_indices_to_pool = token_indices_to_pool
self.decision_layout = decision_layout
# For req-level memory management
self.kv = ReqKvInfo()
+1
View File
@@ -3465,6 +3465,7 @@ class Scheduler(
return_pooled_hidden_states=recv_req.return_pooled_hidden_states,
multi_item_delimiter_indices=recv_req.multi_item_delimiter_indices,
token_indices_to_pool=recv_req.token_indices_to_pool,
decision_layout=recv_req.decision_layout,
)
req.tokenizer = self.tokenizer
self._maybe_namespace_elastic_radix_cache(req)
@@ -64,6 +64,10 @@ from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX
from sglang.srt.disaggregation.encoder.receiver import create_mm_receiver
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.environ import envs
from sglang.srt.layers.joint_schema_head import (
max_joint_prompt_tokens,
parse_decision_layout,
)
from sglang.srt.lora.lora_registry import LoRARef, LoRARegistry
from sglang.srt.managers.async_dynamic_batch_tokenizer import AsyncDynamicbatchTokenizer
from sglang.srt.managers.disagg_service import start_disagg_service
@@ -1295,6 +1299,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
raise ValueError(
"encoder SWA replay cannot return cached prompt logprobs"
)
# A decision layout indexes the whole prompt, so it is checked before any
# truncation below could cut the prompt.
self._validate_joint_schema_request(obj, input_ids)
_max_req_len = self.context_len
input_token_num = len(input_ids) if input_ids is not None else 0
input_token_num += self.num_reserved_tokens
@@ -1399,6 +1406,36 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
"return_sampling_mask=true."
)
def _validate_joint_schema_request(
self, obj: Union[GenerateReqInput, EmbeddingReqInput], input_ids: List[int]
) -> None:
"""A Clef checkpoint scores the spans of a decision layout, and nothing else."""
if self.model_config.joint_head_config is None:
return
if isinstance(obj, GenerateReqInput) or obj.decision_layout is None:
# Health checks only need a response, which is an empty embedding.
if (
isinstance(obj, EmbeddingReqInput)
and isinstance(obj.rid, str)
and obj.rid.startswith(HEALTH_CHECK_RID_PREFIX)
):
return
raise ValueError(
"This checkpoint answers schema decisions, send them to /v1/systemone"
)
limit = max_joint_prompt_tokens(
context_len=self.context_len,
num_reserved_tokens=self.num_reserved_tokens,
max_req_input_len=self.max_req_input_len,
max_prefill_tokens=get_schedule().max_prefill_tokens,
)
if len(input_ids) > limit:
raise ValueError(
f"The prompt has {len(input_ids)} tokens, but this server scores "
f"at most {limit} in one prefill"
)
parse_decision_layout(obj.decision_layout, len(input_ids))
def _validate_mm_limits(
self, obj: Union[GenerateReqInput, EmbeddingReqInput]
) -> None:
@@ -1575,6 +1612,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
return_pooled_hidden_states=obj.return_pooled_hidden_states,
multi_item_delimiter_indices=obj.multi_item_delimiter_indices,
token_indices_to_pool=obj.token_indices_to_pool,
decision_layout=obj.decision_layout,
)
tokenized_obj.time_stats = self.rid_to_state[obj.rid].time_stats
@@ -614,6 +614,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# Setwise pooling readout positions (CPU tensors, one per request)
token_indices_to_pool: Optional[List[torch.Tensor]] = None
# Joint schema head decision layouts (one per request, None if absent)
decision_layouts: Optional[List[Optional[List[int]]]] = None
# === Borrowed from ScheduleBatch: compound (carry their own device tensors) ===
# Sampling info
sampling_info: SamplingBatchInfo = None
@@ -1264,6 +1267,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
for r in batch.reqs
]
if any(r.decision_layout is not None for r in batch.reqs):
self.decision_layouts = [r.decision_layout for r in batch.reqs]
token_type_ids = [
r.token_type_ids for r in batch.reqs if r.token_type_ids is not None
]
+30 -1
View File
@@ -34,7 +34,10 @@ from sglang.kernels.ops.attention.triton_gdn_fused_proj import (
from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul
# Configs
from sglang.srt.configs.model_config import load_decision_config
from sglang.srt.configs.model_config import (
load_decision_config,
load_joint_head_config,
)
from sglang.srt.configs.qwen3_5 import (
Qwen3_5Config,
Qwen3_5MoeConfig,
@@ -54,6 +57,7 @@ from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
from sglang.srt.layers.joint_schema_head import JointSchemaHead, JointSchemaPooler
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
@@ -2330,6 +2334,16 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration):
self.visual.deepstack_visual_indexes if self.visual is not None else []
)
if config.architectures == ["Qwen3_5ForConditionalGeneration"]:
model = get_model()
joint_head_config = load_joint_head_config(model.model_path, model.revision)
if joint_head_config is not None:
# A Clef checkpoint, whose joint schema head scores every option
# of every question from the final hidden states and LM head rows.
self.pooler = JointSchemaPooler(
JointSchemaHead(**joint_head_config), lambda: self.lm_head.weight
)
def get_hidden_dim(self, module_name: str, layer_idx: int):
return self.model.get_hidden_dim(module_name, layer_idx)
@@ -2470,8 +2484,23 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration):
if bare_backbone and self.pp_group.is_last_rank:
self._load_decision_readout()
loaded_params.add("lm_head.weight")
if isinstance(self.pooler, JointSchemaPooler):
self._load_joint_head()
loaded_params.update(
f"pooler.head.{name}" for name, _ in self.pooler.head.named_parameters()
)
return loaded_params
def _load_joint_head(self) -> None:
"""Load joint_head.safetensors, which the checkpoint index leaves out."""
model = get_model()
state_dict = load_file(
cached_file(
model.model_path, "joint_head.safetensors", revision=model.revision
)
)
self.pooler.head.load_state_dict(state_dict, strict=True)
def _load_decision_readout(self) -> None:
"""Place the checkpoint's decision readout in the LM head rows of its codes."""
model = get_model()