[model] support Clef and Clef-Flash decision models on /v1/systemone (#42721)
Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com>
This commit is contained in:
co-authored by
Jimmy Shong
parent
51f7ea84c7
commit
bd2d73daa5
@@ -977,6 +977,37 @@ def handle_model_capability_adjustments(server_args: Any):
|
||||
"prefill; using breakable CUDA graph for CUDA prefill."
|
||||
)
|
||||
|
||||
# A Clef checkpoint's joint schema head reads the final hidden state of
|
||||
# every prompt token in one pass, so each request is one complete prefill
|
||||
# with no decode: embedding mode, without prefix reuse or chunking. This
|
||||
# also puts the FA backend on its raw K/V path, which skips the KV pool.
|
||||
if model_config.joint_head_config is not None:
|
||||
if cfg.tp_size != 1 or cfg.pp_size != 1:
|
||||
raise ValueError(
|
||||
"Clef checkpoints are served with --tp-size 1 and --pp-size 1, "
|
||||
"since the joint schema head reads full LM head rows"
|
||||
)
|
||||
for key, value in (
|
||||
("is_embedding", True),
|
||||
("disable_radix_cache", True),
|
||||
("chunked_prefill_size", -1),
|
||||
):
|
||||
declare_resolution(
|
||||
server_args, "_handle_model_capability_adjustments", **{key: value}
|
||||
)
|
||||
for phase in (Phase.DECODE, Phase.PREFILL):
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_model_capability_adjustments",
|
||||
cuda_graph_config=with_phase(
|
||||
cfg.cuda_graph_config, phase, backend=Backend.DISABLED
|
||||
),
|
||||
)
|
||||
logger.info(
|
||||
"Clef joint schema head detected: serving /v1/systemone decisions in "
|
||||
"embedding mode without radix cache, chunked prefill, or CUDA graphs."
|
||||
)
|
||||
|
||||
if (
|
||||
model_config.is_multimodal
|
||||
and not model_config.is_multimodal_chunked_prefill_supported
|
||||
|
||||
@@ -291,6 +291,23 @@ def load_decision_config(model_path: str, revision: Optional[str]) -> Optional[d
|
||||
return config
|
||||
|
||||
|
||||
@lru_cache
|
||||
def load_joint_head_config(model_path: str, revision: Optional[str]) -> Optional[dict]:
|
||||
"""joint_head_config.json of a Clef checkpoint, whose joint schema head
|
||||
answers /v1/systemone decisions from the backbone's hidden states."""
|
||||
path = cached_file(
|
||||
model_path,
|
||||
"joint_head_config.json",
|
||||
revision=revision,
|
||||
_raise_exceptions_for_missing_entries=False,
|
||||
_raise_exceptions_for_connection_errors=False,
|
||||
)
|
||||
if path is None:
|
||||
return None
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def is_qwen3_5(config) -> bool:
|
||||
return _hf_arch(config) in (
|
||||
"Qwen3_5ForConditionalGeneration",
|
||||
@@ -544,6 +561,9 @@ class ModelConfig:
|
||||
):
|
||||
# Full attention sees the whole prompt, while linear attention stays causal.
|
||||
self.hf_text_config.is_causal = False
|
||||
self.joint_head_config = None
|
||||
if self.hf_config.architectures == ["Qwen3_5ForConditionalGeneration"]:
|
||||
self.joint_head_config = load_joint_head_config(self.model_path, revision)
|
||||
self.requires_mm_token_modalities = requires_mm_token_modalities(
|
||||
self.hf_config.architectures, self.hf_text_config
|
||||
)
|
||||
|
||||
@@ -66,6 +66,7 @@ from fastapi.routing import APIRoute
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolving_view
|
||||
from sglang.srt.configs.embedding_model_spec import resolved_embedding_plan
|
||||
from sglang.srt.configs.model_config import load_joint_head_config
|
||||
from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX
|
||||
from sglang.srt.disaggregation.utils import FAKE_BOOTSTRAP_HOST, DisaggregationMode
|
||||
from sglang.srt.entrypoints.anthropic.protocol import (
|
||||
@@ -972,6 +973,7 @@ async def serve_generate_request(obj: GenerateReqInput, request: Request):
|
||||
async def encode_request(obj: EmbeddingReqInput, request: Request):
|
||||
"""Handle an embedding request."""
|
||||
try:
|
||||
_refuse_decision_layout(obj)
|
||||
ret = await _global_state.tokenizer_manager.generate_request(
|
||||
obj, request
|
||||
).__anext__()
|
||||
@@ -984,6 +986,7 @@ async def encode_request(obj: EmbeddingReqInput, request: Request):
|
||||
async def classify_request(obj: EmbeddingReqInput, request: Request):
|
||||
"""Handle a reward model request. Now the arguments and return values are the same as embedding models."""
|
||||
try:
|
||||
_refuse_decision_layout(obj)
|
||||
ret = await _global_state.tokenizer_manager.generate_request(
|
||||
obj, request
|
||||
).__anext__()
|
||||
@@ -992,6 +995,12 @@ async def classify_request(obj: EmbeddingReqInput, request: Request):
|
||||
return _create_error_response(e)
|
||||
|
||||
|
||||
def _refuse_decision_layout(obj: EmbeddingReqInput) -> None:
|
||||
# Only /v1/systemone builds a decision layout, from the prompt it compiles.
|
||||
if obj.decision_layout is not None:
|
||||
raise ValueError("decision_layout is set only by /v1/systemone")
|
||||
|
||||
|
||||
@app.api_route("/flush_cache", methods=["GET", "POST"])
|
||||
@auth_level(AuthLevel.ADMIN_OPTIONAL)
|
||||
async def flush_cache(timeout: float = Query(0.0, ge=0.0)):
|
||||
@@ -2379,6 +2388,22 @@ def _execute_server_warmup(server_args: ServerArgs):
|
||||
).tolist()
|
||||
json_data["sampling_params"]["max_new_tokens"] = 0
|
||||
|
||||
if (
|
||||
not model_info["is_generation"]
|
||||
and model_info.get("architectures") == ["Qwen3_5ForConditionalGeneration"]
|
||||
and load_joint_head_config(get_model().model_path, get_model().revision)
|
||||
is not None
|
||||
):
|
||||
# A Clef checkpoint answers /v1/systemone decisions only.
|
||||
request_name = "/v1/systemone"
|
||||
json_data = {
|
||||
"model": "warmup",
|
||||
"state": "The capital city of France is Paris.",
|
||||
"questions": {
|
||||
"warmup": {"type": "noul", "instructions": "Is the state true?"}
|
||||
},
|
||||
}
|
||||
|
||||
# Send a warmup request
|
||||
warmup_timeout = envs.SGLANG_WARMUP_TIMEOUT.get()
|
||||
try:
|
||||
|
||||
@@ -83,6 +83,7 @@ class OpenAIServingDecisions(OpenAIServingBase):
|
||||
self.chat_encoding_spec = chat_serving.chat_encoding_spec
|
||||
self.prompt_text_is_lossy = chat_serving._prompt_text_round_trip_is_lossy
|
||||
self.decision_config = self.tokenizer_manager.model_config.decision_config
|
||||
self.joint_head_config = self.tokenizer_manager.model_config.joint_head_config
|
||||
tokenizer = self.tokenizer_manager.tokenizer
|
||||
# Other tokenizers skip the shortcut in _encode_labels and check the full prompt.
|
||||
self.added_tokens = (
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
"""The prompt that a Clef checkpoint's joint schema head reads.
|
||||
|
||||
Clef answers /v1/systemone questions from the backbone's hidden states instead
|
||||
of from generated text. One prompt holds the state and every question with its
|
||||
allowed options, and the head scores the span of each option. This follows
|
||||
encode_record in the checkpoint's joint_schema_model.py, piece by piece, since
|
||||
each piece is tokenized on its own.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, List, Sequence, Tuple
|
||||
|
||||
from sglang.srt.entrypoints.systemone.protocol import (
|
||||
SystemOneChoiceQuestion,
|
||||
SystemOneNoulQuestion,
|
||||
SystemOneQuestion,
|
||||
SystemOneRequest,
|
||||
)
|
||||
from sglang.srt.layers.joint_schema_head import (
|
||||
QUESTION_TYPES,
|
||||
LayoutQuestion,
|
||||
pack_decision_layout,
|
||||
)
|
||||
|
||||
_SYSTEM_PROMPT = (
|
||||
"Read the complete state and schema. Decide every field jointly. Each answer "
|
||||
"must be exactly one of that field's allowed options."
|
||||
)
|
||||
_PREFIX = f"<|im_start|>system\n{_SYSTEM_PROMPT}<|im_end|>\n<|im_start|>user\nSTATE:\n"
|
||||
_IMAGE = "<|vision_start|><|image_pad|><|vision_end|>"
|
||||
_SUFFIX = (
|
||||
"\n<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
|
||||
"JOINT SCHEMA DECISIONS:"
|
||||
)
|
||||
_NOUL_CRITERIA = {
|
||||
"true": "The proposition is true or the answer is yes.",
|
||||
"false": "The proposition is false or the answer is no.",
|
||||
}
|
||||
# The default max_length of encode_record and systemone in joint_schema_model.py.
|
||||
REFERENCE_MAX_LENGTH = 16384
|
||||
|
||||
|
||||
def _render(value: Any) -> str:
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
||||
|
||||
|
||||
def joint_schema_options(question: SystemOneQuestion) -> List[Tuple[str, Any]]:
|
||||
"""Each option's name and description, in the order the head scores them."""
|
||||
if isinstance(question, SystemOneNoulQuestion):
|
||||
criteria = dict(_NOUL_CRITERIA)
|
||||
if question.criteria is not None:
|
||||
# A description sent as null drops the default, as in encode_record.
|
||||
criteria.update(question.criteria.model_dump(exclude_unset=True))
|
||||
return [(name, criteria[name]) for name in ("true", "false")]
|
||||
if isinstance(question, SystemOneChoiceQuestion):
|
||||
return sorted(question.criteria.items(), key=lambda item: item[0])
|
||||
return [(str(level), detail) for level, detail in enumerate(question.criteria)]
|
||||
|
||||
|
||||
def encode_joint_schema(
|
||||
tokenizer: Any,
|
||||
request: SystemOneRequest,
|
||||
max_length: int,
|
||||
image_token_counts: Sequence[int],
|
||||
) -> Tuple[List[int], List[int]]:
|
||||
"""The prompt ids and their decision layout.
|
||||
|
||||
Each image's placeholder expands to its count in ``image_token_counts``, and
|
||||
the state is truncated so that the expanded prompt has at most
|
||||
``max_length`` tokens, as encode_record budgets the expanded media.
|
||||
"""
|
||||
|
||||
def tokens(text: str) -> List[int]:
|
||||
return tokenizer.encode(text, add_special_tokens=False)
|
||||
|
||||
schema = tokens("\n\nSCHEMA FIELDS:\n")
|
||||
questions = []
|
||||
for index, (question_id, question) in enumerate(request.questions.items()):
|
||||
schema += tokens(
|
||||
f"\nFIELD {index + 1}\nID: {question_id}\nTYPE: {question.type}\n"
|
||||
"INSTRUCTION: "
|
||||
)
|
||||
instructions = question.instructions
|
||||
if instructions is None or instructions == "":
|
||||
instructions = question_id
|
||||
question_start = len(schema)
|
||||
schema += tokens(_render(instructions))
|
||||
question_span = (question_start, len(schema))
|
||||
schema += tokens("\nALLOWED OPTIONS:\n")
|
||||
option_spans = []
|
||||
for option_index, (name, description) in enumerate(
|
||||
joint_schema_options(question)
|
||||
):
|
||||
schema += tokens(f"OPTION {option_index + 1}: ")
|
||||
option_start = len(schema)
|
||||
semantics = {"option_id": name}
|
||||
if description is not None:
|
||||
semantics["description"] = description
|
||||
schema += tokens(_render(semantics))
|
||||
option_spans.append((option_start, len(schema)))
|
||||
schema += tokens("\n")
|
||||
schema += tokens("END FIELD\n")
|
||||
questions.append(
|
||||
(QUESTION_TYPES.index(question.type), question_span, option_spans)
|
||||
)
|
||||
|
||||
prefix = tokens(_PREFIX)
|
||||
if request.images:
|
||||
prefix += tokens(_IMAGE * len(request.images) + "\n")
|
||||
suffix = tokens(_SUFFIX)
|
||||
# Each image's single <|image_pad|> becomes that image's tokens.
|
||||
expansion = sum(count - 1 for count in image_token_counts)
|
||||
fixed_length = len(prefix) + expansion + len(schema) + len(suffix)
|
||||
if fixed_length > max_length:
|
||||
raise ValueError(
|
||||
f"the questions need {fixed_length} prompt tokens before the state, "
|
||||
f"but the prompt holds at most {max_length}"
|
||||
)
|
||||
state = tokens(_render(request.state))[: max_length - fixed_length]
|
||||
offset = len(prefix) + len(state)
|
||||
input_ids = prefix + state + schema + suffix
|
||||
layout = pack_decision_layout(
|
||||
len(input_ids),
|
||||
[
|
||||
LayoutQuestion(
|
||||
question_type,
|
||||
(question_span[0] + offset, question_span[1] + offset),
|
||||
tuple((start + offset, end + offset) for start, end in option_spans),
|
||||
)
|
||||
for question_type, question_span, option_spans in questions
|
||||
],
|
||||
)
|
||||
return input_ids, layout
|
||||
@@ -2,14 +2,20 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import json
|
||||
import math
|
||||
import string
|
||||
from io import BytesIO
|
||||
from typing import Any, Dict, Iterator, List, Optional, Tuple
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
import orjson
|
||||
import pybase64
|
||||
from fastapi import Request
|
||||
from fastapi.responses import ORJSONResponse
|
||||
from PIL import Image
|
||||
|
||||
from sglang.srt.entrypoints.openai.serving_decisions import (
|
||||
EncodedQuestion,
|
||||
@@ -22,6 +28,11 @@ from sglang.srt.entrypoints.openai.serving_decisions import (
|
||||
render_question,
|
||||
render_text,
|
||||
)
|
||||
from sglang.srt.entrypoints.systemone.joint_schema import (
|
||||
REFERENCE_MAX_LENGTH,
|
||||
encode_joint_schema,
|
||||
joint_schema_options,
|
||||
)
|
||||
from sglang.srt.entrypoints.systemone.protocol import (
|
||||
SystemOneChoiceAnswer,
|
||||
SystemOneChoiceQuestion,
|
||||
@@ -33,6 +44,10 @@ from sglang.srt.entrypoints.systemone.protocol import (
|
||||
SystemOneScoreAnswer,
|
||||
SystemOneUsage,
|
||||
)
|
||||
from sglang.srt.layers.joint_schema_head import max_joint_prompt_tokens
|
||||
from sglang.srt.managers.io_struct import EmbeddingReqInput
|
||||
from sglang.srt.runtime_context import get_schedule
|
||||
from sglang.srt.utils import CLIENT_MEDIA_EXCEPTIONS, ImageData, get_image_bytes
|
||||
|
||||
# Beyond A to Z, every option gets a two-letter label, in this fixed order.
|
||||
_PAIR_LABELS = [a + b for a in string.ascii_uppercase for b in string.ascii_uppercase]
|
||||
@@ -44,9 +59,16 @@ _DECIDER_SYSTEM = (
|
||||
"Reply with only the selected option code."
|
||||
)
|
||||
|
||||
# Image sources whose content can change between reads, as load_image reads them.
|
||||
_MUTABLE_SOURCES = ("http://", "https://", "file://", "/")
|
||||
# Base64 characters decoded to size an inline image, 48 KiB of image data.
|
||||
# Arbitrary; an image whose header runs longer is decoded in full.
|
||||
_INLINE_HEADER_CHARS = 1 << 16
|
||||
|
||||
|
||||
class SystemOneServing(OpenAIServingDecisions):
|
||||
"""Answers System One questions with the rendering, label checks, and scoring of /v1/decisions."""
|
||||
"""Answers System One questions with the rendering, label checks, and scoring of
|
||||
/v1/decisions, or, for a Clef checkpoint, with its joint schema head."""
|
||||
|
||||
route = "/v1/systemone"
|
||||
|
||||
@@ -54,12 +76,36 @@ class SystemOneServing(OpenAIServingDecisions):
|
||||
return "systemone-"
|
||||
|
||||
def _validate_request(self, request: SystemOneRequest) -> Optional[str]:
|
||||
if self.joint_head_config is not None:
|
||||
return self._validate_joint_schema_request(request)
|
||||
return (
|
||||
self._validate_server(request.model)
|
||||
or self._validate_images(request.images)
|
||||
or self._validate_reasoning(request.chat_template_kwargs)
|
||||
)
|
||||
|
||||
def _validate_joint_schema_request(
|
||||
self, request: SystemOneRequest
|
||||
) -> Optional[str]:
|
||||
"""A Clef checkpoint renders its own prompt, without the chat template."""
|
||||
route = self.route
|
||||
if self.tokenizer_manager.tokenizer is None:
|
||||
return f"{route} requires the server tokenizer"
|
||||
_, adapter = self._parse_model_parameter(request.model)
|
||||
if adapter is not None:
|
||||
return (
|
||||
f"model names the LoRA adapter {adapter!r}, which {route} "
|
||||
"does not support"
|
||||
)
|
||||
if request.images and self.tokenizer_manager.mm_processor is None:
|
||||
return f"{route} images require a model served with multimodal input"
|
||||
if request.chat_template_kwargs:
|
||||
return (
|
||||
f"{route} renders the prompt this checkpoint's joint schema head "
|
||||
"was trained with, which takes no chat_template_kwargs"
|
||||
)
|
||||
return None
|
||||
|
||||
def _convert_to_internal_request(
|
||||
self,
|
||||
request: SystemOneRequest,
|
||||
@@ -69,6 +115,8 @@ class SystemOneServing(OpenAIServingDecisions):
|
||||
Tuple[SystemOneRequest, List[QuestionView]],
|
||||
]:
|
||||
views = [_view(question) for question in request.questions.values()]
|
||||
if self.joint_head_config is not None:
|
||||
return self._joint_schema_request(request, views), (request, views)
|
||||
encode = (
|
||||
self._encoded_systemone_questions
|
||||
if self.decision_config is None
|
||||
@@ -77,6 +125,118 @@ class SystemOneServing(OpenAIServingDecisions):
|
||||
# Lazy, so the async handler can yield to other requests between questions.
|
||||
return encode(request, views), (request, views)
|
||||
|
||||
def _joint_schema_request(
|
||||
self, request: SystemOneRequest, views: List[QuestionView]
|
||||
) -> EmbeddingReqInput:
|
||||
"""All questions in one prompt, which the joint schema head scores in one prefill.
|
||||
|
||||
The handler encodes the prompt once it has read the image sizes."""
|
||||
for question_id, view in zip(request.questions, views):
|
||||
if view.kind == "score":
|
||||
_check_legend(question_id, view)
|
||||
return EmbeddingReqInput(
|
||||
image_data=[
|
||||
ImageData(url=image.url, detail=image.detail or "auto")
|
||||
for image in request.images
|
||||
]
|
||||
or None,
|
||||
)
|
||||
|
||||
async def _encode_joint_schema_prompt(
|
||||
self, adapted_request: EmbeddingReqInput, request: SystemOneRequest
|
||||
) -> None:
|
||||
tokenizer_manager = self.tokenizer_manager
|
||||
max_length = min(
|
||||
REFERENCE_MAX_LENGTH,
|
||||
max_joint_prompt_tokens(
|
||||
context_len=tokenizer_manager.context_len,
|
||||
num_reserved_tokens=tokenizer_manager.num_reserved_tokens,
|
||||
max_req_input_len=tokenizer_manager.max_req_input_len,
|
||||
max_prefill_tokens=get_schedule().max_prefill_tokens,
|
||||
),
|
||||
)
|
||||
image_token_counts = []
|
||||
if adapted_request.image_data:
|
||||
adapted_request.image_data, image_token_counts = await self._read_images(
|
||||
adapted_request.image_data
|
||||
)
|
||||
input_ids, layout = encode_joint_schema(
|
||||
tokenizer_manager.tokenizer,
|
||||
request,
|
||||
max_length=max_length,
|
||||
image_token_counts=image_token_counts,
|
||||
)
|
||||
adapted_request.input_ids = input_ids
|
||||
adapted_request.decision_layout = layout
|
||||
|
||||
async def _read_images(
|
||||
self, images: List[ImageData]
|
||||
) -> Tuple[List[ImageData], List[int]]:
|
||||
"""Each image read once, and the tokens it expands to at the size the
|
||||
processor resizes it to."""
|
||||
read = await asyncio.gather(
|
||||
*(
|
||||
asyncio.to_thread(_read_image, index, image)
|
||||
for index, image in enumerate(images)
|
||||
)
|
||||
)
|
||||
counts = self.tokenizer_manager.mm_processor.resolve_image_token_counts(
|
||||
[header for _, header in read]
|
||||
)
|
||||
return [image for image, _ in read], counts
|
||||
|
||||
async def _answer_joint_schema(
|
||||
self,
|
||||
adapted_request: EmbeddingReqInput,
|
||||
request: SystemOneRequest,
|
||||
views: List[QuestionView],
|
||||
raw_request: Request,
|
||||
) -> ORJSONResponse:
|
||||
await self._encode_joint_schema_prompt(adapted_request, request)
|
||||
ret = await self.tokenizer_manager.generate_request(
|
||||
adapted_request, raw_request
|
||||
).__anext__()
|
||||
logits = ret["embedding"]
|
||||
names = [
|
||||
[name for name, _ in joint_schema_options(question)]
|
||||
for question in request.questions.values()
|
||||
]
|
||||
if len(logits) != sum(map(len, names)):
|
||||
raise RuntimeError("the joint schema head did not score this request")
|
||||
answers = {}
|
||||
offset = 0
|
||||
for question_id, view, question_names in zip(request.questions, views, names):
|
||||
scores = logits[offset : offset + len(question_names)]
|
||||
offset += len(question_names)
|
||||
top = max(scores)
|
||||
weights = [math.exp(score - top) for score in scores]
|
||||
total = math.fsum(weights)
|
||||
by_name = {
|
||||
name: weight / total for name, weight in zip(question_names, weights)
|
||||
}
|
||||
# The head scores true then false, which the view names yes and no.
|
||||
answers[question_id] = _answer(
|
||||
view=view,
|
||||
probabilities=(
|
||||
[by_name["true"], by_name["false"]]
|
||||
if view.kind == "yes_no"
|
||||
else [by_name[name] for name in view.names]
|
||||
),
|
||||
mass=1.0,
|
||||
question_id=question_id,
|
||||
)
|
||||
response = SystemOneResponse(
|
||||
model=self.tokenizer_manager.served_model_name,
|
||||
answers=answers,
|
||||
usage=SystemOneUsage(input_tokens=ret["meta_info"]["prompt_tokens"]),
|
||||
)
|
||||
# The head scores only the allowed options, so there is no label mass.
|
||||
return ORJSONResponse(
|
||||
content=response.model_dump(
|
||||
exclude={"answers": {"__all__": {"x_label_mass"}}}
|
||||
)
|
||||
)
|
||||
|
||||
def _encoded_systemone_questions(
|
||||
self, request: SystemOneRequest, views: List[QuestionView]
|
||||
) -> Iterator[EncodedQuestion]:
|
||||
@@ -179,6 +339,10 @@ class SystemOneServing(OpenAIServingDecisions):
|
||||
raw_request: Request,
|
||||
) -> ORJSONResponse:
|
||||
request, views = processed
|
||||
if self.joint_head_config is not None:
|
||||
return await self._answer_joint_schema(
|
||||
adapted_request, request, views, raw_request
|
||||
)
|
||||
_, _, result = await self._score(
|
||||
adapted_request=adapted_request,
|
||||
raw_request=raw_request,
|
||||
@@ -344,3 +508,38 @@ def _score_confidence(q: List[float]) -> float:
|
||||
spread = math.fsum(p * abs(i - top) for i, p in enumerate(q))
|
||||
uniform_spread = math.fsum(abs(i - (n - 1) / 2) for i in range(n)) / n
|
||||
return max(0.0, 1 - spread / uniform_spread)
|
||||
|
||||
|
||||
def _read_image(index: int, image: ImageData) -> Tuple[ImageData, Image.Image]:
|
||||
"""The image to forward, and its header opened only as far as its size, read
|
||||
once from the sources load_image reads."""
|
||||
url = image.url
|
||||
if not url.startswith(_MUTABLE_SOURCES):
|
||||
# An inline image is forwarded as sent, and the processor decodes all of it.
|
||||
header = _inline_header(url)
|
||||
if header is not None:
|
||||
return image, header
|
||||
source = unquote(urlparse(url).path) if url.startswith("file://") else url
|
||||
try:
|
||||
data = get_image_bytes(source)
|
||||
header = Image.open(BytesIO(data))
|
||||
except CLIENT_MEDIA_EXCEPTIONS as e:
|
||||
raise ValueError(f"image {index} could not be read: {e}") from e
|
||||
if url.startswith(_MUTABLE_SOURCES):
|
||||
# A source that can change between reads is forwarded as the bytes counted.
|
||||
mime = Image.MIME.get(header.format, "application/octet-stream")
|
||||
encoded = pybase64.b64encode(data).decode()
|
||||
image = dataclasses.replace(image, url=f"data:{mime};base64,{encoded}")
|
||||
return image, header
|
||||
|
||||
|
||||
def _inline_header(url: str) -> Optional[Image.Image]:
|
||||
"""An inline image's header from a prefix of its base64, or None when the
|
||||
header does not open from that prefix."""
|
||||
start = url.find(",") + 1 if url.startswith("data:") else 0
|
||||
prefix = url[start : start + _INLINE_HEADER_CHARS]
|
||||
try:
|
||||
return Image.open(BytesIO(pybase64.b64decode(prefix, validate=True)))
|
||||
except Exception:
|
||||
# A header past the prefix, or malformed data that the full read reports.
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,539 @@
|
||||
"""Joint schema head of Clef decision checkpoints.
|
||||
|
||||
A Clef checkpoint is a Qwen3.5 backbone saved with ``joint_head_config.json``
|
||||
and ``joint_head.safetensors``. The head reads the backbone's final hidden
|
||||
state at every prompt token and scores every allowed option of every question
|
||||
together: each option gathers evidence from the prompt, the questions attend
|
||||
to each other and to the prompt, and a lexical prior built from the LM head
|
||||
rows of each option's tokens keeps the option's meaning.
|
||||
|
||||
One head pass serves a whole prefill batch. Token-wise steps run on the flat
|
||||
batch, attention is variable-length per request, and span means come from
|
||||
per-request prefix sums over the schema region only, so a long state costs no
|
||||
more than its memory projection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Callable, List, Optional, Sequence, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.pooler import EmbeddingPoolerOutput
|
||||
|
||||
# Question types in the order of the head's type embedding.
|
||||
QUESTION_TYPES = ("noul", "choice", "score")
|
||||
|
||||
# Bounds of one head pass, so a large prefill batch is scored in a few passes.
|
||||
_MAX_GROUP_TOKENS = 65536
|
||||
_MAX_GROUP_SLOTS = 1 << 18
|
||||
|
||||
|
||||
class LayoutQuestion(msgspec.Struct, frozen=True):
|
||||
question_type: int
|
||||
question_span: Tuple[int, int]
|
||||
option_spans: Tuple[Tuple[int, int], ...]
|
||||
|
||||
|
||||
def max_joint_prompt_tokens(
|
||||
*,
|
||||
context_len: int,
|
||||
num_reserved_tokens: int,
|
||||
max_req_input_len: int,
|
||||
max_prefill_tokens: int,
|
||||
) -> int:
|
||||
"""The longest prompt, images expanded, that the server runs as one prefill.
|
||||
|
||||
The tokenizer manager refuses a prompt of context_len - num_reserved_tokens
|
||||
tokens, the scheduler one of max_req_input_len, and without chunked prefill
|
||||
activation memory is reserved for max_prefill_tokens.
|
||||
"""
|
||||
return min(
|
||||
context_len - num_reserved_tokens - 1,
|
||||
max_req_input_len - 1,
|
||||
max_prefill_tokens,
|
||||
)
|
||||
|
||||
|
||||
def pack_decision_layout(
|
||||
prompt_length: int, questions: Sequence[LayoutQuestion]
|
||||
) -> List[int]:
|
||||
"""Flatten a prompt's questions into the ints a request carries.
|
||||
|
||||
The layout is ``[prompt_length, num_questions]`` followed, per question,
|
||||
by ``type, question_start, question_end, num_options`` and one
|
||||
``option_start, option_end`` pair per option. Positions index the prompt
|
||||
before its image placeholders expand.
|
||||
"""
|
||||
layout = [prompt_length, len(questions)]
|
||||
for question in questions:
|
||||
layout += [
|
||||
question.question_type,
|
||||
*question.question_span,
|
||||
len(question.option_spans),
|
||||
]
|
||||
for start, end in question.option_spans:
|
||||
layout += [start, end]
|
||||
return layout
|
||||
|
||||
|
||||
def parse_decision_layout(
|
||||
layout: Sequence[int], num_tokens: int
|
||||
) -> List[LayoutQuestion]:
|
||||
"""The questions of a layout, placed on the sequence the model reads.
|
||||
|
||||
Every span follows the prompt's images, so expanding their placeholders
|
||||
shifts all spans by the number of added tokens. Raises ValueError when the
|
||||
layout does not describe the sequence.
|
||||
"""
|
||||
if len(layout) < 2 or any(type(value) is not int for value in layout):
|
||||
raise ValueError("decision_layout must be a list of ints")
|
||||
prompt_length, num_questions = layout[0], layout[1]
|
||||
shift = num_tokens - prompt_length
|
||||
if prompt_length < 1 or shift < 0 or num_questions < 1:
|
||||
raise ValueError("decision_layout does not match the prompt length")
|
||||
questions = []
|
||||
cursor = 2
|
||||
for _ in range(num_questions):
|
||||
if cursor + 4 > len(layout):
|
||||
raise ValueError("decision_layout is truncated")
|
||||
question_type, start, end, num_options = layout[cursor : cursor + 4]
|
||||
cursor += 4
|
||||
if (
|
||||
not 0 <= question_type < len(QUESTION_TYPES)
|
||||
or num_options < 1
|
||||
or not 0 <= start < end <= prompt_length
|
||||
or cursor + 2 * num_options > len(layout)
|
||||
):
|
||||
raise ValueError("decision_layout has an invalid question")
|
||||
spans = []
|
||||
for _ in range(num_options):
|
||||
option_start, option_end = layout[cursor : cursor + 2]
|
||||
cursor += 2
|
||||
if not 0 <= option_start < option_end <= prompt_length:
|
||||
raise ValueError("decision_layout has an invalid option span")
|
||||
spans.append((option_start + shift, option_end + shift))
|
||||
questions.append(
|
||||
LayoutQuestion(question_type, (start + shift, end + shift), tuple(spans))
|
||||
)
|
||||
if cursor != len(layout):
|
||||
raise ValueError("decision_layout has trailing values")
|
||||
return questions
|
||||
|
||||
|
||||
class EvidenceRoutingLayer(nn.Module):
|
||||
def __init__(self, width: int, heads: int, feedforward: int) -> None:
|
||||
super().__init__()
|
||||
self.query_norm = nn.LayerNorm(width)
|
||||
self.memory_norm = nn.LayerNorm(width)
|
||||
self.attention = nn.MultiheadAttention(width, heads, batch_first=True)
|
||||
self.feedforward_norm = nn.LayerNorm(width)
|
||||
# Identity slots keep the checkpoint's Sequential indices (dropout at 2 and 4).
|
||||
self.feedforward = nn.Sequential(
|
||||
nn.Linear(width, feedforward),
|
||||
nn.GELU(),
|
||||
nn.Identity(),
|
||||
nn.Linear(feedforward, width),
|
||||
nn.Identity(),
|
||||
)
|
||||
|
||||
|
||||
class JointSchemaHead(nn.Module):
|
||||
"""Parameters named as in the checkpoint's ``joint_head.safetensors``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
width: int,
|
||||
routing_layers: int,
|
||||
layers: int,
|
||||
heads: int,
|
||||
feedforward: int,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_norm = nn.LayerNorm(hidden_size)
|
||||
self.memory_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.question_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_question_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.global_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_context_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_lexical_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.type_embedding = nn.Embedding(len(QUESTION_TYPES), width)
|
||||
self.evidence_layers = nn.ModuleList(
|
||||
EvidenceRoutingLayer(width, heads, feedforward)
|
||||
for _ in range(routing_layers)
|
||||
)
|
||||
self.option_summary_norm = nn.LayerNorm(width)
|
||||
self.layers = nn.ModuleList(
|
||||
nn.TransformerDecoderLayer(
|
||||
d_model=width,
|
||||
nhead=heads,
|
||||
dim_feedforward=feedforward,
|
||||
dropout=0.0,
|
||||
activation="gelu",
|
||||
batch_first=True,
|
||||
norm_first=True,
|
||||
)
|
||||
for _ in range(layers)
|
||||
)
|
||||
self.field_norm = nn.LayerNorm(width)
|
||||
self.option_norm = nn.LayerNorm(width)
|
||||
self.residual_scorer = nn.Sequential(
|
||||
nn.Linear(width * 4, width), nn.GELU(), nn.Identity(), nn.Linear(width, 1)
|
||||
)
|
||||
self.prior_logit_scale = nn.Parameter(torch.zeros(()))
|
||||
self.joint_logit_scale = nn.Parameter(torch.zeros(()))
|
||||
self.residual_gate = nn.Parameter(torch.zeros(()))
|
||||
|
||||
@torch.no_grad()
|
||||
def forward_batch(
|
||||
self,
|
||||
hidden: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
lengths: List[int],
|
||||
layouts: List[List[LayoutQuestion]],
|
||||
output_embedding_weight: torch.Tensor,
|
||||
) -> List[torch.Tensor]:
|
||||
"""One FP32 logit vector per request, its questions' options in layout order.
|
||||
|
||||
``hidden`` holds the final hidden states of the requests back to back.
|
||||
"""
|
||||
outputs: List[torch.Tensor] = []
|
||||
start = 0
|
||||
for group in _groups(lengths, layouts):
|
||||
group_lengths = [lengths[i] for i in group]
|
||||
end = start + sum(group_lengths)
|
||||
outputs += self._forward_group(
|
||||
hidden[start:end],
|
||||
input_ids[start:end],
|
||||
group_lengths,
|
||||
[layouts[i] for i in group],
|
||||
output_embedding_weight,
|
||||
)
|
||||
start = end
|
||||
return outputs
|
||||
|
||||
def _forward_group(
|
||||
self,
|
||||
hidden: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
lengths: List[int],
|
||||
layouts: List[List[LayoutQuestion]],
|
||||
output_embedding_weight: torch.Tensor,
|
||||
) -> List[torch.Tensor]:
|
||||
device, dtype = hidden.device, hidden.dtype
|
||||
batch = len(layouts)
|
||||
question_bounds, option_bounds = [], []
|
||||
type_ids, owner, slot, question_request, last_tokens = [], [], [], [], []
|
||||
regions, region_offsets = [], [0]
|
||||
token_cu, question_cu, option_cu, option_counts = [0], [0], [0], []
|
||||
for request, (length, questions) in enumerate(zip(lengths, layouts)):
|
||||
offset = token_cu[-1]
|
||||
low = min(min(q.question_span[0], q.option_spans[0][0]) for q in questions)
|
||||
high = max(
|
||||
max(q.question_span[1], *(e for _, e in q.option_spans))
|
||||
for q in questions
|
||||
)
|
||||
# Prefix row of span position p in this request's schema region.
|
||||
base = region_offsets[-1] + request - low
|
||||
regions.append((offset + low, offset + high))
|
||||
request_options = 0
|
||||
for question in questions:
|
||||
count = len(question.option_spans)
|
||||
question_bounds += (
|
||||
base + question.question_span[0],
|
||||
base + question.question_span[1],
|
||||
)
|
||||
for option_slot, (option_start, option_end) in enumerate(
|
||||
question.option_spans
|
||||
):
|
||||
option_bounds += (base + option_start, base + option_end)
|
||||
owner.append(len(type_ids))
|
||||
slot.append(option_slot)
|
||||
type_ids.append(question.question_type)
|
||||
question_request.append(request)
|
||||
request_options += count
|
||||
last_tokens.append(offset + length - 1)
|
||||
token_cu.append(offset + length)
|
||||
region_offsets.append(region_offsets[-1] + high - low)
|
||||
question_cu.append(len(type_ids))
|
||||
option_cu.append(option_cu[-1] + request_options)
|
||||
option_counts.append(request_options)
|
||||
|
||||
num_questions = len(type_ids)
|
||||
widest = max(len(q.option_spans) for questions in layouts for q in questions)
|
||||
sizes = [
|
||||
2 * num_questions,
|
||||
len(option_bounds),
|
||||
num_questions,
|
||||
len(owner),
|
||||
len(slot),
|
||||
num_questions,
|
||||
batch,
|
||||
3 * (batch + 1),
|
||||
]
|
||||
packed = torch.tensor(
|
||||
question_bounds
|
||||
+ option_bounds
|
||||
+ type_ids
|
||||
+ owner
|
||||
+ slot
|
||||
+ question_request
|
||||
+ last_tokens
|
||||
+ token_cu
|
||||
+ question_cu
|
||||
+ option_cu,
|
||||
dtype=torch.long,
|
||||
).to(device, non_blocking=True)
|
||||
(
|
||||
question_bounds_t,
|
||||
option_bounds_t,
|
||||
type_ids_t,
|
||||
owner_t,
|
||||
slot_t,
|
||||
question_request_t,
|
||||
last_tokens_t,
|
||||
cu_t,
|
||||
) = torch.split(packed, sizes)
|
||||
question_bounds_t = question_bounds_t.view(-1, 2)
|
||||
option_bounds_t = option_bounds_t.view(-1, 2)
|
||||
token_cu_t, question_cu_t, option_cu_t = (
|
||||
cu_t.to(torch.int32).view(3, batch + 1).unbind(0)
|
||||
)
|
||||
token_bounds = (token_cu_t, max(lengths))
|
||||
question_bounds_cu = (question_cu_t, max(len(q) for q in layouts))
|
||||
option_bounds_cu = (option_cu_t, max(option_counts))
|
||||
|
||||
normalized = self.hidden_norm(hidden)
|
||||
# Running sums over each schema region, after one zero row per request.
|
||||
region_total = region_offsets[-1] + batch
|
||||
hidden_prefix = torch.zeros(
|
||||
region_total, normalized.shape[-1], dtype=torch.float32, device=device
|
||||
)
|
||||
lexical_prefix = torch.zeros(
|
||||
region_total,
|
||||
output_embedding_weight.shape[-1],
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
for request, (low, high) in enumerate(regions):
|
||||
row = region_offsets[request] + request + 1
|
||||
torch.cumsum(
|
||||
normalized[low:high].float(),
|
||||
dim=0,
|
||||
out=hidden_prefix[row : row + high - low],
|
||||
)
|
||||
torch.cumsum(
|
||||
output_embedding_weight[input_ids[low:high]].float(),
|
||||
dim=0,
|
||||
out=lexical_prefix[row : row + high - low],
|
||||
)
|
||||
|
||||
def span_means(prefix: torch.Tensor, bounds: torch.Tensor) -> torch.Tensor:
|
||||
sums = prefix[bounds[:, 1]] - prefix[bounds[:, 0]]
|
||||
return (sums / (bounds[:, 1] - bounds[:, 0]).unsqueeze(-1)).to(dtype)
|
||||
|
||||
memory = self.memory_projection(normalized)
|
||||
global_vectors = normalized[last_tokens_t]
|
||||
question_vectors = span_means(hidden_prefix, question_bounds_t)
|
||||
option_contexts = span_means(hidden_prefix, option_bounds_t)
|
||||
lexical = span_means(lexical_prefix, option_bounds_t)
|
||||
|
||||
routed = (
|
||||
self.option_context_projection(option_contexts)
|
||||
+ self.option_lexical_projection(lexical)
|
||||
+ self.option_question_projection(question_vectors)[owner_t]
|
||||
)
|
||||
for layer in self.evidence_layers:
|
||||
routed = routed + _attention(
|
||||
layer.attention,
|
||||
layer.query_norm(routed),
|
||||
layer.memory_norm(memory),
|
||||
option_bounds_cu,
|
||||
token_bounds,
|
||||
)
|
||||
routed = routed + layer.feedforward(layer.feedforward_norm(routed))
|
||||
|
||||
base_fields = self.question_projection(question_vectors)
|
||||
scores = torch.matmul(
|
||||
routed.unsqueeze(1), base_fields[owner_t].unsqueeze(2)
|
||||
).view(-1) / math.sqrt(routed.shape[-1])
|
||||
padded_scores = torch.full(
|
||||
(num_questions, widest), float("-inf"), dtype=torch.float32, device=device
|
||||
)
|
||||
padded_scores[owner_t, slot_t] = scores.float()
|
||||
weights = torch.softmax(padded_scores, dim=1).to(dtype)
|
||||
padded_options = routed.new_zeros(num_questions, widest, routed.shape[-1])
|
||||
padded_options[owner_t, slot_t] = routed
|
||||
summaries = torch.sum(weights.unsqueeze(-1) * padded_options, dim=1)
|
||||
fields = (
|
||||
base_fields
|
||||
+ self.option_summary_norm(summaries)
|
||||
+ self.global_projection(global_vectors)[question_request_t]
|
||||
+ self.type_embedding(type_ids_t)
|
||||
)
|
||||
for layer in self.layers:
|
||||
fields = fields + _attention(
|
||||
layer.self_attn,
|
||||
layer.norm1(fields),
|
||||
None,
|
||||
question_bounds_cu,
|
||||
question_bounds_cu,
|
||||
)
|
||||
fields = fields + _attention(
|
||||
layer.multihead_attn,
|
||||
layer.norm2(fields),
|
||||
memory,
|
||||
question_bounds_cu,
|
||||
token_bounds,
|
||||
)
|
||||
fields = fields + layer.linear2(
|
||||
layer.activation(layer.linear1(layer.norm3(fields)))
|
||||
)
|
||||
fields = self.field_norm(fields)
|
||||
|
||||
prior_scale = self.prior_logit_scale.clamp(max=math.log(100.0)).exp()
|
||||
joint_scale = self.joint_logit_scale.clamp(max=math.log(100.0)).exp()
|
||||
anchor = F.normalize(
|
||||
question_vectors + global_vectors[question_request_t], dim=-1
|
||||
)
|
||||
prior = prior_scale * torch.matmul(
|
||||
F.normalize(lexical, dim=-1).unsqueeze(1), anchor[owner_t].unsqueeze(2)
|
||||
).view(-1)
|
||||
options = self.option_norm(routed)
|
||||
repeated = fields[owner_t]
|
||||
cosine = F.cosine_similarity(repeated, options, dim=-1)
|
||||
features = torch.cat(
|
||||
[repeated, options, repeated * options, torch.abs(repeated - options)],
|
||||
dim=-1,
|
||||
)
|
||||
residual = self.residual_scorer(features).squeeze(-1)
|
||||
logits = prior + torch.sigmoid(self.residual_gate) * (
|
||||
joint_scale * cosine + residual
|
||||
)
|
||||
return list(torch.split(logits.float(), option_counts))
|
||||
|
||||
|
||||
def _attention(
|
||||
module: nn.MultiheadAttention,
|
||||
queries: torch.Tensor,
|
||||
keys: Optional[torch.Tensor],
|
||||
query_bounds: Tuple[torch.Tensor, int],
|
||||
key_bounds: Tuple[torch.Tensor, int],
|
||||
) -> torch.Tensor:
|
||||
"""nn.MultiheadAttention over packed sequences, keys and values from ``keys``
|
||||
(or from ``queries`` when None), attending within each request."""
|
||||
width = queries.shape[-1]
|
||||
heads = module.num_heads
|
||||
if keys is None:
|
||||
query, key, value = F.linear(
|
||||
queries, module.in_proj_weight, module.in_proj_bias
|
||||
).chunk(3, dim=-1)
|
||||
else:
|
||||
query_weight, key_value_weight = module.in_proj_weight.split([width, 2 * width])
|
||||
query_bias, key_value_bias = module.in_proj_bias.split([width, 2 * width])
|
||||
query = F.linear(queries, query_weight, query_bias)
|
||||
key, value = F.linear(keys, key_value_weight, key_value_bias).chunk(2, dim=-1)
|
||||
head_dim = width // heads
|
||||
output = torch.ops.aten._flash_attention_forward(
|
||||
query.reshape(-1, heads, head_dim),
|
||||
key.reshape(-1, heads, head_dim),
|
||||
value.reshape(-1, heads, head_dim),
|
||||
query_bounds[0],
|
||||
key_bounds[0],
|
||||
query_bounds[1],
|
||||
key_bounds[1],
|
||||
0.0,
|
||||
False,
|
||||
False,
|
||||
)[0]
|
||||
return module.out_proj(output.reshape(-1, width))
|
||||
|
||||
|
||||
def _groups(lengths: List[int], layouts: List[List[LayoutQuestion]]) -> List[range]:
|
||||
"""Consecutive requests per head pass, bounded in tokens and padded option slots."""
|
||||
groups, start, tokens, questions, widest = [], 0, 0, 0, 0
|
||||
for index, (length, layout) in enumerate(zip(lengths, layouts)):
|
||||
layout_widest = max(len(question.option_spans) for question in layout)
|
||||
grown = (questions + len(layout)) * max(widest, layout_widest)
|
||||
if index > start and (
|
||||
tokens + length > _MAX_GROUP_TOKENS or grown > _MAX_GROUP_SLOTS
|
||||
):
|
||||
groups.append(range(start, index))
|
||||
start, tokens, questions, widest = index, 0, 0, 0
|
||||
tokens += length
|
||||
questions += len(layout)
|
||||
widest = max(widest, layout_widest)
|
||||
groups.append(range(start, len(lengths)))
|
||||
return groups
|
||||
|
||||
|
||||
class JointSchemaPooler(nn.Module):
|
||||
"""Pooler of a Clef checkpoint: the joint schema head's option logits per request."""
|
||||
|
||||
def __init__(
|
||||
self, head: JointSchemaHead, output_embedding: Callable[[], torch.Tensor]
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.head = head
|
||||
# A callable, so the LM head is not registered twice as a parameter.
|
||||
self._output_embedding = output_embedding
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, forward_batch
|
||||
) -> EmbeddingPoolerOutput:
|
||||
if (
|
||||
forward_batch.decision_layouts is None
|
||||
or forward_batch.extend_seq_lens_cpu is None
|
||||
):
|
||||
return EmbeddingPoolerOutput(
|
||||
embeddings=[
|
||||
hidden_states.new_empty(0, dtype=torch.float32)
|
||||
for _ in range(forward_batch.batch_size)
|
||||
]
|
||||
)
|
||||
lengths = list(forward_batch.extend_seq_lens_cpu)
|
||||
prefix_lengths = forward_batch.extend_prefix_lens_cpu or [0] * len(lengths)
|
||||
raw_layouts = forward_batch.decision_layouts
|
||||
embeddings: List[torch.Tensor] = [
|
||||
hidden_states.new_empty(0, dtype=torch.float32) for _ in lengths
|
||||
]
|
||||
# The tokenizer manager checks every layout against its prompt, and the
|
||||
# server runs each prompt in one uncached prefill. A layout that no longer
|
||||
# fits its prompt, as after a truncation, gets no scores, which the
|
||||
# decision route reports as a server error.
|
||||
layouts = {}
|
||||
for i, (layout, prefix, length) in enumerate(
|
||||
zip(raw_layouts, prefix_lengths, lengths)
|
||||
):
|
||||
if layout is None or prefix != 0:
|
||||
continue
|
||||
try:
|
||||
layouts[i] = parse_decision_layout(layout, length)
|
||||
except ValueError:
|
||||
continue
|
||||
offsets = [0]
|
||||
for length in lengths:
|
||||
offsets.append(offsets[-1] + length)
|
||||
runs: List[List[int]] = []
|
||||
for i in layouts:
|
||||
if runs and runs[-1][-1] == i - 1:
|
||||
runs[-1].append(i)
|
||||
else:
|
||||
runs.append([i])
|
||||
for run in runs:
|
||||
start, end = offsets[run[0]], offsets[run[-1] + 1]
|
||||
logits = self.head.forward_batch(
|
||||
hidden_states[start:end],
|
||||
forward_batch.input_ids[start:end],
|
||||
[lengths[i] for i in run],
|
||||
[layouts[i] for i in run],
|
||||
self._output_embedding(),
|
||||
)
|
||||
for i, values in zip(run, logits):
|
||||
embeddings[i] = values
|
||||
return EmbeddingPoolerOutput(embeddings=embeddings)
|
||||
@@ -1253,6 +1253,11 @@ class EmbeddingReqInput:
|
||||
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
|
||||
token_indices_to_pool: Optional[Union[List[List[int]], List[int]]] = None
|
||||
|
||||
# Question and option spans that a Clef checkpoint's joint schema head
|
||||
# scores, as packed by layers.joint_schema_head.pack_decision_layout.
|
||||
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
|
||||
decision_layout: Optional[Union[List[List[int]], List[int]]] = None
|
||||
|
||||
def regenerate_rid(self):
|
||||
"""Generate a new request ID and return it."""
|
||||
if isinstance(self.rid, list):
|
||||
@@ -1469,6 +1474,11 @@ class EmbeddingReqInput:
|
||||
if self.token_indices_to_pool is not None
|
||||
else None
|
||||
),
|
||||
decision_layout=(
|
||||
self.decision_layout[i]
|
||||
if self.decision_layout is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
cache[i] = sub
|
||||
return sub
|
||||
@@ -1500,6 +1510,8 @@ class TokenizedEmbeddingReqInput(BaseReq, kw_only=True):
|
||||
multi_item_delimiter_indices: Optional[List[int]] = None
|
||||
# Token positions for setwise pooling readout
|
||||
token_indices_to_pool: Optional[List[int]] = None
|
||||
# Question and option spans for a joint schema head
|
||||
decision_layout: Optional[List[int]] = None
|
||||
# For observability
|
||||
# Pickled Optional[Union[APIServerReqTimeStats, DPControllerReqTimeStats]]
|
||||
time_stats: Optional[PickleWrapper] = None
|
||||
|
||||
@@ -1050,6 +1050,7 @@ class Req(ReqDllmMixin):
|
||||
return_pooled_hidden_states: bool = False,
|
||||
multi_item_delimiter_indices: Optional[List[int]] = None,
|
||||
token_indices_to_pool: Optional[List[int]] = None,
|
||||
decision_layout: Optional[List[int]] = None,
|
||||
session_id: Optional[str] = None,
|
||||
cache_salt: Optional[str] = None,
|
||||
):
|
||||
@@ -1083,6 +1084,7 @@ class Req(ReqDllmMixin):
|
||||
self.positional_embed_overrides = positional_embed_overrides
|
||||
self.multi_item_delimiter_indices = multi_item_delimiter_indices
|
||||
self.token_indices_to_pool = token_indices_to_pool
|
||||
self.decision_layout = decision_layout
|
||||
|
||||
# For req-level memory management
|
||||
self.kv = ReqKvInfo()
|
||||
|
||||
@@ -3465,6 +3465,7 @@ class Scheduler(
|
||||
return_pooled_hidden_states=recv_req.return_pooled_hidden_states,
|
||||
multi_item_delimiter_indices=recv_req.multi_item_delimiter_indices,
|
||||
token_indices_to_pool=recv_req.token_indices_to_pool,
|
||||
decision_layout=recv_req.decision_layout,
|
||||
)
|
||||
req.tokenizer = self.tokenizer
|
||||
self._maybe_namespace_elastic_radix_cache(req)
|
||||
|
||||
@@ -64,6 +64,10 @@ from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX
|
||||
from sglang.srt.disaggregation.encoder.receiver import create_mm_receiver
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.joint_schema_head import (
|
||||
max_joint_prompt_tokens,
|
||||
parse_decision_layout,
|
||||
)
|
||||
from sglang.srt.lora.lora_registry import LoRARef, LoRARegistry
|
||||
from sglang.srt.managers.async_dynamic_batch_tokenizer import AsyncDynamicbatchTokenizer
|
||||
from sglang.srt.managers.disagg_service import start_disagg_service
|
||||
@@ -1295,6 +1299,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
raise ValueError(
|
||||
"encoder SWA replay cannot return cached prompt logprobs"
|
||||
)
|
||||
# A decision layout indexes the whole prompt, so it is checked before any
|
||||
# truncation below could cut the prompt.
|
||||
self._validate_joint_schema_request(obj, input_ids)
|
||||
_max_req_len = self.context_len
|
||||
input_token_num = len(input_ids) if input_ids is not None else 0
|
||||
input_token_num += self.num_reserved_tokens
|
||||
@@ -1399,6 +1406,36 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
"return_sampling_mask=true."
|
||||
)
|
||||
|
||||
def _validate_joint_schema_request(
|
||||
self, obj: Union[GenerateReqInput, EmbeddingReqInput], input_ids: List[int]
|
||||
) -> None:
|
||||
"""A Clef checkpoint scores the spans of a decision layout, and nothing else."""
|
||||
if self.model_config.joint_head_config is None:
|
||||
return
|
||||
if isinstance(obj, GenerateReqInput) or obj.decision_layout is None:
|
||||
# Health checks only need a response, which is an empty embedding.
|
||||
if (
|
||||
isinstance(obj, EmbeddingReqInput)
|
||||
and isinstance(obj.rid, str)
|
||||
and obj.rid.startswith(HEALTH_CHECK_RID_PREFIX)
|
||||
):
|
||||
return
|
||||
raise ValueError(
|
||||
"This checkpoint answers schema decisions, send them to /v1/systemone"
|
||||
)
|
||||
limit = max_joint_prompt_tokens(
|
||||
context_len=self.context_len,
|
||||
num_reserved_tokens=self.num_reserved_tokens,
|
||||
max_req_input_len=self.max_req_input_len,
|
||||
max_prefill_tokens=get_schedule().max_prefill_tokens,
|
||||
)
|
||||
if len(input_ids) > limit:
|
||||
raise ValueError(
|
||||
f"The prompt has {len(input_ids)} tokens, but this server scores "
|
||||
f"at most {limit} in one prefill"
|
||||
)
|
||||
parse_decision_layout(obj.decision_layout, len(input_ids))
|
||||
|
||||
def _validate_mm_limits(
|
||||
self, obj: Union[GenerateReqInput, EmbeddingReqInput]
|
||||
) -> None:
|
||||
@@ -1575,6 +1612,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
return_pooled_hidden_states=obj.return_pooled_hidden_states,
|
||||
multi_item_delimiter_indices=obj.multi_item_delimiter_indices,
|
||||
token_indices_to_pool=obj.token_indices_to_pool,
|
||||
decision_layout=obj.decision_layout,
|
||||
)
|
||||
|
||||
tokenized_obj.time_stats = self.rid_to_state[obj.rid].time_stats
|
||||
|
||||
@@ -614,6 +614,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# Setwise pooling readout positions (CPU tensors, one per request)
|
||||
token_indices_to_pool: Optional[List[torch.Tensor]] = None
|
||||
|
||||
# Joint schema head decision layouts (one per request, None if absent)
|
||||
decision_layouts: Optional[List[Optional[List[int]]]] = None
|
||||
|
||||
# === Borrowed from ScheduleBatch: compound (carry their own device tensors) ===
|
||||
# Sampling info
|
||||
sampling_info: SamplingBatchInfo = None
|
||||
@@ -1264,6 +1267,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
for r in batch.reqs
|
||||
]
|
||||
|
||||
if any(r.decision_layout is not None for r in batch.reqs):
|
||||
self.decision_layouts = [r.decision_layout for r in batch.reqs]
|
||||
|
||||
token_type_ids = [
|
||||
r.token_type_ids for r in batch.reqs if r.token_type_ids is not None
|
||||
]
|
||||
|
||||
@@ -34,7 +34,10 @@ from sglang.kernels.ops.attention.triton_gdn_fused_proj import (
|
||||
from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul
|
||||
|
||||
# Configs
|
||||
from sglang.srt.configs.model_config import load_decision_config
|
||||
from sglang.srt.configs.model_config import (
|
||||
load_decision_config,
|
||||
load_joint_head_config,
|
||||
)
|
||||
from sglang.srt.configs.qwen3_5 import (
|
||||
Qwen3_5Config,
|
||||
Qwen3_5MoeConfig,
|
||||
@@ -54,6 +57,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
from sglang.srt.layers.joint_schema_head import JointSchemaHead, JointSchemaPooler
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
@@ -2330,6 +2334,16 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration):
|
||||
self.visual.deepstack_visual_indexes if self.visual is not None else []
|
||||
)
|
||||
|
||||
if config.architectures == ["Qwen3_5ForConditionalGeneration"]:
|
||||
model = get_model()
|
||||
joint_head_config = load_joint_head_config(model.model_path, model.revision)
|
||||
if joint_head_config is not None:
|
||||
# A Clef checkpoint, whose joint schema head scores every option
|
||||
# of every question from the final hidden states and LM head rows.
|
||||
self.pooler = JointSchemaPooler(
|
||||
JointSchemaHead(**joint_head_config), lambda: self.lm_head.weight
|
||||
)
|
||||
|
||||
def get_hidden_dim(self, module_name: str, layer_idx: int):
|
||||
return self.model.get_hidden_dim(module_name, layer_idx)
|
||||
|
||||
@@ -2470,8 +2484,23 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration):
|
||||
if bare_backbone and self.pp_group.is_last_rank:
|
||||
self._load_decision_readout()
|
||||
loaded_params.add("lm_head.weight")
|
||||
if isinstance(self.pooler, JointSchemaPooler):
|
||||
self._load_joint_head()
|
||||
loaded_params.update(
|
||||
f"pooler.head.{name}" for name, _ in self.pooler.head.named_parameters()
|
||||
)
|
||||
return loaded_params
|
||||
|
||||
def _load_joint_head(self) -> None:
|
||||
"""Load joint_head.safetensors, which the checkpoint index leaves out."""
|
||||
model = get_model()
|
||||
state_dict = load_file(
|
||||
cached_file(
|
||||
model.model_path, "joint_head.safetensors", revision=model.revision
|
||||
)
|
||||
)
|
||||
self.pooler.head.load_state_dict(state_dict, strict=True)
|
||||
|
||||
def _load_decision_readout(self) -> None:
|
||||
"""Place the checkpoint's decision readout in the LM head rows of its codes."""
|
||||
model = get_model()
|
||||
|
||||
@@ -169,6 +169,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
|
||||
args = ServerArgs(model_path="dummy")
|
||||
args._model_config = SimpleNamespace(
|
||||
is_embedding_gemma=True,
|
||||
joint_head_config=None,
|
||||
is_multimodal=False,
|
||||
context_len=2048,
|
||||
hf_config=SimpleNamespace(architectures=["Gemma3TextModel"]),
|
||||
@@ -205,6 +206,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
|
||||
is_embedding_requested=False,
|
||||
is_embedding_gemma=False,
|
||||
),
|
||||
joint_head_config=None,
|
||||
is_multimodal=False,
|
||||
hf_config=SimpleNamespace(architectures=["BertModel"]),
|
||||
)
|
||||
|
||||
@@ -159,6 +159,7 @@ class ScoringManager(TokenizerManagerScoreMixin):
|
||||
model_type="qwen3_5_moe",
|
||||
),
|
||||
decision_config=None,
|
||||
joint_head_config=None,
|
||||
)
|
||||
pad, video = (
|
||||
tokenizer.encode(token, add_special_tokens=False)[0]
|
||||
|
||||
@@ -0,0 +1,474 @@
|
||||
"""Unit tests for /v1/systemone on Clef checkpoints: the joint schema prompt and its
|
||||
decision layout, and answers built from the joint schema head's option logits."""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import math
|
||||
import random
|
||||
import unittest
|
||||
from io import BytesIO
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import pybase64
|
||||
from PIL import Image
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from sglang.srt.entrypoints import http_server
|
||||
from sglang.srt.entrypoints.openai.protocol import DecisionRequest
|
||||
from sglang.srt.entrypoints.openai.serving_decisions import OpenAIServingDecisions
|
||||
from sglang.srt.entrypoints.systemone.joint_schema import encode_joint_schema
|
||||
from sglang.srt.entrypoints.systemone.protocol import SystemOneRequest
|
||||
from sglang.srt.entrypoints.systemone.serving import SystemOneServing, _read_image
|
||||
from sglang.srt.layers.joint_schema_head import (
|
||||
LayoutQuestion,
|
||||
pack_decision_layout,
|
||||
parse_decision_layout,
|
||||
)
|
||||
from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput
|
||||
from sglang.srt.managers.tokenizer_manager import TokenizerManager
|
||||
from sglang.srt.runtime_context import (
|
||||
get_context,
|
||||
get_schedule,
|
||||
publish,
|
||||
restore_context,
|
||||
snapshot_context,
|
||||
)
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.utils import ImageData
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
|
||||
|
||||
# Tokenizer files only, with the Qwen3.5 vocabulary of Clef checkpoints.
|
||||
TOKENIZER = "Qwen/Qwen3.5-35B-A3B"
|
||||
|
||||
REQUEST = {
|
||||
"model": "clef",
|
||||
"state": {"ticket": "Refund for order 1182", "amount": 42},
|
||||
"questions": {
|
||||
"team": {
|
||||
"type": "choice",
|
||||
"instructions": "Which team handles it?",
|
||||
"criteria": {
|
||||
"support": "Customer support",
|
||||
"billing": "Payments and refunds",
|
||||
"legal": None,
|
||||
},
|
||||
},
|
||||
"urgent": {
|
||||
"type": "noul",
|
||||
"instructions": "Is it urgent?",
|
||||
"criteria": {"false": None},
|
||||
},
|
||||
"severity": {"type": "score", "criteria": ["none", "low", "high"]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class PatchCounter:
|
||||
"""Stands in for the multimodal processor: one image token per 28x28 patch."""
|
||||
|
||||
@staticmethod
|
||||
def resolve_image_token_counts(images):
|
||||
return [(image.height // 28) * (image.width // 28) for image in images]
|
||||
|
||||
|
||||
def _png_bytes(width, height):
|
||||
buffer = BytesIO()
|
||||
Image.new("RGB", (width, height), "red").save(buffer, format="PNG")
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def _png(width, height):
|
||||
return (
|
||||
"data:image/png;base64," + base64.b64encode(_png_bytes(width, height)).decode()
|
||||
)
|
||||
|
||||
|
||||
def _expanded_length(request):
|
||||
images = [
|
||||
Image.open(BytesIO(base64.b64decode(image.url.split(",", 1)[1])))
|
||||
for image in request.image_data or []
|
||||
]
|
||||
counts = PatchCounter.resolve_image_token_counts(images)
|
||||
return len(request.input_ids) + sum(count - 1 for count in counts)
|
||||
|
||||
|
||||
class HeadManager:
|
||||
"""Replace model execution with option logits 0, 1, 2, ... per question, in head order,
|
||||
after expanding images and refusing prompts that the tokenizer manager, the
|
||||
scheduler, or the memory reserved for one prefill cannot take."""
|
||||
|
||||
def __init__(self, tokenizer):
|
||||
self.server_args = ServerArgs(model_path="dummy")
|
||||
publish(self.server_args, role="test")
|
||||
self.tokenizer = tokenizer
|
||||
self.model_config = SimpleNamespace(
|
||||
is_multimodal=True,
|
||||
hf_config=SimpleNamespace(
|
||||
architectures=["Qwen3_5ForConditionalGeneration"], model_type="qwen3_5"
|
||||
),
|
||||
decision_config=None,
|
||||
joint_head_config={"hidden_size": 4096},
|
||||
)
|
||||
self.is_generation = False
|
||||
self.allow_auto_truncate = False
|
||||
self.context_len = 4096
|
||||
self.num_reserved_tokens = 0
|
||||
# min(context_len - 1, kv_capacity - 1) - 5, as the scheduler reports it.
|
||||
self.max_req_input_len = 4090
|
||||
self.mm_processor = PatchCounter()
|
||||
self.request_logger = SimpleNamespace(log_requests=False)
|
||||
self.served_model_name = "served-model"
|
||||
self.requests = []
|
||||
|
||||
def config_value(self, name):
|
||||
return None
|
||||
|
||||
async def generate_request(self, request, raw_request):
|
||||
self.requests.append(request)
|
||||
num_tokens = _expanded_length(request)
|
||||
if num_tokens + self.num_reserved_tokens >= self.context_len:
|
||||
raise ValueError(f"The input ({num_tokens} tokens) is too long")
|
||||
if num_tokens >= self.max_req_input_len:
|
||||
raise ValueError(f"The scheduler refuses a prompt of {num_tokens} tokens")
|
||||
if num_tokens > get_schedule().max_prefill_tokens:
|
||||
raise ValueError(f"A prefill of {num_tokens} tokens outgrows its memory")
|
||||
questions = parse_decision_layout(request.decision_layout, num_tokens)
|
||||
logits = [float(i) for q in questions for i in range(len(q.option_spans))]
|
||||
yield {
|
||||
"embedding": logits,
|
||||
"meta_info": {"prompt_tokens": num_tokens},
|
||||
}
|
||||
|
||||
|
||||
def _handler(manager, serving_class=SystemOneServing):
|
||||
template_manager = SimpleNamespace(
|
||||
chat_template_name=None,
|
||||
jinja_template_content_format="openai",
|
||||
reasoning_config=None,
|
||||
suggested_reasoning_parser=None,
|
||||
)
|
||||
chat_serving = SimpleNamespace(
|
||||
tokenizer_manager=manager,
|
||||
template_manager=template_manager,
|
||||
default_chat_template_kwargs={},
|
||||
chat_encoding_spec=None,
|
||||
_prompt_text_round_trip_is_lossy=False,
|
||||
reasoning_parser=None,
|
||||
)
|
||||
return serving_class(chat_serving)
|
||||
|
||||
|
||||
def _softmax(values):
|
||||
weights = [math.exp(value - max(values)) for value in values]
|
||||
return [weight / sum(weights) for weight in weights]
|
||||
|
||||
|
||||
class TestJointSchemaPrompt(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
|
||||
|
||||
def _encode(self, request=REQUEST, max_length=4096):
|
||||
input_ids, layout = encode_joint_schema(
|
||||
self.tokenizer,
|
||||
SystemOneRequest(**request),
|
||||
max_length=max_length,
|
||||
image_token_counts=[],
|
||||
)
|
||||
return input_ids, parse_decision_layout(layout, len(input_ids))
|
||||
|
||||
def _text(self, input_ids, span):
|
||||
return self.tokenizer.decode(input_ids[span[0] : span[1]])
|
||||
|
||||
def test_prompt_is_the_trained_joint_schema_prompt(self):
|
||||
input_ids, _ = self._encode()
|
||||
prompt = self.tokenizer.decode(input_ids)
|
||||
self.assertTrue(
|
||||
prompt.startswith(
|
||||
"<|im_start|>system\nRead the complete state and schema. Decide every "
|
||||
"field jointly. Each answer must be exactly one of that field's allowed "
|
||||
"options.<|im_end|>\n<|im_start|>user\nSTATE:\n"
|
||||
'{"amount":42,"ticket":"Refund for order 1182"}\n\nSCHEMA FIELDS:\n'
|
||||
"\nFIELD 1\nID: team\nTYPE: choice\nINSTRUCTION: Which team handles it?"
|
||||
"\nALLOWED OPTIONS:\n"
|
||||
'OPTION 1: {"description":"Payments and refunds","option_id":"billing"}\n'
|
||||
)
|
||||
)
|
||||
self.assertTrue(
|
||||
prompt.endswith(
|
||||
"END FIELD\n\n<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
|
||||
"JOINT SCHEMA DECISIONS:"
|
||||
)
|
||||
)
|
||||
|
||||
def test_layout_spans_hold_questions_and_options_in_head_order(self):
|
||||
input_ids, questions = self._encode()
|
||||
self.assertEqual([q.question_type for q in questions], [1, 0, 2])
|
||||
self.assertEqual(
|
||||
[self._text(input_ids, q.question_span) for q in questions],
|
||||
["Which team handles it?", "Is it urgent?", "severity"],
|
||||
)
|
||||
options = [
|
||||
[json.loads(self._text(input_ids, span)) for span in q.option_spans]
|
||||
for q in questions
|
||||
]
|
||||
self.assertEqual(
|
||||
options[0],
|
||||
[
|
||||
{"description": "Payments and refunds", "option_id": "billing"},
|
||||
{"option_id": "legal"},
|
||||
{"description": "Customer support", "option_id": "support"},
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
options[1],
|
||||
[
|
||||
{
|
||||
"description": "The proposition is true or the answer is yes.",
|
||||
"option_id": "true",
|
||||
},
|
||||
{"option_id": "false"},
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
options[2],
|
||||
[
|
||||
{"description": level, "option_id": str(index)}
|
||||
for index, level in enumerate(["none", "low", "high"])
|
||||
],
|
||||
)
|
||||
|
||||
def test_state_is_truncated_to_fit(self):
|
||||
input_ids, questions = self._encode()
|
||||
shorter_ids, shorter = self._encode(max_length=len(input_ids) - 3)
|
||||
self.assertEqual(len(shorter_ids), len(input_ids) - 3)
|
||||
self.assertEqual(
|
||||
[self._text(shorter_ids, q.question_span) for q in shorter],
|
||||
[self._text(input_ids, q.question_span) for q in questions],
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "before the state"):
|
||||
self._encode(max_length=100)
|
||||
|
||||
def test_image_placeholders_precede_the_state(self):
|
||||
request = {**REQUEST, "images": ["data:image/png;base64,AAAA"]}
|
||||
input_ids, layout = encode_joint_schema(
|
||||
self.tokenizer,
|
||||
SystemOneRequest(**request),
|
||||
max_length=4096,
|
||||
image_token_counts=[100],
|
||||
)
|
||||
self.assertIn(
|
||||
"STATE:\n<|vision_start|><|image_pad|><|vision_end|>\n{",
|
||||
self.tokenizer.decode(input_ids),
|
||||
)
|
||||
unexpanded = parse_decision_layout(layout, len(input_ids))
|
||||
expanded = parse_decision_layout(layout, len(input_ids) + 99)
|
||||
self.assertEqual(
|
||||
[q.option_spans[0][0] + 99 for q in unexpanded],
|
||||
[q.option_spans[0][0] for q in expanded],
|
||||
)
|
||||
|
||||
|
||||
class TestJointSchemaAnswers(unittest.IsolatedAsyncioTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
|
||||
|
||||
def setUp(self):
|
||||
self.addCleanup(restore_context, snapshot_context())
|
||||
|
||||
async def test_answers_map_head_order_back_to_the_request(self):
|
||||
manager = HeadManager(self.tokenizer)
|
||||
response = await _handler(manager).handle_request(
|
||||
SystemOneRequest(**REQUEST), None
|
||||
)
|
||||
self.assertEqual(response.status_code, 200)
|
||||
body = json.loads(response.body)
|
||||
input_ids, layout = encode_joint_schema(
|
||||
self.tokenizer,
|
||||
SystemOneRequest(**REQUEST),
|
||||
max_length=4089,
|
||||
image_token_counts=[],
|
||||
)
|
||||
(sent,) = manager.requests
|
||||
self.assertEqual((sent.input_ids, sent.decision_layout), (input_ids, layout))
|
||||
self.assertIsNone(sent.image_data)
|
||||
self.assertEqual(body["model"], "served-model")
|
||||
self.assertEqual(body["usage"]["input_tokens"], len(input_ids))
|
||||
|
||||
three, two = _softmax([0.0, 1.0, 2.0]), _softmax([0.0, 1.0])
|
||||
team, urgent, severity = (body["answers"][q] for q in REQUEST["questions"])
|
||||
# The head scores billing, legal, support; answers keep the request order.
|
||||
self.assertEqual(team["choice"], "support")
|
||||
self.assertEqual(list(team["probabilities"]), ["support", "billing", "legal"])
|
||||
for name, expected in zip(["billing", "legal", "support"], three):
|
||||
self.assertAlmostEqual(team["probabilities"][name], expected)
|
||||
self.assertAlmostEqual(urgent["noul"], two[0])
|
||||
self.assertAlmostEqual(severity["score"], three[1] + 2 * three[2])
|
||||
self.assertEqual(severity["legend"], {"0": "none", "1": "low", "2": "high"})
|
||||
for answer in (team, urgent, severity):
|
||||
self.assertNotIn("x_label_mass", answer)
|
||||
|
||||
async def test_url_images_are_read_once_and_forwarded_as_counted(self):
|
||||
"""An image URL whose content changes between reads must be scored as the
|
||||
image that was counted against the prompt budget."""
|
||||
manager = HeadManager(self.tokenizer)
|
||||
url = "http://images.example/chart.png"
|
||||
data = _png_bytes(224, 224)
|
||||
image = {"url": url, "detail": "high"}
|
||||
request = {**REQUEST, "state": "word " * 5000, "images": [image]}
|
||||
target = "sglang.srt.entrypoints.systemone.serving.get_image_bytes"
|
||||
with mock.patch(target, return_value=data) as fetch:
|
||||
response = await _handler(manager).handle_request(
|
||||
SystemOneRequest(**request), None
|
||||
)
|
||||
fetch.assert_called_once_with(url)
|
||||
(forwarded,) = manager.requests[0].image_data
|
||||
self.assertTrue(forwarded.url.startswith("data:image/png;base64,"))
|
||||
self.assertEqual(base64.b64decode(forwarded.url.split(",", 1)[1]), data)
|
||||
self.assertEqual(forwarded.detail, "high")
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(json.loads(response.body)["usage"]["input_tokens"], 4089)
|
||||
|
||||
async def test_refusals(self):
|
||||
manager = HeadManager(self.tokenizer)
|
||||
cases = {
|
||||
"takes no chat_template_kwargs": (
|
||||
SystemOneServing,
|
||||
SystemOneRequest(**REQUEST, chat_template_kwargs={"x": 1}),
|
||||
),
|
||||
"LoRA adapter": (
|
||||
SystemOneServing,
|
||||
SystemOneRequest(**{**REQUEST, "model": "clef:a"}),
|
||||
),
|
||||
# Its 5329 image tokens leave no room for the questions.
|
||||
"before the state": (
|
||||
SystemOneServing,
|
||||
SystemOneRequest(**{**REQUEST, "images": [_png(2048, 2048)]}),
|
||||
),
|
||||
"requires a generation model": (
|
||||
OpenAIServingDecisions,
|
||||
DecisionRequest(
|
||||
model="m",
|
||||
input="x",
|
||||
questions=[{"id": "q", "type": "yes_no", "question": "Q?"}],
|
||||
),
|
||||
),
|
||||
}
|
||||
for message, (serving_class, request) in cases.items():
|
||||
with self.subTest(message):
|
||||
response = await _handler(manager, serving_class).handle_request(
|
||||
request, None
|
||||
)
|
||||
self.assertEqual(response.status_code, 400)
|
||||
self.assertIn(message, json.loads(response.body)["message"])
|
||||
self.assertEqual(manager.requests, [])
|
||||
|
||||
|
||||
def _admission_manager(joint_head_config):
|
||||
manager = TokenizerManager.__new__(TokenizerManager)
|
||||
manager.model_config = SimpleNamespace(joint_head_config=joint_head_config)
|
||||
manager.is_generation = joint_head_config is None
|
||||
manager.context_len = 4096
|
||||
manager.num_reserved_tokens = 0
|
||||
manager.max_req_input_len = 4090
|
||||
# A layout must be refused before auto-truncation could cut its prompt.
|
||||
manager.allow_auto_truncate = True
|
||||
manager.validate_total_tokens = False
|
||||
manager._validate_token_ids_logprob = mock.Mock()
|
||||
return manager
|
||||
|
||||
|
||||
class TestJointSchemaAdmission(CustomTestCase):
|
||||
def setUp(self):
|
||||
override = get_context().override_server_args()
|
||||
override.install()
|
||||
self.addCleanup(override.restore)
|
||||
|
||||
def test_tokenizer_refuses_generation_and_prompts_past_one_prefill(self):
|
||||
clef = _admission_manager({"hidden_size": 4096})
|
||||
layout = [LayoutQuestion(0, (10, 12), ((13, 14), (15, 16)))]
|
||||
|
||||
def embedding(length):
|
||||
return EmbeddingReqInput(
|
||||
input_ids=[1] * length,
|
||||
decision_layout=pack_decision_layout(length, layout),
|
||||
sampling_params={},
|
||||
)
|
||||
|
||||
generation = GenerateReqInput(input_ids=[1, 2, 3], sampling_params={})
|
||||
refused = {
|
||||
"generation": (generation, "/v1/systemone"),
|
||||
# 4095 tokens pass the context check, but the scheduler takes 4089.
|
||||
"past one prefill": (embedding(4095), "at most 4089"),
|
||||
}
|
||||
for name, (obj, message) in refused.items():
|
||||
with self.subTest(name), self.assertRaisesRegex(ValueError, message):
|
||||
clef._validate_one_request(obj, obj.input_ids)
|
||||
clef._validate_one_request(embedding(4089), [1] * 4089)
|
||||
# A server without a joint schema head keeps generating.
|
||||
_admission_manager(None)._validate_one_request(generation, [1, 2, 3])
|
||||
|
||||
def test_client_decision_layouts_are_refused_at_encode_and_classify(self):
|
||||
manager = SimpleNamespace(requests=[])
|
||||
|
||||
async def generate_request(obj, request):
|
||||
manager.requests.append(obj)
|
||||
yield {"embedding": [0.0]}
|
||||
|
||||
manager.generate_request = generate_request
|
||||
self.addCleanup(
|
||||
setattr, http_server, "_global_state", http_server.get_global_state()
|
||||
)
|
||||
http_server.set_global_state(SimpleNamespace(tokenizer_manager=manager))
|
||||
layout = pack_decision_layout(16, [LayoutQuestion(0, (0, 1), ((1, 2),))])
|
||||
responses = [
|
||||
asyncio.run(
|
||||
handler(
|
||||
EmbeddingReqInput(
|
||||
input_ids=list(range(16)), decision_layout=layout
|
||||
),
|
||||
None,
|
||||
)
|
||||
)
|
||||
for handler in (http_server.encode_request, http_server.classify_request)
|
||||
]
|
||||
self.assertEqual(manager.requests, [])
|
||||
self.assertEqual([response.status_code for response in responses], [400, 400])
|
||||
|
||||
def test_inline_images_are_sized_from_a_header_prefix(self):
|
||||
"""Sizing an inline image must not decode its whole payload, and an image
|
||||
whose header runs past the decoded prefix must still be sized."""
|
||||
noise = BytesIO()
|
||||
pixels = random.Random(0).randbytes(256 * 192 * 3)
|
||||
Image.frombytes("RGB", (256, 192), pixels).save(noise, format="PNG")
|
||||
late = BytesIO()
|
||||
# A 60 KB EXIF segment puts the JPEG frame header far into the payload.
|
||||
exif = b"Exif\x00\x00" + bytes(60000)
|
||||
Image.new("RGB", (64, 48)).save(late, format="JPEG", exif=exif)
|
||||
cases = {
|
||||
"png": ("image/png", noise.getvalue(), (256, 192)),
|
||||
"late jpeg header": ("image/jpeg", late.getvalue(), (64, 48)),
|
||||
}
|
||||
for name, (mime, data, size) in cases.items():
|
||||
with self.subTest(name):
|
||||
payload = base64.b64encode(data).decode()
|
||||
image = ImageData(url=f"data:{mime};base64,{payload}")
|
||||
with mock.patch("pybase64.b64decode", wraps=pybase64.b64decode) as read:
|
||||
forwarded, header = _read_image(0, image)
|
||||
self.assertIs(forwarded, image)
|
||||
self.assertEqual(header.size, size)
|
||||
decoded = [len(call.args[0]) for call in read.call_args_list]
|
||||
if name == "png":
|
||||
self.assertLess(max(decoded), len(payload))
|
||||
else:
|
||||
self.assertEqual(decoded[-1], len(payload))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,240 @@
|
||||
"""Unit tests for the joint schema head of Clef checkpoints: decision layouts, and
|
||||
the batched forward against a per-request loop over the stock torch modules."""
|
||||
|
||||
import math
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.srt.layers.joint_schema_head import (
|
||||
JointSchemaHead,
|
||||
JointSchemaPooler,
|
||||
LayoutQuestion,
|
||||
pack_decision_layout,
|
||||
parse_decision_layout,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cpu_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small")
|
||||
|
||||
|
||||
def _questions(offset=0):
|
||||
return [
|
||||
LayoutQuestion(
|
||||
1,
|
||||
(10 + offset, 14 + offset),
|
||||
(
|
||||
(16 + offset, 17 + offset),
|
||||
(17 + offset, 19 + offset),
|
||||
(19 + offset, 21 + offset),
|
||||
),
|
||||
),
|
||||
LayoutQuestion(
|
||||
0,
|
||||
(22 + offset, 25 + offset),
|
||||
((26 + offset, 28 + offset), (29 + offset, 31 + offset)),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class TestDecisionLayout(CustomTestCase):
|
||||
def test_round_trip(self):
|
||||
layout = pack_decision_layout(40, _questions())
|
||||
self.assertEqual(parse_decision_layout(layout, 40), _questions())
|
||||
|
||||
def test_expanded_images_shift_every_span(self):
|
||||
layout = pack_decision_layout(40, _questions())
|
||||
self.assertEqual(parse_decision_layout(layout, 140), _questions(offset=100))
|
||||
|
||||
def test_rejects_layouts_that_do_not_fit_the_prompt(self):
|
||||
layout = pack_decision_layout(40, _questions())
|
||||
invalid = {
|
||||
"shorter prompt": (layout, 39),
|
||||
"not ints": ([float(value) for value in layout], 40),
|
||||
"truncated": (layout[:-1], 40),
|
||||
"trailing values": (layout + [0], 40),
|
||||
"no questions": ([40, 0], 40),
|
||||
"empty span": (
|
||||
pack_decision_layout(40, [LayoutQuestion(0, (5, 5), ((6, 7),))]),
|
||||
40,
|
||||
),
|
||||
"past the prompt": (
|
||||
pack_decision_layout(40, [LayoutQuestion(0, (5, 6), ((6, 41),))]),
|
||||
40,
|
||||
),
|
||||
"unknown type": (
|
||||
pack_decision_layout(40, [LayoutQuestion(3, (5, 6), ((6, 7),))]),
|
||||
40,
|
||||
),
|
||||
}
|
||||
for name, (values, num_tokens) in invalid.items():
|
||||
with self.subTest(name), self.assertRaises(ValueError):
|
||||
parse_decision_layout(values, num_tokens)
|
||||
|
||||
|
||||
class _RecordingHead:
|
||||
"""Stands in for the head, scoring each option with its request's length."""
|
||||
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def forward_batch(self, hidden, input_ids, lengths, layouts, embedding_weight):
|
||||
self.calls.append(lengths)
|
||||
return [
|
||||
torch.full((sum(len(q.option_spans) for q in layout),), float(length))
|
||||
for length, layout in zip(lengths, layouts)
|
||||
]
|
||||
|
||||
|
||||
class TestJointSchemaPooler(CustomTestCase):
|
||||
def test_a_layout_cut_from_its_prompt_is_skipped_and_neighbors_are_scored(self):
|
||||
"""A prompt shorter than its layout, as after a truncation, must not stop the
|
||||
scheduler or the other requests of its batch."""
|
||||
head = _RecordingHead()
|
||||
pooler = JointSchemaPooler(head, lambda: torch.zeros(1))
|
||||
layout = pack_decision_layout(40, _questions())
|
||||
lengths = [40, 35, 40]
|
||||
forward_batch = SimpleNamespace(
|
||||
decision_layouts=[layout, layout, layout],
|
||||
extend_seq_lens_cpu=lengths,
|
||||
extend_prefix_lens_cpu=[0, 0, 0],
|
||||
batch_size=3,
|
||||
input_ids=torch.zeros(sum(lengths), dtype=torch.long),
|
||||
)
|
||||
output = pooler(torch.zeros(sum(lengths), 8), forward_batch)
|
||||
self.assertEqual([len(scores) for scores in output.embeddings], [5, 0, 5])
|
||||
self.assertEqual(head.calls, [[40], [40]])
|
||||
|
||||
|
||||
def _reference_logits(head, hidden, input_ids, questions, lm_head):
|
||||
"""One request through the modules' own forward methods, as the checkpoint's loop does."""
|
||||
normalized = head.hidden_norm(hidden)
|
||||
memory = head.memory_projection(normalized).unsqueeze(0)
|
||||
global_vector = normalized[-1]
|
||||
|
||||
def mean(values, span):
|
||||
return values[span[0] : span[1]].mean(dim=0)
|
||||
|
||||
question_vectors = torch.stack(
|
||||
[mean(normalized, q.question_span) for q in questions]
|
||||
)
|
||||
queries, lexicals = [], []
|
||||
for index, question in enumerate(questions):
|
||||
context = torch.stack(
|
||||
[mean(normalized, span) for span in question.option_spans]
|
||||
)
|
||||
lexical = torch.stack(
|
||||
[
|
||||
lm_head[input_ids[start:end]].mean(dim=0)
|
||||
for start, end in question.option_spans
|
||||
]
|
||||
)
|
||||
queries.append(
|
||||
head.option_context_projection(context)
|
||||
+ head.option_lexical_projection(lexical)
|
||||
+ head.option_question_projection(question_vectors[index]).unsqueeze(0)
|
||||
)
|
||||
lexicals.append(lexical)
|
||||
routed = torch.cat(queries).unsqueeze(0)
|
||||
for layer in head.evidence_layers:
|
||||
attended, _ = layer.attention(
|
||||
layer.query_norm(routed),
|
||||
layer.memory_norm(memory),
|
||||
layer.memory_norm(memory),
|
||||
need_weights=False,
|
||||
)
|
||||
routed = routed + attended
|
||||
routed = routed + layer.feedforward(layer.feedforward_norm(routed))
|
||||
options = list(torch.split(routed[0], [len(q.option_spans) for q in questions]))
|
||||
base_fields = head.question_projection(question_vectors)
|
||||
summaries = []
|
||||
for field, routed_options in zip(base_fields, options):
|
||||
weights = torch.softmax(
|
||||
routed_options @ field / math.sqrt(field.shape[-1]), dim=0
|
||||
)
|
||||
summaries.append((weights.unsqueeze(-1) * routed_options).sum(dim=0))
|
||||
fields = (
|
||||
base_fields
|
||||
+ head.option_summary_norm(torch.stack(summaries))
|
||||
+ head.global_projection(global_vector).unsqueeze(0)
|
||||
+ head.type_embedding(
|
||||
torch.tensor([q.question_type for q in questions], device=hidden.device)
|
||||
)
|
||||
).unsqueeze(0)
|
||||
for layer in head.layers:
|
||||
fields = layer(fields, memory)
|
||||
fields = head.field_norm(fields[0])
|
||||
prior_scale = head.prior_logit_scale.clamp(max=math.log(100.0)).exp()
|
||||
joint_scale = head.joint_logit_scale.clamp(max=math.log(100.0)).exp()
|
||||
logits = []
|
||||
for index, (field, lexical, routed_options) in enumerate(
|
||||
zip(fields, lexicals, options)
|
||||
):
|
||||
anchor = F.normalize(question_vectors[index] + global_vector, dim=-1)
|
||||
prior = prior_scale * (F.normalize(lexical, dim=-1) @ anchor)
|
||||
normed = head.option_norm(routed_options)
|
||||
repeated = field.unsqueeze(0).expand_as(normed)
|
||||
features = torch.cat(
|
||||
[repeated, normed, repeated * normed, torch.abs(repeated - normed)], dim=-1
|
||||
)
|
||||
residual = head.residual_scorer(features).squeeze(-1)
|
||||
cosine = F.cosine_similarity(repeated, normed, dim=-1)
|
||||
logits.append(
|
||||
prior
|
||||
+ torch.sigmoid(head.residual_gate) * (joint_scale * cosine + residual)
|
||||
)
|
||||
return torch.cat(logits).float()
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "the batched head uses FlashAttention")
|
||||
class TestJointSchemaHeadBatch(CustomTestCase):
|
||||
def test_batched_forward_matches_each_request_alone(self):
|
||||
torch.manual_seed(0)
|
||||
hidden_size, vocab = 256, 512
|
||||
head = JointSchemaHead(
|
||||
hidden_size, width=128, routing_layers=2, layers=2, heads=4, feedforward=256
|
||||
)
|
||||
for parameter in (
|
||||
head.prior_logit_scale,
|
||||
head.joint_logit_scale,
|
||||
head.residual_gate,
|
||||
):
|
||||
parameter.data.fill_(0.5)
|
||||
head = head.to("cuda", torch.bfloat16).eval()
|
||||
lm_head = torch.randn(vocab, hidden_size, device="cuda", dtype=torch.bfloat16)
|
||||
lengths = [40, 140, 64]
|
||||
layouts = [
|
||||
_questions(),
|
||||
_questions(offset=100),
|
||||
[
|
||||
LayoutQuestion(
|
||||
2, (3, 9), tuple((10 + 2 * i, 12 + 2 * i) for i in range(20))
|
||||
)
|
||||
],
|
||||
]
|
||||
hidden = torch.randn(
|
||||
sum(lengths), hidden_size, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
input_ids = torch.randint(0, vocab, (sum(lengths),), device="cuda")
|
||||
with torch.no_grad():
|
||||
batched = head.forward_batch(hidden, input_ids, lengths, layouts, lm_head)
|
||||
start = 0
|
||||
for length, questions, logits in zip(lengths, layouts, batched):
|
||||
end = start + length
|
||||
expected = _reference_logits(
|
||||
head, hidden[start:end], input_ids[start:end], questions, lm_head
|
||||
)
|
||||
self.assertEqual(logits.shape, expected.shape)
|
||||
torch.testing.assert_close(logits, expected, atol=5e-2, rtol=5e-2)
|
||||
torch.testing.assert_close(
|
||||
logits.softmax(-1), expected.softmax(-1), atol=1e-2, rtol=0
|
||||
)
|
||||
start = end
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -26,6 +26,7 @@ class TestHiddenStateServerMode(CustomTestCase):
|
||||
manager.allow_auto_truncate = False
|
||||
manager.validate_total_tokens = False
|
||||
manager.is_generation = True
|
||||
manager.model_config = SimpleNamespace(joint_head_config=None)
|
||||
manager.server_args = SimpleNamespace(enable_custom_logit_processor=False)
|
||||
manager._validate_token_ids_logprob = Mock()
|
||||
return manager
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from sglang.srt.managers.io_struct import GenerateReqInput
|
||||
@@ -28,6 +29,7 @@ class TestSamplingMaskValidation(CustomTestCase):
|
||||
self.manager.allow_auto_truncate = False
|
||||
self.manager.validate_total_tokens = False
|
||||
self.manager.is_generation = True
|
||||
self.manager.model_config = SimpleNamespace(joint_head_config=None)
|
||||
|
||||
def _validate(
|
||||
self,
|
||||
|
||||
@@ -987,6 +987,7 @@ class TestSchedulerMmTransportBoundary(CustomTestCase):
|
||||
return_pooled_hidden_states=False,
|
||||
multi_item_delimiter_indices=None,
|
||||
token_indices_to_pool=None,
|
||||
decision_layout=None,
|
||||
mm_inputs=object(),
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user