[model] support Clef and Clef-Flash decision models on /v1/systemone (#42721)
Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com>
This commit is contained in:
co-authored by
Jimmy Shong
parent
51f7ea84c7
commit
bd2d73daa5
@@ -977,6 +977,37 @@ def handle_model_capability_adjustments(server_args: Any):
|
||||
"prefill; using breakable CUDA graph for CUDA prefill."
|
||||
)
|
||||
|
||||
# A Clef checkpoint's joint schema head reads the final hidden state of
|
||||
# every prompt token in one pass, so each request is one complete prefill
|
||||
# with no decode: embedding mode, without prefix reuse or chunking. This
|
||||
# also puts the FA backend on its raw K/V path, which skips the KV pool.
|
||||
if model_config.joint_head_config is not None:
|
||||
if cfg.tp_size != 1 or cfg.pp_size != 1:
|
||||
raise ValueError(
|
||||
"Clef checkpoints are served with --tp-size 1 and --pp-size 1, "
|
||||
"since the joint schema head reads full LM head rows"
|
||||
)
|
||||
for key, value in (
|
||||
("is_embedding", True),
|
||||
("disable_radix_cache", True),
|
||||
("chunked_prefill_size", -1),
|
||||
):
|
||||
declare_resolution(
|
||||
server_args, "_handle_model_capability_adjustments", **{key: value}
|
||||
)
|
||||
for phase in (Phase.DECODE, Phase.PREFILL):
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_model_capability_adjustments",
|
||||
cuda_graph_config=with_phase(
|
||||
cfg.cuda_graph_config, phase, backend=Backend.DISABLED
|
||||
),
|
||||
)
|
||||
logger.info(
|
||||
"Clef joint schema head detected: serving /v1/systemone decisions in "
|
||||
"embedding mode without radix cache, chunked prefill, or CUDA graphs."
|
||||
)
|
||||
|
||||
if (
|
||||
model_config.is_multimodal
|
||||
and not model_config.is_multimodal_chunked_prefill_supported
|
||||
|
||||
@@ -291,6 +291,23 @@ def load_decision_config(model_path: str, revision: Optional[str]) -> Optional[d
|
||||
return config
|
||||
|
||||
|
||||
@lru_cache
|
||||
def load_joint_head_config(model_path: str, revision: Optional[str]) -> Optional[dict]:
|
||||
"""joint_head_config.json of a Clef checkpoint, whose joint schema head
|
||||
answers /v1/systemone decisions from the backbone's hidden states."""
|
||||
path = cached_file(
|
||||
model_path,
|
||||
"joint_head_config.json",
|
||||
revision=revision,
|
||||
_raise_exceptions_for_missing_entries=False,
|
||||
_raise_exceptions_for_connection_errors=False,
|
||||
)
|
||||
if path is None:
|
||||
return None
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def is_qwen3_5(config) -> bool:
|
||||
return _hf_arch(config) in (
|
||||
"Qwen3_5ForConditionalGeneration",
|
||||
@@ -544,6 +561,9 @@ class ModelConfig:
|
||||
):
|
||||
# Full attention sees the whole prompt, while linear attention stays causal.
|
||||
self.hf_text_config.is_causal = False
|
||||
self.joint_head_config = None
|
||||
if self.hf_config.architectures == ["Qwen3_5ForConditionalGeneration"]:
|
||||
self.joint_head_config = load_joint_head_config(self.model_path, revision)
|
||||
self.requires_mm_token_modalities = requires_mm_token_modalities(
|
||||
self.hf_config.architectures, self.hf_text_config
|
||||
)
|
||||
|
||||
@@ -66,6 +66,7 @@ from fastapi.routing import APIRoute
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolving_view
|
||||
from sglang.srt.configs.embedding_model_spec import resolved_embedding_plan
|
||||
from sglang.srt.configs.model_config import load_joint_head_config
|
||||
from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX
|
||||
from sglang.srt.disaggregation.utils import FAKE_BOOTSTRAP_HOST, DisaggregationMode
|
||||
from sglang.srt.entrypoints.anthropic.protocol import (
|
||||
@@ -972,6 +973,7 @@ async def serve_generate_request(obj: GenerateReqInput, request: Request):
|
||||
async def encode_request(obj: EmbeddingReqInput, request: Request):
|
||||
"""Handle an embedding request."""
|
||||
try:
|
||||
_refuse_decision_layout(obj)
|
||||
ret = await _global_state.tokenizer_manager.generate_request(
|
||||
obj, request
|
||||
).__anext__()
|
||||
@@ -984,6 +986,7 @@ async def encode_request(obj: EmbeddingReqInput, request: Request):
|
||||
async def classify_request(obj: EmbeddingReqInput, request: Request):
|
||||
"""Handle a reward model request. Now the arguments and return values are the same as embedding models."""
|
||||
try:
|
||||
_refuse_decision_layout(obj)
|
||||
ret = await _global_state.tokenizer_manager.generate_request(
|
||||
obj, request
|
||||
).__anext__()
|
||||
@@ -992,6 +995,12 @@ async def classify_request(obj: EmbeddingReqInput, request: Request):
|
||||
return _create_error_response(e)
|
||||
|
||||
|
||||
def _refuse_decision_layout(obj: EmbeddingReqInput) -> None:
|
||||
# Only /v1/systemone builds a decision layout, from the prompt it compiles.
|
||||
if obj.decision_layout is not None:
|
||||
raise ValueError("decision_layout is set only by /v1/systemone")
|
||||
|
||||
|
||||
@app.api_route("/flush_cache", methods=["GET", "POST"])
|
||||
@auth_level(AuthLevel.ADMIN_OPTIONAL)
|
||||
async def flush_cache(timeout: float = Query(0.0, ge=0.0)):
|
||||
@@ -2379,6 +2388,22 @@ def _execute_server_warmup(server_args: ServerArgs):
|
||||
).tolist()
|
||||
json_data["sampling_params"]["max_new_tokens"] = 0
|
||||
|
||||
if (
|
||||
not model_info["is_generation"]
|
||||
and model_info.get("architectures") == ["Qwen3_5ForConditionalGeneration"]
|
||||
and load_joint_head_config(get_model().model_path, get_model().revision)
|
||||
is not None
|
||||
):
|
||||
# A Clef checkpoint answers /v1/systemone decisions only.
|
||||
request_name = "/v1/systemone"
|
||||
json_data = {
|
||||
"model": "warmup",
|
||||
"state": "The capital city of France is Paris.",
|
||||
"questions": {
|
||||
"warmup": {"type": "noul", "instructions": "Is the state true?"}
|
||||
},
|
||||
}
|
||||
|
||||
# Send a warmup request
|
||||
warmup_timeout = envs.SGLANG_WARMUP_TIMEOUT.get()
|
||||
try:
|
||||
|
||||
@@ -83,6 +83,7 @@ class OpenAIServingDecisions(OpenAIServingBase):
|
||||
self.chat_encoding_spec = chat_serving.chat_encoding_spec
|
||||
self.prompt_text_is_lossy = chat_serving._prompt_text_round_trip_is_lossy
|
||||
self.decision_config = self.tokenizer_manager.model_config.decision_config
|
||||
self.joint_head_config = self.tokenizer_manager.model_config.joint_head_config
|
||||
tokenizer = self.tokenizer_manager.tokenizer
|
||||
# Other tokenizers skip the shortcut in _encode_labels and check the full prompt.
|
||||
self.added_tokens = (
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
"""The prompt that a Clef checkpoint's joint schema head reads.
|
||||
|
||||
Clef answers /v1/systemone questions from the backbone's hidden states instead
|
||||
of from generated text. One prompt holds the state and every question with its
|
||||
allowed options, and the head scores the span of each option. This follows
|
||||
encode_record in the checkpoint's joint_schema_model.py, piece by piece, since
|
||||
each piece is tokenized on its own.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, List, Sequence, Tuple
|
||||
|
||||
from sglang.srt.entrypoints.systemone.protocol import (
|
||||
SystemOneChoiceQuestion,
|
||||
SystemOneNoulQuestion,
|
||||
SystemOneQuestion,
|
||||
SystemOneRequest,
|
||||
)
|
||||
from sglang.srt.layers.joint_schema_head import (
|
||||
QUESTION_TYPES,
|
||||
LayoutQuestion,
|
||||
pack_decision_layout,
|
||||
)
|
||||
|
||||
_SYSTEM_PROMPT = (
|
||||
"Read the complete state and schema. Decide every field jointly. Each answer "
|
||||
"must be exactly one of that field's allowed options."
|
||||
)
|
||||
_PREFIX = f"<|im_start|>system\n{_SYSTEM_PROMPT}<|im_end|>\n<|im_start|>user\nSTATE:\n"
|
||||
_IMAGE = "<|vision_start|><|image_pad|><|vision_end|>"
|
||||
_SUFFIX = (
|
||||
"\n<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
|
||||
"JOINT SCHEMA DECISIONS:"
|
||||
)
|
||||
_NOUL_CRITERIA = {
|
||||
"true": "The proposition is true or the answer is yes.",
|
||||
"false": "The proposition is false or the answer is no.",
|
||||
}
|
||||
# The default max_length of encode_record and systemone in joint_schema_model.py.
|
||||
REFERENCE_MAX_LENGTH = 16384
|
||||
|
||||
|
||||
def _render(value: Any) -> str:
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
||||
|
||||
|
||||
def joint_schema_options(question: SystemOneQuestion) -> List[Tuple[str, Any]]:
|
||||
"""Each option's name and description, in the order the head scores them."""
|
||||
if isinstance(question, SystemOneNoulQuestion):
|
||||
criteria = dict(_NOUL_CRITERIA)
|
||||
if question.criteria is not None:
|
||||
# A description sent as null drops the default, as in encode_record.
|
||||
criteria.update(question.criteria.model_dump(exclude_unset=True))
|
||||
return [(name, criteria[name]) for name in ("true", "false")]
|
||||
if isinstance(question, SystemOneChoiceQuestion):
|
||||
return sorted(question.criteria.items(), key=lambda item: item[0])
|
||||
return [(str(level), detail) for level, detail in enumerate(question.criteria)]
|
||||
|
||||
|
||||
def encode_joint_schema(
|
||||
tokenizer: Any,
|
||||
request: SystemOneRequest,
|
||||
max_length: int,
|
||||
image_token_counts: Sequence[int],
|
||||
) -> Tuple[List[int], List[int]]:
|
||||
"""The prompt ids and their decision layout.
|
||||
|
||||
Each image's placeholder expands to its count in ``image_token_counts``, and
|
||||
the state is truncated so that the expanded prompt has at most
|
||||
``max_length`` tokens, as encode_record budgets the expanded media.
|
||||
"""
|
||||
|
||||
def tokens(text: str) -> List[int]:
|
||||
return tokenizer.encode(text, add_special_tokens=False)
|
||||
|
||||
schema = tokens("\n\nSCHEMA FIELDS:\n")
|
||||
questions = []
|
||||
for index, (question_id, question) in enumerate(request.questions.items()):
|
||||
schema += tokens(
|
||||
f"\nFIELD {index + 1}\nID: {question_id}\nTYPE: {question.type}\n"
|
||||
"INSTRUCTION: "
|
||||
)
|
||||
instructions = question.instructions
|
||||
if instructions is None or instructions == "":
|
||||
instructions = question_id
|
||||
question_start = len(schema)
|
||||
schema += tokens(_render(instructions))
|
||||
question_span = (question_start, len(schema))
|
||||
schema += tokens("\nALLOWED OPTIONS:\n")
|
||||
option_spans = []
|
||||
for option_index, (name, description) in enumerate(
|
||||
joint_schema_options(question)
|
||||
):
|
||||
schema += tokens(f"OPTION {option_index + 1}: ")
|
||||
option_start = len(schema)
|
||||
semantics = {"option_id": name}
|
||||
if description is not None:
|
||||
semantics["description"] = description
|
||||
schema += tokens(_render(semantics))
|
||||
option_spans.append((option_start, len(schema)))
|
||||
schema += tokens("\n")
|
||||
schema += tokens("END FIELD\n")
|
||||
questions.append(
|
||||
(QUESTION_TYPES.index(question.type), question_span, option_spans)
|
||||
)
|
||||
|
||||
prefix = tokens(_PREFIX)
|
||||
if request.images:
|
||||
prefix += tokens(_IMAGE * len(request.images) + "\n")
|
||||
suffix = tokens(_SUFFIX)
|
||||
# Each image's single <|image_pad|> becomes that image's tokens.
|
||||
expansion = sum(count - 1 for count in image_token_counts)
|
||||
fixed_length = len(prefix) + expansion + len(schema) + len(suffix)
|
||||
if fixed_length > max_length:
|
||||
raise ValueError(
|
||||
f"the questions need {fixed_length} prompt tokens before the state, "
|
||||
f"but the prompt holds at most {max_length}"
|
||||
)
|
||||
state = tokens(_render(request.state))[: max_length - fixed_length]
|
||||
offset = len(prefix) + len(state)
|
||||
input_ids = prefix + state + schema + suffix
|
||||
layout = pack_decision_layout(
|
||||
len(input_ids),
|
||||
[
|
||||
LayoutQuestion(
|
||||
question_type,
|
||||
(question_span[0] + offset, question_span[1] + offset),
|
||||
tuple((start + offset, end + offset) for start, end in option_spans),
|
||||
)
|
||||
for question_type, question_span, option_spans in questions
|
||||
],
|
||||
)
|
||||
return input_ids, layout
|
||||
@@ -2,14 +2,20 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import json
|
||||
import math
|
||||
import string
|
||||
from io import BytesIO
|
||||
from typing import Any, Dict, Iterator, List, Optional, Tuple
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
import orjson
|
||||
import pybase64
|
||||
from fastapi import Request
|
||||
from fastapi.responses import ORJSONResponse
|
||||
from PIL import Image
|
||||
|
||||
from sglang.srt.entrypoints.openai.serving_decisions import (
|
||||
EncodedQuestion,
|
||||
@@ -22,6 +28,11 @@ from sglang.srt.entrypoints.openai.serving_decisions import (
|
||||
render_question,
|
||||
render_text,
|
||||
)
|
||||
from sglang.srt.entrypoints.systemone.joint_schema import (
|
||||
REFERENCE_MAX_LENGTH,
|
||||
encode_joint_schema,
|
||||
joint_schema_options,
|
||||
)
|
||||
from sglang.srt.entrypoints.systemone.protocol import (
|
||||
SystemOneChoiceAnswer,
|
||||
SystemOneChoiceQuestion,
|
||||
@@ -33,6 +44,10 @@ from sglang.srt.entrypoints.systemone.protocol import (
|
||||
SystemOneScoreAnswer,
|
||||
SystemOneUsage,
|
||||
)
|
||||
from sglang.srt.layers.joint_schema_head import max_joint_prompt_tokens
|
||||
from sglang.srt.managers.io_struct import EmbeddingReqInput
|
||||
from sglang.srt.runtime_context import get_schedule
|
||||
from sglang.srt.utils import CLIENT_MEDIA_EXCEPTIONS, ImageData, get_image_bytes
|
||||
|
||||
# Beyond A to Z, every option gets a two-letter label, in this fixed order.
|
||||
_PAIR_LABELS = [a + b for a in string.ascii_uppercase for b in string.ascii_uppercase]
|
||||
@@ -44,9 +59,16 @@ _DECIDER_SYSTEM = (
|
||||
"Reply with only the selected option code."
|
||||
)
|
||||
|
||||
# Image sources whose content can change between reads, as load_image reads them.
|
||||
_MUTABLE_SOURCES = ("http://", "https://", "file://", "/")
|
||||
# Base64 characters decoded to size an inline image, 48 KiB of image data.
|
||||
# Arbitrary; an image whose header runs longer is decoded in full.
|
||||
_INLINE_HEADER_CHARS = 1 << 16
|
||||
|
||||
|
||||
class SystemOneServing(OpenAIServingDecisions):
|
||||
"""Answers System One questions with the rendering, label checks, and scoring of /v1/decisions."""
|
||||
"""Answers System One questions with the rendering, label checks, and scoring of
|
||||
/v1/decisions, or, for a Clef checkpoint, with its joint schema head."""
|
||||
|
||||
route = "/v1/systemone"
|
||||
|
||||
@@ -54,12 +76,36 @@ class SystemOneServing(OpenAIServingDecisions):
|
||||
return "systemone-"
|
||||
|
||||
def _validate_request(self, request: SystemOneRequest) -> Optional[str]:
|
||||
if self.joint_head_config is not None:
|
||||
return self._validate_joint_schema_request(request)
|
||||
return (
|
||||
self._validate_server(request.model)
|
||||
or self._validate_images(request.images)
|
||||
or self._validate_reasoning(request.chat_template_kwargs)
|
||||
)
|
||||
|
||||
def _validate_joint_schema_request(
|
||||
self, request: SystemOneRequest
|
||||
) -> Optional[str]:
|
||||
"""A Clef checkpoint renders its own prompt, without the chat template."""
|
||||
route = self.route
|
||||
if self.tokenizer_manager.tokenizer is None:
|
||||
return f"{route} requires the server tokenizer"
|
||||
_, adapter = self._parse_model_parameter(request.model)
|
||||
if adapter is not None:
|
||||
return (
|
||||
f"model names the LoRA adapter {adapter!r}, which {route} "
|
||||
"does not support"
|
||||
)
|
||||
if request.images and self.tokenizer_manager.mm_processor is None:
|
||||
return f"{route} images require a model served with multimodal input"
|
||||
if request.chat_template_kwargs:
|
||||
return (
|
||||
f"{route} renders the prompt this checkpoint's joint schema head "
|
||||
"was trained with, which takes no chat_template_kwargs"
|
||||
)
|
||||
return None
|
||||
|
||||
def _convert_to_internal_request(
|
||||
self,
|
||||
request: SystemOneRequest,
|
||||
@@ -69,6 +115,8 @@ class SystemOneServing(OpenAIServingDecisions):
|
||||
Tuple[SystemOneRequest, List[QuestionView]],
|
||||
]:
|
||||
views = [_view(question) for question in request.questions.values()]
|
||||
if self.joint_head_config is not None:
|
||||
return self._joint_schema_request(request, views), (request, views)
|
||||
encode = (
|
||||
self._encoded_systemone_questions
|
||||
if self.decision_config is None
|
||||
@@ -77,6 +125,118 @@ class SystemOneServing(OpenAIServingDecisions):
|
||||
# Lazy, so the async handler can yield to other requests between questions.
|
||||
return encode(request, views), (request, views)
|
||||
|
||||
def _joint_schema_request(
|
||||
self, request: SystemOneRequest, views: List[QuestionView]
|
||||
) -> EmbeddingReqInput:
|
||||
"""All questions in one prompt, which the joint schema head scores in one prefill.
|
||||
|
||||
The handler encodes the prompt once it has read the image sizes."""
|
||||
for question_id, view in zip(request.questions, views):
|
||||
if view.kind == "score":
|
||||
_check_legend(question_id, view)
|
||||
return EmbeddingReqInput(
|
||||
image_data=[
|
||||
ImageData(url=image.url, detail=image.detail or "auto")
|
||||
for image in request.images
|
||||
]
|
||||
or None,
|
||||
)
|
||||
|
||||
async def _encode_joint_schema_prompt(
|
||||
self, adapted_request: EmbeddingReqInput, request: SystemOneRequest
|
||||
) -> None:
|
||||
tokenizer_manager = self.tokenizer_manager
|
||||
max_length = min(
|
||||
REFERENCE_MAX_LENGTH,
|
||||
max_joint_prompt_tokens(
|
||||
context_len=tokenizer_manager.context_len,
|
||||
num_reserved_tokens=tokenizer_manager.num_reserved_tokens,
|
||||
max_req_input_len=tokenizer_manager.max_req_input_len,
|
||||
max_prefill_tokens=get_schedule().max_prefill_tokens,
|
||||
),
|
||||
)
|
||||
image_token_counts = []
|
||||
if adapted_request.image_data:
|
||||
adapted_request.image_data, image_token_counts = await self._read_images(
|
||||
adapted_request.image_data
|
||||
)
|
||||
input_ids, layout = encode_joint_schema(
|
||||
tokenizer_manager.tokenizer,
|
||||
request,
|
||||
max_length=max_length,
|
||||
image_token_counts=image_token_counts,
|
||||
)
|
||||
adapted_request.input_ids = input_ids
|
||||
adapted_request.decision_layout = layout
|
||||
|
||||
async def _read_images(
|
||||
self, images: List[ImageData]
|
||||
) -> Tuple[List[ImageData], List[int]]:
|
||||
"""Each image read once, and the tokens it expands to at the size the
|
||||
processor resizes it to."""
|
||||
read = await asyncio.gather(
|
||||
*(
|
||||
asyncio.to_thread(_read_image, index, image)
|
||||
for index, image in enumerate(images)
|
||||
)
|
||||
)
|
||||
counts = self.tokenizer_manager.mm_processor.resolve_image_token_counts(
|
||||
[header for _, header in read]
|
||||
)
|
||||
return [image for image, _ in read], counts
|
||||
|
||||
async def _answer_joint_schema(
|
||||
self,
|
||||
adapted_request: EmbeddingReqInput,
|
||||
request: SystemOneRequest,
|
||||
views: List[QuestionView],
|
||||
raw_request: Request,
|
||||
) -> ORJSONResponse:
|
||||
await self._encode_joint_schema_prompt(adapted_request, request)
|
||||
ret = await self.tokenizer_manager.generate_request(
|
||||
adapted_request, raw_request
|
||||
).__anext__()
|
||||
logits = ret["embedding"]
|
||||
names = [
|
||||
[name for name, _ in joint_schema_options(question)]
|
||||
for question in request.questions.values()
|
||||
]
|
||||
if len(logits) != sum(map(len, names)):
|
||||
raise RuntimeError("the joint schema head did not score this request")
|
||||
answers = {}
|
||||
offset = 0
|
||||
for question_id, view, question_names in zip(request.questions, views, names):
|
||||
scores = logits[offset : offset + len(question_names)]
|
||||
offset += len(question_names)
|
||||
top = max(scores)
|
||||
weights = [math.exp(score - top) for score in scores]
|
||||
total = math.fsum(weights)
|
||||
by_name = {
|
||||
name: weight / total for name, weight in zip(question_names, weights)
|
||||
}
|
||||
# The head scores true then false, which the view names yes and no.
|
||||
answers[question_id] = _answer(
|
||||
view=view,
|
||||
probabilities=(
|
||||
[by_name["true"], by_name["false"]]
|
||||
if view.kind == "yes_no"
|
||||
else [by_name[name] for name in view.names]
|
||||
),
|
||||
mass=1.0,
|
||||
question_id=question_id,
|
||||
)
|
||||
response = SystemOneResponse(
|
||||
model=self.tokenizer_manager.served_model_name,
|
||||
answers=answers,
|
||||
usage=SystemOneUsage(input_tokens=ret["meta_info"]["prompt_tokens"]),
|
||||
)
|
||||
# The head scores only the allowed options, so there is no label mass.
|
||||
return ORJSONResponse(
|
||||
content=response.model_dump(
|
||||
exclude={"answers": {"__all__": {"x_label_mass"}}}
|
||||
)
|
||||
)
|
||||
|
||||
def _encoded_systemone_questions(
|
||||
self, request: SystemOneRequest, views: List[QuestionView]
|
||||
) -> Iterator[EncodedQuestion]:
|
||||
@@ -179,6 +339,10 @@ class SystemOneServing(OpenAIServingDecisions):
|
||||
raw_request: Request,
|
||||
) -> ORJSONResponse:
|
||||
request, views = processed
|
||||
if self.joint_head_config is not None:
|
||||
return await self._answer_joint_schema(
|
||||
adapted_request, request, views, raw_request
|
||||
)
|
||||
_, _, result = await self._score(
|
||||
adapted_request=adapted_request,
|
||||
raw_request=raw_request,
|
||||
@@ -344,3 +508,38 @@ def _score_confidence(q: List[float]) -> float:
|
||||
spread = math.fsum(p * abs(i - top) for i, p in enumerate(q))
|
||||
uniform_spread = math.fsum(abs(i - (n - 1) / 2) for i in range(n)) / n
|
||||
return max(0.0, 1 - spread / uniform_spread)
|
||||
|
||||
|
||||
def _read_image(index: int, image: ImageData) -> Tuple[ImageData, Image.Image]:
|
||||
"""The image to forward, and its header opened only as far as its size, read
|
||||
once from the sources load_image reads."""
|
||||
url = image.url
|
||||
if not url.startswith(_MUTABLE_SOURCES):
|
||||
# An inline image is forwarded as sent, and the processor decodes all of it.
|
||||
header = _inline_header(url)
|
||||
if header is not None:
|
||||
return image, header
|
||||
source = unquote(urlparse(url).path) if url.startswith("file://") else url
|
||||
try:
|
||||
data = get_image_bytes(source)
|
||||
header = Image.open(BytesIO(data))
|
||||
except CLIENT_MEDIA_EXCEPTIONS as e:
|
||||
raise ValueError(f"image {index} could not be read: {e}") from e
|
||||
if url.startswith(_MUTABLE_SOURCES):
|
||||
# A source that can change between reads is forwarded as the bytes counted.
|
||||
mime = Image.MIME.get(header.format, "application/octet-stream")
|
||||
encoded = pybase64.b64encode(data).decode()
|
||||
image = dataclasses.replace(image, url=f"data:{mime};base64,{encoded}")
|
||||
return image, header
|
||||
|
||||
|
||||
def _inline_header(url: str) -> Optional[Image.Image]:
|
||||
"""An inline image's header from a prefix of its base64, or None when the
|
||||
header does not open from that prefix."""
|
||||
start = url.find(",") + 1 if url.startswith("data:") else 0
|
||||
prefix = url[start : start + _INLINE_HEADER_CHARS]
|
||||
try:
|
||||
return Image.open(BytesIO(pybase64.b64decode(prefix, validate=True)))
|
||||
except Exception:
|
||||
# A header past the prefix, or malformed data that the full read reports.
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,539 @@
|
||||
"""Joint schema head of Clef decision checkpoints.
|
||||
|
||||
A Clef checkpoint is a Qwen3.5 backbone saved with ``joint_head_config.json``
|
||||
and ``joint_head.safetensors``. The head reads the backbone's final hidden
|
||||
state at every prompt token and scores every allowed option of every question
|
||||
together: each option gathers evidence from the prompt, the questions attend
|
||||
to each other and to the prompt, and a lexical prior built from the LM head
|
||||
rows of each option's tokens keeps the option's meaning.
|
||||
|
||||
One head pass serves a whole prefill batch. Token-wise steps run on the flat
|
||||
batch, attention is variable-length per request, and span means come from
|
||||
per-request prefix sums over the schema region only, so a long state costs no
|
||||
more than its memory projection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Callable, List, Optional, Sequence, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.pooler import EmbeddingPoolerOutput
|
||||
|
||||
# Question types in the order of the head's type embedding.
|
||||
QUESTION_TYPES = ("noul", "choice", "score")
|
||||
|
||||
# Bounds of one head pass, so a large prefill batch is scored in a few passes.
|
||||
_MAX_GROUP_TOKENS = 65536
|
||||
_MAX_GROUP_SLOTS = 1 << 18
|
||||
|
||||
|
||||
class LayoutQuestion(msgspec.Struct, frozen=True):
|
||||
question_type: int
|
||||
question_span: Tuple[int, int]
|
||||
option_spans: Tuple[Tuple[int, int], ...]
|
||||
|
||||
|
||||
def max_joint_prompt_tokens(
|
||||
*,
|
||||
context_len: int,
|
||||
num_reserved_tokens: int,
|
||||
max_req_input_len: int,
|
||||
max_prefill_tokens: int,
|
||||
) -> int:
|
||||
"""The longest prompt, images expanded, that the server runs as one prefill.
|
||||
|
||||
The tokenizer manager refuses a prompt of context_len - num_reserved_tokens
|
||||
tokens, the scheduler one of max_req_input_len, and without chunked prefill
|
||||
activation memory is reserved for max_prefill_tokens.
|
||||
"""
|
||||
return min(
|
||||
context_len - num_reserved_tokens - 1,
|
||||
max_req_input_len - 1,
|
||||
max_prefill_tokens,
|
||||
)
|
||||
|
||||
|
||||
def pack_decision_layout(
|
||||
prompt_length: int, questions: Sequence[LayoutQuestion]
|
||||
) -> List[int]:
|
||||
"""Flatten a prompt's questions into the ints a request carries.
|
||||
|
||||
The layout is ``[prompt_length, num_questions]`` followed, per question,
|
||||
by ``type, question_start, question_end, num_options`` and one
|
||||
``option_start, option_end`` pair per option. Positions index the prompt
|
||||
before its image placeholders expand.
|
||||
"""
|
||||
layout = [prompt_length, len(questions)]
|
||||
for question in questions:
|
||||
layout += [
|
||||
question.question_type,
|
||||
*question.question_span,
|
||||
len(question.option_spans),
|
||||
]
|
||||
for start, end in question.option_spans:
|
||||
layout += [start, end]
|
||||
return layout
|
||||
|
||||
|
||||
def parse_decision_layout(
|
||||
layout: Sequence[int], num_tokens: int
|
||||
) -> List[LayoutQuestion]:
|
||||
"""The questions of a layout, placed on the sequence the model reads.
|
||||
|
||||
Every span follows the prompt's images, so expanding their placeholders
|
||||
shifts all spans by the number of added tokens. Raises ValueError when the
|
||||
layout does not describe the sequence.
|
||||
"""
|
||||
if len(layout) < 2 or any(type(value) is not int for value in layout):
|
||||
raise ValueError("decision_layout must be a list of ints")
|
||||
prompt_length, num_questions = layout[0], layout[1]
|
||||
shift = num_tokens - prompt_length
|
||||
if prompt_length < 1 or shift < 0 or num_questions < 1:
|
||||
raise ValueError("decision_layout does not match the prompt length")
|
||||
questions = []
|
||||
cursor = 2
|
||||
for _ in range(num_questions):
|
||||
if cursor + 4 > len(layout):
|
||||
raise ValueError("decision_layout is truncated")
|
||||
question_type, start, end, num_options = layout[cursor : cursor + 4]
|
||||
cursor += 4
|
||||
if (
|
||||
not 0 <= question_type < len(QUESTION_TYPES)
|
||||
or num_options < 1
|
||||
or not 0 <= start < end <= prompt_length
|
||||
or cursor + 2 * num_options > len(layout)
|
||||
):
|
||||
raise ValueError("decision_layout has an invalid question")
|
||||
spans = []
|
||||
for _ in range(num_options):
|
||||
option_start, option_end = layout[cursor : cursor + 2]
|
||||
cursor += 2
|
||||
if not 0 <= option_start < option_end <= prompt_length:
|
||||
raise ValueError("decision_layout has an invalid option span")
|
||||
spans.append((option_start + shift, option_end + shift))
|
||||
questions.append(
|
||||
LayoutQuestion(question_type, (start + shift, end + shift), tuple(spans))
|
||||
)
|
||||
if cursor != len(layout):
|
||||
raise ValueError("decision_layout has trailing values")
|
||||
return questions
|
||||
|
||||
|
||||
class EvidenceRoutingLayer(nn.Module):
|
||||
def __init__(self, width: int, heads: int, feedforward: int) -> None:
|
||||
super().__init__()
|
||||
self.query_norm = nn.LayerNorm(width)
|
||||
self.memory_norm = nn.LayerNorm(width)
|
||||
self.attention = nn.MultiheadAttention(width, heads, batch_first=True)
|
||||
self.feedforward_norm = nn.LayerNorm(width)
|
||||
# Identity slots keep the checkpoint's Sequential indices (dropout at 2 and 4).
|
||||
self.feedforward = nn.Sequential(
|
||||
nn.Linear(width, feedforward),
|
||||
nn.GELU(),
|
||||
nn.Identity(),
|
||||
nn.Linear(feedforward, width),
|
||||
nn.Identity(),
|
||||
)
|
||||
|
||||
|
||||
class JointSchemaHead(nn.Module):
|
||||
"""Parameters named as in the checkpoint's ``joint_head.safetensors``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
width: int,
|
||||
routing_layers: int,
|
||||
layers: int,
|
||||
heads: int,
|
||||
feedforward: int,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_norm = nn.LayerNorm(hidden_size)
|
||||
self.memory_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.question_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_question_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.global_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_context_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_lexical_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.type_embedding = nn.Embedding(len(QUESTION_TYPES), width)
|
||||
self.evidence_layers = nn.ModuleList(
|
||||
EvidenceRoutingLayer(width, heads, feedforward)
|
||||
for _ in range(routing_layers)
|
||||
)
|
||||
self.option_summary_norm = nn.LayerNorm(width)
|
||||
self.layers = nn.ModuleList(
|
||||
nn.TransformerDecoderLayer(
|
||||
d_model=width,
|
||||
nhead=heads,
|
||||
dim_feedforward=feedforward,
|
||||
dropout=0.0,
|
||||
activation="gelu",
|
||||
batch_first=True,
|
||||
norm_first=True,
|
||||
)
|
||||
for _ in range(layers)
|
||||
)
|
||||
self.field_norm = nn.LayerNorm(width)
|
||||
self.option_norm = nn.LayerNorm(width)
|
||||
self.residual_scorer = nn.Sequential(
|
||||
nn.Linear(width * 4, width), nn.GELU(), nn.Identity(), nn.Linear(width, 1)
|
||||
)
|
||||
self.prior_logit_scale = nn.Parameter(torch.zeros(()))
|
||||
self.joint_logit_scale = nn.Parameter(torch.zeros(()))
|
||||
self.residual_gate = nn.Parameter(torch.zeros(()))
|
||||
|
||||
@torch.no_grad()
|
||||
def forward_batch(
|
||||
self,
|
||||
hidden: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
lengths: List[int],
|
||||
layouts: List[List[LayoutQuestion]],
|
||||
output_embedding_weight: torch.Tensor,
|
||||
) -> List[torch.Tensor]:
|
||||
"""One FP32 logit vector per request, its questions' options in layout order.
|
||||
|
||||
``hidden`` holds the final hidden states of the requests back to back.
|
||||
"""
|
||||
outputs: List[torch.Tensor] = []
|
||||
start = 0
|
||||
for group in _groups(lengths, layouts):
|
||||
group_lengths = [lengths[i] for i in group]
|
||||
end = start + sum(group_lengths)
|
||||
outputs += self._forward_group(
|
||||
hidden[start:end],
|
||||
input_ids[start:end],
|
||||
group_lengths,
|
||||
[layouts[i] for i in group],
|
||||
output_embedding_weight,
|
||||
)
|
||||
start = end
|
||||
return outputs
|
||||
|
||||
def _forward_group(
|
||||
self,
|
||||
hidden: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
lengths: List[int],
|
||||
layouts: List[List[LayoutQuestion]],
|
||||
output_embedding_weight: torch.Tensor,
|
||||
) -> List[torch.Tensor]:
|
||||
device, dtype = hidden.device, hidden.dtype
|
||||
batch = len(layouts)
|
||||
question_bounds, option_bounds = [], []
|
||||
type_ids, owner, slot, question_request, last_tokens = [], [], [], [], []
|
||||
regions, region_offsets = [], [0]
|
||||
token_cu, question_cu, option_cu, option_counts = [0], [0], [0], []
|
||||
for request, (length, questions) in enumerate(zip(lengths, layouts)):
|
||||
offset = token_cu[-1]
|
||||
low = min(min(q.question_span[0], q.option_spans[0][0]) for q in questions)
|
||||
high = max(
|
||||
max(q.question_span[1], *(e for _, e in q.option_spans))
|
||||
for q in questions
|
||||
)
|
||||
# Prefix row of span position p in this request's schema region.
|
||||
base = region_offsets[-1] + request - low
|
||||
regions.append((offset + low, offset + high))
|
||||
request_options = 0
|
||||
for question in questions:
|
||||
count = len(question.option_spans)
|
||||
question_bounds += (
|
||||
base + question.question_span[0],
|
||||
base + question.question_span[1],
|
||||
)
|
||||
for option_slot, (option_start, option_end) in enumerate(
|
||||
question.option_spans
|
||||
):
|
||||
option_bounds += (base + option_start, base + option_end)
|
||||
owner.append(len(type_ids))
|
||||
slot.append(option_slot)
|
||||
type_ids.append(question.question_type)
|
||||
question_request.append(request)
|
||||
request_options += count
|
||||
last_tokens.append(offset + length - 1)
|
||||
token_cu.append(offset + length)
|
||||
region_offsets.append(region_offsets[-1] + high - low)
|
||||
question_cu.append(len(type_ids))
|
||||
option_cu.append(option_cu[-1] + request_options)
|
||||
option_counts.append(request_options)
|
||||
|
||||
num_questions = len(type_ids)
|
||||
widest = max(len(q.option_spans) for questions in layouts for q in questions)
|
||||
sizes = [
|
||||
2 * num_questions,
|
||||
len(option_bounds),
|
||||
num_questions,
|
||||
len(owner),
|
||||
len(slot),
|
||||
num_questions,
|
||||
batch,
|
||||
3 * (batch + 1),
|
||||
]
|
||||
packed = torch.tensor(
|
||||
question_bounds
|
||||
+ option_bounds
|
||||
+ type_ids
|
||||
+ owner
|
||||
+ slot
|
||||
+ question_request
|
||||
+ last_tokens
|
||||
+ token_cu
|
||||
+ question_cu
|
||||
+ option_cu,
|
||||
dtype=torch.long,
|
||||
).to(device, non_blocking=True)
|
||||
(
|
||||
question_bounds_t,
|
||||
option_bounds_t,
|
||||
type_ids_t,
|
||||
owner_t,
|
||||
slot_t,
|
||||
question_request_t,
|
||||
last_tokens_t,
|
||||
cu_t,
|
||||
) = torch.split(packed, sizes)
|
||||
question_bounds_t = question_bounds_t.view(-1, 2)
|
||||
option_bounds_t = option_bounds_t.view(-1, 2)
|
||||
token_cu_t, question_cu_t, option_cu_t = (
|
||||
cu_t.to(torch.int32).view(3, batch + 1).unbind(0)
|
||||
)
|
||||
token_bounds = (token_cu_t, max(lengths))
|
||||
question_bounds_cu = (question_cu_t, max(len(q) for q in layouts))
|
||||
option_bounds_cu = (option_cu_t, max(option_counts))
|
||||
|
||||
normalized = self.hidden_norm(hidden)
|
||||
# Running sums over each schema region, after one zero row per request.
|
||||
region_total = region_offsets[-1] + batch
|
||||
hidden_prefix = torch.zeros(
|
||||
region_total, normalized.shape[-1], dtype=torch.float32, device=device
|
||||
)
|
||||
lexical_prefix = torch.zeros(
|
||||
region_total,
|
||||
output_embedding_weight.shape[-1],
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
for request, (low, high) in enumerate(regions):
|
||||
row = region_offsets[request] + request + 1
|
||||
torch.cumsum(
|
||||
normalized[low:high].float(),
|
||||
dim=0,
|
||||
out=hidden_prefix[row : row + high - low],
|
||||
)
|
||||
torch.cumsum(
|
||||
output_embedding_weight[input_ids[low:high]].float(),
|
||||
dim=0,
|
||||
out=lexical_prefix[row : row + high - low],
|
||||
)
|
||||
|
||||
def span_means(prefix: torch.Tensor, bounds: torch.Tensor) -> torch.Tensor:
|
||||
sums = prefix[bounds[:, 1]] - prefix[bounds[:, 0]]
|
||||
return (sums / (bounds[:, 1] - bounds[:, 0]).unsqueeze(-1)).to(dtype)
|
||||
|
||||
memory = self.memory_projection(normalized)
|
||||
global_vectors = normalized[last_tokens_t]
|
||||
question_vectors = span_means(hidden_prefix, question_bounds_t)
|
||||
option_contexts = span_means(hidden_prefix, option_bounds_t)
|
||||
lexical = span_means(lexical_prefix, option_bounds_t)
|
||||
|
||||
routed = (
|
||||
self.option_context_projection(option_contexts)
|
||||
+ self.option_lexical_projection(lexical)
|
||||
+ self.option_question_projection(question_vectors)[owner_t]
|
||||
)
|
||||
for layer in self.evidence_layers:
|
||||
routed = routed + _attention(
|
||||
layer.attention,
|
||||
layer.query_norm(routed),
|
||||
layer.memory_norm(memory),
|
||||
option_bounds_cu,
|
||||
token_bounds,
|
||||
)
|
||||
routed = routed + layer.feedforward(layer.feedforward_norm(routed))
|
||||
|
||||
base_fields = self.question_projection(question_vectors)
|
||||
scores = torch.matmul(
|
||||
routed.unsqueeze(1), base_fields[owner_t].unsqueeze(2)
|
||||
).view(-1) / math.sqrt(routed.shape[-1])
|
||||
padded_scores = torch.full(
|
||||
(num_questions, widest), float("-inf"), dtype=torch.float32, device=device
|
||||
)
|
||||
padded_scores[owner_t, slot_t] = scores.float()
|
||||
weights = torch.softmax(padded_scores, dim=1).to(dtype)
|
||||
padded_options = routed.new_zeros(num_questions, widest, routed.shape[-1])
|
||||
padded_options[owner_t, slot_t] = routed
|
||||
summaries = torch.sum(weights.unsqueeze(-1) * padded_options, dim=1)
|
||||
fields = (
|
||||
base_fields
|
||||
+ self.option_summary_norm(summaries)
|
||||
+ self.global_projection(global_vectors)[question_request_t]
|
||||
+ self.type_embedding(type_ids_t)
|
||||
)
|
||||
for layer in self.layers:
|
||||
fields = fields + _attention(
|
||||
layer.self_attn,
|
||||
layer.norm1(fields),
|
||||
None,
|
||||
question_bounds_cu,
|
||||
question_bounds_cu,
|
||||
)
|
||||
fields = fields + _attention(
|
||||
layer.multihead_attn,
|
||||
layer.norm2(fields),
|
||||
memory,
|
||||
question_bounds_cu,
|
||||
token_bounds,
|
||||
)
|
||||
fields = fields + layer.linear2(
|
||||
layer.activation(layer.linear1(layer.norm3(fields)))
|
||||
)
|
||||
fields = self.field_norm(fields)
|
||||
|
||||
prior_scale = self.prior_logit_scale.clamp(max=math.log(100.0)).exp()
|
||||
joint_scale = self.joint_logit_scale.clamp(max=math.log(100.0)).exp()
|
||||
anchor = F.normalize(
|
||||
question_vectors + global_vectors[question_request_t], dim=-1
|
||||
)
|
||||
prior = prior_scale * torch.matmul(
|
||||
F.normalize(lexical, dim=-1).unsqueeze(1), anchor[owner_t].unsqueeze(2)
|
||||
).view(-1)
|
||||
options = self.option_norm(routed)
|
||||
repeated = fields[owner_t]
|
||||
cosine = F.cosine_similarity(repeated, options, dim=-1)
|
||||
features = torch.cat(
|
||||
[repeated, options, repeated * options, torch.abs(repeated - options)],
|
||||
dim=-1,
|
||||
)
|
||||
residual = self.residual_scorer(features).squeeze(-1)
|
||||
logits = prior + torch.sigmoid(self.residual_gate) * (
|
||||
joint_scale * cosine + residual
|
||||
)
|
||||
return list(torch.split(logits.float(), option_counts))
|
||||
|
||||
|
||||
def _attention(
|
||||
module: nn.MultiheadAttention,
|
||||
queries: torch.Tensor,
|
||||
keys: Optional[torch.Tensor],
|
||||
query_bounds: Tuple[torch.Tensor, int],
|
||||
key_bounds: Tuple[torch.Tensor, int],
|
||||
) -> torch.Tensor:
|
||||
"""nn.MultiheadAttention over packed sequences, keys and values from ``keys``
|
||||
(or from ``queries`` when None), attending within each request."""
|
||||
width = queries.shape[-1]
|
||||
heads = module.num_heads
|
||||
if keys is None:
|
||||
query, key, value = F.linear(
|
||||
queries, module.in_proj_weight, module.in_proj_bias
|
||||
).chunk(3, dim=-1)
|
||||
else:
|
||||
query_weight, key_value_weight = module.in_proj_weight.split([width, 2 * width])
|
||||
query_bias, key_value_bias = module.in_proj_bias.split([width, 2 * width])
|
||||
query = F.linear(queries, query_weight, query_bias)
|
||||
key, value = F.linear(keys, key_value_weight, key_value_bias).chunk(2, dim=-1)
|
||||
head_dim = width // heads
|
||||
output = torch.ops.aten._flash_attention_forward(
|
||||
query.reshape(-1, heads, head_dim),
|
||||
key.reshape(-1, heads, head_dim),
|
||||
value.reshape(-1, heads, head_dim),
|
||||
query_bounds[0],
|
||||
key_bounds[0],
|
||||
query_bounds[1],
|
||||
key_bounds[1],
|
||||
0.0,
|
||||
False,
|
||||
False,
|
||||
)[0]
|
||||
return module.out_proj(output.reshape(-1, width))
|
||||
|
||||
|
||||
def _groups(lengths: List[int], layouts: List[List[LayoutQuestion]]) -> List[range]:
|
||||
"""Consecutive requests per head pass, bounded in tokens and padded option slots."""
|
||||
groups, start, tokens, questions, widest = [], 0, 0, 0, 0
|
||||
for index, (length, layout) in enumerate(zip(lengths, layouts)):
|
||||
layout_widest = max(len(question.option_spans) for question in layout)
|
||||
grown = (questions + len(layout)) * max(widest, layout_widest)
|
||||
if index > start and (
|
||||
tokens + length > _MAX_GROUP_TOKENS or grown > _MAX_GROUP_SLOTS
|
||||
):
|
||||
groups.append(range(start, index))
|
||||
start, tokens, questions, widest = index, 0, 0, 0
|
||||
tokens += length
|
||||
questions += len(layout)
|
||||
widest = max(widest, layout_widest)
|
||||
groups.append(range(start, len(lengths)))
|
||||
return groups
|
||||
|
||||
|
||||
class JointSchemaPooler(nn.Module):
|
||||
"""Pooler of a Clef checkpoint: the joint schema head's option logits per request."""
|
||||
|
||||
def __init__(
|
||||
self, head: JointSchemaHead, output_embedding: Callable[[], torch.Tensor]
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.head = head
|
||||
# A callable, so the LM head is not registered twice as a parameter.
|
||||
self._output_embedding = output_embedding
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, forward_batch
|
||||
) -> EmbeddingPoolerOutput:
|
||||
if (
|
||||
forward_batch.decision_layouts is None
|
||||
or forward_batch.extend_seq_lens_cpu is None
|
||||
):
|
||||
return EmbeddingPoolerOutput(
|
||||
embeddings=[
|
||||
hidden_states.new_empty(0, dtype=torch.float32)
|
||||
for _ in range(forward_batch.batch_size)
|
||||
]
|
||||
)
|
||||
lengths = list(forward_batch.extend_seq_lens_cpu)
|
||||
prefix_lengths = forward_batch.extend_prefix_lens_cpu or [0] * len(lengths)
|
||||
raw_layouts = forward_batch.decision_layouts
|
||||
embeddings: List[torch.Tensor] = [
|
||||
hidden_states.new_empty(0, dtype=torch.float32) for _ in lengths
|
||||
]
|
||||
# The tokenizer manager checks every layout against its prompt, and the
|
||||
# server runs each prompt in one uncached prefill. A layout that no longer
|
||||
# fits its prompt, as after a truncation, gets no scores, which the
|
||||
# decision route reports as a server error.
|
||||
layouts = {}
|
||||
for i, (layout, prefix, length) in enumerate(
|
||||
zip(raw_layouts, prefix_lengths, lengths)
|
||||
):
|
||||
if layout is None or prefix != 0:
|
||||
continue
|
||||
try:
|
||||
layouts[i] = parse_decision_layout(layout, length)
|
||||
except ValueError:
|
||||
continue
|
||||
offsets = [0]
|
||||
for length in lengths:
|
||||
offsets.append(offsets[-1] + length)
|
||||
runs: List[List[int]] = []
|
||||
for i in layouts:
|
||||
if runs and runs[-1][-1] == i - 1:
|
||||
runs[-1].append(i)
|
||||
else:
|
||||
runs.append([i])
|
||||
for run in runs:
|
||||
start, end = offsets[run[0]], offsets[run[-1] + 1]
|
||||
logits = self.head.forward_batch(
|
||||
hidden_states[start:end],
|
||||
forward_batch.input_ids[start:end],
|
||||
[lengths[i] for i in run],
|
||||
[layouts[i] for i in run],
|
||||
self._output_embedding(),
|
||||
)
|
||||
for i, values in zip(run, logits):
|
||||
embeddings[i] = values
|
||||
return EmbeddingPoolerOutput(embeddings=embeddings)
|
||||
@@ -1253,6 +1253,11 @@ class EmbeddingReqInput:
|
||||
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
|
||||
token_indices_to_pool: Optional[Union[List[List[int]], List[int]]] = None
|
||||
|
||||
# Question and option spans that a Clef checkpoint's joint schema head
|
||||
# scores, as packed by layers.joint_schema_head.pack_decision_layout.
|
||||
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
|
||||
decision_layout: Optional[Union[List[List[int]], List[int]]] = None
|
||||
|
||||
def regenerate_rid(self):
|
||||
"""Generate a new request ID and return it."""
|
||||
if isinstance(self.rid, list):
|
||||
@@ -1469,6 +1474,11 @@ class EmbeddingReqInput:
|
||||
if self.token_indices_to_pool is not None
|
||||
else None
|
||||
),
|
||||
decision_layout=(
|
||||
self.decision_layout[i]
|
||||
if self.decision_layout is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
cache[i] = sub
|
||||
return sub
|
||||
@@ -1500,6 +1510,8 @@ class TokenizedEmbeddingReqInput(BaseReq, kw_only=True):
|
||||
multi_item_delimiter_indices: Optional[List[int]] = None
|
||||
# Token positions for setwise pooling readout
|
||||
token_indices_to_pool: Optional[List[int]] = None
|
||||
# Question and option spans for a joint schema head
|
||||
decision_layout: Optional[List[int]] = None
|
||||
# For observability
|
||||
# Pickled Optional[Union[APIServerReqTimeStats, DPControllerReqTimeStats]]
|
||||
time_stats: Optional[PickleWrapper] = None
|
||||
|
||||
@@ -1050,6 +1050,7 @@ class Req(ReqDllmMixin):
|
||||
return_pooled_hidden_states: bool = False,
|
||||
multi_item_delimiter_indices: Optional[List[int]] = None,
|
||||
token_indices_to_pool: Optional[List[int]] = None,
|
||||
decision_layout: Optional[List[int]] = None,
|
||||
session_id: Optional[str] = None,
|
||||
cache_salt: Optional[str] = None,
|
||||
):
|
||||
@@ -1083,6 +1084,7 @@ class Req(ReqDllmMixin):
|
||||
self.positional_embed_overrides = positional_embed_overrides
|
||||
self.multi_item_delimiter_indices = multi_item_delimiter_indices
|
||||
self.token_indices_to_pool = token_indices_to_pool
|
||||
self.decision_layout = decision_layout
|
||||
|
||||
# For req-level memory management
|
||||
self.kv = ReqKvInfo()
|
||||
|
||||
@@ -3465,6 +3465,7 @@ class Scheduler(
|
||||
return_pooled_hidden_states=recv_req.return_pooled_hidden_states,
|
||||
multi_item_delimiter_indices=recv_req.multi_item_delimiter_indices,
|
||||
token_indices_to_pool=recv_req.token_indices_to_pool,
|
||||
decision_layout=recv_req.decision_layout,
|
||||
)
|
||||
req.tokenizer = self.tokenizer
|
||||
self._maybe_namespace_elastic_radix_cache(req)
|
||||
|
||||
@@ -64,6 +64,10 @@ from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX
|
||||
from sglang.srt.disaggregation.encoder.receiver import create_mm_receiver
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.joint_schema_head import (
|
||||
max_joint_prompt_tokens,
|
||||
parse_decision_layout,
|
||||
)
|
||||
from sglang.srt.lora.lora_registry import LoRARef, LoRARegistry
|
||||
from sglang.srt.managers.async_dynamic_batch_tokenizer import AsyncDynamicbatchTokenizer
|
||||
from sglang.srt.managers.disagg_service import start_disagg_service
|
||||
@@ -1295,6 +1299,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
raise ValueError(
|
||||
"encoder SWA replay cannot return cached prompt logprobs"
|
||||
)
|
||||
# A decision layout indexes the whole prompt, so it is checked before any
|
||||
# truncation below could cut the prompt.
|
||||
self._validate_joint_schema_request(obj, input_ids)
|
||||
_max_req_len = self.context_len
|
||||
input_token_num = len(input_ids) if input_ids is not None else 0
|
||||
input_token_num += self.num_reserved_tokens
|
||||
@@ -1399,6 +1406,36 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
"return_sampling_mask=true."
|
||||
)
|
||||
|
||||
def _validate_joint_schema_request(
|
||||
self, obj: Union[GenerateReqInput, EmbeddingReqInput], input_ids: List[int]
|
||||
) -> None:
|
||||
"""A Clef checkpoint scores the spans of a decision layout, and nothing else."""
|
||||
if self.model_config.joint_head_config is None:
|
||||
return
|
||||
if isinstance(obj, GenerateReqInput) or obj.decision_layout is None:
|
||||
# Health checks only need a response, which is an empty embedding.
|
||||
if (
|
||||
isinstance(obj, EmbeddingReqInput)
|
||||
and isinstance(obj.rid, str)
|
||||
and obj.rid.startswith(HEALTH_CHECK_RID_PREFIX)
|
||||
):
|
||||
return
|
||||
raise ValueError(
|
||||
"This checkpoint answers schema decisions, send them to /v1/systemone"
|
||||
)
|
||||
limit = max_joint_prompt_tokens(
|
||||
context_len=self.context_len,
|
||||
num_reserved_tokens=self.num_reserved_tokens,
|
||||
max_req_input_len=self.max_req_input_len,
|
||||
max_prefill_tokens=get_schedule().max_prefill_tokens,
|
||||
)
|
||||
if len(input_ids) > limit:
|
||||
raise ValueError(
|
||||
f"The prompt has {len(input_ids)} tokens, but this server scores "
|
||||
f"at most {limit} in one prefill"
|
||||
)
|
||||
parse_decision_layout(obj.decision_layout, len(input_ids))
|
||||
|
||||
def _validate_mm_limits(
|
||||
self, obj: Union[GenerateReqInput, EmbeddingReqInput]
|
||||
) -> None:
|
||||
@@ -1575,6 +1612,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
return_pooled_hidden_states=obj.return_pooled_hidden_states,
|
||||
multi_item_delimiter_indices=obj.multi_item_delimiter_indices,
|
||||
token_indices_to_pool=obj.token_indices_to_pool,
|
||||
decision_layout=obj.decision_layout,
|
||||
)
|
||||
|
||||
tokenized_obj.time_stats = self.rid_to_state[obj.rid].time_stats
|
||||
|
||||
@@ -614,6 +614,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# Setwise pooling readout positions (CPU tensors, one per request)
|
||||
token_indices_to_pool: Optional[List[torch.Tensor]] = None
|
||||
|
||||
# Joint schema head decision layouts (one per request, None if absent)
|
||||
decision_layouts: Optional[List[Optional[List[int]]]] = None
|
||||
|
||||
# === Borrowed from ScheduleBatch: compound (carry their own device tensors) ===
|
||||
# Sampling info
|
||||
sampling_info: SamplingBatchInfo = None
|
||||
@@ -1264,6 +1267,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
for r in batch.reqs
|
||||
]
|
||||
|
||||
if any(r.decision_layout is not None for r in batch.reqs):
|
||||
self.decision_layouts = [r.decision_layout for r in batch.reqs]
|
||||
|
||||
token_type_ids = [
|
||||
r.token_type_ids for r in batch.reqs if r.token_type_ids is not None
|
||||
]
|
||||
|
||||
@@ -34,7 +34,10 @@ from sglang.kernels.ops.attention.triton_gdn_fused_proj import (
|
||||
from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul
|
||||
|
||||
# Configs
|
||||
from sglang.srt.configs.model_config import load_decision_config
|
||||
from sglang.srt.configs.model_config import (
|
||||
load_decision_config,
|
||||
load_joint_head_config,
|
||||
)
|
||||
from sglang.srt.configs.qwen3_5 import (
|
||||
Qwen3_5Config,
|
||||
Qwen3_5MoeConfig,
|
||||
@@ -54,6 +57,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
from sglang.srt.layers.joint_schema_head import JointSchemaHead, JointSchemaPooler
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
@@ -2330,6 +2334,16 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration):
|
||||
self.visual.deepstack_visual_indexes if self.visual is not None else []
|
||||
)
|
||||
|
||||
if config.architectures == ["Qwen3_5ForConditionalGeneration"]:
|
||||
model = get_model()
|
||||
joint_head_config = load_joint_head_config(model.model_path, model.revision)
|
||||
if joint_head_config is not None:
|
||||
# A Clef checkpoint, whose joint schema head scores every option
|
||||
# of every question from the final hidden states and LM head rows.
|
||||
self.pooler = JointSchemaPooler(
|
||||
JointSchemaHead(**joint_head_config), lambda: self.lm_head.weight
|
||||
)
|
||||
|
||||
def get_hidden_dim(self, module_name: str, layer_idx: int):
|
||||
return self.model.get_hidden_dim(module_name, layer_idx)
|
||||
|
||||
@@ -2470,8 +2484,23 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration):
|
||||
if bare_backbone and self.pp_group.is_last_rank:
|
||||
self._load_decision_readout()
|
||||
loaded_params.add("lm_head.weight")
|
||||
if isinstance(self.pooler, JointSchemaPooler):
|
||||
self._load_joint_head()
|
||||
loaded_params.update(
|
||||
f"pooler.head.{name}" for name, _ in self.pooler.head.named_parameters()
|
||||
)
|
||||
return loaded_params
|
||||
|
||||
def _load_joint_head(self) -> None:
|
||||
"""Load joint_head.safetensors, which the checkpoint index leaves out."""
|
||||
model = get_model()
|
||||
state_dict = load_file(
|
||||
cached_file(
|
||||
model.model_path, "joint_head.safetensors", revision=model.revision
|
||||
)
|
||||
)
|
||||
self.pooler.head.load_state_dict(state_dict, strict=True)
|
||||
|
||||
def _load_decision_readout(self) -> None:
|
||||
"""Place the checkpoint's decision readout in the LM head rows of its codes."""
|
||||
model = get_model()
|
||||
|
||||
Reference in New Issue
Block a user