From bd2d73daa5afda6bcad8d479a834f2e70bbe9569 Mon Sep 17 00:00:00 2001 From: areneau-cf Date: Fri, 9 Oct 2026 01:25:52 -0500 Subject: [PATCH] [model] support Clef and Clef-Flash decision models on /v1/systemone (#42721) Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> --- python/sglang/srt/arg_groups/model_hook.py | 31 + python/sglang/srt/configs/model_config.py | 20 + python/sglang/srt/entrypoints/http_server.py | 25 + .../entrypoints/openai/serving_decisions.py | 1 + .../srt/entrypoints/systemone/joint_schema.py | 137 +++++ .../srt/entrypoints/systemone/serving.py | 201 ++++++- python/sglang/srt/layers/joint_schema_head.py | 539 ++++++++++++++++++ python/sglang/srt/managers/io_struct.py | 12 + python/sglang/srt/managers/schedule_batch.py | 2 + python/sglang/srt/managers/scheduler.py | 1 + .../sglang/srt/managers/tokenizer_manager.py | 38 ++ .../srt/model_executor/forward_batch_info.py | 6 + python/sglang/srt/models/qwen3_5.py | 31 +- .../test_multimodal_piecewise_cuda_graph.py | 2 + .../openai/test_serving_decisions.py | 1 + .../systemone/test_joint_schema.py | 474 +++++++++++++++ .../unit/layers/test_joint_schema_head.py | 240 ++++++++ .../managers/test_hidden_state_server_mode.py | 1 + .../managers/test_sampling_mask_validation.py | 2 + .../multimodal/test_gpu_feature_transport.py | 1 + 20 files changed, 1763 insertions(+), 2 deletions(-) create mode 100644 python/sglang/srt/entrypoints/systemone/joint_schema.py create mode 100644 python/sglang/srt/layers/joint_schema_head.py create mode 100644 test/registered/unit/entrypoints/systemone/test_joint_schema.py create mode 100644 test/registered/unit/layers/test_joint_schema_head.py diff --git a/python/sglang/srt/arg_groups/model_hook.py b/python/sglang/srt/arg_groups/model_hook.py index dc1a9be316..485addec92 100644 --- a/python/sglang/srt/arg_groups/model_hook.py +++ b/python/sglang/srt/arg_groups/model_hook.py @@ -977,6 +977,37 @@ def handle_model_capability_adjustments(server_args: Any): "prefill; using breakable CUDA graph for CUDA prefill." ) + # A Clef checkpoint's joint schema head reads the final hidden state of + # every prompt token in one pass, so each request is one complete prefill + # with no decode: embedding mode, without prefix reuse or chunking. This + # also puts the FA backend on its raw K/V path, which skips the KV pool. + if model_config.joint_head_config is not None: + if cfg.tp_size != 1 or cfg.pp_size != 1: + raise ValueError( + "Clef checkpoints are served with --tp-size 1 and --pp-size 1, " + "since the joint schema head reads full LM head rows" + ) + for key, value in ( + ("is_embedding", True), + ("disable_radix_cache", True), + ("chunked_prefill_size", -1), + ): + declare_resolution( + server_args, "_handle_model_capability_adjustments", **{key: value} + ) + for phase in (Phase.DECODE, Phase.PREFILL): + declare_resolution( + server_args, + "_handle_model_capability_adjustments", + cuda_graph_config=with_phase( + cfg.cuda_graph_config, phase, backend=Backend.DISABLED + ), + ) + logger.info( + "Clef joint schema head detected: serving /v1/systemone decisions in " + "embedding mode without radix cache, chunked prefill, or CUDA graphs." + ) + if ( model_config.is_multimodal and not model_config.is_multimodal_chunked_prefill_supported diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index 37cbbe45ff..2f5da7fc36 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -291,6 +291,23 @@ def load_decision_config(model_path: str, revision: Optional[str]) -> Optional[d return config +@lru_cache +def load_joint_head_config(model_path: str, revision: Optional[str]) -> Optional[dict]: + """joint_head_config.json of a Clef checkpoint, whose joint schema head + answers /v1/systemone decisions from the backbone's hidden states.""" + path = cached_file( + model_path, + "joint_head_config.json", + revision=revision, + _raise_exceptions_for_missing_entries=False, + _raise_exceptions_for_connection_errors=False, + ) + if path is None: + return None + with open(path) as f: + return json.load(f) + + def is_qwen3_5(config) -> bool: return _hf_arch(config) in ( "Qwen3_5ForConditionalGeneration", @@ -544,6 +561,9 @@ class ModelConfig: ): # Full attention sees the whole prompt, while linear attention stays causal. self.hf_text_config.is_causal = False + self.joint_head_config = None + if self.hf_config.architectures == ["Qwen3_5ForConditionalGeneration"]: + self.joint_head_config = load_joint_head_config(self.model_path, revision) self.requires_mm_token_modalities = requires_mm_token_modalities( self.hf_config.architectures, self.hf_text_config ) diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index ea9addb0ba..c662f6a2f5 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -66,6 +66,7 @@ from fastapi.routing import APIRoute from sglang.srt.arg_groups.overrides import resolving_view from sglang.srt.configs.embedding_model_spec import resolved_embedding_plan +from sglang.srt.configs.model_config import load_joint_head_config from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX from sglang.srt.disaggregation.utils import FAKE_BOOTSTRAP_HOST, DisaggregationMode from sglang.srt.entrypoints.anthropic.protocol import ( @@ -972,6 +973,7 @@ async def serve_generate_request(obj: GenerateReqInput, request: Request): async def encode_request(obj: EmbeddingReqInput, request: Request): """Handle an embedding request.""" try: + _refuse_decision_layout(obj) ret = await _global_state.tokenizer_manager.generate_request( obj, request ).__anext__() @@ -984,6 +986,7 @@ async def encode_request(obj: EmbeddingReqInput, request: Request): async def classify_request(obj: EmbeddingReqInput, request: Request): """Handle a reward model request. Now the arguments and return values are the same as embedding models.""" try: + _refuse_decision_layout(obj) ret = await _global_state.tokenizer_manager.generate_request( obj, request ).__anext__() @@ -992,6 +995,12 @@ async def classify_request(obj: EmbeddingReqInput, request: Request): return _create_error_response(e) +def _refuse_decision_layout(obj: EmbeddingReqInput) -> None: + # Only /v1/systemone builds a decision layout, from the prompt it compiles. + if obj.decision_layout is not None: + raise ValueError("decision_layout is set only by /v1/systemone") + + @app.api_route("/flush_cache", methods=["GET", "POST"]) @auth_level(AuthLevel.ADMIN_OPTIONAL) async def flush_cache(timeout: float = Query(0.0, ge=0.0)): @@ -2379,6 +2388,22 @@ def _execute_server_warmup(server_args: ServerArgs): ).tolist() json_data["sampling_params"]["max_new_tokens"] = 0 + if ( + not model_info["is_generation"] + and model_info.get("architectures") == ["Qwen3_5ForConditionalGeneration"] + and load_joint_head_config(get_model().model_path, get_model().revision) + is not None + ): + # A Clef checkpoint answers /v1/systemone decisions only. + request_name = "/v1/systemone" + json_data = { + "model": "warmup", + "state": "The capital city of France is Paris.", + "questions": { + "warmup": {"type": "noul", "instructions": "Is the state true?"} + }, + } + # Send a warmup request warmup_timeout = envs.SGLANG_WARMUP_TIMEOUT.get() try: diff --git a/python/sglang/srt/entrypoints/openai/serving_decisions.py b/python/sglang/srt/entrypoints/openai/serving_decisions.py index 960219d5ac..edf7698786 100644 --- a/python/sglang/srt/entrypoints/openai/serving_decisions.py +++ b/python/sglang/srt/entrypoints/openai/serving_decisions.py @@ -83,6 +83,7 @@ class OpenAIServingDecisions(OpenAIServingBase): self.chat_encoding_spec = chat_serving.chat_encoding_spec self.prompt_text_is_lossy = chat_serving._prompt_text_round_trip_is_lossy self.decision_config = self.tokenizer_manager.model_config.decision_config + self.joint_head_config = self.tokenizer_manager.model_config.joint_head_config tokenizer = self.tokenizer_manager.tokenizer # Other tokenizers skip the shortcut in _encode_labels and check the full prompt. self.added_tokens = ( diff --git a/python/sglang/srt/entrypoints/systemone/joint_schema.py b/python/sglang/srt/entrypoints/systemone/joint_schema.py new file mode 100644 index 0000000000..66d9568fbf --- /dev/null +++ b/python/sglang/srt/entrypoints/systemone/joint_schema.py @@ -0,0 +1,137 @@ +"""The prompt that a Clef checkpoint's joint schema head reads. + +Clef answers /v1/systemone questions from the backbone's hidden states instead +of from generated text. One prompt holds the state and every question with its +allowed options, and the head scores the span of each option. This follows +encode_record in the checkpoint's joint_schema_model.py, piece by piece, since +each piece is tokenized on its own. +""" + +from __future__ import annotations + +import json +from typing import Any, List, Sequence, Tuple + +from sglang.srt.entrypoints.systemone.protocol import ( + SystemOneChoiceQuestion, + SystemOneNoulQuestion, + SystemOneQuestion, + SystemOneRequest, +) +from sglang.srt.layers.joint_schema_head import ( + QUESTION_TYPES, + LayoutQuestion, + pack_decision_layout, +) + +_SYSTEM_PROMPT = ( + "Read the complete state and schema. Decide every field jointly. Each answer " + "must be exactly one of that field's allowed options." +) +_PREFIX = f"<|im_start|>system\n{_SYSTEM_PROMPT}<|im_end|>\n<|im_start|>user\nSTATE:\n" +_IMAGE = "<|vision_start|><|image_pad|><|vision_end|>" +_SUFFIX = ( + "\n<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" + "JOINT SCHEMA DECISIONS:" +) +_NOUL_CRITERIA = { + "true": "The proposition is true or the answer is yes.", + "false": "The proposition is false or the answer is no.", +} +# The default max_length of encode_record and systemone in joint_schema_model.py. +REFERENCE_MAX_LENGTH = 16384 + + +def _render(value: Any) -> str: + if isinstance(value, str): + return value + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def joint_schema_options(question: SystemOneQuestion) -> List[Tuple[str, Any]]: + """Each option's name and description, in the order the head scores them.""" + if isinstance(question, SystemOneNoulQuestion): + criteria = dict(_NOUL_CRITERIA) + if question.criteria is not None: + # A description sent as null drops the default, as in encode_record. + criteria.update(question.criteria.model_dump(exclude_unset=True)) + return [(name, criteria[name]) for name in ("true", "false")] + if isinstance(question, SystemOneChoiceQuestion): + return sorted(question.criteria.items(), key=lambda item: item[0]) + return [(str(level), detail) for level, detail in enumerate(question.criteria)] + + +def encode_joint_schema( + tokenizer: Any, + request: SystemOneRequest, + max_length: int, + image_token_counts: Sequence[int], +) -> Tuple[List[int], List[int]]: + """The prompt ids and their decision layout. + + Each image's placeholder expands to its count in ``image_token_counts``, and + the state is truncated so that the expanded prompt has at most + ``max_length`` tokens, as encode_record budgets the expanded media. + """ + + def tokens(text: str) -> List[int]: + return tokenizer.encode(text, add_special_tokens=False) + + schema = tokens("\n\nSCHEMA FIELDS:\n") + questions = [] + for index, (question_id, question) in enumerate(request.questions.items()): + schema += tokens( + f"\nFIELD {index + 1}\nID: {question_id}\nTYPE: {question.type}\n" + "INSTRUCTION: " + ) + instructions = question.instructions + if instructions is None or instructions == "": + instructions = question_id + question_start = len(schema) + schema += tokens(_render(instructions)) + question_span = (question_start, len(schema)) + schema += tokens("\nALLOWED OPTIONS:\n") + option_spans = [] + for option_index, (name, description) in enumerate( + joint_schema_options(question) + ): + schema += tokens(f"OPTION {option_index + 1}: ") + option_start = len(schema) + semantics = {"option_id": name} + if description is not None: + semantics["description"] = description + schema += tokens(_render(semantics)) + option_spans.append((option_start, len(schema))) + schema += tokens("\n") + schema += tokens("END FIELD\n") + questions.append( + (QUESTION_TYPES.index(question.type), question_span, option_spans) + ) + + prefix = tokens(_PREFIX) + if request.images: + prefix += tokens(_IMAGE * len(request.images) + "\n") + suffix = tokens(_SUFFIX) + # Each image's single <|image_pad|> becomes that image's tokens. + expansion = sum(count - 1 for count in image_token_counts) + fixed_length = len(prefix) + expansion + len(schema) + len(suffix) + if fixed_length > max_length: + raise ValueError( + f"the questions need {fixed_length} prompt tokens before the state, " + f"but the prompt holds at most {max_length}" + ) + state = tokens(_render(request.state))[: max_length - fixed_length] + offset = len(prefix) + len(state) + input_ids = prefix + state + schema + suffix + layout = pack_decision_layout( + len(input_ids), + [ + LayoutQuestion( + question_type, + (question_span[0] + offset, question_span[1] + offset), + tuple((start + offset, end + offset) for start, end in option_spans), + ) + for question_type, question_span, option_spans in questions + ], + ) + return input_ids, layout diff --git a/python/sglang/srt/entrypoints/systemone/serving.py b/python/sglang/srt/entrypoints/systemone/serving.py index 4496d50933..c966e3b670 100644 --- a/python/sglang/srt/entrypoints/systemone/serving.py +++ b/python/sglang/srt/entrypoints/systemone/serving.py @@ -2,14 +2,20 @@ from __future__ import annotations +import asyncio +import dataclasses import json import math import string +from io import BytesIO from typing import Any, Dict, Iterator, List, Optional, Tuple +from urllib.parse import unquote, urlparse import orjson +import pybase64 from fastapi import Request from fastapi.responses import ORJSONResponse +from PIL import Image from sglang.srt.entrypoints.openai.serving_decisions import ( EncodedQuestion, @@ -22,6 +28,11 @@ from sglang.srt.entrypoints.openai.serving_decisions import ( render_question, render_text, ) +from sglang.srt.entrypoints.systemone.joint_schema import ( + REFERENCE_MAX_LENGTH, + encode_joint_schema, + joint_schema_options, +) from sglang.srt.entrypoints.systemone.protocol import ( SystemOneChoiceAnswer, SystemOneChoiceQuestion, @@ -33,6 +44,10 @@ from sglang.srt.entrypoints.systemone.protocol import ( SystemOneScoreAnswer, SystemOneUsage, ) +from sglang.srt.layers.joint_schema_head import max_joint_prompt_tokens +from sglang.srt.managers.io_struct import EmbeddingReqInput +from sglang.srt.runtime_context import get_schedule +from sglang.srt.utils import CLIENT_MEDIA_EXCEPTIONS, ImageData, get_image_bytes # Beyond A to Z, every option gets a two-letter label, in this fixed order. _PAIR_LABELS = [a + b for a in string.ascii_uppercase for b in string.ascii_uppercase] @@ -44,9 +59,16 @@ _DECIDER_SYSTEM = ( "Reply with only the selected option code." ) +# Image sources whose content can change between reads, as load_image reads them. +_MUTABLE_SOURCES = ("http://", "https://", "file://", "/") +# Base64 characters decoded to size an inline image, 48 KiB of image data. +# Arbitrary; an image whose header runs longer is decoded in full. +_INLINE_HEADER_CHARS = 1 << 16 + class SystemOneServing(OpenAIServingDecisions): - """Answers System One questions with the rendering, label checks, and scoring of /v1/decisions.""" + """Answers System One questions with the rendering, label checks, and scoring of + /v1/decisions, or, for a Clef checkpoint, with its joint schema head.""" route = "/v1/systemone" @@ -54,12 +76,36 @@ class SystemOneServing(OpenAIServingDecisions): return "systemone-" def _validate_request(self, request: SystemOneRequest) -> Optional[str]: + if self.joint_head_config is not None: + return self._validate_joint_schema_request(request) return ( self._validate_server(request.model) or self._validate_images(request.images) or self._validate_reasoning(request.chat_template_kwargs) ) + def _validate_joint_schema_request( + self, request: SystemOneRequest + ) -> Optional[str]: + """A Clef checkpoint renders its own prompt, without the chat template.""" + route = self.route + if self.tokenizer_manager.tokenizer is None: + return f"{route} requires the server tokenizer" + _, adapter = self._parse_model_parameter(request.model) + if adapter is not None: + return ( + f"model names the LoRA adapter {adapter!r}, which {route} " + "does not support" + ) + if request.images and self.tokenizer_manager.mm_processor is None: + return f"{route} images require a model served with multimodal input" + if request.chat_template_kwargs: + return ( + f"{route} renders the prompt this checkpoint's joint schema head " + "was trained with, which takes no chat_template_kwargs" + ) + return None + def _convert_to_internal_request( self, request: SystemOneRequest, @@ -69,6 +115,8 @@ class SystemOneServing(OpenAIServingDecisions): Tuple[SystemOneRequest, List[QuestionView]], ]: views = [_view(question) for question in request.questions.values()] + if self.joint_head_config is not None: + return self._joint_schema_request(request, views), (request, views) encode = ( self._encoded_systemone_questions if self.decision_config is None @@ -77,6 +125,118 @@ class SystemOneServing(OpenAIServingDecisions): # Lazy, so the async handler can yield to other requests between questions. return encode(request, views), (request, views) + def _joint_schema_request( + self, request: SystemOneRequest, views: List[QuestionView] + ) -> EmbeddingReqInput: + """All questions in one prompt, which the joint schema head scores in one prefill. + + The handler encodes the prompt once it has read the image sizes.""" + for question_id, view in zip(request.questions, views): + if view.kind == "score": + _check_legend(question_id, view) + return EmbeddingReqInput( + image_data=[ + ImageData(url=image.url, detail=image.detail or "auto") + for image in request.images + ] + or None, + ) + + async def _encode_joint_schema_prompt( + self, adapted_request: EmbeddingReqInput, request: SystemOneRequest + ) -> None: + tokenizer_manager = self.tokenizer_manager + max_length = min( + REFERENCE_MAX_LENGTH, + max_joint_prompt_tokens( + context_len=tokenizer_manager.context_len, + num_reserved_tokens=tokenizer_manager.num_reserved_tokens, + max_req_input_len=tokenizer_manager.max_req_input_len, + max_prefill_tokens=get_schedule().max_prefill_tokens, + ), + ) + image_token_counts = [] + if adapted_request.image_data: + adapted_request.image_data, image_token_counts = await self._read_images( + adapted_request.image_data + ) + input_ids, layout = encode_joint_schema( + tokenizer_manager.tokenizer, + request, + max_length=max_length, + image_token_counts=image_token_counts, + ) + adapted_request.input_ids = input_ids + adapted_request.decision_layout = layout + + async def _read_images( + self, images: List[ImageData] + ) -> Tuple[List[ImageData], List[int]]: + """Each image read once, and the tokens it expands to at the size the + processor resizes it to.""" + read = await asyncio.gather( + *( + asyncio.to_thread(_read_image, index, image) + for index, image in enumerate(images) + ) + ) + counts = self.tokenizer_manager.mm_processor.resolve_image_token_counts( + [header for _, header in read] + ) + return [image for image, _ in read], counts + + async def _answer_joint_schema( + self, + adapted_request: EmbeddingReqInput, + request: SystemOneRequest, + views: List[QuestionView], + raw_request: Request, + ) -> ORJSONResponse: + await self._encode_joint_schema_prompt(adapted_request, request) + ret = await self.tokenizer_manager.generate_request( + adapted_request, raw_request + ).__anext__() + logits = ret["embedding"] + names = [ + [name for name, _ in joint_schema_options(question)] + for question in request.questions.values() + ] + if len(logits) != sum(map(len, names)): + raise RuntimeError("the joint schema head did not score this request") + answers = {} + offset = 0 + for question_id, view, question_names in zip(request.questions, views, names): + scores = logits[offset : offset + len(question_names)] + offset += len(question_names) + top = max(scores) + weights = [math.exp(score - top) for score in scores] + total = math.fsum(weights) + by_name = { + name: weight / total for name, weight in zip(question_names, weights) + } + # The head scores true then false, which the view names yes and no. + answers[question_id] = _answer( + view=view, + probabilities=( + [by_name["true"], by_name["false"]] + if view.kind == "yes_no" + else [by_name[name] for name in view.names] + ), + mass=1.0, + question_id=question_id, + ) + response = SystemOneResponse( + model=self.tokenizer_manager.served_model_name, + answers=answers, + usage=SystemOneUsage(input_tokens=ret["meta_info"]["prompt_tokens"]), + ) + # The head scores only the allowed options, so there is no label mass. + return ORJSONResponse( + content=response.model_dump( + exclude={"answers": {"__all__": {"x_label_mass"}}} + ) + ) + def _encoded_systemone_questions( self, request: SystemOneRequest, views: List[QuestionView] ) -> Iterator[EncodedQuestion]: @@ -179,6 +339,10 @@ class SystemOneServing(OpenAIServingDecisions): raw_request: Request, ) -> ORJSONResponse: request, views = processed + if self.joint_head_config is not None: + return await self._answer_joint_schema( + adapted_request, request, views, raw_request + ) _, _, result = await self._score( adapted_request=adapted_request, raw_request=raw_request, @@ -344,3 +508,38 @@ def _score_confidence(q: List[float]) -> float: spread = math.fsum(p * abs(i - top) for i, p in enumerate(q)) uniform_spread = math.fsum(abs(i - (n - 1) / 2) for i in range(n)) / n return max(0.0, 1 - spread / uniform_spread) + + +def _read_image(index: int, image: ImageData) -> Tuple[ImageData, Image.Image]: + """The image to forward, and its header opened only as far as its size, read + once from the sources load_image reads.""" + url = image.url + if not url.startswith(_MUTABLE_SOURCES): + # An inline image is forwarded as sent, and the processor decodes all of it. + header = _inline_header(url) + if header is not None: + return image, header + source = unquote(urlparse(url).path) if url.startswith("file://") else url + try: + data = get_image_bytes(source) + header = Image.open(BytesIO(data)) + except CLIENT_MEDIA_EXCEPTIONS as e: + raise ValueError(f"image {index} could not be read: {e}") from e + if url.startswith(_MUTABLE_SOURCES): + # A source that can change between reads is forwarded as the bytes counted. + mime = Image.MIME.get(header.format, "application/octet-stream") + encoded = pybase64.b64encode(data).decode() + image = dataclasses.replace(image, url=f"data:{mime};base64,{encoded}") + return image, header + + +def _inline_header(url: str) -> Optional[Image.Image]: + """An inline image's header from a prefix of its base64, or None when the + header does not open from that prefix.""" + start = url.find(",") + 1 if url.startswith("data:") else 0 + prefix = url[start : start + _INLINE_HEADER_CHARS] + try: + return Image.open(BytesIO(pybase64.b64decode(prefix, validate=True))) + except Exception: + # A header past the prefix, or malformed data that the full read reports. + return None diff --git a/python/sglang/srt/layers/joint_schema_head.py b/python/sglang/srt/layers/joint_schema_head.py new file mode 100644 index 0000000000..0e939bdee1 --- /dev/null +++ b/python/sglang/srt/layers/joint_schema_head.py @@ -0,0 +1,539 @@ +"""Joint schema head of Clef decision checkpoints. + +A Clef checkpoint is a Qwen3.5 backbone saved with ``joint_head_config.json`` +and ``joint_head.safetensors``. The head reads the backbone's final hidden +state at every prompt token and scores every allowed option of every question +together: each option gathers evidence from the prompt, the questions attend +to each other and to the prompt, and a lexical prior built from the LM head +rows of each option's tokens keeps the option's meaning. + +One head pass serves a whole prefill batch. Token-wise steps run on the flat +batch, attention is variable-length per request, and span means come from +per-request prefix sums over the schema region only, so a long state costs no +more than its memory projection. +""" + +from __future__ import annotations + +import math +from typing import Callable, List, Optional, Sequence, Tuple + +import msgspec +import torch +import torch.nn.functional as F +from torch import nn + +from sglang.srt.layers.pooler import EmbeddingPoolerOutput + +# Question types in the order of the head's type embedding. +QUESTION_TYPES = ("noul", "choice", "score") + +# Bounds of one head pass, so a large prefill batch is scored in a few passes. +_MAX_GROUP_TOKENS = 65536 +_MAX_GROUP_SLOTS = 1 << 18 + + +class LayoutQuestion(msgspec.Struct, frozen=True): + question_type: int + question_span: Tuple[int, int] + option_spans: Tuple[Tuple[int, int], ...] + + +def max_joint_prompt_tokens( + *, + context_len: int, + num_reserved_tokens: int, + max_req_input_len: int, + max_prefill_tokens: int, +) -> int: + """The longest prompt, images expanded, that the server runs as one prefill. + + The tokenizer manager refuses a prompt of context_len - num_reserved_tokens + tokens, the scheduler one of max_req_input_len, and without chunked prefill + activation memory is reserved for max_prefill_tokens. + """ + return min( + context_len - num_reserved_tokens - 1, + max_req_input_len - 1, + max_prefill_tokens, + ) + + +def pack_decision_layout( + prompt_length: int, questions: Sequence[LayoutQuestion] +) -> List[int]: + """Flatten a prompt's questions into the ints a request carries. + + The layout is ``[prompt_length, num_questions]`` followed, per question, + by ``type, question_start, question_end, num_options`` and one + ``option_start, option_end`` pair per option. Positions index the prompt + before its image placeholders expand. + """ + layout = [prompt_length, len(questions)] + for question in questions: + layout += [ + question.question_type, + *question.question_span, + len(question.option_spans), + ] + for start, end in question.option_spans: + layout += [start, end] + return layout + + +def parse_decision_layout( + layout: Sequence[int], num_tokens: int +) -> List[LayoutQuestion]: + """The questions of a layout, placed on the sequence the model reads. + + Every span follows the prompt's images, so expanding their placeholders + shifts all spans by the number of added tokens. Raises ValueError when the + layout does not describe the sequence. + """ + if len(layout) < 2 or any(type(value) is not int for value in layout): + raise ValueError("decision_layout must be a list of ints") + prompt_length, num_questions = layout[0], layout[1] + shift = num_tokens - prompt_length + if prompt_length < 1 or shift < 0 or num_questions < 1: + raise ValueError("decision_layout does not match the prompt length") + questions = [] + cursor = 2 + for _ in range(num_questions): + if cursor + 4 > len(layout): + raise ValueError("decision_layout is truncated") + question_type, start, end, num_options = layout[cursor : cursor + 4] + cursor += 4 + if ( + not 0 <= question_type < len(QUESTION_TYPES) + or num_options < 1 + or not 0 <= start < end <= prompt_length + or cursor + 2 * num_options > len(layout) + ): + raise ValueError("decision_layout has an invalid question") + spans = [] + for _ in range(num_options): + option_start, option_end = layout[cursor : cursor + 2] + cursor += 2 + if not 0 <= option_start < option_end <= prompt_length: + raise ValueError("decision_layout has an invalid option span") + spans.append((option_start + shift, option_end + shift)) + questions.append( + LayoutQuestion(question_type, (start + shift, end + shift), tuple(spans)) + ) + if cursor != len(layout): + raise ValueError("decision_layout has trailing values") + return questions + + +class EvidenceRoutingLayer(nn.Module): + def __init__(self, width: int, heads: int, feedforward: int) -> None: + super().__init__() + self.query_norm = nn.LayerNorm(width) + self.memory_norm = nn.LayerNorm(width) + self.attention = nn.MultiheadAttention(width, heads, batch_first=True) + self.feedforward_norm = nn.LayerNorm(width) + # Identity slots keep the checkpoint's Sequential indices (dropout at 2 and 4). + self.feedforward = nn.Sequential( + nn.Linear(width, feedforward), + nn.GELU(), + nn.Identity(), + nn.Linear(feedforward, width), + nn.Identity(), + ) + + +class JointSchemaHead(nn.Module): + """Parameters named as in the checkpoint's ``joint_head.safetensors``.""" + + def __init__( + self, + hidden_size: int, + width: int, + routing_layers: int, + layers: int, + heads: int, + feedforward: int, + ) -> None: + super().__init__() + self.hidden_norm = nn.LayerNorm(hidden_size) + self.memory_projection = nn.Linear(hidden_size, width, bias=False) + self.question_projection = nn.Linear(hidden_size, width, bias=False) + self.option_question_projection = nn.Linear(hidden_size, width, bias=False) + self.global_projection = nn.Linear(hidden_size, width, bias=False) + self.option_context_projection = nn.Linear(hidden_size, width, bias=False) + self.option_lexical_projection = nn.Linear(hidden_size, width, bias=False) + self.type_embedding = nn.Embedding(len(QUESTION_TYPES), width) + self.evidence_layers = nn.ModuleList( + EvidenceRoutingLayer(width, heads, feedforward) + for _ in range(routing_layers) + ) + self.option_summary_norm = nn.LayerNorm(width) + self.layers = nn.ModuleList( + nn.TransformerDecoderLayer( + d_model=width, + nhead=heads, + dim_feedforward=feedforward, + dropout=0.0, + activation="gelu", + batch_first=True, + norm_first=True, + ) + for _ in range(layers) + ) + self.field_norm = nn.LayerNorm(width) + self.option_norm = nn.LayerNorm(width) + self.residual_scorer = nn.Sequential( + nn.Linear(width * 4, width), nn.GELU(), nn.Identity(), nn.Linear(width, 1) + ) + self.prior_logit_scale = nn.Parameter(torch.zeros(())) + self.joint_logit_scale = nn.Parameter(torch.zeros(())) + self.residual_gate = nn.Parameter(torch.zeros(())) + + @torch.no_grad() + def forward_batch( + self, + hidden: torch.Tensor, + input_ids: torch.Tensor, + lengths: List[int], + layouts: List[List[LayoutQuestion]], + output_embedding_weight: torch.Tensor, + ) -> List[torch.Tensor]: + """One FP32 logit vector per request, its questions' options in layout order. + + ``hidden`` holds the final hidden states of the requests back to back. + """ + outputs: List[torch.Tensor] = [] + start = 0 + for group in _groups(lengths, layouts): + group_lengths = [lengths[i] for i in group] + end = start + sum(group_lengths) + outputs += self._forward_group( + hidden[start:end], + input_ids[start:end], + group_lengths, + [layouts[i] for i in group], + output_embedding_weight, + ) + start = end + return outputs + + def _forward_group( + self, + hidden: torch.Tensor, + input_ids: torch.Tensor, + lengths: List[int], + layouts: List[List[LayoutQuestion]], + output_embedding_weight: torch.Tensor, + ) -> List[torch.Tensor]: + device, dtype = hidden.device, hidden.dtype + batch = len(layouts) + question_bounds, option_bounds = [], [] + type_ids, owner, slot, question_request, last_tokens = [], [], [], [], [] + regions, region_offsets = [], [0] + token_cu, question_cu, option_cu, option_counts = [0], [0], [0], [] + for request, (length, questions) in enumerate(zip(lengths, layouts)): + offset = token_cu[-1] + low = min(min(q.question_span[0], q.option_spans[0][0]) for q in questions) + high = max( + max(q.question_span[1], *(e for _, e in q.option_spans)) + for q in questions + ) + # Prefix row of span position p in this request's schema region. + base = region_offsets[-1] + request - low + regions.append((offset + low, offset + high)) + request_options = 0 + for question in questions: + count = len(question.option_spans) + question_bounds += ( + base + question.question_span[0], + base + question.question_span[1], + ) + for option_slot, (option_start, option_end) in enumerate( + question.option_spans + ): + option_bounds += (base + option_start, base + option_end) + owner.append(len(type_ids)) + slot.append(option_slot) + type_ids.append(question.question_type) + question_request.append(request) + request_options += count + last_tokens.append(offset + length - 1) + token_cu.append(offset + length) + region_offsets.append(region_offsets[-1] + high - low) + question_cu.append(len(type_ids)) + option_cu.append(option_cu[-1] + request_options) + option_counts.append(request_options) + + num_questions = len(type_ids) + widest = max(len(q.option_spans) for questions in layouts for q in questions) + sizes = [ + 2 * num_questions, + len(option_bounds), + num_questions, + len(owner), + len(slot), + num_questions, + batch, + 3 * (batch + 1), + ] + packed = torch.tensor( + question_bounds + + option_bounds + + type_ids + + owner + + slot + + question_request + + last_tokens + + token_cu + + question_cu + + option_cu, + dtype=torch.long, + ).to(device, non_blocking=True) + ( + question_bounds_t, + option_bounds_t, + type_ids_t, + owner_t, + slot_t, + question_request_t, + last_tokens_t, + cu_t, + ) = torch.split(packed, sizes) + question_bounds_t = question_bounds_t.view(-1, 2) + option_bounds_t = option_bounds_t.view(-1, 2) + token_cu_t, question_cu_t, option_cu_t = ( + cu_t.to(torch.int32).view(3, batch + 1).unbind(0) + ) + token_bounds = (token_cu_t, max(lengths)) + question_bounds_cu = (question_cu_t, max(len(q) for q in layouts)) + option_bounds_cu = (option_cu_t, max(option_counts)) + + normalized = self.hidden_norm(hidden) + # Running sums over each schema region, after one zero row per request. + region_total = region_offsets[-1] + batch + hidden_prefix = torch.zeros( + region_total, normalized.shape[-1], dtype=torch.float32, device=device + ) + lexical_prefix = torch.zeros( + region_total, + output_embedding_weight.shape[-1], + dtype=torch.float32, + device=device, + ) + for request, (low, high) in enumerate(regions): + row = region_offsets[request] + request + 1 + torch.cumsum( + normalized[low:high].float(), + dim=0, + out=hidden_prefix[row : row + high - low], + ) + torch.cumsum( + output_embedding_weight[input_ids[low:high]].float(), + dim=0, + out=lexical_prefix[row : row + high - low], + ) + + def span_means(prefix: torch.Tensor, bounds: torch.Tensor) -> torch.Tensor: + sums = prefix[bounds[:, 1]] - prefix[bounds[:, 0]] + return (sums / (bounds[:, 1] - bounds[:, 0]).unsqueeze(-1)).to(dtype) + + memory = self.memory_projection(normalized) + global_vectors = normalized[last_tokens_t] + question_vectors = span_means(hidden_prefix, question_bounds_t) + option_contexts = span_means(hidden_prefix, option_bounds_t) + lexical = span_means(lexical_prefix, option_bounds_t) + + routed = ( + self.option_context_projection(option_contexts) + + self.option_lexical_projection(lexical) + + self.option_question_projection(question_vectors)[owner_t] + ) + for layer in self.evidence_layers: + routed = routed + _attention( + layer.attention, + layer.query_norm(routed), + layer.memory_norm(memory), + option_bounds_cu, + token_bounds, + ) + routed = routed + layer.feedforward(layer.feedforward_norm(routed)) + + base_fields = self.question_projection(question_vectors) + scores = torch.matmul( + routed.unsqueeze(1), base_fields[owner_t].unsqueeze(2) + ).view(-1) / math.sqrt(routed.shape[-1]) + padded_scores = torch.full( + (num_questions, widest), float("-inf"), dtype=torch.float32, device=device + ) + padded_scores[owner_t, slot_t] = scores.float() + weights = torch.softmax(padded_scores, dim=1).to(dtype) + padded_options = routed.new_zeros(num_questions, widest, routed.shape[-1]) + padded_options[owner_t, slot_t] = routed + summaries = torch.sum(weights.unsqueeze(-1) * padded_options, dim=1) + fields = ( + base_fields + + self.option_summary_norm(summaries) + + self.global_projection(global_vectors)[question_request_t] + + self.type_embedding(type_ids_t) + ) + for layer in self.layers: + fields = fields + _attention( + layer.self_attn, + layer.norm1(fields), + None, + question_bounds_cu, + question_bounds_cu, + ) + fields = fields + _attention( + layer.multihead_attn, + layer.norm2(fields), + memory, + question_bounds_cu, + token_bounds, + ) + fields = fields + layer.linear2( + layer.activation(layer.linear1(layer.norm3(fields))) + ) + fields = self.field_norm(fields) + + prior_scale = self.prior_logit_scale.clamp(max=math.log(100.0)).exp() + joint_scale = self.joint_logit_scale.clamp(max=math.log(100.0)).exp() + anchor = F.normalize( + question_vectors + global_vectors[question_request_t], dim=-1 + ) + prior = prior_scale * torch.matmul( + F.normalize(lexical, dim=-1).unsqueeze(1), anchor[owner_t].unsqueeze(2) + ).view(-1) + options = self.option_norm(routed) + repeated = fields[owner_t] + cosine = F.cosine_similarity(repeated, options, dim=-1) + features = torch.cat( + [repeated, options, repeated * options, torch.abs(repeated - options)], + dim=-1, + ) + residual = self.residual_scorer(features).squeeze(-1) + logits = prior + torch.sigmoid(self.residual_gate) * ( + joint_scale * cosine + residual + ) + return list(torch.split(logits.float(), option_counts)) + + +def _attention( + module: nn.MultiheadAttention, + queries: torch.Tensor, + keys: Optional[torch.Tensor], + query_bounds: Tuple[torch.Tensor, int], + key_bounds: Tuple[torch.Tensor, int], +) -> torch.Tensor: + """nn.MultiheadAttention over packed sequences, keys and values from ``keys`` + (or from ``queries`` when None), attending within each request.""" + width = queries.shape[-1] + heads = module.num_heads + if keys is None: + query, key, value = F.linear( + queries, module.in_proj_weight, module.in_proj_bias + ).chunk(3, dim=-1) + else: + query_weight, key_value_weight = module.in_proj_weight.split([width, 2 * width]) + query_bias, key_value_bias = module.in_proj_bias.split([width, 2 * width]) + query = F.linear(queries, query_weight, query_bias) + key, value = F.linear(keys, key_value_weight, key_value_bias).chunk(2, dim=-1) + head_dim = width // heads + output = torch.ops.aten._flash_attention_forward( + query.reshape(-1, heads, head_dim), + key.reshape(-1, heads, head_dim), + value.reshape(-1, heads, head_dim), + query_bounds[0], + key_bounds[0], + query_bounds[1], + key_bounds[1], + 0.0, + False, + False, + )[0] + return module.out_proj(output.reshape(-1, width)) + + +def _groups(lengths: List[int], layouts: List[List[LayoutQuestion]]) -> List[range]: + """Consecutive requests per head pass, bounded in tokens and padded option slots.""" + groups, start, tokens, questions, widest = [], 0, 0, 0, 0 + for index, (length, layout) in enumerate(zip(lengths, layouts)): + layout_widest = max(len(question.option_spans) for question in layout) + grown = (questions + len(layout)) * max(widest, layout_widest) + if index > start and ( + tokens + length > _MAX_GROUP_TOKENS or grown > _MAX_GROUP_SLOTS + ): + groups.append(range(start, index)) + start, tokens, questions, widest = index, 0, 0, 0 + tokens += length + questions += len(layout) + widest = max(widest, layout_widest) + groups.append(range(start, len(lengths))) + return groups + + +class JointSchemaPooler(nn.Module): + """Pooler of a Clef checkpoint: the joint schema head's option logits per request.""" + + def __init__( + self, head: JointSchemaHead, output_embedding: Callable[[], torch.Tensor] + ) -> None: + super().__init__() + self.head = head + # A callable, so the LM head is not registered twice as a parameter. + self._output_embedding = output_embedding + + def forward( + self, hidden_states: torch.Tensor, forward_batch + ) -> EmbeddingPoolerOutput: + if ( + forward_batch.decision_layouts is None + or forward_batch.extend_seq_lens_cpu is None + ): + return EmbeddingPoolerOutput( + embeddings=[ + hidden_states.new_empty(0, dtype=torch.float32) + for _ in range(forward_batch.batch_size) + ] + ) + lengths = list(forward_batch.extend_seq_lens_cpu) + prefix_lengths = forward_batch.extend_prefix_lens_cpu or [0] * len(lengths) + raw_layouts = forward_batch.decision_layouts + embeddings: List[torch.Tensor] = [ + hidden_states.new_empty(0, dtype=torch.float32) for _ in lengths + ] + # The tokenizer manager checks every layout against its prompt, and the + # server runs each prompt in one uncached prefill. A layout that no longer + # fits its prompt, as after a truncation, gets no scores, which the + # decision route reports as a server error. + layouts = {} + for i, (layout, prefix, length) in enumerate( + zip(raw_layouts, prefix_lengths, lengths) + ): + if layout is None or prefix != 0: + continue + try: + layouts[i] = parse_decision_layout(layout, length) + except ValueError: + continue + offsets = [0] + for length in lengths: + offsets.append(offsets[-1] + length) + runs: List[List[int]] = [] + for i in layouts: + if runs and runs[-1][-1] == i - 1: + runs[-1].append(i) + else: + runs.append([i]) + for run in runs: + start, end = offsets[run[0]], offsets[run[-1] + 1] + logits = self.head.forward_batch( + hidden_states[start:end], + forward_batch.input_ids[start:end], + [lengths[i] for i in run], + [layouts[i] for i in run], + self._output_embedding(), + ) + for i, values in zip(run, logits): + embeddings[i] = values + return EmbeddingPoolerOutput(embeddings=embeddings) diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index 3ba6045ff6..028226a683 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -1253,6 +1253,11 @@ class EmbeddingReqInput: # Batch-level: List[List[int]] (one per request). After __getitem__: List[int]. token_indices_to_pool: Optional[Union[List[List[int]], List[int]]] = None + # Question and option spans that a Clef checkpoint's joint schema head + # scores, as packed by layers.joint_schema_head.pack_decision_layout. + # Batch-level: List[List[int]] (one per request). After __getitem__: List[int]. + decision_layout: Optional[Union[List[List[int]], List[int]]] = None + def regenerate_rid(self): """Generate a new request ID and return it.""" if isinstance(self.rid, list): @@ -1469,6 +1474,11 @@ class EmbeddingReqInput: if self.token_indices_to_pool is not None else None ), + decision_layout=( + self.decision_layout[i] + if self.decision_layout is not None + else None + ), ) cache[i] = sub return sub @@ -1500,6 +1510,8 @@ class TokenizedEmbeddingReqInput(BaseReq, kw_only=True): multi_item_delimiter_indices: Optional[List[int]] = None # Token positions for setwise pooling readout token_indices_to_pool: Optional[List[int]] = None + # Question and option spans for a joint schema head + decision_layout: Optional[List[int]] = None # For observability # Pickled Optional[Union[APIServerReqTimeStats, DPControllerReqTimeStats]] time_stats: Optional[PickleWrapper] = None diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 8346c93c3d..a964f40505 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1050,6 +1050,7 @@ class Req(ReqDllmMixin): return_pooled_hidden_states: bool = False, multi_item_delimiter_indices: Optional[List[int]] = None, token_indices_to_pool: Optional[List[int]] = None, + decision_layout: Optional[List[int]] = None, session_id: Optional[str] = None, cache_salt: Optional[str] = None, ): @@ -1083,6 +1084,7 @@ class Req(ReqDllmMixin): self.positional_embed_overrides = positional_embed_overrides self.multi_item_delimiter_indices = multi_item_delimiter_indices self.token_indices_to_pool = token_indices_to_pool + self.decision_layout = decision_layout # For req-level memory management self.kv = ReqKvInfo() diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 4622e32faf..1a3f60b857 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -3465,6 +3465,7 @@ class Scheduler( return_pooled_hidden_states=recv_req.return_pooled_hidden_states, multi_item_delimiter_indices=recv_req.multi_item_delimiter_indices, token_indices_to_pool=recv_req.token_indices_to_pool, + decision_layout=recv_req.decision_layout, ) req.tokenizer = self.tokenizer self._maybe_namespace_elastic_radix_cache(req) diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index b3975bf3f8..aebe9ea6b8 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -64,6 +64,10 @@ from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX from sglang.srt.disaggregation.encoder.receiver import create_mm_receiver from sglang.srt.disaggregation.utils import DisaggregationMode from sglang.srt.environ import envs +from sglang.srt.layers.joint_schema_head import ( + max_joint_prompt_tokens, + parse_decision_layout, +) from sglang.srt.lora.lora_registry import LoRARef, LoRARegistry from sglang.srt.managers.async_dynamic_batch_tokenizer import AsyncDynamicbatchTokenizer from sglang.srt.managers.disagg_service import start_disagg_service @@ -1295,6 +1299,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): raise ValueError( "encoder SWA replay cannot return cached prompt logprobs" ) + # A decision layout indexes the whole prompt, so it is checked before any + # truncation below could cut the prompt. + self._validate_joint_schema_request(obj, input_ids) _max_req_len = self.context_len input_token_num = len(input_ids) if input_ids is not None else 0 input_token_num += self.num_reserved_tokens @@ -1399,6 +1406,36 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): "return_sampling_mask=true." ) + def _validate_joint_schema_request( + self, obj: Union[GenerateReqInput, EmbeddingReqInput], input_ids: List[int] + ) -> None: + """A Clef checkpoint scores the spans of a decision layout, and nothing else.""" + if self.model_config.joint_head_config is None: + return + if isinstance(obj, GenerateReqInput) or obj.decision_layout is None: + # Health checks only need a response, which is an empty embedding. + if ( + isinstance(obj, EmbeddingReqInput) + and isinstance(obj.rid, str) + and obj.rid.startswith(HEALTH_CHECK_RID_PREFIX) + ): + return + raise ValueError( + "This checkpoint answers schema decisions, send them to /v1/systemone" + ) + limit = max_joint_prompt_tokens( + context_len=self.context_len, + num_reserved_tokens=self.num_reserved_tokens, + max_req_input_len=self.max_req_input_len, + max_prefill_tokens=get_schedule().max_prefill_tokens, + ) + if len(input_ids) > limit: + raise ValueError( + f"The prompt has {len(input_ids)} tokens, but this server scores " + f"at most {limit} in one prefill" + ) + parse_decision_layout(obj.decision_layout, len(input_ids)) + def _validate_mm_limits( self, obj: Union[GenerateReqInput, EmbeddingReqInput] ) -> None: @@ -1575,6 +1612,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): return_pooled_hidden_states=obj.return_pooled_hidden_states, multi_item_delimiter_indices=obj.multi_item_delimiter_indices, token_indices_to_pool=obj.token_indices_to_pool, + decision_layout=obj.decision_layout, ) tokenized_obj.time_stats = self.rid_to_state[obj.rid].time_stats diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index a7cbd93559..1efa971f40 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -614,6 +614,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # Setwise pooling readout positions (CPU tensors, one per request) token_indices_to_pool: Optional[List[torch.Tensor]] = None + # Joint schema head decision layouts (one per request, None if absent) + decision_layouts: Optional[List[Optional[List[int]]]] = None + # === Borrowed from ScheduleBatch: compound (carry their own device tensors) === # Sampling info sampling_info: SamplingBatchInfo = None @@ -1264,6 +1267,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): for r in batch.reqs ] + if any(r.decision_layout is not None for r in batch.reqs): + self.decision_layouts = [r.decision_layout for r in batch.reqs] + token_type_ids = [ r.token_type_ids for r in batch.reqs if r.token_type_ids is not None ] diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 81bdfbb56f..1f1e30dd52 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -34,7 +34,10 @@ from sglang.kernels.ops.attention.triton_gdn_fused_proj import ( from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul # Configs -from sglang.srt.configs.model_config import load_decision_config +from sglang.srt.configs.model_config import ( + load_decision_config, + load_joint_head_config, +) from sglang.srt.configs.qwen3_5 import ( Qwen3_5Config, Qwen3_5MoeConfig, @@ -54,6 +57,7 @@ from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion +from sglang.srt.layers.joint_schema_head import JointSchemaHead, JointSchemaPooler from sglang.srt.layers.layer_boundary import ( append_stages, declare_attn, @@ -2330,6 +2334,16 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration): self.visual.deepstack_visual_indexes if self.visual is not None else [] ) + if config.architectures == ["Qwen3_5ForConditionalGeneration"]: + model = get_model() + joint_head_config = load_joint_head_config(model.model_path, model.revision) + if joint_head_config is not None: + # A Clef checkpoint, whose joint schema head scores every option + # of every question from the final hidden states and LM head rows. + self.pooler = JointSchemaPooler( + JointSchemaHead(**joint_head_config), lambda: self.lm_head.weight + ) + def get_hidden_dim(self, module_name: str, layer_idx: int): return self.model.get_hidden_dim(module_name, layer_idx) @@ -2470,8 +2484,23 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration): if bare_backbone and self.pp_group.is_last_rank: self._load_decision_readout() loaded_params.add("lm_head.weight") + if isinstance(self.pooler, JointSchemaPooler): + self._load_joint_head() + loaded_params.update( + f"pooler.head.{name}" for name, _ in self.pooler.head.named_parameters() + ) return loaded_params + def _load_joint_head(self) -> None: + """Load joint_head.safetensors, which the checkpoint index leaves out.""" + model = get_model() + state_dict = load_file( + cached_file( + model.model_path, "joint_head.safetensors", revision=model.revision + ) + ) + self.pooler.head.load_state_dict(state_dict, strict=True) + def _load_decision_readout(self) -> None: """Place the checkpoint's decision readout in the LM head rows of its codes.""" model = get_model() diff --git a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py index 035cd667a0..5e9c92d103 100644 --- a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py +++ b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py @@ -169,6 +169,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase): args = ServerArgs(model_path="dummy") args._model_config = SimpleNamespace( is_embedding_gemma=True, + joint_head_config=None, is_multimodal=False, context_len=2048, hf_config=SimpleNamespace(architectures=["Gemma3TextModel"]), @@ -205,6 +206,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase): is_embedding_requested=False, is_embedding_gemma=False, ), + joint_head_config=None, is_multimodal=False, hf_config=SimpleNamespace(architectures=["BertModel"]), ) diff --git a/test/registered/unit/entrypoints/openai/test_serving_decisions.py b/test/registered/unit/entrypoints/openai/test_serving_decisions.py index 9b6dcf8bad..a561a04431 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_decisions.py +++ b/test/registered/unit/entrypoints/openai/test_serving_decisions.py @@ -159,6 +159,7 @@ class ScoringManager(TokenizerManagerScoreMixin): model_type="qwen3_5_moe", ), decision_config=None, + joint_head_config=None, ) pad, video = ( tokenizer.encode(token, add_special_tokens=False)[0] diff --git a/test/registered/unit/entrypoints/systemone/test_joint_schema.py b/test/registered/unit/entrypoints/systemone/test_joint_schema.py new file mode 100644 index 0000000000..3d03aada1d --- /dev/null +++ b/test/registered/unit/entrypoints/systemone/test_joint_schema.py @@ -0,0 +1,474 @@ +"""Unit tests for /v1/systemone on Clef checkpoints: the joint schema prompt and its +decision layout, and answers built from the joint schema head's option logits.""" + +import asyncio +import base64 +import json +import math +import random +import unittest +from io import BytesIO +from types import SimpleNamespace +from unittest import mock + +import pybase64 +from PIL import Image +from transformers import AutoTokenizer + +from sglang.srt.entrypoints import http_server +from sglang.srt.entrypoints.openai.protocol import DecisionRequest +from sglang.srt.entrypoints.openai.serving_decisions import OpenAIServingDecisions +from sglang.srt.entrypoints.systemone.joint_schema import encode_joint_schema +from sglang.srt.entrypoints.systemone.protocol import SystemOneRequest +from sglang.srt.entrypoints.systemone.serving import SystemOneServing, _read_image +from sglang.srt.layers.joint_schema_head import ( + LayoutQuestion, + pack_decision_layout, + parse_decision_layout, +) +from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput +from sglang.srt.managers.tokenizer_manager import TokenizerManager +from sglang.srt.runtime_context import ( + get_context, + get_schedule, + publish, + restore_context, + snapshot_context, +) +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import ImageData +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + +# Tokenizer files only, with the Qwen3.5 vocabulary of Clef checkpoints. +TOKENIZER = "Qwen/Qwen3.5-35B-A3B" + +REQUEST = { + "model": "clef", + "state": {"ticket": "Refund for order 1182", "amount": 42}, + "questions": { + "team": { + "type": "choice", + "instructions": "Which team handles it?", + "criteria": { + "support": "Customer support", + "billing": "Payments and refunds", + "legal": None, + }, + }, + "urgent": { + "type": "noul", + "instructions": "Is it urgent?", + "criteria": {"false": None}, + }, + "severity": {"type": "score", "criteria": ["none", "low", "high"]}, + }, +} + + +class PatchCounter: + """Stands in for the multimodal processor: one image token per 28x28 patch.""" + + @staticmethod + def resolve_image_token_counts(images): + return [(image.height // 28) * (image.width // 28) for image in images] + + +def _png_bytes(width, height): + buffer = BytesIO() + Image.new("RGB", (width, height), "red").save(buffer, format="PNG") + return buffer.getvalue() + + +def _png(width, height): + return ( + "data:image/png;base64," + base64.b64encode(_png_bytes(width, height)).decode() + ) + + +def _expanded_length(request): + images = [ + Image.open(BytesIO(base64.b64decode(image.url.split(",", 1)[1]))) + for image in request.image_data or [] + ] + counts = PatchCounter.resolve_image_token_counts(images) + return len(request.input_ids) + sum(count - 1 for count in counts) + + +class HeadManager: + """Replace model execution with option logits 0, 1, 2, ... per question, in head order, + after expanding images and refusing prompts that the tokenizer manager, the + scheduler, or the memory reserved for one prefill cannot take.""" + + def __init__(self, tokenizer): + self.server_args = ServerArgs(model_path="dummy") + publish(self.server_args, role="test") + self.tokenizer = tokenizer + self.model_config = SimpleNamespace( + is_multimodal=True, + hf_config=SimpleNamespace( + architectures=["Qwen3_5ForConditionalGeneration"], model_type="qwen3_5" + ), + decision_config=None, + joint_head_config={"hidden_size": 4096}, + ) + self.is_generation = False + self.allow_auto_truncate = False + self.context_len = 4096 + self.num_reserved_tokens = 0 + # min(context_len - 1, kv_capacity - 1) - 5, as the scheduler reports it. + self.max_req_input_len = 4090 + self.mm_processor = PatchCounter() + self.request_logger = SimpleNamespace(log_requests=False) + self.served_model_name = "served-model" + self.requests = [] + + def config_value(self, name): + return None + + async def generate_request(self, request, raw_request): + self.requests.append(request) + num_tokens = _expanded_length(request) + if num_tokens + self.num_reserved_tokens >= self.context_len: + raise ValueError(f"The input ({num_tokens} tokens) is too long") + if num_tokens >= self.max_req_input_len: + raise ValueError(f"The scheduler refuses a prompt of {num_tokens} tokens") + if num_tokens > get_schedule().max_prefill_tokens: + raise ValueError(f"A prefill of {num_tokens} tokens outgrows its memory") + questions = parse_decision_layout(request.decision_layout, num_tokens) + logits = [float(i) for q in questions for i in range(len(q.option_spans))] + yield { + "embedding": logits, + "meta_info": {"prompt_tokens": num_tokens}, + } + + +def _handler(manager, serving_class=SystemOneServing): + template_manager = SimpleNamespace( + chat_template_name=None, + jinja_template_content_format="openai", + reasoning_config=None, + suggested_reasoning_parser=None, + ) + chat_serving = SimpleNamespace( + tokenizer_manager=manager, + template_manager=template_manager, + default_chat_template_kwargs={}, + chat_encoding_spec=None, + _prompt_text_round_trip_is_lossy=False, + reasoning_parser=None, + ) + return serving_class(chat_serving) + + +def _softmax(values): + weights = [math.exp(value - max(values)) for value in values] + return [weight / sum(weights) for weight in weights] + + +class TestJointSchemaPrompt(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tokenizer = AutoTokenizer.from_pretrained(TOKENIZER) + + def _encode(self, request=REQUEST, max_length=4096): + input_ids, layout = encode_joint_schema( + self.tokenizer, + SystemOneRequest(**request), + max_length=max_length, + image_token_counts=[], + ) + return input_ids, parse_decision_layout(layout, len(input_ids)) + + def _text(self, input_ids, span): + return self.tokenizer.decode(input_ids[span[0] : span[1]]) + + def test_prompt_is_the_trained_joint_schema_prompt(self): + input_ids, _ = self._encode() + prompt = self.tokenizer.decode(input_ids) + self.assertTrue( + prompt.startswith( + "<|im_start|>system\nRead the complete state and schema. Decide every " + "field jointly. Each answer must be exactly one of that field's allowed " + "options.<|im_end|>\n<|im_start|>user\nSTATE:\n" + '{"amount":42,"ticket":"Refund for order 1182"}\n\nSCHEMA FIELDS:\n' + "\nFIELD 1\nID: team\nTYPE: choice\nINSTRUCTION: Which team handles it?" + "\nALLOWED OPTIONS:\n" + 'OPTION 1: {"description":"Payments and refunds","option_id":"billing"}\n' + ) + ) + self.assertTrue( + prompt.endswith( + "END FIELD\n\n<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" + "JOINT SCHEMA DECISIONS:" + ) + ) + + def test_layout_spans_hold_questions_and_options_in_head_order(self): + input_ids, questions = self._encode() + self.assertEqual([q.question_type for q in questions], [1, 0, 2]) + self.assertEqual( + [self._text(input_ids, q.question_span) for q in questions], + ["Which team handles it?", "Is it urgent?", "severity"], + ) + options = [ + [json.loads(self._text(input_ids, span)) for span in q.option_spans] + for q in questions + ] + self.assertEqual( + options[0], + [ + {"description": "Payments and refunds", "option_id": "billing"}, + {"option_id": "legal"}, + {"description": "Customer support", "option_id": "support"}, + ], + ) + self.assertEqual( + options[1], + [ + { + "description": "The proposition is true or the answer is yes.", + "option_id": "true", + }, + {"option_id": "false"}, + ], + ) + self.assertEqual( + options[2], + [ + {"description": level, "option_id": str(index)} + for index, level in enumerate(["none", "low", "high"]) + ], + ) + + def test_state_is_truncated_to_fit(self): + input_ids, questions = self._encode() + shorter_ids, shorter = self._encode(max_length=len(input_ids) - 3) + self.assertEqual(len(shorter_ids), len(input_ids) - 3) + self.assertEqual( + [self._text(shorter_ids, q.question_span) for q in shorter], + [self._text(input_ids, q.question_span) for q in questions], + ) + with self.assertRaisesRegex(ValueError, "before the state"): + self._encode(max_length=100) + + def test_image_placeholders_precede_the_state(self): + request = {**REQUEST, "images": ["data:image/png;base64,AAAA"]} + input_ids, layout = encode_joint_schema( + self.tokenizer, + SystemOneRequest(**request), + max_length=4096, + image_token_counts=[100], + ) + self.assertIn( + "STATE:\n<|vision_start|><|image_pad|><|vision_end|>\n{", + self.tokenizer.decode(input_ids), + ) + unexpanded = parse_decision_layout(layout, len(input_ids)) + expanded = parse_decision_layout(layout, len(input_ids) + 99) + self.assertEqual( + [q.option_spans[0][0] + 99 for q in unexpanded], + [q.option_spans[0][0] for q in expanded], + ) + + +class TestJointSchemaAnswers(unittest.IsolatedAsyncioTestCase): + @classmethod + def setUpClass(cls): + cls.tokenizer = AutoTokenizer.from_pretrained(TOKENIZER) + + def setUp(self): + self.addCleanup(restore_context, snapshot_context()) + + async def test_answers_map_head_order_back_to_the_request(self): + manager = HeadManager(self.tokenizer) + response = await _handler(manager).handle_request( + SystemOneRequest(**REQUEST), None + ) + self.assertEqual(response.status_code, 200) + body = json.loads(response.body) + input_ids, layout = encode_joint_schema( + self.tokenizer, + SystemOneRequest(**REQUEST), + max_length=4089, + image_token_counts=[], + ) + (sent,) = manager.requests + self.assertEqual((sent.input_ids, sent.decision_layout), (input_ids, layout)) + self.assertIsNone(sent.image_data) + self.assertEqual(body["model"], "served-model") + self.assertEqual(body["usage"]["input_tokens"], len(input_ids)) + + three, two = _softmax([0.0, 1.0, 2.0]), _softmax([0.0, 1.0]) + team, urgent, severity = (body["answers"][q] for q in REQUEST["questions"]) + # The head scores billing, legal, support; answers keep the request order. + self.assertEqual(team["choice"], "support") + self.assertEqual(list(team["probabilities"]), ["support", "billing", "legal"]) + for name, expected in zip(["billing", "legal", "support"], three): + self.assertAlmostEqual(team["probabilities"][name], expected) + self.assertAlmostEqual(urgent["noul"], two[0]) + self.assertAlmostEqual(severity["score"], three[1] + 2 * three[2]) + self.assertEqual(severity["legend"], {"0": "none", "1": "low", "2": "high"}) + for answer in (team, urgent, severity): + self.assertNotIn("x_label_mass", answer) + + async def test_url_images_are_read_once_and_forwarded_as_counted(self): + """An image URL whose content changes between reads must be scored as the + image that was counted against the prompt budget.""" + manager = HeadManager(self.tokenizer) + url = "http://images.example/chart.png" + data = _png_bytes(224, 224) + image = {"url": url, "detail": "high"} + request = {**REQUEST, "state": "word " * 5000, "images": [image]} + target = "sglang.srt.entrypoints.systemone.serving.get_image_bytes" + with mock.patch(target, return_value=data) as fetch: + response = await _handler(manager).handle_request( + SystemOneRequest(**request), None + ) + fetch.assert_called_once_with(url) + (forwarded,) = manager.requests[0].image_data + self.assertTrue(forwarded.url.startswith("data:image/png;base64,")) + self.assertEqual(base64.b64decode(forwarded.url.split(",", 1)[1]), data) + self.assertEqual(forwarded.detail, "high") + self.assertEqual(response.status_code, 200) + self.assertEqual(json.loads(response.body)["usage"]["input_tokens"], 4089) + + async def test_refusals(self): + manager = HeadManager(self.tokenizer) + cases = { + "takes no chat_template_kwargs": ( + SystemOneServing, + SystemOneRequest(**REQUEST, chat_template_kwargs={"x": 1}), + ), + "LoRA adapter": ( + SystemOneServing, + SystemOneRequest(**{**REQUEST, "model": "clef:a"}), + ), + # Its 5329 image tokens leave no room for the questions. + "before the state": ( + SystemOneServing, + SystemOneRequest(**{**REQUEST, "images": [_png(2048, 2048)]}), + ), + "requires a generation model": ( + OpenAIServingDecisions, + DecisionRequest( + model="m", + input="x", + questions=[{"id": "q", "type": "yes_no", "question": "Q?"}], + ), + ), + } + for message, (serving_class, request) in cases.items(): + with self.subTest(message): + response = await _handler(manager, serving_class).handle_request( + request, None + ) + self.assertEqual(response.status_code, 400) + self.assertIn(message, json.loads(response.body)["message"]) + self.assertEqual(manager.requests, []) + + +def _admission_manager(joint_head_config): + manager = TokenizerManager.__new__(TokenizerManager) + manager.model_config = SimpleNamespace(joint_head_config=joint_head_config) + manager.is_generation = joint_head_config is None + manager.context_len = 4096 + manager.num_reserved_tokens = 0 + manager.max_req_input_len = 4090 + # A layout must be refused before auto-truncation could cut its prompt. + manager.allow_auto_truncate = True + manager.validate_total_tokens = False + manager._validate_token_ids_logprob = mock.Mock() + return manager + + +class TestJointSchemaAdmission(CustomTestCase): + def setUp(self): + override = get_context().override_server_args() + override.install() + self.addCleanup(override.restore) + + def test_tokenizer_refuses_generation_and_prompts_past_one_prefill(self): + clef = _admission_manager({"hidden_size": 4096}) + layout = [LayoutQuestion(0, (10, 12), ((13, 14), (15, 16)))] + + def embedding(length): + return EmbeddingReqInput( + input_ids=[1] * length, + decision_layout=pack_decision_layout(length, layout), + sampling_params={}, + ) + + generation = GenerateReqInput(input_ids=[1, 2, 3], sampling_params={}) + refused = { + "generation": (generation, "/v1/systemone"), + # 4095 tokens pass the context check, but the scheduler takes 4089. + "past one prefill": (embedding(4095), "at most 4089"), + } + for name, (obj, message) in refused.items(): + with self.subTest(name), self.assertRaisesRegex(ValueError, message): + clef._validate_one_request(obj, obj.input_ids) + clef._validate_one_request(embedding(4089), [1] * 4089) + # A server without a joint schema head keeps generating. + _admission_manager(None)._validate_one_request(generation, [1, 2, 3]) + + def test_client_decision_layouts_are_refused_at_encode_and_classify(self): + manager = SimpleNamespace(requests=[]) + + async def generate_request(obj, request): + manager.requests.append(obj) + yield {"embedding": [0.0]} + + manager.generate_request = generate_request + self.addCleanup( + setattr, http_server, "_global_state", http_server.get_global_state() + ) + http_server.set_global_state(SimpleNamespace(tokenizer_manager=manager)) + layout = pack_decision_layout(16, [LayoutQuestion(0, (0, 1), ((1, 2),))]) + responses = [ + asyncio.run( + handler( + EmbeddingReqInput( + input_ids=list(range(16)), decision_layout=layout + ), + None, + ) + ) + for handler in (http_server.encode_request, http_server.classify_request) + ] + self.assertEqual(manager.requests, []) + self.assertEqual([response.status_code for response in responses], [400, 400]) + + def test_inline_images_are_sized_from_a_header_prefix(self): + """Sizing an inline image must not decode its whole payload, and an image + whose header runs past the decoded prefix must still be sized.""" + noise = BytesIO() + pixels = random.Random(0).randbytes(256 * 192 * 3) + Image.frombytes("RGB", (256, 192), pixels).save(noise, format="PNG") + late = BytesIO() + # A 60 KB EXIF segment puts the JPEG frame header far into the payload. + exif = b"Exif\x00\x00" + bytes(60000) + Image.new("RGB", (64, 48)).save(late, format="JPEG", exif=exif) + cases = { + "png": ("image/png", noise.getvalue(), (256, 192)), + "late jpeg header": ("image/jpeg", late.getvalue(), (64, 48)), + } + for name, (mime, data, size) in cases.items(): + with self.subTest(name): + payload = base64.b64encode(data).decode() + image = ImageData(url=f"data:{mime};base64,{payload}") + with mock.patch("pybase64.b64decode", wraps=pybase64.b64decode) as read: + forwarded, header = _read_image(0, image) + self.assertIs(forwarded, image) + self.assertEqual(header.size, size) + decoded = [len(call.args[0]) for call in read.call_args_list] + if name == "png": + self.assertLess(max(decoded), len(payload)) + else: + self.assertEqual(decoded[-1], len(payload)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_joint_schema_head.py b/test/registered/unit/layers/test_joint_schema_head.py new file mode 100644 index 0000000000..a6fc11a28d --- /dev/null +++ b/test/registered/unit/layers/test_joint_schema_head.py @@ -0,0 +1,240 @@ +"""Unit tests for the joint schema head of Clef checkpoints: decision layouts, and +the batched forward against a per-request loop over the stock torch modules.""" + +import math +import unittest +from types import SimpleNamespace + +import torch +import torch.nn.functional as F + +from sglang.srt.layers.joint_schema_head import ( + JointSchemaHead, + JointSchemaPooler, + LayoutQuestion, + pack_decision_layout, + parse_decision_layout, +) +from sglang.test.ci.ci_register import register_cpu_ci, register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") +register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small") + + +def _questions(offset=0): + return [ + LayoutQuestion( + 1, + (10 + offset, 14 + offset), + ( + (16 + offset, 17 + offset), + (17 + offset, 19 + offset), + (19 + offset, 21 + offset), + ), + ), + LayoutQuestion( + 0, + (22 + offset, 25 + offset), + ((26 + offset, 28 + offset), (29 + offset, 31 + offset)), + ), + ] + + +class TestDecisionLayout(CustomTestCase): + def test_round_trip(self): + layout = pack_decision_layout(40, _questions()) + self.assertEqual(parse_decision_layout(layout, 40), _questions()) + + def test_expanded_images_shift_every_span(self): + layout = pack_decision_layout(40, _questions()) + self.assertEqual(parse_decision_layout(layout, 140), _questions(offset=100)) + + def test_rejects_layouts_that_do_not_fit_the_prompt(self): + layout = pack_decision_layout(40, _questions()) + invalid = { + "shorter prompt": (layout, 39), + "not ints": ([float(value) for value in layout], 40), + "truncated": (layout[:-1], 40), + "trailing values": (layout + [0], 40), + "no questions": ([40, 0], 40), + "empty span": ( + pack_decision_layout(40, [LayoutQuestion(0, (5, 5), ((6, 7),))]), + 40, + ), + "past the prompt": ( + pack_decision_layout(40, [LayoutQuestion(0, (5, 6), ((6, 41),))]), + 40, + ), + "unknown type": ( + pack_decision_layout(40, [LayoutQuestion(3, (5, 6), ((6, 7),))]), + 40, + ), + } + for name, (values, num_tokens) in invalid.items(): + with self.subTest(name), self.assertRaises(ValueError): + parse_decision_layout(values, num_tokens) + + +class _RecordingHead: + """Stands in for the head, scoring each option with its request's length.""" + + def __init__(self): + self.calls = [] + + def forward_batch(self, hidden, input_ids, lengths, layouts, embedding_weight): + self.calls.append(lengths) + return [ + torch.full((sum(len(q.option_spans) for q in layout),), float(length)) + for length, layout in zip(lengths, layouts) + ] + + +class TestJointSchemaPooler(CustomTestCase): + def test_a_layout_cut_from_its_prompt_is_skipped_and_neighbors_are_scored(self): + """A prompt shorter than its layout, as after a truncation, must not stop the + scheduler or the other requests of its batch.""" + head = _RecordingHead() + pooler = JointSchemaPooler(head, lambda: torch.zeros(1)) + layout = pack_decision_layout(40, _questions()) + lengths = [40, 35, 40] + forward_batch = SimpleNamespace( + decision_layouts=[layout, layout, layout], + extend_seq_lens_cpu=lengths, + extend_prefix_lens_cpu=[0, 0, 0], + batch_size=3, + input_ids=torch.zeros(sum(lengths), dtype=torch.long), + ) + output = pooler(torch.zeros(sum(lengths), 8), forward_batch) + self.assertEqual([len(scores) for scores in output.embeddings], [5, 0, 5]) + self.assertEqual(head.calls, [[40], [40]]) + + +def _reference_logits(head, hidden, input_ids, questions, lm_head): + """One request through the modules' own forward methods, as the checkpoint's loop does.""" + normalized = head.hidden_norm(hidden) + memory = head.memory_projection(normalized).unsqueeze(0) + global_vector = normalized[-1] + + def mean(values, span): + return values[span[0] : span[1]].mean(dim=0) + + question_vectors = torch.stack( + [mean(normalized, q.question_span) for q in questions] + ) + queries, lexicals = [], [] + for index, question in enumerate(questions): + context = torch.stack( + [mean(normalized, span) for span in question.option_spans] + ) + lexical = torch.stack( + [ + lm_head[input_ids[start:end]].mean(dim=0) + for start, end in question.option_spans + ] + ) + queries.append( + head.option_context_projection(context) + + head.option_lexical_projection(lexical) + + head.option_question_projection(question_vectors[index]).unsqueeze(0) + ) + lexicals.append(lexical) + routed = torch.cat(queries).unsqueeze(0) + for layer in head.evidence_layers: + attended, _ = layer.attention( + layer.query_norm(routed), + layer.memory_norm(memory), + layer.memory_norm(memory), + need_weights=False, + ) + routed = routed + attended + routed = routed + layer.feedforward(layer.feedforward_norm(routed)) + options = list(torch.split(routed[0], [len(q.option_spans) for q in questions])) + base_fields = head.question_projection(question_vectors) + summaries = [] + for field, routed_options in zip(base_fields, options): + weights = torch.softmax( + routed_options @ field / math.sqrt(field.shape[-1]), dim=0 + ) + summaries.append((weights.unsqueeze(-1) * routed_options).sum(dim=0)) + fields = ( + base_fields + + head.option_summary_norm(torch.stack(summaries)) + + head.global_projection(global_vector).unsqueeze(0) + + head.type_embedding( + torch.tensor([q.question_type for q in questions], device=hidden.device) + ) + ).unsqueeze(0) + for layer in head.layers: + fields = layer(fields, memory) + fields = head.field_norm(fields[0]) + prior_scale = head.prior_logit_scale.clamp(max=math.log(100.0)).exp() + joint_scale = head.joint_logit_scale.clamp(max=math.log(100.0)).exp() + logits = [] + for index, (field, lexical, routed_options) in enumerate( + zip(fields, lexicals, options) + ): + anchor = F.normalize(question_vectors[index] + global_vector, dim=-1) + prior = prior_scale * (F.normalize(lexical, dim=-1) @ anchor) + normed = head.option_norm(routed_options) + repeated = field.unsqueeze(0).expand_as(normed) + features = torch.cat( + [repeated, normed, repeated * normed, torch.abs(repeated - normed)], dim=-1 + ) + residual = head.residual_scorer(features).squeeze(-1) + cosine = F.cosine_similarity(repeated, normed, dim=-1) + logits.append( + prior + + torch.sigmoid(head.residual_gate) * (joint_scale * cosine + residual) + ) + return torch.cat(logits).float() + + +@unittest.skipUnless(torch.cuda.is_available(), "the batched head uses FlashAttention") +class TestJointSchemaHeadBatch(CustomTestCase): + def test_batched_forward_matches_each_request_alone(self): + torch.manual_seed(0) + hidden_size, vocab = 256, 512 + head = JointSchemaHead( + hidden_size, width=128, routing_layers=2, layers=2, heads=4, feedforward=256 + ) + for parameter in ( + head.prior_logit_scale, + head.joint_logit_scale, + head.residual_gate, + ): + parameter.data.fill_(0.5) + head = head.to("cuda", torch.bfloat16).eval() + lm_head = torch.randn(vocab, hidden_size, device="cuda", dtype=torch.bfloat16) + lengths = [40, 140, 64] + layouts = [ + _questions(), + _questions(offset=100), + [ + LayoutQuestion( + 2, (3, 9), tuple((10 + 2 * i, 12 + 2 * i) for i in range(20)) + ) + ], + ] + hidden = torch.randn( + sum(lengths), hidden_size, device="cuda", dtype=torch.bfloat16 + ) + input_ids = torch.randint(0, vocab, (sum(lengths),), device="cuda") + with torch.no_grad(): + batched = head.forward_batch(hidden, input_ids, lengths, layouts, lm_head) + start = 0 + for length, questions, logits in zip(lengths, layouts, batched): + end = start + length + expected = _reference_logits( + head, hidden[start:end], input_ids[start:end], questions, lm_head + ) + self.assertEqual(logits.shape, expected.shape) + torch.testing.assert_close(logits, expected, atol=5e-2, rtol=5e-2) + torch.testing.assert_close( + logits.softmax(-1), expected.softmax(-1), atol=1e-2, rtol=0 + ) + start = end + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/managers/test_hidden_state_server_mode.py b/test/registered/unit/managers/test_hidden_state_server_mode.py index 5aa30c20a8..47a8c617f5 100644 --- a/test/registered/unit/managers/test_hidden_state_server_mode.py +++ b/test/registered/unit/managers/test_hidden_state_server_mode.py @@ -26,6 +26,7 @@ class TestHiddenStateServerMode(CustomTestCase): manager.allow_auto_truncate = False manager.validate_total_tokens = False manager.is_generation = True + manager.model_config = SimpleNamespace(joint_head_config=None) manager.server_args = SimpleNamespace(enable_custom_logit_processor=False) manager._validate_token_ids_logprob = Mock() return manager diff --git a/test/registered/unit/managers/test_sampling_mask_validation.py b/test/registered/unit/managers/test_sampling_mask_validation.py index 963bd94d23..8b7bbb85a5 100644 --- a/test/registered/unit/managers/test_sampling_mask_validation.py +++ b/test/registered/unit/managers/test_sampling_mask_validation.py @@ -1,4 +1,5 @@ import unittest +from types import SimpleNamespace from unittest.mock import patch from sglang.srt.managers.io_struct import GenerateReqInput @@ -28,6 +29,7 @@ class TestSamplingMaskValidation(CustomTestCase): self.manager.allow_auto_truncate = False self.manager.validate_total_tokens = False self.manager.is_generation = True + self.manager.model_config = SimpleNamespace(joint_head_config=None) def _validate( self, diff --git a/test/registered/unit/multimodal/test_gpu_feature_transport.py b/test/registered/unit/multimodal/test_gpu_feature_transport.py index 8ad893112e..afa13fb6e6 100644 --- a/test/registered/unit/multimodal/test_gpu_feature_transport.py +++ b/test/registered/unit/multimodal/test_gpu_feature_transport.py @@ -987,6 +987,7 @@ class TestSchedulerMmTransportBoundary(CustomTestCase): return_pooled_hidden_states=False, multi_item_delimiter_indices=None, token_indices_to_pool=None, + decision_layout=None, mm_inputs=object(), )