[Feat] systemone: serve Kev pointer head decision models (#45413)
This commit is contained in:
@@ -984,12 +984,21 @@ def handle_model_capability_adjustments(server_args: Any):
|
||||
# A Clef checkpoint's joint schema head reads the final hidden state of
|
||||
# every prompt token in one pass, so each request is one complete prefill
|
||||
# with no decode: embedding mode, without prefix reuse or chunking. This
|
||||
# also puts the FA backend on its raw K/V path, which skips the KV pool.
|
||||
# also puts the FA backend on its raw K/V path, which skips the KV pool. A
|
||||
# Kev checkpoint's pointer head reads two positions of each question's
|
||||
# prompt and is served the same way.
|
||||
if model_config.joint_head_config is not None:
|
||||
name, head = "Clef", "joint schema head"
|
||||
tp_reason = ", since the joint schema head reads full LM head rows"
|
||||
elif model_config.pointer_head is not None:
|
||||
name, head, tp_reason = "Kev", "pointer head", ""
|
||||
else:
|
||||
name = None
|
||||
if name is not None:
|
||||
if cfg.tp_size != 1 or cfg.pp_size != 1:
|
||||
raise ValueError(
|
||||
"Clef checkpoints are served with --tp-size 1 and --pp-size 1, "
|
||||
"since the joint schema head reads full LM head rows"
|
||||
f"{name} checkpoints are served with --tp-size 1 and --pp-size 1"
|
||||
f"{tp_reason}"
|
||||
)
|
||||
for key, value in (
|
||||
("is_embedding", True),
|
||||
@@ -1008,7 +1017,7 @@ def handle_model_capability_adjustments(server_args: Any):
|
||||
),
|
||||
)
|
||||
logger.info(
|
||||
"Clef joint schema head detected: serving /v1/systemone decisions in "
|
||||
f"{name} {head} detected: serving /v1/systemone decisions in "
|
||||
"embedding mode without radix cache, chunked prefill, or CUDA graphs."
|
||||
)
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@ from sglang.srt.layers.quantization import QUANTIZATION_METHODS
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.utils import is_hip, retry
|
||||
from sglang.srt.utils.hf_transformers.config import POINTER_ADAPTER_BASES
|
||||
from sglang.srt.utils.hf_transformers_utils import (
|
||||
get_config,
|
||||
get_context_length,
|
||||
@@ -308,6 +309,71 @@ def load_joint_head_config(model_path: str, revision: Optional[str]) -> Optional
|
||||
return json.load(f)
|
||||
|
||||
|
||||
# The values the Kev reference reads for head.pt fields that older checkpoints
|
||||
# did not write.
|
||||
_POINTER_HEAD_DEFAULTS = {
|
||||
"base_revision": None,
|
||||
"head_dim": 256,
|
||||
"option_isolation": False,
|
||||
"special_embeddings": False,
|
||||
"weights_dtype": "fp32",
|
||||
"temperature": 1.0,
|
||||
}
|
||||
|
||||
|
||||
@lru_cache
|
||||
def load_pointer_head(model_path: str, revision: Optional[str]) -> Optional[dict]:
|
||||
"""head.pt of a Kev checkpoint, whose pointer head answers /v1/systemone
|
||||
decisions from the backbone's hidden states."""
|
||||
path = cached_file(
|
||||
model_path,
|
||||
"head.pt",
|
||||
revision=revision,
|
||||
_raise_exceptions_for_missing_entries=False,
|
||||
_raise_exceptions_for_connection_errors=False,
|
||||
)
|
||||
if path is None:
|
||||
return None
|
||||
meta = {
|
||||
**_POINTER_HEAD_DEFAULTS,
|
||||
**torch.load(path, map_location="cpu", weights_only=True),
|
||||
}
|
||||
# Both change the prompt or the attention mask the head was trained with.
|
||||
for name in ("option_isolation", "special_embeddings"):
|
||||
if meta[name]:
|
||||
raise ValueError(f"head.pt {name} is not supported")
|
||||
return meta
|
||||
|
||||
|
||||
@lru_cache
|
||||
def load_pointer_adapter(model_path: str, revision: Optional[str]) -> Optional[dict]:
|
||||
"""head.pt of a Kev LoRA checkpoint with its merge scale. Such a checkpoint has
|
||||
no model config and is served as the base backbone its head.pt pins, with the
|
||||
adapter merged into the base weights at load."""
|
||||
meta = load_pointer_head(model_path, revision)
|
||||
if meta is None:
|
||||
return None
|
||||
path = cached_file(
|
||||
model_path,
|
||||
"adapter_config.json",
|
||||
revision=revision,
|
||||
_raise_exceptions_for_missing_entries=False,
|
||||
_raise_exceptions_for_connection_errors=False,
|
||||
)
|
||||
if path is None:
|
||||
return None
|
||||
# Kev applies an adapter trained on a BF16 backbone unmerged, which a merge
|
||||
# into the base weights does not reproduce.
|
||||
if meta["weights_dtype"] != "fp32":
|
||||
raise ValueError(
|
||||
f"{model_path} is a Kev adapter trained on a {meta['weights_dtype']} "
|
||||
"backbone, which SGLang does not serve. Serve its full weight release."
|
||||
)
|
||||
with open(path) as f:
|
||||
adapter = json.load(f)
|
||||
return {**meta, "lora_scale": adapter["lora_alpha"] / adapter["r"]}
|
||||
|
||||
|
||||
def is_qwen3_5(config) -> bool:
|
||||
return _hf_arch(config) in (
|
||||
"Qwen3_5ForConditionalGeneration",
|
||||
@@ -548,6 +614,13 @@ class ModelConfig:
|
||||
**kwargs,
|
||||
)
|
||||
)
|
||||
self.pointer_head = None
|
||||
base = POINTER_ADAPTER_BASES.get((self.model_path, revision))
|
||||
if base is not None:
|
||||
# A Kev LoRA checkpoint, whose base weights load with its adapter merged.
|
||||
self.pointer_head = load_pointer_adapter(self.model_path, revision)
|
||||
self.model_path, self.revision = base
|
||||
revision = self.revision
|
||||
self.hf_text_config = get_hf_text_config(self.hf_config)
|
||||
self.decision_config = None
|
||||
if self.hf_config.architectures == ["Qwen3_5Model"]:
|
||||
@@ -564,6 +637,11 @@ class ModelConfig:
|
||||
self.joint_head_config = None
|
||||
if self.hf_config.architectures == ["Qwen3_5ForConditionalGeneration"]:
|
||||
self.joint_head_config = load_joint_head_config(self.model_path, revision)
|
||||
if self.hf_config.architectures == ["Qwen3_5TextModel"]:
|
||||
self.pointer_head = load_pointer_head(self.model_path, revision)
|
||||
if self.pointer_head is not None:
|
||||
# A Kev backbone without an LM head, served by the text-only class.
|
||||
self.hf_config.architectures = ["Qwen3_5ForCausalLM"]
|
||||
self.requires_mm_token_modalities = requires_mm_token_modalities(
|
||||
self.hf_config.architectures, self.hf_text_config
|
||||
)
|
||||
|
||||
@@ -70,7 +70,7 @@ from uvicorn.supervisors.multiprocess import Multiprocess
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolving_view
|
||||
from sglang.srt.configs.embedding_model_spec import resolved_embedding_plan
|
||||
from sglang.srt.configs.model_config import load_joint_head_config
|
||||
from sglang.srt.configs.model_config import load_joint_head_config, load_pointer_head
|
||||
from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX
|
||||
from sglang.srt.disaggregation.utils import FAKE_BOOTSTRAP_HOST, DisaggregationMode
|
||||
from sglang.srt.entrypoints.anthropic.protocol import (
|
||||
@@ -2406,13 +2406,18 @@ def _execute_server_warmup(server_args: ServerArgs):
|
||||
).tolist()
|
||||
json_data["sampling_params"]["max_new_tokens"] = 0
|
||||
|
||||
if (
|
||||
not model_info["is_generation"]
|
||||
and model_info.get("architectures") == ["Qwen3_5ForConditionalGeneration"]
|
||||
and load_joint_head_config(get_model().model_path, get_model().revision)
|
||||
is not None
|
||||
model = get_model()
|
||||
if not model_info["is_generation"] and (
|
||||
(
|
||||
model_info.get("architectures") == ["Qwen3_5ForConditionalGeneration"]
|
||||
and load_joint_head_config(model.model_path, model.revision) is not None
|
||||
)
|
||||
or (
|
||||
model_info.get("architectures") == ["Qwen3_5ForCausalLM"]
|
||||
and load_pointer_head(model.model_path, model.revision) is not None
|
||||
)
|
||||
):
|
||||
# A Clef checkpoint answers /v1/systemone decisions only.
|
||||
# A Clef or Kev checkpoint answers /v1/systemone decisions only.
|
||||
request_name = "/v1/systemone"
|
||||
json_data = {
|
||||
"model": "warmup",
|
||||
|
||||
@@ -84,6 +84,7 @@ class OpenAIServingDecisions(OpenAIServingBase):
|
||||
self.prompt_text_is_lossy = chat_serving._prompt_text_round_trip_is_lossy
|
||||
self.decision_config = self.tokenizer_manager.model_config.decision_config
|
||||
self.joint_head_config = self.tokenizer_manager.model_config.joint_head_config
|
||||
self.pointer_head = self.tokenizer_manager.model_config.pointer_head
|
||||
tokenizer = self.tokenizer_manager.tokenizer
|
||||
# Other tokenizers skip the shortcut in _encode_labels and check the full prompt.
|
||||
self.added_tokens = (
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
"""The prompts that a Kev checkpoint's pointer head reads.
|
||||
|
||||
Kev answers /v1/systemone questions from the backbone's hidden states instead
|
||||
of from generated text. Each question is its own prompt, the shared state
|
||||
followed by the question and its options between reserved delimiter tokens,
|
||||
and the head scores each option at its closing token. This follows encode and
|
||||
to_record in the checkpoint's reference code, kev/model.py and kev/api.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, List, Tuple
|
||||
|
||||
from sglang.srt.entrypoints.systemone.protocol import (
|
||||
SystemOneChoiceQuestion,
|
||||
SystemOneNoulQuestion,
|
||||
SystemOneQuestion,
|
||||
SystemOneRequest,
|
||||
)
|
||||
from sglang.srt.layers.joint_schema_head import (
|
||||
QUESTION_TYPES,
|
||||
LayoutQuestion,
|
||||
pack_decision_layout,
|
||||
)
|
||||
|
||||
# Rarely used Qwen tokens that the checkpoint reads as state, question, option,
|
||||
# option end, and decide delimiters.
|
||||
_DELIMITERS = (
|
||||
"<|fim_prefix|>",
|
||||
"<|fim_middle|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|fim_suffix|>",
|
||||
)
|
||||
# Text that looks like a delimiter is rewritten, so callers cannot forge one.
|
||||
_SPECIAL = re.compile(r"<\|([A-Za-z0-9_]+)\|>")
|
||||
_ESCAPED = "<¦\\1¦>"
|
||||
# The reference server's limits: the state with its delimiter, and one prompt.
|
||||
MAX_STATE_TOKENS = 65536
|
||||
MAX_ROW_TOKENS = MAX_STATE_TOKENS + 8192
|
||||
|
||||
|
||||
def _render(value: Any, indent: int = 0) -> str:
|
||||
"""Flattens JSON into the labeled text the checkpoint was trained on."""
|
||||
pad = " " * indent
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, (str, int, float, bool)):
|
||||
return str(value)
|
||||
if isinstance(value, list):
|
||||
return "\n".join(f"{pad}- {_render(x, indent + 1).lstrip()}" for x in value)
|
||||
return "\n".join(
|
||||
(
|
||||
f"{pad}{key}:\n{_render(x, indent + 1)}"
|
||||
if isinstance(x, (dict, list))
|
||||
else f"{pad}{key}: {_render(x)}"
|
||||
)
|
||||
for key, x in value.items()
|
||||
)
|
||||
|
||||
|
||||
def _option(name: str, description: Any) -> str:
|
||||
if description is None or description == "":
|
||||
return name
|
||||
return f"{name}: {_render(description)}"
|
||||
|
||||
|
||||
def kev_options(question: SystemOneQuestion) -> List[str]:
|
||||
"""Each option's text, in the order the head scores them."""
|
||||
if isinstance(question, SystemOneNoulQuestion):
|
||||
criteria = question.criteria
|
||||
return [
|
||||
_option("no", criteria.false if criteria else None),
|
||||
_option("yes", criteria.true if criteria else None),
|
||||
]
|
||||
if isinstance(question, SystemOneChoiceQuestion):
|
||||
return [_option(name, detail) for name, detail in question.criteria.items()]
|
||||
return [_render(level) for level in question.criteria]
|
||||
|
||||
|
||||
def encode_kev(
|
||||
tokenizer: Any, request: SystemOneRequest, max_prompt_tokens: int
|
||||
) -> List[Tuple[List[int], List[int]]]:
|
||||
"""Each question's prompt ids and decision layout, in request order."""
|
||||
|
||||
def tokens(value: Any) -> List[int]:
|
||||
text = _SPECIAL.sub(_ESCAPED, _render(value))
|
||||
return tokenizer.encode(text, add_special_tokens=False)
|
||||
|
||||
state_id, question_id, option_id, close_id, decide_id = (
|
||||
tokenizer.convert_tokens_to_ids(token) for token in _DELIMITERS
|
||||
)
|
||||
prefix = [state_id] + tokens(request.state)
|
||||
if len(prefix) > MAX_STATE_TOKENS:
|
||||
raise ValueError(
|
||||
f"the state is {len(prefix)} tokens with its delimiter, over the "
|
||||
f"{MAX_STATE_TOKENS} token limit of this checkpoint"
|
||||
)
|
||||
limit = min(MAX_ROW_TOKENS, max_prompt_tokens)
|
||||
rows = []
|
||||
for name, question in request.questions.items():
|
||||
input_ids = prefix + [question_id] + tokens(question.instructions)
|
||||
option_spans = []
|
||||
for option in kev_options(question):
|
||||
start = len(input_ids)
|
||||
input_ids += [option_id] + tokens(option) + [close_id]
|
||||
option_spans.append((start, len(input_ids)))
|
||||
input_ids.append(decide_id)
|
||||
if len(input_ids) > limit:
|
||||
raise ValueError(
|
||||
f"question {name!r} needs a {len(input_ids)} token prompt, but "
|
||||
f"this server scores at most {limit} in one prefill"
|
||||
)
|
||||
decide = len(input_ids) - 1
|
||||
layout = pack_decision_layout(
|
||||
len(input_ids),
|
||||
[
|
||||
LayoutQuestion(
|
||||
QUESTION_TYPES.index(question.type),
|
||||
(decide, decide + 1),
|
||||
tuple(option_spans),
|
||||
)
|
||||
],
|
||||
)
|
||||
rows.append((input_ids, layout))
|
||||
return rows
|
||||
@@ -33,6 +33,7 @@ from sglang.srt.entrypoints.systemone.joint_schema import (
|
||||
encode_joint_schema,
|
||||
joint_schema_options,
|
||||
)
|
||||
from sglang.srt.entrypoints.systemone.kev import encode_kev
|
||||
from sglang.srt.entrypoints.systemone.protocol import (
|
||||
SystemOneChoiceAnswer,
|
||||
SystemOneChoiceQuestion,
|
||||
@@ -68,7 +69,7 @@ _INLINE_HEADER_CHARS = 1 << 16
|
||||
|
||||
class SystemOneServing(OpenAIServingDecisions):
|
||||
"""Answers System One questions with the rendering, label checks, and scoring of
|
||||
/v1/decisions, or, for a Clef checkpoint, with its joint schema head."""
|
||||
/v1/decisions, or, for a Clef or Kev checkpoint, with its own head."""
|
||||
|
||||
route = "/v1/systemone"
|
||||
|
||||
@@ -76,7 +77,7 @@ class SystemOneServing(OpenAIServingDecisions):
|
||||
return "systemone-"
|
||||
|
||||
def _validate_request(self, request: SystemOneRequest) -> Optional[str]:
|
||||
if self.joint_head_config is not None:
|
||||
if self.joint_head_config is not None or self.pointer_head is not None:
|
||||
return self._validate_joint_schema_request(request)
|
||||
return (
|
||||
self._validate_server(request.model)
|
||||
@@ -87,7 +88,7 @@ class SystemOneServing(OpenAIServingDecisions):
|
||||
def _validate_joint_schema_request(
|
||||
self, request: SystemOneRequest
|
||||
) -> Optional[str]:
|
||||
"""A Clef checkpoint renders its own prompt, without the chat template."""
|
||||
"""A Clef or Kev checkpoint renders its own prompt, without the chat template."""
|
||||
route = self.route
|
||||
if self.tokenizer_manager.tokenizer is None:
|
||||
return f"{route} requires the server tokenizer"
|
||||
@@ -101,8 +102,9 @@ class SystemOneServing(OpenAIServingDecisions):
|
||||
return f"{route} images require a model served with multimodal input"
|
||||
if request.chat_template_kwargs:
|
||||
return (
|
||||
f"{route} renders the prompt this checkpoint's joint schema head "
|
||||
"was trained with, which takes no chat_template_kwargs"
|
||||
f"{route} renders the prompt this checkpoint's "
|
||||
f"{'pointer' if self.pointer_head is not None else 'joint schema'} "
|
||||
"head was trained with, which takes no chat_template_kwargs"
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -117,6 +119,8 @@ class SystemOneServing(OpenAIServingDecisions):
|
||||
views = [_view(question) for question in request.questions.values()]
|
||||
if self.joint_head_config is not None:
|
||||
return self._joint_schema_request(request, views), (request, views)
|
||||
if self.pointer_head is not None:
|
||||
return self._pointer_request(request, views), (request, views)
|
||||
encode = (
|
||||
self._encoded_systemone_questions
|
||||
if self.decision_config is None
|
||||
@@ -142,6 +146,71 @@ class SystemOneServing(OpenAIServingDecisions):
|
||||
or None,
|
||||
)
|
||||
|
||||
def _pointer_request(
|
||||
self, request: SystemOneRequest, views: List[QuestionView]
|
||||
) -> EmbeddingReqInput:
|
||||
"""One prompt per question, which the pointer head scores in its own prefill."""
|
||||
for question_id, view in zip(request.questions, views):
|
||||
if view.kind == "score":
|
||||
_check_legend(question_id, view)
|
||||
tokenizer_manager = self.tokenizer_manager
|
||||
rows = encode_kev(
|
||||
tokenizer_manager.tokenizer,
|
||||
request,
|
||||
max_prompt_tokens=max_joint_prompt_tokens(
|
||||
context_len=tokenizer_manager.context_len,
|
||||
num_reserved_tokens=tokenizer_manager.num_reserved_tokens,
|
||||
max_req_input_len=tokenizer_manager.max_req_input_len,
|
||||
max_prefill_tokens=get_schedule().max_prefill_tokens,
|
||||
),
|
||||
)
|
||||
return EmbeddingReqInput(
|
||||
input_ids=[input_ids for input_ids, _ in rows],
|
||||
decision_layout=[layout for _, layout in rows],
|
||||
)
|
||||
|
||||
async def _answer_pointer(
|
||||
self,
|
||||
adapted_request: EmbeddingReqInput,
|
||||
request: SystemOneRequest,
|
||||
views: List[QuestionView],
|
||||
raw_request: Request,
|
||||
) -> ORJSONResponse:
|
||||
rets = await self.tokenizer_manager.generate_request(
|
||||
adapted_request, raw_request
|
||||
).__anext__()
|
||||
answers = {}
|
||||
for question_id, view, ret in zip(request.questions, views, rets):
|
||||
logits = ret["embedding"]
|
||||
if len(logits) != len(view.names):
|
||||
raise RuntimeError("the pointer head did not score this request")
|
||||
top = max(logits)
|
||||
weights = [math.exp(logit - top) for logit in logits]
|
||||
total = math.fsum(weights)
|
||||
probabilities = [weight / total for weight in weights]
|
||||
# The head scores no then yes, which the view names yes and no.
|
||||
answers[question_id] = _answer(
|
||||
view=view,
|
||||
probabilities=(
|
||||
probabilities[::-1] if view.kind == "yes_no" else probabilities
|
||||
),
|
||||
mass=1.0,
|
||||
question_id=question_id,
|
||||
)
|
||||
response = SystemOneResponse(
|
||||
model=self.tokenizer_manager.served_model_name,
|
||||
answers=answers,
|
||||
usage=SystemOneUsage(
|
||||
input_tokens=sum(ret["meta_info"]["prompt_tokens"] for ret in rets)
|
||||
),
|
||||
)
|
||||
# The head scores only the allowed options, so there is no label mass.
|
||||
return ORJSONResponse(
|
||||
content=response.model_dump(
|
||||
exclude={"answers": {"__all__": {"x_label_mass"}}}
|
||||
)
|
||||
)
|
||||
|
||||
async def _encode_joint_schema_prompt(
|
||||
self, adapted_request: EmbeddingReqInput, request: SystemOneRequest
|
||||
) -> None:
|
||||
@@ -343,6 +412,10 @@ class SystemOneServing(OpenAIServingDecisions):
|
||||
return await self._answer_joint_schema(
|
||||
adapted_request, request, views, raw_request
|
||||
)
|
||||
if self.pointer_head is not None:
|
||||
return await self._answer_pointer(
|
||||
adapted_request, request, views, raw_request
|
||||
)
|
||||
_, _, result = await self._score(
|
||||
adapted_request=adapted_request,
|
||||
raw_request=raw_request,
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Pointer head of Kev decision checkpoints.
|
||||
|
||||
A Kev checkpoint is a Qwen3.5 backbone saved without an LM head, with its head
|
||||
in ``head.pt``. Each question runs as its own prompt after the shared state,
|
||||
and the head scores every option by the final hidden state at the option's
|
||||
closing token against the one at the prompt's last token.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import List
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.layers.joint_schema_head import parse_decision_layout
|
||||
from sglang.srt.layers.pooler import EmbeddingPoolerOutput
|
||||
|
||||
|
||||
class PointerPooler(nn.Module):
|
||||
"""Option logits per request, from a one-question decision layout whose
|
||||
question span ends at the prompt's last token and whose option spans end
|
||||
at the options' closing tokens."""
|
||||
|
||||
def __init__(self, hidden_size: int, meta: dict) -> None:
|
||||
super().__init__()
|
||||
head_dim = meta["head_dim"]
|
||||
# The checkpoint trains and serves the head in FP32.
|
||||
self.q = nn.Linear(hidden_size, head_dim, dtype=torch.float32)
|
||||
self.k = nn.Linear(hidden_size, head_dim, dtype=torch.float32)
|
||||
self.scale = 1 / math.sqrt(head_dim)
|
||||
self.temperature = float(meta["temperature"])
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, forward_batch
|
||||
) -> EmbeddingPoolerOutput:
|
||||
lengths = forward_batch.extend_seq_lens_cpu
|
||||
embeddings: List[torch.Tensor] = [
|
||||
hidden_states.new_empty(0, dtype=torch.float32)
|
||||
for _ in range(forward_batch.batch_size)
|
||||
]
|
||||
if forward_batch.decision_layouts is None or lengths is None:
|
||||
return EmbeddingPoolerOutput(embeddings=embeddings)
|
||||
prefix_lengths = forward_batch.extend_prefix_lens_cpu or [0] * len(lengths)
|
||||
scored, decide, options, owners = [], [], [], []
|
||||
offset = 0
|
||||
for i, (layout, prefix, length) in enumerate(
|
||||
zip(forward_batch.decision_layouts, prefix_lengths, lengths)
|
||||
):
|
||||
# A layout that no longer fits its prompt gets no scores, which the
|
||||
# decision route reports as a server error.
|
||||
questions = None
|
||||
if layout is not None and prefix == 0:
|
||||
try:
|
||||
questions = parse_decision_layout(layout, length)
|
||||
except ValueError:
|
||||
pass
|
||||
if questions is not None and len(questions) == 1:
|
||||
question = questions[0]
|
||||
owners += [len(scored)] * len(question.option_spans)
|
||||
scored.append((i, len(question.option_spans)))
|
||||
decide.append(offset + question.question_span[1] - 1)
|
||||
options += [offset + end - 1 for _, end in question.option_spans]
|
||||
offset += length
|
||||
if not scored:
|
||||
return EmbeddingPoolerOutput(embeddings=embeddings)
|
||||
device = hidden_states.device
|
||||
h_decide = hidden_states[torch.tensor(decide, device=device)].float()
|
||||
h_options = hidden_states[torch.tensor(options, device=device)].float()
|
||||
owner = torch.tensor(owners, device=device)
|
||||
logits = (self.k(h_options) * self.q(h_decide)[owner]).sum(-1) * self.scale
|
||||
logits = logits / self.temperature
|
||||
for (i, _), values in zip(scored, logits.split([n for _, n in scored])):
|
||||
embeddings[i] = values
|
||||
return EmbeddingPoolerOutput(embeddings=embeddings)
|
||||
@@ -1452,8 +1452,11 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
def _validate_joint_schema_request(
|
||||
self, obj: Union[GenerateReqInput, EmbeddingReqInput], input_ids: List[int]
|
||||
) -> None:
|
||||
"""A Clef checkpoint scores the spans of a decision layout, and nothing else."""
|
||||
if self.model_config.joint_head_config is None:
|
||||
"""A Clef or Kev checkpoint scores the spans of a decision layout, and nothing else."""
|
||||
if (
|
||||
self.model_config.joint_head_config is None
|
||||
and self.model_config.pointer_head is None
|
||||
):
|
||||
return
|
||||
if isinstance(obj, GenerateReqInput) or obj.decision_layout is None:
|
||||
# Health checks only need a response, which is an empty embedding.
|
||||
|
||||
@@ -17,10 +17,14 @@ import logging
|
||||
from typing import Iterable, Optional, Set, Tuple, Union
|
||||
|
||||
import torch
|
||||
from safetensors.torch import load_file
|
||||
from torch import nn
|
||||
from transformers.utils import cached_file
|
||||
|
||||
from sglang.srt.configs.model_config import load_pointer_adapter, load_pointer_head
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.pointer_head import PointerPooler
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.utils import PPMissingLayer
|
||||
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
@@ -29,12 +33,16 @@ from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models import qwen3_5
|
||||
from sglang.srt.models.qwen2_moe import Qwen2MoeSparseMoeBlock
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.runtime_context import get_model, get_parallel
|
||||
from sglang.srt.utils import LazyValue, add_prefix
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MODEL_PREFIX = "model."
|
||||
# Where a Kev LoRA checkpoint's base keeps the text backbone, and where the
|
||||
# adapter keeps its factors.
|
||||
_BASE_TEXT_PREFIX = "model.language_model."
|
||||
_ADAPTER_PREFIX = "base_model.model."
|
||||
|
||||
|
||||
class Qwen3_5ForCausalLM(nn.Module):
|
||||
@@ -70,7 +78,19 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
prefix=add_prefix("model", prefix),
|
||||
)
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
self.pooler = None
|
||||
model = get_model()
|
||||
meta = (
|
||||
load_pointer_head(model.model_path, model.revision)
|
||||
if model.is_embedding
|
||||
else None
|
||||
)
|
||||
if meta is not None:
|
||||
# A Kev checkpoint, whose pointer head scores options from the final
|
||||
# hidden states, and which has no LM head.
|
||||
self.pooler = PointerPooler(config.hidden_size, meta)
|
||||
self.lm_head = PPMissingLayer()
|
||||
elif self.pp_group.is_last_rank:
|
||||
if self.pp_group.world_size == 1 and config.tie_word_embeddings:
|
||||
self.lm_head = self.model.embed_tokens
|
||||
else:
|
||||
@@ -165,6 +185,7 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
input_embeds: Optional[torch.Tensor] = None,
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
get_embedding: bool = False,
|
||||
**kwargs,
|
||||
) -> Union[torch.Tensor, PPProxyTensors]:
|
||||
if self.is_mrope_enabled:
|
||||
@@ -185,19 +206,55 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
if isinstance(hidden_states, tuple):
|
||||
hidden_states, aux_hidden_states = hidden_states
|
||||
|
||||
if get_embedding and self.pooler is not None:
|
||||
return self.pooler(hidden_states, forward_batch)
|
||||
return self.logits_processor(
|
||||
input_ids, hidden_states, self.lm_head, forward_batch, aux_hidden_states
|
||||
)
|
||||
|
||||
def _pointer_adapter(self) -> dict:
|
||||
"""LoRA factors of a Kev LoRA checkpoint by backbone module, none for full weights."""
|
||||
model = get_model()
|
||||
meta = load_pointer_adapter(model.model_path, model.revision)
|
||||
if meta is None:
|
||||
return {}
|
||||
tensors = load_file(
|
||||
cached_file(
|
||||
model.model_path, "adapter_model.safetensors", revision=model.revision
|
||||
)
|
||||
)
|
||||
return {
|
||||
name[len(_ADAPTER_PREFIX) : -len(".lora_A.weight")]: (
|
||||
a,
|
||||
tensors[name.replace(".lora_A.", ".lora_B.")],
|
||||
meta["lora_scale"],
|
||||
)
|
||||
for name, a in tensors.items()
|
||||
if name.endswith(".lora_A.weight")
|
||||
}
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]) -> Set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: Set[str] = set()
|
||||
|
||||
lora = self._pointer_adapter() if self.pooler is not None else {}
|
||||
|
||||
# Keep prefix stripping lazy: materializing mmap-backed checkpoint tensors
|
||||
# across model processes can exceed host memory before GPU copies finish.
|
||||
def body_weights():
|
||||
for name, loaded_weight in weights:
|
||||
if name.startswith(_MODEL_PREFIX):
|
||||
if self.pooler is not None:
|
||||
# Kev saves the backbone under its own names, and the base of a
|
||||
# Kev LoRA checkpoint under its text model, next to an LM head
|
||||
# that Kev does not use.
|
||||
name = name.removeprefix(_BASE_TEXT_PREFIX)
|
||||
if name.startswith("lm_head."):
|
||||
continue
|
||||
factors = lora.pop(name.removesuffix(".weight"), None)
|
||||
if factors is not None:
|
||||
loaded_weight = _merge_lora(loaded_weight, *factors)
|
||||
yield name, loaded_weight
|
||||
elif name.startswith(_MODEL_PREFIX):
|
||||
yield name[len(_MODEL_PREFIX) :], loaded_weight
|
||||
elif name == "lm_head.weight":
|
||||
if self.config.tie_word_embeddings:
|
||||
@@ -212,9 +269,21 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
loaded_params.add("lm_head.weight")
|
||||
|
||||
body_loaded = self.model.load_weights(body_weights())
|
||||
if lora:
|
||||
raise ValueError(
|
||||
f"The Kev adapter has factors for {len(lora)} modules that the "
|
||||
f"base checkpoint does not have, such as {next(iter(lora))}"
|
||||
)
|
||||
loaded_params.update(f"{_MODEL_PREFIX}{n}" for n in body_loaded)
|
||||
|
||||
if self.config.tie_word_embeddings and self.pp_group.is_last_rank:
|
||||
if self.pooler is not None:
|
||||
model = get_model()
|
||||
meta = load_pointer_head(model.model_path, model.revision)
|
||||
self.pooler.load_state_dict(meta["head"], strict=True)
|
||||
loaded_params.update(
|
||||
f"pooler.{n}" for n, _ in self.pooler.named_parameters()
|
||||
)
|
||||
elif self.config.tie_word_embeddings and self.pp_group.is_last_rank:
|
||||
loaded_params.add("lm_head.weight")
|
||||
|
||||
return loaded_params
|
||||
@@ -255,4 +324,12 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM):
|
||||
)
|
||||
|
||||
|
||||
def _merge_lora(
|
||||
weight: torch.Tensor, a: torch.Tensor, b: torch.Tensor, scale: float
|
||||
) -> torch.Tensor:
|
||||
"""The weight plus its LoRA delta, added in FP32 and rounded once, as the Kev
|
||||
reference merges an adapter."""
|
||||
return (weight.float() + scale * (b.float() @ a.float())).to(weight.dtype)
|
||||
|
||||
|
||||
EntryClass = [Qwen3_5MoeForCausalLM, Qwen3_5ForCausalLM]
|
||||
|
||||
@@ -13,8 +13,9 @@
|
||||
# ==============================================================================
|
||||
"""Config loading utilities."""
|
||||
|
||||
import copy
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
|
||||
@@ -47,6 +48,10 @@ from .common import (
|
||||
from .gguf_native import build_gguf_config, has_native_gguf_support
|
||||
from .mistral_utils import is_mistral_model, load_mistral_config
|
||||
|
||||
# The base and revision that get_config read for each Kev LoRA checkpoint, which
|
||||
# has no model config of its own, keyed by checkpoint path and revision.
|
||||
POINTER_ADAPTER_BASES: Dict[Tuple[str, Optional[str]], Tuple[str, str]] = {}
|
||||
|
||||
|
||||
def _set_architectures(config, arch_name):
|
||||
config.update({"architectures": [arch_name]})
|
||||
@@ -370,9 +375,32 @@ def get_config(
|
||||
model_config_parser = "mistral" if is_mistral_model(model) else "hf"
|
||||
|
||||
parser = get_model_config_parser(model_config_parser)
|
||||
config = parser.parse(
|
||||
model, trust_remote_code=trust_remote_code, revision=revision, **kwargs
|
||||
)
|
||||
try:
|
||||
config = parser.parse(
|
||||
model, trust_remote_code=trust_remote_code, revision=revision, **kwargs
|
||||
)
|
||||
except (OSError, ValueError):
|
||||
# A Kev LoRA checkpoint has no model config. It serves the text backbone
|
||||
# of the base its head.pt pins.
|
||||
from sglang.srt.configs.model_config import load_pointer_adapter
|
||||
|
||||
meta = load_pointer_adapter(model, revision)
|
||||
if meta is None:
|
||||
raise
|
||||
base = parser.parse(
|
||||
meta["base"],
|
||||
trust_remote_code=trust_remote_code,
|
||||
revision=meta["base_revision"],
|
||||
**kwargs,
|
||||
)
|
||||
if base.model_type != "qwen3_5":
|
||||
raise ValueError(
|
||||
f"Kev checkpoints on a {base.model_type} base are not supported, "
|
||||
"only on Qwen3.5 and Qwen3.8 bases"
|
||||
)
|
||||
config = copy.deepcopy(base.text_config)
|
||||
_set_architectures(config, "Qwen3_5ForCausalLM")
|
||||
POINTER_ADAPTER_BASES[(model, revision)] = (meta["base"], meta["base_revision"])
|
||||
|
||||
if model_override_args:
|
||||
if isinstance(config, DeepseekV41Config):
|
||||
|
||||
Reference in New Issue
Block a user