[Feat] systemone: serve Kev pointer head decision models (#45413)

This commit is contained in:
Richard Wang
2026-10-22 14:16:52 +08:00
committed by GitHub
parent 12370da115
commit ce66b3ab59
20 changed files with 751 additions and 32 deletions
+13 -4
View File
@@ -984,12 +984,21 @@ def handle_model_capability_adjustments(server_args: Any):
# A Clef checkpoint's joint schema head reads the final hidden state of
# every prompt token in one pass, so each request is one complete prefill
# with no decode: embedding mode, without prefix reuse or chunking. This
# also puts the FA backend on its raw K/V path, which skips the KV pool.
# also puts the FA backend on its raw K/V path, which skips the KV pool. A
# Kev checkpoint's pointer head reads two positions of each question's
# prompt and is served the same way.
if model_config.joint_head_config is not None:
name, head = "Clef", "joint schema head"
tp_reason = ", since the joint schema head reads full LM head rows"
elif model_config.pointer_head is not None:
name, head, tp_reason = "Kev", "pointer head", ""
else:
name = None
if name is not None:
if cfg.tp_size != 1 or cfg.pp_size != 1:
raise ValueError(
"Clef checkpoints are served with --tp-size 1 and --pp-size 1, "
"since the joint schema head reads full LM head rows"
f"{name} checkpoints are served with --tp-size 1 and --pp-size 1"
f"{tp_reason}"
)
for key, value in (
("is_embedding", True),
@@ -1008,7 +1017,7 @@ def handle_model_capability_adjustments(server_args: Any):
),
)
logger.info(
"Clef joint schema head detected: serving /v1/systemone decisions in "
f"{name} {head} detected: serving /v1/systemone decisions in "
"embedding mode without radix cache, chunked prefill, or CUDA graphs."
)
+78
View File
@@ -35,6 +35,7 @@ from sglang.srt.layers.quantization import QUANTIZATION_METHODS
from sglang.srt.runtime_context import get_platform
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import is_hip, retry
from sglang.srt.utils.hf_transformers.config import POINTER_ADAPTER_BASES
from sglang.srt.utils.hf_transformers_utils import (
get_config,
get_context_length,
@@ -308,6 +309,71 @@ def load_joint_head_config(model_path: str, revision: Optional[str]) -> Optional
return json.load(f)
# The values the Kev reference reads for head.pt fields that older checkpoints
# did not write.
_POINTER_HEAD_DEFAULTS = {
"base_revision": None,
"head_dim": 256,
"option_isolation": False,
"special_embeddings": False,
"weights_dtype": "fp32",
"temperature": 1.0,
}
@lru_cache
def load_pointer_head(model_path: str, revision: Optional[str]) -> Optional[dict]:
"""head.pt of a Kev checkpoint, whose pointer head answers /v1/systemone
decisions from the backbone's hidden states."""
path = cached_file(
model_path,
"head.pt",
revision=revision,
_raise_exceptions_for_missing_entries=False,
_raise_exceptions_for_connection_errors=False,
)
if path is None:
return None
meta = {
**_POINTER_HEAD_DEFAULTS,
**torch.load(path, map_location="cpu", weights_only=True),
}
# Both change the prompt or the attention mask the head was trained with.
for name in ("option_isolation", "special_embeddings"):
if meta[name]:
raise ValueError(f"head.pt {name} is not supported")
return meta
@lru_cache
def load_pointer_adapter(model_path: str, revision: Optional[str]) -> Optional[dict]:
"""head.pt of a Kev LoRA checkpoint with its merge scale. Such a checkpoint has
no model config and is served as the base backbone its head.pt pins, with the
adapter merged into the base weights at load."""
meta = load_pointer_head(model_path, revision)
if meta is None:
return None
path = cached_file(
model_path,
"adapter_config.json",
revision=revision,
_raise_exceptions_for_missing_entries=False,
_raise_exceptions_for_connection_errors=False,
)
if path is None:
return None
# Kev applies an adapter trained on a BF16 backbone unmerged, which a merge
# into the base weights does not reproduce.
if meta["weights_dtype"] != "fp32":
raise ValueError(
f"{model_path} is a Kev adapter trained on a {meta['weights_dtype']} "
"backbone, which SGLang does not serve. Serve its full weight release."
)
with open(path) as f:
adapter = json.load(f)
return {**meta, "lora_scale": adapter["lora_alpha"] / adapter["r"]}
def is_qwen3_5(config) -> bool:
return _hf_arch(config) in (
"Qwen3_5ForConditionalGeneration",
@@ -548,6 +614,13 @@ class ModelConfig:
**kwargs,
)
)
self.pointer_head = None
base = POINTER_ADAPTER_BASES.get((self.model_path, revision))
if base is not None:
# A Kev LoRA checkpoint, whose base weights load with its adapter merged.
self.pointer_head = load_pointer_adapter(self.model_path, revision)
self.model_path, self.revision = base
revision = self.revision
self.hf_text_config = get_hf_text_config(self.hf_config)
self.decision_config = None
if self.hf_config.architectures == ["Qwen3_5Model"]:
@@ -564,6 +637,11 @@ class ModelConfig:
self.joint_head_config = None
if self.hf_config.architectures == ["Qwen3_5ForConditionalGeneration"]:
self.joint_head_config = load_joint_head_config(self.model_path, revision)
if self.hf_config.architectures == ["Qwen3_5TextModel"]:
self.pointer_head = load_pointer_head(self.model_path, revision)
if self.pointer_head is not None:
# A Kev backbone without an LM head, served by the text-only class.
self.hf_config.architectures = ["Qwen3_5ForCausalLM"]
self.requires_mm_token_modalities = requires_mm_token_modalities(
self.hf_config.architectures, self.hf_text_config
)
+12 -7
View File
@@ -70,7 +70,7 @@ from uvicorn.supervisors.multiprocess import Multiprocess
from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.configs.embedding_model_spec import resolved_embedding_plan
from sglang.srt.configs.model_config import load_joint_head_config
from sglang.srt.configs.model_config import load_joint_head_config, load_pointer_head
from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX
from sglang.srt.disaggregation.utils import FAKE_BOOTSTRAP_HOST, DisaggregationMode
from sglang.srt.entrypoints.anthropic.protocol import (
@@ -2406,13 +2406,18 @@ def _execute_server_warmup(server_args: ServerArgs):
).tolist()
json_data["sampling_params"]["max_new_tokens"] = 0
if (
not model_info["is_generation"]
and model_info.get("architectures") == ["Qwen3_5ForConditionalGeneration"]
and load_joint_head_config(get_model().model_path, get_model().revision)
is not None
model = get_model()
if not model_info["is_generation"] and (
(
model_info.get("architectures") == ["Qwen3_5ForConditionalGeneration"]
and load_joint_head_config(model.model_path, model.revision) is not None
)
or (
model_info.get("architectures") == ["Qwen3_5ForCausalLM"]
and load_pointer_head(model.model_path, model.revision) is not None
)
):
# A Clef checkpoint answers /v1/systemone decisions only.
# A Clef or Kev checkpoint answers /v1/systemone decisions only.
request_name = "/v1/systemone"
json_data = {
"model": "warmup",
@@ -84,6 +84,7 @@ class OpenAIServingDecisions(OpenAIServingBase):
self.prompt_text_is_lossy = chat_serving._prompt_text_round_trip_is_lossy
self.decision_config = self.tokenizer_manager.model_config.decision_config
self.joint_head_config = self.tokenizer_manager.model_config.joint_head_config
self.pointer_head = self.tokenizer_manager.model_config.pointer_head
tokenizer = self.tokenizer_manager.tokenizer
# Other tokenizers skip the shortcut in _encode_labels and check the full prompt.
self.added_tokens = (
@@ -0,0 +1,127 @@
"""The prompts that a Kev checkpoint's pointer head reads.
Kev answers /v1/systemone questions from the backbone's hidden states instead
of from generated text. Each question is its own prompt, the shared state
followed by the question and its options between reserved delimiter tokens,
and the head scores each option at its closing token. This follows encode and
to_record in the checkpoint's reference code, kev/model.py and kev/api.py.
"""
from __future__ import annotations
import re
from typing import Any, List, Tuple
from sglang.srt.entrypoints.systemone.protocol import (
SystemOneChoiceQuestion,
SystemOneNoulQuestion,
SystemOneQuestion,
SystemOneRequest,
)
from sglang.srt.layers.joint_schema_head import (
QUESTION_TYPES,
LayoutQuestion,
pack_decision_layout,
)
# Rarely used Qwen tokens that the checkpoint reads as state, question, option,
# option end, and decide delimiters.
_DELIMITERS = (
"<|fim_prefix|>",
"<|fim_middle|>",
"<|box_start|>",
"<|box_end|>",
"<|fim_suffix|>",
)
# Text that looks like a delimiter is rewritten, so callers cannot forge one.
_SPECIAL = re.compile(r"<\|([A-Za-z0-9_]+)\|>")
_ESCAPED = "<¦\\1¦>"
# The reference server's limits: the state with its delimiter, and one prompt.
MAX_STATE_TOKENS = 65536
MAX_ROW_TOKENS = MAX_STATE_TOKENS + 8192
def _render(value: Any, indent: int = 0) -> str:
"""Flattens JSON into the labeled text the checkpoint was trained on."""
pad = " " * indent
if value is None:
return ""
if isinstance(value, (str, int, float, bool)):
return str(value)
if isinstance(value, list):
return "\n".join(f"{pad}- {_render(x, indent + 1).lstrip()}" for x in value)
return "\n".join(
(
f"{pad}{key}:\n{_render(x, indent + 1)}"
if isinstance(x, (dict, list))
else f"{pad}{key}: {_render(x)}"
)
for key, x in value.items()
)
def _option(name: str, description: Any) -> str:
if description is None or description == "":
return name
return f"{name}: {_render(description)}"
def kev_options(question: SystemOneQuestion) -> List[str]:
"""Each option's text, in the order the head scores them."""
if isinstance(question, SystemOneNoulQuestion):
criteria = question.criteria
return [
_option("no", criteria.false if criteria else None),
_option("yes", criteria.true if criteria else None),
]
if isinstance(question, SystemOneChoiceQuestion):
return [_option(name, detail) for name, detail in question.criteria.items()]
return [_render(level) for level in question.criteria]
def encode_kev(
tokenizer: Any, request: SystemOneRequest, max_prompt_tokens: int
) -> List[Tuple[List[int], List[int]]]:
"""Each question's prompt ids and decision layout, in request order."""
def tokens(value: Any) -> List[int]:
text = _SPECIAL.sub(_ESCAPED, _render(value))
return tokenizer.encode(text, add_special_tokens=False)
state_id, question_id, option_id, close_id, decide_id = (
tokenizer.convert_tokens_to_ids(token) for token in _DELIMITERS
)
prefix = [state_id] + tokens(request.state)
if len(prefix) > MAX_STATE_TOKENS:
raise ValueError(
f"the state is {len(prefix)} tokens with its delimiter, over the "
f"{MAX_STATE_TOKENS} token limit of this checkpoint"
)
limit = min(MAX_ROW_TOKENS, max_prompt_tokens)
rows = []
for name, question in request.questions.items():
input_ids = prefix + [question_id] + tokens(question.instructions)
option_spans = []
for option in kev_options(question):
start = len(input_ids)
input_ids += [option_id] + tokens(option) + [close_id]
option_spans.append((start, len(input_ids)))
input_ids.append(decide_id)
if len(input_ids) > limit:
raise ValueError(
f"question {name!r} needs a {len(input_ids)} token prompt, but "
f"this server scores at most {limit} in one prefill"
)
decide = len(input_ids) - 1
layout = pack_decision_layout(
len(input_ids),
[
LayoutQuestion(
QUESTION_TYPES.index(question.type),
(decide, decide + 1),
tuple(option_spans),
)
],
)
rows.append((input_ids, layout))
return rows
@@ -33,6 +33,7 @@ from sglang.srt.entrypoints.systemone.joint_schema import (
encode_joint_schema,
joint_schema_options,
)
from sglang.srt.entrypoints.systemone.kev import encode_kev
from sglang.srt.entrypoints.systemone.protocol import (
SystemOneChoiceAnswer,
SystemOneChoiceQuestion,
@@ -68,7 +69,7 @@ _INLINE_HEADER_CHARS = 1 << 16
class SystemOneServing(OpenAIServingDecisions):
"""Answers System One questions with the rendering, label checks, and scoring of
/v1/decisions, or, for a Clef checkpoint, with its joint schema head."""
/v1/decisions, or, for a Clef or Kev checkpoint, with its own head."""
route = "/v1/systemone"
@@ -76,7 +77,7 @@ class SystemOneServing(OpenAIServingDecisions):
return "systemone-"
def _validate_request(self, request: SystemOneRequest) -> Optional[str]:
if self.joint_head_config is not None:
if self.joint_head_config is not None or self.pointer_head is not None:
return self._validate_joint_schema_request(request)
return (
self._validate_server(request.model)
@@ -87,7 +88,7 @@ class SystemOneServing(OpenAIServingDecisions):
def _validate_joint_schema_request(
self, request: SystemOneRequest
) -> Optional[str]:
"""A Clef checkpoint renders its own prompt, without the chat template."""
"""A Clef or Kev checkpoint renders its own prompt, without the chat template."""
route = self.route
if self.tokenizer_manager.tokenizer is None:
return f"{route} requires the server tokenizer"
@@ -101,8 +102,9 @@ class SystemOneServing(OpenAIServingDecisions):
return f"{route} images require a model served with multimodal input"
if request.chat_template_kwargs:
return (
f"{route} renders the prompt this checkpoint's joint schema head "
"was trained with, which takes no chat_template_kwargs"
f"{route} renders the prompt this checkpoint's "
f"{'pointer' if self.pointer_head is not None else 'joint schema'} "
"head was trained with, which takes no chat_template_kwargs"
)
return None
@@ -117,6 +119,8 @@ class SystemOneServing(OpenAIServingDecisions):
views = [_view(question) for question in request.questions.values()]
if self.joint_head_config is not None:
return self._joint_schema_request(request, views), (request, views)
if self.pointer_head is not None:
return self._pointer_request(request, views), (request, views)
encode = (
self._encoded_systemone_questions
if self.decision_config is None
@@ -142,6 +146,71 @@ class SystemOneServing(OpenAIServingDecisions):
or None,
)
def _pointer_request(
self, request: SystemOneRequest, views: List[QuestionView]
) -> EmbeddingReqInput:
"""One prompt per question, which the pointer head scores in its own prefill."""
for question_id, view in zip(request.questions, views):
if view.kind == "score":
_check_legend(question_id, view)
tokenizer_manager = self.tokenizer_manager
rows = encode_kev(
tokenizer_manager.tokenizer,
request,
max_prompt_tokens=max_joint_prompt_tokens(
context_len=tokenizer_manager.context_len,
num_reserved_tokens=tokenizer_manager.num_reserved_tokens,
max_req_input_len=tokenizer_manager.max_req_input_len,
max_prefill_tokens=get_schedule().max_prefill_tokens,
),
)
return EmbeddingReqInput(
input_ids=[input_ids for input_ids, _ in rows],
decision_layout=[layout for _, layout in rows],
)
async def _answer_pointer(
self,
adapted_request: EmbeddingReqInput,
request: SystemOneRequest,
views: List[QuestionView],
raw_request: Request,
) -> ORJSONResponse:
rets = await self.tokenizer_manager.generate_request(
adapted_request, raw_request
).__anext__()
answers = {}
for question_id, view, ret in zip(request.questions, views, rets):
logits = ret["embedding"]
if len(logits) != len(view.names):
raise RuntimeError("the pointer head did not score this request")
top = max(logits)
weights = [math.exp(logit - top) for logit in logits]
total = math.fsum(weights)
probabilities = [weight / total for weight in weights]
# The head scores no then yes, which the view names yes and no.
answers[question_id] = _answer(
view=view,
probabilities=(
probabilities[::-1] if view.kind == "yes_no" else probabilities
),
mass=1.0,
question_id=question_id,
)
response = SystemOneResponse(
model=self.tokenizer_manager.served_model_name,
answers=answers,
usage=SystemOneUsage(
input_tokens=sum(ret["meta_info"]["prompt_tokens"] for ret in rets)
),
)
# The head scores only the allowed options, so there is no label mass.
return ORJSONResponse(
content=response.model_dump(
exclude={"answers": {"__all__": {"x_label_mass"}}}
)
)
async def _encode_joint_schema_prompt(
self, adapted_request: EmbeddingReqInput, request: SystemOneRequest
) -> None:
@@ -343,6 +412,10 @@ class SystemOneServing(OpenAIServingDecisions):
return await self._answer_joint_schema(
adapted_request, request, views, raw_request
)
if self.pointer_head is not None:
return await self._answer_pointer(
adapted_request, request, views, raw_request
)
_, _, result = await self._score(
adapted_request=adapted_request,
raw_request=raw_request,
+76
View File
@@ -0,0 +1,76 @@
"""Pointer head of Kev decision checkpoints.
A Kev checkpoint is a Qwen3.5 backbone saved without an LM head, with its head
in ``head.pt``. Each question runs as its own prompt after the shared state,
and the head scores every option by the final hidden state at the option's
closing token against the one at the prompt's last token.
"""
from __future__ import annotations
import math
from typing import List
import torch
from torch import nn
from sglang.srt.layers.joint_schema_head import parse_decision_layout
from sglang.srt.layers.pooler import EmbeddingPoolerOutput
class PointerPooler(nn.Module):
"""Option logits per request, from a one-question decision layout whose
question span ends at the prompt's last token and whose option spans end
at the options' closing tokens."""
def __init__(self, hidden_size: int, meta: dict) -> None:
super().__init__()
head_dim = meta["head_dim"]
# The checkpoint trains and serves the head in FP32.
self.q = nn.Linear(hidden_size, head_dim, dtype=torch.float32)
self.k = nn.Linear(hidden_size, head_dim, dtype=torch.float32)
self.scale = 1 / math.sqrt(head_dim)
self.temperature = float(meta["temperature"])
def forward(
self, hidden_states: torch.Tensor, forward_batch
) -> EmbeddingPoolerOutput:
lengths = forward_batch.extend_seq_lens_cpu
embeddings: List[torch.Tensor] = [
hidden_states.new_empty(0, dtype=torch.float32)
for _ in range(forward_batch.batch_size)
]
if forward_batch.decision_layouts is None or lengths is None:
return EmbeddingPoolerOutput(embeddings=embeddings)
prefix_lengths = forward_batch.extend_prefix_lens_cpu or [0] * len(lengths)
scored, decide, options, owners = [], [], [], []
offset = 0
for i, (layout, prefix, length) in enumerate(
zip(forward_batch.decision_layouts, prefix_lengths, lengths)
):
# A layout that no longer fits its prompt gets no scores, which the
# decision route reports as a server error.
questions = None
if layout is not None and prefix == 0:
try:
questions = parse_decision_layout(layout, length)
except ValueError:
pass
if questions is not None and len(questions) == 1:
question = questions[0]
owners += [len(scored)] * len(question.option_spans)
scored.append((i, len(question.option_spans)))
decide.append(offset + question.question_span[1] - 1)
options += [offset + end - 1 for _, end in question.option_spans]
offset += length
if not scored:
return EmbeddingPoolerOutput(embeddings=embeddings)
device = hidden_states.device
h_decide = hidden_states[torch.tensor(decide, device=device)].float()
h_options = hidden_states[torch.tensor(options, device=device)].float()
owner = torch.tensor(owners, device=device)
logits = (self.k(h_options) * self.q(h_decide)[owner]).sum(-1) * self.scale
logits = logits / self.temperature
for (i, _), values in zip(scored, logits.split([n for _, n in scored])):
embeddings[i] = values
return EmbeddingPoolerOutput(embeddings=embeddings)
@@ -1452,8 +1452,11 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
def _validate_joint_schema_request(
self, obj: Union[GenerateReqInput, EmbeddingReqInput], input_ids: List[int]
) -> None:
"""A Clef checkpoint scores the spans of a decision layout, and nothing else."""
if self.model_config.joint_head_config is None:
"""A Clef or Kev checkpoint scores the spans of a decision layout, and nothing else."""
if (
self.model_config.joint_head_config is None
and self.model_config.pointer_head is None
):
return
if isinstance(obj, GenerateReqInput) or obj.decision_layout is None:
# Health checks only need a response, which is an empty embedding.
+81 -4
View File
@@ -17,10 +17,14 @@ import logging
from typing import Iterable, Optional, Set, Tuple, Union
import torch
from safetensors.torch import load_file
from torch import nn
from transformers.utils import cached_file
from sglang.srt.configs.model_config import load_pointer_adapter, load_pointer_head
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.pointer_head import PointerPooler
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.utils import PPMissingLayer
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
@@ -29,12 +33,16 @@ from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models import qwen3_5
from sglang.srt.models.qwen2_moe import Qwen2MoeSparseMoeBlock
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import get_parallel
from sglang.srt.runtime_context import get_model, get_parallel
from sglang.srt.utils import LazyValue, add_prefix
logger = logging.getLogger(__name__)
_MODEL_PREFIX = "model."
# Where a Kev LoRA checkpoint's base keeps the text backbone, and where the
# adapter keeps its factors.
_BASE_TEXT_PREFIX = "model.language_model."
_ADAPTER_PREFIX = "base_model.model."
class Qwen3_5ForCausalLM(nn.Module):
@@ -70,7 +78,19 @@ class Qwen3_5ForCausalLM(nn.Module):
prefix=add_prefix("model", prefix),
)
if self.pp_group.is_last_rank:
self.pooler = None
model = get_model()
meta = (
load_pointer_head(model.model_path, model.revision)
if model.is_embedding
else None
)
if meta is not None:
# A Kev checkpoint, whose pointer head scores options from the final
# hidden states, and which has no LM head.
self.pooler = PointerPooler(config.hidden_size, meta)
self.lm_head = PPMissingLayer()
elif self.pp_group.is_last_rank:
if self.pp_group.world_size == 1 and config.tie_word_embeddings:
self.lm_head = self.model.embed_tokens
else:
@@ -165,6 +185,7 @@ class Qwen3_5ForCausalLM(nn.Module):
forward_batch: ForwardBatch,
input_embeds: Optional[torch.Tensor] = None,
pp_proxy_tensors: Optional[PPProxyTensors] = None,
get_embedding: bool = False,
**kwargs,
) -> Union[torch.Tensor, PPProxyTensors]:
if self.is_mrope_enabled:
@@ -185,19 +206,55 @@ class Qwen3_5ForCausalLM(nn.Module):
if isinstance(hidden_states, tuple):
hidden_states, aux_hidden_states = hidden_states
if get_embedding and self.pooler is not None:
return self.pooler(hidden_states, forward_batch)
return self.logits_processor(
input_ids, hidden_states, self.lm_head, forward_batch, aux_hidden_states
)
def _pointer_adapter(self) -> dict:
"""LoRA factors of a Kev LoRA checkpoint by backbone module, none for full weights."""
model = get_model()
meta = load_pointer_adapter(model.model_path, model.revision)
if meta is None:
return {}
tensors = load_file(
cached_file(
model.model_path, "adapter_model.safetensors", revision=model.revision
)
)
return {
name[len(_ADAPTER_PREFIX) : -len(".lora_A.weight")]: (
a,
tensors[name.replace(".lora_A.", ".lora_B.")],
meta["lora_scale"],
)
for name, a in tensors.items()
if name.endswith(".lora_A.weight")
}
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]) -> Set[str]:
params_dict = dict(self.named_parameters())
loaded_params: Set[str] = set()
lora = self._pointer_adapter() if self.pooler is not None else {}
# Keep prefix stripping lazy: materializing mmap-backed checkpoint tensors
# across model processes can exceed host memory before GPU copies finish.
def body_weights():
for name, loaded_weight in weights:
if name.startswith(_MODEL_PREFIX):
if self.pooler is not None:
# Kev saves the backbone under its own names, and the base of a
# Kev LoRA checkpoint under its text model, next to an LM head
# that Kev does not use.
name = name.removeprefix(_BASE_TEXT_PREFIX)
if name.startswith("lm_head."):
continue
factors = lora.pop(name.removesuffix(".weight"), None)
if factors is not None:
loaded_weight = _merge_lora(loaded_weight, *factors)
yield name, loaded_weight
elif name.startswith(_MODEL_PREFIX):
yield name[len(_MODEL_PREFIX) :], loaded_weight
elif name == "lm_head.weight":
if self.config.tie_word_embeddings:
@@ -212,9 +269,21 @@ class Qwen3_5ForCausalLM(nn.Module):
loaded_params.add("lm_head.weight")
body_loaded = self.model.load_weights(body_weights())
if lora:
raise ValueError(
f"The Kev adapter has factors for {len(lora)} modules that the "
f"base checkpoint does not have, such as {next(iter(lora))}"
)
loaded_params.update(f"{_MODEL_PREFIX}{n}" for n in body_loaded)
if self.config.tie_word_embeddings and self.pp_group.is_last_rank:
if self.pooler is not None:
model = get_model()
meta = load_pointer_head(model.model_path, model.revision)
self.pooler.load_state_dict(meta["head"], strict=True)
loaded_params.update(
f"pooler.{n}" for n, _ in self.pooler.named_parameters()
)
elif self.config.tie_word_embeddings and self.pp_group.is_last_rank:
loaded_params.add("lm_head.weight")
return loaded_params
@@ -255,4 +324,12 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM):
)
def _merge_lora(
weight: torch.Tensor, a: torch.Tensor, b: torch.Tensor, scale: float
) -> torch.Tensor:
"""The weight plus its LoRA delta, added in FP32 and rounded once, as the Kev
reference merges an adapter."""
return (weight.float() + scale * (b.float() @ a.float())).to(weight.dtype)
EntryClass = [Qwen3_5MoeForCausalLM, Qwen3_5ForCausalLM]
@@ -13,8 +13,9 @@
# ==============================================================================
"""Config loading utilities."""
import copy
from pathlib import Path
from typing import Optional
from typing import Dict, Optional, Tuple
from transformers import PretrainedConfig
from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
@@ -47,6 +48,10 @@ from .common import (
from .gguf_native import build_gguf_config, has_native_gguf_support
from .mistral_utils import is_mistral_model, load_mistral_config
# The base and revision that get_config read for each Kev LoRA checkpoint, which
# has no model config of its own, keyed by checkpoint path and revision.
POINTER_ADAPTER_BASES: Dict[Tuple[str, Optional[str]], Tuple[str, str]] = {}
def _set_architectures(config, arch_name):
config.update({"architectures": [arch_name]})
@@ -370,9 +375,32 @@ def get_config(
model_config_parser = "mistral" if is_mistral_model(model) else "hf"
parser = get_model_config_parser(model_config_parser)
config = parser.parse(
model, trust_remote_code=trust_remote_code, revision=revision, **kwargs
)
try:
config = parser.parse(
model, trust_remote_code=trust_remote_code, revision=revision, **kwargs
)
except (OSError, ValueError):
# A Kev LoRA checkpoint has no model config. It serves the text backbone
# of the base its head.pt pins.
from sglang.srt.configs.model_config import load_pointer_adapter
meta = load_pointer_adapter(model, revision)
if meta is None:
raise
base = parser.parse(
meta["base"],
trust_remote_code=trust_remote_code,
revision=meta["base_revision"],
**kwargs,
)
if base.model_type != "qwen3_5":
raise ValueError(
f"Kev checkpoints on a {base.model_type} base are not supported, "
"only on Qwen3.5 and Qwen3.8 bases"
)
config = copy.deepcopy(base.text_config)
_set_architectures(config, "Qwen3_5ForCausalLM")
POINTER_ADAPTER_BASES[(model, revision)] = (meta["base"], meta["base_revision"])
if model_override_args:
if isinstance(config, DeepseekV41Config):