[Model] Add IQuest Q1 support and MTP draft (#41590)

Co-authored-by: zelong huang <yzhu@ubiquant.com>
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
Co-authored-by: zelong518 <zelonghuang05@gmail.com>
Co-authored-by: chenwenxiaolive <cocoshirolive@gmail.com>
This commit is contained in:
sglang-bot
2026-09-28 17:21:31 -07:00
committed by GitHub
co-authored by zelong huang hnyls2002 zelong518 chenwenxiaolive
parent b95329a74d
commit 79cafec013
33 changed files with 2778 additions and 29 deletions
@@ -22,6 +22,7 @@ from sglang.srt.arg_groups.model_overrides import gpt_oss # noqa: F401
from sglang.srt.arg_groups.model_overrides import granitemoehybrid # noqa: F401
from sglang.srt.arg_groups.model_overrides import inkling # noqa: F401
from sglang.srt.arg_groups.model_overrides import interns2_mobius # noqa: F401
from sglang.srt.arg_groups.model_overrides import iquest_q1 # noqa: F401
from sglang.srt.arg_groups.model_overrides import kimi_k3 # noqa: F401
from sglang.srt.arg_groups.model_overrides import lfm2 # noqa: F401
from sglang.srt.arg_groups.model_overrides import llama4 # noqa: F401
@@ -0,0 +1,48 @@
from typing import Any
from sglang.srt.arg_groups.model_override_base import (
_register_for,
is_attention_backend_not_set,
resolving_view,
)
@_register_for("IQuestQ1ForCausalLM", "IQuestQ1MTP")
def _iquest_q1_overrides(server_args: Any, hf_config: Any) -> dict:
cfg = resolving_view(server_args)
requested = (
cfg.attention_backend,
cfg.prefill_attention_backend,
cfg.decode_attention_backend,
)
if any(backend not in (None, "fa3") for backend in requested):
raise ValueError(
"IQuestQ1 applies its attention sink from the softmax LSE and "
"requires FA3 attention for both prefill and decode."
)
if cfg.speculative_algorithm is not None:
if cfg.speculative_algorithm.upper() not in ("EAGLE", "NEXTN"):
raise ValueError("IQuest Q1 MTP requires serial EAGLE or NEXTN drafting.")
if cfg.speculative_draft_attention_backend not in (None, "fa3"):
raise ValueError("IQuestQ1 requires FA3 attention for the MTP draft.")
if cfg.attn_cp_size > 1 or cfg.dcp_size > 1 or cfg.enable_prefill_cp:
raise ValueError("IQuestQ1 does not support context parallelism.")
if cfg.speculative_eagle_topk is not None and cfg.speculative_eagle_topk > 1:
raise ValueError(
"IQuestQ1 speculative decoding requires --speculative-eagle-topk 1."
)
if cfg.enable_multi_layer_eagle:
raise ValueError("IQuest Q1 MTP drafting does not support multi-layer EAGLE.")
if cfg.enable_dp_attention:
raise ValueError("IQuest Q1 does not support DP attention.")
if cfg.pp_size > 1:
raise ValueError("IQuest Q1 does not support pipeline parallelism.")
overrides = {}
if is_attention_backend_not_set(cfg):
overrides["attention_backend"] = "fa3"
return overrides
if cfg.attention_backend is None and cfg.prefill_attention_backend is None:
overrides["prefill_attention_backend"] = "fa3"
if cfg.attention_backend is None and cfg.decode_attention_backend is None:
overrides["decode_attention_backend"] = "fa3"
return overrides
@@ -910,6 +910,54 @@ def _handle_frozen_kv_mtp(server_args: ServerArgs) -> None:
)
def _handle_iquest_q1_mtp_draft(server_args: ServerArgs) -> bool:
from sglang.srt.configs.iquest_q1 import IQuestQ1MTPConfig
from sglang.srt.utils.hf_transformers_utils import get_config
target = model_config_of(server_args).hf_config
if target.architectures[0] != "IQuestQ1ForCausalLM":
return False
cfg = resolving_view(server_args)
if cfg.speculative_draft_model_path in (None, cfg.model_path):
raise ValueError("IQuest Q1 requires an independent draft model path.")
draft = get_config(
cfg.speculative_draft_model_path,
trust_remote_code=cfg.trust_remote_code,
revision=cfg.speculative_draft_model_revision,
)
if not isinstance(draft, IQuestQ1MTPConfig):
raise ValueError("IQuest Q1 requires an independent MTP draft checkpoint.")
if draft.hidden_size != target.hidden_size or draft.vocab_size != target.vocab_size:
raise ValueError("MTP draft hidden size and vocabulary must match the target.")
if draft.num_target_layers != target.num_hidden_layers:
raise ValueError("MTP draft target layer count must match the target.")
if cfg.speculative_token_map is not None:
raise ValueError("MTP uses its own full-vocabulary embedding and head.")
steps = cfg.speculative_num_steps
if steps is None:
steps = draft.num_draft_slots
if steps < 1:
raise ValueError("MTP requires positive speculative_num_steps.")
if steps > draft.num_draft_slots:
logger.warning(
"MTP depth %d exceeds the checkpoint's num_draft_slots=%d.",
steps,
draft.num_draft_slots,
)
if cfg.speculative_eagle_topk not in (None, 1):
raise ValueError("MTP requires --speculative-eagle-topk 1.")
if cfg.speculative_num_draft_tokens not in (None, steps + 1):
raise ValueError("MTP verification width must equal draft steps + 1.")
declare_resolution(
server_args,
"_handle_iquest_q1_mtp_draft",
speculative_num_steps=steps,
speculative_eagle_topk=1,
speculative_num_draft_tokens=steps + 1,
)
return True
def _handle_eagle_family(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
@@ -994,6 +1042,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
"DeepSeek MTP does not require setting speculative_draft_model_path."
)
_handle_iquest_q1_mtp_draft(server_args)
if not cfg.speculative_adaptive and cfg.speculative_num_steps is None:
assert (
cfg.speculative_eagle_topk is None
+6
View File
@@ -33,6 +33,10 @@ from sglang.srt.configs.interns2_mobius import (
InternS2MobiusVisionConfig,
)
from sglang.srt.configs.interns2preview import InternS2PreviewConfig
from sglang.srt.configs.iquest_q1 import (
IQuestQ1Config,
IQuestQ1MTPConfig,
)
from sglang.srt.configs.janus_pro import MultiModalityConfig
from sglang.srt.configs.jet_nemotron import JetNemotronConfig
from sglang.srt.configs.jet_vlm import JetVLMConfig
@@ -125,6 +129,8 @@ __all__ = [
"Qwen3_5TextConfig",
"Qwen3_5MoeTextConfig",
"InternS2PreviewConfig",
"IQuestQ1Config",
"IQuestQ1MTPConfig",
"InternS2MobiusConfig",
"InternS2MobiusTextConfig",
"InternS2MobiusVisionConfig",
+217
View File
@@ -0,0 +1,217 @@
from typing import List, Optional
from transformers import PretrainedConfig
class IQuestQ1Config(PretrainedConfig):
model_type = "iquest_q1"
def __init__(
self,
vocab_size: int = 160000,
hidden_size: int = 3072,
dense_intermediate_size: int = 12288,
intermediate_size: int = 1536,
num_hidden_layers: int = 88,
num_attention_heads: int = 48,
num_key_value_heads: int = 8,
head_dim: int = 128,
hidden_act: str = "silu",
max_position_embeddings: int = 524288,
rms_norm_eps: float = 1e-6,
use_cache: bool = True,
pad_token_id: int = 0,
bos_token_id: Optional[int] = None,
eos_token_id: int = 0,
tie_word_embeddings: bool = False,
rope_theta: float = 1000000.0,
rope_scaling: Optional[dict] = None,
attention_bias: bool = False,
attention_dropout: float = 0.0,
rotary_dim: int = 32,
rope_parameters: Optional[dict] = None,
no_rope_layers: Optional[List[int]] = None,
swa_rope_theta: float = 10000.0,
sliding_window: int = 4096,
use_hybrid_layers: bool = True,
use_sliding_window: bool = True,
first_layers_types: Optional[List[str]] = None,
hybrid_layers_types_block: Optional[List[str]] = None,
num_hybrid_layers_block: int = 21,
last_layers_types: Optional[List[str]] = None,
mlp_only_layers: Optional[List[int]] = None,
num_experts: int = 256,
num_experts_per_tok: int = 8,
moe_router_dtype: str = "fp32",
enable_sink_attention: bool = True,
first_layer_attn_out_scale: float = 1.0,
first_layer_ffn_out_scale: float = 1.0,
attn_out_scale: float = 1.0,
ffn_out_scale: float = 0.53881590608,
softmax_scale: Optional[float] = None,
logit_scale: float = 1.0,
enable_lm_head_fp32: bool = True,
num_mtp_layers: int = 0,
**kwargs,
):
if first_layers_types is None:
first_layers_types = ["full_attention"]
if hybrid_layers_types_block is None:
hybrid_layers_types_block = [
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
]
if last_layers_types is None:
last_layers_types = [
"full_attention",
"full_attention",
"full_attention",
]
if mlp_only_layers is None:
mlp_only_layers = [0]
layer_types = (
list(first_layers_types)
+ list(hybrid_layers_types_block) * num_hybrid_layers_block
+ list(last_layers_types)
)
if not use_hybrid_layers:
layer_types = ["full_attention"] * num_hidden_layers
if len(layer_types) != num_hidden_layers:
raise ValueError(
"IQuest Q1 layer type pattern has length "
f"{len(layer_types)}, expected {num_hidden_layers}"
)
if any(t not in ("full_attention", "sliding_attention") for t in layer_types):
raise ValueError(
"IQuest Q1 supports full_attention and sliding_attention layers"
)
if not use_sliding_window:
layer_types = ["full_attention"] * num_hidden_layers
if rotary_dim is not None and (
rotary_dim <= 0 or rotary_dim > head_dim or rotary_dim % 2
):
raise ValueError("rotary_dim must be even, positive, and at most head_dim")
if moe_router_dtype != "fp32":
raise ValueError("IQuest Q1 requires moe_router_dtype='fp32'.")
if kwargs.get("use_over_encoding", False):
raise ValueError("IQuest Q1 over-encoding embeddings are not supported.")
if enable_lm_head_fp32 is not True:
raise ValueError("IQuest Q1 requires enable_lm_head_fp32=True.")
kwargs.pop("layer_types", None)
kwargs.setdefault("architectures", ["IQuestQ1ForCausalLM"])
kwargs.setdefault("dtype", "bfloat16")
super().__init__(
pad_token_id=pad_token_id,
bos_token_id=bos_token_id,
eos_token_id=eos_token_id,
tie_word_embeddings=tie_word_embeddings,
rope_scaling=rope_scaling,
**kwargs,
)
self.vocab_size = vocab_size
self.hidden_size = hidden_size
self.dense_intermediate_size = dense_intermediate_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.num_key_value_heads = num_key_value_heads
self.head_dim = head_dim
self.hidden_act = hidden_act
self.max_position_embeddings = max_position_embeddings
self.rms_norm_eps = rms_norm_eps
self.use_cache = use_cache
self.rope_theta = rope_theta
self.rope_scaling = rope_scaling
self.attention_bias = attention_bias
self.attention_dropout = attention_dropout
self.rotary_dim = rotary_dim
self.swa_rope_theta = swa_rope_theta
self.sliding_window = sliding_window if use_sliding_window else None
self.use_hybrid_layers = use_hybrid_layers
self.use_sliding_window = use_sliding_window
self.no_rope_layers = list(no_rope_layers or [])
self.first_layers_types = list(first_layers_types)
self.hybrid_layers_types_block = list(hybrid_layers_types_block)
self.num_hybrid_layers_block = num_hybrid_layers_block
self.last_layers_types = list(last_layers_types)
self.layer_types = layer_types
self.hybrid_layer_pattern = [
int(layer_type == "sliding_attention") for layer_type in layer_types
]
self.is_hybrid_swa = any(self.hybrid_layer_pattern)
self.mlp_only_layers = list(mlp_only_layers)
self.num_experts = num_experts
self.num_experts_per_tok = num_experts_per_tok
self.moe_router_dtype = moe_router_dtype
self.enable_sink_attention = enable_sink_attention
self.first_layer_attn_out_scale = first_layer_attn_out_scale
self.first_layer_ffn_out_scale = first_layer_ffn_out_scale
self.attn_out_scale = attn_out_scale
self.ffn_out_scale = ffn_out_scale
self.softmax_scale = softmax_scale
self.logit_scale = logit_scale
self.enable_lm_head_fp32 = enable_lm_head_fp32
self.num_mtp_layers = num_mtp_layers
self.rope_parameters = dict(rope_parameters or rope_scaling or {})
self.rope_parameters.setdefault(
"rope_type", self.rope_parameters.get("type", "default")
)
self.rope_parameters.setdefault("rope_theta", rope_theta)
if rotary_dim is not None:
self.rope_parameters["partial_rotary_factor"] = rotary_dim / head_dim
class IQuestQ1MTPConfig(IQuestQ1Config):
model_type = "iquest_q1_mtp"
def __init__(
self,
target_config: Optional[dict] = None,
num_draft_slots: int = 7,
num_target_layers: Optional[int] = None,
sliding_window: Optional[int] = 512,
swa_rope_theta: Optional[float] = 10000.0,
fp32_residual_connection: bool = True,
**kwargs,
):
if kwargs.get("num_hidden_layers", 1) != 1:
raise ValueError("MTP supports one physical draft layer.")
if num_draft_slots < 1:
raise ValueError("num_draft_slots must be positive.")
window = sliding_window or None
if window is not None and (window < 0 or not swa_rope_theta):
raise ValueError("A positive draft window requires swa_rope_theta.")
config = dict(target_config or {})
if num_target_layers is None:
num_target_layers = config.get("num_hidden_layers", 88)
if num_target_layers < 1:
raise ValueError("num_target_layers must be positive.")
if config.get("num_hidden_layers", num_target_layers) != num_target_layers:
raise ValueError("num_target_layers must match target_config.")
for key in ("model_type", "architectures", "auto_map", "_name_or_path"):
config.pop(key, None)
config.update(kwargs)
config.update(
architectures=["IQuestQ1MTP"],
num_hidden_layers=1,
mlp_only_layers=[],
no_rope_layers=[],
use_hybrid_layers=True,
use_sliding_window=window is not None,
sliding_window=window,
first_layers_types=[
"sliding_attention" if window is not None else "full_attention"
],
hybrid_layers_types_block=[],
num_hybrid_layers_block=0,
last_layers_types=[],
)
if swa_rope_theta is not None:
config["swa_rope_theta"] = swa_rope_theta
super().__init__(**config)
self.target_config = dict(target_config or {})
self.num_draft_slots = num_draft_slots
self.num_target_layers = num_target_layers
self.fp32_residual_connection = fp32_residual_connection
@@ -88,6 +88,8 @@ SWA_SINK_ARCHS = frozenset(
"GptOssForCausalLM",
"GraniteSWAForCausalLM",
"GraniteMoeSWAForCausalLM",
"IQuestQ1ForCausalLM",
"IQuestQ1MTP",
}
)
@@ -106,7 +106,10 @@ from sglang.srt.parser.jinja_template_utils import (
MEDIA_URL_PART_TYPES,
process_content_for_template_format,
)
from sglang.srt.parser.reasoning_parser import ReasoningParser
from sglang.srt.parser.reasoning_parser import (
IQuestQ1ReasoningDetector,
ReasoningParser,
)
from sglang.srt.sampling.sampling_params import (
set_request_reasoning_end_token_ids,
)
@@ -2564,12 +2567,12 @@ class OpenAIServingChat(OpenAIServingBase):
# as constraint (mirrors the streaming path). For auto: always try.
if self.tool_call_parser:
parser = FunctionCallParser(
tools, self.tool_call_parser, tokenizer=self.tokenizer_manager.tokenizer
)
detector_owns_format = (
parser.detector.supports_structural_tag()
or parser.detector.parses_required_natively()
tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
tool_choice=tool_choice,
)
detector_owns_format = parser.owns_tool_format()
should_try_parser = not is_required or detector_owns_format
if should_try_parser and parser.has_tool_call(text):
try:
@@ -2844,6 +2847,14 @@ class OpenAIServingChat(OpenAIServingBase):
request.reasoning_effort = "medium" if enabled else "no_think"
return
if self.reasoning_parser == "iquest_q1":
request.chat_template_kwargs = {
**(request.chat_template_kwargs or {}),
"thinking": enabled,
"enable_thinking": enabled,
}
return
if self.reasoning_parser == "inkling":
# Effort-conditioned, not toggled: "none" (0.0) is the off switch.
if not enabled:
@@ -2902,6 +2913,11 @@ class OpenAIServingChat(OpenAIServingBase):
if not self.reasoning_parser:
return False
if self.reasoning_parser == "iquest_q1":
return IQuestQ1ReasoningDetector.thinking_enabled(
request.chat_template_kwargs or {}
)
if self.reasoning_parser == "minimax-m3":
# M3 template prefills <mm:think> for thinking_mode=enabled, so it never
# appears in output and reasoning must be forced. Mirrors reasoning_parser.py.
@@ -3008,11 +3024,9 @@ class OpenAIServingChat(OpenAIServingBase):
tools=effective_tools,
tool_call_parser=self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
tool_choice=request.tool_choice,
)
use_native_parser = (
probe.detector.supports_structural_tag()
or probe.detector.parses_required_natively()
)
use_native_parser = probe.owns_tool_format()
if use_native_parser:
parser_dict[index] = probe
else:
@@ -1081,6 +1081,7 @@ class OpenAIServingResponses(OpenAIServingChat):
chat_tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
tool_choice=tool_choice,
)
detector_owns_format = self._tool_parser_owns_format(parser)
should_try_native = not is_required or detector_owns_format
@@ -1166,10 +1167,7 @@ class OpenAIServingResponses(OpenAIServingChat):
@staticmethod
def _tool_parser_owns_format(parser: FunctionCallParser) -> bool:
return (
parser.detector.supports_structural_tag()
or parser.detector.parses_required_natively()
)
return parser.owns_tool_format()
@staticmethod
def _chat_tool_choice(tool_choice: Any) -> Any:
@@ -2011,6 +2009,7 @@ class OpenAIServingResponses(OpenAIServingChat):
chat_tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
tool_choice=tool_choice,
)
detector_owns_format = self._tool_parser_owns_format(probe)
if is_required and not detector_owns_format:
@@ -2020,6 +2019,7 @@ class OpenAIServingResponses(OpenAIServingChat):
chat_tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
tool_choice=tool_choice,
)
reasoning_parser_obj: Optional[ReasoningParser] = None
if self.reasoning_parser:
@@ -367,6 +367,9 @@ class BaseFormatDetector(ABC):
constraints and parse the model's native output format instead."""
return False
def get_required_tool_parser(self, tool_choice):
return None
@abstractmethod
def structure_info(self) -> _GetInfoFunc:
"""
@@ -35,6 +35,7 @@ from sglang.srt.function_call.hermes_detector import HermesDetector
from sglang.srt.function_call.hunyuan_detector import HunyuanDetector
from sglang.srt.function_call.inkling_detector import InklingDetector
from sglang.srt.function_call.internlm_detector import InternlmDetector
from sglang.srt.function_call.iquest_q1_detector import IQuestQ1Detector
from sglang.srt.function_call.k2_v3_detector import K2V3Detector
from sglang.srt.function_call.kimik2_detector import KimiK2Detector
from sglang.srt.function_call.kimik3_detector import KimiK3Detector
@@ -113,9 +114,12 @@ class FunctionCallParser:
"gigachat35": GigaChat35Detector,
"gemma4": Gemma4Detector,
"inkling": InklingDetector,
"iquest_q1": IQuestQ1Detector,
}
def __init__(self, tools: List[Tool], tool_call_parser: str, tokenizer=None):
def __init__(
self, tools: List[Tool], tool_call_parser: str, tokenizer=None, tool_choice=None
):
detector_class = self.ToolCallParserEnum.get(tool_call_parser)
if detector_class:
kwargs = {}
@@ -131,10 +135,20 @@ class FunctionCallParser:
detector.use_full_assistant_constraint = not any(
tool.function.strict for tool in tools
)
self.detector = detector
if isinstance(tool_choice, dict):
tool_choice = ToolChoice(function={"name": tool_choice["name"]})
self._required_tool_parser = detector.get_required_tool_parser(tool_choice)
self.detector = self._required_tool_parser or detector
self.tools = tools
self.tool_strict_level = envs.SGLANG_TOOL_STRICT_LEVEL.get()
def owns_tool_format(self) -> bool:
return (
self._required_tool_parser is not None
or self.detector.supports_structural_tag()
or self.detector.parses_required_natively()
)
def has_tool_call(self, text: str) -> bool:
"""
Check if the given text contains a tool call in the format supported by this parser.
@@ -287,7 +301,7 @@ class FunctionCallParser:
try:
if (
isinstance(self.detector, Glm47MoeDetector)
and self.detector.use_full_assistant_constraint
and self.detector.parses_required_natively()
):
functions = (
[
@@ -356,6 +370,14 @@ class FunctionCallParser:
if (
tool_choice == "required" or isinstance(tool_choice, ToolChoice)
) and not self.detector.parses_required_natively():
required_parser = self.detector.get_required_tool_parser(tool_choice)
if required_parser is not None:
return (
"json_schema",
required_parser.get_json_schema_constraint(
self.tools, tool_choice, parallel_tool_calls
),
)
json_schema = get_json_schema_constraint(
self.tools, tool_choice, parallel_tool_calls=parallel_tool_calls
)
@@ -0,0 +1,176 @@
import orjson
from sglang.srt.entrypoints.openai.protocol import ToolChoice
from sglang.srt.function_call.core_types import StreamingParseResult, ToolCallItem
from sglang.srt.function_call.glm47_moe_detector import (
Glm47MoeDetector,
get_argument_type,
)
from sglang.srt.function_call.json_array_parser import JsonArrayParser
from sglang.srt.function_call.utils import get_json_schema_constraint
class IQuestQ1Detector(Glm47MoeDetector):
def __init__(self):
super().__init__()
self.bot_token = "<iquest_tool_call>"
self.eot_token = "</iquest_tool_call>"
self._inside_call = False
def _parse_body(self, body, tools):
cursor = body.find("<arg_key>")
name = body[:cursor].strip() if cursor >= 0 else body.strip()
if not name:
return None
raw = body[cursor:] if cursor >= 0 else ""
pairs = self.func_arg_regex.findall(raw)
if self.func_arg_regex.sub("", raw).strip() or any(
not key.strip() for key, _ in pairs
):
return None
arguments = self._parse_argument_pairs(pairs, name, tools)
for key, value in pairs:
key = key.strip()
if get_argument_type(name, key, tools, arguments) == "string":
arguments[key] = value
return {"name": name, "arguments": arguments}
def detect_and_parse(self, text, tools):
calls = []
normal = []
content_start = 0
search = 0
while (start := text.find(self.bot_token, search)) >= 0:
begin = start + len(self.bot_token)
end = text.find(self.eot_token, begin)
if end < 0:
break
action = self._parse_body(text[begin:end], tools)
search = end + len(self.eot_token)
if action is not None:
for call in self.parse_base_json(action, tools):
call.tool_index = len(calls)
calls.append(call)
normal.append(text[content_start:start])
content_start = search
normal.append(text[content_start:])
return StreamingParseResult(
normal_text="".join(normal),
calls=calls,
)
def parse_streaming_increment(self, new_text, tools):
self._buffer += new_text
normal, calls = [], []
while True:
if not self._inside_call:
start = self._buffer.find(self.bot_token)
if start < 0:
keep = self._ends_with_partial_token(self._buffer, self.bot_token)
end = len(self._buffer) - keep
normal.append(self._buffer[:end])
self._buffer = self._buffer[end:]
break
normal.append(self._buffer[:start])
self._buffer = self._buffer[start + len(self.bot_token) :]
self._inside_call = True
end = self._buffer.find(self.eot_token)
if end < 0:
break
body = self._buffer[:end]
self._buffer = self._buffer[end + len(self.eot_token) :]
self._inside_call = False
action = self._parse_body(body, tools)
if action is None:
normal.append(self.bot_token + body + self.eot_token)
continue
for call in self.parse_base_json(action, tools):
call.tool_index = len(self.prev_tool_call_arr)
calls.append(call)
self.prev_tool_call_arr.append(action)
self.streamed_args_for_tool.append(call.parameters)
self.current_tool_id = call.tool_index
self.current_tool_name_sent = True
return StreamingParseResult(normal_text="".join(normal), calls=calls)
def finish(self, tools):
text = (self.bot_token if self._inside_call else "") + self._buffer
self._buffer = ""
self._inside_call = False
return StreamingParseResult(normal_text=text)
def parses_required_natively(self):
return False
def get_required_tool_parser(self, tool_choice):
if tool_choice != "required" and not isinstance(tool_choice, ToolChoice):
return None
name = (
tool_choice.function.name if isinstance(tool_choice, ToolChoice) else None
)
return IQuestQ1JsonToolParser(name)
def supports_structural_tag(self):
return False
def structure_info(self):
raise NotImplementedError("IQuest Q1 uses XML argument pairs")
class IQuestQ1JsonToolParser(JsonArrayParser):
def __init__(self, tool_name=None):
super().__init__()
self.tool_name = tool_name
def supports_structural_tag(self):
return False
def get_required_tool_parser(self, tool_choice):
return self
def get_json_schema_constraint(self, tools, tool_choice, parallel_tool_calls=True):
if self.tool_name is not None:
return next(
tool.function.parameters or {}
for tool in tools
if tool.function.name == self.tool_name
)
return get_json_schema_constraint(
tools, tool_choice, parallel_tool_calls=parallel_tool_calls
)
def has_tool_call(self, text):
return True
def detect_and_parse(self, text, tools):
value = orjson.loads(text)
if self.tool_name is not None:
if not isinstance(value, dict):
raise ValueError("Named tool arguments must be a JSON object.")
return StreamingParseResult(
calls=[ToolCallItem(tool_index=0, name=self.tool_name, parameters=text)]
)
if not isinstance(value, list) or not all(
isinstance(call, dict) and "name" in call for call in value
):
raise ValueError("Expected a JSON array of named tool calls.")
return StreamingParseResult(calls=self.parse_base_json(value, tools))
def parse_streaming_increment(self, new_text, tools):
if self.tool_name is not None:
if not new_text:
return StreamingParseResult()
name = None if self.current_tool_name_sent else self.tool_name
self.current_tool_name_sent = True
return StreamingParseResult(
calls=[ToolCallItem(tool_index=0, name=name, parameters=new_text)]
)
calls = []
while True:
result = super().parse_streaming_increment(new_text, tools)
calls.extend(result.calls)
if not result.calls:
break
new_text = ""
return StreamingParseResult(calls=calls)
@@ -49,4 +49,5 @@ TOOL_CALL_PARSER_NAMES = [
"gigachat35",
"gemma4",
"inkling",
"iquest_q1",
]
@@ -42,6 +42,7 @@ from sglang.srt.runtime_context import (
get_schedule,
get_spec,
)
from sglang.srt.speculative.eagle_utils import per_step_draft_out_cache_loc
from sglang.srt.speculative.ragged_verify import build_ragged_target_verify_geometry
from sglang.srt.speculative.spec_info import SpecInput, SpeculativeAlgorithm
from sglang.srt.speculative.spec_utils import resolve_num_tokens_per_req
@@ -518,6 +519,23 @@ class FlashAttentionBackend(AttentionBackend):
else:
self._init_full_cg_decode_metadata(forward_batch, in_capture)
def _get_swa_write_locations(self, forward_batch: ForwardBatch):
locations = getattr(forward_batch, "out_cache_loc", None)
if (
locations is not None
and forward_batch.forward_mode.is_decode_or_idle()
and forward_batch.spec_info is not None
and self.topk > 0
and self.speculative_num_steps > 1
):
locations = per_step_draft_out_cache_loc(
locations,
forward_batch.batch_size,
self.topk,
self.speculative_num_steps,
)[self.speculative_step_id]
return locations
def _init_full_cg_decode_metadata(
self, forward_batch: ForwardBatch, in_capture: bool
):
@@ -534,13 +552,21 @@ class FlashAttentionBackend(AttentionBackend):
# Refill the SWA write-target buffer (bound as a metadata view in
# _bind_metadata_buffers) from the live out_cache_loc before replay.
if self.use_sliding_window_kv_pool and out_cache_loc is not None:
n = out_cache_loc.shape[0]
swa_out_cache_loc = self._get_swa_write_locations(forward_batch)
n = swa_out_cache_loc.shape[0]
assert_buffer_fits(
n,
self.cuda_graph_swa_out_cache_loc.shape[0],
"full decode SWA write locations",
)
self.cuda_graph_swa_out_cache_loc[n:].zero_()
if in_capture:
self.cuda_graph_swa_out_cache_loc[:n].zero_()
else:
self.cuda_graph_swa_out_cache_loc[:n].copy_(
self.kv_index_translator.sliding_window_write_loc_for(out_cache_loc)
self.kv_index_translator.sliding_window_write_loc_for(
swa_out_cache_loc
)
)
if in_capture:
@@ -1148,18 +1174,23 @@ class FlashAttentionBackend(AttentionBackend):
_unified_read = (
self.kv_index_translator.is_translating and metadata.page_table is not None
)
swa_out_cache_loc = (
self._get_swa_write_locations(forward_batch)
if self.use_sliding_window_kv_pool
else None
)
if _unified_read:
kv_view = self.kv_index_translator.index_table_for_batch(forward_batch)
metadata.page_table = kv_view.ids
if self.use_sliding_window_kv_pool:
metadata.swa_page_table = kv_view.sliding_window_ids
if forward_batch.out_cache_loc is not None:
if swa_out_cache_loc is not None:
# The swa write loc was computed from the still-VIRTUAL
# loc at ForwardBatch construction; re-running the
# full->swa map on the kernel-facing loc would be garbage.
metadata.swa_out_cache_loc = (
self.kv_index_translator.sliding_window_write_loc_for(
forward_batch.out_cache_loc
swa_out_cache_loc
)
)
elif self.use_sliding_window_kv_pool:
@@ -1169,10 +1200,10 @@ class FlashAttentionBackend(AttentionBackend):
metadata.page_table
).to(torch.int32)
)
if forward_batch.out_cache_loc is not None:
if swa_out_cache_loc is not None:
metadata.swa_out_cache_loc = (
self.token_to_kv_pool.translate_loc_from_full_to_swa(
forward_batch.out_cache_loc
swa_out_cache_loc
)
)
@@ -1322,9 +1353,13 @@ class FlashAttentionBackend(AttentionBackend):
aux_tensors=None,
rel_bias=None,
rel_bias_event=None,
return_lse: bool = False,
):
if score_mod is not None and self.fa_impl_ver != 4:
raise RuntimeError("score_mod is only supported by the FA4 backend.")
if return_lse and self.use_mla:
raise NotImplementedError("MLA extend does not return an LSE.")
extend_lse = None
cp_active = is_cp_active(forward_batch)
if k is not None:
@@ -1602,13 +1637,19 @@ class FlashAttentionBackend(AttentionBackend):
causal=False if use_cascade_attn else causal,
window_size=window_size,
softcap=layer.logit_cap,
return_softmax_lse=use_cascade_attn,
return_softmax_lse=use_cascade_attn or return_lse,
num_splits=self.num_splits,
out=_fa_out,
ver=self.fa_impl_ver,
**kwargs,
)
if return_lse and not use_cascade_attn:
if not isinstance(result, tuple):
raise RuntimeError("This extend path does not return an LSE.")
o, softmax_lse, *_ = result
extend_lse = softmax_lse.T.contiguous()
result = o
if use_cascade_attn:
o, softmax_lse, *rest = result
if (
@@ -1862,7 +1903,12 @@ class FlashAttentionBackend(AttentionBackend):
else:
o = result
return o.view(-1, layer.tp_q_head_num * layer.v_head_dim)
o = o.view(-1, layer.tp_q_head_num * layer.v_head_dim)
if return_lse:
if extend_lse is None:
raise RuntimeError("This extend path does not return an LSE.")
return o, extend_lse
return o
def forward_decode(
self,
@@ -1883,9 +1929,13 @@ class FlashAttentionBackend(AttentionBackend):
aux_tensors=None,
rel_bias=None,
rel_bias_event=None,
return_lse: bool = False,
) -> torch.Tensor:
if score_mod is not None and self.fa_impl_ver != 4:
raise RuntimeError("score_mod is only supported by the FA4 backend.")
if return_lse and self.use_mla:
raise NotImplementedError("MLA decode does not return an LSE.")
decode_lse = None
if k is not None:
assert v is not None
if save_kv_cache:
@@ -2089,7 +2139,7 @@ class FlashAttentionBackend(AttentionBackend):
causal=False if use_cascade_attn else causal,
window_size=window_size,
softcap=layer.logit_cap,
return_softmax_lse=use_cascade_attn,
return_softmax_lse=use_cascade_attn or return_lse,
num_splits=(
self.decode_num_splits
if not is_swa_layer
@@ -2102,6 +2152,10 @@ class FlashAttentionBackend(AttentionBackend):
scheduler_metadata=sched_meta,
**kwargs,
)
if return_lse and not use_cascade_attn:
o, softmax_lse, *_ = result
decode_lse = softmax_lse.T.contiguous()
result = o
if use_cascade_attn:
o, softmax_lse, *rest = result
if self._decode_uses_static_max_seqlen_k:
@@ -2211,7 +2265,12 @@ class FlashAttentionBackend(AttentionBackend):
else:
o = result
return o.view(-1, layer.tp_q_head_num * layer.v_head_dim)
o = o.view(-1, layer.tp_q_head_num * layer.v_head_dim)
if return_lse:
if decode_lse is None:
raise RuntimeError("This decode path does not return an LSE.")
return o, decode_lse
return o
def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int):
"""Initialize CUDA graph state for the attention backend.
@@ -193,7 +193,7 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
return output
matched = finished_reason.get("matched", None)
if not matched:
if matched is None:
return output
# TODO(lmzheng): handle the case where multiple stop strs are hit
+732
View File
@@ -0,0 +1,732 @@
import logging
from typing import Iterable, Optional, Tuple
import torch
import triton
import triton.language as tl
from torch import nn
from transformers import PretrainedConfig
from triton.language.extra import libdevice
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import post_experts_all_reduce
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.utils import RoutingMethodType
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.fp8 import Fp8Config
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers, set_weight_attrs
logger = logging.getLogger(__name__)
@triton.jit(do_not_specialize=["num_tokens"])
def _apply_learned_sink_kernel(
query,
sink_key,
attn_output,
lse,
result,
num_tokens,
query_stride_t: tl.constexpr,
query_stride_h: tl.constexpr,
query_stride_d: tl.constexpr,
sink_stride_h: tl.constexpr,
sink_stride_d: tl.constexpr,
output_stride_t: tl.constexpr,
output_stride_h: tl.constexpr,
output_stride_d: tl.constexpr,
lse_stride_t: tl.constexpr,
lse_stride_h: tl.constexpr,
NUM_HEADS: tl.constexpr,
QUERIES_PER_KV: tl.constexpr,
HEAD_DIM: tl.constexpr,
SCALE: tl.constexpr,
BLOCK_ROWS: tl.constexpr,
BLOCK_D: tl.constexpr,
):
rows = tl.program_id(0).to(tl.int64) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
tokens = rows // NUM_HEADS
heads = rows % NUM_HEADS
dims = tl.arange(0, BLOCK_D)
mask = (tokens[:, None] < num_tokens) & (dims[None, :] < HEAD_DIM)
q = tl.load(
query
+ tokens[:, None] * query_stride_t
+ heads[:, None] * query_stride_h
+ dims[None, :] * query_stride_d,
mask,
0,
)
sink = tl.load(
sink_key
+ (heads[:, None] // QUERIES_PER_KV) * sink_stride_h
+ dims[None, :] * sink_stride_d,
mask,
0,
).to(q.dtype)
products = q.to(tl.float32) * sink.to(tl.float32)
if HEAD_DIM == 128:
# Keep the reference reduction grouping to limit FP32 rounding drift.
even, odd = tl.split(products.reshape(BLOCK_ROWS, HEAD_DIM // 2, 2))
first, third = tl.split(even.reshape(BLOCK_ROWS, HEAD_DIM // 4, 2))
second, fourth = tl.split(odd.reshape(BLOCK_ROWS, HEAD_DIM // 4, 2))
sink_dot = tl.sum(((first + second) + third) + fourth, axis=1)
else:
sink_dot = tl.sum(products, axis=1)
sink_logit = sink_dot * SCALE
normal_lse = tl.load(
lse + tokens * lse_stride_t + heads * lse_stride_h,
tokens < num_tokens,
0,
).to(tl.float32)
factor = tl.div_rn(1.0, 1.0 + libdevice.exp(sink_logit - normal_lse))
values = tl.load(
attn_output
+ tokens[:, None] * output_stride_t
+ heads[:, None] * output_stride_h
+ dims[None, :] * output_stride_d,
mask,
0,
).to(tl.float32)
tl.store(
result + rows[:, None] * HEAD_DIM + dims[None, :],
values * factor[:, None],
mask,
)
def _apply_learned_sink(
query: torch.Tensor,
sink_key: torch.Tensor,
attn_output: torch.Tensor,
lse: torch.Tensor,
scale: float,
) -> torch.Tensor:
num_tokens, num_heads, head_dim = query.shape
assert attn_output.shape == query.shape
assert sink_key.ndim == 2 and sink_key.shape[1] == head_dim
assert num_heads % sink_key.shape[0] == 0
assert lse.shape == (num_tokens, num_heads)
result = torch.empty(query.shape, dtype=attn_output.dtype, device=query.device)
if num_tokens == 0:
return result
# Larger tiles reduce prefill overhead but cost more on short decode batches.
block_rows = 64 if num_tokens * num_heads >= 32768 else 4
_apply_learned_sink_kernel[(triton.cdiv(num_tokens * num_heads, block_rows),)](
query,
sink_key,
attn_output,
lse,
result,
num_tokens,
*query.stride(),
*sink_key.stride(),
*attn_output.stride(),
*lse.stride(),
NUM_HEADS=num_heads,
QUERIES_PER_KV=num_heads // sink_key.shape[0],
HEAD_DIM=head_dim,
SCALE=scale,
BLOCK_ROWS=block_rows,
BLOCK_D=triton.next_power_of_2(head_dim),
num_warps=4,
# Avoid fused multiply-add rounding differences from the reference.
enable_fp_fusion=False,
)
return result
class IQuestQ1RMSNorm(nn.Module):
def __init__(self, hidden_size: int, eps: float = 1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.variance_epsilon = eps
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
# The draft passes FP32 residuals with BF16 weights, which the fused CUDA
# RMSNorm rejects; it also rounds BF16 inputs differently.
input_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float32)
variance = hidden_states.pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
result = (self.weight * hidden_states).to(input_dtype)
return result
def extra_repr(self):
return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
class IQuestQ1MoEBlock(nn.Module):
def __init__(
self,
config: PretrainedConfig,
layer_id: int,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
):
super().__init__()
self.hidden_size = config.hidden_size
self.num_experts = config.num_experts
self.top_k = config.num_experts_per_tok
self.gate = ReplicatedLinear(
config.hidden_size,
config.num_experts,
bias=False,
params_dtype=torch.float32,
quant_config=None,
prefix=add_prefix("gate", prefix),
)
self.topk = TopK(
top_k=self.top_k,
renormalize=True,
layer_id=layer_id,
)
self.experts = get_moe_impl_class(quant_config)(
num_experts=config.num_experts,
top_k=self.top_k,
layer_id=layer_id,
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
quant_config=quant_config,
prefix=add_prefix("experts", prefix),
routing_method_type=RoutingMethodType.Renormalize,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
original_shape = hidden_states.shape
hidden_states = hidden_states.reshape(-1, self.hidden_size)
router_logits, _ = self.gate(hidden_states.float())
topk_output = self.topk(hidden_states, router_logits)
output = self.experts(hidden_states, topk_output)
output = post_experts_all_reduce(output)
return output.reshape(original_shape)
class IQuestQ1DenseMLP(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
):
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
config.hidden_size,
[config.dense_intermediate_size] * 2,
bias=False,
quant_config=quant_config,
prefix=add_prefix("gate_up_proj", prefix),
)
self.down_proj = RowParallelLinear(
config.dense_intermediate_size,
config.hidden_size,
bias=False,
quant_config=quant_config,
prefix=add_prefix("down_proj", prefix),
)
if config.hidden_act != "silu":
raise ValueError(
f"Unsupported activation: {config.hidden_act}. Only silu is supported."
)
self.act_fn = SiluAndMul()
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
gate_up, _ = self.gate_up_proj(hidden_states)
hidden_states = self.act_fn(gate_up)
hidden_states, _ = self.down_proj(hidden_states)
return hidden_states
class IQuestQ1Attention(nn.Module):
def __init__(
self,
config: PretrainedConfig,
layer_id: int,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
):
super().__init__()
parallel = get_parallel()
attn_tp_size = parallel.attn_tp_size
attn_tp_rank = parallel.attn_tp_rank
self.total_num_heads = config.num_attention_heads
self.total_num_kv_heads = config.num_key_value_heads
assert self.total_num_heads % attn_tp_size == 0
if self.total_num_kv_heads >= attn_tp_size:
assert self.total_num_kv_heads % attn_tp_size == 0
else:
assert attn_tp_size % self.total_num_kv_heads == 0
self.num_heads = self.total_num_heads // attn_tp_size
self.num_kv_heads = max(1, self.total_num_kv_heads // attn_tp_size)
self.head_dim = config.head_dim
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim
self.scaling = (
self.head_dim**-0.5
if config.softmax_scale is None
else config.softmax_scale
)
self.attn_tp_rank = attn_tp_rank
self.attn_tp_size = attn_tp_size
self.qkv_proj = QKVParallelLinear(
config.hidden_size,
self.head_dim,
self.total_num_heads,
self.total_num_kv_heads,
bias=False,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
kv_tp_rank=attn_tp_rank,
kv_tp_size=attn_tp_size,
prefix=add_prefix("qkv_proj", prefix),
)
self.o_proj = RowParallelLinear(
self.total_num_heads * self.head_dim,
config.hidden_size,
bias=False,
quant_config=quant_config,
tp_rank=attn_tp_rank,
tp_size=attn_tp_size,
prefix=add_prefix("o_proj", prefix),
)
self.q_norm = IQuestQ1RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = IQuestQ1RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.is_sliding = (
config.layer_types[layer_id] == "sliding_attention"
and config.sliding_window is not None
)
rope_theta = (
config.swa_rope_theta
if self.is_sliding
else config.rope_parameters["rope_theta"]
)
self.rotary_emb = (
get_rope(
self.head_dim,
rotary_dim=int(
self.head_dim
* config.rope_parameters.get("partial_rotary_factor", 1.0)
),
max_position=config.max_position_embeddings,
base=rope_theta,
rope_scaling=config.rope_parameters,
is_neox_style=True,
)
if layer_id not in config.no_rope_layers
else None
)
sliding_window_size = config.sliding_window - 1 if self.is_sliding else -1
self.attn = RadixAttention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
layer_id=layer_id,
quant_config=quant_config,
prefix=add_prefix("attn", prefix),
sliding_window_size=sliding_window_size,
)
self.enable_sink_attention = config.enable_sink_attention
if self.enable_sink_attention:
self.sink_k = nn.Parameter(
torch.zeros(self.num_kv_heads, self.head_dim), requires_grad=False
)
set_weight_attrs(self.sink_k, {"weight_loader": self._sink_k_loader})
def _sink_k_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor) -> None:
if loaded_weight.dim() == 3:
loaded_weight = loaded_weight.squeeze(0)
if self.total_num_kv_heads >= self.attn_tp_size:
start = self.attn_tp_rank * self.num_kv_heads
else:
replicas = self.attn_tp_size // self.total_num_kv_heads
start = self.attn_tp_rank // replicas
local_weight = loaded_weight[start : start + self.num_kv_heads]
assert local_weight.shape == param.shape, (
f"Unexpected sink_k shard shape {tuple(local_weight.shape)} for "
f"rank-local parameter {tuple(param.shape)}"
)
param.data.copy_(local_weight)
def _apply_zero_value_sink(
self, q: torch.Tensor, attn_output: torch.Tensor, lse: torch.Tensor
) -> torch.Tensor:
q = q.view(-1, self.num_heads, self.head_dim)
return _apply_learned_sink(
q,
self.sink_k,
attn_output.view(q.shape),
lse,
self.scaling,
).view(-1, self.q_size)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q = self.q_norm(q.view(-1, self.num_heads, self.head_dim)).view(q.shape)
k = self.k_norm(k.view(-1, self.num_kv_heads, self.head_dim)).view(k.shape)
if self.rotary_emb is not None:
q, k = self.rotary_emb(positions, q, k)
if self.enable_sink_attention:
attn_output, lse = self.attn(q, k, v, forward_batch, return_lse=True)
attn_output = self._apply_zero_value_sink(q, attn_output, lse)
else:
attn_output = self.attn(q, k, v, forward_batch)
output, _ = self.o_proj(attn_output)
return output
class IQuestQ1DecoderLayer(nn.Module):
def __init__(
self,
config: PretrainedConfig,
layer_id: int,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
):
super().__init__()
self.attention_norm = IQuestQ1RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_out_norm = IQuestQ1RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.feed_forward_norm = IQuestQ1RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.self_attn = IQuestQ1Attention(
config=config,
layer_id=layer_id,
quant_config=quant_config,
prefix=add_prefix("self_attn", prefix),
)
if layer_id in (config.mlp_only_layers or []):
self.mlp = IQuestQ1DenseMLP(
config=config,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
)
else:
self.mlp = IQuestQ1MoEBlock(
config=config,
layer_id=layer_id,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
)
self.is_first_layer = layer_id == 0
if self.is_first_layer:
self.ffn_out_norm = IQuestQ1RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_out_scale = config.first_layer_attn_out_scale
self.ffn_out_scale = config.first_layer_ffn_out_scale
else:
self.attn_out_scale = config.attn_out_scale
self.ffn_out_scale = config.ffn_out_scale
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
) -> torch.Tensor:
# Intentional: layer 0 adds the raw input and an FFN output norm, later layers
# add the normalized input without one; the MTP draft follows layer 0.
if self.is_first_layer:
norm_hidden_states = self.attention_norm(hidden_states)
attn_output = self.self_attn(positions, norm_hidden_states, forward_batch)
hidden_states = hidden_states + self.attn_out_norm(attn_output) * (
self.attn_out_scale
)
mlp_output = self.mlp(self.feed_forward_norm(hidden_states))
return hidden_states + self.ffn_out_norm(mlp_output) * self.ffn_out_scale
norm_hidden_states = self.attention_norm(hidden_states)
attn_output = self.self_attn(positions, norm_hidden_states, forward_batch)
hidden_states = norm_hidden_states + self.attn_out_norm(attn_output) * (
self.attn_out_scale
)
mlp_output = self.mlp(self.feed_forward_norm(hidden_states))
return hidden_states + mlp_output * self.ffn_out_scale
class IQuestQ1Model(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
):
super().__init__()
self.config = config
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
prefix=add_prefix("embed_tokens", prefix),
)
self.layers = make_layers(
config.num_hidden_layers,
lambda idx, prefix: IQuestQ1DecoderLayer(
config=config,
layer_id=idx,
quant_config=quant_config,
prefix=prefix,
),
prefix=add_prefix("layers", prefix),
)
self.norm = IQuestQ1RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: Optional[torch.Tensor] = None,
) -> torch.Tensor:
if input_embeds is None:
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
for layer in self.layers:
hidden_states = layer(positions, hidden_states, forward_batch)
return self.norm(hidden_states)
_FUSED_SHARD_LAYOUT = (
("qkv_proj", ("q", "k", "v")),
("gate_up_proj", (0, 1)),
)
def _is_created_by_quantization(
name: str, param: torch.nn.Parameter, quant_config: Optional[QuantizationConfig]
) -> bool:
if getattr(param, "_skip_weight_check", False):
return True
return (
isinstance(quant_config, Fp8Config)
and not quant_config.is_checkpoint_fp8_serialized
and name.rsplit(".", 1)[-1]
in {
"w13_weight_scale",
"w2_weight_scale",
"w13_weight_scale_inv",
"w2_weight_scale_inv",
}
)
def check_all_params_loaded(
params_dict: dict,
loaded: set,
model: str,
loaded_shards: Optional[dict] = None,
quant_config: Optional[QuantizationConfig] = None,
) -> None:
missing = sorted(
name
for name, param in params_dict.items()
if name not in loaded
and not _is_created_by_quantization(name, param, quant_config)
)
partial = []
for name, shards in sorted((loaded_shards or {}).items()):
for projection, expected in _FUSED_SHARD_LAYOUT:
if not name.endswith(f".{projection}.weight"):
continue
absent = [shard for shard in expected if shard not in shards]
if absent:
partial.append(f"{name} without {absent}")
if not missing and not partial:
return
problems = []
if missing:
problems.append(
f"{len(missing)} parameters were not found in the checkpoint, "
f"e.g. {missing[:8]}"
)
if partial:
problems.append(
f"{len(partial)} fused parameters were only partially loaded, "
f"e.g. {partial[:8]}"
)
message = f"{model}: " + "; ".join(problems)
if quant_config is not None and not partial:
logger.warning(
"%s; verify whether these parameters are generated by quantization.",
message,
)
return
raise ValueError(message)
def load_iquest_q1_weights(
model: nn.Module, weights: Iterable[Tuple[str, torch.Tensor]]
):
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(model.named_parameters())
loaded = set()
loaded_shards = {}
for name, loaded_weight in weights:
if (
name == "lm_head.weight"
and name not in params_dict
and "model.embed_tokens.weight" in params_dict
):
name = "model.embed_tokens.weight"
if "rotary_emb.inv_freq" in name:
continue
if name.endswith("experts.fc"):
param_name = name.replace("experts.fc", "experts.w13_weight")
if param_name not in params_dict:
raise ValueError(f"Unexpected IQuest Q1 weight: {name}")
param = params_dict[param_name]
loaded.add(param_name)
for expert_id in range(model.config.num_experts):
weight_loader = param.weight_loader
weight_loader(
param,
loaded_weight[expert_id][: model.config.intermediate_size],
param_name,
shard_id="w1",
expert_id=expert_id,
)
weight_loader(
param,
loaded_weight[expert_id][model.config.intermediate_size :],
param_name,
shard_id="w3",
expert_id=expert_id,
)
continue
if name.endswith("experts.proj"):
param_name = name.replace("experts.proj", "experts.w2_weight")
if param_name not in params_dict:
raise ValueError(f"Unexpected IQuest Q1 weight: {name}")
param = params_dict[param_name]
loaded.add(param_name)
for expert_id in range(model.config.num_experts):
param.weight_loader(
param,
loaded_weight[expert_id],
param_name,
shard_id="w2",
expert_id=expert_id,
)
continue
if "router.weight" in name:
name = name.replace("router.weight", "gate.weight")
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
mapped_name = name.replace(weight_name, param_name)
if mapped_name not in params_dict:
continue
param = params_dict[mapped_name]
param.weight_loader(param, loaded_weight, shard_id)
if mapped_name not in loaded or mapped_name in loaded_shards:
loaded_shards.setdefault(mapped_name, set()).add(shard_id)
loaded.add(mapped_name)
break
else:
if name not in params_dict:
raise ValueError(f"Unexpected IQuest Q1 weight: {name}")
param = params_dict[name]
getattr(param, "weight_loader", default_weight_loader)(param, loaded_weight)
loaded.add(name)
loaded_shards.pop(name, None)
check_all_params_loaded(
params_dict,
loaded,
type(model).__name__,
loaded_shards=loaded_shards,
quant_config=model.quant_config,
)
class IQuestQ1ForCausalLM(nn.Module):
fall_back_to_pt_during_load = False
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
):
super().__init__()
self.config = config
self.quant_config = quant_config
self.model = IQuestQ1Model(
config, quant_config=quant_config, prefix=add_prefix("model", prefix)
)
self.lm_head = (
self.model.embed_tokens
if config.tie_word_embeddings
else ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=add_prefix("lm_head", prefix),
)
)
self.logits_processor = LogitsProcessor(config, logit_scale=config.logit_scale)
def get_input_embeddings(self):
return self.model.embed_tokens
@torch.no_grad()
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: Optional[torch.Tensor] = None,
) -> torch.Tensor:
hidden_states = self.model(input_ids, positions, forward_batch, input_embeds)
return self.logits_processor(
input_ids, hidden_states, self.lm_head, forward_batch
)
def get_embed_and_head(self):
return self.model.embed_tokens.weight, self.lm_head.weight
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
load_iquest_q1_weights(
self,
(
(name, weight)
for name, weight in weights
if not name.startswith("mtp_layers.")
),
)
EntryClass = [IQuestQ1ForCausalLM]
+204
View File
@@ -0,0 +1,204 @@
from typing import Any, Iterable, Optional, Tuple
import torch
from torch import nn
from sglang.srt.configs.iquest_q1 import IQuestQ1MTPConfig
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.models.iquest_q1 import (
IQuestQ1Attention,
IQuestQ1MoEBlock,
IQuestQ1RMSNorm,
load_iquest_q1_weights,
)
from sglang.srt.utils import add_prefix
class IQuestQ1MTPInnerLayer(nn.Module):
def __init__(
self,
config: IQuestQ1MTPConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
):
super().__init__()
self.self_attn = IQuestQ1Attention(
config=config,
layer_id=0,
quant_config=quant_config,
prefix=add_prefix("self_attn", prefix),
)
self.mlp = IQuestQ1MoEBlock(
config=config,
layer_id=0,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
)
self.attention_norm = IQuestQ1RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.attn_out_norm = IQuestQ1RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.feed_forward_norm = IQuestQ1RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.ffn_out_norm = IQuestQ1RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.attn_out_scale = config.first_layer_attn_out_scale
self.ffn_out_scale = config.first_layer_ffn_out_scale
self.fp32_residual_connection = config.fp32_residual_connection
def forward(self, positions, hidden_states, forward_batch):
dtype = hidden_states.dtype
attention = self.self_attn(
positions, self.attention_norm(hidden_states), forward_batch
)
residual = (
hidden_states.float() if self.fp32_residual_connection else hidden_states
)
hidden_states = residual + self.attn_out_norm(attention) * self.attn_out_scale
output = self.mlp(self.feed_forward_norm(hidden_states).to(dtype))
return hidden_states + self.ffn_out_norm(output) * self.ffn_out_scale
class IQuestQ1MTPLayer(nn.Module):
def __init__(
self,
config: IQuestQ1MTPConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
):
super().__init__()
self.enorm = IQuestQ1RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = IQuestQ1RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
self.mtp_model_layer = IQuestQ1MTPInnerLayer(
config=config,
quant_config=quant_config,
prefix=add_prefix("mtp_model_layer", prefix),
)
self.final_layernorm = IQuestQ1RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
def forward(self, positions, previous_hidden_states, input_embeds, forward_batch):
dtype = self.eh_proj.weight.dtype
hidden_states = self.eh_proj(
torch.cat(
[
self.enorm(input_embeds).to(dtype),
self.hnorm(previous_hidden_states).to(dtype),
],
dim=-1,
)
)
hidden_states = self.mtp_model_layer(positions, hidden_states, forward_batch)
return self.final_layernorm(hidden_states).to(dtype)
class IQuestQ1MTPModel(nn.Module):
def __init__(
self,
config: IQuestQ1MTPConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
):
super().__init__()
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
prefix=add_prefix("embed_tokens", prefix),
)
self.mtp_layer = IQuestQ1MTPLayer(
config=config,
quant_config=quant_config,
prefix=add_prefix("mtp_layer", prefix),
)
@property
def layers(self):
return (self.mtp_layer.mtp_model_layer,)
def forward(self, input_ids, positions, forward_batch, input_embeds=None):
if input_embeds is None:
input_embeds = self.embed_tokens(input_ids)
return self.mtp_layer(
positions,
forward_batch.spec_info.hidden_states,
input_embeds,
forward_batch,
)
class IQuestQ1MTP(nn.Module):
fall_back_to_pt_during_load = False
def __init__(
self,
config: IQuestQ1MTPConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
):
super().__init__()
self.config = config
self.quant_config = quant_config
self.model = IQuestQ1MTPModel(
config=config,
quant_config=quant_config,
prefix=add_prefix("model", prefix),
)
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=add_prefix("lm_head", prefix),
)
self.logits_processor = LogitsProcessor(config)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_tokens(input_ids)
def get_embed_and_head(self):
return self.model.embed_tokens.weight, self.lm_head.weight
def set_embed_and_head(self, embed, head):
return
@torch.no_grad()
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: Optional[torch.Tensor] = None,
**kwargs: Any,
) -> torch.Tensor:
hidden_states = self.model(input_ids, positions, forward_batch, input_embeds)
return self.logits_processor(
input_ids, hidden_states, self.lm_head, forward_batch
)
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
def mapped_weights():
for name, weight in weights:
if name == "target_final_norm.weight":
continue
if name == "embed_tokens.weight":
yield "model.embed_tokens.weight", weight
elif name.startswith("mtp."):
yield "model.mtp_layer." + name.removeprefix("mtp."), weight
elif name == "lm_head.weight":
yield name, weight
else:
raise ValueError(f"Unexpected MTP weight: {name}")
load_iquest_q1_weights(self, mapped_weights())
EntryClass = [IQuestQ1MTP]
@@ -396,6 +396,38 @@ class DeepSeekR1Detector(BaseReasoningFormatDetector):
# https://github.com/sgl-project/sglang/pull/3202#discussion_r1950153599
class IQuestQ1ReasoningDetector(BaseReasoningFormatDetector):
@staticmethod
def thinking_enabled(chat_kwargs):
thinking = chat_kwargs.get("thinking")
enable_thinking = chat_kwargs.get("enable_thinking")
return (
True
if thinking is None and enable_thinking is None
else bool(thinking or enable_thinking)
)
def __init__(
self,
stream_reasoning=True,
force_reasoning=True,
continue_final_message=False,
previous_content="",
force_nonempty_content=False,
):
super().__init__(
"<think>",
"</think>",
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
force_nonempty_content=force_nonempty_content,
thinks_internally=True,
reasoning_default="enable_thinking",
)
class Qwen3Detector(BaseReasoningFormatDetector):
"""
Detector for Qwen3 models (e.g., Qwen/Qwen3-235B-A22B).
@@ -2255,6 +2287,7 @@ class ReasoningParser:
"gemma4": Gemma4Detector,
"gigachat35": DeepSeekR1Detector,
"inkling": InklingDetector,
"iquest_q1": IQuestQ1ReasoningDetector,
"cohere_command4": CohereCommand4Detector,
}
@@ -2276,6 +2309,13 @@ class ReasoningParser:
chat_template_kwargs = getattr(request, "chat_template_kwargs", None) or {}
if model_type.lower() == "iquest_q1" and (
request is not None or force_reasoning is None
):
force_reasoning = IQuestQ1ReasoningDetector.thinking_enabled(
chat_template_kwargs
)
# Special cases where we override force_reasoning
if model_type.lower() in {
"qwen3-thinking",
@@ -35,6 +35,7 @@ REASONING_PARSER_NAMES = [
"interns1",
"gemma4",
"inkling",
"iquest_q1",
"cohere_command4",
"gigachat35",
]
@@ -526,11 +526,16 @@ def _is_gigachat35(ctx):
return ctx.has_text("<|GCML|tool_calls>")
def _is_iquest_q1(ctx):
return ctx.has_text("<|iquest_assistant|>")
# ---------------------------------------------------------------------------
# Reasoning parser rules
# ---------------------------------------------------------------------------
REASONING_PARSER_RULES = (
DetectionRule(name="iquest_q1", value="iquest_q1", predicate=_is_iquest_q1),
DetectionRule(name="k2_horizon", value="k2_horizon", predicate=_is_k2_v3),
DetectionRule(name="apertus2509", value="apertus2509", predicate=_is_apertus2509),
DetectionRule(name="gemma4", value="gemma4", predicate=_is_gemma4),
@@ -576,6 +581,7 @@ REASONING_PARSER_RULES = (
# ---------------------------------------------------------------------------
TOOL_CALL_PARSER_RULES = (
DetectionRule(name="iquest_q1", value="iquest_q1", predicate=_is_iquest_q1),
DetectionRule(name="gigachat35", value="gigachat35", predicate=_is_gigachat35),
DetectionRule(name="k2_horizon", value="k2_horizon", predicate=_is_k2_v3),
DetectionRule(name="apertus2509", value="apertus2509", predicate=_is_apertus2509),
@@ -795,6 +801,8 @@ def _architecture_auto_parsers(server_args, needs: Tuple[str, ...]) -> Dict[str,
if "KimiK3" in arch or model_type == "kimi_k3":
reasoning_parser, tool_call_parser = "kimi_k3", "kimi_k3"
elif arch == "IQuestQ1ForCausalLM":
reasoning_parser, tool_call_parser = "iquest_q1", "iquest_q1"
elif arch in (
"BailingMoeV3ForCausalLM",
"BailingMoeV3VLForConditionalGeneration",
@@ -52,6 +52,8 @@ from sglang.srt.configs import (
InternS2MobiusConfig,
InternS2MobiusTextConfig,
InternS2PreviewConfig,
IQuestQ1Config,
IQuestQ1MTPConfig,
JetNemotronConfig,
JetVLMConfig,
K2HorizonConfig,
@@ -162,6 +164,8 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
Qwen3_5TextConfig,
Qwen3_5MoeTextConfig,
InternS2PreviewConfig,
IQuestQ1Config,
IQuestQ1MTPConfig,
InternS2MobiusConfig,
InternS2MobiusTextConfig,
JetNemotronConfig,