diff --git a/docs/cookbook/autoregressive/IQuestLab/IQuest-Q1.mdx b/docs/cookbook/autoregressive/IQuestLab/IQuest-Q1.mdx
new file mode 100644
index 0000000000..8ed4a8107f
--- /dev/null
+++ b/docs/cookbook/autoregressive/IQuestLab/IQuest-Q1.mdx
@@ -0,0 +1,47 @@
+---
+title: IQuest-Q1
+description: "Deploy IQuest-Q1 with SGLang — BF16 recipes for the 320B Mixture-of-Experts model on 8× NVIDIA H200, with optional MTP speculative decoding."
+tag: NEW
+---
+
+## Deployment
+
+
+
+
+
+IQuest-Q1 needs an SGLang build that includes IQuest-Q1 support. The nightly image carries it:
+
+```bash Command
+docker pull lmsysorg/sglang:dev
+```
+
+For other methods, see the [official SGLang installation guide](../../../docs/get-started/install).
+
+
+
+Download the checkpoint first; the MTP draft lives in its `mtp/` subdirectory:
+
+```bash Command
+hf download IQuestLab/IQuest-Q1 --local-dir /model/IQuest-Q1
+```
+
+Pick a recipe to generate the launch command. **Low-Latency** enables MTP speculative decoding; **High-Throughput** serves the target model alone.
+
+import { Deployment } from "/src/snippets/_deployment.jsx";
+import { config } from "/src/snippets/configs/IQuestLab/iquest-q1.jsx";
+
+
+
+## 1. Model Introduction
+
+**IQuest-Q1** is a Mixture-of-Experts language model from IQuestLab with about 320B total parameters and 15B active per token. It has 88 layers that interleave three sliding-window attention layers (4,096-token window) with one full-attention layer, uses learned attention sinks, and routes each token to 8 of 256 experts. It ships with a single-layer MTP draft that SGLang applies recursively for speculative decoding.
+
+**Resources:** [HuggingFace](https://huggingface.co/IQuestLab/IQuest-Q1).
+
+## 2. Configuration Tips
+
+- **Hardware.** IQuest-Q1 requires the FA3 attention backend for both the target and the MTP draft, because the attention sink is applied from the FA3 softmax LSE. SGLang builds FA3 for Hopper, so Blackwell GPUs are not supported. The BF16 weights take about 640 GB, which fits 8× H200 with tensor parallelism 8.
+- **Reasoning.** Thinking is on by default. Pass `"chat_template_kwargs": {"thinking": false}` to turn it off for a request. The `iquest_q1` reasoning parser returns the thinking in `reasoning_content`.
+- **Tool calling.** The `iquest_q1` tool-call parser supports `tool_choice` values `auto`, `required`, and a named function.
+- **MTP depth.** The single draft layer runs 5 sequential steps, one token each, and each round verifies 6 tokens. Keep `--speculative-num-draft-tokens` equal to `--speculative-num-steps` + 1.
diff --git a/docs/cookbook/autoregressive/intro.mdx b/docs/cookbook/autoregressive/intro.mdx
index c8457ecc82..248c5f48b9 100644
--- a/docs/cookbook/autoregressive/intro.mdx
+++ b/docs/cookbook/autoregressive/intro.mdx
@@ -7,6 +7,11 @@ metatags:
---
+
" },
+ CURL_HOST: { target: "curl", label: "Server host", default: "localhost" },
+ CURL_PORT: { target: "curl", label: "Server port", default: "30000" },
+ },
+ curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\
+ -H 'Content-Type: application/json' \\
+ -d '{"model":"{{MODEL_NAME}}","messages":[{"role":"user","content":"What is 15% of 240?"}]}'`,
+ dockerImages: { h200: "lmsysorg/sglang:dev" },
+ dockerMounts: ["\"{{MODEL_ROOT}}:/model:ro\""],
+ github: { cookbookModel: "IQuestLab/IQuest-Q1" },
+ cells: [
+ {
+ match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" },
+ env: [],
+ flags: [
+ "--model-path '{{MODEL_PATH}}'",
+ "--served-model-name {{MODEL_NAME}}",
+ "--tp 8",
+ "--attention-backend fa3",
+ "--mem-fraction-static 0.85",
+ "--disable-prefill-cuda-graph",
+ "--enable-torch-compile",
+ "--reasoning-parser iquest_q1",
+ "--tool-call-parser iquest_q1",
+ "--speculative-algorithm EAGLE",
+ "--speculative-num-steps 5",
+ "--speculative-eagle-topk 1",
+ "--speculative-num-draft-tokens 6",
+ "--speculative-draft-model-path '{{DRAFT_PATH}}'",
+ "--speculative-draft-attention-backend fa3",
+ "--speculative-use-rejection-sampling",
+ "--host {{HOST_IP}}",
+ "--port {{PORT}}",
+ ],
+ },
+ {
+ match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
+ env: [],
+ flags: [
+ "--model-path '{{MODEL_PATH}}'",
+ "--served-model-name {{MODEL_NAME}}",
+ "--tp 8",
+ "--attention-backend fa3",
+ "--mem-fraction-static 0.85",
+ "--disable-prefill-cuda-graph",
+ "--enable-torch-compile",
+ "--reasoning-parser iquest_q1",
+ "--tool-call-parser iquest_q1",
+ "--host {{HOST_IP}}",
+ "--port {{PORT}}",
+ ],
+ },
+ ],
+};
diff --git a/python/sglang/srt/arg_groups/model_overrides/__init__.py b/python/sglang/srt/arg_groups/model_overrides/__init__.py
index 1605d1770e..4a64a30b31 100644
--- a/python/sglang/srt/arg_groups/model_overrides/__init__.py
+++ b/python/sglang/srt/arg_groups/model_overrides/__init__.py
@@ -22,6 +22,7 @@ from sglang.srt.arg_groups.model_overrides import gpt_oss # noqa: F401
from sglang.srt.arg_groups.model_overrides import granitemoehybrid # noqa: F401
from sglang.srt.arg_groups.model_overrides import inkling # noqa: F401
from sglang.srt.arg_groups.model_overrides import interns2_mobius # noqa: F401
+from sglang.srt.arg_groups.model_overrides import iquest_q1 # noqa: F401
from sglang.srt.arg_groups.model_overrides import kimi_k3 # noqa: F401
from sglang.srt.arg_groups.model_overrides import lfm2 # noqa: F401
from sglang.srt.arg_groups.model_overrides import llama4 # noqa: F401
diff --git a/python/sglang/srt/arg_groups/model_overrides/iquest_q1.py b/python/sglang/srt/arg_groups/model_overrides/iquest_q1.py
new file mode 100644
index 0000000000..7e8f1608e5
--- /dev/null
+++ b/python/sglang/srt/arg_groups/model_overrides/iquest_q1.py
@@ -0,0 +1,48 @@
+from typing import Any
+
+from sglang.srt.arg_groups.model_override_base import (
+ _register_for,
+ is_attention_backend_not_set,
+ resolving_view,
+)
+
+
+@_register_for("IQuestQ1ForCausalLM", "IQuestQ1MTP")
+def _iquest_q1_overrides(server_args: Any, hf_config: Any) -> dict:
+ cfg = resolving_view(server_args)
+ requested = (
+ cfg.attention_backend,
+ cfg.prefill_attention_backend,
+ cfg.decode_attention_backend,
+ )
+ if any(backend not in (None, "fa3") for backend in requested):
+ raise ValueError(
+ "IQuestQ1 applies its attention sink from the softmax LSE and "
+ "requires FA3 attention for both prefill and decode."
+ )
+ if cfg.speculative_algorithm is not None:
+ if cfg.speculative_algorithm.upper() not in ("EAGLE", "NEXTN"):
+ raise ValueError("IQuest Q1 MTP requires serial EAGLE or NEXTN drafting.")
+ if cfg.speculative_draft_attention_backend not in (None, "fa3"):
+ raise ValueError("IQuestQ1 requires FA3 attention for the MTP draft.")
+ if cfg.attn_cp_size > 1 or cfg.dcp_size > 1 or cfg.enable_prefill_cp:
+ raise ValueError("IQuestQ1 does not support context parallelism.")
+ if cfg.speculative_eagle_topk is not None and cfg.speculative_eagle_topk > 1:
+ raise ValueError(
+ "IQuestQ1 speculative decoding requires --speculative-eagle-topk 1."
+ )
+ if cfg.enable_multi_layer_eagle:
+ raise ValueError("IQuest Q1 MTP drafting does not support multi-layer EAGLE.")
+ if cfg.enable_dp_attention:
+ raise ValueError("IQuest Q1 does not support DP attention.")
+ if cfg.pp_size > 1:
+ raise ValueError("IQuest Q1 does not support pipeline parallelism.")
+ overrides = {}
+ if is_attention_backend_not_set(cfg):
+ overrides["attention_backend"] = "fa3"
+ return overrides
+ if cfg.attention_backend is None and cfg.prefill_attention_backend is None:
+ overrides["prefill_attention_backend"] = "fa3"
+ if cfg.attention_backend is None and cfg.decode_attention_backend is None:
+ overrides["decode_attention_backend"] = "fa3"
+ return overrides
diff --git a/python/sglang/srt/arg_groups/speculative_hook.py b/python/sglang/srt/arg_groups/speculative_hook.py
index 09078eed9f..539188ba15 100644
--- a/python/sglang/srt/arg_groups/speculative_hook.py
+++ b/python/sglang/srt/arg_groups/speculative_hook.py
@@ -910,6 +910,54 @@ def _handle_frozen_kv_mtp(server_args: ServerArgs) -> None:
)
+def _handle_iquest_q1_mtp_draft(server_args: ServerArgs) -> bool:
+ from sglang.srt.configs.iquest_q1 import IQuestQ1MTPConfig
+ from sglang.srt.utils.hf_transformers_utils import get_config
+
+ target = model_config_of(server_args).hf_config
+ if target.architectures[0] != "IQuestQ1ForCausalLM":
+ return False
+ cfg = resolving_view(server_args)
+ if cfg.speculative_draft_model_path in (None, cfg.model_path):
+ raise ValueError("IQuest Q1 requires an independent draft model path.")
+ draft = get_config(
+ cfg.speculative_draft_model_path,
+ trust_remote_code=cfg.trust_remote_code,
+ revision=cfg.speculative_draft_model_revision,
+ )
+ if not isinstance(draft, IQuestQ1MTPConfig):
+ raise ValueError("IQuest Q1 requires an independent MTP draft checkpoint.")
+ if draft.hidden_size != target.hidden_size or draft.vocab_size != target.vocab_size:
+ raise ValueError("MTP draft hidden size and vocabulary must match the target.")
+ if draft.num_target_layers != target.num_hidden_layers:
+ raise ValueError("MTP draft target layer count must match the target.")
+ if cfg.speculative_token_map is not None:
+ raise ValueError("MTP uses its own full-vocabulary embedding and head.")
+ steps = cfg.speculative_num_steps
+ if steps is None:
+ steps = draft.num_draft_slots
+ if steps < 1:
+ raise ValueError("MTP requires positive speculative_num_steps.")
+ if steps > draft.num_draft_slots:
+ logger.warning(
+ "MTP depth %d exceeds the checkpoint's num_draft_slots=%d.",
+ steps,
+ draft.num_draft_slots,
+ )
+ if cfg.speculative_eagle_topk not in (None, 1):
+ raise ValueError("MTP requires --speculative-eagle-topk 1.")
+ if cfg.speculative_num_draft_tokens not in (None, steps + 1):
+ raise ValueError("MTP verification width must equal draft steps + 1.")
+ declare_resolution(
+ server_args,
+ "_handle_iquest_q1_mtp_draft",
+ speculative_num_steps=steps,
+ speculative_eagle_topk=1,
+ speculative_num_draft_tokens=steps + 1,
+ )
+ return True
+
+
def _handle_eagle_family(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
@@ -994,6 +1042,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
"DeepSeek MTP does not require setting speculative_draft_model_path."
)
+ _handle_iquest_q1_mtp_draft(server_args)
if not cfg.speculative_adaptive and cfg.speculative_num_steps is None:
assert (
cfg.speculative_eagle_topk is None
diff --git a/python/sglang/srt/configs/__init__.py b/python/sglang/srt/configs/__init__.py
index 6c404b17d1..06239f17ee 100644
--- a/python/sglang/srt/configs/__init__.py
+++ b/python/sglang/srt/configs/__init__.py
@@ -33,6 +33,10 @@ from sglang.srt.configs.interns2_mobius import (
InternS2MobiusVisionConfig,
)
from sglang.srt.configs.interns2preview import InternS2PreviewConfig
+from sglang.srt.configs.iquest_q1 import (
+ IQuestQ1Config,
+ IQuestQ1MTPConfig,
+)
from sglang.srt.configs.janus_pro import MultiModalityConfig
from sglang.srt.configs.jet_nemotron import JetNemotronConfig
from sglang.srt.configs.jet_vlm import JetVLMConfig
@@ -125,6 +129,8 @@ __all__ = [
"Qwen3_5TextConfig",
"Qwen3_5MoeTextConfig",
"InternS2PreviewConfig",
+ "IQuestQ1Config",
+ "IQuestQ1MTPConfig",
"InternS2MobiusConfig",
"InternS2MobiusTextConfig",
"InternS2MobiusVisionConfig",
diff --git a/python/sglang/srt/configs/iquest_q1.py b/python/sglang/srt/configs/iquest_q1.py
new file mode 100644
index 0000000000..d0cc8b8dbd
--- /dev/null
+++ b/python/sglang/srt/configs/iquest_q1.py
@@ -0,0 +1,217 @@
+from typing import List, Optional
+
+from transformers import PretrainedConfig
+
+
+class IQuestQ1Config(PretrainedConfig):
+ model_type = "iquest_q1"
+
+ def __init__(
+ self,
+ vocab_size: int = 160000,
+ hidden_size: int = 3072,
+ dense_intermediate_size: int = 12288,
+ intermediate_size: int = 1536,
+ num_hidden_layers: int = 88,
+ num_attention_heads: int = 48,
+ num_key_value_heads: int = 8,
+ head_dim: int = 128,
+ hidden_act: str = "silu",
+ max_position_embeddings: int = 524288,
+ rms_norm_eps: float = 1e-6,
+ use_cache: bool = True,
+ pad_token_id: int = 0,
+ bos_token_id: Optional[int] = None,
+ eos_token_id: int = 0,
+ tie_word_embeddings: bool = False,
+ rope_theta: float = 1000000.0,
+ rope_scaling: Optional[dict] = None,
+ attention_bias: bool = False,
+ attention_dropout: float = 0.0,
+ rotary_dim: int = 32,
+ rope_parameters: Optional[dict] = None,
+ no_rope_layers: Optional[List[int]] = None,
+ swa_rope_theta: float = 10000.0,
+ sliding_window: int = 4096,
+ use_hybrid_layers: bool = True,
+ use_sliding_window: bool = True,
+ first_layers_types: Optional[List[str]] = None,
+ hybrid_layers_types_block: Optional[List[str]] = None,
+ num_hybrid_layers_block: int = 21,
+ last_layers_types: Optional[List[str]] = None,
+ mlp_only_layers: Optional[List[int]] = None,
+ num_experts: int = 256,
+ num_experts_per_tok: int = 8,
+ moe_router_dtype: str = "fp32",
+ enable_sink_attention: bool = True,
+ first_layer_attn_out_scale: float = 1.0,
+ first_layer_ffn_out_scale: float = 1.0,
+ attn_out_scale: float = 1.0,
+ ffn_out_scale: float = 0.53881590608,
+ softmax_scale: Optional[float] = None,
+ logit_scale: float = 1.0,
+ enable_lm_head_fp32: bool = True,
+ num_mtp_layers: int = 0,
+ **kwargs,
+ ):
+ if first_layers_types is None:
+ first_layers_types = ["full_attention"]
+ if hybrid_layers_types_block is None:
+ hybrid_layers_types_block = [
+ "full_attention",
+ "sliding_attention",
+ "sliding_attention",
+ "sliding_attention",
+ ]
+ if last_layers_types is None:
+ last_layers_types = [
+ "full_attention",
+ "full_attention",
+ "full_attention",
+ ]
+ if mlp_only_layers is None:
+ mlp_only_layers = [0]
+ layer_types = (
+ list(first_layers_types)
+ + list(hybrid_layers_types_block) * num_hybrid_layers_block
+ + list(last_layers_types)
+ )
+ if not use_hybrid_layers:
+ layer_types = ["full_attention"] * num_hidden_layers
+ if len(layer_types) != num_hidden_layers:
+ raise ValueError(
+ "IQuest Q1 layer type pattern has length "
+ f"{len(layer_types)}, expected {num_hidden_layers}"
+ )
+ if any(t not in ("full_attention", "sliding_attention") for t in layer_types):
+ raise ValueError(
+ "IQuest Q1 supports full_attention and sliding_attention layers"
+ )
+ if not use_sliding_window:
+ layer_types = ["full_attention"] * num_hidden_layers
+ if rotary_dim is not None and (
+ rotary_dim <= 0 or rotary_dim > head_dim or rotary_dim % 2
+ ):
+ raise ValueError("rotary_dim must be even, positive, and at most head_dim")
+ if moe_router_dtype != "fp32":
+ raise ValueError("IQuest Q1 requires moe_router_dtype='fp32'.")
+ if kwargs.get("use_over_encoding", False):
+ raise ValueError("IQuest Q1 over-encoding embeddings are not supported.")
+ if enable_lm_head_fp32 is not True:
+ raise ValueError("IQuest Q1 requires enable_lm_head_fp32=True.")
+ kwargs.pop("layer_types", None)
+ kwargs.setdefault("architectures", ["IQuestQ1ForCausalLM"])
+ kwargs.setdefault("dtype", "bfloat16")
+ super().__init__(
+ pad_token_id=pad_token_id,
+ bos_token_id=bos_token_id,
+ eos_token_id=eos_token_id,
+ tie_word_embeddings=tie_word_embeddings,
+ rope_scaling=rope_scaling,
+ **kwargs,
+ )
+ self.vocab_size = vocab_size
+ self.hidden_size = hidden_size
+ self.dense_intermediate_size = dense_intermediate_size
+ self.intermediate_size = intermediate_size
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.num_key_value_heads = num_key_value_heads
+ self.head_dim = head_dim
+ self.hidden_act = hidden_act
+ self.max_position_embeddings = max_position_embeddings
+ self.rms_norm_eps = rms_norm_eps
+ self.use_cache = use_cache
+ self.rope_theta = rope_theta
+ self.rope_scaling = rope_scaling
+ self.attention_bias = attention_bias
+ self.attention_dropout = attention_dropout
+ self.rotary_dim = rotary_dim
+ self.swa_rope_theta = swa_rope_theta
+ self.sliding_window = sliding_window if use_sliding_window else None
+ self.use_hybrid_layers = use_hybrid_layers
+ self.use_sliding_window = use_sliding_window
+ self.no_rope_layers = list(no_rope_layers or [])
+ self.first_layers_types = list(first_layers_types)
+ self.hybrid_layers_types_block = list(hybrid_layers_types_block)
+ self.num_hybrid_layers_block = num_hybrid_layers_block
+ self.last_layers_types = list(last_layers_types)
+ self.layer_types = layer_types
+ self.hybrid_layer_pattern = [
+ int(layer_type == "sliding_attention") for layer_type in layer_types
+ ]
+ self.is_hybrid_swa = any(self.hybrid_layer_pattern)
+ self.mlp_only_layers = list(mlp_only_layers)
+ self.num_experts = num_experts
+ self.num_experts_per_tok = num_experts_per_tok
+ self.moe_router_dtype = moe_router_dtype
+ self.enable_sink_attention = enable_sink_attention
+ self.first_layer_attn_out_scale = first_layer_attn_out_scale
+ self.first_layer_ffn_out_scale = first_layer_ffn_out_scale
+ self.attn_out_scale = attn_out_scale
+ self.ffn_out_scale = ffn_out_scale
+ self.softmax_scale = softmax_scale
+ self.logit_scale = logit_scale
+ self.enable_lm_head_fp32 = enable_lm_head_fp32
+ self.num_mtp_layers = num_mtp_layers
+ self.rope_parameters = dict(rope_parameters or rope_scaling or {})
+ self.rope_parameters.setdefault(
+ "rope_type", self.rope_parameters.get("type", "default")
+ )
+ self.rope_parameters.setdefault("rope_theta", rope_theta)
+ if rotary_dim is not None:
+ self.rope_parameters["partial_rotary_factor"] = rotary_dim / head_dim
+
+
+class IQuestQ1MTPConfig(IQuestQ1Config):
+ model_type = "iquest_q1_mtp"
+
+ def __init__(
+ self,
+ target_config: Optional[dict] = None,
+ num_draft_slots: int = 7,
+ num_target_layers: Optional[int] = None,
+ sliding_window: Optional[int] = 512,
+ swa_rope_theta: Optional[float] = 10000.0,
+ fp32_residual_connection: bool = True,
+ **kwargs,
+ ):
+ if kwargs.get("num_hidden_layers", 1) != 1:
+ raise ValueError("MTP supports one physical draft layer.")
+ if num_draft_slots < 1:
+ raise ValueError("num_draft_slots must be positive.")
+ window = sliding_window or None
+ if window is not None and (window < 0 or not swa_rope_theta):
+ raise ValueError("A positive draft window requires swa_rope_theta.")
+ config = dict(target_config or {})
+ if num_target_layers is None:
+ num_target_layers = config.get("num_hidden_layers", 88)
+ if num_target_layers < 1:
+ raise ValueError("num_target_layers must be positive.")
+ if config.get("num_hidden_layers", num_target_layers) != num_target_layers:
+ raise ValueError("num_target_layers must match target_config.")
+ for key in ("model_type", "architectures", "auto_map", "_name_or_path"):
+ config.pop(key, None)
+ config.update(kwargs)
+ config.update(
+ architectures=["IQuestQ1MTP"],
+ num_hidden_layers=1,
+ mlp_only_layers=[],
+ no_rope_layers=[],
+ use_hybrid_layers=True,
+ use_sliding_window=window is not None,
+ sliding_window=window,
+ first_layers_types=[
+ "sliding_attention" if window is not None else "full_attention"
+ ],
+ hybrid_layers_types_block=[],
+ num_hybrid_layers_block=0,
+ last_layers_types=[],
+ )
+ if swa_rope_theta is not None:
+ config["swa_rope_theta"] = swa_rope_theta
+ super().__init__(**config)
+ self.target_config = dict(target_config or {})
+ self.num_draft_slots = num_draft_slots
+ self.num_target_layers = num_target_layers
+ self.fp32_residual_connection = fp32_residual_connection
diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py
index ad5fa0e15b..b7d31afaef 100644
--- a/python/sglang/srt/configs/model_config.py
+++ b/python/sglang/srt/configs/model_config.py
@@ -88,6 +88,8 @@ SWA_SINK_ARCHS = frozenset(
"GptOssForCausalLM",
"GraniteSWAForCausalLM",
"GraniteMoeSWAForCausalLM",
+ "IQuestQ1ForCausalLM",
+ "IQuestQ1MTP",
}
)
diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py
index cd1ba3c8d3..6845aa4936 100644
--- a/python/sglang/srt/entrypoints/openai/serving_chat.py
+++ b/python/sglang/srt/entrypoints/openai/serving_chat.py
@@ -106,7 +106,10 @@ from sglang.srt.parser.jinja_template_utils import (
MEDIA_URL_PART_TYPES,
process_content_for_template_format,
)
-from sglang.srt.parser.reasoning_parser import ReasoningParser
+from sglang.srt.parser.reasoning_parser import (
+ IQuestQ1ReasoningDetector,
+ ReasoningParser,
+)
from sglang.srt.sampling.sampling_params import (
set_request_reasoning_end_token_ids,
)
@@ -2564,12 +2567,12 @@ class OpenAIServingChat(OpenAIServingBase):
# as constraint (mirrors the streaming path). For auto: always try.
if self.tool_call_parser:
parser = FunctionCallParser(
- tools, self.tool_call_parser, tokenizer=self.tokenizer_manager.tokenizer
- )
- detector_owns_format = (
- parser.detector.supports_structural_tag()
- or parser.detector.parses_required_natively()
+ tools,
+ self.tool_call_parser,
+ tokenizer=self.tokenizer_manager.tokenizer,
+ tool_choice=tool_choice,
)
+ detector_owns_format = parser.owns_tool_format()
should_try_parser = not is_required or detector_owns_format
if should_try_parser and parser.has_tool_call(text):
try:
@@ -2844,6 +2847,14 @@ class OpenAIServingChat(OpenAIServingBase):
request.reasoning_effort = "medium" if enabled else "no_think"
return
+ if self.reasoning_parser == "iquest_q1":
+ request.chat_template_kwargs = {
+ **(request.chat_template_kwargs or {}),
+ "thinking": enabled,
+ "enable_thinking": enabled,
+ }
+ return
+
if self.reasoning_parser == "inkling":
# Effort-conditioned, not toggled: "none" (0.0) is the off switch.
if not enabled:
@@ -2902,6 +2913,11 @@ class OpenAIServingChat(OpenAIServingBase):
if not self.reasoning_parser:
return False
+ if self.reasoning_parser == "iquest_q1":
+ return IQuestQ1ReasoningDetector.thinking_enabled(
+ request.chat_template_kwargs or {}
+ )
+
if self.reasoning_parser == "minimax-m3":
# M3 template prefills for thinking_mode=enabled, so it never
# appears in output and reasoning must be forced. Mirrors reasoning_parser.py.
@@ -3008,11 +3024,9 @@ class OpenAIServingChat(OpenAIServingBase):
tools=effective_tools,
tool_call_parser=self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
+ tool_choice=request.tool_choice,
)
- use_native_parser = (
- probe.detector.supports_structural_tag()
- or probe.detector.parses_required_natively()
- )
+ use_native_parser = probe.owns_tool_format()
if use_native_parser:
parser_dict[index] = probe
else:
diff --git a/python/sglang/srt/entrypoints/openai/serving_responses.py b/python/sglang/srt/entrypoints/openai/serving_responses.py
index 9fdcbe3935..72ac87c09c 100644
--- a/python/sglang/srt/entrypoints/openai/serving_responses.py
+++ b/python/sglang/srt/entrypoints/openai/serving_responses.py
@@ -1081,6 +1081,7 @@ class OpenAIServingResponses(OpenAIServingChat):
chat_tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
+ tool_choice=tool_choice,
)
detector_owns_format = self._tool_parser_owns_format(parser)
should_try_native = not is_required or detector_owns_format
@@ -1166,10 +1167,7 @@ class OpenAIServingResponses(OpenAIServingChat):
@staticmethod
def _tool_parser_owns_format(parser: FunctionCallParser) -> bool:
- return (
- parser.detector.supports_structural_tag()
- or parser.detector.parses_required_natively()
- )
+ return parser.owns_tool_format()
@staticmethod
def _chat_tool_choice(tool_choice: Any) -> Any:
@@ -2011,6 +2009,7 @@ class OpenAIServingResponses(OpenAIServingChat):
chat_tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
+ tool_choice=tool_choice,
)
detector_owns_format = self._tool_parser_owns_format(probe)
if is_required and not detector_owns_format:
@@ -2020,6 +2019,7 @@ class OpenAIServingResponses(OpenAIServingChat):
chat_tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
+ tool_choice=tool_choice,
)
reasoning_parser_obj: Optional[ReasoningParser] = None
if self.reasoning_parser:
diff --git a/python/sglang/srt/function_call/base_format_detector.py b/python/sglang/srt/function_call/base_format_detector.py
index 53c306714a..6905f42d58 100644
--- a/python/sglang/srt/function_call/base_format_detector.py
+++ b/python/sglang/srt/function_call/base_format_detector.py
@@ -367,6 +367,9 @@ class BaseFormatDetector(ABC):
constraints and parse the model's native output format instead."""
return False
+ def get_required_tool_parser(self, tool_choice):
+ return None
+
@abstractmethod
def structure_info(self) -> _GetInfoFunc:
"""
diff --git a/python/sglang/srt/function_call/function_call_parser.py b/python/sglang/srt/function_call/function_call_parser.py
index 2932cfc126..3184ac7dc3 100644
--- a/python/sglang/srt/function_call/function_call_parser.py
+++ b/python/sglang/srt/function_call/function_call_parser.py
@@ -35,6 +35,7 @@ from sglang.srt.function_call.hermes_detector import HermesDetector
from sglang.srt.function_call.hunyuan_detector import HunyuanDetector
from sglang.srt.function_call.inkling_detector import InklingDetector
from sglang.srt.function_call.internlm_detector import InternlmDetector
+from sglang.srt.function_call.iquest_q1_detector import IQuestQ1Detector
from sglang.srt.function_call.k2_v3_detector import K2V3Detector
from sglang.srt.function_call.kimik2_detector import KimiK2Detector
from sglang.srt.function_call.kimik3_detector import KimiK3Detector
@@ -113,9 +114,12 @@ class FunctionCallParser:
"gigachat35": GigaChat35Detector,
"gemma4": Gemma4Detector,
"inkling": InklingDetector,
+ "iquest_q1": IQuestQ1Detector,
}
- def __init__(self, tools: List[Tool], tool_call_parser: str, tokenizer=None):
+ def __init__(
+ self, tools: List[Tool], tool_call_parser: str, tokenizer=None, tool_choice=None
+ ):
detector_class = self.ToolCallParserEnum.get(tool_call_parser)
if detector_class:
kwargs = {}
@@ -131,10 +135,20 @@ class FunctionCallParser:
detector.use_full_assistant_constraint = not any(
tool.function.strict for tool in tools
)
- self.detector = detector
+ if isinstance(tool_choice, dict):
+ tool_choice = ToolChoice(function={"name": tool_choice["name"]})
+ self._required_tool_parser = detector.get_required_tool_parser(tool_choice)
+ self.detector = self._required_tool_parser or detector
self.tools = tools
self.tool_strict_level = envs.SGLANG_TOOL_STRICT_LEVEL.get()
+ def owns_tool_format(self) -> bool:
+ return (
+ self._required_tool_parser is not None
+ or self.detector.supports_structural_tag()
+ or self.detector.parses_required_natively()
+ )
+
def has_tool_call(self, text: str) -> bool:
"""
Check if the given text contains a tool call in the format supported by this parser.
@@ -287,7 +301,7 @@ class FunctionCallParser:
try:
if (
isinstance(self.detector, Glm47MoeDetector)
- and self.detector.use_full_assistant_constraint
+ and self.detector.parses_required_natively()
):
functions = (
[
@@ -356,6 +370,14 @@ class FunctionCallParser:
if (
tool_choice == "required" or isinstance(tool_choice, ToolChoice)
) and not self.detector.parses_required_natively():
+ required_parser = self.detector.get_required_tool_parser(tool_choice)
+ if required_parser is not None:
+ return (
+ "json_schema",
+ required_parser.get_json_schema_constraint(
+ self.tools, tool_choice, parallel_tool_calls
+ ),
+ )
json_schema = get_json_schema_constraint(
self.tools, tool_choice, parallel_tool_calls=parallel_tool_calls
)
diff --git a/python/sglang/srt/function_call/iquest_q1_detector.py b/python/sglang/srt/function_call/iquest_q1_detector.py
new file mode 100644
index 0000000000..2e13dff2dd
--- /dev/null
+++ b/python/sglang/srt/function_call/iquest_q1_detector.py
@@ -0,0 +1,176 @@
+import orjson
+
+from sglang.srt.entrypoints.openai.protocol import ToolChoice
+from sglang.srt.function_call.core_types import StreamingParseResult, ToolCallItem
+from sglang.srt.function_call.glm47_moe_detector import (
+ Glm47MoeDetector,
+ get_argument_type,
+)
+from sglang.srt.function_call.json_array_parser import JsonArrayParser
+from sglang.srt.function_call.utils import get_json_schema_constraint
+
+
+class IQuestQ1Detector(Glm47MoeDetector):
+ def __init__(self):
+ super().__init__()
+ self.bot_token = ""
+ self.eot_token = ""
+ self._inside_call = False
+
+ def _parse_body(self, body, tools):
+ cursor = body.find("")
+ name = body[:cursor].strip() if cursor >= 0 else body.strip()
+ if not name:
+ return None
+ raw = body[cursor:] if cursor >= 0 else ""
+ pairs = self.func_arg_regex.findall(raw)
+ if self.func_arg_regex.sub("", raw).strip() or any(
+ not key.strip() for key, _ in pairs
+ ):
+ return None
+ arguments = self._parse_argument_pairs(pairs, name, tools)
+ for key, value in pairs:
+ key = key.strip()
+ if get_argument_type(name, key, tools, arguments) == "string":
+ arguments[key] = value
+ return {"name": name, "arguments": arguments}
+
+ def detect_and_parse(self, text, tools):
+ calls = []
+ normal = []
+ content_start = 0
+ search = 0
+ while (start := text.find(self.bot_token, search)) >= 0:
+ begin = start + len(self.bot_token)
+ end = text.find(self.eot_token, begin)
+ if end < 0:
+ break
+ action = self._parse_body(text[begin:end], tools)
+ search = end + len(self.eot_token)
+ if action is not None:
+ for call in self.parse_base_json(action, tools):
+ call.tool_index = len(calls)
+ calls.append(call)
+ normal.append(text[content_start:start])
+ content_start = search
+ normal.append(text[content_start:])
+ return StreamingParseResult(
+ normal_text="".join(normal),
+ calls=calls,
+ )
+
+ def parse_streaming_increment(self, new_text, tools):
+ self._buffer += new_text
+ normal, calls = [], []
+ while True:
+ if not self._inside_call:
+ start = self._buffer.find(self.bot_token)
+ if start < 0:
+ keep = self._ends_with_partial_token(self._buffer, self.bot_token)
+ end = len(self._buffer) - keep
+ normal.append(self._buffer[:end])
+ self._buffer = self._buffer[end:]
+ break
+ normal.append(self._buffer[:start])
+ self._buffer = self._buffer[start + len(self.bot_token) :]
+ self._inside_call = True
+ end = self._buffer.find(self.eot_token)
+ if end < 0:
+ break
+ body = self._buffer[:end]
+ self._buffer = self._buffer[end + len(self.eot_token) :]
+ self._inside_call = False
+ action = self._parse_body(body, tools)
+ if action is None:
+ normal.append(self.bot_token + body + self.eot_token)
+ continue
+ for call in self.parse_base_json(action, tools):
+ call.tool_index = len(self.prev_tool_call_arr)
+ calls.append(call)
+ self.prev_tool_call_arr.append(action)
+ self.streamed_args_for_tool.append(call.parameters)
+ self.current_tool_id = call.tool_index
+ self.current_tool_name_sent = True
+ return StreamingParseResult(normal_text="".join(normal), calls=calls)
+
+ def finish(self, tools):
+ text = (self.bot_token if self._inside_call else "") + self._buffer
+ self._buffer = ""
+ self._inside_call = False
+ return StreamingParseResult(normal_text=text)
+
+ def parses_required_natively(self):
+ return False
+
+ def get_required_tool_parser(self, tool_choice):
+ if tool_choice != "required" and not isinstance(tool_choice, ToolChoice):
+ return None
+ name = (
+ tool_choice.function.name if isinstance(tool_choice, ToolChoice) else None
+ )
+ return IQuestQ1JsonToolParser(name)
+
+ def supports_structural_tag(self):
+ return False
+
+ def structure_info(self):
+ raise NotImplementedError("IQuest Q1 uses XML argument pairs")
+
+
+class IQuestQ1JsonToolParser(JsonArrayParser):
+ def __init__(self, tool_name=None):
+ super().__init__()
+ self.tool_name = tool_name
+
+ def supports_structural_tag(self):
+ return False
+
+ def get_required_tool_parser(self, tool_choice):
+ return self
+
+ def get_json_schema_constraint(self, tools, tool_choice, parallel_tool_calls=True):
+ if self.tool_name is not None:
+ return next(
+ tool.function.parameters or {}
+ for tool in tools
+ if tool.function.name == self.tool_name
+ )
+ return get_json_schema_constraint(
+ tools, tool_choice, parallel_tool_calls=parallel_tool_calls
+ )
+
+ def has_tool_call(self, text):
+ return True
+
+ def detect_and_parse(self, text, tools):
+ value = orjson.loads(text)
+ if self.tool_name is not None:
+ if not isinstance(value, dict):
+ raise ValueError("Named tool arguments must be a JSON object.")
+ return StreamingParseResult(
+ calls=[ToolCallItem(tool_index=0, name=self.tool_name, parameters=text)]
+ )
+ if not isinstance(value, list) or not all(
+ isinstance(call, dict) and "name" in call for call in value
+ ):
+ raise ValueError("Expected a JSON array of named tool calls.")
+ return StreamingParseResult(calls=self.parse_base_json(value, tools))
+
+ def parse_streaming_increment(self, new_text, tools):
+ if self.tool_name is not None:
+ if not new_text:
+ return StreamingParseResult()
+ name = None if self.current_tool_name_sent else self.tool_name
+ self.current_tool_name_sent = True
+ return StreamingParseResult(
+ calls=[ToolCallItem(tool_index=0, name=name, parameters=new_text)]
+ )
+
+ calls = []
+ while True:
+ result = super().parse_streaming_increment(new_text, tools)
+ calls.extend(result.calls)
+ if not result.calls:
+ break
+ new_text = ""
+ return StreamingParseResult(calls=calls)
diff --git a/python/sglang/srt/function_call/parser_names.py b/python/sglang/srt/function_call/parser_names.py
index 4b35581ca0..3da09d6b59 100644
--- a/python/sglang/srt/function_call/parser_names.py
+++ b/python/sglang/srt/function_call/parser_names.py
@@ -49,4 +49,5 @@ TOOL_CALL_PARSER_NAMES = [
"gigachat35",
"gemma4",
"inkling",
+ "iquest_q1",
]
diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py
index a53ff0af39..867bcf9367 100644
--- a/python/sglang/srt/layers/attention/flashattention_backend.py
+++ b/python/sglang/srt/layers/attention/flashattention_backend.py
@@ -42,6 +42,7 @@ from sglang.srt.runtime_context import (
get_schedule,
get_spec,
)
+from sglang.srt.speculative.eagle_utils import per_step_draft_out_cache_loc
from sglang.srt.speculative.ragged_verify import build_ragged_target_verify_geometry
from sglang.srt.speculative.spec_info import SpecInput, SpeculativeAlgorithm
from sglang.srt.speculative.spec_utils import resolve_num_tokens_per_req
@@ -518,6 +519,23 @@ class FlashAttentionBackend(AttentionBackend):
else:
self._init_full_cg_decode_metadata(forward_batch, in_capture)
+ def _get_swa_write_locations(self, forward_batch: ForwardBatch):
+ locations = getattr(forward_batch, "out_cache_loc", None)
+ if (
+ locations is not None
+ and forward_batch.forward_mode.is_decode_or_idle()
+ and forward_batch.spec_info is not None
+ and self.topk > 0
+ and self.speculative_num_steps > 1
+ ):
+ locations = per_step_draft_out_cache_loc(
+ locations,
+ forward_batch.batch_size,
+ self.topk,
+ self.speculative_num_steps,
+ )[self.speculative_step_id]
+ return locations
+
def _init_full_cg_decode_metadata(
self, forward_batch: ForwardBatch, in_capture: bool
):
@@ -534,13 +552,21 @@ class FlashAttentionBackend(AttentionBackend):
# Refill the SWA write-target buffer (bound as a metadata view in
# _bind_metadata_buffers) from the live out_cache_loc before replay.
if self.use_sliding_window_kv_pool and out_cache_loc is not None:
- n = out_cache_loc.shape[0]
+ swa_out_cache_loc = self._get_swa_write_locations(forward_batch)
+ n = swa_out_cache_loc.shape[0]
+ assert_buffer_fits(
+ n,
+ self.cuda_graph_swa_out_cache_loc.shape[0],
+ "full decode SWA write locations",
+ )
self.cuda_graph_swa_out_cache_loc[n:].zero_()
if in_capture:
self.cuda_graph_swa_out_cache_loc[:n].zero_()
else:
self.cuda_graph_swa_out_cache_loc[:n].copy_(
- self.kv_index_translator.sliding_window_write_loc_for(out_cache_loc)
+ self.kv_index_translator.sliding_window_write_loc_for(
+ swa_out_cache_loc
+ )
)
if in_capture:
@@ -1148,18 +1174,23 @@ class FlashAttentionBackend(AttentionBackend):
_unified_read = (
self.kv_index_translator.is_translating and metadata.page_table is not None
)
+ swa_out_cache_loc = (
+ self._get_swa_write_locations(forward_batch)
+ if self.use_sliding_window_kv_pool
+ else None
+ )
if _unified_read:
kv_view = self.kv_index_translator.index_table_for_batch(forward_batch)
metadata.page_table = kv_view.ids
if self.use_sliding_window_kv_pool:
metadata.swa_page_table = kv_view.sliding_window_ids
- if forward_batch.out_cache_loc is not None:
+ if swa_out_cache_loc is not None:
# The swa write loc was computed from the still-VIRTUAL
# loc at ForwardBatch construction; re-running the
# full->swa map on the kernel-facing loc would be garbage.
metadata.swa_out_cache_loc = (
self.kv_index_translator.sliding_window_write_loc_for(
- forward_batch.out_cache_loc
+ swa_out_cache_loc
)
)
elif self.use_sliding_window_kv_pool:
@@ -1169,10 +1200,10 @@ class FlashAttentionBackend(AttentionBackend):
metadata.page_table
).to(torch.int32)
)
- if forward_batch.out_cache_loc is not None:
+ if swa_out_cache_loc is not None:
metadata.swa_out_cache_loc = (
self.token_to_kv_pool.translate_loc_from_full_to_swa(
- forward_batch.out_cache_loc
+ swa_out_cache_loc
)
)
@@ -1322,9 +1353,13 @@ class FlashAttentionBackend(AttentionBackend):
aux_tensors=None,
rel_bias=None,
rel_bias_event=None,
+ return_lse: bool = False,
):
if score_mod is not None and self.fa_impl_ver != 4:
raise RuntimeError("score_mod is only supported by the FA4 backend.")
+ if return_lse and self.use_mla:
+ raise NotImplementedError("MLA extend does not return an LSE.")
+ extend_lse = None
cp_active = is_cp_active(forward_batch)
if k is not None:
@@ -1602,13 +1637,19 @@ class FlashAttentionBackend(AttentionBackend):
causal=False if use_cascade_attn else causal,
window_size=window_size,
softcap=layer.logit_cap,
- return_softmax_lse=use_cascade_attn,
+ return_softmax_lse=use_cascade_attn or return_lse,
num_splits=self.num_splits,
out=_fa_out,
ver=self.fa_impl_ver,
**kwargs,
)
+ if return_lse and not use_cascade_attn:
+ if not isinstance(result, tuple):
+ raise RuntimeError("This extend path does not return an LSE.")
+ o, softmax_lse, *_ = result
+ extend_lse = softmax_lse.T.contiguous()
+ result = o
if use_cascade_attn:
o, softmax_lse, *rest = result
if (
@@ -1862,7 +1903,12 @@ class FlashAttentionBackend(AttentionBackend):
else:
o = result
- return o.view(-1, layer.tp_q_head_num * layer.v_head_dim)
+ o = o.view(-1, layer.tp_q_head_num * layer.v_head_dim)
+ if return_lse:
+ if extend_lse is None:
+ raise RuntimeError("This extend path does not return an LSE.")
+ return o, extend_lse
+ return o
def forward_decode(
self,
@@ -1883,9 +1929,13 @@ class FlashAttentionBackend(AttentionBackend):
aux_tensors=None,
rel_bias=None,
rel_bias_event=None,
+ return_lse: bool = False,
) -> torch.Tensor:
if score_mod is not None and self.fa_impl_ver != 4:
raise RuntimeError("score_mod is only supported by the FA4 backend.")
+ if return_lse and self.use_mla:
+ raise NotImplementedError("MLA decode does not return an LSE.")
+ decode_lse = None
if k is not None:
assert v is not None
if save_kv_cache:
@@ -2089,7 +2139,7 @@ class FlashAttentionBackend(AttentionBackend):
causal=False if use_cascade_attn else causal,
window_size=window_size,
softcap=layer.logit_cap,
- return_softmax_lse=use_cascade_attn,
+ return_softmax_lse=use_cascade_attn or return_lse,
num_splits=(
self.decode_num_splits
if not is_swa_layer
@@ -2102,6 +2152,10 @@ class FlashAttentionBackend(AttentionBackend):
scheduler_metadata=sched_meta,
**kwargs,
)
+ if return_lse and not use_cascade_attn:
+ o, softmax_lse, *_ = result
+ decode_lse = softmax_lse.T.contiguous()
+ result = o
if use_cascade_attn:
o, softmax_lse, *rest = result
if self._decode_uses_static_max_seqlen_k:
@@ -2211,7 +2265,12 @@ class FlashAttentionBackend(AttentionBackend):
else:
o = result
- return o.view(-1, layer.tp_q_head_num * layer.v_head_dim)
+ o = o.view(-1, layer.tp_q_head_num * layer.v_head_dim)
+ if return_lse:
+ if decode_lse is None:
+ raise RuntimeError("This decode path does not return an LSE.")
+ return o, decode_lse
+ return o
def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int):
"""Initialize CUDA graph state for the attention backend.
diff --git a/python/sglang/srt/managers/detokenizer_manager.py b/python/sglang/srt/managers/detokenizer_manager.py
index e27726f695..00236933cc 100644
--- a/python/sglang/srt/managers/detokenizer_manager.py
+++ b/python/sglang/srt/managers/detokenizer_manager.py
@@ -193,7 +193,7 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
return output
matched = finished_reason.get("matched", None)
- if not matched:
+ if matched is None:
return output
# TODO(lmzheng): handle the case where multiple stop strs are hit
diff --git a/python/sglang/srt/models/iquest_q1.py b/python/sglang/srt/models/iquest_q1.py
new file mode 100644
index 0000000000..26bdffefed
--- /dev/null
+++ b/python/sglang/srt/models/iquest_q1.py
@@ -0,0 +1,732 @@
+import logging
+from typing import Iterable, Optional, Tuple
+
+import torch
+import triton
+import triton.language as tl
+from torch import nn
+from transformers import PretrainedConfig
+from triton.language.extra import libdevice
+
+from sglang.srt.layers.activation import SiluAndMul
+from sglang.srt.layers.linear import (
+ MergedColumnParallelLinear,
+ QKVParallelLinear,
+ ReplicatedLinear,
+ RowParallelLinear,
+)
+from sglang.srt.layers.logits_processor import LogitsProcessor
+from sglang.srt.layers.moe import post_experts_all_reduce
+from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
+from sglang.srt.layers.moe.topk import TopK
+from sglang.srt.layers.moe.utils import RoutingMethodType
+from sglang.srt.layers.quantization.base_config import QuantizationConfig
+from sglang.srt.layers.quantization.fp8 import Fp8Config
+from sglang.srt.layers.radix_attention import RadixAttention
+from sglang.srt.layers.rotary_embedding import get_rope
+from sglang.srt.layers.vocab_parallel_embedding import (
+ ParallelLMHead,
+ VocabParallelEmbedding,
+)
+from sglang.srt.model_executor.forward_batch_info import ForwardBatch
+from sglang.srt.model_loader.weight_utils import default_weight_loader
+from sglang.srt.runtime_context import get_parallel
+from sglang.srt.utils import add_prefix, make_layers, set_weight_attrs
+
+logger = logging.getLogger(__name__)
+
+
+@triton.jit(do_not_specialize=["num_tokens"])
+def _apply_learned_sink_kernel(
+ query,
+ sink_key,
+ attn_output,
+ lse,
+ result,
+ num_tokens,
+ query_stride_t: tl.constexpr,
+ query_stride_h: tl.constexpr,
+ query_stride_d: tl.constexpr,
+ sink_stride_h: tl.constexpr,
+ sink_stride_d: tl.constexpr,
+ output_stride_t: tl.constexpr,
+ output_stride_h: tl.constexpr,
+ output_stride_d: tl.constexpr,
+ lse_stride_t: tl.constexpr,
+ lse_stride_h: tl.constexpr,
+ NUM_HEADS: tl.constexpr,
+ QUERIES_PER_KV: tl.constexpr,
+ HEAD_DIM: tl.constexpr,
+ SCALE: tl.constexpr,
+ BLOCK_ROWS: tl.constexpr,
+ BLOCK_D: tl.constexpr,
+):
+ rows = tl.program_id(0).to(tl.int64) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
+ tokens = rows // NUM_HEADS
+ heads = rows % NUM_HEADS
+ dims = tl.arange(0, BLOCK_D)
+ mask = (tokens[:, None] < num_tokens) & (dims[None, :] < HEAD_DIM)
+ q = tl.load(
+ query
+ + tokens[:, None] * query_stride_t
+ + heads[:, None] * query_stride_h
+ + dims[None, :] * query_stride_d,
+ mask,
+ 0,
+ )
+ sink = tl.load(
+ sink_key
+ + (heads[:, None] // QUERIES_PER_KV) * sink_stride_h
+ + dims[None, :] * sink_stride_d,
+ mask,
+ 0,
+ ).to(q.dtype)
+ products = q.to(tl.float32) * sink.to(tl.float32)
+ if HEAD_DIM == 128:
+ # Keep the reference reduction grouping to limit FP32 rounding drift.
+ even, odd = tl.split(products.reshape(BLOCK_ROWS, HEAD_DIM // 2, 2))
+ first, third = tl.split(even.reshape(BLOCK_ROWS, HEAD_DIM // 4, 2))
+ second, fourth = tl.split(odd.reshape(BLOCK_ROWS, HEAD_DIM // 4, 2))
+ sink_dot = tl.sum(((first + second) + third) + fourth, axis=1)
+ else:
+ sink_dot = tl.sum(products, axis=1)
+ sink_logit = sink_dot * SCALE
+ normal_lse = tl.load(
+ lse + tokens * lse_stride_t + heads * lse_stride_h,
+ tokens < num_tokens,
+ 0,
+ ).to(tl.float32)
+ factor = tl.div_rn(1.0, 1.0 + libdevice.exp(sink_logit - normal_lse))
+ values = tl.load(
+ attn_output
+ + tokens[:, None] * output_stride_t
+ + heads[:, None] * output_stride_h
+ + dims[None, :] * output_stride_d,
+ mask,
+ 0,
+ ).to(tl.float32)
+ tl.store(
+ result + rows[:, None] * HEAD_DIM + dims[None, :],
+ values * factor[:, None],
+ mask,
+ )
+
+
+def _apply_learned_sink(
+ query: torch.Tensor,
+ sink_key: torch.Tensor,
+ attn_output: torch.Tensor,
+ lse: torch.Tensor,
+ scale: float,
+) -> torch.Tensor:
+ num_tokens, num_heads, head_dim = query.shape
+ assert attn_output.shape == query.shape
+ assert sink_key.ndim == 2 and sink_key.shape[1] == head_dim
+ assert num_heads % sink_key.shape[0] == 0
+ assert lse.shape == (num_tokens, num_heads)
+ result = torch.empty(query.shape, dtype=attn_output.dtype, device=query.device)
+ if num_tokens == 0:
+ return result
+ # Larger tiles reduce prefill overhead but cost more on short decode batches.
+ block_rows = 64 if num_tokens * num_heads >= 32768 else 4
+ _apply_learned_sink_kernel[(triton.cdiv(num_tokens * num_heads, block_rows),)](
+ query,
+ sink_key,
+ attn_output,
+ lse,
+ result,
+ num_tokens,
+ *query.stride(),
+ *sink_key.stride(),
+ *attn_output.stride(),
+ *lse.stride(),
+ NUM_HEADS=num_heads,
+ QUERIES_PER_KV=num_heads // sink_key.shape[0],
+ HEAD_DIM=head_dim,
+ SCALE=scale,
+ BLOCK_ROWS=block_rows,
+ BLOCK_D=triton.next_power_of_2(head_dim),
+ num_warps=4,
+ # Avoid fused multiply-add rounding differences from the reference.
+ enable_fp_fusion=False,
+ )
+ return result
+
+
+class IQuestQ1RMSNorm(nn.Module):
+ def __init__(self, hidden_size: int, eps: float = 1e-6):
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(hidden_size))
+ self.variance_epsilon = eps
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ # The draft passes FP32 residuals with BF16 weights, which the fused CUDA
+ # RMSNorm rejects; it also rounds BF16 inputs differently.
+ input_dtype = hidden_states.dtype
+ hidden_states = hidden_states.to(torch.float32)
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+ result = (self.weight * hidden_states).to(input_dtype)
+ return result
+
+ def extra_repr(self):
+ return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
+
+
+class IQuestQ1MoEBlock(nn.Module):
+ def __init__(
+ self,
+ config: PretrainedConfig,
+ layer_id: int,
+ quant_config: Optional[QuantizationConfig] = None,
+ prefix: str = "",
+ ):
+ super().__init__()
+ self.hidden_size = config.hidden_size
+ self.num_experts = config.num_experts
+ self.top_k = config.num_experts_per_tok
+ self.gate = ReplicatedLinear(
+ config.hidden_size,
+ config.num_experts,
+ bias=False,
+ params_dtype=torch.float32,
+ quant_config=None,
+ prefix=add_prefix("gate", prefix),
+ )
+ self.topk = TopK(
+ top_k=self.top_k,
+ renormalize=True,
+ layer_id=layer_id,
+ )
+ self.experts = get_moe_impl_class(quant_config)(
+ num_experts=config.num_experts,
+ top_k=self.top_k,
+ layer_id=layer_id,
+ hidden_size=config.hidden_size,
+ intermediate_size=config.intermediate_size,
+ quant_config=quant_config,
+ prefix=add_prefix("experts", prefix),
+ routing_method_type=RoutingMethodType.Renormalize,
+ )
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ original_shape = hidden_states.shape
+ hidden_states = hidden_states.reshape(-1, self.hidden_size)
+ router_logits, _ = self.gate(hidden_states.float())
+ topk_output = self.topk(hidden_states, router_logits)
+ output = self.experts(hidden_states, topk_output)
+ output = post_experts_all_reduce(output)
+ return output.reshape(original_shape)
+
+
+class IQuestQ1DenseMLP(nn.Module):
+ def __init__(
+ self,
+ config: PretrainedConfig,
+ quant_config: Optional[QuantizationConfig] = None,
+ prefix: str = "",
+ ):
+ super().__init__()
+ self.gate_up_proj = MergedColumnParallelLinear(
+ config.hidden_size,
+ [config.dense_intermediate_size] * 2,
+ bias=False,
+ quant_config=quant_config,
+ prefix=add_prefix("gate_up_proj", prefix),
+ )
+ self.down_proj = RowParallelLinear(
+ config.dense_intermediate_size,
+ config.hidden_size,
+ bias=False,
+ quant_config=quant_config,
+ prefix=add_prefix("down_proj", prefix),
+ )
+ if config.hidden_act != "silu":
+ raise ValueError(
+ f"Unsupported activation: {config.hidden_act}. Only silu is supported."
+ )
+ self.act_fn = SiluAndMul()
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ gate_up, _ = self.gate_up_proj(hidden_states)
+ hidden_states = self.act_fn(gate_up)
+ hidden_states, _ = self.down_proj(hidden_states)
+ return hidden_states
+
+
+class IQuestQ1Attention(nn.Module):
+ def __init__(
+ self,
+ config: PretrainedConfig,
+ layer_id: int,
+ quant_config: Optional[QuantizationConfig] = None,
+ prefix: str = "",
+ ):
+ super().__init__()
+ parallel = get_parallel()
+ attn_tp_size = parallel.attn_tp_size
+ attn_tp_rank = parallel.attn_tp_rank
+ self.total_num_heads = config.num_attention_heads
+ self.total_num_kv_heads = config.num_key_value_heads
+ assert self.total_num_heads % attn_tp_size == 0
+ if self.total_num_kv_heads >= attn_tp_size:
+ assert self.total_num_kv_heads % attn_tp_size == 0
+ else:
+ assert attn_tp_size % self.total_num_kv_heads == 0
+ self.num_heads = self.total_num_heads // attn_tp_size
+ self.num_kv_heads = max(1, self.total_num_kv_heads // attn_tp_size)
+ self.head_dim = config.head_dim
+ self.q_size = self.num_heads * self.head_dim
+ self.kv_size = self.num_kv_heads * self.head_dim
+ self.scaling = (
+ self.head_dim**-0.5
+ if config.softmax_scale is None
+ else config.softmax_scale
+ )
+ self.attn_tp_rank = attn_tp_rank
+ self.attn_tp_size = attn_tp_size
+ self.qkv_proj = QKVParallelLinear(
+ config.hidden_size,
+ self.head_dim,
+ self.total_num_heads,
+ self.total_num_kv_heads,
+ bias=False,
+ quant_config=quant_config,
+ tp_rank=attn_tp_rank,
+ tp_size=attn_tp_size,
+ kv_tp_rank=attn_tp_rank,
+ kv_tp_size=attn_tp_size,
+ prefix=add_prefix("qkv_proj", prefix),
+ )
+ self.o_proj = RowParallelLinear(
+ self.total_num_heads * self.head_dim,
+ config.hidden_size,
+ bias=False,
+ quant_config=quant_config,
+ tp_rank=attn_tp_rank,
+ tp_size=attn_tp_size,
+ prefix=add_prefix("o_proj", prefix),
+ )
+ self.q_norm = IQuestQ1RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+ self.k_norm = IQuestQ1RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+ self.is_sliding = (
+ config.layer_types[layer_id] == "sliding_attention"
+ and config.sliding_window is not None
+ )
+ rope_theta = (
+ config.swa_rope_theta
+ if self.is_sliding
+ else config.rope_parameters["rope_theta"]
+ )
+ self.rotary_emb = (
+ get_rope(
+ self.head_dim,
+ rotary_dim=int(
+ self.head_dim
+ * config.rope_parameters.get("partial_rotary_factor", 1.0)
+ ),
+ max_position=config.max_position_embeddings,
+ base=rope_theta,
+ rope_scaling=config.rope_parameters,
+ is_neox_style=True,
+ )
+ if layer_id not in config.no_rope_layers
+ else None
+ )
+ sliding_window_size = config.sliding_window - 1 if self.is_sliding else -1
+ self.attn = RadixAttention(
+ self.num_heads,
+ self.head_dim,
+ self.scaling,
+ num_kv_heads=self.num_kv_heads,
+ layer_id=layer_id,
+ quant_config=quant_config,
+ prefix=add_prefix("attn", prefix),
+ sliding_window_size=sliding_window_size,
+ )
+ self.enable_sink_attention = config.enable_sink_attention
+ if self.enable_sink_attention:
+ self.sink_k = nn.Parameter(
+ torch.zeros(self.num_kv_heads, self.head_dim), requires_grad=False
+ )
+ set_weight_attrs(self.sink_k, {"weight_loader": self._sink_k_loader})
+
+ def _sink_k_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor) -> None:
+ if loaded_weight.dim() == 3:
+ loaded_weight = loaded_weight.squeeze(0)
+ if self.total_num_kv_heads >= self.attn_tp_size:
+ start = self.attn_tp_rank * self.num_kv_heads
+ else:
+ replicas = self.attn_tp_size // self.total_num_kv_heads
+ start = self.attn_tp_rank // replicas
+ local_weight = loaded_weight[start : start + self.num_kv_heads]
+ assert local_weight.shape == param.shape, (
+ f"Unexpected sink_k shard shape {tuple(local_weight.shape)} for "
+ f"rank-local parameter {tuple(param.shape)}"
+ )
+ param.data.copy_(local_weight)
+
+ def _apply_zero_value_sink(
+ self, q: torch.Tensor, attn_output: torch.Tensor, lse: torch.Tensor
+ ) -> torch.Tensor:
+ q = q.view(-1, self.num_heads, self.head_dim)
+ return _apply_learned_sink(
+ q,
+ self.sink_k,
+ attn_output.view(q.shape),
+ lse,
+ self.scaling,
+ ).view(-1, self.q_size)
+
+ def forward(
+ self,
+ positions: torch.Tensor,
+ hidden_states: torch.Tensor,
+ forward_batch: ForwardBatch,
+ ) -> torch.Tensor:
+ qkv, _ = self.qkv_proj(hidden_states)
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
+ q = self.q_norm(q.view(-1, self.num_heads, self.head_dim)).view(q.shape)
+ k = self.k_norm(k.view(-1, self.num_kv_heads, self.head_dim)).view(k.shape)
+ if self.rotary_emb is not None:
+ q, k = self.rotary_emb(positions, q, k)
+ if self.enable_sink_attention:
+ attn_output, lse = self.attn(q, k, v, forward_batch, return_lse=True)
+ attn_output = self._apply_zero_value_sink(q, attn_output, lse)
+ else:
+ attn_output = self.attn(q, k, v, forward_batch)
+ output, _ = self.o_proj(attn_output)
+ return output
+
+
+class IQuestQ1DecoderLayer(nn.Module):
+ def __init__(
+ self,
+ config: PretrainedConfig,
+ layer_id: int,
+ quant_config: Optional[QuantizationConfig] = None,
+ prefix: str = "",
+ ):
+ super().__init__()
+ self.attention_norm = IQuestQ1RMSNorm(
+ config.hidden_size, eps=config.rms_norm_eps
+ )
+ self.attn_out_norm = IQuestQ1RMSNorm(
+ config.hidden_size, eps=config.rms_norm_eps
+ )
+ self.feed_forward_norm = IQuestQ1RMSNorm(
+ config.hidden_size, eps=config.rms_norm_eps
+ )
+ self.self_attn = IQuestQ1Attention(
+ config=config,
+ layer_id=layer_id,
+ quant_config=quant_config,
+ prefix=add_prefix("self_attn", prefix),
+ )
+ if layer_id in (config.mlp_only_layers or []):
+ self.mlp = IQuestQ1DenseMLP(
+ config=config,
+ quant_config=quant_config,
+ prefix=add_prefix("mlp", prefix),
+ )
+ else:
+ self.mlp = IQuestQ1MoEBlock(
+ config=config,
+ layer_id=layer_id,
+ quant_config=quant_config,
+ prefix=add_prefix("mlp", prefix),
+ )
+ self.is_first_layer = layer_id == 0
+ if self.is_first_layer:
+ self.ffn_out_norm = IQuestQ1RMSNorm(
+ config.hidden_size, eps=config.rms_norm_eps
+ )
+ self.attn_out_scale = config.first_layer_attn_out_scale
+ self.ffn_out_scale = config.first_layer_ffn_out_scale
+ else:
+ self.attn_out_scale = config.attn_out_scale
+ self.ffn_out_scale = config.ffn_out_scale
+
+ def forward(
+ self,
+ positions: torch.Tensor,
+ hidden_states: torch.Tensor,
+ forward_batch: ForwardBatch,
+ ) -> torch.Tensor:
+ # Intentional: layer 0 adds the raw input and an FFN output norm, later layers
+ # add the normalized input without one; the MTP draft follows layer 0.
+ if self.is_first_layer:
+ norm_hidden_states = self.attention_norm(hidden_states)
+ attn_output = self.self_attn(positions, norm_hidden_states, forward_batch)
+ hidden_states = hidden_states + self.attn_out_norm(attn_output) * (
+ self.attn_out_scale
+ )
+ mlp_output = self.mlp(self.feed_forward_norm(hidden_states))
+ return hidden_states + self.ffn_out_norm(mlp_output) * self.ffn_out_scale
+ norm_hidden_states = self.attention_norm(hidden_states)
+ attn_output = self.self_attn(positions, norm_hidden_states, forward_batch)
+ hidden_states = norm_hidden_states + self.attn_out_norm(attn_output) * (
+ self.attn_out_scale
+ )
+ mlp_output = self.mlp(self.feed_forward_norm(hidden_states))
+ return hidden_states + mlp_output * self.ffn_out_scale
+
+
+class IQuestQ1Model(nn.Module):
+ def __init__(
+ self,
+ config: PretrainedConfig,
+ quant_config: Optional[QuantizationConfig] = None,
+ prefix: str = "",
+ ):
+ super().__init__()
+ self.config = config
+ self.embed_tokens = VocabParallelEmbedding(
+ config.vocab_size,
+ config.hidden_size,
+ prefix=add_prefix("embed_tokens", prefix),
+ )
+ self.layers = make_layers(
+ config.num_hidden_layers,
+ lambda idx, prefix: IQuestQ1DecoderLayer(
+ config=config,
+ layer_id=idx,
+ quant_config=quant_config,
+ prefix=prefix,
+ ),
+ prefix=add_prefix("layers", prefix),
+ )
+ self.norm = IQuestQ1RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+ def forward(
+ self,
+ input_ids: torch.Tensor,
+ positions: torch.Tensor,
+ forward_batch: ForwardBatch,
+ input_embeds: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
+ if input_embeds is None:
+ hidden_states = self.embed_tokens(input_ids)
+ else:
+ hidden_states = input_embeds
+ for layer in self.layers:
+ hidden_states = layer(positions, hidden_states, forward_batch)
+ return self.norm(hidden_states)
+
+
+_FUSED_SHARD_LAYOUT = (
+ ("qkv_proj", ("q", "k", "v")),
+ ("gate_up_proj", (0, 1)),
+)
+
+
+def _is_created_by_quantization(
+ name: str, param: torch.nn.Parameter, quant_config: Optional[QuantizationConfig]
+) -> bool:
+ if getattr(param, "_skip_weight_check", False):
+ return True
+ return (
+ isinstance(quant_config, Fp8Config)
+ and not quant_config.is_checkpoint_fp8_serialized
+ and name.rsplit(".", 1)[-1]
+ in {
+ "w13_weight_scale",
+ "w2_weight_scale",
+ "w13_weight_scale_inv",
+ "w2_weight_scale_inv",
+ }
+ )
+
+
+def check_all_params_loaded(
+ params_dict: dict,
+ loaded: set,
+ model: str,
+ loaded_shards: Optional[dict] = None,
+ quant_config: Optional[QuantizationConfig] = None,
+) -> None:
+ missing = sorted(
+ name
+ for name, param in params_dict.items()
+ if name not in loaded
+ and not _is_created_by_quantization(name, param, quant_config)
+ )
+ partial = []
+ for name, shards in sorted((loaded_shards or {}).items()):
+ for projection, expected in _FUSED_SHARD_LAYOUT:
+ if not name.endswith(f".{projection}.weight"):
+ continue
+ absent = [shard for shard in expected if shard not in shards]
+ if absent:
+ partial.append(f"{name} without {absent}")
+ if not missing and not partial:
+ return
+ problems = []
+ if missing:
+ problems.append(
+ f"{len(missing)} parameters were not found in the checkpoint, "
+ f"e.g. {missing[:8]}"
+ )
+ if partial:
+ problems.append(
+ f"{len(partial)} fused parameters were only partially loaded, "
+ f"e.g. {partial[:8]}"
+ )
+ message = f"{model}: " + "; ".join(problems)
+ if quant_config is not None and not partial:
+ logger.warning(
+ "%s; verify whether these parameters are generated by quantization.",
+ message,
+ )
+ return
+ raise ValueError(message)
+
+
+def load_iquest_q1_weights(
+ model: nn.Module, weights: Iterable[Tuple[str, torch.Tensor]]
+):
+ stacked_params_mapping = [
+ ("qkv_proj", "q_proj", "q"),
+ ("qkv_proj", "k_proj", "k"),
+ ("qkv_proj", "v_proj", "v"),
+ ("gate_up_proj", "gate_proj", 0),
+ ("gate_up_proj", "up_proj", 1),
+ ]
+ params_dict = dict(model.named_parameters())
+ loaded = set()
+ loaded_shards = {}
+ for name, loaded_weight in weights:
+ if (
+ name == "lm_head.weight"
+ and name not in params_dict
+ and "model.embed_tokens.weight" in params_dict
+ ):
+ name = "model.embed_tokens.weight"
+ if "rotary_emb.inv_freq" in name:
+ continue
+ if name.endswith("experts.fc"):
+ param_name = name.replace("experts.fc", "experts.w13_weight")
+ if param_name not in params_dict:
+ raise ValueError(f"Unexpected IQuest Q1 weight: {name}")
+ param = params_dict[param_name]
+ loaded.add(param_name)
+ for expert_id in range(model.config.num_experts):
+ weight_loader = param.weight_loader
+ weight_loader(
+ param,
+ loaded_weight[expert_id][: model.config.intermediate_size],
+ param_name,
+ shard_id="w1",
+ expert_id=expert_id,
+ )
+ weight_loader(
+ param,
+ loaded_weight[expert_id][model.config.intermediate_size :],
+ param_name,
+ shard_id="w3",
+ expert_id=expert_id,
+ )
+ continue
+ if name.endswith("experts.proj"):
+ param_name = name.replace("experts.proj", "experts.w2_weight")
+ if param_name not in params_dict:
+ raise ValueError(f"Unexpected IQuest Q1 weight: {name}")
+ param = params_dict[param_name]
+ loaded.add(param_name)
+ for expert_id in range(model.config.num_experts):
+ param.weight_loader(
+ param,
+ loaded_weight[expert_id],
+ param_name,
+ shard_id="w2",
+ expert_id=expert_id,
+ )
+ continue
+ if "router.weight" in name:
+ name = name.replace("router.weight", "gate.weight")
+ for param_name, weight_name, shard_id in stacked_params_mapping:
+ if weight_name not in name:
+ continue
+ mapped_name = name.replace(weight_name, param_name)
+ if mapped_name not in params_dict:
+ continue
+ param = params_dict[mapped_name]
+ param.weight_loader(param, loaded_weight, shard_id)
+ if mapped_name not in loaded or mapped_name in loaded_shards:
+ loaded_shards.setdefault(mapped_name, set()).add(shard_id)
+ loaded.add(mapped_name)
+ break
+ else:
+ if name not in params_dict:
+ raise ValueError(f"Unexpected IQuest Q1 weight: {name}")
+ param = params_dict[name]
+ getattr(param, "weight_loader", default_weight_loader)(param, loaded_weight)
+ loaded.add(name)
+ loaded_shards.pop(name, None)
+ check_all_params_loaded(
+ params_dict,
+ loaded,
+ type(model).__name__,
+ loaded_shards=loaded_shards,
+ quant_config=model.quant_config,
+ )
+
+
+class IQuestQ1ForCausalLM(nn.Module):
+ fall_back_to_pt_during_load = False
+
+ def __init__(
+ self,
+ config: PretrainedConfig,
+ quant_config: Optional[QuantizationConfig] = None,
+ prefix: str = "",
+ ):
+ super().__init__()
+ self.config = config
+ self.quant_config = quant_config
+ self.model = IQuestQ1Model(
+ config, quant_config=quant_config, prefix=add_prefix("model", prefix)
+ )
+ self.lm_head = (
+ self.model.embed_tokens
+ if config.tie_word_embeddings
+ else ParallelLMHead(
+ config.vocab_size,
+ config.hidden_size,
+ quant_config=quant_config,
+ prefix=add_prefix("lm_head", prefix),
+ )
+ )
+ self.logits_processor = LogitsProcessor(config, logit_scale=config.logit_scale)
+
+ def get_input_embeddings(self):
+ return self.model.embed_tokens
+
+ @torch.no_grad()
+ def forward(
+ self,
+ input_ids: torch.Tensor,
+ positions: torch.Tensor,
+ forward_batch: ForwardBatch,
+ input_embeds: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
+ hidden_states = self.model(input_ids, positions, forward_batch, input_embeds)
+ return self.logits_processor(
+ input_ids, hidden_states, self.lm_head, forward_batch
+ )
+
+ def get_embed_and_head(self):
+ return self.model.embed_tokens.weight, self.lm_head.weight
+
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
+ load_iquest_q1_weights(
+ self,
+ (
+ (name, weight)
+ for name, weight in weights
+ if not name.startswith("mtp_layers.")
+ ),
+ )
+
+
+EntryClass = [IQuestQ1ForCausalLM]
diff --git a/python/sglang/srt/models/iquest_q1_mtp.py b/python/sglang/srt/models/iquest_q1_mtp.py
new file mode 100644
index 0000000000..af90efad5d
--- /dev/null
+++ b/python/sglang/srt/models/iquest_q1_mtp.py
@@ -0,0 +1,204 @@
+from typing import Any, Iterable, Optional, Tuple
+
+import torch
+from torch import nn
+
+from sglang.srt.configs.iquest_q1 import IQuestQ1MTPConfig
+from sglang.srt.layers.logits_processor import LogitsProcessor
+from sglang.srt.layers.quantization.base_config import QuantizationConfig
+from sglang.srt.layers.vocab_parallel_embedding import (
+ ParallelLMHead,
+ VocabParallelEmbedding,
+)
+from sglang.srt.model_executor.forward_batch_info import ForwardBatch
+from sglang.srt.models.iquest_q1 import (
+ IQuestQ1Attention,
+ IQuestQ1MoEBlock,
+ IQuestQ1RMSNorm,
+ load_iquest_q1_weights,
+)
+from sglang.srt.utils import add_prefix
+
+
+class IQuestQ1MTPInnerLayer(nn.Module):
+ def __init__(
+ self,
+ config: IQuestQ1MTPConfig,
+ quant_config: Optional[QuantizationConfig] = None,
+ prefix: str = "",
+ ):
+ super().__init__()
+ self.self_attn = IQuestQ1Attention(
+ config=config,
+ layer_id=0,
+ quant_config=quant_config,
+ prefix=add_prefix("self_attn", prefix),
+ )
+ self.mlp = IQuestQ1MoEBlock(
+ config=config,
+ layer_id=0,
+ quant_config=quant_config,
+ prefix=add_prefix("mlp", prefix),
+ )
+ self.attention_norm = IQuestQ1RMSNorm(
+ config.hidden_size, eps=config.rms_norm_eps
+ )
+ self.attn_out_norm = IQuestQ1RMSNorm(
+ config.hidden_size, eps=config.rms_norm_eps
+ )
+ self.feed_forward_norm = IQuestQ1RMSNorm(
+ config.hidden_size, eps=config.rms_norm_eps
+ )
+ self.ffn_out_norm = IQuestQ1RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ self.attn_out_scale = config.first_layer_attn_out_scale
+ self.ffn_out_scale = config.first_layer_ffn_out_scale
+ self.fp32_residual_connection = config.fp32_residual_connection
+
+ def forward(self, positions, hidden_states, forward_batch):
+ dtype = hidden_states.dtype
+ attention = self.self_attn(
+ positions, self.attention_norm(hidden_states), forward_batch
+ )
+ residual = (
+ hidden_states.float() if self.fp32_residual_connection else hidden_states
+ )
+ hidden_states = residual + self.attn_out_norm(attention) * self.attn_out_scale
+ output = self.mlp(self.feed_forward_norm(hidden_states).to(dtype))
+ return hidden_states + self.ffn_out_norm(output) * self.ffn_out_scale
+
+
+class IQuestQ1MTPLayer(nn.Module):
+ def __init__(
+ self,
+ config: IQuestQ1MTPConfig,
+ quant_config: Optional[QuantizationConfig] = None,
+ prefix: str = "",
+ ):
+ super().__init__()
+ self.enorm = IQuestQ1RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ self.hnorm = IQuestQ1RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
+ self.mtp_model_layer = IQuestQ1MTPInnerLayer(
+ config=config,
+ quant_config=quant_config,
+ prefix=add_prefix("mtp_model_layer", prefix),
+ )
+ self.final_layernorm = IQuestQ1RMSNorm(
+ config.hidden_size, eps=config.rms_norm_eps
+ )
+
+ def forward(self, positions, previous_hidden_states, input_embeds, forward_batch):
+ dtype = self.eh_proj.weight.dtype
+ hidden_states = self.eh_proj(
+ torch.cat(
+ [
+ self.enorm(input_embeds).to(dtype),
+ self.hnorm(previous_hidden_states).to(dtype),
+ ],
+ dim=-1,
+ )
+ )
+ hidden_states = self.mtp_model_layer(positions, hidden_states, forward_batch)
+ return self.final_layernorm(hidden_states).to(dtype)
+
+
+class IQuestQ1MTPModel(nn.Module):
+ def __init__(
+ self,
+ config: IQuestQ1MTPConfig,
+ quant_config: Optional[QuantizationConfig] = None,
+ prefix: str = "",
+ ):
+ super().__init__()
+ self.embed_tokens = VocabParallelEmbedding(
+ config.vocab_size,
+ config.hidden_size,
+ prefix=add_prefix("embed_tokens", prefix),
+ )
+ self.mtp_layer = IQuestQ1MTPLayer(
+ config=config,
+ quant_config=quant_config,
+ prefix=add_prefix("mtp_layer", prefix),
+ )
+
+ @property
+ def layers(self):
+ return (self.mtp_layer.mtp_model_layer,)
+
+ def forward(self, input_ids, positions, forward_batch, input_embeds=None):
+ if input_embeds is None:
+ input_embeds = self.embed_tokens(input_ids)
+ return self.mtp_layer(
+ positions,
+ forward_batch.spec_info.hidden_states,
+ input_embeds,
+ forward_batch,
+ )
+
+
+class IQuestQ1MTP(nn.Module):
+ fall_back_to_pt_during_load = False
+
+ def __init__(
+ self,
+ config: IQuestQ1MTPConfig,
+ quant_config: Optional[QuantizationConfig] = None,
+ prefix: str = "",
+ ):
+ super().__init__()
+ self.config = config
+ self.quant_config = quant_config
+ self.model = IQuestQ1MTPModel(
+ config=config,
+ quant_config=quant_config,
+ prefix=add_prefix("model", prefix),
+ )
+ self.lm_head = ParallelLMHead(
+ config.vocab_size,
+ config.hidden_size,
+ quant_config=quant_config,
+ prefix=add_prefix("lm_head", prefix),
+ )
+ self.logits_processor = LogitsProcessor(config)
+
+ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
+ return self.model.embed_tokens(input_ids)
+
+ def get_embed_and_head(self):
+ return self.model.embed_tokens.weight, self.lm_head.weight
+
+ def set_embed_and_head(self, embed, head):
+ return
+
+ @torch.no_grad()
+ def forward(
+ self,
+ input_ids: torch.Tensor,
+ positions: torch.Tensor,
+ forward_batch: ForwardBatch,
+ input_embeds: Optional[torch.Tensor] = None,
+ **kwargs: Any,
+ ) -> torch.Tensor:
+ hidden_states = self.model(input_ids, positions, forward_batch, input_embeds)
+ return self.logits_processor(
+ input_ids, hidden_states, self.lm_head, forward_batch
+ )
+
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
+ def mapped_weights():
+ for name, weight in weights:
+ if name == "target_final_norm.weight":
+ continue
+ if name == "embed_tokens.weight":
+ yield "model.embed_tokens.weight", weight
+ elif name.startswith("mtp."):
+ yield "model.mtp_layer." + name.removeprefix("mtp."), weight
+ elif name == "lm_head.weight":
+ yield name, weight
+ else:
+ raise ValueError(f"Unexpected MTP weight: {name}")
+
+ load_iquest_q1_weights(self, mapped_weights())
+
+
+EntryClass = [IQuestQ1MTP]
diff --git a/python/sglang/srt/parser/reasoning_parser.py b/python/sglang/srt/parser/reasoning_parser.py
index 57c996ffd5..4dec02573a 100644
--- a/python/sglang/srt/parser/reasoning_parser.py
+++ b/python/sglang/srt/parser/reasoning_parser.py
@@ -396,6 +396,38 @@ class DeepSeekR1Detector(BaseReasoningFormatDetector):
# https://github.com/sgl-project/sglang/pull/3202#discussion_r1950153599
+class IQuestQ1ReasoningDetector(BaseReasoningFormatDetector):
+ @staticmethod
+ def thinking_enabled(chat_kwargs):
+ thinking = chat_kwargs.get("thinking")
+ enable_thinking = chat_kwargs.get("enable_thinking")
+ return (
+ True
+ if thinking is None and enable_thinking is None
+ else bool(thinking or enable_thinking)
+ )
+
+ def __init__(
+ self,
+ stream_reasoning=True,
+ force_reasoning=True,
+ continue_final_message=False,
+ previous_content="",
+ force_nonempty_content=False,
+ ):
+ super().__init__(
+ "",
+ "",
+ force_reasoning=force_reasoning,
+ stream_reasoning=stream_reasoning,
+ continue_final_message=continue_final_message,
+ previous_content=previous_content,
+ force_nonempty_content=force_nonempty_content,
+ thinks_internally=True,
+ reasoning_default="enable_thinking",
+ )
+
+
class Qwen3Detector(BaseReasoningFormatDetector):
"""
Detector for Qwen3 models (e.g., Qwen/Qwen3-235B-A22B).
@@ -2255,6 +2287,7 @@ class ReasoningParser:
"gemma4": Gemma4Detector,
"gigachat35": DeepSeekR1Detector,
"inkling": InklingDetector,
+ "iquest_q1": IQuestQ1ReasoningDetector,
"cohere_command4": CohereCommand4Detector,
}
@@ -2276,6 +2309,13 @@ class ReasoningParser:
chat_template_kwargs = getattr(request, "chat_template_kwargs", None) or {}
+ if model_type.lower() == "iquest_q1" and (
+ request is not None or force_reasoning is None
+ ):
+ force_reasoning = IQuestQ1ReasoningDetector.thinking_enabled(
+ chat_template_kwargs
+ )
+
# Special cases where we override force_reasoning
if model_type.lower() in {
"qwen3-thinking",
diff --git a/python/sglang/srt/parser/reasoning_parser_names.py b/python/sglang/srt/parser/reasoning_parser_names.py
index d6c4099e63..c061572129 100644
--- a/python/sglang/srt/parser/reasoning_parser_names.py
+++ b/python/sglang/srt/parser/reasoning_parser_names.py
@@ -35,6 +35,7 @@ REASONING_PARSER_NAMES = [
"interns1",
"gemma4",
"inkling",
+ "iquest_q1",
"cohere_command4",
"gigachat35",
]
diff --git a/python/sglang/srt/parser/template_detection.py b/python/sglang/srt/parser/template_detection.py
index a739fb33c2..7c329aad92 100644
--- a/python/sglang/srt/parser/template_detection.py
+++ b/python/sglang/srt/parser/template_detection.py
@@ -526,11 +526,16 @@ def _is_gigachat35(ctx):
return ctx.has_text("<|GCML|tool_calls>")
+def _is_iquest_q1(ctx):
+ return ctx.has_text("<|iquest_assistant|>")
+
+
# ---------------------------------------------------------------------------
# Reasoning parser rules
# ---------------------------------------------------------------------------
REASONING_PARSER_RULES = (
+ DetectionRule(name="iquest_q1", value="iquest_q1", predicate=_is_iquest_q1),
DetectionRule(name="k2_horizon", value="k2_horizon", predicate=_is_k2_v3),
DetectionRule(name="apertus2509", value="apertus2509", predicate=_is_apertus2509),
DetectionRule(name="gemma4", value="gemma4", predicate=_is_gemma4),
@@ -576,6 +581,7 @@ REASONING_PARSER_RULES = (
# ---------------------------------------------------------------------------
TOOL_CALL_PARSER_RULES = (
+ DetectionRule(name="iquest_q1", value="iquest_q1", predicate=_is_iquest_q1),
DetectionRule(name="gigachat35", value="gigachat35", predicate=_is_gigachat35),
DetectionRule(name="k2_horizon", value="k2_horizon", predicate=_is_k2_v3),
DetectionRule(name="apertus2509", value="apertus2509", predicate=_is_apertus2509),
@@ -795,6 +801,8 @@ def _architecture_auto_parsers(server_args, needs: Tuple[str, ...]) -> Dict[str,
if "KimiK3" in arch or model_type == "kimi_k3":
reasoning_parser, tool_call_parser = "kimi_k3", "kimi_k3"
+ elif arch == "IQuestQ1ForCausalLM":
+ reasoning_parser, tool_call_parser = "iquest_q1", "iquest_q1"
elif arch in (
"BailingMoeV3ForCausalLM",
"BailingMoeV3VLForConditionalGeneration",
diff --git a/python/sglang/srt/utils/hf_transformers/common.py b/python/sglang/srt/utils/hf_transformers/common.py
index c6f94f3a4a..b8d202b750 100644
--- a/python/sglang/srt/utils/hf_transformers/common.py
+++ b/python/sglang/srt/utils/hf_transformers/common.py
@@ -52,6 +52,8 @@ from sglang.srt.configs import (
InternS2MobiusConfig,
InternS2MobiusTextConfig,
InternS2PreviewConfig,
+ IQuestQ1Config,
+ IQuestQ1MTPConfig,
JetNemotronConfig,
JetVLMConfig,
K2HorizonConfig,
@@ -162,6 +164,8 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
Qwen3_5TextConfig,
Qwen3_5MoeTextConfig,
InternS2PreviewConfig,
+ IQuestQ1Config,
+ IQuestQ1MTPConfig,
InternS2MobiusConfig,
InternS2MobiusTextConfig,
JetNemotronConfig,
diff --git a/test/registered/kernels/ops/attention/test_iquest_q1_learned_sink.py b/test/registered/kernels/ops/attention/test_iquest_q1_learned_sink.py
new file mode 100644
index 0000000000..5caafb0a98
--- /dev/null
+++ b/test/registered/kernels/ops/attention/test_iquest_q1_learned_sink.py
@@ -0,0 +1,104 @@
+import unittest
+
+import torch
+
+from sglang.srt.models.iquest_q1 import _apply_learned_sink
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="1-gpu-large")
+
+
+def torch_reference(query, sink_key, attn_output, lse, scale):
+ group = query.shape[1] // sink_key.shape[0]
+ sink = sink_key.to(query.dtype).repeat_interleave(group, dim=0)
+ sink_logit = (query.float() * sink.float()).sum(-1) * scale
+ factor = torch.sigmoid(lse.float() - sink_logit)
+ return (attn_output.float() * factor.unsqueeze(-1)).to(attn_output.dtype)
+
+
+class TestIQuestQ1LearnedSink(CustomTestCase):
+ def test_matches_torch_across_layouts_and_dtypes(self):
+ torch.manual_seed(17)
+ # (tokens, heads, kv_heads, head_dim): empty batch, row-tail mask, model GQA,
+ # MHA with head_dim 80, the non-128 reduction path, the 64-row prefill tile.
+ shapes = (
+ (0, 6, 1, 128),
+ (7, 6, 1, 128),
+ (9, 48, 8, 128),
+ (37, 16, 16, 80),
+ (17, 8, 1, 256),
+ (16384, 6, 1, 128),
+ )
+ cases = [
+ (shape, dtype, strided)
+ for shape in shapes
+ for dtype, strided in (
+ (torch.float32, False),
+ (torch.bfloat16, False),
+ (torch.bfloat16, True),
+ )
+ ]
+ for shape, dtype, strided in cases:
+ with self.subTest(shape=shape, dtype=dtype, strided=strided):
+ tokens, heads, kv_heads, head_dim = shape
+ step = 2 if strided else 1
+ q = torch.randn(
+ tokens, heads, head_dim * step, device="cuda", dtype=dtype
+ )[..., ::step]
+ output = torch.randn_like(q)
+ if strided:
+ output = torch.randn(
+ heads, tokens, head_dim * step, device="cuda", dtype=dtype
+ ).transpose(0, 1)[..., ::step]
+ sink = torch.randn(kv_heads, head_dim * step, device="cuda")[:, ::step]
+ lse = torch.randn(heads, tokens, device="cuda").T
+ if not strided:
+ lse = lse.contiguous()
+ original = output.clone()
+ scale = head_dim**-0.5
+ expected = torch_reference(q, sink, output, lse, scale)
+ actual = _apply_learned_sink(q, sink, output, lse, scale)
+ rtol = 3e-5 if dtype == torch.float32 else torch.finfo(dtype).eps
+ torch.testing.assert_close(actual, expected, rtol=rtol, atol=5e-7)
+ torch.testing.assert_close(output, original, rtol=0, atol=0)
+ self.assertEqual(actual.shape, q.shape)
+ self.assertEqual(actual.dtype, output.dtype)
+ self.assertTrue(actual.is_contiguous())
+
+ def test_extreme_lse(self):
+ q = torch.ones(5, 6, 128, device="cuda")
+ sink = torch.zeros(1, 128, device="cuda")
+ output = torch.randn_like(q)
+ lse = (
+ torch.tensor([-float("inf"), -1000, 0, 1000, float("inf")], device="cuda")
+ .unsqueeze(-1)
+ .expand(5, 6)
+ )
+ actual = _apply_learned_sink(q, sink, output, lse, 128**-0.5)
+ factors = torch.tensor([0, 0, 0.5, 1, 1], device="cuda")
+ torch.testing.assert_close(
+ actual, output * factors[:, None, None], rtol=0, atol=0
+ )
+
+ def test_torch_compile_fullgraph_preserves_outputs(self):
+ torch.manual_seed(41)
+ compiled = torch.compile(_apply_learned_sink, fullgraph=True)
+ for tokens in (1, 8):
+ with self.subTest(tokens=tokens):
+ q = torch.randn(tokens, 6, 128, device="cuda", dtype=torch.bfloat16)
+ sink = torch.randn(1, 128, device="cuda", dtype=torch.bfloat16)
+ output = torch.randn_like(q)
+ lse = torch.randn(tokens, 6, device="cuda")
+ for _ in range(2):
+ q.normal_()
+ sink.normal_()
+ output.normal_()
+ lse.normal_()
+ expected = _apply_learned_sink(q, sink, output, lse, 128**-0.5)
+ actual = compiled(q, sink, output, lse, 128**-0.5)
+ torch.testing.assert_close(actual, expected, rtol=0, atol=0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py
index 116459e7b4..c5f8e24f1e 100644
--- a/test/registered/unit/entrypoints/openai/test_serving_chat.py
+++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py
@@ -272,7 +272,7 @@ class TestChatTemplateCache(CustomTestCase):
self.tokenizer_manager.tokenizer.decode.assert_not_called()
-class ServingChatTestCase(unittest.TestCase):
+class ServingChatTestCase(CustomTestCase):
# ------------- common fixtures -------------
def setUp(self):
# The serving layer reads its config from the bags, so the fixture has
@@ -2047,6 +2047,150 @@ class ServingChatTestCase(unittest.TestCase):
)
self.assertEqual(finish_reason["type"], "tool_calls")
+ def test_iquest_release_tool_modes_through_streaming_serving(self):
+ self.chat.tool_call_parser = "iquest_q1"
+ self.chat.reasoning_parser = "iquest_q1"
+ tools = [
+ {
+ "type": "function",
+ "function": {
+ "name": "run",
+ "parameters": {
+ "type": "object",
+ "properties": {"code": {"type": "string"}},
+ "required": ["code"],
+ },
+ },
+ }
+ ]
+ named = ToolChoice(function=ToolChoiceFuncName(name="run"))
+ payloads = (
+ (
+ "auto",
+ "beforeruncode"
+ "你好after",
+ "beforeafter",
+ ),
+ ("required", '[{"name":"run","parameters":{"code":"你好"}}]', ""),
+ (named, '{ "code": "你好" }', ""),
+ )
+ for choice, payload, expected_content in payloads:
+ for thinking in (True, False):
+ wire = ("work" if thinking else "") + payload
+ for finish in ("stop", "length"):
+ for size in (1,):
+ with self.subTest(
+ choice=choice, thinking=thinking, finish=finish, size=size
+ ):
+ req = ChatCompletionRequest(
+ model="x",
+ messages=[{"role": "user", "content": "run"}],
+ input_ids=[1],
+ tools=tools,
+ tool_choice=choice,
+ chat_template_kwargs={"thinking": thinking},
+ stream=True,
+ )
+ processed = self.chat._process_messages(req, False)
+ self.assertFalse(req.skip_special_tokens)
+ constraint = processed.tool_call_constraint
+ if choice == "auto":
+ self.assertIsNone(constraint)
+ else:
+ self.assertEqual(constraint[0], "json_schema")
+ self.assertEqual(
+ constraint[1]["type"],
+ "object" if choice == named else "array",
+ )
+
+ async def generate():
+ for end in range(size, len(wire) + size, size):
+ yield {
+ "text": wire[:end],
+ "meta_info": {
+ "id": "chatcmpl-iquest-release",
+ "prompt_tokens": 1,
+ "completion_tokens": min(end, len(wire)),
+ "finish_reason": (
+ {"type": finish, "matched": None}
+ if end >= len(wire)
+ else None
+ ),
+ },
+ "index": 0,
+ }
+
+ self.tm.generate_request.return_value = generate()
+ chunks = self._parse_chunks(
+ self._run_chat_stream(None, req)
+ )
+ self.assertFalse(any("error" in chunk for chunk in chunks))
+ choices = [
+ c for chunk in chunks for c in chunk.get("choices", [])
+ ]
+ deltas = [c.get("delta", {}) for c in choices]
+ self.assertEqual(
+ "".join(d.get("content") or "" for d in deltas),
+ expected_content,
+ )
+ self.assertEqual(
+ "".join(
+ d.get("reasoning_content") or "" for d in deltas
+ ),
+ "work" if thinking else "",
+ )
+ calls = [
+ c for d in deltas for c in (d.get("tool_calls") or [])
+ ]
+ self.assertEqual(
+ [
+ c["function"]["name"]
+ for c in calls
+ if c["function"].get("name")
+ ],
+ ["run"],
+ )
+ self.assertTrue(all(c["index"] == 0 for c in calls))
+ self.assertEqual(
+ json.loads(
+ "".join(
+ c["function"].get("arguments") or ""
+ for c in calls
+ )
+ ),
+ {"code": "你好"},
+ )
+ self.assertEqual(
+ [
+ c["finish_reason"]
+ for c in choices
+ if c.get("finish_reason")
+ ],
+ ["tool_calls" if finish == "stop" else finish],
+ )
+
+ def test_iquest_named_tool_rejects_invalid_arguments(self):
+ self.chat.tool_call_parser = "iquest_q1"
+ choice = ToolChoice(function=ToolChoiceFuncName(name="run"))
+ for payload, finish in (
+ ('{"code":', {"type": "length", "length": 16}),
+ ("42", {"type": "stop", "matched": "stop-marker"}),
+ ):
+ with self.subTest(payload=payload):
+ calls, text, finish_reason = self.chat._process_tool_calls(
+ text=payload,
+ tools=ChatCompletionRequest(
+ model="test",
+ messages=[],
+ tools=[{"type": "function", "function": {"name": "run"}}],
+ ).tools,
+ finish_reason=dict(finish),
+ tool_choice=choice,
+ )
+ self.assertIsNone(calls)
+ self.assertEqual(text, payload)
+ self.assertEqual(finish_reason, finish)
+
def test_required_tool_choice_skips_json_fallback_for_native_parser(self):
"""A structural-tag parser owns the output format, so a missing tool
call must not be pushed through the json_schema array fallback."""
diff --git a/test/registered/unit/entrypoints/openai/test_serving_responses_stream.py b/test/registered/unit/entrypoints/openai/test_serving_responses_stream.py
index c9d6974cc1..f2e30ff028 100644
--- a/test/registered/unit/entrypoints/openai/test_serving_responses_stream.py
+++ b/test/registered/unit/entrypoints/openai/test_serving_responses_stream.py
@@ -1,4 +1,5 @@
import asyncio
+import json
import unittest
from types import SimpleNamespace
from unittest.mock import Mock, patch
@@ -253,6 +254,55 @@ class NonHarmonyStreamTestCase(CustomTestCase):
)
self.assertEqual(deltas, full_item.arguments)
+ def test_detector_selected_tool_parser_matches_full_and_streamed_arguments(self):
+ serving = make_serving()
+ serving.reasoning_parser = None
+ serving.tool_call_parser = "iquest_q1"
+ arguments = '{"city":"北京"}'
+ for choice, raw in (
+ ({"type": "function", "name": "get_weather"}, arguments),
+ ("required", '[{"name":"get_weather","parameters":' + arguments + "}]"),
+ ):
+ with self.subTest(choice=choice):
+ request = ResponsesRequest(
+ model="x",
+ input="weather?",
+ stream=True,
+ store=False,
+ tool_choice=choice,
+ tools=[
+ {
+ "type": "function",
+ "name": "get_weather",
+ "parameters": {"type": "object"},
+ }
+ ],
+ )
+ (full_item,) = serving._make_response_output_items(
+ request,
+ raw,
+ serving.tokenizer_manager.tokenizer,
+ require_reasoning=False,
+ )
+ events = StreamFixture(serving, request).run(
+ [
+ engine_chunk(raw[:i], i, finish=i == len(raw))
+ for i in range(1, len(raw) + 1)
+ ]
+ )
+ (stream_item,) = find_completed_event(events)["response"]["output"]
+ for item in (full_item.model_dump(), stream_item):
+ self.assertEqual(item["type"], "function_call")
+ self.assertEqual(item["name"], "get_weather")
+ self.assertEqual(json.loads(item["arguments"]), {"city": "北京"})
+ self.assertTrue(item["call_id"])
+ deltas = "".join(
+ p["delta"]
+ for p in event_payloads(events)
+ if p["type"] == "response.function_call_arguments.delta"
+ )
+ self.assertEqual(deltas, stream_item["arguments"])
+
def test_final_output_preserves_text_tool_text_order(self):
from sglang.srt.function_call.core_types import (
StreamingParseResult,
diff --git a/test/registered/unit/function_call/test_iquest_q1_detector.py b/test/registered/unit/function_call/test_iquest_q1_detector.py
new file mode 100644
index 0000000000..88ed7af572
--- /dev/null
+++ b/test/registered/unit/function_call/test_iquest_q1_detector.py
@@ -0,0 +1,245 @@
+import json
+import unittest
+from types import SimpleNamespace
+
+from sglang.srt.entrypoints.openai.protocol import (
+ Function,
+ Tool,
+ ToolChoice,
+ ToolChoiceFuncName,
+)
+from sglang.srt.function_call.function_call_parser import FunctionCallParser
+from sglang.srt.function_call.iquest_q1_detector import IQuestQ1Detector
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=2, suite="base-a-test-cpu")
+
+
+class TestIQuestQ1Detector(CustomTestCase):
+ def setUp(self):
+ self.tools = [
+ Tool(
+ type="function",
+ function=Function(
+ name="run",
+ parameters={
+ "type": "object",
+ "properties": {
+ "code": {"type": "string"},
+ "count": {"type": "integer"},
+ "enabled": {"type": "boolean"},
+ "items": {"type": "array"},
+ },
+ },
+ ),
+ )
+ ]
+ self.call = (
+ "run\n"
+ 'code "123"\n'
+ "count 42\n"
+ "enabledtrue"
+ "items[1, null]"
+ ""
+ )
+ self.expected = {
+ "code": ' "123"\n',
+ "count": 42,
+ "enabled": True,
+ "items": [1, None],
+ }
+
+ def test_surrounding_text_is_preserved_in_each_mode(self):
+ for wire, expected_text, call_count in (
+ ("before" + self.call + "after", "beforeafter", 1),
+ (self.call + "\nafter", "\nafter", 1),
+ (
+ "before" + self.call + "between" + self.call + "after",
+ "beforebetweenafter",
+ 2,
+ ),
+ ):
+ with self.subTest(wire=wire):
+ parser = FunctionCallParser(self.tools, "iquest_q1")
+ text, calls = parser.parse_non_stream(wire)
+ self.assertEqual(text, expected_text)
+ self.assertEqual(
+ [json.loads(call.parameters) for call in calls],
+ [self.expected] * call_count,
+ )
+ for size in (1, 5, len(wire)):
+ with self.subTest(size=size):
+ parser = FunctionCallParser(self.tools, "iquest_q1")
+ normal, streamed = [], []
+ for start in range(0, len(wire), size):
+ chunk, chunk_calls = parser.parse_stream_chunk(
+ wire[start : start + size]
+ )
+ normal.append(chunk)
+ streamed.extend(chunk_calls)
+ tail, tail_calls = parser.parse_stream_end()
+ normal.append(tail)
+ streamed.extend(tail_calls)
+ self.assertEqual("".join(normal), expected_text)
+ self.assertEqual(streamed, calls)
+
+ def test_non_stream_preserves_unparsed_text_around_valid_calls(self):
+ malformed = (
+ "runcount"
+ "1"
+ )
+ incomplete = "runcode"
+ for wire, expected_text in (
+ (malformed + self.call + "after", malformed + "after"),
+ (self.call + malformed + self.call, malformed),
+ (self.call + "after" + incomplete, "after" + incomplete),
+ ):
+ with self.subTest(wire=wire):
+ parser = FunctionCallParser(self.tools, "iquest_q1")
+ text, calls = parser.parse_non_stream(wire)
+ self.assertEqual(text, expected_text)
+ self.assertTrue(calls)
+ for call in calls:
+ self.assertEqual(json.loads(call.parameters), self.expected)
+
+ def test_empty_arguments_and_malformed_call(self):
+ detector = IQuestQ1Detector()
+ result = detector.detect_and_parse(
+ "run", self.tools
+ )
+ self.assertEqual(json.loads(result.calls[0].parameters), {})
+ malformed = (
+ "runcount"
+ "1"
+ )
+ result = detector.parse_streaming_increment(malformed, self.tools)
+ self.assertEqual(result.normal_text, malformed)
+ self.assertFalse(result.calls)
+
+ def test_truncated_marker_is_flushed_at_stream_end(self):
+ parser = FunctionCallParser(self.tools, "iquest_q1")
+ text, calls = parser.parse_stream_chunk("Text hts", q.float(), k.float().expand(-1, heads, -1)
+ )
+ * layer.scaling
+ )
+ mask = positions[None, :] <= positions[:, None]
+ if window >= 0:
+ mask &= positions[None, :] >= positions[:, None] - window
+ logits = logits.masked_fill(~mask, -float("inf"))
+ torch.testing.assert_close(
+ lse, logits.logsumexp(-1).T, rtol=1e-5, atol=1e-5
+ )
+ sink_logits = (
+ torch.einsum("thd,kd->ht", q.float(), sink.float()) * layer.scaling
+ )
+ joint = torch.cat([logits, sink_logits[..., None]], -1).softmax(-1)
+ expected = torch.einsum(
+ "hts,shd->thd", joint[..., :-1], v.float().expand(-1, heads, -1)
+ )
+ actual = _apply_learned_sink(
+ q, sink, output.view_as(q), lse, layer.scaling
+ )
+ torch.testing.assert_close(
+ actual.float(), expected, rtol=0.02, atol=0.01
+ )
+ decode = ForwardBatch(
+ forward_mode=ForwardMode.DECODE,
+ batch_size=1,
+ input_ids=batch.input_ids[-1:],
+ req_pool_indices=batch.req_pool_indices,
+ seq_lens=batch.seq_lens,
+ seq_lens_cpu=batch.seq_lens_cpu,
+ seq_lens_sum=tokens,
+ out_cache_loc=batch.out_cache_loc[-1:],
+ )
+ backend.init_forward_metadata(decode)
+ decoded, decode_lse = backend.forward_decode(
+ q[-1:], None, None, layer, decode, return_lse=True
+ )
+ torch.testing.assert_close(decode_lse, lse[-1:], rtol=1e-5, atol=1e-5)
+ torch.testing.assert_close(decoded, output[-1:], rtol=0.02, atol=0.01)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/managers/test_detokenizer_stop_trim.py b/test/registered/unit/managers/test_detokenizer_stop_trim.py
new file mode 100644
index 0000000000..2fefb5c0bd
--- /dev/null
+++ b/test/registered/unit/managers/test_detokenizer_stop_trim.py
@@ -0,0 +1,36 @@
+import unittest
+from types import SimpleNamespace
+
+from sglang.srt.managers.detokenizer_manager import DetokenizerManager
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=1, suite="base-a-test-cpu")
+
+
+class TestDetokenizerStopTrim(CustomTestCase):
+ def trim(self, output, reason, no_stop_trim=False, gpt_oss=False):
+ return DetokenizerManager.trim_matched_stop(
+ SimpleNamespace(is_tool_call_parser_gpt_oss=gpt_oss),
+ output,
+ reason,
+ no_stop_trim,
+ )
+
+ def test_zero_and_nonzero_stop_token_ids(self):
+ for token_id in (0, 30):
+ with self.subTest(token_id=token_id):
+ reason = {"type": "stop", "matched": token_id}
+ self.assertEqual(self.trim([123, token_id], reason), [123])
+ self.assertEqual(self.trim([token_id], reason), [])
+ self.assertEqual(
+ self.trim([123, token_id], reason, no_stop_trim=True),
+ [123, token_id],
+ )
+ self.assertEqual(
+ self.trim([123, 0], {"type": "stop", "matched": None}), [123, 0]
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/models/test_iquest_q1.py b/test/registered/unit/models/test_iquest_q1.py
new file mode 100644
index 0000000000..99f4d75356
--- /dev/null
+++ b/test/registered/unit/models/test_iquest_q1.py
@@ -0,0 +1,223 @@
+import unittest
+
+import torch
+
+from sglang.srt.configs.iquest_q1 import IQuestQ1Config
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=3, suite="base-a-test-cpu")
+
+
+class TestIQuestQ1WeightLoading(CustomTestCase):
+ def test_target_ignores_native_mtp_weights(self):
+ from sglang.srt.models.iquest_q1 import IQuestQ1ForCausalLM
+
+ native_weights = [
+ (f"mtp_layers.{layer}.{suffix}", torch.empty(1, device="meta"))
+ for layer in range(2)
+ for suffix in (
+ "eh_proj.weight",
+ "enorm.weight",
+ "final_layernorm.weight",
+ "hnorm.weight",
+ "mtp_model_layer.mlp.experts.fc",
+ "mtp_model_layer.mlp.experts.proj",
+ "mtp_model_layer.mlp.router.weight",
+ "mtp_model_layer.self_attn.q_proj.weight",
+ "mtp_model_layer.self_attn.sink_k",
+ )
+ ]
+ model = IQuestQ1ForCausalLM.__new__(IQuestQ1ForCausalLM)
+ torch.nn.Module.__init__(model)
+ model.config = IQuestQ1Config(num_mtp_layers=2)
+ model.quant_config = None
+ model.model = torch.nn.Module()
+ model.model.embed_tokens = torch.nn.Embedding(4, 2)
+ model.lm_head = torch.nn.Linear(2, 4, bias=False)
+ weights = [
+ ("model.embed_tokens.weight", torch.full((4, 2), 2.0)),
+ ("lm_head.weight", torch.full((4, 2), 3.0)),
+ ]
+ model.load_weights(iter(native_weights + weights))
+ self.assertEqual(set(model.state_dict()), {name for name, _ in weights})
+ for name, expected in weights:
+ torch.testing.assert_close(model.state_dict()[name], expected)
+ with self.assertRaisesRegex(ValueError, "lm_head.weight"):
+ model.load_weights(iter(weights[:1] + native_weights))
+ for name in ("mtp_layers_extra.weight", "mtp.eh_proj.weight"):
+ with (
+ self.subTest(name=name),
+ self.assertRaisesRegex(ValueError, "Unexpected IQuest Q1 weight"),
+ ):
+ model.load_weights(iter(weights + [(name, torch.ones(1))]))
+
+ def test_unrecognized_quantized_parameters_warn_but_partial_weights_fail(self):
+ from sglang.srt.layers.quantization.fp8 import Fp8Config
+ from sglang.srt.models.iquest_q1 import check_all_params_loaded
+
+ for name, config in (
+ (
+ "model.layers.0.mlp.experts.w13_weight_scale",
+ Fp8Config(is_checkpoint_fp8_serialized=True),
+ ),
+ ("model.learned_scale", Fp8Config(is_checkpoint_fp8_serialized=False)),
+ ):
+ with self.subTest(name=name):
+ with self.assertLogs(
+ "sglang.srt.models.iquest_q1", level="WARNING"
+ ) as logs:
+ check_all_params_loaded(
+ {name: torch.nn.Parameter(torch.zeros(1))},
+ set(),
+ "IQuestQ1ForCausalLM",
+ quant_config=config,
+ )
+ self.assertIn(name, logs.output[0])
+ self.assertIn("not found", logs.output[0])
+ with self.assertRaisesRegex(ValueError, "not found"):
+ check_all_params_loaded(
+ {name: torch.nn.Parameter(torch.zeros(1))},
+ set(),
+ "IQuestQ1ForCausalLM",
+ )
+ name = "model.layers.0.self_attn.qkv_proj.weight"
+ with self.assertRaisesRegex(ValueError, "partially loaded"):
+ check_all_params_loaded(
+ {name: torch.nn.Parameter(torch.zeros(1))},
+ {name},
+ "IQuestQ1ForCausalLM",
+ loaded_shards={name: {"q"}},
+ quant_config=Fp8Config(),
+ )
+ generated = torch.nn.Parameter(torch.zeros(1))
+ generated._skip_weight_check = True
+ check_all_params_loaded(
+ {"generated_parameter": generated}, set(), "IQuestQ1ForCausalLM"
+ )
+
+ def test_target_fused_loaders_require_every_checkpoint_shard(self):
+ from sglang.srt.models.iquest_q1 import IQuestQ1ForCausalLM
+
+ for projection, components in (
+ ("qkv_proj", ("q_proj", "k_proj", "v_proj")),
+ ("gate_up_proj", ("gate_proj", "up_proj")),
+ ):
+ with self.subTest(projection=projection):
+ model = IQuestQ1ForCausalLM.__new__(IQuestQ1ForCausalLM)
+ torch.nn.Module.__init__(model)
+ model.quant_config = None
+ layer = torch.nn.Linear(2, 2 * len(components), bias=False)
+ layer.weight.requires_grad_(False)
+
+ def loader(param, weight, shard):
+ index = {"q": 0, "k": 1, "v": 2}.get(shard, shard)
+ param.data[index * 2 : (index + 1) * 2].copy_(weight)
+
+ layer.weight.weight_loader = loader
+ model.model = torch.nn.Module()
+ setattr(model.model, projection, layer)
+ weights = [
+ (f"model.{name}.weight", torch.full((2, 2), float(i + 1)))
+ for i, name in enumerate(components)
+ ]
+ with self.assertRaisesRegex(ValueError, "partially loaded"):
+ model.load_weights(weights[:1])
+ model.load_weights(weights)
+ torch.testing.assert_close(
+ layer.weight, torch.cat([w for _, w in weights])
+ )
+
+
+class TestIQuestQ1MTPDraft(CustomTestCase):
+ def test_mtp_loads_own_weights(self):
+ from sglang.srt.models.iquest_q1_mtp import IQuestQ1MTP
+
+ model = IQuestQ1MTP.__new__(IQuestQ1MTP)
+ torch.nn.Module.__init__(model)
+ model.quant_config = None
+ model.model = torch.nn.Module()
+ model.model.embed_tokens = torch.nn.Embedding(4, 2)
+ model.lm_head = torch.nn.Linear(2, 4, bias=False)
+ model.model.mtp_layer = torch.nn.Module()
+ model.model.mtp_layer.self_attn = torch.nn.Module()
+ qkv = torch.nn.Linear(2, 6, bias=False)
+ model.model.mtp_layer.self_attn.qkv_proj = qkv
+
+ def loader(param, weight, shard):
+ offset = {"q": 0, "k": 2, "v": 4}[shard]
+ param.data[offset : offset + 2].copy_(weight)
+
+ qkv.weight.weight_loader = loader
+ weights = [
+ ("embed_tokens.weight", torch.full((4, 2), 2.0)),
+ ("lm_head.weight", torch.full((4, 2), 3.0)),
+ ("target_final_norm.weight", torch.ones(2)),
+ ] + [
+ (f"mtp.self_attn.{name}_proj.weight", torch.full((2, 2), float(i)))
+ for i, name in enumerate(("q", "k", "v"), 1)
+ ]
+ model.load_weights(weights)
+ torch.testing.assert_close(qkv.weight, torch.cat([w for _, w in weights[-3:]]))
+ torch.testing.assert_close(model.model.embed_tokens.weight, weights[0][1])
+ torch.testing.assert_close(model.lm_head.weight, weights[1][1])
+ with self.assertRaisesRegex(ValueError, "Unexpected MTP weight"):
+ model.load_weights(weights + [("unknown.weight", torch.ones(1))])
+ with self.assertRaisesRegex(ValueError, "Unexpected MTP weight"):
+ model.load_weights(
+ weights + [("mtp_layers.0.eh_proj.weight", torch.ones(1))]
+ )
+ with self.assertRaisesRegex(ValueError, "lm_head.weight"):
+ model.load_weights(
+ [item for item in weights if item[0] != "lm_head.weight"]
+ )
+
+ def test_mtp_keeps_its_own_embedding_and_head(self):
+ from sglang.srt.models.iquest_q1_mtp import IQuestQ1MTP
+
+ model = IQuestQ1MTP.__new__(IQuestQ1MTP)
+ torch.nn.Module.__init__(model)
+ model.model = torch.nn.Module()
+ model.model.embed_tokens = torch.nn.Embedding(4, 2)
+ model.lm_head = torch.nn.Linear(2, 4, bias=False)
+ embed, head = model.get_embed_and_head()
+ model.set_embed_and_head(torch.zeros(4, 2), torch.zeros(4, 2))
+ current_embed, current_head = model.get_embed_and_head()
+ self.assertIs(current_embed, embed)
+ self.assertIs(current_head, head)
+
+ def test_mtp_residual_is_fp32_but_moe_input_is_bf16(self):
+ from sglang.srt.models.iquest_q1_mtp import (
+ IQuestQ1MTPInnerLayer,
+ )
+
+ class Attention(torch.nn.Module):
+ def forward(self, positions, hidden_states, forward_batch):
+ return torch.full_like(hidden_states, 0.25)
+
+ class Experts(torch.nn.Module):
+ def forward(self, hidden_states):
+ self.input_dtype = hidden_states.dtype
+ return torch.full_like(hidden_states, 0.25)
+
+ layer = IQuestQ1MTPInnerLayer.__new__(IQuestQ1MTPInnerLayer)
+ torch.nn.Module.__init__(layer)
+ layer.attention_norm = torch.nn.Identity()
+ layer.attn_out_norm = torch.nn.Identity()
+ layer.feed_forward_norm = torch.nn.Identity()
+ layer.ffn_out_norm = torch.nn.Identity()
+ layer.self_attn = Attention()
+ layer.mlp = Experts()
+ layer.attn_out_scale = 1.0
+ layer.ffn_out_scale = 1.0
+ hidden = torch.full((2, 2), 256.0, dtype=torch.bfloat16)
+ for fp32 in (False, True):
+ layer.fp32_residual_connection = fp32
+ result = layer(torch.tensor([0, 1]), hidden, None)
+ self.assertEqual(layer.mlp.input_dtype, torch.bfloat16)
+ expected = hidden.float() + 0.5 if fp32 else hidden
+ torch.testing.assert_close(result, expected)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/parser/test_iquest_q1_reasoning_parser.py b/test/registered/unit/parser/test_iquest_q1_reasoning_parser.py
new file mode 100644
index 0000000000..7aeee406ce
--- /dev/null
+++ b/test/registered/unit/parser/test_iquest_q1_reasoning_parser.py
@@ -0,0 +1,63 @@
+import unittest
+from types import SimpleNamespace
+
+from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
+from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
+from sglang.srt.parser.reasoning_parser import ReasoningParser
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=2, suite="base-a-test-cpu")
+
+
+class TestIQuestQ1ReasoningParser(CustomTestCase):
+ def test_thinking_aliases_match_release(self):
+ for kwargs, enabled in (
+ ({}, True),
+ ({"thinking": False}, False),
+ ({"enable_thinking": False}, False),
+ ({"thinking": False, "enable_thinking": True}, True),
+ ({"thinking": True, "enable_thinking": False}, True),
+ ({"enable_thinking": None}, True),
+ ({"thinking": False, "enable_thinking": None}, False),
+ ({"thinking": None, "enable_thinking": False}, False),
+ ):
+ with self.subTest(kwargs=kwargs):
+ request = ChatCompletionRequest(
+ model="iquest-q1",
+ messages=[{"role": "user", "content": "hi"}],
+ chat_template_kwargs=kwargs,
+ )
+ serving = OpenAIServingChat.__new__(OpenAIServingChat)
+ serving.reasoning_parser = "iquest_q1"
+ serving.chat_encoding_spec = None
+ serving.template_manager = SimpleNamespace(reasoning_config=None)
+ serving._reasoning_detector = ReasoningParser("iquest_q1").detector
+ self.assertEqual(
+ OpenAIServingChat._get_reasoning_from_request(serving, request),
+ enabled,
+ )
+
+ def test_request_thinking_toggle_controls_reasoning(self):
+ serving = OpenAIServingChat.__new__(OpenAIServingChat)
+ serving.reasoning_parser = "iquest_q1"
+ request = ChatCompletionRequest(
+ model="iquest-q1",
+ messages=[{"role": "user", "content": "hi"}],
+ chat_template_kwargs={"thinking": True, "enable_thinking": False},
+ )
+ for enabled, wire, expected in (
+ (False, "answer", ("", "answer")),
+ (True, "workanswer", ("work", "answer")),
+ ):
+ with self.subTest(enabled=enabled):
+ serving.apply_reasoning_enabled(request, enabled)
+ self.assertEqual(serving._get_reasoning_from_request(request), enabled)
+ parser = ReasoningParser(
+ "iquest_q1", force_reasoning=not enabled, request=request
+ )
+ self.assertEqual(parser.parse_non_stream(wire), expected)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/registered/unit/parser/test_template_manager.py b/test/registered/unit/parser/test_template_manager.py
index f477f2ccd6..8d00219284 100644
--- a/test/registered/unit/parser/test_template_manager.py
+++ b/test/registered/unit/parser/test_template_manager.py
@@ -67,6 +67,15 @@ class TestTemplateManagerReasoningDetection(CustomTestCase):
)
return force, config, parser
+ def test_iquest_template_wins_over_generic_think_and_xml_rules(self):
+ template = "<|iquest_assistant|>"
+ force, config, parser = self._detect(template, [])
+ self.assertEqual(parser, "iquest_q1")
+ self.assertEqual(
+ detect_tool_call_parser(template, _DummyTokenizer([]), config, force),
+ "iquest_q1",
+ )
+
def test_gigachat35_gcml_template_wins_over_generic_think_tags(self):
"""The GCML marker must map both parsers to gigachat35; the template also
carries , so the rule has to outrank the generic deepseek-r1