[Model] Add IQuest Q1 support and MTP draft (#41590)
Co-authored-by: zelong huang <yzhu@ubiquant.com> Co-authored-by: hnyls2002 <lsyincs@gmail.com> Co-authored-by: zelong518 <zelonghuang05@gmail.com> Co-authored-by: chenwenxiaolive <cocoshirolive@gmail.com>
This commit is contained in:
co-authored by
zelong huang
hnyls2002
zelong518
chenwenxiaolive
parent
b95329a74d
commit
79cafec013
@@ -22,6 +22,7 @@ from sglang.srt.arg_groups.model_overrides import gpt_oss # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import granitemoehybrid # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import inkling # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import interns2_mobius # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import iquest_q1 # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import kimi_k3 # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import lfm2 # noqa: F401
|
||||
from sglang.srt.arg_groups.model_overrides import llama4 # noqa: F401
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
from typing import Any
|
||||
|
||||
from sglang.srt.arg_groups.model_override_base import (
|
||||
_register_for,
|
||||
is_attention_backend_not_set,
|
||||
resolving_view,
|
||||
)
|
||||
|
||||
|
||||
@_register_for("IQuestQ1ForCausalLM", "IQuestQ1MTP")
|
||||
def _iquest_q1_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
cfg = resolving_view(server_args)
|
||||
requested = (
|
||||
cfg.attention_backend,
|
||||
cfg.prefill_attention_backend,
|
||||
cfg.decode_attention_backend,
|
||||
)
|
||||
if any(backend not in (None, "fa3") for backend in requested):
|
||||
raise ValueError(
|
||||
"IQuestQ1 applies its attention sink from the softmax LSE and "
|
||||
"requires FA3 attention for both prefill and decode."
|
||||
)
|
||||
if cfg.speculative_algorithm is not None:
|
||||
if cfg.speculative_algorithm.upper() not in ("EAGLE", "NEXTN"):
|
||||
raise ValueError("IQuest Q1 MTP requires serial EAGLE or NEXTN drafting.")
|
||||
if cfg.speculative_draft_attention_backend not in (None, "fa3"):
|
||||
raise ValueError("IQuestQ1 requires FA3 attention for the MTP draft.")
|
||||
if cfg.attn_cp_size > 1 or cfg.dcp_size > 1 or cfg.enable_prefill_cp:
|
||||
raise ValueError("IQuestQ1 does not support context parallelism.")
|
||||
if cfg.speculative_eagle_topk is not None and cfg.speculative_eagle_topk > 1:
|
||||
raise ValueError(
|
||||
"IQuestQ1 speculative decoding requires --speculative-eagle-topk 1."
|
||||
)
|
||||
if cfg.enable_multi_layer_eagle:
|
||||
raise ValueError("IQuest Q1 MTP drafting does not support multi-layer EAGLE.")
|
||||
if cfg.enable_dp_attention:
|
||||
raise ValueError("IQuest Q1 does not support DP attention.")
|
||||
if cfg.pp_size > 1:
|
||||
raise ValueError("IQuest Q1 does not support pipeline parallelism.")
|
||||
overrides = {}
|
||||
if is_attention_backend_not_set(cfg):
|
||||
overrides["attention_backend"] = "fa3"
|
||||
return overrides
|
||||
if cfg.attention_backend is None and cfg.prefill_attention_backend is None:
|
||||
overrides["prefill_attention_backend"] = "fa3"
|
||||
if cfg.attention_backend is None and cfg.decode_attention_backend is None:
|
||||
overrides["decode_attention_backend"] = "fa3"
|
||||
return overrides
|
||||
@@ -910,6 +910,54 @@ def _handle_frozen_kv_mtp(server_args: ServerArgs) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _handle_iquest_q1_mtp_draft(server_args: ServerArgs) -> bool:
|
||||
from sglang.srt.configs.iquest_q1 import IQuestQ1MTPConfig
|
||||
from sglang.srt.utils.hf_transformers_utils import get_config
|
||||
|
||||
target = model_config_of(server_args).hf_config
|
||||
if target.architectures[0] != "IQuestQ1ForCausalLM":
|
||||
return False
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.speculative_draft_model_path in (None, cfg.model_path):
|
||||
raise ValueError("IQuest Q1 requires an independent draft model path.")
|
||||
draft = get_config(
|
||||
cfg.speculative_draft_model_path,
|
||||
trust_remote_code=cfg.trust_remote_code,
|
||||
revision=cfg.speculative_draft_model_revision,
|
||||
)
|
||||
if not isinstance(draft, IQuestQ1MTPConfig):
|
||||
raise ValueError("IQuest Q1 requires an independent MTP draft checkpoint.")
|
||||
if draft.hidden_size != target.hidden_size or draft.vocab_size != target.vocab_size:
|
||||
raise ValueError("MTP draft hidden size and vocabulary must match the target.")
|
||||
if draft.num_target_layers != target.num_hidden_layers:
|
||||
raise ValueError("MTP draft target layer count must match the target.")
|
||||
if cfg.speculative_token_map is not None:
|
||||
raise ValueError("MTP uses its own full-vocabulary embedding and head.")
|
||||
steps = cfg.speculative_num_steps
|
||||
if steps is None:
|
||||
steps = draft.num_draft_slots
|
||||
if steps < 1:
|
||||
raise ValueError("MTP requires positive speculative_num_steps.")
|
||||
if steps > draft.num_draft_slots:
|
||||
logger.warning(
|
||||
"MTP depth %d exceeds the checkpoint's num_draft_slots=%d.",
|
||||
steps,
|
||||
draft.num_draft_slots,
|
||||
)
|
||||
if cfg.speculative_eagle_topk not in (None, 1):
|
||||
raise ValueError("MTP requires --speculative-eagle-topk 1.")
|
||||
if cfg.speculative_num_draft_tokens not in (None, steps + 1):
|
||||
raise ValueError("MTP verification width must equal draft steps + 1.")
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_iquest_q1_mtp_draft",
|
||||
speculative_num_steps=steps,
|
||||
speculative_eagle_topk=1,
|
||||
speculative_num_draft_tokens=steps + 1,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _handle_eagle_family(server_args: ServerArgs) -> None:
|
||||
cfg = resolving_view(server_args)
|
||||
|
||||
@@ -994,6 +1042,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
|
||||
"DeepSeek MTP does not require setting speculative_draft_model_path."
|
||||
)
|
||||
|
||||
_handle_iquest_q1_mtp_draft(server_args)
|
||||
if not cfg.speculative_adaptive and cfg.speculative_num_steps is None:
|
||||
assert (
|
||||
cfg.speculative_eagle_topk is None
|
||||
|
||||
@@ -33,6 +33,10 @@ from sglang.srt.configs.interns2_mobius import (
|
||||
InternS2MobiusVisionConfig,
|
||||
)
|
||||
from sglang.srt.configs.interns2preview import InternS2PreviewConfig
|
||||
from sglang.srt.configs.iquest_q1 import (
|
||||
IQuestQ1Config,
|
||||
IQuestQ1MTPConfig,
|
||||
)
|
||||
from sglang.srt.configs.janus_pro import MultiModalityConfig
|
||||
from sglang.srt.configs.jet_nemotron import JetNemotronConfig
|
||||
from sglang.srt.configs.jet_vlm import JetVLMConfig
|
||||
@@ -125,6 +129,8 @@ __all__ = [
|
||||
"Qwen3_5TextConfig",
|
||||
"Qwen3_5MoeTextConfig",
|
||||
"InternS2PreviewConfig",
|
||||
"IQuestQ1Config",
|
||||
"IQuestQ1MTPConfig",
|
||||
"InternS2MobiusConfig",
|
||||
"InternS2MobiusTextConfig",
|
||||
"InternS2MobiusVisionConfig",
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
from typing import List, Optional
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
|
||||
class IQuestQ1Config(PretrainedConfig):
|
||||
model_type = "iquest_q1"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size: int = 160000,
|
||||
hidden_size: int = 3072,
|
||||
dense_intermediate_size: int = 12288,
|
||||
intermediate_size: int = 1536,
|
||||
num_hidden_layers: int = 88,
|
||||
num_attention_heads: int = 48,
|
||||
num_key_value_heads: int = 8,
|
||||
head_dim: int = 128,
|
||||
hidden_act: str = "silu",
|
||||
max_position_embeddings: int = 524288,
|
||||
rms_norm_eps: float = 1e-6,
|
||||
use_cache: bool = True,
|
||||
pad_token_id: int = 0,
|
||||
bos_token_id: Optional[int] = None,
|
||||
eos_token_id: int = 0,
|
||||
tie_word_embeddings: bool = False,
|
||||
rope_theta: float = 1000000.0,
|
||||
rope_scaling: Optional[dict] = None,
|
||||
attention_bias: bool = False,
|
||||
attention_dropout: float = 0.0,
|
||||
rotary_dim: int = 32,
|
||||
rope_parameters: Optional[dict] = None,
|
||||
no_rope_layers: Optional[List[int]] = None,
|
||||
swa_rope_theta: float = 10000.0,
|
||||
sliding_window: int = 4096,
|
||||
use_hybrid_layers: bool = True,
|
||||
use_sliding_window: bool = True,
|
||||
first_layers_types: Optional[List[str]] = None,
|
||||
hybrid_layers_types_block: Optional[List[str]] = None,
|
||||
num_hybrid_layers_block: int = 21,
|
||||
last_layers_types: Optional[List[str]] = None,
|
||||
mlp_only_layers: Optional[List[int]] = None,
|
||||
num_experts: int = 256,
|
||||
num_experts_per_tok: int = 8,
|
||||
moe_router_dtype: str = "fp32",
|
||||
enable_sink_attention: bool = True,
|
||||
first_layer_attn_out_scale: float = 1.0,
|
||||
first_layer_ffn_out_scale: float = 1.0,
|
||||
attn_out_scale: float = 1.0,
|
||||
ffn_out_scale: float = 0.53881590608,
|
||||
softmax_scale: Optional[float] = None,
|
||||
logit_scale: float = 1.0,
|
||||
enable_lm_head_fp32: bool = True,
|
||||
num_mtp_layers: int = 0,
|
||||
**kwargs,
|
||||
):
|
||||
if first_layers_types is None:
|
||||
first_layers_types = ["full_attention"]
|
||||
if hybrid_layers_types_block is None:
|
||||
hybrid_layers_types_block = [
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
]
|
||||
if last_layers_types is None:
|
||||
last_layers_types = [
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
]
|
||||
if mlp_only_layers is None:
|
||||
mlp_only_layers = [0]
|
||||
layer_types = (
|
||||
list(first_layers_types)
|
||||
+ list(hybrid_layers_types_block) * num_hybrid_layers_block
|
||||
+ list(last_layers_types)
|
||||
)
|
||||
if not use_hybrid_layers:
|
||||
layer_types = ["full_attention"] * num_hidden_layers
|
||||
if len(layer_types) != num_hidden_layers:
|
||||
raise ValueError(
|
||||
"IQuest Q1 layer type pattern has length "
|
||||
f"{len(layer_types)}, expected {num_hidden_layers}"
|
||||
)
|
||||
if any(t not in ("full_attention", "sliding_attention") for t in layer_types):
|
||||
raise ValueError(
|
||||
"IQuest Q1 supports full_attention and sliding_attention layers"
|
||||
)
|
||||
if not use_sliding_window:
|
||||
layer_types = ["full_attention"] * num_hidden_layers
|
||||
if rotary_dim is not None and (
|
||||
rotary_dim <= 0 or rotary_dim > head_dim or rotary_dim % 2
|
||||
):
|
||||
raise ValueError("rotary_dim must be even, positive, and at most head_dim")
|
||||
if moe_router_dtype != "fp32":
|
||||
raise ValueError("IQuest Q1 requires moe_router_dtype='fp32'.")
|
||||
if kwargs.get("use_over_encoding", False):
|
||||
raise ValueError("IQuest Q1 over-encoding embeddings are not supported.")
|
||||
if enable_lm_head_fp32 is not True:
|
||||
raise ValueError("IQuest Q1 requires enable_lm_head_fp32=True.")
|
||||
kwargs.pop("layer_types", None)
|
||||
kwargs.setdefault("architectures", ["IQuestQ1ForCausalLM"])
|
||||
kwargs.setdefault("dtype", "bfloat16")
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
rope_scaling=rope_scaling,
|
||||
**kwargs,
|
||||
)
|
||||
self.vocab_size = vocab_size
|
||||
self.hidden_size = hidden_size
|
||||
self.dense_intermediate_size = dense_intermediate_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.head_dim = head_dim
|
||||
self.hidden_act = hidden_act
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.use_cache = use_cache
|
||||
self.rope_theta = rope_theta
|
||||
self.rope_scaling = rope_scaling
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
self.rotary_dim = rotary_dim
|
||||
self.swa_rope_theta = swa_rope_theta
|
||||
self.sliding_window = sliding_window if use_sliding_window else None
|
||||
self.use_hybrid_layers = use_hybrid_layers
|
||||
self.use_sliding_window = use_sliding_window
|
||||
self.no_rope_layers = list(no_rope_layers or [])
|
||||
self.first_layers_types = list(first_layers_types)
|
||||
self.hybrid_layers_types_block = list(hybrid_layers_types_block)
|
||||
self.num_hybrid_layers_block = num_hybrid_layers_block
|
||||
self.last_layers_types = list(last_layers_types)
|
||||
self.layer_types = layer_types
|
||||
self.hybrid_layer_pattern = [
|
||||
int(layer_type == "sliding_attention") for layer_type in layer_types
|
||||
]
|
||||
self.is_hybrid_swa = any(self.hybrid_layer_pattern)
|
||||
self.mlp_only_layers = list(mlp_only_layers)
|
||||
self.num_experts = num_experts
|
||||
self.num_experts_per_tok = num_experts_per_tok
|
||||
self.moe_router_dtype = moe_router_dtype
|
||||
self.enable_sink_attention = enable_sink_attention
|
||||
self.first_layer_attn_out_scale = first_layer_attn_out_scale
|
||||
self.first_layer_ffn_out_scale = first_layer_ffn_out_scale
|
||||
self.attn_out_scale = attn_out_scale
|
||||
self.ffn_out_scale = ffn_out_scale
|
||||
self.softmax_scale = softmax_scale
|
||||
self.logit_scale = logit_scale
|
||||
self.enable_lm_head_fp32 = enable_lm_head_fp32
|
||||
self.num_mtp_layers = num_mtp_layers
|
||||
self.rope_parameters = dict(rope_parameters or rope_scaling or {})
|
||||
self.rope_parameters.setdefault(
|
||||
"rope_type", self.rope_parameters.get("type", "default")
|
||||
)
|
||||
self.rope_parameters.setdefault("rope_theta", rope_theta)
|
||||
if rotary_dim is not None:
|
||||
self.rope_parameters["partial_rotary_factor"] = rotary_dim / head_dim
|
||||
|
||||
|
||||
class IQuestQ1MTPConfig(IQuestQ1Config):
|
||||
model_type = "iquest_q1_mtp"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
target_config: Optional[dict] = None,
|
||||
num_draft_slots: int = 7,
|
||||
num_target_layers: Optional[int] = None,
|
||||
sliding_window: Optional[int] = 512,
|
||||
swa_rope_theta: Optional[float] = 10000.0,
|
||||
fp32_residual_connection: bool = True,
|
||||
**kwargs,
|
||||
):
|
||||
if kwargs.get("num_hidden_layers", 1) != 1:
|
||||
raise ValueError("MTP supports one physical draft layer.")
|
||||
if num_draft_slots < 1:
|
||||
raise ValueError("num_draft_slots must be positive.")
|
||||
window = sliding_window or None
|
||||
if window is not None and (window < 0 or not swa_rope_theta):
|
||||
raise ValueError("A positive draft window requires swa_rope_theta.")
|
||||
config = dict(target_config or {})
|
||||
if num_target_layers is None:
|
||||
num_target_layers = config.get("num_hidden_layers", 88)
|
||||
if num_target_layers < 1:
|
||||
raise ValueError("num_target_layers must be positive.")
|
||||
if config.get("num_hidden_layers", num_target_layers) != num_target_layers:
|
||||
raise ValueError("num_target_layers must match target_config.")
|
||||
for key in ("model_type", "architectures", "auto_map", "_name_or_path"):
|
||||
config.pop(key, None)
|
||||
config.update(kwargs)
|
||||
config.update(
|
||||
architectures=["IQuestQ1MTP"],
|
||||
num_hidden_layers=1,
|
||||
mlp_only_layers=[],
|
||||
no_rope_layers=[],
|
||||
use_hybrid_layers=True,
|
||||
use_sliding_window=window is not None,
|
||||
sliding_window=window,
|
||||
first_layers_types=[
|
||||
"sliding_attention" if window is not None else "full_attention"
|
||||
],
|
||||
hybrid_layers_types_block=[],
|
||||
num_hybrid_layers_block=0,
|
||||
last_layers_types=[],
|
||||
)
|
||||
if swa_rope_theta is not None:
|
||||
config["swa_rope_theta"] = swa_rope_theta
|
||||
super().__init__(**config)
|
||||
self.target_config = dict(target_config or {})
|
||||
self.num_draft_slots = num_draft_slots
|
||||
self.num_target_layers = num_target_layers
|
||||
self.fp32_residual_connection = fp32_residual_connection
|
||||
@@ -88,6 +88,8 @@ SWA_SINK_ARCHS = frozenset(
|
||||
"GptOssForCausalLM",
|
||||
"GraniteSWAForCausalLM",
|
||||
"GraniteMoeSWAForCausalLM",
|
||||
"IQuestQ1ForCausalLM",
|
||||
"IQuestQ1MTP",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -106,7 +106,10 @@ from sglang.srt.parser.jinja_template_utils import (
|
||||
MEDIA_URL_PART_TYPES,
|
||||
process_content_for_template_format,
|
||||
)
|
||||
from sglang.srt.parser.reasoning_parser import ReasoningParser
|
||||
from sglang.srt.parser.reasoning_parser import (
|
||||
IQuestQ1ReasoningDetector,
|
||||
ReasoningParser,
|
||||
)
|
||||
from sglang.srt.sampling.sampling_params import (
|
||||
set_request_reasoning_end_token_ids,
|
||||
)
|
||||
@@ -2564,12 +2567,12 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
# as constraint (mirrors the streaming path). For auto: always try.
|
||||
if self.tool_call_parser:
|
||||
parser = FunctionCallParser(
|
||||
tools, self.tool_call_parser, tokenizer=self.tokenizer_manager.tokenizer
|
||||
)
|
||||
detector_owns_format = (
|
||||
parser.detector.supports_structural_tag()
|
||||
or parser.detector.parses_required_natively()
|
||||
tools,
|
||||
self.tool_call_parser,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
detector_owns_format = parser.owns_tool_format()
|
||||
should_try_parser = not is_required or detector_owns_format
|
||||
if should_try_parser and parser.has_tool_call(text):
|
||||
try:
|
||||
@@ -2844,6 +2847,14 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
request.reasoning_effort = "medium" if enabled else "no_think"
|
||||
return
|
||||
|
||||
if self.reasoning_parser == "iquest_q1":
|
||||
request.chat_template_kwargs = {
|
||||
**(request.chat_template_kwargs or {}),
|
||||
"thinking": enabled,
|
||||
"enable_thinking": enabled,
|
||||
}
|
||||
return
|
||||
|
||||
if self.reasoning_parser == "inkling":
|
||||
# Effort-conditioned, not toggled: "none" (0.0) is the off switch.
|
||||
if not enabled:
|
||||
@@ -2902,6 +2913,11 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
if not self.reasoning_parser:
|
||||
return False
|
||||
|
||||
if self.reasoning_parser == "iquest_q1":
|
||||
return IQuestQ1ReasoningDetector.thinking_enabled(
|
||||
request.chat_template_kwargs or {}
|
||||
)
|
||||
|
||||
if self.reasoning_parser == "minimax-m3":
|
||||
# M3 template prefills <mm:think> for thinking_mode=enabled, so it never
|
||||
# appears in output and reasoning must be forced. Mirrors reasoning_parser.py.
|
||||
@@ -3008,11 +3024,9 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
tools=effective_tools,
|
||||
tool_call_parser=self.tool_call_parser,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
tool_choice=request.tool_choice,
|
||||
)
|
||||
use_native_parser = (
|
||||
probe.detector.supports_structural_tag()
|
||||
or probe.detector.parses_required_natively()
|
||||
)
|
||||
use_native_parser = probe.owns_tool_format()
|
||||
if use_native_parser:
|
||||
parser_dict[index] = probe
|
||||
else:
|
||||
|
||||
@@ -1081,6 +1081,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
chat_tools,
|
||||
self.tool_call_parser,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
detector_owns_format = self._tool_parser_owns_format(parser)
|
||||
should_try_native = not is_required or detector_owns_format
|
||||
@@ -1166,10 +1167,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
|
||||
@staticmethod
|
||||
def _tool_parser_owns_format(parser: FunctionCallParser) -> bool:
|
||||
return (
|
||||
parser.detector.supports_structural_tag()
|
||||
or parser.detector.parses_required_natively()
|
||||
)
|
||||
return parser.owns_tool_format()
|
||||
|
||||
@staticmethod
|
||||
def _chat_tool_choice(tool_choice: Any) -> Any:
|
||||
@@ -2011,6 +2009,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
chat_tools,
|
||||
self.tool_call_parser,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
detector_owns_format = self._tool_parser_owns_format(probe)
|
||||
if is_required and not detector_owns_format:
|
||||
@@ -2020,6 +2019,7 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
chat_tools,
|
||||
self.tool_call_parser,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
reasoning_parser_obj: Optional[ReasoningParser] = None
|
||||
if self.reasoning_parser:
|
||||
|
||||
@@ -367,6 +367,9 @@ class BaseFormatDetector(ABC):
|
||||
constraints and parse the model's native output format instead."""
|
||||
return False
|
||||
|
||||
def get_required_tool_parser(self, tool_choice):
|
||||
return None
|
||||
|
||||
@abstractmethod
|
||||
def structure_info(self) -> _GetInfoFunc:
|
||||
"""
|
||||
|
||||
@@ -35,6 +35,7 @@ from sglang.srt.function_call.hermes_detector import HermesDetector
|
||||
from sglang.srt.function_call.hunyuan_detector import HunyuanDetector
|
||||
from sglang.srt.function_call.inkling_detector import InklingDetector
|
||||
from sglang.srt.function_call.internlm_detector import InternlmDetector
|
||||
from sglang.srt.function_call.iquest_q1_detector import IQuestQ1Detector
|
||||
from sglang.srt.function_call.k2_v3_detector import K2V3Detector
|
||||
from sglang.srt.function_call.kimik2_detector import KimiK2Detector
|
||||
from sglang.srt.function_call.kimik3_detector import KimiK3Detector
|
||||
@@ -113,9 +114,12 @@ class FunctionCallParser:
|
||||
"gigachat35": GigaChat35Detector,
|
||||
"gemma4": Gemma4Detector,
|
||||
"inkling": InklingDetector,
|
||||
"iquest_q1": IQuestQ1Detector,
|
||||
}
|
||||
|
||||
def __init__(self, tools: List[Tool], tool_call_parser: str, tokenizer=None):
|
||||
def __init__(
|
||||
self, tools: List[Tool], tool_call_parser: str, tokenizer=None, tool_choice=None
|
||||
):
|
||||
detector_class = self.ToolCallParserEnum.get(tool_call_parser)
|
||||
if detector_class:
|
||||
kwargs = {}
|
||||
@@ -131,10 +135,20 @@ class FunctionCallParser:
|
||||
detector.use_full_assistant_constraint = not any(
|
||||
tool.function.strict for tool in tools
|
||||
)
|
||||
self.detector = detector
|
||||
if isinstance(tool_choice, dict):
|
||||
tool_choice = ToolChoice(function={"name": tool_choice["name"]})
|
||||
self._required_tool_parser = detector.get_required_tool_parser(tool_choice)
|
||||
self.detector = self._required_tool_parser or detector
|
||||
self.tools = tools
|
||||
self.tool_strict_level = envs.SGLANG_TOOL_STRICT_LEVEL.get()
|
||||
|
||||
def owns_tool_format(self) -> bool:
|
||||
return (
|
||||
self._required_tool_parser is not None
|
||||
or self.detector.supports_structural_tag()
|
||||
or self.detector.parses_required_natively()
|
||||
)
|
||||
|
||||
def has_tool_call(self, text: str) -> bool:
|
||||
"""
|
||||
Check if the given text contains a tool call in the format supported by this parser.
|
||||
@@ -287,7 +301,7 @@ class FunctionCallParser:
|
||||
try:
|
||||
if (
|
||||
isinstance(self.detector, Glm47MoeDetector)
|
||||
and self.detector.use_full_assistant_constraint
|
||||
and self.detector.parses_required_natively()
|
||||
):
|
||||
functions = (
|
||||
[
|
||||
@@ -356,6 +370,14 @@ class FunctionCallParser:
|
||||
if (
|
||||
tool_choice == "required" or isinstance(tool_choice, ToolChoice)
|
||||
) and not self.detector.parses_required_natively():
|
||||
required_parser = self.detector.get_required_tool_parser(tool_choice)
|
||||
if required_parser is not None:
|
||||
return (
|
||||
"json_schema",
|
||||
required_parser.get_json_schema_constraint(
|
||||
self.tools, tool_choice, parallel_tool_calls
|
||||
),
|
||||
)
|
||||
json_schema = get_json_schema_constraint(
|
||||
self.tools, tool_choice, parallel_tool_calls=parallel_tool_calls
|
||||
)
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
import orjson
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import ToolChoice
|
||||
from sglang.srt.function_call.core_types import StreamingParseResult, ToolCallItem
|
||||
from sglang.srt.function_call.glm47_moe_detector import (
|
||||
Glm47MoeDetector,
|
||||
get_argument_type,
|
||||
)
|
||||
from sglang.srt.function_call.json_array_parser import JsonArrayParser
|
||||
from sglang.srt.function_call.utils import get_json_schema_constraint
|
||||
|
||||
|
||||
class IQuestQ1Detector(Glm47MoeDetector):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.bot_token = "<iquest_tool_call>"
|
||||
self.eot_token = "</iquest_tool_call>"
|
||||
self._inside_call = False
|
||||
|
||||
def _parse_body(self, body, tools):
|
||||
cursor = body.find("<arg_key>")
|
||||
name = body[:cursor].strip() if cursor >= 0 else body.strip()
|
||||
if not name:
|
||||
return None
|
||||
raw = body[cursor:] if cursor >= 0 else ""
|
||||
pairs = self.func_arg_regex.findall(raw)
|
||||
if self.func_arg_regex.sub("", raw).strip() or any(
|
||||
not key.strip() for key, _ in pairs
|
||||
):
|
||||
return None
|
||||
arguments = self._parse_argument_pairs(pairs, name, tools)
|
||||
for key, value in pairs:
|
||||
key = key.strip()
|
||||
if get_argument_type(name, key, tools, arguments) == "string":
|
||||
arguments[key] = value
|
||||
return {"name": name, "arguments": arguments}
|
||||
|
||||
def detect_and_parse(self, text, tools):
|
||||
calls = []
|
||||
normal = []
|
||||
content_start = 0
|
||||
search = 0
|
||||
while (start := text.find(self.bot_token, search)) >= 0:
|
||||
begin = start + len(self.bot_token)
|
||||
end = text.find(self.eot_token, begin)
|
||||
if end < 0:
|
||||
break
|
||||
action = self._parse_body(text[begin:end], tools)
|
||||
search = end + len(self.eot_token)
|
||||
if action is not None:
|
||||
for call in self.parse_base_json(action, tools):
|
||||
call.tool_index = len(calls)
|
||||
calls.append(call)
|
||||
normal.append(text[content_start:start])
|
||||
content_start = search
|
||||
normal.append(text[content_start:])
|
||||
return StreamingParseResult(
|
||||
normal_text="".join(normal),
|
||||
calls=calls,
|
||||
)
|
||||
|
||||
def parse_streaming_increment(self, new_text, tools):
|
||||
self._buffer += new_text
|
||||
normal, calls = [], []
|
||||
while True:
|
||||
if not self._inside_call:
|
||||
start = self._buffer.find(self.bot_token)
|
||||
if start < 0:
|
||||
keep = self._ends_with_partial_token(self._buffer, self.bot_token)
|
||||
end = len(self._buffer) - keep
|
||||
normal.append(self._buffer[:end])
|
||||
self._buffer = self._buffer[end:]
|
||||
break
|
||||
normal.append(self._buffer[:start])
|
||||
self._buffer = self._buffer[start + len(self.bot_token) :]
|
||||
self._inside_call = True
|
||||
end = self._buffer.find(self.eot_token)
|
||||
if end < 0:
|
||||
break
|
||||
body = self._buffer[:end]
|
||||
self._buffer = self._buffer[end + len(self.eot_token) :]
|
||||
self._inside_call = False
|
||||
action = self._parse_body(body, tools)
|
||||
if action is None:
|
||||
normal.append(self.bot_token + body + self.eot_token)
|
||||
continue
|
||||
for call in self.parse_base_json(action, tools):
|
||||
call.tool_index = len(self.prev_tool_call_arr)
|
||||
calls.append(call)
|
||||
self.prev_tool_call_arr.append(action)
|
||||
self.streamed_args_for_tool.append(call.parameters)
|
||||
self.current_tool_id = call.tool_index
|
||||
self.current_tool_name_sent = True
|
||||
return StreamingParseResult(normal_text="".join(normal), calls=calls)
|
||||
|
||||
def finish(self, tools):
|
||||
text = (self.bot_token if self._inside_call else "") + self._buffer
|
||||
self._buffer = ""
|
||||
self._inside_call = False
|
||||
return StreamingParseResult(normal_text=text)
|
||||
|
||||
def parses_required_natively(self):
|
||||
return False
|
||||
|
||||
def get_required_tool_parser(self, tool_choice):
|
||||
if tool_choice != "required" and not isinstance(tool_choice, ToolChoice):
|
||||
return None
|
||||
name = (
|
||||
tool_choice.function.name if isinstance(tool_choice, ToolChoice) else None
|
||||
)
|
||||
return IQuestQ1JsonToolParser(name)
|
||||
|
||||
def supports_structural_tag(self):
|
||||
return False
|
||||
|
||||
def structure_info(self):
|
||||
raise NotImplementedError("IQuest Q1 uses XML argument pairs")
|
||||
|
||||
|
||||
class IQuestQ1JsonToolParser(JsonArrayParser):
|
||||
def __init__(self, tool_name=None):
|
||||
super().__init__()
|
||||
self.tool_name = tool_name
|
||||
|
||||
def supports_structural_tag(self):
|
||||
return False
|
||||
|
||||
def get_required_tool_parser(self, tool_choice):
|
||||
return self
|
||||
|
||||
def get_json_schema_constraint(self, tools, tool_choice, parallel_tool_calls=True):
|
||||
if self.tool_name is not None:
|
||||
return next(
|
||||
tool.function.parameters or {}
|
||||
for tool in tools
|
||||
if tool.function.name == self.tool_name
|
||||
)
|
||||
return get_json_schema_constraint(
|
||||
tools, tool_choice, parallel_tool_calls=parallel_tool_calls
|
||||
)
|
||||
|
||||
def has_tool_call(self, text):
|
||||
return True
|
||||
|
||||
def detect_and_parse(self, text, tools):
|
||||
value = orjson.loads(text)
|
||||
if self.tool_name is not None:
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError("Named tool arguments must be a JSON object.")
|
||||
return StreamingParseResult(
|
||||
calls=[ToolCallItem(tool_index=0, name=self.tool_name, parameters=text)]
|
||||
)
|
||||
if not isinstance(value, list) or not all(
|
||||
isinstance(call, dict) and "name" in call for call in value
|
||||
):
|
||||
raise ValueError("Expected a JSON array of named tool calls.")
|
||||
return StreamingParseResult(calls=self.parse_base_json(value, tools))
|
||||
|
||||
def parse_streaming_increment(self, new_text, tools):
|
||||
if self.tool_name is not None:
|
||||
if not new_text:
|
||||
return StreamingParseResult()
|
||||
name = None if self.current_tool_name_sent else self.tool_name
|
||||
self.current_tool_name_sent = True
|
||||
return StreamingParseResult(
|
||||
calls=[ToolCallItem(tool_index=0, name=name, parameters=new_text)]
|
||||
)
|
||||
|
||||
calls = []
|
||||
while True:
|
||||
result = super().parse_streaming_increment(new_text, tools)
|
||||
calls.extend(result.calls)
|
||||
if not result.calls:
|
||||
break
|
||||
new_text = ""
|
||||
return StreamingParseResult(calls=calls)
|
||||
@@ -49,4 +49,5 @@ TOOL_CALL_PARSER_NAMES = [
|
||||
"gigachat35",
|
||||
"gemma4",
|
||||
"inkling",
|
||||
"iquest_q1",
|
||||
]
|
||||
|
||||
@@ -42,6 +42,7 @@ from sglang.srt.runtime_context import (
|
||||
get_schedule,
|
||||
get_spec,
|
||||
)
|
||||
from sglang.srt.speculative.eagle_utils import per_step_draft_out_cache_loc
|
||||
from sglang.srt.speculative.ragged_verify import build_ragged_target_verify_geometry
|
||||
from sglang.srt.speculative.spec_info import SpecInput, SpeculativeAlgorithm
|
||||
from sglang.srt.speculative.spec_utils import resolve_num_tokens_per_req
|
||||
@@ -518,6 +519,23 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
else:
|
||||
self._init_full_cg_decode_metadata(forward_batch, in_capture)
|
||||
|
||||
def _get_swa_write_locations(self, forward_batch: ForwardBatch):
|
||||
locations = getattr(forward_batch, "out_cache_loc", None)
|
||||
if (
|
||||
locations is not None
|
||||
and forward_batch.forward_mode.is_decode_or_idle()
|
||||
and forward_batch.spec_info is not None
|
||||
and self.topk > 0
|
||||
and self.speculative_num_steps > 1
|
||||
):
|
||||
locations = per_step_draft_out_cache_loc(
|
||||
locations,
|
||||
forward_batch.batch_size,
|
||||
self.topk,
|
||||
self.speculative_num_steps,
|
||||
)[self.speculative_step_id]
|
||||
return locations
|
||||
|
||||
def _init_full_cg_decode_metadata(
|
||||
self, forward_batch: ForwardBatch, in_capture: bool
|
||||
):
|
||||
@@ -534,13 +552,21 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
# Refill the SWA write-target buffer (bound as a metadata view in
|
||||
# _bind_metadata_buffers) from the live out_cache_loc before replay.
|
||||
if self.use_sliding_window_kv_pool and out_cache_loc is not None:
|
||||
n = out_cache_loc.shape[0]
|
||||
swa_out_cache_loc = self._get_swa_write_locations(forward_batch)
|
||||
n = swa_out_cache_loc.shape[0]
|
||||
assert_buffer_fits(
|
||||
n,
|
||||
self.cuda_graph_swa_out_cache_loc.shape[0],
|
||||
"full decode SWA write locations",
|
||||
)
|
||||
self.cuda_graph_swa_out_cache_loc[n:].zero_()
|
||||
if in_capture:
|
||||
self.cuda_graph_swa_out_cache_loc[:n].zero_()
|
||||
else:
|
||||
self.cuda_graph_swa_out_cache_loc[:n].copy_(
|
||||
self.kv_index_translator.sliding_window_write_loc_for(out_cache_loc)
|
||||
self.kv_index_translator.sliding_window_write_loc_for(
|
||||
swa_out_cache_loc
|
||||
)
|
||||
)
|
||||
|
||||
if in_capture:
|
||||
@@ -1148,18 +1174,23 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
_unified_read = (
|
||||
self.kv_index_translator.is_translating and metadata.page_table is not None
|
||||
)
|
||||
swa_out_cache_loc = (
|
||||
self._get_swa_write_locations(forward_batch)
|
||||
if self.use_sliding_window_kv_pool
|
||||
else None
|
||||
)
|
||||
if _unified_read:
|
||||
kv_view = self.kv_index_translator.index_table_for_batch(forward_batch)
|
||||
metadata.page_table = kv_view.ids
|
||||
if self.use_sliding_window_kv_pool:
|
||||
metadata.swa_page_table = kv_view.sliding_window_ids
|
||||
if forward_batch.out_cache_loc is not None:
|
||||
if swa_out_cache_loc is not None:
|
||||
# The swa write loc was computed from the still-VIRTUAL
|
||||
# loc at ForwardBatch construction; re-running the
|
||||
# full->swa map on the kernel-facing loc would be garbage.
|
||||
metadata.swa_out_cache_loc = (
|
||||
self.kv_index_translator.sliding_window_write_loc_for(
|
||||
forward_batch.out_cache_loc
|
||||
swa_out_cache_loc
|
||||
)
|
||||
)
|
||||
elif self.use_sliding_window_kv_pool:
|
||||
@@ -1169,10 +1200,10 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
metadata.page_table
|
||||
).to(torch.int32)
|
||||
)
|
||||
if forward_batch.out_cache_loc is not None:
|
||||
if swa_out_cache_loc is not None:
|
||||
metadata.swa_out_cache_loc = (
|
||||
self.token_to_kv_pool.translate_loc_from_full_to_swa(
|
||||
forward_batch.out_cache_loc
|
||||
swa_out_cache_loc
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1322,9 +1353,13 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
aux_tensors=None,
|
||||
rel_bias=None,
|
||||
rel_bias_event=None,
|
||||
return_lse: bool = False,
|
||||
):
|
||||
if score_mod is not None and self.fa_impl_ver != 4:
|
||||
raise RuntimeError("score_mod is only supported by the FA4 backend.")
|
||||
if return_lse and self.use_mla:
|
||||
raise NotImplementedError("MLA extend does not return an LSE.")
|
||||
extend_lse = None
|
||||
cp_active = is_cp_active(forward_batch)
|
||||
|
||||
if k is not None:
|
||||
@@ -1602,13 +1637,19 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
causal=False if use_cascade_attn else causal,
|
||||
window_size=window_size,
|
||||
softcap=layer.logit_cap,
|
||||
return_softmax_lse=use_cascade_attn,
|
||||
return_softmax_lse=use_cascade_attn or return_lse,
|
||||
num_splits=self.num_splits,
|
||||
out=_fa_out,
|
||||
ver=self.fa_impl_ver,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if return_lse and not use_cascade_attn:
|
||||
if not isinstance(result, tuple):
|
||||
raise RuntimeError("This extend path does not return an LSE.")
|
||||
o, softmax_lse, *_ = result
|
||||
extend_lse = softmax_lse.T.contiguous()
|
||||
result = o
|
||||
if use_cascade_attn:
|
||||
o, softmax_lse, *rest = result
|
||||
if (
|
||||
@@ -1862,7 +1903,12 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
else:
|
||||
o = result
|
||||
|
||||
return o.view(-1, layer.tp_q_head_num * layer.v_head_dim)
|
||||
o = o.view(-1, layer.tp_q_head_num * layer.v_head_dim)
|
||||
if return_lse:
|
||||
if extend_lse is None:
|
||||
raise RuntimeError("This extend path does not return an LSE.")
|
||||
return o, extend_lse
|
||||
return o
|
||||
|
||||
def forward_decode(
|
||||
self,
|
||||
@@ -1883,9 +1929,13 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
aux_tensors=None,
|
||||
rel_bias=None,
|
||||
rel_bias_event=None,
|
||||
return_lse: bool = False,
|
||||
) -> torch.Tensor:
|
||||
if score_mod is not None and self.fa_impl_ver != 4:
|
||||
raise RuntimeError("score_mod is only supported by the FA4 backend.")
|
||||
if return_lse and self.use_mla:
|
||||
raise NotImplementedError("MLA decode does not return an LSE.")
|
||||
decode_lse = None
|
||||
if k is not None:
|
||||
assert v is not None
|
||||
if save_kv_cache:
|
||||
@@ -2089,7 +2139,7 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
causal=False if use_cascade_attn else causal,
|
||||
window_size=window_size,
|
||||
softcap=layer.logit_cap,
|
||||
return_softmax_lse=use_cascade_attn,
|
||||
return_softmax_lse=use_cascade_attn or return_lse,
|
||||
num_splits=(
|
||||
self.decode_num_splits
|
||||
if not is_swa_layer
|
||||
@@ -2102,6 +2152,10 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
scheduler_metadata=sched_meta,
|
||||
**kwargs,
|
||||
)
|
||||
if return_lse and not use_cascade_attn:
|
||||
o, softmax_lse, *_ = result
|
||||
decode_lse = softmax_lse.T.contiguous()
|
||||
result = o
|
||||
if use_cascade_attn:
|
||||
o, softmax_lse, *rest = result
|
||||
if self._decode_uses_static_max_seqlen_k:
|
||||
@@ -2211,7 +2265,12 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
else:
|
||||
o = result
|
||||
|
||||
return o.view(-1, layer.tp_q_head_num * layer.v_head_dim)
|
||||
o = o.view(-1, layer.tp_q_head_num * layer.v_head_dim)
|
||||
if return_lse:
|
||||
if decode_lse is None:
|
||||
raise RuntimeError("This decode path does not return an LSE.")
|
||||
return o, decode_lse
|
||||
return o
|
||||
|
||||
def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int):
|
||||
"""Initialize CUDA graph state for the attention backend.
|
||||
|
||||
@@ -193,7 +193,7 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
|
||||
return output
|
||||
|
||||
matched = finished_reason.get("matched", None)
|
||||
if not matched:
|
||||
if matched is None:
|
||||
return output
|
||||
|
||||
# TODO(lmzheng): handle the case where multiple stop strs are hit
|
||||
|
||||
@@ -0,0 +1,732 @@
|
||||
import logging
|
||||
from typing import Iterable, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
from triton.language.extra import libdevice
|
||||
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import post_experts_all_reduce
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.moe.utils import RoutingMethodType
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8Config
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers, set_weight_attrs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_tokens"])
|
||||
def _apply_learned_sink_kernel(
|
||||
query,
|
||||
sink_key,
|
||||
attn_output,
|
||||
lse,
|
||||
result,
|
||||
num_tokens,
|
||||
query_stride_t: tl.constexpr,
|
||||
query_stride_h: tl.constexpr,
|
||||
query_stride_d: tl.constexpr,
|
||||
sink_stride_h: tl.constexpr,
|
||||
sink_stride_d: tl.constexpr,
|
||||
output_stride_t: tl.constexpr,
|
||||
output_stride_h: tl.constexpr,
|
||||
output_stride_d: tl.constexpr,
|
||||
lse_stride_t: tl.constexpr,
|
||||
lse_stride_h: tl.constexpr,
|
||||
NUM_HEADS: tl.constexpr,
|
||||
QUERIES_PER_KV: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
SCALE: tl.constexpr,
|
||||
BLOCK_ROWS: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
rows = tl.program_id(0).to(tl.int64) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
|
||||
tokens = rows // NUM_HEADS
|
||||
heads = rows % NUM_HEADS
|
||||
dims = tl.arange(0, BLOCK_D)
|
||||
mask = (tokens[:, None] < num_tokens) & (dims[None, :] < HEAD_DIM)
|
||||
q = tl.load(
|
||||
query
|
||||
+ tokens[:, None] * query_stride_t
|
||||
+ heads[:, None] * query_stride_h
|
||||
+ dims[None, :] * query_stride_d,
|
||||
mask,
|
||||
0,
|
||||
)
|
||||
sink = tl.load(
|
||||
sink_key
|
||||
+ (heads[:, None] // QUERIES_PER_KV) * sink_stride_h
|
||||
+ dims[None, :] * sink_stride_d,
|
||||
mask,
|
||||
0,
|
||||
).to(q.dtype)
|
||||
products = q.to(tl.float32) * sink.to(tl.float32)
|
||||
if HEAD_DIM == 128:
|
||||
# Keep the reference reduction grouping to limit FP32 rounding drift.
|
||||
even, odd = tl.split(products.reshape(BLOCK_ROWS, HEAD_DIM // 2, 2))
|
||||
first, third = tl.split(even.reshape(BLOCK_ROWS, HEAD_DIM // 4, 2))
|
||||
second, fourth = tl.split(odd.reshape(BLOCK_ROWS, HEAD_DIM // 4, 2))
|
||||
sink_dot = tl.sum(((first + second) + third) + fourth, axis=1)
|
||||
else:
|
||||
sink_dot = tl.sum(products, axis=1)
|
||||
sink_logit = sink_dot * SCALE
|
||||
normal_lse = tl.load(
|
||||
lse + tokens * lse_stride_t + heads * lse_stride_h,
|
||||
tokens < num_tokens,
|
||||
0,
|
||||
).to(tl.float32)
|
||||
factor = tl.div_rn(1.0, 1.0 + libdevice.exp(sink_logit - normal_lse))
|
||||
values = tl.load(
|
||||
attn_output
|
||||
+ tokens[:, None] * output_stride_t
|
||||
+ heads[:, None] * output_stride_h
|
||||
+ dims[None, :] * output_stride_d,
|
||||
mask,
|
||||
0,
|
||||
).to(tl.float32)
|
||||
tl.store(
|
||||
result + rows[:, None] * HEAD_DIM + dims[None, :],
|
||||
values * factor[:, None],
|
||||
mask,
|
||||
)
|
||||
|
||||
|
||||
def _apply_learned_sink(
|
||||
query: torch.Tensor,
|
||||
sink_key: torch.Tensor,
|
||||
attn_output: torch.Tensor,
|
||||
lse: torch.Tensor,
|
||||
scale: float,
|
||||
) -> torch.Tensor:
|
||||
num_tokens, num_heads, head_dim = query.shape
|
||||
assert attn_output.shape == query.shape
|
||||
assert sink_key.ndim == 2 and sink_key.shape[1] == head_dim
|
||||
assert num_heads % sink_key.shape[0] == 0
|
||||
assert lse.shape == (num_tokens, num_heads)
|
||||
result = torch.empty(query.shape, dtype=attn_output.dtype, device=query.device)
|
||||
if num_tokens == 0:
|
||||
return result
|
||||
# Larger tiles reduce prefill overhead but cost more on short decode batches.
|
||||
block_rows = 64 if num_tokens * num_heads >= 32768 else 4
|
||||
_apply_learned_sink_kernel[(triton.cdiv(num_tokens * num_heads, block_rows),)](
|
||||
query,
|
||||
sink_key,
|
||||
attn_output,
|
||||
lse,
|
||||
result,
|
||||
num_tokens,
|
||||
*query.stride(),
|
||||
*sink_key.stride(),
|
||||
*attn_output.stride(),
|
||||
*lse.stride(),
|
||||
NUM_HEADS=num_heads,
|
||||
QUERIES_PER_KV=num_heads // sink_key.shape[0],
|
||||
HEAD_DIM=head_dim,
|
||||
SCALE=scale,
|
||||
BLOCK_ROWS=block_rows,
|
||||
BLOCK_D=triton.next_power_of_2(head_dim),
|
||||
num_warps=4,
|
||||
# Avoid fused multiply-add rounding differences from the reference.
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
class IQuestQ1RMSNorm(nn.Module):
|
||||
def __init__(self, hidden_size: int, eps: float = 1e-6):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size))
|
||||
self.variance_epsilon = eps
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
# The draft passes FP32 residuals with BF16 weights, which the fused CUDA
|
||||
# RMSNorm rejects; it also rounds BF16 inputs differently.
|
||||
input_dtype = hidden_states.dtype
|
||||
hidden_states = hidden_states.to(torch.float32)
|
||||
variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
||||
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
|
||||
result = (self.weight * hidden_states).to(input_dtype)
|
||||
return result
|
||||
|
||||
def extra_repr(self):
|
||||
return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
|
||||
|
||||
|
||||
class IQuestQ1MoEBlock(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
layer_id: int,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
self.num_experts = config.num_experts
|
||||
self.top_k = config.num_experts_per_tok
|
||||
self.gate = ReplicatedLinear(
|
||||
config.hidden_size,
|
||||
config.num_experts,
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
quant_config=None,
|
||||
prefix=add_prefix("gate", prefix),
|
||||
)
|
||||
self.topk = TopK(
|
||||
top_k=self.top_k,
|
||||
renormalize=True,
|
||||
layer_id=layer_id,
|
||||
)
|
||||
self.experts = get_moe_impl_class(quant_config)(
|
||||
num_experts=config.num_experts,
|
||||
top_k=self.top_k,
|
||||
layer_id=layer_id,
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("experts", prefix),
|
||||
routing_method_type=RoutingMethodType.Renormalize,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
original_shape = hidden_states.shape
|
||||
hidden_states = hidden_states.reshape(-1, self.hidden_size)
|
||||
router_logits, _ = self.gate(hidden_states.float())
|
||||
topk_output = self.topk(hidden_states, router_logits)
|
||||
output = self.experts(hidden_states, topk_output)
|
||||
output = post_experts_all_reduce(output)
|
||||
return output.reshape(original_shape)
|
||||
|
||||
|
||||
class IQuestQ1DenseMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
config.hidden_size,
|
||||
[config.dense_intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
config.dense_intermediate_size,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
)
|
||||
if config.hidden_act != "silu":
|
||||
raise ValueError(
|
||||
f"Unsupported activation: {config.hidden_act}. Only silu is supported."
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
gate_up, _ = self.gate_up_proj(hidden_states)
|
||||
hidden_states = self.act_fn(gate_up)
|
||||
hidden_states, _ = self.down_proj(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class IQuestQ1Attention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
layer_id: int,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
parallel = get_parallel()
|
||||
attn_tp_size = parallel.attn_tp_size
|
||||
attn_tp_rank = parallel.attn_tp_rank
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
self.total_num_kv_heads = config.num_key_value_heads
|
||||
assert self.total_num_heads % attn_tp_size == 0
|
||||
if self.total_num_kv_heads >= attn_tp_size:
|
||||
assert self.total_num_kv_heads % attn_tp_size == 0
|
||||
else:
|
||||
assert attn_tp_size % self.total_num_kv_heads == 0
|
||||
self.num_heads = self.total_num_heads // attn_tp_size
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // attn_tp_size)
|
||||
self.head_dim = config.head_dim
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
self.scaling = (
|
||||
self.head_dim**-0.5
|
||||
if config.softmax_scale is None
|
||||
else config.softmax_scale
|
||||
)
|
||||
self.attn_tp_rank = attn_tp_rank
|
||||
self.attn_tp_size = attn_tp_size
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
config.hidden_size,
|
||||
self.head_dim,
|
||||
self.total_num_heads,
|
||||
self.total_num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
kv_tp_rank=attn_tp_rank,
|
||||
kv_tp_size=attn_tp_size,
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.total_num_heads * self.head_dim,
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
self.q_norm = IQuestQ1RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = IQuestQ1RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.is_sliding = (
|
||||
config.layer_types[layer_id] == "sliding_attention"
|
||||
and config.sliding_window is not None
|
||||
)
|
||||
rope_theta = (
|
||||
config.swa_rope_theta
|
||||
if self.is_sliding
|
||||
else config.rope_parameters["rope_theta"]
|
||||
)
|
||||
self.rotary_emb = (
|
||||
get_rope(
|
||||
self.head_dim,
|
||||
rotary_dim=int(
|
||||
self.head_dim
|
||||
* config.rope_parameters.get("partial_rotary_factor", 1.0)
|
||||
),
|
||||
max_position=config.max_position_embeddings,
|
||||
base=rope_theta,
|
||||
rope_scaling=config.rope_parameters,
|
||||
is_neox_style=True,
|
||||
)
|
||||
if layer_id not in config.no_rope_layers
|
||||
else None
|
||||
)
|
||||
sliding_window_size = config.sliding_window - 1 if self.is_sliding else -1
|
||||
self.attn = RadixAttention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
layer_id=layer_id,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("attn", prefix),
|
||||
sliding_window_size=sliding_window_size,
|
||||
)
|
||||
self.enable_sink_attention = config.enable_sink_attention
|
||||
if self.enable_sink_attention:
|
||||
self.sink_k = nn.Parameter(
|
||||
torch.zeros(self.num_kv_heads, self.head_dim), requires_grad=False
|
||||
)
|
||||
set_weight_attrs(self.sink_k, {"weight_loader": self._sink_k_loader})
|
||||
|
||||
def _sink_k_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor) -> None:
|
||||
if loaded_weight.dim() == 3:
|
||||
loaded_weight = loaded_weight.squeeze(0)
|
||||
if self.total_num_kv_heads >= self.attn_tp_size:
|
||||
start = self.attn_tp_rank * self.num_kv_heads
|
||||
else:
|
||||
replicas = self.attn_tp_size // self.total_num_kv_heads
|
||||
start = self.attn_tp_rank // replicas
|
||||
local_weight = loaded_weight[start : start + self.num_kv_heads]
|
||||
assert local_weight.shape == param.shape, (
|
||||
f"Unexpected sink_k shard shape {tuple(local_weight.shape)} for "
|
||||
f"rank-local parameter {tuple(param.shape)}"
|
||||
)
|
||||
param.data.copy_(local_weight)
|
||||
|
||||
def _apply_zero_value_sink(
|
||||
self, q: torch.Tensor, attn_output: torch.Tensor, lse: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
q = q.view(-1, self.num_heads, self.head_dim)
|
||||
return _apply_learned_sink(
|
||||
q,
|
||||
self.sink_k,
|
||||
attn_output.view(q.shape),
|
||||
lse,
|
||||
self.scaling,
|
||||
).view(-1, self.q_size)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
q = self.q_norm(q.view(-1, self.num_heads, self.head_dim)).view(q.shape)
|
||||
k = self.k_norm(k.view(-1, self.num_kv_heads, self.head_dim)).view(k.shape)
|
||||
if self.rotary_emb is not None:
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
if self.enable_sink_attention:
|
||||
attn_output, lse = self.attn(q, k, v, forward_batch, return_lse=True)
|
||||
attn_output = self._apply_zero_value_sink(q, attn_output, lse)
|
||||
else:
|
||||
attn_output = self.attn(q, k, v, forward_batch)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
class IQuestQ1DecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
layer_id: int,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.attention_norm = IQuestQ1RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_out_norm = IQuestQ1RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.feed_forward_norm = IQuestQ1RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.self_attn = IQuestQ1Attention(
|
||||
config=config,
|
||||
layer_id=layer_id,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("self_attn", prefix),
|
||||
)
|
||||
if layer_id in (config.mlp_only_layers or []):
|
||||
self.mlp = IQuestQ1DenseMLP(
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
else:
|
||||
self.mlp = IQuestQ1MoEBlock(
|
||||
config=config,
|
||||
layer_id=layer_id,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
self.is_first_layer = layer_id == 0
|
||||
if self.is_first_layer:
|
||||
self.ffn_out_norm = IQuestQ1RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_out_scale = config.first_layer_attn_out_scale
|
||||
self.ffn_out_scale = config.first_layer_ffn_out_scale
|
||||
else:
|
||||
self.attn_out_scale = config.attn_out_scale
|
||||
self.ffn_out_scale = config.ffn_out_scale
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
# Intentional: layer 0 adds the raw input and an FFN output norm, later layers
|
||||
# add the normalized input without one; the MTP draft follows layer 0.
|
||||
if self.is_first_layer:
|
||||
norm_hidden_states = self.attention_norm(hidden_states)
|
||||
attn_output = self.self_attn(positions, norm_hidden_states, forward_batch)
|
||||
hidden_states = hidden_states + self.attn_out_norm(attn_output) * (
|
||||
self.attn_out_scale
|
||||
)
|
||||
mlp_output = self.mlp(self.feed_forward_norm(hidden_states))
|
||||
return hidden_states + self.ffn_out_norm(mlp_output) * self.ffn_out_scale
|
||||
norm_hidden_states = self.attention_norm(hidden_states)
|
||||
attn_output = self.self_attn(positions, norm_hidden_states, forward_batch)
|
||||
hidden_states = norm_hidden_states + self.attn_out_norm(attn_output) * (
|
||||
self.attn_out_scale
|
||||
)
|
||||
mlp_output = self.mlp(self.feed_forward_norm(hidden_states))
|
||||
return hidden_states + mlp_output * self.ffn_out_scale
|
||||
|
||||
|
||||
class IQuestQ1Model(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=add_prefix("embed_tokens", prefix),
|
||||
)
|
||||
self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: IQuestQ1DecoderLayer(
|
||||
config=config,
|
||||
layer_id=idx,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
self.norm = IQuestQ1RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_embeds: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
if input_embeds is None:
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
for layer in self.layers:
|
||||
hidden_states = layer(positions, hidden_states, forward_batch)
|
||||
return self.norm(hidden_states)
|
||||
|
||||
|
||||
_FUSED_SHARD_LAYOUT = (
|
||||
("qkv_proj", ("q", "k", "v")),
|
||||
("gate_up_proj", (0, 1)),
|
||||
)
|
||||
|
||||
|
||||
def _is_created_by_quantization(
|
||||
name: str, param: torch.nn.Parameter, quant_config: Optional[QuantizationConfig]
|
||||
) -> bool:
|
||||
if getattr(param, "_skip_weight_check", False):
|
||||
return True
|
||||
return (
|
||||
isinstance(quant_config, Fp8Config)
|
||||
and not quant_config.is_checkpoint_fp8_serialized
|
||||
and name.rsplit(".", 1)[-1]
|
||||
in {
|
||||
"w13_weight_scale",
|
||||
"w2_weight_scale",
|
||||
"w13_weight_scale_inv",
|
||||
"w2_weight_scale_inv",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def check_all_params_loaded(
|
||||
params_dict: dict,
|
||||
loaded: set,
|
||||
model: str,
|
||||
loaded_shards: Optional[dict] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
) -> None:
|
||||
missing = sorted(
|
||||
name
|
||||
for name, param in params_dict.items()
|
||||
if name not in loaded
|
||||
and not _is_created_by_quantization(name, param, quant_config)
|
||||
)
|
||||
partial = []
|
||||
for name, shards in sorted((loaded_shards or {}).items()):
|
||||
for projection, expected in _FUSED_SHARD_LAYOUT:
|
||||
if not name.endswith(f".{projection}.weight"):
|
||||
continue
|
||||
absent = [shard for shard in expected if shard not in shards]
|
||||
if absent:
|
||||
partial.append(f"{name} without {absent}")
|
||||
if not missing and not partial:
|
||||
return
|
||||
problems = []
|
||||
if missing:
|
||||
problems.append(
|
||||
f"{len(missing)} parameters were not found in the checkpoint, "
|
||||
f"e.g. {missing[:8]}"
|
||||
)
|
||||
if partial:
|
||||
problems.append(
|
||||
f"{len(partial)} fused parameters were only partially loaded, "
|
||||
f"e.g. {partial[:8]}"
|
||||
)
|
||||
message = f"{model}: " + "; ".join(problems)
|
||||
if quant_config is not None and not partial:
|
||||
logger.warning(
|
||||
"%s; verify whether these parameters are generated by quantization.",
|
||||
message,
|
||||
)
|
||||
return
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def load_iquest_q1_weights(
|
||||
model: nn.Module, weights: Iterable[Tuple[str, torch.Tensor]]
|
||||
):
|
||||
stacked_params_mapping = [
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
params_dict = dict(model.named_parameters())
|
||||
loaded = set()
|
||||
loaded_shards = {}
|
||||
for name, loaded_weight in weights:
|
||||
if (
|
||||
name == "lm_head.weight"
|
||||
and name not in params_dict
|
||||
and "model.embed_tokens.weight" in params_dict
|
||||
):
|
||||
name = "model.embed_tokens.weight"
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
if name.endswith("experts.fc"):
|
||||
param_name = name.replace("experts.fc", "experts.w13_weight")
|
||||
if param_name not in params_dict:
|
||||
raise ValueError(f"Unexpected IQuest Q1 weight: {name}")
|
||||
param = params_dict[param_name]
|
||||
loaded.add(param_name)
|
||||
for expert_id in range(model.config.num_experts):
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight[expert_id][: model.config.intermediate_size],
|
||||
param_name,
|
||||
shard_id="w1",
|
||||
expert_id=expert_id,
|
||||
)
|
||||
weight_loader(
|
||||
param,
|
||||
loaded_weight[expert_id][model.config.intermediate_size :],
|
||||
param_name,
|
||||
shard_id="w3",
|
||||
expert_id=expert_id,
|
||||
)
|
||||
continue
|
||||
if name.endswith("experts.proj"):
|
||||
param_name = name.replace("experts.proj", "experts.w2_weight")
|
||||
if param_name not in params_dict:
|
||||
raise ValueError(f"Unexpected IQuest Q1 weight: {name}")
|
||||
param = params_dict[param_name]
|
||||
loaded.add(param_name)
|
||||
for expert_id in range(model.config.num_experts):
|
||||
param.weight_loader(
|
||||
param,
|
||||
loaded_weight[expert_id],
|
||||
param_name,
|
||||
shard_id="w2",
|
||||
expert_id=expert_id,
|
||||
)
|
||||
continue
|
||||
if "router.weight" in name:
|
||||
name = name.replace("router.weight", "gate.weight")
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
mapped_name = name.replace(weight_name, param_name)
|
||||
if mapped_name not in params_dict:
|
||||
continue
|
||||
param = params_dict[mapped_name]
|
||||
param.weight_loader(param, loaded_weight, shard_id)
|
||||
if mapped_name not in loaded or mapped_name in loaded_shards:
|
||||
loaded_shards.setdefault(mapped_name, set()).add(shard_id)
|
||||
loaded.add(mapped_name)
|
||||
break
|
||||
else:
|
||||
if name not in params_dict:
|
||||
raise ValueError(f"Unexpected IQuest Q1 weight: {name}")
|
||||
param = params_dict[name]
|
||||
getattr(param, "weight_loader", default_weight_loader)(param, loaded_weight)
|
||||
loaded.add(name)
|
||||
loaded_shards.pop(name, None)
|
||||
check_all_params_loaded(
|
||||
params_dict,
|
||||
loaded,
|
||||
type(model).__name__,
|
||||
loaded_shards=loaded_shards,
|
||||
quant_config=model.quant_config,
|
||||
)
|
||||
|
||||
|
||||
class IQuestQ1ForCausalLM(nn.Module):
|
||||
fall_back_to_pt_during_load = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.model = IQuestQ1Model(
|
||||
config, quant_config=quant_config, prefix=add_prefix("model", prefix)
|
||||
)
|
||||
self.lm_head = (
|
||||
self.model.embed_tokens
|
||||
if config.tie_word_embeddings
|
||||
else ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("lm_head", prefix),
|
||||
)
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(config, logit_scale=config.logit_scale)
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.model.embed_tokens
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_embeds: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.model(input_ids, positions, forward_batch, input_embeds)
|
||||
return self.logits_processor(
|
||||
input_ids, hidden_states, self.lm_head, forward_batch
|
||||
)
|
||||
|
||||
def get_embed_and_head(self):
|
||||
return self.model.embed_tokens.weight, self.lm_head.weight
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
||||
load_iquest_q1_weights(
|
||||
self,
|
||||
(
|
||||
(name, weight)
|
||||
for name, weight in weights
|
||||
if not name.startswith("mtp_layers.")
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
EntryClass = [IQuestQ1ForCausalLM]
|
||||
@@ -0,0 +1,204 @@
|
||||
from typing import Any, Iterable, Optional, Tuple
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.configs.iquest_q1 import IQuestQ1MTPConfig
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.models.iquest_q1 import (
|
||||
IQuestQ1Attention,
|
||||
IQuestQ1MoEBlock,
|
||||
IQuestQ1RMSNorm,
|
||||
load_iquest_q1_weights,
|
||||
)
|
||||
from sglang.srt.utils import add_prefix
|
||||
|
||||
|
||||
class IQuestQ1MTPInnerLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: IQuestQ1MTPConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.self_attn = IQuestQ1Attention(
|
||||
config=config,
|
||||
layer_id=0,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("self_attn", prefix),
|
||||
)
|
||||
self.mlp = IQuestQ1MoEBlock(
|
||||
config=config,
|
||||
layer_id=0,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
)
|
||||
self.attention_norm = IQuestQ1RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attn_out_norm = IQuestQ1RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.feed_forward_norm = IQuestQ1RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.ffn_out_norm = IQuestQ1RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.attn_out_scale = config.first_layer_attn_out_scale
|
||||
self.ffn_out_scale = config.first_layer_ffn_out_scale
|
||||
self.fp32_residual_connection = config.fp32_residual_connection
|
||||
|
||||
def forward(self, positions, hidden_states, forward_batch):
|
||||
dtype = hidden_states.dtype
|
||||
attention = self.self_attn(
|
||||
positions, self.attention_norm(hidden_states), forward_batch
|
||||
)
|
||||
residual = (
|
||||
hidden_states.float() if self.fp32_residual_connection else hidden_states
|
||||
)
|
||||
hidden_states = residual + self.attn_out_norm(attention) * self.attn_out_scale
|
||||
output = self.mlp(self.feed_forward_norm(hidden_states).to(dtype))
|
||||
return hidden_states + self.ffn_out_norm(output) * self.ffn_out_scale
|
||||
|
||||
|
||||
class IQuestQ1MTPLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: IQuestQ1MTPConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.enorm = IQuestQ1RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = IQuestQ1RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
|
||||
self.mtp_model_layer = IQuestQ1MTPInnerLayer(
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mtp_model_layer", prefix),
|
||||
)
|
||||
self.final_layernorm = IQuestQ1RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
def forward(self, positions, previous_hidden_states, input_embeds, forward_batch):
|
||||
dtype = self.eh_proj.weight.dtype
|
||||
hidden_states = self.eh_proj(
|
||||
torch.cat(
|
||||
[
|
||||
self.enorm(input_embeds).to(dtype),
|
||||
self.hnorm(previous_hidden_states).to(dtype),
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
)
|
||||
hidden_states = self.mtp_model_layer(positions, hidden_states, forward_batch)
|
||||
return self.final_layernorm(hidden_states).to(dtype)
|
||||
|
||||
|
||||
class IQuestQ1MTPModel(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: IQuestQ1MTPConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=add_prefix("embed_tokens", prefix),
|
||||
)
|
||||
self.mtp_layer = IQuestQ1MTPLayer(
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mtp_layer", prefix),
|
||||
)
|
||||
|
||||
@property
|
||||
def layers(self):
|
||||
return (self.mtp_layer.mtp_model_layer,)
|
||||
|
||||
def forward(self, input_ids, positions, forward_batch, input_embeds=None):
|
||||
if input_embeds is None:
|
||||
input_embeds = self.embed_tokens(input_ids)
|
||||
return self.mtp_layer(
|
||||
positions,
|
||||
forward_batch.spec_info.hidden_states,
|
||||
input_embeds,
|
||||
forward_batch,
|
||||
)
|
||||
|
||||
|
||||
class IQuestQ1MTP(nn.Module):
|
||||
fall_back_to_pt_during_load = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: IQuestQ1MTPConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.model = IQuestQ1MTPModel(
|
||||
config=config,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("model", prefix),
|
||||
)
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("lm_head", prefix),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(config)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_tokens(input_ids)
|
||||
|
||||
def get_embed_and_head(self):
|
||||
return self.model.embed_tokens.weight, self.lm_head.weight
|
||||
|
||||
def set_embed_and_head(self, embed, head):
|
||||
return
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_embeds: Optional[torch.Tensor] = None,
|
||||
**kwargs: Any,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.model(input_ids, positions, forward_batch, input_embeds)
|
||||
return self.logits_processor(
|
||||
input_ids, hidden_states, self.lm_head, forward_batch
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
||||
def mapped_weights():
|
||||
for name, weight in weights:
|
||||
if name == "target_final_norm.weight":
|
||||
continue
|
||||
if name == "embed_tokens.weight":
|
||||
yield "model.embed_tokens.weight", weight
|
||||
elif name.startswith("mtp."):
|
||||
yield "model.mtp_layer." + name.removeprefix("mtp."), weight
|
||||
elif name == "lm_head.weight":
|
||||
yield name, weight
|
||||
else:
|
||||
raise ValueError(f"Unexpected MTP weight: {name}")
|
||||
|
||||
load_iquest_q1_weights(self, mapped_weights())
|
||||
|
||||
|
||||
EntryClass = [IQuestQ1MTP]
|
||||
@@ -396,6 +396,38 @@ class DeepSeekR1Detector(BaseReasoningFormatDetector):
|
||||
# https://github.com/sgl-project/sglang/pull/3202#discussion_r1950153599
|
||||
|
||||
|
||||
class IQuestQ1ReasoningDetector(BaseReasoningFormatDetector):
|
||||
@staticmethod
|
||||
def thinking_enabled(chat_kwargs):
|
||||
thinking = chat_kwargs.get("thinking")
|
||||
enable_thinking = chat_kwargs.get("enable_thinking")
|
||||
return (
|
||||
True
|
||||
if thinking is None and enable_thinking is None
|
||||
else bool(thinking or enable_thinking)
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
stream_reasoning=True,
|
||||
force_reasoning=True,
|
||||
continue_final_message=False,
|
||||
previous_content="",
|
||||
force_nonempty_content=False,
|
||||
):
|
||||
super().__init__(
|
||||
"<think>",
|
||||
"</think>",
|
||||
force_reasoning=force_reasoning,
|
||||
stream_reasoning=stream_reasoning,
|
||||
continue_final_message=continue_final_message,
|
||||
previous_content=previous_content,
|
||||
force_nonempty_content=force_nonempty_content,
|
||||
thinks_internally=True,
|
||||
reasoning_default="enable_thinking",
|
||||
)
|
||||
|
||||
|
||||
class Qwen3Detector(BaseReasoningFormatDetector):
|
||||
"""
|
||||
Detector for Qwen3 models (e.g., Qwen/Qwen3-235B-A22B).
|
||||
@@ -2255,6 +2287,7 @@ class ReasoningParser:
|
||||
"gemma4": Gemma4Detector,
|
||||
"gigachat35": DeepSeekR1Detector,
|
||||
"inkling": InklingDetector,
|
||||
"iquest_q1": IQuestQ1ReasoningDetector,
|
||||
"cohere_command4": CohereCommand4Detector,
|
||||
}
|
||||
|
||||
@@ -2276,6 +2309,13 @@ class ReasoningParser:
|
||||
|
||||
chat_template_kwargs = getattr(request, "chat_template_kwargs", None) or {}
|
||||
|
||||
if model_type.lower() == "iquest_q1" and (
|
||||
request is not None or force_reasoning is None
|
||||
):
|
||||
force_reasoning = IQuestQ1ReasoningDetector.thinking_enabled(
|
||||
chat_template_kwargs
|
||||
)
|
||||
|
||||
# Special cases where we override force_reasoning
|
||||
if model_type.lower() in {
|
||||
"qwen3-thinking",
|
||||
|
||||
@@ -35,6 +35,7 @@ REASONING_PARSER_NAMES = [
|
||||
"interns1",
|
||||
"gemma4",
|
||||
"inkling",
|
||||
"iquest_q1",
|
||||
"cohere_command4",
|
||||
"gigachat35",
|
||||
]
|
||||
|
||||
@@ -526,11 +526,16 @@ def _is_gigachat35(ctx):
|
||||
return ctx.has_text("<|GCML|tool_calls>")
|
||||
|
||||
|
||||
def _is_iquest_q1(ctx):
|
||||
return ctx.has_text("<|iquest_assistant|>")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reasoning parser rules
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
REASONING_PARSER_RULES = (
|
||||
DetectionRule(name="iquest_q1", value="iquest_q1", predicate=_is_iquest_q1),
|
||||
DetectionRule(name="k2_horizon", value="k2_horizon", predicate=_is_k2_v3),
|
||||
DetectionRule(name="apertus2509", value="apertus2509", predicate=_is_apertus2509),
|
||||
DetectionRule(name="gemma4", value="gemma4", predicate=_is_gemma4),
|
||||
@@ -576,6 +581,7 @@ REASONING_PARSER_RULES = (
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TOOL_CALL_PARSER_RULES = (
|
||||
DetectionRule(name="iquest_q1", value="iquest_q1", predicate=_is_iquest_q1),
|
||||
DetectionRule(name="gigachat35", value="gigachat35", predicate=_is_gigachat35),
|
||||
DetectionRule(name="k2_horizon", value="k2_horizon", predicate=_is_k2_v3),
|
||||
DetectionRule(name="apertus2509", value="apertus2509", predicate=_is_apertus2509),
|
||||
@@ -795,6 +801,8 @@ def _architecture_auto_parsers(server_args, needs: Tuple[str, ...]) -> Dict[str,
|
||||
|
||||
if "KimiK3" in arch or model_type == "kimi_k3":
|
||||
reasoning_parser, tool_call_parser = "kimi_k3", "kimi_k3"
|
||||
elif arch == "IQuestQ1ForCausalLM":
|
||||
reasoning_parser, tool_call_parser = "iquest_q1", "iquest_q1"
|
||||
elif arch in (
|
||||
"BailingMoeV3ForCausalLM",
|
||||
"BailingMoeV3VLForConditionalGeneration",
|
||||
|
||||
@@ -52,6 +52,8 @@ from sglang.srt.configs import (
|
||||
InternS2MobiusConfig,
|
||||
InternS2MobiusTextConfig,
|
||||
InternS2PreviewConfig,
|
||||
IQuestQ1Config,
|
||||
IQuestQ1MTPConfig,
|
||||
JetNemotronConfig,
|
||||
JetVLMConfig,
|
||||
K2HorizonConfig,
|
||||
@@ -162,6 +164,8 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
|
||||
Qwen3_5TextConfig,
|
||||
Qwen3_5MoeTextConfig,
|
||||
InternS2PreviewConfig,
|
||||
IQuestQ1Config,
|
||||
IQuestQ1MTPConfig,
|
||||
InternS2MobiusConfig,
|
||||
InternS2MobiusTextConfig,
|
||||
JetNemotronConfig,
|
||||
|
||||
Reference in New Issue
Block a user