From e9dbbd19e925008f2e8c975f03ffa72cf4dfedea Mon Sep 17 00:00:00 2001 From: EduardDurech <39579228+EduardDurech@users.noreply.github.com> Date: Sat, 6 Jun 2026 09:04:31 +0200 Subject: [PATCH] =?UTF-8?q?[model]=C2=A0Apertus=20Tool/Function=20and=20Re?= =?UTF-8?q?asoning=20parser=20(#25100)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../separate_reasoning.ipynb | 4 + .../advanced_features/separate_reasoning.mdx | 11 +- .../docs/advanced_features/tool_parser.ipynb | 1 + .../docs/advanced_features/tool_parser.mdx | 7 +- python/sglang/srt/entrypoints/http_server.py | 13 +- .../srt/entrypoints/openai/serving_chat.py | 14 +- .../srt/function_call/apertus2509_detector.py | 267 ++++++++++++++++++ .../srt/function_call/function_call_parser.py | 2 + python/sglang/srt/managers/io_struct.py | 1 + .../sglang/srt/managers/template_detection.py | 6 + python/sglang/srt/parser/reasoning_parser.py | 229 +++++++++++++++ .../unit/managers/test_template_manager.py | 7 + 12 files changed, 555 insertions(+), 7 deletions(-) create mode 100644 python/sglang/srt/function_call/apertus2509_detector.py diff --git a/docs_new/docs/advanced_features/separate_reasoning.ipynb b/docs_new/docs/advanced_features/separate_reasoning.ipynb index 6277dd8bd..e6d8873e2 100644 --- a/docs_new/docs/advanced_features/separate_reasoning.ipynb +++ b/docs_new/docs/advanced_features/separate_reasoning.ipynb @@ -12,6 +12,7 @@ "\n", "| Model | Reasoning tags | Parser | Notes |\n", "|---------|-----------------------------|------------------|-------|\n", + "| [Apertus 2509 models](https://huggingface.co/swiss-ai/Apertus-8B-Instruct-2509) | `<\\|inner_prefix\\|>` … `<\\|inner_suffix\\|>` | `apertus2509` | For tool use, also set `--tool-call-parser apertus2509`. |\n", "| [DeepSeek‑R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d) | `` … `` | `deepseek-r1` | Supports all variants (R1, R1-0528, R1-Distill) |\n", "| [DeepSeek‑V3 series](https://huggingface.co/deepseek-ai/DeepSeek-V3.1) | `` … `` | `deepseek-v3` | Including [DeepSeek‑V3.2](https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp). Supports `thinking` parameter |\n", "| [Standard Qwen3 models](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) | `` … `` | `qwen3` | Supports `enable_thinking` parameter |\n", @@ -20,6 +21,9 @@ "| [GPT OSS](https://huggingface.co/openai/gpt-oss-120b) | `<\\|channel\\|>analysis<\\|message\\|>` … `<\\|end\\|>` | `gpt-oss` | N/A |\n", "### Model-Specific Behaviors\n", "\n", + "**Apertus 2509:**\n", + "- Uses `<|inner_prefix|>` and `<|inner_suffix|>` to delimit reasoning content. For agentic tool use, also specify `--tool-call-parser apertus2509`.\n", + "\n", "**DeepSeek-R1 Family:**\n", "- DeepSeek-R1: No `` start tag, jumps directly to thinking content\n", "- DeepSeek-R1-0528: Generates both `` start and `` end tags\n", diff --git a/docs_new/docs/advanced_features/separate_reasoning.mdx b/docs_new/docs/advanced_features/separate_reasoning.mdx index e0bea35ee..86412b8a2 100644 --- a/docs_new/docs/advanced_features/separate_reasoning.mdx +++ b/docs_new/docs/advanced_features/separate_reasoning.mdx @@ -1,7 +1,7 @@ --- title: "Reasoning Parser" metatags: - description: "SGLang reasoning parser: separate thinking content from output for DeepSeek R1, Qwen3, Kimi K2, GPT-OSS reasoning models." + description: "SGLang reasoning parser: separate thinking content from output for DeepSeek R1, Qwen3, Kimi K2, GPT-OSS, and Apertus reasoning models." --- SGLang supports parsing reasoning content out from "normal" content for reasoning models such as [DeepSeek R1](https://huggingface.co/deepseek-ai/DeepSeek-R1). @@ -23,6 +23,12 @@ SGLang supports parsing reasoning content out from "normal" content for reasonin + + [Apertus 2509 models](https://huggingface.co/swiss-ai/Apertus-8B-Instruct-2509) + `<|inner_prefix|>` … `<|inner_suffix|>` + `apertus2509` + Supports `enable_thinking` parameter + [DeepSeek‑R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d) `` … `` @@ -63,6 +69,9 @@ SGLang supports parsing reasoning content out from "normal" content for reasonin ### Model-Specific Behaviors +**Apertus 2509:** +- Uses `<|inner_prefix|>` and `<|inner_suffix|>` to delimit reasoning content. For agentic tool use, also specify `--tool-call-parser apertus2509`. + **DeepSeek-R1 Family:** - DeepSeek-R1: No `` start tag, jumps directly to thinking content - DeepSeek-R1-0528: Generates both `` start and `` end tags diff --git a/docs_new/docs/advanced_features/tool_parser.ipynb b/docs_new/docs/advanced_features/tool_parser.ipynb index 9afc9663e..60ffa2443 100644 --- a/docs_new/docs/advanced_features/tool_parser.ipynb +++ b/docs_new/docs/advanced_features/tool_parser.ipynb @@ -17,6 +17,7 @@ "\n", "| Parser | Supported Models | Notes |\n", "|---|---|---|\n", + "| `apertus2509` | Apertus 2509 (e.g., `swiss-ai/Apertus-{8,70}B-Instruct-2509`) | Tool calls are emitted as a JSON list of single-key objects: `<\\|tools_prefix\\|>[{\"tool\": {...}}]<\\|tools_suffix\\|>`. |\n", "| `deepseekv3` | DeepSeek-v3 (e.g., `deepseek-ai/DeepSeek-V3-0324`) | Recommend adding `--chat-template ./examples/chat_template/tool_chat_template_deepseekv3.jinja` to launch command. |\n", "| `deepseekv31` | DeepSeek-V3.1 and DeepSeek-V3.2-Exp (e.g. `deepseek-ai/DeepSeek-V3.1`, `deepseek-ai/DeepSeek-V3.2-Exp`) | Recommend adding `--chat-template ./examples/chat_template/tool_chat_template_deepseekv31.jinja` (Or ..deepseekv32.jinja for DeepSeek-V3.2) to launch command. |\n", "| `deepseekv32` | DeepSeek-V3.2 (`deepseek-ai/DeepSeek-V3.2`) | |\n", diff --git a/docs_new/docs/advanced_features/tool_parser.mdx b/docs_new/docs/advanced_features/tool_parser.mdx index a5fcc169b..27b935b3a 100644 --- a/docs_new/docs/advanced_features/tool_parser.mdx +++ b/docs_new/docs/advanced_features/tool_parser.mdx @@ -1,7 +1,7 @@ --- title: "Tool Parser" metatags: - description: "SGLang function calling: tool parsers for DeepSeek, Llama, Qwen, Mistral, GLM, Kimi K2. OpenAI-compatible tool use API." + description: "SGLang function calling: tool parsers for DeepSeek, Llama, Qwen, Mistral, GLM, Kimi K2, and Apertus. OpenAI-compatible tool use API." --- This guide demonstrates how to use SGLang’s [Function calling](https://platform.openai.com/docs/guides/function-calling) functionality. @@ -22,6 +22,11 @@ This guide demonstrates how to use SGLang’s [Function calling](https://platfor + + `apertus2509` + Apertus 2509 (e.g., `swiss-ai/Apertus-{8,70}B-Instruct-2509`) + Tool calls are emitted as a JSON list of single-key objects: `<|tools_prefix|>[{"tool": {...}}]<|tools_suffix|>`. + `deepseekv3` DeepSeek-v3 (e.g., `deepseek-ai/DeepSeek-V3-0324`) diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index fdacd0ad5..d9d649776 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -1463,13 +1463,24 @@ async def separate_reasoning_request(obj: SeparateReasoningReqInput, request: Re parser = ReasoningParser(model_type=obj.reasoning_parser, request=request) # 2) Call the non-stream parsing method (non-stream) - reasoning_text, normal_text = parser.parse_non_stream(obj.text) + if getattr(obj, "return_blocks", False): + blocks = parser.parse_non_stream_blocks(obj.text) + reasoning_blocks = [b["text"] for b in blocks if b["type"] == "reasoning"] + text_blocks = [b["text"] for b in blocks if b["type"] == "text"] + reasoning_text = "".join(reasoning_blocks) + normal_text = "".join(text_blocks) + else: + reasoning_text, normal_text = parser.parse_non_stream(obj.text) # 3) Organize the response content response_data = { "reasoning_text": reasoning_text, "text": normal_text, } + if getattr(obj, "return_blocks", False): + response_data["reasoning_blocks"] = reasoning_blocks + response_data["text_blocks"] = text_blocks + response_data["blocks"] = blocks return ORJSONResponse(content=response_data, status_code=200) diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index a24b48f54..2ea4c4909 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -564,7 +564,7 @@ class OpenAIServingChat(OpenAIServingBase): if self.is_gpt_oss or self.is_gemma4: request.skip_special_tokens = False - self._patch_mistral_skip_special_tokens(request) + self._patch_reasoning_skip_special_tokens(request) thinking_mode = self._get_reasoning_from_request(request) # SGLang's ReasonerGrammarBackend owns the reasoning prefix @@ -1546,11 +1546,17 @@ class OpenAIServingChat(OpenAIServingBase): idx += len(list(tool_calls)) if tool_calls is not None else 0 # noqa return idx - def _patch_mistral_skip_special_tokens( + def _patch_reasoning_skip_special_tokens( self, request: ChatCompletionRequest ) -> None: - """Mistral uses special tokens ([THINK]/[/THINK]) for reasoning markers, - which get stripped when skip_special_tokens=True.""" + """Keep parser-specific reasoning markers in the decoded text. + + Some reasoning parsers rely on special-token delimiters that would be + removed during detokenization when ``skip_special_tokens=True``. + """ + if self.reasoning_parser == "apertus2509": + request.skip_special_tokens = False + if ( self.reasoning_parser in ["mistral"] and request.reasoning_effort is not None diff --git a/python/sglang/srt/function_call/apertus2509_detector.py b/python/sglang/srt/function_call/apertus2509_detector.py new file mode 100644 index 000000000..4c791af2b --- /dev/null +++ b/python/sglang/srt/function_call/apertus2509_detector.py @@ -0,0 +1,267 @@ +import json +import logging +from typing import Any, List, Optional, Tuple + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.environ import envs +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + StructureInfo, + ToolCallItem, + _GetInfoFunc, +) + +logger = logging.getLogger(__name__) + + +class Apertus2509Detector(BaseFormatDetector): + """ + Detector for Apertus 2509 tool/function call format + ``` + <|tools_prefix|>[{"tool1": {...}}, {"tool2": {...}}]<|tools_suffix|> + ``` + + Notes: + - Each list element is a single-key object: {"": } + - The list can contain multiple tool calls separated by ", " + - This is distinct from the OpenAI-style {"name": "...", "arguments": {...}} objects + """ + + def __init__(self): + super().__init__() + self.bot = "<|tools_prefix|>[" + self.suffix = "<|tools_suffix|>" + self._in_tools_block: bool = False + + def has_tool_call(self, text: str) -> bool: + return self.bot in text + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + """ + One-time parsing: Extract all Apertus tools blocks and parse their JSON payloads. + """ + if not self.has_tool_call(text): + return StreamingParseResult(normal_text=text, calls=[]) + + calls: List[ToolCallItem] = [] + normal_parts: List[str] = [] + cursor = 0 + + while True: + if (start := text.find(self.bot, cursor)) == -1: + normal_parts.append(text[cursor:]) + break + + normal_parts.append(text[cursor:start]) + tool_part = text[start:] + parsed_arr, json_end = self._try_parse_json_array(tool_part) + if parsed_arr is None: + normal_parts.append(tool_part) + break + + if (suffix_pos := tool_part.find(self.suffix, json_end)) == -1: + normal_parts.append(tool_part) + break + + calls.extend( + self._parse_apertus_call_list( + parsed_arr, tools, tool_index_offset=len(calls) + ) + ) + + cursor = start + suffix_pos + len(self.suffix) + + return StreamingParseResult( + normal_text="".join(normal_parts).strip(), calls=calls + ) + + def parse_streaming_increment( + self, new_text: str, tools: List[Tool] + ) -> StreamingParseResult: + """ + Streaming incremental parsing for Apertus tool calls. + + - Streams any normal text before `<|tools_prefix|>[` immediately. + - Buffers tool calls until we have a complete tools block, then emits: + - Tool name (empty args), then + - Full JSON arguments string + """ + self._buffer += new_text + out_normal = "" + out_calls: List[ToolCallItem] = [] + + if not hasattr(self, "_tool_indices"): + self._tool_indices = self._get_tool_indices(tools) + + while True: + if not self._in_tools_block: + if (pos := self._buffer.find(self.bot)) > 0: + out_normal += self._buffer[:pos] + self._buffer = self._buffer[pos:] + elif pos == -1: + if partial_bot := self._ends_with_partial_token( + self._buffer, self.bot + ): + out_normal += self._buffer[:-partial_bot] + self._buffer = self._buffer[-partial_bot:] + else: + out_normal += self._buffer + self._buffer = "" + return StreamingParseResult(normal_text=out_normal, calls=out_calls) + + self._in_tools_block = True + + if not self._buffer.startswith(self.bot): + if (marker_pos := self._buffer.find(self.bot)) == -1: + out_normal += self._buffer + self._buffer = "" + self._in_tools_block = False + return StreamingParseResult(normal_text=out_normal, calls=out_calls) + out_normal += self._buffer[:marker_pos] + self._buffer = self._buffer[marker_pos:] + continue + + parsed_arr, suffix_pos = self._try_parse_json_array(self._buffer) + if parsed_arr is None: + if self.suffix in self._buffer: + out_normal += self._buffer + self._buffer = "" + self._in_tools_block = False + return StreamingParseResult(normal_text=out_normal, calls=out_calls) + return StreamingParseResult(normal_text=out_normal, calls=out_calls) + + while suffix_pos < len(self._buffer) and self._buffer[suffix_pos].isspace(): + suffix_pos += 1 + + if not self._buffer.startswith(self.suffix, suffix_pos): + return StreamingParseResult(normal_text=out_normal, calls=out_calls) + + if self.current_tool_id == -1: + self.current_tool_id = 0 + + for item in parsed_arr: + name, args = self._apertus_obj_to_call(item) + if name is None: + continue + if args is None: + args = {} + + if ( + name not in self._tool_indices + and not envs.SGLANG_FORWARD_UNKNOWN_TOOLS.get() + ): + logger.warning( + f"Model attempted to call undefined function: {name}" + ) + continue + + tool_id = self.current_tool_id + self.current_tool_id += 1 + + args_json = json.dumps(args, ensure_ascii=False) + + while len(self.prev_tool_call_arr) <= tool_id: + self.prev_tool_call_arr.append({}) + while len(self.streamed_args_for_tool) <= tool_id: + self.streamed_args_for_tool.append("") + + self.prev_tool_call_arr[tool_id] = {"name": name, "arguments": args} + self.streamed_args_for_tool[tool_id] = args_json + + # Emit tool name first, then full args (OpenAI streaming semantics) + out_calls.append( + ToolCallItem(tool_index=tool_id, name=name, parameters="") + ) + out_calls.append( + ToolCallItem(tool_index=tool_id, name=None, parameters=args_json) + ) + + # Consume the parsed tools block and reset state + self._buffer = self._buffer[suffix_pos + len(self.suffix) :] + self._in_tools_block = False + + if out_calls: + # Flush normal text after the tools block, but keep a tool marker or its partial prefix in the buffer for the next stream + if (marker_pos := self._buffer.find(self.bot)) > 0: + out_normal += self._buffer[:marker_pos] + self._buffer = self._buffer[marker_pos:] + elif marker_pos == -1: + if partial_bot := self._ends_with_partial_token( + self._buffer, self.bot + ): + out_normal += self._buffer[:-partial_bot] + self._buffer = self._buffer[-partial_bot:] + else: + out_normal += self._buffer + self._buffer = "" + return StreamingParseResult(normal_text=out_normal, calls=out_calls) + + continue + + def structure_info(self) -> _GetInfoFunc: + return lambda name: StructureInfo( + begin='<|tools_prefix|>[{"' + name + '": ', + end="}]<|tools_suffix|>", + trigger="<|tools_prefix|>", + ) + + def _apertus_obj_to_call(self, obj: Any) -> Tuple[Optional[str], Optional[Any]]: + """ + Convert a single Apertus tool-call object to (name, arguments). + + Expected shape: {"tool_name": {...}}. + """ + if not isinstance(obj, dict) or not obj: + return None, None + name = next(iter(obj.keys())) + return name, obj.get(name) + + def _parse_apertus_call_list( + self, arr: Any, tools: List[Tool], tool_index_offset: int = 0 + ) -> List[ToolCallItem]: + + if not hasattr(self, "_tool_indices"): + self._tool_indices = self._get_tool_indices(tools) + + calls: List[ToolCallItem] = [] + for item in arr: + name, args = self._apertus_obj_to_call(item) + if name is None: + continue + if args is None: + args = {} + + if ( + name not in self._tool_indices + and not envs.SGLANG_FORWARD_UNKNOWN_TOOLS.get() + ): + logger.warning(f"Model attempted to call undefined function: {name}") + continue + + calls.append( + ToolCallItem( + tool_index=tool_index_offset + len(calls), + name=name, + parameters=json.dumps(args, ensure_ascii=False), + ) + ) + + return calls + + def _try_parse_json_array(self, text: str) -> Tuple[Optional[Any], int]: + """ + Returns: (parsed_array_or_None, end_index_exclusive_in_text) + """ + if (start_idx := text.find(self.bot)) == -1: + return None, 0 + + json_start = start_idx + len(self.bot) - 1 # points to '[' + try: + parsed, end_idx = json.JSONDecoder().raw_decode(text, json_start) + except json.JSONDecodeError: + return None, 0 + + if isinstance(parsed, list): + return parsed, end_idx + return [parsed], end_idx diff --git a/python/sglang/srt/function_call/function_call_parser.py b/python/sglang/srt/function_call/function_call_parser.py index ab2e1c4a1..834aad9d5 100644 --- a/python/sglang/srt/function_call/function_call_parser.py +++ b/python/sglang/srt/function_call/function_call_parser.py @@ -10,6 +10,7 @@ from sglang.srt.entrypoints.openai.protocol import ( ToolChoice, ) from sglang.srt.environ import ToolStrictLevel, envs +from sglang.srt.function_call.apertus2509_detector import Apertus2509Detector from sglang.srt.function_call.base_format_detector import BaseFormatDetector from sglang.srt.function_call.cohere_command4_detector import CohereCommand4Detector from sglang.srt.function_call.core_types import ToolCallItem @@ -56,6 +57,7 @@ class FunctionCallParser: """ ToolCallParserEnum: Dict[str, Type[BaseFormatDetector]] = { + "apertus2509": Apertus2509Detector, "cohere_command4": CohereCommand4Detector, "deepseekv3": DeepSeekV3Detector, "deepseekv31": DeepSeekV31Detector, diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index 782955944..bd6daa997 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -1866,6 +1866,7 @@ class ParseFunctionCallReq(BaseReq): class SeparateReasoningReqInput(BaseReq): text: str # The text to parse. reasoning_parser: str # Specify the parser type, e.g., "deepseek-r1". + return_blocks: bool = False # If True, also return segmented reasoning blocks. @dataclass diff --git a/python/sglang/srt/managers/template_detection.py b/python/sglang/srt/managers/template_detection.py index 190cec67d..51b408a1f 100644 --- a/python/sglang/srt/managers/template_detection.py +++ b/python/sglang/srt/managers/template_detection.py @@ -150,6 +150,10 @@ REASONING_MODE_RULES = ( # --------------------------------------------------------------------------- +def _is_apertus2509(ctx): + return ctx.has_vocab("<|inner_prefix|>") + + def _is_gemma4(ctx): return ctx.has_text("<|channel>") @@ -250,6 +254,7 @@ def _is_deepseek_r1_think_tags(ctx): # --------------------------------------------------------------------------- REASONING_PARSER_RULES = ( + DetectionRule(name="apertus2509", value="apertus2509", predicate=_is_apertus2509), DetectionRule(name="gemma4", value="gemma4", predicate=_is_gemma4), DetectionRule(name="kimi", value="kimi", predicate=_is_kimi), DetectionRule(name="interns1", value="interns1", predicate=_is_interns1), @@ -277,6 +282,7 @@ REASONING_PARSER_RULES = ( # --------------------------------------------------------------------------- TOOL_CALL_PARSER_RULES = ( + DetectionRule(name="apertus2509", value="apertus2509", predicate=_is_apertus2509), DetectionRule(name="gemma4", value="gemma4", predicate=_is_gemma4), DetectionRule(name="gpt_oss", value="gpt-oss", predicate=_is_gpt_oss), DetectionRule(name="kimi_k2", value="kimi_k2", predicate=_is_kimi_k2), diff --git a/python/sglang/srt/parser/reasoning_parser.py b/python/sglang/srt/parser/reasoning_parser.py index 035cb7a75..b36a8fa8f 100644 --- a/python/sglang/srt/parser/reasoning_parser.py +++ b/python/sglang/srt/parser/reasoning_parser.py @@ -597,6 +597,221 @@ class _PoolsideV1Detector(Qwen3Detector): self.reasoning_default = "explicit_enable_thinking" +class Apertus2509Detector(BaseReasoningFormatDetector): + """ + Detector for Apertus 2509 models + + Reasoning blocks are delimited by: + <|inner_prefix|> ... <|inner_suffix|> + """ + + def __init__( + self, + stream_reasoning: bool = True, + force_reasoning: bool = False, + continue_final_message: bool = False, + previous_content: str = "", + force_nonempty_content: bool = False, + ): + super().__init__( + "<|inner_prefix|>", + "<|inner_suffix|>", + force_reasoning=False, + stream_reasoning=stream_reasoning, + continue_final_message=continue_final_message, + previous_content=previous_content, + ) + self._force_reasoning = force_reasoning + self._force_nonempty_content = force_nonempty_content + self._tool_start_token = "<|tools_prefix|>[" + self._tool_end_token = "<|tools_suffix|>" + self._reasoning_acc: str = "" + self._in_inner_tool: bool = False + + @staticmethod + def _ends_with_partial_token(buffer: str, token: str) -> int: + for i in range(1, min(len(buffer) + 1, len(token))): + if token.startswith(buffer[-i:]): + return i + return 0 + + def detect_and_parse(self, text: str) -> StreamingParseResult: + blocks = self.detect_and_parse_block_sequence(text) + reasoning_parts = [t for k, t in blocks if k == "reasoning"] + text_parts = [t for k, t in blocks if k == "text"] + ret = StreamingParseResult( + normal_text="".join(text_parts), + reasoning_text="".join(reasoning_parts), + ) + if self._force_nonempty_content and not ret.normal_text: + ret.normal_text, ret.reasoning_text = ret.reasoning_text, ret.normal_text + return ret + + def detect_and_parse_block_sequence(self, text: str) -> list[tuple[str, str]]: + """Return an ordered sequence of blocks: [("reasoning"|"text", content), ...]""" + start_tok = self.think_start_token + end_tok = self.think_end_token + blocks: list[tuple[str, str]] = [] + cursor = 0 + + # continue_final_message can resume inside an existing inner + if self._in_reasoning: + if (e := text.find(end_tok, cursor)) == -1: + blocks.extend(self._split_inner_reasoning(text[cursor:])) + blocks.append(("text", "")) + return blocks + blocks.extend(self._split_inner_reasoning(text[cursor:e])) + cursor = e + len(end_tok) + + while True: + if (s := text.find(start_tok, cursor)) == -1: + # Always include the trailing text block (may be empty) + blocks.append(("text", text[cursor:])) + break + if s > cursor: + blocks.append(("text", text[cursor:s])) + + cursor = s + len(start_tok) + if (e := text.find(end_tok, cursor)) == -1: + blocks.extend(self._split_inner_reasoning(text[cursor:])) + blocks.append(("text", "")) + break + blocks.extend(self._split_inner_reasoning(text[cursor:e])) + cursor = e + len(end_tok) + + last_idx = len(blocks) - 1 + blocks = [ + (k, t) + for i, (k, t) in enumerate(blocks) + if not (k == "text" and t == "" and i != last_idx) + ] + + return blocks + + def _split_inner_reasoning(self, inner_text: str) -> list[tuple[str, str]]: + """ + Split content inside <|inner_prefix|>...<|inner_suffix|> into: + - ("reasoning", ) + - ("text", <|tools_prefix|>[...]<|tools_suffix|>) for any tool calls inside reasoning + """ + tool_start = self._tool_start_token + tool_end = self._tool_end_token + out: list[tuple[str, str]] = [] + cursor = 0 + + while True: + if (s := inner_text.find(tool_start, cursor)) == -1: + if (tail := inner_text[cursor:]) != "": + out.append(("reasoning", tail)) + break + if s > cursor: + out.append(("reasoning", inner_text[cursor:s])) + + if (e := inner_text.find(tool_end, s)) == -1: + out.append(("text", inner_text[s:])) + break + + out.append(("text", inner_text[s : e + len(tool_end)])) + cursor = e + len(tool_end) + + return out + + def parse_streaming_increment(self, new_text: str) -> StreamingParseResult: + self._buffer += new_text + + out_reasoning = "" + out_normal = "" + + start_tok = self.think_start_token + end_tok = self.think_end_token + tool_start = self._tool_start_token + tool_end = self._tool_end_token + + while True: + if not self._in_reasoning: + if (s := self._buffer.find(start_tok)) == -1: + if partial := self._ends_with_partial_token( + self._buffer, start_tok + ): + out_normal += self._buffer[:-partial] + self._buffer = self._buffer[-partial:] + else: + out_normal += self._buffer + self._buffer = "" + return StreamingParseResult( + normal_text=out_normal, reasoning_text=out_reasoning + ) + + out_normal += self._buffer[:s] + self._buffer = self._buffer[s + len(start_tok) :] + self._in_reasoning = True + self._reasoning_acc = "" + self._in_inner_tool = False + continue + + if self._in_inner_tool: + if (end_pos := self._buffer.find(tool_end)) == -1: + if ( + hold := self._ends_with_partial_token(self._buffer, tool_end) + ) != 0: + out_normal += self._buffer[:-hold] + self._buffer = self._buffer[-hold:] + else: + out_normal += self._buffer + self._buffer = "" + return StreamingParseResult( + normal_text=out_normal, reasoning_text=out_reasoning + ) + + out_normal += self._buffer[: end_pos + len(tool_end)] + self._buffer = self._buffer[end_pos + len(tool_end) :] + self._in_inner_tool = False + continue + + pos_tool = self._buffer.find(tool_start) + pos_end = self._buffer.find(end_tok) + + if pos_tool == -1 and pos_end == -1: + if self.stream_reasoning: + if ( + hold := max( + self._ends_with_partial_token(self._buffer, end_tok), + self._ends_with_partial_token(self._buffer, tool_start), + ) + ) != 0: + out_reasoning += self._buffer[:-hold] + self._buffer = self._buffer[-hold:] + else: + out_reasoning += self._buffer + self._buffer = "" + return StreamingParseResult( + normal_text=out_normal, reasoning_text=out_reasoning + ) + + next_pos = min(p for p in [pos_tool, pos_end] if p != -1) + + if pos_end != -1 and pos_end == next_pos: + reasoning_chunk = self._buffer[:pos_end] + if self.stream_reasoning: + out_reasoning += reasoning_chunk + else: + self._reasoning_acc += reasoning_chunk + out_reasoning += self._reasoning_acc + self._reasoning_acc = "" + self._buffer = self._buffer[pos_end + len(end_tok) :] + self._in_reasoning = False + continue + + reasoning_chunk = self._buffer[:pos_tool] + if self.stream_reasoning: + out_reasoning += reasoning_chunk + else: + self._reasoning_acc += reasoning_chunk + self._buffer = self._buffer[pos_tool:] + self._in_inner_tool = True + continue + + class CohereCommand4Detector(BaseReasoningFormatDetector): """Detector for Cohere Command4 / Command-A family (incl. cohere2_moe and cohere2_vision Command-A-Plus). @@ -848,6 +1063,7 @@ class ReasoningParser: """ DetectorMap: Dict[str, Type[BaseReasoningFormatDetector]] = { + "apertus2509": Apertus2509Detector, "deepseek-r1": DeepSeekR1Detector, "deepseek-v3": _DeepSeekV3Detector, "deepseek-v4": _DeepSeekV3Detector, @@ -918,6 +1134,19 @@ class ReasoningParser: ret = self.detector.detect_and_parse(full_text) return ret.reasoning_text, ret.normal_text + def parse_non_stream_blocks(self, full_text: str) -> list[dict]: + """Non-streaming call: return an ordered sequence of reasoning/text blocks""" + if hasattr(self.detector, "detect_and_parse_block_sequence"): + seq = self.detector.detect_and_parse_block_sequence(full_text) + return [{"type": k, "text": t} for k, t in seq] + + ret = self.detector.detect_and_parse(full_text) + blocks: list[dict] = [] + if ret.reasoning_text: + blocks.append({"type": "reasoning", "text": ret.reasoning_text}) + blocks.append({"type": "text", "text": ret.normal_text or ""}) + return blocks + def parse_stream_chunk( self, chunk_text: str ) -> Tuple[Optional[str], Optional[str]]: diff --git a/test/registered/unit/managers/test_template_manager.py b/test/registered/unit/managers/test_template_manager.py index 9ab82c97c..df0b8d905 100644 --- a/test/registered/unit/managers/test_template_manager.py +++ b/test/registered/unit/managers/test_template_manager.py @@ -114,6 +114,13 @@ class TestTemplateDetectionRuleMatrix(unittest.TestCase): PARSER_RULES_MATRIX = [ # (name, template_snippet, vocab, expected_parser, expected_toggle_param) + ( + "apertus2509_via_unique_vocab_token", + "{% set enable_thinking = enable_thinking if enable_thinking is defined else true %}\n", + ["<|inner_prefix|>"], + "apertus2509", + "enable_thinking", + ), ( "deepseek_r1_think_tags", "\nLet me reason about this\n\nAnswer here",