[Feat] Inkling: render and parse chat via tml-renderers (#45340)
This commit is contained in:
@@ -83,6 +83,7 @@ dependencies = [
|
||||
"tiktoken",
|
||||
"tilelang==0.1.12",
|
||||
"timm==1.0.16",
|
||||
"tml-renderers==0.1.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or (sys_platform == 'darwin' and platform_machine == 'arm64')",
|
||||
"tokenizers==0.23.2",
|
||||
"torch==2.14.1",
|
||||
"torch_memory_saver>=0.0.9.post1",
|
||||
|
||||
@@ -250,13 +250,8 @@ def encode_simple_chat(
|
||||
"""
|
||||
if spec == "inkling":
|
||||
from sglang.srt.parser.inkling_renderer import render_inkling_messages
|
||||
from sglang.srt.parser.inkling_tokenizer import InklingTokenizer
|
||||
|
||||
return render_inkling_messages(
|
||||
messages,
|
||||
InklingTokenizer(tokenizer=tokenizer),
|
||||
add_generation_prompt=False,
|
||||
)
|
||||
return render_inkling_messages(messages)
|
||||
|
||||
if spec in ("dsv4", "dsv32", "dsv41"):
|
||||
if spec != "dsv41" and messages and messages[0]["role"] != "system":
|
||||
|
||||
@@ -119,7 +119,12 @@ class ChatEncodingAdapterContext(msgspec.Struct, frozen=True, kw_only=True):
|
||||
def _build_inkling(ctx: ChatEncodingAdapterContext) -> ChatEncodingAdapter:
|
||||
from sglang.srt.entrypoints.openai.serving_inkling import InklingChatEncodingAdapter
|
||||
|
||||
return InklingChatEncodingAdapter(tokenizer=ctx.tokenizer)
|
||||
return InklingChatEncodingAdapter(
|
||||
tokenizer=ctx.tokenizer,
|
||||
reasoning_parser=ctx.reasoning_parser,
|
||||
tool_call_parser=ctx.tool_call_parser,
|
||||
chat_hooks=ctx.chat_hooks,
|
||||
)
|
||||
|
||||
|
||||
_BUILDERS: dict[
|
||||
|
||||
@@ -3,7 +3,6 @@ from __future__ import annotations
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
import uuid
|
||||
from collections import OrderedDict
|
||||
@@ -687,63 +686,6 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
return prompt_ids
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _pop_inkling_assistant_prefix(
|
||||
messages: list[dict[str, Any]],
|
||||
request: ChatCompletionRequest,
|
||||
) -> str | None:
|
||||
"""Extract the trailing assistant text for ``continue_final_message``.
|
||||
|
||||
Only a plain-string assistant message with no tool calls and no
|
||||
reasoning content can be continued; anything else renders as a closed
|
||||
historical turn. Mutates ``messages`` in place (callers pass a copy).
|
||||
"""
|
||||
if not request.continue_final_message or not messages:
|
||||
return None
|
||||
last = messages[-1]
|
||||
if (
|
||||
last.get("role") != "assistant"
|
||||
or not isinstance(last.get("content"), str)
|
||||
or last.get("tool_calls")
|
||||
or last.get("reasoning_content")
|
||||
):
|
||||
return None
|
||||
messages.pop()
|
||||
return last["content"]
|
||||
|
||||
@staticmethod
|
||||
def _parse_inkling_reasoning_effort(
|
||||
value: str | float | None,
|
||||
) -> float | None:
|
||||
"""Convert an OpenAI-style reasoning_effort to an Inkling float."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool):
|
||||
raise ValueError("Inkling reasoning_effort must not be a boolean")
|
||||
if isinstance(value, (int, float)):
|
||||
parsed = float(value)
|
||||
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
|
||||
raise ValueError("Inkling reasoning_effort must be in [0.0, 0.99]")
|
||||
return parsed
|
||||
_EFFORT_MAP = {
|
||||
"none": 0.0,
|
||||
"minimal": 0.1,
|
||||
"low": 0.2,
|
||||
"medium": 0.7,
|
||||
"high": 0.9,
|
||||
"xhigh": 0.99,
|
||||
"max": 0.99,
|
||||
}
|
||||
if value in _EFFORT_MAP:
|
||||
return _EFFORT_MAP[value]
|
||||
try:
|
||||
parsed = float(value)
|
||||
except (ValueError, TypeError) as exc:
|
||||
raise ValueError(f"invalid Inkling reasoning_effort: {value!r}") from exc
|
||||
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
|
||||
raise ValueError("Inkling reasoning_effort must be in [0.0, 0.99]")
|
||||
return parsed
|
||||
|
||||
def _resolve_dsv41_reasoning_effort(self, value: Any) -> Union[str, int]:
|
||||
"""Request effort for the V4.1 encoder; unsupported values warn and fall back."""
|
||||
effort = chat_encoding.parse_dsv41_reasoning_effort(value)
|
||||
@@ -759,26 +701,6 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
)
|
||||
return self._dsv41_default_reasoning_effort
|
||||
|
||||
@staticmethod
|
||||
def _get_inkling_default_reasoning_effort() -> float:
|
||||
"""Read the default Inkling reasoning effort from the environment."""
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
val = envs.SGLANG_INKLING_DEFAULT_REASONING_EFFORT.get()
|
||||
if not val:
|
||||
return 0.9
|
||||
try:
|
||||
parsed = float(val)
|
||||
except (ValueError, TypeError) as exc:
|
||||
raise ValueError(
|
||||
"SGLANG_INKLING_DEFAULT_REASONING_EFFORT must be numeric"
|
||||
) from exc
|
||||
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
|
||||
raise ValueError(
|
||||
"SGLANG_INKLING_DEFAULT_REASONING_EFFORT must be in [0.0, 0.99]"
|
||||
)
|
||||
return parsed
|
||||
|
||||
def _decode_response(self, ret_item: dict[str, Any]) -> str | ErrorResponse:
|
||||
"""Extract text from response."""
|
||||
return ret_item["text"]
|
||||
|
||||
@@ -1,67 +1,390 @@
|
||||
"""Inkling serving glue: prompt rendering and token-level output parsing for the
|
||||
Chat Completions and Responses APIs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import TYPE_CHECKING, Any, Optional
|
||||
|
||||
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponseStreamChoice,
|
||||
ChatCompletionStreamResponse,
|
||||
DeltaMessage,
|
||||
FunctionResponse,
|
||||
ToolCall,
|
||||
)
|
||||
from sglang.srt.entrypoints.openai.sse_utils import build_sse_content
|
||||
from sglang.srt.function_call.core_types import ToolCallItem
|
||||
from sglang.srt.parser.inkling_output import (
|
||||
InklingOutputDelta,
|
||||
InklingOutputParser,
|
||||
InklingToolCall,
|
||||
)
|
||||
from sglang.srt.parser.inkling_renderer import (
|
||||
load_tml_renderers,
|
||||
render_inkling_assistant_prefix,
|
||||
render_inkling_messages,
|
||||
)
|
||||
from sglang.srt.parser.inkling_tokenizer import INKLING_SPECIAL_TOKEN_IDS
|
||||
from sglang.srt.runtime_context import get_serving
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
|
||||
from sglang.srt.entrypoints.openai.chat_encoding_adapters import ChatOutputHooks
|
||||
|
||||
|
||||
class InklingChatEncodingAdapter:
|
||||
"""Inkling prompt encoding through ``inkling_renderer``; output stays with
|
||||
the registered ``inkling`` reasoning and tool-call parsers."""
|
||||
|
||||
collects_openai_media = True
|
||||
output = None
|
||||
|
||||
def __init__(self, *, tokenizer: Any):
|
||||
self._tokenizer = tokenizer
|
||||
# Resolved at boot: a misconfigured env default fails startup, not requests.
|
||||
self._default_reasoning_effort = (
|
||||
OpenAIServingChat._get_inkling_default_reasoning_effort()
|
||||
)
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
tokenizer: Any,
|
||||
reasoning_parser: str | None,
|
||||
tool_call_parser: str | None,
|
||||
chat_hooks: ChatOutputHooks,
|
||||
):
|
||||
# Validate the frozen effort default at startup,
|
||||
# so configuration errors fail before requests.
|
||||
self._default_reasoning_effort = get_default_reasoning_effort()
|
||||
load_tml_renderers()
|
||||
if tokenizer is not None:
|
||||
_check_served_tokenizer(tokenizer)
|
||||
parsers = {reasoning_parser, tool_call_parser} - {None}
|
||||
if "inkling" in parsers and parsers != {"inkling"}:
|
||||
# The token parser emits Inkling calls while constraints would come
|
||||
# from the other parser's format.
|
||||
raise ValueError(
|
||||
"Inkling token parsing requires --reasoning-parser and "
|
||||
"--tool-call-parser to be inkling or unset, got "
|
||||
f"{reasoning_parser!r} and {tool_call_parser!r}"
|
||||
)
|
||||
self._token_output = "inkling" in parsers
|
||||
self._separate_reasoning = bool(reasoning_parser)
|
||||
self._hooks = chat_hooks
|
||||
|
||||
@property
|
||||
def output(self) -> Optional[InklingChatEncodingAdapter]:
|
||||
return self if self._token_output else None
|
||||
|
||||
def encode_messages(
|
||||
self,
|
||||
*,
|
||||
messages: list[dict[str, Any]],
|
||||
request: ChatCompletionRequest,
|
||||
tools: Optional[list[dict]],
|
||||
tools: list[dict] | None,
|
||||
) -> list[int]:
|
||||
from sglang.srt.parser.inkling_renderer import render_inkling_messages
|
||||
from sglang.srt.parser.inkling_tokenizer import (
|
||||
CONTENT_TEXT,
|
||||
MESSAGE_MODEL,
|
||||
InklingTokenizer,
|
||||
)
|
||||
|
||||
# Framing tokens + ONE placeholder per media (encoding/expansion happens later
|
||||
# in InklingMultimodalProcessor). The server's tokenizer is the base tiktoken
|
||||
# backend; wrap it so encode_special supplies the framing-token overlay.
|
||||
inkling_tokenizer = InklingTokenizer(tokenizer=self._tokenizer)
|
||||
reasoning_effort = OpenAIServingChat._parse_inkling_reasoning_effort(
|
||||
request.reasoning_effort
|
||||
)
|
||||
reasoning_effort = parse_reasoning_effort(request.reasoning_effort)
|
||||
if reasoning_effort is None:
|
||||
reasoning_effort = self._default_reasoning_effort
|
||||
assistant_prefix = OpenAIServingChat._pop_inkling_assistant_prefix(
|
||||
messages, request
|
||||
)
|
||||
assistant_prefix = _pop_assistant_prefix(messages, request)
|
||||
prompt_ids = render_inkling_messages(
|
||||
messages,
|
||||
inkling_tokenizer,
|
||||
add_generation_prompt=False,
|
||||
tools=tools,
|
||||
reasoning_effort=reasoning_effort,
|
||||
messages, tools=tools, reasoning_effort=reasoning_effort
|
||||
)
|
||||
if assistant_prefix is not None:
|
||||
# Continue the final assistant message inside an OPEN model text
|
||||
# block: header + payload, no <|end_message|> and no
|
||||
# <|content_model_end_sampling|>, so the model resumes the turn.
|
||||
prompt_ids += [
|
||||
inkling_tokenizer.encode_special(MESSAGE_MODEL),
|
||||
inkling_tokenizer.encode_special(CONTENT_TEXT),
|
||||
*inkling_tokenizer.encode_text(assistant_prefix),
|
||||
]
|
||||
prompt_ids += render_inkling_assistant_prefix(assistant_prefix)
|
||||
return prompt_ids
|
||||
|
||||
def parse_chat_response(
|
||||
self,
|
||||
*,
|
||||
request: ChatCompletionRequest,
|
||||
output_ids: list[int],
|
||||
finish_reason: dict[str, Any],
|
||||
) -> tuple[str | None, str, list[ToolCall] | None, dict[str, Any]]:
|
||||
parsed = self._new_output_parser(request).finish(
|
||||
output_ids,
|
||||
matched_stop=finish_reason.get("matched"),
|
||||
keep_matched_stop=request.no_stop_trim,
|
||||
)
|
||||
if not parsed.tool_calls:
|
||||
return parsed.reasoning or None, parsed.content, None, finish_reason
|
||||
history_tool_calls_cnt = self._hooks.history_tool_calls_count(request)
|
||||
tool_calls = [
|
||||
self._tool_call(call, history_tool_calls_cnt) for call in parsed.tool_calls
|
||||
]
|
||||
if finish_reason["type"] == "stop":
|
||||
finish_reason = {**finish_reason, "type": "tool_calls", "matched": None}
|
||||
return parsed.reasoning or None, parsed.content, tool_calls, finish_reason
|
||||
|
||||
def stream_chat_chunks(
|
||||
self,
|
||||
*,
|
||||
content: dict[str, Any],
|
||||
index: int,
|
||||
request: ChatCompletionRequest,
|
||||
created: int,
|
||||
parser_dict: dict,
|
||||
has_tool_calls: dict[int, bool],
|
||||
choice_logprobs: dict | None,
|
||||
finish_reason_type: str | None,
|
||||
usage: dict[str, Any] | None,
|
||||
) -> list[str]:
|
||||
if index not in parser_dict:
|
||||
parser_dict[index] = self._new_output_parser(request)
|
||||
parser = parser_dict[index]
|
||||
new_output_ids = select_new_output_ids(
|
||||
output_ids=content["output_ids"],
|
||||
num_consumed_tokens=parser.num_consumed_tokens,
|
||||
completion_tokens=content["meta_info"]["completion_tokens"],
|
||||
finish_reason_type=finish_reason_type,
|
||||
incremental=get_serving().incremental_streaming_output,
|
||||
)
|
||||
if finish_reason_type is None:
|
||||
delta = parser.feed(new_output_ids)
|
||||
else:
|
||||
delta = parser.finish(
|
||||
new_output_ids,
|
||||
matched_stop=content["meta_info"]["finish_reason"].get("matched"),
|
||||
keep_matched_stop=request.no_stop_trim,
|
||||
)
|
||||
|
||||
chunk_fields = dict(
|
||||
chunk_id=content["meta_info"]["id"],
|
||||
created=created,
|
||||
model=request.model,
|
||||
index=index,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
chunks = []
|
||||
remaining_logprobs = choice_logprobs
|
||||
if delta.reasoning:
|
||||
chunks.append(
|
||||
build_sse_content(
|
||||
reasoning_content=delta.reasoning,
|
||||
logprobs=remaining_logprobs,
|
||||
**chunk_fields,
|
||||
)
|
||||
)
|
||||
remaining_logprobs = None
|
||||
if delta.content:
|
||||
chunks.append(
|
||||
build_sse_content(
|
||||
content=delta.content, logprobs=remaining_logprobs, **chunk_fields
|
||||
)
|
||||
)
|
||||
remaining_logprobs = None
|
||||
history_tool_calls_cnt = self._hooks.history_tool_calls_count(request)
|
||||
for call in delta.tool_calls:
|
||||
has_tool_calls[index] = True
|
||||
tool_call_chunk = ChatCompletionStreamResponse(
|
||||
id=content["meta_info"]["id"],
|
||||
created=chunk_fields["created"],
|
||||
choices=[
|
||||
ChatCompletionResponseStreamChoice(
|
||||
index=index,
|
||||
delta=DeltaMessage(
|
||||
tool_calls=[self._tool_call(call, history_tool_calls_cnt)]
|
||||
),
|
||||
finish_reason=None,
|
||||
)
|
||||
],
|
||||
model=request.model,
|
||||
usage=usage,
|
||||
)
|
||||
chunks.append(f"data: {tool_call_chunk.model_dump_json()}\n\n")
|
||||
if remaining_logprobs is not None:
|
||||
chunks.append(
|
||||
build_sse_content(logprobs=remaining_logprobs, **chunk_fields)
|
||||
)
|
||||
return chunks
|
||||
|
||||
def parse_responses_output(
|
||||
self,
|
||||
*,
|
||||
output_ids: list[int],
|
||||
finish_reason: dict[str, Any] | None,
|
||||
separate_reasoning: bool,
|
||||
parse_tool_calls: bool,
|
||||
) -> tuple[str, str, list[ToolCallItem]]:
|
||||
parser = InklingOutputParser(
|
||||
separate_reasoning=separate_reasoning, parse_tool_calls=parse_tool_calls
|
||||
)
|
||||
delta = parser.finish(
|
||||
output_ids,
|
||||
matched_stop=(finish_reason or {}).get("matched"),
|
||||
keep_matched_stop=False,
|
||||
)
|
||||
return delta.reasoning, delta.content, _tool_call_items(delta)
|
||||
|
||||
def new_responses_stream(
|
||||
self, *, separate_reasoning: bool, parse_tool_calls: bool
|
||||
) -> InklingResponsesStream:
|
||||
return InklingResponsesStream(
|
||||
separate_reasoning=separate_reasoning, parse_tool_calls=parse_tool_calls
|
||||
)
|
||||
|
||||
def _new_output_parser(self, request: ChatCompletionRequest) -> InklingOutputParser:
|
||||
return InklingOutputParser(
|
||||
separate_reasoning=self._separate_reasoning and request.separate_reasoning,
|
||||
parse_tool_calls=self._hooks.tool_call_parsing_active(request),
|
||||
stream_reasoning=request.stream_reasoning,
|
||||
continues_text_block=bool(
|
||||
request.continue_final_message
|
||||
and request.messages
|
||||
and _is_continuable_message(request.messages[-1].model_dump())
|
||||
),
|
||||
)
|
||||
|
||||
def _tool_call(
|
||||
self, call: InklingToolCall, history_tool_calls_cnt: int
|
||||
) -> ToolCall:
|
||||
call_item = ToolCallItem(
|
||||
tool_index=call.index, name=call.name, parameters=call.arguments
|
||||
)
|
||||
return ToolCall(
|
||||
id=self._hooks.tool_call_id(call_item, history_tool_calls_cnt),
|
||||
index=call.index,
|
||||
function=FunctionResponse(name=call.name, arguments=call.arguments),
|
||||
)
|
||||
|
||||
|
||||
class InklingResponsesStream:
|
||||
def __init__(self, *, separate_reasoning: bool, parse_tool_calls: bool):
|
||||
self._parser = InklingOutputParser(
|
||||
separate_reasoning=separate_reasoning, parse_tool_calls=parse_tool_calls
|
||||
)
|
||||
self._incremental = get_serving().incremental_streaming_output
|
||||
|
||||
def step(
|
||||
self, chunk: dict[str, Any], finish_reason: dict[str, Any] | None
|
||||
) -> tuple[str, str, list[ToolCallItem]]:
|
||||
new_output_ids = select_new_output_ids(
|
||||
output_ids=chunk.get("output_ids") or [],
|
||||
num_consumed_tokens=self._parser.num_consumed_tokens,
|
||||
completion_tokens=chunk["meta_info"]["completion_tokens"],
|
||||
finish_reason_type=finish_reason["type"] if finish_reason else None,
|
||||
incremental=self._incremental,
|
||||
)
|
||||
if finish_reason is None:
|
||||
delta = self._parser.feed(new_output_ids)
|
||||
else:
|
||||
delta = self._parser.finish(
|
||||
new_output_ids,
|
||||
matched_stop=finish_reason.get("matched"),
|
||||
keep_matched_stop=False,
|
||||
)
|
||||
return delta.reasoning, delta.content, _tool_call_items(delta)
|
||||
|
||||
|
||||
def _tool_call_items(delta: InklingOutputDelta) -> list[ToolCallItem]:
|
||||
return [
|
||||
ToolCallItem(tool_index=call.index, name=call.name, parameters=call.arguments)
|
||||
for call in delta.tool_calls
|
||||
]
|
||||
|
||||
|
||||
def select_new_output_ids(
|
||||
*,
|
||||
output_ids: list[int],
|
||||
num_consumed_tokens: int,
|
||||
completion_tokens: int,
|
||||
finish_reason_type: str | None,
|
||||
incremental: bool,
|
||||
) -> list[int]:
|
||||
if not incremental:
|
||||
# Intermediate chunks share the live output list; completion_tokens is
|
||||
# this chunk's snapshot, so later tokens wait for their own chunk.
|
||||
return output_ids[num_consumed_tokens:completion_tokens]
|
||||
if finish_reason_type == "abort":
|
||||
# The abort chunk re-sends already-streamed tokens.
|
||||
return output_ids[: max(completion_tokens - num_consumed_tokens, 0)]
|
||||
return output_ids
|
||||
|
||||
|
||||
_TOKENIZER_PROBE = "Hello, 世界! def f(x): return x**2"
|
||||
|
||||
|
||||
def _check_served_tokenizer(tokenizer: Any) -> None:
|
||||
# Prompts and outputs use tml-renderers' tokenizer, while the scheduler matches
|
||||
# stop strings and detokenizes with the served one; they must agree.
|
||||
tml_tokenizer = load_tml_renderers().tokenizer
|
||||
# encode() is the lookup both the HF and the tiktoken tokenizer wrappers share.
|
||||
mismatched = [
|
||||
token
|
||||
for token in INKLING_SPECIAL_TOKEN_IDS
|
||||
if tokenizer.encode(token, add_special_tokens=False)
|
||||
!= [tml_tokenizer.encode_special(token[2:-2])]
|
||||
]
|
||||
if tokenizer.encode(_TOKENIZER_PROBE, add_special_tokens=False) != list(
|
||||
tml_tokenizer.encode_ordinary(_TOKENIZER_PROBE)
|
||||
):
|
||||
mismatched.append("ordinary text")
|
||||
if mismatched:
|
||||
raise ValueError(
|
||||
"The served tokenizer disagrees with tml-renderers' o200k_base_chat on "
|
||||
f"{', '.join(mismatched)}; Inkling prompts and outputs would be misencoded."
|
||||
)
|
||||
|
||||
|
||||
def _pop_assistant_prefix(
|
||||
messages: list[dict[str, Any]],
|
||||
request: ChatCompletionRequest,
|
||||
) -> str | None:
|
||||
"""Pops only a continuable text turn; callers pass a message-list copy."""
|
||||
if not request.continue_final_message or not messages:
|
||||
return None
|
||||
if not _is_continuable_message(messages[-1]):
|
||||
return None
|
||||
return messages.pop()["content"]
|
||||
|
||||
|
||||
def _is_continuable_message(message: dict[str, Any]) -> bool:
|
||||
return (
|
||||
message.get("role") == "assistant"
|
||||
and isinstance(message.get("content"), str)
|
||||
and not message.get("tool_calls")
|
||||
and not message.get("reasoning_content")
|
||||
)
|
||||
|
||||
|
||||
def parse_reasoning_effort(
|
||||
value: str | float | None,
|
||||
) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool):
|
||||
raise ValueError("Inkling reasoning_effort must not be a boolean")
|
||||
if isinstance(value, (int, float)):
|
||||
parsed = float(value)
|
||||
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
|
||||
raise ValueError("Inkling reasoning_effort must be in [0.0, 0.99]")
|
||||
return parsed
|
||||
_EFFORT_MAP = {
|
||||
"none": 0.0,
|
||||
"minimal": 0.1,
|
||||
"low": 0.2,
|
||||
"medium": 0.7,
|
||||
"high": 0.9,
|
||||
"xhigh": 0.99,
|
||||
"max": 0.99,
|
||||
}
|
||||
if value in _EFFORT_MAP:
|
||||
return _EFFORT_MAP[value]
|
||||
try:
|
||||
parsed = float(value)
|
||||
except (ValueError, TypeError) as exc:
|
||||
raise ValueError(f"invalid Inkling reasoning_effort: {value!r}") from exc
|
||||
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
|
||||
raise ValueError("Inkling reasoning_effort must be in [0.0, 0.99]")
|
||||
return parsed
|
||||
|
||||
|
||||
def get_default_reasoning_effort() -> float:
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
val = envs.SGLANG_INKLING_DEFAULT_REASONING_EFFORT.get()
|
||||
if not val:
|
||||
return 0.9
|
||||
try:
|
||||
parsed = float(val)
|
||||
except (ValueError, TypeError) as exc:
|
||||
raise ValueError(
|
||||
"SGLANG_INKLING_DEFAULT_REASONING_EFFORT must be numeric"
|
||||
) from exc
|
||||
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
|
||||
raise ValueError(
|
||||
"SGLANG_INKLING_DEFAULT_REASONING_EFFORT must be in [0.0, 0.99]"
|
||||
)
|
||||
return parsed
|
||||
|
||||
@@ -1224,6 +1224,20 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
def _tool_parser_owns_format(parser: FunctionCallParser) -> bool:
|
||||
return parser.owns_tool_format()
|
||||
|
||||
@staticmethod
|
||||
def _drain_json_array(
|
||||
parser: JsonArrayParser, delta: str, tools: list[Tool]
|
||||
) -> tuple[str, list[ToolCallItem]]:
|
||||
# Each call emits one name or one argument diff, so a chunk that holds
|
||||
# a whole call needs more calls; the stream may end with no more chunks.
|
||||
result = parser.parse_streaming_increment(delta, tools)
|
||||
normal_text, calls = result.normal_text or "", list(result.calls)
|
||||
while result.calls:
|
||||
result = parser.parse_streaming_increment("", tools)
|
||||
normal_text += result.normal_text or ""
|
||||
calls.extend(result.calls)
|
||||
return normal_text, calls
|
||||
|
||||
def _uses_required_tool_json(
|
||||
self, request: ResponsesRequest, chat_tools: list[Tool]
|
||||
) -> bool:
|
||||
@@ -2510,8 +2524,9 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
continue
|
||||
|
||||
if isinstance(tool_parser, JsonArrayParser):
|
||||
sp = tool_parser.parse_streaming_increment(delta, chat_tools)
|
||||
normal_text, tool_calls = sp.normal_text or "", sp.calls
|
||||
normal_text, tool_calls = self._drain_json_array(
|
||||
tool_parser, delta, chat_tools
|
||||
)
|
||||
elif token_stream is not None:
|
||||
normal_text, tool_calls = delta, token_calls
|
||||
elif tool_parser is not None:
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.parser.inkling_renderer import load_tml_renderers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CONTENT_KIND_NAMES = (
|
||||
"content_text",
|
||||
"content_thinking",
|
||||
"content_xml",
|
||||
"content_invoke_tool_json",
|
||||
"content_invoke_tool_text",
|
||||
"content_tool_error",
|
||||
"content_image",
|
||||
"content_audio_input",
|
||||
)
|
||||
|
||||
|
||||
class InklingToolCall(msgspec.Struct, frozen=True):
|
||||
index: int
|
||||
name: str
|
||||
arguments: str
|
||||
|
||||
|
||||
class InklingOutputDelta(msgspec.Struct, frozen=True):
|
||||
reasoning: str = ""
|
||||
content: str = ""
|
||||
tool_calls: tuple[InklingToolCall, ...] = ()
|
||||
|
||||
def merge(self, other: InklingOutputDelta) -> InklingOutputDelta:
|
||||
return InklingOutputDelta(
|
||||
reasoning=self.reasoning + other.reasoning,
|
||||
content=self.content + other.content,
|
||||
tool_calls=self.tool_calls + other.tool_calls,
|
||||
)
|
||||
|
||||
|
||||
class InklingOutputParser:
|
||||
"""Control tokens are identified by ID; ordinary tokens spelling a marker remain text."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
separate_reasoning: bool,
|
||||
parse_tool_calls: bool,
|
||||
stream_reasoning: bool = True,
|
||||
continues_text_block: bool = False,
|
||||
):
|
||||
self._tml = load_tml_renderers()
|
||||
self._separate_reasoning = separate_reasoning
|
||||
self._hold_reasoning = separate_reasoning and not stream_reasoning
|
||||
self._parse_tool_calls = parse_tool_calls
|
||||
_spans, self._parser = self._tml.renderer.render_for_completion([])
|
||||
tokenizer = self._tml.tokenizer
|
||||
self._content_kind_ids = frozenset(
|
||||
tokenizer.encode_special(name) for name in _CONTENT_KIND_NAMES
|
||||
)
|
||||
self._text_opener_ids = (
|
||||
tokenizer.encode_special("message_model"),
|
||||
tokenizer.encode_special("content_text"),
|
||||
)
|
||||
self._end_message_id = tokenizer.encode_special("end_message")
|
||||
self._message_ids: list[int] = []
|
||||
self._streamed = ""
|
||||
self._held_reasoning = ""
|
||||
self._model_authored = True
|
||||
self._at_message_boundary = True
|
||||
self._in_unframed_text = False
|
||||
self._num_tool_calls = 0
|
||||
self._finished = False
|
||||
self.num_consumed_tokens = 0
|
||||
if continues_text_block:
|
||||
# The prompt ended inside an open model text block, so the sampled
|
||||
# tokens carry no header; replay the opener so they parse as text.
|
||||
for token_id in self._text_opener_ids:
|
||||
self._parse(token_id, _DeltaBuilder())
|
||||
|
||||
def feed(self, token_ids: Sequence[int]) -> InklingOutputDelta:
|
||||
delta = _DeltaBuilder()
|
||||
for token_id in token_ids:
|
||||
self.num_consumed_tokens += 1
|
||||
self._feed_token(token_id, delta)
|
||||
return delta.build()
|
||||
|
||||
def finish(
|
||||
self,
|
||||
token_ids: Sequence[int] = (),
|
||||
*,
|
||||
matched_stop: int | str | None = None,
|
||||
keep_matched_stop: bool = False,
|
||||
) -> InklingOutputDelta:
|
||||
# Callers see a sticky finish_reason on later chunks; flushing again would
|
||||
# re-emit an unclosed message's payload.
|
||||
if self._finished:
|
||||
return InklingOutputDelta()
|
||||
self._finished = True
|
||||
if isinstance(matched_stop, str):
|
||||
token_ids = self._cut_at_stop_string(
|
||||
token_ids, matched_stop, keep_stop=keep_matched_stop
|
||||
)
|
||||
elif (
|
||||
not keep_matched_stop
|
||||
and token_ids
|
||||
and token_ids[-1] == matched_stop
|
||||
and matched_stop != self._end_message_id
|
||||
):
|
||||
# Inside an open block TML renders even a special stop as text;
|
||||
# only end_message closes a block without rendering.
|
||||
token_ids = token_ids[:-1]
|
||||
return self.feed(token_ids).merge(self._flush())
|
||||
|
||||
def _cut_at_stop_string(
|
||||
self, token_ids: Sequence[int], stop: str, *, keep_stop: bool
|
||||
) -> list[int]:
|
||||
# The scheduler matched ``stop`` on the decoded ids, framing included,
|
||||
# and ended on the token completing it; cut the raw stream there.
|
||||
tokenizer = self._tml.tokenizer
|
||||
if not token_ids:
|
||||
return []
|
||||
stop_bytes = stop.encode()
|
||||
pieces = [tokenizer.decode_bytes([token_ids[-1]])]
|
||||
last_len = region_len = len(pieces[0])
|
||||
start = len(token_ids) - 1
|
||||
# A match ending in the last token starts at most len(stop) - 1 bytes before it.
|
||||
while start > 0 and region_len < last_len + len(stop_bytes) - 1:
|
||||
start -= 1
|
||||
pieces.insert(0, tokenizer.decode_bytes([token_ids[start]]))
|
||||
region_len += len(pieces[0])
|
||||
cut = b"".join(pieces).find(stop_bytes)
|
||||
if cut == -1:
|
||||
return list(token_ids)
|
||||
if keep_stop:
|
||||
cut += len(stop_bytes)
|
||||
kept = list(token_ids[:start])
|
||||
offset = 0
|
||||
for token_id, piece in zip(token_ids[start:], pieces):
|
||||
if offset + len(piece) <= cut:
|
||||
kept.append(token_id)
|
||||
elif offset < cut and not tokenizer.is_special_token(token_id):
|
||||
partial = piece[: cut - offset].decode(errors="ignore")
|
||||
kept += tokenizer.encode_ordinary(partial)
|
||||
offset += len(piece)
|
||||
return kept
|
||||
|
||||
def _flush(self) -> InklingOutputDelta:
|
||||
delta = _DeltaBuilder()
|
||||
try:
|
||||
updates = self._parser.flush_updates()
|
||||
except self._tml.v0.ParseError as exc:
|
||||
self._recover_message(delta, exc)
|
||||
return delta.build()
|
||||
for update in updates:
|
||||
self._apply(update.update, delta)
|
||||
self._surface_unclosed_message(delta)
|
||||
self._release_held_reasoning(delta)
|
||||
return delta.build()
|
||||
|
||||
def _surface_unclosed_message(self, delta: _DeltaBuilder) -> None:
|
||||
# A message cut off (e.g. by max_tokens) never completes; only text kinds
|
||||
# streamed, so a call payload would otherwise vanish.
|
||||
if not self._model_authored or not any(
|
||||
token_id in self._content_kind_ids for token_id in self._message_ids
|
||||
):
|
||||
return
|
||||
text = self._payload_text()
|
||||
if text.startswith(self._streamed):
|
||||
delta.content.append(text[len(self._streamed) :])
|
||||
|
||||
def _feed_token(self, token_id: int, delta: _DeltaBuilder) -> None:
|
||||
is_special = self._tml.tokenizer.is_special_token(token_id)
|
||||
if self._in_unframed_text and is_special:
|
||||
# Inside a real text block TML reads most specials as text; the
|
||||
# sampler never emitted an opener here, so a special is framing.
|
||||
self._in_unframed_text = False
|
||||
if token_id != self._end_message_id:
|
||||
self._parse(self._end_message_id, delta)
|
||||
elif self._at_message_boundary and not is_special:
|
||||
# Headerless fallback output is model text; synthesize its missing opener.
|
||||
for opener_id in self._text_opener_ids:
|
||||
self._parse(opener_id, delta)
|
||||
self._in_unframed_text = True
|
||||
self._parse(token_id, delta)
|
||||
|
||||
def _parse(self, token_id: int, delta: _DeltaBuilder) -> None:
|
||||
self._at_message_boundary = False
|
||||
self._message_ids.append(token_id)
|
||||
try:
|
||||
updates = self._parser.parse_token(token_id)
|
||||
except self._tml.v0.ParseError as exc:
|
||||
self._recover_message(delta, exc)
|
||||
return
|
||||
for update in updates:
|
||||
self._apply(update.update, delta)
|
||||
|
||||
def _apply(self, update: Any, delta: _DeltaBuilder) -> None:
|
||||
chat = self._tml.chat
|
||||
if isinstance(update, chat.StreamingMessageHeader):
|
||||
self._model_authored = update.author.kind == chat.AuthorKind.Model
|
||||
self._streamed = ""
|
||||
elif isinstance(update, chat.StreamingContent):
|
||||
if self._model_authored:
|
||||
self._emit_text(update.content, update.content.text, delta)
|
||||
self._streamed += update.content.text
|
||||
else:
|
||||
self._complete_message(update, delta)
|
||||
self._message_ids = []
|
||||
self._streamed = ""
|
||||
self._at_message_boundary = True
|
||||
|
||||
def _complete_message(self, message: Any, delta: _DeltaBuilder) -> None:
|
||||
chat = self._tml.chat
|
||||
content = message.content
|
||||
if message.author.kind != chat.AuthorKind.Model:
|
||||
logger.debug("Dropping non-model Inkling output message: %r", message)
|
||||
elif isinstance(content, (chat.Text, chat.Thinking)):
|
||||
if content.text.startswith(self._streamed):
|
||||
self._emit_text(content, content.text[len(self._streamed) :], delta)
|
||||
self._release_held_reasoning(delta)
|
||||
elif isinstance(content, chat.InvokeTool):
|
||||
if self._parse_tool_calls and content.text is None:
|
||||
delta.tool_calls.append(self._tool_call(message))
|
||||
self._num_tool_calls += 1
|
||||
else:
|
||||
delta.content.append(self._payload_text())
|
||||
|
||||
def _emit_text(self, content: Any, text: str, delta: _DeltaBuilder) -> None:
|
||||
if not (
|
||||
self._separate_reasoning and isinstance(content, self._tml.chat.Thinking)
|
||||
):
|
||||
delta.content.append(text)
|
||||
elif self._hold_reasoning:
|
||||
self._held_reasoning += text
|
||||
else:
|
||||
delta.reasoning.append(text)
|
||||
|
||||
def _release_held_reasoning(self, delta: _DeltaBuilder) -> None:
|
||||
if self._held_reasoning:
|
||||
delta.reasoning.append(self._held_reasoning)
|
||||
self._held_reasoning = ""
|
||||
|
||||
def _tool_call(self, message: Any) -> InklingToolCall:
|
||||
chat = self._tml.chat
|
||||
oss = chat.OpenAIMessage.to_oss_messages(
|
||||
chat.OpenAIMessage.from_messages([message])
|
||||
)
|
||||
function = oss[0]["tool_calls"][0]["function"]
|
||||
# The bridge re-serializes args with sorted keys, ASCII escapes and big ints
|
||||
# as floats; build them from the sampled payload instead.
|
||||
args = json.loads(self._payload_json())["args"]
|
||||
return InklingToolCall(
|
||||
index=self._num_tool_calls,
|
||||
name=function["name"],
|
||||
arguments=json.dumps(args, ensure_ascii=False),
|
||||
)
|
||||
|
||||
def _recover_message(self, delta: _DeltaBuilder, exc: Exception) -> None:
|
||||
logger.warning(
|
||||
"Inkling output failed TMLv0 parsing; surfacing as text: %s", exc
|
||||
)
|
||||
text = self._payload_text()
|
||||
emitted = "" if self._held_reasoning else self._streamed
|
||||
if text.startswith(emitted):
|
||||
delta.content.append(text[len(emitted) :])
|
||||
self._held_reasoning = ""
|
||||
self._parser.reset()
|
||||
self._message_ids = []
|
||||
self._streamed = ""
|
||||
self._model_authored = True
|
||||
self._at_message_boundary = True
|
||||
self._in_unframed_text = False
|
||||
|
||||
def _payload_ids(self) -> list[int]:
|
||||
ids = self._message_ids
|
||||
kind_index = next(
|
||||
(i for i, token_id in enumerate(ids) if token_id in self._content_kind_ids),
|
||||
-1,
|
||||
)
|
||||
return ids[kind_index + 1 :]
|
||||
|
||||
def _payload_text(self) -> str:
|
||||
tokenizer = self._tml.tokenizer
|
||||
return tokenizer.decode(
|
||||
[t for t in self._payload_ids() if not tokenizer.is_special_token(t)]
|
||||
)
|
||||
|
||||
def _payload_json(self) -> str:
|
||||
# A special inside the payload keeps its spelling, as in the library's parse.
|
||||
ids = self._payload_ids()
|
||||
if ids and ids[-1] == self._end_message_id:
|
||||
ids = ids[:-1]
|
||||
return self._tml.tokenizer.decode(ids)
|
||||
|
||||
|
||||
class _DeltaBuilder:
|
||||
def __init__(self):
|
||||
self.reasoning: list[str] = []
|
||||
self.content: list[str] = []
|
||||
self.tool_calls: list[InklingToolCall] = []
|
||||
|
||||
def build(self) -> InklingOutputDelta:
|
||||
return InklingOutputDelta(
|
||||
reasoning="".join(self.reasoning),
|
||||
content="".join(self.content),
|
||||
tool_calls=tuple(self.tool_calls),
|
||||
)
|
||||
@@ -1,332 +1,242 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import json
|
||||
import math
|
||||
import uuid
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Protocol
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from sglang.srt.parser.inkling_tokenizer import (
|
||||
AUDIO_END,
|
||||
AUDIO_TOKEN_ID,
|
||||
CONTENT_AUDIO_INPUT,
|
||||
CONTENT_IMAGE,
|
||||
CONTENT_INVOKE_TOOL_JSON,
|
||||
CONTENT_MODEL_END_SAMPLING,
|
||||
CONTENT_TEXT,
|
||||
CONTENT_THINKING,
|
||||
CONTENT_XML,
|
||||
END_MESSAGE,
|
||||
IMAGE_TOKEN_ID,
|
||||
MESSAGE_MODEL,
|
||||
ROLE_MESSAGE_TOKENS,
|
||||
)
|
||||
|
||||
INKLING_DEFAULT_REASONING_EFFORT = 0.9
|
||||
|
||||
class InklingTextTokenizer(Protocol):
|
||||
def encode_text(self, text: str) -> list[int]: ...
|
||||
TML_RENDERERS_INSTALL_HINT = (
|
||||
"Serving Inkling requires the `tml-renderers` package (TML's reference "
|
||||
"TMLv0 renderer/parser; wheels for Linux and macOS arm64). Install it with "
|
||||
"`pip install tml-renderers`."
|
||||
)
|
||||
|
||||
def encode_special(self, token: str) -> int: ...
|
||||
|
||||
|
||||
# OpenAI content-part type spellings that mean image / audio (render only needs the kind,
|
||||
# not the bytes — the bytes are encoded later in the MM processor).
|
||||
_IMAGE_PART_TYPES = frozenset({"image", "input_image", "image_url"})
|
||||
_AUDIO_PART_TYPES = frozenset({"audio", "input_audio", "audio_url"})
|
||||
INKLING_DEFAULT_REASONING_EFFORT = 0.9
|
||||
_TEXT_PART_TYPES = frozenset({None, "text", "input_text"})
|
||||
_THINKING_PART_TYPES = frozenset({"thinking", "reasoning"})
|
||||
|
||||
_IMAGE_LOCATION_PREFIX = "sglang-inkling-image://"
|
||||
_AUDIO_LOCATION_PREFIX = "sglang-inkling-audio://"
|
||||
|
||||
|
||||
@functools.cache
|
||||
def load_tml_renderers() -> SimpleNamespace:
|
||||
try:
|
||||
from tml_renderers import chat, tokenizers, v0
|
||||
except ImportError as exc:
|
||||
raise ImportError(TML_RENDERERS_INSTALL_HINT) from exc
|
||||
tokenizer = tokenizers.o200k_base_chat()
|
||||
return SimpleNamespace(
|
||||
chat=chat, v0=v0, tokenizer=tokenizer, renderer=v0.Renderer(tokenizer)
|
||||
)
|
||||
|
||||
|
||||
def render_inkling_messages(
|
||||
messages: Sequence[Mapping[str, Any]],
|
||||
tokenizer: InklingTextTokenizer,
|
||||
*,
|
||||
add_generation_prompt: bool = False,
|
||||
tools: Sequence[Mapping[str, Any]] | None = None,
|
||||
reasoning_effort: float | None = None,
|
||||
) -> list[int]:
|
||||
"""Render chat messages to Inkling input_ids with ONE placeholder per media item.
|
||||
|
||||
PURE renderer: emits Inkling framing + a single IMAGE_TOKEN_ID / AUDIO_TOKEN_ID
|
||||
per image / audio part. Media encoding and 1->N placeholder expansion happen
|
||||
later in the MM processor. Inkling normally emits its own assistant turn
|
||||
opener; ``add_generation_prompt`` is retained only for legacy callers. The
|
||||
conversation-level effort directive is emitted once in the initial prefix
|
||||
and defaults to 0.9.
|
||||
"""
|
||||
input_ids: list[int] = []
|
||||
tool_call_id_to_name: dict[str, str] = {}
|
||||
|
||||
"""Media parts emit one placeholder each for later MM expansion.
|
||||
Inkling samples its own assistant header."""
|
||||
tml = load_tml_renderers()
|
||||
media_marker = f"sglang-inkling-media-{uuid.uuid4().hex}:"
|
||||
oss_messages, media_kinds = _to_oss_messages(messages, media_marker)
|
||||
if tools:
|
||||
_append_message(
|
||||
input_ids,
|
||||
tokenizer,
|
||||
"system",
|
||||
"xml",
|
||||
_tool_declare_json(tools),
|
||||
author_name="tool_declare",
|
||||
oss_messages.insert(
|
||||
_leading_system_count(oss_messages), _tool_declare_message(tools)
|
||||
)
|
||||
|
||||
# Normalize the OpenAI "developer" role to "system" (as the Responses API
|
||||
# does in _normalize_response_message_for_chat) so developer-instruction
|
||||
# messages render instead of tripping _expect_role; the leading-system
|
||||
# grouping below then also sees them. Shallow-copy only affected messages so
|
||||
# the caller's list is left untouched.
|
||||
message_list = [
|
||||
{**message, "role": "system"} if message.get("role") == "developer" else message
|
||||
for message in messages
|
||||
native = [
|
||||
_resolve_media(message, tml.chat, media_marker, media_kinds)
|
||||
for oss_message in tml.chat.OpenAIMessage.from_oss_messages(oss_messages)
|
||||
for message in oss_message.to_messages()
|
||||
]
|
||||
leading_system_count = 0
|
||||
for message in message_list:
|
||||
if message.get("role") != "system":
|
||||
break
|
||||
leading_system_count += 1
|
||||
effort = (
|
||||
INKLING_DEFAULT_REASONING_EFFORT
|
||||
if reasoning_effort is None
|
||||
else reasoning_effort
|
||||
)
|
||||
spans, _parser = tml.renderer.render_for_completion_with_effort(native, effort)
|
||||
return _spans_to_input_ids(spans, tml)
|
||||
|
||||
def append_effort() -> None:
|
||||
effort = (
|
||||
INKLING_DEFAULT_REASONING_EFFORT
|
||||
if reasoning_effort is None
|
||||
else reasoning_effort
|
||||
)
|
||||
_append_message(
|
||||
input_ids,
|
||||
tokenizer,
|
||||
"system",
|
||||
"text",
|
||||
f"Thinking effort level: {_format_reasoning_effort(effort)}",
|
||||
)
|
||||
|
||||
for message_index, message in enumerate(message_list):
|
||||
if message_index == leading_system_count:
|
||||
append_effort()
|
||||
role = _expect_role(message)
|
||||
if role == "tool":
|
||||
tool_name = str(
|
||||
message.get("name")
|
||||
or tool_call_id_to_name.get(message.get("tool_call_id") or "", "")
|
||||
def render_inkling_assistant_prefix(prefix: str) -> list[int]:
|
||||
"""TMLv0 has no prefill; continuation opens a model text block without a terminator."""
|
||||
tokenizer = load_tml_renderers().tokenizer
|
||||
return [
|
||||
tokenizer.encode_special(_special_name(MESSAGE_MODEL)),
|
||||
tokenizer.encode_special(_special_name(CONTENT_TEXT)),
|
||||
*tokenizer.encode_ordinary(prefix),
|
||||
]
|
||||
|
||||
|
||||
def _to_oss_messages(
|
||||
messages: Sequence[Mapping[str, Any]], media_marker: str
|
||||
) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
media_kinds: list[str] = []
|
||||
oss_messages = []
|
||||
call_names: dict[str, str] = {}
|
||||
for message in messages:
|
||||
oss_message = {
|
||||
key: value for key, value in message.items() if value is not None
|
||||
}
|
||||
if oss_message.get("role") == "developer":
|
||||
oss_message["role"] = "system"
|
||||
reasoning = oss_message.pop("reasoning_content", None)
|
||||
if reasoning and oss_message.get("role") == "assistant":
|
||||
oss_message["content"] = _with_leading_thinking(
|
||||
reasoning, oss_message.get("content")
|
||||
)
|
||||
# The MM processor harvests media from tool messages, so coercing content
|
||||
# to one string would drop images and desync the placeholder count.
|
||||
tool_parts = list(_iter_render_parts(message.get("content", "")))
|
||||
if not tool_parts:
|
||||
tool_parts = [("text", "")] # else the answered tool_call dangles
|
||||
for kind, text in tool_parts:
|
||||
if kind == "thinking":
|
||||
raise ValueError("Inkling thinking parts require role='assistant'")
|
||||
_append_message(
|
||||
input_ids,
|
||||
tokenizer,
|
||||
"tool",
|
||||
kind,
|
||||
text,
|
||||
author_name=tool_name,
|
||||
)
|
||||
continue
|
||||
for call in oss_message.get("tool_calls") or ():
|
||||
if call.get("id") and call["function"].get("name"):
|
||||
call_names[call["id"]] = call["function"]["name"]
|
||||
# The library takes a tool author only from ``name``; spec Chat clients and the
|
||||
# Responses/Anthropic adapters send just ``tool_call_id``.
|
||||
if oss_message.get("role") == "tool" and not oss_message.get("name"):
|
||||
name = call_names.get(oss_message.get("tool_call_id"))
|
||||
if name:
|
||||
oss_message["name"] = name
|
||||
content = oss_message.get("content")
|
||||
if isinstance(content, Sequence) and not isinstance(
|
||||
content, (str, bytes, bytearray)
|
||||
):
|
||||
oss_message["content"] = [
|
||||
_to_oss_part(part, media_marker, media_kinds) for part in content
|
||||
]
|
||||
oss_messages.append(oss_message)
|
||||
return oss_messages, media_kinds
|
||||
|
||||
parts = list(_iter_render_parts(message.get("content", "")))
|
||||
turn_start = len(input_ids)
|
||||
if role == "assistant":
|
||||
reasoning_content = message.get("reasoning_content")
|
||||
if reasoning_content:
|
||||
if not isinstance(reasoning_content, str):
|
||||
raise TypeError(
|
||||
"assistant reasoning_content must be a string for Inkling rendering"
|
||||
)
|
||||
if any(kind == "thinking" for kind, _ in parts):
|
||||
raise ValueError(
|
||||
"assistant message cannot mix reasoning_content with ordered thinking parts"
|
||||
)
|
||||
_append_message(
|
||||
input_ids,
|
||||
tokenizer,
|
||||
"assistant",
|
||||
"thinking",
|
||||
reasoning_content,
|
||||
)
|
||||
|
||||
for kind, text in parts:
|
||||
if kind == "thinking" and role != "assistant":
|
||||
raise ValueError("Inkling thinking parts require role='assistant'")
|
||||
_append_message(input_ids, tokenizer, role, kind, text)
|
||||
def _with_leading_thinking(reasoning: str, content: Any) -> list[Any]:
|
||||
# tml-renderers reads prior reasoning only from thinking content parts.
|
||||
parts: list[Any] = [{"type": "thinking", "thinking": reasoning}]
|
||||
if isinstance(content, str):
|
||||
if content:
|
||||
parts.append({"type": "text", "text": content})
|
||||
elif content:
|
||||
if any(
|
||||
isinstance(part, Mapping) and part.get("type") in _THINKING_PART_TYPES
|
||||
for part in content
|
||||
):
|
||||
raise ValueError(
|
||||
"assistant message cannot mix reasoning_content with ordered "
|
||||
"thinking parts"
|
||||
)
|
||||
parts.extend(content)
|
||||
return parts
|
||||
|
||||
if role == "assistant":
|
||||
for tool_call in message.get("tool_calls") or []:
|
||||
name, args = _tool_call_name_and_args(tool_call)
|
||||
tool_call_id = _as_mapping(tool_call).get("id")
|
||||
if tool_call_id:
|
||||
tool_call_id_to_name[str(tool_call_id)] = name
|
||||
_append_message(
|
||||
input_ids,
|
||||
tokenizer,
|
||||
"assistant",
|
||||
"invoke_tool_json",
|
||||
_tool_call_json(name, args),
|
||||
author_name=name,
|
||||
)
|
||||
if len(input_ids) > turn_start:
|
||||
# Close the historical model turn — but never emit a bare
|
||||
# terminator for an assistant message that rendered no blocks.
|
||||
input_ids.append(tokenizer.encode_special(CONTENT_MODEL_END_SAMPLING))
|
||||
|
||||
if leading_system_count == len(message_list):
|
||||
append_effort()
|
||||
def _to_oss_part(
|
||||
part: Any, media_marker: str, media_kinds: list[str]
|
||||
) -> dict[str, Any]:
|
||||
if isinstance(part, str):
|
||||
return {"type": "text", "text": part}
|
||||
if not isinstance(part, Mapping):
|
||||
raise TypeError(f"content part must be mapping, got {type(part).__name__}")
|
||||
part_type = part.get("type")
|
||||
if part_type in _TEXT_PART_TYPES:
|
||||
return {"type": "text", "text": part.get("text") or ""}
|
||||
if part_type in _THINKING_PART_TYPES:
|
||||
text = part.get("thinking")
|
||||
if text is None:
|
||||
text = part.get("text", "")
|
||||
if not isinstance(text, str):
|
||||
raise TypeError("Inkling thinking part payload must be a string")
|
||||
return {"type": "thinking", "thinking": text}
|
||||
if part_type in _IMAGE_PART_TYPES or part_type in _AUDIO_PART_TYPES:
|
||||
media_kinds.append("image" if part_type in _IMAGE_PART_TYPES else "audio")
|
||||
return {"type": "text", "text": f"{media_marker}{len(media_kinds) - 1}"}
|
||||
raise ValueError(f"unsupported content part type: {part_type!r}")
|
||||
|
||||
if add_generation_prompt:
|
||||
input_ids.append(tokenizer.encode_special(MESSAGE_MODEL))
|
||||
|
||||
def _resolve_media(message: Any, chat: Any, media_marker: str, media_kinds: list[str]):
|
||||
# Media bytes are encoded later by the MM processor, so each media part
|
||||
# travels through the renderer as a pointer whose span becomes ONE placeholder.
|
||||
content = message.content
|
||||
if not isinstance(content, chat.Text) or not content.text.startswith(media_marker):
|
||||
return message
|
||||
index = int(content.text.removeprefix(media_marker))
|
||||
if media_kinds[index] == "audio" and message.author.kind != chat.AuthorKind.User:
|
||||
raise ValueError("Inkling audio input is only supported in user messages")
|
||||
prefix = (
|
||||
_IMAGE_LOCATION_PREFIX
|
||||
if media_kinds[index] == "image"
|
||||
else _AUDIO_LOCATION_PREFIX
|
||||
)
|
||||
pointer = chat.ImagePointer(f"{prefix}{index}", chat.ImageFormat.Png, 1, 1)
|
||||
return message.copy(content=pointer)
|
||||
|
||||
|
||||
def _leading_system_count(messages: Sequence[Mapping[str, Any]]) -> int:
|
||||
count = 0
|
||||
while count < len(messages) and messages[count].get("role") == "system":
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def _tool_declare_message(tools: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
||||
declared = []
|
||||
for tool in tools:
|
||||
function = tool.get("function") or {}
|
||||
declared.append(
|
||||
{
|
||||
"type": tool.get("type") or "function",
|
||||
"function": {
|
||||
key: value for key, value in function.items() if value is not None
|
||||
},
|
||||
}
|
||||
)
|
||||
return {
|
||||
"role": "tool_declare",
|
||||
"content": json.dumps(declared, ensure_ascii=False, separators=(",", ":")),
|
||||
}
|
||||
|
||||
|
||||
def _spans_to_input_ids(spans: Sequence[Any], tml: SimpleNamespace) -> list[int]:
|
||||
input_ids: list[int] = []
|
||||
for wrapped in spans:
|
||||
span = wrapped.span
|
||||
if isinstance(span, tml.chat.EncodedTextTokenSpan):
|
||||
input_ids.extend(span.tokens)
|
||||
elif isinstance(span, tml.chat.ImageAssetPointerTokenSpan):
|
||||
_append_media_placeholder(input_ids, span.location, tml.tokenizer)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"unexpected TMLv0 span in Inkling prompt: {type(span).__name__}"
|
||||
)
|
||||
return input_ids
|
||||
|
||||
|
||||
def _append_message(
|
||||
input_ids: list[int],
|
||||
tokenizer: InklingTextTokenizer,
|
||||
role: str,
|
||||
kind: str,
|
||||
text: str,
|
||||
*,
|
||||
author_name: str | None = None,
|
||||
def _append_media_placeholder(
|
||||
input_ids: list[int], location: str, tokenizer: Any
|
||||
) -> None:
|
||||
input_ids.append(tokenizer.encode_special(ROLE_MESSAGE_TOKENS[role]))
|
||||
if author_name:
|
||||
input_ids.extend(tokenizer.encode_text(author_name))
|
||||
|
||||
if kind == "text":
|
||||
input_ids.append(tokenizer.encode_special(CONTENT_TEXT))
|
||||
input_ids.extend(tokenizer.encode_text(text))
|
||||
elif kind == "image":
|
||||
input_ids.append(tokenizer.encode_special(CONTENT_IMAGE))
|
||||
if location.startswith(_IMAGE_LOCATION_PREFIX):
|
||||
input_ids.append(IMAGE_TOKEN_ID)
|
||||
elif kind == "audio":
|
||||
input_ids.append(tokenizer.encode_special(CONTENT_AUDIO_INPUT))
|
||||
input_ids.append(AUDIO_TOKEN_ID)
|
||||
input_ids.append(tokenizer.encode_special(AUDIO_END))
|
||||
elif kind == "thinking":
|
||||
input_ids.append(tokenizer.encode_special(CONTENT_THINKING))
|
||||
input_ids.extend(tokenizer.encode_text(text))
|
||||
elif kind == "xml":
|
||||
input_ids.append(tokenizer.encode_special(CONTENT_XML))
|
||||
input_ids.extend(tokenizer.encode_text(text))
|
||||
elif kind == "invoke_tool_json":
|
||||
input_ids.append(tokenizer.encode_special(CONTENT_INVOKE_TOOL_JSON))
|
||||
input_ids.extend(tokenizer.encode_text(text))
|
||||
else:
|
||||
raise ValueError(f"unsupported Inkling render part kind: {kind!r}")
|
||||
|
||||
input_ids.append(tokenizer.encode_special(END_MESSAGE))
|
||||
|
||||
|
||||
def _iter_render_parts(content: Any):
|
||||
"""Yield ordered ``(kind, text)`` pairs from message content."""
|
||||
if content is None:
|
||||
return
|
||||
if isinstance(content, str):
|
||||
if content:
|
||||
yield ("text", content)
|
||||
return
|
||||
if not isinstance(content, Sequence) or isinstance(content, (bytes, bytearray)):
|
||||
raise TypeError("message content must be a string or a sequence of parts")
|
||||
for part in content:
|
||||
if isinstance(part, str):
|
||||
yield ("text", part)
|
||||
continue
|
||||
if not isinstance(part, Mapping):
|
||||
raise TypeError(f"content part must be mapping, got {type(part).__name__}")
|
||||
ptype = part.get("type")
|
||||
if ptype in (None, "text", "input_text"):
|
||||
text = part.get("text", "")
|
||||
yield ("text", text if isinstance(text, str) else "")
|
||||
elif ptype in ("thinking", "reasoning"):
|
||||
text = part.get("thinking")
|
||||
if text is None:
|
||||
text = part.get("text", "")
|
||||
if not isinstance(text, str):
|
||||
raise TypeError("Inkling thinking part payload must be a string")
|
||||
yield ("thinking", text)
|
||||
elif ptype in _IMAGE_PART_TYPES:
|
||||
yield ("image", "")
|
||||
elif ptype in _AUDIO_PART_TYPES:
|
||||
yield ("audio", "")
|
||||
else:
|
||||
raise ValueError(f"unsupported content part type: {ptype!r}")
|
||||
|
||||
|
||||
def _format_reasoning_effort(reasoning_effort: float) -> str:
|
||||
if isinstance(reasoning_effort, bool) or not isinstance(
|
||||
reasoning_effort, (int, float)
|
||||
):
|
||||
raise TypeError("Inkling reasoning_effort must be a number")
|
||||
value = float(reasoning_effort)
|
||||
if not math.isfinite(value) or not 0.0 <= value <= 0.99:
|
||||
raise ValueError("Inkling reasoning_effort must be finite and in [0.0, 0.99]")
|
||||
return f"{round(value, 2):g}"
|
||||
|
||||
|
||||
def _expect_role(message: Mapping[str, Any]) -> str:
|
||||
role = message.get("role")
|
||||
if role not in ROLE_MESSAGE_TOKENS:
|
||||
raise ValueError(
|
||||
f"unsupported Inkling message role {role!r}; expected one of {sorted(ROLE_MESSAGE_TOKENS)}"
|
||||
)
|
||||
return str(role)
|
||||
|
||||
|
||||
def _as_mapping(value: Any) -> Mapping[str, Any]:
|
||||
if isinstance(value, Mapping):
|
||||
return value
|
||||
if hasattr(value, "model_dump"):
|
||||
dumped = value.model_dump()
|
||||
if isinstance(dumped, Mapping):
|
||||
return dumped
|
||||
raise TypeError(f"expected mapping, got {type(value).__name__}")
|
||||
|
||||
|
||||
def _canonical_json(value: Any) -> str:
|
||||
return json.dumps(
|
||||
_sort_json(value),
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
separators=(",", ":"),
|
||||
# tml-renderers DMel-encodes audio at render time, so audio rides in an image
|
||||
# slot and gets the MM processor's audio framing here.
|
||||
if input_ids[-1] != tokenizer.encode_special(_special_name(CONTENT_IMAGE)):
|
||||
raise ValueError("Inkling audio placeholder is not framed as a content block")
|
||||
input_ids[-1] = tokenizer.encode_special(_special_name(CONTENT_AUDIO_INPUT))
|
||||
input_ids.extend(
|
||||
[AUDIO_TOKEN_ID, tokenizer.encode_special(_special_name(AUDIO_END))]
|
||||
)
|
||||
|
||||
|
||||
def _sort_json(value: Any) -> Any:
|
||||
if isinstance(value, Mapping):
|
||||
return {str(key): _sort_json(value[key]) for key in sorted(value)}
|
||||
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
||||
return [_sort_json(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _tool_declare_json(tools: Sequence[Mapping[str, Any]]) -> str:
|
||||
tool_specs = []
|
||||
for tool_value in tools:
|
||||
tool = _as_mapping(tool_value)
|
||||
function = _as_mapping(tool.get("function", {}))
|
||||
tool_specs.append(
|
||||
{
|
||||
"description": function.get("description") or "",
|
||||
"name": function["name"],
|
||||
"parameters": function.get("parameters") or {},
|
||||
"type": tool.get("type", "function"),
|
||||
}
|
||||
)
|
||||
return _canonical_json(tool_specs)
|
||||
|
||||
|
||||
def _tool_call_name_and_args(tool_call_value: Any) -> tuple[str, Mapping[str, Any]]:
|
||||
tool_call = _as_mapping(tool_call_value)
|
||||
function = _as_mapping(tool_call.get("function", {}))
|
||||
name = function.get("name")
|
||||
if not isinstance(name, str):
|
||||
raise TypeError("tool call function name must be a string")
|
||||
|
||||
raw_args = function.get("arguments") or {}
|
||||
if isinstance(raw_args, str):
|
||||
args = json.loads(raw_args) if raw_args else {}
|
||||
else:
|
||||
args = raw_args
|
||||
if not isinstance(args, Mapping):
|
||||
raise TypeError("tool call function arguments must decode to an object")
|
||||
return name, args
|
||||
|
||||
|
||||
def _tool_call_json(name: str, args: Mapping[str, Any]) -> str:
|
||||
name_json = json.dumps(name, ensure_ascii=False, allow_nan=False)
|
||||
return f'{{"name":{name_json},"args":{_canonical_json(args)}}}'
|
||||
def _special_name(token: str) -> str:
|
||||
return token.removeprefix("<|").removesuffix("|>")
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
END_OF_TEXT = "<|endoftext|>"
|
||||
MESSAGE_USER = "<|message_user|>"
|
||||
MESSAGE_MODEL = "<|message_model|>"
|
||||
@@ -56,50 +52,3 @@ INKLING_CONTROL_TOKENS: frozenset[str] = frozenset(
|
||||
"<|model_trigger_generation|>",
|
||||
}
|
||||
)
|
||||
|
||||
INKLING_SPECIAL_TOKEN_NAMES: dict[str, str] = {
|
||||
token.removeprefix("<|").removesuffix("|>"): token
|
||||
for token in INKLING_SPECIAL_TOKENS
|
||||
}
|
||||
|
||||
ROLE_MESSAGE_TOKENS: dict[str, str] = {
|
||||
"user": MESSAGE_USER,
|
||||
"assistant": MESSAGE_MODEL,
|
||||
"system": MESSAGE_SYSTEM,
|
||||
"tool": MESSAGE_TOOL,
|
||||
}
|
||||
|
||||
|
||||
def normalize_special_token(token: str) -> str:
|
||||
"""Accept either message_user or <|message_user|> spellings."""
|
||||
if token in INKLING_SPECIAL_TOKENS:
|
||||
return token
|
||||
try:
|
||||
return INKLING_SPECIAL_TOKEN_NAMES[token]
|
||||
except KeyError as exc:
|
||||
raise KeyError(f"unknown Inkling special token: {token!r}") from exc
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InklingTokenizer:
|
||||
"""Small wrapper around a base text tokenizer plus Inkling framing IDs.
|
||||
|
||||
Plain text is encoded by the base tokenizer, while the minimal chat
|
||||
framing tokens are inserted from the fixed overlay map.
|
||||
"""
|
||||
|
||||
tokenizer: Any
|
||||
special_token_ids: Mapping[str, int] | None = None
|
||||
|
||||
def encode_text(self, text: str) -> list[int]:
|
||||
if not isinstance(text, str):
|
||||
raise TypeError(f"text must be str, got {type(text).__name__}")
|
||||
return list(self.tokenizer.encode(text, add_special_tokens=False))
|
||||
|
||||
def encode_special(self, token: str) -> int:
|
||||
special = normalize_special_token(token)
|
||||
token_ids = self.special_token_ids or INKLING_SPECIAL_TOKEN_IDS
|
||||
return int(token_ids[special])
|
||||
|
||||
def decode(self, token_ids: list[int]) -> str:
|
||||
return self.tokenizer.decode(token_ids)
|
||||
|
||||
Reference in New Issue
Block a user