[Feat] Inkling: render and parse chat via tml-renderers (#45340)

This commit is contained in:
Xinyuan Tong
2026-10-23 16:43:38 +08:00
committed by GitHub
parent bb72b47daf
commit 8b9f72417f
16 changed files with 3149 additions and 970 deletions
+1
View File
@@ -83,6 +83,7 @@ dependencies = [
"tiktoken",
"tilelang==0.1.12",
"timm==1.0.16",
"tml-renderers==0.1.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or (sys_platform == 'darwin' and platform_machine == 'arm64')",
"tokenizers==0.23.2",
"torch==2.14.1",
"torch_memory_saver>=0.0.9.post1",
@@ -250,13 +250,8 @@ def encode_simple_chat(
"""
if spec == "inkling":
from sglang.srt.parser.inkling_renderer import render_inkling_messages
from sglang.srt.parser.inkling_tokenizer import InklingTokenizer
return render_inkling_messages(
messages,
InklingTokenizer(tokenizer=tokenizer),
add_generation_prompt=False,
)
return render_inkling_messages(messages)
if spec in ("dsv4", "dsv32", "dsv41"):
if spec != "dsv41" and messages and messages[0]["role"] != "system":
@@ -119,7 +119,12 @@ class ChatEncodingAdapterContext(msgspec.Struct, frozen=True, kw_only=True):
def _build_inkling(ctx: ChatEncodingAdapterContext) -> ChatEncodingAdapter:
from sglang.srt.entrypoints.openai.serving_inkling import InklingChatEncodingAdapter
return InklingChatEncodingAdapter(tokenizer=ctx.tokenizer)
return InklingChatEncodingAdapter(
tokenizer=ctx.tokenizer,
reasoning_parser=ctx.reasoning_parser,
tool_call_parser=ctx.tool_call_parser,
chat_hooks=ctx.chat_hooks,
)
_BUILDERS: dict[
@@ -3,7 +3,6 @@ from __future__ import annotations
import copy
import json
import logging
import math
import time
import uuid
from collections import OrderedDict
@@ -687,63 +686,6 @@ class OpenAIServingChat(OpenAIServingBase):
return prompt_ids
return None
@staticmethod
def _pop_inkling_assistant_prefix(
messages: list[dict[str, Any]],
request: ChatCompletionRequest,
) -> str | None:
"""Extract the trailing assistant text for ``continue_final_message``.
Only a plain-string assistant message with no tool calls and no
reasoning content can be continued; anything else renders as a closed
historical turn. Mutates ``messages`` in place (callers pass a copy).
"""
if not request.continue_final_message or not messages:
return None
last = messages[-1]
if (
last.get("role") != "assistant"
or not isinstance(last.get("content"), str)
or last.get("tool_calls")
or last.get("reasoning_content")
):
return None
messages.pop()
return last["content"]
@staticmethod
def _parse_inkling_reasoning_effort(
value: str | float | None,
) -> float | None:
"""Convert an OpenAI-style reasoning_effort to an Inkling float."""
if value is None:
return None
if isinstance(value, bool):
raise ValueError("Inkling reasoning_effort must not be a boolean")
if isinstance(value, (int, float)):
parsed = float(value)
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
raise ValueError("Inkling reasoning_effort must be in [0.0, 0.99]")
return parsed
_EFFORT_MAP = {
"none": 0.0,
"minimal": 0.1,
"low": 0.2,
"medium": 0.7,
"high": 0.9,
"xhigh": 0.99,
"max": 0.99,
}
if value in _EFFORT_MAP:
return _EFFORT_MAP[value]
try:
parsed = float(value)
except (ValueError, TypeError) as exc:
raise ValueError(f"invalid Inkling reasoning_effort: {value!r}") from exc
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
raise ValueError("Inkling reasoning_effort must be in [0.0, 0.99]")
return parsed
def _resolve_dsv41_reasoning_effort(self, value: Any) -> Union[str, int]:
"""Request effort for the V4.1 encoder; unsupported values warn and fall back."""
effort = chat_encoding.parse_dsv41_reasoning_effort(value)
@@ -759,26 +701,6 @@ class OpenAIServingChat(OpenAIServingBase):
)
return self._dsv41_default_reasoning_effort
@staticmethod
def _get_inkling_default_reasoning_effort() -> float:
"""Read the default Inkling reasoning effort from the environment."""
from sglang.srt.environ import envs
val = envs.SGLANG_INKLING_DEFAULT_REASONING_EFFORT.get()
if not val:
return 0.9
try:
parsed = float(val)
except (ValueError, TypeError) as exc:
raise ValueError(
"SGLANG_INKLING_DEFAULT_REASONING_EFFORT must be numeric"
) from exc
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
raise ValueError(
"SGLANG_INKLING_DEFAULT_REASONING_EFFORT must be in [0.0, 0.99]"
)
return parsed
def _decode_response(self, ret_item: dict[str, Any]) -> str | ErrorResponse:
"""Extract text from response."""
return ret_item["text"]
@@ -1,67 +1,390 @@
"""Inkling serving glue: prompt rendering and token-level output parsing for the
Chat Completions and Responses APIs."""
from __future__ import annotations
import math
from typing import TYPE_CHECKING, Any, Optional
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
from sglang.srt.entrypoints.openai.protocol import (
ChatCompletionRequest,
ChatCompletionResponseStreamChoice,
ChatCompletionStreamResponse,
DeltaMessage,
FunctionResponse,
ToolCall,
)
from sglang.srt.entrypoints.openai.sse_utils import build_sse_content
from sglang.srt.function_call.core_types import ToolCallItem
from sglang.srt.parser.inkling_output import (
InklingOutputDelta,
InklingOutputParser,
InklingToolCall,
)
from sglang.srt.parser.inkling_renderer import (
load_tml_renderers,
render_inkling_assistant_prefix,
render_inkling_messages,
)
from sglang.srt.parser.inkling_tokenizer import INKLING_SPECIAL_TOKEN_IDS
from sglang.srt.runtime_context import get_serving
if TYPE_CHECKING:
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
from sglang.srt.entrypoints.openai.chat_encoding_adapters import ChatOutputHooks
class InklingChatEncodingAdapter:
"""Inkling prompt encoding through ``inkling_renderer``; output stays with
the registered ``inkling`` reasoning and tool-call parsers."""
collects_openai_media = True
output = None
def __init__(self, *, tokenizer: Any):
self._tokenizer = tokenizer
# Resolved at boot: a misconfigured env default fails startup, not requests.
self._default_reasoning_effort = (
OpenAIServingChat._get_inkling_default_reasoning_effort()
)
def __init__(
self,
*,
tokenizer: Any,
reasoning_parser: str | None,
tool_call_parser: str | None,
chat_hooks: ChatOutputHooks,
):
# Validate the frozen effort default at startup,
# so configuration errors fail before requests.
self._default_reasoning_effort = get_default_reasoning_effort()
load_tml_renderers()
if tokenizer is not None:
_check_served_tokenizer(tokenizer)
parsers = {reasoning_parser, tool_call_parser} - {None}
if "inkling" in parsers and parsers != {"inkling"}:
# The token parser emits Inkling calls while constraints would come
# from the other parser's format.
raise ValueError(
"Inkling token parsing requires --reasoning-parser and "
"--tool-call-parser to be inkling or unset, got "
f"{reasoning_parser!r} and {tool_call_parser!r}"
)
self._token_output = "inkling" in parsers
self._separate_reasoning = bool(reasoning_parser)
self._hooks = chat_hooks
@property
def output(self) -> Optional[InklingChatEncodingAdapter]:
return self if self._token_output else None
def encode_messages(
self,
*,
messages: list[dict[str, Any]],
request: ChatCompletionRequest,
tools: Optional[list[dict]],
tools: list[dict] | None,
) -> list[int]:
from sglang.srt.parser.inkling_renderer import render_inkling_messages
from sglang.srt.parser.inkling_tokenizer import (
CONTENT_TEXT,
MESSAGE_MODEL,
InklingTokenizer,
)
# Framing tokens + ONE placeholder per media (encoding/expansion happens later
# in InklingMultimodalProcessor). The server's tokenizer is the base tiktoken
# backend; wrap it so encode_special supplies the framing-token overlay.
inkling_tokenizer = InklingTokenizer(tokenizer=self._tokenizer)
reasoning_effort = OpenAIServingChat._parse_inkling_reasoning_effort(
request.reasoning_effort
)
reasoning_effort = parse_reasoning_effort(request.reasoning_effort)
if reasoning_effort is None:
reasoning_effort = self._default_reasoning_effort
assistant_prefix = OpenAIServingChat._pop_inkling_assistant_prefix(
messages, request
)
assistant_prefix = _pop_assistant_prefix(messages, request)
prompt_ids = render_inkling_messages(
messages,
inkling_tokenizer,
add_generation_prompt=False,
tools=tools,
reasoning_effort=reasoning_effort,
messages, tools=tools, reasoning_effort=reasoning_effort
)
if assistant_prefix is not None:
# Continue the final assistant message inside an OPEN model text
# block: header + payload, no <|end_message|> and no
# <|content_model_end_sampling|>, so the model resumes the turn.
prompt_ids += [
inkling_tokenizer.encode_special(MESSAGE_MODEL),
inkling_tokenizer.encode_special(CONTENT_TEXT),
*inkling_tokenizer.encode_text(assistant_prefix),
]
prompt_ids += render_inkling_assistant_prefix(assistant_prefix)
return prompt_ids
def parse_chat_response(
self,
*,
request: ChatCompletionRequest,
output_ids: list[int],
finish_reason: dict[str, Any],
) -> tuple[str | None, str, list[ToolCall] | None, dict[str, Any]]:
parsed = self._new_output_parser(request).finish(
output_ids,
matched_stop=finish_reason.get("matched"),
keep_matched_stop=request.no_stop_trim,
)
if not parsed.tool_calls:
return parsed.reasoning or None, parsed.content, None, finish_reason
history_tool_calls_cnt = self._hooks.history_tool_calls_count(request)
tool_calls = [
self._tool_call(call, history_tool_calls_cnt) for call in parsed.tool_calls
]
if finish_reason["type"] == "stop":
finish_reason = {**finish_reason, "type": "tool_calls", "matched": None}
return parsed.reasoning or None, parsed.content, tool_calls, finish_reason
def stream_chat_chunks(
self,
*,
content: dict[str, Any],
index: int,
request: ChatCompletionRequest,
created: int,
parser_dict: dict,
has_tool_calls: dict[int, bool],
choice_logprobs: dict | None,
finish_reason_type: str | None,
usage: dict[str, Any] | None,
) -> list[str]:
if index not in parser_dict:
parser_dict[index] = self._new_output_parser(request)
parser = parser_dict[index]
new_output_ids = select_new_output_ids(
output_ids=content["output_ids"],
num_consumed_tokens=parser.num_consumed_tokens,
completion_tokens=content["meta_info"]["completion_tokens"],
finish_reason_type=finish_reason_type,
incremental=get_serving().incremental_streaming_output,
)
if finish_reason_type is None:
delta = parser.feed(new_output_ids)
else:
delta = parser.finish(
new_output_ids,
matched_stop=content["meta_info"]["finish_reason"].get("matched"),
keep_matched_stop=request.no_stop_trim,
)
chunk_fields = dict(
chunk_id=content["meta_info"]["id"],
created=created,
model=request.model,
index=index,
usage=usage,
)
chunks = []
remaining_logprobs = choice_logprobs
if delta.reasoning:
chunks.append(
build_sse_content(
reasoning_content=delta.reasoning,
logprobs=remaining_logprobs,
**chunk_fields,
)
)
remaining_logprobs = None
if delta.content:
chunks.append(
build_sse_content(
content=delta.content, logprobs=remaining_logprobs, **chunk_fields
)
)
remaining_logprobs = None
history_tool_calls_cnt = self._hooks.history_tool_calls_count(request)
for call in delta.tool_calls:
has_tool_calls[index] = True
tool_call_chunk = ChatCompletionStreamResponse(
id=content["meta_info"]["id"],
created=chunk_fields["created"],
choices=[
ChatCompletionResponseStreamChoice(
index=index,
delta=DeltaMessage(
tool_calls=[self._tool_call(call, history_tool_calls_cnt)]
),
finish_reason=None,
)
],
model=request.model,
usage=usage,
)
chunks.append(f"data: {tool_call_chunk.model_dump_json()}\n\n")
if remaining_logprobs is not None:
chunks.append(
build_sse_content(logprobs=remaining_logprobs, **chunk_fields)
)
return chunks
def parse_responses_output(
self,
*,
output_ids: list[int],
finish_reason: dict[str, Any] | None,
separate_reasoning: bool,
parse_tool_calls: bool,
) -> tuple[str, str, list[ToolCallItem]]:
parser = InklingOutputParser(
separate_reasoning=separate_reasoning, parse_tool_calls=parse_tool_calls
)
delta = parser.finish(
output_ids,
matched_stop=(finish_reason or {}).get("matched"),
keep_matched_stop=False,
)
return delta.reasoning, delta.content, _tool_call_items(delta)
def new_responses_stream(
self, *, separate_reasoning: bool, parse_tool_calls: bool
) -> InklingResponsesStream:
return InklingResponsesStream(
separate_reasoning=separate_reasoning, parse_tool_calls=parse_tool_calls
)
def _new_output_parser(self, request: ChatCompletionRequest) -> InklingOutputParser:
return InklingOutputParser(
separate_reasoning=self._separate_reasoning and request.separate_reasoning,
parse_tool_calls=self._hooks.tool_call_parsing_active(request),
stream_reasoning=request.stream_reasoning,
continues_text_block=bool(
request.continue_final_message
and request.messages
and _is_continuable_message(request.messages[-1].model_dump())
),
)
def _tool_call(
self, call: InklingToolCall, history_tool_calls_cnt: int
) -> ToolCall:
call_item = ToolCallItem(
tool_index=call.index, name=call.name, parameters=call.arguments
)
return ToolCall(
id=self._hooks.tool_call_id(call_item, history_tool_calls_cnt),
index=call.index,
function=FunctionResponse(name=call.name, arguments=call.arguments),
)
class InklingResponsesStream:
def __init__(self, *, separate_reasoning: bool, parse_tool_calls: bool):
self._parser = InklingOutputParser(
separate_reasoning=separate_reasoning, parse_tool_calls=parse_tool_calls
)
self._incremental = get_serving().incremental_streaming_output
def step(
self, chunk: dict[str, Any], finish_reason: dict[str, Any] | None
) -> tuple[str, str, list[ToolCallItem]]:
new_output_ids = select_new_output_ids(
output_ids=chunk.get("output_ids") or [],
num_consumed_tokens=self._parser.num_consumed_tokens,
completion_tokens=chunk["meta_info"]["completion_tokens"],
finish_reason_type=finish_reason["type"] if finish_reason else None,
incremental=self._incremental,
)
if finish_reason is None:
delta = self._parser.feed(new_output_ids)
else:
delta = self._parser.finish(
new_output_ids,
matched_stop=finish_reason.get("matched"),
keep_matched_stop=False,
)
return delta.reasoning, delta.content, _tool_call_items(delta)
def _tool_call_items(delta: InklingOutputDelta) -> list[ToolCallItem]:
return [
ToolCallItem(tool_index=call.index, name=call.name, parameters=call.arguments)
for call in delta.tool_calls
]
def select_new_output_ids(
*,
output_ids: list[int],
num_consumed_tokens: int,
completion_tokens: int,
finish_reason_type: str | None,
incremental: bool,
) -> list[int]:
if not incremental:
# Intermediate chunks share the live output list; completion_tokens is
# this chunk's snapshot, so later tokens wait for their own chunk.
return output_ids[num_consumed_tokens:completion_tokens]
if finish_reason_type == "abort":
# The abort chunk re-sends already-streamed tokens.
return output_ids[: max(completion_tokens - num_consumed_tokens, 0)]
return output_ids
_TOKENIZER_PROBE = "Hello, 世界! def f(x): return x**2"
def _check_served_tokenizer(tokenizer: Any) -> None:
# Prompts and outputs use tml-renderers' tokenizer, while the scheduler matches
# stop strings and detokenizes with the served one; they must agree.
tml_tokenizer = load_tml_renderers().tokenizer
# encode() is the lookup both the HF and the tiktoken tokenizer wrappers share.
mismatched = [
token
for token in INKLING_SPECIAL_TOKEN_IDS
if tokenizer.encode(token, add_special_tokens=False)
!= [tml_tokenizer.encode_special(token[2:-2])]
]
if tokenizer.encode(_TOKENIZER_PROBE, add_special_tokens=False) != list(
tml_tokenizer.encode_ordinary(_TOKENIZER_PROBE)
):
mismatched.append("ordinary text")
if mismatched:
raise ValueError(
"The served tokenizer disagrees with tml-renderers' o200k_base_chat on "
f"{', '.join(mismatched)}; Inkling prompts and outputs would be misencoded."
)
def _pop_assistant_prefix(
messages: list[dict[str, Any]],
request: ChatCompletionRequest,
) -> str | None:
"""Pops only a continuable text turn; callers pass a message-list copy."""
if not request.continue_final_message or not messages:
return None
if not _is_continuable_message(messages[-1]):
return None
return messages.pop()["content"]
def _is_continuable_message(message: dict[str, Any]) -> bool:
return (
message.get("role") == "assistant"
and isinstance(message.get("content"), str)
and not message.get("tool_calls")
and not message.get("reasoning_content")
)
def parse_reasoning_effort(
value: str | float | None,
) -> float | None:
if value is None:
return None
if isinstance(value, bool):
raise ValueError("Inkling reasoning_effort must not be a boolean")
if isinstance(value, (int, float)):
parsed = float(value)
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
raise ValueError("Inkling reasoning_effort must be in [0.0, 0.99]")
return parsed
_EFFORT_MAP = {
"none": 0.0,
"minimal": 0.1,
"low": 0.2,
"medium": 0.7,
"high": 0.9,
"xhigh": 0.99,
"max": 0.99,
}
if value in _EFFORT_MAP:
return _EFFORT_MAP[value]
try:
parsed = float(value)
except (ValueError, TypeError) as exc:
raise ValueError(f"invalid Inkling reasoning_effort: {value!r}") from exc
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
raise ValueError("Inkling reasoning_effort must be in [0.0, 0.99]")
return parsed
def get_default_reasoning_effort() -> float:
from sglang.srt.environ import envs
val = envs.SGLANG_INKLING_DEFAULT_REASONING_EFFORT.get()
if not val:
return 0.9
try:
parsed = float(val)
except (ValueError, TypeError) as exc:
raise ValueError(
"SGLANG_INKLING_DEFAULT_REASONING_EFFORT must be numeric"
) from exc
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
raise ValueError(
"SGLANG_INKLING_DEFAULT_REASONING_EFFORT must be in [0.0, 0.99]"
)
return parsed
@@ -1224,6 +1224,20 @@ class OpenAIServingResponses(OpenAIServingChat):
def _tool_parser_owns_format(parser: FunctionCallParser) -> bool:
return parser.owns_tool_format()
@staticmethod
def _drain_json_array(
parser: JsonArrayParser, delta: str, tools: list[Tool]
) -> tuple[str, list[ToolCallItem]]:
# Each call emits one name or one argument diff, so a chunk that holds
# a whole call needs more calls; the stream may end with no more chunks.
result = parser.parse_streaming_increment(delta, tools)
normal_text, calls = result.normal_text or "", list(result.calls)
while result.calls:
result = parser.parse_streaming_increment("", tools)
normal_text += result.normal_text or ""
calls.extend(result.calls)
return normal_text, calls
def _uses_required_tool_json(
self, request: ResponsesRequest, chat_tools: list[Tool]
) -> bool:
@@ -2510,8 +2524,9 @@ class OpenAIServingResponses(OpenAIServingChat):
continue
if isinstance(tool_parser, JsonArrayParser):
sp = tool_parser.parse_streaming_increment(delta, chat_tools)
normal_text, tool_calls = sp.normal_text or "", sp.calls
normal_text, tool_calls = self._drain_json_array(
tool_parser, delta, chat_tools
)
elif token_stream is not None:
normal_text, tool_calls = delta, token_calls
elif tool_parser is not None:
+312
View File
@@ -0,0 +1,312 @@
from __future__ import annotations
import json
import logging
from collections.abc import Sequence
from typing import Any
import msgspec
from sglang.srt.parser.inkling_renderer import load_tml_renderers
logger = logging.getLogger(__name__)
_CONTENT_KIND_NAMES = (
"content_text",
"content_thinking",
"content_xml",
"content_invoke_tool_json",
"content_invoke_tool_text",
"content_tool_error",
"content_image",
"content_audio_input",
)
class InklingToolCall(msgspec.Struct, frozen=True):
index: int
name: str
arguments: str
class InklingOutputDelta(msgspec.Struct, frozen=True):
reasoning: str = ""
content: str = ""
tool_calls: tuple[InklingToolCall, ...] = ()
def merge(self, other: InklingOutputDelta) -> InklingOutputDelta:
return InklingOutputDelta(
reasoning=self.reasoning + other.reasoning,
content=self.content + other.content,
tool_calls=self.tool_calls + other.tool_calls,
)
class InklingOutputParser:
"""Control tokens are identified by ID; ordinary tokens spelling a marker remain text."""
def __init__(
self,
*,
separate_reasoning: bool,
parse_tool_calls: bool,
stream_reasoning: bool = True,
continues_text_block: bool = False,
):
self._tml = load_tml_renderers()
self._separate_reasoning = separate_reasoning
self._hold_reasoning = separate_reasoning and not stream_reasoning
self._parse_tool_calls = parse_tool_calls
_spans, self._parser = self._tml.renderer.render_for_completion([])
tokenizer = self._tml.tokenizer
self._content_kind_ids = frozenset(
tokenizer.encode_special(name) for name in _CONTENT_KIND_NAMES
)
self._text_opener_ids = (
tokenizer.encode_special("message_model"),
tokenizer.encode_special("content_text"),
)
self._end_message_id = tokenizer.encode_special("end_message")
self._message_ids: list[int] = []
self._streamed = ""
self._held_reasoning = ""
self._model_authored = True
self._at_message_boundary = True
self._in_unframed_text = False
self._num_tool_calls = 0
self._finished = False
self.num_consumed_tokens = 0
if continues_text_block:
# The prompt ended inside an open model text block, so the sampled
# tokens carry no header; replay the opener so they parse as text.
for token_id in self._text_opener_ids:
self._parse(token_id, _DeltaBuilder())
def feed(self, token_ids: Sequence[int]) -> InklingOutputDelta:
delta = _DeltaBuilder()
for token_id in token_ids:
self.num_consumed_tokens += 1
self._feed_token(token_id, delta)
return delta.build()
def finish(
self,
token_ids: Sequence[int] = (),
*,
matched_stop: int | str | None = None,
keep_matched_stop: bool = False,
) -> InklingOutputDelta:
# Callers see a sticky finish_reason on later chunks; flushing again would
# re-emit an unclosed message's payload.
if self._finished:
return InklingOutputDelta()
self._finished = True
if isinstance(matched_stop, str):
token_ids = self._cut_at_stop_string(
token_ids, matched_stop, keep_stop=keep_matched_stop
)
elif (
not keep_matched_stop
and token_ids
and token_ids[-1] == matched_stop
and matched_stop != self._end_message_id
):
# Inside an open block TML renders even a special stop as text;
# only end_message closes a block without rendering.
token_ids = token_ids[:-1]
return self.feed(token_ids).merge(self._flush())
def _cut_at_stop_string(
self, token_ids: Sequence[int], stop: str, *, keep_stop: bool
) -> list[int]:
# The scheduler matched ``stop`` on the decoded ids, framing included,
# and ended on the token completing it; cut the raw stream there.
tokenizer = self._tml.tokenizer
if not token_ids:
return []
stop_bytes = stop.encode()
pieces = [tokenizer.decode_bytes([token_ids[-1]])]
last_len = region_len = len(pieces[0])
start = len(token_ids) - 1
# A match ending in the last token starts at most len(stop) - 1 bytes before it.
while start > 0 and region_len < last_len + len(stop_bytes) - 1:
start -= 1
pieces.insert(0, tokenizer.decode_bytes([token_ids[start]]))
region_len += len(pieces[0])
cut = b"".join(pieces).find(stop_bytes)
if cut == -1:
return list(token_ids)
if keep_stop:
cut += len(stop_bytes)
kept = list(token_ids[:start])
offset = 0
for token_id, piece in zip(token_ids[start:], pieces):
if offset + len(piece) <= cut:
kept.append(token_id)
elif offset < cut and not tokenizer.is_special_token(token_id):
partial = piece[: cut - offset].decode(errors="ignore")
kept += tokenizer.encode_ordinary(partial)
offset += len(piece)
return kept
def _flush(self) -> InklingOutputDelta:
delta = _DeltaBuilder()
try:
updates = self._parser.flush_updates()
except self._tml.v0.ParseError as exc:
self._recover_message(delta, exc)
return delta.build()
for update in updates:
self._apply(update.update, delta)
self._surface_unclosed_message(delta)
self._release_held_reasoning(delta)
return delta.build()
def _surface_unclosed_message(self, delta: _DeltaBuilder) -> None:
# A message cut off (e.g. by max_tokens) never completes; only text kinds
# streamed, so a call payload would otherwise vanish.
if not self._model_authored or not any(
token_id in self._content_kind_ids for token_id in self._message_ids
):
return
text = self._payload_text()
if text.startswith(self._streamed):
delta.content.append(text[len(self._streamed) :])
def _feed_token(self, token_id: int, delta: _DeltaBuilder) -> None:
is_special = self._tml.tokenizer.is_special_token(token_id)
if self._in_unframed_text and is_special:
# Inside a real text block TML reads most specials as text; the
# sampler never emitted an opener here, so a special is framing.
self._in_unframed_text = False
if token_id != self._end_message_id:
self._parse(self._end_message_id, delta)
elif self._at_message_boundary and not is_special:
# Headerless fallback output is model text; synthesize its missing opener.
for opener_id in self._text_opener_ids:
self._parse(opener_id, delta)
self._in_unframed_text = True
self._parse(token_id, delta)
def _parse(self, token_id: int, delta: _DeltaBuilder) -> None:
self._at_message_boundary = False
self._message_ids.append(token_id)
try:
updates = self._parser.parse_token(token_id)
except self._tml.v0.ParseError as exc:
self._recover_message(delta, exc)
return
for update in updates:
self._apply(update.update, delta)
def _apply(self, update: Any, delta: _DeltaBuilder) -> None:
chat = self._tml.chat
if isinstance(update, chat.StreamingMessageHeader):
self._model_authored = update.author.kind == chat.AuthorKind.Model
self._streamed = ""
elif isinstance(update, chat.StreamingContent):
if self._model_authored:
self._emit_text(update.content, update.content.text, delta)
self._streamed += update.content.text
else:
self._complete_message(update, delta)
self._message_ids = []
self._streamed = ""
self._at_message_boundary = True
def _complete_message(self, message: Any, delta: _DeltaBuilder) -> None:
chat = self._tml.chat
content = message.content
if message.author.kind != chat.AuthorKind.Model:
logger.debug("Dropping non-model Inkling output message: %r", message)
elif isinstance(content, (chat.Text, chat.Thinking)):
if content.text.startswith(self._streamed):
self._emit_text(content, content.text[len(self._streamed) :], delta)
self._release_held_reasoning(delta)
elif isinstance(content, chat.InvokeTool):
if self._parse_tool_calls and content.text is None:
delta.tool_calls.append(self._tool_call(message))
self._num_tool_calls += 1
else:
delta.content.append(self._payload_text())
def _emit_text(self, content: Any, text: str, delta: _DeltaBuilder) -> None:
if not (
self._separate_reasoning and isinstance(content, self._tml.chat.Thinking)
):
delta.content.append(text)
elif self._hold_reasoning:
self._held_reasoning += text
else:
delta.reasoning.append(text)
def _release_held_reasoning(self, delta: _DeltaBuilder) -> None:
if self._held_reasoning:
delta.reasoning.append(self._held_reasoning)
self._held_reasoning = ""
def _tool_call(self, message: Any) -> InklingToolCall:
chat = self._tml.chat
oss = chat.OpenAIMessage.to_oss_messages(
chat.OpenAIMessage.from_messages([message])
)
function = oss[0]["tool_calls"][0]["function"]
# The bridge re-serializes args with sorted keys, ASCII escapes and big ints
# as floats; build them from the sampled payload instead.
args = json.loads(self._payload_json())["args"]
return InklingToolCall(
index=self._num_tool_calls,
name=function["name"],
arguments=json.dumps(args, ensure_ascii=False),
)
def _recover_message(self, delta: _DeltaBuilder, exc: Exception) -> None:
logger.warning(
"Inkling output failed TMLv0 parsing; surfacing as text: %s", exc
)
text = self._payload_text()
emitted = "" if self._held_reasoning else self._streamed
if text.startswith(emitted):
delta.content.append(text[len(emitted) :])
self._held_reasoning = ""
self._parser.reset()
self._message_ids = []
self._streamed = ""
self._model_authored = True
self._at_message_boundary = True
self._in_unframed_text = False
def _payload_ids(self) -> list[int]:
ids = self._message_ids
kind_index = next(
(i for i, token_id in enumerate(ids) if token_id in self._content_kind_ids),
-1,
)
return ids[kind_index + 1 :]
def _payload_text(self) -> str:
tokenizer = self._tml.tokenizer
return tokenizer.decode(
[t for t in self._payload_ids() if not tokenizer.is_special_token(t)]
)
def _payload_json(self) -> str:
# A special inside the payload keeps its spelling, as in the library's parse.
ids = self._payload_ids()
if ids and ids[-1] == self._end_message_id:
ids = ids[:-1]
return self._tml.tokenizer.decode(ids)
class _DeltaBuilder:
def __init__(self):
self.reasoning: list[str] = []
self.content: list[str] = []
self.tool_calls: list[InklingToolCall] = []
def build(self) -> InklingOutputDelta:
return InklingOutputDelta(
reasoning="".join(self.reasoning),
content="".join(self.content),
tool_calls=tuple(self.tool_calls),
)
+195 -285
View File
@@ -1,332 +1,242 @@
from __future__ import annotations
import functools
import json
import math
import uuid
from collections.abc import Mapping, Sequence
from typing import Any, Protocol
from types import SimpleNamespace
from typing import Any
from sglang.srt.parser.inkling_tokenizer import (
AUDIO_END,
AUDIO_TOKEN_ID,
CONTENT_AUDIO_INPUT,
CONTENT_IMAGE,
CONTENT_INVOKE_TOOL_JSON,
CONTENT_MODEL_END_SAMPLING,
CONTENT_TEXT,
CONTENT_THINKING,
CONTENT_XML,
END_MESSAGE,
IMAGE_TOKEN_ID,
MESSAGE_MODEL,
ROLE_MESSAGE_TOKENS,
)
INKLING_DEFAULT_REASONING_EFFORT = 0.9
class InklingTextTokenizer(Protocol):
def encode_text(self, text: str) -> list[int]: ...
TML_RENDERERS_INSTALL_HINT = (
"Serving Inkling requires the `tml-renderers` package (TML's reference "
"TMLv0 renderer/parser; wheels for Linux and macOS arm64). Install it with "
"`pip install tml-renderers`."
)
def encode_special(self, token: str) -> int: ...
# OpenAI content-part type spellings that mean image / audio (render only needs the kind,
# not the bytes — the bytes are encoded later in the MM processor).
_IMAGE_PART_TYPES = frozenset({"image", "input_image", "image_url"})
_AUDIO_PART_TYPES = frozenset({"audio", "input_audio", "audio_url"})
INKLING_DEFAULT_REASONING_EFFORT = 0.9
_TEXT_PART_TYPES = frozenset({None, "text", "input_text"})
_THINKING_PART_TYPES = frozenset({"thinking", "reasoning"})
_IMAGE_LOCATION_PREFIX = "sglang-inkling-image://"
_AUDIO_LOCATION_PREFIX = "sglang-inkling-audio://"
@functools.cache
def load_tml_renderers() -> SimpleNamespace:
try:
from tml_renderers import chat, tokenizers, v0
except ImportError as exc:
raise ImportError(TML_RENDERERS_INSTALL_HINT) from exc
tokenizer = tokenizers.o200k_base_chat()
return SimpleNamespace(
chat=chat, v0=v0, tokenizer=tokenizer, renderer=v0.Renderer(tokenizer)
)
def render_inkling_messages(
messages: Sequence[Mapping[str, Any]],
tokenizer: InklingTextTokenizer,
*,
add_generation_prompt: bool = False,
tools: Sequence[Mapping[str, Any]] | None = None,
reasoning_effort: float | None = None,
) -> list[int]:
"""Render chat messages to Inkling input_ids with ONE placeholder per media item.
PURE renderer: emits Inkling framing + a single IMAGE_TOKEN_ID / AUDIO_TOKEN_ID
per image / audio part. Media encoding and 1->N placeholder expansion happen
later in the MM processor. Inkling normally emits its own assistant turn
opener; ``add_generation_prompt`` is retained only for legacy callers. The
conversation-level effort directive is emitted once in the initial prefix
and defaults to 0.9.
"""
input_ids: list[int] = []
tool_call_id_to_name: dict[str, str] = {}
"""Media parts emit one placeholder each for later MM expansion.
Inkling samples its own assistant header."""
tml = load_tml_renderers()
media_marker = f"sglang-inkling-media-{uuid.uuid4().hex}:"
oss_messages, media_kinds = _to_oss_messages(messages, media_marker)
if tools:
_append_message(
input_ids,
tokenizer,
"system",
"xml",
_tool_declare_json(tools),
author_name="tool_declare",
oss_messages.insert(
_leading_system_count(oss_messages), _tool_declare_message(tools)
)
# Normalize the OpenAI "developer" role to "system" (as the Responses API
# does in _normalize_response_message_for_chat) so developer-instruction
# messages render instead of tripping _expect_role; the leading-system
# grouping below then also sees them. Shallow-copy only affected messages so
# the caller's list is left untouched.
message_list = [
{**message, "role": "system"} if message.get("role") == "developer" else message
for message in messages
native = [
_resolve_media(message, tml.chat, media_marker, media_kinds)
for oss_message in tml.chat.OpenAIMessage.from_oss_messages(oss_messages)
for message in oss_message.to_messages()
]
leading_system_count = 0
for message in message_list:
if message.get("role") != "system":
break
leading_system_count += 1
effort = (
INKLING_DEFAULT_REASONING_EFFORT
if reasoning_effort is None
else reasoning_effort
)
spans, _parser = tml.renderer.render_for_completion_with_effort(native, effort)
return _spans_to_input_ids(spans, tml)
def append_effort() -> None:
effort = (
INKLING_DEFAULT_REASONING_EFFORT
if reasoning_effort is None
else reasoning_effort
)
_append_message(
input_ids,
tokenizer,
"system",
"text",
f"Thinking effort level: {_format_reasoning_effort(effort)}",
)
for message_index, message in enumerate(message_list):
if message_index == leading_system_count:
append_effort()
role = _expect_role(message)
if role == "tool":
tool_name = str(
message.get("name")
or tool_call_id_to_name.get(message.get("tool_call_id") or "", "")
def render_inkling_assistant_prefix(prefix: str) -> list[int]:
"""TMLv0 has no prefill; continuation opens a model text block without a terminator."""
tokenizer = load_tml_renderers().tokenizer
return [
tokenizer.encode_special(_special_name(MESSAGE_MODEL)),
tokenizer.encode_special(_special_name(CONTENT_TEXT)),
*tokenizer.encode_ordinary(prefix),
]
def _to_oss_messages(
messages: Sequence[Mapping[str, Any]], media_marker: str
) -> tuple[list[dict[str, Any]], list[str]]:
media_kinds: list[str] = []
oss_messages = []
call_names: dict[str, str] = {}
for message in messages:
oss_message = {
key: value for key, value in message.items() if value is not None
}
if oss_message.get("role") == "developer":
oss_message["role"] = "system"
reasoning = oss_message.pop("reasoning_content", None)
if reasoning and oss_message.get("role") == "assistant":
oss_message["content"] = _with_leading_thinking(
reasoning, oss_message.get("content")
)
# The MM processor harvests media from tool messages, so coercing content
# to one string would drop images and desync the placeholder count.
tool_parts = list(_iter_render_parts(message.get("content", "")))
if not tool_parts:
tool_parts = [("text", "")] # else the answered tool_call dangles
for kind, text in tool_parts:
if kind == "thinking":
raise ValueError("Inkling thinking parts require role='assistant'")
_append_message(
input_ids,
tokenizer,
"tool",
kind,
text,
author_name=tool_name,
)
continue
for call in oss_message.get("tool_calls") or ():
if call.get("id") and call["function"].get("name"):
call_names[call["id"]] = call["function"]["name"]
# The library takes a tool author only from ``name``; spec Chat clients and the
# Responses/Anthropic adapters send just ``tool_call_id``.
if oss_message.get("role") == "tool" and not oss_message.get("name"):
name = call_names.get(oss_message.get("tool_call_id"))
if name:
oss_message["name"] = name
content = oss_message.get("content")
if isinstance(content, Sequence) and not isinstance(
content, (str, bytes, bytearray)
):
oss_message["content"] = [
_to_oss_part(part, media_marker, media_kinds) for part in content
]
oss_messages.append(oss_message)
return oss_messages, media_kinds
parts = list(_iter_render_parts(message.get("content", "")))
turn_start = len(input_ids)
if role == "assistant":
reasoning_content = message.get("reasoning_content")
if reasoning_content:
if not isinstance(reasoning_content, str):
raise TypeError(
"assistant reasoning_content must be a string for Inkling rendering"
)
if any(kind == "thinking" for kind, _ in parts):
raise ValueError(
"assistant message cannot mix reasoning_content with ordered thinking parts"
)
_append_message(
input_ids,
tokenizer,
"assistant",
"thinking",
reasoning_content,
)
for kind, text in parts:
if kind == "thinking" and role != "assistant":
raise ValueError("Inkling thinking parts require role='assistant'")
_append_message(input_ids, tokenizer, role, kind, text)
def _with_leading_thinking(reasoning: str, content: Any) -> list[Any]:
# tml-renderers reads prior reasoning only from thinking content parts.
parts: list[Any] = [{"type": "thinking", "thinking": reasoning}]
if isinstance(content, str):
if content:
parts.append({"type": "text", "text": content})
elif content:
if any(
isinstance(part, Mapping) and part.get("type") in _THINKING_PART_TYPES
for part in content
):
raise ValueError(
"assistant message cannot mix reasoning_content with ordered "
"thinking parts"
)
parts.extend(content)
return parts
if role == "assistant":
for tool_call in message.get("tool_calls") or []:
name, args = _tool_call_name_and_args(tool_call)
tool_call_id = _as_mapping(tool_call).get("id")
if tool_call_id:
tool_call_id_to_name[str(tool_call_id)] = name
_append_message(
input_ids,
tokenizer,
"assistant",
"invoke_tool_json",
_tool_call_json(name, args),
author_name=name,
)
if len(input_ids) > turn_start:
# Close the historical model turn — but never emit a bare
# terminator for an assistant message that rendered no blocks.
input_ids.append(tokenizer.encode_special(CONTENT_MODEL_END_SAMPLING))
if leading_system_count == len(message_list):
append_effort()
def _to_oss_part(
part: Any, media_marker: str, media_kinds: list[str]
) -> dict[str, Any]:
if isinstance(part, str):
return {"type": "text", "text": part}
if not isinstance(part, Mapping):
raise TypeError(f"content part must be mapping, got {type(part).__name__}")
part_type = part.get("type")
if part_type in _TEXT_PART_TYPES:
return {"type": "text", "text": part.get("text") or ""}
if part_type in _THINKING_PART_TYPES:
text = part.get("thinking")
if text is None:
text = part.get("text", "")
if not isinstance(text, str):
raise TypeError("Inkling thinking part payload must be a string")
return {"type": "thinking", "thinking": text}
if part_type in _IMAGE_PART_TYPES or part_type in _AUDIO_PART_TYPES:
media_kinds.append("image" if part_type in _IMAGE_PART_TYPES else "audio")
return {"type": "text", "text": f"{media_marker}{len(media_kinds) - 1}"}
raise ValueError(f"unsupported content part type: {part_type!r}")
if add_generation_prompt:
input_ids.append(tokenizer.encode_special(MESSAGE_MODEL))
def _resolve_media(message: Any, chat: Any, media_marker: str, media_kinds: list[str]):
# Media bytes are encoded later by the MM processor, so each media part
# travels through the renderer as a pointer whose span becomes ONE placeholder.
content = message.content
if not isinstance(content, chat.Text) or not content.text.startswith(media_marker):
return message
index = int(content.text.removeprefix(media_marker))
if media_kinds[index] == "audio" and message.author.kind != chat.AuthorKind.User:
raise ValueError("Inkling audio input is only supported in user messages")
prefix = (
_IMAGE_LOCATION_PREFIX
if media_kinds[index] == "image"
else _AUDIO_LOCATION_PREFIX
)
pointer = chat.ImagePointer(f"{prefix}{index}", chat.ImageFormat.Png, 1, 1)
return message.copy(content=pointer)
def _leading_system_count(messages: Sequence[Mapping[str, Any]]) -> int:
count = 0
while count < len(messages) and messages[count].get("role") == "system":
count += 1
return count
def _tool_declare_message(tools: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
declared = []
for tool in tools:
function = tool.get("function") or {}
declared.append(
{
"type": tool.get("type") or "function",
"function": {
key: value for key, value in function.items() if value is not None
},
}
)
return {
"role": "tool_declare",
"content": json.dumps(declared, ensure_ascii=False, separators=(",", ":")),
}
def _spans_to_input_ids(spans: Sequence[Any], tml: SimpleNamespace) -> list[int]:
input_ids: list[int] = []
for wrapped in spans:
span = wrapped.span
if isinstance(span, tml.chat.EncodedTextTokenSpan):
input_ids.extend(span.tokens)
elif isinstance(span, tml.chat.ImageAssetPointerTokenSpan):
_append_media_placeholder(input_ids, span.location, tml.tokenizer)
else:
raise ValueError(
f"unexpected TMLv0 span in Inkling prompt: {type(span).__name__}"
)
return input_ids
def _append_message(
input_ids: list[int],
tokenizer: InklingTextTokenizer,
role: str,
kind: str,
text: str,
*,
author_name: str | None = None,
def _append_media_placeholder(
input_ids: list[int], location: str, tokenizer: Any
) -> None:
input_ids.append(tokenizer.encode_special(ROLE_MESSAGE_TOKENS[role]))
if author_name:
input_ids.extend(tokenizer.encode_text(author_name))
if kind == "text":
input_ids.append(tokenizer.encode_special(CONTENT_TEXT))
input_ids.extend(tokenizer.encode_text(text))
elif kind == "image":
input_ids.append(tokenizer.encode_special(CONTENT_IMAGE))
if location.startswith(_IMAGE_LOCATION_PREFIX):
input_ids.append(IMAGE_TOKEN_ID)
elif kind == "audio":
input_ids.append(tokenizer.encode_special(CONTENT_AUDIO_INPUT))
input_ids.append(AUDIO_TOKEN_ID)
input_ids.append(tokenizer.encode_special(AUDIO_END))
elif kind == "thinking":
input_ids.append(tokenizer.encode_special(CONTENT_THINKING))
input_ids.extend(tokenizer.encode_text(text))
elif kind == "xml":
input_ids.append(tokenizer.encode_special(CONTENT_XML))
input_ids.extend(tokenizer.encode_text(text))
elif kind == "invoke_tool_json":
input_ids.append(tokenizer.encode_special(CONTENT_INVOKE_TOOL_JSON))
input_ids.extend(tokenizer.encode_text(text))
else:
raise ValueError(f"unsupported Inkling render part kind: {kind!r}")
input_ids.append(tokenizer.encode_special(END_MESSAGE))
def _iter_render_parts(content: Any):
"""Yield ordered ``(kind, text)`` pairs from message content."""
if content is None:
return
if isinstance(content, str):
if content:
yield ("text", content)
return
if not isinstance(content, Sequence) or isinstance(content, (bytes, bytearray)):
raise TypeError("message content must be a string or a sequence of parts")
for part in content:
if isinstance(part, str):
yield ("text", part)
continue
if not isinstance(part, Mapping):
raise TypeError(f"content part must be mapping, got {type(part).__name__}")
ptype = part.get("type")
if ptype in (None, "text", "input_text"):
text = part.get("text", "")
yield ("text", text if isinstance(text, str) else "")
elif ptype in ("thinking", "reasoning"):
text = part.get("thinking")
if text is None:
text = part.get("text", "")
if not isinstance(text, str):
raise TypeError("Inkling thinking part payload must be a string")
yield ("thinking", text)
elif ptype in _IMAGE_PART_TYPES:
yield ("image", "")
elif ptype in _AUDIO_PART_TYPES:
yield ("audio", "")
else:
raise ValueError(f"unsupported content part type: {ptype!r}")
def _format_reasoning_effort(reasoning_effort: float) -> str:
if isinstance(reasoning_effort, bool) or not isinstance(
reasoning_effort, (int, float)
):
raise TypeError("Inkling reasoning_effort must be a number")
value = float(reasoning_effort)
if not math.isfinite(value) or not 0.0 <= value <= 0.99:
raise ValueError("Inkling reasoning_effort must be finite and in [0.0, 0.99]")
return f"{round(value, 2):g}"
def _expect_role(message: Mapping[str, Any]) -> str:
role = message.get("role")
if role not in ROLE_MESSAGE_TOKENS:
raise ValueError(
f"unsupported Inkling message role {role!r}; expected one of {sorted(ROLE_MESSAGE_TOKENS)}"
)
return str(role)
def _as_mapping(value: Any) -> Mapping[str, Any]:
if isinstance(value, Mapping):
return value
if hasattr(value, "model_dump"):
dumped = value.model_dump()
if isinstance(dumped, Mapping):
return dumped
raise TypeError(f"expected mapping, got {type(value).__name__}")
def _canonical_json(value: Any) -> str:
return json.dumps(
_sort_json(value),
ensure_ascii=False,
allow_nan=False,
separators=(",", ":"),
# tml-renderers DMel-encodes audio at render time, so audio rides in an image
# slot and gets the MM processor's audio framing here.
if input_ids[-1] != tokenizer.encode_special(_special_name(CONTENT_IMAGE)):
raise ValueError("Inkling audio placeholder is not framed as a content block")
input_ids[-1] = tokenizer.encode_special(_special_name(CONTENT_AUDIO_INPUT))
input_ids.extend(
[AUDIO_TOKEN_ID, tokenizer.encode_special(_special_name(AUDIO_END))]
)
def _sort_json(value: Any) -> Any:
if isinstance(value, Mapping):
return {str(key): _sort_json(value[key]) for key in sorted(value)}
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
return [_sort_json(item) for item in value]
return value
def _tool_declare_json(tools: Sequence[Mapping[str, Any]]) -> str:
tool_specs = []
for tool_value in tools:
tool = _as_mapping(tool_value)
function = _as_mapping(tool.get("function", {}))
tool_specs.append(
{
"description": function.get("description") or "",
"name": function["name"],
"parameters": function.get("parameters") or {},
"type": tool.get("type", "function"),
}
)
return _canonical_json(tool_specs)
def _tool_call_name_and_args(tool_call_value: Any) -> tuple[str, Mapping[str, Any]]:
tool_call = _as_mapping(tool_call_value)
function = _as_mapping(tool_call.get("function", {}))
name = function.get("name")
if not isinstance(name, str):
raise TypeError("tool call function name must be a string")
raw_args = function.get("arguments") or {}
if isinstance(raw_args, str):
args = json.loads(raw_args) if raw_args else {}
else:
args = raw_args
if not isinstance(args, Mapping):
raise TypeError("tool call function arguments must decode to an object")
return name, args
def _tool_call_json(name: str, args: Mapping[str, Any]) -> str:
name_json = json.dumps(name, ensure_ascii=False, allow_nan=False)
return f'{{"name":{name_json},"args":{_canonical_json(args)}}}'
def _special_name(token: str) -> str:
return token.removeprefix("<|").removesuffix("|>")
@@ -1,9 +1,5 @@
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any
END_OF_TEXT = "<|endoftext|>"
MESSAGE_USER = "<|message_user|>"
MESSAGE_MODEL = "<|message_model|>"
@@ -56,50 +52,3 @@ INKLING_CONTROL_TOKENS: frozenset[str] = frozenset(
"<|model_trigger_generation|>",
}
)
INKLING_SPECIAL_TOKEN_NAMES: dict[str, str] = {
token.removeprefix("<|").removesuffix("|>"): token
for token in INKLING_SPECIAL_TOKENS
}
ROLE_MESSAGE_TOKENS: dict[str, str] = {
"user": MESSAGE_USER,
"assistant": MESSAGE_MODEL,
"system": MESSAGE_SYSTEM,
"tool": MESSAGE_TOOL,
}
def normalize_special_token(token: str) -> str:
"""Accept either message_user or <|message_user|> spellings."""
if token in INKLING_SPECIAL_TOKENS:
return token
try:
return INKLING_SPECIAL_TOKEN_NAMES[token]
except KeyError as exc:
raise KeyError(f"unknown Inkling special token: {token!r}") from exc
@dataclass(frozen=True)
class InklingTokenizer:
"""Small wrapper around a base text tokenizer plus Inkling framing IDs.
Plain text is encoded by the base tokenizer, while the minimal chat
framing tokens are inserted from the fixed overlay map.
"""
tokenizer: Any
special_token_ids: Mapping[str, int] | None = None
def encode_text(self, text: str) -> list[int]:
if not isinstance(text, str):
raise TypeError(f"text must be str, got {type(text).__name__}")
return list(self.tokenizer.encode(text, add_special_tokens=False))
def encode_special(self, token: str) -> int:
special = normalize_special_token(token)
token_ids = self.special_token_ids or INKLING_SPECIAL_TOKEN_IDS
return int(token_ids[special])
def decode(self, token_ids: list[int]) -> str:
return self.tokenizer.decode(token_ids)