Co-authored-by: hnyls2002 <lsyincs@gmail.com> Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com> Co-authored-by: A-transformer <cl5743590921@gmail.com>
56 lines
1.7 KiB
Python
56 lines
1.7 KiB
Python
from typing import List
|
|
|
|
THINK_OPEN = "<|open|>think<|sep|>"
|
|
THINK_CLOSE = "<|close|>think<|sep|>"
|
|
RESPONSE_OPEN = "<|open|>response<|sep|>"
|
|
RESPONSE_CLOSE = "<|close|>response<|sep|>"
|
|
TOOLS_OPEN = "<|open|>tools<|sep|>"
|
|
TOOLS_CLOSE = "<|close|>tools<|sep|>"
|
|
MESSAGE_CLOSE = "<|close|>message<|sep|>"
|
|
CALL_OPEN = "<|open|>call"
|
|
CALL_CLOSE = "<|close|>call<|sep|>"
|
|
ARGUMENT_CLOSE = "<|close|>argument<|sep|>"
|
|
|
|
# max_tokens can stop after an XTML control token or channel name, before <|sep|>.
|
|
_PARTIAL_MARKER_SUFFIXES = (
|
|
"<|open|>",
|
|
"<|close|>",
|
|
THINK_CLOSE.removesuffix("<|sep|>"),
|
|
RESPONSE_OPEN.removesuffix("<|sep|>"),
|
|
RESPONSE_CLOSE.removesuffix("<|sep|>"),
|
|
TOOLS_OPEN.removesuffix("<|sep|>"),
|
|
TOOLS_CLOSE.removesuffix("<|sep|>"),
|
|
MESSAGE_CLOSE.removesuffix("<|sep|>"),
|
|
)
|
|
|
|
|
|
def partial_suffix_len(text: str, markers: List[str]) -> int:
|
|
best = 0
|
|
for marker in markers:
|
|
for length in range(min(len(marker) - 1, len(text)), best, -1):
|
|
if text.endswith(marker[:length]):
|
|
best = length
|
|
break
|
|
return best
|
|
|
|
|
|
def strip_partial_marker_suffix(text: str) -> str:
|
|
for suffix in _PARTIAL_MARKER_SUFFIXES:
|
|
if text.endswith(suffix):
|
|
return text[: -len(suffix)]
|
|
return text
|
|
|
|
|
|
def strip_response_wrappers(text: str) -> str:
|
|
open_idx = text.find(RESPONSE_OPEN)
|
|
if open_idx != -1:
|
|
close_idx = text.find(RESPONSE_CLOSE, open_idx + len(RESPONSE_OPEN))
|
|
if close_idx != -1:
|
|
text = text[open_idx + len(RESPONSE_OPEN) : close_idx]
|
|
else:
|
|
text = text[open_idx + len(RESPONSE_OPEN) :]
|
|
else:
|
|
text = text.replace(RESPONSE_CLOSE, "")
|
|
text = text.replace(MESSAGE_CLOSE, "")
|
|
return strip_partial_marker_suffix(text)
|