Files
sglang/test/registered/unit/tokenizer/test_tiktoken_tokenizer.py
T

65 lines
2.1 KiB
Python

"""Unit tests for tiktoken_tokenizer — no server, no model loading."""
import unittest
from unittest.mock import MagicMock
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
from sglang.srt.tokenizer.tiktoken_tokenizer import (
TiktokenTokenizer,
)
class TestTiktokenTokenizer(CustomTestCase):
def setUp(self):
from jinja2 import Template
self.tok = TiktokenTokenizer.__new__(TiktokenTokenizer)
self.mock_tokenizer = MagicMock()
self.tok.tokenizer = self.mock_tokenizer
self.tok.chat_template = "dummy"
self.tok.chat_template_jinja = Template(
"{% for message in messages %}"
"{{ message['role'] }}: {{ message['content'] }}"
"{% endfor %}"
"{% if add_generation_prompt %}assistant:{% endif %}"
)
def test_batch_decode_flat_list_wraps_each(self):
self.mock_tokenizer.decode_batch.return_value = ["a", "b"]
self.tok.batch_decode([1, 2])
self.mock_tokenizer.decode_batch.assert_called_once_with([[1], [2]])
def test_call_returns_input_ids(self):
self.mock_tokenizer.encode.return_value = [1, 2, 3]
result = self.tok(["hello", "world"])
self.assertIn("input_ids", result)
self.assertEqual(len(result["input_ids"]), 2)
def test_apply_chat_template_no_tokenize(self):
messages = [{"role": "user", "content": "hello"}]
result = self.tok.apply_chat_template(
messages=messages,
tokenize=False,
add_generation_prompt=False,
)
self.assertIsInstance(result, str)
self.assertIn("hello", result)
def test_apply_chat_template_with_tokenize(self):
self.mock_tokenizer.encode.return_value = [1, 2, 3]
messages = [{"role": "user", "content": "hello"}]
result = self.tok.apply_chat_template(
messages=messages,
tokenize=True,
add_generation_prompt=False,
)
self.assertIsInstance(result, list)
if __name__ == "__main__":
unittest.main()