From 94de28764c88a3df26ddb111c7523c3893770368 Mon Sep 17 00:00:00 2001 From: sushil Dubey Date: Fri, 10 Jul 2026 18:07:30 +0530 Subject: [PATCH] Fix TiktokenTokenizer missing num_special_tokens_to_add (#30643) Co-authored-by: Sushil Dubey Co-authored-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com> --- python/sglang/srt/tokenizer/tiktoken_tokenizer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/sglang/srt/tokenizer/tiktoken_tokenizer.py b/python/sglang/srt/tokenizer/tiktoken_tokenizer.py index b29015547..c1f9ad134 100644 --- a/python/sglang/srt/tokenizer/tiktoken_tokenizer.py +++ b/python/sglang/srt/tokenizer/tiktoken_tokenizer.py @@ -113,6 +113,10 @@ class TiktokenTokenizer: def decode(self, x, *args, **kwargs): return self.tokenizer.decode(x) + def num_special_tokens_to_add(self, *args, **kwargs) -> int: + # tiktoken's encode adds no special tokens (add_special_tokens is ignored). + return 0 + def batch_decode( self, batch, skip_special_tokens=True, spaces_between_special_tokens=False ):