diff --git a/python/sglang/srt/parser/deepseek_recipe_renderer.py b/python/sglang/srt/parser/deepseek_recipe_renderer.py index 44d7c8d9d2..9f9427c473 100644 --- a/python/sglang/srt/parser/deepseek_recipe_renderer.py +++ b/python/sglang/srt/parser/deepseek_recipe_renderer.py @@ -6,6 +6,7 @@ from __future__ import annotations import copy import functools import json +import logging from types import ModuleType from typing import Any, Optional, Union @@ -15,6 +16,8 @@ DEEPSEEK_RECIPE_INSTALL_HINT = ( "`pip install deepseek-recipe`." ) +logger = logging.getLogger(__name__) + MODEL_NAMES = {"dsv4": "DeepSeek-V4-Pro", "dsv41": "DeepSeek-V4.1-Flash"} # The checkpoint tokenizer's image token; recipe's bundled tokenizer spells it <|image|>. IMAGE_TOKEN = "<|deepseek_image|>" @@ -127,6 +130,20 @@ class DeepSeekRecipe: recipe = load_deepseek_recipe() self.model = MODEL_NAMES[chat_encoding_spec] self.hf_tokenizer = tokenizer + self._fast_prompt_tokenizer = None + if type(tokenizer).__module__.partition(".")[0] == "fastokens": + from fastokens import Tokenizer + + if hasattr(Tokenizer, "encode_segments"): + self._fast_prompt_tokenizer = Tokenizer.from_json_str( + tokenizer.to_str() + ) + logger.info("DeepSeek recipe prompt encoder: fastokens segmented") + else: + logger.warning( + "deepseek-recipe prompt acceleration requires fastokens.encode_segments; " + "using the reference prompt tokenizer" + ) self.special_tokens = recipe.SpecialTokens(image=IMAGE_TOKEN) self._options = recipe.ConversionOptions().with_special_tokens( self.special_tokens diff --git a/test/registered/unit/parser/test_deepseek_recipe_renderer.py b/test/registered/unit/parser/test_deepseek_recipe_renderer.py index d8b83e8c09..53da20c48d 100644 --- a/test/registered/unit/parser/test_deepseek_recipe_renderer.py +++ b/test/registered/unit/parser/test_deepseek_recipe_renderer.py @@ -204,6 +204,25 @@ class TestDeepSeekRecipe(CustomTestCase): } ) + def test_fastokens_preserves_prompt_ids(self): + try: + from fastokens import Tokenizer, _ConfiguredTokenizerShim + except ImportError: + self.skipTest("fastokens is not installed") + if not hasattr(Tokenizer, "encode_segments"): + self.skipTest("fastokens has no segmented encoder") + candidate = renderer.DeepSeekRecipe( + "dsv41", _ConfiguredTokenizerShim(self.tokenizer) + ) + self.assertIsNotNone(candidate._fast_prompt_tokenizer) + message = {"role": "user", "content": "中文 👩🏽‍💻 x"} + for thinking in (False, True): + body = {"messages": [message], "thinking": thinking} + with self.subTest(thinking=thinking): + self.assertEqual( + candidate.encode(body), self.deepseek_recipe.encode(body) + ) + if __name__ == "__main__": unittest.main()