[Perf] DeepSeek-V4.1-Flash: Use segmented fastokens for prompt encoding (#48928)

(cherry picked from commit 219b7e7a688363bac9ef09eacf80e44eef9efe4b)
This commit is contained in:
leon
2026-11-15 10:51:40 +08:00
committed by minke.yu
parent a7c668a936
commit 7980a09c0a
2 changed files with 36 additions and 0 deletions
@@ -6,6 +6,7 @@ from __future__ import annotations
import copy import copy
import functools import functools
import json import json
import logging
from types import ModuleType from types import ModuleType
from typing import Any, Optional, Union from typing import Any, Optional, Union
@@ -15,6 +16,8 @@ DEEPSEEK_RECIPE_INSTALL_HINT = (
"`pip install deepseek-recipe`." "`pip install deepseek-recipe`."
) )
logger = logging.getLogger(__name__)
MODEL_NAMES = {"dsv4": "DeepSeek-V4-Pro", "dsv41": "DeepSeek-V4.1-Flash"} MODEL_NAMES = {"dsv4": "DeepSeek-V4-Pro", "dsv41": "DeepSeek-V4.1-Flash"}
# The checkpoint tokenizer's image token; recipe's bundled tokenizer spells it <|image|>. # The checkpoint tokenizer's image token; recipe's bundled tokenizer spells it <|image|>.
IMAGE_TOKEN = "<|deepseek_image|>" IMAGE_TOKEN = "<|deepseek_image|>"
@@ -127,6 +130,20 @@ class DeepSeekRecipe:
recipe = load_deepseek_recipe() recipe = load_deepseek_recipe()
self.model = MODEL_NAMES[chat_encoding_spec] self.model = MODEL_NAMES[chat_encoding_spec]
self.hf_tokenizer = tokenizer self.hf_tokenizer = tokenizer
self._fast_prompt_tokenizer = None
if type(tokenizer).__module__.partition(".")[0] == "fastokens":
from fastokens import Tokenizer
if hasattr(Tokenizer, "encode_segments"):
self._fast_prompt_tokenizer = Tokenizer.from_json_str(
tokenizer.to_str()
)
logger.info("DeepSeek recipe prompt encoder: fastokens segmented")
else:
logger.warning(
"deepseek-recipe prompt acceleration requires fastokens.encode_segments; "
"using the reference prompt tokenizer"
)
self.special_tokens = recipe.SpecialTokens(image=IMAGE_TOKEN) self.special_tokens = recipe.SpecialTokens(image=IMAGE_TOKEN)
self._options = recipe.ConversionOptions().with_special_tokens( self._options = recipe.ConversionOptions().with_special_tokens(
self.special_tokens self.special_tokens
@@ -204,6 +204,25 @@ class TestDeepSeekRecipe(CustomTestCase):
} }
) )
def test_fastokens_preserves_prompt_ids(self):
try:
from fastokens import Tokenizer, _ConfiguredTokenizerShim
except ImportError:
self.skipTest("fastokens is not installed")
if not hasattr(Tokenizer, "encode_segments"):
self.skipTest("fastokens has no segmented encoder")
candidate = renderer.DeepSeekRecipe(
"dsv41", _ConfiguredTokenizerShim(self.tokenizer)
)
self.assertIsNotNone(candidate._fast_prompt_tokenizer)
message = {"role": "user", "content": "中文 👩🏽‍💻 <think>x</think>"}
for thinking in (False, True):
body = {"messages": [message], "thinking": thinking}
with self.subTest(thinking=thinking):
self.assertEqual(
candidate.encode(body), self.deepseek_recipe.encode(body)
)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()