[Perf] DeepSeek-V4.1-Flash: Use segmented fastokens for prompt encoding (#48928)

(cherry picked from commit 219b7e7a688363bac9ef09eacf80e44eef9efe4b)
This commit is contained in:
leon
2026-11-15 10:51:40 +08:00
committed by minke.yu
parent a7c668a936
commit 7980a09c0a
2 changed files with 36 additions and 0 deletions
@@ -6,6 +6,7 @@ from __future__ import annotations
import copy
import functools
import json
import logging
from types import ModuleType
from typing import Any, Optional, Union
@@ -15,6 +16,8 @@ DEEPSEEK_RECIPE_INSTALL_HINT = (
"`pip install deepseek-recipe`."
)
logger = logging.getLogger(__name__)
MODEL_NAMES = {"dsv4": "DeepSeek-V4-Pro", "dsv41": "DeepSeek-V4.1-Flash"}
# The checkpoint tokenizer's image token; recipe's bundled tokenizer spells it <|image|>.
IMAGE_TOKEN = "<|deepseek_image|>"
@@ -127,6 +130,20 @@ class DeepSeekRecipe:
recipe = load_deepseek_recipe()
self.model = MODEL_NAMES[chat_encoding_spec]
self.hf_tokenizer = tokenizer
self._fast_prompt_tokenizer = None
if type(tokenizer).__module__.partition(".")[0] == "fastokens":
from fastokens import Tokenizer
if hasattr(Tokenizer, "encode_segments"):
self._fast_prompt_tokenizer = Tokenizer.from_json_str(
tokenizer.to_str()
)
logger.info("DeepSeek recipe prompt encoder: fastokens segmented")
else:
logger.warning(
"deepseek-recipe prompt acceleration requires fastokens.encode_segments; "
"using the reference prompt tokenizer"
)
self.special_tokens = recipe.SpecialTokens(image=IMAGE_TOKEN)
self._options = recipe.ConversionOptions().with_special_tokens(
self.special_tokens