[Perf] DeepSeek-V4.1-Flash: Use segmented fastokens for prompt encoding (#48928)
(cherry picked from commit 219b7e7a688363bac9ef09eacf80e44eef9efe4b)
This commit is contained in:
@@ -6,6 +6,7 @@ from __future__ import annotations
|
||||
import copy
|
||||
import functools
|
||||
import json
|
||||
import logging
|
||||
from types import ModuleType
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
@@ -15,6 +16,8 @@ DEEPSEEK_RECIPE_INSTALL_HINT = (
|
||||
"`pip install deepseek-recipe`."
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MODEL_NAMES = {"dsv4": "DeepSeek-V4-Pro", "dsv41": "DeepSeek-V4.1-Flash"}
|
||||
# The checkpoint tokenizer's image token; recipe's bundled tokenizer spells it <|image|>.
|
||||
IMAGE_TOKEN = "<|deepseek_image|>"
|
||||
@@ -127,6 +130,20 @@ class DeepSeekRecipe:
|
||||
recipe = load_deepseek_recipe()
|
||||
self.model = MODEL_NAMES[chat_encoding_spec]
|
||||
self.hf_tokenizer = tokenizer
|
||||
self._fast_prompt_tokenizer = None
|
||||
if type(tokenizer).__module__.partition(".")[0] == "fastokens":
|
||||
from fastokens import Tokenizer
|
||||
|
||||
if hasattr(Tokenizer, "encode_segments"):
|
||||
self._fast_prompt_tokenizer = Tokenizer.from_json_str(
|
||||
tokenizer.to_str()
|
||||
)
|
||||
logger.info("DeepSeek recipe prompt encoder: fastokens segmented")
|
||||
else:
|
||||
logger.warning(
|
||||
"deepseek-recipe prompt acceleration requires fastokens.encode_segments; "
|
||||
"using the reference prompt tokenizer"
|
||||
)
|
||||
self.special_tokens = recipe.SpecialTokens(image=IMAGE_TOKEN)
|
||||
self._options = recipe.ConversionOptions().with_special_tokens(
|
||||
self.special_tokens
|
||||
|
||||
Reference in New Issue
Block a user