[Perf] DeepSeek-V4.1-Flash: Use segmented fastokens for prompt encoding (#48928)
(cherry picked from commit 219b7e7a688363bac9ef09eacf80e44eef9efe4b)
This commit is contained in:
@@ -6,6 +6,7 @@ from __future__ import annotations
|
|||||||
import copy
|
import copy
|
||||||
import functools
|
import functools
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
from types import ModuleType
|
from types import ModuleType
|
||||||
from typing import Any, Optional, Union
|
from typing import Any, Optional, Union
|
||||||
|
|
||||||
@@ -15,6 +16,8 @@ DEEPSEEK_RECIPE_INSTALL_HINT = (
|
|||||||
"`pip install deepseek-recipe`."
|
"`pip install deepseek-recipe`."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
MODEL_NAMES = {"dsv4": "DeepSeek-V4-Pro", "dsv41": "DeepSeek-V4.1-Flash"}
|
MODEL_NAMES = {"dsv4": "DeepSeek-V4-Pro", "dsv41": "DeepSeek-V4.1-Flash"}
|
||||||
# The checkpoint tokenizer's image token; recipe's bundled tokenizer spells it <|image|>.
|
# The checkpoint tokenizer's image token; recipe's bundled tokenizer spells it <|image|>.
|
||||||
IMAGE_TOKEN = "<|deepseek_image|>"
|
IMAGE_TOKEN = "<|deepseek_image|>"
|
||||||
@@ -127,6 +130,20 @@ class DeepSeekRecipe:
|
|||||||
recipe = load_deepseek_recipe()
|
recipe = load_deepseek_recipe()
|
||||||
self.model = MODEL_NAMES[chat_encoding_spec]
|
self.model = MODEL_NAMES[chat_encoding_spec]
|
||||||
self.hf_tokenizer = tokenizer
|
self.hf_tokenizer = tokenizer
|
||||||
|
self._fast_prompt_tokenizer = None
|
||||||
|
if type(tokenizer).__module__.partition(".")[0] == "fastokens":
|
||||||
|
from fastokens import Tokenizer
|
||||||
|
|
||||||
|
if hasattr(Tokenizer, "encode_segments"):
|
||||||
|
self._fast_prompt_tokenizer = Tokenizer.from_json_str(
|
||||||
|
tokenizer.to_str()
|
||||||
|
)
|
||||||
|
logger.info("DeepSeek recipe prompt encoder: fastokens segmented")
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
"deepseek-recipe prompt acceleration requires fastokens.encode_segments; "
|
||||||
|
"using the reference prompt tokenizer"
|
||||||
|
)
|
||||||
self.special_tokens = recipe.SpecialTokens(image=IMAGE_TOKEN)
|
self.special_tokens = recipe.SpecialTokens(image=IMAGE_TOKEN)
|
||||||
self._options = recipe.ConversionOptions().with_special_tokens(
|
self._options = recipe.ConversionOptions().with_special_tokens(
|
||||||
self.special_tokens
|
self.special_tokens
|
||||||
|
|||||||
@@ -204,6 +204,25 @@ class TestDeepSeekRecipe(CustomTestCase):
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_fastokens_preserves_prompt_ids(self):
|
||||||
|
try:
|
||||||
|
from fastokens import Tokenizer, _ConfiguredTokenizerShim
|
||||||
|
except ImportError:
|
||||||
|
self.skipTest("fastokens is not installed")
|
||||||
|
if not hasattr(Tokenizer, "encode_segments"):
|
||||||
|
self.skipTest("fastokens has no segmented encoder")
|
||||||
|
candidate = renderer.DeepSeekRecipe(
|
||||||
|
"dsv41", _ConfiguredTokenizerShim(self.tokenizer)
|
||||||
|
)
|
||||||
|
self.assertIsNotNone(candidate._fast_prompt_tokenizer)
|
||||||
|
message = {"role": "user", "content": "中文 👩🏽💻 <think>x</think>"}
|
||||||
|
for thinking in (False, True):
|
||||||
|
body = {"messages": [message], "thinking": thinking}
|
||||||
|
with self.subTest(thinking=thinking):
|
||||||
|
self.assertEqual(
|
||||||
|
candidate.encode(body), self.deepseek_recipe.encode(body)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user