fix(recipe): define _fast_prompt_tokenizer fallback on this base
This commit is contained in:
@@ -141,6 +141,9 @@ class DeepSeekRecipe:
|
||||
self._encoding = encoding.with_special_tokens(
|
||||
self.special_tokens
|
||||
).with_tokenizer(self.tokenizer)
|
||||
# This base has no segment-level fast prompt tokenizer; `encode_conversation`
|
||||
# falls back to `self._encoding.encode`.
|
||||
self._fast_prompt_tokenizer = None
|
||||
|
||||
@classmethod
|
||||
def from_hf_tokenizer(
|
||||
|
||||
Reference in New Issue
Block a user