fix(recipe): drop the pre-48928 _fast_prompt_tokenizer fallback stub
This commit is contained in:
@@ -158,9 +158,6 @@ class DeepSeekRecipe:
|
|||||||
self._encoding = encoding.with_special_tokens(
|
self._encoding = encoding.with_special_tokens(
|
||||||
self.special_tokens
|
self.special_tokens
|
||||||
).with_tokenizer(self.tokenizer)
|
).with_tokenizer(self.tokenizer)
|
||||||
# This base has no segment-level fast prompt tokenizer; `encode_conversation`
|
|
||||||
# falls back to `self._encoding.encode`.
|
|
||||||
self._fast_prompt_tokenizer = None
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_hf_tokenizer(
|
def from_hf_tokenizer(
|
||||||
|
|||||||
Reference in New Issue
Block a user