bugfix for npu Grok2 model --detokenizer without all special ids (#29853)

2026-07-02 22:12:48 +08:00
parent bdd3515389
commit c05c48b35e
1 changed files with 4 additions and 2 deletions
@@ -33,8 +33,10 @@ def decode_without_hf_kwargs(tokenizer, token_ids, skip_special_tokens):
    if skip_special_tokens:
        special_ids = getattr(tokenizer, "all_special_ids_set", None)
        if special_ids is None:
-            special_ids = set(tokenizer.all_special_ids)
-        token_ids = [tid for tid in token_ids if tid not in special_ids]
+            special_ids = getattr(tokenizer, "all_special_ids", None)
+        if special_ids is not None:
+            special_ids_set = set(special_ids)
+            token_ids = [tid for tid in token_ids if tid not in special_ids_set]
    return tokenizer.decode(token_ids)