minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

bugfix for npu Grok2 model --detokenizer without all special ids (#29853)

Browse Source
This commit is contained in:
McZyWu
2026-07-02 22:12:48 +08:00
committed by GitHub
parent bdd3515389
commit c05c48b35e
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/utils/patch_tokenizer.py
+4 -2
View File
@@ -33,8 +33,10 @@ def decode_without_hf_kwargs(tokenizer, token_ids, skip_special_tokens):
if skip_special_tokens: if skip_special_tokens:
special_ids = getattr(tokenizer, "all_special_ids_set", None) special_ids = getattr(tokenizer, "all_special_ids_set", None)
if special_ids is None: if special_ids is None:
special_ids = set(tokenizer.all_special_ids) special_ids = getattr(tokenizer, "all_special_ids", None)
token_ids = [tid for tid in token_ids if tid not in special_ids] if special_ids is not None:
special_ids_set = set(special_ids)
token_ids = [tid for tid in token_ids if tid not in special_ids_set]
return tokenizer.decode(token_ids) return tokenizer.decode(token_ids)