diff --git a/python/sglang/srt/utils/patch_tokenizer.py b/python/sglang/srt/utils/patch_tokenizer.py
index 1d083180f..ad6407a4f 100644
--- a/python/sglang/srt/utils/patch_tokenizer.py
+++ b/python/sglang/srt/utils/patch_tokenizer.py
@@ -33,8 +33,10 @@ def decode_without_hf_kwargs(tokenizer, token_ids, skip_special_tokens):
     if skip_special_tokens:
         special_ids = getattr(tokenizer, "all_special_ids_set", None)
         if special_ids is None:
-            special_ids = set(tokenizer.all_special_ids)
-        token_ids = [tid for tid in token_ids if tid not in special_ids]
+            special_ids = getattr(tokenizer, "all_special_ids", None)
+        if special_ids is not None:
+            special_ids_set = set(special_ids)
+            token_ids = [tid for tid in token_ids if tid not in special_ids_set]
     return tokenizer.decode(token_ids)