Fix tokenizer warning filtering for processors (#34500)

python/sglang/srt/utils/hf_transformers/processor.py

@@ -48,6 +48,7 @@ from .tokenizer import (
     _TOKENIZERS_BACKEND,
     _fix_added_tokens_encoding,
     _fix_special_tokens_pattern,
+    _install_tokenizer_warnings_filter,
 )
 
 _IMAGE_PROCESSOR_BACKENDS = {"auto", "torchvision", "pil"}
@@ -368,6 +369,8 @@ def get_processor(
         else:
             processor.tokenizer = tokenizer
 
+    _install_tokenizer_warnings_filter(tokenizer)
+
     if tokenizer.chat_template is None:
         local_path = download_from_hf(
             tokenizer_name, allow_patterns=["*.json", "*.jinja", "*.model"]

python/sglang/srt/utils/hf_transformers/tokenizer.py

@@ -132,6 +132,15 @@ class TokenizerWarningsFilter(logging.Filter):
         return "Calling super().encode with" not in record.getMessage()
 
 
+_tokenizer_warnings_filter = TokenizerWarningsFilter()
+
+
+def _install_tokenizer_warnings_filter(tokenizer):
+    logging.getLogger(tokenizer.__class__.__module__).addFilter(
+        _tokenizer_warnings_filter
+    )
+
+
 # ---------------------------------------------------------------------------
 # Helpers for get_tokenizer
 # ---------------------------------------------------------------------------
@@ -169,9 +178,6 @@ def _auto_tokenizer_from_pretrained(tokenizer_name, *args, **common_kwargs):
         tokenizer = AutoTokenizer.from_pretrained(
             tokenizer_name, *args, **common_kwargs
         )
-        logging.getLogger(tokenizer.__class__.__module__).addFilter(
-            TokenizerWarningsFilter()
-        )
         return tokenizer
     except TypeError as e:
         err_msg = (
@@ -419,6 +425,7 @@ def _fix_special_tokens_pattern(tokenizer):
 
 def _apply_post_load_fixes(tokenizer, tokenizer_name, revision):
     """Apply all post-load patches and return the final tokenizer."""
+    _install_tokenizer_warnings_filter(tokenizer)
     _fix_v5_tokenizer_components(tokenizer, tokenizer_name, revision)
     _fix_v5_add_bos_eos_token(tokenizer, tokenizer_name, revision)