fix: get_processor fails when --tokenizer-path lacks model config.json (#25643)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: ronnie_zheng <zl19940307@163.com>

python/sglang/srt/managers/scheduler.py

@@ -688,6 +688,7 @@ class Scheduler(
                     revision=server_args.revision,
                     use_fast=not server_args.disable_fast_image_processor,
                     tokenizer_backend=server_args.tokenizer_backend,
+                    model_name=server_args.model_path,
                 )
                 self.tokenizer = get_tokenizer_from_processor(self.processor)
             else:

python/sglang/srt/managers/tokenizer_manager.py

@@ -3015,6 +3015,7 @@ def _get_processor_wrapper(server_args):
             revision=server_args.revision,
             use_fast=not server_args.disable_fast_image_processor,
             tokenizer_backend=server_args.tokenizer_backend,
+            model_name=server_args.model_path,
         )
     except ValueError as e:
         error_message = str(e)
@@ -3029,6 +3030,7 @@ def _get_processor_wrapper(server_args):
                 revision=server_args.revision,
                 use_fast=True,
                 tokenizer_backend=server_args.tokenizer_backend,
+                model_name=server_args.model_path,
             )
         else:
             raise e

python/sglang/srt/managers/tp_worker.py

@@ -276,6 +276,7 @@ class TpModelWorker(BaseTpWorker):
                     trust_remote_code=server_args.trust_remote_code,
                     revision=server_args.revision,
                     tokenizer_backend=server_args.tokenizer_backend,
+                    model_name=server_args.model_path,
                 )
                 self.tokenizer = get_tokenizer_from_processor(self.processor)
             else:

python/sglang/srt/utils/hf_transformers/processor.py

@@ -143,6 +143,7 @@ def get_processor(
     tokenizer_revision: Optional[str] = None,
     use_fast: Optional[bool] = True,
     tokenizer_backend: str = "huggingface",
+    model_name: Optional[str] = None,
     **kwargs,
 ):
     if tokenizer_backend == "fastokens":
@@ -159,6 +160,13 @@ def get_processor(
             trust_remote_code=trust_remote_code,
             revision=revision,
         )
+    elif model_name is not None:
+        config = AutoConfig.from_pretrained(
+            model_name,
+            trust_remote_code=trust_remote_code,
+            revision=revision,
+            **kwargs,
+        )
     else:
         config = AutoConfig.from_pretrained(
             tokenizer_name,