diff --git a/python/sglang/__init__.py b/python/sglang/__init__.py index 7b6756b39..d074dabc3 100644 --- a/python/sglang/__init__.py +++ b/python/sglang/__init__.py @@ -24,6 +24,11 @@ if _sys.platform == "darwin": pass del _sys +from sglang.srt.utils.hf_transformers_patches import apply_all as _apply_hf_patches + +_apply_hf_patches() +del _apply_hf_patches + # Frontend Language APIs from sglang.global_config import global_config from sglang.lang.api import ( diff --git a/python/sglang/srt/utils/hf_transformers/__init__.py b/python/sglang/srt/utils/hf_transformers/__init__.py index 86da65690..3e6b3fa78 100644 --- a/python/sglang/srt/utils/hf_transformers/__init__.py +++ b/python/sglang/srt/utils/hf_transformers/__init__.py @@ -14,17 +14,16 @@ """Hugging Face Transformers utilities. This package provides HF Transformers helpers, split into submodules -(common, compat, config, tokenizer, processor, mistral_utils). +(common, config, tokenizer, processor, mistral_utils). Compatibility +monkey-patches live in the sibling ``sglang.srt.utils.hf_transformers_patches`` +module and are applied at sglang import time. All public symbols are re-exported here for convenience. The old import path ``sglang.srt.utils.hf_transformers_utils`` is preserved by a separate shim module. """ -from .compat import apply_all as _apply_compat - -_apply_compat() - -from .common import ( # noqa: E402 +from ..hf_transformers_patches import normalize_rope_scaling_compat +from .common import ( CONTEXT_LENGTH_KEYS, AutoConfig, attach_additional_stop_token_ids, @@ -37,10 +36,9 @@ from .common import ( # noqa: E402 get_sparse_attention_config, get_tokenizer_from_processor, ) -from .compat import normalize_rope_scaling_compat # noqa: E402 -from .config import get_config # noqa: E402 -from .processor import get_processor # noqa: E402 -from .tokenizer import ( # noqa: E402 +from .config import get_config +from .processor import get_processor +from .tokenizer import ( _fix_added_tokens_encoding, _fix_v5_add_bos_eos_token, get_tokenizer, diff --git a/python/sglang/srt/utils/hf_transformers/common.py b/python/sglang/srt/utils/hf_transformers/common.py index 40b73c123..0f3ccf907 100644 --- a/python/sglang/srt/utils/hf_transformers/common.py +++ b/python/sglang/srt/utils/hf_transformers/common.py @@ -52,7 +52,7 @@ from sglang.srt.configs.deepseek_ocr import DeepseekVLV2Config from sglang.srt.configs.internvl import InternVLChatConfig from sglang.srt.utils import get_bool_env_var, logger, lru_cache_frozenset -from .compat import normalize_rope_scaling_compat +from ..hf_transformers_patches import normalize_rope_scaling_compat if get_bool_env_var("SGLANG_USE_MODELSCOPE"): from modelscope import AutoConfig, GenerationConfig diff --git a/python/sglang/srt/utils/hf_transformers/config.py b/python/sglang/srt/utils/hf_transformers/config.py index f5c800646..759dade67 100644 --- a/python/sglang/srt/utils/hf_transformers/config.py +++ b/python/sglang/srt/utils/hf_transformers/config.py @@ -23,6 +23,7 @@ from sglang.srt.connector import create_remote_connector from sglang.srt.utils import is_remote_url, logger, lru_cache_frozenset from sglang.srt.utils.runai_utils import ObjectStorageModel, is_runai_obj_uri +from ..hf_transformers_patches import _ensure_gguf_version from .common import ( _CONFIG_REGISTRY, AutoConfig, @@ -34,7 +35,6 @@ from .common import ( check_gguf_file, get_hf_text_config, ) -from .compat import _ensure_gguf_version from .mistral_utils import is_mistral_model, load_mistral_config diff --git a/python/sglang/srt/utils/hf_transformers/tokenizer.py b/python/sglang/srt/utils/hf_transformers/tokenizer.py index 8c82145e6..b965b804e 100644 --- a/python/sglang/srt/utils/hf_transformers/tokenizer.py +++ b/python/sglang/srt/utils/hf_transformers/tokenizer.py @@ -30,12 +30,12 @@ from sglang.srt.utils import is_remote_url, logger from sglang.srt.utils.patch_tokenizer import patch_tokenizer from sglang.srt.utils.runai_utils import ObjectStorageModel, is_runai_obj_uri +from ..hf_transformers_patches import _ensure_gguf_version from .common import ( _resolve_local_or_cached_file, attach_additional_stop_token_ids, check_gguf_file, ) -from .compat import _ensure_gguf_version, patch_is_base_mistral_in_ci from .mistral_utils import ( _MISTRAL_TOKENIZER_REDIRECTS, patch_mistral_common_tokenizer, @@ -462,7 +462,6 @@ def get_tokenizer( kwargs["use_fast"] = True tokenizer_name = _resolve_tokenizer_name(tokenizer_name, kwargs) - patch_is_base_mistral_in_ci() common_kwargs = dict( trust_remote_code=trust_remote_code, diff --git a/python/sglang/srt/utils/hf_transformers/compat.py b/python/sglang/srt/utils/hf_transformers_patches.py similarity index 95% rename from python/sglang/srt/utils/hf_transformers/compat.py rename to python/sglang/srt/utils/hf_transformers_patches.py index 570096e9d..5e05f638e 100644 --- a/python/sglang/srt/utils/hf_transformers/compat.py +++ b/python/sglang/srt/utils/hf_transformers_patches.py @@ -11,21 +11,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Compatibility patches for transformers v5.x. +"""Monkey-patches on transformers internals. -This module applies monkey-patches to work around breaking changes in -transformers v5. Each patch is tagged with the upstream issue it works -around so it can be removed once the upstream fix lands. +Mix of backward-compat shims (re-add symbols removed in v5), workarounds +for transformers v5 bugs, fixes for remote-model-code (trust_remote_code) +that hasn't been updated for v5 yet, and CI-only patches (e.g. neutralize +HF API calls to avoid rate limits). Import this module early (before any ``from_pretrained`` call) to activate all patches. It is safe to import multiple times -- patches are idempotent. - -Patches fall into two categories: - -1. **Transformers bugs / regressions** -- issues in transformers itself. -2. **Remote-model-code compat** -- remote model code (trust_remote_code) - that hasn't been updated for v5 yet. These should be removed once - the model authors publish fixes. """ import inspect @@ -44,10 +38,19 @@ def apply_all(): """Apply all transformers compatibility patches (idempotent). Call this once at import time. It is safe to call multiple times. + + No-op when the ``transformers`` package is not installed -- frontend-only + sglang users should not be forced to install transformers just to import + the top-level ``sglang`` package. """ global _applied if _applied: return + try: + import transformers # noqa: F401 + except ImportError: + _applied = True + return _applied = True # v5.4 patches @@ -62,6 +65,9 @@ def apply_all(): _ensure_clean_up_tokenization_compat() _ensure_is_torch_fx_available_compat() + # CI-only: neutralize HF API calls inside tokenizer from_pretrained + patch_is_base_mistral_in_ci() + logger.debug("transformers compatibility patches applied") diff --git a/test/registered/unit/utils/test_hf_transformers.py b/test/registered/unit/utils/test_hf_transformers.py index e5a0a3e06..cc6226a73 100644 --- a/test/registered/unit/utils/test_hf_transformers.py +++ b/test/registered/unit/utils/test_hf_transformers.py @@ -20,8 +20,8 @@ from sglang.srt.utils.hf_transformers.common import ( get_hf_text_config, get_rope_config, ) -from sglang.srt.utils.hf_transformers.compat import normalize_rope_scaling_compat from sglang.srt.utils.hf_transformers.tokenizer import _fix_special_tokens_pattern +from sglang.srt.utils.hf_transformers_patches import normalize_rope_scaling_compat from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=5, suite="stage-a-test-cpu")