diff --git a/python/sglang/srt/layers/rotary_embedding/base.py b/python/sglang/srt/layers/rotary_embedding/base.py index 67bd3fcc56..8dbd5ff6a7 100644 --- a/python/sglang/srt/layers/rotary_embedding/base.py +++ b/python/sglang/srt/layers/rotary_embedding/base.py @@ -565,6 +565,8 @@ class RotaryEmbedding(BaseFusedOp): self.is_neox_style, ) return query, key + elif self.head_size != self.rotary_dim: + return self.forward_native(positions, query, key) else: self._match_cos_sin_cache_dtype(query) # Use fallback kernel of 'rotary_embedding'. diff --git a/python/sglang/srt/utils/patch_tokenizer.py b/python/sglang/srt/utils/patch_tokenizer.py index 3da8c2e644..e855691948 100644 --- a/python/sglang/srt/utils/patch_tokenizer.py +++ b/python/sglang/srt/utils/patch_tokenizer.py @@ -1,3 +1,4 @@ +import inspect import logging from sglang.srt.environ import envs @@ -15,10 +16,31 @@ def patch_tokenizer(tokenizer): ) return _SpecialTokensCachePatcher.patch(tokenizer) + if _needs_pad_padding_side_shim(tokenizer): + logger.info( + f"Applying _pad(padding_side=...) compat shim for {type(tokenizer)}" + ) + return _PadPaddingSideShim.patch(tokenizer) + return tokenizer +def _needs_pad_padding_side_shim(tokenizer) -> bool: + pad_fn = getattr(type(tokenizer), "_pad", None) + if pad_fn is None: + return False + try: + sig = inspect.signature(pad_fn) + except (TypeError, ValueError): + return False + params = sig.parameters + if "padding_side" in params: + return False + return not any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values()) + + def unpatch_tokenizer(tokenizer): + tokenizer = _PadPaddingSideShim.unpatch(tokenizer) return _SpecialTokensCachePatcher.unpatch(tokenizer) @@ -40,6 +62,49 @@ def decode_without_hf_kwargs(tokenizer, token_ids, skip_special_tokens): return tokenizer.decode(token_ids) +class _PadPaddingSideShim: + _PATCHED_FLAG = "_sglang_pad_padding_side_patched" + + @classmethod + def patch(cls, tokenizer): + tokenizer_cls = type(tokenizer) + + if getattr(tokenizer_cls, cls._PATCHED_FLAG, False): + return tokenizer + + original_pad = tokenizer_cls._pad + + def patched_pad(self, *args, **kwargs): + padding_side = kwargs.pop("padding_side", None) + if padding_side is None or padding_side == self.padding_side: + return original_pad(self, *args, **kwargs) + previous_padding_side = self.padding_side + self.padding_side = padding_side + try: + return original_pad(self, *args, **kwargs) + finally: + self.padding_side = previous_padding_side + + tokenizer_cls._original_pad = original_pad + tokenizer_cls._pad = patched_pad + setattr(tokenizer_cls, cls._PATCHED_FLAG, True) + + return tokenizer + + @classmethod + def unpatch(cls, tokenizer): + tokenizer_cls = type(tokenizer) + + if not getattr(tokenizer_cls, cls._PATCHED_FLAG, False): + return tokenizer + + tokenizer_cls._pad = tokenizer_cls._original_pad + del tokenizer_cls._original_pad + delattr(tokenizer_cls, cls._PATCHED_FLAG) + + return tokenizer + + class _SpecialTokensCachePatcher: _PATCHED_FLAG = "_sglang_special_tokens_patched" _CACHED_TOKENS_ATTR = "_sglang_cached_special_tokens" diff --git a/test/registered/unit/utils/test_patch_tokenizer.py b/test/registered/unit/utils/test_patch_tokenizer.py index f5e912d5a7..e538bf6a12 100644 --- a/test/registered/unit/utils/test_patch_tokenizer.py +++ b/test/registered/unit/utils/test_patch_tokenizer.py @@ -5,6 +5,7 @@ from contextlib import contextmanager from transformers import AutoTokenizer from sglang.srt.utils.patch_tokenizer import ( + _PadPaddingSideShim, _SpecialTokensCachePatcher, decode_without_hf_kwargs, unpatch_tokenizer, @@ -166,6 +167,113 @@ class TestPatchTokenizerUnitTest(unittest.TestCase): self.assertEqual(tokenizer.decode_calls, [[1, 2], [1, 99, 2]]) +class TestPadPaddingSideShimUnitTest(unittest.TestCase): + def test_patch_unpatch_restores_original(self): + tokenizer = _FakeStaleTokenizer() + cls = type(tokenizer) + original_pad = cls._pad + + _PadPaddingSideShim.patch(tokenizer) + self.addCleanup(unpatch_tokenizer, tokenizer) + self.assertTrue(getattr(cls, "_sglang_pad_padding_side_patched", False)) + self.assertIsNot(cls._pad, original_pad) + + unpatch_tokenizer(tokenizer) + self.assertFalse(getattr(cls, "_sglang_pad_padding_side_patched", False)) + self.assertIs(cls._pad, original_pad) + + def test_double_patch_is_idempotent(self): + tokenizer = _FakeStaleTokenizer() + cls = type(tokenizer) + + _PadPaddingSideShim.patch(tokenizer) + self.addCleanup(unpatch_tokenizer, tokenizer) + patched_once = cls._pad + _PadPaddingSideShim.patch(tokenizer) + + self.assertIs(cls._pad, patched_once) + self.assertTrue(getattr(cls, "_sglang_pad_padding_side_patched", False)) + + def test_patch_applies_to_all_instances_of_the_class(self): + tokenizer_a = _FakeStaleTokenizer(padding_side="left") + tokenizer_b = _FakeStaleTokenizer(padding_side="right") + self.assertIs(type(tokenizer_a), type(tokenizer_b)) + + _PadPaddingSideShim.patch(tokenizer_a) + try: + self.assertTrue( + getattr(type(tokenizer_b), "_sglang_pad_padding_side_patched", False) + ) + + tokenizer_a._pad({"input_ids": [1]}) + tokenizer_b._pad({"input_ids": [2]}) + + self.assertEqual(tokenizer_a.pad_calls[-1][1], "left") + self.assertEqual(tokenizer_b.pad_calls[-1][1], "right") + finally: + unpatch_tokenizer(tokenizer_a) + + def test_per_call_padding_side_override_applied_and_restored(self): + tokenizer = _FakeStaleTokenizer(padding_side="right") + _PadPaddingSideShim.patch(tokenizer) + try: + tokenizer._pad({"input_ids": [1]}, padding_side="left") + self.assertEqual(tokenizer.pad_calls[-1][1], "left") + self.assertEqual(tokenizer.padding_side, "right") + finally: + unpatch_tokenizer(tokenizer) + + def test_padding_side_override_matching_default_is_noop(self): + tokenizer = _FakeStaleTokenizer(padding_side="right") + _PadPaddingSideShim.patch(tokenizer) + try: + tokenizer._pad({"input_ids": [1]}, padding_side="right") + self.assertEqual(tokenizer.pad_calls[-1][1], "right") + self.assertEqual(tokenizer.padding_side, "right") + finally: + unpatch_tokenizer(tokenizer) + + def test_none_padding_side_passes_through(self): + tokenizer = _FakeStaleTokenizer(padding_side="right") + _PadPaddingSideShim.patch(tokenizer) + try: + tokenizer._pad({"input_ids": [1]}) + self.assertEqual(tokenizer.pad_calls[-1][1], "right") + finally: + unpatch_tokenizer(tokenizer) + + def test_padding_side_restored_after_pad_raises(self): + tokenizer = _FakeRaisingTokenizer(padding_side="right") + _PadPaddingSideShim.patch(tokenizer) + try: + with self.assertRaises(RuntimeError): + tokenizer._pad(padding_side="left") + self.assertEqual(tokenizer.padding_side, "right") + finally: + unpatch_tokenizer(tokenizer) + + +class _FakeStaleTokenizer: + """Stand-in for a custom remote-code tokenizer whose _pad() predates + transformers forwarding padding_side -- no padding_side param, no **kwargs.""" + + def __init__(self, padding_side="right"): + self.padding_side = padding_side + self.pad_calls = [] + + def _pad(self, encoded_inputs, max_length=None): + self.pad_calls.append((encoded_inputs, self.padding_side)) + return encoded_inputs + + +class _FakeRaisingTokenizer: + def __init__(self, padding_side="right"): + self.padding_side = padding_side + + def _pad(self, *args, **kwargs): + raise RuntimeError("boom") + + def _get_class_attr_ids(cls): return { n: id(v.fget if isinstance(v, property) else v) for n, v in vars(cls).items()