[Fix] [XPU] runtime: Fix XPU rotary embeddings and tokenizer padding (#41213)
This commit is contained in:
@@ -565,6 +565,8 @@ class RotaryEmbedding(BaseFusedOp):
|
||||
self.is_neox_style,
|
||||
)
|
||||
return query, key
|
||||
elif self.head_size != self.rotary_dim:
|
||||
return self.forward_native(positions, query, key)
|
||||
else:
|
||||
self._match_cos_sin_cache_dtype(query)
|
||||
# Use fallback kernel of 'rotary_embedding'.
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import inspect
|
||||
import logging
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
@@ -15,10 +16,31 @@ def patch_tokenizer(tokenizer):
|
||||
)
|
||||
return _SpecialTokensCachePatcher.patch(tokenizer)
|
||||
|
||||
if _needs_pad_padding_side_shim(tokenizer):
|
||||
logger.info(
|
||||
f"Applying _pad(padding_side=...) compat shim for {type(tokenizer)}"
|
||||
)
|
||||
return _PadPaddingSideShim.patch(tokenizer)
|
||||
|
||||
return tokenizer
|
||||
|
||||
|
||||
def _needs_pad_padding_side_shim(tokenizer) -> bool:
|
||||
pad_fn = getattr(type(tokenizer), "_pad", None)
|
||||
if pad_fn is None:
|
||||
return False
|
||||
try:
|
||||
sig = inspect.signature(pad_fn)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
params = sig.parameters
|
||||
if "padding_side" in params:
|
||||
return False
|
||||
return not any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
|
||||
|
||||
|
||||
def unpatch_tokenizer(tokenizer):
|
||||
tokenizer = _PadPaddingSideShim.unpatch(tokenizer)
|
||||
return _SpecialTokensCachePatcher.unpatch(tokenizer)
|
||||
|
||||
|
||||
@@ -40,6 +62,49 @@ def decode_without_hf_kwargs(tokenizer, token_ids, skip_special_tokens):
|
||||
return tokenizer.decode(token_ids)
|
||||
|
||||
|
||||
class _PadPaddingSideShim:
|
||||
_PATCHED_FLAG = "_sglang_pad_padding_side_patched"
|
||||
|
||||
@classmethod
|
||||
def patch(cls, tokenizer):
|
||||
tokenizer_cls = type(tokenizer)
|
||||
|
||||
if getattr(tokenizer_cls, cls._PATCHED_FLAG, False):
|
||||
return tokenizer
|
||||
|
||||
original_pad = tokenizer_cls._pad
|
||||
|
||||
def patched_pad(self, *args, **kwargs):
|
||||
padding_side = kwargs.pop("padding_side", None)
|
||||
if padding_side is None or padding_side == self.padding_side:
|
||||
return original_pad(self, *args, **kwargs)
|
||||
previous_padding_side = self.padding_side
|
||||
self.padding_side = padding_side
|
||||
try:
|
||||
return original_pad(self, *args, **kwargs)
|
||||
finally:
|
||||
self.padding_side = previous_padding_side
|
||||
|
||||
tokenizer_cls._original_pad = original_pad
|
||||
tokenizer_cls._pad = patched_pad
|
||||
setattr(tokenizer_cls, cls._PATCHED_FLAG, True)
|
||||
|
||||
return tokenizer
|
||||
|
||||
@classmethod
|
||||
def unpatch(cls, tokenizer):
|
||||
tokenizer_cls = type(tokenizer)
|
||||
|
||||
if not getattr(tokenizer_cls, cls._PATCHED_FLAG, False):
|
||||
return tokenizer
|
||||
|
||||
tokenizer_cls._pad = tokenizer_cls._original_pad
|
||||
del tokenizer_cls._original_pad
|
||||
delattr(tokenizer_cls, cls._PATCHED_FLAG)
|
||||
|
||||
return tokenizer
|
||||
|
||||
|
||||
class _SpecialTokensCachePatcher:
|
||||
_PATCHED_FLAG = "_sglang_special_tokens_patched"
|
||||
_CACHED_TOKENS_ATTR = "_sglang_cached_special_tokens"
|
||||
|
||||
Reference in New Issue
Block a user