[Fix] [XPU] runtime: Fix XPU rotary embeddings and tokenizer padding (#41213)

This commit is contained in:
vikram singh shekhawat
2026-10-23 14:09:59 +08:00
committed by GitHub
parent 0e338777b0
commit 3f36fc0833
3 changed files with 175 additions and 0 deletions
@@ -565,6 +565,8 @@ class RotaryEmbedding(BaseFusedOp):
self.is_neox_style,
)
return query, key
elif self.head_size != self.rotary_dim:
return self.forward_native(positions, query, key)
else:
self._match_cos_sin_cache_dtype(query)
# Use fallback kernel of 'rotary_embedding'.
@@ -1,3 +1,4 @@
import inspect
import logging
from sglang.srt.environ import envs
@@ -15,10 +16,31 @@ def patch_tokenizer(tokenizer):
)
return _SpecialTokensCachePatcher.patch(tokenizer)
if _needs_pad_padding_side_shim(tokenizer):
logger.info(
f"Applying _pad(padding_side=...) compat shim for {type(tokenizer)}"
)
return _PadPaddingSideShim.patch(tokenizer)
return tokenizer
def _needs_pad_padding_side_shim(tokenizer) -> bool:
pad_fn = getattr(type(tokenizer), "_pad", None)
if pad_fn is None:
return False
try:
sig = inspect.signature(pad_fn)
except (TypeError, ValueError):
return False
params = sig.parameters
if "padding_side" in params:
return False
return not any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
def unpatch_tokenizer(tokenizer):
tokenizer = _PadPaddingSideShim.unpatch(tokenizer)
return _SpecialTokensCachePatcher.unpatch(tokenizer)
@@ -40,6 +62,49 @@ def decode_without_hf_kwargs(tokenizer, token_ids, skip_special_tokens):
return tokenizer.decode(token_ids)
class _PadPaddingSideShim:
_PATCHED_FLAG = "_sglang_pad_padding_side_patched"
@classmethod
def patch(cls, tokenizer):
tokenizer_cls = type(tokenizer)
if getattr(tokenizer_cls, cls._PATCHED_FLAG, False):
return tokenizer
original_pad = tokenizer_cls._pad
def patched_pad(self, *args, **kwargs):
padding_side = kwargs.pop("padding_side", None)
if padding_side is None or padding_side == self.padding_side:
return original_pad(self, *args, **kwargs)
previous_padding_side = self.padding_side
self.padding_side = padding_side
try:
return original_pad(self, *args, **kwargs)
finally:
self.padding_side = previous_padding_side
tokenizer_cls._original_pad = original_pad
tokenizer_cls._pad = patched_pad
setattr(tokenizer_cls, cls._PATCHED_FLAG, True)
return tokenizer
@classmethod
def unpatch(cls, tokenizer):
tokenizer_cls = type(tokenizer)
if not getattr(tokenizer_cls, cls._PATCHED_FLAG, False):
return tokenizer
tokenizer_cls._pad = tokenizer_cls._original_pad
del tokenizer_cls._original_pad
delattr(tokenizer_cls, cls._PATCHED_FLAG)
return tokenizer
class _SpecialTokensCachePatcher:
_PATCHED_FLAG = "_sglang_special_tokens_patched"
_CACHED_TOKENS_ATTR = "_sglang_cached_special_tokens"