[perf] reduce overhead of fill_ids list reconstruction and decref (#27965)
This commit is contained in:
@@ -56,13 +56,14 @@ if TYPE_CHECKING:
|
||||
class RadixKey:
|
||||
"""is_bigram=True: token_ids holds raw tokens (N+1 for N bigrams); slices share one boundary token."""
|
||||
|
||||
__slots__ = ("token_ids", "extra_key", "is_bigram")
|
||||
__slots__ = ("token_ids", "extra_key", "is_bigram", "limit")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
token_ids: array[int],
|
||||
extra_key: Optional[str] = None,
|
||||
is_bigram: bool = False,
|
||||
limit: Optional[int] = None,
|
||||
):
|
||||
# token ids sequence (raw ints in both modes)
|
||||
self.token_ids = token_ids
|
||||
@@ -70,21 +71,40 @@ class RadixKey:
|
||||
self.extra_key = extra_key
|
||||
# bigram view over token_ids: length = max(0, len(token_ids) - 1)
|
||||
self.is_bigram = is_bigram
|
||||
# Optional cap on raw tokens: behave as if token_ids were sliced to
|
||||
# token_ids[:limit], without the O(n) copy. None = use all tokens.
|
||||
self.limit = limit
|
||||
|
||||
def _raw_len(self) -> int:
|
||||
n = len(self.token_ids)
|
||||
if self.limit is not None and self.limit < n:
|
||||
return self.limit
|
||||
return n
|
||||
|
||||
def raw_token_ids(self) -> array:
|
||||
"""token_ids honoring `limit` (copies only when capped)."""
|
||||
n = self._raw_len()
|
||||
t = self.token_ids
|
||||
return t if n == len(t) else t[:n]
|
||||
|
||||
def __len__(self) -> int:
|
||||
n = self._raw_len()
|
||||
if self.is_bigram:
|
||||
n = len(self.token_ids)
|
||||
return n - 1 if n > 0 else 0
|
||||
return len(self.token_ids)
|
||||
return n
|
||||
|
||||
# TODO(Jialin): vectorize with numpy without PyLong boxing
|
||||
def __iter__(self) -> Iterator:
|
||||
t = self.token_ids
|
||||
n = self._raw_len()
|
||||
if self.is_bigram:
|
||||
t = self.token_ids
|
||||
for i in range(len(t) - 1):
|
||||
for i in range(n - 1 if n > 0 else 0):
|
||||
yield (t[i], t[i + 1])
|
||||
elif n == len(t):
|
||||
yield from t
|
||||
else:
|
||||
yield from self.token_ids
|
||||
for i in range(n):
|
||||
yield t[i]
|
||||
|
||||
def __getitem__(self, idx: Union[int, slice]) -> RadixKey:
|
||||
# Normalize int -> 1-element slice so the rest handles one shape.
|
||||
@@ -166,6 +186,7 @@ class RadixKey:
|
||||
matched = max(0, min(matched_tokens - 1, len(self), len(other)))
|
||||
return (matched // page_size) * page_size if page_size > 1 else matched
|
||||
|
||||
matched_tokens = min(matched_tokens, len(self), len(other))
|
||||
if page_size == 1:
|
||||
return matched_tokens
|
||||
return (matched_tokens // page_size) * page_size
|
||||
|
||||
Reference in New Issue
Block a user