[perf] reduce overhead of fill_ids list reconstruction and decref (#27965)

This commit is contained in:
Qiaolin Yu
2026-06-14 00:41:11 -07:00
committed by GitHub
parent f2d7d67603
commit f293ddf3ce
8 changed files with 363 additions and 66 deletions
+27 -6
View File
@@ -56,13 +56,14 @@ if TYPE_CHECKING:
class RadixKey:
"""is_bigram=True: token_ids holds raw tokens (N+1 for N bigrams); slices share one boundary token."""
__slots__ = ("token_ids", "extra_key", "is_bigram")
__slots__ = ("token_ids", "extra_key", "is_bigram", "limit")
def __init__(
self,
token_ids: array[int],
extra_key: Optional[str] = None,
is_bigram: bool = False,
limit: Optional[int] = None,
):
# token ids sequence (raw ints in both modes)
self.token_ids = token_ids
@@ -70,21 +71,40 @@ class RadixKey:
self.extra_key = extra_key
# bigram view over token_ids: length = max(0, len(token_ids) - 1)
self.is_bigram = is_bigram
# Optional cap on raw tokens: behave as if token_ids were sliced to
# token_ids[:limit], without the O(n) copy. None = use all tokens.
self.limit = limit
def _raw_len(self) -> int:
n = len(self.token_ids)
if self.limit is not None and self.limit < n:
return self.limit
return n
def raw_token_ids(self) -> array:
"""token_ids honoring `limit` (copies only when capped)."""
n = self._raw_len()
t = self.token_ids
return t if n == len(t) else t[:n]
def __len__(self) -> int:
n = self._raw_len()
if self.is_bigram:
n = len(self.token_ids)
return n - 1 if n > 0 else 0
return len(self.token_ids)
return n
# TODO(Jialin): vectorize with numpy without PyLong boxing
def __iter__(self) -> Iterator:
t = self.token_ids
n = self._raw_len()
if self.is_bigram:
t = self.token_ids
for i in range(len(t) - 1):
for i in range(n - 1 if n > 0 else 0):
yield (t[i], t[i + 1])
elif n == len(t):
yield from t
else:
yield from self.token_ids
for i in range(n):
yield t[i]
def __getitem__(self, idx: Union[int, slice]) -> RadixKey:
# Normalize int -> 1-element slice so the rest handles one shape.
@@ -166,6 +186,7 @@ class RadixKey:
matched = max(0, min(matched_tokens - 1, len(self), len(other)))
return (matched // page_size) * page_size if page_size > 1 else matched
matched_tokens = min(matched_tokens, len(self), len(other))
if page_size == 1:
return matched_tokens
return (matched_tokens // page_size) * page_size