Filter non-int token ids in benchmark and observe decode-side bootstrap/alloc metrics (#24684)

2026-05-08 11:45:37 -07:00
parent 73b8eda103
commit e40e339c72
3 changed files with 18 additions and 4 deletions
@@ -66,8 +66,12 @@ def compute_random_lens(full_len: int, range_ratio: float, num: int) -> List[int

@lru_cache(maxsize=1)
 def get_available_tokens(tokenizer):
-    """Get all available token ids from the tokenizer vocabulary."""
-    return list(tokenizer.get_vocab().values())
+    """Get valid token ids from the tokenizer vocabulary."""
+    return [
+        token_id
+        for token_id in tokenizer.get_vocab().values()
+        if isinstance(token_id, int)
+    ]


 def gen_prompt(tokenizer, token_num):