tokenizer: Add fastokens support (#23753)

2026-04-28 11:43:10 -07:00
parent ad785a2299
commit 66ea0aee7f
11 changed files with 152 additions and 4 deletions
@@ -142,6 +142,10 @@ http2 = [
  "granian>=2.6.0",
 ]

+fastokens = [
+  "fastokens>=0.1.1,<0.2.0",
+]
+
 test = [
  "accelerate",
  "addict",
@@ -158,6 +162,7 @@ test = [
  "pytest-cov",
  "diff-cover",
  "sentence_transformers",
+  "sglang[fastokens]",
  "tabulate",
  "granian>=2.6.0",
 ]