bump dynamo-tokenizers to 1.7.0 (#32981)

2026-07-31 15:02:28 -07:00
parent 70cec31378
commit 1d640aaea2
4 changed files with 887 additions and 74 deletions
@@ -33,9 +33,8 @@ uuid = { workspace = true }

 axum = { version = "0.8.9", features = ["json", "tokio"] }
 core_affinity = "0.8"
-# pin the dynamo-tokenizers for now, 1.7.0 enables`serde_json/preserve_order`
-# transitively, which reorders every JSON object.
-dynamo-tokenizers = "=1.5.3"
+# the dynamo-tokenizers is deps on hf-hub, should bump version together.
+dynamo-tokenizers = "1.7.0"
 flume = "0.12.0"
 itertools = "0.14"
 hf-hub = { version = "0.4", default-features = false }