[smg] use official tokenizer crate instead of manually built one (#17721)

2026-01-25 13:35:16 -08:00
parent 2c1e2674cc
commit f5ac1ca10b
4 changed files with 2 additions and 2794 deletions
@@ -86,6 +86,7 @@ tiktoken-rs = { version = "0.7.0" }
 reasoning-parser = "1.0.0"
 openai-protocol = { version = "1.0.0", features = ["axum"] }
 tool-parser = "1.0.0"
+llm-tokenizer = "1.0.0"
 minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins"] }
 minijinja-contrib = { version = "2.0", features = ["pycompat"] }
 rustls = { version = "0.23", default-features = false, features = ["ring", "std"] }
@@ -172,21 +173,6 @@ path = "benches/request_processing.rs"
 name = "router_registry_bench"
 harness = false

-[[bench]]
-name = "tokenizer_benchmark"
-harness = false
-path = "benches/tokenizer_benchmark.rs"
-
-[[bench]]
-name = "tool_parser_benchmark"
-harness = false
-path = "benches/tool_parser_benchmark.rs"
-
-[[bench]]
-name = "tree_benchmark"
-harness = false
-path = "benches/tree_benchmark.rs"
-
 [[bench]]
 name = "manual_policy_benchmark"
 harness = false