[router] Apply chat template before cache-aware hashing (fix overlap=0 on chat traffic) (#27386)

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Kangyan-Zhou
2026-06-10 10:27:54 -07:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 276c98c6cf
commit 21647f1f5d
9 changed files with 1289 additions and 76 deletions
+10
View File
@@ -43,6 +43,16 @@ serde_json = { version = "1", features = ["preserve_order"] }
# this pulls no openssl/native-tls (matching reqwest's rustls-tls above).
hf-hub = { version = "0.4", default-features = false, features = ["ureq"] }
# Chat-template rendering for cache-aware routing: the engine caches tokens
# AFTER applying the model's chat template, so the router renders the same
# template (from tokenizer_config.json) before hashing — otherwise its query
# token_ids diverge from the engine's stored blocks. `pycompat` supplies the
# Python str/dict methods HF chat templates rely on (.startswith, .items, ...).
minijinja = { version = "2", features = ["loop_controls", "json"] }
minijinja-contrib = { version = "2", features = ["pycompat"] }
# `strftime_now` chat-template helper (some templates inject the current date).
chrono = { version = "0.4", default-features = false, features = ["clock"] }
# Utilities
anyhow = "1"
thiserror = "2"