[router] Apply chat template before cache-aware hashing (fix overlap=0 on chat traffic) (#27386)
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
276c98c6cf
commit
21647f1f5d
@@ -43,6 +43,16 @@ serde_json = { version = "1", features = ["preserve_order"] }
|
||||
# this pulls no openssl/native-tls (matching reqwest's rustls-tls above).
|
||||
hf-hub = { version = "0.4", default-features = false, features = ["ureq"] }
|
||||
|
||||
# Chat-template rendering for cache-aware routing: the engine caches tokens
|
||||
# AFTER applying the model's chat template, so the router renders the same
|
||||
# template (from tokenizer_config.json) before hashing — otherwise its query
|
||||
# token_ids diverge from the engine's stored blocks. `pycompat` supplies the
|
||||
# Python str/dict methods HF chat templates rely on (.startswith, .items, ...).
|
||||
minijinja = { version = "2", features = ["loop_controls", "json"] }
|
||||
minijinja-contrib = { version = "2", features = ["pycompat"] }
|
||||
# `strftime_now` chat-template helper (some templates inject the current date).
|
||||
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
||||
|
||||
# Utilities
|
||||
anyhow = "1"
|
||||
thiserror = "2"
|
||||
|
||||
Reference in New Issue
Block a user