[router] Apply chat template before cache-aware hashing (fix overlap=0 on chat traffic) (#27386)

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-10 10:27:54 -07:00
parent 276c98c6cf
commit 21647f1f5d
9 changed files with 1289 additions and 76 deletions
@@ -26,7 +26,6 @@ import socket
 import subprocess
 import time
 from dataclasses import dataclass, field
-from pathlib import Path

 import httpx

@@ -34,16 +33,6 @@ from .model_specs import get_model_spec

 logger = logging.getLogger(__name__)

-# Passthrough Jinja chat template that emits ONLY `messages[*].content`
-# joined with `\n` — matching the router's cache_aware_zmq prompt
-# extraction. A worker launched with
-# ``--chat-template <PASSTHROUGH_CHAT_TEMPLATE_PATH>`` tokenizes the
-# raw content string, so its KV-block hashes align with what the
-# router computes from the same chat-completions request. Test-only.
-PASSTHROUGH_CHAT_TEMPLATE_PATH = str(
-    Path(__file__).parent / "passthrough_chat_template.jinja"
-)
-

 def _get_open_port() -> int:
    """Allocate an ephemeral TCP port in the range [20000, 55535].
@@ -1,13 +0,0 @@
-{#-
-  Passthrough chat template for cache-aware-zmq e2e tests.
-
-  Emits ONLY `messages[*].content` joined with `\n` — no role markers,
-  no special tokens, no generation prompt. This is the SAME shape the
-  router's cache_aware_zmq policy produces in `extract_prompt_text`,
-  so a worker launched with `--chat-template <this file>` tokenizes the
-  same string the router will tokenize for routing — making block
-  hashes align across worker KV cache and router HashTree.
-
-  Use only for tests; not appropriate for any real chat workload.
-#}
-{{- messages | map(attribute='content') | join('\n') -}}