[router] Apply chat template before cache-aware hashing (fix overlap=0 on chat traffic) (#27386)

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Kangyan-Zhou
2026-06-10 10:27:54 -07:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 276c98c6cf
commit 21647f1f5d
9 changed files with 1289 additions and 76 deletions
@@ -26,7 +26,6 @@ import socket
import subprocess
import time
from dataclasses import dataclass, field
from pathlib import Path
import httpx
@@ -34,16 +33,6 @@ from .model_specs import get_model_spec
logger = logging.getLogger(__name__)
# Passthrough Jinja chat template that emits ONLY `messages[*].content`
# joined with `\n` — matching the router's cache_aware_zmq prompt
# extraction. A worker launched with
# ``--chat-template <PASSTHROUGH_CHAT_TEMPLATE_PATH>`` tokenizes the
# raw content string, so its KV-block hashes align with what the
# router computes from the same chat-completions request. Test-only.
PASSTHROUGH_CHAT_TEMPLATE_PATH = str(
Path(__file__).parent / "passthrough_chat_template.jinja"
)
def _get_open_port() -> int:
"""Allocate an ephemeral TCP port in the range [20000, 55535].