[router] Apply chat template before cache-aware hashing (fix overlap=0 on chat traffic) (#27386)
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
276c98c6cf
commit
21647f1f5d
@@ -26,7 +26,6 @@ import socket
|
||||
import subprocess
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
@@ -34,16 +33,6 @@ from .model_specs import get_model_spec
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Passthrough Jinja chat template that emits ONLY `messages[*].content`
|
||||
# joined with `\n` — matching the router's cache_aware_zmq prompt
|
||||
# extraction. A worker launched with
|
||||
# ``--chat-template <PASSTHROUGH_CHAT_TEMPLATE_PATH>`` tokenizes the
|
||||
# raw content string, so its KV-block hashes align with what the
|
||||
# router computes from the same chat-completions request. Test-only.
|
||||
PASSTHROUGH_CHAT_TEMPLATE_PATH = str(
|
||||
Path(__file__).parent / "passthrough_chat_template.jinja"
|
||||
)
|
||||
|
||||
|
||||
def _get_open_port() -> int:
|
||||
"""Allocate an ephemeral TCP port in the range [20000, 55535].
|
||||
|
||||
Reference in New Issue
Block a user