Compare commits
60
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c8d06424fa | ||
|
|
97c7eabb33 | ||
|
|
e9b436dc2d | ||
|
|
fc63ae5392 | ||
|
|
9df2d40324 | ||
|
|
6a8a6e399f | ||
|
|
e6c0737541 | ||
|
|
9374e03f13 | ||
|
|
86883ae89f | ||
|
|
43c2844c18 | ||
|
|
9e288079e4 | ||
|
|
738d5a50d0 | ||
|
|
e80c8608dc | ||
|
|
5351d5491b | ||
|
|
8a2e1403a3 | ||
|
|
1910eccb8e | ||
|
|
6217b9de46 | ||
|
|
9dd446b5c1 | ||
|
|
f76eaf0f92 | ||
|
|
45fdc97082 | ||
|
|
f690861265 | ||
|
|
e4bfdc949b | ||
|
|
59e3aee6d9 | ||
|
|
406081d3e3 | ||
|
|
1975265315 | ||
|
|
534417b82e | ||
|
|
ed0a454730 | ||
|
|
04ee2d03d7 | ||
|
|
9d5db13e53 | ||
|
|
7e20f0af79 | ||
|
|
0b74ab2ac4 | ||
|
|
3c7ec33d26 | ||
|
|
09ca4957ee | ||
|
|
7921442a5c | ||
|
|
05ef104208 | ||
|
|
d44620f285 | ||
|
|
5df4f2c036 | ||
|
|
db1e8cdbe3 | ||
|
|
f3ac513d92 | ||
|
|
0054c5b8fe | ||
|
|
6da0a40b02 | ||
|
|
caf3403086 | ||
|
|
617f19ba61 | ||
|
|
ba3318ffbd | ||
|
|
8279d3d6df | ||
|
|
ef4cccb6a0 | ||
|
|
98b950366d | ||
|
|
87ab6c0155 | ||
|
|
0308920bdb | ||
|
|
57db2a4ae9 | ||
|
|
7fd7fd58a5 | ||
|
|
71a72b8fea | ||
|
|
6c9518fcd8 | ||
|
|
3a4ab451ac | ||
|
|
5d38fc96a9 | ||
|
|
261ebbc55b | ||
|
|
53fb9be7ee | ||
|
|
e5be84f795 | ||
|
|
39ceba8a3d | ||
|
|
b9d259441c |
@@ -25,7 +25,7 @@
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-6612af9e0-local-20261011-0740
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-e7dc175e9-local-20261011-0940
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
|
||||
@@ -2998,12 +2998,22 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
# add further chunks into the transfer queue.
|
||||
return
|
||||
|
||||
# NOTE(shangming): sharding according to the dst_infos to make sure
|
||||
# requests with the same dst_sessions will be added into the same
|
||||
# queue, which enables early abort with failed sessions.
|
||||
# Shard by dst session ports AND bootstrap_room. The port term alone
|
||||
# degenerates to a constant when all requests target a single decode
|
||||
# instance (its session ports are fixed), collapsing every transfer
|
||||
# onto 1/16 of the lanes and throttling P->D intake (measured at
|
||||
# client concurrency 300: #transfer-req piled at 100+ while decode
|
||||
# itself stayed healthy). Mixing in bootstrap_room keeps all chunks
|
||||
# of one request on the same lane (per-request FIFO order preserved)
|
||||
# while spreading distinct rooms across all lanes. Abort/cleanup is
|
||||
# unaffected: chunks are dropped per-room at dequeue via
|
||||
# request_status (see the worker loop) and per-request via
|
||||
# failed_sessions before send — no lane-level purge exists, and the
|
||||
# deferred abort ack is room-scoped (_staging_outstanding). Same
|
||||
# trade-off as mori/conn.py which shards by bootstrap_room alone.
|
||||
dst_infos = self.transfer_infos[bootstrap_room].keys()
|
||||
session_port_sum = sum(int(session.rsplit(":", 1)[1]) for session in dst_infos)
|
||||
shard_idx = session_port_sum % len(self.transfer_queues)
|
||||
shard_idx = (session_port_sum + bootstrap_room) % len(self.transfer_queues)
|
||||
|
||||
if trace_ctx is None:
|
||||
trace_ctx = TraceNullContext()
|
||||
|
||||
@@ -21,6 +21,8 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import zlib
|
||||
from array import array
|
||||
from collections import deque
|
||||
from http import HTTPStatus
|
||||
@@ -105,6 +107,64 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
# DEBUG-ONLY (p2 wedge trap 2026-10-13): per-step batch-decision fingerprint
|
||||
# and event-loop phase logging. Same env as custom_all_reduce_v2 ARDBG.
|
||||
# 1 = sampled/one-line, 2 = + stacks on ONE_SHOT_PUSH. Off by default.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
_plan_seq = 0
|
||||
_plan_none_seq = 0
|
||||
|
||||
|
||||
def _dbg_plan(self, batch) -> None:
|
||||
global _plan_seq, _plan_none_seq
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
_plan_seq += 1
|
||||
rank = torch.distributed.get_rank()
|
||||
if batch is None:
|
||||
# 空转步采样记录,避免 idle 自旋刷爆日志
|
||||
_plan_none_seq += 1
|
||||
if _plan_none_seq & 511:
|
||||
return
|
||||
logger.info("[ARDBG] plan rank=%s step=%d batch=None", rank, _plan_seq)
|
||||
return
|
||||
reqs = batch.reqs
|
||||
fp = 0
|
||||
ext_total = 0
|
||||
for r in reqs:
|
||||
# crc32 而非 hash():后者按进程随机化种子,跨 rank 不可比
|
||||
fp = (fp * 131 + zlib.crc32(r.rid.encode())) & 0xFFFFFFFFFFFFFFFF
|
||||
er = getattr(r, "extend_range", None)
|
||||
el = int(er.length) if er is not None else -1
|
||||
fp = (fp * 131 + el) & 0xFFFFFFFFFFFFFFFF
|
||||
fp = (fp * 131 + int(getattr(r, "prefix_len", -1))) & 0xFFFFFFFFFFFFFFFF
|
||||
if el > 0:
|
||||
ext_total += el
|
||||
logger.info(
|
||||
"[ARDBG] plan rank=%s step=%d nreq=%d ext=%d fp=%016x",
|
||||
rank,
|
||||
_plan_seq,
|
||||
len(reqs),
|
||||
ext_total,
|
||||
fp,
|
||||
)
|
||||
|
||||
|
||||
def _dbg_phase(label: str, batch=None) -> None:
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
if batch is None and label in ("inflight_enter", "inflight_exit"):
|
||||
# 空转迭代的 inflight 相位价值低,采样记录
|
||||
if _plan_seq & 255:
|
||||
return
|
||||
logger.info(
|
||||
"[ARDBG] phase rank=%s step=%s %s nreq=%s",
|
||||
torch.distributed.get_rank(),
|
||||
_plan_seq,
|
||||
label,
|
||||
len(batch.reqs) if batch is not None else "-",
|
||||
)
|
||||
|
||||
|
||||
def should_force_retry(req: Req) -> bool:
|
||||
"""Test hook to force a request into optimistic prefill retry."""
|
||||
@@ -628,6 +688,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
set_schedule_time_batch(batch)
|
||||
|
||||
_dbg_plan(self, batch)
|
||||
return NextBatchPlan(batch_to_run=batch, running_batch=running_batch)
|
||||
|
||||
@torch.no_grad()
|
||||
@@ -658,13 +719,17 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
_dbg_phase("run_batch_enter", batch)
|
||||
result = self.run_batch(batch)
|
||||
_dbg_phase("run_batch_exit", batch)
|
||||
self.process_batch_result(batch, result)
|
||||
else:
|
||||
self._sched_idled = True
|
||||
self.on_idle()
|
||||
|
||||
_dbg_phase("inflight_enter")
|
||||
self.process_disagg_prefill_inflight_queue()
|
||||
_dbg_phase("inflight_exit")
|
||||
|
||||
# Update last_batch
|
||||
self.last_batch = batch
|
||||
@@ -698,7 +763,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
_dbg_phase("run_batch_enter", batch)
|
||||
batch_result = self.run_batch(batch)
|
||||
_dbg_phase("run_batch_exit", batch)
|
||||
self._apply_war_barrier()
|
||||
self.result_queue.append((batch.copy(), batch_result))
|
||||
else:
|
||||
@@ -713,7 +780,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# When the server is idle, do self-check and re-init some states
|
||||
self.on_idle()
|
||||
|
||||
_dbg_phase("inflight_enter")
|
||||
self.process_disagg_prefill_inflight_queue()
|
||||
_dbg_phase("inflight_exit")
|
||||
|
||||
# Run sample of the current batch
|
||||
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from collections import deque
|
||||
from contextlib import nullcontext
|
||||
@@ -43,6 +45,32 @@ if is_npu():
|
||||
DSV4NPUTokenToKVPool,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# DEBUG-ONLY (p2 wedge repro 2026-10-13): env-gated seq logging for host-side
|
||||
# gloo all-reduce sync points. Mirrors SGLANG_AR_SEQ_DEBUG in
|
||||
# custom_all_reduce_v2.py. 1 = one-line, 2 = + python stack. Not for prod.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
_host_ar_seq = 0
|
||||
|
||||
|
||||
def _dbg_log_host_ar(phase: str, polls) -> None:
|
||||
global _host_ar_seq
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
seq = _host_ar_seq
|
||||
_host_ar_seq += 1
|
||||
# 空转迭代(无 poller)采样 1/256,避免 idle 自旋刷爆日志
|
||||
if (not polls) and seq & 255:
|
||||
return
|
||||
logger.info(
|
||||
"[ARDBG] host rank=%s seq=%d phase=%s polls=%s",
|
||||
dist.get_rank(),
|
||||
seq,
|
||||
phase,
|
||||
polls,
|
||||
)
|
||||
|
||||
#########################
|
||||
# Constants & Enums
|
||||
#########################
|
||||
@@ -213,13 +241,20 @@ def poll_and_all_reduce_attn_cp_tp_group(
|
||||
attn_cp_cpu_group: dist.ProcessGroup,
|
||||
attn_tp_cpu_group: dist.ProcessGroup,
|
||||
):
|
||||
# DEBUG-ONLY
|
||||
_dbg_log_host_ar("enter", len(pollers))
|
||||
# First sync across attn-tp ranks so all TP participants for a given (dp, cp)
|
||||
# shard observe the same status transitions.
|
||||
polls = poll_and_all_reduce(pollers, attn_tp_cpu_group)
|
||||
# DEBUG-ONLY
|
||||
_dbg_log_host_ar("tp-done", polls)
|
||||
|
||||
# Then sync across attn-cp ranks, so all TPxCP participants in one DP shard
|
||||
# converge to the same global status.
|
||||
return _all_reduce_polls(polls, attn_cp_cpu_group)
|
||||
result = _all_reduce_polls(polls, attn_cp_cpu_group)
|
||||
# DEBUG-ONLY
|
||||
_dbg_log_host_ar("cp-done", result)
|
||||
return result
|
||||
|
||||
|
||||
def poll_and_all_reduce_with_staging(
|
||||
|
||||
@@ -25,6 +25,8 @@ the kernel captured in the graph dereferences its row at replay time.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import traceback
|
||||
from contextlib import contextmanager
|
||||
from typing import List, NamedTuple, Optional, Tuple
|
||||
|
||||
@@ -68,6 +70,9 @@ MB = 1024 * 1024
|
||||
_ALIGN_BYTES = 1024
|
||||
_SEMAPHORE_BYTES = 128
|
||||
_MAX_GRAPH_INPUTS = 131072
|
||||
# DEBUG-ONLY (p2 wedge repro 2026-10-13): env-gated seq logging for TP device
|
||||
# all-reduce calls. 1 = one-line per call, 2 = + python stack. Not for prod.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
# resolved once at import time; explicit constructor sizes take precedence
|
||||
_DEFAULT_MAX_SIZE = envs.SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB.get() * 1024
|
||||
# forced per-direction workspace sizes; highest priority, override both the
|
||||
@@ -297,6 +302,9 @@ class CustomAllReduceV2:
|
||||
self.pull_enabled,
|
||||
)
|
||||
dist.barrier(group=self.group)
|
||||
# DEBUG-ONLY: per-instance seq counter for SGLANG_AR_SEQ_DEBUG logging
|
||||
self._dbg_seq = 0
|
||||
self._dbg_rank = self.rank
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Algo selection
|
||||
@@ -376,6 +384,27 @@ class CustomAllReduceV2:
|
||||
assert config is not None, f"No config for {nbytes = }"
|
||||
algo, use_graph, use_multicast = config
|
||||
graph_params = self._allocate_graph_row(input, nbytes) if use_graph else None
|
||||
if _AR_SEQ_DEBUG:
|
||||
seq = self._dbg_seq
|
||||
self._dbg_seq += 1
|
||||
# 采样 1/1024:生产高频路径下全量日志会拖垮 P 节点
|
||||
if seq & 1023 == 0:
|
||||
logger.info(
|
||||
"[ARDBG] dev rank=%s seq=%d nbytes=%d algo=%s graph=%s capturing=%s",
|
||||
self._dbg_rank,
|
||||
seq,
|
||||
nbytes,
|
||||
getattr(algo, "name", algo),
|
||||
use_graph,
|
||||
torch.cuda.is_current_stream_capturing(),
|
||||
)
|
||||
if _AR_SEQ_DEBUG >= 2 and getattr(algo, "name", algo) == "ONE_SHOT_PUSH":
|
||||
logger.info(
|
||||
"[ARDBG] stack rank=%s seq=%d\n%s",
|
||||
self._dbg_rank,
|
||||
seq,
|
||||
"".join(traceback.format_stack(limit=15)),
|
||||
)
|
||||
return custom_all_reduce(
|
||||
self.obj,
|
||||
input,
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import replace
|
||||
@@ -132,6 +133,11 @@ COMPONENT_REGISTRY: dict[ComponentType, type[TreeComponent]] = {
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# DEBUG-ONLY (p2 wedge trap 2026-10-13): env-gated logging for the per-step
|
||||
# hicache event sync. Same SGLANG_AR_SEQ_DEBUG switch as the ARDBG sites.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
_hc_step = 0
|
||||
|
||||
|
||||
class _OngoingWriteThrough(NamedTuple):
|
||||
"""Tracks an in-flight D→H write-through operation."""
|
||||
@@ -3533,6 +3539,16 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
def check_hicache_events(self) -> None:
|
||||
"""Called per scheduler step to poll async HiCache events."""
|
||||
global _hc_step
|
||||
if _AR_SEQ_DEBUG:
|
||||
_hc_step += 1
|
||||
if _hc_step & 255 == 0:
|
||||
logger.info(
|
||||
"[ARDBG] hcev rank=%s step=%d linker=%s",
|
||||
torch.distributed.get_rank(),
|
||||
_hc_step,
|
||||
self.linker is not None,
|
||||
)
|
||||
if self.linker is not None:
|
||||
finish_counts = torch.tensor(
|
||||
[
|
||||
@@ -3566,6 +3582,15 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
storage_queue_sizes,
|
||||
extra_pool_names,
|
||||
) = self._sync_hicache_ready_counts()
|
||||
if _AR_SEQ_DEBUG and _hc_step & 255 == 0:
|
||||
logger.info(
|
||||
"[ARDBG] hcsync rank=%s step=%d write=%s load=%s storage=%s",
|
||||
torch.distributed.get_rank(),
|
||||
_hc_step,
|
||||
write_finish_count,
|
||||
load_finish_count,
|
||||
storage_queue_sizes,
|
||||
)
|
||||
self.writing_check(finish_count=write_finish_count)
|
||||
self.loading_check(finish_count=load_finish_count)
|
||||
|
||||
|
||||
@@ -378,6 +378,12 @@ impl RouterConfigBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// G9i pool-aware admission control + fast-P routing config (PD mode).
|
||||
pub fn g9i(mut self, g9i: Option<crate::routers::http::g9i::G9iConfig>) -> Self {
|
||||
self.config.g9i = g9i;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn model_path<S: Into<String>>(mut self, path: S) -> Self {
|
||||
self.config.model_path = Some(path.into());
|
||||
self
|
||||
|
||||
@@ -99,6 +99,9 @@ pub struct RouterConfig {
|
||||
/// Enable WASM support
|
||||
#[serde(default)]
|
||||
pub enable_wasm: bool,
|
||||
/// G9i pool-aware admission control + fast-P routing (PD mode, HTTP only).
|
||||
#[serde(default)]
|
||||
pub g9i: Option<crate::routers::http::g9i::G9iConfig>,
|
||||
}
|
||||
|
||||
/// Tokenizer cache configuration
|
||||
@@ -551,6 +554,7 @@ impl Default for RouterConfig {
|
||||
ca_certificates: vec![],
|
||||
mcp_config: None,
|
||||
enable_wasm: false,
|
||||
g9i: None,
|
||||
server_cert: None,
|
||||
server_key: None,
|
||||
}
|
||||
|
||||
@@ -219,6 +219,155 @@ struct CliArgs {
|
||||
#[arg(long, default_value_t = 30, help_heading = "PD Disaggregation")]
|
||||
worker_startup_check_interval: u64,
|
||||
|
||||
// ==================== G9i: Pool Admission + Fast-P Routing ====================
|
||||
/// Enable G9i pool-aware admission control + fast-P routing (PD HTTP mode)
|
||||
#[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")]
|
||||
g9i_enabled: bool,
|
||||
|
||||
/// Fast-P pool member base URLs (repeatable; router-side role, engine unaware)
|
||||
#[arg(long, action = ArgAction::Append, help_heading = "G9i Pool Admission")]
|
||||
g9i_fast_prefill_url: Vec<String>,
|
||||
|
||||
/// Small-prefill cutoff on NEW tokens (total - prefix hit). Tentative; tune by bench.
|
||||
#[arg(long, default_value_t = 4096, help_heading = "G9i Pool Admission")]
|
||||
g9i_max_new_tokens: u64,
|
||||
|
||||
/// 429 when the least-wait fast-P queue estimate exceeds this many seconds. Tentative.
|
||||
#[arg(long, default_value_t = 5.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_fast_p_queue_reject_secs: f64,
|
||||
|
||||
/// D-pool admission ceiling: sum of in-processing requests across decode nodes. Tentative.
|
||||
#[arg(long, default_value_t = 256, help_heading = "G9i Pool Admission")]
|
||||
g9i_decode_max_running: u64,
|
||||
|
||||
/// 429 when the least-wait REGULAR pool queue estimate exceeds this many seconds. Tentative 10.0; 0=off.
|
||||
#[arg(long, default_value_t = 10.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_regular_p_queue_reject_secs: f64,
|
||||
|
||||
/// Drain-rate floor (tok/s) for queue-wait estimates (div-by-zero / cold-start guard)
|
||||
#[arg(long, default_value_t = 1000.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_drain_rate_floor: f64,
|
||||
|
||||
/// Snapshot poll interval (seconds)
|
||||
#[arg(long, default_value_t = 1, help_heading = "G9i Pool Admission")]
|
||||
g9i_poll_interval_secs: u64,
|
||||
|
||||
/// Snapshot TTL — stale nodes leave pool decisions (seconds)
|
||||
#[arg(long, default_value_t = 5, help_heading = "G9i Pool Admission")]
|
||||
g9i_snapshot_ttl_secs: u64,
|
||||
|
||||
/// No fresh D-pool snapshot: "allow" (default, fail-open + warn) or "reject"
|
||||
#[arg(long, default_value = "allow", value_parser = ["allow", "reject"], help_heading = "G9i Pool Admission")]
|
||||
g9i_d_no_snapshot: String,
|
||||
|
||||
/// /query_cache_hit probe timeout per request (ms)
|
||||
#[arg(long, default_value_t = 2000, help_heading = "G9i Pool Admission")]
|
||||
g9i_cache_hit_query_timeout_ms: u64,
|
||||
|
||||
/// Also least-wait route the regular P pool (default: keep configured policy)
|
||||
#[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")]
|
||||
g9i_regular_pool_least_wait: bool,
|
||||
|
||||
/// Sliding window for TTFT p95/p99 statistics publication (seconds; observability only, never gates)
|
||||
#[arg(long, default_value_t = 300, help_heading = "G9i Pool Admission")]
|
||||
g9i_ttft_window_s: u64,
|
||||
|
||||
/// G9i v2 shadow mode: prefix-chain bookkeeping + shadow classifier + queue model (observability only, zero routing effect; requires --g9i-enabled)
|
||||
#[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")]
|
||||
g9i_shadow_enabled: bool,
|
||||
|
||||
/// Shadow prefix-table capacity in entries (LRU + TTL evicted)
|
||||
#[arg(long, default_value_t = 100_000, help_heading = "G9i Pool Admission")]
|
||||
g9i_shadow_table_max_entries: usize,
|
||||
|
||||
/// Shadow prefix-table entry TTL (seconds)
|
||||
#[arg(long, default_value_t = 86400, help_heading = "G9i Pool Admission")]
|
||||
g9i_shadow_table_ttl_secs: u64,
|
||||
|
||||
/// est_T_D shadow estimate (D-side prealloc wait folded into shadow est_TTFT; observability only, default on; set false to roll back to the P-only model exactly)
|
||||
#[arg(long, default_value_t = true, help_heading = "G9i Pool Admission")]
|
||||
g9i_est_td_enabled: bool,
|
||||
|
||||
/// Shadow-consistent routing: live prefill pick follows the shadow decision's matched_node (requires --g9i-shadow-enabled; rejection gates unchanged; default off)
|
||||
#[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")]
|
||||
g9i_shadow_route: bool,
|
||||
|
||||
/// T_D admission mode: off (default, ≡ current behavior) | shadow (log/count would-rejects, allow) | enforce (real 429s; requires --g9i-shadow-enabled)
|
||||
#[arg(long, default_value = "off", help_heading = "G9i Pool Admission")]
|
||||
g9i_td_admit: String,
|
||||
|
||||
/// T_D hard gate (seconds): when min over decode nodes of T_D exceeds this, every class gets 429 d_prealloc_saturated (0 = disabled)
|
||||
#[arg(long, default_value_t = 5.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_td_gate_secs: f64,
|
||||
|
||||
/// T_PREFER (seconds): prefix-holder affinity yields to a cheaper prefill candidate when the holder's est_TTFT exceeds this
|
||||
#[arg(long, default_value_t = 2.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_t_prefer_secs: f64,
|
||||
|
||||
/// Hard P-queue token cap (physical backstop; effective cap = min(this, drain_ewma x budget)) — 429 p_queue_cap only when ALL nodes over (0 = disabled; follows --g9i-td-admit mode)
|
||||
#[arg(long, default_value_t = 16_000_000, help_heading = "G9i Pool Admission")]
|
||||
g9i_p_queue_token_cap: u64,
|
||||
|
||||
/// P-queue real-queue gate: skip prefill nodes whose engine-reported waiting queue (num_waiting_reqs) exceeds this (0 = disabled; follows --g9i-td-admit mode)
|
||||
#[arg(long, default_value_t = 10, help_heading = "G9i Pool Admission")]
|
||||
g9i_p_queue_reqs_cap: u64,
|
||||
|
||||
/// Admission hold queue: park holdable rejections (d_window_full / p_queue_cap) up to this many seconds instead of an immediate 429 (0 = disabled; enforce mode only; hold time counts into TTFT)
|
||||
#[arg(long, default_value_t = 1.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_admit_hold_secs: f64,
|
||||
|
||||
/// Admission hold queue bound: at most this many parked requests (full queue = immediate 429)
|
||||
#[arg(long, default_value_t = 512, help_heading = "G9i Pool Admission")]
|
||||
g9i_admit_hold_max: u64,
|
||||
|
||||
/// D-ledger pipeline booking TTL (seconds): force-release bookings that never saw a first token (wedge backstop; 0 = disabled)
|
||||
#[arg(long, default_value_t = 120, help_heading = "G9i Pool Admission")]
|
||||
g9i_pipeline_ttl_secs: u64,
|
||||
|
||||
/// D-ledger streaming booking TTL (seconds): force-release bookings stuck mid-decode (0 = disabled)
|
||||
#[arg(long, default_value_t = 600, help_heading = "G9i Pool Admission")]
|
||||
g9i_streaming_ttl_secs: u64,
|
||||
|
||||
/// Ledger reconcile: clamp the D ledger down to the engine-side in-flight when the excess is sustained >60s (subtract-only)
|
||||
#[arg(long, default_value_t = true, help_heading = "G9i Pool Admission")]
|
||||
g9i_ledger_reconcile: bool,
|
||||
|
||||
/// AIMD window initial pipeline depth per decode node (warmup start; clamped to [16, 256])
|
||||
#[arg(long, default_value_t = 128.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_window_init: f64,
|
||||
|
||||
/// AIMD additive-increase step per no-congestion tick (slo_ok / blind_recover; cut & holds unchanged)
|
||||
#[arg(long, default_value_t = 64.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_window_add: f64,
|
||||
|
||||
/// Pin the AIMD admission window at this value (clamped to [16, 256]); skips all window dynamics (0 = normal AIMD)
|
||||
#[arg(long, default_value_t = 0.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_window_fixed: f64,
|
||||
|
||||
/// Prompt-length hard validation: 400 prompt_too_long at the front of admission when even the lower-bound estimate exceeds the context limit (default on)
|
||||
#[arg(long, default_value_t = true, help_heading = "G9i Pool Admission")]
|
||||
g9i_prompt_validation: bool,
|
||||
|
||||
/// Context-length limit (tokens) for the prompt_too_long lower-bound check (0 = disabled)
|
||||
#[arg(long, default_value_t = 1_048_576, help_heading = "G9i Pool Admission")]
|
||||
g9i_prompt_token_limit: u64,
|
||||
|
||||
/// T25 hot-channel admission A/B policy: off (default ≡ current behavior) | reservoir (Hot-class holdable rejections park in a priority reservoir up to --g9i-reservoir-max-ms instead of 429) | hardlane (cold classes may use only (1 - --g9i-hot-reserve-frac) of the d_window/p_queue budgets)
|
||||
#[arg(long, default_value = "off", help_heading = "G9i Pool Admission")]
|
||||
g9i_hot_policy: String,
|
||||
|
||||
/// T25-A reservoir wait budget in milliseconds (timeout → 429 reservoir_timeout; the wait counts into TTFT)
|
||||
#[arg(long, default_value_t = 1000, help_heading = "G9i Pool Admission")]
|
||||
g9i_reservoir_max_ms: u64,
|
||||
|
||||
/// T25-A reservoir capacity: at most this many parked requests (full → immediate 429 reservoir_full)
|
||||
#[arg(long, default_value_t = 512, help_heading = "G9i Pool Admission")]
|
||||
g9i_reservoir_cap: u64,
|
||||
|
||||
/// T25-B hardlane cold-budget reserve fraction: cold classes use at most (1 - frac) of the AIMD d_window and p_queue caps (fixed value, no adaptation)
|
||||
#[arg(long, default_value_t = 0.25, help_heading = "G9i Pool Admission")]
|
||||
g9i_hot_reserve_frac: f64,
|
||||
|
||||
// ==================== Service Discovery (Kubernetes) ====================
|
||||
/// Enable Kubernetes service discovery
|
||||
#[arg(
|
||||
@@ -1070,11 +1219,60 @@ impl CliArgs {
|
||||
.circuit_breaker(!self.disable_circuit_breaker)
|
||||
.enable_wasm(self.enable_wasm)
|
||||
.igw(self.enable_igw)
|
||||
.g9i(self.build_g9i_config()?)
|
||||
.maybe_server_cert_and_key(self.tls_cert_path.as_ref(), self.tls_key_path.as_ref());
|
||||
|
||||
builder.build()
|
||||
}
|
||||
|
||||
/// G9i pool admission + fast-P routing config; None when the feature is off.
|
||||
fn build_g9i_config(&self) -> ConfigResult<Option<smg::routers::http::g9i::G9iConfig>> {
|
||||
if !self.g9i_enabled && self.g9i_fast_prefill_url.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let config = smg::routers::http::g9i::G9iConfig {
|
||||
enabled: self.g9i_enabled,
|
||||
fast_prefill_urls: self.g9i_fast_prefill_url.clone(),
|
||||
max_new_tokens: self.g9i_max_new_tokens,
|
||||
fast_p_queue_reject_secs: self.g9i_fast_p_queue_reject_secs,
|
||||
regular_p_queue_reject_secs: self.g9i_regular_p_queue_reject_secs,
|
||||
decode_max_running: self.g9i_decode_max_running,
|
||||
drain_rate_floor: self.g9i_drain_rate_floor,
|
||||
poll_interval_secs: self.g9i_poll_interval_secs,
|
||||
snapshot_ttl_secs: self.g9i_snapshot_ttl_secs,
|
||||
d_no_snapshot: self.g9i_d_no_snapshot.clone(),
|
||||
cache_hit_query_timeout_ms: self.g9i_cache_hit_query_timeout_ms,
|
||||
regular_pool_least_wait: self.g9i_regular_pool_least_wait,
|
||||
ttft_window_s: self.g9i_ttft_window_s,
|
||||
shadow_enabled: self.g9i_shadow_enabled,
|
||||
shadow_table_max_entries: self.g9i_shadow_table_max_entries,
|
||||
shadow_table_ttl_secs: self.g9i_shadow_table_ttl_secs,
|
||||
est_td_enabled: self.g9i_est_td_enabled,
|
||||
shadow_route: self.g9i_shadow_route,
|
||||
td_admit: self.g9i_td_admit.clone(),
|
||||
td_gate_secs: self.g9i_td_gate_secs,
|
||||
t_prefer_secs: self.g9i_t_prefer_secs,
|
||||
p_queue_token_cap: self.g9i_p_queue_token_cap,
|
||||
p_queue_reqs_cap: self.g9i_p_queue_reqs_cap,
|
||||
admit_hold_secs: self.g9i_admit_hold_secs,
|
||||
admit_hold_max: self.g9i_admit_hold_max,
|
||||
pipeline_ttl_secs: self.g9i_pipeline_ttl_secs,
|
||||
streaming_ttl_secs: self.g9i_streaming_ttl_secs,
|
||||
ledger_reconcile: self.g9i_ledger_reconcile,
|
||||
window_init: self.g9i_window_init,
|
||||
window_add: self.g9i_window_add,
|
||||
window_fixed: self.g9i_window_fixed,
|
||||
prompt_validation: self.g9i_prompt_validation,
|
||||
prompt_token_limit: self.g9i_prompt_token_limit,
|
||||
hot_policy: self.g9i_hot_policy.clone(),
|
||||
reservoir_max_ms: self.g9i_reservoir_max_ms,
|
||||
reservoir_cap: self.g9i_reservoir_cap,
|
||||
hot_reserve_frac: self.g9i_hot_reserve_frac,
|
||||
};
|
||||
config.validate().map_err(|reason| ConfigError::ValidationFailed { reason })?;
|
||||
Ok(Some(config))
|
||||
}
|
||||
|
||||
fn to_server_config(&self, router_config: RouterConfig) -> ServerConfig {
|
||||
let service_discovery_config = if self.service_discovery {
|
||||
// Get router discovery config from router_config.discovery if available
|
||||
|
||||
@@ -334,6 +334,64 @@ pub(crate) fn init_metrics() {
|
||||
smg_mesh::metrics::init_mesh_metrics();
|
||||
}
|
||||
|
||||
/// T25b: explicit Prometheus histogram buckets for every gateway histogram
|
||||
/// NOT covered by the `duration_seconds` suffix matcher (those keep the
|
||||
/// configurable duration_buckets). Root cause being fixed: without explicit
|
||||
/// buckets the exporter renders histograms as client-side SUMMARIES
|
||||
/// (DDSketch over a 3×20s rolling window); in the v16 deployment those
|
||||
/// summaries exported all-zero quantiles while _sum/_count stayed healthy
|
||||
/// (the rolling-window path silently drops samples when the quanta clock
|
||||
/// reading falls behind the newest bucket begin, poisoning the summary
|
||||
/// permanently). Explicit buckets render real Prometheus histograms —
|
||||
/// aggregable across time and replicas with histogram_quantile, and immune
|
||||
/// to the summary path entirely.
|
||||
///
|
||||
/// Groups by scale (keep in sync when adding a histogram!):
|
||||
/// * TTFT-scale seconds (0.5–120s): TTFT, e2e, est_T_P/T_D, residuals
|
||||
/// (signed values fall into the lowest bucket — acceptable),
|
||||
/// bootstrap actuals;
|
||||
/// * fine seconds (5ms–5s): TPOT, hold/reservoir waits, retry backoff;
|
||||
/// * prefix lookup depth (ring count, 1–400);
|
||||
/// * shadow-table evicted entry age (1s–1d).
|
||||
fn histogram_bucket_overrides() -> Vec<(Matcher, Vec<f64>)> {
|
||||
let ttft: Vec<f64> = vec![0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 60.0, 120.0];
|
||||
let fine: Vec<f64> = vec![0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0];
|
||||
let depth: Vec<f64> = vec![1.0, 2.0, 3.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 400.0];
|
||||
let age: Vec<f64> = vec![1.0, 10.0, 60.0, 300.0, 900.0, 3600.0, 21600.0, 86400.0];
|
||||
let mut out: Vec<(Matcher, Vec<f64>)> = Vec::new();
|
||||
for name in [
|
||||
"smg_g9i_ttft_seconds",
|
||||
"smg_g9i_class_ttft_seconds",
|
||||
"smg_g9i_nonstream_e2e_secs",
|
||||
"smg_g9i_est_tp_secs",
|
||||
"smg_g9i_est_td_secs",
|
||||
"smg_g9i_est_td_slot_secs",
|
||||
"smg_g9i_shadow_ttft_est_error_seconds",
|
||||
"smg_g9i_est_vs_actual_residual",
|
||||
"smg_g9i_bootstrap_actual_secs",
|
||||
"smg_router_ttft_seconds",
|
||||
] {
|
||||
out.push((Matcher::Full(name.to_string()), ttft.clone()));
|
||||
}
|
||||
for name in [
|
||||
"smg_router_tpot_seconds",
|
||||
"smg_g9i_admit_hold_secs",
|
||||
"smg_g9i_reservoir_wait_seconds",
|
||||
"smg_worker_retry_backoff_seconds",
|
||||
] {
|
||||
out.push((Matcher::Full(name.to_string()), fine.clone()));
|
||||
}
|
||||
out.push((
|
||||
Matcher::Full("smg_g9i_shadow_prefix_lookup_depth".to_string()),
|
||||
depth,
|
||||
));
|
||||
out.push((
|
||||
Matcher::Full("smg_g9i_shadow_prefix_table_evicted_age_seconds".to_string()),
|
||||
age,
|
||||
));
|
||||
out
|
||||
}
|
||||
|
||||
pub fn start_prometheus(config: PrometheusConfig) {
|
||||
init_metrics();
|
||||
|
||||
@@ -351,11 +409,20 @@ pub fn start_prometheus(config: PrometheusConfig) {
|
||||
.unwrap_or(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)));
|
||||
let socket_addr = SocketAddr::new(ip_addr, config.port);
|
||||
|
||||
PrometheusBuilder::new()
|
||||
let mut builder = PrometheusBuilder::new()
|
||||
.with_http_listener(socket_addr)
|
||||
.upkeep_timeout(Duration::from_secs(5 * 60))
|
||||
.set_buckets_for_metric(duration_matcher, &duration_bucket)
|
||||
.expect("failed to set duration bucket")
|
||||
.expect("failed to set duration bucket");
|
||||
// T25b: every remaining gateway histogram gets explicit buckets (see
|
||||
// histogram_bucket_overrides) — no metric falls back to the broken
|
||||
// client-side summary path.
|
||||
for (matcher, buckets) in histogram_bucket_overrides() {
|
||||
builder = builder
|
||||
.set_buckets_for_metric(matcher, &buckets)
|
||||
.expect("failed to set g9i histogram buckets");
|
||||
}
|
||||
builder
|
||||
.install()
|
||||
.expect("failed to install Prometheus metrics exporter");
|
||||
}
|
||||
@@ -1513,4 +1580,61 @@ mod tests {
|
||||
assert_eq!(method_to_static_str("POST"), "POST");
|
||||
assert_eq!(method_to_static_str("UNKNOWN"), "OTHER");
|
||||
}
|
||||
|
||||
/// T25b smoke: with the production bucket overrides applied, g9i
|
||||
/// histograms render as real Prometheus histograms with non-zero le
|
||||
/// bucket counts (histogram_quantile-able) — never the client-side
|
||||
/// summary path that exported all-zero quantiles in v16.
|
||||
#[test]
|
||||
fn t25b_g9i_histograms_render_bucketed() {
|
||||
let mut builder = PrometheusBuilder::new();
|
||||
for (matcher, buckets) in histogram_bucket_overrides() {
|
||||
builder = builder
|
||||
.set_buckets_for_metric(matcher, &buckets)
|
||||
.expect("bucket override must apply");
|
||||
}
|
||||
let recorder = builder.build_recorder();
|
||||
let handle = recorder.handle();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
let ttft = histogram!("smg_g9i_class_ttft_seconds", "class" => "hot");
|
||||
for v in [0.8, 1.5, 3.0, 12.0, 45.0] {
|
||||
ttft.record(v);
|
||||
}
|
||||
histogram!("smg_g9i_shadow_prefix_lookup_depth").record(7.0);
|
||||
histogram!("smg_g9i_shadow_prefix_lookup_depth").record(120.0);
|
||||
});
|
||||
let out = handle.render();
|
||||
// Real histogram exposition, not summary.
|
||||
assert!(
|
||||
out.contains("# TYPE smg_g9i_class_ttft_seconds histogram"),
|
||||
"class ttft renders as histogram:\n{out}"
|
||||
);
|
||||
assert!(
|
||||
!out.contains("smg_g9i_class_ttft_seconds{quantile="),
|
||||
"no summary quantile lines:\n{out}"
|
||||
);
|
||||
// 3 of the 5 samples (0.8, 1.5, 3.0) land at or below le=5.
|
||||
assert!(
|
||||
out.contains("smg_g9i_class_ttft_seconds_bucket{class=\"hot\",le=\"5\"} 3"),
|
||||
"le=5 bucket counts 3 samples:\n{out}"
|
||||
);
|
||||
assert!(
|
||||
out.contains("smg_g9i_class_ttft_seconds_bucket{class=\"hot\",le=\"+Inf\"} 5"),
|
||||
"+Inf bucket counts all 5:\n{out}"
|
||||
);
|
||||
// lookup_depth: 7 -> le=10; both samples (7, 120) <= 200
|
||||
// (Prometheus buckets are cumulative).
|
||||
assert!(
|
||||
out.contains("smg_g9i_shadow_prefix_lookup_depth_bucket{le=\"10\"} 1"),
|
||||
"depth 7 lands in le=10:\n{out}"
|
||||
);
|
||||
assert!(
|
||||
out.contains("smg_g9i_shadow_prefix_lookup_depth_bucket{le=\"200\"} 2"),
|
||||
"both depths land at or below le=200 (cumulative):\n{out}"
|
||||
);
|
||||
assert!(
|
||||
out.contains("smg_g9i_shadow_prefix_lookup_depth_count 2"),
|
||||
"count line present:\n{out}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,979 @@
|
||||
//! G9i v2 P1 shadow mode — T1: prefix hash chain (pure function layer).
|
||||
//!
|
||||
//! Design source: `experiments/g9i/g9i-v2-design-20261013.md` §3.2/§4.1,
|
||||
//! content-handling constraints C4/C10. This module is **read-only with
|
||||
//! respect to routing**: it computes hashes, it never touches dispatch.
|
||||
//!
|
||||
//! Why a chain: a chat body with `m` messages has exactly `m` prefix
|
||||
//! boundaries, and the session bookkeeping table ("prefix → last serving
|
||||
//! node + exact token count") needs a stable key per boundary. Rolling the
|
||||
//! hash forward per message gives all `m` keys in one pass; lookup then
|
||||
//! probes from deep to shallow and takes the deepest hit. In practice a new
|
||||
//! turn is `previous body + assistant reply + new user message`, so hits
|
||||
//! usually land at `h[m-2]`.
|
||||
//!
|
||||
//! seed = xxh3(canonical_json({model, chat_template_kwargs, tools_hash, system}))
|
||||
//! h[0] = seed
|
||||
//! h[k] = xxh3(h[k-1] LE bytes ‖ len(role) ‖ role ‖ tagged content
|
||||
//! ‖ tagged canonical(tool_calls) ‖ tagged meta) k = 1..=m
|
||||
//!
|
||||
//! Encoding decisions (why, not what):
|
||||
//! * Streaming `Xxh3` instead of concatenating into a `Vec`: chat bodies
|
||||
//! with base64 images are routinely hundreds of KB; copying them per
|
||||
//! chain step would be pure waste.
|
||||
//! * Every variable-length field is length-prefixed (u64 LE) and every
|
||||
//! content piece carries a type tag. Without that, `["ab","c"]` and
|
||||
//! `["a","bc"]` (or role/content, part/part splits) would hash
|
||||
//! identically — silent false cache hits.
|
||||
//! * Missing role defaults to "user" (OpenAI clients routinely omit it);
|
||||
//! missing `model` / `chat_template_kwargs` / `tools` / system message
|
||||
//! serialize as JSON `null` in the seed document — a fixed placeholder
|
||||
//! that cannot collide with any real value because `null` is not a
|
||||
//! string/object the field could otherwise hold.
|
||||
//! * Byte-level semantics for multimodal (C4): `image_url` URLs are
|
||||
//! hashed as the raw string — `http(s):` URLs as-is, `data:` URLs
|
||||
//! including the base64 payload, **never fetched, never decoded**.
|
||||
//! Same image at different URLs hashes differently; that is correct —
|
||||
//! the rendered prompt bytes differ.
|
||||
//! * `is_unclassifiable` and `chain_hashes` share one traversal
|
||||
//! (`walk_messages`) so the "weird content" judgement can never drift
|
||||
//! between the two call sites (C10). Unknown part types still
|
||||
//! contribute a deterministic tag to the chain — the chain stays
|
||||
//! well-defined, the caller just must not trust it for table lookups
|
||||
//! when `is_unclassifiable` is true.
|
||||
//! * `content: null` / missing renders as empty content in every chat
|
||||
//! template, so it hashes exactly like `""` — NOT unclassifiable.
|
||||
//! Flagging it would disenfranchise tool-use conversations (assistant
|
||||
//! turns with `content: null` + `tool_calls`), our main traffic shape.
|
||||
//! Genuinely malformed content (number/object/bool) keeps the
|
||||
//! Empty + unclassifiable treatment.
|
||||
//! * `tool_calls` are rendered into the prompt by the chat template, so
|
||||
//! they MUST enter the chain (as canonical JSON, key-order
|
||||
//! insensitive) — otherwise two conversations with different tool
|
||||
//! calls could share a hash: a false hit in the dangerous direction.
|
||||
//! `reasoning_content` is deliberately excluded: templates never
|
||||
//! render it, so it cannot change prompt bytes.
|
||||
//! * `tool_call_id` (on `role:"tool"` results) and `name` (any role)
|
||||
//! are rendered by most templates, so they enter the chain as one
|
||||
//! canonical meta object. The META segment is unconditional — a
|
||||
//! message without these fields contributes `TAG_META ‖ len=0`,
|
||||
//! keeping the formula single-path (no presence/absence ambiguity).
|
||||
//!
|
||||
//! T23 (intra-message chunking): a text piece longer than `CHUNK_BYTES`
|
||||
//! (2048) is first rolled into bare chunk rings `xxh3(prev ‖ chunk)` —
|
||||
//! no tag, no length prefix — in content order, and the message-final
|
||||
//! ring then carries only the remaining ≤2048-byte tail in its content
|
||||
//! segment. Why: a per-message chain is structurally blind inside one
|
||||
//! long message (one message = one ring, so a shared 100KB prefix with
|
||||
//! a different tail hits nothing — v13 measured 9034/9034 depth-0
|
||||
//! misses on single-message long-prompt traffic). Chunk rings give the
|
||||
//! lookup table mid-message anchors at a fixed granularity. Texts of
|
||||
//! ≤2048 bytes emit no chunk rings, so short-message chains stay
|
||||
//! byte-identical to the pre-T23 formula (pinned by
|
||||
//! `short_messages_match_legacy_chain_byte_for_byte`). Because one
|
||||
//! message may now own several rings, `chain_hashes_indexed` also
|
||||
//! returns each ring's message index; `classify` needs it to map a
|
||||
//! hit's ring position back into the message index space of
|
||||
//! `msg_bytes` / `msg_image`.
|
||||
|
||||
use serde_json::Value;
|
||||
use xxhash_rust::xxh3::{xxh3_64, Xxh3};
|
||||
|
||||
/// Part-type tags mixed in before each content piece. Terminated with `\0`
|
||||
/// so tags are self-delimiting; content bytes are length-prefixed on top.
|
||||
const TAG_TEXT: &[u8] = b"text\0";
|
||||
const TAG_IMAGE_URL: &[u8] = b"image_url\0";
|
||||
const TAG_UNKNOWN: &[u8] = b"unknown\0";
|
||||
const TAG_TOOL_CALLS: &[u8] = b"tool_calls\0";
|
||||
const TAG_META: &[u8] = b"meta\0";
|
||||
|
||||
/// T23: intra-message chunk size in bytes. A text piece longer than this
|
||||
/// contributes `ceil`-style bare chunk rings before its message-final
|
||||
/// ring (see module docs). 2048 bytes ≈ 500-600 tokens of English/JSON —
|
||||
/// fine enough to rescue shared-prefix hits, coarse enough that typical
|
||||
/// multi-turn chats (short messages) never pay the extra hash passes.
|
||||
const CHUNK_BYTES: usize = 2048;
|
||||
|
||||
/// Chain seed: folds every body field that changes the rendered prompt but
|
||||
/// lives outside `messages` into one u64.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub struct ChainSeed {
|
||||
pub value: u64,
|
||||
}
|
||||
|
||||
impl ChainSeed {
|
||||
/// Build the seed from a chat request body.
|
||||
///
|
||||
/// Inputs: `model`, `chat_template_kwargs`, `tools` (folded to a single
|
||||
/// hash of its canonical form), and the first `role=="system"` message's
|
||||
/// string content. Missing fields become JSON `null` in the seed
|
||||
/// document (fixed placeholder, see module docs). Canonicalization
|
||||
/// sorts object keys recursively and emits no whitespace, so semantically
|
||||
/// equal bodies from different clients produce the same seed.
|
||||
///
|
||||
/// Note: a system message is *also* hashed as `messages[0]` inside the
|
||||
/// chain — deliberately. The chain must stay 1:1 with `messages`
|
||||
/// indices (the lookup table keys are `h[k]` by position), and the
|
||||
/// redundancy only costs one extra mix of already-known bytes.
|
||||
pub fn from_body(body: &Value) -> ChainSeed {
|
||||
let system_content: Option<&str> = body
|
||||
.get("messages")
|
||||
.and_then(Value::as_array)
|
||||
.and_then(|msgs| {
|
||||
msgs.iter()
|
||||
.find(|m| m.get("role").and_then(Value::as_str) == Some("system"))
|
||||
})
|
||||
.and_then(|m| m.get("content"))
|
||||
.and_then(Value::as_str);
|
||||
|
||||
let tools_hash: Option<u64> = body
|
||||
.get("tools")
|
||||
.map(|t| xxh3_64(canonical_json(t).as_bytes()));
|
||||
|
||||
let seed_doc = serde_json::json!({
|
||||
"model": body.get("model").cloned().unwrap_or(Value::Null),
|
||||
"chat_template_kwargs": body
|
||||
.get("chat_template_kwargs")
|
||||
.cloned()
|
||||
.unwrap_or(Value::Null),
|
||||
"tools_hash": tools_hash,
|
||||
"system": system_content,
|
||||
});
|
||||
ChainSeed {
|
||||
value: xxh3_64(canonical_json(&seed_doc).as_bytes()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the chain hashes `h[1..=m]`, one ring per message (plus T23
|
||||
/// intra-message chunk rings), in order.
|
||||
///
|
||||
/// `messages` missing / not an array / empty → empty `Vec`. The caller
|
||||
/// treats an empty chain as "cannot classify" (same fall-back path as
|
||||
/// `is_unclassifiable`); we never panic on malformed bodies because this
|
||||
/// sits on the hot request path.
|
||||
///
|
||||
/// The returned chain is deterministic even when `is_unclassifiable(body)`
|
||||
/// is true — unknown parts contribute their tag — but such chains are only
|
||||
/// suitable for logging/shadow comparison, not for bookkeeping lookups.
|
||||
///
|
||||
/// This is the chain-only view of `chain_hashes_indexed`; callers that
|
||||
/// index into per-message arrays (`msg_bytes`, `msg_image`) need the
|
||||
/// indexed form to translate ring positions to message indices.
|
||||
pub fn chain_hashes(seed: &ChainSeed, body: &Value) -> Vec<u64> {
|
||||
chain_hashes_indexed(seed, body).0
|
||||
}
|
||||
|
||||
/// `chain_hashes` plus, for each ring, the index of the message it
|
||||
/// belongs to (chunk rings and the message-final ring of message `k`
|
||||
/// all carry `k`). Both returned vecs always have the same length.
|
||||
pub fn chain_hashes_indexed(seed: &ChainSeed, body: &Value) -> (Vec<u64>, Vec<usize>) {
|
||||
let (msgs, _) = walk_messages(body);
|
||||
let mut chain = Vec::with_capacity(msgs.len());
|
||||
let mut ring_msg = Vec::with_capacity(msgs.len());
|
||||
let mut prev = seed.value;
|
||||
for (mi, msg) in msgs.iter().enumerate() {
|
||||
// T23 pass 1: roll oversized text pieces into bare chunk rings
|
||||
// (h = xxh3(prev ‖ chunk), no tag/length) and keep the ≤2048-byte
|
||||
// tails for the message-final ring. Short texts emit nothing here
|
||||
// and their tails ARE the originals — the pre-T23 byte stream.
|
||||
let content = match &msg.content {
|
||||
ContentView::Str(s) => {
|
||||
ContentView::Str(roll_chunks(&mut prev, &mut chain, &mut ring_msg, mi, s))
|
||||
}
|
||||
ContentView::Parts(parts) => {
|
||||
let mut viewed = Vec::with_capacity(parts.len());
|
||||
for part in parts {
|
||||
viewed.push(match part {
|
||||
PartView::Text(t) => PartView::Text(roll_chunks(
|
||||
&mut prev,
|
||||
&mut chain,
|
||||
&mut ring_msg,
|
||||
mi,
|
||||
t,
|
||||
)),
|
||||
PartView::ImageUrl(u) => PartView::ImageUrl(u),
|
||||
PartView::Unknown => PartView::Unknown,
|
||||
});
|
||||
}
|
||||
ContentView::Parts(viewed)
|
||||
}
|
||||
ContentView::Empty => ContentView::Empty,
|
||||
};
|
||||
|
||||
// Pass 2: the message-final ring — the pre-T23 formula unchanged,
|
||||
// except its content segment now hashes the pass-1 tails.
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&prev.to_le_bytes());
|
||||
|
||||
let role = msg.role.as_bytes();
|
||||
h.update(&(role.len() as u64).to_le_bytes());
|
||||
h.update(role);
|
||||
|
||||
match &content {
|
||||
ContentView::Str(s) => {
|
||||
update_tagged(&mut h, TAG_TEXT, s.as_bytes());
|
||||
}
|
||||
ContentView::Parts(parts) => {
|
||||
for part in parts {
|
||||
match part {
|
||||
PartView::Text(t) => update_tagged(&mut h, TAG_TEXT, t.as_bytes()),
|
||||
PartView::ImageUrl(u) => {
|
||||
update_tagged(&mut h, TAG_IMAGE_URL, u.as_bytes())
|
||||
}
|
||||
PartView::Unknown => h.update(TAG_UNKNOWN),
|
||||
}
|
||||
}
|
||||
}
|
||||
ContentView::Empty => {}
|
||||
}
|
||||
|
||||
// Third segment: tool_calls are rendered into prompt bytes by the
|
||||
// chat template, so they must be in the chain.
|
||||
if let Some(tc) = &msg.tool_calls {
|
||||
update_tagged(&mut h, TAG_TOOL_CALLS, tc.as_bytes());
|
||||
}
|
||||
|
||||
// Fourth segment (unconditional): name / tool_call_id meta. Empty
|
||||
// meta hashes as TAG_META ‖ len=0 — one formula for all messages.
|
||||
update_tagged(&mut h, TAG_META, msg.extra.as_bytes());
|
||||
|
||||
prev = h.digest();
|
||||
chain.push(prev);
|
||||
ring_msg.push(mi);
|
||||
}
|
||||
(chain, ring_msg)
|
||||
}
|
||||
|
||||
/// Roll `prev` forward over `text`'s full chunks, emitting one bare ring
|
||||
/// per chunk (`xxh3(prev ‖ chunk bytes)`), and return the remaining tail
|
||||
/// (≤ CHUNK_BYTES, possibly empty) for the message-final ring. Texts of
|
||||
/// ≤ CHUNK_BYTES emit nothing and are returned untouched.
|
||||
fn roll_chunks<'a>(
|
||||
prev: &mut u64,
|
||||
chain: &mut Vec<u64>,
|
||||
ring_msg: &mut Vec<usize>,
|
||||
mi: usize,
|
||||
text: &'a str,
|
||||
) -> &'a str {
|
||||
let mut rest = text;
|
||||
while rest.len() > CHUNK_BYTES {
|
||||
let (chunk, tail) = take_chunk(rest);
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&prev.to_le_bytes());
|
||||
h.update(chunk.as_bytes());
|
||||
*prev = h.digest();
|
||||
chain.push(*prev);
|
||||
ring_msg.push(mi);
|
||||
rest = tail;
|
||||
}
|
||||
rest
|
||||
}
|
||||
|
||||
/// Split off the leading chunk of at most CHUNK_BYTES bytes, backing off
|
||||
/// to the previous UTF-8 char boundary when the cut lands mid-character.
|
||||
/// The caller guarantees `s.len() > CHUNK_BYTES`, so the boundary walk
|
||||
/// can never reach 0 (the first char is at most 4 bytes).
|
||||
fn take_chunk(s: &str) -> (&str, &str) {
|
||||
debug_assert!(s.len() > CHUNK_BYTES);
|
||||
let mut end = CHUNK_BYTES;
|
||||
while !s.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
s.split_at(end)
|
||||
}
|
||||
|
||||
/// Whether the body contains content this module has no hashing semantics
|
||||
/// for: unknown part types (`audio_url`, `video_url`, `file`, ...),
|
||||
/// malformed parts, unsupported `image_url` schemes, or genuinely
|
||||
/// malformed message content (number/object/bool). Null/missing content
|
||||
/// and `tool_calls`-carrying assistant turns return false — both have
|
||||
/// well-defined rendering semantics. See `walk_messages` — this is the
|
||||
/// same traversal, so the answer cannot disagree with what `chain_hashes`
|
||||
/// actually hashed.
|
||||
pub fn is_unclassifiable(body: &Value) -> bool {
|
||||
walk_messages(body).1
|
||||
}
|
||||
|
||||
/// Mix `tag ‖ len(bytes) ‖ bytes` into the hasher. Length prefix kills
|
||||
/// concatenation ambiguity between adjacent pieces.
|
||||
fn update_tagged(h: &mut Xxh3, tag: &[u8], bytes: &[u8]) {
|
||||
h.update(tag);
|
||||
h.update(&(bytes.len() as u64).to_le_bytes());
|
||||
h.update(bytes);
|
||||
}
|
||||
|
||||
/// One message, flattened to just what the hash needs. Borrows from the
|
||||
/// body — zero copies on the hot path, except `tool_calls` / `extra`
|
||||
/// whose canonical forms must be owned.
|
||||
struct MessageView<'a> {
|
||||
role: &'a str,
|
||||
content: ContentView<'a>,
|
||||
/// Canonical JSON of `tool_calls` when the field exists and is an
|
||||
/// array (a non-array shape can't be rendered by the template either,
|
||||
/// so it's treated as absent).
|
||||
tool_calls: Option<String>,
|
||||
/// Canonical JSON of the meta object {"name"?, "tool_call_id"?}, or
|
||||
/// empty when neither field applies. See `walk_messages` for which
|
||||
/// fields qualify per role.
|
||||
extra: String,
|
||||
}
|
||||
|
||||
enum ContentView<'a> {
|
||||
Str(&'a str),
|
||||
Parts(Vec<PartView<'a>>),
|
||||
/// Genuinely malformed content (number/object/bool): contributes
|
||||
/// nothing, flagged unclassifiable. Null/missing content never lands
|
||||
/// here — it maps to `Str("")`.
|
||||
Empty,
|
||||
}
|
||||
|
||||
enum PartView<'a> {
|
||||
Text(&'a str),
|
||||
ImageUrl(&'a str),
|
||||
/// Unrecognized or malformed part: deterministic placeholder.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// Single traversal shared by `chain_hashes` and `is_unclassifiable`
|
||||
/// (C10: one source of truth for "what can we hash").
|
||||
fn walk_messages(body: &Value) -> (Vec<MessageView<'_>>, bool) {
|
||||
let Some(msgs) = body.get("messages").and_then(Value::as_array) else {
|
||||
return (Vec::new(), false);
|
||||
};
|
||||
let mut unclassifiable = false;
|
||||
let mut out = Vec::with_capacity(msgs.len());
|
||||
for m in msgs {
|
||||
let role = m.get("role").and_then(Value::as_str).unwrap_or("user");
|
||||
// `reasoning_content` is deliberately NOT hashed: chat templates do
|
||||
// not render it, so it cannot change prompt bytes — mixing it in
|
||||
// would only manufacture misses.
|
||||
let tool_calls = match m.get("tool_calls") {
|
||||
Some(v @ Value::Array(_)) => Some(canonical_json(v)),
|
||||
_ => None,
|
||||
};
|
||||
// Meta segment: `name` (any role) and `tool_call_id` (tool results
|
||||
// only) are rendered by most templates -> they must be in the
|
||||
// chain. String and non-string values alike go through the
|
||||
// canonical serializer, so wire-shape differences can't collide.
|
||||
// Explicit null counts as absent (templates render it as nothing).
|
||||
let mut meta = serde_json::Map::new();
|
||||
if let Some(v) = m.get("name").filter(|v| !v.is_null()) {
|
||||
meta.insert("name".to_string(), v.clone());
|
||||
}
|
||||
if role == "tool" {
|
||||
if let Some(v) = m.get("tool_call_id").filter(|v| !v.is_null()) {
|
||||
meta.insert("tool_call_id".to_string(), v.clone());
|
||||
}
|
||||
}
|
||||
let extra = if meta.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
canonical_json(&Value::Object(meta))
|
||||
};
|
||||
let content = match m.get("content") {
|
||||
Some(Value::String(s)) => ContentView::Str(s.as_str()),
|
||||
Some(Value::Array(parts)) => {
|
||||
let mut views = Vec::with_capacity(parts.len());
|
||||
for p in parts {
|
||||
match classify_part(p) {
|
||||
Some(view) => views.push(view),
|
||||
None => {
|
||||
unclassifiable = true;
|
||||
views.push(PartView::Unknown);
|
||||
}
|
||||
}
|
||||
}
|
||||
ContentView::Parts(views)
|
||||
}
|
||||
// null / absent renders as empty content in every chat
|
||||
// template — hash exactly like "" and stay classifiable. This
|
||||
// is the tool-use conversation shape (assistant turn with
|
||||
// content:null + tool_calls), our main traffic.
|
||||
None | Some(Value::Null) => ContentView::Str(""),
|
||||
// number/object/bool content: genuinely malformed, no defined
|
||||
// rendering semantics — fail closed.
|
||||
_ => {
|
||||
unclassifiable = true;
|
||||
ContentView::Empty
|
||||
}
|
||||
};
|
||||
out.push(MessageView {
|
||||
role,
|
||||
content,
|
||||
tool_calls,
|
||||
extra,
|
||||
});
|
||||
}
|
||||
(out, unclassifiable)
|
||||
}
|
||||
|
||||
/// Classify one content part. `None` = no hashing semantics for this part
|
||||
/// (unknown type, malformed shape, or an `image_url` scheme outside
|
||||
/// http/https/data — e.g. `file://` would tempt a local read we must never
|
||||
/// do).
|
||||
fn classify_part(p: &Value) -> Option<PartView<'_>> {
|
||||
let ty = p.get("type").and_then(Value::as_str)?;
|
||||
match ty {
|
||||
"text" => p
|
||||
.get("text")
|
||||
.and_then(Value::as_str)
|
||||
.map(PartView::Text),
|
||||
"image_url" => {
|
||||
// Both wire shapes exist in the wild:
|
||||
// {"image_url":{"url":...}} and {"image_url":"..."}.
|
||||
let url = match p.get("image_url") {
|
||||
Some(Value::String(s)) => Some(s.as_str()),
|
||||
Some(v @ Value::Object(_)) => v.get("url").and_then(Value::as_str),
|
||||
_ => None,
|
||||
}?;
|
||||
if url.starts_with("http://") || url.starts_with("https://") || url.starts_with("data:")
|
||||
{
|
||||
Some(PartView::ImageUrl(url))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical JSON: object keys sorted recursively, no whitespace. Hand-rolled
|
||||
/// (no new crate) because we only need it for seed inputs, which are small —
|
||||
/// `tools` is hashed from its canonical form, and the seed doc itself is
|
||||
/// four fields.
|
||||
fn canonical_json(v: &Value) -> String {
|
||||
let mut out = String::new();
|
||||
write_canonical(v, &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
fn write_canonical(v: &Value, out: &mut String) {
|
||||
match v {
|
||||
Value::Object(map) => {
|
||||
out.push('{');
|
||||
let mut keys: Vec<&String> = map.keys().collect();
|
||||
keys.sort();
|
||||
for (i, k) in keys.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
// Serializing a &str Value is infallible; avoid unwrap anyway.
|
||||
if let Ok(k_json) = serde_json::to_string(k) {
|
||||
out.push_str(&k_json);
|
||||
}
|
||||
out.push(':');
|
||||
write_canonical(&map[*k], out);
|
||||
}
|
||||
out.push('}');
|
||||
}
|
||||
Value::Array(arr) => {
|
||||
out.push('[');
|
||||
for (i, item) in arr.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
write_canonical(item, out);
|
||||
}
|
||||
out.push(']');
|
||||
}
|
||||
other => {
|
||||
if let Ok(s) = serde_json::to_string(other) {
|
||||
out.push_str(&s);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn text_body(msgs: Vec<Value>) -> Value {
|
||||
json!({"model": "dsv41-flash", "messages": msgs})
|
||||
}
|
||||
|
||||
fn user(t: &str) -> Value {
|
||||
json!({"role": "user", "content": t})
|
||||
}
|
||||
|
||||
fn assistant(t: &str) -> Value {
|
||||
json!({"role": "assistant", "content": t})
|
||||
}
|
||||
|
||||
fn chain(body: &Value) -> Vec<u64> {
|
||||
chain_hashes(&ChainSeed::from_body(body), body)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_prefix_different_tail() {
|
||||
let a = text_body(vec![user("hello"), assistant("hi"), user("tail A")]);
|
||||
let b = text_body(vec![user("hello"), assistant("hi"), user("tail B")]);
|
||||
let ca = chain(&a);
|
||||
let cb = chain(&b);
|
||||
assert_eq!(ca.len(), 3);
|
||||
assert_eq!(cb.len(), 3);
|
||||
assert_eq!(ca[..2], cb[..2], "shared prefix must hash identically");
|
||||
assert_ne!(ca[2], cb[2], "different tail must diverge");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn forked_conversation_shares_prefix() {
|
||||
let base = vec![user("q1"), assistant("a1"), user("q2"), assistant("a2")];
|
||||
let mut fork_x = base.clone();
|
||||
fork_x.push(user("branch X"));
|
||||
let mut fork_y = base;
|
||||
fork_y.push(user("branch Y"));
|
||||
let bx = text_body(fork_x);
|
||||
let by = text_body(fork_y);
|
||||
let cx = chain(&bx);
|
||||
let cy = chain(&by);
|
||||
assert_eq!(cx[..4], cy[..4]);
|
||||
assert_ne!(cx[4], cy[4]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn editing_history_rewrites_from_that_point_on() {
|
||||
let a = text_body(vec![user("q1"), assistant("old answer"), user("q2")]);
|
||||
let b = text_body(vec![user("q1"), assistant("edited answer"), user("q2")]);
|
||||
let ca = chain(&a);
|
||||
let cb = chain(&b);
|
||||
assert_eq!(ca[0], cb[0], "before the edit: unchanged");
|
||||
assert_ne!(ca[1], cb[1], "at the edit: changed");
|
||||
assert_ne!(ca[2], cb[2], "after the edit: still changed even with identical text");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multimodal_byte_level_semantics() {
|
||||
let mk = |url: &str| {
|
||||
text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "what is in this image?"},
|
||||
{"type": "image_url", "image_url": {"url": url}}
|
||||
]
|
||||
})])
|
||||
};
|
||||
// Same image content, different URLs -> different chain (the rendered
|
||||
// prompt bytes differ; this is the correct byte-level behaviour).
|
||||
let http_a = mk("https://example.com/a.png");
|
||||
let http_b = mk("https://example.com/b.png");
|
||||
assert_ne!(chain(&http_a), chain(&http_b));
|
||||
|
||||
// Identical base64 payload -> identical chain (hit).
|
||||
let data_1 = mk("data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==");
|
||||
let data_2 = mk("data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==");
|
||||
assert_eq!(chain(&data_1), chain(&data_2));
|
||||
|
||||
// base64 vs http form of the "same" image -> different (bytes differ).
|
||||
assert_ne!(chain(&data_1), chain(&http_a));
|
||||
|
||||
// Wire-shape variant {"image_url":"..."} hashes like the object form.
|
||||
let flat = text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "what is in this image?"},
|
||||
{"type": "image_url", "image_url": "https://example.com/a.png"}
|
||||
]
|
||||
})]);
|
||||
assert_eq!(chain(&flat), chain(&http_a));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seed_fields_change_whole_chain() {
|
||||
let msgs = || vec![user("same"), assistant("same")];
|
||||
let base = text_body(msgs());
|
||||
let other_model = json!({"model": "other-model", "messages": msgs()});
|
||||
let with_kwargs = json!({
|
||||
"model": "dsv41-flash",
|
||||
"chat_template_kwargs": {"thinking": true},
|
||||
"messages": msgs()
|
||||
});
|
||||
let with_tools = json!({
|
||||
"model": "dsv41-flash",
|
||||
"tools": [{"type": "function", "function": {"name": "f"}}],
|
||||
"messages": msgs()
|
||||
});
|
||||
|
||||
let c_base = chain(&base);
|
||||
assert_ne!(ChainSeed::from_body(&base), ChainSeed::from_body(&other_model));
|
||||
assert_ne!(chain(&other_model), c_base);
|
||||
assert_ne!(chain(&with_kwargs), c_base);
|
||||
assert_ne!(chain(&with_tools), c_base);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seed_canonicalization_is_key_order_insensitive() {
|
||||
let a = json!({"model": "m", "chat_template_kwargs": {"x": 1, "y": 2}, "messages": [user("hi")]});
|
||||
let b = json!({"messages": [user("hi")], "chat_template_kwargs": {"y": 2, "x": 1}, "model": "m"});
|
||||
assert_eq!(ChainSeed::from_body(&a), ChainSeed::from_body(&b));
|
||||
assert_eq!(chain(&a), chain(&b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_or_malformed_messages_give_empty_chain() {
|
||||
let no_messages = json!({"model": "m"});
|
||||
let empty_messages = json!({"model": "m", "messages": []});
|
||||
let wrong_type = json!({"model": "m", "messages": "not-an-array"});
|
||||
for b in [&no_messages, &empty_messages, &wrong_type] {
|
||||
assert!(chain(b).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_part_types_flag_unclassifiable() {
|
||||
let audio = text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [{"type": "audio_url", "audio_url": {"url": "https://x/a.mp3"}}]
|
||||
})]);
|
||||
assert!(is_unclassifiable(&audio));
|
||||
// Chain is still full-length and deterministic — just not to be trusted.
|
||||
assert_eq!(chain(&audio).len(), 1);
|
||||
assert_eq!(chain(&audio), chain(&audio));
|
||||
|
||||
let file_scheme = text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [{"type": "image_url", "image_url": {"url": "file:///etc/passwd"}}]
|
||||
})]);
|
||||
assert!(is_unclassifiable(&file_scheme));
|
||||
|
||||
let plain = text_body(vec![user("hello"), assistant("hi")]);
|
||||
assert!(!is_unclassifiable(&plain));
|
||||
|
||||
let multimodal_ok = text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "look"},
|
||||
{"type": "image_url", "image_url": {"url": "https://x/a.png"}}
|
||||
]
|
||||
})]);
|
||||
assert!(!is_unclassifiable(&multimodal_ok));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_role_defaults_to_user() {
|
||||
let implicit = text_body(vec![json!({"content": "hi"})]);
|
||||
let explicit = text_body(vec![user("hi")]);
|
||||
assert_eq!(chain(&implicit), chain(&explicit));
|
||||
assert!(!is_unclassifiable(&implicit));
|
||||
}
|
||||
|
||||
fn tool_call(name: &str, args: &str) -> Value {
|
||||
json!([{
|
||||
"id": format!("call_{}", name),
|
||||
"type": "function",
|
||||
"function": {"name": name, "arguments": args}
|
||||
}])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_calls_enter_the_chain() {
|
||||
let mk = |tc: Value| {
|
||||
text_body(vec![
|
||||
user("weather in Shanghai?"),
|
||||
json!({"role": "assistant", "content": null, "tool_calls": tc}),
|
||||
user("and tomorrow?"),
|
||||
])
|
||||
};
|
||||
let a = mk(tool_call("get_weather", "{\"city\":\"sh\"}"));
|
||||
let b = mk(tool_call("get_weather", "{\"city\":\"bj\"}"));
|
||||
let c = mk(tool_call("get_weather", "{\"city\":\"sh\"}"));
|
||||
let (ca, cb, cc) = (chain(&a), chain(&b), chain(&c));
|
||||
assert_eq!(ca[0], cb[0], "before the tool call: unchanged");
|
||||
assert_ne!(ca[1], cb[1], "different tool_calls must diverge");
|
||||
assert_ne!(ca[2], cb[2], "and stay diverged afterwards");
|
||||
assert_eq!(ca, cc, "identical tool_calls must hit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_null_with_tool_calls_is_classifiable() {
|
||||
let tc = tool_call("get_weather", "{}");
|
||||
let null_content = text_body(vec![json!({
|
||||
"role": "assistant", "content": null, "tool_calls": tc.clone()
|
||||
})]);
|
||||
let empty_content = text_body(vec![json!({
|
||||
"role": "assistant", "content": "", "tool_calls": tc
|
||||
})]);
|
||||
assert!(!is_unclassifiable(&null_content));
|
||||
assert_eq!(
|
||||
chain(&null_content),
|
||||
chain(&empty_content),
|
||||
"content:null renders as empty -> must hash like content:\"\""
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_null_without_tool_calls_is_empty() {
|
||||
let null_content = text_body(vec![json!({"role": "assistant", "content": null})]);
|
||||
let missing_content = text_body(vec![json!({"role": "assistant"})]);
|
||||
let empty_content = text_body(vec![assistant("")]);
|
||||
assert!(!is_unclassifiable(&null_content));
|
||||
assert!(!is_unclassifiable(&missing_content));
|
||||
assert_eq!(chain(&null_content), chain(&empty_content));
|
||||
assert_eq!(chain(&missing_content), chain(&empty_content));
|
||||
|
||||
// Genuinely malformed content types keep the fail-closed treatment.
|
||||
let malformed = text_body(vec![json!({"role": "assistant", "content": 42})]);
|
||||
assert!(is_unclassifiable(&malformed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_calls_canonicalization_key_order_insensitive() {
|
||||
let a = text_body(vec![json!({
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"tool_calls": [{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "f", "arguments": "{}"}
|
||||
}]
|
||||
})]);
|
||||
let b = text_body(vec![json!({
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"tool_calls": [{
|
||||
"function": {"arguments": "{}", "name": "f"},
|
||||
"type": "function",
|
||||
"id": "call_1"
|
||||
}]
|
||||
})]);
|
||||
assert_eq!(chain(&a), chain(&b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_call_id_enters_chain() {
|
||||
let mk = |id: &str| {
|
||||
text_body(vec![
|
||||
user("weather in Shanghai?"),
|
||||
json!({
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"tool_calls": tool_call("get_weather", "{}")
|
||||
}),
|
||||
json!({"role": "tool", "tool_call_id": id, "content": "sunny"}),
|
||||
user("thanks"),
|
||||
])
|
||||
};
|
||||
let a = mk("call_1");
|
||||
let b = mk("call_2");
|
||||
let c = mk("call_1");
|
||||
let (ca, cb, cc) = (chain(&a), chain(&b), chain(&c));
|
||||
assert_eq!(ca[..2], cb[..2], "before the tool result: unchanged");
|
||||
assert_ne!(ca[2], cb[2], "different tool_call_id must diverge");
|
||||
assert_ne!(ca[3], cb[3], "and stay diverged afterwards");
|
||||
assert_eq!(ca, cc, "identical tool_call_id must hit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn name_field_enters_chain() {
|
||||
let a = text_body(vec![json!({"role": "user", "name": "alice", "content": "hi"})]);
|
||||
let b = text_body(vec![json!({"role": "user", "name": "bob", "content": "hi"})]);
|
||||
let c = text_body(vec![json!({"role": "user", "name": "alice", "content": "hi"})]);
|
||||
assert_ne!(chain(&a), chain(&b), "different name must diverge");
|
||||
assert_eq!(chain(&a), chain(&c), "identical name must hit");
|
||||
|
||||
// Absent name and empty meta stay equivalent to a plain message.
|
||||
let plain = text_body(vec![user("hi")]);
|
||||
let no_name = text_body(vec![json!({"role": "user", "content": "hi"})]);
|
||||
assert_eq!(chain(&plain), chain(&no_name));
|
||||
}
|
||||
|
||||
/// `reasoning_content` must NOT change the chain: chat templates do not
|
||||
/// render historical reasoning into the prompt, so hashing it would
|
||||
/// manufacture misses for thinking-enabled clients that echo it back.
|
||||
/// Pins the walk_messages assumption (thinking:true is our default).
|
||||
#[test]
|
||||
fn reasoning_content_does_not_enter_chain() {
|
||||
let plain = text_body(vec![user("q"), assistant("a"), user("q2")]);
|
||||
let with_reasoning = text_body(vec![
|
||||
user("q"),
|
||||
json!({"role": "assistant", "content": "a", "reasoning_content": "let me think..."}),
|
||||
user("q2"),
|
||||
]);
|
||||
assert_eq!(
|
||||
chain(&plain),
|
||||
chain(&with_reasoning),
|
||||
"reasoning_content is not rendered -> must not change the chain"
|
||||
);
|
||||
assert!(
|
||||
!is_unclassifiable(&with_reasoning),
|
||||
"reasoning-carrying turns stay classifiable"
|
||||
);
|
||||
}
|
||||
|
||||
// ==================== T23: intra-message chunking ====================
|
||||
|
||||
/// The pre-T23 `chain_hashes`, verbatim copy. Short-message bodies must
|
||||
/// produce byte-identical chains under the chunked implementation.
|
||||
fn legacy_chain_hashes(seed: &ChainSeed, body: &Value) -> Vec<u64> {
|
||||
let (msgs, _) = walk_messages(body);
|
||||
let mut chain = Vec::with_capacity(msgs.len());
|
||||
let mut prev = seed.value;
|
||||
for msg in &msgs {
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&prev.to_le_bytes());
|
||||
|
||||
let role = msg.role.as_bytes();
|
||||
h.update(&(role.len() as u64).to_le_bytes());
|
||||
h.update(role);
|
||||
|
||||
match &msg.content {
|
||||
ContentView::Str(s) => {
|
||||
update_tagged(&mut h, TAG_TEXT, s.as_bytes());
|
||||
}
|
||||
ContentView::Parts(parts) => {
|
||||
for part in parts {
|
||||
match part {
|
||||
PartView::Text(t) => update_tagged(&mut h, TAG_TEXT, t.as_bytes()),
|
||||
PartView::ImageUrl(u) => {
|
||||
update_tagged(&mut h, TAG_IMAGE_URL, u.as_bytes())
|
||||
}
|
||||
PartView::Unknown => h.update(TAG_UNKNOWN),
|
||||
}
|
||||
}
|
||||
}
|
||||
ContentView::Empty => {}
|
||||
}
|
||||
|
||||
if let Some(tc) = &msg.tool_calls {
|
||||
update_tagged(&mut h, TAG_TOOL_CALLS, tc.as_bytes());
|
||||
}
|
||||
|
||||
update_tagged(&mut h, TAG_META, msg.extra.as_bytes());
|
||||
|
||||
prev = h.digest();
|
||||
chain.push(prev);
|
||||
}
|
||||
chain
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_messages_match_legacy_chain_byte_for_byte() {
|
||||
let exact_boundary = "x".repeat(CHUNK_BYTES);
|
||||
let bodies = vec![
|
||||
// pure-text multi-turn
|
||||
text_body(vec![user("q1"), assistant("a1"), user("q2")]),
|
||||
// multimodal parts (short text + image)
|
||||
text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "look"},
|
||||
{"type": "image_url", "image_url": {"url": "https://x/a.png"}}
|
||||
]
|
||||
})]),
|
||||
// tool_calls + content:null + tool result (meta on both)
|
||||
text_body(vec![
|
||||
user("weather?"),
|
||||
json!({"role": "assistant", "content": null,
|
||||
"tool_calls": tool_call("get_weather", "{\"city\":\"sh\"}")}),
|
||||
json!({"role": "tool", "tool_call_id": "call_1", "content": "sunny"}),
|
||||
]),
|
||||
// name meta
|
||||
text_body(vec![json!({"role": "user", "name": "alice", "content": "hi"})]),
|
||||
// null content without tool_calls
|
||||
text_body(vec![json!({"role": "assistant", "content": null})]),
|
||||
// exactly CHUNK_BYTES: never splits (only > CHUNK_BYTES does)
|
||||
text_body(vec![user(&exact_boundary)]),
|
||||
];
|
||||
for b in &bodies {
|
||||
let seed = ChainSeed::from_body(b);
|
||||
assert_eq!(
|
||||
chain_hashes(&seed, b),
|
||||
legacy_chain_hashes(&seed, b),
|
||||
"short-message chain must stay byte-identical to pre-T23"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_message_splits_into_chunk_rings() {
|
||||
// CHUNK_BYTES + 1 -> one chunk ring + the message-final ring.
|
||||
let text = "x".repeat(CHUNK_BYTES + 1);
|
||||
let body = text_body(vec![user(&text)]);
|
||||
let seed = ChainSeed::from_body(&body);
|
||||
let (c, idx) = chain_hashes_indexed(&seed, &body);
|
||||
assert_eq!(c.len(), 2);
|
||||
assert_eq!(idx, vec![0, 0], "both rings belong to message 0");
|
||||
// The chunk ring is bare: xxh3(prev_le ‖ chunk), no tag, no length.
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&seed.value.to_le_bytes());
|
||||
h.update("x".repeat(CHUNK_BYTES).as_bytes());
|
||||
assert_eq!(c[0], h.digest());
|
||||
|
||||
// Exactly CHUNK_BYTES -> single ring (only > CHUNK_BYTES splits);
|
||||
// already pinned by the legacy-equivalence battery above.
|
||||
assert_eq!(chain(&text_body(vec![user(&"y".repeat(CHUNK_BYTES))])).len(), 1);
|
||||
}
|
||||
|
||||
/// The T23 acceptance shape: two single-message bodies sharing a 4KB
|
||||
/// prefix but differing in the tail now share their chunk rings — the
|
||||
/// pre-T23 chain (one ring per message) could never hit here.
|
||||
#[test]
|
||||
fn shared_long_prefix_different_tail_shares_chunk_rings() {
|
||||
let prefix = "a".repeat(2 * CHUNK_BYTES);
|
||||
let a = text_body(vec![user(&format!("{prefix}{}", "b".repeat(100)))]);
|
||||
let b = text_body(vec![user(&format!("{prefix}{}", "c".repeat(200)))]);
|
||||
let (ca, ia) = chain_hashes_indexed(&ChainSeed::from_body(&a), &a);
|
||||
let (cb, ib) = chain_hashes_indexed(&ChainSeed::from_body(&b), &b);
|
||||
assert_eq!(ca.len(), 3, "2 chunk rings + message-final ring");
|
||||
assert_eq!(cb.len(), 3);
|
||||
assert_eq!(ia, vec![0, 0, 0]);
|
||||
assert_eq!(ib, vec![0, 0, 0]);
|
||||
assert_eq!(
|
||||
ca[..2],
|
||||
cb[..2],
|
||||
"identical 4KB prefix -> identical chunk rings"
|
||||
);
|
||||
assert_ne!(ca[2], cb[2], "different tail -> final ring diverges");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utf8_boundary_chunk_never_splits_a_character() {
|
||||
// 700 x 3 bytes = 2100 > 2048; the 2048 cut lands mid-汉
|
||||
// (2048 % 3 = 2), so the chunk must back off to 2046 bytes.
|
||||
let text = "汉".repeat(700);
|
||||
let (chunk, tail) = take_chunk(&text);
|
||||
assert_eq!(chunk.len(), 2046);
|
||||
assert_eq!(tail.len(), 54);
|
||||
assert_eq!(format!("{chunk}{tail}"), text, "chunk + tail reconstruct");
|
||||
|
||||
let body = text_body(vec![user(&text)]);
|
||||
let seed = ChainSeed::from_body(&body);
|
||||
let (c, idx) = chain_hashes_indexed(&seed, &body);
|
||||
assert_eq!(c.len(), 2);
|
||||
assert_eq!(idx, vec![0, 0]);
|
||||
// The chunk ring carries exactly the boundary-aligned bytes.
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&seed.value.to_le_bytes());
|
||||
h.update(chunk.as_bytes());
|
||||
assert_eq!(c[0], h.digest());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_part_long_text_chunks_per_part_in_order() {
|
||||
let t1 = "p".repeat(CHUNK_BYTES + 10); // 1 chunk + 10-byte tail
|
||||
let t2 = "q".repeat(CHUNK_BYTES + 20); // 1 chunk + 20-byte tail
|
||||
let body = text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": t1},
|
||||
{"type": "image_url", "image_url": {"url": "https://x/a.png"}},
|
||||
{"type": "text", "text": t2}
|
||||
]
|
||||
})]);
|
||||
let seed = ChainSeed::from_body(&body);
|
||||
let (c, idx) = chain_hashes_indexed(&seed, &body);
|
||||
assert_eq!(c.len(), 3, "t1 chunk + t2 chunk + message-final ring");
|
||||
assert_eq!(idx, vec![0, 0, 0]);
|
||||
// Chunk rings follow part order: t1's chunk first, then t2's.
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&seed.value.to_le_bytes());
|
||||
h.update("p".repeat(CHUNK_BYTES).as_bytes());
|
||||
assert_eq!(c[0], h.digest());
|
||||
let mut h2 = Xxh3::new();
|
||||
h2.update(&c[0].to_le_bytes());
|
||||
h2.update("q".repeat(CHUNK_BYTES).as_bytes());
|
||||
assert_eq!(c[1], h2.digest());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,8 @@
|
||||
//! HTTP router implementations
|
||||
|
||||
pub mod g9i;
|
||||
pub mod g9i_prefix_chain;
|
||||
pub mod g9i_shadow;
|
||||
pub mod pd_router;
|
||||
pub mod pd_types;
|
||||
pub mod router;
|
||||
|
||||
@@ -18,6 +18,7 @@ use serde_json::{json, Value};
|
||||
use tokio_stream::wrappers::UnboundedReceiverStream;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use super::g9i_shadow::{ShadowRequestContext, ShadowResponseContext};
|
||||
use super::pd_types::api_path;
|
||||
use crate::{
|
||||
config::types::RetryConfig,
|
||||
@@ -58,6 +59,8 @@ pub struct PDRouter {
|
||||
pub retry_config: RetryConfig,
|
||||
pub api_key: Option<String>,
|
||||
pub enable_igw: bool,
|
||||
/// G9i pool admission + fast-P routing state; None = feature off.
|
||||
pub g9i: Option<Arc<super::g9i::G9iState>>,
|
||||
}
|
||||
|
||||
struct PreparedWorkerRequest<'a> {
|
||||
@@ -182,6 +185,14 @@ impl PDRouter {
|
||||
}
|
||||
|
||||
pub async fn new(ctx: &Arc<crate::app_context::AppContext>) -> Result<Self, String> {
|
||||
let g9i = match &ctx.router_config.g9i {
|
||||
Some(config) if config.enabled => {
|
||||
let state = Arc::new(super::g9i::G9iState::new(config.clone()));
|
||||
state.start_monitor(Arc::clone(&ctx.worker_registry), ctx.client.clone());
|
||||
Some(state)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
Ok(PDRouter {
|
||||
worker_registry: Arc::clone(&ctx.worker_registry),
|
||||
policy_registry: Arc::clone(&ctx.policy_registry),
|
||||
@@ -189,6 +200,7 @@ impl PDRouter {
|
||||
retry_config: ctx.router_config.effective_retry_config(),
|
||||
api_key: ctx.router_config.api_key.clone(),
|
||||
enable_igw: ctx.router_config.enable_igw,
|
||||
g9i,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -366,6 +378,92 @@ impl PDRouter {
|
||||
Ok((prefill_request, decode_request))
|
||||
}
|
||||
|
||||
/// G9i admission + shadow booking, ONE attempt (T15: the caller retries
|
||||
/// holdable rejections through the hold queue). None = G9i not
|
||||
/// configured ("untracked"). Err = rejected (the caller counts +
|
||||
/// answers 429); the rejections are structured so that a rejected
|
||||
/// attempt never entered the shadow bookkeeping (T11 拒绝路径零记账).
|
||||
/// T25-A: `prior_wait_secs` is the time the request already spent
|
||||
/// parked in the reservoir across re-runs — it counts into TTFT, so
|
||||
/// the re-judged est_TTFT carries it.
|
||||
async fn g9i_admission<T: Serialize + Clone>(
|
||||
&self,
|
||||
context: &PDRequestContext<'_>,
|
||||
original_request: &T,
|
||||
prior_wait_secs: f64,
|
||||
) -> Option<Result<(Option<String>, &'static str, Option<ShadowRequestContext>), super::g9i::RejectInfo>>
|
||||
{
|
||||
let g9i = self.g9i.as_ref()?;
|
||||
if let Err(info) = g9i.check_decode_pool(&self.worker_registry).await {
|
||||
return Some(Err(info));
|
||||
}
|
||||
if context.route != "/v1/chat/completions" {
|
||||
// Non-chat routes skip classification but still land on
|
||||
// the regular pool — the regular queue gate applies.
|
||||
if let Err(info) = g9i.check_regular_queue(&self.worker_registry) {
|
||||
return Some(Err(info));
|
||||
}
|
||||
return Some(Ok((None, "non_chat", None)));
|
||||
}
|
||||
let body = match serde_json::to_value(original_request) {
|
||||
Ok(body) => body,
|
||||
Err(_) => return Some(Ok((None, "regular", None))),
|
||||
};
|
||||
// Shadow T2: compute the prefix chain once, here, alongside the v1
|
||||
// classification. Skipped entirely when shadow mode is off.
|
||||
let shadow_ctx = if g9i.shadow_enabled() {
|
||||
ShadowRequestContext::from_body(&body, context.is_stream)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let (plan, path, probe_new_work) = g9i
|
||||
.classify_prefill(&self.worker_registry, &self.client, &body)
|
||||
.await;
|
||||
// T11: v1 rejection FIRST — a request rejected by any gate must
|
||||
// never enter the shadow bookkeeping (拒绝路径零记账).
|
||||
let v1_override = match plan.into_override() {
|
||||
Ok(o) => o,
|
||||
Err(info) => return Some(Err(info)),
|
||||
};
|
||||
let shadow_ctx = match shadow_ctx {
|
||||
Some(mut ctx) => {
|
||||
// T11: classify + candidate pick + cap verdict + admission
|
||||
// in ONE queue-model critical section (atomic
|
||||
// check-and-act) — the check→act window no longer spans
|
||||
// the TTFT wait. Err = p_queue_cap (enforce), nothing
|
||||
// admitted.
|
||||
match g9i.shadow_classify_and_admit(&mut ctx) {
|
||||
Ok(Some(mut decision)) => {
|
||||
// T25-A: time already parked in the reservoir counts
|
||||
// into TTFT — the re-judged estimate carries it.
|
||||
if prior_wait_secs > 0.0 {
|
||||
decision.est_ttft_secs += prior_wait_secs;
|
||||
ctx.decision = Some(decision.clone());
|
||||
}
|
||||
super::g9i_shadow::record_agreement(
|
||||
path,
|
||||
probe_new_work,
|
||||
&ctx,
|
||||
&decision,
|
||||
);
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(info) => return Some(Err(info)),
|
||||
}
|
||||
Some(ctx)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
// T7 shadow-consistent routing (--g9i-shadow-route): the live
|
||||
// prefill override follows the shadow decision's matched_node (=
|
||||
// the booked node), so est vs actual is pure estimation error.
|
||||
let override_url = g9i.shadow_route_pick(
|
||||
v1_override,
|
||||
shadow_ctx.as_ref().and_then(|c| c.decision.as_ref()),
|
||||
);
|
||||
Some(Ok((override_url, path, shadow_ctx)))
|
||||
}
|
||||
|
||||
async fn execute_dual_dispatch<T: Serialize + Clone>(
|
||||
&self,
|
||||
headers: Option<&HeaderMap>,
|
||||
@@ -390,6 +488,138 @@ impl PDRouter {
|
||||
// Clone request once outside the retry loop, then use Arc to share across attempts
|
||||
// This avoids O(retries) clones by sharing the same data
|
||||
let shared_request = Arc::new(original_request.clone());
|
||||
|
||||
// G9i admission (router-side, zero engine changes). Computed once per
|
||||
// request, outside the retry loop: our own 429s must never be
|
||||
// retried. `ttft_path` labels the TTFT histogram dimension for this
|
||||
// request's eventual first-token sample. `shadow_ctx` is the G9i v2
|
||||
// shadow bookkeeping carrier (T2): Some only for chat requests with
|
||||
// a classifiable prefix chain while shadow mode is on; it never
|
||||
// influences routing.
|
||||
//
|
||||
// T15: holdable rejections (d_window_full / p_queue_cap) park the
|
||||
// request in the FIFO hold queue instead of an immediate 429
|
||||
// (429 太伤吞吐 — 测试端把 429 当失败). On pass the COMPLETE
|
||||
// admission re-runs (every gate re-checked; the T11 booking happens
|
||||
// only on a passing attempt — a parked request occupies no ledger).
|
||||
// T20 (B): prompt_too_long hard validation — the FRONT-MOST check,
|
||||
// before any admission gate / est_TTFT / ledger / queue bookkeeping
|
||||
// and before the hold loop (an invalid request must never park).
|
||||
// 400 (client bug), counted in smg_g9i_invalid_request_total, never
|
||||
// in smg_g9i_rejects_total. Chat-completions only.
|
||||
if context.route == "/v1/chat/completions" {
|
||||
if let Some(g9i) = &self.g9i {
|
||||
if let Ok(body) = serde_json::to_value(original_request) {
|
||||
if let Some(resp) = g9i.prompt_too_long_rejection(&body) {
|
||||
return resp;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut hold_deadline: Option<Instant> = None;
|
||||
// T25-A reservoir park state: (deadline, seq id) across re-runs +
|
||||
// the accumulated wait (counts into the est_TTFT re-judgment).
|
||||
let mut reservoir_state: Option<(Instant, u64)> = None;
|
||||
let mut reservoir_waited_secs: f64 = 0.0;
|
||||
let (g9i_prefill_override, g9i_ttft_path, shadow_ctx): (
|
||||
Option<String>,
|
||||
&'static str,
|
||||
Option<ShadowRequestContext>,
|
||||
) = loop {
|
||||
match self
|
||||
.g9i_admission(&context, original_request, reservoir_waited_secs)
|
||||
.await
|
||||
{
|
||||
None => {
|
||||
// T25: per-request admission outcome (untracked = no
|
||||
// G9i configured — class "none").
|
||||
metrics::counter!(
|
||||
super::g9i::METRIC_ADMISSION_TOTAL,
|
||||
"class" => "none",
|
||||
"result" => "admitted",
|
||||
"reason" => "none",
|
||||
)
|
||||
.increment(1);
|
||||
break (None, "untracked", None);
|
||||
}
|
||||
Some(Ok(outcome)) => {
|
||||
// T25: per-request admission outcome — the shadow
|
||||
// decision's class (the experiment's primary counter).
|
||||
let class = outcome
|
||||
.2
|
||||
.as_ref()
|
||||
.and_then(|c| c.decision.as_ref())
|
||||
.map(|d| d.class.as_str())
|
||||
.unwrap_or("none");
|
||||
metrics::counter!(
|
||||
super::g9i::METRIC_ADMISSION_TOTAL,
|
||||
"class" => class,
|
||||
"result" => "admitted",
|
||||
"reason" => "none",
|
||||
)
|
||||
.increment(1);
|
||||
break outcome;
|
||||
}
|
||||
Some(Err(mut info)) => {
|
||||
let mut retry = false;
|
||||
if let Some(g9i) = &self.g9i {
|
||||
// T25-A: under hot_policy=reservoir, HOT-class
|
||||
// holdable rejections park in the priority
|
||||
// reservoir INSTEAD of the T15 hold (the A/B stays
|
||||
// clean — every other class keeps the hold path).
|
||||
if g9i.reservoir_eligible(&info) {
|
||||
match g9i
|
||||
.reservoir_or_reject(&info, &mut reservoir_state)
|
||||
.await
|
||||
{
|
||||
Ok(waited) => {
|
||||
reservoir_waited_secs += waited;
|
||||
retry = true;
|
||||
}
|
||||
Err(final_info) => info = final_info,
|
||||
}
|
||||
} else if info.holdable() {
|
||||
let body_bytes = context
|
||||
.request_text
|
||||
.as_deref()
|
||||
.map(|s| s.len() as u64)
|
||||
.unwrap_or(0);
|
||||
match g9i
|
||||
.hold_or_reject(&info, body_bytes, &mut hold_deadline)
|
||||
.await
|
||||
{
|
||||
Ok(()) => retry = true,
|
||||
Err(final_info) => info = final_info,
|
||||
}
|
||||
}
|
||||
}
|
||||
if retry {
|
||||
continue;
|
||||
}
|
||||
// T25: per-request rejection outcome (class from the
|
||||
// rejecting gate; "none" on non-chat/early gates).
|
||||
metrics::counter!(
|
||||
super::g9i::METRIC_ADMISSION_TOTAL,
|
||||
"class" => info.hold_class.unwrap_or("none"),
|
||||
"result" => "rejected",
|
||||
"reason" => info.reason,
|
||||
)
|
||||
.increment(1);
|
||||
metrics::counter!("smg_g9i_rejects_total", "reason" => info.reason)
|
||||
.increment(1);
|
||||
return super::g9i::reject_response(&info);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// T11: handle of the decision-time booking, extracted before the
|
||||
// retry closure moves `shadow_ctx`. If the dispatch never reached
|
||||
// the response path (all attempts failed before headers), the
|
||||
// booking is released after the loop — exactly once, paired with
|
||||
// shadow_admit's consume mark.
|
||||
let admission_mark = shadow_ctx.as_ref().and_then(|c| c.admission.clone());
|
||||
|
||||
let response = RetryExecutor::execute_response_with_retry(
|
||||
&self.retry_config,
|
||||
{
|
||||
@@ -397,6 +627,8 @@ impl PDRouter {
|
||||
// Clone Arc (cheap reference count increment) instead of cloning the entire request
|
||||
let shared_request = Arc::clone(&shared_request);
|
||||
let context = context.clone();
|
||||
let g9i_prefill_override = g9i_prefill_override.clone();
|
||||
let shadow_ctx = shadow_ctx.clone();
|
||||
async move {
|
||||
let (prefill, decode) = match self
|
||||
.select_pd_pair(
|
||||
@@ -412,6 +644,38 @@ impl PDRouter {
|
||||
}
|
||||
};
|
||||
|
||||
// G9i least-wait override: route onto the classified
|
||||
// fast-P node instead of the policy pick.
|
||||
let (prefill, decode) =
|
||||
if let Some(override_url) = g9i_prefill_override.as_ref() {
|
||||
let matched = self
|
||||
.worker_registry
|
||||
.get_prefill_workers()
|
||||
.into_iter()
|
||||
.find(|w| {
|
||||
w.url().trim_end_matches('/') == override_url.as_str()
|
||||
&& w.is_available()
|
||||
});
|
||||
match matched {
|
||||
Some(w) => {
|
||||
debug!(
|
||||
"G9i: overriding prefill selection with {}",
|
||||
override_url
|
||||
);
|
||||
(w, decode)
|
||||
}
|
||||
None => {
|
||||
warn!(
|
||||
"G9i: override target {} unavailable, keeping policy pick",
|
||||
override_url
|
||||
);
|
||||
(prefill, decode)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
(prefill, decode)
|
||||
};
|
||||
|
||||
debug!(
|
||||
"PD retry attempt {} using prefill={} decode={}",
|
||||
attempt,
|
||||
@@ -428,6 +692,15 @@ impl PDRouter {
|
||||
json_request["stream"] = Value::Bool(context.is_stream);
|
||||
}
|
||||
|
||||
// G9i shadow T2: inject the usage / cache-details
|
||||
// flags the client didn't set (chat + shadow mode
|
||||
// only; `shadow_ctx` is None otherwise). Paired with
|
||||
// response-side stripping: we inject exactly what we
|
||||
// later strip, so clients see no change.
|
||||
if let Some(sctx) = &shadow_ctx {
|
||||
sctx.inject(&mut json_request);
|
||||
}
|
||||
|
||||
json_request = match Self::inject_bootstrap_into_value(
|
||||
json_request,
|
||||
prefill.as_ref(),
|
||||
@@ -446,6 +719,8 @@ impl PDRouter {
|
||||
Arc::clone(&prefill),
|
||||
Arc::clone(&decode),
|
||||
start_time,
|
||||
g9i_ttft_path,
|
||||
shadow_ctx,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -524,6 +799,13 @@ impl PDRouter {
|
||||
);
|
||||
}
|
||||
|
||||
// T11: release a decision-time booking that no response path ever
|
||||
// consumed (all attempts failed before headers). No-op when the
|
||||
// stream/non-stream path already took it over.
|
||||
if let Some(g9i) = &self.g9i {
|
||||
g9i.shadow_release_unconsumed(admission_mark);
|
||||
}
|
||||
|
||||
response
|
||||
}
|
||||
|
||||
@@ -575,6 +857,9 @@ impl PDRouter {
|
||||
Some(response_headers),
|
||||
prefill,
|
||||
decode,
|
||||
None,
|
||||
// Error envelope: no usage will ever arrive — no shadow hook.
|
||||
None,
|
||||
)
|
||||
} else {
|
||||
// Handle non-streaming error response
|
||||
@@ -649,6 +934,7 @@ impl PDRouter {
|
||||
}
|
||||
|
||||
// Internal method that performs the actual dual dispatch (without retry logic)
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn execute_dual_dispatch_internal(
|
||||
&self,
|
||||
headers: Option<&HeaderMap>,
|
||||
@@ -656,7 +942,11 @@ impl PDRouter {
|
||||
context: PDRequestContext<'_>,
|
||||
prefill: Arc<dyn Worker>,
|
||||
decode: Arc<dyn Worker>,
|
||||
_start_time: Instant,
|
||||
start_time: Instant,
|
||||
g9i_path: &'static str,
|
||||
// G9i shadow T2: prefix chain + injection flags; None = shadow off
|
||||
// or unclassifiable request — zero behavior change.
|
||||
shadow_ctx: Option<ShadowRequestContext>,
|
||||
) -> Response {
|
||||
// For non-streaming: use guard for automatic load management
|
||||
// For streaming: load will be managed in create_streaming_response
|
||||
@@ -868,6 +1158,24 @@ impl PDRouter {
|
||||
|
||||
let response_headers = header_utils::preserve_response_headers(res.headers());
|
||||
|
||||
// Shadow T2: hand the chain + the ACTUAL prefill landing
|
||||
// node (post-override) to the stream wrapper, which
|
||||
// harvests the usage event and strips injected fields.
|
||||
let shadow = shadow_ctx.map(|ctx| {
|
||||
ShadowResponseContext::from_request(
|
||||
ctx,
|
||||
super::g9i::normalize_url(prefill.url()),
|
||||
super::g9i::normalize_url(decode.url()),
|
||||
)
|
||||
});
|
||||
// Shadow T3: admit into the queue model keyed by the
|
||||
// ACTUAL landing node; the stream wrapper pairs this with
|
||||
// exactly one complete (usage recorded) or release
|
||||
// (abort/disconnect/error) before teardown.
|
||||
if let (Some(g9i), Some(s)) = (&self.g9i, &shadow) {
|
||||
g9i.shadow_admit(s);
|
||||
}
|
||||
|
||||
self.create_streaming_response(
|
||||
res.bytes_stream(),
|
||||
status,
|
||||
@@ -876,17 +1184,41 @@ impl PDRouter {
|
||||
Some(response_headers),
|
||||
prefill,
|
||||
decode,
|
||||
Some((start_time, g9i_path)),
|
||||
shadow,
|
||||
)
|
||||
} else {
|
||||
// Non-streaming response
|
||||
if context.return_logprob {
|
||||
self.process_non_streaming_response(
|
||||
res,
|
||||
status,
|
||||
context.return_logprob,
|
||||
prefill_body,
|
||||
let shadow = shadow_ctx.map(|ctx| {
|
||||
ShadowResponseContext::from_request(
|
||||
ctx,
|
||||
super::g9i::normalize_url(prefill.url()),
|
||||
super::g9i::normalize_url(decode.url()),
|
||||
)
|
||||
.await
|
||||
});
|
||||
// Shadow T3: admit into the queue model keyed by the
|
||||
// ACTUAL landing node; paired with complete/release in
|
||||
// shadow_record_non_stream (or the read-failure paths
|
||||
// below / in process_non_streaming_response).
|
||||
if let (Some(g9i), Some(s)) = (&self.g9i, &shadow) {
|
||||
g9i.shadow_admit(s);
|
||||
}
|
||||
if context.return_logprob {
|
||||
let resp = self
|
||||
.process_non_streaming_response(
|
||||
res,
|
||||
status,
|
||||
context.return_logprob,
|
||||
prefill_body,
|
||||
shadow,
|
||||
)
|
||||
.await;
|
||||
// T17: non-streaming e2e (observability only — the
|
||||
// AIMD blind state reads local signals instead).
|
||||
if let Some(g9i) = &self.g9i {
|
||||
g9i.record_nonstream_e2e(start_time.elapsed().as_secs_f64());
|
||||
}
|
||||
resp
|
||||
} else {
|
||||
// Direct passthrough when no logprobs needed
|
||||
let response_headers =
|
||||
@@ -894,13 +1226,26 @@ impl PDRouter {
|
||||
|
||||
match res.bytes().await {
|
||||
Ok(decode_body) => {
|
||||
let mut response = Response::new(Body::from(decode_body));
|
||||
*response.status_mut() = status;
|
||||
*response.headers_mut() = response_headers;
|
||||
let decode_body =
|
||||
self.shadow_record_non_stream(decode_body, status, &shadow);
|
||||
let response = Self::non_stream_passthrough_response(
|
||||
status,
|
||||
decode_body,
|
||||
response_headers,
|
||||
);
|
||||
// T17: non-streaming e2e (observability only).
|
||||
if let Some(g9i) = &self.g9i {
|
||||
g9i.record_nonstream_e2e(start_time.elapsed().as_secs_f64());
|
||||
}
|
||||
response
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to read decode response: {}", e);
|
||||
// Shadow T3: the admitted dispatch produced no
|
||||
// readable body — release the queue slot.
|
||||
if let (Some(g9i), Some(s)) = (&self.g9i, &shadow) {
|
||||
g9i.shadow_release(s);
|
||||
}
|
||||
error::internal_error(
|
||||
"read_response_failed",
|
||||
"Failed to read response",
|
||||
@@ -1116,11 +1461,39 @@ impl PDRouter {
|
||||
headers: Option<HeaderMap>,
|
||||
prefill: Arc<dyn Worker>,
|
||||
decode: Arc<dyn Worker>,
|
||||
// TTFT sampling hook: Some((dispatch-entry instant, path label)) for
|
||||
// real upstream streams; None for synthetic error envelopes (their
|
||||
// "first chunk" is not a token — sampling them would poison the
|
||||
// window with near-zero error samples) and for tests.
|
||||
ttft: Option<(Instant, &'static str)>,
|
||||
// G9i shadow T2 hook: Some only for shadowable chat requests on a
|
||||
// success-status stream; None on error envelopes and tests.
|
||||
shadow: Option<ShadowResponseContext>,
|
||||
) -> Response {
|
||||
use crate::core::AttachedBody;
|
||||
|
||||
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
|
||||
// G9i TTFT sample: record on the first upstream chunk — that moment is
|
||||
// the request's time-to-first-token as seen by the gateway (dispatch
|
||||
// entry -> first token). Non-streaming requests have no observable
|
||||
// first-token moment and are not sampled.
|
||||
let g9i_ttft = self.g9i.clone().zip(ttft);
|
||||
let mut ttft_recorded = false;
|
||||
|
||||
// G9i shadow T2: zip the state handle with the per-request context
|
||||
// (chain + node + strip flags) for the stream loop below.
|
||||
let g9i_shadow = self.g9i.clone().zip(shadow);
|
||||
// Shadow T3 queue bookkeeping state. `ttft` is Option<(Instant,
|
||||
// &'static str)> which is Copy, so zip() above left it usable:
|
||||
// mirror the dispatch-entry instant so the first chunk can also feed
|
||||
// the shadow TTFT calibration. `shadow_done` latches once usage is
|
||||
// recorded; teardown releases the admission only when no completion
|
||||
// happened (abort / client disconnect / upstream error).
|
||||
let ttft_start = ttft.map(|(t, _)| t);
|
||||
let mut shadow_ttft_secs: Option<f64> = None;
|
||||
let mut shadow_done = false;
|
||||
|
||||
// Uses select! to race stream.next() against tx.closed() so that
|
||||
// when the client disconnects the upstream HTTP connection is dropped
|
||||
// promptly, allowing the engine to abort the request.
|
||||
@@ -1149,8 +1522,81 @@ impl PDRouter {
|
||||
chunk_result = tracked.next() => {
|
||||
match chunk_result {
|
||||
Some(Ok(chunk)) => {
|
||||
// G9i shadow T2: harvest the usage event for
|
||||
// prefix-table bookkeeping and strip exactly
|
||||
// the fields we injected on the way in.
|
||||
// Chunks without a usage field pass through
|
||||
// byte-identically. A chunk reduced to nothing
|
||||
// (usage-only event we must hide from the
|
||||
// client) is dropped entirely.
|
||||
let chunk = if let Some((g9i, sctx)) = &g9i_shadow {
|
||||
let (usage, forward) =
|
||||
super::g9i_shadow::process_sse_chunk(
|
||||
&chunk,
|
||||
sctx.strip_usage,
|
||||
sctx.strip_cached_details,
|
||||
);
|
||||
if let Some(json) = usage {
|
||||
// Shadow T3: exactly one completion
|
||||
// per admission — the latch keeps a
|
||||
// (malformed) repeat usage event from
|
||||
// double-completing the queue slot.
|
||||
if !shadow_done {
|
||||
// Streaming harvests the usage
|
||||
// event pre-strip, so the
|
||||
// breakdown extraction inside
|
||||
// works unaided (None).
|
||||
g9i.shadow_record(sctx, &json, shadow_ttft_secs, None);
|
||||
shadow_done = true;
|
||||
}
|
||||
}
|
||||
match forward {
|
||||
Some(bytes) => bytes,
|
||||
None => continue,
|
||||
}
|
||||
} else {
|
||||
chunk
|
||||
};
|
||||
|
||||
let is_done = memmem::find(&chunk, b"data: [DONE]").is_some();
|
||||
|
||||
if !ttft_recorded {
|
||||
ttft_recorded = true;
|
||||
if let Some((g9i, (start_time, g9i_path))) = &g9i_ttft {
|
||||
g9i.record_ttft(
|
||||
g9i_path,
|
||||
start_time.elapsed().as_secs_f64(),
|
||||
);
|
||||
// T25: TTFT by shadow class — the
|
||||
// same gateway-observed first-token
|
||||
// moment (start_time predates the
|
||||
// admission loop, so reservoir /
|
||||
// hold waits are included).
|
||||
if let Some((_, sctx)) = &g9i_shadow {
|
||||
if let Some(class) =
|
||||
sctx.class.map(|c| c.as_str())
|
||||
{
|
||||
metrics::histogram!(
|
||||
super::g9i::METRIC_CLASS_TTFT,
|
||||
"class" => class,
|
||||
)
|
||||
.record(start_time.elapsed().as_secs_f64());
|
||||
}
|
||||
}
|
||||
}
|
||||
// Shadow T3: the same first-token moment
|
||||
// calibrates est_TTFT (est − actual).
|
||||
shadow_ttft_secs =
|
||||
ttft_start.map(|t| t.elapsed().as_secs_f64());
|
||||
// T16: first token -> the D booking moves
|
||||
// from the pipeline to the streaming
|
||||
// sub-ledger (the window gates pipeline
|
||||
// depth only).
|
||||
if let Some((g9i, sctx)) = &g9i_shadow {
|
||||
g9i.shadow_promote_streaming(sctx);
|
||||
}
|
||||
}
|
||||
|
||||
let result = if return_logprob && prefill_logprobs.is_some() {
|
||||
Self::merge_streaming_logprobs(prefill_logprobs.clone(), &chunk)
|
||||
.unwrap_or(chunk)
|
||||
@@ -1199,6 +1645,15 @@ impl PDRouter {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Shadow T3: the stream ended without a recorded usage event
|
||||
// (abort, client disconnect, or upstream error) — release the
|
||||
// queue admission so outstanding stays conserved. No-op when the
|
||||
// usage event already completed it.
|
||||
if !shadow_done {
|
||||
if let Some((g9i, sctx)) = &g9i_shadow {
|
||||
g9i.shadow_release(sctx);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let stream = UnboundedReceiverStream::new(rx);
|
||||
@@ -1226,16 +1681,26 @@ impl PDRouter {
|
||||
status: StatusCode,
|
||||
return_logprob: bool,
|
||||
prefill_body: Option<bytes::Bytes>,
|
||||
shadow: Option<ShadowResponseContext>,
|
||||
) -> Response {
|
||||
let response = res.bytes().await;
|
||||
let decode_body = match response {
|
||||
Ok(decode_body) => decode_body,
|
||||
Err(e) => {
|
||||
error!("Failed to read decode response: {}", e);
|
||||
// Shadow T3: the admitted dispatch produced no readable
|
||||
// body — release the queue slot.
|
||||
if let (Some(g9i), Some(s)) = (&self.g9i, &shadow) {
|
||||
g9i.shadow_release(s);
|
||||
}
|
||||
return error::internal_error("read_response_failed", "Failed to read response");
|
||||
}
|
||||
};
|
||||
|
||||
// Shadow T2: record usage + strip injected fields before any
|
||||
// logprob merging (merge touches choices only, usage is untouched).
|
||||
let decode_body = self.shadow_record_non_stream(decode_body, status, &shadow);
|
||||
|
||||
if !return_logprob {
|
||||
return (status, decode_body).into_response();
|
||||
}
|
||||
@@ -1265,6 +1730,71 @@ impl PDRouter {
|
||||
}
|
||||
}
|
||||
|
||||
/// G9i shadow bookkeeping for non-streaming responses: parse the body,
|
||||
/// record usage into the prefix table (T2), pair the T3 queue admission
|
||||
/// with exactly one complete (parseable body) or release (non-success /
|
||||
/// unparseable), and strip `cached_tokens_details` when (and only when)
|
||||
/// we injected the request flag ourselves.
|
||||
///
|
||||
/// Passthrough guarantees: shadow off / non-success status / unparseable
|
||||
/// body / nothing stripped -> the ORIGINAL bytes are returned untouched,
|
||||
/// so clients never see a re-serialized response unless we actually
|
||||
/// removed an injected field.
|
||||
fn shadow_record_non_stream(
|
||||
&self,
|
||||
body: bytes::Bytes,
|
||||
status: StatusCode,
|
||||
shadow: &Option<ShadowResponseContext>,
|
||||
) -> bytes::Bytes {
|
||||
let (Some(g9i), Some(ctx)) = (&self.g9i, shadow) else {
|
||||
return body;
|
||||
};
|
||||
// Only 200 responses with usage may write the table (design §3.3-3).
|
||||
// A non-success response still releases the T3 queue admission.
|
||||
if !status.is_success() {
|
||||
g9i.shadow_release(ctx);
|
||||
return body;
|
||||
}
|
||||
let (parsed, rewritten, breakdown) =
|
||||
super::g9i_shadow::strip_and_extract(&body, ctx.strip_cached_details);
|
||||
match &parsed {
|
||||
// TTFT calibration is streaming-only (non-streaming has no
|
||||
// observable first-token moment) -> None. The breakdown was
|
||||
// extracted pre-strip and is passed through — the parsed JSON
|
||||
// here is post-strip, so re-extracting from it would always
|
||||
// miss (the 2026-10-17 order bug).
|
||||
Some(json) => g9i.shadow_record(ctx, json, None, breakdown),
|
||||
// Unparseable success body: nothing to record, but the dispatch
|
||||
// is over — release rather than leak the outstanding count.
|
||||
None => g9i.shadow_release(ctx),
|
||||
}
|
||||
match rewritten {
|
||||
Some(bytes) => bytes::Bytes::from(bytes),
|
||||
None => body,
|
||||
}
|
||||
}
|
||||
|
||||
/// Assemble the non-streaming passthrough response. The shadow strip
|
||||
/// may have REWRITTEN (shortened) the body, so the upstream
|
||||
/// Content-Length in `headers` is stale; forwarding it verbatim
|
||||
/// truncated cache-hit responses at exactly the stripped length (the
|
||||
/// 2026-10-17 IncompleteRead incident). Pin Content-Length to the
|
||||
/// actual body length — correct for rewritten and untouched bodies
|
||||
/// alike.
|
||||
fn non_stream_passthrough_response(
|
||||
status: StatusCode,
|
||||
body: bytes::Bytes,
|
||||
mut headers: HeaderMap,
|
||||
) -> Response {
|
||||
if let Ok(v) = HeaderValue::from_str(body.len().to_string().as_str()) {
|
||||
headers.insert(CONTENT_LENGTH, v);
|
||||
}
|
||||
let mut response = Response::new(Body::from(body));
|
||||
*response.status_mut() = status;
|
||||
*response.headers_mut() = headers;
|
||||
response
|
||||
}
|
||||
|
||||
// Helper to process prefill response and extract body if needed for logprobs
|
||||
async fn process_prefill_response(
|
||||
&self,
|
||||
@@ -1780,6 +2310,7 @@ mod tests {
|
||||
retry_config: RetryConfig::default(),
|
||||
api_key: Some("test_api_key".to_string()),
|
||||
enable_igw: false,
|
||||
g9i: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1791,6 +2322,39 @@ mod tests {
|
||||
Box::new(worker)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_stream_passthrough_pins_content_length_to_body() {
|
||||
// 2026-10-17 IncompleteRead regression: the shadow strip shortens
|
||||
// the body on engine cache hits; the upstream Content-Length must
|
||||
// never survive onto the rewritten body.
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_LENGTH, HeaderValue::from_static("637"));
|
||||
let body = bytes::Bytes::from_static(b"{}");
|
||||
let resp =
|
||||
PDRouter::non_stream_passthrough_response(StatusCode::OK, body.clone(), headers);
|
||||
assert_eq!(
|
||||
resp.headers().get(CONTENT_LENGTH).unwrap(),
|
||||
&HeaderValue::from_static("2"),
|
||||
"stale upstream length is replaced by the actual body length"
|
||||
);
|
||||
|
||||
// Untouched body: the pinned length matches the original.
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_LENGTH, HeaderValue::from_static("2"));
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
let resp = PDRouter::non_stream_passthrough_response(StatusCode::OK, body, headers);
|
||||
assert_eq!(
|
||||
resp.headers().get(CONTENT_LENGTH).unwrap(),
|
||||
&HeaderValue::from_static("2")
|
||||
);
|
||||
assert_eq!(
|
||||
resp.headers().get(CONTENT_TYPE).unwrap(),
|
||||
&HeaderValue::from_static("application/json"),
|
||||
"other preserved headers survive"
|
||||
);
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chat_request_text_uses_full_conversation() {
|
||||
// Regression test for https://github.com/sgl-project/sglang/issues/26263
|
||||
@@ -2048,6 +2612,8 @@ mod tests {
|
||||
None,
|
||||
prefill_ref.clone(),
|
||||
decode_ref.clone(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
// Guards are now attached to response body, so load should be 1
|
||||
|
||||
@@ -321,6 +321,7 @@ mod pd_responses_routing_tests {
|
||||
retry_config: RetryConfig::default(),
|
||||
api_key: None,
|
||||
enable_igw: false,
|
||||
g9i: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user