Compare commits
40
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c8d06424fa | ||
|
|
97c7eabb33 | ||
|
|
e9b436dc2d | ||
|
|
fc63ae5392 | ||
|
|
9df2d40324 | ||
|
|
6a8a6e399f | ||
|
|
e6c0737541 | ||
|
|
9374e03f13 | ||
|
|
86883ae89f | ||
|
|
43c2844c18 | ||
|
|
9e288079e4 | ||
|
|
738d5a50d0 | ||
|
|
e80c8608dc | ||
|
|
5351d5491b | ||
|
|
8a2e1403a3 | ||
|
|
1910eccb8e | ||
|
|
6217b9de46 | ||
|
|
9dd446b5c1 | ||
|
|
f76eaf0f92 | ||
|
|
45fdc97082 | ||
|
|
f690861265 | ||
|
|
e4bfdc949b | ||
|
|
59e3aee6d9 | ||
|
|
406081d3e3 | ||
|
|
1975265315 | ||
|
|
534417b82e | ||
|
|
ed0a454730 | ||
|
|
04ee2d03d7 | ||
|
|
9d5db13e53 | ||
|
|
7e20f0af79 | ||
|
|
0b74ab2ac4 | ||
|
|
3c7ec33d26 | ||
|
|
09ca4957ee | ||
|
|
7921442a5c | ||
|
|
05ef104208 | ||
|
|
d44620f285 | ||
|
|
5df4f2c036 | ||
|
|
db1e8cdbe3 | ||
|
|
f3ac513d92 | ||
|
|
0054c5b8fe |
@@ -21,6 +21,8 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import zlib
|
||||
from array import array
|
||||
from collections import deque
|
||||
from http import HTTPStatus
|
||||
@@ -105,6 +107,64 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
# DEBUG-ONLY (p2 wedge trap 2026-10-13): per-step batch-decision fingerprint
|
||||
# and event-loop phase logging. Same env as custom_all_reduce_v2 ARDBG.
|
||||
# 1 = sampled/one-line, 2 = + stacks on ONE_SHOT_PUSH. Off by default.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
_plan_seq = 0
|
||||
_plan_none_seq = 0
|
||||
|
||||
|
||||
def _dbg_plan(self, batch) -> None:
|
||||
global _plan_seq, _plan_none_seq
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
_plan_seq += 1
|
||||
rank = torch.distributed.get_rank()
|
||||
if batch is None:
|
||||
# 空转步采样记录,避免 idle 自旋刷爆日志
|
||||
_plan_none_seq += 1
|
||||
if _plan_none_seq & 511:
|
||||
return
|
||||
logger.info("[ARDBG] plan rank=%s step=%d batch=None", rank, _plan_seq)
|
||||
return
|
||||
reqs = batch.reqs
|
||||
fp = 0
|
||||
ext_total = 0
|
||||
for r in reqs:
|
||||
# crc32 而非 hash():后者按进程随机化种子,跨 rank 不可比
|
||||
fp = (fp * 131 + zlib.crc32(r.rid.encode())) & 0xFFFFFFFFFFFFFFFF
|
||||
er = getattr(r, "extend_range", None)
|
||||
el = int(er.length) if er is not None else -1
|
||||
fp = (fp * 131 + el) & 0xFFFFFFFFFFFFFFFF
|
||||
fp = (fp * 131 + int(getattr(r, "prefix_len", -1))) & 0xFFFFFFFFFFFFFFFF
|
||||
if el > 0:
|
||||
ext_total += el
|
||||
logger.info(
|
||||
"[ARDBG] plan rank=%s step=%d nreq=%d ext=%d fp=%016x",
|
||||
rank,
|
||||
_plan_seq,
|
||||
len(reqs),
|
||||
ext_total,
|
||||
fp,
|
||||
)
|
||||
|
||||
|
||||
def _dbg_phase(label: str, batch=None) -> None:
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
if batch is None and label in ("inflight_enter", "inflight_exit"):
|
||||
# 空转迭代的 inflight 相位价值低,采样记录
|
||||
if _plan_seq & 255:
|
||||
return
|
||||
logger.info(
|
||||
"[ARDBG] phase rank=%s step=%s %s nreq=%s",
|
||||
torch.distributed.get_rank(),
|
||||
_plan_seq,
|
||||
label,
|
||||
len(batch.reqs) if batch is not None else "-",
|
||||
)
|
||||
|
||||
|
||||
def should_force_retry(req: Req) -> bool:
|
||||
"""Test hook to force a request into optimistic prefill retry."""
|
||||
@@ -628,6 +688,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
set_schedule_time_batch(batch)
|
||||
|
||||
_dbg_plan(self, batch)
|
||||
return NextBatchPlan(batch_to_run=batch, running_batch=running_batch)
|
||||
|
||||
@torch.no_grad()
|
||||
@@ -658,13 +719,17 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
_dbg_phase("run_batch_enter", batch)
|
||||
result = self.run_batch(batch)
|
||||
_dbg_phase("run_batch_exit", batch)
|
||||
self.process_batch_result(batch, result)
|
||||
else:
|
||||
self._sched_idled = True
|
||||
self.on_idle()
|
||||
|
||||
_dbg_phase("inflight_enter")
|
||||
self.process_disagg_prefill_inflight_queue()
|
||||
_dbg_phase("inflight_exit")
|
||||
|
||||
# Update last_batch
|
||||
self.last_batch = batch
|
||||
@@ -698,7 +763,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
_dbg_phase("run_batch_enter", batch)
|
||||
batch_result = self.run_batch(batch)
|
||||
_dbg_phase("run_batch_exit", batch)
|
||||
self._apply_war_barrier()
|
||||
self.result_queue.append((batch.copy(), batch_result))
|
||||
else:
|
||||
@@ -713,7 +780,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# When the server is idle, do self-check and re-init some states
|
||||
self.on_idle()
|
||||
|
||||
_dbg_phase("inflight_enter")
|
||||
self.process_disagg_prefill_inflight_queue()
|
||||
_dbg_phase("inflight_exit")
|
||||
|
||||
# Run sample of the current batch
|
||||
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from collections import deque
|
||||
from contextlib import nullcontext
|
||||
@@ -43,6 +45,32 @@ if is_npu():
|
||||
DSV4NPUTokenToKVPool,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# DEBUG-ONLY (p2 wedge repro 2026-10-13): env-gated seq logging for host-side
|
||||
# gloo all-reduce sync points. Mirrors SGLANG_AR_SEQ_DEBUG in
|
||||
# custom_all_reduce_v2.py. 1 = one-line, 2 = + python stack. Not for prod.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
_host_ar_seq = 0
|
||||
|
||||
|
||||
def _dbg_log_host_ar(phase: str, polls) -> None:
|
||||
global _host_ar_seq
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
seq = _host_ar_seq
|
||||
_host_ar_seq += 1
|
||||
# 空转迭代(无 poller)采样 1/256,避免 idle 自旋刷爆日志
|
||||
if (not polls) and seq & 255:
|
||||
return
|
||||
logger.info(
|
||||
"[ARDBG] host rank=%s seq=%d phase=%s polls=%s",
|
||||
dist.get_rank(),
|
||||
seq,
|
||||
phase,
|
||||
polls,
|
||||
)
|
||||
|
||||
#########################
|
||||
# Constants & Enums
|
||||
#########################
|
||||
@@ -213,13 +241,20 @@ def poll_and_all_reduce_attn_cp_tp_group(
|
||||
attn_cp_cpu_group: dist.ProcessGroup,
|
||||
attn_tp_cpu_group: dist.ProcessGroup,
|
||||
):
|
||||
# DEBUG-ONLY
|
||||
_dbg_log_host_ar("enter", len(pollers))
|
||||
# First sync across attn-tp ranks so all TP participants for a given (dp, cp)
|
||||
# shard observe the same status transitions.
|
||||
polls = poll_and_all_reduce(pollers, attn_tp_cpu_group)
|
||||
# DEBUG-ONLY
|
||||
_dbg_log_host_ar("tp-done", polls)
|
||||
|
||||
# Then sync across attn-cp ranks, so all TPxCP participants in one DP shard
|
||||
# converge to the same global status.
|
||||
return _all_reduce_polls(polls, attn_cp_cpu_group)
|
||||
result = _all_reduce_polls(polls, attn_cp_cpu_group)
|
||||
# DEBUG-ONLY
|
||||
_dbg_log_host_ar("cp-done", result)
|
||||
return result
|
||||
|
||||
|
||||
def poll_and_all_reduce_with_staging(
|
||||
|
||||
@@ -25,6 +25,8 @@ the kernel captured in the graph dereferences its row at replay time.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import traceback
|
||||
from contextlib import contextmanager
|
||||
from typing import List, NamedTuple, Optional, Tuple
|
||||
|
||||
@@ -68,6 +70,9 @@ MB = 1024 * 1024
|
||||
_ALIGN_BYTES = 1024
|
||||
_SEMAPHORE_BYTES = 128
|
||||
_MAX_GRAPH_INPUTS = 131072
|
||||
# DEBUG-ONLY (p2 wedge repro 2026-10-13): env-gated seq logging for TP device
|
||||
# all-reduce calls. 1 = one-line per call, 2 = + python stack. Not for prod.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
# resolved once at import time; explicit constructor sizes take precedence
|
||||
_DEFAULT_MAX_SIZE = envs.SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB.get() * 1024
|
||||
# forced per-direction workspace sizes; highest priority, override both the
|
||||
@@ -297,6 +302,9 @@ class CustomAllReduceV2:
|
||||
self.pull_enabled,
|
||||
)
|
||||
dist.barrier(group=self.group)
|
||||
# DEBUG-ONLY: per-instance seq counter for SGLANG_AR_SEQ_DEBUG logging
|
||||
self._dbg_seq = 0
|
||||
self._dbg_rank = self.rank
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Algo selection
|
||||
@@ -376,6 +384,27 @@ class CustomAllReduceV2:
|
||||
assert config is not None, f"No config for {nbytes = }"
|
||||
algo, use_graph, use_multicast = config
|
||||
graph_params = self._allocate_graph_row(input, nbytes) if use_graph else None
|
||||
if _AR_SEQ_DEBUG:
|
||||
seq = self._dbg_seq
|
||||
self._dbg_seq += 1
|
||||
# 采样 1/1024:生产高频路径下全量日志会拖垮 P 节点
|
||||
if seq & 1023 == 0:
|
||||
logger.info(
|
||||
"[ARDBG] dev rank=%s seq=%d nbytes=%d algo=%s graph=%s capturing=%s",
|
||||
self._dbg_rank,
|
||||
seq,
|
||||
nbytes,
|
||||
getattr(algo, "name", algo),
|
||||
use_graph,
|
||||
torch.cuda.is_current_stream_capturing(),
|
||||
)
|
||||
if _AR_SEQ_DEBUG >= 2 and getattr(algo, "name", algo) == "ONE_SHOT_PUSH":
|
||||
logger.info(
|
||||
"[ARDBG] stack rank=%s seq=%d\n%s",
|
||||
self._dbg_rank,
|
||||
seq,
|
||||
"".join(traceback.format_stack(limit=15)),
|
||||
)
|
||||
return custom_all_reduce(
|
||||
self.obj,
|
||||
input,
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import replace
|
||||
@@ -132,6 +133,11 @@ COMPONENT_REGISTRY: dict[ComponentType, type[TreeComponent]] = {
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# DEBUG-ONLY (p2 wedge trap 2026-10-13): env-gated logging for the per-step
|
||||
# hicache event sync. Same SGLANG_AR_SEQ_DEBUG switch as the ARDBG sites.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
_hc_step = 0
|
||||
|
||||
|
||||
class _OngoingWriteThrough(NamedTuple):
|
||||
"""Tracks an in-flight D→H write-through operation."""
|
||||
@@ -3533,6 +3539,16 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
def check_hicache_events(self) -> None:
|
||||
"""Called per scheduler step to poll async HiCache events."""
|
||||
global _hc_step
|
||||
if _AR_SEQ_DEBUG:
|
||||
_hc_step += 1
|
||||
if _hc_step & 255 == 0:
|
||||
logger.info(
|
||||
"[ARDBG] hcev rank=%s step=%d linker=%s",
|
||||
torch.distributed.get_rank(),
|
||||
_hc_step,
|
||||
self.linker is not None,
|
||||
)
|
||||
if self.linker is not None:
|
||||
finish_counts = torch.tensor(
|
||||
[
|
||||
@@ -3566,6 +3582,15 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
storage_queue_sizes,
|
||||
extra_pool_names,
|
||||
) = self._sync_hicache_ready_counts()
|
||||
if _AR_SEQ_DEBUG and _hc_step & 255 == 0:
|
||||
logger.info(
|
||||
"[ARDBG] hcsync rank=%s step=%d write=%s load=%s storage=%s",
|
||||
torch.distributed.get_rank(),
|
||||
_hc_step,
|
||||
write_finish_count,
|
||||
load_finish_count,
|
||||
storage_queue_sizes,
|
||||
)
|
||||
self.writing_check(finish_count=write_finish_count)
|
||||
self.loading_check(finish_count=load_finish_count)
|
||||
|
||||
|
||||
@@ -284,6 +284,90 @@ struct CliArgs {
|
||||
#[arg(long, default_value_t = 86400, help_heading = "G9i Pool Admission")]
|
||||
g9i_shadow_table_ttl_secs: u64,
|
||||
|
||||
/// est_T_D shadow estimate (D-side prealloc wait folded into shadow est_TTFT; observability only, default on; set false to roll back to the P-only model exactly)
|
||||
#[arg(long, default_value_t = true, help_heading = "G9i Pool Admission")]
|
||||
g9i_est_td_enabled: bool,
|
||||
|
||||
/// Shadow-consistent routing: live prefill pick follows the shadow decision's matched_node (requires --g9i-shadow-enabled; rejection gates unchanged; default off)
|
||||
#[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")]
|
||||
g9i_shadow_route: bool,
|
||||
|
||||
/// T_D admission mode: off (default, ≡ current behavior) | shadow (log/count would-rejects, allow) | enforce (real 429s; requires --g9i-shadow-enabled)
|
||||
#[arg(long, default_value = "off", help_heading = "G9i Pool Admission")]
|
||||
g9i_td_admit: String,
|
||||
|
||||
/// T_D hard gate (seconds): when min over decode nodes of T_D exceeds this, every class gets 429 d_prealloc_saturated (0 = disabled)
|
||||
#[arg(long, default_value_t = 5.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_td_gate_secs: f64,
|
||||
|
||||
/// T_PREFER (seconds): prefix-holder affinity yields to a cheaper prefill candidate when the holder's est_TTFT exceeds this
|
||||
#[arg(long, default_value_t = 2.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_t_prefer_secs: f64,
|
||||
|
||||
/// Hard P-queue token cap (physical backstop; effective cap = min(this, drain_ewma x budget)) — 429 p_queue_cap only when ALL nodes over (0 = disabled; follows --g9i-td-admit mode)
|
||||
#[arg(long, default_value_t = 16_000_000, help_heading = "G9i Pool Admission")]
|
||||
g9i_p_queue_token_cap: u64,
|
||||
|
||||
/// P-queue real-queue gate: skip prefill nodes whose engine-reported waiting queue (num_waiting_reqs) exceeds this (0 = disabled; follows --g9i-td-admit mode)
|
||||
#[arg(long, default_value_t = 10, help_heading = "G9i Pool Admission")]
|
||||
g9i_p_queue_reqs_cap: u64,
|
||||
|
||||
/// Admission hold queue: park holdable rejections (d_window_full / p_queue_cap) up to this many seconds instead of an immediate 429 (0 = disabled; enforce mode only; hold time counts into TTFT)
|
||||
#[arg(long, default_value_t = 1.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_admit_hold_secs: f64,
|
||||
|
||||
/// Admission hold queue bound: at most this many parked requests (full queue = immediate 429)
|
||||
#[arg(long, default_value_t = 512, help_heading = "G9i Pool Admission")]
|
||||
g9i_admit_hold_max: u64,
|
||||
|
||||
/// D-ledger pipeline booking TTL (seconds): force-release bookings that never saw a first token (wedge backstop; 0 = disabled)
|
||||
#[arg(long, default_value_t = 120, help_heading = "G9i Pool Admission")]
|
||||
g9i_pipeline_ttl_secs: u64,
|
||||
|
||||
/// D-ledger streaming booking TTL (seconds): force-release bookings stuck mid-decode (0 = disabled)
|
||||
#[arg(long, default_value_t = 600, help_heading = "G9i Pool Admission")]
|
||||
g9i_streaming_ttl_secs: u64,
|
||||
|
||||
/// Ledger reconcile: clamp the D ledger down to the engine-side in-flight when the excess is sustained >60s (subtract-only)
|
||||
#[arg(long, default_value_t = true, help_heading = "G9i Pool Admission")]
|
||||
g9i_ledger_reconcile: bool,
|
||||
|
||||
/// AIMD window initial pipeline depth per decode node (warmup start; clamped to [16, 256])
|
||||
#[arg(long, default_value_t = 128.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_window_init: f64,
|
||||
|
||||
/// AIMD additive-increase step per no-congestion tick (slo_ok / blind_recover; cut & holds unchanged)
|
||||
#[arg(long, default_value_t = 64.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_window_add: f64,
|
||||
|
||||
/// Pin the AIMD admission window at this value (clamped to [16, 256]); skips all window dynamics (0 = normal AIMD)
|
||||
#[arg(long, default_value_t = 0.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_window_fixed: f64,
|
||||
|
||||
/// Prompt-length hard validation: 400 prompt_too_long at the front of admission when even the lower-bound estimate exceeds the context limit (default on)
|
||||
#[arg(long, default_value_t = true, help_heading = "G9i Pool Admission")]
|
||||
g9i_prompt_validation: bool,
|
||||
|
||||
/// Context-length limit (tokens) for the prompt_too_long lower-bound check (0 = disabled)
|
||||
#[arg(long, default_value_t = 1_048_576, help_heading = "G9i Pool Admission")]
|
||||
g9i_prompt_token_limit: u64,
|
||||
|
||||
/// T25 hot-channel admission A/B policy: off (default ≡ current behavior) | reservoir (Hot-class holdable rejections park in a priority reservoir up to --g9i-reservoir-max-ms instead of 429) | hardlane (cold classes may use only (1 - --g9i-hot-reserve-frac) of the d_window/p_queue budgets)
|
||||
#[arg(long, default_value = "off", help_heading = "G9i Pool Admission")]
|
||||
g9i_hot_policy: String,
|
||||
|
||||
/// T25-A reservoir wait budget in milliseconds (timeout → 429 reservoir_timeout; the wait counts into TTFT)
|
||||
#[arg(long, default_value_t = 1000, help_heading = "G9i Pool Admission")]
|
||||
g9i_reservoir_max_ms: u64,
|
||||
|
||||
/// T25-A reservoir capacity: at most this many parked requests (full → immediate 429 reservoir_full)
|
||||
#[arg(long, default_value_t = 512, help_heading = "G9i Pool Admission")]
|
||||
g9i_reservoir_cap: u64,
|
||||
|
||||
/// T25-B hardlane cold-budget reserve fraction: cold classes use at most (1 - frac) of the AIMD d_window and p_queue caps (fixed value, no adaptation)
|
||||
#[arg(long, default_value_t = 0.25, help_heading = "G9i Pool Admission")]
|
||||
g9i_hot_reserve_frac: f64,
|
||||
|
||||
// ==================== Service Discovery (Kubernetes) ====================
|
||||
/// Enable Kubernetes service discovery
|
||||
#[arg(
|
||||
@@ -1163,6 +1247,27 @@ impl CliArgs {
|
||||
shadow_enabled: self.g9i_shadow_enabled,
|
||||
shadow_table_max_entries: self.g9i_shadow_table_max_entries,
|
||||
shadow_table_ttl_secs: self.g9i_shadow_table_ttl_secs,
|
||||
est_td_enabled: self.g9i_est_td_enabled,
|
||||
shadow_route: self.g9i_shadow_route,
|
||||
td_admit: self.g9i_td_admit.clone(),
|
||||
td_gate_secs: self.g9i_td_gate_secs,
|
||||
t_prefer_secs: self.g9i_t_prefer_secs,
|
||||
p_queue_token_cap: self.g9i_p_queue_token_cap,
|
||||
p_queue_reqs_cap: self.g9i_p_queue_reqs_cap,
|
||||
admit_hold_secs: self.g9i_admit_hold_secs,
|
||||
admit_hold_max: self.g9i_admit_hold_max,
|
||||
pipeline_ttl_secs: self.g9i_pipeline_ttl_secs,
|
||||
streaming_ttl_secs: self.g9i_streaming_ttl_secs,
|
||||
ledger_reconcile: self.g9i_ledger_reconcile,
|
||||
window_init: self.g9i_window_init,
|
||||
window_add: self.g9i_window_add,
|
||||
window_fixed: self.g9i_window_fixed,
|
||||
prompt_validation: self.g9i_prompt_validation,
|
||||
prompt_token_limit: self.g9i_prompt_token_limit,
|
||||
hot_policy: self.g9i_hot_policy.clone(),
|
||||
reservoir_max_ms: self.g9i_reservoir_max_ms,
|
||||
reservoir_cap: self.g9i_reservoir_cap,
|
||||
hot_reserve_frac: self.g9i_hot_reserve_frac,
|
||||
};
|
||||
config.validate().map_err(|reason| ConfigError::ValidationFailed { reason })?;
|
||||
Ok(Some(config))
|
||||
|
||||
@@ -334,6 +334,64 @@ pub(crate) fn init_metrics() {
|
||||
smg_mesh::metrics::init_mesh_metrics();
|
||||
}
|
||||
|
||||
/// T25b: explicit Prometheus histogram buckets for every gateway histogram
|
||||
/// NOT covered by the `duration_seconds` suffix matcher (those keep the
|
||||
/// configurable duration_buckets). Root cause being fixed: without explicit
|
||||
/// buckets the exporter renders histograms as client-side SUMMARIES
|
||||
/// (DDSketch over a 3×20s rolling window); in the v16 deployment those
|
||||
/// summaries exported all-zero quantiles while _sum/_count stayed healthy
|
||||
/// (the rolling-window path silently drops samples when the quanta clock
|
||||
/// reading falls behind the newest bucket begin, poisoning the summary
|
||||
/// permanently). Explicit buckets render real Prometheus histograms —
|
||||
/// aggregable across time and replicas with histogram_quantile, and immune
|
||||
/// to the summary path entirely.
|
||||
///
|
||||
/// Groups by scale (keep in sync when adding a histogram!):
|
||||
/// * TTFT-scale seconds (0.5–120s): TTFT, e2e, est_T_P/T_D, residuals
|
||||
/// (signed values fall into the lowest bucket — acceptable),
|
||||
/// bootstrap actuals;
|
||||
/// * fine seconds (5ms–5s): TPOT, hold/reservoir waits, retry backoff;
|
||||
/// * prefix lookup depth (ring count, 1–400);
|
||||
/// * shadow-table evicted entry age (1s–1d).
|
||||
fn histogram_bucket_overrides() -> Vec<(Matcher, Vec<f64>)> {
|
||||
let ttft: Vec<f64> = vec![0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 60.0, 120.0];
|
||||
let fine: Vec<f64> = vec![0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0];
|
||||
let depth: Vec<f64> = vec![1.0, 2.0, 3.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 400.0];
|
||||
let age: Vec<f64> = vec![1.0, 10.0, 60.0, 300.0, 900.0, 3600.0, 21600.0, 86400.0];
|
||||
let mut out: Vec<(Matcher, Vec<f64>)> = Vec::new();
|
||||
for name in [
|
||||
"smg_g9i_ttft_seconds",
|
||||
"smg_g9i_class_ttft_seconds",
|
||||
"smg_g9i_nonstream_e2e_secs",
|
||||
"smg_g9i_est_tp_secs",
|
||||
"smg_g9i_est_td_secs",
|
||||
"smg_g9i_est_td_slot_secs",
|
||||
"smg_g9i_shadow_ttft_est_error_seconds",
|
||||
"smg_g9i_est_vs_actual_residual",
|
||||
"smg_g9i_bootstrap_actual_secs",
|
||||
"smg_router_ttft_seconds",
|
||||
] {
|
||||
out.push((Matcher::Full(name.to_string()), ttft.clone()));
|
||||
}
|
||||
for name in [
|
||||
"smg_router_tpot_seconds",
|
||||
"smg_g9i_admit_hold_secs",
|
||||
"smg_g9i_reservoir_wait_seconds",
|
||||
"smg_worker_retry_backoff_seconds",
|
||||
] {
|
||||
out.push((Matcher::Full(name.to_string()), fine.clone()));
|
||||
}
|
||||
out.push((
|
||||
Matcher::Full("smg_g9i_shadow_prefix_lookup_depth".to_string()),
|
||||
depth,
|
||||
));
|
||||
out.push((
|
||||
Matcher::Full("smg_g9i_shadow_prefix_table_evicted_age_seconds".to_string()),
|
||||
age,
|
||||
));
|
||||
out
|
||||
}
|
||||
|
||||
pub fn start_prometheus(config: PrometheusConfig) {
|
||||
init_metrics();
|
||||
|
||||
@@ -351,11 +409,20 @@ pub fn start_prometheus(config: PrometheusConfig) {
|
||||
.unwrap_or(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)));
|
||||
let socket_addr = SocketAddr::new(ip_addr, config.port);
|
||||
|
||||
PrometheusBuilder::new()
|
||||
let mut builder = PrometheusBuilder::new()
|
||||
.with_http_listener(socket_addr)
|
||||
.upkeep_timeout(Duration::from_secs(5 * 60))
|
||||
.set_buckets_for_metric(duration_matcher, &duration_bucket)
|
||||
.expect("failed to set duration bucket")
|
||||
.expect("failed to set duration bucket");
|
||||
// T25b: every remaining gateway histogram gets explicit buckets (see
|
||||
// histogram_bucket_overrides) — no metric falls back to the broken
|
||||
// client-side summary path.
|
||||
for (matcher, buckets) in histogram_bucket_overrides() {
|
||||
builder = builder
|
||||
.set_buckets_for_metric(matcher, &buckets)
|
||||
.expect("failed to set g9i histogram buckets");
|
||||
}
|
||||
builder
|
||||
.install()
|
||||
.expect("failed to install Prometheus metrics exporter");
|
||||
}
|
||||
@@ -1513,4 +1580,61 @@ mod tests {
|
||||
assert_eq!(method_to_static_str("POST"), "POST");
|
||||
assert_eq!(method_to_static_str("UNKNOWN"), "OTHER");
|
||||
}
|
||||
|
||||
/// T25b smoke: with the production bucket overrides applied, g9i
|
||||
/// histograms render as real Prometheus histograms with non-zero le
|
||||
/// bucket counts (histogram_quantile-able) — never the client-side
|
||||
/// summary path that exported all-zero quantiles in v16.
|
||||
#[test]
|
||||
fn t25b_g9i_histograms_render_bucketed() {
|
||||
let mut builder = PrometheusBuilder::new();
|
||||
for (matcher, buckets) in histogram_bucket_overrides() {
|
||||
builder = builder
|
||||
.set_buckets_for_metric(matcher, &buckets)
|
||||
.expect("bucket override must apply");
|
||||
}
|
||||
let recorder = builder.build_recorder();
|
||||
let handle = recorder.handle();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
let ttft = histogram!("smg_g9i_class_ttft_seconds", "class" => "hot");
|
||||
for v in [0.8, 1.5, 3.0, 12.0, 45.0] {
|
||||
ttft.record(v);
|
||||
}
|
||||
histogram!("smg_g9i_shadow_prefix_lookup_depth").record(7.0);
|
||||
histogram!("smg_g9i_shadow_prefix_lookup_depth").record(120.0);
|
||||
});
|
||||
let out = handle.render();
|
||||
// Real histogram exposition, not summary.
|
||||
assert!(
|
||||
out.contains("# TYPE smg_g9i_class_ttft_seconds histogram"),
|
||||
"class ttft renders as histogram:\n{out}"
|
||||
);
|
||||
assert!(
|
||||
!out.contains("smg_g9i_class_ttft_seconds{quantile="),
|
||||
"no summary quantile lines:\n{out}"
|
||||
);
|
||||
// 3 of the 5 samples (0.8, 1.5, 3.0) land at or below le=5.
|
||||
assert!(
|
||||
out.contains("smg_g9i_class_ttft_seconds_bucket{class=\"hot\",le=\"5\"} 3"),
|
||||
"le=5 bucket counts 3 samples:\n{out}"
|
||||
);
|
||||
assert!(
|
||||
out.contains("smg_g9i_class_ttft_seconds_bucket{class=\"hot\",le=\"+Inf\"} 5"),
|
||||
"+Inf bucket counts all 5:\n{out}"
|
||||
);
|
||||
// lookup_depth: 7 -> le=10; both samples (7, 120) <= 200
|
||||
// (Prometheus buckets are cumulative).
|
||||
assert!(
|
||||
out.contains("smg_g9i_shadow_prefix_lookup_depth_bucket{le=\"10\"} 1"),
|
||||
"depth 7 lands in le=10:\n{out}"
|
||||
);
|
||||
assert!(
|
||||
out.contains("smg_g9i_shadow_prefix_lookup_depth_bucket{le=\"200\"} 2"),
|
||||
"both depths land at or below le=200 (cumulative):\n{out}"
|
||||
);
|
||||
assert!(
|
||||
out.contains("smg_g9i_shadow_prefix_lookup_depth_count 2"),
|
||||
"count line present:\n{out}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -58,6 +58,23 @@
|
||||
//! canonical meta object. The META segment is unconditional — a
|
||||
//! message without these fields contributes `TAG_META ‖ len=0`,
|
||||
//! keeping the formula single-path (no presence/absence ambiguity).
|
||||
//!
|
||||
//! T23 (intra-message chunking): a text piece longer than `CHUNK_BYTES`
|
||||
//! (2048) is first rolled into bare chunk rings `xxh3(prev ‖ chunk)` —
|
||||
//! no tag, no length prefix — in content order, and the message-final
|
||||
//! ring then carries only the remaining ≤2048-byte tail in its content
|
||||
//! segment. Why: a per-message chain is structurally blind inside one
|
||||
//! long message (one message = one ring, so a shared 100KB prefix with
|
||||
//! a different tail hits nothing — v13 measured 9034/9034 depth-0
|
||||
//! misses on single-message long-prompt traffic). Chunk rings give the
|
||||
//! lookup table mid-message anchors at a fixed granularity. Texts of
|
||||
//! ≤2048 bytes emit no chunk rings, so short-message chains stay
|
||||
//! byte-identical to the pre-T23 formula (pinned by
|
||||
//! `short_messages_match_legacy_chain_byte_for_byte`). Because one
|
||||
//! message may now own several rings, `chain_hashes_indexed` also
|
||||
//! returns each ring's message index; `classify` needs it to map a
|
||||
//! hit's ring position back into the message index space of
|
||||
//! `msg_bytes` / `msg_image`.
|
||||
|
||||
use serde_json::Value;
|
||||
use xxhash_rust::xxh3::{xxh3_64, Xxh3};
|
||||
@@ -70,6 +87,13 @@ const TAG_UNKNOWN: &[u8] = b"unknown\0";
|
||||
const TAG_TOOL_CALLS: &[u8] = b"tool_calls\0";
|
||||
const TAG_META: &[u8] = b"meta\0";
|
||||
|
||||
/// T23: intra-message chunk size in bytes. A text piece longer than this
|
||||
/// contributes `ceil`-style bare chunk rings before its message-final
|
||||
/// ring (see module docs). 2048 bytes ≈ 500-600 tokens of English/JSON —
|
||||
/// fine enough to rescue shared-prefix hits, coarse enough that typical
|
||||
/// multi-turn chats (short messages) never pay the extra hash passes.
|
||||
const CHUNK_BYTES: usize = 2048;
|
||||
|
||||
/// Chain seed: folds every body field that changes the rendered prompt but
|
||||
/// lives outside `messages` into one u64.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
@@ -121,7 +145,8 @@ impl ChainSeed {
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the chain hashes `h[1..=m]`, one per message, in order.
|
||||
/// Compute the chain hashes `h[1..=m]`, one ring per message (plus T23
|
||||
/// intra-message chunk rings), in order.
|
||||
///
|
||||
/// `messages` missing / not an array / empty → empty `Vec`. The caller
|
||||
/// treats an empty chain as "cannot classify" (same fall-back path as
|
||||
@@ -131,11 +156,53 @@ impl ChainSeed {
|
||||
/// The returned chain is deterministic even when `is_unclassifiable(body)`
|
||||
/// is true — unknown parts contribute their tag — but such chains are only
|
||||
/// suitable for logging/shadow comparison, not for bookkeeping lookups.
|
||||
///
|
||||
/// This is the chain-only view of `chain_hashes_indexed`; callers that
|
||||
/// index into per-message arrays (`msg_bytes`, `msg_image`) need the
|
||||
/// indexed form to translate ring positions to message indices.
|
||||
pub fn chain_hashes(seed: &ChainSeed, body: &Value) -> Vec<u64> {
|
||||
chain_hashes_indexed(seed, body).0
|
||||
}
|
||||
|
||||
/// `chain_hashes` plus, for each ring, the index of the message it
|
||||
/// belongs to (chunk rings and the message-final ring of message `k`
|
||||
/// all carry `k`). Both returned vecs always have the same length.
|
||||
pub fn chain_hashes_indexed(seed: &ChainSeed, body: &Value) -> (Vec<u64>, Vec<usize>) {
|
||||
let (msgs, _) = walk_messages(body);
|
||||
let mut chain = Vec::with_capacity(msgs.len());
|
||||
let mut ring_msg = Vec::with_capacity(msgs.len());
|
||||
let mut prev = seed.value;
|
||||
for msg in &msgs {
|
||||
for (mi, msg) in msgs.iter().enumerate() {
|
||||
// T23 pass 1: roll oversized text pieces into bare chunk rings
|
||||
// (h = xxh3(prev ‖ chunk), no tag/length) and keep the ≤2048-byte
|
||||
// tails for the message-final ring. Short texts emit nothing here
|
||||
// and their tails ARE the originals — the pre-T23 byte stream.
|
||||
let content = match &msg.content {
|
||||
ContentView::Str(s) => {
|
||||
ContentView::Str(roll_chunks(&mut prev, &mut chain, &mut ring_msg, mi, s))
|
||||
}
|
||||
ContentView::Parts(parts) => {
|
||||
let mut viewed = Vec::with_capacity(parts.len());
|
||||
for part in parts {
|
||||
viewed.push(match part {
|
||||
PartView::Text(t) => PartView::Text(roll_chunks(
|
||||
&mut prev,
|
||||
&mut chain,
|
||||
&mut ring_msg,
|
||||
mi,
|
||||
t,
|
||||
)),
|
||||
PartView::ImageUrl(u) => PartView::ImageUrl(u),
|
||||
PartView::Unknown => PartView::Unknown,
|
||||
});
|
||||
}
|
||||
ContentView::Parts(viewed)
|
||||
}
|
||||
ContentView::Empty => ContentView::Empty,
|
||||
};
|
||||
|
||||
// Pass 2: the message-final ring — the pre-T23 formula unchanged,
|
||||
// except its content segment now hashes the pass-1 tails.
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&prev.to_le_bytes());
|
||||
|
||||
@@ -143,7 +210,7 @@ pub fn chain_hashes(seed: &ChainSeed, body: &Value) -> Vec<u64> {
|
||||
h.update(&(role.len() as u64).to_le_bytes());
|
||||
h.update(role);
|
||||
|
||||
match &msg.content {
|
||||
match &content {
|
||||
ContentView::Str(s) => {
|
||||
update_tagged(&mut h, TAG_TEXT, s.as_bytes());
|
||||
}
|
||||
@@ -173,8 +240,47 @@ pub fn chain_hashes(seed: &ChainSeed, body: &Value) -> Vec<u64> {
|
||||
|
||||
prev = h.digest();
|
||||
chain.push(prev);
|
||||
ring_msg.push(mi);
|
||||
}
|
||||
chain
|
||||
(chain, ring_msg)
|
||||
}
|
||||
|
||||
/// Roll `prev` forward over `text`'s full chunks, emitting one bare ring
|
||||
/// per chunk (`xxh3(prev ‖ chunk bytes)`), and return the remaining tail
|
||||
/// (≤ CHUNK_BYTES, possibly empty) for the message-final ring. Texts of
|
||||
/// ≤ CHUNK_BYTES emit nothing and are returned untouched.
|
||||
fn roll_chunks<'a>(
|
||||
prev: &mut u64,
|
||||
chain: &mut Vec<u64>,
|
||||
ring_msg: &mut Vec<usize>,
|
||||
mi: usize,
|
||||
text: &'a str,
|
||||
) -> &'a str {
|
||||
let mut rest = text;
|
||||
while rest.len() > CHUNK_BYTES {
|
||||
let (chunk, tail) = take_chunk(rest);
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&prev.to_le_bytes());
|
||||
h.update(chunk.as_bytes());
|
||||
*prev = h.digest();
|
||||
chain.push(*prev);
|
||||
ring_msg.push(mi);
|
||||
rest = tail;
|
||||
}
|
||||
rest
|
||||
}
|
||||
|
||||
/// Split off the leading chunk of at most CHUNK_BYTES bytes, backing off
|
||||
/// to the previous UTF-8 char boundary when the cut lands mid-character.
|
||||
/// The caller guarantees `s.len() > CHUNK_BYTES`, so the boundary walk
|
||||
/// can never reach 0 (the first char is at most 4 bytes).
|
||||
fn take_chunk(s: &str) -> (&str, &str) {
|
||||
debug_assert!(s.len() > CHUNK_BYTES);
|
||||
let mut end = CHUNK_BYTES;
|
||||
while !s.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
s.split_at(end)
|
||||
}
|
||||
|
||||
/// Whether the body contains content this module has no hashing semantics
|
||||
@@ -672,4 +778,202 @@ mod tests {
|
||||
let no_name = text_body(vec![json!({"role": "user", "content": "hi"})]);
|
||||
assert_eq!(chain(&plain), chain(&no_name));
|
||||
}
|
||||
|
||||
/// `reasoning_content` must NOT change the chain: chat templates do not
|
||||
/// render historical reasoning into the prompt, so hashing it would
|
||||
/// manufacture misses for thinking-enabled clients that echo it back.
|
||||
/// Pins the walk_messages assumption (thinking:true is our default).
|
||||
#[test]
|
||||
fn reasoning_content_does_not_enter_chain() {
|
||||
let plain = text_body(vec![user("q"), assistant("a"), user("q2")]);
|
||||
let with_reasoning = text_body(vec![
|
||||
user("q"),
|
||||
json!({"role": "assistant", "content": "a", "reasoning_content": "let me think..."}),
|
||||
user("q2"),
|
||||
]);
|
||||
assert_eq!(
|
||||
chain(&plain),
|
||||
chain(&with_reasoning),
|
||||
"reasoning_content is not rendered -> must not change the chain"
|
||||
);
|
||||
assert!(
|
||||
!is_unclassifiable(&with_reasoning),
|
||||
"reasoning-carrying turns stay classifiable"
|
||||
);
|
||||
}
|
||||
|
||||
// ==================== T23: intra-message chunking ====================
|
||||
|
||||
/// The pre-T23 `chain_hashes`, verbatim copy. Short-message bodies must
|
||||
/// produce byte-identical chains under the chunked implementation.
|
||||
fn legacy_chain_hashes(seed: &ChainSeed, body: &Value) -> Vec<u64> {
|
||||
let (msgs, _) = walk_messages(body);
|
||||
let mut chain = Vec::with_capacity(msgs.len());
|
||||
let mut prev = seed.value;
|
||||
for msg in &msgs {
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&prev.to_le_bytes());
|
||||
|
||||
let role = msg.role.as_bytes();
|
||||
h.update(&(role.len() as u64).to_le_bytes());
|
||||
h.update(role);
|
||||
|
||||
match &msg.content {
|
||||
ContentView::Str(s) => {
|
||||
update_tagged(&mut h, TAG_TEXT, s.as_bytes());
|
||||
}
|
||||
ContentView::Parts(parts) => {
|
||||
for part in parts {
|
||||
match part {
|
||||
PartView::Text(t) => update_tagged(&mut h, TAG_TEXT, t.as_bytes()),
|
||||
PartView::ImageUrl(u) => {
|
||||
update_tagged(&mut h, TAG_IMAGE_URL, u.as_bytes())
|
||||
}
|
||||
PartView::Unknown => h.update(TAG_UNKNOWN),
|
||||
}
|
||||
}
|
||||
}
|
||||
ContentView::Empty => {}
|
||||
}
|
||||
|
||||
if let Some(tc) = &msg.tool_calls {
|
||||
update_tagged(&mut h, TAG_TOOL_CALLS, tc.as_bytes());
|
||||
}
|
||||
|
||||
update_tagged(&mut h, TAG_META, msg.extra.as_bytes());
|
||||
|
||||
prev = h.digest();
|
||||
chain.push(prev);
|
||||
}
|
||||
chain
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_messages_match_legacy_chain_byte_for_byte() {
|
||||
let exact_boundary = "x".repeat(CHUNK_BYTES);
|
||||
let bodies = vec![
|
||||
// pure-text multi-turn
|
||||
text_body(vec![user("q1"), assistant("a1"), user("q2")]),
|
||||
// multimodal parts (short text + image)
|
||||
text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "look"},
|
||||
{"type": "image_url", "image_url": {"url": "https://x/a.png"}}
|
||||
]
|
||||
})]),
|
||||
// tool_calls + content:null + tool result (meta on both)
|
||||
text_body(vec![
|
||||
user("weather?"),
|
||||
json!({"role": "assistant", "content": null,
|
||||
"tool_calls": tool_call("get_weather", "{\"city\":\"sh\"}")}),
|
||||
json!({"role": "tool", "tool_call_id": "call_1", "content": "sunny"}),
|
||||
]),
|
||||
// name meta
|
||||
text_body(vec![json!({"role": "user", "name": "alice", "content": "hi"})]),
|
||||
// null content without tool_calls
|
||||
text_body(vec![json!({"role": "assistant", "content": null})]),
|
||||
// exactly CHUNK_BYTES: never splits (only > CHUNK_BYTES does)
|
||||
text_body(vec![user(&exact_boundary)]),
|
||||
];
|
||||
for b in &bodies {
|
||||
let seed = ChainSeed::from_body(b);
|
||||
assert_eq!(
|
||||
chain_hashes(&seed, b),
|
||||
legacy_chain_hashes(&seed, b),
|
||||
"short-message chain must stay byte-identical to pre-T23"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_message_splits_into_chunk_rings() {
|
||||
// CHUNK_BYTES + 1 -> one chunk ring + the message-final ring.
|
||||
let text = "x".repeat(CHUNK_BYTES + 1);
|
||||
let body = text_body(vec![user(&text)]);
|
||||
let seed = ChainSeed::from_body(&body);
|
||||
let (c, idx) = chain_hashes_indexed(&seed, &body);
|
||||
assert_eq!(c.len(), 2);
|
||||
assert_eq!(idx, vec![0, 0], "both rings belong to message 0");
|
||||
// The chunk ring is bare: xxh3(prev_le ‖ chunk), no tag, no length.
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&seed.value.to_le_bytes());
|
||||
h.update("x".repeat(CHUNK_BYTES).as_bytes());
|
||||
assert_eq!(c[0], h.digest());
|
||||
|
||||
// Exactly CHUNK_BYTES -> single ring (only > CHUNK_BYTES splits);
|
||||
// already pinned by the legacy-equivalence battery above.
|
||||
assert_eq!(chain(&text_body(vec![user(&"y".repeat(CHUNK_BYTES))])).len(), 1);
|
||||
}
|
||||
|
||||
/// The T23 acceptance shape: two single-message bodies sharing a 4KB
|
||||
/// prefix but differing in the tail now share their chunk rings — the
|
||||
/// pre-T23 chain (one ring per message) could never hit here.
|
||||
#[test]
|
||||
fn shared_long_prefix_different_tail_shares_chunk_rings() {
|
||||
let prefix = "a".repeat(2 * CHUNK_BYTES);
|
||||
let a = text_body(vec![user(&format!("{prefix}{}", "b".repeat(100)))]);
|
||||
let b = text_body(vec![user(&format!("{prefix}{}", "c".repeat(200)))]);
|
||||
let (ca, ia) = chain_hashes_indexed(&ChainSeed::from_body(&a), &a);
|
||||
let (cb, ib) = chain_hashes_indexed(&ChainSeed::from_body(&b), &b);
|
||||
assert_eq!(ca.len(), 3, "2 chunk rings + message-final ring");
|
||||
assert_eq!(cb.len(), 3);
|
||||
assert_eq!(ia, vec![0, 0, 0]);
|
||||
assert_eq!(ib, vec![0, 0, 0]);
|
||||
assert_eq!(
|
||||
ca[..2],
|
||||
cb[..2],
|
||||
"identical 4KB prefix -> identical chunk rings"
|
||||
);
|
||||
assert_ne!(ca[2], cb[2], "different tail -> final ring diverges");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utf8_boundary_chunk_never_splits_a_character() {
|
||||
// 700 x 3 bytes = 2100 > 2048; the 2048 cut lands mid-汉
|
||||
// (2048 % 3 = 2), so the chunk must back off to 2046 bytes.
|
||||
let text = "汉".repeat(700);
|
||||
let (chunk, tail) = take_chunk(&text);
|
||||
assert_eq!(chunk.len(), 2046);
|
||||
assert_eq!(tail.len(), 54);
|
||||
assert_eq!(format!("{chunk}{tail}"), text, "chunk + tail reconstruct");
|
||||
|
||||
let body = text_body(vec![user(&text)]);
|
||||
let seed = ChainSeed::from_body(&body);
|
||||
let (c, idx) = chain_hashes_indexed(&seed, &body);
|
||||
assert_eq!(c.len(), 2);
|
||||
assert_eq!(idx, vec![0, 0]);
|
||||
// The chunk ring carries exactly the boundary-aligned bytes.
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&seed.value.to_le_bytes());
|
||||
h.update(chunk.as_bytes());
|
||||
assert_eq!(c[0], h.digest());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_part_long_text_chunks_per_part_in_order() {
|
||||
let t1 = "p".repeat(CHUNK_BYTES + 10); // 1 chunk + 10-byte tail
|
||||
let t2 = "q".repeat(CHUNK_BYTES + 20); // 1 chunk + 20-byte tail
|
||||
let body = text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": t1},
|
||||
{"type": "image_url", "image_url": {"url": "https://x/a.png"}},
|
||||
{"type": "text", "text": t2}
|
||||
]
|
||||
})]);
|
||||
let seed = ChainSeed::from_body(&body);
|
||||
let (c, idx) = chain_hashes_indexed(&seed, &body);
|
||||
assert_eq!(c.len(), 3, "t1 chunk + t2 chunk + message-final ring");
|
||||
assert_eq!(idx, vec![0, 0, 0]);
|
||||
// Chunk rings follow part order: t1's chunk first, then t2's.
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&seed.value.to_le_bytes());
|
||||
h.update("p".repeat(CHUNK_BYTES).as_bytes());
|
||||
assert_eq!(c[0], h.digest());
|
||||
let mut h2 = Xxh3::new();
|
||||
h2.update(&c[0].to_le_bytes());
|
||||
h2.update("q".repeat(CHUNK_BYTES).as_bytes());
|
||||
assert_eq!(c[1], h2.digest());
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -378,6 +378,92 @@ impl PDRouter {
|
||||
Ok((prefill_request, decode_request))
|
||||
}
|
||||
|
||||
/// G9i admission + shadow booking, ONE attempt (T15: the caller retries
|
||||
/// holdable rejections through the hold queue). None = G9i not
|
||||
/// configured ("untracked"). Err = rejected (the caller counts +
|
||||
/// answers 429); the rejections are structured so that a rejected
|
||||
/// attempt never entered the shadow bookkeeping (T11 拒绝路径零记账).
|
||||
/// T25-A: `prior_wait_secs` is the time the request already spent
|
||||
/// parked in the reservoir across re-runs — it counts into TTFT, so
|
||||
/// the re-judged est_TTFT carries it.
|
||||
async fn g9i_admission<T: Serialize + Clone>(
|
||||
&self,
|
||||
context: &PDRequestContext<'_>,
|
||||
original_request: &T,
|
||||
prior_wait_secs: f64,
|
||||
) -> Option<Result<(Option<String>, &'static str, Option<ShadowRequestContext>), super::g9i::RejectInfo>>
|
||||
{
|
||||
let g9i = self.g9i.as_ref()?;
|
||||
if let Err(info) = g9i.check_decode_pool(&self.worker_registry).await {
|
||||
return Some(Err(info));
|
||||
}
|
||||
if context.route != "/v1/chat/completions" {
|
||||
// Non-chat routes skip classification but still land on
|
||||
// the regular pool — the regular queue gate applies.
|
||||
if let Err(info) = g9i.check_regular_queue(&self.worker_registry) {
|
||||
return Some(Err(info));
|
||||
}
|
||||
return Some(Ok((None, "non_chat", None)));
|
||||
}
|
||||
let body = match serde_json::to_value(original_request) {
|
||||
Ok(body) => body,
|
||||
Err(_) => return Some(Ok((None, "regular", None))),
|
||||
};
|
||||
// Shadow T2: compute the prefix chain once, here, alongside the v1
|
||||
// classification. Skipped entirely when shadow mode is off.
|
||||
let shadow_ctx = if g9i.shadow_enabled() {
|
||||
ShadowRequestContext::from_body(&body, context.is_stream)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let (plan, path, probe_new_work) = g9i
|
||||
.classify_prefill(&self.worker_registry, &self.client, &body)
|
||||
.await;
|
||||
// T11: v1 rejection FIRST — a request rejected by any gate must
|
||||
// never enter the shadow bookkeeping (拒绝路径零记账).
|
||||
let v1_override = match plan.into_override() {
|
||||
Ok(o) => o,
|
||||
Err(info) => return Some(Err(info)),
|
||||
};
|
||||
let shadow_ctx = match shadow_ctx {
|
||||
Some(mut ctx) => {
|
||||
// T11: classify + candidate pick + cap verdict + admission
|
||||
// in ONE queue-model critical section (atomic
|
||||
// check-and-act) — the check→act window no longer spans
|
||||
// the TTFT wait. Err = p_queue_cap (enforce), nothing
|
||||
// admitted.
|
||||
match g9i.shadow_classify_and_admit(&mut ctx) {
|
||||
Ok(Some(mut decision)) => {
|
||||
// T25-A: time already parked in the reservoir counts
|
||||
// into TTFT — the re-judged estimate carries it.
|
||||
if prior_wait_secs > 0.0 {
|
||||
decision.est_ttft_secs += prior_wait_secs;
|
||||
ctx.decision = Some(decision.clone());
|
||||
}
|
||||
super::g9i_shadow::record_agreement(
|
||||
path,
|
||||
probe_new_work,
|
||||
&ctx,
|
||||
&decision,
|
||||
);
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(info) => return Some(Err(info)),
|
||||
}
|
||||
Some(ctx)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
// T7 shadow-consistent routing (--g9i-shadow-route): the live
|
||||
// prefill override follows the shadow decision's matched_node (=
|
||||
// the booked node), so est vs actual is pure estimation error.
|
||||
let override_url = g9i.shadow_route_pick(
|
||||
v1_override,
|
||||
shadow_ctx.as_ref().and_then(|c| c.decision.as_ref()),
|
||||
);
|
||||
Some(Ok((override_url, path, shadow_ctx)))
|
||||
}
|
||||
|
||||
async fn execute_dual_dispatch<T: Serialize + Clone>(
|
||||
&self,
|
||||
headers: Option<&HeaderMap>,
|
||||
@@ -410,82 +496,130 @@ impl PDRouter {
|
||||
// shadow bookkeeping carrier (T2): Some only for chat requests with
|
||||
// a classifiable prefix chain while shadow mode is on; it never
|
||||
// influences routing.
|
||||
//
|
||||
// T15: holdable rejections (d_window_full / p_queue_cap) park the
|
||||
// request in the FIFO hold queue instead of an immediate 429
|
||||
// (429 太伤吞吐 — 测试端把 429 当失败). On pass the COMPLETE
|
||||
// admission re-runs (every gate re-checked; the T11 booking happens
|
||||
// only on a passing attempt — a parked request occupies no ledger).
|
||||
// T20 (B): prompt_too_long hard validation — the FRONT-MOST check,
|
||||
// before any admission gate / est_TTFT / ledger / queue bookkeeping
|
||||
// and before the hold loop (an invalid request must never park).
|
||||
// 400 (client bug), counted in smg_g9i_invalid_request_total, never
|
||||
// in smg_g9i_rejects_total. Chat-completions only.
|
||||
if context.route == "/v1/chat/completions" {
|
||||
if let Some(g9i) = &self.g9i {
|
||||
if let Ok(body) = serde_json::to_value(original_request) {
|
||||
if let Some(resp) = g9i.prompt_too_long_rejection(&body) {
|
||||
return resp;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut hold_deadline: Option<Instant> = None;
|
||||
// T25-A reservoir park state: (deadline, seq id) across re-runs +
|
||||
// the accumulated wait (counts into the est_TTFT re-judgment).
|
||||
let mut reservoir_state: Option<(Instant, u64)> = None;
|
||||
let mut reservoir_waited_secs: f64 = 0.0;
|
||||
let (g9i_prefill_override, g9i_ttft_path, shadow_ctx): (
|
||||
Option<String>,
|
||||
&'static str,
|
||||
Option<ShadowRequestContext>,
|
||||
) = if let Some(g9i) = &self.g9i {
|
||||
if let Err(info) = g9i.check_decode_pool(&self.worker_registry).await {
|
||||
metrics::counter!("smg_g9i_rejects_total", "reason" => info.reason)
|
||||
) = loop {
|
||||
match self
|
||||
.g9i_admission(&context, original_request, reservoir_waited_secs)
|
||||
.await
|
||||
{
|
||||
None => {
|
||||
// T25: per-request admission outcome (untracked = no
|
||||
// G9i configured — class "none").
|
||||
metrics::counter!(
|
||||
super::g9i::METRIC_ADMISSION_TOTAL,
|
||||
"class" => "none",
|
||||
"result" => "admitted",
|
||||
"reason" => "none",
|
||||
)
|
||||
.increment(1);
|
||||
return super::g9i::reject_response(&info);
|
||||
}
|
||||
if context.route == "/v1/chat/completions" {
|
||||
match serde_json::to_value(original_request) {
|
||||
Ok(body) => {
|
||||
// Shadow T2: compute the prefix chain once, here,
|
||||
// alongside the v1 classification. Skipped entirely
|
||||
// when shadow mode is off.
|
||||
let shadow_ctx = if g9i.shadow_enabled() {
|
||||
ShadowRequestContext::from_body(&body, context.is_stream)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
match g9i
|
||||
.classify_prefill(&self.worker_registry, &self.client, &body)
|
||||
.await
|
||||
{
|
||||
(plan, path, probe_new_work) => {
|
||||
// Shadow T3: shadow-classify alongside the v1
|
||||
// verdict and record agreement/divergence
|
||||
// counters. Routing still follows the v1 plan
|
||||
// alone; the decision only rides the context
|
||||
// into dispatch for queue bookkeeping.
|
||||
let shadow_ctx = match shadow_ctx {
|
||||
Some(mut ctx) => {
|
||||
if let Some(decision) = g9i.shadow_classify(&ctx) {
|
||||
super::g9i_shadow::record_agreement(
|
||||
path,
|
||||
probe_new_work,
|
||||
&ctx,
|
||||
&decision,
|
||||
);
|
||||
ctx.decision = Some(decision);
|
||||
}
|
||||
Some(ctx)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
match plan.into_override() {
|
||||
Ok(override_url) => (override_url, path, shadow_ctx),
|
||||
Err(info) => {
|
||||
metrics::counter!(
|
||||
"smg_g9i_rejects_total",
|
||||
"reason" => info.reason
|
||||
)
|
||||
.increment(1);
|
||||
return super::g9i::reject_response(&info);
|
||||
}
|
||||
break (None, "untracked", None);
|
||||
}
|
||||
Some(Ok(outcome)) => {
|
||||
// T25: per-request admission outcome — the shadow
|
||||
// decision's class (the experiment's primary counter).
|
||||
let class = outcome
|
||||
.2
|
||||
.as_ref()
|
||||
.and_then(|c| c.decision.as_ref())
|
||||
.map(|d| d.class.as_str())
|
||||
.unwrap_or("none");
|
||||
metrics::counter!(
|
||||
super::g9i::METRIC_ADMISSION_TOTAL,
|
||||
"class" => class,
|
||||
"result" => "admitted",
|
||||
"reason" => "none",
|
||||
)
|
||||
.increment(1);
|
||||
break outcome;
|
||||
}
|
||||
Some(Err(mut info)) => {
|
||||
let mut retry = false;
|
||||
if let Some(g9i) = &self.g9i {
|
||||
// T25-A: under hot_policy=reservoir, HOT-class
|
||||
// holdable rejections park in the priority
|
||||
// reservoir INSTEAD of the T15 hold (the A/B stays
|
||||
// clean — every other class keeps the hold path).
|
||||
if g9i.reservoir_eligible(&info) {
|
||||
match g9i
|
||||
.reservoir_or_reject(&info, &mut reservoir_state)
|
||||
.await
|
||||
{
|
||||
Ok(waited) => {
|
||||
reservoir_waited_secs += waited;
|
||||
retry = true;
|
||||
}
|
||||
Err(final_info) => info = final_info,
|
||||
}
|
||||
} else if info.holdable() {
|
||||
let body_bytes = context
|
||||
.request_text
|
||||
.as_deref()
|
||||
.map(|s| s.len() as u64)
|
||||
.unwrap_or(0);
|
||||
match g9i
|
||||
.hold_or_reject(&info, body_bytes, &mut hold_deadline)
|
||||
.await
|
||||
{
|
||||
Ok(()) => retry = true,
|
||||
Err(final_info) => info = final_info,
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(_) => (None, "regular", None),
|
||||
}
|
||||
} else {
|
||||
// Non-chat routes skip classification but still land on
|
||||
// the regular pool — the regular queue gate applies.
|
||||
if let Err(info) = g9i.check_regular_queue(&self.worker_registry) {
|
||||
if retry {
|
||||
continue;
|
||||
}
|
||||
// T25: per-request rejection outcome (class from the
|
||||
// rejecting gate; "none" on non-chat/early gates).
|
||||
metrics::counter!(
|
||||
super::g9i::METRIC_ADMISSION_TOTAL,
|
||||
"class" => info.hold_class.unwrap_or("none"),
|
||||
"result" => "rejected",
|
||||
"reason" => info.reason,
|
||||
)
|
||||
.increment(1);
|
||||
metrics::counter!("smg_g9i_rejects_total", "reason" => info.reason)
|
||||
.increment(1);
|
||||
return super::g9i::reject_response(&info);
|
||||
}
|
||||
(None, "non_chat", None)
|
||||
}
|
||||
} else {
|
||||
(None, "untracked", None)
|
||||
};
|
||||
|
||||
// T11: handle of the decision-time booking, extracted before the
|
||||
// retry closure moves `shadow_ctx`. If the dispatch never reached
|
||||
// the response path (all attempts failed before headers), the
|
||||
// booking is released after the loop — exactly once, paired with
|
||||
// shadow_admit's consume mark.
|
||||
let admission_mark = shadow_ctx.as_ref().and_then(|c| c.admission.clone());
|
||||
|
||||
let response = RetryExecutor::execute_response_with_retry(
|
||||
&self.retry_config,
|
||||
{
|
||||
@@ -665,6 +799,13 @@ impl PDRouter {
|
||||
);
|
||||
}
|
||||
|
||||
// T11: release a decision-time booking that no response path ever
|
||||
// consumed (all attempts failed before headers). No-op when the
|
||||
// stream/non-stream path already took it over.
|
||||
if let Some(g9i) = &self.g9i {
|
||||
g9i.shadow_release_unconsumed(admission_mark);
|
||||
}
|
||||
|
||||
response
|
||||
}
|
||||
|
||||
@@ -1024,6 +1165,7 @@ impl PDRouter {
|
||||
ShadowResponseContext::from_request(
|
||||
ctx,
|
||||
super::g9i::normalize_url(prefill.url()),
|
||||
super::g9i::normalize_url(decode.url()),
|
||||
)
|
||||
});
|
||||
// Shadow T3: admit into the queue model keyed by the
|
||||
@@ -1051,6 +1193,7 @@ impl PDRouter {
|
||||
ShadowResponseContext::from_request(
|
||||
ctx,
|
||||
super::g9i::normalize_url(prefill.url()),
|
||||
super::g9i::normalize_url(decode.url()),
|
||||
)
|
||||
});
|
||||
// Shadow T3: admit into the queue model keyed by the
|
||||
@@ -1061,14 +1204,21 @@ impl PDRouter {
|
||||
g9i.shadow_admit(s);
|
||||
}
|
||||
if context.return_logprob {
|
||||
self.process_non_streaming_response(
|
||||
res,
|
||||
status,
|
||||
context.return_logprob,
|
||||
prefill_body,
|
||||
shadow,
|
||||
)
|
||||
.await
|
||||
let resp = self
|
||||
.process_non_streaming_response(
|
||||
res,
|
||||
status,
|
||||
context.return_logprob,
|
||||
prefill_body,
|
||||
shadow,
|
||||
)
|
||||
.await;
|
||||
// T17: non-streaming e2e (observability only — the
|
||||
// AIMD blind state reads local signals instead).
|
||||
if let Some(g9i) = &self.g9i {
|
||||
g9i.record_nonstream_e2e(start_time.elapsed().as_secs_f64());
|
||||
}
|
||||
resp
|
||||
} else {
|
||||
// Direct passthrough when no logprobs needed
|
||||
let response_headers =
|
||||
@@ -1078,9 +1228,15 @@ impl PDRouter {
|
||||
Ok(decode_body) => {
|
||||
let decode_body =
|
||||
self.shadow_record_non_stream(decode_body, status, &shadow);
|
||||
let mut response = Response::new(Body::from(decode_body));
|
||||
*response.status_mut() = status;
|
||||
*response.headers_mut() = response_headers;
|
||||
let response = Self::non_stream_passthrough_response(
|
||||
status,
|
||||
decode_body,
|
||||
response_headers,
|
||||
);
|
||||
// T17: non-streaming e2e (observability only).
|
||||
if let Some(g9i) = &self.g9i {
|
||||
g9i.record_nonstream_e2e(start_time.elapsed().as_secs_f64());
|
||||
}
|
||||
response
|
||||
}
|
||||
Err(e) => {
|
||||
@@ -1386,7 +1542,11 @@ impl PDRouter {
|
||||
// (malformed) repeat usage event from
|
||||
// double-completing the queue slot.
|
||||
if !shadow_done {
|
||||
g9i.shadow_record(sctx, &json, shadow_ttft_secs);
|
||||
// Streaming harvests the usage
|
||||
// event pre-strip, so the
|
||||
// breakdown extraction inside
|
||||
// works unaided (None).
|
||||
g9i.shadow_record(sctx, &json, shadow_ttft_secs, None);
|
||||
shadow_done = true;
|
||||
}
|
||||
}
|
||||
@@ -1407,11 +1567,34 @@ impl PDRouter {
|
||||
g9i_path,
|
||||
start_time.elapsed().as_secs_f64(),
|
||||
);
|
||||
// T25: TTFT by shadow class — the
|
||||
// same gateway-observed first-token
|
||||
// moment (start_time predates the
|
||||
// admission loop, so reservoir /
|
||||
// hold waits are included).
|
||||
if let Some((_, sctx)) = &g9i_shadow {
|
||||
if let Some(class) =
|
||||
sctx.class.map(|c| c.as_str())
|
||||
{
|
||||
metrics::histogram!(
|
||||
super::g9i::METRIC_CLASS_TTFT,
|
||||
"class" => class,
|
||||
)
|
||||
.record(start_time.elapsed().as_secs_f64());
|
||||
}
|
||||
}
|
||||
}
|
||||
// Shadow T3: the same first-token moment
|
||||
// calibrates est_TTFT (est − actual).
|
||||
shadow_ttft_secs =
|
||||
ttft_start.map(|t| t.elapsed().as_secs_f64());
|
||||
// T16: first token -> the D booking moves
|
||||
// from the pipeline to the streaming
|
||||
// sub-ledger (the window gates pipeline
|
||||
// depth only).
|
||||
if let Some((g9i, sctx)) = &g9i_shadow {
|
||||
g9i.shadow_promote_streaming(sctx);
|
||||
}
|
||||
}
|
||||
|
||||
let result = if return_logprob && prefill_logprobs.is_some() {
|
||||
@@ -1572,12 +1755,15 @@ impl PDRouter {
|
||||
g9i.shadow_release(ctx);
|
||||
return body;
|
||||
}
|
||||
let (parsed, rewritten) =
|
||||
let (parsed, rewritten, breakdown) =
|
||||
super::g9i_shadow::strip_and_extract(&body, ctx.strip_cached_details);
|
||||
match &parsed {
|
||||
// TTFT calibration is streaming-only (non-streaming has no
|
||||
// observable first-token moment) -> None.
|
||||
Some(json) => g9i.shadow_record(ctx, json, None),
|
||||
// observable first-token moment) -> None. The breakdown was
|
||||
// extracted pre-strip and is passed through — the parsed JSON
|
||||
// here is post-strip, so re-extracting from it would always
|
||||
// miss (the 2026-10-17 order bug).
|
||||
Some(json) => g9i.shadow_record(ctx, json, None, breakdown),
|
||||
// Unparseable success body: nothing to record, but the dispatch
|
||||
// is over — release rather than leak the outstanding count.
|
||||
None => g9i.shadow_release(ctx),
|
||||
@@ -1588,6 +1774,27 @@ impl PDRouter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Assemble the non-streaming passthrough response. The shadow strip
|
||||
/// may have REWRITTEN (shortened) the body, so the upstream
|
||||
/// Content-Length in `headers` is stale; forwarding it verbatim
|
||||
/// truncated cache-hit responses at exactly the stripped length (the
|
||||
/// 2026-10-17 IncompleteRead incident). Pin Content-Length to the
|
||||
/// actual body length — correct for rewritten and untouched bodies
|
||||
/// alike.
|
||||
fn non_stream_passthrough_response(
|
||||
status: StatusCode,
|
||||
body: bytes::Bytes,
|
||||
mut headers: HeaderMap,
|
||||
) -> Response {
|
||||
if let Ok(v) = HeaderValue::from_str(body.len().to_string().as_str()) {
|
||||
headers.insert(CONTENT_LENGTH, v);
|
||||
}
|
||||
let mut response = Response::new(Body::from(body));
|
||||
*response.status_mut() = status;
|
||||
*response.headers_mut() = headers;
|
||||
response
|
||||
}
|
||||
|
||||
// Helper to process prefill response and extract body if needed for logprobs
|
||||
async fn process_prefill_response(
|
||||
&self,
|
||||
@@ -2115,6 +2322,39 @@ mod tests {
|
||||
Box::new(worker)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_stream_passthrough_pins_content_length_to_body() {
|
||||
// 2026-10-17 IncompleteRead regression: the shadow strip shortens
|
||||
// the body on engine cache hits; the upstream Content-Length must
|
||||
// never survive onto the rewritten body.
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_LENGTH, HeaderValue::from_static("637"));
|
||||
let body = bytes::Bytes::from_static(b"{}");
|
||||
let resp =
|
||||
PDRouter::non_stream_passthrough_response(StatusCode::OK, body.clone(), headers);
|
||||
assert_eq!(
|
||||
resp.headers().get(CONTENT_LENGTH).unwrap(),
|
||||
&HeaderValue::from_static("2"),
|
||||
"stale upstream length is replaced by the actual body length"
|
||||
);
|
||||
|
||||
// Untouched body: the pinned length matches the original.
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_LENGTH, HeaderValue::from_static("2"));
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
let resp = PDRouter::non_stream_passthrough_response(StatusCode::OK, body, headers);
|
||||
assert_eq!(
|
||||
resp.headers().get(CONTENT_LENGTH).unwrap(),
|
||||
&HeaderValue::from_static("2")
|
||||
);
|
||||
assert_eq!(
|
||||
resp.headers().get(CONTENT_TYPE).unwrap(),
|
||||
&HeaderValue::from_static("application/json"),
|
||||
"other preserved headers survive"
|
||||
);
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chat_request_text_uses_full_conversation() {
|
||||
// Regression test for https://github.com/sgl-project/sglang/issues/26263
|
||||
|
||||
Reference in New Issue
Block a user