gateway g9i: D-side in-flight ledger + Little's-law slot release rate (P2 T13)
This commit is contained in:
@@ -334,6 +334,9 @@ pub struct NodeLoad {
|
||||
/// summed (T12: the slot dimension of T_D — c512 实测 d1 槽位 128/128
|
||||
/// 打满而内存仅 36%,内存口径完全看不见).
|
||||
pub free_slots: u64,
|
||||
/// Slot capacity: sum of max_running_requests across dp ranks (T13
|
||||
/// ledger口径 needs capacity, not just headroom).
|
||||
pub max_running_reqs: u64,
|
||||
/// True when any rank answered with a disaggregation section. False on
|
||||
/// old engines (no disagg include support / non-PD deployment) — the
|
||||
/// est_T_D path then degrades to the P-only model exactly (C13).
|
||||
@@ -604,6 +607,14 @@ impl G9iState {
|
||||
"worker" => url.clone(),
|
||||
)
|
||||
.set(q.slot_release_rate(url));
|
||||
// T13: the gateway-side in-flight ledger (vs the
|
||||
// snapshot's running+prealloc — divergence shows
|
||||
// poll-lag / bookkeeping drift).
|
||||
metrics::gauge!(
|
||||
"smg_g9i_d_inflight_ledger",
|
||||
"worker" => url.clone(),
|
||||
)
|
||||
.set(q.d_inflight(url) as f64);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -846,7 +857,7 @@ impl G9iState {
|
||||
let lock = self.shadow_queue.as_ref()?;
|
||||
let mut q = lock.write().ok()?;
|
||||
q.est_td_min(&d_snaps, own_ctx_tokens, self.config.est_td_enabled)
|
||||
.map(|(t_d, _slot)| t_d)
|
||||
.map(|(t_d, _slot, _node)| t_d)
|
||||
}
|
||||
|
||||
/// P2 T_D hard gate (design §2.4-2, user-ruled): when min(T_D) over ALL
|
||||
@@ -1071,20 +1082,36 @@ impl G9iState {
|
||||
let actual = super::g9i_shadow::extract_prompt_tokens(resp_json)
|
||||
.unwrap_or(ctx.est_new_tokens);
|
||||
q.complete(&ctx.node, class, ctx.est_new_tokens, actual);
|
||||
// T13: pair the D-side ledger booking (admit→complete).
|
||||
// Booked iff the decision-time admission carried a
|
||||
// d_node, or the deferred admit booked at response time.
|
||||
let d_booked = match &ctx.admission {
|
||||
Some(adm) => adm.d_node.is_some(),
|
||||
None => true,
|
||||
};
|
||||
if d_booked {
|
||||
q.d_ledger_release(&ctx.decode_node);
|
||||
}
|
||||
}
|
||||
// est_T_D P1 (design 2026-10-14 §2.2): a successful response
|
||||
// frees its ctx tokens of KV pool on the decode node — fold
|
||||
// the freed amount into that node's pool_free_rate EWMA.
|
||||
// Only responses with real usage count: the release path
|
||||
// (errors/aborts) freed nothing we can measure.
|
||||
// T13: dwell = admit→complete wall time (Little's-law
|
||||
// residency sample behind the slot release rate).
|
||||
if let Some(ctx_tokens) = super::g9i_shadow::extract_prompt_tokens(resp_json) {
|
||||
q.record_decode_completion(&ctx.decode_node, ctx_tokens);
|
||||
let dwell = ctx
|
||||
.admission
|
||||
.as_ref()
|
||||
.map(|a| a.created.elapsed().as_secs_f64());
|
||||
q.record_decode_completion(&ctx.decode_node, ctx_tokens, dwell);
|
||||
metrics::gauge!(
|
||||
super::g9i_shadow::METRIC_POOL_FREE_RATE,
|
||||
"worker" => ctx.decode_node.clone(),
|
||||
)
|
||||
.set(q.pool_free_rate(&ctx.decode_node));
|
||||
// T12: the same completion is one slot release.
|
||||
// T13: Little's-law slot release rate beside it.
|
||||
metrics::gauge!(
|
||||
super::g9i_shadow::METRIC_SLOT_RELEASE_RATE,
|
||||
"worker" => ctx.decode_node.clone(),
|
||||
@@ -1111,11 +1138,19 @@ impl G9iState {
|
||||
/// Undo a queue admission whose response will never carry usage
|
||||
/// (non-200, aborted stream, unparseable body). Keeps outstanding
|
||||
/// conservative without polluting the drain EWMA with failure samples.
|
||||
/// T13: pairs the D-side ledger booking too (same bookkeeping scope).
|
||||
pub fn shadow_release(&self, ctx: &ShadowResponseContext) {
|
||||
if let Some(class) = ctx.class {
|
||||
if let Some(lock) = &self.shadow_queue {
|
||||
if let Ok(mut q) = lock.write() {
|
||||
q.release(&ctx.node, class, ctx.est_new_tokens);
|
||||
let d_booked = match &ctx.admission {
|
||||
Some(adm) => adm.d_node.is_some(),
|
||||
None => true, // deferred admit booked the ledger
|
||||
};
|
||||
if d_booked {
|
||||
q.d_ledger_release(&ctx.decode_node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1147,9 +1182,17 @@ impl G9iState {
|
||||
return;
|
||||
}
|
||||
q.rekey(&adm.node, &ctx.node, class, adm.est_tokens);
|
||||
// T13: same for the D-side ledger (decision-time pick vs
|
||||
// actual decode landing).
|
||||
if let Some(dn) = &adm.d_node {
|
||||
q.d_ledger_rekey(dn, &ctx.decode_node);
|
||||
}
|
||||
}
|
||||
// Deferred admission (decision had no node to book).
|
||||
None => q.admit(&ctx.node, class, ctx.est_new_tokens),
|
||||
None => {
|
||||
q.admit(&ctx.node, class, ctx.est_new_tokens);
|
||||
q.d_ledger_admit(&ctx.decode_node);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1171,6 +1214,11 @@ impl G9iState {
|
||||
if let Some(lock) = &self.shadow_queue {
|
||||
if let Ok(mut q) = lock.write() {
|
||||
q.release(&adm.node, adm.class, adm.est_tokens);
|
||||
// T13: pair the D-side ledger booking (never dispatched —
|
||||
// the booking was intent-only, released here).
|
||||
if let Some(dn) = &adm.d_node {
|
||||
q.d_ledger_release(dn);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1189,7 +1237,8 @@ impl G9iState {
|
||||
awaiting_tokens: load.queued_total_input_tokens,
|
||||
free_tokens: load.free_tokens,
|
||||
prealloc_queue_reqs: load.decode_prealloc_queue_reqs,
|
||||
free_slots: load.free_slots,
|
||||
running_reqs: load.running_reqs,
|
||||
max_running_total: load.max_running_reqs,
|
||||
usable: load.has_disagg && load.is_fresh(ttl),
|
||||
})
|
||||
.collect()
|
||||
@@ -1275,10 +1324,21 @@ impl G9iState {
|
||||
out.est.total,
|
||||
)?;
|
||||
q.admit(&out.node, decision.class, decision.est_new_tokens);
|
||||
// T13: book the D-side in-flight ledger on the
|
||||
// min-T_D node — same write lock, so the slot
|
||||
// gate's check→act is atomic too (快照竞态消灭).
|
||||
// The booking reads the ledger BEFORE this request.
|
||||
let d_node = q
|
||||
.est_td_min(&d_snaps, own_ctx, self.config.est_td_enabled)
|
||||
.map(|(_, _, n)| n);
|
||||
if let Some(dn) = &d_node {
|
||||
q.d_ledger_admit(dn);
|
||||
}
|
||||
ctx.admission = Some(Arc::new(super::g9i_shadow::AdmissionMark::new(
|
||||
out.node,
|
||||
decision.class,
|
||||
decision.est_new_tokens,
|
||||
d_node,
|
||||
)));
|
||||
out.est
|
||||
}
|
||||
@@ -1822,6 +1882,7 @@ fn parse_node_load(json: &Value) -> Result<NodeLoad, String> {
|
||||
.get("max_running_requests")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(0);
|
||||
node.max_running_reqs += max_running;
|
||||
node.free_slots += max_running.saturating_sub(running);
|
||||
node.queued_total_input_tokens += rank
|
||||
.get("queued_total_input_tokens")
|
||||
@@ -2022,6 +2083,7 @@ mod tests {
|
||||
assert_eq!(node.free_tokens, 2_000_000);
|
||||
// (128-3) + (128-5) = 248 free slots (per-rank, then summed).
|
||||
assert_eq!(node.free_slots, 248);
|
||||
assert_eq!(node.max_running_reqs, 256);
|
||||
assert!(node.has_disagg);
|
||||
}
|
||||
|
||||
@@ -2199,8 +2261,8 @@ sglang:token_usage{dp_rank=\"0\"} 0.5
|
||||
}
|
||||
|
||||
/// Install decode snapshots: (url, awaiting_tokens, free_tokens,
|
||||
/// has_disagg, stale). free_slots pinned at 128 with an empty prealloc
|
||||
/// queue so the T12 slot dimension stays inert in the pre-T12 tests.
|
||||
/// has_disagg, stale). Slot view pinned at empty+128 capacity so the
|
||||
/// slot dimension stays inert in the pre-T13 tests.
|
||||
fn set_d_snapshots(st: &G9iState, entries: &[(&str, u64, u64, bool, bool)]) {
|
||||
let mapped: Vec<(String, NodeLoad)> = entries
|
||||
.iter()
|
||||
@@ -2210,7 +2272,8 @@ sglang:token_usage{dp_rank=\"0\"} 0.5
|
||||
NodeLoad {
|
||||
queued_total_input_tokens: *awaiting,
|
||||
free_tokens: *free,
|
||||
free_slots: 128,
|
||||
running_reqs: 0,
|
||||
max_running_reqs: 128,
|
||||
has_disagg: *has_disagg,
|
||||
fetch_ok: true,
|
||||
fetched_at: Some(if *stale {
|
||||
@@ -2320,18 +2383,21 @@ sglang:token_usage{dp_rank=\"0\"} 0.5
|
||||
assert!((err.estimated_wait_secs.unwrap() - 1000.0).abs() < 1e-6);
|
||||
}
|
||||
|
||||
/// T12: the slot dimension drives the admission gate even with memory
|
||||
/// wide open — the c512 blind spot (slots 128/128, memory 36%), covered.
|
||||
/// T12/T13: the slot dimension drives the admission gate even with
|
||||
/// memory wide open — the c512 blind spot (slots 128/128 + 224 queued,
|
||||
/// memory 36%), covered.
|
||||
#[test]
|
||||
fn td_gate_slot_bound_rejects_in_enforce() {
|
||||
let st = p2_state("enforce");
|
||||
// Memory idle (10M free tokens), slots full: 224 queued, 0 free.
|
||||
// Cold-start slot rate floor 0.01 req/s -> (224+1)/0.01 = 22500s.
|
||||
// Memory idle (10M free tokens), slots full: running 128/128,
|
||||
// 224 queued. Cold-start slot rate floor 0.01 req/s ->
|
||||
// (352+1-128)/0.01 = 22500s.
|
||||
let load = NodeLoad {
|
||||
queued_total_input_tokens: 0,
|
||||
free_tokens: 10_000_000,
|
||||
decode_prealloc_queue_reqs: 224,
|
||||
free_slots: 0,
|
||||
running_reqs: 128,
|
||||
max_running_reqs: 128,
|
||||
has_disagg: true,
|
||||
fetch_ok: true,
|
||||
fetched_at: Some(Instant::now()),
|
||||
@@ -2678,4 +2744,66 @@ sglang:token_usage{dp_rank=\"0\"} 0.5
|
||||
assert!(!d.all_over_cap, "cap inert when td_admit=off (C13)");
|
||||
assert!(ctx.admission.is_some(), "bookkeeping still happens");
|
||||
}
|
||||
|
||||
// ============ T13: D-side in-flight ledger wiring ============
|
||||
|
||||
fn d_inflight_of(st: &G9iState, node: &str) -> u64 {
|
||||
st.shadow_queue
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.read()
|
||||
.unwrap()
|
||||
.d_inflight(node)
|
||||
}
|
||||
|
||||
/// The D ledger is booked at DECISION time (same critical section as
|
||||
/// the gates — the snapshot race is closed), released by the unconsumed
|
||||
/// fallback, and rekeyed to the actual decode node on consume.
|
||||
#[test]
|
||||
fn t13_d_ledger_booked_at_decision_and_paired() {
|
||||
let st = t11_state("enforce", 4_000_000);
|
||||
*st.p_nodes.write().unwrap() = vec!["http://p1:30000".to_string()];
|
||||
set_d_snapshots(&st, &[("http://d1:30001", 0, 10_000_000, true, false)]);
|
||||
let seed = chat_ctx_turn1();
|
||||
seed_table(&st, &seed.chain, "http://p1:30000", 1000);
|
||||
let mut ctx = chat_ctx();
|
||||
st.shadow_classify_and_admit(&mut ctx).unwrap().unwrap();
|
||||
assert_eq!(
|
||||
d_inflight_of(&st, "http://d1:30001"),
|
||||
1,
|
||||
"ledger booked at decision time, before dispatch"
|
||||
);
|
||||
// Never dispatched -> fallback release pairs it (P and D both).
|
||||
st.shadow_release_unconsumed(ctx.admission.clone());
|
||||
assert_eq!(d_inflight_of(&st, "http://d1:30001"), 0);
|
||||
assert_eq!(outstanding_of(&st, "http://p1:30000"), 0);
|
||||
// Exactly-once.
|
||||
st.shadow_release_unconsumed(ctx.admission.clone());
|
||||
assert_eq!(d_inflight_of(&st, "http://d1:30001"), 0);
|
||||
}
|
||||
|
||||
/// Consume rekeys the D booking when the actual decode node differs.
|
||||
#[test]
|
||||
fn t13_d_ledger_rekey_on_consume() {
|
||||
let st = t11_state("enforce", 4_000_000);
|
||||
*st.p_nodes.write().unwrap() = vec!["http://p1:30000".to_string()];
|
||||
set_d_snapshots(&st, &[("http://d1:30001", 0, 10_000_000, true, false)]);
|
||||
let seed = chat_ctx_turn1();
|
||||
seed_table(&st, &seed.chain, "http://p1:30000", 1000);
|
||||
let mut ctx = chat_ctx();
|
||||
st.shadow_classify_and_admit(&mut ctx).unwrap().unwrap();
|
||||
assert_eq!(d_inflight_of(&st, "http://d1:30001"), 1);
|
||||
// Actual decode landing differs (multi-D future shape).
|
||||
let resp_ctx = ShadowResponseContext::from_request(
|
||||
ctx,
|
||||
"http://p1:30000".to_string(),
|
||||
"http://d2:30002".to_string(),
|
||||
);
|
||||
st.shadow_admit(&resp_ctx);
|
||||
assert_eq!(d_inflight_of(&st, "http://d1:30001"), 0, "rekeyed off d1");
|
||||
assert_eq!(d_inflight_of(&st, "http://d2:30002"), 1, "rekeyed onto d2");
|
||||
// Abort-path release pairs the rekeyed booking.
|
||||
st.shadow_release(&resp_ctx);
|
||||
assert_eq!(d_inflight_of(&st, "http://d2:30002"), 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -508,11 +508,15 @@ pub const POOL_FREE_RATE_MIN: f64 = 1_000.0;
|
||||
pub const POOL_FREE_RATE_MAX: f64 = 2_000_000.0;
|
||||
/// Fixed EWMA weight per completion event (design §2.2).
|
||||
const POOL_FREE_RATE_ALPHA: f64 = 0.3;
|
||||
/// T12 slot_release_rate EWMA hard bounds in req/s: [0.01, 50] — one
|
||||
/// release per 100s at the floor (unknown = conservative), 50 req/s cap
|
||||
/// against burst skew.
|
||||
/// T13 slot_release_rate bounds in req/s: [0.01, 50]. The rate itself is
|
||||
/// Little's law (ledger / dwell_ewma); dwell samples are clamped to
|
||||
/// [0.5, 500]s before the EWMA. Cold start (no dwell sample yet) reads the
|
||||
/// slow floor — unknown is conservative.
|
||||
pub const SLOT_RELEASE_RATE_MIN: f64 = 0.01;
|
||||
pub const SLOT_RELEASE_RATE_MAX: f64 = 50.0;
|
||||
/// Decode residency (dwell) sample clamp, seconds.
|
||||
const DWELL_MIN_SECS: f64 = 0.5;
|
||||
const DWELL_MAX_SECS: f64 = 500.0;
|
||||
|
||||
/// est_T_D shadow metrics (design §2.5). Emitted from G9iState; the names
|
||||
/// live here next to the model that produces the values.
|
||||
@@ -539,18 +543,32 @@ pub const METRIC_BOOTSTRAP_ACTUAL: &str = "smg_g9i_bootstrap_actual_secs";
|
||||
struct DPoolEntry {
|
||||
/// Completion-driven pool free-rate EWMA (tokens/s), clamped on write.
|
||||
free_rate_ewma: f64,
|
||||
/// T12: completion-driven request-rate EWMA (req/s) — the release rate
|
||||
/// of the max_running slot dimension. Same event stream, same α.
|
||||
slot_rate_ewma: f64,
|
||||
/// T13: decode-residency EWMA (seconds per request, admit→complete as
|
||||
/// seen by the gateway — directly measured, no right-censoring). The
|
||||
/// slot release rate is Little's law: ledger_inflight / dwell_ewma,
|
||||
/// so a short-request burst cannot pump the rate (the T12 incident:
|
||||
/// EWMA of req/s read 8.6 while true long-tail release was ~2).
|
||||
dwell_ewma: f64,
|
||||
/// T13 gateway-side in-flight ledger: requests admitted at decision
|
||||
/// time minus completions/releases. Kills the snapshot race (the first
|
||||
/// poll interval after a burst start reads an empty pool).
|
||||
inflight_ledger: u64,
|
||||
/// Admit time of the oldest ledger resident (set on the 0→1 transition,
|
||||
/// cleared at 0). Live lower bound on current decode dwell — visible
|
||||
/// even when NOTHING completes (the stall case).
|
||||
oldest_inflight_since: Option<Instant>,
|
||||
last_complete: Option<Instant>,
|
||||
/// C4 high-water: max(0, awaiting - free) tokens at the last FRESH
|
||||
/// disagg snapshot; reused when the snapshot goes stale or loses its
|
||||
/// disagg section (misjudgment is asymmetric — overestimating the wait
|
||||
/// is the safe direction).
|
||||
last_deficit_tokens: Option<u64>,
|
||||
/// C4 high-water for the slot dimension (T12): max(0,
|
||||
/// prealloc_queue_reqs + 1 - free_slots) at the last fresh snapshot.
|
||||
last_slot_numerator: Option<u64>,
|
||||
/// C4 high-water for the slot dimension (T13): snapshot-side pressure
|
||||
/// (num_running + prealloc_queue) and the pool's max_running total at
|
||||
/// the last fresh snapshot. On stale reads the LIVE ledger still wins
|
||||
/// via max().
|
||||
last_pressure_snap: Option<u64>,
|
||||
last_max_running_total: Option<u64>,
|
||||
}
|
||||
|
||||
/// Minimal in-flight queue model, per (node, class) bucket. Metrics-only in
|
||||
@@ -649,11 +667,14 @@ pub struct DSnapshot {
|
||||
/// Free KV-pool tokens: per-rank (1 - token_usage) * max_total, summed.
|
||||
pub free_tokens: u64,
|
||||
/// Requests queued at the D-side prealloc gate (disagg section), summed
|
||||
/// over dp ranks — the queue length of the SLOT dimension (T12).
|
||||
/// over dp ranks — the engine-side queue of the SLOT dimension (T12/T13).
|
||||
pub prealloc_queue_reqs: u64,
|
||||
/// Free running slots: per-rank max_running_requests - num_running_reqs,
|
||||
/// summed (core section).
|
||||
pub free_slots: u64,
|
||||
/// Requests in processing: Σ num_running_reqs (core section). With
|
||||
/// prealloc_queue_reqs this is the engine-side slot occupancy.
|
||||
pub running_reqs: u64,
|
||||
/// Slot capacity: Σ max_running_requests (core section). 0 = unknown
|
||||
/// (old engine) — the slot dimension then degrades out (C13).
|
||||
pub max_running_total: u64,
|
||||
/// Snapshot carries disagg data AND is fresh. Otherwise the C4
|
||||
/// high-water fallback applies (last known deficit, never 0).
|
||||
pub usable: bool,
|
||||
@@ -712,7 +733,14 @@ impl QueueModel {
|
||||
/// T12: per node, T_D = max(memory wait, slot wait) — the prealloc
|
||||
/// memory gate and the max_running slot gate are in series, either one
|
||||
/// stalls the request (c512 实测盲区:槽位 128/128 打满而内存 36%).
|
||||
/// Returns (t_d_min, slot_wait_of_the_min_node).
|
||||
/// Returns (t_d_min, slot_wait_of_the_min_node, min_node).
|
||||
///
|
||||
/// T13 slot口径: pressure = max(台账 inflight_ledger, 快照
|
||||
/// num_running + prealloc_queue) — both sides are LOWER bounds of the
|
||||
/// true occupancy (the ledger misses what the gateway never saw, the
|
||||
/// snapshot misses in-flight-to-engine and lags by a poll interval),
|
||||
/// so the max is the conservative join (C4). Slot wait = max(0,
|
||||
/// pressure + 1 − max_running_total) / slot_release_rate.
|
||||
///
|
||||
/// None = est_T_D inapplicable (est_td_enabled false, no D snapshots,
|
||||
/// or no node has EVER reported disagg data): the C13 "behave exactly
|
||||
@@ -723,55 +751,78 @@ impl QueueModel {
|
||||
d_snaps: &[DSnapshot],
|
||||
own_ctx_tokens: u64,
|
||||
est_td_enabled: bool,
|
||||
) -> Option<(f64, f64)> {
|
||||
) -> Option<(f64, f64, String)> {
|
||||
if !est_td_enabled || d_snaps.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut saw_d = false;
|
||||
let mut min_td = f64::MAX;
|
||||
let mut min_slot = 0.0f64;
|
||||
let mut min_node = String::new();
|
||||
for snap in d_snaps {
|
||||
let rate = self.pool_free_rate(&snap.node);
|
||||
let slot_rate = self.slot_release_rate(&snap.node);
|
||||
let (mem_wait, slot_wait) = if snap.usable {
|
||||
saw_d = true;
|
||||
// C4 high-water: remember the request-independent deficit
|
||||
// for reuse when this node's snapshot later goes stale.
|
||||
let deficit_base = snap.awaiting_tokens.saturating_sub(snap.free_tokens);
|
||||
// T12 slot numerator: queued prealloc reqs + this request
|
||||
// − free slots (a full queue with free slots waits ~0).
|
||||
let slot_numer = snap
|
||||
.prealloc_queue_reqs
|
||||
.saturating_add(1)
|
||||
.saturating_sub(snap.free_slots);
|
||||
// T13: engine-side slot pressure (running + queued).
|
||||
let pressure_snap = snap
|
||||
.running_reqs
|
||||
.saturating_add(snap.prealloc_queue_reqs);
|
||||
{
|
||||
let e = self.d_pools.entry(snap.node.clone()).or_default();
|
||||
e.last_deficit_tokens = Some(deficit_base);
|
||||
e.last_slot_numerator = Some(slot_numer);
|
||||
e.last_pressure_snap = Some(pressure_snap);
|
||||
if snap.max_running_total > 0 {
|
||||
e.last_max_running_total = Some(snap.max_running_total);
|
||||
}
|
||||
}
|
||||
let rate = self.pool_free_rate(&snap.node);
|
||||
let slot_rate = self.slot_release_rate(&snap.node);
|
||||
let ledger = self.d_inflight(&snap.node);
|
||||
let pressure = pressure_snap.max(ledger); // C4: 冲突取大
|
||||
// T_D(mem) = max(0, awaiting + own_ctx - free) / pool_free_rate.
|
||||
let cur = snap
|
||||
.awaiting_tokens
|
||||
.saturating_add(own_ctx_tokens)
|
||||
.saturating_sub(snap.free_tokens);
|
||||
(cur as f64 / rate, slot_numer as f64 / slot_rate)
|
||||
let slot = if snap.max_running_total > 0 {
|
||||
pressure.saturating_add(1).saturating_sub(snap.max_running_total) as f64
|
||||
/ slot_rate
|
||||
} else {
|
||||
0.0 // capacity unknown -> slot dimension degrades out (C13)
|
||||
};
|
||||
(cur as f64 / rate, slot)
|
||||
} else {
|
||||
match self.d_pools.get(&snap.node) {
|
||||
// C4: stale snapshot -> last known high-water deficits,
|
||||
// both dimensions (mem still charges this request's ctx).
|
||||
// both dimensions; the LIVE ledger still wins via max().
|
||||
Some(e)
|
||||
if e.last_deficit_tokens.is_some()
|
||||
|| e.last_slot_numerator.is_some() =>
|
||||
|| e.last_pressure_snap.is_some() =>
|
||||
{
|
||||
saw_d = true;
|
||||
let rate = e
|
||||
.free_rate_ewma
|
||||
.clamp(POOL_FREE_RATE_MIN, POOL_FREE_RATE_MAX);
|
||||
let dwell = self.dwell_for_rate(e, Instant::now());
|
||||
let slot_rate = if e.inflight_ledger > 0 && dwell > 0.0 {
|
||||
(e.inflight_ledger as f64 / dwell)
|
||||
.clamp(SLOT_RELEASE_RATE_MIN, SLOT_RELEASE_RATE_MAX)
|
||||
} else {
|
||||
SLOT_RELEASE_RATE_MIN
|
||||
};
|
||||
let mem = e
|
||||
.last_deficit_tokens
|
||||
.map(|d| d.saturating_add(own_ctx_tokens) as f64 / rate)
|
||||
.unwrap_or(0.0);
|
||||
let slot = e
|
||||
.last_slot_numerator
|
||||
.map(|n| n as f64 / slot_rate)
|
||||
.unwrap_or(0.0);
|
||||
let slot = match (e.last_pressure_snap, e.last_max_running_total) {
|
||||
(Some(p_snap), Some(mr)) if mr > 0 => {
|
||||
let pressure = p_snap.max(e.inflight_ledger);
|
||||
pressure.saturating_add(1).saturating_sub(mr) as f64 / slot_rate
|
||||
}
|
||||
_ => 0.0,
|
||||
};
|
||||
(mem, slot)
|
||||
}
|
||||
// Never reported disagg (old engine / non-PD): not a D
|
||||
@@ -783,9 +834,10 @@ impl QueueModel {
|
||||
if td < min_td {
|
||||
min_td = td;
|
||||
min_slot = slot_wait;
|
||||
min_node = snap.node.clone();
|
||||
}
|
||||
}
|
||||
saw_d.then_some((min_td, min_slot))
|
||||
saw_d.then_some((min_td, min_slot, min_node))
|
||||
}
|
||||
|
||||
/// est_TTFT with the D-side prealloc wait folded in (design §2.1).
|
||||
@@ -804,7 +856,8 @@ impl QueueModel {
|
||||
est_td_enabled: bool,
|
||||
) -> EstTtft {
|
||||
let t_p = self.t_p_of(node, class);
|
||||
let Some((t_d, t_d_slot)) = self.est_td_min(d_snaps, own_ctx_tokens, est_td_enabled)
|
||||
let Some((t_d, t_d_slot, _d_node)) =
|
||||
self.est_td_min(d_snaps, own_ctx_tokens, est_td_enabled)
|
||||
else {
|
||||
return EstTtft {
|
||||
total: t_p,
|
||||
@@ -1015,12 +1068,19 @@ impl QueueModel {
|
||||
/// the decode node. Rate sample = ctx / dt since the previous completion
|
||||
/// on the same node; fixed α=0.3 per event, hard-clamped to
|
||||
/// [POOL_FREE_RATE_MIN, POOL_FREE_RATE_MAX]. The first completion only
|
||||
/// arms the clock (no dt to divide by).
|
||||
pub fn record_decode_completion(&mut self, node: &str, ctx_tokens: u64) {
|
||||
self.record_decode_completion_at(node, ctx_tokens, Instant::now());
|
||||
/// arms the clock (no dt to divide by). `dwell_secs` (T13) feeds the
|
||||
/// decode-residency EWMA behind the Little's-law slot release rate.
|
||||
pub fn record_decode_completion(&mut self, node: &str, ctx_tokens: u64, dwell_secs: Option<f64>) {
|
||||
self.record_decode_completion_at(node, ctx_tokens, dwell_secs, Instant::now());
|
||||
}
|
||||
|
||||
fn record_decode_completion_at(&mut self, node: &str, ctx_tokens: u64, now: Instant) {
|
||||
fn record_decode_completion_at(
|
||||
&mut self,
|
||||
node: &str,
|
||||
ctx_tokens: u64,
|
||||
dwell_secs: Option<f64>,
|
||||
now: Instant,
|
||||
) {
|
||||
let e = self.d_pools.entry(node.to_string()).or_default();
|
||||
if let Some(last) = e.last_complete {
|
||||
let dt = now.duration_since(last).as_secs_f64().max(0.001);
|
||||
@@ -1028,12 +1088,17 @@ impl QueueModel {
|
||||
e.free_rate_ewma = ((1.0 - POOL_FREE_RATE_ALPHA) * e.free_rate_ewma
|
||||
+ POOL_FREE_RATE_ALPHA * sample)
|
||||
.clamp(POOL_FREE_RATE_MIN, POOL_FREE_RATE_MAX);
|
||||
// T12: the same event is one slot release — req/s EWMA beside
|
||||
// the tokens/s one (α shared, own hard bounds).
|
||||
let slot_sample = 1.0 / dt;
|
||||
e.slot_rate_ewma = ((1.0 - POOL_FREE_RATE_ALPHA) * e.slot_rate_ewma
|
||||
+ POOL_FREE_RATE_ALPHA * slot_sample)
|
||||
.clamp(SLOT_RELEASE_RATE_MIN, SLOT_RELEASE_RATE_MAX);
|
||||
}
|
||||
// T13: decode-residency sample (admit→complete wall time, directly
|
||||
// measured by the gateway — no right-censoring, unlike a completion
|
||||
// -rate EWMA which only sees the requests that finished EARLY).
|
||||
if let Some(dwell) = dwell_secs {
|
||||
let dwell = dwell.clamp(DWELL_MIN_SECS, DWELL_MAX_SECS);
|
||||
e.dwell_ewma = if e.dwell_ewma <= 0.0 {
|
||||
dwell
|
||||
} else {
|
||||
(1.0 - POOL_FREE_RATE_ALPHA) * e.dwell_ewma + POOL_FREE_RATE_ALPHA * dwell
|
||||
};
|
||||
}
|
||||
e.last_complete = Some(now);
|
||||
}
|
||||
@@ -1049,15 +1114,96 @@ impl QueueModel {
|
||||
.clamp(POOL_FREE_RATE_MIN, POOL_FREE_RATE_MAX)
|
||||
}
|
||||
|
||||
/// T12: current slot release rate (req/s) for a decode node — the
|
||||
/// completion rate EWMA. Floor doubles as cold start (0.01 req/s =
|
||||
/// one release per 100s: unknown is conservative).
|
||||
/// T12/T13: current slot release rate (req/s) for a decode node —
|
||||
/// Little's law over gateway-measured state: ledger_inflight /
|
||||
/// max(dwell_ewma, oldest resident age). Both factors are
|
||||
/// gateway-observed and live; a completion-only rate estimator is
|
||||
/// right-censored during stalls (the T12 incident: EWMA of req/s read
|
||||
/// 8.6 from the opening short-request burst while true long-tail
|
||||
/// release was ~2 req/s). Cold start (no dwell, empty ledger) reads
|
||||
/// the slow floor — unknown is conservative.
|
||||
pub fn slot_release_rate(&self, node: &str) -> f64 {
|
||||
self.slot_release_rate_at(node, Instant::now())
|
||||
}
|
||||
|
||||
fn slot_release_rate_at(&self, node: &str, now: Instant) -> f64 {
|
||||
let Some(e) = self.d_pools.get(node) else {
|
||||
return SLOT_RELEASE_RATE_MIN;
|
||||
};
|
||||
if e.inflight_ledger == 0 {
|
||||
return SLOT_RELEASE_RATE_MIN;
|
||||
}
|
||||
let dwell = self.dwell_for_rate(e, now);
|
||||
if dwell <= 0.0 {
|
||||
return SLOT_RELEASE_RATE_MIN;
|
||||
}
|
||||
(e.inflight_ledger as f64 / dwell).clamp(SLOT_RELEASE_RATE_MIN, SLOT_RELEASE_RATE_MAX)
|
||||
}
|
||||
|
||||
/// T13 D-side in-flight ledger: +1 at admission-decision time (same
|
||||
/// queue-model write lock as the P booking — the check→act race is
|
||||
/// closed on both sides). Tracks the oldest resident's admit time
|
||||
/// (0→1 transition) for the live dwell lower bound.
|
||||
pub fn d_ledger_admit(&mut self, node: &str) {
|
||||
self.d_ledger_admit_at(node, Instant::now());
|
||||
}
|
||||
|
||||
fn d_ledger_admit_at(&mut self, node: &str, now: Instant) {
|
||||
let e = self.d_pools.entry(node.to_string()).or_default();
|
||||
if e.inflight_ledger == 0 {
|
||||
e.oldest_inflight_since = Some(now);
|
||||
}
|
||||
e.inflight_ledger = e.inflight_ledger.saturating_add(1);
|
||||
}
|
||||
|
||||
/// Ledger −1 (completion/release paths; saturating — a stray release
|
||||
/// must never underflow the ledger). Hitting 0 clears the oldest-age
|
||||
/// tracker; otherwise the oldest timestamp stays sticky (after the
|
||||
/// oldest completes we over-estimate age briefly — the conservative
|
||||
/// direction).
|
||||
pub fn d_ledger_release(&mut self, node: &str) {
|
||||
self.d_ledger_release_at(node, Instant::now());
|
||||
}
|
||||
|
||||
fn d_ledger_release_at(&mut self, node: &str, _now: Instant) {
|
||||
if let Some(e) = self.d_pools.get_mut(node) {
|
||||
e.inflight_ledger = e.inflight_ledger.saturating_sub(1);
|
||||
if e.inflight_ledger == 0 {
|
||||
e.oldest_inflight_since = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Move a ledger booking between D nodes (decision-time pick vs actual
|
||||
/// landing, mirrors the P-side rekey).
|
||||
pub fn d_ledger_rekey(&mut self, from: &str, to: &str) {
|
||||
if from == to {
|
||||
return;
|
||||
}
|
||||
self.d_ledger_release(from);
|
||||
self.d_ledger_admit(to);
|
||||
}
|
||||
|
||||
/// Current ledger count for a decode node.
|
||||
pub fn d_inflight(&self, node: &str) -> u64 {
|
||||
self.d_pools
|
||||
.get(node)
|
||||
.map(|e| e.slot_rate_ewma)
|
||||
.unwrap_or(0.0)
|
||||
.clamp(SLOT_RELEASE_RATE_MIN, SLOT_RELEASE_RATE_MAX)
|
||||
.map(|e| e.inflight_ledger)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Effective decode residency (s) behind the slot release rate:
|
||||
/// max(dwell_ewma from completions, oldest in-flight resident's age).
|
||||
/// The second term is what a completion-only estimator cannot see:
|
||||
/// during a long-request stall NOTHING completes, and the oldest
|
||||
/// resident's age is a live, conservative lower bound on dwell (kills
|
||||
/// the T12 right-censoring pump — 短请求潮灌高速率).
|
||||
fn dwell_for_rate(&self, e: &DPoolEntry, now: Instant) -> f64 {
|
||||
let oldest_age = e
|
||||
.oldest_inflight_since
|
||||
.map(|t| now.duration_since(t).as_secs_f64())
|
||||
.unwrap_or(0.0);
|
||||
e.dwell_ewma.max(oldest_age)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1103,15 +1249,24 @@ pub struct AdmissionMark {
|
||||
pub node: String,
|
||||
pub class: ShadowClass,
|
||||
pub est_tokens: u64,
|
||||
/// T13: the D node this request was ledger-booked against at decision
|
||||
/// time (the min-T_D node). None when no D view existed (C13) — no
|
||||
/// ledger ops then. Rekeyed to the actual decode node at response time.
|
||||
pub d_node: Option<String>,
|
||||
/// Decision time; the admit→complete span is the decode-residency
|
||||
/// (dwell) sample behind the Little's-law slot release rate.
|
||||
pub created: Instant,
|
||||
consumed: AtomicBool,
|
||||
}
|
||||
|
||||
impl AdmissionMark {
|
||||
pub fn new(node: String, class: ShadowClass, est_tokens: u64) -> Self {
|
||||
pub fn new(node: String, class: ShadowClass, est_tokens: u64, d_node: Option<String>) -> Self {
|
||||
Self {
|
||||
node,
|
||||
class,
|
||||
est_tokens,
|
||||
d_node,
|
||||
created: Instant::now(),
|
||||
consumed: AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
@@ -2045,16 +2200,22 @@ mod tests {
|
||||
// ============ est_T_D P1 (T3): T_D estimation + max() composition ============
|
||||
|
||||
fn d_snap(node: &str, awaiting: u64, free: u64, usable: bool) -> DSnapshot {
|
||||
// Slots never constrain in the pre-T12 tests: 128 free, empty queue.
|
||||
d_snap_slots(node, awaiting, free, 0, 128, usable)
|
||||
// Slots never constrain in the pre-T12 tests: empty engine view,
|
||||
// capacity 128 -> pressure 0, wait 0.
|
||||
d_snap_slots(node, awaiting, free, 0, 0, 128, usable)
|
||||
}
|
||||
|
||||
/// (node, awaiting_tokens, free_tokens, prealloc_queue, running,
|
||||
/// max_running_total, usable). T13口径: slot pressure = max(ledger,
|
||||
/// running + prealloc_queue); slot wait = max(0, pressure + 1 -
|
||||
/// max_running_total) / slot_release_rate.
|
||||
fn d_snap_slots(
|
||||
node: &str,
|
||||
awaiting: u64,
|
||||
free: u64,
|
||||
prealloc_queue_reqs: u64,
|
||||
free_slots: u64,
|
||||
running_reqs: u64,
|
||||
max_running_total: u64,
|
||||
usable: bool,
|
||||
) -> DSnapshot {
|
||||
DSnapshot {
|
||||
@@ -2062,7 +2223,8 @@ mod tests {
|
||||
awaiting_tokens: awaiting,
|
||||
free_tokens: free,
|
||||
prealloc_queue_reqs,
|
||||
free_slots,
|
||||
running_reqs,
|
||||
max_running_total,
|
||||
usable,
|
||||
}
|
||||
}
|
||||
@@ -2136,8 +2298,8 @@ mod tests {
|
||||
let mut q = QueueModel::new();
|
||||
let t0 = Instant::now();
|
||||
// 200k tok/s measured release rate on d1.
|
||||
q.record_decode_completion_at("d1", 200_000, t0);
|
||||
q.record_decode_completion_at("d1", 200_000, t0 + Duration::from_secs(1));
|
||||
q.record_decode_completion_at("d1", 200_000, None, t0);
|
||||
q.record_decode_completion_at("d1", 200_000, None, t0 + Duration::from_secs(1));
|
||||
let snaps = [d_snap("d1", 900_000, 100_000, true)]; // deficit 800k
|
||||
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
|
||||
// rate = 0.3*200k (first real sample after arming) = 60k tok/s.
|
||||
@@ -2148,17 +2310,17 @@ mod tests {
|
||||
|
||||
/// Slot-bound blind spot (the c512 shape): memory wide idle, slots full
|
||||
/// → T_D is slot-dominated and grows linearly with the prealloc queue.
|
||||
/// Engine view: running 128/128 (full), queue 10; cold-start slot rate
|
||||
/// floor 0.01 req/s -> (138+1-128)/0.01 = 1100s.
|
||||
#[test]
|
||||
fn est_td_slot_bound_dominates_and_scales() {
|
||||
let mut q = QueueModel::new();
|
||||
// Memory: 10M free (mem wait 0). Slots: 0 free, 10 queued; cold-start
|
||||
// slot rate floor 0.01 req/s -> (10+1)/0.01 = 1100s.
|
||||
let snaps = [d_snap_slots("d1", 0, 10_000_000, 10, 0, true)];
|
||||
let snaps = [d_snap_slots("d1", 0, 10_000_000, 10, 128, 128, true)];
|
||||
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
|
||||
assert_eq!(est.t_d, 1100.0, "slot wait dominates: {}", est.t_d);
|
||||
assert_eq!(est.t_d_slot, 1100.0);
|
||||
// Linear in the queue: 20 queued -> 2100s.
|
||||
let snaps = [d_snap_slots("d1", 0, 10_000_000, 20, 0, true)];
|
||||
// Linear in the queue: 20 queued -> (148+1-128)/0.01 = 2100s.
|
||||
let snaps = [d_snap_slots("d1", 0, 10_000_000, 20, 128, 128, true)];
|
||||
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
|
||||
assert_eq!(est.t_d, 2100.0);
|
||||
}
|
||||
@@ -2168,71 +2330,212 @@ mod tests {
|
||||
#[test]
|
||||
fn est_td_takes_max_of_memory_and_slot() {
|
||||
let mut q = QueueModel::new();
|
||||
// Memory: deficit 100k/1k floor = 100s. Slots: full + 10 queued =
|
||||
// 1100s -> slot wins.
|
||||
let snaps = [d_snap_slots("d1", 100_000, 0, 10, 0, true)];
|
||||
// Memory: deficit 100k/1k floor = 100s. Slots: full (128/128) +
|
||||
// 10 queued = (138+1-128)/0.01 = 1100s -> slot wins.
|
||||
let snaps = [d_snap_slots("d1", 100_000, 0, 10, 128, 128, true)];
|
||||
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
|
||||
assert_eq!(est.t_d, 1100.0, "max picks the slot side");
|
||||
// Same memory deficit, slots wide open -> memory wins (unchanged
|
||||
// pre-T12 behavior: 100s).
|
||||
let snaps = [d_snap_slots("d1", 100_000, 0, 0, 128, true)];
|
||||
let snaps = [d_snap_slots("d1", 100_000, 0, 0, 0, 128, true)];
|
||||
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
|
||||
assert_eq!(est.t_d, 100.0, "max picks the memory side");
|
||||
assert_eq!(est.t_d_slot, 0.0);
|
||||
}
|
||||
|
||||
/// Slot release rate EWMA: convergence, both clamps, cold-start floor.
|
||||
/// T13 Little's-law slot release rate: ledger / max(dwell_ewma,
|
||||
/// oldest resident age). The short-burst pump is impossible: during a
|
||||
/// long-request stall nothing completes, and the oldest resident's age
|
||||
/// drives the rate down live.
|
||||
#[test]
|
||||
fn slot_release_rate_ewma_converges_and_clamps() {
|
||||
fn slot_release_rate_littles_law_not_pumped_by_short_burst() {
|
||||
let mut q = QueueModel::new();
|
||||
let t0 = Instant::now();
|
||||
// Cold start: the floor.
|
||||
assert_eq!(q.slot_release_rate("d1"), SLOT_RELEASE_RATE_MIN);
|
||||
q.record_decode_completion_at("d1", 100_000, t0); // arms the clock
|
||||
assert_eq!(q.slot_release_rate("d1"), SLOT_RELEASE_RATE_MIN);
|
||||
// 10 completions at 1/s -> sample 1 req/s; EWMA α=0.3 -> 1-0.7^10.
|
||||
for i in 1..=10u64 {
|
||||
q.record_decode_completion_at("d1", 100_000, t0 + Duration::from_secs(i));
|
||||
// Cold start: floor.
|
||||
assert_eq!(q.slot_release_rate_at("d1", t0), SLOT_RELEASE_RATE_MIN);
|
||||
// Opening burst: 256 shorts admitted over 10s, 20 complete fast
|
||||
// (dwell 0.5s). Rate right after the burst: dwell_ewma 0.5s,
|
||||
// oldest age ~0 -> high rate (the burst IS fast, that's true).
|
||||
for i in 0..256u64 {
|
||||
q.d_ledger_admit_at("d1", t0 + Duration::from_millis(i * 40));
|
||||
}
|
||||
let rate = q.slot_release_rate("d1");
|
||||
assert!((rate - (1.0 - 0.7f64.powi(10))).abs() < 1e-6, "rate={rate}");
|
||||
// Burst: 1ms apart -> sample 1000 req/s clamps to 50.
|
||||
let mut t = t0 + Duration::from_secs(20);
|
||||
for _ in 0..20 {
|
||||
t += Duration::from_millis(1);
|
||||
q.record_decode_completion_at("d2", 1000, t);
|
||||
for i in 0..20u64 {
|
||||
let t = t0 + Duration::from_millis(256 * 40 + i * 25);
|
||||
q.record_decode_completion_at("d1", 10_000, Some(0.5), t);
|
||||
// Completions pair their ledger bookings (production does this
|
||||
// in shadow_record; the QueueModel completion hook deliberately
|
||||
// does NOT touch the ledger).
|
||||
q.d_ledger_release_at("d1", t);
|
||||
}
|
||||
assert_eq!(q.slot_release_rate("d2"), SLOT_RELEASE_RATE_MAX);
|
||||
// 236 still resident, nothing completing (long tail). At +120s the
|
||||
// oldest age dominates: rate = 236 / max(0.5, 120) ≈ 1.97 req/s —
|
||||
// NOT the burst's implied hundreds.
|
||||
let t_later = t0 + Duration::from_secs(120);
|
||||
let rate = q.slot_release_rate_at("d1", t_later);
|
||||
assert!(
|
||||
(rate - 236.0 / 120.0).abs() < 0.05,
|
||||
"rate={rate} must collapse with resident age, not stay pumped"
|
||||
);
|
||||
// Healthy steady state: dwell_ewma 30s from completions dominates
|
||||
// over a young oldest age.
|
||||
let mut q2 = QueueModel::new();
|
||||
for i in 0..10u64 {
|
||||
q2.d_ledger_admit_at("d1", t0 + Duration::from_secs(i));
|
||||
}
|
||||
for i in 0..5u64 {
|
||||
q2.record_decode_completion_at(
|
||||
"d1",
|
||||
10_000,
|
||||
Some(30.0),
|
||||
t0 + Duration::from_secs(10 + i),
|
||||
);
|
||||
}
|
||||
// ledger 10, dwell_ewma 30 (first sample sets), oldest age ~5s.
|
||||
let rate2 = q2.slot_release_rate_at("d1", t0 + Duration::from_secs(15));
|
||||
assert!((rate2 - 10.0 / 30.0).abs() < 1e-6, "rate={rate2}");
|
||||
// Ledger empty -> floor (unknown = conservative).
|
||||
let q3 = QueueModel::new();
|
||||
assert_eq!(q3.slot_release_rate_at("d1", t0), SLOT_RELEASE_RATE_MIN);
|
||||
}
|
||||
|
||||
/// Rate clamps: absurdly fast turnover caps at 50 req/s; a permanent
|
||||
/// stall sinks to the 0.01 floor.
|
||||
#[test]
|
||||
fn slot_release_rate_hard_bounds() {
|
||||
let mut q = QueueModel::new();
|
||||
let t0 = Instant::now();
|
||||
for i in 0..100u64 {
|
||||
q.d_ledger_admit_at("d1", t0 + Duration::from_millis(i));
|
||||
}
|
||||
// 100 residents aged 100ms -> 100/0.1 = 1000 req/s -> clamp 50.
|
||||
assert_eq!(
|
||||
q.slot_release_rate_at("d1", t0 + Duration::from_millis(100)),
|
||||
SLOT_RELEASE_RATE_MAX
|
||||
);
|
||||
// Same ledger a year later -> 100/31.5M s -> floor 0.01.
|
||||
assert_eq!(
|
||||
q.slot_release_rate_at("d1", t0 + Duration::from_secs(31_536_000)),
|
||||
SLOT_RELEASE_RATE_MIN
|
||||
);
|
||||
}
|
||||
|
||||
/// C4 for the slot dimension: a stale snapshot keeps the last known
|
||||
/// slot-queue high-water — it must NOT clear the gate.
|
||||
/// slot-pressure high-water — it must NOT clear the gate; the LIVE
|
||||
/// ledger still wins via max().
|
||||
#[test]
|
||||
fn est_td_slot_stale_uses_high_water() {
|
||||
let mut q = QueueModel::new();
|
||||
// Fresh: 10 queued, 0 free slots -> slot numerator 11 stored.
|
||||
let snaps = [d_snap_slots("d1", 0, 10_000_000, 10, 0, true)];
|
||||
// Fresh: full (128/128) + 10 queued -> pressure_snap 138 stored.
|
||||
let snaps = [d_snap_slots("d1", 0, 10_000_000, 10, 128, 128, true)];
|
||||
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
|
||||
assert_eq!(est.t_d, 1100.0);
|
||||
// Stale: snapshot now claims empty queue + free slots — ignored,
|
||||
// the stored high-water still drives T_D.
|
||||
let snaps = [d_snap_slots("d1", 0, 10_000_000, 0, 128, false)];
|
||||
// Stale: snapshot now claims empty + idle — ignored, stored 138
|
||||
// still drives T_D.
|
||||
let snaps = [d_snap_slots("d1", 0, 10_000_000, 0, 0, 128, false)];
|
||||
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
|
||||
assert_eq!(est.t_d, 1100.0, "stale slot view falls back to high-water");
|
||||
assert_eq!(est.t_d_slot, 1100.0);
|
||||
// T13: a live ledger ABOVE the stored snapshot wins even stale.
|
||||
for _ in 0..200 {
|
||||
q.d_ledger_admit("d1");
|
||||
}
|
||||
let snaps = [d_snap_slots("d1", 0, 10_000_000, 0, 0, 128, false)];
|
||||
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
|
||||
// pressure = max(200, 138) = 200 -> (200+1-128)/rate. rate: ledger
|
||||
// 200, no dwell, oldest age ~0 -> dwell ~0 -> floor... read the
|
||||
// actual rate to keep the assertion honest.
|
||||
let rate = q.slot_release_rate("d1");
|
||||
let expect = 73.0 / rate;
|
||||
assert!((est.t_d - expect).abs() < 1e-6, "t_d={} expect={expect}", est.t_d);
|
||||
}
|
||||
|
||||
/// C13 regression with slot fields: a node without disagg data (old
|
||||
/// engine) is not a D source — est stays exactly the P-only model.
|
||||
/// Capacity unknown (max_running 0) also degrades the slot term to 0.
|
||||
#[test]
|
||||
fn est_td_slot_c13_unchanged() {
|
||||
let mut q = QueueModel::new();
|
||||
q.admit("p1", ShadowClass::Hot, 2000); // t_p = 2.5
|
||||
let base = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &[], true);
|
||||
let snaps = [d_snap_slots("d1", 0, 0, 224, 0, false)]; // unusable
|
||||
let snaps = [d_snap_slots("d1", 0, 0, 224, 128, 128, false)]; // unusable
|
||||
let degraded = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
|
||||
assert_eq!(degraded, base);
|
||||
assert_eq!(degraded.t_d_slot, 0.0);
|
||||
// max_running_total unknown (0): slot term degrades out, memory
|
||||
// dimension still works.
|
||||
let snaps = [d_snap_slots("d1", 100_000, 0, 224, 128, 0, true)];
|
||||
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
|
||||
assert_eq!(est.t_d, 100.0, "memory only; slot degraded");
|
||||
assert_eq!(est.t_d_slot, 0.0);
|
||||
}
|
||||
|
||||
// ============ T13: D-side in-flight ledger ============
|
||||
|
||||
/// Ledger balance: decision-time admit, completion/release pairing,
|
||||
/// rekey between D nodes, saturating release.
|
||||
#[test]
|
||||
fn t13_d_ledger_balance_and_rekey() {
|
||||
let mut q = QueueModel::new();
|
||||
q.d_ledger_admit("d1");
|
||||
q.d_ledger_admit("d1");
|
||||
q.d_ledger_admit("d2");
|
||||
assert_eq!(q.d_inflight("d1"), 2);
|
||||
assert_eq!(q.d_inflight("d2"), 1);
|
||||
q.d_ledger_rekey("d1", "d2");
|
||||
assert_eq!(q.d_inflight("d1"), 1);
|
||||
assert_eq!(q.d_inflight("d2"), 2);
|
||||
q.d_ledger_release("d1");
|
||||
q.d_ledger_release("d1"); // saturating at 0, no underflow
|
||||
q.d_ledger_release("d1");
|
||||
assert_eq!(q.d_inflight("d1"), 0);
|
||||
assert_eq!(q.d_inflight("d2"), 2);
|
||||
}
|
||||
|
||||
/// Burst oversell test: 300 concurrent check-and-admit at the slot
|
||||
/// boundary. mr=256, dwell pinned 100s -> rate = ledger/100, gate
|
||||
/// wait = (ledger+1-256)/rate > 5s iff ledger > 268.4 -> exactly 268
|
||||
/// admitted, deterministically.
|
||||
#[test]
|
||||
fn t13_slot_gate_no_oversell_multithreaded() {
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
let q = Arc::new(std::sync::RwLock::new(QueueModel::new()));
|
||||
let t0 = Instant::now();
|
||||
// Pin dwell at 100s via completion samples (before any ledger).
|
||||
{
|
||||
let mut qq = q.write().unwrap();
|
||||
qq.record_decode_completion_at("d1", 10_000, Some(100.0), t0);
|
||||
qq.record_decode_completion_at("d1", 10_000, Some(100.0), t0 + Duration::from_secs(1));
|
||||
}
|
||||
let t_gate = t0 + Duration::from_secs(100); // oldest_age = 100s too
|
||||
let admitted = Arc::new(AtomicUsize::new(0));
|
||||
let barrier = Arc::new(std::sync::Barrier::new(300));
|
||||
let mut handles = Vec::new();
|
||||
for _ in 0..300 {
|
||||
let q = Arc::clone(&q);
|
||||
let admitted = Arc::clone(&admitted);
|
||||
let barrier = Arc::clone(&barrier);
|
||||
handles.push(std::thread::spawn(move || {
|
||||
barrier.wait();
|
||||
// Mirrors the slot gate inside shadow_classify_and_admit's
|
||||
// write lock: pressure = ledger (engine view absent here).
|
||||
let mut q = q.write().unwrap();
|
||||
let ledger = q.d_inflight("d1");
|
||||
let rate = q.slot_release_rate_at("d1", t_gate);
|
||||
let wait = ledger.saturating_add(1).saturating_sub(256) as f64 / rate;
|
||||
if wait <= 5.0 {
|
||||
q.d_ledger_admit_at("d1", t0);
|
||||
admitted.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}));
|
||||
}
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
// wait > 5 iff (l+1-256)*100/l > 5 iff l > 268.42 -> admits while
|
||||
// ledger l <= 268, so exactly 269 bookings land (the 270th is the
|
||||
// first rejection).
|
||||
assert_eq!(admitted.load(Ordering::SeqCst), 269, "deterministic gate boundary");
|
||||
assert_eq!(q.read().unwrap().d_inflight("d1"), 269);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2270,13 +2573,14 @@ mod tests {
|
||||
let mut q = QueueModel::new();
|
||||
let t0 = Instant::now();
|
||||
// First completion only arms the clock: cold start reads the floor.
|
||||
q.record_decode_completion_at("http://d1:30001", 100_000, t0);
|
||||
q.record_decode_completion_at("http://d1:30001", 100_000, None, t0);
|
||||
assert_eq!(q.pool_free_rate("http://d1:30001"), POOL_FREE_RATE_MIN);
|
||||
// Ten completions of 100k ctx at 1s spacing -> 100k tok/s samples.
|
||||
for i in 1..=10u64 {
|
||||
q.record_decode_completion_at(
|
||||
"http://d1:30001",
|
||||
100_000,
|
||||
None,
|
||||
t0 + Duration::from_secs(i),
|
||||
);
|
||||
}
|
||||
@@ -2296,12 +2600,12 @@ mod tests {
|
||||
let mut q = QueueModel::new();
|
||||
let t0 = Instant::now();
|
||||
// Enormous sample (100M tokens in 1ms) clamps to the 2M upper bound.
|
||||
q.record_decode_completion_at("d1", 1_000, t0);
|
||||
q.record_decode_completion_at("d1", 100_000_000, t0 + Duration::from_millis(1));
|
||||
q.record_decode_completion_at("d1", 1_000, None, t0);
|
||||
q.record_decode_completion_at("d1", 100_000_000, None, t0 + Duration::from_millis(1));
|
||||
assert_eq!(q.pool_free_rate("d1"), POOL_FREE_RATE_MAX);
|
||||
// Tiny sample on a fresh node clamps to the 1k floor.
|
||||
q.record_decode_completion_at("d2", 1_000, t0);
|
||||
q.record_decode_completion_at("d2", 1, t0 + Duration::from_secs(100));
|
||||
q.record_decode_completion_at("d2", 1_000, None, t0);
|
||||
q.record_decode_completion_at("d2", 1, None, t0 + Duration::from_secs(100));
|
||||
assert_eq!(q.pool_free_rate("d2"), POOL_FREE_RATE_MIN);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user