gateway g9i: D-side in-flight ledger + Little's-law slot release rate (P2 T13)

This commit is contained in:
2026-10-15 23:21:42 +08:00
parent ed0a454730
commit 534417b82e
2 changed files with 539 additions and 107 deletions
+141 -13
View File
@@ -334,6 +334,9 @@ pub struct NodeLoad {
/// summed (T12: the slot dimension of T_D — c512 实测 d1 槽位 128/128
/// 打满而内存仅 36%,内存口径完全看不见).
pub free_slots: u64,
/// Slot capacity: sum of max_running_requests across dp ranks (T13
/// ledger口径 needs capacity, not just headroom).
pub max_running_reqs: u64,
/// True when any rank answered with a disaggregation section. False on
/// old engines (no disagg include support / non-PD deployment) — the
/// est_T_D path then degrades to the P-only model exactly (C13).
@@ -604,6 +607,14 @@ impl G9iState {
"worker" => url.clone(),
)
.set(q.slot_release_rate(url));
// T13: the gateway-side in-flight ledger (vs the
// snapshot's running+prealloc — divergence shows
// poll-lag / bookkeeping drift).
metrics::gauge!(
"smg_g9i_d_inflight_ledger",
"worker" => url.clone(),
)
.set(q.d_inflight(url) as f64);
}
}
}
@@ -846,7 +857,7 @@ impl G9iState {
let lock = self.shadow_queue.as_ref()?;
let mut q = lock.write().ok()?;
q.est_td_min(&d_snaps, own_ctx_tokens, self.config.est_td_enabled)
.map(|(t_d, _slot)| t_d)
.map(|(t_d, _slot, _node)| t_d)
}
/// P2 T_D hard gate (design §2.4-2, user-ruled): when min(T_D) over ALL
@@ -1071,20 +1082,36 @@ impl G9iState {
let actual = super::g9i_shadow::extract_prompt_tokens(resp_json)
.unwrap_or(ctx.est_new_tokens);
q.complete(&ctx.node, class, ctx.est_new_tokens, actual);
// T13: pair the D-side ledger booking (admit→complete).
// Booked iff the decision-time admission carried a
// d_node, or the deferred admit booked at response time.
let d_booked = match &ctx.admission {
Some(adm) => adm.d_node.is_some(),
None => true,
};
if d_booked {
q.d_ledger_release(&ctx.decode_node);
}
}
// est_T_D P1 (design 2026-10-14 §2.2): a successful response
// frees its ctx tokens of KV pool on the decode node — fold
// the freed amount into that node's pool_free_rate EWMA.
// Only responses with real usage count: the release path
// (errors/aborts) freed nothing we can measure.
// T13: dwell = admit→complete wall time (Little's-law
// residency sample behind the slot release rate).
if let Some(ctx_tokens) = super::g9i_shadow::extract_prompt_tokens(resp_json) {
q.record_decode_completion(&ctx.decode_node, ctx_tokens);
let dwell = ctx
.admission
.as_ref()
.map(|a| a.created.elapsed().as_secs_f64());
q.record_decode_completion(&ctx.decode_node, ctx_tokens, dwell);
metrics::gauge!(
super::g9i_shadow::METRIC_POOL_FREE_RATE,
"worker" => ctx.decode_node.clone(),
)
.set(q.pool_free_rate(&ctx.decode_node));
// T12: the same completion is one slot release.
// T13: Little's-law slot release rate beside it.
metrics::gauge!(
super::g9i_shadow::METRIC_SLOT_RELEASE_RATE,
"worker" => ctx.decode_node.clone(),
@@ -1111,11 +1138,19 @@ impl G9iState {
/// Undo a queue admission whose response will never carry usage
/// (non-200, aborted stream, unparseable body). Keeps outstanding
/// conservative without polluting the drain EWMA with failure samples.
/// T13: pairs the D-side ledger booking too (same bookkeeping scope).
pub fn shadow_release(&self, ctx: &ShadowResponseContext) {
if let Some(class) = ctx.class {
if let Some(lock) = &self.shadow_queue {
if let Ok(mut q) = lock.write() {
q.release(&ctx.node, class, ctx.est_new_tokens);
let d_booked = match &ctx.admission {
Some(adm) => adm.d_node.is_some(),
None => true, // deferred admit booked the ledger
};
if d_booked {
q.d_ledger_release(&ctx.decode_node);
}
}
}
}
@@ -1147,9 +1182,17 @@ impl G9iState {
return;
}
q.rekey(&adm.node, &ctx.node, class, adm.est_tokens);
// T13: same for the D-side ledger (decision-time pick vs
// actual decode landing).
if let Some(dn) = &adm.d_node {
q.d_ledger_rekey(dn, &ctx.decode_node);
}
}
// Deferred admission (decision had no node to book).
None => q.admit(&ctx.node, class, ctx.est_new_tokens),
None => {
q.admit(&ctx.node, class, ctx.est_new_tokens);
q.d_ledger_admit(&ctx.decode_node);
}
}
}
@@ -1171,6 +1214,11 @@ impl G9iState {
if let Some(lock) = &self.shadow_queue {
if let Ok(mut q) = lock.write() {
q.release(&adm.node, adm.class, adm.est_tokens);
// T13: pair the D-side ledger booking (never dispatched —
// the booking was intent-only, released here).
if let Some(dn) = &adm.d_node {
q.d_ledger_release(dn);
}
}
}
}
@@ -1189,7 +1237,8 @@ impl G9iState {
awaiting_tokens: load.queued_total_input_tokens,
free_tokens: load.free_tokens,
prealloc_queue_reqs: load.decode_prealloc_queue_reqs,
free_slots: load.free_slots,
running_reqs: load.running_reqs,
max_running_total: load.max_running_reqs,
usable: load.has_disagg && load.is_fresh(ttl),
})
.collect()
@@ -1275,10 +1324,21 @@ impl G9iState {
out.est.total,
)?;
q.admit(&out.node, decision.class, decision.est_new_tokens);
// T13: book the D-side in-flight ledger on the
// min-T_D node — same write lock, so the slot
// gate's check→act is atomic too (快照竞态消灭).
// The booking reads the ledger BEFORE this request.
let d_node = q
.est_td_min(&d_snaps, own_ctx, self.config.est_td_enabled)
.map(|(_, _, n)| n);
if let Some(dn) = &d_node {
q.d_ledger_admit(dn);
}
ctx.admission = Some(Arc::new(super::g9i_shadow::AdmissionMark::new(
out.node,
decision.class,
decision.est_new_tokens,
d_node,
)));
out.est
}
@@ -1822,6 +1882,7 @@ fn parse_node_load(json: &Value) -> Result<NodeLoad, String> {
.get("max_running_requests")
.and_then(|v| v.as_u64())
.unwrap_or(0);
node.max_running_reqs += max_running;
node.free_slots += max_running.saturating_sub(running);
node.queued_total_input_tokens += rank
.get("queued_total_input_tokens")
@@ -2022,6 +2083,7 @@ mod tests {
assert_eq!(node.free_tokens, 2_000_000);
// (128-3) + (128-5) = 248 free slots (per-rank, then summed).
assert_eq!(node.free_slots, 248);
assert_eq!(node.max_running_reqs, 256);
assert!(node.has_disagg);
}
@@ -2199,8 +2261,8 @@ sglang:token_usage{dp_rank=\"0\"} 0.5
}
/// Install decode snapshots: (url, awaiting_tokens, free_tokens,
/// has_disagg, stale). free_slots pinned at 128 with an empty prealloc
/// queue so the T12 slot dimension stays inert in the pre-T12 tests.
/// has_disagg, stale). Slot view pinned at empty+128 capacity so the
/// slot dimension stays inert in the pre-T13 tests.
fn set_d_snapshots(st: &G9iState, entries: &[(&str, u64, u64, bool, bool)]) {
let mapped: Vec<(String, NodeLoad)> = entries
.iter()
@@ -2210,7 +2272,8 @@ sglang:token_usage{dp_rank=\"0\"} 0.5
NodeLoad {
queued_total_input_tokens: *awaiting,
free_tokens: *free,
free_slots: 128,
running_reqs: 0,
max_running_reqs: 128,
has_disagg: *has_disagg,
fetch_ok: true,
fetched_at: Some(if *stale {
@@ -2320,18 +2383,21 @@ sglang:token_usage{dp_rank=\"0\"} 0.5
assert!((err.estimated_wait_secs.unwrap() - 1000.0).abs() < 1e-6);
}
/// T12: the slot dimension drives the admission gate even with memory
/// wide open — the c512 blind spot (slots 128/128, memory 36%), covered.
/// T12/T13: the slot dimension drives the admission gate even with
/// memory wide open — the c512 blind spot (slots 128/128 + 224 queued,
/// memory 36%), covered.
#[test]
fn td_gate_slot_bound_rejects_in_enforce() {
let st = p2_state("enforce");
// Memory idle (10M free tokens), slots full: 224 queued, 0 free.
// Cold-start slot rate floor 0.01 req/s -> (224+1)/0.01 = 22500s.
// Memory idle (10M free tokens), slots full: running 128/128,
// 224 queued. Cold-start slot rate floor 0.01 req/s ->
// (352+1-128)/0.01 = 22500s.
let load = NodeLoad {
queued_total_input_tokens: 0,
free_tokens: 10_000_000,
decode_prealloc_queue_reqs: 224,
free_slots: 0,
running_reqs: 128,
max_running_reqs: 128,
has_disagg: true,
fetch_ok: true,
fetched_at: Some(Instant::now()),
@@ -2678,4 +2744,66 @@ sglang:token_usage{dp_rank=\"0\"} 0.5
assert!(!d.all_over_cap, "cap inert when td_admit=off (C13)");
assert!(ctx.admission.is_some(), "bookkeeping still happens");
}
// ============ T13: D-side in-flight ledger wiring ============
fn d_inflight_of(st: &G9iState, node: &str) -> u64 {
st.shadow_queue
.as_ref()
.unwrap()
.read()
.unwrap()
.d_inflight(node)
}
/// The D ledger is booked at DECISION time (same critical section as
/// the gates — the snapshot race is closed), released by the unconsumed
/// fallback, and rekeyed to the actual decode node on consume.
#[test]
fn t13_d_ledger_booked_at_decision_and_paired() {
let st = t11_state("enforce", 4_000_000);
*st.p_nodes.write().unwrap() = vec!["http://p1:30000".to_string()];
set_d_snapshots(&st, &[("http://d1:30001", 0, 10_000_000, true, false)]);
let seed = chat_ctx_turn1();
seed_table(&st, &seed.chain, "http://p1:30000", 1000);
let mut ctx = chat_ctx();
st.shadow_classify_and_admit(&mut ctx).unwrap().unwrap();
assert_eq!(
d_inflight_of(&st, "http://d1:30001"),
1,
"ledger booked at decision time, before dispatch"
);
// Never dispatched -> fallback release pairs it (P and D both).
st.shadow_release_unconsumed(ctx.admission.clone());
assert_eq!(d_inflight_of(&st, "http://d1:30001"), 0);
assert_eq!(outstanding_of(&st, "http://p1:30000"), 0);
// Exactly-once.
st.shadow_release_unconsumed(ctx.admission.clone());
assert_eq!(d_inflight_of(&st, "http://d1:30001"), 0);
}
/// Consume rekeys the D booking when the actual decode node differs.
#[test]
fn t13_d_ledger_rekey_on_consume() {
let st = t11_state("enforce", 4_000_000);
*st.p_nodes.write().unwrap() = vec!["http://p1:30000".to_string()];
set_d_snapshots(&st, &[("http://d1:30001", 0, 10_000_000, true, false)]);
let seed = chat_ctx_turn1();
seed_table(&st, &seed.chain, "http://p1:30000", 1000);
let mut ctx = chat_ctx();
st.shadow_classify_and_admit(&mut ctx).unwrap().unwrap();
assert_eq!(d_inflight_of(&st, "http://d1:30001"), 1);
// Actual decode landing differs (multi-D future shape).
let resp_ctx = ShadowResponseContext::from_request(
ctx,
"http://p1:30000".to_string(),
"http://d2:30002".to_string(),
);
st.shadow_admit(&resp_ctx);
assert_eq!(d_inflight_of(&st, "http://d1:30001"), 0, "rekeyed off d1");
assert_eq!(d_inflight_of(&st, "http://d2:30002"), 1, "rekeyed onto d2");
// Abort-path release pairs the rekeyed booking.
st.shadow_release(&resp_ctx);
assert_eq!(d_inflight_of(&st, "http://d2:30002"), 0);
}
}
+398 -94
View File
@@ -508,11 +508,15 @@ pub const POOL_FREE_RATE_MIN: f64 = 1_000.0;
pub const POOL_FREE_RATE_MAX: f64 = 2_000_000.0;
/// Fixed EWMA weight per completion event (design §2.2).
const POOL_FREE_RATE_ALPHA: f64 = 0.3;
/// T12 slot_release_rate EWMA hard bounds in req/s: [0.01, 50] — one
/// release per 100s at the floor (unknown = conservative), 50 req/s cap
/// against burst skew.
/// T13 slot_release_rate bounds in req/s: [0.01, 50]. The rate itself is
/// Little's law (ledger / dwell_ewma); dwell samples are clamped to
/// [0.5, 500]s before the EWMA. Cold start (no dwell sample yet) reads the
/// slow floor — unknown is conservative.
pub const SLOT_RELEASE_RATE_MIN: f64 = 0.01;
pub const SLOT_RELEASE_RATE_MAX: f64 = 50.0;
/// Decode residency (dwell) sample clamp, seconds.
const DWELL_MIN_SECS: f64 = 0.5;
const DWELL_MAX_SECS: f64 = 500.0;
/// est_T_D shadow metrics (design §2.5). Emitted from G9iState; the names
/// live here next to the model that produces the values.
@@ -539,18 +543,32 @@ pub const METRIC_BOOTSTRAP_ACTUAL: &str = "smg_g9i_bootstrap_actual_secs";
struct DPoolEntry {
/// Completion-driven pool free-rate EWMA (tokens/s), clamped on write.
free_rate_ewma: f64,
/// T12: completion-driven request-rate EWMA (req/s) — the release rate
/// of the max_running slot dimension. Same event stream, same α.
slot_rate_ewma: f64,
/// T13: decode-residency EWMA (seconds per request, admit→complete as
/// seen by the gateway — directly measured, no right-censoring). The
/// slot release rate is Little's law: ledger_inflight / dwell_ewma,
/// so a short-request burst cannot pump the rate (the T12 incident:
/// EWMA of req/s read 8.6 while true long-tail release was ~2).
dwell_ewma: f64,
/// T13 gateway-side in-flight ledger: requests admitted at decision
/// time minus completions/releases. Kills the snapshot race (the first
/// poll interval after a burst start reads an empty pool).
inflight_ledger: u64,
/// Admit time of the oldest ledger resident (set on the 0→1 transition,
/// cleared at 0). Live lower bound on current decode dwell — visible
/// even when NOTHING completes (the stall case).
oldest_inflight_since: Option<Instant>,
last_complete: Option<Instant>,
/// C4 high-water: max(0, awaiting - free) tokens at the last FRESH
/// disagg snapshot; reused when the snapshot goes stale or loses its
/// disagg section (misjudgment is asymmetric — overestimating the wait
/// is the safe direction).
last_deficit_tokens: Option<u64>,
/// C4 high-water for the slot dimension (T12): max(0,
/// prealloc_queue_reqs + 1 - free_slots) at the last fresh snapshot.
last_slot_numerator: Option<u64>,
/// C4 high-water for the slot dimension (T13): snapshot-side pressure
/// (num_running + prealloc_queue) and the pool's max_running total at
/// the last fresh snapshot. On stale reads the LIVE ledger still wins
/// via max().
last_pressure_snap: Option<u64>,
last_max_running_total: Option<u64>,
}
/// Minimal in-flight queue model, per (node, class) bucket. Metrics-only in
@@ -649,11 +667,14 @@ pub struct DSnapshot {
/// Free KV-pool tokens: per-rank (1 - token_usage) * max_total, summed.
pub free_tokens: u64,
/// Requests queued at the D-side prealloc gate (disagg section), summed
/// over dp ranks — the queue length of the SLOT dimension (T12).
/// over dp ranks — the engine-side queue of the SLOT dimension (T12/T13).
pub prealloc_queue_reqs: u64,
/// Free running slots: per-rank max_running_requests - num_running_reqs,
/// summed (core section).
pub free_slots: u64,
/// Requests in processing: Σ num_running_reqs (core section). With
/// prealloc_queue_reqs this is the engine-side slot occupancy.
pub running_reqs: u64,
/// Slot capacity: Σ max_running_requests (core section). 0 = unknown
/// (old engine) — the slot dimension then degrades out (C13).
pub max_running_total: u64,
/// Snapshot carries disagg data AND is fresh. Otherwise the C4
/// high-water fallback applies (last known deficit, never 0).
pub usable: bool,
@@ -712,7 +733,14 @@ impl QueueModel {
/// T12: per node, T_D = max(memory wait, slot wait) — the prealloc
/// memory gate and the max_running slot gate are in series, either one
/// stalls the request (c512 实测盲区:槽位 128/128 打满而内存 36%).
/// Returns (t_d_min, slot_wait_of_the_min_node).
/// Returns (t_d_min, slot_wait_of_the_min_node, min_node).
///
/// T13 slot口径: pressure = max(台账 inflight_ledger, 快照
/// num_running + prealloc_queue) — both sides are LOWER bounds of the
/// true occupancy (the ledger misses what the gateway never saw, the
/// snapshot misses in-flight-to-engine and lags by a poll interval),
/// so the max is the conservative join (C4). Slot wait = max(0,
/// pressure + 1 − max_running_total) / slot_release_rate.
///
/// None = est_T_D inapplicable (est_td_enabled false, no D snapshots,
/// or no node has EVER reported disagg data): the C13 "behave exactly
@@ -723,55 +751,78 @@ impl QueueModel {
d_snaps: &[DSnapshot],
own_ctx_tokens: u64,
est_td_enabled: bool,
) -> Option<(f64, f64)> {
) -> Option<(f64, f64, String)> {
if !est_td_enabled || d_snaps.is_empty() {
return None;
}
let mut saw_d = false;
let mut min_td = f64::MAX;
let mut min_slot = 0.0f64;
let mut min_node = String::new();
for snap in d_snaps {
let rate = self.pool_free_rate(&snap.node);
let slot_rate = self.slot_release_rate(&snap.node);
let (mem_wait, slot_wait) = if snap.usable {
saw_d = true;
// C4 high-water: remember the request-independent deficit
// for reuse when this node's snapshot later goes stale.
let deficit_base = snap.awaiting_tokens.saturating_sub(snap.free_tokens);
// T12 slot numerator: queued prealloc reqs + this request
// − free slots (a full queue with free slots waits ~0).
let slot_numer = snap
.prealloc_queue_reqs
.saturating_add(1)
.saturating_sub(snap.free_slots);
// T13: engine-side slot pressure (running + queued).
let pressure_snap = snap
.running_reqs
.saturating_add(snap.prealloc_queue_reqs);
{
let e = self.d_pools.entry(snap.node.clone()).or_default();
e.last_deficit_tokens = Some(deficit_base);
e.last_slot_numerator = Some(slot_numer);
e.last_pressure_snap = Some(pressure_snap);
if snap.max_running_total > 0 {
e.last_max_running_total = Some(snap.max_running_total);
}
}
let rate = self.pool_free_rate(&snap.node);
let slot_rate = self.slot_release_rate(&snap.node);
let ledger = self.d_inflight(&snap.node);
let pressure = pressure_snap.max(ledger); // C4: 冲突取大
// T_D(mem) = max(0, awaiting + own_ctx - free) / pool_free_rate.
let cur = snap
.awaiting_tokens
.saturating_add(own_ctx_tokens)
.saturating_sub(snap.free_tokens);
(cur as f64 / rate, slot_numer as f64 / slot_rate)
let slot = if snap.max_running_total > 0 {
pressure.saturating_add(1).saturating_sub(snap.max_running_total) as f64
/ slot_rate
} else {
0.0 // capacity unknown -> slot dimension degrades out (C13)
};
(cur as f64 / rate, slot)
} else {
match self.d_pools.get(&snap.node) {
// C4: stale snapshot -> last known high-water deficits,
// both dimensions (mem still charges this request's ctx).
// both dimensions; the LIVE ledger still wins via max().
Some(e)
if e.last_deficit_tokens.is_some()
|| e.last_slot_numerator.is_some() =>
|| e.last_pressure_snap.is_some() =>
{
saw_d = true;
let rate = e
.free_rate_ewma
.clamp(POOL_FREE_RATE_MIN, POOL_FREE_RATE_MAX);
let dwell = self.dwell_for_rate(e, Instant::now());
let slot_rate = if e.inflight_ledger > 0 && dwell > 0.0 {
(e.inflight_ledger as f64 / dwell)
.clamp(SLOT_RELEASE_RATE_MIN, SLOT_RELEASE_RATE_MAX)
} else {
SLOT_RELEASE_RATE_MIN
};
let mem = e
.last_deficit_tokens
.map(|d| d.saturating_add(own_ctx_tokens) as f64 / rate)
.unwrap_or(0.0);
let slot = e
.last_slot_numerator
.map(|n| n as f64 / slot_rate)
.unwrap_or(0.0);
let slot = match (e.last_pressure_snap, e.last_max_running_total) {
(Some(p_snap), Some(mr)) if mr > 0 => {
let pressure = p_snap.max(e.inflight_ledger);
pressure.saturating_add(1).saturating_sub(mr) as f64 / slot_rate
}
_ => 0.0,
};
(mem, slot)
}
// Never reported disagg (old engine / non-PD): not a D
@@ -783,9 +834,10 @@ impl QueueModel {
if td < min_td {
min_td = td;
min_slot = slot_wait;
min_node = snap.node.clone();
}
}
saw_d.then_some((min_td, min_slot))
saw_d.then_some((min_td, min_slot, min_node))
}
/// est_TTFT with the D-side prealloc wait folded in (design §2.1).
@@ -804,7 +856,8 @@ impl QueueModel {
est_td_enabled: bool,
) -> EstTtft {
let t_p = self.t_p_of(node, class);
let Some((t_d, t_d_slot)) = self.est_td_min(d_snaps, own_ctx_tokens, est_td_enabled)
let Some((t_d, t_d_slot, _d_node)) =
self.est_td_min(d_snaps, own_ctx_tokens, est_td_enabled)
else {
return EstTtft {
total: t_p,
@@ -1015,12 +1068,19 @@ impl QueueModel {
/// the decode node. Rate sample = ctx / dt since the previous completion
/// on the same node; fixed α=0.3 per event, hard-clamped to
/// [POOL_FREE_RATE_MIN, POOL_FREE_RATE_MAX]. The first completion only
/// arms the clock (no dt to divide by).
pub fn record_decode_completion(&mut self, node: &str, ctx_tokens: u64) {
self.record_decode_completion_at(node, ctx_tokens, Instant::now());
/// arms the clock (no dt to divide by). `dwell_secs` (T13) feeds the
/// decode-residency EWMA behind the Little's-law slot release rate.
pub fn record_decode_completion(&mut self, node: &str, ctx_tokens: u64, dwell_secs: Option<f64>) {
self.record_decode_completion_at(node, ctx_tokens, dwell_secs, Instant::now());
}
fn record_decode_completion_at(&mut self, node: &str, ctx_tokens: u64, now: Instant) {
fn record_decode_completion_at(
&mut self,
node: &str,
ctx_tokens: u64,
dwell_secs: Option<f64>,
now: Instant,
) {
let e = self.d_pools.entry(node.to_string()).or_default();
if let Some(last) = e.last_complete {
let dt = now.duration_since(last).as_secs_f64().max(0.001);
@@ -1028,12 +1088,17 @@ impl QueueModel {
e.free_rate_ewma = ((1.0 - POOL_FREE_RATE_ALPHA) * e.free_rate_ewma
+ POOL_FREE_RATE_ALPHA * sample)
.clamp(POOL_FREE_RATE_MIN, POOL_FREE_RATE_MAX);
// T12: the same event is one slot release — req/s EWMA beside
// the tokens/s one (α shared, own hard bounds).
let slot_sample = 1.0 / dt;
e.slot_rate_ewma = ((1.0 - POOL_FREE_RATE_ALPHA) * e.slot_rate_ewma
+ POOL_FREE_RATE_ALPHA * slot_sample)
.clamp(SLOT_RELEASE_RATE_MIN, SLOT_RELEASE_RATE_MAX);
}
// T13: decode-residency sample (admit→complete wall time, directly
// measured by the gateway — no right-censoring, unlike a completion
// -rate EWMA which only sees the requests that finished EARLY).
if let Some(dwell) = dwell_secs {
let dwell = dwell.clamp(DWELL_MIN_SECS, DWELL_MAX_SECS);
e.dwell_ewma = if e.dwell_ewma <= 0.0 {
dwell
} else {
(1.0 - POOL_FREE_RATE_ALPHA) * e.dwell_ewma + POOL_FREE_RATE_ALPHA * dwell
};
}
e.last_complete = Some(now);
}
@@ -1049,15 +1114,96 @@ impl QueueModel {
.clamp(POOL_FREE_RATE_MIN, POOL_FREE_RATE_MAX)
}
/// T12: current slot release rate (req/s) for a decode node — the
/// completion rate EWMA. Floor doubles as cold start (0.01 req/s =
/// one release per 100s: unknown is conservative).
/// T12/T13: current slot release rate (req/s) for a decode node —
/// Little's law over gateway-measured state: ledger_inflight /
/// max(dwell_ewma, oldest resident age). Both factors are
/// gateway-observed and live; a completion-only rate estimator is
/// right-censored during stalls (the T12 incident: EWMA of req/s read
/// 8.6 from the opening short-request burst while true long-tail
/// release was ~2 req/s). Cold start (no dwell, empty ledger) reads
/// the slow floor — unknown is conservative.
pub fn slot_release_rate(&self, node: &str) -> f64 {
self.slot_release_rate_at(node, Instant::now())
}
fn slot_release_rate_at(&self, node: &str, now: Instant) -> f64 {
let Some(e) = self.d_pools.get(node) else {
return SLOT_RELEASE_RATE_MIN;
};
if e.inflight_ledger == 0 {
return SLOT_RELEASE_RATE_MIN;
}
let dwell = self.dwell_for_rate(e, now);
if dwell <= 0.0 {
return SLOT_RELEASE_RATE_MIN;
}
(e.inflight_ledger as f64 / dwell).clamp(SLOT_RELEASE_RATE_MIN, SLOT_RELEASE_RATE_MAX)
}
/// T13 D-side in-flight ledger: +1 at admission-decision time (same
/// queue-model write lock as the P booking — the check→act race is
/// closed on both sides). Tracks the oldest resident's admit time
/// (0→1 transition) for the live dwell lower bound.
pub fn d_ledger_admit(&mut self, node: &str) {
self.d_ledger_admit_at(node, Instant::now());
}
fn d_ledger_admit_at(&mut self, node: &str, now: Instant) {
let e = self.d_pools.entry(node.to_string()).or_default();
if e.inflight_ledger == 0 {
e.oldest_inflight_since = Some(now);
}
e.inflight_ledger = e.inflight_ledger.saturating_add(1);
}
/// Ledger −1 (completion/release paths; saturating — a stray release
/// must never underflow the ledger). Hitting 0 clears the oldest-age
/// tracker; otherwise the oldest timestamp stays sticky (after the
/// oldest completes we over-estimate age briefly — the conservative
/// direction).
pub fn d_ledger_release(&mut self, node: &str) {
self.d_ledger_release_at(node, Instant::now());
}
fn d_ledger_release_at(&mut self, node: &str, _now: Instant) {
if let Some(e) = self.d_pools.get_mut(node) {
e.inflight_ledger = e.inflight_ledger.saturating_sub(1);
if e.inflight_ledger == 0 {
e.oldest_inflight_since = None;
}
}
}
/// Move a ledger booking between D nodes (decision-time pick vs actual
/// landing, mirrors the P-side rekey).
pub fn d_ledger_rekey(&mut self, from: &str, to: &str) {
if from == to {
return;
}
self.d_ledger_release(from);
self.d_ledger_admit(to);
}
/// Current ledger count for a decode node.
pub fn d_inflight(&self, node: &str) -> u64 {
self.d_pools
.get(node)
.map(|e| e.slot_rate_ewma)
.unwrap_or(0.0)
.clamp(SLOT_RELEASE_RATE_MIN, SLOT_RELEASE_RATE_MAX)
.map(|e| e.inflight_ledger)
.unwrap_or(0)
}
/// Effective decode residency (s) behind the slot release rate:
/// max(dwell_ewma from completions, oldest in-flight resident's age).
/// The second term is what a completion-only estimator cannot see:
/// during a long-request stall NOTHING completes, and the oldest
/// resident's age is a live, conservative lower bound on dwell (kills
/// the T12 right-censoring pump — 短请求潮灌高速率).
fn dwell_for_rate(&self, e: &DPoolEntry, now: Instant) -> f64 {
let oldest_age = e
.oldest_inflight_since
.map(|t| now.duration_since(t).as_secs_f64())
.unwrap_or(0.0);
e.dwell_ewma.max(oldest_age)
}
}
@@ -1103,15 +1249,24 @@ pub struct AdmissionMark {
pub node: String,
pub class: ShadowClass,
pub est_tokens: u64,
/// T13: the D node this request was ledger-booked against at decision
/// time (the min-T_D node). None when no D view existed (C13) — no
/// ledger ops then. Rekeyed to the actual decode node at response time.
pub d_node: Option<String>,
/// Decision time; the admit→complete span is the decode-residency
/// (dwell) sample behind the Little's-law slot release rate.
pub created: Instant,
consumed: AtomicBool,
}
impl AdmissionMark {
pub fn new(node: String, class: ShadowClass, est_tokens: u64) -> Self {
pub fn new(node: String, class: ShadowClass, est_tokens: u64, d_node: Option<String>) -> Self {
Self {
node,
class,
est_tokens,
d_node,
created: Instant::now(),
consumed: AtomicBool::new(false),
}
}
@@ -2045,16 +2200,22 @@ mod tests {
// ============ est_T_D P1 (T3): T_D estimation + max() composition ============
fn d_snap(node: &str, awaiting: u64, free: u64, usable: bool) -> DSnapshot {
// Slots never constrain in the pre-T12 tests: 128 free, empty queue.
d_snap_slots(node, awaiting, free, 0, 128, usable)
// Slots never constrain in the pre-T12 tests: empty engine view,
// capacity 128 -> pressure 0, wait 0.
d_snap_slots(node, awaiting, free, 0, 0, 128, usable)
}
/// (node, awaiting_tokens, free_tokens, prealloc_queue, running,
/// max_running_total, usable). T13口径: slot pressure = max(ledger,
/// running + prealloc_queue); slot wait = max(0, pressure + 1 -
/// max_running_total) / slot_release_rate.
fn d_snap_slots(
node: &str,
awaiting: u64,
free: u64,
prealloc_queue_reqs: u64,
free_slots: u64,
running_reqs: u64,
max_running_total: u64,
usable: bool,
) -> DSnapshot {
DSnapshot {
@@ -2062,7 +2223,8 @@ mod tests {
awaiting_tokens: awaiting,
free_tokens: free,
prealloc_queue_reqs,
free_slots,
running_reqs,
max_running_total,
usable,
}
}
@@ -2136,8 +2298,8 @@ mod tests {
let mut q = QueueModel::new();
let t0 = Instant::now();
// 200k tok/s measured release rate on d1.
q.record_decode_completion_at("d1", 200_000, t0);
q.record_decode_completion_at("d1", 200_000, t0 + Duration::from_secs(1));
q.record_decode_completion_at("d1", 200_000, None, t0);
q.record_decode_completion_at("d1", 200_000, None, t0 + Duration::from_secs(1));
let snaps = [d_snap("d1", 900_000, 100_000, true)]; // deficit 800k
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
// rate = 0.3*200k (first real sample after arming) = 60k tok/s.
@@ -2148,17 +2310,17 @@ mod tests {
/// Slot-bound blind spot (the c512 shape): memory wide idle, slots full
/// → T_D is slot-dominated and grows linearly with the prealloc queue.
/// Engine view: running 128/128 (full), queue 10; cold-start slot rate
/// floor 0.01 req/s -> (138+1-128)/0.01 = 1100s.
#[test]
fn est_td_slot_bound_dominates_and_scales() {
let mut q = QueueModel::new();
// Memory: 10M free (mem wait 0). Slots: 0 free, 10 queued; cold-start
// slot rate floor 0.01 req/s -> (10+1)/0.01 = 1100s.
let snaps = [d_snap_slots("d1", 0, 10_000_000, 10, 0, true)];
let snaps = [d_snap_slots("d1", 0, 10_000_000, 10, 128, 128, true)];
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
assert_eq!(est.t_d, 1100.0, "slot wait dominates: {}", est.t_d);
assert_eq!(est.t_d_slot, 1100.0);
// Linear in the queue: 20 queued -> 2100s.
let snaps = [d_snap_slots("d1", 0, 10_000_000, 20, 0, true)];
// Linear in the queue: 20 queued -> (148+1-128)/0.01 = 2100s.
let snaps = [d_snap_slots("d1", 0, 10_000_000, 20, 128, 128, true)];
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
assert_eq!(est.t_d, 2100.0);
}
@@ -2168,71 +2330,212 @@ mod tests {
#[test]
fn est_td_takes_max_of_memory_and_slot() {
let mut q = QueueModel::new();
// Memory: deficit 100k/1k floor = 100s. Slots: full + 10 queued =
// 1100s -> slot wins.
let snaps = [d_snap_slots("d1", 100_000, 0, 10, 0, true)];
// Memory: deficit 100k/1k floor = 100s. Slots: full (128/128) +
// 10 queued = (138+1-128)/0.01 = 1100s -> slot wins.
let snaps = [d_snap_slots("d1", 100_000, 0, 10, 128, 128, true)];
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
assert_eq!(est.t_d, 1100.0, "max picks the slot side");
// Same memory deficit, slots wide open -> memory wins (unchanged
// pre-T12 behavior: 100s).
let snaps = [d_snap_slots("d1", 100_000, 0, 0, 128, true)];
let snaps = [d_snap_slots("d1", 100_000, 0, 0, 0, 128, true)];
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
assert_eq!(est.t_d, 100.0, "max picks the memory side");
assert_eq!(est.t_d_slot, 0.0);
}
/// Slot release rate EWMA: convergence, both clamps, cold-start floor.
/// T13 Little's-law slot release rate: ledger / max(dwell_ewma,
/// oldest resident age). The short-burst pump is impossible: during a
/// long-request stall nothing completes, and the oldest resident's age
/// drives the rate down live.
#[test]
fn slot_release_rate_ewma_converges_and_clamps() {
fn slot_release_rate_littles_law_not_pumped_by_short_burst() {
let mut q = QueueModel::new();
let t0 = Instant::now();
// Cold start: the floor.
assert_eq!(q.slot_release_rate("d1"), SLOT_RELEASE_RATE_MIN);
q.record_decode_completion_at("d1", 100_000, t0); // arms the clock
assert_eq!(q.slot_release_rate("d1"), SLOT_RELEASE_RATE_MIN);
// 10 completions at 1/s -> sample 1 req/s; EWMA α=0.3 -> 1-0.7^10.
for i in 1..=10u64 {
q.record_decode_completion_at("d1", 100_000, t0 + Duration::from_secs(i));
// Cold start: floor.
assert_eq!(q.slot_release_rate_at("d1", t0), SLOT_RELEASE_RATE_MIN);
// Opening burst: 256 shorts admitted over 10s, 20 complete fast
// (dwell 0.5s). Rate right after the burst: dwell_ewma 0.5s,
// oldest age ~0 -> high rate (the burst IS fast, that's true).
for i in 0..256u64 {
q.d_ledger_admit_at("d1", t0 + Duration::from_millis(i * 40));
}
let rate = q.slot_release_rate("d1");
assert!((rate - (1.0 - 0.7f64.powi(10))).abs() < 1e-6, "rate={rate}");
// Burst: 1ms apart -> sample 1000 req/s clamps to 50.
let mut t = t0 + Duration::from_secs(20);
for _ in 0..20 {
t += Duration::from_millis(1);
q.record_decode_completion_at("d2", 1000, t);
for i in 0..20u64 {
let t = t0 + Duration::from_millis(256 * 40 + i * 25);
q.record_decode_completion_at("d1", 10_000, Some(0.5), t);
// Completions pair their ledger bookings (production does this
// in shadow_record; the QueueModel completion hook deliberately
// does NOT touch the ledger).
q.d_ledger_release_at("d1", t);
}
assert_eq!(q.slot_release_rate("d2"), SLOT_RELEASE_RATE_MAX);
// 236 still resident, nothing completing (long tail). At +120s the
// oldest age dominates: rate = 236 / max(0.5, 120) ≈ 1.97 req/s —
// NOT the burst's implied hundreds.
let t_later = t0 + Duration::from_secs(120);
let rate = q.slot_release_rate_at("d1", t_later);
assert!(
(rate - 236.0 / 120.0).abs() < 0.05,
"rate={rate} must collapse with resident age, not stay pumped"
);
// Healthy steady state: dwell_ewma 30s from completions dominates
// over a young oldest age.
let mut q2 = QueueModel::new();
for i in 0..10u64 {
q2.d_ledger_admit_at("d1", t0 + Duration::from_secs(i));
}
for i in 0..5u64 {
q2.record_decode_completion_at(
"d1",
10_000,
Some(30.0),
t0 + Duration::from_secs(10 + i),
);
}
// ledger 10, dwell_ewma 30 (first sample sets), oldest age ~5s.
let rate2 = q2.slot_release_rate_at("d1", t0 + Duration::from_secs(15));
assert!((rate2 - 10.0 / 30.0).abs() < 1e-6, "rate={rate2}");
// Ledger empty -> floor (unknown = conservative).
let q3 = QueueModel::new();
assert_eq!(q3.slot_release_rate_at("d1", t0), SLOT_RELEASE_RATE_MIN);
}
/// Rate clamps: absurdly fast turnover caps at 50 req/s; a permanent
/// stall sinks to the 0.01 floor.
#[test]
fn slot_release_rate_hard_bounds() {
let mut q = QueueModel::new();
let t0 = Instant::now();
for i in 0..100u64 {
q.d_ledger_admit_at("d1", t0 + Duration::from_millis(i));
}
// 100 residents aged 100ms -> 100/0.1 = 1000 req/s -> clamp 50.
assert_eq!(
q.slot_release_rate_at("d1", t0 + Duration::from_millis(100)),
SLOT_RELEASE_RATE_MAX
);
// Same ledger a year later -> 100/31.5M s -> floor 0.01.
assert_eq!(
q.slot_release_rate_at("d1", t0 + Duration::from_secs(31_536_000)),
SLOT_RELEASE_RATE_MIN
);
}
/// C4 for the slot dimension: a stale snapshot keeps the last known
/// slot-queue high-water — it must NOT clear the gate.
/// slot-pressure high-water — it must NOT clear the gate; the LIVE
/// ledger still wins via max().
#[test]
fn est_td_slot_stale_uses_high_water() {
let mut q = QueueModel::new();
// Fresh: 10 queued, 0 free slots -> slot numerator 11 stored.
let snaps = [d_snap_slots("d1", 0, 10_000_000, 10, 0, true)];
// Fresh: full (128/128) + 10 queued -> pressure_snap 138 stored.
let snaps = [d_snap_slots("d1", 0, 10_000_000, 10, 128, 128, true)];
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
assert_eq!(est.t_d, 1100.0);
// Stale: snapshot now claims empty queue + free slots — ignored,
// the stored high-water still drives T_D.
let snaps = [d_snap_slots("d1", 0, 10_000_000, 0, 128, false)];
// Stale: snapshot now claims empty + idle — ignored, stored 138
// still drives T_D.
let snaps = [d_snap_slots("d1", 0, 10_000_000, 0, 0, 128, false)];
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
assert_eq!(est.t_d, 1100.0, "stale slot view falls back to high-water");
assert_eq!(est.t_d_slot, 1100.0);
// T13: a live ledger ABOVE the stored snapshot wins even stale.
for _ in 0..200 {
q.d_ledger_admit("d1");
}
let snaps = [d_snap_slots("d1", 0, 10_000_000, 0, 0, 128, false)];
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
// pressure = max(200, 138) = 200 -> (200+1-128)/rate. rate: ledger
// 200, no dwell, oldest age ~0 -> dwell ~0 -> floor... read the
// actual rate to keep the assertion honest.
let rate = q.slot_release_rate("d1");
let expect = 73.0 / rate;
assert!((est.t_d - expect).abs() < 1e-6, "t_d={} expect={expect}", est.t_d);
}
/// C13 regression with slot fields: a node without disagg data (old
/// engine) is not a D source — est stays exactly the P-only model.
/// Capacity unknown (max_running 0) also degrades the slot term to 0.
#[test]
fn est_td_slot_c13_unchanged() {
let mut q = QueueModel::new();
q.admit("p1", ShadowClass::Hot, 2000); // t_p = 2.5
let base = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &[], true);
let snaps = [d_snap_slots("d1", 0, 0, 224, 0, false)]; // unusable
let snaps = [d_snap_slots("d1", 0, 0, 224, 128, 128, false)]; // unusable
let degraded = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
assert_eq!(degraded, base);
assert_eq!(degraded.t_d_slot, 0.0);
// max_running_total unknown (0): slot term degrades out, memory
// dimension still works.
let snaps = [d_snap_slots("d1", 100_000, 0, 224, 128, 0, true)];
let est = q.est_ttft(Some("p1"), ShadowClass::Hot, 0, &snaps, true);
assert_eq!(est.t_d, 100.0, "memory only; slot degraded");
assert_eq!(est.t_d_slot, 0.0);
}
// ============ T13: D-side in-flight ledger ============
/// Ledger balance: decision-time admit, completion/release pairing,
/// rekey between D nodes, saturating release.
#[test]
fn t13_d_ledger_balance_and_rekey() {
let mut q = QueueModel::new();
q.d_ledger_admit("d1");
q.d_ledger_admit("d1");
q.d_ledger_admit("d2");
assert_eq!(q.d_inflight("d1"), 2);
assert_eq!(q.d_inflight("d2"), 1);
q.d_ledger_rekey("d1", "d2");
assert_eq!(q.d_inflight("d1"), 1);
assert_eq!(q.d_inflight("d2"), 2);
q.d_ledger_release("d1");
q.d_ledger_release("d1"); // saturating at 0, no underflow
q.d_ledger_release("d1");
assert_eq!(q.d_inflight("d1"), 0);
assert_eq!(q.d_inflight("d2"), 2);
}
/// Burst oversell test: 300 concurrent check-and-admit at the slot
/// boundary. mr=256, dwell pinned 100s -> rate = ledger/100, gate
/// wait = (ledger+1-256)/rate > 5s iff ledger > 268.4 -> exactly 268
/// admitted, deterministically.
#[test]
fn t13_slot_gate_no_oversell_multithreaded() {
use std::sync::atomic::AtomicUsize;
let q = Arc::new(std::sync::RwLock::new(QueueModel::new()));
let t0 = Instant::now();
// Pin dwell at 100s via completion samples (before any ledger).
{
let mut qq = q.write().unwrap();
qq.record_decode_completion_at("d1", 10_000, Some(100.0), t0);
qq.record_decode_completion_at("d1", 10_000, Some(100.0), t0 + Duration::from_secs(1));
}
let t_gate = t0 + Duration::from_secs(100); // oldest_age = 100s too
let admitted = Arc::new(AtomicUsize::new(0));
let barrier = Arc::new(std::sync::Barrier::new(300));
let mut handles = Vec::new();
for _ in 0..300 {
let q = Arc::clone(&q);
let admitted = Arc::clone(&admitted);
let barrier = Arc::clone(&barrier);
handles.push(std::thread::spawn(move || {
barrier.wait();
// Mirrors the slot gate inside shadow_classify_and_admit's
// write lock: pressure = ledger (engine view absent here).
let mut q = q.write().unwrap();
let ledger = q.d_inflight("d1");
let rate = q.slot_release_rate_at("d1", t_gate);
let wait = ledger.saturating_add(1).saturating_sub(256) as f64 / rate;
if wait <= 5.0 {
q.d_ledger_admit_at("d1", t0);
admitted.fetch_add(1, Ordering::SeqCst);
}
}));
}
for h in handles {
h.join().unwrap();
}
// wait > 5 iff (l+1-256)*100/l > 5 iff l > 268.42 -> admits while
// ledger l <= 268, so exactly 269 bookings land (the 270th is the
// first rejection).
assert_eq!(admitted.load(Ordering::SeqCst), 269, "deterministic gate boundary");
assert_eq!(q.read().unwrap().d_inflight("d1"), 269);
}
#[test]
@@ -2270,13 +2573,14 @@ mod tests {
let mut q = QueueModel::new();
let t0 = Instant::now();
// First completion only arms the clock: cold start reads the floor.
q.record_decode_completion_at("http://d1:30001", 100_000, t0);
q.record_decode_completion_at("http://d1:30001", 100_000, None, t0);
assert_eq!(q.pool_free_rate("http://d1:30001"), POOL_FREE_RATE_MIN);
// Ten completions of 100k ctx at 1s spacing -> 100k tok/s samples.
for i in 1..=10u64 {
q.record_decode_completion_at(
"http://d1:30001",
100_000,
None,
t0 + Duration::from_secs(i),
);
}
@@ -2296,12 +2600,12 @@ mod tests {
let mut q = QueueModel::new();
let t0 = Instant::now();
// Enormous sample (100M tokens in 1ms) clamps to the 2M upper bound.
q.record_decode_completion_at("d1", 1_000, t0);
q.record_decode_completion_at("d1", 100_000_000, t0 + Duration::from_millis(1));
q.record_decode_completion_at("d1", 1_000, None, t0);
q.record_decode_completion_at("d1", 100_000_000, None, t0 + Duration::from_millis(1));
assert_eq!(q.pool_free_rate("d1"), POOL_FREE_RATE_MAX);
// Tiny sample on a fresh node clamps to the 1k floor.
q.record_decode_completion_at("d2", 1_000, t0);
q.record_decode_completion_at("d2", 1, t0 + Duration::from_secs(100));
q.record_decode_completion_at("d2", 1_000, None, t0);
q.record_decode_completion_at("d2", 1, None, t0 + Duration::from_secs(100));
assert_eq!(q.pool_free_rate("d2"), POOL_FREE_RATE_MIN);
}