fix(dp): rank uncached candidates by published backlog only

Deriving each candidate's uncached estimate from its own recent hit rate is
self-reinforcing: a rank that falls behind sees its hit rate drop (fewer
chances to hit), gets charged more for every new request, and falls further
behind. Measured on c580: one rank ended 16% below the others, 30 s cross-rank
dispersion max 952x, and 9.6% lower rps than the total_tokens arm.

Charge the fleet-average uncached size instead and rank by the published
num_waiting_uncached_tokens, which already carries each rank's exact prefix
match.
This commit is contained in:
2026-11-11 02:06:59 +08:00
parent 24b284d4e7
commit c401c614fa
2 changed files with 14 additions and 13 deletions
@@ -158,23 +158,23 @@ class DPBudget:
key=lambda i: (self.total_tokens[i], self.total_requests[i]),
)
elif method == LoadBalanceMethod.UNCACHED_TOKENS:
# Charge every candidate its own estimated uncached share: the same
# prompt costs less on a rank whose cache already holds more of it.
# The exact prefix match is only known inside the scheduler, so the
# rank's recent cache hit rate stands in for it.
estimated_uncached = [
int(num_input_tokens * (1.0 - self.cache_hit_rate[i]))
for i in range(self.num_dp_ranks)
]
# Rank the candidates by the uncached prefill work each already has
# queued (published per rank, exact prefix match). The arriving
# request is charged the fleet-average uncached size: deriving it
# per rank from that rank's own recent hit rate is self-reinforcing
# (a rank that slips behind sees its hit rate drop, is charged more,
# and falls further behind).
miss_rate = 1.0 - sum(self.cache_hit_rate) / self.num_dp_ranks
estimated_uncached = int(num_input_tokens * miss_rate)
target_rank = min(
range(self.num_dp_ranks),
key=lambda i: (
self.total_uncached_tokens[i] + estimated_uncached[i],
self.total_uncached_tokens[i],
self.total_requests[i],
),
)
self.total_requests[target_rank] += 1
self.total_uncached_tokens[target_rank] += estimated_uncached[target_rank]
self.total_uncached_tokens[target_rank] += estimated_uncached
return target_rank
else:
return None
@@ -506,12 +506,13 @@ class TestUncachedTokensScheduler(CustomTestCase):
self.assertEqual(ctl.dp_budget.total_uncached_tokens, [100, 50, 900, 1010])
self.assertEqual(ctl.dp_budget.total_requests, [0, 0, 0, 1])
def test_charge_scales_with_target_hit_rate(self):
def test_charge_uses_fleet_average_miss_rate(self):
ctl = _make_controller(num_dp_ranks=2)
ctl.dp_budget.total_uncached_tokens = [0, 0]
ctl.dp_budget.cache_hit_rate = [0.5, 0.0]
# Fleet-average hit rate 0.5 -> every candidate charges 500 tokens, so
# the pick is decided by queued work alone (no per-rank feedback).
ctl.dp_budget.cache_hit_rate = [0.9, 0.1]
ctl.total_uncached_tokens_scheduler(_req(input_ids=[0] * 1000))
# rank 0 estimates 500 uncached tokens, rank 1 estimates 1000.
ctl.workers[0].send_pyobj.assert_called_once()
self.assertEqual(ctl.dp_budget.total_uncached_tokens, [500, 0])