diff --git a/python/sglang/srt/managers/data_parallel_controller.py b/python/sglang/srt/managers/data_parallel_controller.py index 2f9eff34d3..3be77f7091 100644 --- a/python/sglang/srt/managers/data_parallel_controller.py +++ b/python/sglang/srt/managers/data_parallel_controller.py @@ -158,23 +158,23 @@ class DPBudget: key=lambda i: (self.total_tokens[i], self.total_requests[i]), ) elif method == LoadBalanceMethod.UNCACHED_TOKENS: - # Charge every candidate its own estimated uncached share: the same - # prompt costs less on a rank whose cache already holds more of it. - # The exact prefix match is only known inside the scheduler, so the - # rank's recent cache hit rate stands in for it. - estimated_uncached = [ - int(num_input_tokens * (1.0 - self.cache_hit_rate[i])) - for i in range(self.num_dp_ranks) - ] + # Rank the candidates by the uncached prefill work each already has + # queued (published per rank, exact prefix match). The arriving + # request is charged the fleet-average uncached size: deriving it + # per rank from that rank's own recent hit rate is self-reinforcing + # (a rank that slips behind sees its hit rate drop, is charged more, + # and falls further behind). + miss_rate = 1.0 - sum(self.cache_hit_rate) / self.num_dp_ranks + estimated_uncached = int(num_input_tokens * miss_rate) target_rank = min( range(self.num_dp_ranks), key=lambda i: ( - self.total_uncached_tokens[i] + estimated_uncached[i], + self.total_uncached_tokens[i], self.total_requests[i], ), ) self.total_requests[target_rank] += 1 - self.total_uncached_tokens[target_rank] += estimated_uncached[target_rank] + self.total_uncached_tokens[target_rank] += estimated_uncached return target_rank else: return None diff --git a/test/registered/unit/managers/test_data_parallel_controller.py b/test/registered/unit/managers/test_data_parallel_controller.py index 5dffb9ba29..0ae7cf5c23 100644 --- a/test/registered/unit/managers/test_data_parallel_controller.py +++ b/test/registered/unit/managers/test_data_parallel_controller.py @@ -506,12 +506,13 @@ class TestUncachedTokensScheduler(CustomTestCase): self.assertEqual(ctl.dp_budget.total_uncached_tokens, [100, 50, 900, 1010]) self.assertEqual(ctl.dp_budget.total_requests, [0, 0, 0, 1]) - def test_charge_scales_with_target_hit_rate(self): + def test_charge_uses_fleet_average_miss_rate(self): ctl = _make_controller(num_dp_ranks=2) ctl.dp_budget.total_uncached_tokens = [0, 0] - ctl.dp_budget.cache_hit_rate = [0.5, 0.0] + # Fleet-average hit rate 0.5 -> every candidate charges 500 tokens, so + # the pick is decided by queued work alone (no per-rank feedback). + ctl.dp_budget.cache_hit_rate = [0.9, 0.1] ctl.total_uncached_tokens_scheduler(_req(input_ids=[0] * 1000)) - # rank 0 estimates 500 uncached tokens, rank 1 estimates 1000. ctl.workers[0].send_pyobj.assert_called_once() self.assertEqual(ctl.dp_budget.total_uncached_tokens, [500, 0])