feat(dp): uncached_tokens load-balance method

DP balancing on total_tokens charges each request's whole prompt, but with
prefix caching only the unmatched part costs prefill compute. On a workload
with a 95% hit rate the prompt is ~95% noise: per request the uncached part
is a few percent of the prompt yet varies by more than an order of magnitude
(p50 1.9k / p99 59k tokens against a 95k prompt), so one cold request
outweighs many hot ones and per-rank prefill work drifts apart.

Add --load-balance-method uncached_tokens: DPBudget tracks the published
LoadSnapshot.num_waiting_uncached_tokens (prefix match minus accepted storage
hits) plus the rank's recent cache_hit_rate for a newly dispatched request.
This commit is contained in:
2026-11-11 00:03:55 +08:00
parent fef023b64f
commit 24b284d4e7
4 changed files with 1302 additions and 1190 deletions
@@ -98,6 +98,7 @@ class Parallel(msgspec.Struct):
"follow_bootstrap_room",
"total_requests",
"total_tokens",
"uncached_tokens",
],
),
] = "auto"
File diff suppressed because it is too large Load Diff
@@ -265,6 +265,27 @@ class TestDPBudgetUpdateBudget(CustomTestCase):
)
self.assertEqual(budget.total_tokens, [150, 80])
def test_maps_waiting_uncached_tokens_and_hit_rate(self):
budget = DPBudget(num_dp_ranks=2)
budget.update_budget(
[
_load(
dp_rank=0,
timestamp=1.0,
num_waiting_uncached_tokens=900,
cache_hit_rate=0.95,
),
_load(
dp_rank=1,
timestamp=1.0,
num_waiting_uncached_tokens=100,
cache_hit_rate=0.5,
),
]
)
self.assertEqual(budget.total_uncached_tokens, [900, 100])
self.assertEqual(budget.cache_hit_rate, [0.95, 0.5])
def test_partial_update_only_affects_reported_rank(self):
budget = DPBudget(num_dp_ranks=3)
budget.update_budget(
@@ -473,6 +494,43 @@ class TestTotalRequestsScheduler(CustomTestCase):
)
class TestUncachedTokensScheduler(CustomTestCase):
def test_dispatches_to_rank_with_least_uncached_work(self):
ctl = _make_controller(num_dp_ranks=4)
ctl.dp_budget.total_uncached_tokens = [100, 50, 900, 10]
ctl.total_uncached_tokens_scheduler(_req(input_ids=[0] * 1000))
ctl.workers[3].send_pyobj.assert_called_once()
for i in (0, 1, 2):
ctl.workers[i].send_pyobj.assert_not_called()
# No hit rate reported -> the whole prompt is charged as uncached.
self.assertEqual(ctl.dp_budget.total_uncached_tokens, [100, 50, 900, 1010])
self.assertEqual(ctl.dp_budget.total_requests, [0, 0, 0, 1])
def test_charge_scales_with_target_hit_rate(self):
ctl = _make_controller(num_dp_ranks=2)
ctl.dp_budget.total_uncached_tokens = [0, 0]
ctl.dp_budget.cache_hit_rate = [0.5, 0.0]
ctl.total_uncached_tokens_scheduler(_req(input_ids=[0] * 1000))
# rank 0 estimates 500 uncached tokens, rank 1 estimates 1000.
ctl.workers[0].send_pyobj.assert_called_once()
self.assertEqual(ctl.dp_budget.total_uncached_tokens, [500, 0])
def test_pending_backlog_beats_a_cheaper_candidate(self):
ctl = _make_controller(num_dp_ranks=2)
ctl.dp_budget.total_uncached_tokens = [500, 0]
ctl.dp_budget.cache_hit_rate = [0.0, 0.0]
ctl.total_uncached_tokens_scheduler(_req(input_ids=[0] * 100))
ctl.workers[1].send_pyobj.assert_called_once()
self.assertEqual(ctl.dp_budget.total_uncached_tokens, [500, 100])
def test_routed_dp_rank_bypasses_budget(self):
ctl = _make_controller(num_dp_ranks=4)
ctl.dp_budget.total_uncached_tokens = [5, 3, 1, 4]
ctl.total_uncached_tokens_scheduler(_req(routed_dp_rank=0))
ctl.workers[0].send_pyobj.assert_called_once()
self.assertEqual(ctl.dp_budget.total_uncached_tokens, [5, 3, 1, 4])
class TestStatusAwarenessInconsistency(CustomTestCase):
"""Document a divergence: ``round_robin_scheduler`` skips workers whose
``status`` is False, but ``total_requests_scheduler`` /
@@ -1,202 +1,203 @@
import unittest
from dataclasses import replace
from types import SimpleNamespace
from unittest.mock import Mock
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.managers.schedule_policy import (
CacheAwarePolicy,
SchedulePolicy,
)
from sglang.srt.managers.scheduler_components.load_inquirer import (
SchedulerLoadInquirer,
)
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
class TestSchedulePolicyWaitingQueueMatching(unittest.TestCase):
def make_policy(self, policy, supports_fast_match_prefix):
schedule_policy = object.__new__(SchedulePolicy)
schedule_policy.policy = policy
schedule_policy.tree_cache = SimpleNamespace(
supports_fast_match_prefix=lambda: supports_fast_match_prefix
)
return schedule_policy
def test_lpm_queue_limit_respects_fast_matching_capability(self):
policy = self.make_policy(CacheAwarePolicy.LPM, False)
self.assertTrue(policy.waiting_queue_prefix_matched([None] * 128))
self.assertFalse(policy.waiting_queue_prefix_matched([None] * 129))
policy.tree_cache = SimpleNamespace(supports_fast_match_prefix=lambda: True)
self.assertTrue(policy.waiting_queue_prefix_matched([None] * 129))
class TestSchedulerLoadInquirer(unittest.TestCase):
def make_inquirer(
self,
waiting_queue_prefix_matched=True,
storage_span=None,
disaggregation_mode=DisaggregationMode.NULL,
):
waiting_req = SimpleNamespace(
seqlen=100,
num_matched_prefix_tokens=20,
cache_request_handle=CacheRequestHandle("waiting", 0),
)
chunked_req = SimpleNamespace(seqlen=50, prefix_len=10)
return SchedulerLoadInquirer(
disaggregation_mode=disaggregation_mode,
server_args=None,
max_total_num_tokens=1000,
max_running_requests=10,
pool_stats_observer=None,
tp_worker=None,
token_to_kv_pool_allocator=None,
spec_algorithm=None,
get_running_batch=None,
get_waiting_queue=lambda: [waiting_req],
waiting_queue_prefix_matched=lambda: waiting_queue_prefix_matched,
get_chunked_req=lambda: chunked_req,
get_recent_cache_hit_rate=lambda: 0.75,
get_stats=None,
get_disagg_prefill_bootstrap_queue=None,
get_disagg_prefill_inflight_queue=None,
get_disagg_decode_prealloc_queue=None,
get_disagg_decode_transfer_queue=None,
get_spec_total_num_accept_tokens=None,
get_spec_total_num_forward_ct=None,
get_total_prefill_uncached_tokens=None,
get_total_prefill_busy_us=None,
get_decode_moment_totals=None,
get_waiting_storage_span=Mock(return_value=storage_span),
)
def test_waiting_tokens_are_estimated_when_prefix_matching_is_skipped(self):
inquirer = self.make_inquirer(waiting_queue_prefix_matched=False)
self.assertEqual(
SchedulerLoadInquirer.get_num_waiting_uncached_tokens(inquirer),
65,
)
def test_waiting_tokens_use_exact_match_when_prefix_matching_is_done(self):
inquirer = self.make_inquirer(waiting_queue_prefix_matched=True)
self.assertEqual(
SchedulerLoadInquirer.get_num_waiting_uncached_tokens(inquirer),
120,
)
def test_storage_exclusion_is_clipped_to_uncached_prompt(self):
for span, expected in (
(None, 120),
((0, 10), 120),
((0, 40), 100),
((20, 80), 60),
((60, 120), 80),
((20, 120), 40),
((20, 20), 120),
((100, 120), 120),
):
with self.subTest(span=span):
inquirer = self.make_inquirer(storage_span=span)
self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), expected)
inquirer.get_waiting_storage_span.assert_called_once_with(
CacheRequestHandle("waiting", 0)
)
def test_storage_exclusion_requires_per_request_matching(self):
for prefix_matched, expected in ((True, 60), (False, 65)):
with self.subTest(prefix_matched=prefix_matched):
inquirer = self.make_inquirer(
waiting_queue_prefix_matched=prefix_matched,
storage_span=(20, 80),
)
self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), expected)
if not prefix_matched:
inquirer.get_waiting_storage_span.assert_not_called()
def test_storage_exclusion_does_not_change_decode(self):
for prefix_matched in (False, True):
with self.subTest(prefix_matched=prefix_matched):
inquirer = self.make_inquirer(
waiting_queue_prefix_matched=prefix_matched,
storage_span=(20, 100),
disaggregation_mode=DisaggregationMode.DECODE,
)
self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 0)
inquirer.get_waiting_storage_span.assert_not_called()
def test_storage_exclusion_preserves_the_historical_hit_rate_estimate(self):
for span in (None, (20, 50), (20, 80), (80, 100), (0, 120)):
with self.subTest(span=span):
inquirer = self.make_inquirer(
waiting_queue_prefix_matched=False,
storage_span=span,
)
self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 65)
inquirer.get_waiting_storage_span.assert_not_called()
def test_historical_estimate_already_includes_storage_hits(self):
requests = [
SimpleNamespace(seqlen=100, cache_request_handle=CacheRequestHandle(rid, 0))
for rid in ("storage_hit", "cold")
]
get_storage_span = Mock(
side_effect=lambda handle: (0, 80) if handle.rid == "storage_hit" else None
)
inquirer = replace(
self.make_inquirer(waiting_queue_prefix_matched=False),
get_waiting_queue=lambda: requests,
get_chunked_req=lambda: None,
get_recent_cache_hit_rate=lambda: 0.4,
get_waiting_storage_span=get_storage_span,
)
# One 80-token storage hit and one cold prompt need 20 + 100 tokens.
self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 120)
get_storage_span.assert_not_called()
def test_storage_exclusion_without_a_provider_keeps_the_original_estimate(self):
inquirer = replace(
self.make_inquirer(),
get_waiting_storage_span=None,
)
self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 120)
def test_scheduler_wires_storage_provider(self):
from sglang.srt.managers.scheduler import Scheduler
req = self.make_inquirer().get_waiting_queue()[0]
get_storage_span = Mock(return_value=(20, 80))
scheduler = SimpleNamespace(
disaggregation_mode=DisaggregationMode.PREFILL,
server_args=None,
max_total_num_tokens=1000,
kv_shard_widening=1,
max_running_requests=10,
pool_stats_observer=None,
tp_worker=None,
token_to_kv_pool_allocator=None,
spec_algorithm=None,
tree_cache=SimpleNamespace(waiting_storage_span=get_storage_span),
policy=SimpleNamespace(waiting_queue_prefix_matched=lambda queue: True),
metrics_reporter=SimpleNamespace(recent_cache_hit_rate=0.75),
waiting_queue=[req],
chunked_req=None,
)
Scheduler.init_load_inquirer(scheduler)
inquirer = scheduler.load_inquirer
self.assertIs(inquirer.get_waiting_storage_span, get_storage_span)
self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 20)
get_storage_span.assert_called_once_with(req.cache_request_handle)
if __name__ == "__main__":
unittest.main()
import unittest
from dataclasses import replace
from types import SimpleNamespace
from unittest.mock import Mock
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.managers.schedule_policy import (
CacheAwarePolicy,
SchedulePolicy,
)
from sglang.srt.managers.scheduler_components.load_inquirer import (
SchedulerLoadInquirer,
)
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
class TestSchedulePolicyWaitingQueueMatching(unittest.TestCase):
def make_policy(self, policy, supports_fast_match_prefix):
schedule_policy = object.__new__(SchedulePolicy)
schedule_policy.policy = policy
schedule_policy.tree_cache = SimpleNamespace(
supports_fast_match_prefix=lambda: supports_fast_match_prefix
)
return schedule_policy
def test_lpm_queue_limit_respects_fast_matching_capability(self):
policy = self.make_policy(CacheAwarePolicy.LPM, False)
self.assertTrue(policy.waiting_queue_prefix_matched([None] * 128))
self.assertFalse(policy.waiting_queue_prefix_matched([None] * 129))
policy.tree_cache = SimpleNamespace(supports_fast_match_prefix=lambda: True)
self.assertTrue(policy.waiting_queue_prefix_matched([None] * 129))
class TestSchedulerLoadInquirer(unittest.TestCase):
def make_inquirer(
self,
waiting_queue_prefix_matched=True,
storage_span=None,
disaggregation_mode=DisaggregationMode.NULL,
):
waiting_req = SimpleNamespace(
seqlen=100,
num_matched_prefix_tokens=20,
cache_request_handle=CacheRequestHandle("waiting", 0),
)
chunked_req = SimpleNamespace(seqlen=50, prefix_len=10)
return SchedulerLoadInquirer(
disaggregation_mode=disaggregation_mode,
server_args=None,
max_total_num_tokens=1000,
max_running_requests=10,
pool_stats_observer=None,
tp_worker=None,
token_to_kv_pool_allocator=None,
spec_algorithm=None,
get_running_batch=None,
get_waiting_queue=lambda: [waiting_req],
waiting_queue_prefix_matched=lambda: waiting_queue_prefix_matched,
get_chunked_req=lambda: chunked_req,
get_grammar_queue=lambda: [],
get_recent_cache_hit_rate=lambda: 0.75,
get_stats=None,
get_disagg_prefill_bootstrap_queue=None,
get_disagg_prefill_inflight_queue=None,
get_disagg_decode_prealloc_queue=None,
get_disagg_decode_transfer_queue=None,
get_spec_total_num_accept_tokens=None,
get_spec_total_num_forward_ct=None,
get_total_prefill_uncached_tokens=None,
get_total_prefill_busy_us=None,
get_decode_moment_totals=None,
get_waiting_storage_span=Mock(return_value=storage_span),
)
def test_waiting_tokens_are_estimated_when_prefix_matching_is_skipped(self):
inquirer = self.make_inquirer(waiting_queue_prefix_matched=False)
self.assertEqual(
SchedulerLoadInquirer.get_num_waiting_uncached_tokens(inquirer),
65,
)
def test_waiting_tokens_use_exact_match_when_prefix_matching_is_done(self):
inquirer = self.make_inquirer(waiting_queue_prefix_matched=True)
self.assertEqual(
SchedulerLoadInquirer.get_num_waiting_uncached_tokens(inquirer),
120,
)
def test_storage_exclusion_is_clipped_to_uncached_prompt(self):
for span, expected in (
(None, 120),
((0, 10), 120),
((0, 40), 100),
((20, 80), 60),
((60, 120), 80),
((20, 120), 40),
((20, 20), 120),
((100, 120), 120),
):
with self.subTest(span=span):
inquirer = self.make_inquirer(storage_span=span)
self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), expected)
inquirer.get_waiting_storage_span.assert_called_once_with(
CacheRequestHandle("waiting", 0)
)
def test_storage_exclusion_requires_per_request_matching(self):
for prefix_matched, expected in ((True, 60), (False, 65)):
with self.subTest(prefix_matched=prefix_matched):
inquirer = self.make_inquirer(
waiting_queue_prefix_matched=prefix_matched,
storage_span=(20, 80),
)
self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), expected)
if not prefix_matched:
inquirer.get_waiting_storage_span.assert_not_called()
def test_storage_exclusion_does_not_change_decode(self):
for prefix_matched in (False, True):
with self.subTest(prefix_matched=prefix_matched):
inquirer = self.make_inquirer(
waiting_queue_prefix_matched=prefix_matched,
storage_span=(20, 100),
disaggregation_mode=DisaggregationMode.DECODE,
)
self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 0)
inquirer.get_waiting_storage_span.assert_not_called()
def test_storage_exclusion_preserves_the_historical_hit_rate_estimate(self):
for span in (None, (20, 50), (20, 80), (80, 100), (0, 120)):
with self.subTest(span=span):
inquirer = self.make_inquirer(
waiting_queue_prefix_matched=False,
storage_span=span,
)
self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 65)
inquirer.get_waiting_storage_span.assert_not_called()
def test_historical_estimate_already_includes_storage_hits(self):
requests = [
SimpleNamespace(seqlen=100, cache_request_handle=CacheRequestHandle(rid, 0))
for rid in ("storage_hit", "cold")
]
get_storage_span = Mock(
side_effect=lambda handle: (0, 80) if handle.rid == "storage_hit" else None
)
inquirer = replace(
self.make_inquirer(waiting_queue_prefix_matched=False),
get_waiting_queue=lambda: requests,
get_chunked_req=lambda: None,
get_recent_cache_hit_rate=lambda: 0.4,
get_waiting_storage_span=get_storage_span,
)
# One 80-token storage hit and one cold prompt need 20 + 100 tokens.
self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 120)
get_storage_span.assert_not_called()
def test_storage_exclusion_without_a_provider_keeps_the_original_estimate(self):
inquirer = replace(
self.make_inquirer(),
get_waiting_storage_span=None,
)
self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 120)
def test_scheduler_wires_storage_provider(self):
from sglang.srt.managers.scheduler import Scheduler
req = self.make_inquirer().get_waiting_queue()[0]
get_storage_span = Mock(return_value=(20, 80))
scheduler = SimpleNamespace(
disaggregation_mode=DisaggregationMode.PREFILL,
server_args=None,
max_total_num_tokens=1000,
kv_shard_widening=1,
max_running_requests=10,
pool_stats_observer=None,
tp_worker=None,
token_to_kv_pool_allocator=None,
spec_algorithm=None,
tree_cache=SimpleNamespace(waiting_storage_span=get_storage_span),
policy=SimpleNamespace(waiting_queue_prefix_matched=lambda queue: True),
metrics_reporter=SimpleNamespace(recent_cache_hit_rate=0.75),
waiting_queue=[req],
chunked_req=None,
)
Scheduler.init_load_inquirer(scheduler)
inquirer = scheduler.load_inquirer
self.assertIs(inquirer.get_waiting_storage_span, get_storage_span)
self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 20)
get_storage_span.assert_called_once_with(req.cache_request_handle)
if __name__ == "__main__":
unittest.main()