From 6ade23696d86855cf69ddb1e2409ec76e141682c Mon Sep 17 00:00:00 2001 From: metamergebot Date: Tue, 13 Oct 2026 11:33:26 -0700 Subject: [PATCH] [Core] Keep request priority within the signed 64-bit range in radix caches (#43920) Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: qandrew --- .../sglang/srt/managers/tokenizer_manager.py | 7 +- python/sglang/srt/mem_cache/radix_cache.py | 2 +- .../srt/mem_cache/unified_radix_cache.py | 4 +- .../test_tokenizer_manager_rid_cleanup.py | 43 +++++++++++ .../unit/mem_cache/test_radix_cache_unit.py | 23 +++++- .../test_rust_tree_core_integration.py | 71 +++++++++++++++++++ 6 files changed, 145 insertions(+), 5 deletions(-) diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 023745c6b6..b0fb98fbf0 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -3902,13 +3902,18 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): return span_attrs def _set_default_priority(self, obj: Union[GenerateReqInput, EmbeddingReqInput]): - """Set the default priority value.""" + """Set the default priority and validate the signed 64-bit range.""" if ( self.enable_priority_scheduling and obj.priority is None and self.default_priority_value is not None ): obj.priority = self.default_priority_value + if obj.priority is not None and not -(1 << 63) <= obj.priority < (1 << 63): + raise ValueError( + "priority must be a signed 64-bit integer " + "between -9223372036854775808 and 9223372036854775807" + ) class ServerStatus(Enum): diff --git a/python/sglang/srt/mem_cache/radix_cache.py b/python/sglang/srt/mem_cache/radix_cache.py index 523d51757e..b034e4e9c2 100644 --- a/python/sglang/srt/mem_cache/radix_cache.py +++ b/python/sglang/srt/mem_cache/radix_cache.py @@ -497,7 +497,7 @@ class RadixCache(BasePrefixCache): key=prompt_key, value=values[: len(prompt_key)], inserted_len=len(prompt_key), - priority=priority + 1, + priority=min(priority + 1, torch.iinfo(torch.int64).max), ) ) diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index ef4d91fd0e..9e36908e8c 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -1170,7 +1170,9 @@ class UnifiedRadixCache(BasePrefixCache): value=values[: len(prompt_key)], prev_prefix_len=len(prompt_key), inserted_len=len(prompt_key), - priority=insert_params.priority + 1, + priority=min( + insert_params.priority + 1, torch.iinfo(torch.int64).max + ), ) ) diff --git a/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py b/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py index f5dfc10b58..19750e5c81 100644 --- a/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py +++ b/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py @@ -1350,5 +1350,48 @@ class TestLoRARequestLifecycle(CustomTestCase): asyncio.run(self._drive_parallel_request(fail_second_warmup=True)) +class TestPriorityRange(CustomTestCase): + """Request priorities reach the radix cache as signed 64-bit integers, so an + out-of-range value is rejected before dispatch instead of failing inside + the scheduler, and the boundary values themselves pass through unchanged.""" + + INT64_MIN = -(1 << 63) + INT64_MAX = (1 << 63) - 1 + + def test_int64_boundaries_are_preserved(self): + tm = _make_tokenizer_manager(self) + tm.enable_priority_scheduling = True + tm.default_priority_value = 0 + for priority in (self.INT64_MIN, self.INT64_MAX): + with self.subTest(priority=priority): + obj = GenerateReqInput(input_ids=[1], priority=priority) + tm._set_default_priority(obj) + self.assertEqual(obj.priority, priority) + + def test_out_of_range_priority_is_rejected_before_dispatch(self): + for enabled in (False, True): + for priority in (self.INT64_MIN - 1, self.INT64_MAX + 1): + for source in ("body", "default"): + if source == "default" and not enabled: + continue + with self.subTest( + enabled=enabled, priority=priority, source=source + ): + tm = _make_tm_for_generate(self) + del tm._set_default_priority # the real one is under test + tm.enable_priority_scheduling = enabled + tm.default_priority_value = ( + priority if source == "default" else 0 + ) + tm._init_req_state = Mock() + obj = GenerateReqInput( + input_ids=[1], + priority=priority if source == "body" else None, + ) + with self.assertRaisesRegex(ValueError, "signed 64-bit"): + asyncio.run(tm.generate_request(obj).__anext__()) + tm._init_req_state.assert_not_called() + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/test/registered/unit/mem_cache/test_radix_cache_unit.py b/test/registered/unit/mem_cache/test_radix_cache_unit.py index d9c5320afb..a1dbd5cb06 100644 --- a/test/registered/unit/mem_cache/test_radix_cache_unit.py +++ b/test/registered/unit/mem_cache/test_radix_cache_unit.py @@ -504,7 +504,9 @@ class TestRadixCache(CustomTestCase): cache.req_to_token_pool.req_to_token[0], tree_indices ) - def test_finished_request_splits_prompt_from_output_for_eviction(self): + def _checkpoint_split_request(self, priority): + """Checkpoint a finished request; returns (cache, prompt_ids, output_ids).""" + class ReqToTokenPool: def __init__(self, row): self.req_to_token = row.unsqueeze(0) @@ -532,7 +534,7 @@ class TestRadixCache(CustomTestCase): kv=ReqKvInfo(req_pool_idx=0, cache_protected_len=0), extra_key=None, cache_salt=None, - priority=0, + priority=priority, last_node=cache.root_node, lock=None, ) @@ -541,9 +543,17 @@ class TestRadixCache(CustomTestCase): cache.checkpoint(req, up_to=up_to) cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)]) cache.unlock(req.lock) + return cache, prompt_ids, output_ids + @staticmethod + def _split_nodes(cache): (prompt_node,) = cache.root_node.children.values() (output_node,) = prompt_node.children.values() + return prompt_node, output_node + + def test_finished_request_splits_prompt_from_output_for_eviction(self): + cache, prompt_ids, output_ids = self._checkpoint_split_request(priority=0) + prompt_node, output_node = self._split_nodes(cache) self.assertEqual(len(prompt_node.key), len(prompt_ids)) self.assertEqual(len(output_node.key), len(output_ids)) @@ -554,6 +564,15 @@ class TestRadixCache(CustomTestCase): ) self.assertEqual(match.device_prefix_len, len(prompt_ids)) + def test_prompt_priority_boost_saturates_at_int64_max(self): + """The prompt is kept one priority above the request; priorities are + signed 64-bit integers, so at INT64_MAX the boost saturates.""" + int64_max = torch.iinfo(torch.int64).max + cache, _, _ = self._checkpoint_split_request(priority=int64_max) + prompt_node, output_node = self._split_nodes(cache) + self.assertEqual(prompt_node.priority, int64_max) + self.assertEqual(output_node.priority, int64_max) + def test_kv_cache_events(self): """Test KV cache events functionality.""" test_cases = [ diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py index 026747a10d..0afe96cbf8 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py @@ -24,6 +24,8 @@ from sglang.srt.disaggregation.kv_events import ( namespaced_block_hash, ) from sglang.srt.environ import envs +from sglang.srt.managers.schedule_batch import Req +from sglang.srt.mem_cache.allocator.token import TokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import ( DecLockRefParams, EvictParams, @@ -34,6 +36,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( TreeLock, ) from sglang.srt.mem_cache.cache_init_params import CacheInitParams +from sglang.srt.mem_cache.common import release_kv_cache from sglang.srt.mem_cache.events import LoRANameTable from sglang.srt.mem_cache.evict_policy import TLRUStrategy from sglang.srt.mem_cache.hicache_storage import ( @@ -42,6 +45,7 @@ from sglang.srt.mem_cache.hicache_storage import ( PoolTransfer, PoolTransferResult, ) +from sglang.srt.mem_cache.memory_pool import ReqToTokenPool from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.rust_tree_core.adapter import ( RustUnifiedTreeCore, @@ -63,8 +67,10 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import ( TreePathNodeInfo, UnifiedTreeCoreInterface, ) +from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache from sglang.srt.mem_cache.utils import get_storage_hash_str, hash_str_to_int64 from sglang.srt.runtime_context import get_context +from sglang.srt.sampling.sampling_params import SamplingParams def _tree_core(backend="rust", **params_overrides) -> UnifiedTreeCoreInterface: @@ -5396,5 +5402,70 @@ def test_insert_reports_whether_it_reached_the_swa_branch_boundary(): assert result.swa_branch_inserted is expected, branching_seqlen +# ---- request priority at the i64 boundary ---- + + +class _RecordingUnifiedRadixCache(UnifiedRadixCache): + def __init__(self, params): + self.insert_priorities = [] + super().__init__(params) + + def insert(self, params): + self.insert_priorities.append(params.priority) + return super().insert(params) + + +@pytest.mark.parametrize("priority", [(1 << 63) - 2, (1 << 63) - 1]) +def test_finished_request_prompt_priority_saturates_at_int64_max(priority): + """Caching a finished request re-inserts its prompt one priority above the + request so eviction drops the output first. The core stores priority as a + signed 64-bit integer, so the boost must saturate at INT64_MAX instead of + overflowing the binding and failing the insert.""" + from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler + + # release_kv_cache reads the serving config off the published bags. + set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) + req_to_token_pool = ReqToTokenPool( + size=2, max_context_len=16, device="cpu", enable_memory_saver=False + ) + allocator = TokenToKVPoolAllocator( + size=16, dtype=torch.float32, device="cpu", kvcache=None, need_sort=False + ) + cache = _RecordingUnifiedRadixCache( + CacheInitParams( + disable=False, + req_to_token_pool=req_to_token_pool, + token_to_kv_pool_allocator=allocator, + page_size=1, + tree_components=(ComponentType.FULL,), + tree_core_backend="rust", + ) + ) + req = Req( + rid="priority", + origin_input_text=None, + origin_input_ids=array("q", [1, 2, 3]), + sampling_params=SamplingParams(max_new_tokens=2), + priority=priority, + ) + req.output_ids = array("q", [4, 5]) + req.finished_reason = "finished" + (req.kv.req_pool_idx,) = req_to_token_pool.alloc([req]) + req.lock = cache.lock(cache.root_node_handle()) + indices = allocator.alloc(5) + assert indices is not None + req_to_token_pool.write((req.kv.req_pool_idx, slice(0, 5)), indices) + req.kv.kv_committed_len = req.kv.kv_allocated_len = 5 + + release_kv_cache(req, cache, checkpoint=True) + + assert req.priority == priority + assert cache.insert_priorities == [priority, (1 << 63) - 1] + match = cache.match_prefix( + MatchPrefixParams(key=RadixKey(req.origin_input_ids + req.output_ids)) + ) + assert torch.equal(cache.path_device_indices(match.last_device_node), indices) + + if __name__ == "__main__": sys.exit(pytest.main([__file__]))