diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index a921e11363..69c189b1bc 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -2201,7 +2201,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): kv_loc, ) - # Truncate fill_len to kv_committed_len so insert_req only + # Truncate fill_len to kv_committed_len so checkpoint only # inserts committed KV into the radix tree. The last output token # hasn't had KV committed yet (output_ids is 1 ahead). req.full_untruncated_fill_ids = req.origin_input_ids + req.output_ids @@ -3125,7 +3125,7 @@ class SchedulerDisaggregationDecodeMixin: else: tree_cache = self.tree_cache req.init_next_round_input(tree_cache) - # Truncate fill_len to kv_committed_len so insert_req + # Truncate fill_len to kv_committed_len so checkpoint # only sees committed KV (full array includes one uncommitted # token because init_next_round_input rebuilt it as full). if req.kv.kv_committed_len is not None: diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index da97552647..3296b3af29 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1220,7 +1220,7 @@ class Req(ReqDllmMixin): # Refreshed at every sharded alloc — read through last_node, or drawn # least-full for a new chain — and consumed by the radix insert to # stamp new tree nodes. Allocation itself must NOT read it back when - # a tree node is available (the insert_req dedup rebind + # a tree node is available (the checkpoint dedup rebind # would make it stale); the only allocation-time reader is the # ChunkCache fallback, which has no tree nodes and no rebind. self.kv_rotation_base: Optional[int] = None diff --git a/python/sglang/srt/mem_cache/allocation.py b/python/sglang/srt/mem_cache/allocation.py index a17695bf86..542f8943f7 100644 --- a/python/sglang/srt/mem_cache/allocation.py +++ b/python/sglang/srt/mem_cache/allocation.py @@ -255,7 +255,7 @@ def _kv_shard_rotation_bases( The owner class of position-page P is ``(b_i + P) % shard_size``. Rules: - Read through ``req.last_node`` at alloc time, never a value cached on - the request: a checkpoint ``insert_req`` can rebind a chunked request onto + the request: a ``checkpoint`` can rebind a chunked request onto another chain's canonical locs between chunks, changing the base. The read goes through ``tree_cache.rotation_base_of`` because the node handle is tree-specific (a NodeId on the unified tree). diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index 56fe67fb1b..7bf22f64e2 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -459,7 +459,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait): return None @abstractmethod - def insert_req(self, req: Req, *, up_to: int, **kwargs): + def checkpoint(self, req: Req, *, up_to: int, **kwargs): """Insert the request's KV up to row position ``up_to`` into the tree, repoint the row onto the tree's copy, re-anchor ``req.last_node`` on the node the insert ended on and advance ``cache_protected_len``. diff --git a/python/sglang/srt/mem_cache/chunk_cache.py b/python/sglang/srt/mem_cache/chunk_cache.py index 4b60dbdb87..ae0e50f5d1 100644 --- a/python/sglang/srt/mem_cache/chunk_cache.py +++ b/python/sglang/srt/mem_cache/chunk_cache.py @@ -76,7 +76,7 @@ class ChunkCache(BasePrefixCache): # ChunkCache does not support prefix caching, so insert is a no-op return InsertResult(prefix_len=0) - def insert_req(self, req: Req, *, up_to: int): + def checkpoint(self, req: Req, *, up_to: int): kv_indices = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :up_to] # `req.prefix_indices` will be used in `PrefillAdder::add_chunked_req` later req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True) diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 7950c66e48..c6d048b974 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -178,7 +178,7 @@ def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None: if req.skip_radix_cache_insert: return - tree_cache.insert_req(req, up_to=req.extend_range.end) + tree_cache.checkpoint(req, up_to=req.extend_range.end) def evict_from_tree_cache( @@ -325,7 +325,7 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr ) # The fill-id array lags output_ids until the next prepare_for_decode. req.refresh_fill_ids() - tree_cache.insert_req(req, up_to=owned_kv_len) + tree_cache.checkpoint(req, up_to=owned_kv_len) # The protected prefix is not this req's to free. tree_cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)]) tree_cache.unpin(req) diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index ed9b19f96f..fd8d4c8b74 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -1051,8 +1051,8 @@ class MambaPool: ReplaySSM invariant: the SOURCE must be a fully-flushed checkpoint (``write_pos[src] == 0``). Only ``temporal`` is copied, not the ring, so an un-flushed source would drop its last ``write_pos`` updates. Callers - comply: COW copies radix checkpoints; a checkpoint ``insert_req`` copies an - active slot only during prefill (ring empty); ``insert_req`` + comply: COW copies radix checkpoints; a ``checkpoint`` copies an + active slot only during prefill (ring empty); ``checkpoint`` caps the donate to the last flush boundary. The dst cursor is reset to 0 (the copied checkpoint has no pending ring entries). """ diff --git a/python/sglang/srt/mem_cache/radix_cache.py b/python/sglang/srt/mem_cache/radix_cache.py index 8a7530c466..1df2aa6c28 100644 --- a/python/sglang/srt/mem_cache/radix_cache.py +++ b/python/sglang/srt/mem_cache/radix_cache.py @@ -494,7 +494,7 @@ class RadixCache(BasePrefixCache): ) return radix_key, kv_indices, result.prefix_len - def insert_req(self, req: Req, *, up_to: int): + def checkpoint(self, req: Req, *, up_to: int): if self.disable: return token_ids = req.full_untruncated_fill_ids[:up_to] @@ -524,7 +524,7 @@ class RadixCache(BasePrefixCache): # With page_size > 1 the partial page sits in req.prefix_indices but not # in the tree; cache_protected_len marks the tree-owned part so the next - # insert_req or release_kv_cache frees the rest. + # checkpoint or release_kv_cache frees the rest. req.kv.cache_protected_len = len(new_indices) self.dec_lock_ref(req.last_node) diff --git a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py index 5c1dc179ed..fbae1235bf 100644 --- a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py +++ b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py @@ -382,16 +382,16 @@ class FlexKVRadixCache(RadixCache): return fetched_slots, new_node # ------------------------------------------------------------------ - # insert_req (STORE) + # checkpoint (STORE) # ------------------------------------------------------------------ def on_release(self, req: Req, *, inserted: bool) -> None: if not inserted: self._load_markers.pop(req.cache_request_handle, None) - def insert_req(self, req: Req, *, up_to: int) -> None: # type: ignore[override] - """Base insert_req; a finished request also fires an async FlexKV store.""" - super().insert_req(req, up_to=up_to) + def checkpoint(self, req: Req, *, up_to: int) -> None: # type: ignore[override] + """Base checkpoint; a finished request also fires an async FlexKV store.""" + super().checkpoint(req, up_to=up_to) if not req.finished(): return diff --git a/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py b/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py index 5d028d6169..57976d5772 100644 --- a/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py @@ -253,9 +253,9 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache): if not inserted: self.release_aborted_request(req.cache_request_handle) - def insert_req(self, req: Req, *, up_to: int, **kwargs) -> None: + def checkpoint(self, req: Req, *, up_to: int, **kwargs) -> None: self._publish_external_loaded_prefix(req, token_ids_len=up_to) - super().insert_req(req, up_to=up_to, **kwargs) + super().checkpoint(req, up_to=up_to, **kwargs) self._retire_loaded_flow(req.rid) self._submit_store(req, req.full_untruncated_fill_ids[:up_to]) if req.finished(): diff --git a/python/sglang/srt/mem_cache/unified_cache/components/README.md b/python/sglang/srt/mem_cache/unified_cache/components/README.md index 35487cd219..d5bc53e642 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/README.md +++ b/python/sglang/srt/mem_cache/unified_cache/components/README.md @@ -228,7 +228,7 @@ receipt proves were taken. The eventual full release must pass --- -### `insert_req(req: Req, *, up_to: int)` +### `checkpoint(req: Req, *, up_to: int)` Insert a request's KV into the tree: at every checkpoint while it runs, and once more when it finishes. diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index b999571adc..0b9c7898be 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -1003,8 +1003,8 @@ class UnifiedRadixCache(BasePrefixCache): comp.cleanup_after_caching_req(req, is_finished=True) @rank_consensus(same_params=["req.rid", "up_to"]) - def insert_req(self, req: Req, *, up_to: int, **kwargs) -> None: - if self.session.try_insert_req(req, up_to=up_to, **kwargs): + def checkpoint(self, req: Req, *, up_to: int, **kwargs) -> None: + if self.session.try_checkpoint(req, up_to=up_to, **kwargs): return # A finished request hands its component state (mamba) to the tree # instead of forking it, and the tree frees what the request still held. diff --git a/python/sglang/srt/session/streaming_session.py b/python/sglang/srt/session/streaming_session.py index 2064d9931d..6049849902 100644 --- a/python/sglang/srt/session/streaming_session.py +++ b/python/sglang/srt/session/streaming_session.py @@ -325,7 +325,7 @@ class StreamingSession(BasePrefixCache): return True - def try_insert_req(self, req: Req, *, up_to: int, **kwargs) -> bool: + def try_checkpoint(self, req: Req, *, up_to: int, **kwargs) -> bool: """A first turn checkpoints into the tree like any request (its prompt prefix is tree-owned and the slot inherits that lock); later turns run on the slot's KV, so only the chunk cursor is kept.""" @@ -349,10 +349,10 @@ class StreamingSession(BasePrefixCache): def on_release(self, req: Req, *, inserted: bool) -> None: self.inner.on_release(req, inserted=inserted) - def insert_req(self, req: Req, **kwargs): - if self.try_insert_req(req, **kwargs): + def checkpoint(self, req: Req, **kwargs): + if self.try_checkpoint(req, **kwargs): return - self.inner.insert_req(req, **kwargs) + self.inner.checkpoint(req, **kwargs) def unpin(self, req: Req) -> None: self.inner.unpin(req) diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 6f80415863..3d60e51af9 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -4727,7 +4727,7 @@ def get_extend_input_len_swa_limit( sliding_window_size: int, chunked_prefill_size: int, page_size: int ) -> int: # 1. a factor of 2x is because each prefill contains chunked_prefill_size tokens, - # and between prefills, we run the tree cache's insert_req(), + # and between prefills, we run the tree cache's checkpoint(), # so we unlock the previously locked nodes. # 2. max is to handle the case that chunked_prefill_size is larger than sliding_window_size. # in that case, each prefill contains chunked_prefill_size tokens, diff --git a/python/sglang/test/mem_cache_utils.py b/python/sglang/test/mem_cache_utils.py index ce155a7413..683f4a4885 100644 --- a/python/sglang/test/mem_cache_utils.py +++ b/python/sglang/test/mem_cache_utils.py @@ -5,10 +5,10 @@ from sglang.srt.managers.schedule_batch import FINISH_LENGTH def finish_req(cache, req, up_to): """What release_kv_cache does for a request that keeps its KV: mark it - finished, insert_req, free the rest, drop the lock.""" + finished, checkpoint, free the rest, drop the lock.""" if getattr(req, "finished_reason", None) is None: req.finished_reason = FINISH_LENGTH(length=0) req.refresh_fill_ids() - cache.insert_req(req, up_to=up_to) + cache.checkpoint(req, up_to=up_to) cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)]) cache.unpin(req) diff --git a/test/manual/chunked_prefill/test_scripted_regression.py b/test/manual/chunked_prefill/test_scripted_regression.py index 259ab29c08..901a0afadc 100644 --- a/test/manual/chunked_prefill/test_scripted_regression.py +++ b/test/manual/chunked_prefill/test_scripted_regression.py @@ -517,7 +517,7 @@ class TestRegressionGptOss(ScriptedTestCase): assert committed > 0 assert len(r.req.prefix_indices) <= committed, ( - f"insert_req over-read past kv_committed_len: " + f"checkpoint over-read past kv_committed_len: " f"prefix_indices_len={len(r.req.prefix_indices)}, " f"kv_committed_len={committed}" ) diff --git a/test/manual/dsv4/test_dsv4_swa_radix_retract.py b/test/manual/dsv4/test_dsv4_swa_radix_retract.py index 36ca376b6c..24a9a64b54 100644 --- a/test/manual/dsv4/test_dsv4_swa_radix_retract.py +++ b/test/manual/dsv4/test_dsv4_swa_radix_retract.py @@ -1,6 +1,6 @@ """DSV4 stress test for SWA radix cache + tombstone + retract interaction. -Regression test for the former SWA `insert_req` assertion: +Regression test for the former SWA `checkpoint` assertion: assert old_prefix_len <= len(new_indices) The unified cache reads `req.kv.cache_protected_len` and tolerates page_size - 1 of alignment slack, so this reproduces the historical trip conditions rather diff --git a/test/registered/kv_canary/test_self_e2e_pd_perturb.py b/test/registered/kv_canary/test_self_e2e_pd_perturb.py index 73f61ace69..ea149c08fc 100644 --- a/test/registered/kv_canary/test_self_e2e_pd_perturb.py +++ b/test/registered/kv_canary/test_self_e2e_pd_perturb.py @@ -42,7 +42,7 @@ class _PDPerturbBase(CanaryPDFixture): # distinct_prompts is required for the violation to surface reliably: # with a shared prompt, P-side radix caching rewrites each request's # req_to_token row to the first-inserted (canonical) copy's slots in - # insert_req BEFORE send_kv_chunk snapshots the indices, so a + # checkpoint BEFORE send_kv_chunk snapshots the indices, so a # flip on a deduped duplicate slot is freed untransferred and never # re-verified on either side. self.send_parallel_short_requests(n=4, distinct_prompts=True) diff --git a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py index 2dd769ee03..fe81e4c4a3 100644 --- a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py +++ b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py @@ -119,7 +119,7 @@ def _scheduler_for_get_next_batch(*, tree_cache, chunked_req) -> Scheduler: class TestStashGatePreservesPrefixIndices(CustomTestCase): - """Consumer side: real ChunkCache.insert_req mutates + """Consumer side: real ChunkCache.checkpoint mutates req.prefix_indices iff stash actually runs, so prefix_indices content is the bug-detection signal. The stash gate is content-based: `fill_len > len(prefix_indices)` means there is freshly computed KV to diff --git a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py index dcb46201ee..a3b5792b02 100644 --- a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py +++ b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py @@ -5,10 +5,10 @@ Verifies that inc_lock_ref / dec_lock_ref are balanced across the four transfer scenarios identified in PR #19746: 1. Incremental transfer & success (prefix match > 0) - inc_lock_ref(pop_preallocated) -> dec+inc(insert_req) -> dec(release_kv_cache) + inc_lock_ref(pop_preallocated) -> dec+inc(checkpoint) -> dec(release_kv_cache) 2. Full transfer & success (prefix match == 0, full KV transferred) - inc_lock_ref(get_new_prebuilt_batch) -> dec+inc(insert_req) -> dec(release_kv_cache) + inc_lock_ref(get_new_prebuilt_batch) -> dec+inc(checkpoint) -> dec(release_kv_cache) 3. Incremental transfer & failure (prefix match > 0, transfer fails) inc_lock_ref(pop_preallocated) -> dec(unpin via release_kv_cache is_insert=False) @@ -60,7 +60,7 @@ from sglang.test.test_utils import CustomTestCase def _make_cache_with_pools(page_size=1): - """Create a RadixCache with mock pools sufficient for insert_req / insert_req.""" + """Create a RadixCache with mock pools sufficient for checkpoint and release_kv_cache.""" mock_allocator = MagicMock() mock_allocator.device = torch.device("cpu") mock_allocator.page_size = page_size @@ -84,7 +84,7 @@ def _make_cache_with_pools(page_size=1): class MockReq: - """Minimal mock Req with fields needed by insert_req / release_kv_cache.""" + """Minimal mock Req with fields needed by checkpoint / release_kv_cache.""" def __init__(self, fill_ids, req_pool_idx=0, cache_protected_len=0, last_node=None): self.full_untruncated_fill_ids = array("q", fill_ids) @@ -224,7 +224,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): """Scenario 1: prefix match > 0, transfer succeeds. Flow: inc_lock_ref(pop_preallocated) - -> dec_lock_ref + inc_lock_ref(insert_req) + -> dec_lock_ref + inc_lock_ref(checkpoint) -> dec_lock_ref(cache_finished_req) """ cache, req_to_token = _make_cache_with_pools() @@ -256,8 +256,8 @@ class TestDecodeLockRefScenarios(CustomTestCase): last_node=matched_node, ) - # Step 2: insert_req (dec old lock, inc new lock) - cache.insert_req(req, up_to=req.extend_range.end) + # Step 2: checkpoint (dec old lock, inc new lock) + cache.checkpoint(req, up_to=req.extend_range.end) # Step 3: release_kv_cache (insert, free the rest, dec lock) req.finished_reason = "finished" @@ -274,7 +274,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): """Scenario 2: no prefix match, full KV transferred, succeeds. Flow: inc_lock_ref(root, via init_next_round_input/get_new_prebuilt_batch) - -> dec_lock_ref + inc_lock_ref(insert_req) + -> dec_lock_ref + inc_lock_ref(checkpoint) -> dec_lock_ref(cache_finished_req) """ cache, req_to_token = _make_cache_with_pools() @@ -306,8 +306,8 @@ class TestDecodeLockRefScenarios(CustomTestCase): last_node=matched_node, ) - # Step 2: insert_req (dec root=no-op, inc new leaf) - cache.insert_req(req, up_to=req.extend_range.end) + # Step 2: checkpoint (dec root=no-op, inc new leaf) + cache.checkpoint(req, up_to=req.extend_range.end) # Step 3: cache_finished_req (dec leaf) req.finished_reason = "finished" @@ -662,7 +662,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): SchedulerDisaggregationDecodeMixin._get_new_prebuilt_batch( scheduler, SimpleNamespace(batch_size=lambda: 0) ) - cache.insert_req(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_range.end) self.assertEqual(shared.lock_ref, 2) self.assertEqual(req_to_token[0, :3].tolist(), prefix_vals) @@ -698,7 +698,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): last_node=matched_node, ) - cache.insert_req(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_range.end) req.finished_reason = "finished" release_kv_cache(req, cache) diff --git a/test/registered/unit/mem_cache/test_decode_retraction_backup.py b/test/registered/unit/mem_cache/test_decode_retraction_backup.py index 9d588be7c6..8732bde51a 100644 --- a/test/registered/unit/mem_cache/test_decode_retraction_backup.py +++ b/test/registered/unit/mem_cache/test_decode_retraction_backup.py @@ -591,7 +591,7 @@ class TestDecodeRetractionBackup(CustomTestCase): cached_indices = queue._pre_alloc(cached) self._seed_pool(env.target_pool, cached_indices, base=500) cached_values = self._snapshot_pool(env.target_pool, cached_indices) - cache.insert_req(cached, up_to=cached.extend_range.end) + cache.checkpoint(cached, up_to=cached.extend_range.end) pressure = env.allocator.alloc(self.pool_size // 2 - 4) req = make_req("receiving", 8) @@ -620,7 +620,7 @@ class TestDecodeRetractionBackup(CustomTestCase): restore_kv_cache(req, cache, env.req_to_token_pool, env.allocator, "host_pool") self.assertEqual(cache.host_pool_group.available_size(), host_free_before) free_before_insert = env.allocator.available_size() - cache.insert_req(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_range.end) row = env.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :8] self.assertTrue(torch.equal(row[:4], cached_indices)) diff --git a/test/registered/unit/mem_cache/test_free_kv_row_coalesce.py b/test/registered/unit/mem_cache/test_free_kv_row_coalesce.py index 687f234983..b438b40f28 100644 --- a/test/registered/unit/mem_cache/test_free_kv_row_coalesce.py +++ b/test/registered/unit/mem_cache/test_free_kv_row_coalesce.py @@ -39,7 +39,7 @@ class _RowCache(BasePrefixCache): def match_prefix(self, params): raise NotImplementedError - def insert_req(self, req, **kwargs): + def checkpoint(self, req, **kwargs): raise NotImplementedError def evict(self, params): diff --git a/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py b/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py index e65cfb2729..9e555e6c19 100644 --- a/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py +++ b/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py @@ -1,7 +1,7 @@ """CPU-only unit tests for the mamba pool ratio vs the prefill->decode peak. Pins the sizing invariant behind MAMBA_CACHE_SIZE_MAX_RUNNING_REQUESTS_RATIO: -at the first insert_req, a request still holds its admission-locked +at the first checkpoint, a request still holds its admission-locked matched-prefix mamba (protected) plus its own COW slot, and then allocates a donated slot. With N distinct-prefix requests that peak is N own + N locked + 1 donated. An effective ratio of 2 (pool = 2N) leaves no evictable victim and diff --git a/test/registered/unit/mem_cache/test_page_interleave_shard.py b/test/registered/unit/mem_cache/test_page_interleave_shard.py index 80997876d3..4216ee6fee 100644 --- a/test/registered/unit/mem_cache/test_page_interleave_shard.py +++ b/test/registered/unit/mem_cache/test_page_interleave_shard.py @@ -775,7 +775,7 @@ class TestShardedCoreGate(_TreeCoreBackendCase): class _GraftReq: - """Minimal Req stand-in for insert_req.""" + """Minimal Req stand-in for checkpoint.""" def __init__(self, fill_ids, req_pool_idx=0): self.fill_ids = list(fill_ids) @@ -917,7 +917,7 @@ class TestRotationGraftDecline(_TreeCoreBackendCase): req = _GraftReq(list(range(8)) + [90, 91, 92, 93]) req.kv_rotation_base = 3 own_locs = self._own_row(tree, req, 12) - tree.insert_req(req, up_to=len(req.fill_ids)) + tree.checkpoint(req, up_to=len(req.fill_ids)) # No dedup free, no rebind: the request keeps its own locs whole. self.assertEqual([t.tolist() for t in freed], []) self.assertTrue(torch.equal(req.prefix_indices, own_locs)) @@ -926,7 +926,7 @@ class TestRotationGraftDecline(_TreeCoreBackendCase): torch.equal(tree.req_to_token_pool.req_to_token[0, :12], own_locs) ) - def test_insert_req_decline_frees_duplicates_and_suffix(self): + def test_checkpoint_decline_frees_duplicates_and_suffix(self): tree, freed = self._tree_with_spy() self._seed_chain(tree, list(range(8)), base=1) req = _GraftReq(list(range(8)) + [90, 91, 92, 93]) @@ -940,7 +940,7 @@ class TestRotationGraftDecline(_TreeCoreBackendCase): self.assertEqual(set(released.tolist()), set(own_locs.tolist())) self.assertEqual(_match_len(tree, req.fill_ids), 8) - def test_insert_req_same_base_keeps_the_tail_cached(self): + def test_checkpoint_same_base_keeps_the_tail_cached(self): """Control for the decline test: with an agreeing base the tail is grafted and only the matched duplicates are freed.""" tree, freed = self._tree_with_spy() diff --git a/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py b/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py index b7e3fe2455..85a4de7d22 100644 --- a/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py +++ b/test/registered/unit/mem_cache/test_pure_swa_chunk_cache.py @@ -61,7 +61,7 @@ class TestPureSWAChunkCache(CustomTestCase): # protected 2, floor 3, cursor 6: [2, 3) and [6, 8) go back, [3, 6) is dead req = _make_req() - cache.insert_req(req, up_to=8) + cache.checkpoint(req, up_to=8) cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, 8)]) cache.unpin(req) diff --git a/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py b/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py index 56c705d479..20ff609e52 100644 --- a/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py +++ b/test/registered/unit/mem_cache/test_pure_swa_radix_cache.py @@ -78,7 +78,7 @@ class TestPureSWARadixCache(CustomTestCase): ), ) - cache.insert_req(req, up_to=8) + cache.checkpoint(req, up_to=8) cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, 8)]) cache.unpin(req) diff --git a/test/registered/unit/mem_cache/test_radix_cache_unit.py b/test/registered/unit/mem_cache/test_radix_cache_unit.py index 2fec456cd3..ba42160b4d 100644 --- a/test/registered/unit/mem_cache/test_radix_cache_unit.py +++ b/test/registered/unit/mem_cache/test_radix_cache_unit.py @@ -413,7 +413,7 @@ class TestRadixCache(CustomTestCase): ) self.assertEqual(cache.total_size(), 5) - def test_insert_req_deferred_free_owns_original_indices(self): + def test_checkpoint_deferred_free_owns_original_indices(self): class ReqToTokenPool: def __init__(self, row): self.req_to_token = row.unsqueeze(0) @@ -454,7 +454,7 @@ class TestRadixCache(CustomTestCase): available_before_free = allocator.available_size() allocator.free_group_begin() - cache.insert_req(req, up_to=len(token_ids)) + cache.checkpoint(req, up_to=len(token_ids)) allocator.free_group_end() self.assertEqual( @@ -499,7 +499,7 @@ class TestRadixCache(CustomTestCase): ) up_to = len(prompt_ids) + len(output_ids) - cache.insert_req(req, up_to=up_to) + cache.checkpoint(req, up_to=up_to) cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)]) cache.unpin(req) diff --git a/test/registered/unit/mem_cache/test_streaming_session_unit.py b/test/registered/unit/mem_cache/test_streaming_session_unit.py index 5c1d66dcd1..06b7b31731 100644 --- a/test/registered/unit/mem_cache/test_streaming_session_unit.py +++ b/test/registered/unit/mem_cache/test_streaming_session_unit.py @@ -63,7 +63,7 @@ class _FakeInnerCache: self.dec_lock_ref_params = [] self.dec_lock_ref_skip_swa = [] - def insert_req(self, *args, **kwargs): + def checkpoint(self, *args, **kwargs): raise AssertionError("Streaming requests should not delegate to inner cache") def match_prefix(self, *args, **kwargs): diff --git a/test/registered/unit/mem_cache/test_swa_locked_full_recover_unified.py b/test/registered/unit/mem_cache/test_swa_locked_full_recover_unified.py index 4909e753e5..63ac884f96 100644 --- a/test/registered/unit/mem_cache/test_swa_locked_full_recover_unified.py +++ b/test/registered/unit/mem_cache/test_swa_locked_full_recover_unified.py @@ -24,7 +24,7 @@ The recovery must succeed rather than decline: the TreeCore insert walk counts the node in `prefix_len` regardless of component consumption, while the SWA match validator rejects a `value is None` node, so a declined recovery reports a prefix `match_prefix` cannot honor and trips -`new_prefix_len <= len(new_indices)` in `insert_req`. +`new_prefix_len <= len(new_indices)` in `checkpoint`. """ import unittest diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py b/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py index a0c2efbc0d..4eb6bcb50e 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py @@ -603,7 +603,7 @@ def bench_release( ): """Request release throughput — full request lifecycle. - Simulates: match_prefix → inc_lock_ref → alloc → fill req_to_token → insert_req + free + unpin. + Simulates: match_prefix → inc_lock_ref → alloc → fill req_to_token → checkpoint + free + unpin. """ env = _make_env(num_seqs, chunk_len, kv_size, components, page_size) diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 1b79f9c575..43e6af9a55 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -1729,7 +1729,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(allocator.available_size(), avail_before - len(seq_3p)) cache.sanity_check() - def test_insert_req(self): + def test_checkpoint(self): cache, allocator, req_to_token_pool = build_fixture(self.cfg) ps = self.cfg.page_size @@ -1785,7 +1785,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(len(prompt_only.device_indices), prompt_aligned_len) cache.sanity_check() - def test_insert_req_strips_thinking(self): + def test_checkpoint_strips_thinking(self): cache, allocator, req_to_token_pool = build_fixture(self.cfg) ps = self.cfg.page_size @@ -1883,7 +1883,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(len(m.device_indices), 0) cache.sanity_check() - def test_insert_req_mid_flight(self): + def test_checkpoint_mid_flight(self): cache, allocator, req_to_token_pool = build_fixture(self.cfg) req = self._make_req(req_to_token_pool) @@ -1905,7 +1905,7 @@ class UnifiedRadixCacheSuite: if self.cfg.has_mamba: req.kv.mamba_last_track_seqlen = kv_len - cache.insert_req(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_range.end) self.assertGreater(len(req.prefix_indices), 0) self.assertEqual(req.kv.cache_protected_len, len(req.prefix_indices)) @@ -1943,7 +1943,7 @@ class UnifiedRadixCacheSuite: req.extra_key = None req.kv.set_evicted_seqlen(ComponentType.SWA, evicted_len) - cache.insert_req(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_range.end) (first,) = _node_children(cache, cache.root_node_handle()) self.assertEqual(_node_key_length(cache, first), evicted_len) @@ -2022,7 +2022,7 @@ class UnifiedRadixCacheSuite: self.assertEqual(len(m.device_indices), 0) cache.sanity_check() - def test_paged_insert_req_unaligned_tail_freed(self): + def test_paged_checkpoint_unaligned_tail_freed(self): if self.cfg.page_size == 1: self.skipTest("page_size > 1 only") if self.cfg.has_swa: @@ -2170,7 +2170,7 @@ class UnifiedRadixCacheSuite: full_available_before_insert = allocator.full_attn_allocator.available_size() - cache.insert_req(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_range.end) self.assertEqual( allocator.full_attn_allocator.available_size(), @@ -2859,7 +2859,7 @@ class UnifiedRadixCacheSuite: swa_avail_before = allocator.swa_attn_allocator.available_size() with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True): - cache.insert_req(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_range.end) cushion = max(self.cfg.sliding_window_size, self.cfg.page_size) expected_evicted = (pre_len - 1) - cushion @@ -2946,7 +2946,7 @@ class UnifiedRadixCacheSuite: req.extra_key = None with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True): - cache.insert_req(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_range.end) self.assertEqual( req.kv.get_evicted_seqlen(ComponentType.SWA), @@ -10289,7 +10289,7 @@ class TestUnifiedRadixPrefetchCorruption(CustomTestCase): class TestSWAWindowUnderBigramKey(CustomTestCase): - """`insert_req` has to leave the leaf it inserts holding a full + """`checkpoint` has to leave the leaf it inserts holding a full sliding window of live SWA. Otherwise the match that follows the insert refuses that leaf, `cache_protected_len` never advances, and the next insert frees KV the tree already owns as if it were the request's duplicate. @@ -10345,7 +10345,7 @@ class TestSWAWindowUnderBigramKey(CustomTestCase): req.extra_key = None with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True): - cache.insert_req(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_range.end) boundary = (seq_len - 1) // page_size * page_size self.assertGreaterEqual(