diff --git a/python/sglang/srt/mem_cache/buffer_mode/pipeline.py b/python/sglang/srt/mem_cache/buffer_mode/pipeline.py index de54689338..db0835a95d 100644 --- a/python/sglang/srt/mem_cache/buffer_mode/pipeline.py +++ b/python/sglang/srt/mem_cache/buffer_mode/pipeline.py @@ -86,9 +86,10 @@ class _UnifiedBackupIntent(msgspec.Struct): Snapshots node identity at enqueue time: a split rewrites the node's key/hash in place while these copies stay intact, so a key-length change - detects a split and a missing FULL device value detects eviction - (``_validate_backup_intent``). ``keys`` is the admission view (it sizes - drop metrics); the D2H launch re-reads rows and keys off the node. + detects a split (repaired during queue refresh) and a missing FULL device + value detects eviction (``_validate_backup_intent``). ``keys`` is the + admission view (it sizes drop metrics); the D2H launch re-reads rows and + keys off the node. """ snapshot: BufferBackupSnapshot @@ -721,35 +722,92 @@ class BufferModePipeline: self, intent: _UnifiedBackupIntent ) -> Optional[BufferBackupState]: # Arena-lookup failure = deleted, key-length mismatch vs the snapshot - # = split, a None FULL device value = evicted. Stale - # intents are counted as dropped; the node re-triggers on a later hit. + # = split, a None FULL device value = evicted. snapshot = intent.snapshot return self._cache.tree_core.validate_buffer_backup( snapshot.node_id, len(snapshot.key) ) - def _sweep_stale_backup_intents(self) -> dict[NodeId, BufferBackupState]: - """Cancel stale intents anywhere in the queue, not just at the head: - a dead intent would otherwise inflate the backlog accounting and - hold FIFO position ahead of live segments.""" + def _split_pieces( + self, snapshot: BufferBackupSnapshot + ) -> Optional[list[BufferBackupSnapshot]]: + """Resolve a split span, parents first, or return None if it is gone. + + The original node retains the tail; its ancestors must cover exactly + the admitted span and end at the original parent. + """ + tree_core = self._cache.tree_core + pass_prefix_keys = self._cache.hicache_storage_pass_prefix_keys + pieces: list[BufferBackupSnapshot] = [] + node_id, remaining = snapshot.node_id, len(snapshot.key) + while remaining > 0: + piece = tree_core.snapshot_buffer_backup(node_id, pass_prefix_keys) + if piece is None or len(piece.key) > remaining: + return None + pieces.append(piece) + remaining -= len(piece.key) + node_id = piece.parent_node_id + if node_id != snapshot.parent_node_id: + return None + pieces.reverse() + return pieces + + def _refresh_pending_backup_intents(self) -> dict[NodeId, BufferBackupState]: + """Drop dead spans and replace split intents in FIFO order. + + The earliest intent for each (node, pool) wins, including split pieces. + """ if not self.pending_write_queue: return {} page_size = self._cache.page_size - survivors: deque[_UnifiedBackupIntent] = deque() + queued = {(i.snapshot.node_id, i.pool) for i in self.pending_write_queue} + survivors: dict[tuple[NodeId, PoolName], _UnifiedBackupIntent] = {} states: dict[NodeId, BufferBackupState] = {} swept_tokens = 0 for intent in self.pending_write_queue: snapshot = intent.snapshot - state = self._validate_backup_intent(intent) - if state is None: - self._finish_inflight(snapshot.node_id, intent.pool) - intent_tokens = len(intent.keys) * page_size + key = (snapshot.node_id, intent.pool) + if key in survivors: + # The replacement retains this intent's inflight ownership. self._release_queued_span(intent) - swept_tokens += intent_tokens continue - survivors.append(intent) - states[snapshot.node_id] = state - self.pending_write_queue = survivors + state = self._validate_backup_intent(intent) + if state is not None: + survivors[key] = intent + states[snapshot.node_id] = state + continue + self._finish_inflight(snapshot.node_id, intent.pool) + self._release_queued_span(intent) + pieces = self._split_pieces(snapshot) + if pieces is None: + swept_tokens += len(intent.keys) * page_size + continue + for piece in pieces: + key = (piece.node_id, intent.pool) + if key in survivors: + continue + if ( + intent.pool in self.inflight_backup_pools.get(piece.node_id, ()) + and key not in queued + ): + continue # staged or awaiting its storage ack + keys = dict(self._write_intents(piece)).get(intent.pool) + if not keys: + continue # covered since admission + repaired = _UnifiedBackupIntent( + snapshot=piece, pool=intent.pool, keys=list(keys) + ) + survivors[key] = repaired + states[piece.node_id] = BufferBackupState( + parent_node_id=piece.parent_node_id, + parent_is_root=piece.parent_is_root, + parent_last_hash=piece.parent_last_hash, + ) + self.inflight_backup_pools.setdefault(piece.node_id, set()).add( + intent.pool + ) + self._retain_queued_span(repaired) + self.pending_write_queue = deque(survivors.values()) self._log_backup_dropped(swept_tokens) return states @@ -764,7 +822,7 @@ class BufferModePipeline: if not self.pending_write_queue: return cc = self._cache.cache_controller - states = self._sweep_stale_backup_intents() + states = self._refresh_pending_backup_intents() # Loads have priority (writes are deferrable): the write window is # the pool minus prefetch occupancy minus a 10% margin, floored at # the configured fraction. diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 07c61d6815..46d9929e2b 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -4350,6 +4350,120 @@ class UnifiedRadixCacheSuite: self.assertEqual(self._host_avail_sizes(cache), avail0) cache.sanity_check() + def _drain_buffer_writes(self, cache): + pipeline = cache.buffer_pipeline + self._pump_hicache_until( + cache, + lambda: ( + not pipeline.inflight_backup_node_ids and not pipeline.ongoing_backup + ), + "buffer backup pipeline did not drain", + ) + self.assertFalse(pipeline.pending_write_queue) + self.assertFalse(pipeline._queued_span_refs) + self.assertEqual(pipeline.write_backlog_tokens_, 0) + self.assertEqual(pipeline.write_staged_tokens_, 0) + + def test_buffer_only_split_intent_writes_every_piece(self): + """Repeated splits preserve every pool's write once, parents first.""" + self._skip_unsupported_hicache_test() + storage_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, storage_dir, ignore_errors=True) + cache, allocator, req_to_token_pool = build_fixture(self.cfg) + self._init_buffer_hicache(cache, storage_dir) + pipeline = cache.buffer_pipeline + ps = self.cfg.page_size + + seq = self._buffer_swa_seq(min_pages=4) + self._insert(cache, allocator, req_to_token_pool, seq) + leaf = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", seq))) + ).last_device_node + chain = self._path_chain(cache, leaf) + span_hashes = [h for n in chain for h in cache.tree_core.get_hash_values(n)] + aux_keys = _aux_storage_key_transfers(cache, leaf) + _write_backup(cache, leaf) + expected_keys = [ + (i.pool, key) for i in pipeline.pending_write_queue for key in i.keys + ] + # SWA may already have split the leaf at its window boundary. + end = len(seq) + for node in reversed(chain): + node_len = _node_key_length(cache, node) + if node_len >= 3 * ps: + break + end -= node_len + else: + self.fail("no node of three pages to split") + for offset in (1, 2): + split_at = end - node_len + offset * ps + self._insert( + cache, + allocator, + req_to_token_pool, + seq[:split_at] + self._make_seq(9000, 2), + ) + self.assertEqual(_node_key_length(cache, node), end - split_at) + _write_backup(cache, _node_parent(cache, node)) + + pipeline._refresh_pending_backup_intents() + kv_order = [ + intent.snapshot.node_id + for intent in pipeline.pending_write_queue + if intent.pool == PoolName.KV + ] + self.assertEqual(kv_order, self._path_chain(cache, leaf)) + self.assertCountEqual( + [(i.pool, key) for i in pipeline.pending_write_queue for key in i.keys], + expected_keys, + ) + self.assertEqual(pipeline.write_backlog_tokens_, len(seq)) + self._drain_buffer_writes(cache) + self.assertEqual( + self._storage_exists_count(cache, span_hashes, aux_keys), len(span_hashes) + ) + cache.sanity_check() + + def test_buffer_only_finish_writes_prompt_and_output(self): + """The finish-path split preserves both prompt and output writes.""" + if self.cfg.components != (ComponentType.FULL,): + self.skipTest("only FULL-only caches split the leaf at the prompt") + storage_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, storage_dir, ignore_errors=True) + cache, allocator, req_to_token_pool = build_fixture(self.cfg) + self._init_buffer_hicache(cache, storage_dir) + cache.tree_core.write_through_threshold = 1 # the write_through trigger + + req = self._make_req(req_to_token_pool) + prompt = self._make_seq(1, 6) + output = self._make_seq(3000, 4) + req.origin_input_ids = array("q", prompt) + req.output_ids = array("q", output) + req.full_untruncated_fill_ids = array("q", prompt + output) + req.extend_end = len(prompt) + len(output) + kv_len = req.extend_end + kv_indices = self._alloc(allocator, kv_len) + req_to_token_pool.write((req.kv.req_pool_idx, slice(0, kv_len)), kv_indices) + req.kv.kv_committed_len = kv_len + req.kv.kv_allocated_len = kv_len + req.last_node = cache.root_node_handle() + req.kv.cache_protected_len = 0 + req.lock = None + req.extra_key = None + finish_req(cache, req, kv_len) + + leaf = cache.match_prefix( + MatchPrefixParams(key=RadixKey(array("q", prompt + output))) + ).last_device_node + chain = self._path_chain(cache, leaf) + self.assertEqual( + [_node_key_length(cache, n) for n in chain], [len(prompt), len(output)] + ) + self._drain_buffer_writes(cache) + hashes = [h for n in chain for h in cache.tree_core.get_hash_values(n)] + self.assertEqual(self._storage_exists_count(cache, hashes), len(hashes)) + cache.sanity_check() + @staticmethod def _writes_by_pool(write_storage, extra): """Key tuples of the KV writes and of the ``extra`` pool's writes