diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index 710eff6167..9abb9d7237 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -373,6 +373,7 @@ class HiCacheController: # Set by the scheduler to the forward stream; gates load-back H2D # behind in-flight forwards (see start_loading). self.load_fence_stream = None + self.write_fence_stream = None self.ack_write_queue: List[HiCacheAck] = [] self.l2_transfer_engine = L2TransferEngine(io_backend) @@ -831,7 +832,7 @@ class HiCacheController: self.start_writing() return host_indices - def start_writing(self) -> None: + def start_writing(self, *, wait_for_forward: bool = False) -> None: if len(self.write_queue) == 0: return @@ -839,6 +840,12 @@ class HiCacheController: host_indices, device_indices, pool_transfers = self._move_write_operation(op) self.write_queue.clear() + if wait_for_forward and self.write_fence_stream is not None: + # An evicted SWA page can belong to an unfinished checkpoint. + self.l2_transfer_engine.device_to_host_stream.wait_stream( + self.write_fence_stream + ) + completion = self.l2_transfer_engine.submit_device_to_host( self._l2_transfers(host_indices, device_indices, pool_transfers) ) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index d3ce7f5c3e..c5824e0e68 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -609,6 +609,9 @@ class Scheduler( cache_controller.load_fence_stream = ( self.tp_worker.model_runner.forward_stream ) + cache_controller.write_fence_stream = ( + self.tp_worker.model_runner.forward_stream + ) if self.enable_unified_memory: # Keep device rows stable until host transfers are acknowledged. # Queue reads and relocation both run on the scheduler thread. diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index 8e2eca094f..0714b191ae 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -1398,18 +1398,16 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): state.phase = _InsertPhase.TAIL def _needs_incremental_component_backup(self, node: UnifiedTreeNode) -> bool: - components = self.components - if self.is_write_back: - swa = self.components_by_type.get(ComponentType.SWA) - components = () if swa is None else (swa,) return any( component.needs_incremental_backup(node) - for component in components + for component in self.components if component.component_type != BASE_COMPONENT_TYPE ) def _should_backup_after_insert(self, state: _InsertWalkState) -> bool: """Check whether the insert target needs a Host backup.""" + if self.is_write_back: + return False if state.is_new_leaf: leaf_end = state.total_prefix_length + len(state.target_node.key) return ( diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index cf098c884c..d43d5445c8 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -3385,7 +3385,7 @@ class UnifiedRadixCache(BasePrefixCache): if write_back: # Blocking: submit what is still queued, then wait for every ack. - cc.start_writing() + cc.start_writing(wait_for_forward=True) while self.ongoing_write_through: for ack in cc.ack_write_queue: ack.finish_event.synchronize() diff --git a/rust/sglang-radix-tree/src/tests/components/swa.rs b/rust/sglang-radix-tree/src/tests/components/swa.rs index ef7c8fabb0..eac6384bc8 100644 --- a/rust/sglang-radix-tree/src/tests/components/swa.rs +++ b/rust/sglang-radix-tree/src/tests/components/swa.rs @@ -6443,7 +6443,7 @@ fn buffer_mode_backup_window_is_the_target_alone_at_both_call_sites() { } #[test] -fn write_back_reinsert_still_backs_up_an_unbacked_swa_window() { +fn write_back_reinsert_defers_swa_backup_while_write_through_publishes() { let mut tc: UnifiedTreeCore> = UnifiedTreeCore::new( CacheInitParams { is_write_back: true, @@ -6462,6 +6462,15 @@ fn write_back_reinsert_still_backs_up_an_unbacked_swa_window() { tc.commit_backup(leaf, Tensor::from_slice(&[100i64, 101]), HashMap::new()) .expect("live test node"); + let result = tc.insert(&insert_params_swa(&key, &[20, 21], 0, 0)); + assert!( + !result + .cache_actions + .iter() + .any(|action| matches!(action, CacheAction::BackupKV(_))) + ); + + tc.is_write_back = false; let result = tc.insert(&insert_params_swa(&key, &[20, 21], 0, 0)); let backups: Vec<_> = result .cache_actions diff --git a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs index 64dfb871a9..1f0489cd36 100644 --- a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs @@ -3014,32 +3014,8 @@ fn backup_kv_action_chains_unbacked_ancestors_first() { assert_eq!(action.node_ids, vec![c]); } -// Mirrors UnifiedRadixCache._execute_and_commit_kv_backup: back up each chain -// node and stamp every node its transfers cover with that node's ack. -// It skips the inc_lock_ref the Python side takes; the asserts do not depend on it. -fn execute_backup_kv_for_test(tc: &mut UnifiedTreeCore>, backup: &BackupKV) { - for &node_id in &backup.node_ids { - let (device_value, mut comp_xfers) = tc.build_backup_spec(node_id).expect("backup spec"); - if device_value.numel() == 0 && comp_xfers.is_empty() { - continue; - } - let mut publish_node_ids = Vec::new(); - for transfer in comp_xfers.values_mut().flatten() { - transfer.host_indices = transfer.device_indices.as_ref().map(Tensor::copy); - publish_node_ids.extend(transfer.nodes_to_load.iter().flatten().copied()); - } - if !publish_node_ids.contains(&node_id) { - publish_node_ids.push(node_id); - } - tc.commit_backup(node_id, device_value.copy(), comp_xfers) - .expect("live test node"); - tc.mark_write_through_pending(publish_node_ids, /* ack_id = */ node_id) - .expect("live test nodes"); - } -} - #[test] -fn write_back_swa_publish_leaves_an_ancestor_pending_under_another_ack_alone() { +fn write_back_defers_swa_window_backup_until_eviction() { // c(Full unbacked) -> b(Full backed) -> t1(Full backed) // -> d(Full unbacked) -> t2(Full backed) // Every node holds a device-only SWA value inside the window. @@ -3079,7 +3055,7 @@ fn write_back_swa_publish_leaves_an_ancestor_pending_under_another_ack_alone() { } // Two requests in one batch finish at t1 and t2; neither ack drains. - for (key, target) in [(&key_t1, t1), (&key_t2, t2)] { + for key in [&key_t1, &key_t2] { let result = tc.insert(&insert_params(key, &[20, 21, 22, 23])); let backups: Vec<&BackupKV> = result .cache_actions @@ -3089,9 +3065,7 @@ fn write_back_swa_publish_leaves_an_ancestor_pending_under_another_ack_alone() { _ => None, }) .collect(); - assert_eq!(backups.len(), 1); - assert_eq!(backups[0].node_ids, vec![target]); - execute_backup_kv_for_test(&mut tc, backups[0]); + assert!(backups.is_empty()); } let pending = |node_id| { @@ -3099,8 +3073,11 @@ fn write_back_swa_publish_leaves_an_ancestor_pending_under_another_ack_alone() { .node(tc.arena.resolve(node_id).expect("live test node")) .write_through_pending_id }; - for (node_id, ack_id) in [(c, t1), (b, t1), (t1, t1), (d, t2), (t2, t2)] { - assert_eq!(pending(node_id), Some(ack_id)); + for node_id in [c, b, t1, d, t2] { + assert_eq!(pending(node_id), None); + } + for idx in [c_idx, b_idx, t1_idx, d_idx, t2_idx] { + assert!(!tc.arena.node(idx).has_host_value(SWA)); } // Write-back defers the unbacked Full prefix to eviction. assert!( diff --git a/rust/sglang-radix-tree/src/unified_tree_core.rs b/rust/sglang-radix-tree/src/unified_tree_core.rs index 2445e15320..6f6fd1ea4c 100644 --- a/rust/sglang-radix-tree/src/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/unified_tree_core.rs @@ -2075,10 +2075,7 @@ impl UnifiedTreeCore { fn needs_incremental_component_backup_(&self, node_id: NodeIdx_) -> bool { self.components.iter().any(|component| { let component_type = component.component_type(); - // Write-back defers Full and Mamba to eviction; SWA still publishes - // its window here because out-of-window frees never reach eviction. component_type != BASE_COMPONENT_TYPE - && (!self.is_write_back || component_type == SWA) && component.needs_incremental_backup(self, node_id) }) } @@ -2089,6 +2086,9 @@ impl UnifiedTreeCore { state: &InsertWalkState, target_node_id: NodeIdx_, ) -> bool { + if self.is_write_back { + return false; + } if state.is_new_leaf { // The new leaf runs to the end of the aligned key. return state.aligned_key_len > state.inserted_len diff --git a/test/registered/unit/mem_cache/test_hicache_write_back_ordering.py b/test/registered/unit/mem_cache/test_hicache_write_back_ordering.py new file mode 100644 index 0000000000..e902cdefd1 --- /dev/null +++ b/test/registered/unit/mem_cache/test_hicache_write_back_ordering.py @@ -0,0 +1,116 @@ +"""Write-back eviction must wait for the checkpoint's KV producer.""" + +import unittest +from array import array +from unittest.mock import patch + +import torch +from test_unified_radix_cache_unittest import ( + CacheConfig, + UnifiedRadixCacheSuite, + build_fixture, +) + +from sglang.srt.mem_cache.base_prefix_cache import DecLockRefParams, EvictParams +from sglang.srt.mem_cache.unified_cache.component_type import ComponentType +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=5, stage="extra-a", runner_config="1-gpu-small") + + +@unittest.skipUnless(torch.cuda.is_available(), "CUDA required") +class TestHiCacheWriteBackOrdering(unittest.TestCase): + _init_hicache = UnifiedRadixCacheSuite._init_hicache + _make_req = UnifiedRadixCacheSuite._make_req + _rid = 0 + + def _request(self, cache, allocator, req_pool, start, length=8): + req = self._make_req(req_pool) + req.origin_input_ids = array("q", range(start, start + 2 * length)) + req.full_untruncated_fill_ids = array("q", range(start, start + length)) + req.output_ids = array("q") + req.pending_bootstrap = False + req.extend_end = length + ids = allocator.alloc(length) + self.assertIsNotNone(ids) + req_pool.write((req.kv.req_pool_idx, slice(0, length)), ids) + req.kv.kv_committed_len = length + req.last_node = cache.root_node_handle() + req.lock_receipt = DecLockRefParams() + return req + + def test_write_back_internal_swa_waits_for_checkpoint_producer(self): + self.cfg = CacheConfig( + components=(ComponentType.FULL, ComponentType.SWA), + page_size=1, + sliding_window_size=4, + num_layers=2, + full_attention_layer_ids=(0,), + kv_size=64, + max_context_len=64, + head_num=1, + head_dim=64, + ) + cache, allocator, req_pool = build_fixture(self.cfg) + self._init_hicache(cache, write_policy="write_back") + controller = cache.cache_controller + forward = torch.cuda.Stream() + controller.write_fence_stream = forward + warm = self._request(cache, allocator, req_pool, 100, length=12) + cache.checkpoint(warm, up_to=12) + warm_eviction = cache.evict(EvictParams(swa_num_tokens=1)) + self.assertGreaterEqual(warm_eviction.swa_num_tokens_evicted, 1) + req = self._request(cache, allocator, req_pool, 0, length=12) + kv = allocator.get_kvcache() + buffers = [ + buf + for pool in (kv.full_kv_pool, kv.swa_kv_pool) + for buf in pool.k_buffer + pool.v_buffer + ] + for buf in buffers: + buf.fill_(-1) + read_done = torch.cuda.Event() + read_done.record() + torch.cuda.synchronize() + engine = controller.l2_transfer_engine + submit = engine.submit_device_to_host + snapshots = [] + + def record_copy(transfers): + completion = submit(transfers) + completion.finish_event.synchronize() + for transfer in transfers: + if transfer.host_indices.numel(): + snapshots.append( + transfer.host_pool.get_data_page( + int(transfer.host_indices[0]) + ).clone() + ) + return completion + + with torch.cuda.stream(forward): + read_done.record() + torch.cuda._sleep(600_000_000) + for buf in buffers: + buf.fill_(42) + torch.cuda.current_stream().wait_event(read_done) + try: + with patch.object(engine, "submit_device_to_host", side_effect=record_copy): + cache.checkpoint(req, up_to=12) + self.assertEqual(controller.write_queue, []) + # The active request pins Full and its final SWA window. An + # older window from this same forward is still evictable. + evicted = cache.evict(EvictParams(swa_num_tokens=1)) + self.assertGreaterEqual(evicted.swa_num_tokens_evicted, 1) + self.assertTrue(snapshots) + for snapshot in snapshots: + self.assertTrue( + torch.equal(snapshot, torch.full_like(snapshot, 42)), + f"expected 42, copied {snapshot.unique().tolist()}", + ) + finally: + forward.synchronize() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py index 41169bfc5e..a31d5bbe60 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py @@ -3657,334 +3657,6 @@ def test_full_load_back_does_not_pin_restored_ancestor_aux( core.sanity_check([], []) -@pytest.mark.parametrize("backend", ["python", "rust"]) -def test_write_back_swa_repair_backup_leaves_the_unbacked_parent_to_eviction(backend): - """Write-back builds the SWA repair backup as the repaired node alone; the - parent's missing Full copy waits for eviction instead of joining at insert.""" - cache, allocator = _swa_cache(window=4, backend=backend) - core = cache.tree_core - values = allocator.alloc(12) - nodes = [ - cache.insert( - InsertParams( - key=_key(list(range(size))), - value=values[:size], - prev_prefix_len=size - 4, - ) - ).last_device_node - for size in (4, 8, 12) - ] - parent, child, leaf = nodes - core.set_hicache_enabled() - core.has_swa_host_pool = True - core.is_write_back = True - for node in nodes: - _complete_backup(core, node, full_offset=10000, aux_offset=20000) - - # Reclaim the two internal SWA states and the parent's Full host copy. - tracker = {} - core.evict_device_start(ComponentType.SWA, 8) - try: - for _ in range(2): - step = core.evict_device_next_node(ComponentType.SWA, tracker) - assert step.node_id is None - for component, count in step.tracker.items(): - tracker[component] = tracker.get(component, 0) + count - cache._free_values(step.device_frees, step.host_frees) - finally: - core.evict_device_end(ComponentType.SWA) - assert tracker[ComponentType.SWA] == 8 - for component, amount in ((ComponentType.SWA, 8), (ComponentType.FULL, 4)): - step = core.drive_host_eviction(component, amount) - assert step.tracker[component] == amount - _accumulate_step(step, {}, {}, {}) - assert not core.is_backuped(parent) and core.is_backuped(child) - - # Mock DMA submission only; insertion, backup publication and FIFO - # acknowledgment all use the real shared controller. - cache.cache_controller = SimpleNamespace(write_policy="write_back") - cache._build_backup_sidecar = Mock(return_value=[]) - submissions = [] - - def write(node, device, aux, sidecars): - submissions.append((node, device.numel())) - for transfers in aux.values(): - for transfer in transfers: - transfer.host_indices = transfer.device_indices + 30000 - return device + 40000 - - cache._execute_kv_backup = write - # An SWA eviction floor of 4 means the request's window already slid past - # its first 4 slots and released their SWA peers. - value = allocator.alloc(8) - allocator.free_swa(value[:4]) - cache.insert( - InsertParams( - key=_key(list(range(8))), - value=value, - component_evicted_seqlens={ComponentType.SWA: 4}, - ) - ) - # Repairing SWA still issues the async backup in write-back mode, but for - # the repaired node alone: a chained parent could already be pending under - # another request's window publish. - assert submissions == [(child, 0)] - assert set(cache.ongoing_write_through) == {child} - core.sanity_check([(child, child)], []) - cache._finish_write_through_ack(child) - core.sanity_check([], []) - - # The existing Full copies keep their order; the parent stays unbacked - # until write-back eviction backs it up. - expected = [ - (values[4:8] + 10000).tolist(), - (values[8:] + 10000).tolist(), - ] - for host_indices in expected: - host_frees = {} - _accumulate_step( - core.drive_host_eviction(ComponentType.FULL, 1), {}, {}, host_frees - ) - assert [value.tolist() for value in host_frees[ComponentType.FULL]] == [ - host_indices - ] - assert all(not core.is_backuped(node) for node in nodes) - core.sanity_check([], []) - - -@pytest.mark.parametrize("backend", ["python", "rust"]) -@pytest.mark.parametrize("pin_ancestor", [False, True]) -def test_pending_swa_backup_does_not_pin_restored_ancestor_mamba(backend, pin_ancestor): - cache, allocator = _swa_mamba_cache(backend, window=8) - core = cache.tree_core - values = allocator.alloc(12) - nodes = [ - cache.insert( - InsertParams( - key=_key(list(range(size))), - value=values[:size], - prev_prefix_len=size - 4, - mamba_value=torch.tensor([slot]), - ) - ).last_device_node - for size, slot in ((4, 201), (8, 202), (12, 203)) - ] - parent, child, leaf = nodes - core.set_hicache_enabled() - core.has_swa_host_pool = True - core.is_write_back = True - for node in nodes: - _complete_backup(core, node, full_offset=10000, aux_offset=20000) - - # SWA eviction also removes ancestor Mamba states. Repair SWA first; - # a separate request below restores the ancestor's Mamba checkpoint. - tracker = {} - core.evict_device_start(ComponentType.SWA, 8) - try: - for _ in range(2): - step = core.evict_device_next_node(ComponentType.SWA, tracker) - assert step.node_id is None - for component, count in step.tracker.items(): - tracker[component] = tracker.get(component, 0) + count - cache._free_values(step.device_frees, step.host_frees) - finally: - core.evict_device_end(ComponentType.SWA) - assert tracker[ComponentType.SWA] == 8 - step = core.drive_host_eviction(ComponentType.SWA, 8) - assert step.tracker[ComponentType.SWA] == 8 - _accumulate_step(step, {}, {}, {}) - assert all(core.is_backuped(node) for node in nodes) - cache.cache_controller = SimpleNamespace(write_policy="write_back") - cache._build_backup_sidecar = Mock(return_value=[]) - submissions = [] - - def write(node, full, aux, sidecars): - submissions.append((node, full.numel(), aux)) - for transfers in aux.values(): - for transfer in transfers: - transfer.host_indices = transfer.device_indices + 30000 - return full + 40000 - - cache._execute_kv_backup = write - cache.insert( - InsertParams( - key=_key(list(range(8))), - value=allocator.alloc(8), - component_evicted_seqlens={ComponentType.SWA: 0}, - mamba_value=torch.tensor([202]), - ) - ) - assert len(submissions) == 1 - node, full_rows, aux = submissions[0] - assert node == child and full_rows == 0 - assert aux[ComponentType.SWA][0].nodes_to_load == [parent, child] - assert aux[ComponentType.MAMBA][0].device_indices.tolist() == [202] - assert list(cache.ongoing_write_through) == [child] - assert cache.ongoing_write_through[child].publish_node_ids == [parent, child] - assert core.get_component_device_value(parent, ComponentType.MAMBA) is None - - cache.insert( - InsertParams( - key=_key(list(range(4))), - value=allocator.alloc(4), - mamba_value=torch.tensor([211]), - ) - ) - assert len(submissions) == 1 - assert core.get_component_device_value(parent, ComponentType.MAMBA).tolist() == [ - 211 - ] - leaf_lock = cache.inc_lock_ref(leaf).to_dec_params() - extra_lock = cache.inc_lock_ref(parent).to_dec_params() if pin_ancestor else None - active = [(leaf, leaf)] + ([(parent, parent)] if pin_ancestor else []) - core.sanity_check([(child, child)] + active, []) - retained = { - (node, component): core.get_component_device_value(node, component).tolist() - for node in nodes - for component in (ComponentType.FULL, ComponentType.SWA) - } - - def evict_one_mamba_state(): - tracker, frees = {ComponentType.MAMBA: 0}, {} - core.evict_device_start(ComponentType.MAMBA, 1) - try: - step = core.evict_device_next_node(ComponentType.MAMBA, tracker) - assert step.node_id is None and step.mamba_backup_node_id is None - _accumulate_step(step, tracker, frees, {}) - finally: - core.evict_device_end(ComponentType.MAMBA) - return [value.tolist() for value in frees.get(ComponentType.MAMBA, [])] - - assert evict_one_mamba_state() == ([] if pin_ancestor else [[211]]) - # The actual Mamba DMA source remains protected by its own device lock. - assert core.get_component_device_value(child, ComponentType.MAMBA).tolist() == [202] - core.sanity_check([(child, child)] + active, []) - for (node, component), indices in retained.items(): - assert core.get_component_device_value(node, component).tolist() == indices - cache._finish_write_through_ack(child) - assert evict_one_mamba_state() == [[202]] - core.sanity_check(active, []) - cache.dec_lock_ref(leaf, leaf_lock) - if extra_lock is not None: - cache.dec_lock_ref(parent, extra_lock) - core.sanity_check([], []) - - -@pytest.mark.parametrize("backend", ["python", "rust"]) -def test_failed_mamba_backup_drops_state_under_unrelated_pending_swa(backend): - cache, allocator = _swa_mamba_cache(backend, window=12) - core = cache.tree_core - values = allocator.alloc(16) - nodes = [ - cache.insert( - InsertParams( - key=_key(list(range(size))), - value=values[:size], - prev_prefix_len=size - 4, - mamba_value=torch.tensor([slot]), - ) - ).last_device_node - for size, slot in ((4, 201), (8, 202), (12, 203), (16, 204)) - ] - grandparent, parent, anchor, leaf = nodes - core.set_hicache_enabled() - core.has_swa_host_pool = True - core.is_write_back = True - cache.cache_controller = SimpleNamespace(write_policy="write_back") - cache._build_backup_sidecar = Mock(return_value=[]) - host_pool = Mock() - host_pool.available_size.return_value = 1 - cache.host_pool_group = Mock() - cache.host_pool_group.get_pool.return_value = host_pool - cache.components[ComponentType.MAMBA]._mamba_pool_host = host_pool - submissions = [] - failed_node = None - - def write(node, full, aux, sidecars): - submissions.append((node, full.numel(), aux)) - if node == failed_node: - return None # Full host allocation failed before submitting DMA. - for transfers in aux.values(): - for transfer in transfers: - transfer.host_indices = transfer.device_indices + 20000 - return full + 10000 - - def complete_physical_writes(write_back=False): - for ack_id in list(cache.ongoing_write_through): - cache._finish_write_through_ack(ack_id) - - cache._execute_kv_backup = write - cache.writing_check = complete_physical_writes - - def evict_component(component, count): - tracker = {ct: 0 for ct in cache.tree_components} - requested = { - ct: count if ct == component else 0 for ct in cache.tree_components - } - cache._evict_components(requested, tracker) - return tracker[component] - - # Reach the host topology through actual state backups and ACKs. Full - # duplicate reclaim then leaves an unbacked grandparent above a backed - # parent; no synthetic node flags or direct backup commits are needed. - assert evict_component(ComponentType.MAMBA, 3) == 3 - assert [(node, count) for node, count, _ in submissions] == [ - (node, 4) for node in nodes[:3] - ] - assert cache.evict_host(4, ComponentType.FULL) == 4 - assert [core.is_backuped(node) for node in nodes] == [False, True, True, False] - core.sanity_check([], []) - - assert evict_component(ComponentType.SWA, 12) == 12 - assert cache.evict_host(12, ComponentType.SWA) == 12 - cache.insert( - InsertParams( - key=_key(list(range(12))), - value=allocator.alloc(12), - mamba_value=torch.tensor([203]), - ) - ) - # Backup ancestry stops at the backed parent, while the expanded SWA - # window reaches the unbacked grandparent. Its pending mark belongs to SWA. - assert list(cache.ongoing_write_through) == [anchor] - assert cache.ongoing_write_through[anchor].publish_node_ids == nodes[:3] - assert submissions[-1][2][ComponentType.SWA][0].nodes_to_load == nodes[:3] - cache.insert( - InsertParams( - key=_key(list(range(4))), - value=allocator.alloc(4), - mamba_value=torch.tensor([211]), - ) - ) - assert not core.is_backuped(grandparent) - assert ComponentType.MAMBA in core.build_backup_spec(grandparent)[1] - leaf_lock = cache.inc_lock_ref(leaf).to_dec_params() - active = [(anchor, anchor), (leaf, leaf)] - core.sanity_check(active, []) - retained = { - (node, ct): core.get_component_device_value(node, ct).clone() - for node in nodes - for ct in (ComponentType.FULL, ComponentType.SWA) - } - - failed_node = grandparent - assert evict_component(ComponentType.MAMBA, 1) == 1 - assert submissions[-1][:2] == (grandparent, 4) - assert core.get_component_device_value(grandparent, ComponentType.MAMBA) is None - # The completed backup and active request retain their other device values. - for node, slot in ((anchor, 203), (leaf, 204)): - assert core.get_component_device_value(node, ComponentType.MAMBA).tolist() == [ - slot - ] - for (node, ct), value in retained.items(): - assert torch.equal(core.get_component_device_value(node, ct), value) - # The earlier SWA ACK is drained before the failed backup attempt. - assert not cache.ongoing_write_through - core.sanity_check([(leaf, leaf)], []) - cache.dec_lock_ref(leaf, leaf_lock) - core.sanity_check([], []) - - @pytest.mark.parametrize("backend", ["python", "rust"]) @pytest.mark.parametrize( "ring,hicache,has_swa_host_pool", diff --git a/test/registered/unit/mem_cache/test_unified_hicache_regressions.py b/test/registered/unit/mem_cache/test_unified_hicache_regressions.py index 64540b0b2c..54a8f2c780 100644 --- a/test/registered/unit/mem_cache/test_unified_hicache_regressions.py +++ b/test/registered/unit/mem_cache/test_unified_hicache_regressions.py @@ -71,6 +71,68 @@ class TestHostGateCapacity(unittest.TestCase): @unittest.skipUnless(torch.cuda.is_available(), "CUDA required") class TestTransferStreamOrdering(unittest.TestCase): + def test_write_back_drain_waits_for_forward_before_copying_queued_nodes(self): + from sglang.srt.managers.cache_controller import ( + CacheOperation, + HiCacheController, + ) + from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache + + values = torch.full((8,), -1, dtype=torch.int64, device="cuda") + expected = torch.arange(8, dtype=torch.int64, device="cuda") + copied = torch.empty(8, dtype=torch.int64, pin_memory=True) + forward = torch.cuda.Stream() + engine = L2TransferEngine("kernel") + host = SimpleNamespace( + prepare_transfer_indices=lambda h, d, backend: (h, d), + backup_from_device_all_layer_physical=lambda pool, h, d, backend: ( + copied.copy_(values[d], non_blocking=True) + ), + ) + controller = HiCacheController.__new__(HiCacheController) + controller.write_fence_stream = forward + controller.l2_transfer_engine = engine + controller.mem_pool_device_allocator = Mock() + controller.ack_write_queue = [] + # A tick's queued backup and the eviction victim share one drain. + controller.write_queue = [ + CacheOperation(torch.arange(4), torch.arange(4, device="cuda"), 1), + CacheOperation(torch.arange(4, 8), torch.arange(4, 8, device="cuda"), 2), + ] + controller._move_write_operation = lambda op: ( + op.host_indices, + op.device_indices, + op.pool_transfers, + ) + controller._l2_transfers = lambda h, d, extra: [ + L2Transfer(host, SimpleNamespace(), h, d) + ] + controller._num_tokens_by_pool = lambda op: {} + controller._transfer_num_bytes = lambda op: 64 + cache = UnifiedRadixCache.__new__(UnifiedRadixCache) + cache.cache_controller = controller + cache.ongoing_write_through = {1: None, 2: None} + cache._finish_write_through_ack = cache.ongoing_write_through.pop + cache.metrics_collector = None + # Warm up temporary allocations so cudaMalloc cannot hide a missing fence. + operations = list(controller.write_queue) + cache.writing_check(write_back=True) + controller.write_queue = operations + cache.ongoing_write_through.update({1: None, 2: None}) + torch.cuda.synchronize() + + with torch.cuda.stream(forward): + torch.cuda._sleep(100_000_000) + values.copy_(expected) + try: + cache.writing_check(write_back=True) + self.assertTrue(torch.equal(copied, torch.arange(8))) + self.assertEqual(controller.write_queue, []) + self.assertEqual(controller.ack_write_queue, []) + self.assertEqual(cache.ongoing_write_through, {}) + finally: + forward.synchronize() + def test_load_translation_follows_supplied_start_event(self): # The start event precedes translation; transfer must wait for both. engine = L2TransferEngine("kernel") diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index b08ad4b9c7..f4eaf35d9c 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -7315,6 +7315,9 @@ class UnifiedRadixCacheSuite: # The backed parent stops the Full backup chain, while the SWA window # reaches the unbacked grandparent and marks it under the anchor's ack. insert(3 * page) + # Queue the asynchronous window backup explicitly: write-back insertion + # no longer publishes it, but Mamba eviction must still respect its ack. + cache._execute_and_commit_kv_backup(BackupKV(node_ids=[anchor])) self.assertEqual( cache.ongoing_write_through[anchor].publish_node_ids, nodes[:3] ) @@ -9564,11 +9567,8 @@ class UnifiedRadixCacheSuite: cache.sanity_check() - def test_write_back_swa_publish_leaves_an_ancestor_pending_under_another_ack_alone( - self, - ): - """Write-back builds the insert backup as [target] only, so a second SWA - window publish never re-marks an ancestor another ack still holds.""" + def test_write_back_defers_swa_window_backup_until_eviction(self): + """Reinserting backed Full nodes must not publish freshly recovered SWA.""" if not self.cfg.has_swa: self.skipTest("requires SWA") if self._skip_unsupported_hicache_test(): @@ -9612,16 +9612,14 @@ class UnifiedRadixCacheSuite: "_execute_and_commit_kv_backup", wraps=cache._execute_and_commit_kv_backup, ) as backups: - for seq, target in ((t1_seq, t1), (t2_seq, t2)): - backups.reset_mock() + for seq in (t1_seq, t2_seq): self._insert(cache, allocator, req_to_token_pool, seq) - self.assertEqual( - [call.args[0].node_ids for call in backups.call_args_list], - [[target]], - ) + backups.assert_not_called() - for node, ack in ((c, t1), (b, t1), (t1, t1), (d, t2), (t2, t2)): - self.assertEqual(cache.tree_core.get_write_through_pending_id(node), ack) + for node in (c, b, t1, d, t2): + self.assertIsNone(cache.tree_core.get_write_through_pending_id(node)) + self.assertIsNone(_host_value(cache, node, ComponentType.SWA)) + self.assertEqual(cache.cache_controller.write_queue, []) # Write-back defers the unbacked Full prefix to eviction. self.assertFalse(cache.tree_core.is_backuped(c)) self.assertFalse(cache.tree_core.is_backuped(d))