[HiCache]: Fix host lock ownership across radix splits (#38480)

Signed-off-by: Lukas Humbel <lhumbel@nvidia.com>
Co-authored-by: Jialin Ouyang <jialino@meta.com>
Co-authored-by: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com>
This commit is contained in:
Lukas Humbel
2026-10-04 11:16:04 +08:00
committed by GitHub
co-authored by Jialin Ouyang Shuwen Wang
parent ff219e159e
commit 4611ea365e
20 changed files with 526 additions and 103 deletions
@@ -215,7 +215,10 @@ so the order is not load-bearing for the leaf sets. Full walks to root; SWA
stops at the receipt boundary; components in `skipped_lock_components` are
left alone. `skip_swa=True` also skips lower-priority components already
released by `dec_swa_lock_only`. The host-side `dec_host_lock_ref` takes the
same required receipt.
same required receipt. A Full host lock begins as a one-node UUID-bounded
segment. A later radix split copies the anchor's `host_lock_ref` to the inserted
prefix and migrates the UUID to that prefix; release walks every fragment until
it finds the receipt's UUID.
---
@@ -289,8 +292,8 @@ Each component implements these hooks. See `base.py` for the ABC and docstrings.
| Hook | Purpose | Called By | Default |
|------|---------|-----------|----------|
| `acquire_component_lock(lock_host=False)` | Increment device or host lock refs; moves device tokens from evictable to protected. Full: path-lock for device, single-node host lock. SWA: window-lock with UUID boundary. Mamba: single-node lock. | `inc_lock_ref`, `inc_host_lock_ref` | *abstract* |
| `release_component_lock(lock_host=False)` | Decrement device or host lock refs; moves device tokens from protected to evictable when `lock_ref` → 0. Full path-unlocks device; SWA walks up to UUID boundary; Mamba unlocks a single node. | `dec_lock_ref`, `dec_host_lock_ref` | *abstract* |
| `acquire_component_lock(lock_host=False)` | Increment device or host lock refs; moves device tokens from evictable to protected. Full: path-lock for device; its host lock starts as a one-node UUID-bounded segment so later split fragments remain protected. SWA: window-lock with UUID boundary. Mamba: single-node lock. | `inc_lock_ref`, `inc_host_lock_ref` | *abstract* |
| `release_component_lock(lock_host=False)` | Decrement device or host lock refs; moves device tokens from protected to evictable when `lock_ref` -> 0. Full path-unlocks device and releases all host split fragments to the receipt boundary; SWA walks up to UUID boundary; Mamba unlocks a single node. | `dec_lock_ref`, `dec_host_lock_ref` | *abstract* |
### Caching Phase
@@ -41,7 +41,18 @@ if TYPE_CHECKING:
_NUM_COMPONENT_TYPES = len(ComponentType)
_LAST_ACCESS_TIME_COUNTER_FLOAT = float64(1.0)
_COMPONENT_UUID_COUNTER = 1
_COMPONENT_UUID_RANGE_SIZE = 100_000_000_000_000
_COMPONENT_UUID_RANGE_INDEX = {
ComponentType.SWA: 0,
ComponentType.FULL: 1,
ComponentType.MAMBA: 2,
ComponentType.C128: 3,
ComponentType.AUXILIARY_SWA: 4,
}
_COMPONENT_UUID_COUNTERS = {
component_type: (range_index + 1) * _COMPONENT_UUID_RANGE_SIZE
for component_type, range_index in _COMPONENT_UUID_RANGE_INDEX.items()
}
@dataclasses.dataclass
@@ -121,10 +132,9 @@ def get_and_increase_time_counter() -> float64:
return ret
def next_component_uuid() -> int:
global _COMPONENT_UUID_COUNTER
_COMPONENT_UUID_COUNTER += 1
return _COMPONENT_UUID_COUNTER
def next_component_uuid(component_type: ComponentType) -> int:
_COMPONENT_UUID_COUNTERS[component_type] += 1
return _COMPONENT_UUID_COUNTERS[component_type]
class TreeComponent(ABC):
@@ -589,7 +599,7 @@ class TreeComponent(ABC):
node itself (mamba state is per-leaf, not per-path).
When ``lock_host`` is True, the lock applies to host-side state:
- Full: single-node host lock.
- Full: a one-node UUID-bounded segment that expands across splits.
- SWA: host window-lock with a dedicated host UUID boundary.
- Mamba: single-node host lock with host LRU detach."""
...
@@ -610,7 +620,8 @@ class TreeComponent(ABC):
- Mamba: single-node unlock — only decrements lock_ref on the
node itself.
When ``lock_host`` is True, the inverse host-side semantics apply."""
When ``lock_host`` is True, Full and SWA replay their host boundary
UUIDs while Mamba retains its single-node semantics."""
...
def prepare_for_caching_req(
@@ -26,6 +26,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import (
ExternalLinkerLoadPhase,
LinkerTransferPhase,
TreeComponent,
next_component_uuid,
)
if TYPE_CHECKING:
@@ -146,6 +147,13 @@ class FullComponent(TreeComponent):
new_parent.component_data[ct].lock_ref = child.component_data[ct].lock_ref
new_parent.component_data[ct].session_ref = child.component_data[ct].session_ref
child_cd = child.component_data[ct]
new_parent.component_data[ct].host_lock_ref = child_cd.host_lock_ref
# A Full host lock starts as a one-node segment. Splitting that node
# extends the segment through the inserted prefix, so its boundary
# moves to the prefix's older edge just like the SWA segment boundary.
host_uuid = child_cd.metadata.pop("host_uuid", None)
if host_uuid is not None:
new_parent.component_data[ct].metadata["host_uuid"] = host_uuid
assert new_parent.component_data[ct].session_ids is None
split_len = len(new_parent.key)
if child_cd.value is not None:
@@ -274,6 +282,9 @@ class FullComponent(TreeComponent):
# write_back mode: the anchor may be device-only (no host_value); pin it anyway.
if cd.host_value is None and not self.tree_core.is_write_back:
return result
if cd.metadata.get("host_uuid") is None:
cd.metadata["host_uuid"] = next_component_uuid(ct)
result.set_lock_uuid(ct, cd.metadata["host_uuid"], lock_host=True)
cd.host_lock_ref += 1
self.tree_core._update_evictable_leaf_sets(node)
return result
@@ -314,13 +325,27 @@ class FullComponent(TreeComponent):
) -> None:
ct = self.component_type
if lock_host:
cd = node.component_data[ct]
if cd.host_lock_ref == 0:
if ct not in params.component_host_lock_uuids:
# This receipt did not acquire the Full host component.
return
if cd.host_value is None and not self.tree_core.is_write_back:
return
cd.host_lock_ref -= 1
self.tree_core._update_evictable_leaf_sets(node)
boundary_uuid = params.get_lock_uuid(ct, lock_host=True)
assert boundary_uuid is not None, "Full host lock receipt has no boundary"
while True:
cd = node.component_data[ct]
if cd.host_value is None and not self.tree_core.is_write_back:
return
assert cd.host_lock_ref > 0, (
f"Full host segment release hit host_lock_ref=0 on node {node.id}"
)
cd.host_lock_ref -= 1
self.tree_core._update_evictable_leaf_sets(node)
if cd.metadata.get("host_uuid") == boundary_uuid:
break
assert node.parent is not None, (
f"Full host lock boundary {boundary_uuid} is not an ancestor "
f"of receipt anchor {params.node_id}"
)
node = node.parent
return
root = self.tree_core.root_node
@@ -843,7 +843,7 @@ class SWAComponent(TreeComponent):
covered += len(cur.key)
if covered >= sliding_window_size:
if comp.metadata.get(uuid_key) is None:
comp.metadata[uuid_key] = next_component_uuid()
comp.metadata[uuid_key] = next_component_uuid(ct)
swa_uuid = comp.metadata[uuid_key]
cur = cur.parent