[HiCache]: Fix host lock ownership across radix splits (#38480)
Signed-off-by: Lukas Humbel <lhumbel@nvidia.com> Co-authored-by: Jialin Ouyang <jialino@meta.com> Co-authored-by: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com>
This commit is contained in:
co-authored by
Jialin Ouyang
Shuwen Wang
parent
ff219e159e
commit
4611ea365e
@@ -215,7 +215,10 @@ so the order is not load-bearing for the leaf sets. Full walks to root; SWA
|
||||
stops at the receipt boundary; components in `skipped_lock_components` are
|
||||
left alone. `skip_swa=True` also skips lower-priority components already
|
||||
released by `dec_swa_lock_only`. The host-side `dec_host_lock_ref` takes the
|
||||
same required receipt.
|
||||
same required receipt. A Full host lock begins as a one-node UUID-bounded
|
||||
segment. A later radix split copies the anchor's `host_lock_ref` to the inserted
|
||||
prefix and migrates the UUID to that prefix; release walks every fragment until
|
||||
it finds the receipt's UUID.
|
||||
|
||||
---
|
||||
|
||||
@@ -289,8 +292,8 @@ Each component implements these hooks. See `base.py` for the ABC and docstrings.
|
||||
|
||||
| Hook | Purpose | Called By | Default |
|
||||
|------|---------|-----------|----------|
|
||||
| `acquire_component_lock(lock_host=False)` | Increment device or host lock refs; moves device tokens from evictable to protected. Full: path-lock for device, single-node host lock. SWA: window-lock with UUID boundary. Mamba: single-node lock. | `inc_lock_ref`, `inc_host_lock_ref` | *abstract* |
|
||||
| `release_component_lock(lock_host=False)` | Decrement device or host lock refs; moves device tokens from protected to evictable when `lock_ref` → 0. Full path-unlocks device; SWA walks up to UUID boundary; Mamba unlocks a single node. | `dec_lock_ref`, `dec_host_lock_ref` | *abstract* |
|
||||
| `acquire_component_lock(lock_host=False)` | Increment device or host lock refs; moves device tokens from evictable to protected. Full: path-lock for device; its host lock starts as a one-node UUID-bounded segment so later split fragments remain protected. SWA: window-lock with UUID boundary. Mamba: single-node lock. | `inc_lock_ref`, `inc_host_lock_ref` | *abstract* |
|
||||
| `release_component_lock(lock_host=False)` | Decrement device or host lock refs; moves device tokens from protected to evictable when `lock_ref` -> 0. Full path-unlocks device and releases all host split fragments to the receipt boundary; SWA walks up to UUID boundary; Mamba unlocks a single node. | `dec_lock_ref`, `dec_host_lock_ref` | *abstract* |
|
||||
|
||||
### Caching Phase
|
||||
|
||||
|
||||
@@ -41,7 +41,18 @@ if TYPE_CHECKING:
|
||||
_NUM_COMPONENT_TYPES = len(ComponentType)
|
||||
|
||||
_LAST_ACCESS_TIME_COUNTER_FLOAT = float64(1.0)
|
||||
_COMPONENT_UUID_COUNTER = 1
|
||||
_COMPONENT_UUID_RANGE_SIZE = 100_000_000_000_000
|
||||
_COMPONENT_UUID_RANGE_INDEX = {
|
||||
ComponentType.SWA: 0,
|
||||
ComponentType.FULL: 1,
|
||||
ComponentType.MAMBA: 2,
|
||||
ComponentType.C128: 3,
|
||||
ComponentType.AUXILIARY_SWA: 4,
|
||||
}
|
||||
_COMPONENT_UUID_COUNTERS = {
|
||||
component_type: (range_index + 1) * _COMPONENT_UUID_RANGE_SIZE
|
||||
for component_type, range_index in _COMPONENT_UUID_RANGE_INDEX.items()
|
||||
}
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
@@ -121,10 +132,9 @@ def get_and_increase_time_counter() -> float64:
|
||||
return ret
|
||||
|
||||
|
||||
def next_component_uuid() -> int:
|
||||
global _COMPONENT_UUID_COUNTER
|
||||
_COMPONENT_UUID_COUNTER += 1
|
||||
return _COMPONENT_UUID_COUNTER
|
||||
def next_component_uuid(component_type: ComponentType) -> int:
|
||||
_COMPONENT_UUID_COUNTERS[component_type] += 1
|
||||
return _COMPONENT_UUID_COUNTERS[component_type]
|
||||
|
||||
|
||||
class TreeComponent(ABC):
|
||||
@@ -589,7 +599,7 @@ class TreeComponent(ABC):
|
||||
node itself (mamba state is per-leaf, not per-path).
|
||||
|
||||
When ``lock_host`` is True, the lock applies to host-side state:
|
||||
- Full: single-node host lock.
|
||||
- Full: a one-node UUID-bounded segment that expands across splits.
|
||||
- SWA: host window-lock with a dedicated host UUID boundary.
|
||||
- Mamba: single-node host lock with host LRU detach."""
|
||||
...
|
||||
@@ -610,7 +620,8 @@ class TreeComponent(ABC):
|
||||
- Mamba: single-node unlock — only decrements lock_ref on the
|
||||
node itself.
|
||||
|
||||
When ``lock_host`` is True, the inverse host-side semantics apply."""
|
||||
When ``lock_host`` is True, Full and SWA replay their host boundary
|
||||
UUIDs while Mamba retains its single-node semantics."""
|
||||
...
|
||||
|
||||
def prepare_for_caching_req(
|
||||
|
||||
@@ -26,6 +26,7 @@ from sglang.srt.mem_cache.unified_cache.components.base import (
|
||||
ExternalLinkerLoadPhase,
|
||||
LinkerTransferPhase,
|
||||
TreeComponent,
|
||||
next_component_uuid,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -146,6 +147,13 @@ class FullComponent(TreeComponent):
|
||||
new_parent.component_data[ct].lock_ref = child.component_data[ct].lock_ref
|
||||
new_parent.component_data[ct].session_ref = child.component_data[ct].session_ref
|
||||
child_cd = child.component_data[ct]
|
||||
new_parent.component_data[ct].host_lock_ref = child_cd.host_lock_ref
|
||||
# A Full host lock starts as a one-node segment. Splitting that node
|
||||
# extends the segment through the inserted prefix, so its boundary
|
||||
# moves to the prefix's older edge just like the SWA segment boundary.
|
||||
host_uuid = child_cd.metadata.pop("host_uuid", None)
|
||||
if host_uuid is not None:
|
||||
new_parent.component_data[ct].metadata["host_uuid"] = host_uuid
|
||||
assert new_parent.component_data[ct].session_ids is None
|
||||
split_len = len(new_parent.key)
|
||||
if child_cd.value is not None:
|
||||
@@ -274,6 +282,9 @@ class FullComponent(TreeComponent):
|
||||
# write_back mode: the anchor may be device-only (no host_value); pin it anyway.
|
||||
if cd.host_value is None and not self.tree_core.is_write_back:
|
||||
return result
|
||||
if cd.metadata.get("host_uuid") is None:
|
||||
cd.metadata["host_uuid"] = next_component_uuid(ct)
|
||||
result.set_lock_uuid(ct, cd.metadata["host_uuid"], lock_host=True)
|
||||
cd.host_lock_ref += 1
|
||||
self.tree_core._update_evictable_leaf_sets(node)
|
||||
return result
|
||||
@@ -314,13 +325,27 @@ class FullComponent(TreeComponent):
|
||||
) -> None:
|
||||
ct = self.component_type
|
||||
if lock_host:
|
||||
cd = node.component_data[ct]
|
||||
if cd.host_lock_ref == 0:
|
||||
if ct not in params.component_host_lock_uuids:
|
||||
# This receipt did not acquire the Full host component.
|
||||
return
|
||||
if cd.host_value is None and not self.tree_core.is_write_back:
|
||||
return
|
||||
cd.host_lock_ref -= 1
|
||||
self.tree_core._update_evictable_leaf_sets(node)
|
||||
boundary_uuid = params.get_lock_uuid(ct, lock_host=True)
|
||||
assert boundary_uuid is not None, "Full host lock receipt has no boundary"
|
||||
while True:
|
||||
cd = node.component_data[ct]
|
||||
if cd.host_value is None and not self.tree_core.is_write_back:
|
||||
return
|
||||
assert cd.host_lock_ref > 0, (
|
||||
f"Full host segment release hit host_lock_ref=0 on node {node.id}"
|
||||
)
|
||||
cd.host_lock_ref -= 1
|
||||
self.tree_core._update_evictable_leaf_sets(node)
|
||||
if cd.metadata.get("host_uuid") == boundary_uuid:
|
||||
break
|
||||
assert node.parent is not None, (
|
||||
f"Full host lock boundary {boundary_uuid} is not an ancestor "
|
||||
f"of receipt anchor {params.node_id}"
|
||||
)
|
||||
node = node.parent
|
||||
return
|
||||
|
||||
root = self.tree_core.root_node
|
||||
|
||||
@@ -843,7 +843,7 @@ class SWAComponent(TreeComponent):
|
||||
covered += len(cur.key)
|
||||
if covered >= sliding_window_size:
|
||||
if comp.metadata.get(uuid_key) is None:
|
||||
comp.metadata[uuid_key] = next_component_uuid()
|
||||
comp.metadata[uuid_key] = next_component_uuid(ct)
|
||||
swa_uuid = comp.metadata[uuid_key]
|
||||
cur = cur.parent
|
||||
|
||||
|
||||
Reference in New Issue
Block a user