[HiCache] Support Mamba in buffer-only host memory mode (#36345)
This commit is contained in:
@@ -797,20 +797,22 @@ class PrefillAdder:
|
||||
chunk_limit=self.rem_chunk_tokens,
|
||||
)
|
||||
|
||||
def _mamba_gap_budget_for_req(self, req: Req) -> int:
|
||||
"""Shared-gap reservation (full-token-equivalents) for a request's new
|
||||
mamba state. Charged only on the SHARED Mamba pool (`_mamba_slot_cost > 0`)
|
||||
and only when the req has no state yet (`mamba_pool_idx is None`, mirroring
|
||||
`HybridReqToTokenPool.alloc`); 0 keeps baseline / SWA / non-Mamba unchanged.
|
||||
def _mamba_gap_budget_for_req(
|
||||
self, req: Req, *, include_host_hit: bool = False
|
||||
) -> int:
|
||||
"""Reserve full-token equivalents for shared-pool Mamba state slots.
|
||||
|
||||
Conservative by design (`_mamba_slot_cost` rounds UP). Does NOT reserve
|
||||
radix COW headroom or locked-but-evictable bytes — that residual is
|
||||
backstopped by the fail-loud RuntimeError in `alloc_req_slots`. FIXME: if
|
||||
over-admission crashes under pressure, make this more conservative (e.g.
|
||||
also account for missing tracking buffers)."""
|
||||
if self._mamba_slot_cost and not req.kv.holds_mamba:
|
||||
return self._mamba_slot_cost
|
||||
return 0
|
||||
Admission includes the host checkpoint before load-back binds the
|
||||
request slot. Continuing chunks only need the request-state budget;
|
||||
their host-hit metadata may still describe an already consumed hit.
|
||||
Tracking buffers and radix CoW headroom are not included here.
|
||||
"""
|
||||
if not self._mamba_slot_cost:
|
||||
return 0
|
||||
slots = int(not req.kv.holds_mamba)
|
||||
if include_host_hit:
|
||||
slots += req.mamba_host_hit_length
|
||||
return slots * self._mamba_slot_cost
|
||||
|
||||
def ceil_paged_tokens(self, tokens: int) -> int:
|
||||
return -(-tokens // self.page_size) * self.page_size
|
||||
@@ -873,10 +875,9 @@ class PrefillAdder:
|
||||
chunk_limit=self.rem_chunk_tokens,
|
||||
is_chunked_continuation=is_chunked_continuation,
|
||||
)
|
||||
# The new mamba slot also consumes one mamba-recoverable slot (gated
|
||||
# separately so full_evictable can't cover it — see __init__).
|
||||
# Debit every reserved state slot, including the load-back checkpoint.
|
||||
if mamba_gap_reserve and self.rem_mamba_slots is not None:
|
||||
self.rem_mamba_slots -= 1
|
||||
self.rem_mamba_slots -= mamba_gap_reserve // self._mamba_slot_cost
|
||||
self.rem_input_tokens -= compute_charge
|
||||
|
||||
if self.dllm_config is not None:
|
||||
@@ -1316,12 +1317,19 @@ class PrefillAdder:
|
||||
)
|
||||
cand_extend_input_len = len(req.full_untruncated_fill_ids) - req.prefix_len
|
||||
total_tokens = cand_extend_input_len + max_new + self.per_req_token_overhead
|
||||
# Shared Mamba pool: fold the new mamba state's shared-gap cost into
|
||||
# `total_tokens` so both `rem_total_tokens` gates reflect the joint budget.
|
||||
# Read before `init_load_back` binds `req.mamba_pool_idx` — after that
|
||||
# this returns 0, so the debit sites below reuse the value.
|
||||
mamba_gap_reserve = self._mamba_gap_budget_for_req(req)
|
||||
# Reserve request and load-back state slots before init_load_back binds
|
||||
# the request slot. Reuse this reservation when committing admission.
|
||||
mamba_gap_reserve = self._mamba_gap_budget_for_req(req, include_host_hit=True)
|
||||
total_tokens += mamba_gap_reserve
|
||||
if self.rem_mamba_slots is not None:
|
||||
required_mamba_slots = mamba_gap_reserve // self._mamba_slot_cost
|
||||
if required_mamba_slots > self.rem_mamba_slots:
|
||||
if (
|
||||
isinstance(self.tree_cache, UnifiedRadixCache)
|
||||
and self.tree_cache.buffer_pipeline is not None
|
||||
):
|
||||
self.tree_cache.buffer_pipeline.defer_staged_admission(req, "mamba")
|
||||
return AddReqResult.NO_TOKEN
|
||||
|
||||
# The temporary pin excludes this prefix from the evictable budget.
|
||||
# Selection itself neither allocates slots nor materializes host hits.
|
||||
|
||||
@@ -1275,7 +1275,7 @@ class BufferModePipeline:
|
||||
return request in self.staged_prefetches
|
||||
|
||||
def prepare_staged_prefetch(self, req: Req) -> bool:
|
||||
"""Rebuild the admission plan from this pass's joint FULL/SWA match."""
|
||||
"""Rebuild the admission plan from this pass's joint component match."""
|
||||
req.staged_prefetch_plan = None
|
||||
f = self.staged_prefetches.get(req.cache_request_handle)
|
||||
if f is None:
|
||||
@@ -1315,24 +1315,31 @@ class BufferModePipeline:
|
||||
for t in f.aux_xfers
|
||||
if t.name == PoolName.SWA and t.host_indices is not None
|
||||
)
|
||||
if full_tokens == 0 and swa_tokens == 0:
|
||||
mamba_slots = sum(
|
||||
len(t.host_indices)
|
||||
for t in f.aux_xfers
|
||||
if t.name == PoolName.MAMBA and t.host_indices is not None
|
||||
)
|
||||
if full_tokens == 0 and swa_tokens == 0 and mamba_slots == 0:
|
||||
self._resolve_device_covered(req)
|
||||
return True
|
||||
req.host_hit_length = full_tokens
|
||||
req.swa_host_hit_length = swa_tokens
|
||||
req.mamba_host_hit_length = mamba_slots
|
||||
# The device alone serves only this pass's joint match; the rest of the
|
||||
# span, fetched FULL or resident FULL the aux tail unlocks, is storage's.
|
||||
req.storage_hit_length = f.matched_len + f.num_tokens - joint_len
|
||||
req.storage_hit_start = joint_len
|
||||
req.host_hit_is_storage = True
|
||||
req.staged_prefetch_plan = StagedPrefetchPlan(
|
||||
f.operation_id, key, matched_len, full_tokens, swa_tokens
|
||||
f.operation_id, key, matched_len, full_tokens, swa_tokens, mamba_slots
|
||||
)
|
||||
return True
|
||||
|
||||
def _resolve_device_covered(self, req: Req) -> None:
|
||||
req.host_hit_length = 0
|
||||
req.swa_host_hit_length = 0
|
||||
req.mamba_host_hit_length = 0
|
||||
self._clear_storage_hit(req)
|
||||
self.release_staged_hold(req.cache_request_handle, reason="device_covered")
|
||||
|
||||
@@ -1430,6 +1437,39 @@ class BufferModePipeline:
|
||||
cache.prefetch_loaded_storage_start_by_reqid[request] = operation.storage_start
|
||||
return True
|
||||
|
||||
def defer_staged_admission(self, req: Req, pool: str) -> None:
|
||||
"""Retain staging for a bounded number of capacity retries."""
|
||||
request = req.cache_request_handle
|
||||
f = self.staged_prefetches.get(request)
|
||||
if f is None:
|
||||
return
|
||||
cache = self._cache
|
||||
defers = self._staged_admission_defers.get(request, 0) + 1
|
||||
self._staged_admission_defers[request] = defers
|
||||
cache._log_storage_prefetch_deferred(f.num_tokens, "device_capacity")
|
||||
if defers < self.max_staged_admission_defers:
|
||||
logger.warning(
|
||||
"HiCache staged prefetch deferred at admission req=%s "
|
||||
"reason=device_capacity pool=%s tokens=%d defers=%d",
|
||||
req.rid,
|
||||
pool,
|
||||
f.num_tokens,
|
||||
defers,
|
||||
)
|
||||
return
|
||||
# Still unmaterializable: drop the hold so the admission loop stops
|
||||
# breaking on this request, which recomputes on its next pass.
|
||||
logger.warning(
|
||||
"HiCache staged prefetch dropped after %d device_capacity "
|
||||
"deferrals req=%s pool=%s tokens=%d",
|
||||
defers,
|
||||
req.rid,
|
||||
pool,
|
||||
f.num_tokens,
|
||||
)
|
||||
self.release_staged_hold(request, reason="device_capacity")
|
||||
req.staged_prefetch_plan = None
|
||||
|
||||
def init_load_back(
|
||||
self, params: InitLoadBackParams
|
||||
) -> Optional[tuple[int, NodeId]]:
|
||||
@@ -1440,10 +1480,10 @@ class BufferModePipeline:
|
||||
prefix lock protects allocation-time eviction. None retains staging
|
||||
and its anchor for the next admission attempt.
|
||||
|
||||
Ownership contract: cc.load queues the H2D before insert adjudicates
|
||||
ownership, so the prepared boundary must ensure the insert can only
|
||||
ADD nodes — a dedup would free slots the in-flight copy still
|
||||
targets (queued use-after-free)."""
|
||||
cc.load queues H2D before insert. Insert must preserve all FULL
|
||||
destinations queued for H2D; freeing them during dedup would let the
|
||||
pending copy overwrite reused slots. Free redundant auxiliary
|
||||
destinations only after the transfer ack."""
|
||||
cache = self._cache
|
||||
req = params.req
|
||||
assert req is not None
|
||||
@@ -1458,38 +1498,16 @@ class BufferModePipeline:
|
||||
assert plan is not None, f"staged prefetch was not planned for {req.rid}"
|
||||
assert f.operation_id == plan.operation_id
|
||||
assert (f.extra_key, f.cache_salt) == (req.extra_key, req.cache_salt)
|
||||
assert (req.host_hit_length, req.swa_host_hit_length) == (
|
||||
assert (
|
||||
req.host_hit_length,
|
||||
req.swa_host_hit_length,
|
||||
req.mamba_host_hit_length,
|
||||
) == (
|
||||
plan.full_tokens,
|
||||
plan.swa_tokens,
|
||||
plan.mamba_slots,
|
||||
), f"staged load-back budget changed for {req.rid}"
|
||||
|
||||
def _defer_for_capacity(pool: str) -> None:
|
||||
defers = self._staged_admission_defers.get(request, 0) + 1
|
||||
self._staged_admission_defers[request] = defers
|
||||
cache._log_storage_prefetch_deferred(f.num_tokens, "device_capacity")
|
||||
if defers < self.max_staged_admission_defers:
|
||||
logger.warning(
|
||||
"HiCache staged prefetch deferred at admission req=%s "
|
||||
"reason=device_capacity pool=%s tokens=%d defers=%d",
|
||||
req.rid,
|
||||
pool,
|
||||
f.num_tokens,
|
||||
defers,
|
||||
)
|
||||
return
|
||||
# Still unmaterializable: drop the hold so the admission loop stops
|
||||
# breaking on this request, which recomputes on its next pass.
|
||||
logger.warning(
|
||||
"HiCache staged prefetch dropped after %d device_capacity "
|
||||
"deferrals req=%s pool=%s tokens=%d",
|
||||
defers,
|
||||
req.rid,
|
||||
pool,
|
||||
f.num_tokens,
|
||||
)
|
||||
self.release_staged_hold(request, reason="device_capacity")
|
||||
req.staged_prefetch_plan = None
|
||||
|
||||
splice_base = plan.device_prefix_len
|
||||
assert req.prefix_len == splice_base
|
||||
prefix_indices = self._cache.prefix_device_indices(req)
|
||||
@@ -1517,7 +1535,7 @@ class BufferModePipeline:
|
||||
else:
|
||||
avail = cache.token_to_kv_pool_allocator.available_size()
|
||||
if avail < load_tokens:
|
||||
return _defer_for_capacity("full")
|
||||
return self.defer_staged_admission(req, "full")
|
||||
|
||||
load_back_id = -(f.operation_id) - 1
|
||||
# The full trailing-window aux transfer is independent of the shorter
|
||||
@@ -1531,16 +1549,30 @@ class BufferModePipeline:
|
||||
),
|
||||
0,
|
||||
)
|
||||
component_load_contexts = []
|
||||
for component in cache.components.values():
|
||||
load_context = component.prepare_buffer_load_back(req, f.aux_xfers)
|
||||
if load_context is None:
|
||||
for load_context in component_load_contexts:
|
||||
load_context.finalize_allocation(success=False)
|
||||
return self.defer_staged_admission(
|
||||
req, component.component_type.name.lower()
|
||||
)
|
||||
component_load_contexts.append(load_context)
|
||||
# Not staging, so these stay out of the aux_xfers the ack frees.
|
||||
load_xfers.extend(load_context.load_xfers)
|
||||
|
||||
device_indices = cc.load(
|
||||
host_indices=f.host_indices[trim_tokens:],
|
||||
node_id=load_back_id,
|
||||
extra_pools=load_xfers or None,
|
||||
)
|
||||
for load_context in component_load_contexts:
|
||||
load_context.finalize_allocation(success=device_indices is not None)
|
||||
if device_indices is None:
|
||||
# load() allocates all pools atomically before queueing H2D, so the
|
||||
# staged host buffers remain reusable after either pool is short.
|
||||
return _defer_for_capacity("full_or_aux")
|
||||
return self.defer_staged_admission(req, "full_or_aux")
|
||||
del self.staged_prefetches[request]
|
||||
self._staged_admission_defers.pop(request, None)
|
||||
req.staged_prefetch_plan = None
|
||||
@@ -1610,6 +1642,9 @@ class BufferModePipeline:
|
||||
# Publish via a plain insert under the admission lock choreography;
|
||||
# the caller's request lock then pins the span (load_back pattern).
|
||||
# prev_prefix_len covers the already-device-resident head.
|
||||
insert_fields = {}
|
||||
for load_context in component_load_contexts:
|
||||
insert_fields.update(load_context.get_insert_fields())
|
||||
insert_result = cache.insert(
|
||||
InsertParams(
|
||||
key=key,
|
||||
@@ -1618,8 +1653,13 @@ class BufferModePipeline:
|
||||
component_evicted_seqlens={
|
||||
ComponentType.SWA: (span_end - staged_swa) if staged_swa else 0
|
||||
},
|
||||
**insert_fields,
|
||||
)
|
||||
)
|
||||
for load_context in component_load_contexts:
|
||||
aux_device_releases.extend(
|
||||
load_context.get_redundant_device_slots(insert_result)
|
||||
)
|
||||
self.ongoing_buffer_load_back[load_back_id] = _OngoingBufferLoadBack(
|
||||
request=f.request,
|
||||
num_tokens=load_tokens,
|
||||
|
||||
@@ -83,3 +83,4 @@ class StagedPrefetchPlan:
|
||||
device_prefix_len: int
|
||||
full_tokens: int
|
||||
swa_tokens: int
|
||||
mamba_slots: int = 0
|
||||
|
||||
@@ -99,6 +99,30 @@ class PrepareLoadBackResult:
|
||||
allocated_mamba_slot: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
class BufferLoadBackContext:
|
||||
"""A component's part in one buffer-mode load-back; the default has none.
|
||||
|
||||
``load_xfers`` are H2D destinations beyond the staged transfers. After
|
||||
``cc.load``, ``finalize_allocation`` commits or rolls back preparation;
|
||||
on success, ``get_insert_fields()`` supplies the node data for insert.
|
||||
``get_redundant_device_slots()`` returns slots to free after the H2D ack.
|
||||
"""
|
||||
|
||||
load_xfers: tuple[PoolTransfer, ...] = ()
|
||||
|
||||
def finalize_allocation(self, success: bool) -> None:
|
||||
"""Commit or roll back after allocation; H2D may still be pending."""
|
||||
pass
|
||||
|
||||
def get_insert_fields(self) -> dict[str, Any]:
|
||||
return {}
|
||||
|
||||
def get_redundant_device_slots(
|
||||
self, insert_result: InsertResult
|
||||
) -> list[tuple[PoolName, torch.Tensor]]:
|
||||
return []
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class PreparePrefetchResult:
|
||||
"""Outcome of prepare_prefetch; default = the component takes no part."""
|
||||
@@ -752,6 +776,16 @@ class TreeComponent(ABC):
|
||||
policy."""
|
||||
return None
|
||||
|
||||
def prepare_buffer_load_back(
|
||||
self, req: Req, staged: list[PoolTransfer]
|
||||
) -> Optional[BufferLoadBackContext]:
|
||||
"""Validate staged data and prepare this component's load-back context.
|
||||
|
||||
Return None on insufficient device capacity. Invalid staged data is
|
||||
an invariant violation, not a capacity failure.
|
||||
"""
|
||||
return BufferLoadBackContext()
|
||||
|
||||
def build_hicache_transfers(
|
||||
self,
|
||||
node: UnifiedTreeNode,
|
||||
|
||||
@@ -27,6 +27,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.components.base import (
|
||||
BASE_COMPONENT_TYPE,
|
||||
BufferLoadBackContext,
|
||||
CacheTransferPhase,
|
||||
ComponentType,
|
||||
EvictLayer,
|
||||
@@ -711,13 +712,18 @@ class MambaComponent(TreeComponent):
|
||||
)
|
||||
):
|
||||
return PrepareLoadBackResult()
|
||||
dst = self._alloc_request_state_slot(req)
|
||||
assert dst is not None, "Cannot alloc mamba for load_back"
|
||||
return PrepareLoadBackResult(allocated_mamba_slot=dst)
|
||||
|
||||
def _alloc_request_state_slot(self, req: Req) -> Optional[torch.Tensor]:
|
||||
dst = self.cache.req_to_token_pool.mamba_allocator.alloc(1)
|
||||
if dst is None:
|
||||
self.cache.evict_for_alloc(EvictParams(num_tokens=0, mamba_num=1))
|
||||
dst = self.cache.req_to_token_pool.mamba_allocator.alloc(1)
|
||||
assert dst is not None, "Cannot alloc mamba for load_back"
|
||||
req.kv.mamba_pool_idx = dst[0]
|
||||
return PrepareLoadBackResult(allocated_mamba_slot=dst)
|
||||
if dst is not None:
|
||||
req.kv.mamba_pool_idx = dst[0]
|
||||
return dst
|
||||
|
||||
def finalize_load_back(
|
||||
self, req: Optional[Req], prep: PrepareLoadBackResult, success: bool
|
||||
@@ -727,6 +733,23 @@ class MambaComponent(TreeComponent):
|
||||
self.cache.req_to_token_pool.mamba_allocator.free(prep.allocated_mamba_slot)
|
||||
req.kv.mamba_pool_idx = None
|
||||
|
||||
def prepare_buffer_load_back(
|
||||
self, req: Req, staged: list[PoolTransfer]
|
||||
) -> Optional[BufferLoadBackContext]:
|
||||
node_copy = next((t for t in staged if t.name == PoolName.MAMBA), None)
|
||||
assert (
|
||||
node_copy is not None
|
||||
and node_copy.host_indices is not None
|
||||
and node_copy.host_indices.numel() == 1
|
||||
), "Mamba buffer load-back requires one staged checkpoint"
|
||||
prep = PrepareLoadBackResult()
|
||||
if not req.kv.holds_mamba:
|
||||
dst = self._alloc_request_state_slot(req)
|
||||
if dst is None:
|
||||
return None
|
||||
prep = PrepareLoadBackResult(allocated_mamba_slot=dst)
|
||||
return _MambaBufferLoadBackContext(self, req, node_copy, prep)
|
||||
|
||||
def prepare_prefetch(
|
||||
self,
|
||||
node_id: NodeId,
|
||||
@@ -982,3 +1005,52 @@ class MambaComponent(TreeComponent):
|
||||
raise AssertionError(
|
||||
f"MambaComponent: unhandled ComponentAction {type(action).__name__}"
|
||||
)
|
||||
|
||||
|
||||
class _MambaBufferLoadBackContext(BufferLoadBackContext):
|
||||
"""Loads the staged state into the published node's slot and, layer-gated,
|
||||
into the request's own: a device CoW would run before the layer gate."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
component: MambaComponent,
|
||||
req: Req,
|
||||
node_copy: PoolTransfer,
|
||||
prep: PrepareLoadBackResult,
|
||||
):
|
||||
self.component = component
|
||||
self.req = req
|
||||
self.node_copy = node_copy
|
||||
self.prep = prep
|
||||
self.load_xfers = (
|
||||
PoolTransfer(
|
||||
name=PoolName.MAMBA,
|
||||
host_indices=node_copy.host_indices,
|
||||
device_indices=req.kv.mamba_pool_idx.unsqueeze(0),
|
||||
),
|
||||
)
|
||||
|
||||
def finalize_allocation(self, success: bool) -> None:
|
||||
req = self.req
|
||||
if not success:
|
||||
self.component.finalize_load_back(req, self.prep, success=False)
|
||||
return
|
||||
# The H2D supersedes the replay cursor and the deferred CoW/clear.
|
||||
req_to_token_pool = self.component.cache.req_to_token_pool
|
||||
write_pos = req_to_token_pool.mamba_pool.replayssm_write_pos
|
||||
if write_pos is not None:
|
||||
slot = req.kv.mamba_pool_idx.unsqueeze(0)
|
||||
write_pos[req_to_token_pool.translate_mamba_indices(slot)] = 0
|
||||
req.kv.mamba_cow_src_index = None
|
||||
req.kv.mamba_needs_clear = False
|
||||
|
||||
def get_insert_fields(self) -> dict[str, torch.Tensor]:
|
||||
return {"mamba_value": self.node_copy.device_indices}
|
||||
|
||||
def get_redundant_device_slots(
|
||||
self, insert_result: InsertResult
|
||||
) -> list[tuple[PoolName, torch.Tensor]]:
|
||||
# A SWA-only repair can keep the tail's existing checkpoint.
|
||||
if not insert_result.mamba_exist:
|
||||
return []
|
||||
return [(PoolName.MAMBA, self.node_copy.device_indices)]
|
||||
|
||||
@@ -424,18 +424,11 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
"""Initialize HiCache infrastructure."""
|
||||
self.host_memory_mode = get_memory().hicache_host_memory_mode
|
||||
if self.host_memory_mode == "buffer_only":
|
||||
# TODO(Jialin): Extend buffer-only state handoff to Mamba in a
|
||||
# follow-up to #34798 and #35769.
|
||||
# FULL and FULL+SWA only: Mamba has no state-handoff channel on
|
||||
# the admission-time load-back read path and is not layer-gated.
|
||||
# Lifting the fence also needs the admission charge: a staged
|
||||
# state slot is request-pinned at consumption and must ride
|
||||
# req.mamba_host_hit_length the way the SWA window does.
|
||||
supported = {ComponentType.FULL, ComponentType.SWA}
|
||||
supported = {ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA}
|
||||
if not set(self.tree_components) <= supported:
|
||||
raise ValueError(
|
||||
"--hicache-host-memory-mode buffer_only supports only "
|
||||
"FULL/SWA unified trees; got components "
|
||||
"FULL/SWA/MAMBA unified trees; got components "
|
||||
f"{sorted(ct.name for ct in self.tree_components)}."
|
||||
)
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
|
||||
|
||||
Reference in New Issue
Block a user