[HiCache] Support Mamba in buffer-only host memory mode (#36345)

This commit is contained in:
Shuwen Wang
2026-10-12 00:35:16 -07:00
committed by GitHub
parent 4195980f5a
commit f5f5003323
11 changed files with 416 additions and 98 deletions
+29 -21
View File
@@ -797,20 +797,22 @@ class PrefillAdder:
chunk_limit=self.rem_chunk_tokens,
)
def _mamba_gap_budget_for_req(self, req: Req) -> int:
"""Shared-gap reservation (full-token-equivalents) for a request's new
mamba state. Charged only on the SHARED Mamba pool (`_mamba_slot_cost > 0`)
and only when the req has no state yet (`mamba_pool_idx is None`, mirroring
`HybridReqToTokenPool.alloc`); 0 keeps baseline / SWA / non-Mamba unchanged.
def _mamba_gap_budget_for_req(
self, req: Req, *, include_host_hit: bool = False
) -> int:
"""Reserve full-token equivalents for shared-pool Mamba state slots.
Conservative by design (`_mamba_slot_cost` rounds UP). Does NOT reserve
radix COW headroom or locked-but-evictable bytes — that residual is
backstopped by the fail-loud RuntimeError in `alloc_req_slots`. FIXME: if
over-admission crashes under pressure, make this more conservative (e.g.
also account for missing tracking buffers)."""
if self._mamba_slot_cost and not req.kv.holds_mamba:
return self._mamba_slot_cost
return 0
Admission includes the host checkpoint before load-back binds the
request slot. Continuing chunks only need the request-state budget;
their host-hit metadata may still describe an already consumed hit.
Tracking buffers and radix CoW headroom are not included here.
"""
if not self._mamba_slot_cost:
return 0
slots = int(not req.kv.holds_mamba)
if include_host_hit:
slots += req.mamba_host_hit_length
return slots * self._mamba_slot_cost
def ceil_paged_tokens(self, tokens: int) -> int:
return -(-tokens // self.page_size) * self.page_size
@@ -873,10 +875,9 @@ class PrefillAdder:
chunk_limit=self.rem_chunk_tokens,
is_chunked_continuation=is_chunked_continuation,
)
# The new mamba slot also consumes one mamba-recoverable slot (gated
# separately so full_evictable can't cover it — see __init__).
# Debit every reserved state slot, including the load-back checkpoint.
if mamba_gap_reserve and self.rem_mamba_slots is not None:
self.rem_mamba_slots -= 1
self.rem_mamba_slots -= mamba_gap_reserve // self._mamba_slot_cost
self.rem_input_tokens -= compute_charge
if self.dllm_config is not None:
@@ -1316,12 +1317,19 @@ class PrefillAdder:
)
cand_extend_input_len = len(req.full_untruncated_fill_ids) - req.prefix_len
total_tokens = cand_extend_input_len + max_new + self.per_req_token_overhead
# Shared Mamba pool: fold the new mamba state's shared-gap cost into
# `total_tokens` so both `rem_total_tokens` gates reflect the joint budget.
# Read before `init_load_back` binds `req.mamba_pool_idx` — after that
# this returns 0, so the debit sites below reuse the value.
mamba_gap_reserve = self._mamba_gap_budget_for_req(req)
# Reserve request and load-back state slots before init_load_back binds
# the request slot. Reuse this reservation when committing admission.
mamba_gap_reserve = self._mamba_gap_budget_for_req(req, include_host_hit=True)
total_tokens += mamba_gap_reserve
if self.rem_mamba_slots is not None:
required_mamba_slots = mamba_gap_reserve // self._mamba_slot_cost
if required_mamba_slots > self.rem_mamba_slots:
if (
isinstance(self.tree_cache, UnifiedRadixCache)
and self.tree_cache.buffer_pipeline is not None
):
self.tree_cache.buffer_pipeline.defer_staged_admission(req, "mamba")
return AddReqResult.NO_TOKEN
# The temporary pin excludes this prefix from the evictable budget.
# Selection itself neither allocates slots nor materializes host hits.
@@ -1275,7 +1275,7 @@ class BufferModePipeline:
return request in self.staged_prefetches
def prepare_staged_prefetch(self, req: Req) -> bool:
"""Rebuild the admission plan from this pass's joint FULL/SWA match."""
"""Rebuild the admission plan from this pass's joint component match."""
req.staged_prefetch_plan = None
f = self.staged_prefetches.get(req.cache_request_handle)
if f is None:
@@ -1315,24 +1315,31 @@ class BufferModePipeline:
for t in f.aux_xfers
if t.name == PoolName.SWA and t.host_indices is not None
)
if full_tokens == 0 and swa_tokens == 0:
mamba_slots = sum(
len(t.host_indices)
for t in f.aux_xfers
if t.name == PoolName.MAMBA and t.host_indices is not None
)
if full_tokens == 0 and swa_tokens == 0 and mamba_slots == 0:
self._resolve_device_covered(req)
return True
req.host_hit_length = full_tokens
req.swa_host_hit_length = swa_tokens
req.mamba_host_hit_length = mamba_slots
# The device alone serves only this pass's joint match; the rest of the
# span, fetched FULL or resident FULL the aux tail unlocks, is storage's.
req.storage_hit_length = f.matched_len + f.num_tokens - joint_len
req.storage_hit_start = joint_len
req.host_hit_is_storage = True
req.staged_prefetch_plan = StagedPrefetchPlan(
f.operation_id, key, matched_len, full_tokens, swa_tokens
f.operation_id, key, matched_len, full_tokens, swa_tokens, mamba_slots
)
return True
def _resolve_device_covered(self, req: Req) -> None:
req.host_hit_length = 0
req.swa_host_hit_length = 0
req.mamba_host_hit_length = 0
self._clear_storage_hit(req)
self.release_staged_hold(req.cache_request_handle, reason="device_covered")
@@ -1430,6 +1437,39 @@ class BufferModePipeline:
cache.prefetch_loaded_storage_start_by_reqid[request] = operation.storage_start
return True
def defer_staged_admission(self, req: Req, pool: str) -> None:
"""Retain staging for a bounded number of capacity retries."""
request = req.cache_request_handle
f = self.staged_prefetches.get(request)
if f is None:
return
cache = self._cache
defers = self._staged_admission_defers.get(request, 0) + 1
self._staged_admission_defers[request] = defers
cache._log_storage_prefetch_deferred(f.num_tokens, "device_capacity")
if defers < self.max_staged_admission_defers:
logger.warning(
"HiCache staged prefetch deferred at admission req=%s "
"reason=device_capacity pool=%s tokens=%d defers=%d",
req.rid,
pool,
f.num_tokens,
defers,
)
return
# Still unmaterializable: drop the hold so the admission loop stops
# breaking on this request, which recomputes on its next pass.
logger.warning(
"HiCache staged prefetch dropped after %d device_capacity "
"deferrals req=%s pool=%s tokens=%d",
defers,
req.rid,
pool,
f.num_tokens,
)
self.release_staged_hold(request, reason="device_capacity")
req.staged_prefetch_plan = None
def init_load_back(
self, params: InitLoadBackParams
) -> Optional[tuple[int, NodeId]]:
@@ -1440,10 +1480,10 @@ class BufferModePipeline:
prefix lock protects allocation-time eviction. None retains staging
and its anchor for the next admission attempt.
Ownership contract: cc.load queues the H2D before insert adjudicates
ownership, so the prepared boundary must ensure the insert can only
ADD nodes — a dedup would free slots the in-flight copy still
targets (queued use-after-free)."""
cc.load queues H2D before insert. Insert must preserve all FULL
destinations queued for H2D; freeing them during dedup would let the
pending copy overwrite reused slots. Free redundant auxiliary
destinations only after the transfer ack."""
cache = self._cache
req = params.req
assert req is not None
@@ -1458,38 +1498,16 @@ class BufferModePipeline:
assert plan is not None, f"staged prefetch was not planned for {req.rid}"
assert f.operation_id == plan.operation_id
assert (f.extra_key, f.cache_salt) == (req.extra_key, req.cache_salt)
assert (req.host_hit_length, req.swa_host_hit_length) == (
assert (
req.host_hit_length,
req.swa_host_hit_length,
req.mamba_host_hit_length,
) == (
plan.full_tokens,
plan.swa_tokens,
plan.mamba_slots,
), f"staged load-back budget changed for {req.rid}"
def _defer_for_capacity(pool: str) -> None:
defers = self._staged_admission_defers.get(request, 0) + 1
self._staged_admission_defers[request] = defers
cache._log_storage_prefetch_deferred(f.num_tokens, "device_capacity")
if defers < self.max_staged_admission_defers:
logger.warning(
"HiCache staged prefetch deferred at admission req=%s "
"reason=device_capacity pool=%s tokens=%d defers=%d",
req.rid,
pool,
f.num_tokens,
defers,
)
return
# Still unmaterializable: drop the hold so the admission loop stops
# breaking on this request, which recomputes on its next pass.
logger.warning(
"HiCache staged prefetch dropped after %d device_capacity "
"deferrals req=%s pool=%s tokens=%d",
defers,
req.rid,
pool,
f.num_tokens,
)
self.release_staged_hold(request, reason="device_capacity")
req.staged_prefetch_plan = None
splice_base = plan.device_prefix_len
assert req.prefix_len == splice_base
prefix_indices = self._cache.prefix_device_indices(req)
@@ -1517,7 +1535,7 @@ class BufferModePipeline:
else:
avail = cache.token_to_kv_pool_allocator.available_size()
if avail < load_tokens:
return _defer_for_capacity("full")
return self.defer_staged_admission(req, "full")
load_back_id = -(f.operation_id) - 1
# The full trailing-window aux transfer is independent of the shorter
@@ -1531,16 +1549,30 @@ class BufferModePipeline:
),
0,
)
component_load_contexts = []
for component in cache.components.values():
load_context = component.prepare_buffer_load_back(req, f.aux_xfers)
if load_context is None:
for load_context in component_load_contexts:
load_context.finalize_allocation(success=False)
return self.defer_staged_admission(
req, component.component_type.name.lower()
)
component_load_contexts.append(load_context)
# Not staging, so these stay out of the aux_xfers the ack frees.
load_xfers.extend(load_context.load_xfers)
device_indices = cc.load(
host_indices=f.host_indices[trim_tokens:],
node_id=load_back_id,
extra_pools=load_xfers or None,
)
for load_context in component_load_contexts:
load_context.finalize_allocation(success=device_indices is not None)
if device_indices is None:
# load() allocates all pools atomically before queueing H2D, so the
# staged host buffers remain reusable after either pool is short.
return _defer_for_capacity("full_or_aux")
return self.defer_staged_admission(req, "full_or_aux")
del self.staged_prefetches[request]
self._staged_admission_defers.pop(request, None)
req.staged_prefetch_plan = None
@@ -1610,6 +1642,9 @@ class BufferModePipeline:
# Publish via a plain insert under the admission lock choreography;
# the caller's request lock then pins the span (load_back pattern).
# prev_prefix_len covers the already-device-resident head.
insert_fields = {}
for load_context in component_load_contexts:
insert_fields.update(load_context.get_insert_fields())
insert_result = cache.insert(
InsertParams(
key=key,
@@ -1618,8 +1653,13 @@ class BufferModePipeline:
component_evicted_seqlens={
ComponentType.SWA: (span_end - staged_swa) if staged_swa else 0
},
**insert_fields,
)
)
for load_context in component_load_contexts:
aux_device_releases.extend(
load_context.get_redundant_device_slots(insert_result)
)
self.ongoing_buffer_load_back[load_back_id] = _OngoingBufferLoadBack(
request=f.request,
num_tokens=load_tokens,
@@ -83,3 +83,4 @@ class StagedPrefetchPlan:
device_prefix_len: int
full_tokens: int
swa_tokens: int
mamba_slots: int = 0
@@ -99,6 +99,30 @@ class PrepareLoadBackResult:
allocated_mamba_slot: Optional[torch.Tensor] = None
class BufferLoadBackContext:
"""A component's part in one buffer-mode load-back; the default has none.
``load_xfers`` are H2D destinations beyond the staged transfers. After
``cc.load``, ``finalize_allocation`` commits or rolls back preparation;
on success, ``get_insert_fields()`` supplies the node data for insert.
``get_redundant_device_slots()`` returns slots to free after the H2D ack.
"""
load_xfers: tuple[PoolTransfer, ...] = ()
def finalize_allocation(self, success: bool) -> None:
"""Commit or roll back after allocation; H2D may still be pending."""
pass
def get_insert_fields(self) -> dict[str, Any]:
return {}
def get_redundant_device_slots(
self, insert_result: InsertResult
) -> list[tuple[PoolName, torch.Tensor]]:
return []
@dataclasses.dataclass(frozen=True)
class PreparePrefetchResult:
"""Outcome of prepare_prefetch; default = the component takes no part."""
@@ -752,6 +776,16 @@ class TreeComponent(ABC):
policy."""
return None
def prepare_buffer_load_back(
self, req: Req, staged: list[PoolTransfer]
) -> Optional[BufferLoadBackContext]:
"""Validate staged data and prepare this component's load-back context.
Return None on insufficient device capacity. Invalid staged data is
an invariant violation, not a capacity failure.
"""
return BufferLoadBackContext()
def build_hicache_transfers(
self,
node: UnifiedTreeNode,
@@ -27,6 +27,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
)
from sglang.srt.mem_cache.unified_cache.components.base import (
BASE_COMPONENT_TYPE,
BufferLoadBackContext,
CacheTransferPhase,
ComponentType,
EvictLayer,
@@ -711,13 +712,18 @@ class MambaComponent(TreeComponent):
)
):
return PrepareLoadBackResult()
dst = self._alloc_request_state_slot(req)
assert dst is not None, "Cannot alloc mamba for load_back"
return PrepareLoadBackResult(allocated_mamba_slot=dst)
def _alloc_request_state_slot(self, req: Req) -> Optional[torch.Tensor]:
dst = self.cache.req_to_token_pool.mamba_allocator.alloc(1)
if dst is None:
self.cache.evict_for_alloc(EvictParams(num_tokens=0, mamba_num=1))
dst = self.cache.req_to_token_pool.mamba_allocator.alloc(1)
assert dst is not None, "Cannot alloc mamba for load_back"
req.kv.mamba_pool_idx = dst[0]
return PrepareLoadBackResult(allocated_mamba_slot=dst)
if dst is not None:
req.kv.mamba_pool_idx = dst[0]
return dst
def finalize_load_back(
self, req: Optional[Req], prep: PrepareLoadBackResult, success: bool
@@ -727,6 +733,23 @@ class MambaComponent(TreeComponent):
self.cache.req_to_token_pool.mamba_allocator.free(prep.allocated_mamba_slot)
req.kv.mamba_pool_idx = None
def prepare_buffer_load_back(
self, req: Req, staged: list[PoolTransfer]
) -> Optional[BufferLoadBackContext]:
node_copy = next((t for t in staged if t.name == PoolName.MAMBA), None)
assert (
node_copy is not None
and node_copy.host_indices is not None
and node_copy.host_indices.numel() == 1
), "Mamba buffer load-back requires one staged checkpoint"
prep = PrepareLoadBackResult()
if not req.kv.holds_mamba:
dst = self._alloc_request_state_slot(req)
if dst is None:
return None
prep = PrepareLoadBackResult(allocated_mamba_slot=dst)
return _MambaBufferLoadBackContext(self, req, node_copy, prep)
def prepare_prefetch(
self,
node_id: NodeId,
@@ -982,3 +1005,52 @@ class MambaComponent(TreeComponent):
raise AssertionError(
f"MambaComponent: unhandled ComponentAction {type(action).__name__}"
)
class _MambaBufferLoadBackContext(BufferLoadBackContext):
"""Loads the staged state into the published node's slot and, layer-gated,
into the request's own: a device CoW would run before the layer gate."""
def __init__(
self,
component: MambaComponent,
req: Req,
node_copy: PoolTransfer,
prep: PrepareLoadBackResult,
):
self.component = component
self.req = req
self.node_copy = node_copy
self.prep = prep
self.load_xfers = (
PoolTransfer(
name=PoolName.MAMBA,
host_indices=node_copy.host_indices,
device_indices=req.kv.mamba_pool_idx.unsqueeze(0),
),
)
def finalize_allocation(self, success: bool) -> None:
req = self.req
if not success:
self.component.finalize_load_back(req, self.prep, success=False)
return
# The H2D supersedes the replay cursor and the deferred CoW/clear.
req_to_token_pool = self.component.cache.req_to_token_pool
write_pos = req_to_token_pool.mamba_pool.replayssm_write_pos
if write_pos is not None:
slot = req.kv.mamba_pool_idx.unsqueeze(0)
write_pos[req_to_token_pool.translate_mamba_indices(slot)] = 0
req.kv.mamba_cow_src_index = None
req.kv.mamba_needs_clear = False
def get_insert_fields(self) -> dict[str, torch.Tensor]:
return {"mamba_value": self.node_copy.device_indices}
def get_redundant_device_slots(
self, insert_result: InsertResult
) -> list[tuple[PoolName, torch.Tensor]]:
# A SWA-only repair can keep the tail's existing checkpoint.
if not insert_result.mamba_exist:
return []
return [(PoolName.MAMBA, self.node_copy.device_indices)]
@@ -424,18 +424,11 @@ class UnifiedRadixCache(BasePrefixCache):
"""Initialize HiCache infrastructure."""
self.host_memory_mode = get_memory().hicache_host_memory_mode
if self.host_memory_mode == "buffer_only":
# TODO(Jialin): Extend buffer-only state handoff to Mamba in a
# follow-up to #34798 and #35769.
# FULL and FULL+SWA only: Mamba has no state-handoff channel on
# the admission-time load-back read path and is not layer-gated.
# Lifting the fence also needs the admission charge: a staged
# state slot is request-pinned at consumption and must ride
# req.mamba_host_hit_length the way the SWA window does.
supported = {ComponentType.FULL, ComponentType.SWA}
supported = {ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA}
if not set(self.tree_components) <= supported:
raise ValueError(
"--hicache-host-memory-mode buffer_only supports only "
"FULL/SWA unified trees; got components "
"FULL/SWA/MAMBA unified trees; got components "
f"{sorted(ct.name for ct in self.tree_components)}."
)
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (