diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index 54d7ff1166..15851c73e2 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -797,20 +797,22 @@ class PrefillAdder: chunk_limit=self.rem_chunk_tokens, ) - def _mamba_gap_budget_for_req(self, req: Req) -> int: - """Shared-gap reservation (full-token-equivalents) for a request's new - mamba state. Charged only on the SHARED Mamba pool (`_mamba_slot_cost > 0`) - and only when the req has no state yet (`mamba_pool_idx is None`, mirroring - `HybridReqToTokenPool.alloc`); 0 keeps baseline / SWA / non-Mamba unchanged. + def _mamba_gap_budget_for_req( + self, req: Req, *, include_host_hit: bool = False + ) -> int: + """Reserve full-token equivalents for shared-pool Mamba state slots. - Conservative by design (`_mamba_slot_cost` rounds UP). Does NOT reserve - radix COW headroom or locked-but-evictable bytes — that residual is - backstopped by the fail-loud RuntimeError in `alloc_req_slots`. FIXME: if - over-admission crashes under pressure, make this more conservative (e.g. - also account for missing tracking buffers).""" - if self._mamba_slot_cost and not req.kv.holds_mamba: - return self._mamba_slot_cost - return 0 + Admission includes the host checkpoint before load-back binds the + request slot. Continuing chunks only need the request-state budget; + their host-hit metadata may still describe an already consumed hit. + Tracking buffers and radix CoW headroom are not included here. + """ + if not self._mamba_slot_cost: + return 0 + slots = int(not req.kv.holds_mamba) + if include_host_hit: + slots += req.mamba_host_hit_length + return slots * self._mamba_slot_cost def ceil_paged_tokens(self, tokens: int) -> int: return -(-tokens // self.page_size) * self.page_size @@ -873,10 +875,9 @@ class PrefillAdder: chunk_limit=self.rem_chunk_tokens, is_chunked_continuation=is_chunked_continuation, ) - # The new mamba slot also consumes one mamba-recoverable slot (gated - # separately so full_evictable can't cover it — see __init__). + # Debit every reserved state slot, including the load-back checkpoint. if mamba_gap_reserve and self.rem_mamba_slots is not None: - self.rem_mamba_slots -= 1 + self.rem_mamba_slots -= mamba_gap_reserve // self._mamba_slot_cost self.rem_input_tokens -= compute_charge if self.dllm_config is not None: @@ -1316,12 +1317,19 @@ class PrefillAdder: ) cand_extend_input_len = len(req.full_untruncated_fill_ids) - req.prefix_len total_tokens = cand_extend_input_len + max_new + self.per_req_token_overhead - # Shared Mamba pool: fold the new mamba state's shared-gap cost into - # `total_tokens` so both `rem_total_tokens` gates reflect the joint budget. - # Read before `init_load_back` binds `req.mamba_pool_idx` — after that - # this returns 0, so the debit sites below reuse the value. - mamba_gap_reserve = self._mamba_gap_budget_for_req(req) + # Reserve request and load-back state slots before init_load_back binds + # the request slot. Reuse this reservation when committing admission. + mamba_gap_reserve = self._mamba_gap_budget_for_req(req, include_host_hit=True) total_tokens += mamba_gap_reserve + if self.rem_mamba_slots is not None: + required_mamba_slots = mamba_gap_reserve // self._mamba_slot_cost + if required_mamba_slots > self.rem_mamba_slots: + if ( + isinstance(self.tree_cache, UnifiedRadixCache) + and self.tree_cache.buffer_pipeline is not None + ): + self.tree_cache.buffer_pipeline.defer_staged_admission(req, "mamba") + return AddReqResult.NO_TOKEN # The temporary pin excludes this prefix from the evictable budget. # Selection itself neither allocates slots nor materializes host hits. diff --git a/python/sglang/srt/mem_cache/buffer_mode/pipeline.py b/python/sglang/srt/mem_cache/buffer_mode/pipeline.py index a196d28348..6d841dca75 100644 --- a/python/sglang/srt/mem_cache/buffer_mode/pipeline.py +++ b/python/sglang/srt/mem_cache/buffer_mode/pipeline.py @@ -1275,7 +1275,7 @@ class BufferModePipeline: return request in self.staged_prefetches def prepare_staged_prefetch(self, req: Req) -> bool: - """Rebuild the admission plan from this pass's joint FULL/SWA match.""" + """Rebuild the admission plan from this pass's joint component match.""" req.staged_prefetch_plan = None f = self.staged_prefetches.get(req.cache_request_handle) if f is None: @@ -1315,24 +1315,31 @@ class BufferModePipeline: for t in f.aux_xfers if t.name == PoolName.SWA and t.host_indices is not None ) - if full_tokens == 0 and swa_tokens == 0: + mamba_slots = sum( + len(t.host_indices) + for t in f.aux_xfers + if t.name == PoolName.MAMBA and t.host_indices is not None + ) + if full_tokens == 0 and swa_tokens == 0 and mamba_slots == 0: self._resolve_device_covered(req) return True req.host_hit_length = full_tokens req.swa_host_hit_length = swa_tokens + req.mamba_host_hit_length = mamba_slots # The device alone serves only this pass's joint match; the rest of the # span, fetched FULL or resident FULL the aux tail unlocks, is storage's. req.storage_hit_length = f.matched_len + f.num_tokens - joint_len req.storage_hit_start = joint_len req.host_hit_is_storage = True req.staged_prefetch_plan = StagedPrefetchPlan( - f.operation_id, key, matched_len, full_tokens, swa_tokens + f.operation_id, key, matched_len, full_tokens, swa_tokens, mamba_slots ) return True def _resolve_device_covered(self, req: Req) -> None: req.host_hit_length = 0 req.swa_host_hit_length = 0 + req.mamba_host_hit_length = 0 self._clear_storage_hit(req) self.release_staged_hold(req.cache_request_handle, reason="device_covered") @@ -1430,6 +1437,39 @@ class BufferModePipeline: cache.prefetch_loaded_storage_start_by_reqid[request] = operation.storage_start return True + def defer_staged_admission(self, req: Req, pool: str) -> None: + """Retain staging for a bounded number of capacity retries.""" + request = req.cache_request_handle + f = self.staged_prefetches.get(request) + if f is None: + return + cache = self._cache + defers = self._staged_admission_defers.get(request, 0) + 1 + self._staged_admission_defers[request] = defers + cache._log_storage_prefetch_deferred(f.num_tokens, "device_capacity") + if defers < self.max_staged_admission_defers: + logger.warning( + "HiCache staged prefetch deferred at admission req=%s " + "reason=device_capacity pool=%s tokens=%d defers=%d", + req.rid, + pool, + f.num_tokens, + defers, + ) + return + # Still unmaterializable: drop the hold so the admission loop stops + # breaking on this request, which recomputes on its next pass. + logger.warning( + "HiCache staged prefetch dropped after %d device_capacity " + "deferrals req=%s pool=%s tokens=%d", + defers, + req.rid, + pool, + f.num_tokens, + ) + self.release_staged_hold(request, reason="device_capacity") + req.staged_prefetch_plan = None + def init_load_back( self, params: InitLoadBackParams ) -> Optional[tuple[int, NodeId]]: @@ -1440,10 +1480,10 @@ class BufferModePipeline: prefix lock protects allocation-time eviction. None retains staging and its anchor for the next admission attempt. - Ownership contract: cc.load queues the H2D before insert adjudicates - ownership, so the prepared boundary must ensure the insert can only - ADD nodes — a dedup would free slots the in-flight copy still - targets (queued use-after-free).""" + cc.load queues H2D before insert. Insert must preserve all FULL + destinations queued for H2D; freeing them during dedup would let the + pending copy overwrite reused slots. Free redundant auxiliary + destinations only after the transfer ack.""" cache = self._cache req = params.req assert req is not None @@ -1458,38 +1498,16 @@ class BufferModePipeline: assert plan is not None, f"staged prefetch was not planned for {req.rid}" assert f.operation_id == plan.operation_id assert (f.extra_key, f.cache_salt) == (req.extra_key, req.cache_salt) - assert (req.host_hit_length, req.swa_host_hit_length) == ( + assert ( + req.host_hit_length, + req.swa_host_hit_length, + req.mamba_host_hit_length, + ) == ( plan.full_tokens, plan.swa_tokens, + plan.mamba_slots, ), f"staged load-back budget changed for {req.rid}" - def _defer_for_capacity(pool: str) -> None: - defers = self._staged_admission_defers.get(request, 0) + 1 - self._staged_admission_defers[request] = defers - cache._log_storage_prefetch_deferred(f.num_tokens, "device_capacity") - if defers < self.max_staged_admission_defers: - logger.warning( - "HiCache staged prefetch deferred at admission req=%s " - "reason=device_capacity pool=%s tokens=%d defers=%d", - req.rid, - pool, - f.num_tokens, - defers, - ) - return - # Still unmaterializable: drop the hold so the admission loop stops - # breaking on this request, which recomputes on its next pass. - logger.warning( - "HiCache staged prefetch dropped after %d device_capacity " - "deferrals req=%s pool=%s tokens=%d", - defers, - req.rid, - pool, - f.num_tokens, - ) - self.release_staged_hold(request, reason="device_capacity") - req.staged_prefetch_plan = None - splice_base = plan.device_prefix_len assert req.prefix_len == splice_base prefix_indices = self._cache.prefix_device_indices(req) @@ -1517,7 +1535,7 @@ class BufferModePipeline: else: avail = cache.token_to_kv_pool_allocator.available_size() if avail < load_tokens: - return _defer_for_capacity("full") + return self.defer_staged_admission(req, "full") load_back_id = -(f.operation_id) - 1 # The full trailing-window aux transfer is independent of the shorter @@ -1531,16 +1549,30 @@ class BufferModePipeline: ), 0, ) + component_load_contexts = [] + for component in cache.components.values(): + load_context = component.prepare_buffer_load_back(req, f.aux_xfers) + if load_context is None: + for load_context in component_load_contexts: + load_context.finalize_allocation(success=False) + return self.defer_staged_admission( + req, component.component_type.name.lower() + ) + component_load_contexts.append(load_context) + # Not staging, so these stay out of the aux_xfers the ack frees. + load_xfers.extend(load_context.load_xfers) device_indices = cc.load( host_indices=f.host_indices[trim_tokens:], node_id=load_back_id, extra_pools=load_xfers or None, ) + for load_context in component_load_contexts: + load_context.finalize_allocation(success=device_indices is not None) if device_indices is None: # load() allocates all pools atomically before queueing H2D, so the # staged host buffers remain reusable after either pool is short. - return _defer_for_capacity("full_or_aux") + return self.defer_staged_admission(req, "full_or_aux") del self.staged_prefetches[request] self._staged_admission_defers.pop(request, None) req.staged_prefetch_plan = None @@ -1610,6 +1642,9 @@ class BufferModePipeline: # Publish via a plain insert under the admission lock choreography; # the caller's request lock then pins the span (load_back pattern). # prev_prefix_len covers the already-device-resident head. + insert_fields = {} + for load_context in component_load_contexts: + insert_fields.update(load_context.get_insert_fields()) insert_result = cache.insert( InsertParams( key=key, @@ -1618,8 +1653,13 @@ class BufferModePipeline: component_evicted_seqlens={ ComponentType.SWA: (span_end - staged_swa) if staged_swa else 0 }, + **insert_fields, ) ) + for load_context in component_load_contexts: + aux_device_releases.extend( + load_context.get_redundant_device_slots(insert_result) + ) self.ongoing_buffer_load_back[load_back_id] = _OngoingBufferLoadBack( request=f.request, num_tokens=load_tokens, diff --git a/python/sglang/srt/mem_cache/storage_prefetch.py b/python/sglang/srt/mem_cache/storage_prefetch.py index c1de15a058..a265c266ff 100644 --- a/python/sglang/srt/mem_cache/storage_prefetch.py +++ b/python/sglang/srt/mem_cache/storage_prefetch.py @@ -83,3 +83,4 @@ class StagedPrefetchPlan: device_prefix_len: int full_tokens: int swa_tokens: int + mamba_slots: int = 0 diff --git a/python/sglang/srt/mem_cache/unified_cache/components/base.py b/python/sglang/srt/mem_cache/unified_cache/components/base.py index c627974c38..d098a8da76 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/base.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/base.py @@ -99,6 +99,30 @@ class PrepareLoadBackResult: allocated_mamba_slot: Optional[torch.Tensor] = None +class BufferLoadBackContext: + """A component's part in one buffer-mode load-back; the default has none. + + ``load_xfers`` are H2D destinations beyond the staged transfers. After + ``cc.load``, ``finalize_allocation`` commits or rolls back preparation; + on success, ``get_insert_fields()`` supplies the node data for insert. + ``get_redundant_device_slots()`` returns slots to free after the H2D ack. + """ + + load_xfers: tuple[PoolTransfer, ...] = () + + def finalize_allocation(self, success: bool) -> None: + """Commit or roll back after allocation; H2D may still be pending.""" + pass + + def get_insert_fields(self) -> dict[str, Any]: + return {} + + def get_redundant_device_slots( + self, insert_result: InsertResult + ) -> list[tuple[PoolName, torch.Tensor]]: + return [] + + @dataclasses.dataclass(frozen=True) class PreparePrefetchResult: """Outcome of prepare_prefetch; default = the component takes no part.""" @@ -752,6 +776,16 @@ class TreeComponent(ABC): policy.""" return None + def prepare_buffer_load_back( + self, req: Req, staged: list[PoolTransfer] + ) -> Optional[BufferLoadBackContext]: + """Validate staged data and prepare this component's load-back context. + + Return None on insufficient device capacity. Invalid staged data is + an invariant violation, not a capacity failure. + """ + return BufferLoadBackContext() + def build_hicache_transfers( self, node: UnifiedTreeNode, diff --git a/python/sglang/srt/mem_cache/unified_cache/components/mamba.py b/python/sglang/srt/mem_cache/unified_cache/components/mamba.py index 4df7314d0f..325cb92dec 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/mamba.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/mamba.py @@ -27,6 +27,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import ( ) from sglang.srt.mem_cache.unified_cache.components.base import ( BASE_COMPONENT_TYPE, + BufferLoadBackContext, CacheTransferPhase, ComponentType, EvictLayer, @@ -711,13 +712,18 @@ class MambaComponent(TreeComponent): ) ): return PrepareLoadBackResult() + dst = self._alloc_request_state_slot(req) + assert dst is not None, "Cannot alloc mamba for load_back" + return PrepareLoadBackResult(allocated_mamba_slot=dst) + + def _alloc_request_state_slot(self, req: Req) -> Optional[torch.Tensor]: dst = self.cache.req_to_token_pool.mamba_allocator.alloc(1) if dst is None: self.cache.evict_for_alloc(EvictParams(num_tokens=0, mamba_num=1)) dst = self.cache.req_to_token_pool.mamba_allocator.alloc(1) - assert dst is not None, "Cannot alloc mamba for load_back" - req.kv.mamba_pool_idx = dst[0] - return PrepareLoadBackResult(allocated_mamba_slot=dst) + if dst is not None: + req.kv.mamba_pool_idx = dst[0] + return dst def finalize_load_back( self, req: Optional[Req], prep: PrepareLoadBackResult, success: bool @@ -727,6 +733,23 @@ class MambaComponent(TreeComponent): self.cache.req_to_token_pool.mamba_allocator.free(prep.allocated_mamba_slot) req.kv.mamba_pool_idx = None + def prepare_buffer_load_back( + self, req: Req, staged: list[PoolTransfer] + ) -> Optional[BufferLoadBackContext]: + node_copy = next((t for t in staged if t.name == PoolName.MAMBA), None) + assert ( + node_copy is not None + and node_copy.host_indices is not None + and node_copy.host_indices.numel() == 1 + ), "Mamba buffer load-back requires one staged checkpoint" + prep = PrepareLoadBackResult() + if not req.kv.holds_mamba: + dst = self._alloc_request_state_slot(req) + if dst is None: + return None + prep = PrepareLoadBackResult(allocated_mamba_slot=dst) + return _MambaBufferLoadBackContext(self, req, node_copy, prep) + def prepare_prefetch( self, node_id: NodeId, @@ -982,3 +1005,52 @@ class MambaComponent(TreeComponent): raise AssertionError( f"MambaComponent: unhandled ComponentAction {type(action).__name__}" ) + + +class _MambaBufferLoadBackContext(BufferLoadBackContext): + """Loads the staged state into the published node's slot and, layer-gated, + into the request's own: a device CoW would run before the layer gate.""" + + def __init__( + self, + component: MambaComponent, + req: Req, + node_copy: PoolTransfer, + prep: PrepareLoadBackResult, + ): + self.component = component + self.req = req + self.node_copy = node_copy + self.prep = prep + self.load_xfers = ( + PoolTransfer( + name=PoolName.MAMBA, + host_indices=node_copy.host_indices, + device_indices=req.kv.mamba_pool_idx.unsqueeze(0), + ), + ) + + def finalize_allocation(self, success: bool) -> None: + req = self.req + if not success: + self.component.finalize_load_back(req, self.prep, success=False) + return + # The H2D supersedes the replay cursor and the deferred CoW/clear. + req_to_token_pool = self.component.cache.req_to_token_pool + write_pos = req_to_token_pool.mamba_pool.replayssm_write_pos + if write_pos is not None: + slot = req.kv.mamba_pool_idx.unsqueeze(0) + write_pos[req_to_token_pool.translate_mamba_indices(slot)] = 0 + req.kv.mamba_cow_src_index = None + req.kv.mamba_needs_clear = False + + def get_insert_fields(self) -> dict[str, torch.Tensor]: + return {"mamba_value": self.node_copy.device_indices} + + def get_redundant_device_slots( + self, insert_result: InsertResult + ) -> list[tuple[PoolName, torch.Tensor]]: + # A SWA-only repair can keep the tail's existing checkpoint. + if not insert_result.mamba_exist: + return [] + return [(PoolName.MAMBA, self.node_copy.device_indices)] diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index f3c7dc6a01..c62cdc08f6 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -424,18 +424,11 @@ class UnifiedRadixCache(BasePrefixCache): """Initialize HiCache infrastructure.""" self.host_memory_mode = get_memory().hicache_host_memory_mode if self.host_memory_mode == "buffer_only": - # TODO(Jialin): Extend buffer-only state handoff to Mamba in a - # follow-up to #34798 and #35769. - # FULL and FULL+SWA only: Mamba has no state-handoff channel on - # the admission-time load-back read path and is not layer-gated. - # Lifting the fence also needs the admission charge: a staged - # state slot is request-pinned at consumption and must ride - # req.mamba_host_hit_length the way the SWA window does. - supported = {ComponentType.FULL, ComponentType.SWA} + supported = {ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA} if not set(self.tree_components) <= supported: raise ValueError( "--hicache-host-memory-mode buffer_only supports only " - "FULL/SWA unified trees; got components " + "FULL/SWA/MAMBA unified trees; got components " f"{sorted(ct.name for ct in self.tree_components)}." ) from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import ( diff --git a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_hybrid_bitexact.py b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_hybrid_bitexact.py index 0365d34f3e..8ef70f352b 100644 --- a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_hybrid_bitexact.py +++ b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_hybrid_bitexact.py @@ -46,12 +46,18 @@ as tests. import os import random import re +import shutil +import tempfile import unittest import requests from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kl_multiturn_utils import ( + _extract_output_logprobs, + _flush_cache, + _generate, + _replay_and_compare_kl, make_mamba_decode_assert, ) from sglang.test.kl_multiturn_utils import ( @@ -78,7 +84,7 @@ from sglang.test.test_utils import ( unified_radix_tree_server_env, ) -register_cuda_ci(est_time=2640, stage="extra-a", runner_config="1-gpu-large") +register_cuda_ci(est_time=3140, stage="extra-a", runner_config="1-gpu-large") _MODEL_PATH = os.environ.get("INKLING_TEST_MODEL_PATH", "thinkingmachines/Inkling") _MODEL_REVISION = os.environ.get("INKLING_TEST_MODEL_REVISION", "test") @@ -100,6 +106,16 @@ PAGE_SIZE = 128 # handover from prompt tokens to generated ones. MAX_NEW_TOKENS = 1024 +# flush_cache drains staged file-backend writes, which can exceed 30 seconds. +_FLUSH_TIMEOUT_S = 600 + +# Seed a checkpoint strictly inside the prompt: prefix reuse excludes its last +# token. Keep file-backend writes bounded while decoding past the 512-token SWA. +BUFFER_ONLY_SAMPLES = 8 +BUFFER_ONLY_SEED_TOKENS = 512 +BUFFER_ONLY_PROMPT_TOKENS = 768 +BUFFER_ONLY_MAX_NEW_TOKENS = 640 + def _random_suffixes(n: int, length: int, seed: int) -> list[list[int]]: rng = random.Random(seed) @@ -107,7 +123,10 @@ def _random_suffixes(n: int, length: int, seed: int) -> list[list[int]]: def _base_args( - mamba_strategy: str = "extra_buffer", *, mem_fraction_static: float = 0.6 + mamba_strategy: str = "extra_buffer", + swa_full_tokens_ratio: str = "0.1", + *, + mem_fraction_static: float = 0.6, ) -> list[str]: return [ "--trust-remote-code", @@ -118,7 +137,7 @@ def _base_args( "--mamba-radix-cache-strategy", mamba_strategy, "--swa-full-tokens-ratio", - "0.1", + swa_full_tokens_ratio, "--mamba-full-memory-ratio", "0.1", # 0.85 was carried over from the 4-GPU B200 test and OOMs an 80 GB card: @@ -444,5 +463,110 @@ class TestRustUnifiedHybridMTPBitExact(TestUnifiedHybridMTPBitExact): tree_core_backend = "rust" +class TestUnifiedHybridBufferOnlyBitExact(CustomTestCase): + """Restore FULL/SWA/Mamba from L3 and compare against recomputed logprobs. + + Flushing the device tree after seeding ensures hits come from storage. + """ + + @classmethod + def setUpClass(cls): + cls.model = _MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.storage_dir = tempfile.mkdtemp(prefix="sgl-buffer-only-kl-") + # Keep all seed SWA windows resident until their backups are staged. + other_args = _base_args(swa_full_tokens_ratio="0.5") + [ + "--enable-hierarchical-cache", + "--hicache-host-memory-mode", + "buffer_only", + "--hicache-storage-backend", + "file", + "--hicache-storage-backend-extra-config", + '{"enable_metadata_cache": true}', + "--hicache-write-policy", + "write_through", + "--hicache-io-backend", + "direct", + # The mamba host pool only supports page_first and page_first_direct. + "--hicache-mem-layout", + "page_first_direct", + # Partial fetches would make the hit assertions timing-dependent. + "--hicache-storage-prefetch-policy", + "wait_complete", + "--chunked-prefill-size", + "2048", + "--max-total-tokens", + "65536", + "--max-mamba-cache-size", + "500", + "--max-running-requests", + "4", + ] + if _MODEL_REVISION: + other_args += ["--revision", _MODEL_REVISION] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + env=unified_radix_tree_server_env( + "python", SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR=cls.storage_dir + ), + ) + cls.input_ids = [ + ids[:BUFFER_ONLY_PROMPT_TOKENS] + for ids in get_input_ids( + tokenizer_path=cls.model, + num_samples=BUFFER_ONLY_SAMPLES, + trust_remote_code=True, + ) + ] + + @classmethod + def tearDownClass(cls): + if getattr(cls, "process", None) is not None: + terminate_and_kill_process_tree(cls.process, wait_timeout=60) + shutil.rmtree(cls.storage_dir, ignore_errors=True) + + def test_prefill_cache_hit_from_storage(self): + full_ids = self.input_ids + seed_ids = [ids[:BUFFER_ONLY_SEED_TOKENS] for ids in full_ids] + + # The second flush also waits for seed writes to reach L3. + _flush_cache(self.base_url, timeout_s=_FLUSH_TIMEOUT_S) + _generate(self.base_url, seed_ids, max_new_tokens=0) + _flush_cache(self.base_url, timeout_s=_FLUSH_TIMEOUT_S) + + results = _generate( + self.base_url, + full_ids, + BUFFER_ONLY_MAX_NEW_TOKENS, + return_logprob=True, + ) + self.assertEqual(len(results), len(full_ids)) + + # The device tree was flushed: every hit comes back from L3, on the + # checkpoint grid and within the seed. + cached = [r["meta_info"]["cached_tokens"] for r in results] + print(f"buffer_only storage hits: {cached}") + self.assertTrue(all(0 < h <= BUFFER_ONLY_SEED_TOKENS for h in cached), cached) + self.assertTrue(all(h % TRACK_INTERVAL == 0 for h in cached), cached) + + # Drain this pass's storage writes here: the replay's own flush uses + # the 30s default, which they can outlast. + _flush_cache(self.base_url, timeout_s=_FLUSH_TIMEOUT_S) + _replay_and_compare_kl( + self.base_url, + self.model, + KL_DIV_THRESHOLD, + [full_ids[i] + results[i]["output_ids"] for i in range(len(results))], + [_extract_output_logprobs(r) for r in results], + label="buffer_only_prefill_cache_hit", + # One batch avoids repeated 30s flushes while replay writes drain. + batch_size=BUFFER_ONLY_SAMPLES, + sampling_temperature=0, + ) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/managers/test_prefill_adder.py b/test/registered/unit/managers/test_prefill_adder.py index a8d11f9e58..6bd25c2312 100644 --- a/test/registered/unit/managers/test_prefill_adder.py +++ b/test/registered/unit/managers/test_prefill_adder.py @@ -135,6 +135,7 @@ class TestPrefillAdder(CustomTestCase): req.retracted_stain = False req.host_hit_length = 0 req.swa_host_hit_length = 0 + req.mamba_host_hit_length = 0 req.storage_hit_length = 0 req.storage_hit_start = None req.host_hit_is_storage = False @@ -1414,6 +1415,72 @@ class TestPrefillAdder(CustomTestCase): req.kv = SimpleNamespace(cache_protected_len=prefix_len) return req + def _build_mamba_restore_admission(self, *, holds_state, available_slots): + self.mock_token_allocator = self.create_token_allocator(available_size=100_000) + self.mock_tree_cache = self.create_tree_cache() + self.mock_tree_cache.mock_add_spec(schedule_policy.UnifiedRadixCache) + self.mock_tree_cache.buffer_pipeline = MagicMock() + adder = self.create_adder(self.create_running_batch()) + adder._mamba_slot_cost = 64 + adder.rem_mamba_slots = available_slots + req = self._create_host_hit_req(host_hit=32, tail=8) + req.mamba_host_hit_length = 1 + req.kv.holds_mamba = holds_state + + def load_back(params): + req.kv.holds_mamba = True + return req.host_hit_length, req.last_node + + self.mock_tree_cache.init_load_back.side_effect = load_back + return adder, req + + def test_mamba_restore_admission_counts_request_and_checkpoint(self): + for holds_state, slots, capacity, admitted in ( + (False, 1, 100_000, False), + (True, 0, 100_000, False), + (False, 2, 100_000, True), + (True, 1, 100_000, True), + (False, 2, 150, False), # Enough shared bytes for one state, not two. + ): + with self.subTest(holds_state=holds_state, slots=slots, capacity=capacity): + adder, req = self._build_mamba_restore_admission( + holds_state=holds_state, available_slots=slots + ) + self.mock_token_allocator.available_size.return_value = capacity + result = adder.add_one_req(req, False, None) + self.assertEqual(adder.can_run_list, [req] if admitted else []) + self.assertEqual(adder.rem_mamba_slots, 0 if admitted else slots) + if admitted: + self.mock_tree_cache.init_load_back.assert_called_once() + self.assertEqual( + adder.memory_budget.total_offset, 8 + 8 + 1 + slots * 64 + ) + else: + self.assertIs(result, AddReqResult.NO_TOKEN) + self.mock_tree_cache.init_load_back.assert_not_called() + self.assertEqual(adder.memory_budget.total_offset, 0) + if capacity == 100_000: + self.mock_tree_cache.buffer_pipeline.defer_staged_admission.assert_called_once_with( + req, "mamba" + ) + + def test_mamba_chunk_continuation_does_not_charge_restored_state_again(self): + adder, req = self._build_mamba_restore_admission( + holds_state=False, available_slots=2 + ) + adder.rem_chunk_tokens = 4 + adder.add_one_req(req, False, None) + self.assertIs(adder.new_chunked_req, req) + req.prefix_len = req.extend_end + # Continuing chunks retain the original host-hit metadata. + self.assertEqual(req.mamba_host_hit_length, 1) + next_adder = self.create_adder(self.create_running_batch(), rem_chunk_tokens=4) + next_adder._mamba_slot_cost = 64 + next_adder.rem_mamba_slots = 0 + self.assertIsNone(next_adder.add_chunked_req(req)) + self.assertEqual(next_adder.rem_mamba_slots, 0) + self.assertEqual(next_adder.memory_budget.total_offset, 4 + 8 + 1) + def test_successful_load_back_commits_the_selected_shape_once(self): cases = ( ("full", 0, 24, None, None, 8, 8), diff --git a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py index 338bd416fb..4689cf4b90 100644 --- a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py +++ b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py @@ -274,6 +274,7 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): handle = CacheRequestHandle("r", 0) pipeline = BufferModePipeline.__new__(BufferModePipeline) pipeline._cache = mock.Mock() + pipeline._cache.components = {} pipeline._cache.cache_controller.mem_pool_host.entry_map = { PoolName.SWA: SimpleNamespace( host_pool=SimpleNamespace(page_size=swa_page_size) diff --git a/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py b/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py index e455559869..f817e27a9a 100644 --- a/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py +++ b/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py @@ -63,6 +63,7 @@ def _staged_fixture(full_match=2): cache.ongoing_prefetch = {} cache._prefetch_outcome_stats = defaultdict(int) cache.tree_components = [] + cache.components = {} cache.prefetch_threshold = 2 cache._build_sidecar_transfers = Mock(return_value=[]) cache.supports_swa = lambda: True diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 3334ace755..203dbe3ac9 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -4010,29 +4010,6 @@ class UnifiedRadixCacheSuite: # Buffer-only host memory mode (host = transient staging, L3 = cache) # ================================================================ - def test_buffer_only_rejects_mamba(self): - if ( - self.cfg.components - != ( - ComponentType.FULL, - ComponentType.MAMBA, - ) - or self.cfg.page_size != 1 - ): - self.skipTest("one FULL+MAMBA page_size=1 fixture covers this guard") - - cache, _, _ = build_fixture(self.cfg) - storage_dir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, storage_dir, ignore_errors=True) - with self.assertRaisesRegex(ValueError, "supports only FULL/SWA"): - self._init_hicache( - cache, - storage_backend="file", - storage_dir=storage_dir, - prefetch_threshold=1, - host_memory_mode="buffer_only", - ) - def _init_buffer_hicache( self, cache, @@ -4043,8 +4020,8 @@ class UnifiedRadixCacheSuite: ): if self.cfg.has_mamba: self.skipTest( - "buffer_only is FULL/SWA-only (no Mamba state-handoff channel " - "on the admission-time load-back read path)" + "load-back requests here are Mocks; buffer_only Mamba is " + "covered by TestUnifiedHybridBufferOnlyBitExact" ) self._init_hicache( cache,