hicache: pipeline prefetch/backup IO lanes + stage timing
- prefetch_io_aux_func: allow up to SGLANG_HICACHE_PREFETCH_IO_THREADS transfers in flight (default 1 = legacy behaviour); PrefetchAck is still emitted strictly FIFO so the per-rank ack stream (and the TP all_reduce in prefetch_sync_thread_func) is unchanged. - backup_thread_func: same pipelining for L2->L3 writes via SGLANG_HICACHE_BACKUP_IO_THREADS. - SGLANG_HICACHE_TIMING=1: emit [HC_TIMING] prefetch/backup queue, inflight, p50/p95 per 100 ops (the prefetch/backup lanes were previously single threaded; this is what caps L3<->L2 at ~1 op at a time).
This commit is contained in:
@@ -62,6 +62,12 @@ def storage_model_name(
|
||||
return f"{model_name}-{tag}" if model_name else tag
|
||||
|
||||
|
||||
# [HC_PARALLEL_PATCH] knobs for the pipelined HiCache prefetch/backup lanes
|
||||
import os as _hc_os
|
||||
_HC_PREFETCH_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_PREFETCH_IO_THREADS", "1") or 1)
|
||||
_HC_BACKUP_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_BACKUP_IO_THREADS", "1") or 1)
|
||||
_HC_TIMING = _hc_os.getenv("SGLANG_HICACHE_TIMING", "0") not in ("", "0", "false", "False")
|
||||
|
||||
class LayerLoadingEvent:
|
||||
def __init__(self, num_layers: int):
|
||||
self._num_layers = num_layers
|
||||
@@ -1152,7 +1158,11 @@ class HiCacheController:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation) -> int:
|
||||
def _page_transfer(
|
||||
self,
|
||||
operation: PrefetchOperation,
|
||||
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
|
||||
) -> int:
|
||||
# Transfer batch by batch
|
||||
prefix_keys = operation.prefix_keys
|
||||
kv_derived_transfers = [
|
||||
@@ -1202,9 +1212,28 @@ class HiCacheController:
|
||||
completed_tokens=completed_pages * self.page_size,
|
||||
operation=operation,
|
||||
)
|
||||
self.prefetch_sync_queue.put(ack)
|
||||
if ack_sink is None:
|
||||
# Keep the direct-call behavior for existing users. The pipelined
|
||||
# coordinator always supplies a sink so workers never share the queue.
|
||||
self.prefetch_sync_queue.put(ack)
|
||||
else:
|
||||
ack_sink(ack)
|
||||
return completed_pages
|
||||
|
||||
def _transfer_prefetch_and_collect_acks(
|
||||
self, operation: PrefetchOperation
|
||||
) -> List[PrefetchAck]:
|
||||
"""Run one worker without publishing ACKs to the shared sync queue."""
|
||||
acks = []
|
||||
try:
|
||||
self._page_transfer(operation, ack_sink=acks.append)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"HiCache prefetch transfer failed for request %s",
|
||||
operation.request_id,
|
||||
)
|
||||
return acks
|
||||
|
||||
def _page_transfer_kv_batch(
|
||||
self,
|
||||
operation: PrefetchOperation,
|
||||
@@ -1247,24 +1276,86 @@ class HiCacheController:
|
||||
def prefetch_io_aux_func(self):
|
||||
"""
|
||||
Auxiliary function conducting IO operations for prefetching.
|
||||
|
||||
# [HC_PARALLEL_PATCH] pipelined: up to SGLANG_HICACHE_PREFETCH_IO_THREADS
|
||||
# transfers in flight. Every ACK of one operation is published before any
|
||||
# ACK of the next operation, keeping per-rank collective pairing unchanged.
|
||||
"""
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
operation = self.prefetch_buffer.get(block=True, timeout=1)
|
||||
except Empty:
|
||||
continue
|
||||
if operation is None:
|
||||
continue
|
||||
try:
|
||||
self._page_transfer(operation)
|
||||
finally:
|
||||
self.prefetch_sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
completed_req=True,
|
||||
operation=operation,
|
||||
import time as _time
|
||||
from collections import deque as _deque
|
||||
from concurrent.futures import ThreadPoolExecutor as _TPE
|
||||
|
||||
K = max(1, _HC_PREFETCH_WORKERS)
|
||||
timing = _HC_TIMING
|
||||
ex = _TPE(max_workers=K, thread_name_prefix="hc-prefetch-io")
|
||||
inflight = _deque()
|
||||
lat = []
|
||||
pages = 0
|
||||
try:
|
||||
while not self.storage_stop_event.is_set():
|
||||
while len(inflight) < K:
|
||||
try:
|
||||
operation = self.prefetch_buffer.get_nowait()
|
||||
except Empty:
|
||||
break
|
||||
if operation is None:
|
||||
continue
|
||||
inflight.append(
|
||||
(
|
||||
operation,
|
||||
_time.perf_counter(),
|
||||
ex.submit(
|
||||
self._transfer_prefetch_and_collect_acks, operation
|
||||
),
|
||||
)
|
||||
)
|
||||
)
|
||||
if not inflight:
|
||||
try:
|
||||
operation = self.prefetch_buffer.get(block=True, timeout=1)
|
||||
except Empty:
|
||||
continue
|
||||
if operation is None:
|
||||
continue
|
||||
inflight.append(
|
||||
(
|
||||
operation,
|
||||
_time.perf_counter(),
|
||||
ex.submit(
|
||||
self._transfer_prefetch_and_collect_acks, operation
|
||||
),
|
||||
)
|
||||
)
|
||||
operation, t0, fut = inflight[0]
|
||||
try:
|
||||
acks = fut.result()
|
||||
except Exception:
|
||||
logger.exception("HiCache prefetch transfer failed")
|
||||
acks = []
|
||||
finally:
|
||||
inflight.popleft()
|
||||
if timing:
|
||||
lat.append((_time.perf_counter() - t0) * 1000.0)
|
||||
pages += len(getattr(operation, "hash_value", []) or [])
|
||||
for ack in acks:
|
||||
self.prefetch_sync_queue.put(ack)
|
||||
self.prefetch_sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
completed_req=True,
|
||||
operation=operation,
|
||||
)
|
||||
)
|
||||
if timing and len(lat) >= 100:
|
||||
lat.sort()
|
||||
print("[HC_TIMING] prefetch queue=%d inflight=%d n=%d transfer_ms p50=%.1f p95=%.1f pages=%d"
|
||||
% (self.prefetch_buffer.qsize(), len(inflight), len(lat),
|
||||
lat[len(lat) // 2], lat[int(len(lat) * 0.95)], pages), flush=True)
|
||||
lat = []
|
||||
pages = 0
|
||||
finally:
|
||||
# Detach closes the backend after joining this outer thread. Do not
|
||||
# let pool workers outlive that join and touch released state.
|
||||
ex.shutdown(wait=True)
|
||||
|
||||
def prefetch_rate_limited(self) -> bool:
|
||||
"""
|
||||
@@ -1443,22 +1534,82 @@ class HiCacheController:
|
||||
"HiCache backup failed after %d tokens", operation.completed_tokens
|
||||
)
|
||||
|
||||
def _should_skip_backup(self) -> bool:
|
||||
"""Whether this rank can ACK without doing any storage IO."""
|
||||
return self.backup_skip
|
||||
|
||||
def backup_thread_func(self):
|
||||
"""
|
||||
Manage backup operations from host memory to storage backend.
|
||||
|
||||
# [HC_PARALLEL_PATCH] pipelined: up to SGLANG_HICACHE_BACKUP_IO_THREADS
|
||||
# writes in flight, acked in FIFO order.
|
||||
"""
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
operation = self.backup_queue.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
import time as _time
|
||||
from collections import deque as _deque
|
||||
from concurrent.futures import ThreadPoolExecutor as _TPE
|
||||
|
||||
if not self.backup_skip:
|
||||
self._page_backup_or_log(operation)
|
||||
K = max(1, _HC_BACKUP_WORKERS)
|
||||
timing = _HC_TIMING
|
||||
ex = _TPE(max_workers=K, thread_name_prefix="hc-backup-io")
|
||||
inflight = _deque()
|
||||
lat = []
|
||||
pages = 0
|
||||
|
||||
def submit(operation) -> bool:
|
||||
if self._should_skip_backup():
|
||||
self.ack_backup_queue.put(operation)
|
||||
return False
|
||||
inflight.append(
|
||||
(
|
||||
operation,
|
||||
_time.perf_counter(),
|
||||
ex.submit(self._page_backup_or_log, operation),
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
except Empty:
|
||||
continue
|
||||
try:
|
||||
while not self.storage_stop_event.is_set():
|
||||
while len(inflight) < K:
|
||||
try:
|
||||
operation = self.backup_queue.get_nowait()
|
||||
except Empty:
|
||||
break
|
||||
if operation is None:
|
||||
continue
|
||||
submit(operation)
|
||||
if not inflight:
|
||||
try:
|
||||
operation = self.backup_queue.get(block=True, timeout=1)
|
||||
except Empty:
|
||||
continue
|
||||
if operation is None:
|
||||
continue
|
||||
if not submit(operation):
|
||||
continue
|
||||
operation, t0, fut = inflight[0]
|
||||
try:
|
||||
fut.result()
|
||||
except Exception:
|
||||
logger.exception("HiCache backup failed")
|
||||
finally:
|
||||
inflight.popleft()
|
||||
if timing:
|
||||
lat.append((_time.perf_counter() - t0) * 1000.0)
|
||||
pages += len(getattr(operation, "hash_value", []) or [])
|
||||
self.ack_backup_queue.put(operation)
|
||||
if timing and len(lat) >= 100:
|
||||
lat.sort()
|
||||
print("[HC_TIMING] backup queue=%d inflight=%d n=%d write_ms p50=%.1f p95=%.1f pages=%d"
|
||||
% (self.backup_queue.qsize(), len(inflight), len(lat),
|
||||
lat[len(lat) // 2], lat[int(len(lat) * 0.95)], pages), flush=True)
|
||||
lat = []
|
||||
pages = 0
|
||||
finally:
|
||||
# The outer thread is joined before backend/process-group teardown.
|
||||
# Wait for every submitted write before reporting that thread stopped.
|
||||
ex.shutdown(wait=True)
|
||||
|
||||
def prefetch_sync_thread_func(self):
|
||||
"""Synchronize prefetch results across all PP and TP ranks."""
|
||||
|
||||
@@ -1547,19 +1547,31 @@ class HybridCacheController(BaseHiCacheController):
|
||||
)
|
||||
return host_indices, device_indices, resolved_pool_transfers
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation) -> bool:
|
||||
def _page_transfer(
|
||||
self,
|
||||
operation: PrefetchOperation,
|
||||
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
|
||||
) -> int:
|
||||
with self.mem_pool_host.layout_lease():
|
||||
return self._page_transfer_with_stable_layout(operation)
|
||||
return self._page_transfer_with_stable_layout(operation, ack_sink)
|
||||
|
||||
def _page_transfer_with_stable_layout(self, operation: PrefetchOperation) -> None:
|
||||
def _page_transfer_with_stable_layout(
|
||||
self,
|
||||
operation: PrefetchOperation,
|
||||
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
|
||||
) -> int:
|
||||
# KV pools and KV-derived pools first — determines actual completed page count
|
||||
kv_completed_pages = super()._page_transfer(operation)
|
||||
kv_completed_pages = super()._page_transfer(operation, ack_sink)
|
||||
|
||||
# Read non-KV derived sidecar pool, e.g. SWA, Mamba.
|
||||
self._page_transfer_sidecar(operation, kv_completed_pages)
|
||||
self._page_transfer_sidecar(operation, kv_completed_pages, ack_sink)
|
||||
return kv_completed_pages
|
||||
|
||||
def _page_transfer_sidecar(
|
||||
self, operation: PrefetchOperation, kv_completed_pages: int
|
||||
self,
|
||||
operation: PrefetchOperation,
|
||||
kv_completed_pages: int,
|
||||
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
|
||||
) -> None:
|
||||
if operation.pool_transfers is None:
|
||||
return
|
||||
@@ -1604,17 +1616,19 @@ class HybridCacheController(BaseHiCacheController):
|
||||
"HiCache sidecar prefetch failed for request %s",
|
||||
operation.request_id,
|
||||
)
|
||||
# Emit PrefetchAck to prefetch_sync_queue, even the operation has been canceled by the
|
||||
# scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects
|
||||
# to perform all_reduce.
|
||||
self.prefetch_sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
operation=operation,
|
||||
pool_hits=pool_hits,
|
||||
)
|
||||
# Emit PrefetchAck even when the operation has been canceled by the
|
||||
# scheduler thread. The sync thread expects the same number of ACKs on
|
||||
# every rank. Pipelined workers buffer this ACK per operation instead of
|
||||
# publishing it directly to the shared queue.
|
||||
ack = PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
operation=operation,
|
||||
pool_hits=pool_hits,
|
||||
)
|
||||
return
|
||||
if ack_sink is None:
|
||||
self.prefetch_sync_queue.put(ack)
|
||||
else:
|
||||
ack_sink(ack)
|
||||
|
||||
def trim_prefetch_full_head(self, operation, trim_tokens: int) -> None:
|
||||
"""Drop a device-resident FULL head after the availability query;
|
||||
@@ -1704,22 +1718,15 @@ class HybridCacheController(BaseHiCacheController):
|
||||
|
||||
return False
|
||||
|
||||
def backup_thread_func(self):
|
||||
"""Back up rank-sharded sidecars on every TP rank.
|
||||
def _should_skip_backup(self) -> bool:
|
||||
# Replicated primary MLA KV may be TP0-only, but rank-sharded sidecars
|
||||
# still require IO on every rank. _page_backup applies should_backup()
|
||||
# to each individual transfer, so the whole operation cannot be skipped.
|
||||
return False
|
||||
|
||||
The base implementation skips the entire operation on non-zero MLA TP
|
||||
ranks. That optimization is valid for replicated MLA KV, but not for
|
||||
hybrid rank-sharded pools such as Kimi-K3 Mamba state.
|
||||
"""
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
operation = self.backup_queue.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
self._page_backup_or_log(operation)
|
||||
self.ack_backup_queue.put(operation)
|
||||
except Empty:
|
||||
continue
|
||||
def backup_thread_func(self):
|
||||
"""Use the base pipeline while retaining per-pool writer selection."""
|
||||
super().backup_thread_func()
|
||||
|
||||
def _resolve_sidecar_kv_derived_pool_transfers(self, operation):
|
||||
for transfer in operation.pool_transfers:
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
"""CPU regressions for pipelined HiCache storage IO ordering and shutdown."""
|
||||
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from queue import Queue
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
|
||||
import sglang.srt.managers.cache_controller as cache_controller_module
|
||||
from sglang.srt.managers.cache_controller import (
|
||||
HiCacheController,
|
||||
StorageOperation,
|
||||
)
|
||||
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
|
||||
HybridCacheController,
|
||||
PrefetchOperation,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
def _drain(queue):
|
||||
items = []
|
||||
while not queue.empty():
|
||||
items.append(queue.get_nowait())
|
||||
return items
|
||||
|
||||
|
||||
def _wait_for_queue_size(queue, size, timeout=5):
|
||||
deadline = time.monotonic() + timeout
|
||||
while queue.qsize() < size and time.monotonic() < deadline:
|
||||
time.sleep(0.005)
|
||||
return queue.qsize() >= size
|
||||
|
||||
|
||||
class TestPrefetchPipelineOrdering(unittest.TestCase):
|
||||
def _controller(self, wait_for, peer_done):
|
||||
controller = HybridCacheController.__new__(HybridCacheController)
|
||||
controller.page_size = 1
|
||||
controller.prefetch_buffer = Queue()
|
||||
controller.prefetch_sync_queue = Queue()
|
||||
controller.storage_stop_event = threading.Event()
|
||||
controller.mem_pool_host = SimpleNamespace(
|
||||
layout_lease=lambda: nullcontext()
|
||||
)
|
||||
controller.storage_backend = Mock()
|
||||
controller.storage_backend.batch_get_v2.return_value = {}
|
||||
|
||||
for rid in ("a", "b"):
|
||||
operation = PrefetchOperation(
|
||||
CacheRequestHandle(rid, 0),
|
||||
list(range(2)),
|
||||
pool_transfers=[],
|
||||
)
|
||||
operation.hash_value = [f"{rid}-0", f"{rid}-1"]
|
||||
operation.host_indices = torch.arange(2)
|
||||
controller.prefetch_buffer.put(operation)
|
||||
|
||||
def page_get(operation, batch_hashes, host_indices, extra_info):
|
||||
if operation.request_id == wait_for:
|
||||
if not peer_done.wait(timeout=5):
|
||||
raise TimeoutError("peer prefetch did not finish")
|
||||
return len(batch_hashes)
|
||||
|
||||
controller.page_get_func = page_get
|
||||
collect = controller._transfer_prefetch_and_collect_acks
|
||||
|
||||
def collect_and_signal(operation):
|
||||
acks = collect(operation)
|
||||
if operation.request_id != wait_for:
|
||||
# This rank's other operation has now completed. The delayed
|
||||
# peer may resume, producing the opposite worker finish order.
|
||||
peer_done.set()
|
||||
return acks
|
||||
|
||||
controller._transfer_prefetch_and_collect_acks = collect_and_signal
|
||||
return controller
|
||||
|
||||
def test_two_tp_ranks_keep_operation_ack_sequences_aligned(self):
|
||||
# Rank 0 finishes A before B; rank 1 finishes B before A. Without
|
||||
# per-operation buffering their progress/sidecar ACKs enter collectives
|
||||
# in opposite orders even though completed_req itself remains FIFO.
|
||||
rank0_done = threading.Event()
|
||||
rank1_done = threading.Event()
|
||||
rank0 = self._controller(wait_for="b", peer_done=rank0_done)
|
||||
rank1 = self._controller(wait_for="a", peer_done=rank1_done)
|
||||
threads = [
|
||||
threading.Thread(target=rank.prefetch_io_aux_func, daemon=True)
|
||||
for rank in (rank0, rank1)
|
||||
]
|
||||
|
||||
with (
|
||||
patch.object(cache_controller_module, "_HC_PREFETCH_WORKERS", 2),
|
||||
patch.object(cache_controller_module, "STORAGE_BATCH_SIZE", 1),
|
||||
):
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
try:
|
||||
self.assertTrue(_wait_for_queue_size(rank0.prefetch_sync_queue, 8))
|
||||
self.assertTrue(_wait_for_queue_size(rank1.prefetch_sync_queue, 8))
|
||||
finally:
|
||||
rank0.storage_stop_event.set()
|
||||
rank1.storage_stop_event.set()
|
||||
rank0_done.set()
|
||||
rank1_done.set()
|
||||
for thread in threads:
|
||||
thread.join(timeout=5)
|
||||
for thread in threads:
|
||||
self.assertFalse(thread.is_alive())
|
||||
|
||||
rank0_acks = _drain(rank0.prefetch_sync_queue)
|
||||
rank1_acks = _drain(rank1.prefetch_sync_queue)
|
||||
|
||||
def ack_kind(ack):
|
||||
if ack.completed_req:
|
||||
return "completed_req"
|
||||
if ack.completed_tokens is not None:
|
||||
return "completed_tokens"
|
||||
if ack.pool_hits is not None:
|
||||
return "pool_hits"
|
||||
raise AssertionError(f"unknown ACK: {ack}")
|
||||
|
||||
self.assertEqual([ack.rid for ack in rank0_acks], ["a"] * 4 + ["b"] * 4)
|
||||
self.assertEqual(
|
||||
[(ack.rid, ack_kind(ack)) for ack in rank0_acks],
|
||||
[(ack.rid, ack_kind(ack)) for ack in rank1_acks],
|
||||
)
|
||||
for acks in (rank0_acks, rank1_acks):
|
||||
self.assertEqual(
|
||||
[ack_kind(ack) for ack in acks[:4]],
|
||||
["completed_tokens", "completed_tokens", "pool_hits", "completed_req"],
|
||||
)
|
||||
self.assertEqual(
|
||||
[ack.completed_tokens for ack in acks[:2]],
|
||||
[1, 2],
|
||||
)
|
||||
|
||||
|
||||
class TestBackupPipeline(unittest.TestCase):
|
||||
@staticmethod
|
||||
def _controller(controller_type, is_storage_writer, backup):
|
||||
controller = controller_type.__new__(controller_type)
|
||||
controller.backup_queue = Queue()
|
||||
controller.ack_backup_queue = Queue()
|
||||
controller.storage_stop_event = threading.Event()
|
||||
controller.storage_config = SimpleNamespace(
|
||||
is_storage_writer=is_storage_writer
|
||||
)
|
||||
controller._page_backup_or_log = backup
|
||||
return controller
|
||||
|
||||
def _run_one(self, controller, operation):
|
||||
thread = threading.Thread(target=controller.backup_thread_func, daemon=True)
|
||||
with patch.object(cache_controller_module, "_HC_BACKUP_WORKERS", 1):
|
||||
controller.backup_queue.put(operation)
|
||||
thread.start()
|
||||
try:
|
||||
ack = controller.ack_backup_queue.get(timeout=5)
|
||||
finally:
|
||||
controller.storage_stop_event.set()
|
||||
thread.join(timeout=5)
|
||||
self.assertFalse(thread.is_alive())
|
||||
self.assertIs(ack, operation)
|
||||
|
||||
def test_non_writer_never_submits_backup_io(self):
|
||||
backup = Mock()
|
||||
controller = self._controller(HiCacheController, False, backup)
|
||||
operation = StorageOperation(None, [])
|
||||
self._run_one(controller, operation)
|
||||
backup.assert_not_called()
|
||||
|
||||
def test_hybrid_non_writer_still_runs_rank_sharded_backup(self):
|
||||
backup = Mock()
|
||||
controller = self._controller(HybridCacheController, False, backup)
|
||||
operation = StorageOperation(None, [])
|
||||
self._run_one(controller, operation)
|
||||
backup.assert_called_once_with(operation)
|
||||
|
||||
def test_stop_waits_for_every_backup_worker(self):
|
||||
started = [threading.Event(), threading.Event()]
|
||||
release = [threading.Event(), threading.Event()]
|
||||
finished = [threading.Event(), threading.Event()]
|
||||
|
||||
def backup(operation):
|
||||
index = operation.token_ids[0]
|
||||
started[index].set()
|
||||
if not release[index].wait(timeout=5):
|
||||
raise TimeoutError(f"backup {index} was not released")
|
||||
finished[index].set()
|
||||
|
||||
controller = self._controller(HiCacheController, True, backup)
|
||||
for index in range(2):
|
||||
controller.backup_queue.put(StorageOperation(None, [index]))
|
||||
thread = threading.Thread(target=controller.backup_thread_func, daemon=True)
|
||||
|
||||
with patch.object(cache_controller_module, "_HC_BACKUP_WORKERS", 2):
|
||||
thread.start()
|
||||
try:
|
||||
self.assertTrue(started[0].wait(timeout=5))
|
||||
self.assertTrue(started[1].wait(timeout=5))
|
||||
controller.storage_stop_event.set()
|
||||
release[0].set()
|
||||
|
||||
# The coordinator may finish operation 0, but shutdown cannot
|
||||
# return while operation 1 still uses the storage backend.
|
||||
thread.join(timeout=0.1)
|
||||
self.assertTrue(thread.is_alive())
|
||||
self.assertFalse(finished[1].is_set())
|
||||
|
||||
release[1].set()
|
||||
thread.join(timeout=5)
|
||||
self.assertFalse(thread.is_alive())
|
||||
self.assertTrue(finished[0].is_set())
|
||||
self.assertTrue(finished[1].is_set())
|
||||
finally:
|
||||
controller.storage_stop_event.set()
|
||||
for event in release:
|
||||
event.set()
|
||||
thread.join(timeout=5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user