[diffusion] feat: recycle warmup-only entries of conditioning-cache at the first served store (#41832)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
7352073ec6
commit
d349e672bc
@@ -154,6 +154,8 @@ class _CacheEntry:
|
||||
owner: int
|
||||
preferred: bool
|
||||
device_resident: bool = False
|
||||
# stored by warmup and not yet hit by a served request
|
||||
provisional: bool = False
|
||||
|
||||
@classmethod
|
||||
def snapshot(cls, output, size, owner, preferred, device_resident):
|
||||
@@ -296,6 +298,19 @@ class ConditioningCache:
|
||||
self._entries.clear()
|
||||
self.bytes = 0
|
||||
|
||||
def _recycle_provisional_entries(self):
|
||||
"""Free warmup-only entries so served snapshots reuse their pinned blocks
|
||||
instead of pinning fresh pages (~0.3 ms/MB). Seeded negatives stay."""
|
||||
for key in [
|
||||
key
|
||||
for key, entry in self._entries.items()
|
||||
if entry.provisional and not entry.preferred
|
||||
]:
|
||||
removed = self._entries.pop(key)
|
||||
removed.wait()
|
||||
self.bytes -= removed.size
|
||||
self.evictions += 1
|
||||
|
||||
def invalidate(self, parameters):
|
||||
if parameters is None:
|
||||
self.clear()
|
||||
@@ -453,6 +468,8 @@ class ConditioningCache:
|
||||
self.hits += 1
|
||||
if entry is not None:
|
||||
entry.preferred |= _prefer_cache.get()
|
||||
if not _refresh_cache.get():
|
||||
entry.provisional = False
|
||||
self._entries.move_to_end(key)
|
||||
if group_entry is not None:
|
||||
self.group_hits += 1
|
||||
@@ -511,6 +528,9 @@ class ConditioningCache:
|
||||
if old is not None:
|
||||
old.wait()
|
||||
self.bytes -= old.size
|
||||
refresh = _refresh_cache.get()
|
||||
if not refresh:
|
||||
self._recycle_provisional_entries()
|
||||
preferred = _prefer_cache.get()
|
||||
evictable = [
|
||||
key
|
||||
@@ -535,6 +555,7 @@ class ConditioningCache:
|
||||
self._entries[key] = _CacheEntry.snapshot(
|
||||
output, size, self._identity(model), preferred, keep_on_device
|
||||
)
|
||||
self._entries[key].provisional = refresh
|
||||
self.bytes += size
|
||||
logger.debug(
|
||||
"Conditioning cache store: %s.%s, %d bytes",
|
||||
|
||||
@@ -180,6 +180,30 @@ def test_warmup_executes_nested_encoders_and_seeds_serving_cache():
|
||||
assert model.calls == 4 and model.vision_calls == 3
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_first_served_store_recycles_warmup_only_entries():
|
||||
cache = ConditioningCache(4096)
|
||||
model = Encoder().eval()
|
||||
synthetic, reused, negative = torch.ones(4), torch.full((4,), 2.0), torch.zeros(4)
|
||||
with cache.scope(refresh=True):
|
||||
model(synthetic)
|
||||
model(reused)
|
||||
with prefer_conditioning_cache():
|
||||
model(negative)
|
||||
with cache.scope():
|
||||
model(reused)
|
||||
assert cache.evictions == 0
|
||||
model(torch.full((4,), 3.0))
|
||||
# only the warmup entry no served request touched is recycled
|
||||
assert cache.evictions == 1 and cache.stats()["entries"] == 3
|
||||
model(reused)
|
||||
with prefer_conditioning_cache():
|
||||
model(negative)
|
||||
assert model.calls == 4
|
||||
model(synthetic)
|
||||
assert model.calls == 5
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bypass", ["disabled", "inactive", "training", "grad", "ar"])
|
||||
def test_bypass(bypass):
|
||||
cache = ConditioningCache(0 if bypass == "disabled" else 1024)
|
||||
|
||||
Reference in New Issue
Block a user