[diffusion] feat: recycle warmup-only entries of conditioning-cache at the first served store (#41832)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-05 19:55:30 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 7352073ec6
commit d349e672bc
2 changed files with 45 additions and 0 deletions
@@ -154,6 +154,8 @@ class _CacheEntry:
owner: int
preferred: bool
device_resident: bool = False
# stored by warmup and not yet hit by a served request
provisional: bool = False
@classmethod
def snapshot(cls, output, size, owner, preferred, device_resident):
@@ -296,6 +298,19 @@ class ConditioningCache:
self._entries.clear()
self.bytes = 0
def _recycle_provisional_entries(self):
"""Free warmup-only entries so served snapshots reuse their pinned blocks
instead of pinning fresh pages (~0.3 ms/MB). Seeded negatives stay."""
for key in [
key
for key, entry in self._entries.items()
if entry.provisional and not entry.preferred
]:
removed = self._entries.pop(key)
removed.wait()
self.bytes -= removed.size
self.evictions += 1
def invalidate(self, parameters):
if parameters is None:
self.clear()
@@ -453,6 +468,8 @@ class ConditioningCache:
self.hits += 1
if entry is not None:
entry.preferred |= _prefer_cache.get()
if not _refresh_cache.get():
entry.provisional = False
self._entries.move_to_end(key)
if group_entry is not None:
self.group_hits += 1
@@ -511,6 +528,9 @@ class ConditioningCache:
if old is not None:
old.wait()
self.bytes -= old.size
refresh = _refresh_cache.get()
if not refresh:
self._recycle_provisional_entries()
preferred = _prefer_cache.get()
evictable = [
key
@@ -535,6 +555,7 @@ class ConditioningCache:
self._entries[key] = _CacheEntry.snapshot(
output, size, self._identity(model), preferred, keep_on_device
)
self._entries[key].provisional = refresh
self.bytes += size
logger.debug(
"Conditioning cache store: %s.%s, %d bytes",
@@ -180,6 +180,30 @@ def test_warmup_executes_nested_encoders_and_seeds_serving_cache():
assert model.calls == 4 and model.vision_calls == 3
@torch.no_grad()
def test_first_served_store_recycles_warmup_only_entries():
cache = ConditioningCache(4096)
model = Encoder().eval()
synthetic, reused, negative = torch.ones(4), torch.full((4,), 2.0), torch.zeros(4)
with cache.scope(refresh=True):
model(synthetic)
model(reused)
with prefer_conditioning_cache():
model(negative)
with cache.scope():
model(reused)
assert cache.evictions == 0
model(torch.full((4,), 3.0))
# only the warmup entry no served request touched is recycled
assert cache.evictions == 1 and cache.stats()["entries"] == 3
model(reused)
with prefer_conditioning_cache():
model(negative)
assert model.calls == 4
model(synthetic)
assert model.calls == 5
@pytest.mark.parametrize("bypass", ["disabled", "inactive", "training", "grad", "ar"])
def test_bypass(bypass):
cache = ConditioningCache(0 if bypass == "disabled" else 1024)