From d349e672bc7eefb532010ec7830bff0ed682bc0e Mon Sep 17 00:00:00 2001 From: Mick Date: Mon, 5 Oct 2026 19:55:30 +0800 Subject: [PATCH] [diffusion] feat: recycle warmup-only entries of conditioning-cache at the first served store (#41832) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5.5 --- .../runtime/cache/conditioning.py | 21 ++++++++++++++++ .../test/unit/test_conditioning_cache.py | 24 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/python/sglang/multimodal_gen/runtime/cache/conditioning.py b/python/sglang/multimodal_gen/runtime/cache/conditioning.py index dc59bbddea..ae95683199 100644 --- a/python/sglang/multimodal_gen/runtime/cache/conditioning.py +++ b/python/sglang/multimodal_gen/runtime/cache/conditioning.py @@ -154,6 +154,8 @@ class _CacheEntry: owner: int preferred: bool device_resident: bool = False + # stored by warmup and not yet hit by a served request + provisional: bool = False @classmethod def snapshot(cls, output, size, owner, preferred, device_resident): @@ -296,6 +298,19 @@ class ConditioningCache: self._entries.clear() self.bytes = 0 + def _recycle_provisional_entries(self): + """Free warmup-only entries so served snapshots reuse their pinned blocks + instead of pinning fresh pages (~0.3 ms/MB). Seeded negatives stay.""" + for key in [ + key + for key, entry in self._entries.items() + if entry.provisional and not entry.preferred + ]: + removed = self._entries.pop(key) + removed.wait() + self.bytes -= removed.size + self.evictions += 1 + def invalidate(self, parameters): if parameters is None: self.clear() @@ -453,6 +468,8 @@ class ConditioningCache: self.hits += 1 if entry is not None: entry.preferred |= _prefer_cache.get() + if not _refresh_cache.get(): + entry.provisional = False self._entries.move_to_end(key) if group_entry is not None: self.group_hits += 1 @@ -511,6 +528,9 @@ class ConditioningCache: if old is not None: old.wait() self.bytes -= old.size + refresh = _refresh_cache.get() + if not refresh: + self._recycle_provisional_entries() preferred = _prefer_cache.get() evictable = [ key @@ -535,6 +555,7 @@ class ConditioningCache: self._entries[key] = _CacheEntry.snapshot( output, size, self._identity(model), preferred, keep_on_device ) + self._entries[key].provisional = refresh self.bytes += size logger.debug( "Conditioning cache store: %s.%s, %d bytes", diff --git a/python/sglang/multimodal_gen/test/unit/test_conditioning_cache.py b/python/sglang/multimodal_gen/test/unit/test_conditioning_cache.py index e804e9d66e..24e0a48f15 100644 --- a/python/sglang/multimodal_gen/test/unit/test_conditioning_cache.py +++ b/python/sglang/multimodal_gen/test/unit/test_conditioning_cache.py @@ -180,6 +180,30 @@ def test_warmup_executes_nested_encoders_and_seeds_serving_cache(): assert model.calls == 4 and model.vision_calls == 3 +@torch.no_grad() +def test_first_served_store_recycles_warmup_only_entries(): + cache = ConditioningCache(4096) + model = Encoder().eval() + synthetic, reused, negative = torch.ones(4), torch.full((4,), 2.0), torch.zeros(4) + with cache.scope(refresh=True): + model(synthetic) + model(reused) + with prefer_conditioning_cache(): + model(negative) + with cache.scope(): + model(reused) + assert cache.evictions == 0 + model(torch.full((4,), 3.0)) + # only the warmup entry no served request touched is recycled + assert cache.evictions == 1 and cache.stats()["entries"] == 3 + model(reused) + with prefer_conditioning_cache(): + model(negative) + assert model.calls == 4 + model(synthetic) + assert model.calls == 5 + + @pytest.mark.parametrize("bypass", ["disabled", "inactive", "training", "grad", "ar"]) def test_bypass(bypass): cache = ConditioningCache(0 if bypass == "disabled" else 1024)