[diffusion] feat: support bounded exact conditioning cache across native models (#40470)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -12,6 +12,7 @@ SGLang Diffusion has the following features:
|
||||
- Broad model support: Wan, FastWan, FLUX, Qwen-Image / Qwen-Image 2.1, LongCat-Image, Z-Image, Anima, Ideogram 4, Krea-2, Cosmos3, LTX-2/LTX-2.3/LTX-2.5, MiniMax-H3, FastH3, VDN-H3, LingBot Video MoE, LingBot World, SANA-Video/SANA-WM, JoyEcho, MOVA, GLM-Image, ERNIE-Image, Hunyuan3D, and more
|
||||
- Fast inference speed: empowered by optimized `sgl-kernel` kernels, scheduler/runtime improvements, caching acceleration, and native diffusion hot-path optimizations
|
||||
- Ease of use: OpenAI-compatible api, CLI, python sdk, and a [ComfyUI plugin](apps/ComfyUI_SGLDiffusion/README.md)
|
||||
- [Conditioning reuse](https://docs.sglang.io/docs/sglang-diffusion/caching-acceleration#conditioning-cache): bounded CPU caching for repeated text/image inputs and VAE posteriors, with `--disable-conditioning-cache` to opt out
|
||||
- Multi-platform support:
|
||||
- NVIDIA GPUs (H100, H200, A100, B200, 4090, 5090)
|
||||
- AMD GPUs (MI300X, MI325X, MI355X)
|
||||
|
||||
@@ -0,0 +1,681 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Exact conditioning reuse within grouped stages and across requests."""
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import pickle
|
||||
import weakref
|
||||
from collections import OrderedDict
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass, fields, is_dataclass
|
||||
from functools import wraps
|
||||
from typing import Callable
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution
|
||||
from PIL import Image
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_replica_group,
|
||||
get_tp_group,
|
||||
get_world_size,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
_active_cache: ContextVar["ConditioningCache | None"] = ContextVar(
|
||||
"conditioning_cache", default=None
|
||||
)
|
||||
_refresh_cache: ContextVar[bool] = ContextVar(
|
||||
"refresh_conditioning_cache", default=False
|
||||
)
|
||||
_cross_request_cache: ContextVar[bool] = ContextVar(
|
||||
"cross_request_conditioning_cache", default=True
|
||||
)
|
||||
_prefer_cache: ContextVar[bool] = ContextVar("prefer_conditioning_cache", default=False)
|
||||
_stage_encoder: ContextVar[object | None] = ContextVar("stage_encoder", default=None)
|
||||
_container_types: set[type] = {DiagonalGaussianDistribution}
|
||||
_live_caches = weakref.WeakSet()
|
||||
_weights_epoch = 0
|
||||
|
||||
|
||||
def invalidate_conditioning_caches(modules=None):
|
||||
"""Invalidate before weight mutations, including partially failed updates."""
|
||||
global _weights_epoch
|
||||
_weights_epoch += 1
|
||||
parameters = (
|
||||
None
|
||||
if modules is None
|
||||
else {id(p) for module in modules for p in module.parameters()}
|
||||
)
|
||||
for cache in _live_caches:
|
||||
cache.invalidate(parameters)
|
||||
|
||||
|
||||
def conditioning_weights_epoch():
|
||||
return _weights_epoch
|
||||
|
||||
|
||||
def register_conditioning_container(cls):
|
||||
"""Opt in a tensor-only posterior container whose fields can be copied."""
|
||||
_container_types.add(cls)
|
||||
return cls
|
||||
|
||||
|
||||
class Uncacheable(TypeError):
|
||||
pass
|
||||
|
||||
|
||||
@contextmanager
|
||||
def conditioning_cache_group(*, enabled):
|
||||
cache = _active_cache.get()
|
||||
if cache is None:
|
||||
yield
|
||||
else:
|
||||
with cache.group_scope(enabled=enabled):
|
||||
yield
|
||||
|
||||
|
||||
@contextmanager
|
||||
def prefer_conditioning_cache():
|
||||
# retain reusable negative conditioning ahead of changing positive prompts
|
||||
token = _prefer_cache.set(True)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_prefer_cache.reset(token)
|
||||
|
||||
|
||||
def _fingerprint(value):
|
||||
if isinstance(value, torch.Tensor):
|
||||
if (
|
||||
type(value) is not torch.Tensor
|
||||
or value.layout != torch.strided
|
||||
or value.requires_grad
|
||||
):
|
||||
raise Uncacheable("only dense inference tensors can be cached")
|
||||
tensor = value.detach().cpu().contiguous()
|
||||
data = tensor.reshape(-1).view(torch.uint8).numpy()
|
||||
return (
|
||||
"tensor",
|
||||
str(value.dtype),
|
||||
str(value.device),
|
||||
tuple(value.shape),
|
||||
tuple(value.stride()),
|
||||
hashlib.sha256(data).digest(),
|
||||
)
|
||||
if isinstance(value, Image.Image):
|
||||
return (
|
||||
"image",
|
||||
value.mode,
|
||||
value.size,
|
||||
hashlib.sha256(value.tobytes()).digest(),
|
||||
_fingerprint(value.getpalette()),
|
||||
_fingerprint(value.info.get("transparency")),
|
||||
)
|
||||
if isinstance(value, np.ndarray):
|
||||
if value.dtype.hasobject:
|
||||
raise Uncacheable("object arrays are not conditioning values")
|
||||
return (
|
||||
"array",
|
||||
value.dtype.str,
|
||||
value.shape,
|
||||
hashlib.sha256(value.tobytes()).digest(),
|
||||
)
|
||||
if type(value) in (str, bytes, int, float, bool, type(None)):
|
||||
return (type(value).__name__, value)
|
||||
if isinstance(value, (torch.dtype, torch.device)):
|
||||
return (type(value).__name__, str(value))
|
||||
if isinstance(value, dict):
|
||||
return (
|
||||
"dict",
|
||||
tuple((_fingerprint(k), _fingerprint(v)) for k, v in value.items()),
|
||||
)
|
||||
if isinstance(value, (tuple, list)):
|
||||
return (type(value).__name__, tuple(_fingerprint(v) for v in value))
|
||||
raise Uncacheable(f"unsupported conditioning input: {type(value).__name__}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class _HostTensor:
|
||||
data: torch.Tensor
|
||||
device: torch.device
|
||||
|
||||
|
||||
@dataclass
|
||||
class _CacheEntry:
|
||||
output: object
|
||||
size: int
|
||||
ready: tuple[torch.cuda.Event, ...]
|
||||
owner: int
|
||||
preferred: bool
|
||||
device_resident: bool = False
|
||||
|
||||
@classmethod
|
||||
def snapshot(cls, output, size, owner, preferred, device_resident):
|
||||
tensors = {}
|
||||
copy_streams = {}
|
||||
|
||||
def snapshot_tensor(t):
|
||||
if id(t) not in tensors:
|
||||
if t.device.type == "cuda":
|
||||
copy_streams[t.device] = torch.cuda.current_stream(t.device)
|
||||
if device_resident:
|
||||
stored = t.detach().clone()
|
||||
elif t.device.type == "cuda":
|
||||
stored = torch.empty_like(t, device="cpu", pin_memory=True)
|
||||
stored.copy_(t.detach(), non_blocking=True)
|
||||
else:
|
||||
stored = t.detach().to("cpu", copy=True)
|
||||
tensors[id(t)] = (
|
||||
stored if device_resident else _HostTensor(stored, t.device)
|
||||
)
|
||||
return tensors[id(t)]
|
||||
|
||||
stored = _map_output(output, snapshot_tensor)
|
||||
# snapshot before downstream mutations on each producing stream
|
||||
ready = []
|
||||
for stream in copy_streams.values():
|
||||
event = torch.cuda.Event()
|
||||
event.record(stream)
|
||||
ready.append(event)
|
||||
return cls(stored, size, tuple(ready), owner, preferred, device_resident)
|
||||
|
||||
def wait(self):
|
||||
if self.device_resident:
|
||||
for event in self.ready:
|
||||
torch.cuda.current_stream(event.device).wait_event(event)
|
||||
|
||||
def record(t):
|
||||
if t.device.type == "cuda":
|
||||
t.record_stream(torch.cuda.current_stream(t.device))
|
||||
return t
|
||||
|
||||
_map_output(self.output, record)
|
||||
return
|
||||
for event in self.ready:
|
||||
event.synchronize()
|
||||
|
||||
|
||||
@dataclass
|
||||
class _GroupEntry:
|
||||
output: object
|
||||
owner: int
|
||||
copied_bytes: int
|
||||
streams: tuple[torch.cuda.Stream, ...]
|
||||
|
||||
def wait(self):
|
||||
consumers = {}
|
||||
for producer in self.streams:
|
||||
consumer = torch.cuda.current_stream(producer.device)
|
||||
if consumer != producer:
|
||||
consumer.wait_stream(producer)
|
||||
consumers[producer.device] = consumer
|
||||
|
||||
if not consumers:
|
||||
return
|
||||
|
||||
def record(t):
|
||||
if t.device in consumers:
|
||||
t.record_stream(consumers[t.device])
|
||||
return t
|
||||
|
||||
_map_output(self.output, record)
|
||||
|
||||
|
||||
def _copy_output(value, *, share_tensors=False):
|
||||
tensors = {}
|
||||
|
||||
def copy_tensor(t):
|
||||
if id(t) not in tensors:
|
||||
tensors[id(t)] = t if share_tensors else t.clone()
|
||||
return tensors[id(t)]
|
||||
|
||||
return _map_output(value, copy_tensor)
|
||||
|
||||
|
||||
def _map_output(value, tensor_fn, *, restore=False):
|
||||
if isinstance(value, _HostTensor if restore else torch.Tensor):
|
||||
return tensor_fn(value)
|
||||
if type(value) in (str, bytes, int, float, bool, type(None)):
|
||||
return value
|
||||
if isinstance(value, (torch.dtype, torch.device)):
|
||||
return value
|
||||
# ModelOutput is both a dataclass and a dict. Reconstruct it as a dataclass
|
||||
# so attribute access and tuple indexing retain their original semantics.
|
||||
if is_dataclass(value) and not isinstance(value, type):
|
||||
return type(value)(
|
||||
**{
|
||||
field.name: _map_output(
|
||||
vars(value)[field.name], tensor_fn, restore=restore
|
||||
)
|
||||
for field in fields(value)
|
||||
}
|
||||
)
|
||||
if type(value) in _container_types:
|
||||
result = copy.copy(value)
|
||||
for name, item in vars(value).items():
|
||||
setattr(result, name, _map_output(item, tensor_fn, restore=restore))
|
||||
return result
|
||||
if type(value) is dict:
|
||||
return {k: _map_output(v, tensor_fn, restore=restore) for k, v in value.items()}
|
||||
if type(value) is list:
|
||||
return [_map_output(v, tensor_fn, restore=restore) for v in value]
|
||||
if isinstance(value, tuple):
|
||||
items = [_map_output(v, tensor_fn, restore=restore) for v in value]
|
||||
return type(value)(*items) if type(value) is not tuple else tuple(items)
|
||||
raise Uncacheable(f"unsupported conditioning output: {type(value).__name__}")
|
||||
|
||||
|
||||
class ConditioningCache:
|
||||
"""One bounded cache per rank with host entries and device-hot conditioning."""
|
||||
|
||||
def __init__(self, max_bytes: int):
|
||||
if max_bytes < 0:
|
||||
raise ValueError("conditioning cache capacity must be nonnegative")
|
||||
_live_caches.add(self)
|
||||
self.max_bytes = max_bytes
|
||||
self.bytes = 0
|
||||
self.hits = self.misses = self.evictions = self.bypasses = 0
|
||||
self._entries = OrderedDict()
|
||||
self._models = weakref.WeakKeyDictionary()
|
||||
self._next_model = 0
|
||||
self._group_entries = ContextVar("conditioning_group_entries", default=None)
|
||||
self.group_hits = 0
|
||||
|
||||
def clear(self):
|
||||
group_entries = self._group_entries.get()
|
||||
if group_entries is not None:
|
||||
group_entries.clear()
|
||||
for entry in self._entries.values():
|
||||
entry.wait()
|
||||
self._entries.clear()
|
||||
self.bytes = 0
|
||||
|
||||
def invalidate(self, parameters):
|
||||
if parameters is None:
|
||||
self.clear()
|
||||
return
|
||||
# shared parameters invalidate both owners, including nested encoders
|
||||
owners = {
|
||||
identity
|
||||
for model, identity in self._models.items()
|
||||
if isinstance(model, torch.nn.Module)
|
||||
and any(id(p) in parameters for p in model.parameters())
|
||||
}
|
||||
for key, entry in list(self._entries.items()):
|
||||
if entry.owner in owners:
|
||||
entry.wait()
|
||||
self.bytes -= entry.size
|
||||
del self._entries[key]
|
||||
group_entries = self._group_entries.get()
|
||||
if group_entries is not None:
|
||||
for key, entry in list(group_entries.items()):
|
||||
if entry.owner in owners:
|
||||
del group_entries[key]
|
||||
|
||||
def stats(self):
|
||||
return dict(
|
||||
hits=self.hits,
|
||||
misses=self.misses,
|
||||
evictions=self.evictions,
|
||||
bypasses=self.bypasses,
|
||||
entries=len(self._entries),
|
||||
bytes=self.bytes,
|
||||
group_hits=self.group_hits,
|
||||
device_bytes=sum(
|
||||
entry.size for entry in self._entries.values() if entry.device_resident
|
||||
),
|
||||
)
|
||||
|
||||
def _identity(self, owner):
|
||||
if owner not in self._models:
|
||||
self._models[owner] = self._next_model
|
||||
self._next_model += 1
|
||||
return self._models[owner]
|
||||
|
||||
@contextmanager
|
||||
def group_scope(self, enabled=True):
|
||||
"""Keep device results only while a stage executes a group of requests."""
|
||||
token = self._group_entries.set({} if enabled else None)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._group_entries.reset(token)
|
||||
|
||||
def _remember_group(self, key, output, owner, share_in_group):
|
||||
entries = self._group_entries.get()
|
||||
if entries is None:
|
||||
return
|
||||
tensors = {}
|
||||
|
||||
def visit(t):
|
||||
tensors[id(t)] = t
|
||||
return t
|
||||
|
||||
_map_output(output, visit)
|
||||
size = (
|
||||
0
|
||||
if share_in_group
|
||||
else sum(t.numel() * t.element_size() for t in tensors.values())
|
||||
)
|
||||
# private snapshots must not retain unbounded intermediate encoder states
|
||||
if not share_in_group and (
|
||||
len(entries) >= 128
|
||||
or size + sum(entry.copied_bytes for entry in entries.values())
|
||||
> 512 * 1024**2
|
||||
):
|
||||
return
|
||||
stored = _copy_output(output, share_tensors=share_in_group)
|
||||
streams = tuple(
|
||||
torch.cuda.current_stream(device)
|
||||
for device in {
|
||||
t.device for t in tensors.values() if t.device.type == "cuda"
|
||||
}
|
||||
)
|
||||
entries[key] = _GroupEntry(stored, owner, size, streams)
|
||||
|
||||
@contextmanager
|
||||
def scope(self, enabled=True, *, refresh=False, cross_request=True):
|
||||
# Zero-capacity ranks still participate in encoder hit consensus.
|
||||
if refresh and self._group_entries.get() is not None:
|
||||
self._group_entries.get().clear()
|
||||
token = _active_cache.set(self if enabled else None)
|
||||
refresh_token = _refresh_cache.set(refresh or _refresh_cache.get())
|
||||
cross_request_token = _cross_request_cache.set(
|
||||
cross_request and _cross_request_cache.get()
|
||||
)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_cross_request_cache.reset(cross_request_token)
|
||||
_refresh_cache.reset(refresh_token)
|
||||
_active_cache.reset(token)
|
||||
|
||||
def run(
|
||||
self,
|
||||
model,
|
||||
method,
|
||||
args,
|
||||
kwargs,
|
||||
compute: Callable,
|
||||
group=None,
|
||||
*,
|
||||
nested=False,
|
||||
namespace=None,
|
||||
share_in_group=False,
|
||||
cross_request=True,
|
||||
keep_on_device=False,
|
||||
):
|
||||
group_entries = self._group_entries.get()
|
||||
cross_request = cross_request and _cross_request_cache.get()
|
||||
if not cross_request and group_entries is None:
|
||||
return compute()
|
||||
try:
|
||||
if (not cross_request or not self.max_bytes) and group_entries is None:
|
||||
raise Uncacheable("cache disabled")
|
||||
if kwargs.get("use_cache") or kwargs.get("past_key_values") is not None:
|
||||
raise Uncacheable("stateful autoregressive encoding")
|
||||
parameter = next(model.parameters(), None)
|
||||
precision = str(parameter.dtype) if parameter is not None else None
|
||||
key = (
|
||||
self._identity(model),
|
||||
self._identity(namespace) if namespace is not None else None,
|
||||
method,
|
||||
share_in_group,
|
||||
precision,
|
||||
torch.is_autocast_enabled("cuda"),
|
||||
torch.get_autocast_dtype("cuda"),
|
||||
torch.is_autocast_enabled("cpu"),
|
||||
torch.get_autocast_dtype("cpu"),
|
||||
_fingerprint(args),
|
||||
_fingerprint(kwargs),
|
||||
)
|
||||
key = hashlib.sha256(pickle.dumps(key)).digest()
|
||||
except Uncacheable:
|
||||
key = None
|
||||
entry = self._entries.get(key) if cross_request else None
|
||||
group_entry = group_entries.get(key) if group_entries is not None else None
|
||||
hit = group_entry is not None or (
|
||||
entry is not None and not _refresh_cache.get()
|
||||
)
|
||||
if group is not None and group.world_size > 1:
|
||||
# Encoders may issue TP/folding collectives. A rank-local eviction
|
||||
# must never leave another rank returning early from the encoder.
|
||||
flag = torch.tensor(int(hit), dtype=torch.int32)
|
||||
dist.all_reduce(flag, op=dist.ReduceOp.MIN, group=group.cpu_group)
|
||||
hit = bool(flag.item())
|
||||
if hit:
|
||||
self.hits += 1
|
||||
if entry is not None:
|
||||
entry.preferred |= _prefer_cache.get()
|
||||
self._entries.move_to_end(key)
|
||||
if group_entry is not None:
|
||||
self.group_hits += 1
|
||||
group_entry.wait()
|
||||
return _copy_output(group_entry.output, share_tensors=share_in_group)
|
||||
entry.wait()
|
||||
logger.debug("Conditioning cache hit: %s.%s", type(model).__name__, method)
|
||||
restored = {}
|
||||
|
||||
def restore(t):
|
||||
if id(t) not in restored:
|
||||
restored[id(t)] = t.data.to(t.device, copy=True, non_blocking=True)
|
||||
return restored[id(t)]
|
||||
|
||||
output = (
|
||||
_copy_output(entry.output)
|
||||
if entry.device_resident
|
||||
else _map_output(entry.output, restore, restore=True)
|
||||
)
|
||||
if keep_on_device and not entry.device_resident:
|
||||
self._entries[key] = _CacheEntry.snapshot(
|
||||
output, entry.size, entry.owner, True, True
|
||||
)
|
||||
self._remember_group(key, output, entry.owner, share_in_group)
|
||||
return output
|
||||
if key is None:
|
||||
self.bypasses += 1
|
||||
return compute()
|
||||
self.misses += 1
|
||||
# A VLM may reuse image features even when its joint text/image key
|
||||
# misses. VAE delegates share one outer posterior entry instead.
|
||||
with self.scope(enabled=nested):
|
||||
output = compute()
|
||||
try:
|
||||
size = 0
|
||||
seen = set()
|
||||
|
||||
def count(t):
|
||||
nonlocal size
|
||||
if id(t) not in seen:
|
||||
size += t.numel() * t.element_size()
|
||||
seen.add(id(t))
|
||||
if type(t) is not torch.Tensor or t.requires_grad:
|
||||
raise Uncacheable("autograd output")
|
||||
return t
|
||||
|
||||
_map_output(output, count)
|
||||
self._remember_group(key, output, self._identity(model), share_in_group)
|
||||
if not cross_request or not size or size > self.max_bytes:
|
||||
self.bypasses += 1
|
||||
return output
|
||||
except Uncacheable:
|
||||
self.bypasses += 1
|
||||
return output
|
||||
old = self._entries.pop(key, None)
|
||||
if old is not None:
|
||||
old.wait()
|
||||
self.bytes -= old.size
|
||||
preferred = _prefer_cache.get()
|
||||
evictable = [
|
||||
key
|
||||
for key, entry in self._entries.items()
|
||||
if preferred or not entry.preferred
|
||||
]
|
||||
available = (
|
||||
self.max_bytes
|
||||
- self.bytes
|
||||
+ sum(self._entries[key].size for key in evictable)
|
||||
)
|
||||
if size > available or (len(self._entries) >= 128 and not evictable):
|
||||
self.bypasses += 1
|
||||
return output
|
||||
for evicted in evictable:
|
||||
if self.bytes + size <= self.max_bytes and len(self._entries) < 128:
|
||||
break
|
||||
removed = self._entries.pop(evicted)
|
||||
removed.wait()
|
||||
self.bytes -= removed.size
|
||||
self.evictions += 1
|
||||
self._entries[key] = _CacheEntry.snapshot(
|
||||
output, size, self._identity(model), preferred, keep_on_device
|
||||
)
|
||||
self.bytes += size
|
||||
logger.debug(
|
||||
"Conditioning cache store: %s.%s, %d bytes",
|
||||
type(model).__name__,
|
||||
method,
|
||||
size,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def _inference_cache(model, *, share_in_group=False, cross_request=True):
|
||||
if torch.compiler.is_compiling():
|
||||
return None
|
||||
cache = _active_cache.get()
|
||||
if cache is None or model.training or torch.is_grad_enabled():
|
||||
return None
|
||||
# FSDP reuses only consumed stage outputs, never sharded intermediate states
|
||||
if not (cross_request and _cross_request_cache.get()) and (
|
||||
not share_in_group or cache._group_entries.get() is None
|
||||
):
|
||||
return None
|
||||
# graph capture cannot hash or copy CUDA values through host memory
|
||||
if torch.cuda.is_available() and torch.cuda.is_current_stream_capturing():
|
||||
return None
|
||||
return cache
|
||||
|
||||
|
||||
def cached_encoder_call(
|
||||
model,
|
||||
args,
|
||||
kwargs,
|
||||
compute,
|
||||
group=None,
|
||||
*,
|
||||
namespace=None,
|
||||
nested=True,
|
||||
share_in_group=False,
|
||||
cross_request=True,
|
||||
):
|
||||
if torch.compiler.is_compiling():
|
||||
return compute()
|
||||
# only consumed negative conditioning is device-hot; raw hidden states and
|
||||
# other encoder boundaries keep their host-cache policy
|
||||
keep_on_device = namespace is not None and share_in_group and _prefer_cache.get()
|
||||
cross_request = cross_request or keep_on_device
|
||||
cache = _inference_cache(
|
||||
model, share_in_group=share_in_group, cross_request=cross_request
|
||||
)
|
||||
if (
|
||||
cache is None
|
||||
or model is _stage_encoder.get()
|
||||
or kwargs.get("use_cache")
|
||||
or kwargs.get("past_key_values") is not None
|
||||
):
|
||||
return compute()
|
||||
|
||||
if not _cross_request_cache.get() and model_parallel_is_initialized():
|
||||
# the encoder's TP group may not cover its FSDP shard group
|
||||
group = get_replica_group()
|
||||
|
||||
def compute_conditioning():
|
||||
# a stage owns the consumed output; preserve nested vision-method caches
|
||||
token = _stage_encoder.set(model)
|
||||
try:
|
||||
return compute()
|
||||
finally:
|
||||
_stage_encoder.reset(token)
|
||||
|
||||
return cache.run(
|
||||
model,
|
||||
"forward",
|
||||
args,
|
||||
kwargs,
|
||||
compute_conditioning if namespace is not None and cross_request else compute,
|
||||
group,
|
||||
nested=nested,
|
||||
namespace=namespace,
|
||||
share_in_group=share_in_group,
|
||||
cross_request=cross_request,
|
||||
keep_on_device=keep_on_device,
|
||||
)
|
||||
|
||||
|
||||
def cached_conditioning(fn):
|
||||
"""Cache a deterministic conditioning method on the current encoder TP group."""
|
||||
|
||||
@wraps(fn)
|
||||
def wrapped(self, *args, **kwargs):
|
||||
cache = _inference_cache(self)
|
||||
if cache is None:
|
||||
return fn(self, *args, **kwargs)
|
||||
group = get_tp_group() if model_parallel_is_initialized() else None
|
||||
return cache.run(
|
||||
self, fn.__name__, args, kwargs, lambda: fn(self, *args, **kwargs), group
|
||||
)
|
||||
|
||||
return wrapped
|
||||
|
||||
|
||||
def cached_vae_encode(fn):
|
||||
"""Cache the posterior, before sample()/mode() and latent normalization.
|
||||
|
||||
Distributed VAEs currently bypass this cache: their encoding methods can
|
||||
run on an owner rank or a model-specific subgroup, unlike encoder TP.
|
||||
"""
|
||||
|
||||
@wraps(fn)
|
||||
def wrapped(self, *args, **kwargs):
|
||||
cache = _inference_cache(self)
|
||||
if cache is None or (model_parallel_is_initialized() and get_world_size() > 1):
|
||||
return fn(self, *args, **kwargs)
|
||||
# Tiling and slicing can change numerical results without changing x.
|
||||
settings = {
|
||||
name: value
|
||||
for name, value in vars(self).items()
|
||||
if name.startswith(
|
||||
(
|
||||
"use_",
|
||||
"tile_",
|
||||
"blend_",
|
||||
"parallel_",
|
||||
"spatial_compression",
|
||||
"temporal_compression",
|
||||
)
|
||||
)
|
||||
and type(value) in (int, float, bool, str, type(None))
|
||||
}
|
||||
return cache.run(
|
||||
self,
|
||||
fn.__name__,
|
||||
(args, settings),
|
||||
kwargs,
|
||||
lambda: fn(self, *args, **kwargs),
|
||||
)
|
||||
|
||||
return wrapped
|
||||
|
||||
|
||||
class ConditioningEncoderMixin:
|
||||
"""Cache replicated encoders without adding a module or altering state_dict."""
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
forward = super().__call__
|
||||
return cached_encoder_call(self, args, kwargs, lambda: forward(*args, **kwargs))
|
||||
@@ -456,6 +456,14 @@ class ComponentResidencyManager:
|
||||
"""Prepare a shared component and wait without making it the active use."""
|
||||
self._prepare_forward_use(use, module=module)
|
||||
|
||||
def finish_unused_component(
|
||||
self, use: ComponentUse, module: nn.Module | None = None
|
||||
) -> None:
|
||||
"""Release retained weights when conditioning reuse skips their use."""
|
||||
if self._active_use is not None and self._same_use(self._active_use, use):
|
||||
return
|
||||
self._finish_use(use, module=module, keep_on_warmup=False, force=True)
|
||||
|
||||
def remove_nvtx_hooks_for_module(self, module: nn.Module | None) -> None:
|
||||
"""Detach NVTX hooks before a component object is deleted or replaced."""
|
||||
if module is None:
|
||||
|
||||
@@ -21,6 +21,7 @@ from sglang.kernels.ops.diffusion import (
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.dits.cosmos3video import Cosmos3VideoConfig
|
||||
from sglang.multimodal_gen.configs.models.fsdp import is_module_list_entry_in
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_conditioning
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_sp_group,
|
||||
get_sp_world_size,
|
||||
@@ -1099,6 +1100,7 @@ class Cosmos3LanguageModel(nn.Module):
|
||||
]
|
||||
)
|
||||
|
||||
@cached_conditioning
|
||||
def forward(
|
||||
self,
|
||||
text_ids: torch.Tensor,
|
||||
|
||||
@@ -13,11 +13,13 @@ from sglang.multimodal_gen.configs.models.encoders import (
|
||||
ImageEncoderConfig,
|
||||
TextEncoderConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_encoder_call
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_replica_group,
|
||||
get_sp_group,
|
||||
get_tp_group,
|
||||
get_world_group,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.group_coordinator import GroupCoordinator
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
@@ -178,10 +180,20 @@ class EncoderTensorParallelMixin:
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
tp_group = self._encoder_tp_group
|
||||
forward = super().__call__
|
||||
cache_group = tp_group
|
||||
if cache_group is None and model_parallel_is_initialized():
|
||||
cache_group = get_tp_group()
|
||||
|
||||
def run():
|
||||
return cached_encoder_call(
|
||||
self, args, kwargs, lambda: forward(*args, **kwargs), cache_group
|
||||
)
|
||||
|
||||
if tp_group is None:
|
||||
return super().__call__(*args, **kwargs)
|
||||
return run()
|
||||
with use_tensor_parallel_group(tp_group):
|
||||
return super().__call__(*args, **kwargs)
|
||||
return run()
|
||||
|
||||
|
||||
class TextEncoder(
|
||||
|
||||
@@ -11,6 +11,7 @@ from torch import nn
|
||||
|
||||
from sglang.multimodal_gen.configs.models.encoders.base import BaseEncoderOutput
|
||||
from sglang.multimodal_gen.configs.models.encoders.gemma_3 import Gemma3Config
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_conditioning
|
||||
from sglang.multimodal_gen.runtime.distributed import get_tp_world_size
|
||||
from sglang.multimodal_gen.runtime.layers.activation import GeluAndMul
|
||||
from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
@@ -772,8 +773,7 @@ class Gemma3ForConditionalGeneration(
|
||||
elif pixel_values.dim() != 4:
|
||||
raise ValueError(f"Unexpected pixel_values shape: {pixel_values.shape}")
|
||||
|
||||
vision_outputs = self.vision_tower(pixel_values)
|
||||
image_features = self.multi_modal_projector(vision_outputs)
|
||||
image_features = self._encode_image_features(pixel_values)
|
||||
image_features = image_features.to(
|
||||
device=inputs_embeds.device, dtype=inputs_embeds.dtype
|
||||
)
|
||||
@@ -788,6 +788,10 @@ class Gemma3ForConditionalGeneration(
|
||||
llm_input_ids, inputs_embeds=inputs_embeds, **kwargs
|
||||
)
|
||||
|
||||
@cached_conditioning
|
||||
def _encode_image_features(self, pixel_values):
|
||||
return self.multi_modal_projector(self.vision_tower(pixel_values))
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]) -> Set[str]:
|
||||
loaded_params: Set[str] = set()
|
||||
params_dict = dict(self.named_parameters())
|
||||
|
||||
@@ -11,6 +11,7 @@ from transformers import (
|
||||
Dinov2Model,
|
||||
)
|
||||
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningEncoderMixin
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
@@ -32,7 +33,9 @@ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
|
||||
return np.concatenate([emb_sin, emb_cos], axis=1)
|
||||
|
||||
|
||||
class ImageEncoder(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
class ImageEncoder(
|
||||
ConditioningEncoderMixin, nn.Module, LayerwiseOffloadableModuleMixin
|
||||
):
|
||||
layerwise_offload_dit_group_enabled = False
|
||||
layer_names = [
|
||||
"model.encoder.layer",
|
||||
@@ -212,7 +215,9 @@ def build_image_encoder(config):
|
||||
raise ValueError(f"Unknown image encoder type: {config['type']}")
|
||||
|
||||
|
||||
class DualImageEncoder(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
class DualImageEncoder(
|
||||
ConditioningEncoderMixin, nn.Module, LayerwiseOffloadableModuleMixin
|
||||
):
|
||||
layerwise_offload_dit_group_enabled = False
|
||||
layer_names = [
|
||||
"main_image_encoder.model.encoder.layer",
|
||||
@@ -249,7 +254,9 @@ class DualImageEncoder(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
return outputs
|
||||
|
||||
|
||||
class SingleImageEncoder(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
class SingleImageEncoder(
|
||||
ConditioningEncoderMixin, nn.Module, LayerwiseOffloadableModuleMixin
|
||||
):
|
||||
layerwise_offload_dit_group_enabled = False
|
||||
layer_names = [
|
||||
"main_image_encoder.model.encoder.layer",
|
||||
|
||||
@@ -81,6 +81,8 @@ from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import (
|
||||
Qwen2_5_VLModelOutputWithPast,
|
||||
)
|
||||
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_conditioning
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -911,6 +913,7 @@ class Qwen2_5_VLModel(nn.Module):
|
||||
video_embeds = torch.split(video_embeds, split_sizes)
|
||||
return video_embeds
|
||||
|
||||
@cached_conditioning
|
||||
def get_image_features(
|
||||
self,
|
||||
pixel_values: torch.FloatTensor,
|
||||
|
||||
@@ -57,6 +57,8 @@ import torch
|
||||
import torch.nn as nn
|
||||
from transformers.activations import ACT2FN
|
||||
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_conditioning
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from transformers.modeling_outputs import BaseModelOutputWithPast
|
||||
@@ -910,6 +912,7 @@ class Qwen3VLModel(nn.Module):
|
||||
# Same implementation as for images
|
||||
return self.get_image_features(pixel_values_videos, video_grid_thw)
|
||||
|
||||
@cached_conditioning
|
||||
def _get_flat_visual_features(
|
||||
self,
|
||||
pixel_values: torch.FloatTensor,
|
||||
|
||||
+12
-6
@@ -13,6 +13,8 @@ from transformers.modeling_outputs import CausalLMOutputWithPast
|
||||
from transformers.modeling_utils import PreTrainedModel
|
||||
from transformers.utils import logging
|
||||
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_conditioning
|
||||
|
||||
from .configuration_neo_chat import NEOChatConfig, NEOMoELLMConfig
|
||||
from .conversation import get_conv_template
|
||||
from .modeling_fm_modules import (
|
||||
@@ -484,12 +486,16 @@ class NEOChatModel(PreTrainedModel):
|
||||
grid_hw=grid_hw,
|
||||
).last_hidden_state
|
||||
else:
|
||||
return self.vision_model(
|
||||
pixel_values=pixel_values,
|
||||
output_hidden_states=False,
|
||||
return_dict=True,
|
||||
grid_hw=grid_hw,
|
||||
).last_hidden_state
|
||||
return self._reference_image_features(pixel_values, grid_hw)
|
||||
|
||||
@cached_conditioning
|
||||
def _reference_image_features(self, pixel_values, grid_hw):
|
||||
return self.vision_model(
|
||||
pixel_values=pixel_values,
|
||||
output_hidden_states=False,
|
||||
return_dict=True,
|
||||
grid_hw=grid_hw,
|
||||
).last_hidden_state
|
||||
|
||||
def batch_chat(
|
||||
self,
|
||||
|
||||
@@ -22,6 +22,7 @@ from diffusers.models.modeling_outputs import AutoencoderKLOutput
|
||||
from torch import nn
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.base import VAEConfig
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
@@ -287,6 +288,7 @@ class AutoencoderKL(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
|
||||
return enc
|
||||
|
||||
@cached_vae_encode
|
||||
def encode(
|
||||
self, x: torch.Tensor, return_dict: bool = True
|
||||
) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]:
|
||||
|
||||
@@ -7,6 +7,7 @@ from diffusers.models.autoencoders.vae import DecoderOutput
|
||||
from torch import nn
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.sana import SanaVAEConfig
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_decode_parallel_rank,
|
||||
get_decode_parallel_world_size,
|
||||
@@ -135,6 +136,7 @@ class AutoencoderDC(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
return next(self._inner_model.parameters()).device
|
||||
return torch.device("cpu")
|
||||
|
||||
@cached_vae_encode
|
||||
def encode(self, x: torch.Tensor, **kwargs):
|
||||
self._ensure_inner_model()
|
||||
return self._inner_model.encode(x, **kwargs)
|
||||
|
||||
@@ -19,6 +19,7 @@ from diffusers.models.autoencoders.vae import (
|
||||
from diffusers.models.modeling_outputs import AutoencoderKLOutput
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.flux import Flux2VAEConfig
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
|
||||
from sglang.multimodal_gen.runtime.models.vaes.common import (
|
||||
ParallelTiledVAE,
|
||||
can_install_spatial_shard_parallel_decode,
|
||||
@@ -238,6 +239,7 @@ class AutoencoderKLFlux2(ParallelTiledVAE):
|
||||
|
||||
return enc
|
||||
|
||||
@cached_vae_encode
|
||||
def encode(
|
||||
self, x: torch.Tensor, return_dict: bool = True
|
||||
) -> Union[DiagonalGaussianDistribution]:
|
||||
|
||||
@@ -14,6 +14,7 @@ from diffusers.models.autoencoders.vae import (
|
||||
from diffusers.models.modeling_outputs import AutoencoderKLOutput
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.qwenimage import QwenImageVAEConfig
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_decode_parallel_rank,
|
||||
@@ -1069,6 +1070,7 @@ class AutoencoderKLQwenImage(ParallelTiledVAE):
|
||||
self.clear_cache()
|
||||
return enc
|
||||
|
||||
@cached_vae_encode
|
||||
def encode(
|
||||
self, x: torch.Tensor, return_dict: bool = True
|
||||
) -> DiagonalGaussianDistribution:
|
||||
|
||||
@@ -18,6 +18,10 @@ from sglang.multimodal_gen.configs.models import VAEConfig
|
||||
from sglang.multimodal_gen.configs.models.vaes.base import (
|
||||
should_use_spatial_shard_parallel_decode,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import (
|
||||
cached_vae_encode,
|
||||
register_conditioning_container,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_decode_parallel_group_coordinator,
|
||||
get_decode_parallel_world_size,
|
||||
@@ -137,6 +141,7 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
def _decode(self, *args, **kwargs) -> torch.Tensor:
|
||||
pass
|
||||
|
||||
@cached_vae_encode
|
||||
def encode(self, x: torch.Tensor) -> DiagonalGaussianDistribution:
|
||||
batch_size, num_channels, num_frames, height, width = x.shape
|
||||
latent_num_frames = (num_frames - 1) // self.temporal_compression_ratio + 1
|
||||
@@ -748,6 +753,7 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
|
||||
|
||||
# adapted from https://github.com/huggingface/diffusers/blob/e7ffeae0a191f710881d1fbde00cd6ff025e81f2/src/diffusers/models/autoencoders/vae.py#L691
|
||||
@register_conditioning_container
|
||||
class DiagonalGaussianDistribution:
|
||||
def __init__(self, parameters: torch.Tensor, deterministic: bool = False):
|
||||
self.parameters = parameters
|
||||
|
||||
@@ -12,6 +12,7 @@ from einops import rearrange
|
||||
from torch import nn
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.dac import DacVAEConfig
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
@@ -506,6 +507,7 @@ class DAC(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
|
||||
return audio_data
|
||||
|
||||
@cached_vae_encode
|
||||
def encode(
|
||||
self,
|
||||
audio_data: torch.Tensor,
|
||||
|
||||
@@ -3,6 +3,7 @@ from typing import Any
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningEncoderMixin
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
@@ -113,7 +114,9 @@ def _make_ltx23_encoder_block(
|
||||
)
|
||||
|
||||
|
||||
class LTX23VideoConditionEncoder(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
class LTX23VideoConditionEncoder(
|
||||
ConditioningEncoderMixin, nn.Module, LayerwiseOffloadableModuleMixin
|
||||
):
|
||||
layerwise_offload_dit_group_enabled = False
|
||||
layer_names = ["down_blocks"]
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ from diffusers.models.modeling_outputs import AutoencoderKLOutput
|
||||
from torch import nn
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.ltx_audio import LTXAudioVAEConfig
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
|
||||
from sglang.multimodal_gen.runtime.models.vaes.common import ParallelTiledVAE
|
||||
|
||||
LATENT_DOWNSAMPLE_FACTOR = 4
|
||||
@@ -868,6 +869,7 @@ class AutoencoderKLLTX2Audio(ParallelTiledVAE):
|
||||
def _encode(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.encoder(x)
|
||||
|
||||
@cached_vae_encode
|
||||
def encode(self, x: torch.Tensor, return_dict: bool = True):
|
||||
if self.use_slicing and x.shape[0] > 1:
|
||||
encoded_slices = [self._encode(x_slice) for x_slice in x.split(1)]
|
||||
|
||||
@@ -12,6 +12,7 @@ from diffusers.models.embeddings import PixArtAlphaCombinedTimestepSizeEmbedding
|
||||
from diffusers.models.modeling_outputs import AutoencoderKLOutput
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.ltx_video import LTXVideoVAEConfig
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_decode_parallel_rank,
|
||||
get_decode_parallel_world_size,
|
||||
@@ -1857,6 +1858,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
|
||||
return enc
|
||||
|
||||
@cached_vae_encode
|
||||
def encode(
|
||||
self, x: torch.Tensor, causal: Optional[bool] = None, return_dict: bool = True
|
||||
) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]:
|
||||
|
||||
@@ -14,6 +14,7 @@ from diffusers.models import ModelMixin
|
||||
from diffusers.utils import logging
|
||||
from PIL import Image
|
||||
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_decode_parallel_group_coordinator,
|
||||
get_decode_parallel_rank,
|
||||
@@ -1281,6 +1282,7 @@ class AutoencoderKLLegacy(AutoencoderKL):
|
||||
self.decoder = ViT3DDecoder(**vit_kwargs)
|
||||
|
||||
@torch.no_grad()
|
||||
@cached_vae_encode
|
||||
def encode(self, x):
|
||||
return self.quant_conv(self.encoder(x))
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ from sglang.multimodal_gen.configs.models.vaes import WanVAEConfig
|
||||
from sglang.multimodal_gen.configs.models.vaes.base import (
|
||||
should_use_spatial_shard_parallel_decode,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_decode_parallel_rank,
|
||||
get_decode_parallel_world_size,
|
||||
@@ -1585,6 +1586,7 @@ class AutoencoderKLWan(ParallelTiledVAE):
|
||||
self._causal_decode_initialized = True
|
||||
return out
|
||||
|
||||
@cached_vae_encode
|
||||
def encode(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self.use_feature_cache:
|
||||
self.clear_cache()
|
||||
|
||||
@@ -62,10 +62,6 @@ class SanaVideoTextEncodingStage(TextEncodingStage):
|
||||
return [item.lower().strip() for item in text]
|
||||
|
||||
def _encode_negative_text(self, batch, server_args, all_indices):
|
||||
cache_key = self._build_negative_text_cache_key(batch, server_args, all_indices)
|
||||
cached = self._get_cached_negative_text_embedding(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
outputs = self.encode_text(
|
||||
self._normalize_text(batch.negative_prompt),
|
||||
server_args,
|
||||
@@ -73,7 +69,6 @@ class SanaVideoTextEncodingStage(TextEncodingStage):
|
||||
return_attention_mask=True,
|
||||
max_length=300,
|
||||
)
|
||||
self._maybe_cache_negative_text_embedding(cache_key, outputs)
|
||||
return outputs
|
||||
|
||||
@torch.no_grad()
|
||||
|
||||
@@ -12,6 +12,10 @@ from typing import TYPE_CHECKING, Any, Callable, List
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import (
|
||||
ConditioningCache,
|
||||
conditioning_cache_group,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import get_world_rank
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
@@ -54,6 +58,11 @@ class PipelineExecutor(ABC):
|
||||
def __init__(self, server_args):
|
||||
self.server_args = server_args
|
||||
self.component_residency_manager = None
|
||||
self.conditioning_cache = ConditioningCache(
|
||||
int(server_args.conditioning_cache_max_size_mb * 1024**2)
|
||||
if not server_args.disable_conditioning_cache
|
||||
else 0
|
||||
)
|
||||
|
||||
def begin_component_residency_request(
|
||||
self,
|
||||
@@ -87,7 +96,11 @@ class PipelineExecutor(ABC):
|
||||
):
|
||||
self.begin_component_residency_request(stages, payload, server_args)
|
||||
try:
|
||||
yield
|
||||
with self.conditioning_cache.scope(
|
||||
cross_request=not server_args.use_fsdp_inference,
|
||||
refresh=self._is_warmup_payload(payload),
|
||||
):
|
||||
yield
|
||||
finally:
|
||||
self.finish_component_residency_request()
|
||||
|
||||
@@ -117,9 +130,13 @@ class PipelineExecutor(ABC):
|
||||
self.before_stage(stage, stage_index, payload, server_args)
|
||||
with maybe_record_function(f"STAGE {stage_name}"):
|
||||
with maybe_nvtx_range(f"stage_{stage_name}", use_nvtx):
|
||||
payload = self.run_stage_with_context(
|
||||
stage, payload, server_args, run_stage
|
||||
)
|
||||
with conditioning_cache_group(
|
||||
enabled=(isinstance(payload, list) and len(payload) > 1)
|
||||
or (isinstance(payload, Req) and payload.batch_size > 1)
|
||||
):
|
||||
payload = self.run_stage_with_context(
|
||||
stage, payload, server_args, run_stage
|
||||
)
|
||||
return payload
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -13,6 +13,9 @@ from safetensors.torch import load_file
|
||||
from torch.distributed.tensor import DTensor
|
||||
|
||||
from sglang.multimodal_gen import envs
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import (
|
||||
invalidate_conditioning_caches,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import (
|
||||
BaseLayerWithLoRA,
|
||||
@@ -1036,6 +1039,14 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
self.loaded_adapter_alphas[lora_nickname] = adapter_lora_alpha
|
||||
logger.info("Rank %d: loaded LoRA adapter %s", rank, lora_path)
|
||||
|
||||
def _invalidate_lora_conditioning(self):
|
||||
# LoRA targets DiTs; unrelated text and VAE weights remain unchanged
|
||||
invalidate_conditioning_caches(
|
||||
module
|
||||
for name in ("transformer", "transformer_2", "fake_score_transformer")
|
||||
if (module := self.modules.get(name)) is not None
|
||||
)
|
||||
|
||||
def set_lora(
|
||||
self,
|
||||
lora_nickname: str | list[str],
|
||||
@@ -1055,6 +1066,7 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
stop costing anonymous host memory; pass it only for the startup
|
||||
(static) adapter, where the merged combination is stable.
|
||||
"""
|
||||
self._invalidate_lora_conditioning()
|
||||
merge_mode = self._resolve_lora_merge_mode(merge_weights, merge_mode)
|
||||
|
||||
# Normalize inputs to lists for multi-LoRA support
|
||||
@@ -1303,6 +1315,7 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
Disable LoRA for the specified target, regardless of whether weights were
|
||||
merged into the base model or are still active in the wrapped LoRA path.
|
||||
"""
|
||||
self._invalidate_lora_conditioning()
|
||||
target_modules, error = self._get_target_lora_layers(target)
|
||||
if error:
|
||||
logger.warning("deactivate_lora_weights: %s", error)
|
||||
@@ -1340,6 +1353,7 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
"transformer_2", "critic".
|
||||
strength: LoRA strength for merge, default 1.0.
|
||||
"""
|
||||
self._invalidate_lora_conditioning()
|
||||
target_modules, error = self._get_target_lora_layers(target)
|
||||
if error:
|
||||
logger.warning("merge_lora_weights: %s", error)
|
||||
@@ -1412,6 +1426,7 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
target: Which transformer(s) to unmerge. One of "all", "transformer",
|
||||
"transformer_2", "critic".
|
||||
"""
|
||||
self._invalidate_lora_conditioning()
|
||||
target_modules, error = self._get_target_lora_layers(target)
|
||||
if error:
|
||||
logger.warning("unmerge_lora_weights: %s", error)
|
||||
|
||||
@@ -287,6 +287,12 @@ class PipelineStage(StageDedupMixin, ABC):
|
||||
)
|
||||
self._component_residency_manager.begin_use(use, module=module)
|
||||
|
||||
def finish_unused_declared_component(self, *, component_name: str, module=None):
|
||||
if self._component_residency_manager is None:
|
||||
return
|
||||
use = self._declared_component_use(component_name=component_name)
|
||||
self._component_residency_manager.finish_unused_component(use, module=module)
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
|
||||
@@ -8,6 +8,8 @@ from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import conditioning_cache_group
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
@@ -91,16 +93,19 @@ class StageDedupMixin:
|
||||
|
||||
results: list[Req | None] = [None] * len(batches)
|
||||
|
||||
for _, group in self._group_requests_by_fingerprint(
|
||||
groups = self._group_requests_by_fingerprint(
|
||||
batches, lambda batch: self.build_dedup_fingerprint(batch, server_args)
|
||||
):
|
||||
first_index, first_batch = group[0]
|
||||
first_result = self(first_batch, server_args)
|
||||
results[first_index] = first_result
|
||||
)
|
||||
# a single computed request needs no group memo or cache-hit collective
|
||||
with conditioning_cache_group(enabled=len(groups) > 1):
|
||||
for _, group in groups:
|
||||
first_index, first_batch = group[0]
|
||||
first_result = self(first_batch, server_args)
|
||||
results[first_index] = first_result
|
||||
|
||||
for index, batch in group[1:]:
|
||||
copy_outputs(first_result, batch)
|
||||
results[index] = batch
|
||||
for index, batch in group[1:]:
|
||||
copy_outputs(first_result, batch)
|
||||
results[index] = batch
|
||||
|
||||
return [result for result in results if result is not None]
|
||||
|
||||
|
||||
+1
@@ -117,6 +117,7 @@ class Ideogram4Scheduler:
|
||||
|
||||
|
||||
class Ideogram4TextEncodingStage(TextEncodingStage):
|
||||
deduplicated_output_fields = ("prompt_embeds", "prompt_embeds_mask")
|
||||
deduplicated_extra_tensor_tree_output_keys = ("ideogram4",)
|
||||
|
||||
def __init__(self, text_encoder, tokenizer) -> None:
|
||||
|
||||
+35
-1
@@ -2,7 +2,13 @@
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_encoder_call
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_local_torch_device,
|
||||
get_tp_group,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
|
||||
TextEncodingStage,
|
||||
@@ -35,6 +41,8 @@ VIDEO_PROMPT_TEMPLATE = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
class LingBotVideoTextEncodingStage(TextEncodingStage):
|
||||
"""Qwen3-VL prompt/negative encoding for LingBot-Video MoE (T2V, base)."""
|
||||
|
||||
deduplicated_output_fields = ()
|
||||
|
||||
def __init__(self, text_encoders, tokenizers, transformer):
|
||||
super().__init__(text_encoders, tokenizers)
|
||||
self.transformer = transformer
|
||||
@@ -106,6 +114,32 @@ class LingBotVideoTextEncodingStage(TextEncodingStage):
|
||||
)
|
||||
|
||||
inputs = self._build_prompt_inputs(prompt)
|
||||
cache_group = (
|
||||
text_encoder._encoder_tp_group
|
||||
if isinstance(text_encoder, TextEncoder)
|
||||
else None
|
||||
)
|
||||
if cache_group is None and model_parallel_is_initialized():
|
||||
cache_group = get_tp_group()
|
||||
return cached_encoder_call(
|
||||
text_encoder,
|
||||
(dict(inputs),),
|
||||
{
|
||||
"device": str(device),
|
||||
"dtype": dtype,
|
||||
"skip_layer": self.hidden_state_skip_layer,
|
||||
"crop_start": self._compute_crop_start(),
|
||||
},
|
||||
lambda: self._encode_inputs(inputs, device, dtype),
|
||||
cache_group,
|
||||
namespace=self,
|
||||
nested=False,
|
||||
share_in_group=True,
|
||||
)
|
||||
|
||||
def _encode_inputs(self, inputs, device, dtype):
|
||||
self._begin_text_encoder_use(0)
|
||||
text_encoder = self.text_encoders[0]
|
||||
inputs = inputs.to(device)
|
||||
outputs = self._forward_text_encoder(
|
||||
text_encoder,
|
||||
|
||||
+1
-9
@@ -51,15 +51,7 @@ def _causal_block_count(batch: Req, server_args: ServerArgs) -> int:
|
||||
|
||||
|
||||
class LongLive2TextEncodingStage(TextEncodingStage):
|
||||
def build_dedup_fingerprint(self, batch: Req, server_args: ServerArgs):
|
||||
base = super().build_dedup_fingerprint(batch, server_args)
|
||||
return (
|
||||
base,
|
||||
self.freeze_for_dedup(getattr(batch, "shot_prompts", None)),
|
||||
self.freeze_for_dedup(getattr(batch, "shot_durations", None)),
|
||||
int(getattr(batch, "chunks_per_shot", 0) or 0),
|
||||
getattr(batch, "scene_cut_prefix", None),
|
||||
)
|
||||
deduplicated_output_fields = ()
|
||||
|
||||
def _block_prompts(self, batch: Req, server_args: ServerArgs) -> list[str] | None:
|
||||
shot_prompts = getattr(batch, "shot_prompts", None)
|
||||
|
||||
+34
-8
@@ -15,8 +15,13 @@ from transformers.models.qwen2_vl.image_processing_pil_qwen2_vl import (
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.ming_image import (
|
||||
MingImageLayerPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_encoder_call
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_local_torch_device,
|
||||
get_tp_group,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
@@ -31,6 +36,8 @@ from sglang.multimodal_gen.runtime.utils.vision import load_image
|
||||
|
||||
|
||||
class MingImageEncodingStage(TextEncodingStage):
|
||||
deduplicated_output_fields = ()
|
||||
|
||||
def __init__(self, text_encoders, tokenizers):
|
||||
super().__init__(text_encoders, tokenizers)
|
||||
self.image_processor = Qwen2VLImageProcessorPil.from_pretrained(
|
||||
@@ -131,13 +138,32 @@ class MingImageEncodingStage(TextEncodingStage):
|
||||
input_ids=torch.tensor([ids], device=device),
|
||||
position_ids=ming_position_ids(ids, grids, encoder.image_token).to(device),
|
||||
)
|
||||
self._begin_text_encoder_use(0)
|
||||
# the official MoE router runs under BF16 autocast, including its logits
|
||||
with torch.autocast(device.type, dtype=encoder.dtype):
|
||||
encoded = self._forward_text_encoder(encoder, inputs)
|
||||
batch.prompt_embeds = [encoded.last_hidden_state]
|
||||
batch.negative_prompt_embeds = [torch.zeros_like(encoded.last_hidden_state)]
|
||||
batch.extra["ming_direct"] = encoded.hidden_states[0]
|
||||
|
||||
def encode_conditioning():
|
||||
self._begin_text_encoder_use(0)
|
||||
# the official MoE router runs under BF16 autocast, including its logits
|
||||
with torch.autocast(device.type, dtype=encoder.dtype):
|
||||
encoded = self._forward_text_encoder(encoder, inputs)
|
||||
return encoded.last_hidden_state, encoded.hidden_states[0]
|
||||
|
||||
cache_group = encoder._encoder_tp_group
|
||||
if cache_group is None and model_parallel_is_initialized():
|
||||
cache_group = get_tp_group()
|
||||
prompt_embeds, direct = cached_encoder_call(
|
||||
encoder,
|
||||
(inputs,),
|
||||
{},
|
||||
encode_conditioning,
|
||||
cache_group,
|
||||
namespace=self,
|
||||
share_in_group=True,
|
||||
)
|
||||
self.finish_unused_declared_component(
|
||||
component_name="text_encoder", module=encoder
|
||||
)
|
||||
batch.prompt_embeds = [prompt_embeds]
|
||||
batch.negative_prompt_embeds = [torch.zeros_like(prompt_embeds)]
|
||||
batch.extra["ming_direct"] = direct
|
||||
return batch
|
||||
|
||||
|
||||
|
||||
+24
-1
@@ -4,7 +4,12 @@ import math
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_encoder_call
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_local_torch_device,
|
||||
get_tp_group,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
@@ -83,6 +88,24 @@ class QwenImage21EncodingStage(PipelineStage):
|
||||
]
|
||||
|
||||
def encode_prompt(self, prompt, images, device):
|
||||
cache_group = self.text_encoder._encoder_tp_group
|
||||
if cache_group is None and model_parallel_is_initialized():
|
||||
cache_group = get_tp_group()
|
||||
conditioning = cached_encoder_call(
|
||||
self.text_encoder,
|
||||
(prompt, images),
|
||||
{"device": str(device)},
|
||||
lambda: self._encode_prompt(prompt, images, device),
|
||||
cache_group,
|
||||
namespace=self,
|
||||
share_in_group=True,
|
||||
)
|
||||
self.finish_unused_declared_component(
|
||||
component_name="text_encoder", module=self.text_encoder
|
||||
)
|
||||
return conditioning
|
||||
|
||||
def _encode_prompt(self, prompt, images, device):
|
||||
prefix = " ".join(
|
||||
f"<image{i + 1}><|vision_start|><|image_pad|><|vision_end|>"
|
||||
for i in range(len(images))
|
||||
|
||||
+2
@@ -637,6 +637,8 @@ class SanaWMTextEncodingStage(TextEncodingStage):
|
||||
model-specific prompt-window contract.
|
||||
"""
|
||||
|
||||
deduplicated_output_fields = ()
|
||||
|
||||
@staticmethod
|
||||
def _text_encoder_max_length(server_args: ServerArgs) -> int:
|
||||
encoder_cfg = server_args.pipeline_config.text_encoder_configs[0]
|
||||
|
||||
+9
-1
@@ -16,6 +16,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import conditioning_weights_epoch
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
@@ -109,6 +110,8 @@ class RealtimeTextState(BaseRealtimeState):
|
||||
class RealtimeTextEncodingStage(TextEncodingStage):
|
||||
"""Cache text encoder outputs across realtime chunks by prompt identity."""
|
||||
|
||||
deduplicated_output_fields = ()
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
@@ -120,6 +123,7 @@ class RealtimeTextEncodingStage(TextEncodingStage):
|
||||
|
||||
def _make_cache_key(self, batch: Req) -> tuple[Any, ...]:
|
||||
return (
|
||||
conditioning_weights_epoch(),
|
||||
_normalize_prompt_value(batch.prompt),
|
||||
bool(batch.do_classifier_free_guidance),
|
||||
(
|
||||
@@ -147,7 +151,11 @@ class RealtimeTextEncodingStage(TextEncodingStage):
|
||||
batch: Req,
|
||||
server_args: ServerArgs,
|
||||
) -> Req:
|
||||
if batch.session is None:
|
||||
if (
|
||||
batch.session is None
|
||||
or server_args.disable_conditioning_cache
|
||||
or server_args.conditioning_cache_max_size_mb == 0
|
||||
):
|
||||
return super().forward(batch, server_args)
|
||||
|
||||
state = batch.session.get_or_create_state(RealtimeTextState)
|
||||
|
||||
@@ -8,17 +8,21 @@ This module contains implementations of prompt encoding stages for diffusion pip
|
||||
"""
|
||||
|
||||
import inspect
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import TextConditioningOutput
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import (
|
||||
cached_encoder_call,
|
||||
prefer_conditioning_cache,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_encoder_data_parallel_group,
|
||||
get_local_torch_device,
|
||||
get_replica_group,
|
||||
get_tp_group,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
@@ -102,27 +106,6 @@ def _data_parallel_text_encode(forward_fn, forward_kwargs: dict, group):
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_model_default_negative_prompt(
|
||||
model_path: str, backend: Any, model_id: str | None
|
||||
):
|
||||
from sglang.multimodal_gen.registry import get_model_info
|
||||
|
||||
model_info = get_model_info(model_path, backend=backend, model_id=model_id)
|
||||
if model_info is None:
|
||||
return None
|
||||
return model_info.sampling_param_cls().negative_prompt
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TextEncodingFingerprint:
|
||||
prompt: Any
|
||||
negative_prompt: Any
|
||||
do_classifier_free_guidance: bool
|
||||
prompt_template: Any
|
||||
max_sequence_length: int | None
|
||||
|
||||
|
||||
def stack_tensors(name: str, tensors: list[torch.Tensor]) -> torch.Tensor:
|
||||
base_shape = list(tensors[0].shape)
|
||||
for tensor in tensors[1:]:
|
||||
@@ -157,6 +140,26 @@ class TextEncodingStage(ConditionEncodingStage):
|
||||
"is_prompt_processed",
|
||||
)
|
||||
|
||||
def build_dedup_fingerprint(self, batch: Req, server_args: ServerArgs):
|
||||
return (
|
||||
self.freeze_for_dedup(batch.prompt),
|
||||
self.freeze_for_dedup(batch.negative_prompt),
|
||||
bool(batch.do_classifier_free_guidance),
|
||||
self.freeze_for_dedup(batch.prompt_template),
|
||||
batch.max_sequence_length,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def copy_stage_output(cls, value):
|
||||
# embeddings stay shared; nested metadata containers belong to each request
|
||||
if isinstance(value, list):
|
||||
return [cls.copy_stage_output(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(cls.copy_stage_output(item) for item in value)
|
||||
if isinstance(value, dict):
|
||||
return {key: cls.copy_stage_output(item) for key, item in value.items()}
|
||||
return value
|
||||
|
||||
def __init__(self, text_encoders, tokenizers) -> None:
|
||||
"""
|
||||
Initialize the prompt encoding stage.
|
||||
@@ -165,8 +168,6 @@ class TextEncodingStage(ConditionEncodingStage):
|
||||
super().__init__()
|
||||
self.tokenizers = tokenizers
|
||||
self.text_encoders = text_encoders
|
||||
self._negative_text_cache_key = None
|
||||
self._negative_text_cache_value = None
|
||||
self._dp_choice_logged = False
|
||||
|
||||
def component_uses(
|
||||
@@ -178,6 +179,7 @@ class TextEncodingStage(ConditionEncodingStage):
|
||||
stage_name=stage_name,
|
||||
component_name="text_encoder" if i == 0 else f"text_encoder_{i + 1}",
|
||||
preferred_ready_after_request=i == 0,
|
||||
start_at_stage_entry=False,
|
||||
target_dtype=resolve_component_precision_override(
|
||||
server_args,
|
||||
"text_encoder" if i == 0 else f"text_encoder_{i + 1}",
|
||||
@@ -189,100 +191,14 @@ class TextEncodingStage(ConditionEncodingStage):
|
||||
def get_or_compute_negative_text_embedding(
|
||||
self, batch: Req, server_args: ServerArgs, all_indices: list[int]
|
||||
):
|
||||
"""Get the cached text embedding result or compute
|
||||
|
||||
this is a one-slot cache for the model-default negative prompt:
|
||||
most requests don't override the negative prompt, the cache hit rate is considerably high
|
||||
|
||||
invariant: hit/miss must match across ranks -- a miss runs encode_text,
|
||||
which may issue collectives (folding, dp encoding), so a split would
|
||||
deadlock; keep any future eviction rank-global
|
||||
"""
|
||||
negative_cache_key = self._build_negative_text_cache_key(
|
||||
batch, server_args, all_indices
|
||||
)
|
||||
cached_negative = self._get_cached_negative_text_embedding(negative_cache_key)
|
||||
if cached_negative is not None:
|
||||
return cached_negative
|
||||
|
||||
negative_text_outputs = self.encode_text(
|
||||
batch.negative_prompt,
|
||||
server_args,
|
||||
encoder_index=all_indices,
|
||||
return_attention_mask=True,
|
||||
)
|
||||
self._maybe_cache_negative_text_embedding(
|
||||
negative_cache_key, negative_text_outputs
|
||||
)
|
||||
return negative_text_outputs
|
||||
|
||||
def _should_cache_negative_text_embedding(
|
||||
self, batch: Req, server_args: ServerArgs
|
||||
) -> bool:
|
||||
if not batch.is_warmup:
|
||||
return True
|
||||
return self._uses_model_default_negative_prompt(batch, server_args)
|
||||
|
||||
def _get_cached_negative_text_embedding(self, negative_cache_key):
|
||||
if negative_cache_key is None:
|
||||
return None
|
||||
if self._negative_text_cache_key == negative_cache_key:
|
||||
return self._negative_text_cache_value
|
||||
return None
|
||||
|
||||
def _maybe_cache_negative_text_embedding(
|
||||
self,
|
||||
negative_cache_key,
|
||||
negative_text_outputs,
|
||||
) -> None:
|
||||
|
||||
# skip caching if None
|
||||
if negative_cache_key is None:
|
||||
return
|
||||
self._negative_text_cache_key = negative_cache_key
|
||||
self._negative_text_cache_value = tuple(
|
||||
tuple(value) for value in negative_text_outputs
|
||||
)
|
||||
|
||||
def _build_negative_text_cache_key(
|
||||
self, batch: Req, server_args: ServerArgs, encoder_indices: list[int]
|
||||
):
|
||||
"""if the current req doesn't worth caching, returns None"""
|
||||
# skip if we don't cache for current req
|
||||
if not self._should_cache_negative_text_embedding(batch, server_args):
|
||||
return None
|
||||
|
||||
# Negative text encoding changes when the template or max length changes,
|
||||
# even if the visible negative prompt string is the same.
|
||||
return (
|
||||
tuple(encoder_indices),
|
||||
self.freeze_for_dedup(batch.negative_prompt),
|
||||
self.freeze_for_dedup(batch.prompt_template),
|
||||
batch.max_sequence_length,
|
||||
)
|
||||
|
||||
def _uses_model_default_negative_prompt(
|
||||
self, batch: Req, server_args: ServerArgs
|
||||
) -> bool:
|
||||
default_negative_prompt = self._get_model_default_negative_prompt(server_args)
|
||||
if default_negative_prompt is None:
|
||||
return False
|
||||
return self._normalize_negative_prompt_for_default_match(
|
||||
batch.negative_prompt
|
||||
) == self._normalize_negative_prompt_for_default_match(default_negative_prompt)
|
||||
|
||||
def _get_model_default_negative_prompt(self, server_args: ServerArgs) -> str | None:
|
||||
return get_model_default_negative_prompt(
|
||||
server_args.model_path,
|
||||
server_args.backend,
|
||||
server_args.model_id,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_negative_prompt_for_default_match(value):
|
||||
if isinstance(value, str) and not value.isspace():
|
||||
return value.strip()
|
||||
return value
|
||||
"""Encode negative conditioning through the shared encoder cache."""
|
||||
with prefer_conditioning_cache():
|
||||
return self.encode_text(
|
||||
batch.negative_prompt,
|
||||
server_args,
|
||||
encoder_index=all_indices,
|
||||
return_attention_mask=True,
|
||||
)
|
||||
|
||||
def _append_positive_text_outputs(
|
||||
self,
|
||||
@@ -462,17 +378,6 @@ class TextEncodingStage(ConditionEncodingStage):
|
||||
|
||||
return batch
|
||||
|
||||
def build_dedup_fingerprint(
|
||||
self, batch: Req, server_args: ServerArgs
|
||||
) -> TextEncodingFingerprint:
|
||||
return TextEncodingFingerprint(
|
||||
prompt=self.freeze_for_dedup(batch.prompt),
|
||||
negative_prompt=self.freeze_for_dedup(batch.negative_prompt),
|
||||
do_classifier_free_guidance=bool(batch.do_classifier_free_guidance),
|
||||
prompt_template=self.freeze_for_dedup(batch.prompt_template),
|
||||
max_sequence_length=batch.max_sequence_length,
|
||||
)
|
||||
|
||||
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
"""Verify text encoding stage inputs."""
|
||||
result = VerificationResult()
|
||||
@@ -696,131 +601,185 @@ class TextEncodingStage(ConditionEncodingStage):
|
||||
|
||||
text_inputs: dict = server_args.pipeline_config.tokenize_prompt(
|
||||
processed_text_list, tokenizer, tok_kwargs
|
||||
).to(target_device)
|
||||
|
||||
input_ids = text_inputs["input_ids"]
|
||||
attention_mask = (
|
||||
server_args.pipeline_config.get_text_encoder_attention_mask(
|
||||
text_inputs, i
|
||||
)
|
||||
)
|
||||
encoder_forward_kwargs = {
|
||||
"input_ids": input_ids,
|
||||
"output_hidden_states": True,
|
||||
}
|
||||
if attention_mask is not None:
|
||||
encoder_forward_kwargs["attention_mask"] = attention_mask
|
||||
if "use_cache" in inspect.signature(text_encoder.forward).parameters:
|
||||
encoder_forward_kwargs["use_cache"] = False
|
||||
self._begin_text_encoder_use(i)
|
||||
# match host tokens before preparing weights or uploading encoder inputs
|
||||
host_inputs = text_inputs
|
||||
cache_inputs = dict(host_inputs)
|
||||
dp_group = self._text_encode_dp_group(
|
||||
server_args, encoder_config, input_ids.shape[0], text_encoder
|
||||
server_args,
|
||||
encoder_config,
|
||||
host_inputs["input_ids"].shape[0],
|
||||
text_encoder,
|
||||
)
|
||||
if dp_group is not None:
|
||||
outputs = _data_parallel_text_encode(
|
||||
lambda kw: self._forward_text_encoder(text_encoder, kw),
|
||||
encoder_forward_kwargs,
|
||||
dp_group,
|
||||
)
|
||||
else:
|
||||
outputs = self._forward_text_encoder(
|
||||
text_encoder, encoder_forward_kwargs
|
||||
)
|
||||
postprocess_sig = inspect.signature(postprocess_func)
|
||||
|
||||
postprocess_kwargs = {}
|
||||
if "pipeline_config" in postprocess_sig.parameters:
|
||||
# required by models like LTX
|
||||
postprocess_kwargs["pipeline_config"] = server_args.pipeline_config
|
||||
if "return_attention_mask" in postprocess_sig.parameters:
|
||||
postprocess_kwargs["return_attention_mask"] = return_attention_mask
|
||||
postprocess_result = postprocess_func(
|
||||
outputs, text_inputs, **postprocess_kwargs
|
||||
)
|
||||
prompt_embeds_mask = None
|
||||
prompt_seq_lens = None
|
||||
if isinstance(postprocess_result, TextConditioningOutput):
|
||||
prompt_embeds = postprocess_result.prompt_embeds
|
||||
prompt_embeds_mask = postprocess_result.prompt_embeds_mask
|
||||
prompt_seq_lens = postprocess_result.prompt_seq_lens
|
||||
elif isinstance(postprocess_result, tuple):
|
||||
if len(postprocess_result) != 2:
|
||||
raise ValueError(
|
||||
"Text postprocess tuple output must be (prompt_embeds, prompt_embeds_mask)"
|
||||
def encode_conditioning():
|
||||
text_inputs = host_inputs.to(target_device, non_blocking=True)
|
||||
|
||||
input_ids = text_inputs["input_ids"]
|
||||
attention_mask = (
|
||||
server_args.pipeline_config.get_text_encoder_attention_mask(
|
||||
text_inputs, i
|
||||
)
|
||||
prompt_embeds, prompt_embeds_mask = postprocess_result
|
||||
else:
|
||||
prompt_embeds = postprocess_result
|
||||
|
||||
if dtype is not None:
|
||||
prompt_embeds = prompt_embeds.to(device=target_device, dtype=dtype)
|
||||
else:
|
||||
prompt_embeds = prompt_embeds.to(device=target_device)
|
||||
|
||||
if prompt_embeds_mask is not None:
|
||||
prompt_embeds_mask = prompt_embeds_mask.to(
|
||||
device=target_device, dtype=torch.bool
|
||||
)
|
||||
encoder_forward_kwargs = {
|
||||
"input_ids": input_ids,
|
||||
"output_hidden_states": True,
|
||||
}
|
||||
if attention_mask is not None:
|
||||
encoder_forward_kwargs["attention_mask"] = attention_mask
|
||||
if "use_cache" in inspect.signature(text_encoder.forward).parameters:
|
||||
encoder_forward_kwargs["use_cache"] = False
|
||||
self._begin_text_encoder_use(i)
|
||||
postprocess_sig = inspect.signature(postprocess_func)
|
||||
|
||||
embeds_list.append(prompt_embeds)
|
||||
postprocess_kwargs = {}
|
||||
if "pipeline_config" in postprocess_sig.parameters:
|
||||
# required by models like LTX
|
||||
postprocess_kwargs["pipeline_config"] = server_args.pipeline_config
|
||||
if "return_attention_mask" in postprocess_sig.parameters:
|
||||
postprocess_kwargs["return_attention_mask"] = return_attention_mask
|
||||
|
||||
pooled_output = server_args.pipeline_config.get_text_encoder_pooler_output(
|
||||
outputs, i
|
||||
)
|
||||
if pooled_output is not None:
|
||||
pooled_embeds_list.append(pooled_output.to(device=target_device))
|
||||
|
||||
if return_attention_mask:
|
||||
if prompt_embeds_mask is not None:
|
||||
mask_to_store = prompt_embeds_mask.to(
|
||||
device=target_device,
|
||||
dtype=(
|
||||
attention_mask.dtype
|
||||
if attention_mask is not None
|
||||
else torch.long
|
||||
),
|
||||
if dp_group is not None:
|
||||
outputs = _data_parallel_text_encode(
|
||||
lambda kw: self._forward_text_encoder(text_encoder, kw),
|
||||
encoder_forward_kwargs,
|
||||
dp_group,
|
||||
)
|
||||
elif attention_mask is not None and list(attention_mask.shape) == list(
|
||||
prompt_embeds.shape[:2]
|
||||
):
|
||||
mask_to_store = attention_mask.to(device=target_device)
|
||||
else:
|
||||
mask_to_store = torch.ones(
|
||||
prompt_embeds.shape[:2],
|
||||
device=target_device,
|
||||
dtype=(
|
||||
attention_mask.dtype
|
||||
if attention_mask is not None
|
||||
else torch.long
|
||||
),
|
||||
outputs = self._forward_text_encoder(
|
||||
text_encoder, encoder_forward_kwargs
|
||||
)
|
||||
attn_masks_list.append(mask_to_store)
|
||||
|
||||
embeds_mask = prompt_embeds_mask
|
||||
if embeds_mask is None:
|
||||
embeds_mask = (
|
||||
server_args.pipeline_config.build_text_conditioning_mask(
|
||||
text_inputs,
|
||||
attention_mask,
|
||||
prompt_embeds,
|
||||
i,
|
||||
postprocess_result = postprocess_func(
|
||||
outputs, text_inputs, **postprocess_kwargs
|
||||
)
|
||||
pooled_output = (
|
||||
server_args.pipeline_config.get_text_encoder_pooler_output(
|
||||
outputs, i
|
||||
)
|
||||
)
|
||||
prompt_embeds_mask = None
|
||||
prompt_seq_lens = None
|
||||
if isinstance(postprocess_result, TextConditioningOutput):
|
||||
prompt_embeds = postprocess_result.prompt_embeds
|
||||
prompt_embeds_mask = postprocess_result.prompt_embeds_mask
|
||||
prompt_seq_lens = postprocess_result.prompt_seq_lens
|
||||
elif isinstance(postprocess_result, tuple):
|
||||
if len(postprocess_result) != 2:
|
||||
raise ValueError(
|
||||
"Text postprocess tuple output must be (prompt_embeds, prompt_embeds_mask)"
|
||||
)
|
||||
prompt_embeds, prompt_embeds_mask = postprocess_result
|
||||
else:
|
||||
prompt_embeds = postprocess_result
|
||||
|
||||
if dtype is not None:
|
||||
prompt_embeds = prompt_embeds.to(device=target_device, dtype=dtype)
|
||||
else:
|
||||
prompt_embeds = prompt_embeds.to(device=target_device)
|
||||
|
||||
if prompt_embeds_mask is not None:
|
||||
prompt_embeds_mask = prompt_embeds_mask.to(
|
||||
device=target_device, dtype=torch.bool
|
||||
)
|
||||
embeds_masks_list.append(embeds_mask)
|
||||
if prompt_seq_lens is not None:
|
||||
seq_lens_list.append([int(x) for x in prompt_seq_lens])
|
||||
elif embeds_mask is not None:
|
||||
seq_lens_list.append(
|
||||
server_args.pipeline_config.seq_lens_from_text_conditioning_mask(
|
||||
|
||||
if pooled_output is not None:
|
||||
pooled_output = pooled_output.to(device=target_device)
|
||||
|
||||
mask_to_store = embeds_mask = seq_lens = None
|
||||
if return_attention_mask:
|
||||
if prompt_embeds_mask is not None:
|
||||
mask_to_store = prompt_embeds_mask.to(
|
||||
device=target_device,
|
||||
dtype=(
|
||||
attention_mask.dtype
|
||||
if attention_mask is not None
|
||||
else torch.long
|
||||
),
|
||||
)
|
||||
elif attention_mask is not None and list(
|
||||
attention_mask.shape
|
||||
) == list(prompt_embeds.shape[:2]):
|
||||
mask_to_store = attention_mask.to(device=target_device)
|
||||
else:
|
||||
mask_to_store = torch.ones(
|
||||
prompt_embeds.shape[:2],
|
||||
device=target_device,
|
||||
dtype=(
|
||||
attention_mask.dtype
|
||||
if attention_mask is not None
|
||||
else torch.long
|
||||
),
|
||||
)
|
||||
|
||||
embeds_mask = prompt_embeds_mask
|
||||
if embeds_mask is None:
|
||||
embeds_mask = (
|
||||
server_args.pipeline_config.build_text_conditioning_mask(
|
||||
text_inputs,
|
||||
attention_mask,
|
||||
prompt_embeds,
|
||||
i,
|
||||
)
|
||||
)
|
||||
if prompt_seq_lens is not None:
|
||||
seq_lens = [int(x) for x in prompt_seq_lens]
|
||||
elif embeds_mask is not None:
|
||||
seq_lens = server_args.pipeline_config.seq_lens_from_text_conditioning_mask(
|
||||
embeds_mask
|
||||
)
|
||||
)
|
||||
elif prompt_embeds.ndim == 2:
|
||||
seq_lens_list.append([int(prompt_embeds.shape[0])])
|
||||
else:
|
||||
seq_lens_list.append(
|
||||
[int(prompt_embeds.shape[1])] * int(prompt_embeds.shape[0])
|
||||
)
|
||||
elif prompt_embeds.ndim == 2:
|
||||
seq_lens = [int(prompt_embeds.shape[0])]
|
||||
else:
|
||||
seq_lens = [int(prompt_embeds.shape[1])] * int(
|
||||
prompt_embeds.shape[0]
|
||||
)
|
||||
return (
|
||||
prompt_embeds,
|
||||
mask_to_store,
|
||||
pooled_output,
|
||||
embeds_mask,
|
||||
seq_lens,
|
||||
)
|
||||
|
||||
native_encoder = isinstance(text_encoder, TextEncoder)
|
||||
cache_group = text_encoder._encoder_tp_group if native_encoder else None
|
||||
if model_parallel_is_initialized():
|
||||
if dp_group is not None:
|
||||
# agree across encoder copies before skipping their gather
|
||||
cache_group = get_replica_group()
|
||||
elif cache_group is None:
|
||||
cache_group = get_tp_group()
|
||||
# library and batch-DP positives retain consumed outputs only within
|
||||
# the group; preferred negatives also use the bounded device cache
|
||||
conditioning = cached_encoder_call(
|
||||
text_encoder,
|
||||
(cache_inputs,),
|
||||
{
|
||||
"encoder_index": i,
|
||||
"return_attention_mask": return_attention_mask,
|
||||
"device": str(target_device),
|
||||
"dtype": dtype,
|
||||
},
|
||||
encode_conditioning,
|
||||
cache_group,
|
||||
namespace=self,
|
||||
nested=dp_group is not None,
|
||||
share_in_group=True,
|
||||
cross_request=native_encoder and dp_group is None,
|
||||
)
|
||||
# a hit can leave weights retained by warmup or the previous request
|
||||
self.finish_unused_declared_component(
|
||||
component_name="text_encoder" if i == 0 else f"text_encoder_{i + 1}",
|
||||
module=text_encoder,
|
||||
)
|
||||
prompt_embeds, mask, pooled_output, embeds_mask, seq_lens = conditioning
|
||||
embeds_list.append(prompt_embeds)
|
||||
if pooled_output is not None:
|
||||
pooled_embeds_list.append(pooled_output)
|
||||
if return_attention_mask:
|
||||
attn_masks_list.append(mask)
|
||||
embeds_masks_list.append(embeds_mask)
|
||||
seq_lens_list.append(seq_lens)
|
||||
|
||||
# Shape results according to return_type
|
||||
if return_type == "list":
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import Any
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import conditioning_weights_epoch
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import (
|
||||
OutputBatch,
|
||||
@@ -84,9 +85,11 @@ def _effective_prefix_cache_enabled(
|
||||
server_args: ServerArgs,
|
||||
) -> bool:
|
||||
options = vla_options(batch)
|
||||
return bool(options.get("enable_prefix_cache", True)) and bool(
|
||||
server_args.pipeline_config.enable_global_prefix_cache
|
||||
)
|
||||
return (
|
||||
not server_args.disable_conditioning_cache
|
||||
and server_args.conditioning_cache_max_size_mb > 0
|
||||
and bool(options.get("enable_prefix_cache", True))
|
||||
) and bool(server_args.pipeline_config.enable_global_prefix_cache)
|
||||
|
||||
|
||||
def _cuda_graph_enabled(batch: Req) -> bool:
|
||||
@@ -264,6 +267,7 @@ class VLAPrefixEncodingStage(PipelineStage):
|
||||
observation,
|
||||
bucket_prompt=_cuda_graph_enabled(batch),
|
||||
)
|
||||
cache_key = f"{conditioning_weights_epoch()}:{cache_key}"
|
||||
cached_context = self.prefix_cache.get(cache_key)
|
||||
else:
|
||||
cache_key = None
|
||||
|
||||
@@ -46,6 +46,9 @@ from typing import Any
|
||||
import torch
|
||||
from torch.distributed.tensor import DTensor, distribute_tensor
|
||||
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import (
|
||||
invalidate_conditioning_caches,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.cache.teacache import TeaCacheMixin
|
||||
from sglang.multimodal_gen.runtime.loader.utils import (
|
||||
_list_safetensors_files,
|
||||
@@ -369,6 +372,7 @@ class WeightsUpdater:
|
||||
target_modules: list[str] | None = None,
|
||||
) -> tuple[bool, str]:
|
||||
"""Update model weights from disk without restarting the server."""
|
||||
invalidate_conditioning_caches()
|
||||
logger.info(f"Updating weights from disk: {model_path}")
|
||||
|
||||
try:
|
||||
@@ -531,6 +535,7 @@ class WeightsUpdater:
|
||||
lora_alpha: int | None = None,
|
||||
lora_rank: int | None = None,
|
||||
) -> tuple[bool, str]:
|
||||
invalidate_conditioning_caches()
|
||||
if weight_update_mode == LORA_MERGE_WEIGHT_UPDATE_MODE:
|
||||
return self._update_lora_from_tensor(
|
||||
named_tensors=named_tensors,
|
||||
|
||||
@@ -609,6 +609,8 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
log_requests_target: Optional[List[str]] = None
|
||||
uvicorn_access_log_exclude_prefixes: list[str] = field(default_factory=list)
|
||||
enable_cache_report: bool = False
|
||||
disable_conditioning_cache: bool = False
|
||||
conditioning_cache_max_size_mb: float = 512.0
|
||||
|
||||
# Tracing
|
||||
enable_trace: bool = False
|
||||
@@ -1932,6 +1934,10 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
raise ValueError(f"Could not parse attention backend config: {config_str}")
|
||||
|
||||
def __post_init__(self):
|
||||
if not 0 <= self.conditioning_cache_max_size_mb < float("inf"):
|
||||
raise ValueError(
|
||||
"conditioning_cache_max_size_mb must be finite and nonnegative"
|
||||
)
|
||||
if not self._explicit_arg_names:
|
||||
self._explicit_arg_names = _infer_direct_constructor_explicit_arg_names(
|
||||
self
|
||||
@@ -3026,6 +3032,18 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
default=ServerArgs.enable_cache_report,
|
||||
help="Return number of cached tokens in usage.prompt_tokens_details for each OpenAI-compatible request.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--disable-conditioning-cache",
|
||||
action="store_true",
|
||||
default=ServerArgs.disable_conditioning_cache,
|
||||
help="Disable cross-request text/image and VAE posterior caching; reuse within a grouped stage remains enabled.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--conditioning-cache-max-size-mb",
|
||||
type=float,
|
||||
default=ServerArgs.conditioning_cache_max_size_mb,
|
||||
help="Per-worker conditioning cache capacity across CPU and device entries in MiB (default: 512; 0 disables cross-request caching).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backend",
|
||||
type=str,
|
||||
|
||||
@@ -0,0 +1,611 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import sys
|
||||
from contextlib import nullcontext
|
||||
from datetime import timedelta
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution
|
||||
from diffusers.models.modeling_outputs import AutoencoderKLOutput
|
||||
|
||||
from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import (
|
||||
ConditioningCache,
|
||||
cached_conditioning,
|
||||
cached_encoder_call,
|
||||
cached_vae_encode,
|
||||
invalidate_conditioning_caches,
|
||||
prefer_conditioning_cache,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.base import (
|
||||
EncoderTensorParallelMixin,
|
||||
)
|
||||
|
||||
|
||||
class Encoder(EncoderTensorParallelMixin, torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.weight = torch.nn.Parameter(torch.ones(()))
|
||||
self.calls = 0
|
||||
|
||||
def forward(self, x, attention_mask=None, **kwargs):
|
||||
self.calls += 1
|
||||
result = x * self.weight
|
||||
if attention_mask is not None:
|
||||
result = result * attention_mask
|
||||
return BaseEncoderOutput(last_hidden_state=result, hidden_states=(result,))
|
||||
|
||||
|
||||
class VAE(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.calls = 0
|
||||
self.use_tiling = False
|
||||
|
||||
@cached_vae_encode
|
||||
def encode(self, x):
|
||||
self.calls += 1
|
||||
return AutoencoderKLOutput(latent_dist=DiagonalGaussianDistribution(x.clone()))
|
||||
|
||||
|
||||
class VisionLanguageEncoder(Encoder):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.vision_calls = 0
|
||||
|
||||
@cached_conditioning
|
||||
def image_features(self, pixels):
|
||||
self.vision_calls += 1
|
||||
return pixels * 2
|
||||
|
||||
def forward(self, text, pixels):
|
||||
self.calls += 1
|
||||
return self.image_features(pixels) + text
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_changed_instruction_reuses_vision_but_not_joint_embedding():
|
||||
cache = ConditioningCache(4096)
|
||||
model = VisionLanguageEncoder().eval()
|
||||
pixels = torch.ones(4)
|
||||
with cache.scope():
|
||||
first = model(torch.ones(4), pixels)
|
||||
changed = model(torch.full((4,), 2.0), pixels)
|
||||
assert not torch.equal(first, changed)
|
||||
assert model.calls == 2 and model.vision_calls == 1
|
||||
torch.testing.assert_close(
|
||||
model(torch.full((4,), 2.0), pixels), changed, rtol=0, atol=0
|
||||
)
|
||||
assert model.calls == 2
|
||||
model(torch.ones(4), pixels + 1)
|
||||
assert model.vision_calls == 2
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_content_masks_model_identity_and_copy_isolation():
|
||||
cache = ConditioningCache(1024 * 1024)
|
||||
model = Encoder().eval()
|
||||
x = torch.arange(1024, dtype=torch.float32).reshape(1, 4, 16, 16)
|
||||
with cache.scope():
|
||||
original = model(x)
|
||||
expected = original.last_hidden_state.clone()
|
||||
original.last_hidden_state.zero_()
|
||||
hit = model(x.clone())
|
||||
assert hit.hidden_states[0] is hit.last_hidden_state
|
||||
torch.testing.assert_close(hit.last_hidden_state, expected, rtol=0, atol=0)
|
||||
hit.hidden_states[0].zero_()
|
||||
torch.testing.assert_close(model(x).last_hidden_state, expected, rtol=0, atol=0)
|
||||
changed_alpha = x.clone()
|
||||
changed_alpha[:, 3, 0, 0] += 1
|
||||
model(changed_alpha)
|
||||
model(x, attention_mask=torch.zeros_like(x))
|
||||
other_model = Encoder().eval()
|
||||
other_model(x)
|
||||
assert model.calls == 3
|
||||
assert other_model.calls == 1
|
||||
assert cache.hits == 2
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_lru_budget_and_oversized_outputs():
|
||||
cache = ConditioningCache(32)
|
||||
model = Encoder().eval()
|
||||
with cache.scope():
|
||||
for value in (1, 2, 1, 3, 2):
|
||||
model(torch.full((4,), float(value)))
|
||||
before = cache.bytes
|
||||
model(torch.ones(100))
|
||||
assert model.calls == 5 # one LRU hit, then an oversized miss
|
||||
assert cache.hits == 1
|
||||
assert cache.evictions == 2
|
||||
assert cache.bytes == before == 32
|
||||
assert cache.bypasses == 1
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_device_hot_promotion_shares_budget_and_eviction_with_host_entries():
|
||||
cache = ConditioningCache(32)
|
||||
model = Encoder().eval()
|
||||
namespace = Encoder().eval()
|
||||
|
||||
def encode(value):
|
||||
return cached_encoder_call(
|
||||
model,
|
||||
(value,),
|
||||
{},
|
||||
lambda: model(torch.full((4,), value)),
|
||||
namespace=namespace,
|
||||
share_in_group=True,
|
||||
)
|
||||
|
||||
with cache.scope():
|
||||
encode(1.0)
|
||||
encode(2.0)
|
||||
assert cache.bytes == 32 and cache.stats()["device_bytes"] == 0
|
||||
with prefer_conditioning_cache():
|
||||
promoted = encode(1.0)
|
||||
promoted.last_hidden_state.zero_()
|
||||
assert cache.bytes == 32 and cache.stats()["device_bytes"] == 16
|
||||
encode(3.0)
|
||||
assert cache.bytes == cache.stats()["device_bytes"] == 32
|
||||
encode(4.0) # a cold positive must not evict either preferred negative
|
||||
assert cache.bytes == cache.stats()["device_bytes"] == 32
|
||||
with prefer_conditioning_cache():
|
||||
torch.testing.assert_close(encode(1.0).last_hidden_state, torch.ones(4))
|
||||
encode(5.0) # replaces the least-recent negative within the same budget
|
||||
assert cache.evictions == 2
|
||||
assert cache.bytes == cache.stats()["device_bytes"] == 32
|
||||
cache.clear()
|
||||
assert cache.bytes == cache.stats()["device_bytes"] == 0
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_warmup_executes_nested_encoders_and_seeds_serving_cache():
|
||||
cache = ConditioningCache(4096)
|
||||
model = VisionLanguageEncoder().eval()
|
||||
text, pixels = torch.ones(4), torch.ones(4)
|
||||
with cache.scope():
|
||||
expected = model(text, pixels)
|
||||
with cache.scope(refresh=True):
|
||||
model(text, pixels)
|
||||
model(text, pixels)
|
||||
assert model.calls == model.vision_calls == 3
|
||||
torch.testing.assert_close(model(text, pixels), expected, rtol=0, atol=0)
|
||||
assert model.calls == model.vision_calls == 3
|
||||
model(text + 1, pixels)
|
||||
assert model.calls == 4 and model.vision_calls == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bypass", ["disabled", "inactive", "training", "grad", "ar"])
|
||||
def test_bypass(bypass):
|
||||
cache = ConditioningCache(0 if bypass == "disabled" else 1024)
|
||||
model = Encoder().eval()
|
||||
if bypass == "training":
|
||||
model.train()
|
||||
kwargs = {"use_cache": True} if bypass == "ar" else {}
|
||||
with (
|
||||
torch.set_grad_enabled(bypass == "grad"),
|
||||
cache.scope(enabled=bypass != "inactive"),
|
||||
):
|
||||
model(torch.ones(4), **kwargs)
|
||||
model(torch.ones(4), **kwargs)
|
||||
assert model.calls == 2
|
||||
assert cache.hits == 0
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_posterior_cache_preserves_rng_and_sampling():
|
||||
cache = ConditioningCache(1024 * 1024)
|
||||
vae = VAE().eval()
|
||||
x = torch.linspace(-1, 1, 64).reshape(1, 4, 4, 4)
|
||||
with cache.scope():
|
||||
posterior = vae.encode(x).latent_dist
|
||||
cold = posterior.sample(generator=torch.Generator().manual_seed(42))
|
||||
posterior.mean.add_(100) # downstream normalization must not poison storage
|
||||
generator = torch.Generator().manual_seed(42)
|
||||
state = generator.get_state().clone()
|
||||
hit = vae.encode(x).latent_dist
|
||||
assert torch.equal(generator.get_state(), state)
|
||||
warm = hit.sample(generator=generator)
|
||||
torch.testing.assert_close(cold, warm, rtol=0, atol=0)
|
||||
assert not torch.equal(
|
||||
warm, hit.sample(generator=torch.Generator().manual_seed(43))
|
||||
)
|
||||
vae.use_tiling = True
|
||||
vae.encode(x)
|
||||
assert vae.calls == 2
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_weight_invalidation_and_precision():
|
||||
cache = ConditioningCache(1024)
|
||||
model = Encoder().eval()
|
||||
x = torch.ones(4)
|
||||
with cache.scope():
|
||||
model(x)
|
||||
model(x)
|
||||
invalidate_conditioning_caches()
|
||||
model.weight.fill_(2)
|
||||
assert torch.equal(model(x).last_hidden_state, x * 2)
|
||||
model.to(torch.float64)
|
||||
model(x)
|
||||
assert model.calls == 3
|
||||
assert cache.hits == 1
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_scoped_invalidation_preserves_independent_encoders_and_shared_weights():
|
||||
cache = ConditioningCache(1024)
|
||||
independent = Encoder().eval()
|
||||
nested = Encoder().eval()
|
||||
alias = Encoder().eval()
|
||||
alias.weight = nested.weight
|
||||
transformer = torch.nn.ModuleList([nested])
|
||||
x = torch.ones(4)
|
||||
with cache.scope():
|
||||
for model in (independent, nested, alias):
|
||||
model(x)
|
||||
invalidate_conditioning_caches([transformer])
|
||||
nested.weight.fill_(2)
|
||||
assert torch.equal(independent(x).last_hidden_state, x)
|
||||
assert torch.equal(nested(x).last_hidden_state, x * 2)
|
||||
assert torch.equal(alias(x).last_hidden_state, x * 2)
|
||||
assert (independent.calls, nested.calls, alias.calls) == (1, 2, 2)
|
||||
assert cache.bytes <= cache.max_bytes
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_negative_conditioning_survives_positive_capacity_pressure():
|
||||
model = Encoder().eval()
|
||||
# only one entry fits, reproducing two LTX embeddings in the host budget
|
||||
cache = ConditioningCache(24)
|
||||
positive, negative = torch.ones(4), torch.zeros(4)
|
||||
with cache.scope(refresh=True):
|
||||
model(positive)
|
||||
with prefer_conditioning_cache():
|
||||
model(negative)
|
||||
with cache.scope():
|
||||
model(positive + 1)
|
||||
with prefer_conditioning_cache():
|
||||
actual = model(negative)
|
||||
assert model.calls == 3
|
||||
assert cache.hits == 1
|
||||
assert cache.bytes == 16
|
||||
assert cache.evictions == 1
|
||||
torch.testing.assert_close(actual.last_hidden_state, negative)
|
||||
# preferred entries still replace each other and remain bounded
|
||||
with prefer_conditioning_cache():
|
||||
model(negative - 1)
|
||||
assert cache.bytes == 16
|
||||
assert cache.evictions == 2
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA transfers")
|
||||
@torch.no_grad()
|
||||
def test_cuda_snapshot_waits_for_producing_stream_before_restore():
|
||||
cache = ConditioningCache(32 * 1024 * 1024)
|
||||
model = Encoder().eval()
|
||||
stream = torch.cuda.Stream()
|
||||
with torch.cuda.stream(stream):
|
||||
value = torch.arange(1024 * 1024, device="cuda").reshape(1024, 1024)
|
||||
output = BaseEncoderOutput(
|
||||
last_hidden_state=value,
|
||||
hidden_states=(value, value.T, value[::2, ::2]),
|
||||
)
|
||||
cache.run(model, "forward", (), {}, lambda: output)
|
||||
value.zero_()
|
||||
restored = cache.run(model, "forward", (), {}, lambda: pytest.fail("cache miss"))
|
||||
expected = torch.arange(1024 * 1024).reshape(1024, 1024)
|
||||
assert restored.last_hidden_state is restored.hidden_states[0]
|
||||
for actual, reference in zip(
|
||||
restored.hidden_states, (expected, expected.T, expected[::2, ::2]), strict=True
|
||||
):
|
||||
torch.testing.assert_close(actual.cpu(), reference, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA transfers")
|
||||
@torch.no_grad()
|
||||
def test_cuda_cache_hit_does_not_wait_for_unrelated_gpu_work():
|
||||
cache = ConditioningCache(4096)
|
||||
model = Encoder().eval()
|
||||
value = torch.arange(64, device="cuda")
|
||||
cache.run(model, "forward", (), {}, lambda: value)
|
||||
torch.cuda.synchronize()
|
||||
# a negative-prompt hit must not synchronize the preceding positive encode
|
||||
torch.cuda._sleep(250_000_000)
|
||||
pending = torch.cuda.Event()
|
||||
pending.record()
|
||||
restored = cache.run(model, "forward", (), {}, lambda: pytest.fail("cache miss"))
|
||||
assert not pending.query()
|
||||
torch.testing.assert_close(restored, value, rtol=0, atol=0)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_compiled_consumed_conditioning_bypasses_context_lookup():
|
||||
model = torch.nn.Linear(2, 2).eval()
|
||||
cache = ConditioningCache(1024)
|
||||
|
||||
@torch.compile(backend="eager", fullgraph=True)
|
||||
def encode(x):
|
||||
return cached_encoder_call(
|
||||
model,
|
||||
(x,),
|
||||
{},
|
||||
lambda: model(x),
|
||||
namespace=model,
|
||||
share_in_group=True,
|
||||
)
|
||||
|
||||
with cache.scope(), prefer_conditioning_cache():
|
||||
x = torch.ones(1, 2)
|
||||
torch.testing.assert_close(encode(x), model(x), rtol=0, atol=0)
|
||||
assert cache.stats()["entries"] == 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA graph capture")
|
||||
@pytest.mark.parametrize("consumed", [False, True])
|
||||
@torch.no_grad()
|
||||
def test_cuda_graph_capture_bypasses_conditioning_cache(consumed):
|
||||
cache = ConditioningCache(1024)
|
||||
model = Encoder().cuda().eval()
|
||||
x = torch.ones(4, device="cuda")
|
||||
|
||||
def encode():
|
||||
return (
|
||||
cached_encoder_call(
|
||||
model,
|
||||
(x,),
|
||||
{},
|
||||
lambda: model(x),
|
||||
namespace=model,
|
||||
share_in_group=True,
|
||||
)
|
||||
if consumed
|
||||
else model(x)
|
||||
)
|
||||
|
||||
with cache.scope(), prefer_conditioning_cache():
|
||||
encode()
|
||||
torch.cuda.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = encode()
|
||||
x.fill_(3)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(output.last_hidden_state, x, rtol=0, atol=0)
|
||||
assert model.calls == 2
|
||||
assert cache.hits == 0
|
||||
|
||||
|
||||
def _rank_eviction(rank, init_method, disabled_rank, grouped=False):
|
||||
dist.init_process_group(
|
||||
"gloo",
|
||||
rank=rank,
|
||||
world_size=2,
|
||||
init_method=init_method,
|
||||
timeout=timedelta(seconds=30),
|
||||
)
|
||||
try:
|
||||
cache = ConditioningCache(0 if disabled_rank and rank == 0 else 1024)
|
||||
model = Encoder().eval()
|
||||
group = SimpleNamespace(world_size=2, cpu_group=dist.group.WORLD)
|
||||
x = torch.ones(4)
|
||||
|
||||
def compute():
|
||||
# The same ordering requirement as a TP encoder forward.
|
||||
flag = torch.ones(1)
|
||||
dist.all_reduce(flag)
|
||||
return model.forward(x)
|
||||
|
||||
with torch.no_grad(), cache.group_scope() if grouped else nullcontext():
|
||||
for attempt in range(3):
|
||||
if attempt == 1 and rank == 0:
|
||||
cache.clear()
|
||||
if attempt == 2 and rank == 0 and grouped and not disabled_rank:
|
||||
# One rank restores a host entry while its peer reuses a device entry.
|
||||
cache._group_entries.get().clear()
|
||||
cache.run(model, "forward", (x,), {}, compute, group)
|
||||
assert model.calls == (3 if disabled_rank and not grouped else 2)
|
||||
assert cache.hits == (0 if disabled_rank and not grouped else 1)
|
||||
finally:
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("disabled_rank", [False, True])
|
||||
@pytest.mark.parametrize("grouped", [False, True])
|
||||
def test_rank_local_eviction_forces_collective_miss(tmp_path, disabled_rank, grouped):
|
||||
mp.spawn(
|
||||
_rank_eviction,
|
||||
args=(f"file://{tmp_path / 'rendezvous'}", disabled_rank, grouped),
|
||||
nprocs=2,
|
||||
join=True,
|
||||
)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_stage_conditioning_owns_output_but_preserves_vision_cache():
|
||||
model = VisionLanguageEncoder().eval()
|
||||
namespace = Encoder()
|
||||
cache = ConditioningCache(4096)
|
||||
pixels = torch.ones(4)
|
||||
hooks = []
|
||||
model.register_forward_hook(lambda *args: hooks.append(1))
|
||||
|
||||
def stage(text):
|
||||
return cached_encoder_call(
|
||||
model,
|
||||
(text, pixels),
|
||||
{},
|
||||
lambda: model(text, pixels)[:1],
|
||||
namespace=namespace,
|
||||
)
|
||||
|
||||
with cache.scope():
|
||||
first = stage(torch.ones(4))
|
||||
# one vision entry and one consumed output, without a full encoder copy
|
||||
assert cache.bytes == 20
|
||||
assert cache.stats()["entries"] == 2
|
||||
first.zero_()
|
||||
assert stage(torch.ones(4)).item() == 3
|
||||
assert model.calls == model.vision_calls == len(hooks) == 1
|
||||
assert stage(torch.full((4,), 2.0)).item() == 4
|
||||
assert model.calls == len(hooks) == 2
|
||||
assert model.vision_calls == 1
|
||||
with cache.scope(refresh=True):
|
||||
stage(torch.ones(4))
|
||||
assert model.calls == len(hooks) == 3
|
||||
assert model.vision_calls == 2
|
||||
|
||||
def fail():
|
||||
raise RuntimeError("encoding failed")
|
||||
|
||||
with pytest.raises(RuntimeError, match="encoding failed"):
|
||||
cached_encoder_call(model, ("failure",), {}, fail, namespace=namespace)
|
||||
# the temporary stage owner must not bypass later encoder caching
|
||||
model(torch.ones(4), pixels)
|
||||
model(torch.ones(4), pixels)
|
||||
assert model.calls == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("capacity", [0, 1, 4096])
|
||||
@torch.no_grad()
|
||||
def test_group_cache_preserves_mutable_outputs_and_posterior_rng(capacity):
|
||||
cache = ConditioningCache(capacity)
|
||||
encoder = Encoder().eval()
|
||||
vae = VAE().eval()
|
||||
x = torch.ones(1, 4, 2, 2)
|
||||
with cache.scope(), cache.group_scope():
|
||||
first = encoder(x)
|
||||
first.last_hidden_state.zero_()
|
||||
second = encoder(x)
|
||||
assert encoder.calls == 1
|
||||
assert second.hidden_states[0] is second.last_hidden_state
|
||||
torch.testing.assert_close(second.last_hidden_state, x, rtol=0, atol=0)
|
||||
second.last_hidden_state.zero_()
|
||||
torch.testing.assert_close(encoder(x).last_hidden_state, x, rtol=0, atol=0)
|
||||
for seed in (13, 27):
|
||||
generator = torch.Generator().manual_seed(seed)
|
||||
control = torch.Generator().manual_seed(seed)
|
||||
posterior = vae.encode(x).latent_dist
|
||||
expected = DiagonalGaussianDistribution(x).sample(control)
|
||||
torch.testing.assert_close(
|
||||
posterior.sample(generator), expected, rtol=0, atol=0
|
||||
)
|
||||
assert torch.equal(generator.get_state(), control.get_state())
|
||||
posterior.mean.zero_()
|
||||
assert vae.calls == 1
|
||||
assert cache._group_entries.get() is None
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_group_hit_preserves_negative_host_entry_under_capacity_pressure():
|
||||
model = Encoder().eval()
|
||||
cache = ConditioningCache(24)
|
||||
negative = torch.zeros(4)
|
||||
with cache.scope(), cache.group_scope():
|
||||
model(negative)
|
||||
with prefer_conditioning_cache():
|
||||
model(negative)
|
||||
model(torch.ones(4))
|
||||
with cache.scope():
|
||||
actual = model(negative)
|
||||
assert model.calls == 2
|
||||
assert cache.hits == 2
|
||||
assert cache.group_hits == 1
|
||||
assert cache.evictions == 0
|
||||
torch.testing.assert_close(actual.last_hidden_state, negative, rtol=0, atol=0)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_group_only_scope_ignores_host_entries_and_nested_encoder_caches():
|
||||
model = VisionLanguageEncoder().eval()
|
||||
cache = ConditioningCache(4096)
|
||||
namespace = Encoder()
|
||||
x = torch.ones(4)
|
||||
|
||||
def stage():
|
||||
return cached_encoder_call(
|
||||
model,
|
||||
(x, x),
|
||||
{},
|
||||
lambda: model(x, x),
|
||||
namespace=namespace,
|
||||
share_in_group=True,
|
||||
)
|
||||
|
||||
with cache.scope():
|
||||
expected = stage()
|
||||
entries = cache.stats()["entries"]
|
||||
with cache.scope(cross_request=False):
|
||||
for _ in range(2):
|
||||
with cache.group_scope():
|
||||
torch.testing.assert_close(stage(), expected, rtol=0, atol=0)
|
||||
stage()
|
||||
assert cache.stats()["entries"] == entries
|
||||
assert model.calls == model.vision_calls == 3
|
||||
stage()
|
||||
stage()
|
||||
assert model.calls == model.vision_calls == 5
|
||||
stage()
|
||||
assert model.calls == 5
|
||||
assert cache.group_hits == 2
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_warmup_refreshes_existing_group_then_reuses_fresh_results():
|
||||
cache = ConditioningCache(4096)
|
||||
encoder = Encoder().eval()
|
||||
x = torch.ones(4)
|
||||
with cache.scope(), cache.group_scope():
|
||||
encoder(x)
|
||||
with cache.scope(refresh=True):
|
||||
encoder(x)
|
||||
encoder(x)
|
||||
assert encoder.calls == 2
|
||||
assert cache.group_hits == 1
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_group_cache_invalidates_and_releases_after_failure():
|
||||
cache = ConditioningCache(0)
|
||||
model = Encoder().eval()
|
||||
x = torch.ones(4)
|
||||
with pytest.raises(RuntimeError, match="stop"), cache.scope(), cache.group_scope():
|
||||
model(x)
|
||||
model(x)
|
||||
invalidate_conditioning_caches([model])
|
||||
model.weight.fill_(2)
|
||||
torch.testing.assert_close(model(x).last_hidden_state, x * 2)
|
||||
assert model.calls == 2
|
||||
raise RuntimeError("stop")
|
||||
assert cache._group_entries.get() is None
|
||||
with cache.scope(), cache.group_scope():
|
||||
model(x)
|
||||
assert model.calls == 3
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@torch.no_grad()
|
||||
def test_group_cache_waits_for_producing_cuda_stream():
|
||||
cache = ConditioningCache(0)
|
||||
model = Encoder().cuda().eval()
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with cache.scope(), cache.group_scope():
|
||||
with torch.cuda.stream(stream):
|
||||
torch.cuda._sleep(10_000_000)
|
||||
x = torch.ones(1024, device="cuda")
|
||||
cache.run(model, "stream", (), {}, lambda: model.forward(x))
|
||||
output = cache.run(model, "stream", (), {}, lambda: pytest.fail("cache miss"))
|
||||
torch.testing.assert_close(
|
||||
output.last_hidden_state, torch.ones_like(output.last_hidden_state)
|
||||
)
|
||||
assert model.calls == 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, *sys.argv[1:]]))
|
||||
@@ -0,0 +1,142 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import (
|
||||
ConditioningCache,
|
||||
invalidate_conditioning_caches,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.cosmos3video import Cosmos3LanguageModel
|
||||
from sglang.multimodal_gen.runtime.models.sensenova_u1.neo_unify.modeling_neo_chat import (
|
||||
NEOChatModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.realtime.text_encoding import (
|
||||
RealtimeTextEncodingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
|
||||
TextEncodingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.vla import (
|
||||
VLAPrefixEncodingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.realtime.session import RealtimeSession
|
||||
|
||||
|
||||
class UndLayer(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.calls = 0
|
||||
|
||||
def forward(self, hidden, cos_sin, positions):
|
||||
self.calls += 1
|
||||
return hidden, hidden + 2, hidden + 3
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_cosmos_understanding_cache_restores_private_kv():
|
||||
model = Cosmos3LanguageModel.__new__(Cosmos3LanguageModel)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.embed_tokens = torch.nn.Embedding(8, 4)
|
||||
model.rotary_emb = SimpleNamespace(
|
||||
build_rope_cache_inputs=lambda positions, cache_dtype: (None, positions)
|
||||
)
|
||||
layer = UndLayer()
|
||||
model.layers = torch.nn.ModuleList([layer])
|
||||
model.eval()
|
||||
ids = torch.tensor([[1, 2]])
|
||||
mask = torch.ones_like(ids)
|
||||
positions = torch.zeros(3, 1, 2, dtype=torch.long)
|
||||
with ConditioningCache(4096).scope():
|
||||
first = model(ids, mask, positions)
|
||||
expected = first[0][0].clone()
|
||||
first[0][0].zero_()
|
||||
hit = model(ids, mask, positions)
|
||||
torch.testing.assert_close(hit[0][0], expected, rtol=0, atol=0)
|
||||
assert layer.calls == 1
|
||||
model(ids, mask, positions + 1)
|
||||
model(ids, mask * 0, positions)
|
||||
assert layer.calls == 3
|
||||
|
||||
|
||||
class Vision(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.calls = 0
|
||||
|
||||
def forward(self, pixel_values, **kwargs):
|
||||
self.calls += 1
|
||||
return SimpleNamespace(last_hidden_state=pixel_values * 2)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_sensenova_caches_references_but_runs_noisy_vision_each_step():
|
||||
model = NEOChatModel.__new__(NEOChatModel)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.vision_model = Vision()
|
||||
model.fm_modules = torch.nn.ModuleDict({"vision_model_mot_gen": Vision()})
|
||||
model.eval()
|
||||
pixels = torch.ones(2, 4)
|
||||
with ConditioningCache(4096).scope():
|
||||
for _ in range(2):
|
||||
model.extract_feature(pixels)
|
||||
model.extract_feature(pixels, gen_model=True)
|
||||
assert model.vision_model.calls == 1
|
||||
assert model.fm_modules["vision_model_mot_gen"].calls == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("disabled, capacity", [(False, 512), (True, 512), (False, 0)])
|
||||
def test_realtime_text_respects_disable_and_weight_invalidation(
|
||||
monkeypatch, disabled, capacity
|
||||
):
|
||||
calls = []
|
||||
|
||||
def encode(self, batch, server_args):
|
||||
calls.append(batch.prompt)
|
||||
batch.prompt_embeds = [torch.ones(1)]
|
||||
return batch
|
||||
|
||||
monkeypatch.setattr(TextEncodingStage, "forward", encode)
|
||||
stage = RealtimeTextEncodingStage([], [])
|
||||
session = RealtimeSession()
|
||||
args = SimpleNamespace(
|
||||
disable_conditioning_cache=disabled,
|
||||
conditioning_cache_max_size_mb=capacity,
|
||||
)
|
||||
for _ in range(2):
|
||||
stage.forward(Req(prompt="a cat", session=session), args)
|
||||
expected = 2 if disabled or capacity == 0 else 1
|
||||
assert len(calls) == expected
|
||||
invalidate_conditioning_caches()
|
||||
stage.forward(Req(prompt="a cat", session=session), args)
|
||||
assert len(calls) == expected + 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("disabled, capacity", [(False, 512), (True, 512), (False, 0)])
|
||||
def test_vla_prefix_respects_disable_and_weight_invalidation(disabled, capacity):
|
||||
stage = VLAPrefixEncodingStage.__new__(VLAPrefixEncodingStage)
|
||||
stage.policy_model = Mock()
|
||||
stage.policy_model.build_prefix_cache_key.return_value = "observation"
|
||||
entries = {}
|
||||
stage.prefix_cache = entries
|
||||
args = SimpleNamespace(
|
||||
disable_conditioning_cache=disabled,
|
||||
conditioning_cache_max_size_mb=capacity,
|
||||
pipeline_config=SimpleNamespace(enable_global_prefix_cache=True),
|
||||
)
|
||||
batch = Req(prompt="pick up the cube", extra={"vla": {}})
|
||||
key, cached = stage.get_cached_context(batch, args, None)
|
||||
assert cached is None
|
||||
if disabled or capacity == 0:
|
||||
assert key is None
|
||||
stage.policy_model.build_prefix_cache_key.assert_not_called()
|
||||
else:
|
||||
context = object()
|
||||
entries[key] = context
|
||||
assert stage.get_cached_context(batch, args, None)[1] is context
|
||||
invalidate_conditioning_caches()
|
||||
assert stage.get_cached_context(batch, args, None)[1] is None
|
||||
@@ -704,6 +704,9 @@ class TestIdeogram4(unittest.TestCase):
|
||||
}
|
||||
stage.copy_deduplicated_outputs(base, same)
|
||||
|
||||
self.assertIs(same.prompt_embeds[0], base.prompt_embeds[0])
|
||||
self.assertIs(same.prompt_embeds_mask[0], base.prompt_embeds_mask[0])
|
||||
self.assertIsNot(same.prompt_embeds, base.prompt_embeds)
|
||||
self.assertIn("ideogram4", same.extra)
|
||||
self.assertTrue(
|
||||
torch.equal(
|
||||
|
||||
@@ -235,6 +235,7 @@ class _FakeQwenProcessor:
|
||||
|
||||
def _text_encoding_stage(processor, encoder):
|
||||
stage = object.__new__(LingBotVideoTextEncodingStage)
|
||||
stage._component_residency_manager = None
|
||||
stage.text_encoders = [encoder]
|
||||
stage.tokenizers = [processor]
|
||||
stage.token_length = 128
|
||||
|
||||
@@ -7,6 +7,7 @@ import pytest
|
||||
import torch
|
||||
from prometheus_client import CollectorRegistry
|
||||
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningCache
|
||||
from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import (
|
||||
BaseLayerWithLoRA,
|
||||
@@ -88,6 +89,35 @@ def test_worker_metrics_count_individual_adapters_in_multi_lora():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operation", ["set", "merge", "unmerge", "deactivate"])
|
||||
@torch.no_grad()
|
||||
def test_lora_mutations_preserve_independent_conditioning(operation):
|
||||
layer = _make_layer()
|
||||
pipeline = _make_pipeline(layer)
|
||||
pipeline.modules["transformer"].add_module("linear", layer)
|
||||
pipeline._temporarily_disable_offload = lambda *args, **kwargs: nullcontext([])
|
||||
encoder = torch.nn.Linear(2, 2).eval()
|
||||
cache = ConditioningCache(1024)
|
||||
x = torch.ones(1, 2)
|
||||
with patch(_RANK_PATCH, return_value=0):
|
||||
pipeline.set_lora("adapter", merge_mode="dynamic")
|
||||
cache.run(encoder, "forward", (x,), {}, lambda: encoder(x))
|
||||
cache.run(layer, "forward", (x,), {}, lambda: x.clone())
|
||||
if operation == "set":
|
||||
pipeline.set_lora("adapter", strength=0.5, merge_mode="dynamic")
|
||||
elif operation == "merge":
|
||||
pipeline.merge_lora_weights()
|
||||
elif operation == "unmerge":
|
||||
pipeline.unmerge_lora_weights()
|
||||
else:
|
||||
pipeline.deactivate_lora_weights()
|
||||
cached = cache.run(encoder, "forward", (x,), {}, lambda: encoder(x))
|
||||
cache.run(layer, "forward", (x,), {}, lambda: x.clone())
|
||||
torch.testing.assert_close(cached, encoder(x), rtol=0, atol=0)
|
||||
assert cache.hits == 1
|
||||
assert cache.misses == 3
|
||||
|
||||
|
||||
def test_merge_cache_only_accepts_cpu_backed_weights():
|
||||
pipeline = _make_pipeline(_make_layer())
|
||||
cpu_cache = pipeline._merge_cache_for(
|
||||
|
||||
@@ -5,6 +5,7 @@ from unittest.mock import Mock
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningEncoderMixin
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency import (
|
||||
COMPONENT_OFFLOAD,
|
||||
LAYERWISE_OFFLOAD,
|
||||
@@ -14,11 +15,12 @@ from sglang.multimodal_gen.runtime.pipelines_core.executors.pipeline_executor im
|
||||
PipelineExecutor,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms.npu import NPUPlatformBase
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
|
||||
class _RecordingExecutor(PipelineExecutor):
|
||||
def __init__(self):
|
||||
super().__init__(server_args=SimpleNamespace())
|
||||
super().__init__(server_args=_server_args())
|
||||
self.single_inference_mode = None
|
||||
self.group_inference_mode = None
|
||||
self.single_grad_enabled = None
|
||||
@@ -47,6 +49,8 @@ class _TestServerArgs(SimpleNamespace):
|
||||
def _server_args(**overrides):
|
||||
values = {
|
||||
"use_fsdp_inference": False,
|
||||
"disable_conditioning_cache": ServerArgs.disable_conditioning_cache,
|
||||
"conditioning_cache_max_size_mb": ServerArgs.conditioning_cache_max_size_mb,
|
||||
"component_modes": {},
|
||||
}
|
||||
values.update(overrides)
|
||||
@@ -115,6 +119,31 @@ def test_execute_group_with_profiling_uses_platform_inference_mode(monkeypatch):
|
||||
assert executor.group_grad_enabled is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fsdp", [False, True])
|
||||
@torch.no_grad()
|
||||
def test_warmup_conditioning_is_reused_by_first_serving_request(fsdp):
|
||||
class Encoder(ConditioningEncoderMixin, torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.calls = 0
|
||||
|
||||
def forward(self, tokens):
|
||||
self.calls += 1
|
||||
return tokens * 2
|
||||
|
||||
executor = _RecordingExecutor()
|
||||
executor.component_residency_manager = Mock()
|
||||
encoder = Encoder().eval()
|
||||
tokens = torch.arange(8)
|
||||
args = _server_args(use_fsdp_inference=fsdp)
|
||||
for warmup in (True, True, False):
|
||||
batch = _batch()
|
||||
batch.is_warmup = warmup
|
||||
with executor._component_residency_request([], batch, args):
|
||||
torch.testing.assert_close(encoder(tokens), tokens * 2, rtol=0, atol=0)
|
||||
assert encoder.calls == (3 if fsdp else 2)
|
||||
|
||||
|
||||
def test_group_payload_is_forwarded_to_component_residency_manager():
|
||||
executor = _RecordingExecutor()
|
||||
executor.component_residency_manager = Mock()
|
||||
|
||||
@@ -22,6 +22,7 @@ from sglang.multimodal_gen.configs.pipeline_configs.qwen_image21 import (
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.qwenimage21 import QwenImage21SamplingParams
|
||||
from sglang.multimodal_gen.registry import _get_config_info
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningCache
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.image_api import (
|
||||
_resolve_image_output_format,
|
||||
)
|
||||
@@ -33,6 +34,9 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency_
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.scheduler import Scheduler
|
||||
from sglang.multimodal_gen.runtime.models.dits.qwen_image21 import build_layout
|
||||
from sglang.multimodal_gen.runtime.models.encoders.base import (
|
||||
EncoderTensorParallelMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.qwen3vl_vision import (
|
||||
Qwen3VLVisionRotaryEmbedding,
|
||||
)
|
||||
@@ -120,6 +124,78 @@ def test_prompt_conditioning_uses_training_template_and_pre_norm(prompt, image_c
|
||||
assert image.getpixel((0, 0)) == (12, 34, 56, 0)
|
||||
|
||||
|
||||
class ConditioningTestEncoder(EncoderTensorParallelMixin, torch.nn.Module):
|
||||
def __init__(self, device):
|
||||
super().__init__()
|
||||
self.weight = torch.nn.Parameter(torch.ones((), device=device))
|
||||
self.model = SimpleNamespace(
|
||||
visual=SimpleNamespace(rotary_pos_emb=SimpleNamespace())
|
||||
)
|
||||
self.calls = 0
|
||||
|
||||
def forward(self, input_ids, **kwargs):
|
||||
self.calls += 1
|
||||
hidden = input_ids[..., None].float().expand(-1, -1, 4) * self.weight
|
||||
return SimpleNamespace(hidden_states=tuple(hidden + i for i in range(32)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"device",
|
||||
[
|
||||
"cpu",
|
||||
pytest.param(
|
||||
"cuda",
|
||||
marks=pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="CUDA required"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@torch.no_grad()
|
||||
def test_prompt_cache_skips_processor_and_weight_preparation(device):
|
||||
def process(text, **kwargs):
|
||||
image_value = sum(kwargs["images"][0].getpixel((0, 0)))
|
||||
return BatchFeature(
|
||||
data={
|
||||
"input_ids": torch.tensor([[1, len(text[0]), 99, 99, image_value, 0]]),
|
||||
"attention_mask": torch.tensor([[1, 1, 1, 1, 1, 0]]),
|
||||
}
|
||||
)
|
||||
|
||||
processor = Mock(side_effect=process)
|
||||
processor.tokenizer.convert_tokens_to_ids.return_value = 99
|
||||
processor.apply_chat_template.return_value = [[1]]
|
||||
encoder = ConditioningTestEncoder(device).eval()
|
||||
stage = QwenImage21EncodingStage(encoder, processor, None, None)
|
||||
stage.use_declared_component = Mock(return_value=nullcontext(encoder))
|
||||
image = Image.new("RGBA", (2, 2), (12, 34, 56, 128))
|
||||
cache = ConditioningCache(1024)
|
||||
with cache.scope():
|
||||
first, slots = stage.encode_prompt("edit", [image], device)
|
||||
expected = first.clone()
|
||||
first.zero_()
|
||||
slots.zero_()
|
||||
restored, slots = stage.encode_prompt("edit", [image.copy()], device)
|
||||
torch.testing.assert_close(restored, expected, atol=0, rtol=0)
|
||||
assert slots.tolist() == [False, True, False]
|
||||
assert encoder.calls == processor.call_count == 1
|
||||
stage.use_declared_component.assert_called_once()
|
||||
assert cache.stats()["entries"] == 1
|
||||
assert cache.bytes == 3 * 4 * 4 + 3
|
||||
assert cache.bypasses == 0 # no oversized duplicate of all hidden states
|
||||
changed, _ = stage.encode_prompt("another edit", [image], device)
|
||||
assert not torch.equal(changed, expected)
|
||||
image.putpixel((0, 0), (12, 34, 56, 255))
|
||||
changed_alpha, _ = stage.encode_prompt("edit", [image], device)
|
||||
assert not torch.equal(changed_alpha, expected)
|
||||
assert encoder.calls == 3
|
||||
with cache.scope(refresh=True):
|
||||
stage.encode_prompt("edit", [image], device)
|
||||
with cache.scope(enabled=False):
|
||||
stage.encode_prompt("edit", [image], device)
|
||||
assert encoder.calls == processor.call_count == 5
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
def test_encoder_component_offload_preserves_loaded_dtypes(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
|
||||
@@ -1299,7 +1299,12 @@ def _make_sensenova_u1_sequential_entrypoint(*, fail=False, fail_request_ids=Non
|
||||
output_file_name="sample.png",
|
||||
trace_ctx=trace_ctx,
|
||||
)
|
||||
server_args = SimpleNamespace(pipeline_config=SenseNovaU1PipelineConfig())
|
||||
server_args = SimpleNamespace(
|
||||
pipeline_config=SenseNovaU1PipelineConfig(),
|
||||
disable_conditioning_cache=False,
|
||||
conditioning_cache_max_size_mb=512,
|
||||
use_fsdp_inference=False,
|
||||
)
|
||||
pipeline = _SequentialTestPipeline(
|
||||
server_args, fail=fail, fail_request_ids=fail_request_ids
|
||||
)
|
||||
|
||||
@@ -1,17 +1,202 @@
|
||||
import os
|
||||
from contextlib import nullcontext
|
||||
from datetime import timedelta
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
from torch.distributed.device_mesh import init_device_mesh
|
||||
from torch.distributed.fsdp import fully_shard
|
||||
from transformers import BatchEncoding
|
||||
|
||||
from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import TextConditioningOutput
|
||||
from sglang.multimodal_gen.configs.sample.longlive2 import LongLive2SamplingParams
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.runtime.cache import conditioning
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningCache
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
destroy_distributed_environment,
|
||||
destroy_model_parallel,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentResidencyManager,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency_strategies import (
|
||||
ComponentOffloadStrategy,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.executors.sync_executor import (
|
||||
SyncExecutor,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.causal_denoising import (
|
||||
CAUSAL_BLOCK_PROMPTS_KEY,
|
||||
CAUSAL_SCENE_CUT_MASK_KEY,
|
||||
CAUSAL_SHOT_INDICES_KEY,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.lingbot_video_moe.text_encoding import (
|
||||
LingBotVideoTextEncodingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.longlive2 import (
|
||||
LongLive2TextEncodingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ming_image import (
|
||||
MingImageEncodingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.realtime.text_encoding import (
|
||||
RealtimeTextEncodingStage,
|
||||
RealtimeTextState,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
|
||||
TextEncodingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.realtime.session import RealtimeSession
|
||||
|
||||
_GLOBAL_ARGS_PATCH = (
|
||||
"sglang.multimodal_gen.runtime.pipelines_core.stages.base.get_global_server_args"
|
||||
)
|
||||
|
||||
|
||||
class FullHiddenStateEncoder(TextEncoder):
|
||||
uses_sglang_forward_context = False
|
||||
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
self.weight = torch.nn.Parameter(torch.ones(()))
|
||||
self.calls = 0
|
||||
|
||||
def forward(self, input_ids, **kwargs):
|
||||
self.calls += 1
|
||||
states = tuple(input_ids[..., None].float() + layer for layer in range(32))
|
||||
return BaseEncoderOutput(
|
||||
last_hidden_state=states[-1],
|
||||
hidden_states=states,
|
||||
pooler_output=states[-1][:, 0].clone(),
|
||||
)
|
||||
|
||||
|
||||
class LibraryEncoder(torch.nn.Module):
|
||||
__init__ = FullHiddenStateEncoder.__init__
|
||||
forward = FullHiddenStateEncoder.forward
|
||||
|
||||
|
||||
def make_text_config(output_type="tensor"):
|
||||
def postprocess(output, text_inputs, return_attention_mask=False):
|
||||
embedding = output.hidden_states[9]
|
||||
mask = text_inputs["attention_mask"].bool()
|
||||
if output_type == "tuple":
|
||||
return embedding, mask
|
||||
if output_type == "structured":
|
||||
return TextConditioningOutput(embedding, mask, [2])
|
||||
return embedding
|
||||
|
||||
def tokenize(texts, _tokenizer, _kwargs):
|
||||
return BatchEncoding(
|
||||
{
|
||||
"input_ids": torch.tensor([[len(text), 2] for text in texts]),
|
||||
"attention_mask": torch.ones((len(texts), 2), dtype=torch.long),
|
||||
}
|
||||
)
|
||||
|
||||
return SimpleNamespace(
|
||||
text_encoder_configs=[SimpleNamespace(tokenizer_kwargs={})],
|
||||
preprocess_text_funcs=[None],
|
||||
postprocess_text_funcs=[postprocess],
|
||||
text_encoder_extra_args=[],
|
||||
is_flux_v1=lambda: False,
|
||||
tokenize_prompt=tokenize,
|
||||
get_text_encoder_attention_mask=lambda inputs, _: inputs["attention_mask"],
|
||||
get_text_encoder_pooler_output=lambda outputs, _: outputs.pooler_output,
|
||||
build_text_conditioning_mask=lambda inputs, mask, embeds, _: mask.bool(),
|
||||
seq_lens_from_text_conditioning_mask=lambda mask: mask.sum(-1).tolist(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_type", ["tensor", "tuple", "structured"])
|
||||
@pytest.mark.parametrize(
|
||||
"device",
|
||||
[
|
||||
"cpu",
|
||||
pytest.param(
|
||||
"cuda",
|
||||
marks=pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires CUDA"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@torch.no_grad()
|
||||
def test_cache_stores_only_consumed_text_conditioning(output_type, device, monkeypatch):
|
||||
encoder = FullHiddenStateEncoder().to(device).eval()
|
||||
fingerprint = conditioning._fingerprint
|
||||
|
||||
def fingerprint_host_inputs(value):
|
||||
if isinstance(value, torch.Tensor):
|
||||
assert value.device.type == "cpu", "token keys must not read GPU inputs"
|
||||
return fingerprint(value)
|
||||
|
||||
monkeypatch.setattr(conditioning, "_fingerprint", fingerprint_host_inputs)
|
||||
|
||||
config = make_text_config(output_type)
|
||||
args = make_server_args(pipeline_config=config)
|
||||
|
||||
def make_stage():
|
||||
with patch(_GLOBAL_ARGS_PATCH, return_value=MagicMock()):
|
||||
stage = TextEncodingStage(text_encoders=[encoder], tokenizers=[object()])
|
||||
stage._begin_text_encoder_use = MagicMock()
|
||||
stage._text_encode_dp_group = MagicMock(return_value=None)
|
||||
return stage
|
||||
|
||||
stage = make_stage()
|
||||
cache = ConditioningCache(4096)
|
||||
with cache.scope():
|
||||
first = stage.encode_text(
|
||||
"hello", args, device=device, return_attention_mask=True
|
||||
)
|
||||
expected = first[0][0].clone()
|
||||
expected_pooled = first[2][0].clone()
|
||||
first[0][0].zero_()
|
||||
first[2][0].zero_()
|
||||
restored = stage.encode_text(
|
||||
"hello", args, device=device, return_attention_mask=True
|
||||
)
|
||||
torch.testing.assert_close(restored[0][0], expected, rtol=0, atol=0)
|
||||
torch.testing.assert_close(restored[2][0], expected_pooled, rtol=0, atol=0)
|
||||
assert restored[4] == [[2]]
|
||||
assert encoder.calls == 1
|
||||
stage._begin_text_encoder_use.assert_called_once_with(0)
|
||||
assert cache.stats()["entries"] == 1
|
||||
assert cache.bytes < 32 # two embeddings, one pooled value, optional mask
|
||||
stage.encode_text("changed", args, device=device, return_attention_mask=True)
|
||||
assert encoder.calls == 2
|
||||
# The same encoder can serve different pipeline postprocessing contracts.
|
||||
make_stage().encode_text(
|
||||
"hello", args, device=device, return_attention_mask=True
|
||||
)
|
||||
assert encoder.calls == 3
|
||||
|
||||
# exercise the real negative-stage boundary with only one entry of capacity
|
||||
cache = ConditioningCache(cache.bytes // 3)
|
||||
stage.encode_text = partial(stage.encode_text, device=device)
|
||||
calls_before = encoder.calls
|
||||
with cache.scope(refresh=True):
|
||||
stage.encode_text("hello", args, return_attention_mask=True)
|
||||
stage.get_or_compute_negative_text_embedding(make_req(), args, [0])
|
||||
with cache.scope():
|
||||
stage.encode_text("changed positive", args, return_attention_mask=True)
|
||||
stage.get_or_compute_negative_text_embedding(make_req(), args, [0])
|
||||
assert encoder.calls == calls_before + 3
|
||||
assert cache.hits == 1
|
||||
assert cache.bytes <= cache.max_bytes
|
||||
|
||||
|
||||
class DummyTextEncodingStage(TextEncodingStage):
|
||||
def __init__(self):
|
||||
with patch(_GLOBAL_ARGS_PATCH) as mock_global_args:
|
||||
@@ -53,6 +238,341 @@ def make_server_args(**kwargs):
|
||||
return SimpleNamespace(**defaults)
|
||||
|
||||
|
||||
def make_group_executor(
|
||||
stage_type, device, capacity, *, fsdp=False, native=True, tokenizer=None
|
||||
):
|
||||
encoder = (
|
||||
(FullHiddenStateEncoder() if native else LibraryEncoder()).to(device).eval()
|
||||
)
|
||||
config = make_text_config()
|
||||
config.supports_auto_residency = False
|
||||
config.vae_config = SimpleNamespace(use_temporal_scaling_frames=False)
|
||||
config.dit_config = SimpleNamespace(
|
||||
arch_config=SimpleNamespace(num_frames_per_block=1)
|
||||
)
|
||||
args = make_server_args(
|
||||
pipeline_config=config,
|
||||
component_precisions={},
|
||||
comfyui_mode=True,
|
||||
enable_layerwise_nvtx_marker=False,
|
||||
use_fsdp_inference=fsdp,
|
||||
disable_conditioning_cache=capacity == 0,
|
||||
conditioning_cache_max_size_mb=capacity / 1024**2,
|
||||
should_cpu_offload_component=lambda _: False,
|
||||
)
|
||||
with patch(_GLOBAL_ARGS_PATCH, return_value=args):
|
||||
stage = stage_type(
|
||||
[encoder], [tokenizer if tokenizer is not None else object()]
|
||||
)
|
||||
stage._text_encode_dp_group = Mock(return_value=None)
|
||||
stage.encode_text = partial(stage.encode_text, device=device)
|
||||
pipeline = SimpleNamespace(
|
||||
modules={"text_encoder": encoder},
|
||||
_stage_name_mapping={"text": stage},
|
||||
component_residency_strategies={},
|
||||
)
|
||||
executor = SyncExecutor(args)
|
||||
manager = ComponentResidencyManager(pipeline, args)
|
||||
strategy = Mock()
|
||||
strategy.prefetch_for_use.return_value = False
|
||||
manager.strategy_for = Mock(return_value=strategy)
|
||||
executor.component_residency_manager = manager
|
||||
return executor, stage, encoder, args
|
||||
|
||||
|
||||
@pytest.mark.parametrize("capacity", [0, 4096])
|
||||
@torch.no_grad()
|
||||
def test_grouped_realtime_text_updates_each_session(capacity):
|
||||
executor, stage, encoder, args = make_group_executor(
|
||||
RealtimeTextEncodingStage, "cpu", capacity
|
||||
)
|
||||
sessions = [RealtimeSession(), RealtimeSession()]
|
||||
requests = [
|
||||
Req(
|
||||
sampling_params=SamplingParams(prompt="hello", num_inference_steps=4),
|
||||
do_classifier_free_guidance=False,
|
||||
session=session,
|
||||
)
|
||||
for session in sessions
|
||||
]
|
||||
|
||||
outputs = executor.execute_group([stage], requests, args)
|
||||
|
||||
assert encoder.calls == 1
|
||||
for session, output in zip(sessions, outputs):
|
||||
state = session.get_or_create_state(RealtimeTextState)
|
||||
if capacity:
|
||||
assert state.cache_key is not None
|
||||
assert state.prompt_embeds[0] is output.prompt_embeds[0]
|
||||
assert state.prompt_seq_lens[0] is not output.prompt_seq_lens[0]
|
||||
else:
|
||||
assert state.cache_key is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fsdp, native", [(True, False), (True, True), (False, False)])
|
||||
@torch.no_grad()
|
||||
def test_one_unique_stage_skips_group_only_cache_lookup(fsdp, native, monkeypatch):
|
||||
executor, stage, encoder, args = make_group_executor(
|
||||
TextEncodingStage, "cpu", 4096, fsdp=fsdp, native=native
|
||||
)
|
||||
fingerprint = Mock(wraps=conditioning._fingerprint)
|
||||
monkeypatch.setattr(conditioning, "_fingerprint", fingerprint)
|
||||
capture_query = Mock(wraps=torch.cuda.is_current_stream_capturing)
|
||||
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", capture_query)
|
||||
stage.forward = Mock(wraps=stage.forward)
|
||||
requests = [
|
||||
Req(
|
||||
sampling_params=SamplingParams(prompt="hello", num_inference_steps=4),
|
||||
do_classifier_free_guidance=False,
|
||||
)
|
||||
for _ in range(3)
|
||||
]
|
||||
|
||||
outputs = executor.execute_group([stage], requests, args)
|
||||
|
||||
assert stage.forward.call_count == 1
|
||||
assert encoder.calls == 1
|
||||
assert outputs[0].prompt_embeds[0] is outputs[2].prompt_embeds[0]
|
||||
fingerprint.assert_not_called()
|
||||
capture_query.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("capacity", [0, 1, 4096])
|
||||
@pytest.mark.parametrize("fsdp", [False, True])
|
||||
@pytest.mark.parametrize("native", [False, True])
|
||||
@pytest.mark.parametrize("warmup", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"device",
|
||||
[
|
||||
"cpu",
|
||||
pytest.param(
|
||||
"cuda",
|
||||
marks=pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires CUDA"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@torch.no_grad()
|
||||
def test_grouped_conditioning_reuses_positive_and_negative_independently(
|
||||
capacity, device, fsdp, native, warmup
|
||||
):
|
||||
executor, stage, encoder, args = make_group_executor(
|
||||
TextEncodingStage, device, capacity, fsdp=fsdp, native=native
|
||||
)
|
||||
stage.forward = Mock(wraps=stage.forward)
|
||||
|
||||
def requests():
|
||||
return [
|
||||
Req(
|
||||
sampling_params=SamplingParams(
|
||||
prompt=prompt, negative_prompt="bad quality", num_inference_steps=4
|
||||
),
|
||||
do_classifier_free_guidance=True,
|
||||
is_warmup=warmup,
|
||||
)
|
||||
for prompt in ("hello", "different", "hello")
|
||||
]
|
||||
|
||||
first = executor.execute_group([stage], requests(), args)
|
||||
assert stage.forward.call_count == 2
|
||||
assert encoder.calls == 3 # two positives, one shared negative
|
||||
assert executor.conditioning_cache.group_hits == 1
|
||||
assert first[0].prompt_embeds[0] is first[2].prompt_embeds[0]
|
||||
assert first[0].negative_prompt_embeds[0] is first[1].negative_prompt_embeds[0]
|
||||
assert first[0].prompt_seq_lens is not first[2].prompt_seq_lens
|
||||
assert first[0].prompt_seq_lens[0] is not first[2].prompt_seq_lens[0]
|
||||
expected = first[0].prompt_embeds[0].clone()
|
||||
first[0].prompt_embeds[0].zero_()
|
||||
first[0].prompt_seq_lens[0][0] = 0
|
||||
assert executor.conditioning_cache._group_entries.get() is None
|
||||
|
||||
second = executor.execute_group([stage], requests(), args)
|
||||
assert stage.forward.call_count == 4
|
||||
persistent = capacity == 4096 and not fsdp and not warmup
|
||||
expected_calls = (3 if native else 5) if persistent else 6
|
||||
assert encoder.calls == expected_calls
|
||||
torch.testing.assert_close(second[0].prompt_embeds[0], expected, rtol=0, atol=0)
|
||||
assert second[0].prompt_seq_lens == [[2]]
|
||||
assert executor.conditioning_cache.bytes <= capacity
|
||||
if fsdp:
|
||||
assert executor.conditioning_cache.bytes == 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("native", [False, True])
|
||||
@torch.no_grad()
|
||||
def test_negative_conditioning_keeps_private_device_snapshot(native):
|
||||
encoder = (FullHiddenStateEncoder() if native else LibraryEncoder()).cuda().eval()
|
||||
config = make_text_config("structured")
|
||||
config.postprocess_text_funcs[0] = Mock(wraps=config.postprocess_text_funcs[0])
|
||||
args = make_server_args(pipeline_config=config)
|
||||
with patch(_GLOBAL_ARGS_PATCH, return_value=args):
|
||||
stage = TextEncodingStage([encoder], [object()])
|
||||
stage._text_encode_dp_group = Mock(return_value=None)
|
||||
stage._begin_text_encoder_use = Mock()
|
||||
stage.encode_text = partial(stage.encode_text, device="cuda")
|
||||
cache = ConditioningCache(4096)
|
||||
producer, consumer = torch.cuda.Stream(), torch.cuda.Stream()
|
||||
expected = torch.tensor([[[20.0], [11.0]]])
|
||||
|
||||
with cache.scope(), torch.cuda.stream(producer):
|
||||
torch.cuda._sleep(250_000_000)
|
||||
first = stage.get_or_compute_negative_text_embedding(make_req(), args, [0])
|
||||
first[0][0].zero_()
|
||||
first[1][0].zero_()
|
||||
first[4][0][0] = 0
|
||||
entry = next(iter(cache._entries.values()))
|
||||
assert entry.device_resident
|
||||
assert entry.output[0].device.type == "cuda"
|
||||
assert entry.output[0].data_ptr() != first[0][0].data_ptr()
|
||||
assert cache.stats()["device_bytes"] == cache.bytes <= 4096
|
||||
|
||||
with cache.scope(), torch.cuda.stream(consumer):
|
||||
for _ in range(2):
|
||||
hit = stage.get_or_compute_negative_text_embedding(make_req(), args, [0])
|
||||
torch.testing.assert_close(hit[0][0].cpu(), expected, rtol=0, atol=0)
|
||||
assert hit[1][0].tolist() == [[1, 1]]
|
||||
assert hit[4] == [[2]]
|
||||
hit[0][0].zero_()
|
||||
assert encoder.calls == config.postprocess_text_funcs[0].call_count == 1
|
||||
stage._begin_text_encoder_use.assert_called_once_with(0)
|
||||
conditioning.invalidate_conditioning_caches([encoder])
|
||||
assert cache.bytes == cache.stats()["device_bytes"] == 0
|
||||
stage.get_or_compute_negative_text_embedding(make_req(), args, [0])
|
||||
assert encoder.calls == 2
|
||||
with cache.scope(refresh=True):
|
||||
stage.get_or_compute_negative_text_embedding(make_req(), args, [0])
|
||||
assert encoder.calls == 3
|
||||
stage.get_or_compute_negative_text_embedding(
|
||||
make_req(negative_prompt="changed negative"), args, [0]
|
||||
)
|
||||
assert encoder.calls == 4
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_batched_request_reuses_conditioning_without_changing_output_seeds():
|
||||
executor, stage, encoder, args = make_group_executor(TextEncodingStage, "cpu", 0)
|
||||
for _ in range(2):
|
||||
request = Req(
|
||||
sampling_params=SamplingParams(
|
||||
prompt="hello",
|
||||
negative_prompt="hello",
|
||||
num_outputs_per_prompt=3,
|
||||
seed=[7, 8, 9],
|
||||
),
|
||||
do_classifier_free_guidance=True,
|
||||
)
|
||||
output = executor.execute([stage], request, args)
|
||||
assert output.seed == [7, 8, 9]
|
||||
assert output.num_outputs_per_prompt == 3
|
||||
assert output.prompt_embeds[0] is output.negative_prompt_embeds[0]
|
||||
assert executor.conditioning_cache._group_entries.get() is None
|
||||
assert encoder.calls == 2
|
||||
assert executor.conditioning_cache.group_hits == 2
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def test_grouped_longlive_conditioning_keeps_per_request_shot_metadata():
|
||||
executor, stage, encoder, args = make_group_executor(
|
||||
LongLive2TextEncodingStage, "cpu", 0
|
||||
)
|
||||
requests = [
|
||||
Req(
|
||||
sampling_params=LongLive2SamplingParams(
|
||||
prompt="hello",
|
||||
shot_prompts=["first", "second"],
|
||||
shot_durations=[1, 1],
|
||||
num_frames=2,
|
||||
),
|
||||
do_classifier_free_guidance=False,
|
||||
)
|
||||
for _ in range(2)
|
||||
]
|
||||
outputs = executor.execute_group([stage], requests, args)
|
||||
assert encoder.calls == 1
|
||||
for key in (
|
||||
CAUSAL_BLOCK_PROMPTS_KEY,
|
||||
CAUSAL_SCENE_CUT_MASK_KEY,
|
||||
CAUSAL_SHOT_INDICES_KEY,
|
||||
):
|
||||
assert key in outputs[0].extra and key in outputs[1].extra
|
||||
assert outputs[0].extra[key] == outputs[1].extra[key]
|
||||
assert outputs[0].extra[key] is not outputs[1].extra[key]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fsdp", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"stage_type", [MingImageEncodingStage, LingBotVideoTextEncodingStage]
|
||||
)
|
||||
@torch.no_grad()
|
||||
def test_custom_text_stages_preserve_group_reuse_and_request_metadata(fsdp, stage_type):
|
||||
tokenizer = Mock(name_or_path="fixture", eos_token="<eos>")
|
||||
tokenizer.side_effect = lambda text, **kwargs: {"input_ids": [len(text), 2]}
|
||||
tokenizer.convert_tokens_to_ids.side_effect = lambda text: (
|
||||
3 if text == "<image>" else 4
|
||||
)
|
||||
factory = stage_type
|
||||
if stage_type is LingBotVideoTextEncodingStage:
|
||||
factory = partial(stage_type, transformer=torch.nn.Linear(1, 1))
|
||||
|
||||
def tokenize(**kwargs):
|
||||
texts = kwargs["text"]
|
||||
ids = (
|
||||
[[1]]
|
||||
if isinstance(texts, str)
|
||||
else [[1, len(text), 2] for text in texts]
|
||||
)
|
||||
return BatchEncoding(
|
||||
{
|
||||
"input_ids": torch.tensor(ids),
|
||||
"attention_mask": torch.ones_like(torch.tensor(ids)),
|
||||
}
|
||||
)
|
||||
|
||||
tokenizer.side_effect = tokenize
|
||||
|
||||
with patch(
|
||||
"sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages."
|
||||
"ming_image.Qwen2VLImageProcessorPil.from_pretrained"
|
||||
):
|
||||
executor, stage, encoder, args = make_group_executor(
|
||||
factory, "cpu", 0, fsdp=fsdp, tokenizer=tokenizer
|
||||
)
|
||||
encoder.dtype = torch.bfloat16
|
||||
encoder.config = SimpleNamespace(projection_config={"img_gen_scales": [1]})
|
||||
encoder.image_token = 99
|
||||
args.pipeline_config.dit_config.arch_config.alignment_padding_mode = "zero_masked"
|
||||
args.pipeline_config.dit_config.arch_config.multi_frame_output = False
|
||||
|
||||
def requests():
|
||||
return [
|
||||
Req(
|
||||
sampling_params=SamplingParams(
|
||||
prompt=prompt, negative_prompt="", height=64, width=64
|
||||
),
|
||||
do_classifier_free_guidance=False,
|
||||
)
|
||||
for prompt in ("hello", "different", "hello")
|
||||
]
|
||||
|
||||
with patch(
|
||||
f"{stage_type.__module__}.get_local_torch_device",
|
||||
return_value=torch.device("cpu"),
|
||||
):
|
||||
for run in range(2):
|
||||
outputs = executor.execute_group([stage], requests(), args)
|
||||
assert encoder.calls == 2 * (run + 1)
|
||||
assert outputs[0].prompt_embeds[0] is outputs[2].prompt_embeds[0]
|
||||
assert outputs[0].prompt_embeds is not outputs[2].prompt_embeds
|
||||
if stage_type is MingImageEncodingStage:
|
||||
assert all(output.extra["ming_frames"] == 1 for output in outputs)
|
||||
assert (
|
||||
outputs[0].extra["ming_direct"] is outputs[2].extra["ming_direct"]
|
||||
)
|
||||
assert executor.conditioning_cache._group_entries.get() is None
|
||||
|
||||
|
||||
def get_negative_embedding_twice(stage, server_args, first_req, second_req=None):
|
||||
stage.get_or_compute_negative_text_embedding(first_req, server_args, [0])
|
||||
stage.get_or_compute_negative_text_embedding(
|
||||
@@ -60,23 +580,62 @@ def get_negative_embedding_twice(stage, server_args, first_req, second_req=None)
|
||||
)
|
||||
|
||||
|
||||
def test_negative_text_cache_key_tracks_encode_options():
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("capacity", [0, 1, 4096])
|
||||
@torch.no_grad()
|
||||
def test_lingbot_cache_miss_prepares_offloaded_encoder(capacity, monkeypatch):
|
||||
device = torch.device("cuda", torch.cuda.current_device())
|
||||
monkeypatch.setattr(
|
||||
"sglang.multimodal_gen.runtime.managers.memory_managers."
|
||||
"component_residency_strategies.get_local_torch_device",
|
||||
lambda: device,
|
||||
)
|
||||
executor, stage, encoder, args = make_group_executor(
|
||||
partial(LingBotVideoTextEncodingStage, transformer=torch.nn.Linear(1, 1)),
|
||||
"cpu",
|
||||
capacity,
|
||||
)
|
||||
encoder.embedding = torch.nn.Embedding(8, 2)
|
||||
|
||||
def encode(input_ids, **kwargs):
|
||||
encoder.calls += 1
|
||||
states = encoder.embedding(input_ids)
|
||||
return BaseEncoderOutput(last_hidden_state=states, hidden_states=(states,))
|
||||
|
||||
encoder.forward = encode
|
||||
stage._crop_start = 0
|
||||
stage._build_prompt_inputs = lambda prompt: BatchEncoding(
|
||||
{"input_ids": torch.tensor([[1, 2]]), "attention_mask": torch.ones(1, 2)}
|
||||
)
|
||||
manager = executor.component_residency_manager
|
||||
strategy = ComponentOffloadStrategy()
|
||||
strategy.prepare_for_use = Mock(wraps=strategy.prepare_for_use)
|
||||
manager.strategy_for = Mock(return_value=strategy)
|
||||
stage.set_component_residency_manager(manager)
|
||||
cache = ConditioningCache(capacity)
|
||||
results = []
|
||||
for _ in range(2):
|
||||
batch = make_req()
|
||||
manager.begin_request([stage], batch, args)
|
||||
manager.before_stage(stage, 0, batch, args)
|
||||
manager.begin_stage()
|
||||
with cache.scope():
|
||||
results.append(stage._encode_prompt("hello", device, torch.float32))
|
||||
manager.end_stage()
|
||||
manager.finish_request()
|
||||
assert encoder.embedding.weight.device.type == "cpu"
|
||||
assert results[-1][0].device == device
|
||||
expected_calls = 1 if capacity == 4096 else 2
|
||||
assert encoder.calls == strategy.prepare_for_use.call_count == expected_calls
|
||||
torch.testing.assert_close(results[0][0], results[1][0], rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_negative_text_encoding_has_no_separate_gpu_cache():
|
||||
stage = DummyTextEncodingStage()
|
||||
server_args = make_server_args()
|
||||
|
||||
get_negative_embedding_twice(stage, server_args, make_req())
|
||||
assert stage.calls == 1
|
||||
|
||||
stage.get_or_compute_negative_text_embedding(
|
||||
make_req(max_sequence_length=512), server_args, [0]
|
||||
)
|
||||
assert stage.calls == 2
|
||||
|
||||
stage.get_or_compute_negative_text_embedding(
|
||||
make_req(prompt_template={"template": "negative: {}"}), server_args, [0]
|
||||
)
|
||||
assert stage.calls == 3
|
||||
|
||||
|
||||
def test_component_uses_exact_encoder_precision():
|
||||
with patch(_GLOBAL_ARGS_PATCH) as mock_global_args:
|
||||
@@ -97,25 +656,280 @@ def test_component_uses_exact_encoder_precision():
|
||||
]
|
||||
|
||||
|
||||
def test_negative_text_cache_skips_warmup():
|
||||
def test_negative_text_encoding_warmup_does_not_seed_a_private_cache():
|
||||
stage = DummyTextEncodingStage()
|
||||
server_args = make_server_args()
|
||||
|
||||
with patch.object(
|
||||
stage, "_get_model_default_negative_prompt", return_value="default negative"
|
||||
):
|
||||
get_negative_embedding_twice(stage, server_args, make_req(is_warmup=True))
|
||||
|
||||
get_negative_embedding_twice(stage, make_server_args(), make_req(is_warmup=True))
|
||||
assert stage.calls == 2
|
||||
|
||||
|
||||
def test_negative_text_cache_keeps_default_warmup():
|
||||
stage = DummyTextEncodingStage()
|
||||
server_args = make_server_args()
|
||||
@pytest.mark.parametrize("encoder_count", [1, 2])
|
||||
@torch.no_grad()
|
||||
def test_cache_hit_skips_encoder_residency_and_input_preparation(encoder_count):
|
||||
encoders = [FullHiddenStateEncoder().eval() for _ in range(encoder_count)]
|
||||
config = make_text_config()
|
||||
config.text_encoder_configs *= encoder_count
|
||||
config.preprocess_text_funcs *= encoder_count
|
||||
config.postprocess_text_funcs *= encoder_count
|
||||
config.supports_auto_residency = False
|
||||
config.get_text_encoder_attention_mask = Mock(
|
||||
wraps=config.get_text_encoder_attention_mask
|
||||
)
|
||||
config.seq_lens_from_text_conditioning_mask = Mock(
|
||||
wraps=config.seq_lens_from_text_conditioning_mask
|
||||
)
|
||||
args = make_server_args(
|
||||
pipeline_config=config,
|
||||
component_precisions={},
|
||||
enable_layerwise_nvtx_marker=False,
|
||||
)
|
||||
with patch(_GLOBAL_ARGS_PATCH, return_value=args):
|
||||
stage = TextEncodingStage(encoders, [object()] * encoder_count)
|
||||
stage._text_encode_dp_group = Mock(return_value=None)
|
||||
pipeline = SimpleNamespace(
|
||||
modules={
|
||||
"text_encoder" if i == 0 else f"text_encoder_{i + 1}": encoder
|
||||
for i, encoder in enumerate(encoders)
|
||||
},
|
||||
_stage_name_mapping={"text": stage},
|
||||
component_residency_strategies={},
|
||||
)
|
||||
manager = ComponentResidencyManager(pipeline, args)
|
||||
strategy = Mock()
|
||||
strategy.prefetch_for_use.return_value = False
|
||||
manager.strategy_for = Mock(return_value=strategy)
|
||||
stage.set_component_residency_manager(manager)
|
||||
cache = ConditioningCache(4096)
|
||||
|
||||
with patch.object(
|
||||
stage, "_get_model_default_negative_prompt", return_value="bad quality"
|
||||
):
|
||||
get_negative_embedding_twice(stage, server_args, make_req(is_warmup=True))
|
||||
def request(prompt, *, refresh=False, enabled=True, dtype=None):
|
||||
batch = make_req(is_warmup=refresh)
|
||||
manager.begin_request([stage], batch, args)
|
||||
manager.before_stage(stage, 0, batch, args)
|
||||
manager.begin_stage()
|
||||
with (cache if enabled else ConditioningCache(0)).scope(refresh=refresh):
|
||||
result = stage.encode_text(
|
||||
prompt,
|
||||
args,
|
||||
encoder_index=list(range(encoder_count)),
|
||||
device="cpu",
|
||||
dtype=dtype,
|
||||
return_attention_mask=True,
|
||||
)
|
||||
manager.end_stage()
|
||||
manager.finish_request()
|
||||
return result
|
||||
|
||||
assert stage.calls == 1
|
||||
first = request("hello")
|
||||
expected = first[0][0].clone()
|
||||
first[0][0].zero_()
|
||||
first[1][0].zero_()
|
||||
first[3][0].zero_()
|
||||
first[4][0][0] = 0
|
||||
strategy.reset_mock()
|
||||
restored = request("hello")
|
||||
strategy.prepare_for_use.assert_not_called()
|
||||
strategy.wait_for_use.assert_not_called()
|
||||
assert config.get_text_encoder_attention_mask.call_count == encoder_count
|
||||
assert config.seq_lens_from_text_conditioning_mask.call_count == encoder_count
|
||||
torch.testing.assert_close(restored[0][0], expected, rtol=0, atol=0)
|
||||
assert restored[1][0].tolist() == [[1, 1]]
|
||||
assert restored[3][0].tolist() == [[True, True]]
|
||||
assert restored[4][0] == [2]
|
||||
|
||||
# dtype is part of the consumed conditioning contract
|
||||
cast = request("hello", dtype=torch.float64)
|
||||
assert cast[0][0].dtype == torch.float64
|
||||
assert encoders[0].calls == 2
|
||||
request("hello", refresh=True)
|
||||
request("changed")
|
||||
request("hello", enabled=False)
|
||||
assert all(encoder.calls == 5 for encoder in encoders)
|
||||
assert strategy.prepare_for_use.call_count == 4 * encoder_count
|
||||
|
||||
|
||||
class TextEncodingDPGroup:
|
||||
world_size = 2
|
||||
|
||||
def __init__(self, rank):
|
||||
self.rank_in_group = rank
|
||||
self.gathers = 0
|
||||
self.cpu_group = dist.group.WORLD
|
||||
|
||||
def all_reduce(self, tensor):
|
||||
dist.all_reduce(tensor)
|
||||
return tensor
|
||||
|
||||
def all_gather(self, tensor, dim):
|
||||
self.gathers += 1
|
||||
outputs = [torch.empty_like(tensor) for _ in range(self.world_size)]
|
||||
dist.all_gather(outputs, tensor)
|
||||
return torch.cat(outputs, dim=dim)
|
||||
|
||||
|
||||
def run_text_encoding_dp(rank, rendezvous, grouped, negative):
|
||||
dist.init_process_group(
|
||||
"gloo",
|
||||
rank=rank,
|
||||
world_size=2,
|
||||
init_method=rendezvous,
|
||||
timeout=timedelta(seconds=30),
|
||||
)
|
||||
try:
|
||||
encoder = FullHiddenStateEncoder().eval()
|
||||
args = make_server_args(pipeline_config=make_text_config())
|
||||
with patch(_GLOBAL_ARGS_PATCH, return_value=args):
|
||||
stage = TextEncodingStage([encoder], [object()])
|
||||
group = TextEncodingDPGroup(rank)
|
||||
stage._text_encode_dp_group = Mock(return_value=group)
|
||||
cache = ConditioningCache(4096)
|
||||
stage_module = TextEncodingStage.__module__
|
||||
with (
|
||||
torch.no_grad(),
|
||||
cache.scope(),
|
||||
cache.group_scope(enabled=grouped),
|
||||
conditioning.prefer_conditioning_cache() if negative else nullcontext(),
|
||||
patch(f"{stage_module}.model_parallel_is_initialized", return_value=True),
|
||||
patch(f"{stage_module}.get_replica_group", return_value=group),
|
||||
):
|
||||
for attempt in range(3):
|
||||
if attempt == 1 and rank == 0:
|
||||
cache.clear()
|
||||
before = group.gathers
|
||||
outputs = stage.encode_text(
|
||||
["a", "bb", "ccc"], args, device="cpu", return_attention_mask=True
|
||||
)
|
||||
if (grouped or negative) and attempt == 2:
|
||||
assert group.gathers == before
|
||||
else:
|
||||
assert group.gathers > before
|
||||
assert outputs[0][0][:, 0, 0].tolist() == [10, 11, 12]
|
||||
assert outputs[4] == [[2, 2, 2]]
|
||||
assert encoder.calls == (2 if negative or rank == 0 else 1)
|
||||
assert cache.hits == (1 if negative or rank == 0 else 2)
|
||||
finally:
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("grouped", [False, True])
|
||||
@pytest.mark.parametrize("negative", [False, True])
|
||||
def test_batch_dp_requires_consensus_to_skip_gather(tmp_path, grouped, negative):
|
||||
mp.spawn(
|
||||
run_text_encoding_dp,
|
||||
args=(f"file://{tmp_path / 'dp-rendezvous'}", grouped, negative),
|
||||
nprocs=2,
|
||||
join=True,
|
||||
)
|
||||
|
||||
|
||||
class ShardedTextEncoder(FullHiddenStateEncoder):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
self.projection = torch.nn.Linear(1, 8, bias=False)
|
||||
torch.nn.init.ones_(self.projection.weight)
|
||||
self.calls = 0
|
||||
|
||||
def forward(self, input_ids, **kwargs):
|
||||
self.calls += 1
|
||||
hidden = self.projection(input_ids[..., None].float())
|
||||
return BaseEncoderOutput(last_hidden_state=hidden, hidden_states=(hidden,) * 32)
|
||||
|
||||
|
||||
def run_fsdp_conditioning(rank, rendezvous):
|
||||
os.environ["LOCAL_RANK"] = str(rank)
|
||||
torch.cuda.set_device(rank)
|
||||
init_distributed_environment(
|
||||
world_size=2,
|
||||
rank=rank,
|
||||
local_rank=rank,
|
||||
distributed_init_method=rendezvous,
|
||||
backend="nccl",
|
||||
timeout=60,
|
||||
)
|
||||
initialize_model_parallel(sequence_parallel_degree=2, ulysses_degree=2)
|
||||
try:
|
||||
encoder = ShardedTextEncoder().cuda().eval()
|
||||
fully_shard(
|
||||
encoder, mesh=init_device_mesh("cuda", (2,)), reshard_after_forward=True
|
||||
)
|
||||
assert encoder.projection.weight.to_local().numel() == 4
|
||||
args = make_server_args(pipeline_config=make_text_config())
|
||||
with patch(_GLOBAL_ARGS_PATCH, return_value=args):
|
||||
stage = TextEncodingStage([encoder], [object()])
|
||||
stage._text_encode_dp_group = Mock(return_value=None)
|
||||
cache = ConditioningCache(4096)
|
||||
with torch.no_grad(), cache.scope(cross_request=False):
|
||||
for _ in range(2):
|
||||
with cache.group_scope():
|
||||
for attempt in range(3):
|
||||
if rank == 0 and attempt == 1:
|
||||
cache.clear()
|
||||
output = stage.encode_text("hello", args, device=f"cuda:{rank}")
|
||||
torch.testing.assert_close(
|
||||
output[0][0][0, :, 0],
|
||||
torch.tensor([5.0, 2.0], device=f"cuda:{rank}"),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
assert cache._group_entries.get() is None
|
||||
assert encoder.calls == 4
|
||||
assert cache.group_hits == 2
|
||||
assert cache.bytes == 0
|
||||
assert encoder.projection.weight.to_local().numel() == 4
|
||||
finally:
|
||||
destroy_model_parallel()
|
||||
destroy_distributed_environment()
|
||||
|
||||
|
||||
@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="requires two CUDA GPUs")
|
||||
def test_fsdp_group_reuse_agrees_across_shards_and_expires(tmp_path):
|
||||
mp.spawn(
|
||||
run_fsdp_conditioning,
|
||||
args=(f"file://{tmp_path / 'fsdp-rendezvous'}",),
|
||||
nprocs=2,
|
||||
join=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
@torch.no_grad()
|
||||
def test_cache_hit_releases_weights_retained_after_warmup(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"sglang.multimodal_gen.runtime.managers.memory_managers."
|
||||
"component_residency_strategies.get_local_torch_device",
|
||||
lambda: torch.device("cuda"),
|
||||
)
|
||||
encoder = FullHiddenStateEncoder().eval()
|
||||
config = make_text_config()
|
||||
config.supports_auto_residency = False
|
||||
args = make_server_args(
|
||||
pipeline_config=config,
|
||||
component_precisions={},
|
||||
enable_layerwise_nvtx_marker=False,
|
||||
)
|
||||
with patch(_GLOBAL_ARGS_PATCH, return_value=args):
|
||||
stage = TextEncodingStage([encoder], [object()])
|
||||
stage._text_encode_dp_group = Mock(return_value=None)
|
||||
pipeline = SimpleNamespace(
|
||||
modules={"text_encoder": encoder},
|
||||
_stage_name_mapping={"text": stage},
|
||||
component_residency_strategies={},
|
||||
)
|
||||
manager = ComponentResidencyManager(pipeline, args)
|
||||
strategy = ComponentOffloadStrategy()
|
||||
manager.strategy_for = Mock(return_value=strategy)
|
||||
stage.set_component_residency_manager(manager)
|
||||
cache = ConditioningCache(4096)
|
||||
for warmup in [True, False]:
|
||||
batch = make_req(is_warmup=warmup)
|
||||
manager.begin_request([stage], batch, args)
|
||||
manager.before_stage(stage, 0, batch, args)
|
||||
manager.begin_stage()
|
||||
with cache.scope(refresh=warmup):
|
||||
result = stage.encode_text(
|
||||
"hello", args, device="cuda", return_attention_mask=True
|
||||
)
|
||||
manager.end_stage()
|
||||
manager.finish_request()
|
||||
assert encoder.weight.device.type == ("cuda" if warmup else "cpu")
|
||||
assert result[0][0].device.type == "cuda"
|
||||
assert encoder.calls == 1
|
||||
assert cache.hits == 1
|
||||
|
||||
Reference in New Issue
Block a user