[diffusion] feat: support bounded exact conditioning cache across native models (#40470)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-09-29 15:27:22 +08:00
committed by GitHub
co-authored by Mick Qian
parent c7be3e935b
commit dc1bd46802
51 changed files with 2990 additions and 332 deletions
+1
View File
@@ -12,6 +12,7 @@ SGLang Diffusion has the following features:
- Broad model support: Wan, FastWan, FLUX, Qwen-Image / Qwen-Image 2.1, LongCat-Image, Z-Image, Anima, Ideogram 4, Krea-2, Cosmos3, LTX-2/LTX-2.3/LTX-2.5, MiniMax-H3, FastH3, VDN-H3, LingBot Video MoE, LingBot World, SANA-Video/SANA-WM, JoyEcho, MOVA, GLM-Image, ERNIE-Image, Hunyuan3D, and more
- Fast inference speed: empowered by optimized `sgl-kernel` kernels, scheduler/runtime improvements, caching acceleration, and native diffusion hot-path optimizations
- Ease of use: OpenAI-compatible api, CLI, python sdk, and a [ComfyUI plugin](apps/ComfyUI_SGLDiffusion/README.md)
- [Conditioning reuse](https://docs.sglang.io/docs/sglang-diffusion/caching-acceleration#conditioning-cache): bounded CPU caching for repeated text/image inputs and VAE posteriors, with `--disable-conditioning-cache` to opt out
- Multi-platform support:
- NVIDIA GPUs (H100, H200, A100, B200, 4090, 5090)
- AMD GPUs (MI300X, MI325X, MI355X)
@@ -0,0 +1,681 @@
# SPDX-License-Identifier: Apache-2.0
"""Exact conditioning reuse within grouped stages and across requests."""
import copy
import hashlib
import pickle
import weakref
from collections import OrderedDict
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass, fields, is_dataclass
from functools import wraps
from typing import Callable
import numpy as np
import torch
import torch.distributed as dist
from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution
from PIL import Image
from sglang.multimodal_gen.runtime.distributed import (
get_replica_group,
get_tp_group,
get_world_size,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
_active_cache: ContextVar["ConditioningCache | None"] = ContextVar(
"conditioning_cache", default=None
)
_refresh_cache: ContextVar[bool] = ContextVar(
"refresh_conditioning_cache", default=False
)
_cross_request_cache: ContextVar[bool] = ContextVar(
"cross_request_conditioning_cache", default=True
)
_prefer_cache: ContextVar[bool] = ContextVar("prefer_conditioning_cache", default=False)
_stage_encoder: ContextVar[object | None] = ContextVar("stage_encoder", default=None)
_container_types: set[type] = {DiagonalGaussianDistribution}
_live_caches = weakref.WeakSet()
_weights_epoch = 0
def invalidate_conditioning_caches(modules=None):
"""Invalidate before weight mutations, including partially failed updates."""
global _weights_epoch
_weights_epoch += 1
parameters = (
None
if modules is None
else {id(p) for module in modules for p in module.parameters()}
)
for cache in _live_caches:
cache.invalidate(parameters)
def conditioning_weights_epoch():
return _weights_epoch
def register_conditioning_container(cls):
"""Opt in a tensor-only posterior container whose fields can be copied."""
_container_types.add(cls)
return cls
class Uncacheable(TypeError):
pass
@contextmanager
def conditioning_cache_group(*, enabled):
cache = _active_cache.get()
if cache is None:
yield
else:
with cache.group_scope(enabled=enabled):
yield
@contextmanager
def prefer_conditioning_cache():
# retain reusable negative conditioning ahead of changing positive prompts
token = _prefer_cache.set(True)
try:
yield
finally:
_prefer_cache.reset(token)
def _fingerprint(value):
if isinstance(value, torch.Tensor):
if (
type(value) is not torch.Tensor
or value.layout != torch.strided
or value.requires_grad
):
raise Uncacheable("only dense inference tensors can be cached")
tensor = value.detach().cpu().contiguous()
data = tensor.reshape(-1).view(torch.uint8).numpy()
return (
"tensor",
str(value.dtype),
str(value.device),
tuple(value.shape),
tuple(value.stride()),
hashlib.sha256(data).digest(),
)
if isinstance(value, Image.Image):
return (
"image",
value.mode,
value.size,
hashlib.sha256(value.tobytes()).digest(),
_fingerprint(value.getpalette()),
_fingerprint(value.info.get("transparency")),
)
if isinstance(value, np.ndarray):
if value.dtype.hasobject:
raise Uncacheable("object arrays are not conditioning values")
return (
"array",
value.dtype.str,
value.shape,
hashlib.sha256(value.tobytes()).digest(),
)
if type(value) in (str, bytes, int, float, bool, type(None)):
return (type(value).__name__, value)
if isinstance(value, (torch.dtype, torch.device)):
return (type(value).__name__, str(value))
if isinstance(value, dict):
return (
"dict",
tuple((_fingerprint(k), _fingerprint(v)) for k, v in value.items()),
)
if isinstance(value, (tuple, list)):
return (type(value).__name__, tuple(_fingerprint(v) for v in value))
raise Uncacheable(f"unsupported conditioning input: {type(value).__name__}")
@dataclass
class _HostTensor:
data: torch.Tensor
device: torch.device
@dataclass
class _CacheEntry:
output: object
size: int
ready: tuple[torch.cuda.Event, ...]
owner: int
preferred: bool
device_resident: bool = False
@classmethod
def snapshot(cls, output, size, owner, preferred, device_resident):
tensors = {}
copy_streams = {}
def snapshot_tensor(t):
if id(t) not in tensors:
if t.device.type == "cuda":
copy_streams[t.device] = torch.cuda.current_stream(t.device)
if device_resident:
stored = t.detach().clone()
elif t.device.type == "cuda":
stored = torch.empty_like(t, device="cpu", pin_memory=True)
stored.copy_(t.detach(), non_blocking=True)
else:
stored = t.detach().to("cpu", copy=True)
tensors[id(t)] = (
stored if device_resident else _HostTensor(stored, t.device)
)
return tensors[id(t)]
stored = _map_output(output, snapshot_tensor)
# snapshot before downstream mutations on each producing stream
ready = []
for stream in copy_streams.values():
event = torch.cuda.Event()
event.record(stream)
ready.append(event)
return cls(stored, size, tuple(ready), owner, preferred, device_resident)
def wait(self):
if self.device_resident:
for event in self.ready:
torch.cuda.current_stream(event.device).wait_event(event)
def record(t):
if t.device.type == "cuda":
t.record_stream(torch.cuda.current_stream(t.device))
return t
_map_output(self.output, record)
return
for event in self.ready:
event.synchronize()
@dataclass
class _GroupEntry:
output: object
owner: int
copied_bytes: int
streams: tuple[torch.cuda.Stream, ...]
def wait(self):
consumers = {}
for producer in self.streams:
consumer = torch.cuda.current_stream(producer.device)
if consumer != producer:
consumer.wait_stream(producer)
consumers[producer.device] = consumer
if not consumers:
return
def record(t):
if t.device in consumers:
t.record_stream(consumers[t.device])
return t
_map_output(self.output, record)
def _copy_output(value, *, share_tensors=False):
tensors = {}
def copy_tensor(t):
if id(t) not in tensors:
tensors[id(t)] = t if share_tensors else t.clone()
return tensors[id(t)]
return _map_output(value, copy_tensor)
def _map_output(value, tensor_fn, *, restore=False):
if isinstance(value, _HostTensor if restore else torch.Tensor):
return tensor_fn(value)
if type(value) in (str, bytes, int, float, bool, type(None)):
return value
if isinstance(value, (torch.dtype, torch.device)):
return value
# ModelOutput is both a dataclass and a dict. Reconstruct it as a dataclass
# so attribute access and tuple indexing retain their original semantics.
if is_dataclass(value) and not isinstance(value, type):
return type(value)(
**{
field.name: _map_output(
vars(value)[field.name], tensor_fn, restore=restore
)
for field in fields(value)
}
)
if type(value) in _container_types:
result = copy.copy(value)
for name, item in vars(value).items():
setattr(result, name, _map_output(item, tensor_fn, restore=restore))
return result
if type(value) is dict:
return {k: _map_output(v, tensor_fn, restore=restore) for k, v in value.items()}
if type(value) is list:
return [_map_output(v, tensor_fn, restore=restore) for v in value]
if isinstance(value, tuple):
items = [_map_output(v, tensor_fn, restore=restore) for v in value]
return type(value)(*items) if type(value) is not tuple else tuple(items)
raise Uncacheable(f"unsupported conditioning output: {type(value).__name__}")
class ConditioningCache:
"""One bounded cache per rank with host entries and device-hot conditioning."""
def __init__(self, max_bytes: int):
if max_bytes < 0:
raise ValueError("conditioning cache capacity must be nonnegative")
_live_caches.add(self)
self.max_bytes = max_bytes
self.bytes = 0
self.hits = self.misses = self.evictions = self.bypasses = 0
self._entries = OrderedDict()
self._models = weakref.WeakKeyDictionary()
self._next_model = 0
self._group_entries = ContextVar("conditioning_group_entries", default=None)
self.group_hits = 0
def clear(self):
group_entries = self._group_entries.get()
if group_entries is not None:
group_entries.clear()
for entry in self._entries.values():
entry.wait()
self._entries.clear()
self.bytes = 0
def invalidate(self, parameters):
if parameters is None:
self.clear()
return
# shared parameters invalidate both owners, including nested encoders
owners = {
identity
for model, identity in self._models.items()
if isinstance(model, torch.nn.Module)
and any(id(p) in parameters for p in model.parameters())
}
for key, entry in list(self._entries.items()):
if entry.owner in owners:
entry.wait()
self.bytes -= entry.size
del self._entries[key]
group_entries = self._group_entries.get()
if group_entries is not None:
for key, entry in list(group_entries.items()):
if entry.owner in owners:
del group_entries[key]
def stats(self):
return dict(
hits=self.hits,
misses=self.misses,
evictions=self.evictions,
bypasses=self.bypasses,
entries=len(self._entries),
bytes=self.bytes,
group_hits=self.group_hits,
device_bytes=sum(
entry.size for entry in self._entries.values() if entry.device_resident
),
)
def _identity(self, owner):
if owner not in self._models:
self._models[owner] = self._next_model
self._next_model += 1
return self._models[owner]
@contextmanager
def group_scope(self, enabled=True):
"""Keep device results only while a stage executes a group of requests."""
token = self._group_entries.set({} if enabled else None)
try:
yield
finally:
self._group_entries.reset(token)
def _remember_group(self, key, output, owner, share_in_group):
entries = self._group_entries.get()
if entries is None:
return
tensors = {}
def visit(t):
tensors[id(t)] = t
return t
_map_output(output, visit)
size = (
0
if share_in_group
else sum(t.numel() * t.element_size() for t in tensors.values())
)
# private snapshots must not retain unbounded intermediate encoder states
if not share_in_group and (
len(entries) >= 128
or size + sum(entry.copied_bytes for entry in entries.values())
> 512 * 1024**2
):
return
stored = _copy_output(output, share_tensors=share_in_group)
streams = tuple(
torch.cuda.current_stream(device)
for device in {
t.device for t in tensors.values() if t.device.type == "cuda"
}
)
entries[key] = _GroupEntry(stored, owner, size, streams)
@contextmanager
def scope(self, enabled=True, *, refresh=False, cross_request=True):
# Zero-capacity ranks still participate in encoder hit consensus.
if refresh and self._group_entries.get() is not None:
self._group_entries.get().clear()
token = _active_cache.set(self if enabled else None)
refresh_token = _refresh_cache.set(refresh or _refresh_cache.get())
cross_request_token = _cross_request_cache.set(
cross_request and _cross_request_cache.get()
)
try:
yield
finally:
_cross_request_cache.reset(cross_request_token)
_refresh_cache.reset(refresh_token)
_active_cache.reset(token)
def run(
self,
model,
method,
args,
kwargs,
compute: Callable,
group=None,
*,
nested=False,
namespace=None,
share_in_group=False,
cross_request=True,
keep_on_device=False,
):
group_entries = self._group_entries.get()
cross_request = cross_request and _cross_request_cache.get()
if not cross_request and group_entries is None:
return compute()
try:
if (not cross_request or not self.max_bytes) and group_entries is None:
raise Uncacheable("cache disabled")
if kwargs.get("use_cache") or kwargs.get("past_key_values") is not None:
raise Uncacheable("stateful autoregressive encoding")
parameter = next(model.parameters(), None)
precision = str(parameter.dtype) if parameter is not None else None
key = (
self._identity(model),
self._identity(namespace) if namespace is not None else None,
method,
share_in_group,
precision,
torch.is_autocast_enabled("cuda"),
torch.get_autocast_dtype("cuda"),
torch.is_autocast_enabled("cpu"),
torch.get_autocast_dtype("cpu"),
_fingerprint(args),
_fingerprint(kwargs),
)
key = hashlib.sha256(pickle.dumps(key)).digest()
except Uncacheable:
key = None
entry = self._entries.get(key) if cross_request else None
group_entry = group_entries.get(key) if group_entries is not None else None
hit = group_entry is not None or (
entry is not None and not _refresh_cache.get()
)
if group is not None and group.world_size > 1:
# Encoders may issue TP/folding collectives. A rank-local eviction
# must never leave another rank returning early from the encoder.
flag = torch.tensor(int(hit), dtype=torch.int32)
dist.all_reduce(flag, op=dist.ReduceOp.MIN, group=group.cpu_group)
hit = bool(flag.item())
if hit:
self.hits += 1
if entry is not None:
entry.preferred |= _prefer_cache.get()
self._entries.move_to_end(key)
if group_entry is not None:
self.group_hits += 1
group_entry.wait()
return _copy_output(group_entry.output, share_tensors=share_in_group)
entry.wait()
logger.debug("Conditioning cache hit: %s.%s", type(model).__name__, method)
restored = {}
def restore(t):
if id(t) not in restored:
restored[id(t)] = t.data.to(t.device, copy=True, non_blocking=True)
return restored[id(t)]
output = (
_copy_output(entry.output)
if entry.device_resident
else _map_output(entry.output, restore, restore=True)
)
if keep_on_device and not entry.device_resident:
self._entries[key] = _CacheEntry.snapshot(
output, entry.size, entry.owner, True, True
)
self._remember_group(key, output, entry.owner, share_in_group)
return output
if key is None:
self.bypasses += 1
return compute()
self.misses += 1
# A VLM may reuse image features even when its joint text/image key
# misses. VAE delegates share one outer posterior entry instead.
with self.scope(enabled=nested):
output = compute()
try:
size = 0
seen = set()
def count(t):
nonlocal size
if id(t) not in seen:
size += t.numel() * t.element_size()
seen.add(id(t))
if type(t) is not torch.Tensor or t.requires_grad:
raise Uncacheable("autograd output")
return t
_map_output(output, count)
self._remember_group(key, output, self._identity(model), share_in_group)
if not cross_request or not size or size > self.max_bytes:
self.bypasses += 1
return output
except Uncacheable:
self.bypasses += 1
return output
old = self._entries.pop(key, None)
if old is not None:
old.wait()
self.bytes -= old.size
preferred = _prefer_cache.get()
evictable = [
key
for key, entry in self._entries.items()
if preferred or not entry.preferred
]
available = (
self.max_bytes
- self.bytes
+ sum(self._entries[key].size for key in evictable)
)
if size > available or (len(self._entries) >= 128 and not evictable):
self.bypasses += 1
return output
for evicted in evictable:
if self.bytes + size <= self.max_bytes and len(self._entries) < 128:
break
removed = self._entries.pop(evicted)
removed.wait()
self.bytes -= removed.size
self.evictions += 1
self._entries[key] = _CacheEntry.snapshot(
output, size, self._identity(model), preferred, keep_on_device
)
self.bytes += size
logger.debug(
"Conditioning cache store: %s.%s, %d bytes",
type(model).__name__,
method,
size,
)
return output
def _inference_cache(model, *, share_in_group=False, cross_request=True):
if torch.compiler.is_compiling():
return None
cache = _active_cache.get()
if cache is None or model.training or torch.is_grad_enabled():
return None
# FSDP reuses only consumed stage outputs, never sharded intermediate states
if not (cross_request and _cross_request_cache.get()) and (
not share_in_group or cache._group_entries.get() is None
):
return None
# graph capture cannot hash or copy CUDA values through host memory
if torch.cuda.is_available() and torch.cuda.is_current_stream_capturing():
return None
return cache
def cached_encoder_call(
model,
args,
kwargs,
compute,
group=None,
*,
namespace=None,
nested=True,
share_in_group=False,
cross_request=True,
):
if torch.compiler.is_compiling():
return compute()
# only consumed negative conditioning is device-hot; raw hidden states and
# other encoder boundaries keep their host-cache policy
keep_on_device = namespace is not None and share_in_group and _prefer_cache.get()
cross_request = cross_request or keep_on_device
cache = _inference_cache(
model, share_in_group=share_in_group, cross_request=cross_request
)
if (
cache is None
or model is _stage_encoder.get()
or kwargs.get("use_cache")
or kwargs.get("past_key_values") is not None
):
return compute()
if not _cross_request_cache.get() and model_parallel_is_initialized():
# the encoder's TP group may not cover its FSDP shard group
group = get_replica_group()
def compute_conditioning():
# a stage owns the consumed output; preserve nested vision-method caches
token = _stage_encoder.set(model)
try:
return compute()
finally:
_stage_encoder.reset(token)
return cache.run(
model,
"forward",
args,
kwargs,
compute_conditioning if namespace is not None and cross_request else compute,
group,
nested=nested,
namespace=namespace,
share_in_group=share_in_group,
cross_request=cross_request,
keep_on_device=keep_on_device,
)
def cached_conditioning(fn):
"""Cache a deterministic conditioning method on the current encoder TP group."""
@wraps(fn)
def wrapped(self, *args, **kwargs):
cache = _inference_cache(self)
if cache is None:
return fn(self, *args, **kwargs)
group = get_tp_group() if model_parallel_is_initialized() else None
return cache.run(
self, fn.__name__, args, kwargs, lambda: fn(self, *args, **kwargs), group
)
return wrapped
def cached_vae_encode(fn):
"""Cache the posterior, before sample()/mode() and latent normalization.
Distributed VAEs currently bypass this cache: their encoding methods can
run on an owner rank or a model-specific subgroup, unlike encoder TP.
"""
@wraps(fn)
def wrapped(self, *args, **kwargs):
cache = _inference_cache(self)
if cache is None or (model_parallel_is_initialized() and get_world_size() > 1):
return fn(self, *args, **kwargs)
# Tiling and slicing can change numerical results without changing x.
settings = {
name: value
for name, value in vars(self).items()
if name.startswith(
(
"use_",
"tile_",
"blend_",
"parallel_",
"spatial_compression",
"temporal_compression",
)
)
and type(value) in (int, float, bool, str, type(None))
}
return cache.run(
self,
fn.__name__,
(args, settings),
kwargs,
lambda: fn(self, *args, **kwargs),
)
return wrapped
class ConditioningEncoderMixin:
"""Cache replicated encoders without adding a module or altering state_dict."""
def __call__(self, *args, **kwargs):
forward = super().__call__
return cached_encoder_call(self, args, kwargs, lambda: forward(*args, **kwargs))
@@ -456,6 +456,14 @@ class ComponentResidencyManager:
"""Prepare a shared component and wait without making it the active use."""
self._prepare_forward_use(use, module=module)
def finish_unused_component(
self, use: ComponentUse, module: nn.Module | None = None
) -> None:
"""Release retained weights when conditioning reuse skips their use."""
if self._active_use is not None and self._same_use(self._active_use, use):
return
self._finish_use(use, module=module, keep_on_warmup=False, force=True)
def remove_nvtx_hooks_for_module(self, module: nn.Module | None) -> None:
"""Detach NVTX hooks before a component object is deleted or replaced."""
if module is None:
@@ -21,6 +21,7 @@ from sglang.kernels.ops.diffusion import (
)
from sglang.multimodal_gen.configs.models.dits.cosmos3video import Cosmos3VideoConfig
from sglang.multimodal_gen.configs.models.fsdp import is_module_list_entry_in
from sglang.multimodal_gen.runtime.cache.conditioning import cached_conditioning
from sglang.multimodal_gen.runtime.distributed import (
get_sp_group,
get_sp_world_size,
@@ -1099,6 +1100,7 @@ class Cosmos3LanguageModel(nn.Module):
]
)
@cached_conditioning
def forward(
self,
text_ids: torch.Tensor,
@@ -13,11 +13,13 @@ from sglang.multimodal_gen.configs.models.encoders import (
ImageEncoderConfig,
TextEncoderConfig,
)
from sglang.multimodal_gen.runtime.cache.conditioning import cached_encoder_call
from sglang.multimodal_gen.runtime.distributed import (
get_replica_group,
get_sp_group,
get_tp_group,
get_world_group,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.distributed.group_coordinator import GroupCoordinator
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
@@ -178,10 +180,20 @@ class EncoderTensorParallelMixin:
def __call__(self, *args, **kwargs):
tp_group = self._encoder_tp_group
forward = super().__call__
cache_group = tp_group
if cache_group is None and model_parallel_is_initialized():
cache_group = get_tp_group()
def run():
return cached_encoder_call(
self, args, kwargs, lambda: forward(*args, **kwargs), cache_group
)
if tp_group is None:
return super().__call__(*args, **kwargs)
return run()
with use_tensor_parallel_group(tp_group):
return super().__call__(*args, **kwargs)
return run()
class TextEncoder(
@@ -11,6 +11,7 @@ from torch import nn
from sglang.multimodal_gen.configs.models.encoders.base import BaseEncoderOutput
from sglang.multimodal_gen.configs.models.encoders.gemma_3 import Gemma3Config
from sglang.multimodal_gen.runtime.cache.conditioning import cached_conditioning
from sglang.multimodal_gen.runtime.distributed import get_tp_world_size
from sglang.multimodal_gen.runtime.layers.activation import GeluAndMul
from sglang.multimodal_gen.runtime.layers.linear import (
@@ -772,8 +773,7 @@ class Gemma3ForConditionalGeneration(
elif pixel_values.dim() != 4:
raise ValueError(f"Unexpected pixel_values shape: {pixel_values.shape}")
vision_outputs = self.vision_tower(pixel_values)
image_features = self.multi_modal_projector(vision_outputs)
image_features = self._encode_image_features(pixel_values)
image_features = image_features.to(
device=inputs_embeds.device, dtype=inputs_embeds.dtype
)
@@ -788,6 +788,10 @@ class Gemma3ForConditionalGeneration(
llm_input_ids, inputs_embeds=inputs_embeds, **kwargs
)
@cached_conditioning
def _encode_image_features(self, pixel_values):
return self.multi_modal_projector(self.vision_tower(pixel_values))
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]) -> Set[str]:
loaded_params: Set[str] = set()
params_dict = dict(self.named_parameters())
@@ -11,6 +11,7 @@ from transformers import (
Dinov2Model,
)
from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningEncoderMixin
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
)
@@ -32,7 +33,9 @@ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
return np.concatenate([emb_sin, emb_cos], axis=1)
class ImageEncoder(nn.Module, LayerwiseOffloadableModuleMixin):
class ImageEncoder(
ConditioningEncoderMixin, nn.Module, LayerwiseOffloadableModuleMixin
):
layerwise_offload_dit_group_enabled = False
layer_names = [
"model.encoder.layer",
@@ -212,7 +215,9 @@ def build_image_encoder(config):
raise ValueError(f"Unknown image encoder type: {config['type']}")
class DualImageEncoder(nn.Module, LayerwiseOffloadableModuleMixin):
class DualImageEncoder(
ConditioningEncoderMixin, nn.Module, LayerwiseOffloadableModuleMixin
):
layerwise_offload_dit_group_enabled = False
layer_names = [
"main_image_encoder.model.encoder.layer",
@@ -249,7 +254,9 @@ class DualImageEncoder(nn.Module, LayerwiseOffloadableModuleMixin):
return outputs
class SingleImageEncoder(nn.Module, LayerwiseOffloadableModuleMixin):
class SingleImageEncoder(
ConditioningEncoderMixin, nn.Module, LayerwiseOffloadableModuleMixin
):
layerwise_offload_dit_group_enabled = False
layer_names = [
"main_image_encoder.model.encoder.layer",
@@ -81,6 +81,8 @@ from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import (
Qwen2_5_VLModelOutputWithPast,
)
from sglang.multimodal_gen.runtime.cache.conditioning import cached_conditioning
logger = logging.getLogger(__name__)
@@ -911,6 +913,7 @@ class Qwen2_5_VLModel(nn.Module):
video_embeds = torch.split(video_embeds, split_sizes)
return video_embeds
@cached_conditioning
def get_image_features(
self,
pixel_values: torch.FloatTensor,
@@ -57,6 +57,8 @@ import torch
import torch.nn as nn
from transformers.activations import ACT2FN
from sglang.multimodal_gen.runtime.cache.conditioning import cached_conditioning
logger = logging.getLogger(__name__)
from transformers.modeling_outputs import BaseModelOutputWithPast
@@ -910,6 +912,7 @@ class Qwen3VLModel(nn.Module):
# Same implementation as for images
return self.get_image_features(pixel_values_videos, video_grid_thw)
@cached_conditioning
def _get_flat_visual_features(
self,
pixel_values: torch.FloatTensor,
@@ -13,6 +13,8 @@ from transformers.modeling_outputs import CausalLMOutputWithPast
from transformers.modeling_utils import PreTrainedModel
from transformers.utils import logging
from sglang.multimodal_gen.runtime.cache.conditioning import cached_conditioning
from .configuration_neo_chat import NEOChatConfig, NEOMoELLMConfig
from .conversation import get_conv_template
from .modeling_fm_modules import (
@@ -484,12 +486,16 @@ class NEOChatModel(PreTrainedModel):
grid_hw=grid_hw,
).last_hidden_state
else:
return self.vision_model(
pixel_values=pixel_values,
output_hidden_states=False,
return_dict=True,
grid_hw=grid_hw,
).last_hidden_state
return self._reference_image_features(pixel_values, grid_hw)
@cached_conditioning
def _reference_image_features(self, pixel_values, grid_hw):
return self.vision_model(
pixel_values=pixel_values,
output_hidden_states=False,
return_dict=True,
grid_hw=grid_hw,
).last_hidden_state
def batch_chat(
self,
@@ -22,6 +22,7 @@ from diffusers.models.modeling_outputs import AutoencoderKLOutput
from torch import nn
from sglang.multimodal_gen.configs.models.vaes.base import VAEConfig
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
)
@@ -287,6 +288,7 @@ class AutoencoderKL(nn.Module, LayerwiseOffloadableModuleMixin):
return enc
@cached_vae_encode
def encode(
self, x: torch.Tensor, return_dict: bool = True
) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]:
@@ -7,6 +7,7 @@ from diffusers.models.autoencoders.vae import DecoderOutput
from torch import nn
from sglang.multimodal_gen.configs.models.vaes.sana import SanaVAEConfig
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_decode_parallel_rank,
get_decode_parallel_world_size,
@@ -135,6 +136,7 @@ class AutoencoderDC(nn.Module, LayerwiseOffloadableModuleMixin):
return next(self._inner_model.parameters()).device
return torch.device("cpu")
@cached_vae_encode
def encode(self, x: torch.Tensor, **kwargs):
self._ensure_inner_model()
return self._inner_model.encode(x, **kwargs)
@@ -19,6 +19,7 @@ from diffusers.models.autoencoders.vae import (
from diffusers.models.modeling_outputs import AutoencoderKLOutput
from sglang.multimodal_gen.configs.models.vaes.flux import Flux2VAEConfig
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
from sglang.multimodal_gen.runtime.models.vaes.common import (
ParallelTiledVAE,
can_install_spatial_shard_parallel_decode,
@@ -238,6 +239,7 @@ class AutoencoderKLFlux2(ParallelTiledVAE):
return enc
@cached_vae_encode
def encode(
self, x: torch.Tensor, return_dict: bool = True
) -> Union[DiagonalGaussianDistribution]:
@@ -14,6 +14,7 @@ from diffusers.models.autoencoders.vae import (
from diffusers.models.modeling_outputs import AutoencoderKLOutput
from sglang.multimodal_gen.configs.models.vaes.qwenimage import QwenImageVAEConfig
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_decode_parallel_rank,
@@ -1069,6 +1070,7 @@ class AutoencoderKLQwenImage(ParallelTiledVAE):
self.clear_cache()
return enc
@cached_vae_encode
def encode(
self, x: torch.Tensor, return_dict: bool = True
) -> DiagonalGaussianDistribution:
@@ -18,6 +18,10 @@ from sglang.multimodal_gen.configs.models import VAEConfig
from sglang.multimodal_gen.configs.models.vaes.base import (
should_use_spatial_shard_parallel_decode,
)
from sglang.multimodal_gen.runtime.cache.conditioning import (
cached_vae_encode,
register_conditioning_container,
)
from sglang.multimodal_gen.runtime.distributed import (
get_decode_parallel_group_coordinator,
get_decode_parallel_world_size,
@@ -137,6 +141,7 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
def _decode(self, *args, **kwargs) -> torch.Tensor:
pass
@cached_vae_encode
def encode(self, x: torch.Tensor) -> DiagonalGaussianDistribution:
batch_size, num_channels, num_frames, height, width = x.shape
latent_num_frames = (num_frames - 1) // self.temporal_compression_ratio + 1
@@ -748,6 +753,7 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
# adapted from https://github.com/huggingface/diffusers/blob/e7ffeae0a191f710881d1fbde00cd6ff025e81f2/src/diffusers/models/autoencoders/vae.py#L691
@register_conditioning_container
class DiagonalGaussianDistribution:
def __init__(self, parameters: torch.Tensor, deterministic: bool = False):
self.parameters = parameters
@@ -12,6 +12,7 @@ from einops import rearrange
from torch import nn
from sglang.multimodal_gen.configs.models.vaes.dac import DacVAEConfig
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
)
@@ -506,6 +507,7 @@ class DAC(nn.Module, LayerwiseOffloadableModuleMixin):
return audio_data
@cached_vae_encode
def encode(
self,
audio_data: torch.Tensor,
@@ -3,6 +3,7 @@ from typing import Any
import torch
import torch.nn as nn
from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningEncoderMixin
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
)
@@ -113,7 +114,9 @@ def _make_ltx23_encoder_block(
)
class LTX23VideoConditionEncoder(nn.Module, LayerwiseOffloadableModuleMixin):
class LTX23VideoConditionEncoder(
ConditioningEncoderMixin, nn.Module, LayerwiseOffloadableModuleMixin
):
layerwise_offload_dit_group_enabled = False
layer_names = ["down_blocks"]
@@ -10,6 +10,7 @@ from diffusers.models.modeling_outputs import AutoencoderKLOutput
from torch import nn
from sglang.multimodal_gen.configs.models.vaes.ltx_audio import LTXAudioVAEConfig
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
from sglang.multimodal_gen.runtime.models.vaes.common import ParallelTiledVAE
LATENT_DOWNSAMPLE_FACTOR = 4
@@ -868,6 +869,7 @@ class AutoencoderKLLTX2Audio(ParallelTiledVAE):
def _encode(self, x: torch.Tensor) -> torch.Tensor:
return self.encoder(x)
@cached_vae_encode
def encode(self, x: torch.Tensor, return_dict: bool = True):
if self.use_slicing and x.shape[0] > 1:
encoded_slices = [self._encode(x_slice) for x_slice in x.split(1)]
@@ -12,6 +12,7 @@ from diffusers.models.embeddings import PixArtAlphaCombinedTimestepSizeEmbedding
from diffusers.models.modeling_outputs import AutoencoderKLOutput
from sglang.multimodal_gen.configs.models.vaes.ltx_video import LTXVideoVAEConfig
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_decode_parallel_rank,
get_decode_parallel_world_size,
@@ -1857,6 +1858,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
return enc
@cached_vae_encode
def encode(
self, x: torch.Tensor, causal: Optional[bool] = None, return_dict: bool = True
) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]:
@@ -14,6 +14,7 @@ from diffusers.models import ModelMixin
from diffusers.utils import logging
from PIL import Image
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
from sglang.multimodal_gen.runtime.distributed import (
get_decode_parallel_group_coordinator,
get_decode_parallel_rank,
@@ -1281,6 +1282,7 @@ class AutoencoderKLLegacy(AutoencoderKL):
self.decoder = ViT3DDecoder(**vit_kwargs)
@torch.no_grad()
@cached_vae_encode
def encode(self, x):
return self.quant_conv(self.encoder(x))
@@ -29,6 +29,7 @@ from sglang.multimodal_gen.configs.models.vaes import WanVAEConfig
from sglang.multimodal_gen.configs.models.vaes.base import (
should_use_spatial_shard_parallel_decode,
)
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_decode_parallel_rank,
get_decode_parallel_world_size,
@@ -1585,6 +1586,7 @@ class AutoencoderKLWan(ParallelTiledVAE):
self._causal_decode_initialized = True
return out
@cached_vae_encode
def encode(self, x: torch.Tensor) -> torch.Tensor:
if self.use_feature_cache:
self.clear_cache()
@@ -62,10 +62,6 @@ class SanaVideoTextEncodingStage(TextEncodingStage):
return [item.lower().strip() for item in text]
def _encode_negative_text(self, batch, server_args, all_indices):
cache_key = self._build_negative_text_cache_key(batch, server_args, all_indices)
cached = self._get_cached_negative_text_embedding(cache_key)
if cached is not None:
return cached
outputs = self.encode_text(
self._normalize_text(batch.negative_prompt),
server_args,
@@ -73,7 +69,6 @@ class SanaVideoTextEncodingStage(TextEncodingStage):
return_attention_mask=True,
max_length=300,
)
self._maybe_cache_negative_text_embedding(cache_key, outputs)
return outputs
@torch.no_grad()
@@ -12,6 +12,10 @@ from typing import TYPE_CHECKING, Any, Callable, List
import torch
from sglang.multimodal_gen.runtime.cache.conditioning import (
ConditioningCache,
conditioning_cache_group,
)
from sglang.multimodal_gen.runtime.distributed import get_world_rank
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
from sglang.multimodal_gen.runtime.platforms import current_platform
@@ -54,6 +58,11 @@ class PipelineExecutor(ABC):
def __init__(self, server_args):
self.server_args = server_args
self.component_residency_manager = None
self.conditioning_cache = ConditioningCache(
int(server_args.conditioning_cache_max_size_mb * 1024**2)
if not server_args.disable_conditioning_cache
else 0
)
def begin_component_residency_request(
self,
@@ -87,7 +96,11 @@ class PipelineExecutor(ABC):
):
self.begin_component_residency_request(stages, payload, server_args)
try:
yield
with self.conditioning_cache.scope(
cross_request=not server_args.use_fsdp_inference,
refresh=self._is_warmup_payload(payload),
):
yield
finally:
self.finish_component_residency_request()
@@ -117,9 +130,13 @@ class PipelineExecutor(ABC):
self.before_stage(stage, stage_index, payload, server_args)
with maybe_record_function(f"STAGE {stage_name}"):
with maybe_nvtx_range(f"stage_{stage_name}", use_nvtx):
payload = self.run_stage_with_context(
stage, payload, server_args, run_stage
)
with conditioning_cache_group(
enabled=(isinstance(payload, list) and len(payload) > 1)
or (isinstance(payload, Req) and payload.batch_size > 1)
):
payload = self.run_stage_with_context(
stage, payload, server_args, run_stage
)
return payload
@staticmethod
@@ -13,6 +13,9 @@ from safetensors.torch import load_file
from torch.distributed.tensor import DTensor
from sglang.multimodal_gen import envs
from sglang.multimodal_gen.runtime.cache.conditioning import (
invalidate_conditioning_caches,
)
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.layers.lora.linear import (
BaseLayerWithLoRA,
@@ -1036,6 +1039,14 @@ class LoRAPipeline(ComposedPipelineBase):
self.loaded_adapter_alphas[lora_nickname] = adapter_lora_alpha
logger.info("Rank %d: loaded LoRA adapter %s", rank, lora_path)
def _invalidate_lora_conditioning(self):
# LoRA targets DiTs; unrelated text and VAE weights remain unchanged
invalidate_conditioning_caches(
module
for name in ("transformer", "transformer_2", "fake_score_transformer")
if (module := self.modules.get(name)) is not None
)
def set_lora(
self,
lora_nickname: str | list[str],
@@ -1055,6 +1066,7 @@ class LoRAPipeline(ComposedPipelineBase):
stop costing anonymous host memory; pass it only for the startup
(static) adapter, where the merged combination is stable.
"""
self._invalidate_lora_conditioning()
merge_mode = self._resolve_lora_merge_mode(merge_weights, merge_mode)
# Normalize inputs to lists for multi-LoRA support
@@ -1303,6 +1315,7 @@ class LoRAPipeline(ComposedPipelineBase):
Disable LoRA for the specified target, regardless of whether weights were
merged into the base model or are still active in the wrapped LoRA path.
"""
self._invalidate_lora_conditioning()
target_modules, error = self._get_target_lora_layers(target)
if error:
logger.warning("deactivate_lora_weights: %s", error)
@@ -1340,6 +1353,7 @@ class LoRAPipeline(ComposedPipelineBase):
"transformer_2", "critic".
strength: LoRA strength for merge, default 1.0.
"""
self._invalidate_lora_conditioning()
target_modules, error = self._get_target_lora_layers(target)
if error:
logger.warning("merge_lora_weights: %s", error)
@@ -1412,6 +1426,7 @@ class LoRAPipeline(ComposedPipelineBase):
target: Which transformer(s) to unmerge. One of "all", "transformer",
"transformer_2", "critic".
"""
self._invalidate_lora_conditioning()
target_modules, error = self._get_target_lora_layers(target)
if error:
logger.warning("unmerge_lora_weights: %s", error)
@@ -287,6 +287,12 @@ class PipelineStage(StageDedupMixin, ABC):
)
self._component_residency_manager.begin_use(use, module=module)
def finish_unused_declared_component(self, *, component_name: str, module=None):
if self._component_residency_manager is None:
return
use = self._declared_component_use(component_name=component_name)
self._component_residency_manager.finish_unused_component(use, module=module)
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
@@ -8,6 +8,8 @@ from typing import TYPE_CHECKING, Any, ClassVar
import torch
from sglang.multimodal_gen.runtime.cache.conditioning import conditioning_cache_group
if TYPE_CHECKING:
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.server_args import ServerArgs
@@ -91,16 +93,19 @@ class StageDedupMixin:
results: list[Req | None] = [None] * len(batches)
for _, group in self._group_requests_by_fingerprint(
groups = self._group_requests_by_fingerprint(
batches, lambda batch: self.build_dedup_fingerprint(batch, server_args)
):
first_index, first_batch = group[0]
first_result = self(first_batch, server_args)
results[first_index] = first_result
)
# a single computed request needs no group memo or cache-hit collective
with conditioning_cache_group(enabled=len(groups) > 1):
for _, group in groups:
first_index, first_batch = group[0]
first_result = self(first_batch, server_args)
results[first_index] = first_result
for index, batch in group[1:]:
copy_outputs(first_result, batch)
results[index] = batch
for index, batch in group[1:]:
copy_outputs(first_result, batch)
results[index] = batch
return [result for result in results if result is not None]
@@ -117,6 +117,7 @@ class Ideogram4Scheduler:
class Ideogram4TextEncodingStage(TextEncodingStage):
deduplicated_output_fields = ("prompt_embeds", "prompt_embeds_mask")
deduplicated_extra_tensor_tree_output_keys = ("ideogram4",)
def __init__(self, text_encoder, tokenizer) -> None:
@@ -2,7 +2,13 @@
import torch
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.cache.conditioning import cached_encoder_call
from sglang.multimodal_gen.runtime.distributed import (
get_local_torch_device,
get_tp_group,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
TextEncodingStage,
@@ -35,6 +41,8 @@ VIDEO_PROMPT_TEMPLATE = "<|vision_start|><|video_pad|><|vision_end|>"
class LingBotVideoTextEncodingStage(TextEncodingStage):
"""Qwen3-VL prompt/negative encoding for LingBot-Video MoE (T2V, base)."""
deduplicated_output_fields = ()
def __init__(self, text_encoders, tokenizers, transformer):
super().__init__(text_encoders, tokenizers)
self.transformer = transformer
@@ -106,6 +114,32 @@ class LingBotVideoTextEncodingStage(TextEncodingStage):
)
inputs = self._build_prompt_inputs(prompt)
cache_group = (
text_encoder._encoder_tp_group
if isinstance(text_encoder, TextEncoder)
else None
)
if cache_group is None and model_parallel_is_initialized():
cache_group = get_tp_group()
return cached_encoder_call(
text_encoder,
(dict(inputs),),
{
"device": str(device),
"dtype": dtype,
"skip_layer": self.hidden_state_skip_layer,
"crop_start": self._compute_crop_start(),
},
lambda: self._encode_inputs(inputs, device, dtype),
cache_group,
namespace=self,
nested=False,
share_in_group=True,
)
def _encode_inputs(self, inputs, device, dtype):
self._begin_text_encoder_use(0)
text_encoder = self.text_encoders[0]
inputs = inputs.to(device)
outputs = self._forward_text_encoder(
text_encoder,
@@ -51,15 +51,7 @@ def _causal_block_count(batch: Req, server_args: ServerArgs) -> int:
class LongLive2TextEncodingStage(TextEncodingStage):
def build_dedup_fingerprint(self, batch: Req, server_args: ServerArgs):
base = super().build_dedup_fingerprint(batch, server_args)
return (
base,
self.freeze_for_dedup(getattr(batch, "shot_prompts", None)),
self.freeze_for_dedup(getattr(batch, "shot_durations", None)),
int(getattr(batch, "chunks_per_shot", 0) or 0),
getattr(batch, "scene_cut_prefix", None),
)
deduplicated_output_fields = ()
def _block_prompts(self, batch: Req, server_args: ServerArgs) -> list[str] | None:
shot_prompts = getattr(batch, "shot_prompts", None)
@@ -15,8 +15,13 @@ from transformers.models.qwen2_vl.image_processing_pil_qwen2_vl import (
from sglang.multimodal_gen.configs.pipeline_configs.ming_image import (
MingImageLayerPipelineConfig,
)
from sglang.multimodal_gen.runtime.cache.conditioning import cached_encoder_call
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.distributed import (
get_local_torch_device,
get_tp_group,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
@@ -31,6 +36,8 @@ from sglang.multimodal_gen.runtime.utils.vision import load_image
class MingImageEncodingStage(TextEncodingStage):
deduplicated_output_fields = ()
def __init__(self, text_encoders, tokenizers):
super().__init__(text_encoders, tokenizers)
self.image_processor = Qwen2VLImageProcessorPil.from_pretrained(
@@ -131,13 +138,32 @@ class MingImageEncodingStage(TextEncodingStage):
input_ids=torch.tensor([ids], device=device),
position_ids=ming_position_ids(ids, grids, encoder.image_token).to(device),
)
self._begin_text_encoder_use(0)
# the official MoE router runs under BF16 autocast, including its logits
with torch.autocast(device.type, dtype=encoder.dtype):
encoded = self._forward_text_encoder(encoder, inputs)
batch.prompt_embeds = [encoded.last_hidden_state]
batch.negative_prompt_embeds = [torch.zeros_like(encoded.last_hidden_state)]
batch.extra["ming_direct"] = encoded.hidden_states[0]
def encode_conditioning():
self._begin_text_encoder_use(0)
# the official MoE router runs under BF16 autocast, including its logits
with torch.autocast(device.type, dtype=encoder.dtype):
encoded = self._forward_text_encoder(encoder, inputs)
return encoded.last_hidden_state, encoded.hidden_states[0]
cache_group = encoder._encoder_tp_group
if cache_group is None and model_parallel_is_initialized():
cache_group = get_tp_group()
prompt_embeds, direct = cached_encoder_call(
encoder,
(inputs,),
{},
encode_conditioning,
cache_group,
namespace=self,
share_in_group=True,
)
self.finish_unused_declared_component(
component_name="text_encoder", module=encoder
)
batch.prompt_embeds = [prompt_embeds]
batch.negative_prompt_embeds = [torch.zeros_like(prompt_embeds)]
batch.extra["ming_direct"] = direct
return batch
@@ -4,7 +4,12 @@ import math
import torch
from PIL import Image
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.cache.conditioning import cached_encoder_call
from sglang.multimodal_gen.runtime.distributed import (
get_local_torch_device,
get_tp_group,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
@@ -83,6 +88,24 @@ class QwenImage21EncodingStage(PipelineStage):
]
def encode_prompt(self, prompt, images, device):
cache_group = self.text_encoder._encoder_tp_group
if cache_group is None and model_parallel_is_initialized():
cache_group = get_tp_group()
conditioning = cached_encoder_call(
self.text_encoder,
(prompt, images),
{"device": str(device)},
lambda: self._encode_prompt(prompt, images, device),
cache_group,
namespace=self,
share_in_group=True,
)
self.finish_unused_declared_component(
component_name="text_encoder", module=self.text_encoder
)
return conditioning
def _encode_prompt(self, prompt, images, device):
prefix = " ".join(
f"<image{i + 1}><|vision_start|><|image_pad|><|vision_end|>"
for i in range(len(images))
@@ -637,6 +637,8 @@ class SanaWMTextEncodingStage(TextEncodingStage):
model-specific prompt-window contract.
"""
deduplicated_output_fields = ()
@staticmethod
def _text_encoder_max_length(server_args: ServerArgs) -> int:
encoder_cfg = server_args.pipeline_config.text_encoder_configs[0]
@@ -16,6 +16,7 @@ from typing import Any
import torch
from sglang.multimodal_gen.runtime.cache.conditioning import conditioning_weights_epoch
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
@@ -109,6 +110,8 @@ class RealtimeTextState(BaseRealtimeState):
class RealtimeTextEncodingStage(TextEncodingStage):
"""Cache text encoder outputs across realtime chunks by prompt identity."""
deduplicated_output_fields = ()
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
@@ -120,6 +123,7 @@ class RealtimeTextEncodingStage(TextEncodingStage):
def _make_cache_key(self, batch: Req) -> tuple[Any, ...]:
return (
conditioning_weights_epoch(),
_normalize_prompt_value(batch.prompt),
bool(batch.do_classifier_free_guidance),
(
@@ -147,7 +151,11 @@ class RealtimeTextEncodingStage(TextEncodingStage):
batch: Req,
server_args: ServerArgs,
) -> Req:
if batch.session is None:
if (
batch.session is None
or server_args.disable_conditioning_cache
or server_args.conditioning_cache_max_size_mb == 0
):
return super().forward(batch, server_args)
state = batch.session.get_or_create_state(RealtimeTextState)
@@ -8,17 +8,21 @@ This module contains implementations of prompt encoding stages for diffusion pip
"""
import inspect
from dataclasses import dataclass
from functools import lru_cache
from typing import Any
import torch
from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput
from sglang.multimodal_gen.configs.pipeline_configs.base import TextConditioningOutput
from sglang.multimodal_gen.runtime.cache.conditioning import (
cached_encoder_call,
prefer_conditioning_cache,
)
from sglang.multimodal_gen.runtime.distributed import (
get_encoder_data_parallel_group,
get_local_torch_device,
get_replica_group,
get_tp_group,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
@@ -102,27 +106,6 @@ def _data_parallel_text_encode(forward_fn, forward_kwargs: dict, group):
)
@lru_cache(maxsize=1)
def get_model_default_negative_prompt(
model_path: str, backend: Any, model_id: str | None
):
from sglang.multimodal_gen.registry import get_model_info
model_info = get_model_info(model_path, backend=backend, model_id=model_id)
if model_info is None:
return None
return model_info.sampling_param_cls().negative_prompt
@dataclass(frozen=True)
class TextEncodingFingerprint:
prompt: Any
negative_prompt: Any
do_classifier_free_guidance: bool
prompt_template: Any
max_sequence_length: int | None
def stack_tensors(name: str, tensors: list[torch.Tensor]) -> torch.Tensor:
base_shape = list(tensors[0].shape)
for tensor in tensors[1:]:
@@ -157,6 +140,26 @@ class TextEncodingStage(ConditionEncodingStage):
"is_prompt_processed",
)
def build_dedup_fingerprint(self, batch: Req, server_args: ServerArgs):
return (
self.freeze_for_dedup(batch.prompt),
self.freeze_for_dedup(batch.negative_prompt),
bool(batch.do_classifier_free_guidance),
self.freeze_for_dedup(batch.prompt_template),
batch.max_sequence_length,
)
@classmethod
def copy_stage_output(cls, value):
# embeddings stay shared; nested metadata containers belong to each request
if isinstance(value, list):
return [cls.copy_stage_output(item) for item in value]
if isinstance(value, tuple):
return tuple(cls.copy_stage_output(item) for item in value)
if isinstance(value, dict):
return {key: cls.copy_stage_output(item) for key, item in value.items()}
return value
def __init__(self, text_encoders, tokenizers) -> None:
"""
Initialize the prompt encoding stage.
@@ -165,8 +168,6 @@ class TextEncodingStage(ConditionEncodingStage):
super().__init__()
self.tokenizers = tokenizers
self.text_encoders = text_encoders
self._negative_text_cache_key = None
self._negative_text_cache_value = None
self._dp_choice_logged = False
def component_uses(
@@ -178,6 +179,7 @@ class TextEncodingStage(ConditionEncodingStage):
stage_name=stage_name,
component_name="text_encoder" if i == 0 else f"text_encoder_{i + 1}",
preferred_ready_after_request=i == 0,
start_at_stage_entry=False,
target_dtype=resolve_component_precision_override(
server_args,
"text_encoder" if i == 0 else f"text_encoder_{i + 1}",
@@ -189,100 +191,14 @@ class TextEncodingStage(ConditionEncodingStage):
def get_or_compute_negative_text_embedding(
self, batch: Req, server_args: ServerArgs, all_indices: list[int]
):
"""Get the cached text embedding result or compute
this is a one-slot cache for the model-default negative prompt:
most requests don't override the negative prompt, the cache hit rate is considerably high
invariant: hit/miss must match across ranks -- a miss runs encode_text,
which may issue collectives (folding, dp encoding), so a split would
deadlock; keep any future eviction rank-global
"""
negative_cache_key = self._build_negative_text_cache_key(
batch, server_args, all_indices
)
cached_negative = self._get_cached_negative_text_embedding(negative_cache_key)
if cached_negative is not None:
return cached_negative
negative_text_outputs = self.encode_text(
batch.negative_prompt,
server_args,
encoder_index=all_indices,
return_attention_mask=True,
)
self._maybe_cache_negative_text_embedding(
negative_cache_key, negative_text_outputs
)
return negative_text_outputs
def _should_cache_negative_text_embedding(
self, batch: Req, server_args: ServerArgs
) -> bool:
if not batch.is_warmup:
return True
return self._uses_model_default_negative_prompt(batch, server_args)
def _get_cached_negative_text_embedding(self, negative_cache_key):
if negative_cache_key is None:
return None
if self._negative_text_cache_key == negative_cache_key:
return self._negative_text_cache_value
return None
def _maybe_cache_negative_text_embedding(
self,
negative_cache_key,
negative_text_outputs,
) -> None:
# skip caching if None
if negative_cache_key is None:
return
self._negative_text_cache_key = negative_cache_key
self._negative_text_cache_value = tuple(
tuple(value) for value in negative_text_outputs
)
def _build_negative_text_cache_key(
self, batch: Req, server_args: ServerArgs, encoder_indices: list[int]
):
"""if the current req doesn't worth caching, returns None"""
# skip if we don't cache for current req
if not self._should_cache_negative_text_embedding(batch, server_args):
return None
# Negative text encoding changes when the template or max length changes,
# even if the visible negative prompt string is the same.
return (
tuple(encoder_indices),
self.freeze_for_dedup(batch.negative_prompt),
self.freeze_for_dedup(batch.prompt_template),
batch.max_sequence_length,
)
def _uses_model_default_negative_prompt(
self, batch: Req, server_args: ServerArgs
) -> bool:
default_negative_prompt = self._get_model_default_negative_prompt(server_args)
if default_negative_prompt is None:
return False
return self._normalize_negative_prompt_for_default_match(
batch.negative_prompt
) == self._normalize_negative_prompt_for_default_match(default_negative_prompt)
def _get_model_default_negative_prompt(self, server_args: ServerArgs) -> str | None:
return get_model_default_negative_prompt(
server_args.model_path,
server_args.backend,
server_args.model_id,
)
@staticmethod
def _normalize_negative_prompt_for_default_match(value):
if isinstance(value, str) and not value.isspace():
return value.strip()
return value
"""Encode negative conditioning through the shared encoder cache."""
with prefer_conditioning_cache():
return self.encode_text(
batch.negative_prompt,
server_args,
encoder_index=all_indices,
return_attention_mask=True,
)
def _append_positive_text_outputs(
self,
@@ -462,17 +378,6 @@ class TextEncodingStage(ConditionEncodingStage):
return batch
def build_dedup_fingerprint(
self, batch: Req, server_args: ServerArgs
) -> TextEncodingFingerprint:
return TextEncodingFingerprint(
prompt=self.freeze_for_dedup(batch.prompt),
negative_prompt=self.freeze_for_dedup(batch.negative_prompt),
do_classifier_free_guidance=bool(batch.do_classifier_free_guidance),
prompt_template=self.freeze_for_dedup(batch.prompt_template),
max_sequence_length=batch.max_sequence_length,
)
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
"""Verify text encoding stage inputs."""
result = VerificationResult()
@@ -696,131 +601,185 @@ class TextEncodingStage(ConditionEncodingStage):
text_inputs: dict = server_args.pipeline_config.tokenize_prompt(
processed_text_list, tokenizer, tok_kwargs
).to(target_device)
input_ids = text_inputs["input_ids"]
attention_mask = (
server_args.pipeline_config.get_text_encoder_attention_mask(
text_inputs, i
)
)
encoder_forward_kwargs = {
"input_ids": input_ids,
"output_hidden_states": True,
}
if attention_mask is not None:
encoder_forward_kwargs["attention_mask"] = attention_mask
if "use_cache" in inspect.signature(text_encoder.forward).parameters:
encoder_forward_kwargs["use_cache"] = False
self._begin_text_encoder_use(i)
# match host tokens before preparing weights or uploading encoder inputs
host_inputs = text_inputs
cache_inputs = dict(host_inputs)
dp_group = self._text_encode_dp_group(
server_args, encoder_config, input_ids.shape[0], text_encoder
server_args,
encoder_config,
host_inputs["input_ids"].shape[0],
text_encoder,
)
if dp_group is not None:
outputs = _data_parallel_text_encode(
lambda kw: self._forward_text_encoder(text_encoder, kw),
encoder_forward_kwargs,
dp_group,
)
else:
outputs = self._forward_text_encoder(
text_encoder, encoder_forward_kwargs
)
postprocess_sig = inspect.signature(postprocess_func)
postprocess_kwargs = {}
if "pipeline_config" in postprocess_sig.parameters:
# required by models like LTX
postprocess_kwargs["pipeline_config"] = server_args.pipeline_config
if "return_attention_mask" in postprocess_sig.parameters:
postprocess_kwargs["return_attention_mask"] = return_attention_mask
postprocess_result = postprocess_func(
outputs, text_inputs, **postprocess_kwargs
)
prompt_embeds_mask = None
prompt_seq_lens = None
if isinstance(postprocess_result, TextConditioningOutput):
prompt_embeds = postprocess_result.prompt_embeds
prompt_embeds_mask = postprocess_result.prompt_embeds_mask
prompt_seq_lens = postprocess_result.prompt_seq_lens
elif isinstance(postprocess_result, tuple):
if len(postprocess_result) != 2:
raise ValueError(
"Text postprocess tuple output must be (prompt_embeds, prompt_embeds_mask)"
def encode_conditioning():
text_inputs = host_inputs.to(target_device, non_blocking=True)
input_ids = text_inputs["input_ids"]
attention_mask = (
server_args.pipeline_config.get_text_encoder_attention_mask(
text_inputs, i
)
prompt_embeds, prompt_embeds_mask = postprocess_result
else:
prompt_embeds = postprocess_result
if dtype is not None:
prompt_embeds = prompt_embeds.to(device=target_device, dtype=dtype)
else:
prompt_embeds = prompt_embeds.to(device=target_device)
if prompt_embeds_mask is not None:
prompt_embeds_mask = prompt_embeds_mask.to(
device=target_device, dtype=torch.bool
)
encoder_forward_kwargs = {
"input_ids": input_ids,
"output_hidden_states": True,
}
if attention_mask is not None:
encoder_forward_kwargs["attention_mask"] = attention_mask
if "use_cache" in inspect.signature(text_encoder.forward).parameters:
encoder_forward_kwargs["use_cache"] = False
self._begin_text_encoder_use(i)
postprocess_sig = inspect.signature(postprocess_func)
embeds_list.append(prompt_embeds)
postprocess_kwargs = {}
if "pipeline_config" in postprocess_sig.parameters:
# required by models like LTX
postprocess_kwargs["pipeline_config"] = server_args.pipeline_config
if "return_attention_mask" in postprocess_sig.parameters:
postprocess_kwargs["return_attention_mask"] = return_attention_mask
pooled_output = server_args.pipeline_config.get_text_encoder_pooler_output(
outputs, i
)
if pooled_output is not None:
pooled_embeds_list.append(pooled_output.to(device=target_device))
if return_attention_mask:
if prompt_embeds_mask is not None:
mask_to_store = prompt_embeds_mask.to(
device=target_device,
dtype=(
attention_mask.dtype
if attention_mask is not None
else torch.long
),
if dp_group is not None:
outputs = _data_parallel_text_encode(
lambda kw: self._forward_text_encoder(text_encoder, kw),
encoder_forward_kwargs,
dp_group,
)
elif attention_mask is not None and list(attention_mask.shape) == list(
prompt_embeds.shape[:2]
):
mask_to_store = attention_mask.to(device=target_device)
else:
mask_to_store = torch.ones(
prompt_embeds.shape[:2],
device=target_device,
dtype=(
attention_mask.dtype
if attention_mask is not None
else torch.long
),
outputs = self._forward_text_encoder(
text_encoder, encoder_forward_kwargs
)
attn_masks_list.append(mask_to_store)
embeds_mask = prompt_embeds_mask
if embeds_mask is None:
embeds_mask = (
server_args.pipeline_config.build_text_conditioning_mask(
text_inputs,
attention_mask,
prompt_embeds,
i,
postprocess_result = postprocess_func(
outputs, text_inputs, **postprocess_kwargs
)
pooled_output = (
server_args.pipeline_config.get_text_encoder_pooler_output(
outputs, i
)
)
prompt_embeds_mask = None
prompt_seq_lens = None
if isinstance(postprocess_result, TextConditioningOutput):
prompt_embeds = postprocess_result.prompt_embeds
prompt_embeds_mask = postprocess_result.prompt_embeds_mask
prompt_seq_lens = postprocess_result.prompt_seq_lens
elif isinstance(postprocess_result, tuple):
if len(postprocess_result) != 2:
raise ValueError(
"Text postprocess tuple output must be (prompt_embeds, prompt_embeds_mask)"
)
prompt_embeds, prompt_embeds_mask = postprocess_result
else:
prompt_embeds = postprocess_result
if dtype is not None:
prompt_embeds = prompt_embeds.to(device=target_device, dtype=dtype)
else:
prompt_embeds = prompt_embeds.to(device=target_device)
if prompt_embeds_mask is not None:
prompt_embeds_mask = prompt_embeds_mask.to(
device=target_device, dtype=torch.bool
)
embeds_masks_list.append(embeds_mask)
if prompt_seq_lens is not None:
seq_lens_list.append([int(x) for x in prompt_seq_lens])
elif embeds_mask is not None:
seq_lens_list.append(
server_args.pipeline_config.seq_lens_from_text_conditioning_mask(
if pooled_output is not None:
pooled_output = pooled_output.to(device=target_device)
mask_to_store = embeds_mask = seq_lens = None
if return_attention_mask:
if prompt_embeds_mask is not None:
mask_to_store = prompt_embeds_mask.to(
device=target_device,
dtype=(
attention_mask.dtype
if attention_mask is not None
else torch.long
),
)
elif attention_mask is not None and list(
attention_mask.shape
) == list(prompt_embeds.shape[:2]):
mask_to_store = attention_mask.to(device=target_device)
else:
mask_to_store = torch.ones(
prompt_embeds.shape[:2],
device=target_device,
dtype=(
attention_mask.dtype
if attention_mask is not None
else torch.long
),
)
embeds_mask = prompt_embeds_mask
if embeds_mask is None:
embeds_mask = (
server_args.pipeline_config.build_text_conditioning_mask(
text_inputs,
attention_mask,
prompt_embeds,
i,
)
)
if prompt_seq_lens is not None:
seq_lens = [int(x) for x in prompt_seq_lens]
elif embeds_mask is not None:
seq_lens = server_args.pipeline_config.seq_lens_from_text_conditioning_mask(
embeds_mask
)
)
elif prompt_embeds.ndim == 2:
seq_lens_list.append([int(prompt_embeds.shape[0])])
else:
seq_lens_list.append(
[int(prompt_embeds.shape[1])] * int(prompt_embeds.shape[0])
)
elif prompt_embeds.ndim == 2:
seq_lens = [int(prompt_embeds.shape[0])]
else:
seq_lens = [int(prompt_embeds.shape[1])] * int(
prompt_embeds.shape[0]
)
return (
prompt_embeds,
mask_to_store,
pooled_output,
embeds_mask,
seq_lens,
)
native_encoder = isinstance(text_encoder, TextEncoder)
cache_group = text_encoder._encoder_tp_group if native_encoder else None
if model_parallel_is_initialized():
if dp_group is not None:
# agree across encoder copies before skipping their gather
cache_group = get_replica_group()
elif cache_group is None:
cache_group = get_tp_group()
# library and batch-DP positives retain consumed outputs only within
# the group; preferred negatives also use the bounded device cache
conditioning = cached_encoder_call(
text_encoder,
(cache_inputs,),
{
"encoder_index": i,
"return_attention_mask": return_attention_mask,
"device": str(target_device),
"dtype": dtype,
},
encode_conditioning,
cache_group,
namespace=self,
nested=dp_group is not None,
share_in_group=True,
cross_request=native_encoder and dp_group is None,
)
# a hit can leave weights retained by warmup or the previous request
self.finish_unused_declared_component(
component_name="text_encoder" if i == 0 else f"text_encoder_{i + 1}",
module=text_encoder,
)
prompt_embeds, mask, pooled_output, embeds_mask, seq_lens = conditioning
embeds_list.append(prompt_embeds)
if pooled_output is not None:
pooled_embeds_list.append(pooled_output)
if return_attention_mask:
attn_masks_list.append(mask)
embeds_masks_list.append(embeds_mask)
seq_lens_list.append(seq_lens)
# Shape results according to return_type
if return_type == "list":
@@ -8,6 +8,7 @@ from typing import Any
import numpy as np
import torch
from sglang.multimodal_gen.runtime.cache.conditioning import conditioning_weights_epoch
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import (
OutputBatch,
@@ -84,9 +85,11 @@ def _effective_prefix_cache_enabled(
server_args: ServerArgs,
) -> bool:
options = vla_options(batch)
return bool(options.get("enable_prefix_cache", True)) and bool(
server_args.pipeline_config.enable_global_prefix_cache
)
return (
not server_args.disable_conditioning_cache
and server_args.conditioning_cache_max_size_mb > 0
and bool(options.get("enable_prefix_cache", True))
) and bool(server_args.pipeline_config.enable_global_prefix_cache)
def _cuda_graph_enabled(batch: Req) -> bool:
@@ -264,6 +267,7 @@ class VLAPrefixEncodingStage(PipelineStage):
observation,
bucket_prompt=_cuda_graph_enabled(batch),
)
cache_key = f"{conditioning_weights_epoch()}:{cache_key}"
cached_context = self.prefix_cache.get(cache_key)
else:
cache_key = None
@@ -46,6 +46,9 @@ from typing import Any
import torch
from torch.distributed.tensor import DTensor, distribute_tensor
from sglang.multimodal_gen.runtime.cache.conditioning import (
invalidate_conditioning_caches,
)
from sglang.multimodal_gen.runtime.cache.teacache import TeaCacheMixin
from sglang.multimodal_gen.runtime.loader.utils import (
_list_safetensors_files,
@@ -369,6 +372,7 @@ class WeightsUpdater:
target_modules: list[str] | None = None,
) -> tuple[bool, str]:
"""Update model weights from disk without restarting the server."""
invalidate_conditioning_caches()
logger.info(f"Updating weights from disk: {model_path}")
try:
@@ -531,6 +535,7 @@ class WeightsUpdater:
lora_alpha: int | None = None,
lora_rank: int | None = None,
) -> tuple[bool, str]:
invalidate_conditioning_caches()
if weight_update_mode == LORA_MERGE_WEIGHT_UPDATE_MODE:
return self._update_lora_from_tensor(
named_tensors=named_tensors,
@@ -609,6 +609,8 @@ class ServerArgs(DisaggServerArgsMixin):
log_requests_target: Optional[List[str]] = None
uvicorn_access_log_exclude_prefixes: list[str] = field(default_factory=list)
enable_cache_report: bool = False
disable_conditioning_cache: bool = False
conditioning_cache_max_size_mb: float = 512.0
# Tracing
enable_trace: bool = False
@@ -1932,6 +1934,10 @@ class ServerArgs(DisaggServerArgsMixin):
raise ValueError(f"Could not parse attention backend config: {config_str}")
def __post_init__(self):
if not 0 <= self.conditioning_cache_max_size_mb < float("inf"):
raise ValueError(
"conditioning_cache_max_size_mb must be finite and nonnegative"
)
if not self._explicit_arg_names:
self._explicit_arg_names = _infer_direct_constructor_explicit_arg_names(
self
@@ -3026,6 +3032,18 @@ class ServerArgs(DisaggServerArgsMixin):
default=ServerArgs.enable_cache_report,
help="Return number of cached tokens in usage.prompt_tokens_details for each OpenAI-compatible request.",
)
parser.add_argument(
"--disable-conditioning-cache",
action="store_true",
default=ServerArgs.disable_conditioning_cache,
help="Disable cross-request text/image and VAE posterior caching; reuse within a grouped stage remains enabled.",
)
parser.add_argument(
"--conditioning-cache-max-size-mb",
type=float,
default=ServerArgs.conditioning_cache_max_size_mb,
help="Per-worker conditioning cache capacity across CPU and device entries in MiB (default: 512; 0 disables cross-request caching).",
)
parser.add_argument(
"--backend",
type=str,
@@ -0,0 +1,611 @@
# SPDX-License-Identifier: Apache-2.0
import sys
from contextlib import nullcontext
from datetime import timedelta
from types import SimpleNamespace
import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution
from diffusers.models.modeling_outputs import AutoencoderKLOutput
from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput
from sglang.multimodal_gen.runtime.cache.conditioning import (
ConditioningCache,
cached_conditioning,
cached_encoder_call,
cached_vae_encode,
invalidate_conditioning_caches,
prefer_conditioning_cache,
)
from sglang.multimodal_gen.runtime.models.encoders.base import (
EncoderTensorParallelMixin,
)
class Encoder(EncoderTensorParallelMixin, torch.nn.Module):
def __init__(self):
super().__init__()
self.weight = torch.nn.Parameter(torch.ones(()))
self.calls = 0
def forward(self, x, attention_mask=None, **kwargs):
self.calls += 1
result = x * self.weight
if attention_mask is not None:
result = result * attention_mask
return BaseEncoderOutput(last_hidden_state=result, hidden_states=(result,))
class VAE(torch.nn.Module):
def __init__(self):
super().__init__()
self.calls = 0
self.use_tiling = False
@cached_vae_encode
def encode(self, x):
self.calls += 1
return AutoencoderKLOutput(latent_dist=DiagonalGaussianDistribution(x.clone()))
class VisionLanguageEncoder(Encoder):
def __init__(self):
super().__init__()
self.vision_calls = 0
@cached_conditioning
def image_features(self, pixels):
self.vision_calls += 1
return pixels * 2
def forward(self, text, pixels):
self.calls += 1
return self.image_features(pixels) + text
@torch.no_grad()
def test_changed_instruction_reuses_vision_but_not_joint_embedding():
cache = ConditioningCache(4096)
model = VisionLanguageEncoder().eval()
pixels = torch.ones(4)
with cache.scope():
first = model(torch.ones(4), pixels)
changed = model(torch.full((4,), 2.0), pixels)
assert not torch.equal(first, changed)
assert model.calls == 2 and model.vision_calls == 1
torch.testing.assert_close(
model(torch.full((4,), 2.0), pixels), changed, rtol=0, atol=0
)
assert model.calls == 2
model(torch.ones(4), pixels + 1)
assert model.vision_calls == 2
@torch.no_grad()
def test_content_masks_model_identity_and_copy_isolation():
cache = ConditioningCache(1024 * 1024)
model = Encoder().eval()
x = torch.arange(1024, dtype=torch.float32).reshape(1, 4, 16, 16)
with cache.scope():
original = model(x)
expected = original.last_hidden_state.clone()
original.last_hidden_state.zero_()
hit = model(x.clone())
assert hit.hidden_states[0] is hit.last_hidden_state
torch.testing.assert_close(hit.last_hidden_state, expected, rtol=0, atol=0)
hit.hidden_states[0].zero_()
torch.testing.assert_close(model(x).last_hidden_state, expected, rtol=0, atol=0)
changed_alpha = x.clone()
changed_alpha[:, 3, 0, 0] += 1
model(changed_alpha)
model(x, attention_mask=torch.zeros_like(x))
other_model = Encoder().eval()
other_model(x)
assert model.calls == 3
assert other_model.calls == 1
assert cache.hits == 2
@torch.no_grad()
def test_lru_budget_and_oversized_outputs():
cache = ConditioningCache(32)
model = Encoder().eval()
with cache.scope():
for value in (1, 2, 1, 3, 2):
model(torch.full((4,), float(value)))
before = cache.bytes
model(torch.ones(100))
assert model.calls == 5 # one LRU hit, then an oversized miss
assert cache.hits == 1
assert cache.evictions == 2
assert cache.bytes == before == 32
assert cache.bypasses == 1
@torch.no_grad()
def test_device_hot_promotion_shares_budget_and_eviction_with_host_entries():
cache = ConditioningCache(32)
model = Encoder().eval()
namespace = Encoder().eval()
def encode(value):
return cached_encoder_call(
model,
(value,),
{},
lambda: model(torch.full((4,), value)),
namespace=namespace,
share_in_group=True,
)
with cache.scope():
encode(1.0)
encode(2.0)
assert cache.bytes == 32 and cache.stats()["device_bytes"] == 0
with prefer_conditioning_cache():
promoted = encode(1.0)
promoted.last_hidden_state.zero_()
assert cache.bytes == 32 and cache.stats()["device_bytes"] == 16
encode(3.0)
assert cache.bytes == cache.stats()["device_bytes"] == 32
encode(4.0) # a cold positive must not evict either preferred negative
assert cache.bytes == cache.stats()["device_bytes"] == 32
with prefer_conditioning_cache():
torch.testing.assert_close(encode(1.0).last_hidden_state, torch.ones(4))
encode(5.0) # replaces the least-recent negative within the same budget
assert cache.evictions == 2
assert cache.bytes == cache.stats()["device_bytes"] == 32
cache.clear()
assert cache.bytes == cache.stats()["device_bytes"] == 0
@torch.no_grad()
def test_warmup_executes_nested_encoders_and_seeds_serving_cache():
cache = ConditioningCache(4096)
model = VisionLanguageEncoder().eval()
text, pixels = torch.ones(4), torch.ones(4)
with cache.scope():
expected = model(text, pixels)
with cache.scope(refresh=True):
model(text, pixels)
model(text, pixels)
assert model.calls == model.vision_calls == 3
torch.testing.assert_close(model(text, pixels), expected, rtol=0, atol=0)
assert model.calls == model.vision_calls == 3
model(text + 1, pixels)
assert model.calls == 4 and model.vision_calls == 3
@pytest.mark.parametrize("bypass", ["disabled", "inactive", "training", "grad", "ar"])
def test_bypass(bypass):
cache = ConditioningCache(0 if bypass == "disabled" else 1024)
model = Encoder().eval()
if bypass == "training":
model.train()
kwargs = {"use_cache": True} if bypass == "ar" else {}
with (
torch.set_grad_enabled(bypass == "grad"),
cache.scope(enabled=bypass != "inactive"),
):
model(torch.ones(4), **kwargs)
model(torch.ones(4), **kwargs)
assert model.calls == 2
assert cache.hits == 0
@torch.no_grad()
def test_posterior_cache_preserves_rng_and_sampling():
cache = ConditioningCache(1024 * 1024)
vae = VAE().eval()
x = torch.linspace(-1, 1, 64).reshape(1, 4, 4, 4)
with cache.scope():
posterior = vae.encode(x).latent_dist
cold = posterior.sample(generator=torch.Generator().manual_seed(42))
posterior.mean.add_(100) # downstream normalization must not poison storage
generator = torch.Generator().manual_seed(42)
state = generator.get_state().clone()
hit = vae.encode(x).latent_dist
assert torch.equal(generator.get_state(), state)
warm = hit.sample(generator=generator)
torch.testing.assert_close(cold, warm, rtol=0, atol=0)
assert not torch.equal(
warm, hit.sample(generator=torch.Generator().manual_seed(43))
)
vae.use_tiling = True
vae.encode(x)
assert vae.calls == 2
@torch.no_grad()
def test_weight_invalidation_and_precision():
cache = ConditioningCache(1024)
model = Encoder().eval()
x = torch.ones(4)
with cache.scope():
model(x)
model(x)
invalidate_conditioning_caches()
model.weight.fill_(2)
assert torch.equal(model(x).last_hidden_state, x * 2)
model.to(torch.float64)
model(x)
assert model.calls == 3
assert cache.hits == 1
@torch.no_grad()
def test_scoped_invalidation_preserves_independent_encoders_and_shared_weights():
cache = ConditioningCache(1024)
independent = Encoder().eval()
nested = Encoder().eval()
alias = Encoder().eval()
alias.weight = nested.weight
transformer = torch.nn.ModuleList([nested])
x = torch.ones(4)
with cache.scope():
for model in (independent, nested, alias):
model(x)
invalidate_conditioning_caches([transformer])
nested.weight.fill_(2)
assert torch.equal(independent(x).last_hidden_state, x)
assert torch.equal(nested(x).last_hidden_state, x * 2)
assert torch.equal(alias(x).last_hidden_state, x * 2)
assert (independent.calls, nested.calls, alias.calls) == (1, 2, 2)
assert cache.bytes <= cache.max_bytes
@torch.no_grad()
def test_negative_conditioning_survives_positive_capacity_pressure():
model = Encoder().eval()
# only one entry fits, reproducing two LTX embeddings in the host budget
cache = ConditioningCache(24)
positive, negative = torch.ones(4), torch.zeros(4)
with cache.scope(refresh=True):
model(positive)
with prefer_conditioning_cache():
model(negative)
with cache.scope():
model(positive + 1)
with prefer_conditioning_cache():
actual = model(negative)
assert model.calls == 3
assert cache.hits == 1
assert cache.bytes == 16
assert cache.evictions == 1
torch.testing.assert_close(actual.last_hidden_state, negative)
# preferred entries still replace each other and remain bounded
with prefer_conditioning_cache():
model(negative - 1)
assert cache.bytes == 16
assert cache.evictions == 2
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA transfers")
@torch.no_grad()
def test_cuda_snapshot_waits_for_producing_stream_before_restore():
cache = ConditioningCache(32 * 1024 * 1024)
model = Encoder().eval()
stream = torch.cuda.Stream()
with torch.cuda.stream(stream):
value = torch.arange(1024 * 1024, device="cuda").reshape(1024, 1024)
output = BaseEncoderOutput(
last_hidden_state=value,
hidden_states=(value, value.T, value[::2, ::2]),
)
cache.run(model, "forward", (), {}, lambda: output)
value.zero_()
restored = cache.run(model, "forward", (), {}, lambda: pytest.fail("cache miss"))
expected = torch.arange(1024 * 1024).reshape(1024, 1024)
assert restored.last_hidden_state is restored.hidden_states[0]
for actual, reference in zip(
restored.hidden_states, (expected, expected.T, expected[::2, ::2]), strict=True
):
torch.testing.assert_close(actual.cpu(), reference, rtol=0, atol=0)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA transfers")
@torch.no_grad()
def test_cuda_cache_hit_does_not_wait_for_unrelated_gpu_work():
cache = ConditioningCache(4096)
model = Encoder().eval()
value = torch.arange(64, device="cuda")
cache.run(model, "forward", (), {}, lambda: value)
torch.cuda.synchronize()
# a negative-prompt hit must not synchronize the preceding positive encode
torch.cuda._sleep(250_000_000)
pending = torch.cuda.Event()
pending.record()
restored = cache.run(model, "forward", (), {}, lambda: pytest.fail("cache miss"))
assert not pending.query()
torch.testing.assert_close(restored, value, rtol=0, atol=0)
@torch.no_grad()
def test_compiled_consumed_conditioning_bypasses_context_lookup():
model = torch.nn.Linear(2, 2).eval()
cache = ConditioningCache(1024)
@torch.compile(backend="eager", fullgraph=True)
def encode(x):
return cached_encoder_call(
model,
(x,),
{},
lambda: model(x),
namespace=model,
share_in_group=True,
)
with cache.scope(), prefer_conditioning_cache():
x = torch.ones(1, 2)
torch.testing.assert_close(encode(x), model(x), rtol=0, atol=0)
assert cache.stats()["entries"] == 0
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA graph capture")
@pytest.mark.parametrize("consumed", [False, True])
@torch.no_grad()
def test_cuda_graph_capture_bypasses_conditioning_cache(consumed):
cache = ConditioningCache(1024)
model = Encoder().cuda().eval()
x = torch.ones(4, device="cuda")
def encode():
return (
cached_encoder_call(
model,
(x,),
{},
lambda: model(x),
namespace=model,
share_in_group=True,
)
if consumed
else model(x)
)
with cache.scope(), prefer_conditioning_cache():
encode()
torch.cuda.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = encode()
x.fill_(3)
graph.replay()
torch.testing.assert_close(output.last_hidden_state, x, rtol=0, atol=0)
assert model.calls == 2
assert cache.hits == 0
def _rank_eviction(rank, init_method, disabled_rank, grouped=False):
dist.init_process_group(
"gloo",
rank=rank,
world_size=2,
init_method=init_method,
timeout=timedelta(seconds=30),
)
try:
cache = ConditioningCache(0 if disabled_rank and rank == 0 else 1024)
model = Encoder().eval()
group = SimpleNamespace(world_size=2, cpu_group=dist.group.WORLD)
x = torch.ones(4)
def compute():
# The same ordering requirement as a TP encoder forward.
flag = torch.ones(1)
dist.all_reduce(flag)
return model.forward(x)
with torch.no_grad(), cache.group_scope() if grouped else nullcontext():
for attempt in range(3):
if attempt == 1 and rank == 0:
cache.clear()
if attempt == 2 and rank == 0 and grouped and not disabled_rank:
# One rank restores a host entry while its peer reuses a device entry.
cache._group_entries.get().clear()
cache.run(model, "forward", (x,), {}, compute, group)
assert model.calls == (3 if disabled_rank and not grouped else 2)
assert cache.hits == (0 if disabled_rank and not grouped else 1)
finally:
dist.destroy_process_group()
@pytest.mark.parametrize("disabled_rank", [False, True])
@pytest.mark.parametrize("grouped", [False, True])
def test_rank_local_eviction_forces_collective_miss(tmp_path, disabled_rank, grouped):
mp.spawn(
_rank_eviction,
args=(f"file://{tmp_path / 'rendezvous'}", disabled_rank, grouped),
nprocs=2,
join=True,
)
@torch.no_grad()
def test_stage_conditioning_owns_output_but_preserves_vision_cache():
model = VisionLanguageEncoder().eval()
namespace = Encoder()
cache = ConditioningCache(4096)
pixels = torch.ones(4)
hooks = []
model.register_forward_hook(lambda *args: hooks.append(1))
def stage(text):
return cached_encoder_call(
model,
(text, pixels),
{},
lambda: model(text, pixels)[:1],
namespace=namespace,
)
with cache.scope():
first = stage(torch.ones(4))
# one vision entry and one consumed output, without a full encoder copy
assert cache.bytes == 20
assert cache.stats()["entries"] == 2
first.zero_()
assert stage(torch.ones(4)).item() == 3
assert model.calls == model.vision_calls == len(hooks) == 1
assert stage(torch.full((4,), 2.0)).item() == 4
assert model.calls == len(hooks) == 2
assert model.vision_calls == 1
with cache.scope(refresh=True):
stage(torch.ones(4))
assert model.calls == len(hooks) == 3
assert model.vision_calls == 2
def fail():
raise RuntimeError("encoding failed")
with pytest.raises(RuntimeError, match="encoding failed"):
cached_encoder_call(model, ("failure",), {}, fail, namespace=namespace)
# the temporary stage owner must not bypass later encoder caching
model(torch.ones(4), pixels)
model(torch.ones(4), pixels)
assert model.calls == 4
@pytest.mark.parametrize("capacity", [0, 1, 4096])
@torch.no_grad()
def test_group_cache_preserves_mutable_outputs_and_posterior_rng(capacity):
cache = ConditioningCache(capacity)
encoder = Encoder().eval()
vae = VAE().eval()
x = torch.ones(1, 4, 2, 2)
with cache.scope(), cache.group_scope():
first = encoder(x)
first.last_hidden_state.zero_()
second = encoder(x)
assert encoder.calls == 1
assert second.hidden_states[0] is second.last_hidden_state
torch.testing.assert_close(second.last_hidden_state, x, rtol=0, atol=0)
second.last_hidden_state.zero_()
torch.testing.assert_close(encoder(x).last_hidden_state, x, rtol=0, atol=0)
for seed in (13, 27):
generator = torch.Generator().manual_seed(seed)
control = torch.Generator().manual_seed(seed)
posterior = vae.encode(x).latent_dist
expected = DiagonalGaussianDistribution(x).sample(control)
torch.testing.assert_close(
posterior.sample(generator), expected, rtol=0, atol=0
)
assert torch.equal(generator.get_state(), control.get_state())
posterior.mean.zero_()
assert vae.calls == 1
assert cache._group_entries.get() is None
@torch.no_grad()
def test_group_hit_preserves_negative_host_entry_under_capacity_pressure():
model = Encoder().eval()
cache = ConditioningCache(24)
negative = torch.zeros(4)
with cache.scope(), cache.group_scope():
model(negative)
with prefer_conditioning_cache():
model(negative)
model(torch.ones(4))
with cache.scope():
actual = model(negative)
assert model.calls == 2
assert cache.hits == 2
assert cache.group_hits == 1
assert cache.evictions == 0
torch.testing.assert_close(actual.last_hidden_state, negative, rtol=0, atol=0)
@torch.no_grad()
def test_group_only_scope_ignores_host_entries_and_nested_encoder_caches():
model = VisionLanguageEncoder().eval()
cache = ConditioningCache(4096)
namespace = Encoder()
x = torch.ones(4)
def stage():
return cached_encoder_call(
model,
(x, x),
{},
lambda: model(x, x),
namespace=namespace,
share_in_group=True,
)
with cache.scope():
expected = stage()
entries = cache.stats()["entries"]
with cache.scope(cross_request=False):
for _ in range(2):
with cache.group_scope():
torch.testing.assert_close(stage(), expected, rtol=0, atol=0)
stage()
assert cache.stats()["entries"] == entries
assert model.calls == model.vision_calls == 3
stage()
stage()
assert model.calls == model.vision_calls == 5
stage()
assert model.calls == 5
assert cache.group_hits == 2
@torch.no_grad()
def test_warmup_refreshes_existing_group_then_reuses_fresh_results():
cache = ConditioningCache(4096)
encoder = Encoder().eval()
x = torch.ones(4)
with cache.scope(), cache.group_scope():
encoder(x)
with cache.scope(refresh=True):
encoder(x)
encoder(x)
assert encoder.calls == 2
assert cache.group_hits == 1
@torch.no_grad()
def test_group_cache_invalidates_and_releases_after_failure():
cache = ConditioningCache(0)
model = Encoder().eval()
x = torch.ones(4)
with pytest.raises(RuntimeError, match="stop"), cache.scope(), cache.group_scope():
model(x)
model(x)
invalidate_conditioning_caches([model])
model.weight.fill_(2)
torch.testing.assert_close(model(x).last_hidden_state, x * 2)
assert model.calls == 2
raise RuntimeError("stop")
assert cache._group_entries.get() is None
with cache.scope(), cache.group_scope():
model(x)
assert model.calls == 3
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@torch.no_grad()
def test_group_cache_waits_for_producing_cuda_stream():
cache = ConditioningCache(0)
model = Encoder().cuda().eval()
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with cache.scope(), cache.group_scope():
with torch.cuda.stream(stream):
torch.cuda._sleep(10_000_000)
x = torch.ones(1024, device="cuda")
cache.run(model, "stream", (), {}, lambda: model.forward(x))
output = cache.run(model, "stream", (), {}, lambda: pytest.fail("cache miss"))
torch.testing.assert_close(
output.last_hidden_state, torch.ones_like(output.last_hidden_state)
)
assert model.calls == 1
if __name__ == "__main__":
sys.exit(pytest.main([__file__, *sys.argv[1:]]))
@@ -0,0 +1,142 @@
# SPDX-License-Identifier: Apache-2.0
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from sglang.multimodal_gen.runtime.cache.conditioning import (
ConditioningCache,
invalidate_conditioning_caches,
)
from sglang.multimodal_gen.runtime.models.dits.cosmos3video import Cosmos3LanguageModel
from sglang.multimodal_gen.runtime.models.sensenova_u1.neo_unify.modeling_neo_chat import (
NEOChatModel,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.realtime.text_encoding import (
RealtimeTextEncodingStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
TextEncodingStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.vla import (
VLAPrefixEncodingStage,
)
from sglang.multimodal_gen.runtime.realtime.session import RealtimeSession
class UndLayer(torch.nn.Module):
def __init__(self):
super().__init__()
self.calls = 0
def forward(self, hidden, cos_sin, positions):
self.calls += 1
return hidden, hidden + 2, hidden + 3
@torch.no_grad()
def test_cosmos_understanding_cache_restores_private_kv():
model = Cosmos3LanguageModel.__new__(Cosmos3LanguageModel)
torch.nn.Module.__init__(model)
model.embed_tokens = torch.nn.Embedding(8, 4)
model.rotary_emb = SimpleNamespace(
build_rope_cache_inputs=lambda positions, cache_dtype: (None, positions)
)
layer = UndLayer()
model.layers = torch.nn.ModuleList([layer])
model.eval()
ids = torch.tensor([[1, 2]])
mask = torch.ones_like(ids)
positions = torch.zeros(3, 1, 2, dtype=torch.long)
with ConditioningCache(4096).scope():
first = model(ids, mask, positions)
expected = first[0][0].clone()
first[0][0].zero_()
hit = model(ids, mask, positions)
torch.testing.assert_close(hit[0][0], expected, rtol=0, atol=0)
assert layer.calls == 1
model(ids, mask, positions + 1)
model(ids, mask * 0, positions)
assert layer.calls == 3
class Vision(torch.nn.Module):
def __init__(self):
super().__init__()
self.calls = 0
def forward(self, pixel_values, **kwargs):
self.calls += 1
return SimpleNamespace(last_hidden_state=pixel_values * 2)
@torch.no_grad()
def test_sensenova_caches_references_but_runs_noisy_vision_each_step():
model = NEOChatModel.__new__(NEOChatModel)
torch.nn.Module.__init__(model)
model.vision_model = Vision()
model.fm_modules = torch.nn.ModuleDict({"vision_model_mot_gen": Vision()})
model.eval()
pixels = torch.ones(2, 4)
with ConditioningCache(4096).scope():
for _ in range(2):
model.extract_feature(pixels)
model.extract_feature(pixels, gen_model=True)
assert model.vision_model.calls == 1
assert model.fm_modules["vision_model_mot_gen"].calls == 2
@pytest.mark.parametrize("disabled, capacity", [(False, 512), (True, 512), (False, 0)])
def test_realtime_text_respects_disable_and_weight_invalidation(
monkeypatch, disabled, capacity
):
calls = []
def encode(self, batch, server_args):
calls.append(batch.prompt)
batch.prompt_embeds = [torch.ones(1)]
return batch
monkeypatch.setattr(TextEncodingStage, "forward", encode)
stage = RealtimeTextEncodingStage([], [])
session = RealtimeSession()
args = SimpleNamespace(
disable_conditioning_cache=disabled,
conditioning_cache_max_size_mb=capacity,
)
for _ in range(2):
stage.forward(Req(prompt="a cat", session=session), args)
expected = 2 if disabled or capacity == 0 else 1
assert len(calls) == expected
invalidate_conditioning_caches()
stage.forward(Req(prompt="a cat", session=session), args)
assert len(calls) == expected + 1
@pytest.mark.parametrize("disabled, capacity", [(False, 512), (True, 512), (False, 0)])
def test_vla_prefix_respects_disable_and_weight_invalidation(disabled, capacity):
stage = VLAPrefixEncodingStage.__new__(VLAPrefixEncodingStage)
stage.policy_model = Mock()
stage.policy_model.build_prefix_cache_key.return_value = "observation"
entries = {}
stage.prefix_cache = entries
args = SimpleNamespace(
disable_conditioning_cache=disabled,
conditioning_cache_max_size_mb=capacity,
pipeline_config=SimpleNamespace(enable_global_prefix_cache=True),
)
batch = Req(prompt="pick up the cube", extra={"vla": {}})
key, cached = stage.get_cached_context(batch, args, None)
assert cached is None
if disabled or capacity == 0:
assert key is None
stage.policy_model.build_prefix_cache_key.assert_not_called()
else:
context = object()
entries[key] = context
assert stage.get_cached_context(batch, args, None)[1] is context
invalidate_conditioning_caches()
assert stage.get_cached_context(batch, args, None)[1] is None
@@ -704,6 +704,9 @@ class TestIdeogram4(unittest.TestCase):
}
stage.copy_deduplicated_outputs(base, same)
self.assertIs(same.prompt_embeds[0], base.prompt_embeds[0])
self.assertIs(same.prompt_embeds_mask[0], base.prompt_embeds_mask[0])
self.assertIsNot(same.prompt_embeds, base.prompt_embeds)
self.assertIn("ideogram4", same.extra)
self.assertTrue(
torch.equal(
@@ -235,6 +235,7 @@ class _FakeQwenProcessor:
def _text_encoding_stage(processor, encoder):
stage = object.__new__(LingBotVideoTextEncodingStage)
stage._component_residency_manager = None
stage.text_encoders = [encoder]
stage.tokenizers = [processor]
stage.token_length = 128
@@ -7,6 +7,7 @@ import pytest
import torch
from prometheus_client import CollectorRegistry
from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningCache
from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear
from sglang.multimodal_gen.runtime.layers.lora.linear import (
BaseLayerWithLoRA,
@@ -88,6 +89,35 @@ def test_worker_metrics_count_individual_adapters_in_multi_lora():
]
@pytest.mark.parametrize("operation", ["set", "merge", "unmerge", "deactivate"])
@torch.no_grad()
def test_lora_mutations_preserve_independent_conditioning(operation):
layer = _make_layer()
pipeline = _make_pipeline(layer)
pipeline.modules["transformer"].add_module("linear", layer)
pipeline._temporarily_disable_offload = lambda *args, **kwargs: nullcontext([])
encoder = torch.nn.Linear(2, 2).eval()
cache = ConditioningCache(1024)
x = torch.ones(1, 2)
with patch(_RANK_PATCH, return_value=0):
pipeline.set_lora("adapter", merge_mode="dynamic")
cache.run(encoder, "forward", (x,), {}, lambda: encoder(x))
cache.run(layer, "forward", (x,), {}, lambda: x.clone())
if operation == "set":
pipeline.set_lora("adapter", strength=0.5, merge_mode="dynamic")
elif operation == "merge":
pipeline.merge_lora_weights()
elif operation == "unmerge":
pipeline.unmerge_lora_weights()
else:
pipeline.deactivate_lora_weights()
cached = cache.run(encoder, "forward", (x,), {}, lambda: encoder(x))
cache.run(layer, "forward", (x,), {}, lambda: x.clone())
torch.testing.assert_close(cached, encoder(x), rtol=0, atol=0)
assert cache.hits == 1
assert cache.misses == 3
def test_merge_cache_only_accepts_cpu_backed_weights():
pipeline = _make_pipeline(_make_layer())
cpu_cache = pipeline._merge_cache_for(
@@ -5,6 +5,7 @@ from unittest.mock import Mock
import pytest
import torch
from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningEncoderMixin
from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency import (
COMPONENT_OFFLOAD,
LAYERWISE_OFFLOAD,
@@ -14,11 +15,12 @@ from sglang.multimodal_gen.runtime.pipelines_core.executors.pipeline_executor im
PipelineExecutor,
)
from sglang.multimodal_gen.runtime.platforms.npu import NPUPlatformBase
from sglang.multimodal_gen.runtime.server_args import ServerArgs
class _RecordingExecutor(PipelineExecutor):
def __init__(self):
super().__init__(server_args=SimpleNamespace())
super().__init__(server_args=_server_args())
self.single_inference_mode = None
self.group_inference_mode = None
self.single_grad_enabled = None
@@ -47,6 +49,8 @@ class _TestServerArgs(SimpleNamespace):
def _server_args(**overrides):
values = {
"use_fsdp_inference": False,
"disable_conditioning_cache": ServerArgs.disable_conditioning_cache,
"conditioning_cache_max_size_mb": ServerArgs.conditioning_cache_max_size_mb,
"component_modes": {},
}
values.update(overrides)
@@ -115,6 +119,31 @@ def test_execute_group_with_profiling_uses_platform_inference_mode(monkeypatch):
assert executor.group_grad_enabled is False
@pytest.mark.parametrize("fsdp", [False, True])
@torch.no_grad()
def test_warmup_conditioning_is_reused_by_first_serving_request(fsdp):
class Encoder(ConditioningEncoderMixin, torch.nn.Module):
def __init__(self):
super().__init__()
self.calls = 0
def forward(self, tokens):
self.calls += 1
return tokens * 2
executor = _RecordingExecutor()
executor.component_residency_manager = Mock()
encoder = Encoder().eval()
tokens = torch.arange(8)
args = _server_args(use_fsdp_inference=fsdp)
for warmup in (True, True, False):
batch = _batch()
batch.is_warmup = warmup
with executor._component_residency_request([], batch, args):
torch.testing.assert_close(encoder(tokens), tokens * 2, rtol=0, atol=0)
assert encoder.calls == (3 if fsdp else 2)
def test_group_payload_is_forwarded_to_component_residency_manager():
executor = _RecordingExecutor()
executor.component_residency_manager = Mock()
@@ -22,6 +22,7 @@ from sglang.multimodal_gen.configs.pipeline_configs.qwen_image21 import (
)
from sglang.multimodal_gen.configs.sample.qwenimage21 import QwenImage21SamplingParams
from sglang.multimodal_gen.registry import _get_config_info
from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningCache
from sglang.multimodal_gen.runtime.entrypoints.openai.image_api import (
_resolve_image_output_format,
)
@@ -33,6 +34,9 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency_
)
from sglang.multimodal_gen.runtime.managers.scheduler import Scheduler
from sglang.multimodal_gen.runtime.models.dits.qwen_image21 import build_layout
from sglang.multimodal_gen.runtime.models.encoders.base import (
EncoderTensorParallelMixin,
)
from sglang.multimodal_gen.runtime.models.encoders.qwen3vl_vision import (
Qwen3VLVisionRotaryEmbedding,
)
@@ -120,6 +124,78 @@ def test_prompt_conditioning_uses_training_template_and_pre_norm(prompt, image_c
assert image.getpixel((0, 0)) == (12, 34, 56, 0)
class ConditioningTestEncoder(EncoderTensorParallelMixin, torch.nn.Module):
def __init__(self, device):
super().__init__()
self.weight = torch.nn.Parameter(torch.ones((), device=device))
self.model = SimpleNamespace(
visual=SimpleNamespace(rotary_pos_emb=SimpleNamespace())
)
self.calls = 0
def forward(self, input_ids, **kwargs):
self.calls += 1
hidden = input_ids[..., None].float().expand(-1, -1, 4) * self.weight
return SimpleNamespace(hidden_states=tuple(hidden + i for i in range(32)))
@pytest.mark.parametrize(
"device",
[
"cpu",
pytest.param(
"cuda",
marks=pytest.mark.skipif(
not torch.cuda.is_available(), reason="CUDA required"
),
),
],
)
@torch.no_grad()
def test_prompt_cache_skips_processor_and_weight_preparation(device):
def process(text, **kwargs):
image_value = sum(kwargs["images"][0].getpixel((0, 0)))
return BatchFeature(
data={
"input_ids": torch.tensor([[1, len(text[0]), 99, 99, image_value, 0]]),
"attention_mask": torch.tensor([[1, 1, 1, 1, 1, 0]]),
}
)
processor = Mock(side_effect=process)
processor.tokenizer.convert_tokens_to_ids.return_value = 99
processor.apply_chat_template.return_value = [[1]]
encoder = ConditioningTestEncoder(device).eval()
stage = QwenImage21EncodingStage(encoder, processor, None, None)
stage.use_declared_component = Mock(return_value=nullcontext(encoder))
image = Image.new("RGBA", (2, 2), (12, 34, 56, 128))
cache = ConditioningCache(1024)
with cache.scope():
first, slots = stage.encode_prompt("edit", [image], device)
expected = first.clone()
first.zero_()
slots.zero_()
restored, slots = stage.encode_prompt("edit", [image.copy()], device)
torch.testing.assert_close(restored, expected, atol=0, rtol=0)
assert slots.tolist() == [False, True, False]
assert encoder.calls == processor.call_count == 1
stage.use_declared_component.assert_called_once()
assert cache.stats()["entries"] == 1
assert cache.bytes == 3 * 4 * 4 + 3
assert cache.bypasses == 0 # no oversized duplicate of all hidden states
changed, _ = stage.encode_prompt("another edit", [image], device)
assert not torch.equal(changed, expected)
image.putpixel((0, 0), (12, 34, 56, 255))
changed_alpha, _ = stage.encode_prompt("edit", [image], device)
assert not torch.equal(changed_alpha, expected)
assert encoder.calls == 3
with cache.scope(refresh=True):
stage.encode_prompt("edit", [image], device)
with cache.scope(enabled=False):
stage.encode_prompt("edit", [image], device)
assert encoder.calls == processor.call_count == 5
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
def test_encoder_component_offload_preserves_loaded_dtypes(monkeypatch):
monkeypatch.setattr(
@@ -1299,7 +1299,12 @@ def _make_sensenova_u1_sequential_entrypoint(*, fail=False, fail_request_ids=Non
output_file_name="sample.png",
trace_ctx=trace_ctx,
)
server_args = SimpleNamespace(pipeline_config=SenseNovaU1PipelineConfig())
server_args = SimpleNamespace(
pipeline_config=SenseNovaU1PipelineConfig(),
disable_conditioning_cache=False,
conditioning_cache_max_size_mb=512,
use_fsdp_inference=False,
)
pipeline = _SequentialTestPipeline(
server_args, fail=fail, fail_request_ids=fail_request_ids
)
@@ -1,17 +1,202 @@
import os
from contextlib import nullcontext
from datetime import timedelta
from functools import partial
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock, Mock, patch
import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from torch.distributed.device_mesh import init_device_mesh
from torch.distributed.fsdp import fully_shard
from transformers import BatchEncoding
from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput
from sglang.multimodal_gen.configs.pipeline_configs.base import TextConditioningOutput
from sglang.multimodal_gen.configs.sample.longlive2 import LongLive2SamplingParams
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.runtime.cache import conditioning
from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningCache
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
destroy_distributed_environment,
destroy_model_parallel,
init_distributed_environment,
initialize_model_parallel,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentResidencyManager,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency_strategies import (
ComponentOffloadStrategy,
)
from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder
from sglang.multimodal_gen.runtime.pipelines_core.executors.sync_executor import (
SyncExecutor,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.causal_denoising import (
CAUSAL_BLOCK_PROMPTS_KEY,
CAUSAL_SCENE_CUT_MASK_KEY,
CAUSAL_SHOT_INDICES_KEY,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.lingbot_video_moe.text_encoding import (
LingBotVideoTextEncodingStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.longlive2 import (
LongLive2TextEncodingStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ming_image import (
MingImageEncodingStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.realtime.text_encoding import (
RealtimeTextEncodingStage,
RealtimeTextState,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
TextEncodingStage,
)
from sglang.multimodal_gen.runtime.realtime.session import RealtimeSession
_GLOBAL_ARGS_PATCH = (
"sglang.multimodal_gen.runtime.pipelines_core.stages.base.get_global_server_args"
)
class FullHiddenStateEncoder(TextEncoder):
uses_sglang_forward_context = False
def __init__(self):
torch.nn.Module.__init__(self)
self.weight = torch.nn.Parameter(torch.ones(()))
self.calls = 0
def forward(self, input_ids, **kwargs):
self.calls += 1
states = tuple(input_ids[..., None].float() + layer for layer in range(32))
return BaseEncoderOutput(
last_hidden_state=states[-1],
hidden_states=states,
pooler_output=states[-1][:, 0].clone(),
)
class LibraryEncoder(torch.nn.Module):
__init__ = FullHiddenStateEncoder.__init__
forward = FullHiddenStateEncoder.forward
def make_text_config(output_type="tensor"):
def postprocess(output, text_inputs, return_attention_mask=False):
embedding = output.hidden_states[9]
mask = text_inputs["attention_mask"].bool()
if output_type == "tuple":
return embedding, mask
if output_type == "structured":
return TextConditioningOutput(embedding, mask, [2])
return embedding
def tokenize(texts, _tokenizer, _kwargs):
return BatchEncoding(
{
"input_ids": torch.tensor([[len(text), 2] for text in texts]),
"attention_mask": torch.ones((len(texts), 2), dtype=torch.long),
}
)
return SimpleNamespace(
text_encoder_configs=[SimpleNamespace(tokenizer_kwargs={})],
preprocess_text_funcs=[None],
postprocess_text_funcs=[postprocess],
text_encoder_extra_args=[],
is_flux_v1=lambda: False,
tokenize_prompt=tokenize,
get_text_encoder_attention_mask=lambda inputs, _: inputs["attention_mask"],
get_text_encoder_pooler_output=lambda outputs, _: outputs.pooler_output,
build_text_conditioning_mask=lambda inputs, mask, embeds, _: mask.bool(),
seq_lens_from_text_conditioning_mask=lambda mask: mask.sum(-1).tolist(),
)
@pytest.mark.parametrize("output_type", ["tensor", "tuple", "structured"])
@pytest.mark.parametrize(
"device",
[
"cpu",
pytest.param(
"cuda",
marks=pytest.mark.skipif(
not torch.cuda.is_available(), reason="requires CUDA"
),
),
],
)
@torch.no_grad()
def test_cache_stores_only_consumed_text_conditioning(output_type, device, monkeypatch):
encoder = FullHiddenStateEncoder().to(device).eval()
fingerprint = conditioning._fingerprint
def fingerprint_host_inputs(value):
if isinstance(value, torch.Tensor):
assert value.device.type == "cpu", "token keys must not read GPU inputs"
return fingerprint(value)
monkeypatch.setattr(conditioning, "_fingerprint", fingerprint_host_inputs)
config = make_text_config(output_type)
args = make_server_args(pipeline_config=config)
def make_stage():
with patch(_GLOBAL_ARGS_PATCH, return_value=MagicMock()):
stage = TextEncodingStage(text_encoders=[encoder], tokenizers=[object()])
stage._begin_text_encoder_use = MagicMock()
stage._text_encode_dp_group = MagicMock(return_value=None)
return stage
stage = make_stage()
cache = ConditioningCache(4096)
with cache.scope():
first = stage.encode_text(
"hello", args, device=device, return_attention_mask=True
)
expected = first[0][0].clone()
expected_pooled = first[2][0].clone()
first[0][0].zero_()
first[2][0].zero_()
restored = stage.encode_text(
"hello", args, device=device, return_attention_mask=True
)
torch.testing.assert_close(restored[0][0], expected, rtol=0, atol=0)
torch.testing.assert_close(restored[2][0], expected_pooled, rtol=0, atol=0)
assert restored[4] == [[2]]
assert encoder.calls == 1
stage._begin_text_encoder_use.assert_called_once_with(0)
assert cache.stats()["entries"] == 1
assert cache.bytes < 32 # two embeddings, one pooled value, optional mask
stage.encode_text("changed", args, device=device, return_attention_mask=True)
assert encoder.calls == 2
# The same encoder can serve different pipeline postprocessing contracts.
make_stage().encode_text(
"hello", args, device=device, return_attention_mask=True
)
assert encoder.calls == 3
# exercise the real negative-stage boundary with only one entry of capacity
cache = ConditioningCache(cache.bytes // 3)
stage.encode_text = partial(stage.encode_text, device=device)
calls_before = encoder.calls
with cache.scope(refresh=True):
stage.encode_text("hello", args, return_attention_mask=True)
stage.get_or_compute_negative_text_embedding(make_req(), args, [0])
with cache.scope():
stage.encode_text("changed positive", args, return_attention_mask=True)
stage.get_or_compute_negative_text_embedding(make_req(), args, [0])
assert encoder.calls == calls_before + 3
assert cache.hits == 1
assert cache.bytes <= cache.max_bytes
class DummyTextEncodingStage(TextEncodingStage):
def __init__(self):
with patch(_GLOBAL_ARGS_PATCH) as mock_global_args:
@@ -53,6 +238,341 @@ def make_server_args(**kwargs):
return SimpleNamespace(**defaults)
def make_group_executor(
stage_type, device, capacity, *, fsdp=False, native=True, tokenizer=None
):
encoder = (
(FullHiddenStateEncoder() if native else LibraryEncoder()).to(device).eval()
)
config = make_text_config()
config.supports_auto_residency = False
config.vae_config = SimpleNamespace(use_temporal_scaling_frames=False)
config.dit_config = SimpleNamespace(
arch_config=SimpleNamespace(num_frames_per_block=1)
)
args = make_server_args(
pipeline_config=config,
component_precisions={},
comfyui_mode=True,
enable_layerwise_nvtx_marker=False,
use_fsdp_inference=fsdp,
disable_conditioning_cache=capacity == 0,
conditioning_cache_max_size_mb=capacity / 1024**2,
should_cpu_offload_component=lambda _: False,
)
with patch(_GLOBAL_ARGS_PATCH, return_value=args):
stage = stage_type(
[encoder], [tokenizer if tokenizer is not None else object()]
)
stage._text_encode_dp_group = Mock(return_value=None)
stage.encode_text = partial(stage.encode_text, device=device)
pipeline = SimpleNamespace(
modules={"text_encoder": encoder},
_stage_name_mapping={"text": stage},
component_residency_strategies={},
)
executor = SyncExecutor(args)
manager = ComponentResidencyManager(pipeline, args)
strategy = Mock()
strategy.prefetch_for_use.return_value = False
manager.strategy_for = Mock(return_value=strategy)
executor.component_residency_manager = manager
return executor, stage, encoder, args
@pytest.mark.parametrize("capacity", [0, 4096])
@torch.no_grad()
def test_grouped_realtime_text_updates_each_session(capacity):
executor, stage, encoder, args = make_group_executor(
RealtimeTextEncodingStage, "cpu", capacity
)
sessions = [RealtimeSession(), RealtimeSession()]
requests = [
Req(
sampling_params=SamplingParams(prompt="hello", num_inference_steps=4),
do_classifier_free_guidance=False,
session=session,
)
for session in sessions
]
outputs = executor.execute_group([stage], requests, args)
assert encoder.calls == 1
for session, output in zip(sessions, outputs):
state = session.get_or_create_state(RealtimeTextState)
if capacity:
assert state.cache_key is not None
assert state.prompt_embeds[0] is output.prompt_embeds[0]
assert state.prompt_seq_lens[0] is not output.prompt_seq_lens[0]
else:
assert state.cache_key is None
@pytest.mark.parametrize("fsdp, native", [(True, False), (True, True), (False, False)])
@torch.no_grad()
def test_one_unique_stage_skips_group_only_cache_lookup(fsdp, native, monkeypatch):
executor, stage, encoder, args = make_group_executor(
TextEncodingStage, "cpu", 4096, fsdp=fsdp, native=native
)
fingerprint = Mock(wraps=conditioning._fingerprint)
monkeypatch.setattr(conditioning, "_fingerprint", fingerprint)
capture_query = Mock(wraps=torch.cuda.is_current_stream_capturing)
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", capture_query)
stage.forward = Mock(wraps=stage.forward)
requests = [
Req(
sampling_params=SamplingParams(prompt="hello", num_inference_steps=4),
do_classifier_free_guidance=False,
)
for _ in range(3)
]
outputs = executor.execute_group([stage], requests, args)
assert stage.forward.call_count == 1
assert encoder.calls == 1
assert outputs[0].prompt_embeds[0] is outputs[2].prompt_embeds[0]
fingerprint.assert_not_called()
capture_query.assert_not_called()
@pytest.mark.parametrize("capacity", [0, 1, 4096])
@pytest.mark.parametrize("fsdp", [False, True])
@pytest.mark.parametrize("native", [False, True])
@pytest.mark.parametrize("warmup", [False, True])
@pytest.mark.parametrize(
"device",
[
"cpu",
pytest.param(
"cuda",
marks=pytest.mark.skipif(
not torch.cuda.is_available(), reason="requires CUDA"
),
),
],
)
@torch.no_grad()
def test_grouped_conditioning_reuses_positive_and_negative_independently(
capacity, device, fsdp, native, warmup
):
executor, stage, encoder, args = make_group_executor(
TextEncodingStage, device, capacity, fsdp=fsdp, native=native
)
stage.forward = Mock(wraps=stage.forward)
def requests():
return [
Req(
sampling_params=SamplingParams(
prompt=prompt, negative_prompt="bad quality", num_inference_steps=4
),
do_classifier_free_guidance=True,
is_warmup=warmup,
)
for prompt in ("hello", "different", "hello")
]
first = executor.execute_group([stage], requests(), args)
assert stage.forward.call_count == 2
assert encoder.calls == 3 # two positives, one shared negative
assert executor.conditioning_cache.group_hits == 1
assert first[0].prompt_embeds[0] is first[2].prompt_embeds[0]
assert first[0].negative_prompt_embeds[0] is first[1].negative_prompt_embeds[0]
assert first[0].prompt_seq_lens is not first[2].prompt_seq_lens
assert first[0].prompt_seq_lens[0] is not first[2].prompt_seq_lens[0]
expected = first[0].prompt_embeds[0].clone()
first[0].prompt_embeds[0].zero_()
first[0].prompt_seq_lens[0][0] = 0
assert executor.conditioning_cache._group_entries.get() is None
second = executor.execute_group([stage], requests(), args)
assert stage.forward.call_count == 4
persistent = capacity == 4096 and not fsdp and not warmup
expected_calls = (3 if native else 5) if persistent else 6
assert encoder.calls == expected_calls
torch.testing.assert_close(second[0].prompt_embeds[0], expected, rtol=0, atol=0)
assert second[0].prompt_seq_lens == [[2]]
assert executor.conditioning_cache.bytes <= capacity
if fsdp:
assert executor.conditioning_cache.bytes == 0
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("native", [False, True])
@torch.no_grad()
def test_negative_conditioning_keeps_private_device_snapshot(native):
encoder = (FullHiddenStateEncoder() if native else LibraryEncoder()).cuda().eval()
config = make_text_config("structured")
config.postprocess_text_funcs[0] = Mock(wraps=config.postprocess_text_funcs[0])
args = make_server_args(pipeline_config=config)
with patch(_GLOBAL_ARGS_PATCH, return_value=args):
stage = TextEncodingStage([encoder], [object()])
stage._text_encode_dp_group = Mock(return_value=None)
stage._begin_text_encoder_use = Mock()
stage.encode_text = partial(stage.encode_text, device="cuda")
cache = ConditioningCache(4096)
producer, consumer = torch.cuda.Stream(), torch.cuda.Stream()
expected = torch.tensor([[[20.0], [11.0]]])
with cache.scope(), torch.cuda.stream(producer):
torch.cuda._sleep(250_000_000)
first = stage.get_or_compute_negative_text_embedding(make_req(), args, [0])
first[0][0].zero_()
first[1][0].zero_()
first[4][0][0] = 0
entry = next(iter(cache._entries.values()))
assert entry.device_resident
assert entry.output[0].device.type == "cuda"
assert entry.output[0].data_ptr() != first[0][0].data_ptr()
assert cache.stats()["device_bytes"] == cache.bytes <= 4096
with cache.scope(), torch.cuda.stream(consumer):
for _ in range(2):
hit = stage.get_or_compute_negative_text_embedding(make_req(), args, [0])
torch.testing.assert_close(hit[0][0].cpu(), expected, rtol=0, atol=0)
assert hit[1][0].tolist() == [[1, 1]]
assert hit[4] == [[2]]
hit[0][0].zero_()
assert encoder.calls == config.postprocess_text_funcs[0].call_count == 1
stage._begin_text_encoder_use.assert_called_once_with(0)
conditioning.invalidate_conditioning_caches([encoder])
assert cache.bytes == cache.stats()["device_bytes"] == 0
stage.get_or_compute_negative_text_embedding(make_req(), args, [0])
assert encoder.calls == 2
with cache.scope(refresh=True):
stage.get_or_compute_negative_text_embedding(make_req(), args, [0])
assert encoder.calls == 3
stage.get_or_compute_negative_text_embedding(
make_req(negative_prompt="changed negative"), args, [0]
)
assert encoder.calls == 4
@torch.no_grad()
def test_batched_request_reuses_conditioning_without_changing_output_seeds():
executor, stage, encoder, args = make_group_executor(TextEncodingStage, "cpu", 0)
for _ in range(2):
request = Req(
sampling_params=SamplingParams(
prompt="hello",
negative_prompt="hello",
num_outputs_per_prompt=3,
seed=[7, 8, 9],
),
do_classifier_free_guidance=True,
)
output = executor.execute([stage], request, args)
assert output.seed == [7, 8, 9]
assert output.num_outputs_per_prompt == 3
assert output.prompt_embeds[0] is output.negative_prompt_embeds[0]
assert executor.conditioning_cache._group_entries.get() is None
assert encoder.calls == 2
assert executor.conditioning_cache.group_hits == 2
@torch.no_grad()
def test_grouped_longlive_conditioning_keeps_per_request_shot_metadata():
executor, stage, encoder, args = make_group_executor(
LongLive2TextEncodingStage, "cpu", 0
)
requests = [
Req(
sampling_params=LongLive2SamplingParams(
prompt="hello",
shot_prompts=["first", "second"],
shot_durations=[1, 1],
num_frames=2,
),
do_classifier_free_guidance=False,
)
for _ in range(2)
]
outputs = executor.execute_group([stage], requests, args)
assert encoder.calls == 1
for key in (
CAUSAL_BLOCK_PROMPTS_KEY,
CAUSAL_SCENE_CUT_MASK_KEY,
CAUSAL_SHOT_INDICES_KEY,
):
assert key in outputs[0].extra and key in outputs[1].extra
assert outputs[0].extra[key] == outputs[1].extra[key]
assert outputs[0].extra[key] is not outputs[1].extra[key]
@pytest.mark.parametrize("fsdp", [False, True])
@pytest.mark.parametrize(
"stage_type", [MingImageEncodingStage, LingBotVideoTextEncodingStage]
)
@torch.no_grad()
def test_custom_text_stages_preserve_group_reuse_and_request_metadata(fsdp, stage_type):
tokenizer = Mock(name_or_path="fixture", eos_token="<eos>")
tokenizer.side_effect = lambda text, **kwargs: {"input_ids": [len(text), 2]}
tokenizer.convert_tokens_to_ids.side_effect = lambda text: (
3 if text == "<image>" else 4
)
factory = stage_type
if stage_type is LingBotVideoTextEncodingStage:
factory = partial(stage_type, transformer=torch.nn.Linear(1, 1))
def tokenize(**kwargs):
texts = kwargs["text"]
ids = (
[[1]]
if isinstance(texts, str)
else [[1, len(text), 2] for text in texts]
)
return BatchEncoding(
{
"input_ids": torch.tensor(ids),
"attention_mask": torch.ones_like(torch.tensor(ids)),
}
)
tokenizer.side_effect = tokenize
with patch(
"sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages."
"ming_image.Qwen2VLImageProcessorPil.from_pretrained"
):
executor, stage, encoder, args = make_group_executor(
factory, "cpu", 0, fsdp=fsdp, tokenizer=tokenizer
)
encoder.dtype = torch.bfloat16
encoder.config = SimpleNamespace(projection_config={"img_gen_scales": [1]})
encoder.image_token = 99
args.pipeline_config.dit_config.arch_config.alignment_padding_mode = "zero_masked"
args.pipeline_config.dit_config.arch_config.multi_frame_output = False
def requests():
return [
Req(
sampling_params=SamplingParams(
prompt=prompt, negative_prompt="", height=64, width=64
),
do_classifier_free_guidance=False,
)
for prompt in ("hello", "different", "hello")
]
with patch(
f"{stage_type.__module__}.get_local_torch_device",
return_value=torch.device("cpu"),
):
for run in range(2):
outputs = executor.execute_group([stage], requests(), args)
assert encoder.calls == 2 * (run + 1)
assert outputs[0].prompt_embeds[0] is outputs[2].prompt_embeds[0]
assert outputs[0].prompt_embeds is not outputs[2].prompt_embeds
if stage_type is MingImageEncodingStage:
assert all(output.extra["ming_frames"] == 1 for output in outputs)
assert (
outputs[0].extra["ming_direct"] is outputs[2].extra["ming_direct"]
)
assert executor.conditioning_cache._group_entries.get() is None
def get_negative_embedding_twice(stage, server_args, first_req, second_req=None):
stage.get_or_compute_negative_text_embedding(first_req, server_args, [0])
stage.get_or_compute_negative_text_embedding(
@@ -60,23 +580,62 @@ def get_negative_embedding_twice(stage, server_args, first_req, second_req=None)
)
def test_negative_text_cache_key_tracks_encode_options():
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("capacity", [0, 1, 4096])
@torch.no_grad()
def test_lingbot_cache_miss_prepares_offloaded_encoder(capacity, monkeypatch):
device = torch.device("cuda", torch.cuda.current_device())
monkeypatch.setattr(
"sglang.multimodal_gen.runtime.managers.memory_managers."
"component_residency_strategies.get_local_torch_device",
lambda: device,
)
executor, stage, encoder, args = make_group_executor(
partial(LingBotVideoTextEncodingStage, transformer=torch.nn.Linear(1, 1)),
"cpu",
capacity,
)
encoder.embedding = torch.nn.Embedding(8, 2)
def encode(input_ids, **kwargs):
encoder.calls += 1
states = encoder.embedding(input_ids)
return BaseEncoderOutput(last_hidden_state=states, hidden_states=(states,))
encoder.forward = encode
stage._crop_start = 0
stage._build_prompt_inputs = lambda prompt: BatchEncoding(
{"input_ids": torch.tensor([[1, 2]]), "attention_mask": torch.ones(1, 2)}
)
manager = executor.component_residency_manager
strategy = ComponentOffloadStrategy()
strategy.prepare_for_use = Mock(wraps=strategy.prepare_for_use)
manager.strategy_for = Mock(return_value=strategy)
stage.set_component_residency_manager(manager)
cache = ConditioningCache(capacity)
results = []
for _ in range(2):
batch = make_req()
manager.begin_request([stage], batch, args)
manager.before_stage(stage, 0, batch, args)
manager.begin_stage()
with cache.scope():
results.append(stage._encode_prompt("hello", device, torch.float32))
manager.end_stage()
manager.finish_request()
assert encoder.embedding.weight.device.type == "cpu"
assert results[-1][0].device == device
expected_calls = 1 if capacity == 4096 else 2
assert encoder.calls == strategy.prepare_for_use.call_count == expected_calls
torch.testing.assert_close(results[0][0], results[1][0], rtol=0, atol=0)
def test_negative_text_encoding_has_no_separate_gpu_cache():
stage = DummyTextEncodingStage()
server_args = make_server_args()
get_negative_embedding_twice(stage, server_args, make_req())
assert stage.calls == 1
stage.get_or_compute_negative_text_embedding(
make_req(max_sequence_length=512), server_args, [0]
)
assert stage.calls == 2
stage.get_or_compute_negative_text_embedding(
make_req(prompt_template={"template": "negative: {}"}), server_args, [0]
)
assert stage.calls == 3
def test_component_uses_exact_encoder_precision():
with patch(_GLOBAL_ARGS_PATCH) as mock_global_args:
@@ -97,25 +656,280 @@ def test_component_uses_exact_encoder_precision():
]
def test_negative_text_cache_skips_warmup():
def test_negative_text_encoding_warmup_does_not_seed_a_private_cache():
stage = DummyTextEncodingStage()
server_args = make_server_args()
with patch.object(
stage, "_get_model_default_negative_prompt", return_value="default negative"
):
get_negative_embedding_twice(stage, server_args, make_req(is_warmup=True))
get_negative_embedding_twice(stage, make_server_args(), make_req(is_warmup=True))
assert stage.calls == 2
def test_negative_text_cache_keeps_default_warmup():
stage = DummyTextEncodingStage()
server_args = make_server_args()
@pytest.mark.parametrize("encoder_count", [1, 2])
@torch.no_grad()
def test_cache_hit_skips_encoder_residency_and_input_preparation(encoder_count):
encoders = [FullHiddenStateEncoder().eval() for _ in range(encoder_count)]
config = make_text_config()
config.text_encoder_configs *= encoder_count
config.preprocess_text_funcs *= encoder_count
config.postprocess_text_funcs *= encoder_count
config.supports_auto_residency = False
config.get_text_encoder_attention_mask = Mock(
wraps=config.get_text_encoder_attention_mask
)
config.seq_lens_from_text_conditioning_mask = Mock(
wraps=config.seq_lens_from_text_conditioning_mask
)
args = make_server_args(
pipeline_config=config,
component_precisions={},
enable_layerwise_nvtx_marker=False,
)
with patch(_GLOBAL_ARGS_PATCH, return_value=args):
stage = TextEncodingStage(encoders, [object()] * encoder_count)
stage._text_encode_dp_group = Mock(return_value=None)
pipeline = SimpleNamespace(
modules={
"text_encoder" if i == 0 else f"text_encoder_{i + 1}": encoder
for i, encoder in enumerate(encoders)
},
_stage_name_mapping={"text": stage},
component_residency_strategies={},
)
manager = ComponentResidencyManager(pipeline, args)
strategy = Mock()
strategy.prefetch_for_use.return_value = False
manager.strategy_for = Mock(return_value=strategy)
stage.set_component_residency_manager(manager)
cache = ConditioningCache(4096)
with patch.object(
stage, "_get_model_default_negative_prompt", return_value="bad quality"
):
get_negative_embedding_twice(stage, server_args, make_req(is_warmup=True))
def request(prompt, *, refresh=False, enabled=True, dtype=None):
batch = make_req(is_warmup=refresh)
manager.begin_request([stage], batch, args)
manager.before_stage(stage, 0, batch, args)
manager.begin_stage()
with (cache if enabled else ConditioningCache(0)).scope(refresh=refresh):
result = stage.encode_text(
prompt,
args,
encoder_index=list(range(encoder_count)),
device="cpu",
dtype=dtype,
return_attention_mask=True,
)
manager.end_stage()
manager.finish_request()
return result
assert stage.calls == 1
first = request("hello")
expected = first[0][0].clone()
first[0][0].zero_()
first[1][0].zero_()
first[3][0].zero_()
first[4][0][0] = 0
strategy.reset_mock()
restored = request("hello")
strategy.prepare_for_use.assert_not_called()
strategy.wait_for_use.assert_not_called()
assert config.get_text_encoder_attention_mask.call_count == encoder_count
assert config.seq_lens_from_text_conditioning_mask.call_count == encoder_count
torch.testing.assert_close(restored[0][0], expected, rtol=0, atol=0)
assert restored[1][0].tolist() == [[1, 1]]
assert restored[3][0].tolist() == [[True, True]]
assert restored[4][0] == [2]
# dtype is part of the consumed conditioning contract
cast = request("hello", dtype=torch.float64)
assert cast[0][0].dtype == torch.float64
assert encoders[0].calls == 2
request("hello", refresh=True)
request("changed")
request("hello", enabled=False)
assert all(encoder.calls == 5 for encoder in encoders)
assert strategy.prepare_for_use.call_count == 4 * encoder_count
class TextEncodingDPGroup:
world_size = 2
def __init__(self, rank):
self.rank_in_group = rank
self.gathers = 0
self.cpu_group = dist.group.WORLD
def all_reduce(self, tensor):
dist.all_reduce(tensor)
return tensor
def all_gather(self, tensor, dim):
self.gathers += 1
outputs = [torch.empty_like(tensor) for _ in range(self.world_size)]
dist.all_gather(outputs, tensor)
return torch.cat(outputs, dim=dim)
def run_text_encoding_dp(rank, rendezvous, grouped, negative):
dist.init_process_group(
"gloo",
rank=rank,
world_size=2,
init_method=rendezvous,
timeout=timedelta(seconds=30),
)
try:
encoder = FullHiddenStateEncoder().eval()
args = make_server_args(pipeline_config=make_text_config())
with patch(_GLOBAL_ARGS_PATCH, return_value=args):
stage = TextEncodingStage([encoder], [object()])
group = TextEncodingDPGroup(rank)
stage._text_encode_dp_group = Mock(return_value=group)
cache = ConditioningCache(4096)
stage_module = TextEncodingStage.__module__
with (
torch.no_grad(),
cache.scope(),
cache.group_scope(enabled=grouped),
conditioning.prefer_conditioning_cache() if negative else nullcontext(),
patch(f"{stage_module}.model_parallel_is_initialized", return_value=True),
patch(f"{stage_module}.get_replica_group", return_value=group),
):
for attempt in range(3):
if attempt == 1 and rank == 0:
cache.clear()
before = group.gathers
outputs = stage.encode_text(
["a", "bb", "ccc"], args, device="cpu", return_attention_mask=True
)
if (grouped or negative) and attempt == 2:
assert group.gathers == before
else:
assert group.gathers > before
assert outputs[0][0][:, 0, 0].tolist() == [10, 11, 12]
assert outputs[4] == [[2, 2, 2]]
assert encoder.calls == (2 if negative or rank == 0 else 1)
assert cache.hits == (1 if negative or rank == 0 else 2)
finally:
dist.destroy_process_group()
@pytest.mark.parametrize("grouped", [False, True])
@pytest.mark.parametrize("negative", [False, True])
def test_batch_dp_requires_consensus_to_skip_gather(tmp_path, grouped, negative):
mp.spawn(
run_text_encoding_dp,
args=(f"file://{tmp_path / 'dp-rendezvous'}", grouped, negative),
nprocs=2,
join=True,
)
class ShardedTextEncoder(FullHiddenStateEncoder):
def __init__(self):
torch.nn.Module.__init__(self)
self.projection = torch.nn.Linear(1, 8, bias=False)
torch.nn.init.ones_(self.projection.weight)
self.calls = 0
def forward(self, input_ids, **kwargs):
self.calls += 1
hidden = self.projection(input_ids[..., None].float())
return BaseEncoderOutput(last_hidden_state=hidden, hidden_states=(hidden,) * 32)
def run_fsdp_conditioning(rank, rendezvous):
os.environ["LOCAL_RANK"] = str(rank)
torch.cuda.set_device(rank)
init_distributed_environment(
world_size=2,
rank=rank,
local_rank=rank,
distributed_init_method=rendezvous,
backend="nccl",
timeout=60,
)
initialize_model_parallel(sequence_parallel_degree=2, ulysses_degree=2)
try:
encoder = ShardedTextEncoder().cuda().eval()
fully_shard(
encoder, mesh=init_device_mesh("cuda", (2,)), reshard_after_forward=True
)
assert encoder.projection.weight.to_local().numel() == 4
args = make_server_args(pipeline_config=make_text_config())
with patch(_GLOBAL_ARGS_PATCH, return_value=args):
stage = TextEncodingStage([encoder], [object()])
stage._text_encode_dp_group = Mock(return_value=None)
cache = ConditioningCache(4096)
with torch.no_grad(), cache.scope(cross_request=False):
for _ in range(2):
with cache.group_scope():
for attempt in range(3):
if rank == 0 and attempt == 1:
cache.clear()
output = stage.encode_text("hello", args, device=f"cuda:{rank}")
torch.testing.assert_close(
output[0][0][0, :, 0],
torch.tensor([5.0, 2.0], device=f"cuda:{rank}"),
rtol=0,
atol=0,
)
assert cache._group_entries.get() is None
assert encoder.calls == 4
assert cache.group_hits == 2
assert cache.bytes == 0
assert encoder.projection.weight.to_local().numel() == 4
finally:
destroy_model_parallel()
destroy_distributed_environment()
@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="requires two CUDA GPUs")
def test_fsdp_group_reuse_agrees_across_shards_and_expires(tmp_path):
mp.spawn(
run_fsdp_conditioning,
args=(f"file://{tmp_path / 'fsdp-rendezvous'}",),
nprocs=2,
join=True,
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
@torch.no_grad()
def test_cache_hit_releases_weights_retained_after_warmup(monkeypatch):
monkeypatch.setattr(
"sglang.multimodal_gen.runtime.managers.memory_managers."
"component_residency_strategies.get_local_torch_device",
lambda: torch.device("cuda"),
)
encoder = FullHiddenStateEncoder().eval()
config = make_text_config()
config.supports_auto_residency = False
args = make_server_args(
pipeline_config=config,
component_precisions={},
enable_layerwise_nvtx_marker=False,
)
with patch(_GLOBAL_ARGS_PATCH, return_value=args):
stage = TextEncodingStage([encoder], [object()])
stage._text_encode_dp_group = Mock(return_value=None)
pipeline = SimpleNamespace(
modules={"text_encoder": encoder},
_stage_name_mapping={"text": stage},
component_residency_strategies={},
)
manager = ComponentResidencyManager(pipeline, args)
strategy = ComponentOffloadStrategy()
manager.strategy_for = Mock(return_value=strategy)
stage.set_component_residency_manager(manager)
cache = ConditioningCache(4096)
for warmup in [True, False]:
batch = make_req(is_warmup=warmup)
manager.begin_request([stage], batch, args)
manager.before_stage(stage, 0, batch, args)
manager.begin_stage()
with cache.scope(refresh=warmup):
result = stage.encode_text(
"hello", args, device="cuda", return_attention_mask=True
)
manager.end_stage()
manager.finish_request()
assert encoder.weight.device.type == ("cuda" if warmup else "cpu")
assert result[0][0].device.type == "cuda"
assert encoder.calls == 1
assert cache.hits == 1