From dc1bd468020f8f543139d3f0e55d4d694e2d928c Mon Sep 17 00:00:00 2001 From: Mick Date: Tue, 29 Sep 2026 15:27:22 +0800 Subject: [PATCH] [diffusion] feat: support bounded exact conditioning cache across native models (#40470) Co-authored-by: Mick Qian --- .../diffusion/Qwen-Image/Qwen-Image-2.1.mdx | 9 +- docs/docs/sglang-diffusion/api/cli.mdx | 7 + .../sglang-diffusion/caching-acceleration.mdx | 56 +- .../performance-optimization.mdx | 1 + .../sglang-diffusion/support_new_models.mdx | 22 + python/sglang/multimodal_gen/README.md | 1 + .../runtime/cache/conditioning.py | 681 ++++++++++++++ .../memory_managers/component_manager.py | 8 + .../runtime/models/dits/cosmos3video.py | 2 + .../runtime/models/encoders/base.py | 16 +- .../runtime/models/encoders/gemma_3.py | 8 +- .../runtime/models/encoders/hunyuan3d.py | 13 +- .../runtime/models/encoders/qwen2_5vl.py | 3 + .../runtime/models/encoders/qwen3vl.py | 3 + .../neo_unify/modeling_neo_chat.py | 18 +- .../runtime/models/vaes/autoencoder.py | 2 + .../runtime/models/vaes/autoencoder_dc.py | 2 + .../models/vaes/autoencoder_kl_flux2.py | 2 + .../models/vaes/autoencoder_kl_qwenimage.py | 2 + .../runtime/models/vaes/common.py | 6 + .../multimodal_gen/runtime/models/vaes/dac.py | 2 + .../models/vaes/ltx_2_3_condition_encoder.py | 5 +- .../runtime/models/vaes/ltx_2_audio.py | 2 + .../runtime/models/vaes/ltx_2_vae.py | 2 + .../models/vaes/minimax_h3_video_vae/klvae.py | 2 + .../runtime/models/vaes/wanvae.py | 2 + .../runtime/pipelines/sana_video.py | 5 - .../executors/pipeline_executor.py | 25 +- .../runtime/pipelines_core/lora/pipeline.py | 15 + .../runtime/pipelines_core/stages/base.py | 6 + .../runtime/pipelines_core/stages/dedup.py | 21 +- .../stages/model_specific_stages/ideogram.py | 1 + .../lingbot_video_moe/text_encoding.py | 36 +- .../stages/model_specific_stages/longlive2.py | 10 +- .../model_specific_stages/ming_image.py | 42 +- .../model_specific_stages/qwen_image21.py | 25 +- .../model_specific_stages/sana_wm/base.py | 2 + .../stages/realtime/text_encoding.py | 10 +- .../pipelines_core/stages/text_encoding.py | 441 ++++----- .../runtime/pipelines_core/stages/vla.py | 10 +- .../runtime/post_training/weights_updater.py | 5 + .../runtime/server_args/server_args.py | 18 + .../test/unit/test_conditioning_cache.py | 611 ++++++++++++ .../test_conditioning_model_boundaries.py | 142 +++ .../test/unit/test_ideogram4.py | 3 + .../test/unit/test_lingbot_video_moe.py | 1 + .../test/unit/test_lora_pipeline.py | 30 + .../test/unit/test_pipeline_executor.py | 31 +- .../test/unit/test_qwen_image21.py | 76 ++ .../test/unit/test_sensenova_u1.py | 7 +- .../test/unit/test_text_encoding_cache.py | 872 +++++++++++++++++- 51 files changed, 2990 insertions(+), 332 deletions(-) create mode 100644 python/sglang/multimodal_gen/runtime/cache/conditioning.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_conditioning_cache.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_conditioning_model_boundaries.py diff --git a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1.mdx b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1.mdx index d91f70def8..95c75cd17a 100644 --- a/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1.mdx +++ b/docs/cookbook/diffusion/Qwen-Image/Qwen-Image-2.1.mdx @@ -117,9 +117,12 @@ Qwen-Image 2.1 supports text-to-image generation, single- and multi-image editin and RGBA output. Use one checkpoint for all modes. For multi-round editing, send the previous output as the next reference image. -The server does not retain conversation state. Condition-prefix KV caches are -reused within one request; cross-request and dialogue-history caching are not -implemented. +The server does not retain conversation state. Repeated prompts and reference +images can reuse exact encoder outputs and VAE posteriors through the default +[conditioning cache](/docs/sglang-diffusion/caching-acceleration#conditioning-cache). +Changing the prompt or image recomputes their joint embedding; unchanged images +can still reuse vision features. Condition-prefix KV reuse stays within each +request. Use `--disable-conditioning-cache` to turn off cross-request reuse. ## 3. Checkpoint layout diff --git a/docs/docs/sglang-diffusion/api/cli.mdx b/docs/docs/sglang-diffusion/api/cli.mdx index faf961c866..120835fb72 100644 --- a/docs/docs/sglang-diffusion/api/cli.mdx +++ b/docs/docs/sglang-diffusion/api/cli.mdx @@ -275,6 +275,13 @@ HTTP server-only arguments are ignored by `sglang generate`. For supported native pipelines, set `SGLANG_CACHE_DIT_ENABLED=true` to enable Cache-DiT. It can run with DiT layerwise offload; it cannot run with FSDP. For the diffusers backend, use `--backend diffusers --cache-dit-config ...`. See [Cache-DiT](/docs/sglang-diffusion/cache_dit). +Native conditioning reuse is enabled by default, independently of Cache-DiT: + +- `--disable-conditioning-cache`: disable cross-request text/image conditioning reuse and VAE posterior caching. Reuse within a grouped stage remains enabled. +- `--conditioning-cache-max-size-mb {MiB}`: shared CPU and device LRU capacity per executor/rank, default `512`; `0` disables cross-request caching. Consumed negative text conditioning stays on its original device; other entries live on the CPU. The cache holds at most 128 entries and bypasses oversized outputs. + +See [conditioning cache](/docs/sglang-diffusion/caching-acceleration#conditioning-cache) for model coverage, invalidation, and the distinction between cold and repeated-input latency. + For supported image pipelines, breakable CUDA graph can be enabled with `--enable-breakable-cuda-graph`, but you must declare every served resolution in `--warmup-resolutions` so warmup captures matching graph signatures. LongCat-Image is supported by this path. Its DiT always consumes a fixed diff --git a/docs/docs/sglang-diffusion/caching-acceleration.mdx b/docs/docs/sglang-diffusion/caching-acceleration.mdx index 041898339f..f96e70ebbc 100644 --- a/docs/docs/sglang-diffusion/caching-acceleration.mdx +++ b/docs/docs/sglang-diffusion/caching-acceleration.mdx @@ -1,13 +1,63 @@ --- title: "Caching Acceleration" description: "Compare caching acceleration strategies for diffusion models." -tag: "approx" --- -SGLang provides three complementary caching strategies for Diffusion Transformer (DiT) models. All reduce denoising cost by skipping redundant computation, but they operate at different levels. +SGLang can reuse exact conditioning across requests and cache intermediate denoising results. Conditioning reuse preserves the encoded values; denoising caches such as Cache-DiT, TeaCache, and Spectrum use approximations. + +## Conditioning cache + +Native pipelines enable a conditioning cache by default. Repeated inputs can reuse text embeddings (including negative prompts), image or vision-language encoder outputs, and VAE posteriors. The shared LRU holds at most **512 MiB per executor/rank** across CPU and device entries, with at most 128 entries. The common text stage keeps consumed negative conditioning on its original device to avoid repeated CPU-to-GPU transfers and postprocessing. Other entries live on the CPU. Oversized results are not admitted. + +Within a grouped or batched stage, the same conditioning boundary can reuse a device result before consulting the cross-request cache. The common text stage and the Qwen-Image 2.1, Ming-Image, and LingBot-Video text stages share read-only consumed tensors and give each request its own containers. Other cached boundaries return private copies, with device snapshots limited to 512 MiB and 128 entries per stage. These temporary references are released when the stage finishes or fails. Group reuse remains available when cross-request caching is disabled or its budget is too small. + +The standard text encoding stage stores the final embeddings, pooled outputs, masks, and sequence lengths. A hit skips encoder weight preparation, token uploads, and mask construction; CPU tokenization still runs to match the inputs. Qwen-Image 2.1 also caches its consumed prompt embeddings and image-token slots before processor execution, while retaining separate vision-feature reuse for changed edit instructions. Other custom stages and batch-DP encoding retain caching at the native encoder boundary. + +CPU hits restore outputs to their original devices; device hits return private device copies. Negative conditioning can therefore retain GPU memory between requests, within the shared cache budget. Measure memory as well as latency when using a tight VRAM budget. + +This is most useful when generating several variations from the same prompt or reference image, especially with a layerwise-offloaded encoder. A changed edit instruction recomputes the joint vision-language embedding, while the unchanged image can still reuse its vision features and VAE posterior. Separate CLI invocations start separate processes and cannot share this cache. Denoising and output decoding still run for every request, so end-to-end gains depend on how much time the original request spends encoding conditioning. + +To disable cross-request conditioning reuse: + +```bash +sglang serve --model-path Qwen/Qwen-Image-2.1 --disable-conditioning-cache +``` + +To increase the shared cache budget: + +```bash +sglang serve --model-path Qwen/Qwen-Image-2.1 --conditioning-cache-max-size-mb 1024 +``` + +The same settings are accepted by `DiffGenerator.from_pretrained()` as `disable_conditioning_cache` and `conditioning_cache_max_size_mb`. A capacity of `0` also disables both cross-request tiers. Neither setting disables conditioning reuse within a grouped stage or changes per-output seeds. + +### Matching and correctness + +- Keys include the loaded encoder instance, actual input contents, shapes, dtypes, masks, and autocast precision. Image tensors and PIL images are matched by content, so replacing a file at the same path cannot reuse stale pixels. Alpha is part of the VAE input. +- Positive and negative inputs use the same bounded cache. Different seeds can reuse conditioning; initial noise is still generated for each request. +- VAE caching stores the posterior before sampling and normalization. Every request still calls `sample()` or `mode()` as its model requires, preserving the random-number sequence. +- Cross-request hits return independent copies. Grouped consumed outputs may share read-only tensors; other boundaries isolate mutable tensors and posteriors. In-place normalization and downstream mutations cannot change a persistent entry. +- Encoder TP and folding groups agree on hits before skipping collective work. Weight updates and LoRA changes invalidate the cache, including failed updates that may have modified weights. +- Warmup refreshes cached results and executes each distinct encoding within a group once. This primes kernels while preserving multi-output reuse; subsequent requests can reuse conditioning such as the model-default negative prompt. Compare disabled, cold, and repeated-input latency separately when benchmarking. + +### Coverage and limits + +The common encoder boundary covers native CLIP, T5/UMT5, Llama, Mistral 3, Gemma 2/3, Qwen 2.5-VL, Qwen 3/3-VL, Ideogram, and MiniMax-H3 encoders, including custom pipeline stages. Qwen-VL and Gemma 3 also cache vision features independently of text. Hunyuan3D image encoders and the LTX-2.3 video condition encoder participate. Native image, video, and audio VAE encode entry points cache deterministic tensors or supported posterior containers. + +Cosmos3 has no separate text encoder: its deterministic understanding pathway caches immutable per-layer K/V results, which are copied into fresh request state. SenseNova-U1 caches reference-image vision features; its noisy-image vision path still runs at each denoising step. VLA keeps its model-owned observation/prefix cache. + +FSDP disables cross-request caching and caching inside sharded modules. Consumed text-conditioning outputs still support group-local reuse, with all ranks agreeing before skipping the encoder. The common text stage also reuses consumed outputs from library fallback encoders within a group and keeps their negative conditioning device-hot across requests. Batch-DP can skip encoding and its output gather only when all participating ranks have the consumed result; this includes device-hot negatives. Otherwise each encoder copy retains its normal cache and gather behavior. + +Stateful autoregressive calls (`use_cache=True` or existing `past_key_values`), training/autograd, CUDA Graph capture, unsupported tensor or output types, and distributed VAE encoding bypass the shared caches. External SRT encoders and the Diffusers pipeline backend keep their own cache behavior. The global disable flag also disables existing realtime text and VLA prefix reuse; those model-specific caches otherwise retain their own memory settings. + +Joint vision-language embeddings depend on both the prompt and image. The cache does not reuse a previous turn's combined embedding when either changes, and does not carry mutable denoising KV state between requests. + +Dynamic batching still decides which requests can execute together. Conditioning reuse does not change encoder batch shapes, padding, or output order: a batched encoder call is matched as a whole, rather than split into individually cached rows. Model-specific grouped execution that also prepares materials, broadcasts results, or manages state retains its own execution rules. + +The common text stage also keeps group-local stage deduplication: equivalent requests execute the stage once and receive their own output containers. This avoids repeating tokenization and stage bookkeeping. When stage deduplication leaves only one computation, it also skips the temporary group cache and its hit synchronization; cross-request caching remains available. The conditioning cache shares encodings between otherwise different requests, including a common negative prompt. Custom stages that prepare request-specific metadata or realtime session state use encoder-result reuse while still preparing each request. ## Overview -SGLang supports three complementary caching approaches: +The following approaches reduce denoising computation: diff --git a/docs/docs/sglang-diffusion/performance-optimization.mdx b/docs/docs/sglang-diffusion/performance-optimization.mdx index abc900c94a..78dcc4ed3a 100644 --- a/docs/docs/sglang-diffusion/performance-optimization.mdx +++ b/docs/docs/sglang-diffusion/performance-optimization.mdx @@ -179,4 +179,5 @@ Performance results are configuration-dependent. Record the exact checkpoint rev - [Fused Kernels](/docs/sglang-diffusion/fused_kernels) - [Sequence Parallelism](/docs/sglang-diffusion/ring_sp_performance) - [Caching Strategies](/docs/sglang-diffusion/caching-acceleration) +- [Exact conditioning reuse across requests](/docs/sglang-diffusion/caching-acceleration#conditioning-cache) - [Profiling](/docs/sglang-diffusion/profiling) diff --git a/docs/docs/sglang-diffusion/support_new_models.mdx b/docs/docs/sglang-diffusion/support_new_models.mdx index 5528211f64..e12d428195 100644 --- a/docs/docs/sglang-diffusion/support_new_models.mdx +++ b/docs/docs/sglang-diffusion/support_new_models.mdx @@ -166,6 +166,28 @@ Use the model's Diffusers pipeline, official implementation, or If the new model is close to Flux, Qwen-Image, GLM-Image, Wan, HunyuanVideo, or LTX, extend that implementation before starting from an empty file. +## Conditioning reuse + +Use the native `TextEncoder` / `ImageEncoder` base classes (or `EncoderTensorParallelMixin`) and call encoders through `encoder(...)`. This preserves the shared conditioning cache, module hooks, and encoder TP/folding group. A replicated custom encoder can use `ConditioningEncoderMixin`; do not apply it to a module with hidden mutable state or random sampling in `forward()`. + +The standard `TextEncodingStage` caches postprocessed embeddings, conditioning masks, sequence lengths, and pooled outputs instead of unused intermediate hidden states. Keep its postprocessing deterministic. When adding a stage-level `cached_encoder_call`, place deterministic preprocessing and component preparation inside the compute callback and give the stage a `namespace`. The namespace avoids a duplicate full encoder entry while preserving nested vision-method caches. Declare conditional component uses with `start_at_stage_entry=False`, so the executor cannot load weights before checking the cache. Keep calling `encoder(...)` inside the callback to preserve hooks and parallel-group handling. After retrieving the conditioning, call `finish_unused_declared_component()` to release weights retained by warmup or the previous request when no encoder use began; an active use remains owned by the residency manager. Batch-DP retains per-copy caches. A gathered group result may skip encoding and the output gather only after hit consensus across the whole replica; a miss keeps every rank in the normal gather path. + +For a custom VAE, apply `cached_vae_encode` only to the deterministic encode method returning a tensor or posterior. Keep posterior sampling, normalization, initial noise, scheduler state, and request-specific KV dictionaries outside that boundary. Unsupported outputs bypass caching; explicitly register a tensor-only posterior container with `register_conditioning_container` when needed. Never register an autoregressive cache or a session object as a posterior container. + +For integrated models without a separate encoder, use `cached_conditioning` on a pure conditioning method, as in Cosmos3's understanding pathway or SenseNova-U1's reference-image features. Every rank in the current encoder TP group must enter that method together. Do not decorate the noisy-image or denoising path. + +Prefer these function boundaries to whole-stage deduplication. A stage may also write per-request metadata, advance RNG state, release materials, or execute collectives; these actions must still run for each request. The executor gives each grouped or batched stage a temporary device-result scope, using the same keys and invalidation as the cross-request cache. Use `share_in_group=True` only for consumed tensors that remain read-only across requests; containers are copied. Other outputs get private tensor snapshots with a bounded temporary budget. The scope ends at stage exit, including exceptions, and remains available with `--disable-conditioning-cache`. + +Wrap negative encoding in `prefer_conditioning_cache()` to prioritize reusable conditioning. At a consumed `cached_encoder_call` boundary with a stage namespace and `share_in_group=True`, this keeps a private device snapshot across requests, including library encoders and gathered batch-DP results. Hits skip encoder preparation and postprocessing, with the same rank consensus and invalidation as CPU entries. Device and CPU entries share one byte and entry budget; raw intermediate encoder states remain on the CPU. Avoid adding a separate model-owned negative cache. + +Do not put `Req`, scheduler instances, sampled latents, or session state into a cross-request cache. Keep model-specific grouped execution for parallel work distribution and material ownership. Preserve batch shapes and padding when matching batched encoder calls; per-row regrouping needs separate numerical validation. + +The common `TextEncodingStage` retains stage deduplication to avoid repeated tokenization and stage bookkeeping. If you override its `forward`, declare a complete stage-output contract or set `deduplicated_output_fields = ()` and use conditioning-result reuse. Do not inherit the common stage's output list when your stage also writes model-specific metadata. Keep request-local containers independent even when their tensors are shared. + +Test cache-on versus cache-off with identical seeds, changed prompts/masks/reference pixels (including alpha), downstream in-place mutations, changed weights, and rank-local eviction under supported parallel groups. Also exercise grouped requests through the executor with a zero or undersized cache budget: check encoder call counts, per-request metadata, output ownership, and group cleanup on failure. Include the new model in the [conditioning cache coverage](/docs/sglang-diffusion/caching-acceleration#coverage-and-limits). + +Preserve existing grouped-request encoding reuse when cross-request caching is unavailable. Use `cross_request=False` for boundaries that support only group-local reuse, such as library fallback encoders and gathered batch-DP outputs. FSDP scopes allow consumed-output group reuse while bypassing nested encoder caches. A boundary containing collectives needs hit consensus across every participating rank, including FSDP shards and encoder DP copies; an encoder TP group alone may be insufficient. Test these bypass paths and single-rank invalidation before removing stage deduplication. + ## Choose a Pipeline Shape SGLang-Diffusion uses `ComposedPipelineBase` to wire stages together. Most diff --git a/python/sglang/multimodal_gen/README.md b/python/sglang/multimodal_gen/README.md index b8b390d484..8cdc788533 100644 --- a/python/sglang/multimodal_gen/README.md +++ b/python/sglang/multimodal_gen/README.md @@ -12,6 +12,7 @@ SGLang Diffusion has the following features: - Broad model support: Wan, FastWan, FLUX, Qwen-Image / Qwen-Image 2.1, LongCat-Image, Z-Image, Anima, Ideogram 4, Krea-2, Cosmos3, LTX-2/LTX-2.3/LTX-2.5, MiniMax-H3, FastH3, VDN-H3, LingBot Video MoE, LingBot World, SANA-Video/SANA-WM, JoyEcho, MOVA, GLM-Image, ERNIE-Image, Hunyuan3D, and more - Fast inference speed: empowered by optimized `sgl-kernel` kernels, scheduler/runtime improvements, caching acceleration, and native diffusion hot-path optimizations - Ease of use: OpenAI-compatible api, CLI, python sdk, and a [ComfyUI plugin](apps/ComfyUI_SGLDiffusion/README.md) + - [Conditioning reuse](https://docs.sglang.io/docs/sglang-diffusion/caching-acceleration#conditioning-cache): bounded CPU caching for repeated text/image inputs and VAE posteriors, with `--disable-conditioning-cache` to opt out - Multi-platform support: - NVIDIA GPUs (H100, H200, A100, B200, 4090, 5090) - AMD GPUs (MI300X, MI325X, MI355X) diff --git a/python/sglang/multimodal_gen/runtime/cache/conditioning.py b/python/sglang/multimodal_gen/runtime/cache/conditioning.py new file mode 100644 index 0000000000..dc59bbddea --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/cache/conditioning.py @@ -0,0 +1,681 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Exact conditioning reuse within grouped stages and across requests.""" + +import copy +import hashlib +import pickle +import weakref +from collections import OrderedDict +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass, fields, is_dataclass +from functools import wraps +from typing import Callable + +import numpy as np +import torch +import torch.distributed as dist +from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution +from PIL import Image + +from sglang.multimodal_gen.runtime.distributed import ( + get_replica_group, + get_tp_group, + get_world_size, + model_parallel_is_initialized, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) +_active_cache: ContextVar["ConditioningCache | None"] = ContextVar( + "conditioning_cache", default=None +) +_refresh_cache: ContextVar[bool] = ContextVar( + "refresh_conditioning_cache", default=False +) +_cross_request_cache: ContextVar[bool] = ContextVar( + "cross_request_conditioning_cache", default=True +) +_prefer_cache: ContextVar[bool] = ContextVar("prefer_conditioning_cache", default=False) +_stage_encoder: ContextVar[object | None] = ContextVar("stage_encoder", default=None) +_container_types: set[type] = {DiagonalGaussianDistribution} +_live_caches = weakref.WeakSet() +_weights_epoch = 0 + + +def invalidate_conditioning_caches(modules=None): + """Invalidate before weight mutations, including partially failed updates.""" + global _weights_epoch + _weights_epoch += 1 + parameters = ( + None + if modules is None + else {id(p) for module in modules for p in module.parameters()} + ) + for cache in _live_caches: + cache.invalidate(parameters) + + +def conditioning_weights_epoch(): + return _weights_epoch + + +def register_conditioning_container(cls): + """Opt in a tensor-only posterior container whose fields can be copied.""" + _container_types.add(cls) + return cls + + +class Uncacheable(TypeError): + pass + + +@contextmanager +def conditioning_cache_group(*, enabled): + cache = _active_cache.get() + if cache is None: + yield + else: + with cache.group_scope(enabled=enabled): + yield + + +@contextmanager +def prefer_conditioning_cache(): + # retain reusable negative conditioning ahead of changing positive prompts + token = _prefer_cache.set(True) + try: + yield + finally: + _prefer_cache.reset(token) + + +def _fingerprint(value): + if isinstance(value, torch.Tensor): + if ( + type(value) is not torch.Tensor + or value.layout != torch.strided + or value.requires_grad + ): + raise Uncacheable("only dense inference tensors can be cached") + tensor = value.detach().cpu().contiguous() + data = tensor.reshape(-1).view(torch.uint8).numpy() + return ( + "tensor", + str(value.dtype), + str(value.device), + tuple(value.shape), + tuple(value.stride()), + hashlib.sha256(data).digest(), + ) + if isinstance(value, Image.Image): + return ( + "image", + value.mode, + value.size, + hashlib.sha256(value.tobytes()).digest(), + _fingerprint(value.getpalette()), + _fingerprint(value.info.get("transparency")), + ) + if isinstance(value, np.ndarray): + if value.dtype.hasobject: + raise Uncacheable("object arrays are not conditioning values") + return ( + "array", + value.dtype.str, + value.shape, + hashlib.sha256(value.tobytes()).digest(), + ) + if type(value) in (str, bytes, int, float, bool, type(None)): + return (type(value).__name__, value) + if isinstance(value, (torch.dtype, torch.device)): + return (type(value).__name__, str(value)) + if isinstance(value, dict): + return ( + "dict", + tuple((_fingerprint(k), _fingerprint(v)) for k, v in value.items()), + ) + if isinstance(value, (tuple, list)): + return (type(value).__name__, tuple(_fingerprint(v) for v in value)) + raise Uncacheable(f"unsupported conditioning input: {type(value).__name__}") + + +@dataclass +class _HostTensor: + data: torch.Tensor + device: torch.device + + +@dataclass +class _CacheEntry: + output: object + size: int + ready: tuple[torch.cuda.Event, ...] + owner: int + preferred: bool + device_resident: bool = False + + @classmethod + def snapshot(cls, output, size, owner, preferred, device_resident): + tensors = {} + copy_streams = {} + + def snapshot_tensor(t): + if id(t) not in tensors: + if t.device.type == "cuda": + copy_streams[t.device] = torch.cuda.current_stream(t.device) + if device_resident: + stored = t.detach().clone() + elif t.device.type == "cuda": + stored = torch.empty_like(t, device="cpu", pin_memory=True) + stored.copy_(t.detach(), non_blocking=True) + else: + stored = t.detach().to("cpu", copy=True) + tensors[id(t)] = ( + stored if device_resident else _HostTensor(stored, t.device) + ) + return tensors[id(t)] + + stored = _map_output(output, snapshot_tensor) + # snapshot before downstream mutations on each producing stream + ready = [] + for stream in copy_streams.values(): + event = torch.cuda.Event() + event.record(stream) + ready.append(event) + return cls(stored, size, tuple(ready), owner, preferred, device_resident) + + def wait(self): + if self.device_resident: + for event in self.ready: + torch.cuda.current_stream(event.device).wait_event(event) + + def record(t): + if t.device.type == "cuda": + t.record_stream(torch.cuda.current_stream(t.device)) + return t + + _map_output(self.output, record) + return + for event in self.ready: + event.synchronize() + + +@dataclass +class _GroupEntry: + output: object + owner: int + copied_bytes: int + streams: tuple[torch.cuda.Stream, ...] + + def wait(self): + consumers = {} + for producer in self.streams: + consumer = torch.cuda.current_stream(producer.device) + if consumer != producer: + consumer.wait_stream(producer) + consumers[producer.device] = consumer + + if not consumers: + return + + def record(t): + if t.device in consumers: + t.record_stream(consumers[t.device]) + return t + + _map_output(self.output, record) + + +def _copy_output(value, *, share_tensors=False): + tensors = {} + + def copy_tensor(t): + if id(t) not in tensors: + tensors[id(t)] = t if share_tensors else t.clone() + return tensors[id(t)] + + return _map_output(value, copy_tensor) + + +def _map_output(value, tensor_fn, *, restore=False): + if isinstance(value, _HostTensor if restore else torch.Tensor): + return tensor_fn(value) + if type(value) in (str, bytes, int, float, bool, type(None)): + return value + if isinstance(value, (torch.dtype, torch.device)): + return value + # ModelOutput is both a dataclass and a dict. Reconstruct it as a dataclass + # so attribute access and tuple indexing retain their original semantics. + if is_dataclass(value) and not isinstance(value, type): + return type(value)( + **{ + field.name: _map_output( + vars(value)[field.name], tensor_fn, restore=restore + ) + for field in fields(value) + } + ) + if type(value) in _container_types: + result = copy.copy(value) + for name, item in vars(value).items(): + setattr(result, name, _map_output(item, tensor_fn, restore=restore)) + return result + if type(value) is dict: + return {k: _map_output(v, tensor_fn, restore=restore) for k, v in value.items()} + if type(value) is list: + return [_map_output(v, tensor_fn, restore=restore) for v in value] + if isinstance(value, tuple): + items = [_map_output(v, tensor_fn, restore=restore) for v in value] + return type(value)(*items) if type(value) is not tuple else tuple(items) + raise Uncacheable(f"unsupported conditioning output: {type(value).__name__}") + + +class ConditioningCache: + """One bounded cache per rank with host entries and device-hot conditioning.""" + + def __init__(self, max_bytes: int): + if max_bytes < 0: + raise ValueError("conditioning cache capacity must be nonnegative") + _live_caches.add(self) + self.max_bytes = max_bytes + self.bytes = 0 + self.hits = self.misses = self.evictions = self.bypasses = 0 + self._entries = OrderedDict() + self._models = weakref.WeakKeyDictionary() + self._next_model = 0 + self._group_entries = ContextVar("conditioning_group_entries", default=None) + self.group_hits = 0 + + def clear(self): + group_entries = self._group_entries.get() + if group_entries is not None: + group_entries.clear() + for entry in self._entries.values(): + entry.wait() + self._entries.clear() + self.bytes = 0 + + def invalidate(self, parameters): + if parameters is None: + self.clear() + return + # shared parameters invalidate both owners, including nested encoders + owners = { + identity + for model, identity in self._models.items() + if isinstance(model, torch.nn.Module) + and any(id(p) in parameters for p in model.parameters()) + } + for key, entry in list(self._entries.items()): + if entry.owner in owners: + entry.wait() + self.bytes -= entry.size + del self._entries[key] + group_entries = self._group_entries.get() + if group_entries is not None: + for key, entry in list(group_entries.items()): + if entry.owner in owners: + del group_entries[key] + + def stats(self): + return dict( + hits=self.hits, + misses=self.misses, + evictions=self.evictions, + bypasses=self.bypasses, + entries=len(self._entries), + bytes=self.bytes, + group_hits=self.group_hits, + device_bytes=sum( + entry.size for entry in self._entries.values() if entry.device_resident + ), + ) + + def _identity(self, owner): + if owner not in self._models: + self._models[owner] = self._next_model + self._next_model += 1 + return self._models[owner] + + @contextmanager + def group_scope(self, enabled=True): + """Keep device results only while a stage executes a group of requests.""" + token = self._group_entries.set({} if enabled else None) + try: + yield + finally: + self._group_entries.reset(token) + + def _remember_group(self, key, output, owner, share_in_group): + entries = self._group_entries.get() + if entries is None: + return + tensors = {} + + def visit(t): + tensors[id(t)] = t + return t + + _map_output(output, visit) + size = ( + 0 + if share_in_group + else sum(t.numel() * t.element_size() for t in tensors.values()) + ) + # private snapshots must not retain unbounded intermediate encoder states + if not share_in_group and ( + len(entries) >= 128 + or size + sum(entry.copied_bytes for entry in entries.values()) + > 512 * 1024**2 + ): + return + stored = _copy_output(output, share_tensors=share_in_group) + streams = tuple( + torch.cuda.current_stream(device) + for device in { + t.device for t in tensors.values() if t.device.type == "cuda" + } + ) + entries[key] = _GroupEntry(stored, owner, size, streams) + + @contextmanager + def scope(self, enabled=True, *, refresh=False, cross_request=True): + # Zero-capacity ranks still participate in encoder hit consensus. + if refresh and self._group_entries.get() is not None: + self._group_entries.get().clear() + token = _active_cache.set(self if enabled else None) + refresh_token = _refresh_cache.set(refresh or _refresh_cache.get()) + cross_request_token = _cross_request_cache.set( + cross_request and _cross_request_cache.get() + ) + try: + yield + finally: + _cross_request_cache.reset(cross_request_token) + _refresh_cache.reset(refresh_token) + _active_cache.reset(token) + + def run( + self, + model, + method, + args, + kwargs, + compute: Callable, + group=None, + *, + nested=False, + namespace=None, + share_in_group=False, + cross_request=True, + keep_on_device=False, + ): + group_entries = self._group_entries.get() + cross_request = cross_request and _cross_request_cache.get() + if not cross_request and group_entries is None: + return compute() + try: + if (not cross_request or not self.max_bytes) and group_entries is None: + raise Uncacheable("cache disabled") + if kwargs.get("use_cache") or kwargs.get("past_key_values") is not None: + raise Uncacheable("stateful autoregressive encoding") + parameter = next(model.parameters(), None) + precision = str(parameter.dtype) if parameter is not None else None + key = ( + self._identity(model), + self._identity(namespace) if namespace is not None else None, + method, + share_in_group, + precision, + torch.is_autocast_enabled("cuda"), + torch.get_autocast_dtype("cuda"), + torch.is_autocast_enabled("cpu"), + torch.get_autocast_dtype("cpu"), + _fingerprint(args), + _fingerprint(kwargs), + ) + key = hashlib.sha256(pickle.dumps(key)).digest() + except Uncacheable: + key = None + entry = self._entries.get(key) if cross_request else None + group_entry = group_entries.get(key) if group_entries is not None else None + hit = group_entry is not None or ( + entry is not None and not _refresh_cache.get() + ) + if group is not None and group.world_size > 1: + # Encoders may issue TP/folding collectives. A rank-local eviction + # must never leave another rank returning early from the encoder. + flag = torch.tensor(int(hit), dtype=torch.int32) + dist.all_reduce(flag, op=dist.ReduceOp.MIN, group=group.cpu_group) + hit = bool(flag.item()) + if hit: + self.hits += 1 + if entry is not None: + entry.preferred |= _prefer_cache.get() + self._entries.move_to_end(key) + if group_entry is not None: + self.group_hits += 1 + group_entry.wait() + return _copy_output(group_entry.output, share_tensors=share_in_group) + entry.wait() + logger.debug("Conditioning cache hit: %s.%s", type(model).__name__, method) + restored = {} + + def restore(t): + if id(t) not in restored: + restored[id(t)] = t.data.to(t.device, copy=True, non_blocking=True) + return restored[id(t)] + + output = ( + _copy_output(entry.output) + if entry.device_resident + else _map_output(entry.output, restore, restore=True) + ) + if keep_on_device and not entry.device_resident: + self._entries[key] = _CacheEntry.snapshot( + output, entry.size, entry.owner, True, True + ) + self._remember_group(key, output, entry.owner, share_in_group) + return output + if key is None: + self.bypasses += 1 + return compute() + self.misses += 1 + # A VLM may reuse image features even when its joint text/image key + # misses. VAE delegates share one outer posterior entry instead. + with self.scope(enabled=nested): + output = compute() + try: + size = 0 + seen = set() + + def count(t): + nonlocal size + if id(t) not in seen: + size += t.numel() * t.element_size() + seen.add(id(t)) + if type(t) is not torch.Tensor or t.requires_grad: + raise Uncacheable("autograd output") + return t + + _map_output(output, count) + self._remember_group(key, output, self._identity(model), share_in_group) + if not cross_request or not size or size > self.max_bytes: + self.bypasses += 1 + return output + except Uncacheable: + self.bypasses += 1 + return output + old = self._entries.pop(key, None) + if old is not None: + old.wait() + self.bytes -= old.size + preferred = _prefer_cache.get() + evictable = [ + key + for key, entry in self._entries.items() + if preferred or not entry.preferred + ] + available = ( + self.max_bytes + - self.bytes + + sum(self._entries[key].size for key in evictable) + ) + if size > available or (len(self._entries) >= 128 and not evictable): + self.bypasses += 1 + return output + for evicted in evictable: + if self.bytes + size <= self.max_bytes and len(self._entries) < 128: + break + removed = self._entries.pop(evicted) + removed.wait() + self.bytes -= removed.size + self.evictions += 1 + self._entries[key] = _CacheEntry.snapshot( + output, size, self._identity(model), preferred, keep_on_device + ) + self.bytes += size + logger.debug( + "Conditioning cache store: %s.%s, %d bytes", + type(model).__name__, + method, + size, + ) + return output + + +def _inference_cache(model, *, share_in_group=False, cross_request=True): + if torch.compiler.is_compiling(): + return None + cache = _active_cache.get() + if cache is None or model.training or torch.is_grad_enabled(): + return None + # FSDP reuses only consumed stage outputs, never sharded intermediate states + if not (cross_request and _cross_request_cache.get()) and ( + not share_in_group or cache._group_entries.get() is None + ): + return None + # graph capture cannot hash or copy CUDA values through host memory + if torch.cuda.is_available() and torch.cuda.is_current_stream_capturing(): + return None + return cache + + +def cached_encoder_call( + model, + args, + kwargs, + compute, + group=None, + *, + namespace=None, + nested=True, + share_in_group=False, + cross_request=True, +): + if torch.compiler.is_compiling(): + return compute() + # only consumed negative conditioning is device-hot; raw hidden states and + # other encoder boundaries keep their host-cache policy + keep_on_device = namespace is not None and share_in_group and _prefer_cache.get() + cross_request = cross_request or keep_on_device + cache = _inference_cache( + model, share_in_group=share_in_group, cross_request=cross_request + ) + if ( + cache is None + or model is _stage_encoder.get() + or kwargs.get("use_cache") + or kwargs.get("past_key_values") is not None + ): + return compute() + + if not _cross_request_cache.get() and model_parallel_is_initialized(): + # the encoder's TP group may not cover its FSDP shard group + group = get_replica_group() + + def compute_conditioning(): + # a stage owns the consumed output; preserve nested vision-method caches + token = _stage_encoder.set(model) + try: + return compute() + finally: + _stage_encoder.reset(token) + + return cache.run( + model, + "forward", + args, + kwargs, + compute_conditioning if namespace is not None and cross_request else compute, + group, + nested=nested, + namespace=namespace, + share_in_group=share_in_group, + cross_request=cross_request, + keep_on_device=keep_on_device, + ) + + +def cached_conditioning(fn): + """Cache a deterministic conditioning method on the current encoder TP group.""" + + @wraps(fn) + def wrapped(self, *args, **kwargs): + cache = _inference_cache(self) + if cache is None: + return fn(self, *args, **kwargs) + group = get_tp_group() if model_parallel_is_initialized() else None + return cache.run( + self, fn.__name__, args, kwargs, lambda: fn(self, *args, **kwargs), group + ) + + return wrapped + + +def cached_vae_encode(fn): + """Cache the posterior, before sample()/mode() and latent normalization. + + Distributed VAEs currently bypass this cache: their encoding methods can + run on an owner rank or a model-specific subgroup, unlike encoder TP. + """ + + @wraps(fn) + def wrapped(self, *args, **kwargs): + cache = _inference_cache(self) + if cache is None or (model_parallel_is_initialized() and get_world_size() > 1): + return fn(self, *args, **kwargs) + # Tiling and slicing can change numerical results without changing x. + settings = { + name: value + for name, value in vars(self).items() + if name.startswith( + ( + "use_", + "tile_", + "blend_", + "parallel_", + "spatial_compression", + "temporal_compression", + ) + ) + and type(value) in (int, float, bool, str, type(None)) + } + return cache.run( + self, + fn.__name__, + (args, settings), + kwargs, + lambda: fn(self, *args, **kwargs), + ) + + return wrapped + + +class ConditioningEncoderMixin: + """Cache replicated encoders without adding a module or altering state_dict.""" + + def __call__(self, *args, **kwargs): + forward = super().__call__ + return cached_encoder_call(self, args, kwargs, lambda: forward(*args, **kwargs)) diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py index 450a3ca746..a83aab6195 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py @@ -456,6 +456,14 @@ class ComponentResidencyManager: """Prepare a shared component and wait without making it the active use.""" self._prepare_forward_use(use, module=module) + def finish_unused_component( + self, use: ComponentUse, module: nn.Module | None = None + ) -> None: + """Release retained weights when conditioning reuse skips their use.""" + if self._active_use is not None and self._same_use(self._active_use, use): + return + self._finish_use(use, module=module, keep_on_warmup=False, force=True) + def remove_nvtx_hooks_for_module(self, module: nn.Module | None) -> None: """Detach NVTX hooks before a component object is deleted or replaced.""" if module is None: diff --git a/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py b/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py index 019b6970d9..bcd45d75c3 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py @@ -21,6 +21,7 @@ from sglang.kernels.ops.diffusion import ( ) from sglang.multimodal_gen.configs.models.dits.cosmos3video import Cosmos3VideoConfig from sglang.multimodal_gen.configs.models.fsdp import is_module_list_entry_in +from sglang.multimodal_gen.runtime.cache.conditioning import cached_conditioning from sglang.multimodal_gen.runtime.distributed import ( get_sp_group, get_sp_world_size, @@ -1099,6 +1100,7 @@ class Cosmos3LanguageModel(nn.Module): ] ) + @cached_conditioning def forward( self, text_ids: torch.Tensor, diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/base.py b/python/sglang/multimodal_gen/runtime/models/encoders/base.py index 7fc01c04b3..21eaae1905 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/base.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/base.py @@ -13,11 +13,13 @@ from sglang.multimodal_gen.configs.models.encoders import ( ImageEncoderConfig, TextEncoderConfig, ) +from sglang.multimodal_gen.runtime.cache.conditioning import cached_encoder_call from sglang.multimodal_gen.runtime.distributed import ( get_replica_group, get_sp_group, get_tp_group, get_world_group, + model_parallel_is_initialized, ) from sglang.multimodal_gen.runtime.distributed.group_coordinator import GroupCoordinator from sglang.multimodal_gen.runtime.distributed.parallel_state import ( @@ -178,10 +180,20 @@ class EncoderTensorParallelMixin: def __call__(self, *args, **kwargs): tp_group = self._encoder_tp_group + forward = super().__call__ + cache_group = tp_group + if cache_group is None and model_parallel_is_initialized(): + cache_group = get_tp_group() + + def run(): + return cached_encoder_call( + self, args, kwargs, lambda: forward(*args, **kwargs), cache_group + ) + if tp_group is None: - return super().__call__(*args, **kwargs) + return run() with use_tensor_parallel_group(tp_group): - return super().__call__(*args, **kwargs) + return run() class TextEncoder( diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py b/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py index a1e21953ee..63da9eda6c 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py @@ -11,6 +11,7 @@ from torch import nn from sglang.multimodal_gen.configs.models.encoders.base import BaseEncoderOutput from sglang.multimodal_gen.configs.models.encoders.gemma_3 import Gemma3Config +from sglang.multimodal_gen.runtime.cache.conditioning import cached_conditioning from sglang.multimodal_gen.runtime.distributed import get_tp_world_size from sglang.multimodal_gen.runtime.layers.activation import GeluAndMul from sglang.multimodal_gen.runtime.layers.linear import ( @@ -772,8 +773,7 @@ class Gemma3ForConditionalGeneration( elif pixel_values.dim() != 4: raise ValueError(f"Unexpected pixel_values shape: {pixel_values.shape}") - vision_outputs = self.vision_tower(pixel_values) - image_features = self.multi_modal_projector(vision_outputs) + image_features = self._encode_image_features(pixel_values) image_features = image_features.to( device=inputs_embeds.device, dtype=inputs_embeds.dtype ) @@ -788,6 +788,10 @@ class Gemma3ForConditionalGeneration( llm_input_ids, inputs_embeds=inputs_embeds, **kwargs ) + @cached_conditioning + def _encode_image_features(self, pixel_values): + return self.multi_modal_projector(self.vision_tower(pixel_values)) + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]) -> Set[str]: loaded_params: Set[str] = set() params_dict = dict(self.named_parameters()) diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/hunyuan3d.py b/python/sglang/multimodal_gen/runtime/models/encoders/hunyuan3d.py index f5548aa54b..69f2431f81 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/hunyuan3d.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/hunyuan3d.py @@ -11,6 +11,7 @@ from transformers import ( Dinov2Model, ) +from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningEncoderMixin from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( LayerwiseOffloadableModuleMixin, ) @@ -32,7 +33,9 @@ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): return np.concatenate([emb_sin, emb_cos], axis=1) -class ImageEncoder(nn.Module, LayerwiseOffloadableModuleMixin): +class ImageEncoder( + ConditioningEncoderMixin, nn.Module, LayerwiseOffloadableModuleMixin +): layerwise_offload_dit_group_enabled = False layer_names = [ "model.encoder.layer", @@ -212,7 +215,9 @@ def build_image_encoder(config): raise ValueError(f"Unknown image encoder type: {config['type']}") -class DualImageEncoder(nn.Module, LayerwiseOffloadableModuleMixin): +class DualImageEncoder( + ConditioningEncoderMixin, nn.Module, LayerwiseOffloadableModuleMixin +): layerwise_offload_dit_group_enabled = False layer_names = [ "main_image_encoder.model.encoder.layer", @@ -249,7 +254,9 @@ class DualImageEncoder(nn.Module, LayerwiseOffloadableModuleMixin): return outputs -class SingleImageEncoder(nn.Module, LayerwiseOffloadableModuleMixin): +class SingleImageEncoder( + ConditioningEncoderMixin, nn.Module, LayerwiseOffloadableModuleMixin +): layerwise_offload_dit_group_enabled = False layer_names = [ "main_image_encoder.model.encoder.layer", diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py b/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py index c995291ca6..a782df3b8a 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py @@ -81,6 +81,8 @@ from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import ( Qwen2_5_VLModelOutputWithPast, ) +from sglang.multimodal_gen.runtime.cache.conditioning import cached_conditioning + logger = logging.getLogger(__name__) @@ -911,6 +913,7 @@ class Qwen2_5_VLModel(nn.Module): video_embeds = torch.split(video_embeds, split_sizes) return video_embeds + @cached_conditioning def get_image_features( self, pixel_values: torch.FloatTensor, diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl.py b/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl.py index 3d1ed6e434..9c8850c666 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl.py @@ -57,6 +57,8 @@ import torch import torch.nn as nn from transformers.activations import ACT2FN +from sglang.multimodal_gen.runtime.cache.conditioning import cached_conditioning + logger = logging.getLogger(__name__) from transformers.modeling_outputs import BaseModelOutputWithPast @@ -910,6 +912,7 @@ class Qwen3VLModel(nn.Module): # Same implementation as for images return self.get_image_features(pixel_values_videos, video_grid_thw) + @cached_conditioning def _get_flat_visual_features( self, pixel_values: torch.FloatTensor, diff --git a/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/modeling_neo_chat.py b/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/modeling_neo_chat.py index 94f06163fb..a7f1614181 100644 --- a/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/modeling_neo_chat.py +++ b/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/modeling_neo_chat.py @@ -13,6 +13,8 @@ from transformers.modeling_outputs import CausalLMOutputWithPast from transformers.modeling_utils import PreTrainedModel from transformers.utils import logging +from sglang.multimodal_gen.runtime.cache.conditioning import cached_conditioning + from .configuration_neo_chat import NEOChatConfig, NEOMoELLMConfig from .conversation import get_conv_template from .modeling_fm_modules import ( @@ -484,12 +486,16 @@ class NEOChatModel(PreTrainedModel): grid_hw=grid_hw, ).last_hidden_state else: - return self.vision_model( - pixel_values=pixel_values, - output_hidden_states=False, - return_dict=True, - grid_hw=grid_hw, - ).last_hidden_state + return self._reference_image_features(pixel_values, grid_hw) + + @cached_conditioning + def _reference_image_features(self, pixel_values, grid_hw): + return self.vision_model( + pixel_values=pixel_values, + output_hidden_states=False, + return_dict=True, + grid_hw=grid_hw, + ).last_hidden_state def batch_chat( self, diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder.py b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder.py index 5e1b834e07..3d48a5fff9 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder.py @@ -22,6 +22,7 @@ from diffusers.models.modeling_outputs import AutoencoderKLOutput from torch import nn from sglang.multimodal_gen.configs.models.vaes.base import VAEConfig +from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( LayerwiseOffloadableModuleMixin, ) @@ -287,6 +288,7 @@ class AutoencoderKL(nn.Module, LayerwiseOffloadableModuleMixin): return enc + @cached_vae_encode def encode( self, x: torch.Tensor, return_dict: bool = True ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_dc.py b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_dc.py index dea31a494b..5941d32d70 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_dc.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_dc.py @@ -7,6 +7,7 @@ from diffusers.models.autoencoders.vae import DecoderOutput from torch import nn from sglang.multimodal_gen.configs.models.vaes.sana import SanaVAEConfig +from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode from sglang.multimodal_gen.runtime.distributed.parallel_state import ( get_decode_parallel_rank, get_decode_parallel_world_size, @@ -135,6 +136,7 @@ class AutoencoderDC(nn.Module, LayerwiseOffloadableModuleMixin): return next(self._inner_model.parameters()).device return torch.device("cpu") + @cached_vae_encode def encode(self, x: torch.Tensor, **kwargs): self._ensure_inner_model() return self._inner_model.encode(x, **kwargs) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_flux2.py b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_flux2.py index e546811ed1..58651f4afa 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_flux2.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_flux2.py @@ -19,6 +19,7 @@ from diffusers.models.autoencoders.vae import ( from diffusers.models.modeling_outputs import AutoencoderKLOutput from sglang.multimodal_gen.configs.models.vaes.flux import Flux2VAEConfig +from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode from sglang.multimodal_gen.runtime.models.vaes.common import ( ParallelTiledVAE, can_install_spatial_shard_parallel_decode, @@ -238,6 +239,7 @@ class AutoencoderKLFlux2(ParallelTiledVAE): return enc + @cached_vae_encode def encode( self, x: torch.Tensor, return_dict: bool = True ) -> Union[DiagonalGaussianDistribution]: diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py index 3aab61538c..da80a3aa25 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py @@ -14,6 +14,7 @@ from diffusers.models.autoencoders.vae import ( from diffusers.models.modeling_outputs import AutoencoderKLOutput from sglang.multimodal_gen.configs.models.vaes.qwenimage import QwenImageVAEConfig +from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode from sglang.multimodal_gen.runtime.distributed import get_local_torch_device from sglang.multimodal_gen.runtime.distributed.parallel_state import ( get_decode_parallel_rank, @@ -1069,6 +1070,7 @@ class AutoencoderKLQwenImage(ParallelTiledVAE): self.clear_cache() return enc + @cached_vae_encode def encode( self, x: torch.Tensor, return_dict: bool = True ) -> DiagonalGaussianDistribution: diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/common.py b/python/sglang/multimodal_gen/runtime/models/vaes/common.py index 28b0321538..851f4ef9fb 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/common.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/common.py @@ -18,6 +18,10 @@ from sglang.multimodal_gen.configs.models import VAEConfig from sglang.multimodal_gen.configs.models.vaes.base import ( should_use_spatial_shard_parallel_decode, ) +from sglang.multimodal_gen.runtime.cache.conditioning import ( + cached_vae_encode, + register_conditioning_container, +) from sglang.multimodal_gen.runtime.distributed import ( get_decode_parallel_group_coordinator, get_decode_parallel_world_size, @@ -137,6 +141,7 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin): def _decode(self, *args, **kwargs) -> torch.Tensor: pass + @cached_vae_encode def encode(self, x: torch.Tensor) -> DiagonalGaussianDistribution: batch_size, num_channels, num_frames, height, width = x.shape latent_num_frames = (num_frames - 1) // self.temporal_compression_ratio + 1 @@ -748,6 +753,7 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin): # adapted from https://github.com/huggingface/diffusers/blob/e7ffeae0a191f710881d1fbde00cd6ff025e81f2/src/diffusers/models/autoencoders/vae.py#L691 +@register_conditioning_container class DiagonalGaussianDistribution: def __init__(self, parameters: torch.Tensor, deterministic: bool = False): self.parameters = parameters diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/dac.py b/python/sglang/multimodal_gen/runtime/models/vaes/dac.py index a944bcb994..6c52484cc4 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/dac.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/dac.py @@ -12,6 +12,7 @@ from einops import rearrange from torch import nn from sglang.multimodal_gen.configs.models.vaes.dac import DacVAEConfig +from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( LayerwiseOffloadableModuleMixin, ) @@ -506,6 +507,7 @@ class DAC(nn.Module, LayerwiseOffloadableModuleMixin): return audio_data + @cached_vae_encode def encode( self, audio_data: torch.Tensor, diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_3_condition_encoder.py b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_3_condition_encoder.py index 75871f10a8..4bd41d8aaa 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_3_condition_encoder.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_3_condition_encoder.py @@ -3,6 +3,7 @@ from typing import Any import torch import torch.nn as nn +from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningEncoderMixin from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( LayerwiseOffloadableModuleMixin, ) @@ -113,7 +114,9 @@ def _make_ltx23_encoder_block( ) -class LTX23VideoConditionEncoder(nn.Module, LayerwiseOffloadableModuleMixin): +class LTX23VideoConditionEncoder( + ConditioningEncoderMixin, nn.Module, LayerwiseOffloadableModuleMixin +): layerwise_offload_dit_group_enabled = False layer_names = ["down_blocks"] diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_audio.py b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_audio.py index 0c7097225c..5d0bbfd997 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_audio.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_audio.py @@ -10,6 +10,7 @@ from diffusers.models.modeling_outputs import AutoencoderKLOutput from torch import nn from sglang.multimodal_gen.configs.models.vaes.ltx_audio import LTXAudioVAEConfig +from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode from sglang.multimodal_gen.runtime.models.vaes.common import ParallelTiledVAE LATENT_DOWNSAMPLE_FACTOR = 4 @@ -868,6 +869,7 @@ class AutoencoderKLLTX2Audio(ParallelTiledVAE): def _encode(self, x: torch.Tensor) -> torch.Tensor: return self.encoder(x) + @cached_vae_encode def encode(self, x: torch.Tensor, return_dict: bool = True): if self.use_slicing and x.shape[0] > 1: encoded_slices = [self._encode(x_slice) for x_slice in x.split(1)] diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py index 45493e5333..192261b37d 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py @@ -12,6 +12,7 @@ from diffusers.models.embeddings import PixArtAlphaCombinedTimestepSizeEmbedding from diffusers.models.modeling_outputs import AutoencoderKLOutput from sglang.multimodal_gen.configs.models.vaes.ltx_video import LTXVideoVAEConfig +from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode from sglang.multimodal_gen.runtime.distributed.parallel_state import ( get_decode_parallel_rank, get_decode_parallel_world_size, @@ -1857,6 +1858,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE): return enc + @cached_vae_encode def encode( self, x: torch.Tensor, causal: Optional[bool] = None, return_dict: bool = True ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py index 74a64733de..a95227c853 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py @@ -14,6 +14,7 @@ from diffusers.models import ModelMixin from diffusers.utils import logging from PIL import Image +from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode from sglang.multimodal_gen.runtime.distributed import ( get_decode_parallel_group_coordinator, get_decode_parallel_rank, @@ -1281,6 +1282,7 @@ class AutoencoderKLLegacy(AutoencoderKL): self.decoder = ViT3DDecoder(**vit_kwargs) @torch.no_grad() + @cached_vae_encode def encode(self, x): return self.quant_conv(self.encoder(x)) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py b/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py index d50d65bcb6..a6b1489300 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py @@ -29,6 +29,7 @@ from sglang.multimodal_gen.configs.models.vaes import WanVAEConfig from sglang.multimodal_gen.configs.models.vaes.base import ( should_use_spatial_shard_parallel_decode, ) +from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode from sglang.multimodal_gen.runtime.distributed.parallel_state import ( get_decode_parallel_rank, get_decode_parallel_world_size, @@ -1585,6 +1586,7 @@ class AutoencoderKLWan(ParallelTiledVAE): self._causal_decode_initialized = True return out + @cached_vae_encode def encode(self, x: torch.Tensor) -> torch.Tensor: if self.use_feature_cache: self.clear_cache() diff --git a/python/sglang/multimodal_gen/runtime/pipelines/sana_video.py b/python/sglang/multimodal_gen/runtime/pipelines/sana_video.py index 54f8564d5f..97e1673511 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/sana_video.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/sana_video.py @@ -62,10 +62,6 @@ class SanaVideoTextEncodingStage(TextEncodingStage): return [item.lower().strip() for item in text] def _encode_negative_text(self, batch, server_args, all_indices): - cache_key = self._build_negative_text_cache_key(batch, server_args, all_indices) - cached = self._get_cached_negative_text_embedding(cache_key) - if cached is not None: - return cached outputs = self.encode_text( self._normalize_text(batch.negative_prompt), server_args, @@ -73,7 +69,6 @@ class SanaVideoTextEncodingStage(TextEncodingStage): return_attention_mask=True, max_length=300, ) - self._maybe_cache_negative_text_embedding(cache_key, outputs) return outputs @torch.no_grad() diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/executors/pipeline_executor.py b/python/sglang/multimodal_gen/runtime/pipelines_core/executors/pipeline_executor.py index 3bfd147cff..55525c0e1d 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/executors/pipeline_executor.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/executors/pipeline_executor.py @@ -12,6 +12,10 @@ from typing import TYPE_CHECKING, Any, Callable, List import torch +from sglang.multimodal_gen.runtime.cache.conditioning import ( + ConditioningCache, + conditioning_cache_group, +) from sglang.multimodal_gen.runtime.distributed import get_world_rank from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req from sglang.multimodal_gen.runtime.platforms import current_platform @@ -54,6 +58,11 @@ class PipelineExecutor(ABC): def __init__(self, server_args): self.server_args = server_args self.component_residency_manager = None + self.conditioning_cache = ConditioningCache( + int(server_args.conditioning_cache_max_size_mb * 1024**2) + if not server_args.disable_conditioning_cache + else 0 + ) def begin_component_residency_request( self, @@ -87,7 +96,11 @@ class PipelineExecutor(ABC): ): self.begin_component_residency_request(stages, payload, server_args) try: - yield + with self.conditioning_cache.scope( + cross_request=not server_args.use_fsdp_inference, + refresh=self._is_warmup_payload(payload), + ): + yield finally: self.finish_component_residency_request() @@ -117,9 +130,13 @@ class PipelineExecutor(ABC): self.before_stage(stage, stage_index, payload, server_args) with maybe_record_function(f"STAGE {stage_name}"): with maybe_nvtx_range(f"stage_{stage_name}", use_nvtx): - payload = self.run_stage_with_context( - stage, payload, server_args, run_stage - ) + with conditioning_cache_group( + enabled=(isinstance(payload, list) and len(payload) > 1) + or (isinstance(payload, Req) and payload.batch_size > 1) + ): + payload = self.run_stage_with_context( + stage, payload, server_args, run_stage + ) return payload @staticmethod diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/lora/pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines_core/lora/pipeline.py index 0afa9dc2b0..9a12ef0799 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/lora/pipeline.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/lora/pipeline.py @@ -13,6 +13,9 @@ from safetensors.torch import load_file from torch.distributed.tensor import DTensor from sglang.multimodal_gen import envs +from sglang.multimodal_gen.runtime.cache.conditioning import ( + invalidate_conditioning_caches, +) from sglang.multimodal_gen.runtime.distributed import get_local_torch_device from sglang.multimodal_gen.runtime.layers.lora.linear import ( BaseLayerWithLoRA, @@ -1036,6 +1039,14 @@ class LoRAPipeline(ComposedPipelineBase): self.loaded_adapter_alphas[lora_nickname] = adapter_lora_alpha logger.info("Rank %d: loaded LoRA adapter %s", rank, lora_path) + def _invalidate_lora_conditioning(self): + # LoRA targets DiTs; unrelated text and VAE weights remain unchanged + invalidate_conditioning_caches( + module + for name in ("transformer", "transformer_2", "fake_score_transformer") + if (module := self.modules.get(name)) is not None + ) + def set_lora( self, lora_nickname: str | list[str], @@ -1055,6 +1066,7 @@ class LoRAPipeline(ComposedPipelineBase): stop costing anonymous host memory; pass it only for the startup (static) adapter, where the merged combination is stable. """ + self._invalidate_lora_conditioning() merge_mode = self._resolve_lora_merge_mode(merge_weights, merge_mode) # Normalize inputs to lists for multi-LoRA support @@ -1303,6 +1315,7 @@ class LoRAPipeline(ComposedPipelineBase): Disable LoRA for the specified target, regardless of whether weights were merged into the base model or are still active in the wrapped LoRA path. """ + self._invalidate_lora_conditioning() target_modules, error = self._get_target_lora_layers(target) if error: logger.warning("deactivate_lora_weights: %s", error) @@ -1340,6 +1353,7 @@ class LoRAPipeline(ComposedPipelineBase): "transformer_2", "critic". strength: LoRA strength for merge, default 1.0. """ + self._invalidate_lora_conditioning() target_modules, error = self._get_target_lora_layers(target) if error: logger.warning("merge_lora_weights: %s", error) @@ -1412,6 +1426,7 @@ class LoRAPipeline(ComposedPipelineBase): target: Which transformer(s) to unmerge. One of "all", "transformer", "transformer_2", "critic". """ + self._invalidate_lora_conditioning() target_modules, error = self._get_target_lora_layers(target) if error: logger.warning("unmerge_lora_weights: %s", error) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py index f2b3a5a96a..91d475b20a 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py @@ -287,6 +287,12 @@ class PipelineStage(StageDedupMixin, ABC): ) self._component_residency_manager.begin_use(use, module=module) + def finish_unused_declared_component(self, *, component_name: str, module=None): + if self._component_residency_manager is None: + return + use = self._declared_component_use(component_name=component_name) + self._component_residency_manager.finish_unused_component(use, module=module) + def component_uses( self, server_args: ServerArgs, stage_name: str | None = None ) -> list[ComponentUse]: diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/dedup.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/dedup.py index 47dbd89d5d..1190f54b41 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/dedup.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/dedup.py @@ -8,6 +8,8 @@ from typing import TYPE_CHECKING, Any, ClassVar import torch +from sglang.multimodal_gen.runtime.cache.conditioning import conditioning_cache_group + if TYPE_CHECKING: from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req from sglang.multimodal_gen.runtime.server_args import ServerArgs @@ -91,16 +93,19 @@ class StageDedupMixin: results: list[Req | None] = [None] * len(batches) - for _, group in self._group_requests_by_fingerprint( + groups = self._group_requests_by_fingerprint( batches, lambda batch: self.build_dedup_fingerprint(batch, server_args) - ): - first_index, first_batch = group[0] - first_result = self(first_batch, server_args) - results[first_index] = first_result + ) + # a single computed request needs no group memo or cache-hit collective + with conditioning_cache_group(enabled=len(groups) > 1): + for _, group in groups: + first_index, first_batch = group[0] + first_result = self(first_batch, server_args) + results[first_index] = first_result - for index, batch in group[1:]: - copy_outputs(first_result, batch) - results[index] = batch + for index, batch in group[1:]: + copy_outputs(first_result, batch) + results[index] = batch return [result for result in results if result is not None] diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ideogram.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ideogram.py index 51f942acf1..c854946563 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ideogram.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ideogram.py @@ -117,6 +117,7 @@ class Ideogram4Scheduler: class Ideogram4TextEncodingStage(TextEncodingStage): + deduplicated_output_fields = ("prompt_embeds", "prompt_embeds_mask") deduplicated_extra_tensor_tree_output_keys = ("ideogram4",) def __init__(self, text_encoder, tokenizer) -> None: diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_video_moe/text_encoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_video_moe/text_encoding.py index ed81d58969..510269c718 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_video_moe/text_encoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_video_moe/text_encoding.py @@ -2,7 +2,13 @@ import torch -from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.cache.conditioning import cached_encoder_call +from sglang.multimodal_gen.runtime.distributed import ( + get_local_torch_device, + get_tp_group, + model_parallel_is_initialized, +) +from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import ( TextEncodingStage, @@ -35,6 +41,8 @@ VIDEO_PROMPT_TEMPLATE = "<|vision_start|><|video_pad|><|vision_end|>" class LingBotVideoTextEncodingStage(TextEncodingStage): """Qwen3-VL prompt/negative encoding for LingBot-Video MoE (T2V, base).""" + deduplicated_output_fields = () + def __init__(self, text_encoders, tokenizers, transformer): super().__init__(text_encoders, tokenizers) self.transformer = transformer @@ -106,6 +114,32 @@ class LingBotVideoTextEncodingStage(TextEncodingStage): ) inputs = self._build_prompt_inputs(prompt) + cache_group = ( + text_encoder._encoder_tp_group + if isinstance(text_encoder, TextEncoder) + else None + ) + if cache_group is None and model_parallel_is_initialized(): + cache_group = get_tp_group() + return cached_encoder_call( + text_encoder, + (dict(inputs),), + { + "device": str(device), + "dtype": dtype, + "skip_layer": self.hidden_state_skip_layer, + "crop_start": self._compute_crop_start(), + }, + lambda: self._encode_inputs(inputs, device, dtype), + cache_group, + namespace=self, + nested=False, + share_in_group=True, + ) + + def _encode_inputs(self, inputs, device, dtype): + self._begin_text_encoder_use(0) + text_encoder = self.text_encoders[0] inputs = inputs.to(device) outputs = self._forward_text_encoder( text_encoder, diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/longlive2.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/longlive2.py index 06f696ff53..59e85ff0fc 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/longlive2.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/longlive2.py @@ -51,15 +51,7 @@ def _causal_block_count(batch: Req, server_args: ServerArgs) -> int: class LongLive2TextEncodingStage(TextEncodingStage): - def build_dedup_fingerprint(self, batch: Req, server_args: ServerArgs): - base = super().build_dedup_fingerprint(batch, server_args) - return ( - base, - self.freeze_for_dedup(getattr(batch, "shot_prompts", None)), - self.freeze_for_dedup(getattr(batch, "shot_durations", None)), - int(getattr(batch, "chunks_per_shot", 0) or 0), - getattr(batch, "scene_cut_prefix", None), - ) + deduplicated_output_fields = () def _block_prompts(self, batch: Req, server_args: ServerArgs) -> list[str] | None: shot_prompts = getattr(batch, "shot_prompts", None) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ming_image.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ming_image.py index 5f7157ead6..59ba311733 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ming_image.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ming_image.py @@ -15,8 +15,13 @@ from transformers.models.qwen2_vl.image_processing_pil_qwen2_vl import ( from sglang.multimodal_gen.configs.pipeline_configs.ming_image import ( MingImageLayerPipelineConfig, ) +from sglang.multimodal_gen.runtime.cache.conditioning import cached_encoder_call from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType -from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.distributed import ( + get_local_torch_device, + get_tp_group, + model_parallel_is_initialized, +) from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( ComponentUse, ) @@ -31,6 +36,8 @@ from sglang.multimodal_gen.runtime.utils.vision import load_image class MingImageEncodingStage(TextEncodingStage): + deduplicated_output_fields = () + def __init__(self, text_encoders, tokenizers): super().__init__(text_encoders, tokenizers) self.image_processor = Qwen2VLImageProcessorPil.from_pretrained( @@ -131,13 +138,32 @@ class MingImageEncodingStage(TextEncodingStage): input_ids=torch.tensor([ids], device=device), position_ids=ming_position_ids(ids, grids, encoder.image_token).to(device), ) - self._begin_text_encoder_use(0) - # the official MoE router runs under BF16 autocast, including its logits - with torch.autocast(device.type, dtype=encoder.dtype): - encoded = self._forward_text_encoder(encoder, inputs) - batch.prompt_embeds = [encoded.last_hidden_state] - batch.negative_prompt_embeds = [torch.zeros_like(encoded.last_hidden_state)] - batch.extra["ming_direct"] = encoded.hidden_states[0] + + def encode_conditioning(): + self._begin_text_encoder_use(0) + # the official MoE router runs under BF16 autocast, including its logits + with torch.autocast(device.type, dtype=encoder.dtype): + encoded = self._forward_text_encoder(encoder, inputs) + return encoded.last_hidden_state, encoded.hidden_states[0] + + cache_group = encoder._encoder_tp_group + if cache_group is None and model_parallel_is_initialized(): + cache_group = get_tp_group() + prompt_embeds, direct = cached_encoder_call( + encoder, + (inputs,), + {}, + encode_conditioning, + cache_group, + namespace=self, + share_in_group=True, + ) + self.finish_unused_declared_component( + component_name="text_encoder", module=encoder + ) + batch.prompt_embeds = [prompt_embeds] + batch.negative_prompt_embeds = [torch.zeros_like(prompt_embeds)] + batch.extra["ming_direct"] = direct return batch diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image21.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image21.py index f1efc17f61..db178e10fe 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image21.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image21.py @@ -4,7 +4,12 @@ import math import torch from PIL import Image -from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.cache.conditioning import cached_encoder_call +from sglang.multimodal_gen.runtime.distributed import ( + get_local_torch_device, + get_tp_group, + model_parallel_is_initialized, +) from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( ComponentUse, @@ -83,6 +88,24 @@ class QwenImage21EncodingStage(PipelineStage): ] def encode_prompt(self, prompt, images, device): + cache_group = self.text_encoder._encoder_tp_group + if cache_group is None and model_parallel_is_initialized(): + cache_group = get_tp_group() + conditioning = cached_encoder_call( + self.text_encoder, + (prompt, images), + {"device": str(device)}, + lambda: self._encode_prompt(prompt, images, device), + cache_group, + namespace=self, + share_in_group=True, + ) + self.finish_unused_declared_component( + component_name="text_encoder", module=self.text_encoder + ) + return conditioning + + def _encode_prompt(self, prompt, images, device): prefix = " ".join( f"<|vision_start|><|image_pad|><|vision_end|>" for i in range(len(images)) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/base.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/base.py index d3d0512432..04e2b4f7f8 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/base.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/base.py @@ -637,6 +637,8 @@ class SanaWMTextEncodingStage(TextEncodingStage): model-specific prompt-window contract. """ + deduplicated_output_fields = () + @staticmethod def _text_encoder_max_length(server_args: ServerArgs) -> int: encoder_cfg = server_args.pipeline_config.text_encoder_configs[0] diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/realtime/text_encoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/realtime/text_encoding.py index 0ef78850f7..895fc2db23 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/realtime/text_encoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/realtime/text_encoding.py @@ -16,6 +16,7 @@ from typing import Any import torch +from sglang.multimodal_gen.runtime.cache.conditioning import conditioning_weights_epoch from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( ComponentUse, ) @@ -109,6 +110,8 @@ class RealtimeTextState(BaseRealtimeState): class RealtimeTextEncodingStage(TextEncodingStage): """Cache text encoder outputs across realtime chunks by prompt identity.""" + deduplicated_output_fields = () + def component_uses( self, server_args: ServerArgs, stage_name: str | None = None ) -> list[ComponentUse]: @@ -120,6 +123,7 @@ class RealtimeTextEncodingStage(TextEncodingStage): def _make_cache_key(self, batch: Req) -> tuple[Any, ...]: return ( + conditioning_weights_epoch(), _normalize_prompt_value(batch.prompt), bool(batch.do_classifier_free_guidance), ( @@ -147,7 +151,11 @@ class RealtimeTextEncodingStage(TextEncodingStage): batch: Req, server_args: ServerArgs, ) -> Req: - if batch.session is None: + if ( + batch.session is None + or server_args.disable_conditioning_cache + or server_args.conditioning_cache_max_size_mb == 0 + ): return super().forward(batch, server_args) state = batch.session.get_or_create_state(RealtimeTextState) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py index 7339680654..98c62e3037 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/text_encoding.py @@ -8,17 +8,21 @@ This module contains implementations of prompt encoding stages for diffusion pip """ import inspect -from dataclasses import dataclass -from functools import lru_cache -from typing import Any import torch from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput from sglang.multimodal_gen.configs.pipeline_configs.base import TextConditioningOutput +from sglang.multimodal_gen.runtime.cache.conditioning import ( + cached_encoder_call, + prefer_conditioning_cache, +) from sglang.multimodal_gen.runtime.distributed import ( get_encoder_data_parallel_group, get_local_torch_device, + get_replica_group, + get_tp_group, + model_parallel_is_initialized, ) from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( @@ -102,27 +106,6 @@ def _data_parallel_text_encode(forward_fn, forward_kwargs: dict, group): ) -@lru_cache(maxsize=1) -def get_model_default_negative_prompt( - model_path: str, backend: Any, model_id: str | None -): - from sglang.multimodal_gen.registry import get_model_info - - model_info = get_model_info(model_path, backend=backend, model_id=model_id) - if model_info is None: - return None - return model_info.sampling_param_cls().negative_prompt - - -@dataclass(frozen=True) -class TextEncodingFingerprint: - prompt: Any - negative_prompt: Any - do_classifier_free_guidance: bool - prompt_template: Any - max_sequence_length: int | None - - def stack_tensors(name: str, tensors: list[torch.Tensor]) -> torch.Tensor: base_shape = list(tensors[0].shape) for tensor in tensors[1:]: @@ -157,6 +140,26 @@ class TextEncodingStage(ConditionEncodingStage): "is_prompt_processed", ) + def build_dedup_fingerprint(self, batch: Req, server_args: ServerArgs): + return ( + self.freeze_for_dedup(batch.prompt), + self.freeze_for_dedup(batch.negative_prompt), + bool(batch.do_classifier_free_guidance), + self.freeze_for_dedup(batch.prompt_template), + batch.max_sequence_length, + ) + + @classmethod + def copy_stage_output(cls, value): + # embeddings stay shared; nested metadata containers belong to each request + if isinstance(value, list): + return [cls.copy_stage_output(item) for item in value] + if isinstance(value, tuple): + return tuple(cls.copy_stage_output(item) for item in value) + if isinstance(value, dict): + return {key: cls.copy_stage_output(item) for key, item in value.items()} + return value + def __init__(self, text_encoders, tokenizers) -> None: """ Initialize the prompt encoding stage. @@ -165,8 +168,6 @@ class TextEncodingStage(ConditionEncodingStage): super().__init__() self.tokenizers = tokenizers self.text_encoders = text_encoders - self._negative_text_cache_key = None - self._negative_text_cache_value = None self._dp_choice_logged = False def component_uses( @@ -178,6 +179,7 @@ class TextEncodingStage(ConditionEncodingStage): stage_name=stage_name, component_name="text_encoder" if i == 0 else f"text_encoder_{i + 1}", preferred_ready_after_request=i == 0, + start_at_stage_entry=False, target_dtype=resolve_component_precision_override( server_args, "text_encoder" if i == 0 else f"text_encoder_{i + 1}", @@ -189,100 +191,14 @@ class TextEncodingStage(ConditionEncodingStage): def get_or_compute_negative_text_embedding( self, batch: Req, server_args: ServerArgs, all_indices: list[int] ): - """Get the cached text embedding result or compute - - this is a one-slot cache for the model-default negative prompt: - most requests don't override the negative prompt, the cache hit rate is considerably high - - invariant: hit/miss must match across ranks -- a miss runs encode_text, - which may issue collectives (folding, dp encoding), so a split would - deadlock; keep any future eviction rank-global - """ - negative_cache_key = self._build_negative_text_cache_key( - batch, server_args, all_indices - ) - cached_negative = self._get_cached_negative_text_embedding(negative_cache_key) - if cached_negative is not None: - return cached_negative - - negative_text_outputs = self.encode_text( - batch.negative_prompt, - server_args, - encoder_index=all_indices, - return_attention_mask=True, - ) - self._maybe_cache_negative_text_embedding( - negative_cache_key, negative_text_outputs - ) - return negative_text_outputs - - def _should_cache_negative_text_embedding( - self, batch: Req, server_args: ServerArgs - ) -> bool: - if not batch.is_warmup: - return True - return self._uses_model_default_negative_prompt(batch, server_args) - - def _get_cached_negative_text_embedding(self, negative_cache_key): - if negative_cache_key is None: - return None - if self._negative_text_cache_key == negative_cache_key: - return self._negative_text_cache_value - return None - - def _maybe_cache_negative_text_embedding( - self, - negative_cache_key, - negative_text_outputs, - ) -> None: - - # skip caching if None - if negative_cache_key is None: - return - self._negative_text_cache_key = negative_cache_key - self._negative_text_cache_value = tuple( - tuple(value) for value in negative_text_outputs - ) - - def _build_negative_text_cache_key( - self, batch: Req, server_args: ServerArgs, encoder_indices: list[int] - ): - """if the current req doesn't worth caching, returns None""" - # skip if we don't cache for current req - if not self._should_cache_negative_text_embedding(batch, server_args): - return None - - # Negative text encoding changes when the template or max length changes, - # even if the visible negative prompt string is the same. - return ( - tuple(encoder_indices), - self.freeze_for_dedup(batch.negative_prompt), - self.freeze_for_dedup(batch.prompt_template), - batch.max_sequence_length, - ) - - def _uses_model_default_negative_prompt( - self, batch: Req, server_args: ServerArgs - ) -> bool: - default_negative_prompt = self._get_model_default_negative_prompt(server_args) - if default_negative_prompt is None: - return False - return self._normalize_negative_prompt_for_default_match( - batch.negative_prompt - ) == self._normalize_negative_prompt_for_default_match(default_negative_prompt) - - def _get_model_default_negative_prompt(self, server_args: ServerArgs) -> str | None: - return get_model_default_negative_prompt( - server_args.model_path, - server_args.backend, - server_args.model_id, - ) - - @staticmethod - def _normalize_negative_prompt_for_default_match(value): - if isinstance(value, str) and not value.isspace(): - return value.strip() - return value + """Encode negative conditioning through the shared encoder cache.""" + with prefer_conditioning_cache(): + return self.encode_text( + batch.negative_prompt, + server_args, + encoder_index=all_indices, + return_attention_mask=True, + ) def _append_positive_text_outputs( self, @@ -462,17 +378,6 @@ class TextEncodingStage(ConditionEncodingStage): return batch - def build_dedup_fingerprint( - self, batch: Req, server_args: ServerArgs - ) -> TextEncodingFingerprint: - return TextEncodingFingerprint( - prompt=self.freeze_for_dedup(batch.prompt), - negative_prompt=self.freeze_for_dedup(batch.negative_prompt), - do_classifier_free_guidance=bool(batch.do_classifier_free_guidance), - prompt_template=self.freeze_for_dedup(batch.prompt_template), - max_sequence_length=batch.max_sequence_length, - ) - def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: """Verify text encoding stage inputs.""" result = VerificationResult() @@ -696,131 +601,185 @@ class TextEncodingStage(ConditionEncodingStage): text_inputs: dict = server_args.pipeline_config.tokenize_prompt( processed_text_list, tokenizer, tok_kwargs - ).to(target_device) - - input_ids = text_inputs["input_ids"] - attention_mask = ( - server_args.pipeline_config.get_text_encoder_attention_mask( - text_inputs, i - ) ) - encoder_forward_kwargs = { - "input_ids": input_ids, - "output_hidden_states": True, - } - if attention_mask is not None: - encoder_forward_kwargs["attention_mask"] = attention_mask - if "use_cache" in inspect.signature(text_encoder.forward).parameters: - encoder_forward_kwargs["use_cache"] = False - self._begin_text_encoder_use(i) + # match host tokens before preparing weights or uploading encoder inputs + host_inputs = text_inputs + cache_inputs = dict(host_inputs) dp_group = self._text_encode_dp_group( - server_args, encoder_config, input_ids.shape[0], text_encoder + server_args, + encoder_config, + host_inputs["input_ids"].shape[0], + text_encoder, ) - if dp_group is not None: - outputs = _data_parallel_text_encode( - lambda kw: self._forward_text_encoder(text_encoder, kw), - encoder_forward_kwargs, - dp_group, - ) - else: - outputs = self._forward_text_encoder( - text_encoder, encoder_forward_kwargs - ) - postprocess_sig = inspect.signature(postprocess_func) - postprocess_kwargs = {} - if "pipeline_config" in postprocess_sig.parameters: - # required by models like LTX - postprocess_kwargs["pipeline_config"] = server_args.pipeline_config - if "return_attention_mask" in postprocess_sig.parameters: - postprocess_kwargs["return_attention_mask"] = return_attention_mask - postprocess_result = postprocess_func( - outputs, text_inputs, **postprocess_kwargs - ) - prompt_embeds_mask = None - prompt_seq_lens = None - if isinstance(postprocess_result, TextConditioningOutput): - prompt_embeds = postprocess_result.prompt_embeds - prompt_embeds_mask = postprocess_result.prompt_embeds_mask - prompt_seq_lens = postprocess_result.prompt_seq_lens - elif isinstance(postprocess_result, tuple): - if len(postprocess_result) != 2: - raise ValueError( - "Text postprocess tuple output must be (prompt_embeds, prompt_embeds_mask)" + def encode_conditioning(): + text_inputs = host_inputs.to(target_device, non_blocking=True) + + input_ids = text_inputs["input_ids"] + attention_mask = ( + server_args.pipeline_config.get_text_encoder_attention_mask( + text_inputs, i ) - prompt_embeds, prompt_embeds_mask = postprocess_result - else: - prompt_embeds = postprocess_result - - if dtype is not None: - prompt_embeds = prompt_embeds.to(device=target_device, dtype=dtype) - else: - prompt_embeds = prompt_embeds.to(device=target_device) - - if prompt_embeds_mask is not None: - prompt_embeds_mask = prompt_embeds_mask.to( - device=target_device, dtype=torch.bool ) + encoder_forward_kwargs = { + "input_ids": input_ids, + "output_hidden_states": True, + } + if attention_mask is not None: + encoder_forward_kwargs["attention_mask"] = attention_mask + if "use_cache" in inspect.signature(text_encoder.forward).parameters: + encoder_forward_kwargs["use_cache"] = False + self._begin_text_encoder_use(i) + postprocess_sig = inspect.signature(postprocess_func) - embeds_list.append(prompt_embeds) + postprocess_kwargs = {} + if "pipeline_config" in postprocess_sig.parameters: + # required by models like LTX + postprocess_kwargs["pipeline_config"] = server_args.pipeline_config + if "return_attention_mask" in postprocess_sig.parameters: + postprocess_kwargs["return_attention_mask"] = return_attention_mask - pooled_output = server_args.pipeline_config.get_text_encoder_pooler_output( - outputs, i - ) - if pooled_output is not None: - pooled_embeds_list.append(pooled_output.to(device=target_device)) - - if return_attention_mask: - if prompt_embeds_mask is not None: - mask_to_store = prompt_embeds_mask.to( - device=target_device, - dtype=( - attention_mask.dtype - if attention_mask is not None - else torch.long - ), + if dp_group is not None: + outputs = _data_parallel_text_encode( + lambda kw: self._forward_text_encoder(text_encoder, kw), + encoder_forward_kwargs, + dp_group, ) - elif attention_mask is not None and list(attention_mask.shape) == list( - prompt_embeds.shape[:2] - ): - mask_to_store = attention_mask.to(device=target_device) else: - mask_to_store = torch.ones( - prompt_embeds.shape[:2], - device=target_device, - dtype=( - attention_mask.dtype - if attention_mask is not None - else torch.long - ), + outputs = self._forward_text_encoder( + text_encoder, encoder_forward_kwargs ) - attn_masks_list.append(mask_to_store) - - embeds_mask = prompt_embeds_mask - if embeds_mask is None: - embeds_mask = ( - server_args.pipeline_config.build_text_conditioning_mask( - text_inputs, - attention_mask, - prompt_embeds, - i, + postprocess_result = postprocess_func( + outputs, text_inputs, **postprocess_kwargs + ) + pooled_output = ( + server_args.pipeline_config.get_text_encoder_pooler_output( + outputs, i + ) + ) + prompt_embeds_mask = None + prompt_seq_lens = None + if isinstance(postprocess_result, TextConditioningOutput): + prompt_embeds = postprocess_result.prompt_embeds + prompt_embeds_mask = postprocess_result.prompt_embeds_mask + prompt_seq_lens = postprocess_result.prompt_seq_lens + elif isinstance(postprocess_result, tuple): + if len(postprocess_result) != 2: + raise ValueError( + "Text postprocess tuple output must be (prompt_embeds, prompt_embeds_mask)" ) + prompt_embeds, prompt_embeds_mask = postprocess_result + else: + prompt_embeds = postprocess_result + + if dtype is not None: + prompt_embeds = prompt_embeds.to(device=target_device, dtype=dtype) + else: + prompt_embeds = prompt_embeds.to(device=target_device) + + if prompt_embeds_mask is not None: + prompt_embeds_mask = prompt_embeds_mask.to( + device=target_device, dtype=torch.bool ) - embeds_masks_list.append(embeds_mask) - if prompt_seq_lens is not None: - seq_lens_list.append([int(x) for x in prompt_seq_lens]) - elif embeds_mask is not None: - seq_lens_list.append( - server_args.pipeline_config.seq_lens_from_text_conditioning_mask( + + if pooled_output is not None: + pooled_output = pooled_output.to(device=target_device) + + mask_to_store = embeds_mask = seq_lens = None + if return_attention_mask: + if prompt_embeds_mask is not None: + mask_to_store = prompt_embeds_mask.to( + device=target_device, + dtype=( + attention_mask.dtype + if attention_mask is not None + else torch.long + ), + ) + elif attention_mask is not None and list( + attention_mask.shape + ) == list(prompt_embeds.shape[:2]): + mask_to_store = attention_mask.to(device=target_device) + else: + mask_to_store = torch.ones( + prompt_embeds.shape[:2], + device=target_device, + dtype=( + attention_mask.dtype + if attention_mask is not None + else torch.long + ), + ) + + embeds_mask = prompt_embeds_mask + if embeds_mask is None: + embeds_mask = ( + server_args.pipeline_config.build_text_conditioning_mask( + text_inputs, + attention_mask, + prompt_embeds, + i, + ) + ) + if prompt_seq_lens is not None: + seq_lens = [int(x) for x in prompt_seq_lens] + elif embeds_mask is not None: + seq_lens = server_args.pipeline_config.seq_lens_from_text_conditioning_mask( embeds_mask ) - ) - elif prompt_embeds.ndim == 2: - seq_lens_list.append([int(prompt_embeds.shape[0])]) - else: - seq_lens_list.append( - [int(prompt_embeds.shape[1])] * int(prompt_embeds.shape[0]) - ) + elif prompt_embeds.ndim == 2: + seq_lens = [int(prompt_embeds.shape[0])] + else: + seq_lens = [int(prompt_embeds.shape[1])] * int( + prompt_embeds.shape[0] + ) + return ( + prompt_embeds, + mask_to_store, + pooled_output, + embeds_mask, + seq_lens, + ) + + native_encoder = isinstance(text_encoder, TextEncoder) + cache_group = text_encoder._encoder_tp_group if native_encoder else None + if model_parallel_is_initialized(): + if dp_group is not None: + # agree across encoder copies before skipping their gather + cache_group = get_replica_group() + elif cache_group is None: + cache_group = get_tp_group() + # library and batch-DP positives retain consumed outputs only within + # the group; preferred negatives also use the bounded device cache + conditioning = cached_encoder_call( + text_encoder, + (cache_inputs,), + { + "encoder_index": i, + "return_attention_mask": return_attention_mask, + "device": str(target_device), + "dtype": dtype, + }, + encode_conditioning, + cache_group, + namespace=self, + nested=dp_group is not None, + share_in_group=True, + cross_request=native_encoder and dp_group is None, + ) + # a hit can leave weights retained by warmup or the previous request + self.finish_unused_declared_component( + component_name="text_encoder" if i == 0 else f"text_encoder_{i + 1}", + module=text_encoder, + ) + prompt_embeds, mask, pooled_output, embeds_mask, seq_lens = conditioning + embeds_list.append(prompt_embeds) + if pooled_output is not None: + pooled_embeds_list.append(pooled_output) + if return_attention_mask: + attn_masks_list.append(mask) + embeds_masks_list.append(embeds_mask) + seq_lens_list.append(seq_lens) # Shape results according to return_type if return_type == "list": diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/vla.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/vla.py index c17a5b5e75..76bccf4396 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/vla.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/vla.py @@ -8,6 +8,7 @@ from typing import Any import numpy as np import torch +from sglang.multimodal_gen.runtime.cache.conditioning import conditioning_weights_epoch from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import ( OutputBatch, @@ -84,9 +85,11 @@ def _effective_prefix_cache_enabled( server_args: ServerArgs, ) -> bool: options = vla_options(batch) - return bool(options.get("enable_prefix_cache", True)) and bool( - server_args.pipeline_config.enable_global_prefix_cache - ) + return ( + not server_args.disable_conditioning_cache + and server_args.conditioning_cache_max_size_mb > 0 + and bool(options.get("enable_prefix_cache", True)) + ) and bool(server_args.pipeline_config.enable_global_prefix_cache) def _cuda_graph_enabled(batch: Req) -> bool: @@ -264,6 +267,7 @@ class VLAPrefixEncodingStage(PipelineStage): observation, bucket_prompt=_cuda_graph_enabled(batch), ) + cache_key = f"{conditioning_weights_epoch()}:{cache_key}" cached_context = self.prefix_cache.get(cache_key) else: cache_key = None diff --git a/python/sglang/multimodal_gen/runtime/post_training/weights_updater.py b/python/sglang/multimodal_gen/runtime/post_training/weights_updater.py index 75359a82ce..e6ef5094ce 100644 --- a/python/sglang/multimodal_gen/runtime/post_training/weights_updater.py +++ b/python/sglang/multimodal_gen/runtime/post_training/weights_updater.py @@ -46,6 +46,9 @@ from typing import Any import torch from torch.distributed.tensor import DTensor, distribute_tensor +from sglang.multimodal_gen.runtime.cache.conditioning import ( + invalidate_conditioning_caches, +) from sglang.multimodal_gen.runtime.cache.teacache import TeaCacheMixin from sglang.multimodal_gen.runtime.loader.utils import ( _list_safetensors_files, @@ -369,6 +372,7 @@ class WeightsUpdater: target_modules: list[str] | None = None, ) -> tuple[bool, str]: """Update model weights from disk without restarting the server.""" + invalidate_conditioning_caches() logger.info(f"Updating weights from disk: {model_path}") try: @@ -531,6 +535,7 @@ class WeightsUpdater: lora_alpha: int | None = None, lora_rank: int | None = None, ) -> tuple[bool, str]: + invalidate_conditioning_caches() if weight_update_mode == LORA_MERGE_WEIGHT_UPDATE_MODE: return self._update_lora_from_tensor( named_tensors=named_tensors, diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index 702a5c7d84..75b683f989 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -609,6 +609,8 @@ class ServerArgs(DisaggServerArgsMixin): log_requests_target: Optional[List[str]] = None uvicorn_access_log_exclude_prefixes: list[str] = field(default_factory=list) enable_cache_report: bool = False + disable_conditioning_cache: bool = False + conditioning_cache_max_size_mb: float = 512.0 # Tracing enable_trace: bool = False @@ -1932,6 +1934,10 @@ class ServerArgs(DisaggServerArgsMixin): raise ValueError(f"Could not parse attention backend config: {config_str}") def __post_init__(self): + if not 0 <= self.conditioning_cache_max_size_mb < float("inf"): + raise ValueError( + "conditioning_cache_max_size_mb must be finite and nonnegative" + ) if not self._explicit_arg_names: self._explicit_arg_names = _infer_direct_constructor_explicit_arg_names( self @@ -3026,6 +3032,18 @@ class ServerArgs(DisaggServerArgsMixin): default=ServerArgs.enable_cache_report, help="Return number of cached tokens in usage.prompt_tokens_details for each OpenAI-compatible request.", ) + parser.add_argument( + "--disable-conditioning-cache", + action="store_true", + default=ServerArgs.disable_conditioning_cache, + help="Disable cross-request text/image and VAE posterior caching; reuse within a grouped stage remains enabled.", + ) + parser.add_argument( + "--conditioning-cache-max-size-mb", + type=float, + default=ServerArgs.conditioning_cache_max_size_mb, + help="Per-worker conditioning cache capacity across CPU and device entries in MiB (default: 512; 0 disables cross-request caching).", + ) parser.add_argument( "--backend", type=str, diff --git a/python/sglang/multimodal_gen/test/unit/test_conditioning_cache.py b/python/sglang/multimodal_gen/test/unit/test_conditioning_cache.py new file mode 100644 index 0000000000..e804e9d66e --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_conditioning_cache.py @@ -0,0 +1,611 @@ +# SPDX-License-Identifier: Apache-2.0 + +import sys +from contextlib import nullcontext +from datetime import timedelta +from types import SimpleNamespace + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp +from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution +from diffusers.models.modeling_outputs import AutoencoderKLOutput + +from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput +from sglang.multimodal_gen.runtime.cache.conditioning import ( + ConditioningCache, + cached_conditioning, + cached_encoder_call, + cached_vae_encode, + invalidate_conditioning_caches, + prefer_conditioning_cache, +) +from sglang.multimodal_gen.runtime.models.encoders.base import ( + EncoderTensorParallelMixin, +) + + +class Encoder(EncoderTensorParallelMixin, torch.nn.Module): + def __init__(self): + super().__init__() + self.weight = torch.nn.Parameter(torch.ones(())) + self.calls = 0 + + def forward(self, x, attention_mask=None, **kwargs): + self.calls += 1 + result = x * self.weight + if attention_mask is not None: + result = result * attention_mask + return BaseEncoderOutput(last_hidden_state=result, hidden_states=(result,)) + + +class VAE(torch.nn.Module): + def __init__(self): + super().__init__() + self.calls = 0 + self.use_tiling = False + + @cached_vae_encode + def encode(self, x): + self.calls += 1 + return AutoencoderKLOutput(latent_dist=DiagonalGaussianDistribution(x.clone())) + + +class VisionLanguageEncoder(Encoder): + def __init__(self): + super().__init__() + self.vision_calls = 0 + + @cached_conditioning + def image_features(self, pixels): + self.vision_calls += 1 + return pixels * 2 + + def forward(self, text, pixels): + self.calls += 1 + return self.image_features(pixels) + text + + +@torch.no_grad() +def test_changed_instruction_reuses_vision_but_not_joint_embedding(): + cache = ConditioningCache(4096) + model = VisionLanguageEncoder().eval() + pixels = torch.ones(4) + with cache.scope(): + first = model(torch.ones(4), pixels) + changed = model(torch.full((4,), 2.0), pixels) + assert not torch.equal(first, changed) + assert model.calls == 2 and model.vision_calls == 1 + torch.testing.assert_close( + model(torch.full((4,), 2.0), pixels), changed, rtol=0, atol=0 + ) + assert model.calls == 2 + model(torch.ones(4), pixels + 1) + assert model.vision_calls == 2 + + +@torch.no_grad() +def test_content_masks_model_identity_and_copy_isolation(): + cache = ConditioningCache(1024 * 1024) + model = Encoder().eval() + x = torch.arange(1024, dtype=torch.float32).reshape(1, 4, 16, 16) + with cache.scope(): + original = model(x) + expected = original.last_hidden_state.clone() + original.last_hidden_state.zero_() + hit = model(x.clone()) + assert hit.hidden_states[0] is hit.last_hidden_state + torch.testing.assert_close(hit.last_hidden_state, expected, rtol=0, atol=0) + hit.hidden_states[0].zero_() + torch.testing.assert_close(model(x).last_hidden_state, expected, rtol=0, atol=0) + changed_alpha = x.clone() + changed_alpha[:, 3, 0, 0] += 1 + model(changed_alpha) + model(x, attention_mask=torch.zeros_like(x)) + other_model = Encoder().eval() + other_model(x) + assert model.calls == 3 + assert other_model.calls == 1 + assert cache.hits == 2 + + +@torch.no_grad() +def test_lru_budget_and_oversized_outputs(): + cache = ConditioningCache(32) + model = Encoder().eval() + with cache.scope(): + for value in (1, 2, 1, 3, 2): + model(torch.full((4,), float(value))) + before = cache.bytes + model(torch.ones(100)) + assert model.calls == 5 # one LRU hit, then an oversized miss + assert cache.hits == 1 + assert cache.evictions == 2 + assert cache.bytes == before == 32 + assert cache.bypasses == 1 + + +@torch.no_grad() +def test_device_hot_promotion_shares_budget_and_eviction_with_host_entries(): + cache = ConditioningCache(32) + model = Encoder().eval() + namespace = Encoder().eval() + + def encode(value): + return cached_encoder_call( + model, + (value,), + {}, + lambda: model(torch.full((4,), value)), + namespace=namespace, + share_in_group=True, + ) + + with cache.scope(): + encode(1.0) + encode(2.0) + assert cache.bytes == 32 and cache.stats()["device_bytes"] == 0 + with prefer_conditioning_cache(): + promoted = encode(1.0) + promoted.last_hidden_state.zero_() + assert cache.bytes == 32 and cache.stats()["device_bytes"] == 16 + encode(3.0) + assert cache.bytes == cache.stats()["device_bytes"] == 32 + encode(4.0) # a cold positive must not evict either preferred negative + assert cache.bytes == cache.stats()["device_bytes"] == 32 + with prefer_conditioning_cache(): + torch.testing.assert_close(encode(1.0).last_hidden_state, torch.ones(4)) + encode(5.0) # replaces the least-recent negative within the same budget + assert cache.evictions == 2 + assert cache.bytes == cache.stats()["device_bytes"] == 32 + cache.clear() + assert cache.bytes == cache.stats()["device_bytes"] == 0 + + +@torch.no_grad() +def test_warmup_executes_nested_encoders_and_seeds_serving_cache(): + cache = ConditioningCache(4096) + model = VisionLanguageEncoder().eval() + text, pixels = torch.ones(4), torch.ones(4) + with cache.scope(): + expected = model(text, pixels) + with cache.scope(refresh=True): + model(text, pixels) + model(text, pixels) + assert model.calls == model.vision_calls == 3 + torch.testing.assert_close(model(text, pixels), expected, rtol=0, atol=0) + assert model.calls == model.vision_calls == 3 + model(text + 1, pixels) + assert model.calls == 4 and model.vision_calls == 3 + + +@pytest.mark.parametrize("bypass", ["disabled", "inactive", "training", "grad", "ar"]) +def test_bypass(bypass): + cache = ConditioningCache(0 if bypass == "disabled" else 1024) + model = Encoder().eval() + if bypass == "training": + model.train() + kwargs = {"use_cache": True} if bypass == "ar" else {} + with ( + torch.set_grad_enabled(bypass == "grad"), + cache.scope(enabled=bypass != "inactive"), + ): + model(torch.ones(4), **kwargs) + model(torch.ones(4), **kwargs) + assert model.calls == 2 + assert cache.hits == 0 + + +@torch.no_grad() +def test_posterior_cache_preserves_rng_and_sampling(): + cache = ConditioningCache(1024 * 1024) + vae = VAE().eval() + x = torch.linspace(-1, 1, 64).reshape(1, 4, 4, 4) + with cache.scope(): + posterior = vae.encode(x).latent_dist + cold = posterior.sample(generator=torch.Generator().manual_seed(42)) + posterior.mean.add_(100) # downstream normalization must not poison storage + generator = torch.Generator().manual_seed(42) + state = generator.get_state().clone() + hit = vae.encode(x).latent_dist + assert torch.equal(generator.get_state(), state) + warm = hit.sample(generator=generator) + torch.testing.assert_close(cold, warm, rtol=0, atol=0) + assert not torch.equal( + warm, hit.sample(generator=torch.Generator().manual_seed(43)) + ) + vae.use_tiling = True + vae.encode(x) + assert vae.calls == 2 + + +@torch.no_grad() +def test_weight_invalidation_and_precision(): + cache = ConditioningCache(1024) + model = Encoder().eval() + x = torch.ones(4) + with cache.scope(): + model(x) + model(x) + invalidate_conditioning_caches() + model.weight.fill_(2) + assert torch.equal(model(x).last_hidden_state, x * 2) + model.to(torch.float64) + model(x) + assert model.calls == 3 + assert cache.hits == 1 + + +@torch.no_grad() +def test_scoped_invalidation_preserves_independent_encoders_and_shared_weights(): + cache = ConditioningCache(1024) + independent = Encoder().eval() + nested = Encoder().eval() + alias = Encoder().eval() + alias.weight = nested.weight + transformer = torch.nn.ModuleList([nested]) + x = torch.ones(4) + with cache.scope(): + for model in (independent, nested, alias): + model(x) + invalidate_conditioning_caches([transformer]) + nested.weight.fill_(2) + assert torch.equal(independent(x).last_hidden_state, x) + assert torch.equal(nested(x).last_hidden_state, x * 2) + assert torch.equal(alias(x).last_hidden_state, x * 2) + assert (independent.calls, nested.calls, alias.calls) == (1, 2, 2) + assert cache.bytes <= cache.max_bytes + + +@torch.no_grad() +def test_negative_conditioning_survives_positive_capacity_pressure(): + model = Encoder().eval() + # only one entry fits, reproducing two LTX embeddings in the host budget + cache = ConditioningCache(24) + positive, negative = torch.ones(4), torch.zeros(4) + with cache.scope(refresh=True): + model(positive) + with prefer_conditioning_cache(): + model(negative) + with cache.scope(): + model(positive + 1) + with prefer_conditioning_cache(): + actual = model(negative) + assert model.calls == 3 + assert cache.hits == 1 + assert cache.bytes == 16 + assert cache.evictions == 1 + torch.testing.assert_close(actual.last_hidden_state, negative) + # preferred entries still replace each other and remain bounded + with prefer_conditioning_cache(): + model(negative - 1) + assert cache.bytes == 16 + assert cache.evictions == 2 + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA transfers") +@torch.no_grad() +def test_cuda_snapshot_waits_for_producing_stream_before_restore(): + cache = ConditioningCache(32 * 1024 * 1024) + model = Encoder().eval() + stream = torch.cuda.Stream() + with torch.cuda.stream(stream): + value = torch.arange(1024 * 1024, device="cuda").reshape(1024, 1024) + output = BaseEncoderOutput( + last_hidden_state=value, + hidden_states=(value, value.T, value[::2, ::2]), + ) + cache.run(model, "forward", (), {}, lambda: output) + value.zero_() + restored = cache.run(model, "forward", (), {}, lambda: pytest.fail("cache miss")) + expected = torch.arange(1024 * 1024).reshape(1024, 1024) + assert restored.last_hidden_state is restored.hidden_states[0] + for actual, reference in zip( + restored.hidden_states, (expected, expected.T, expected[::2, ::2]), strict=True + ): + torch.testing.assert_close(actual.cpu(), reference, rtol=0, atol=0) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA transfers") +@torch.no_grad() +def test_cuda_cache_hit_does_not_wait_for_unrelated_gpu_work(): + cache = ConditioningCache(4096) + model = Encoder().eval() + value = torch.arange(64, device="cuda") + cache.run(model, "forward", (), {}, lambda: value) + torch.cuda.synchronize() + # a negative-prompt hit must not synchronize the preceding positive encode + torch.cuda._sleep(250_000_000) + pending = torch.cuda.Event() + pending.record() + restored = cache.run(model, "forward", (), {}, lambda: pytest.fail("cache miss")) + assert not pending.query() + torch.testing.assert_close(restored, value, rtol=0, atol=0) + + +@torch.no_grad() +def test_compiled_consumed_conditioning_bypasses_context_lookup(): + model = torch.nn.Linear(2, 2).eval() + cache = ConditioningCache(1024) + + @torch.compile(backend="eager", fullgraph=True) + def encode(x): + return cached_encoder_call( + model, + (x,), + {}, + lambda: model(x), + namespace=model, + share_in_group=True, + ) + + with cache.scope(), prefer_conditioning_cache(): + x = torch.ones(1, 2) + torch.testing.assert_close(encode(x), model(x), rtol=0, atol=0) + assert cache.stats()["entries"] == 0 + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA graph capture") +@pytest.mark.parametrize("consumed", [False, True]) +@torch.no_grad() +def test_cuda_graph_capture_bypasses_conditioning_cache(consumed): + cache = ConditioningCache(1024) + model = Encoder().cuda().eval() + x = torch.ones(4, device="cuda") + + def encode(): + return ( + cached_encoder_call( + model, + (x,), + {}, + lambda: model(x), + namespace=model, + share_in_group=True, + ) + if consumed + else model(x) + ) + + with cache.scope(), prefer_conditioning_cache(): + encode() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + output = encode() + x.fill_(3) + graph.replay() + torch.testing.assert_close(output.last_hidden_state, x, rtol=0, atol=0) + assert model.calls == 2 + assert cache.hits == 0 + + +def _rank_eviction(rank, init_method, disabled_rank, grouped=False): + dist.init_process_group( + "gloo", + rank=rank, + world_size=2, + init_method=init_method, + timeout=timedelta(seconds=30), + ) + try: + cache = ConditioningCache(0 if disabled_rank and rank == 0 else 1024) + model = Encoder().eval() + group = SimpleNamespace(world_size=2, cpu_group=dist.group.WORLD) + x = torch.ones(4) + + def compute(): + # The same ordering requirement as a TP encoder forward. + flag = torch.ones(1) + dist.all_reduce(flag) + return model.forward(x) + + with torch.no_grad(), cache.group_scope() if grouped else nullcontext(): + for attempt in range(3): + if attempt == 1 and rank == 0: + cache.clear() + if attempt == 2 and rank == 0 and grouped and not disabled_rank: + # One rank restores a host entry while its peer reuses a device entry. + cache._group_entries.get().clear() + cache.run(model, "forward", (x,), {}, compute, group) + assert model.calls == (3 if disabled_rank and not grouped else 2) + assert cache.hits == (0 if disabled_rank and not grouped else 1) + finally: + dist.destroy_process_group() + + +@pytest.mark.parametrize("disabled_rank", [False, True]) +@pytest.mark.parametrize("grouped", [False, True]) +def test_rank_local_eviction_forces_collective_miss(tmp_path, disabled_rank, grouped): + mp.spawn( + _rank_eviction, + args=(f"file://{tmp_path / 'rendezvous'}", disabled_rank, grouped), + nprocs=2, + join=True, + ) + + +@torch.no_grad() +def test_stage_conditioning_owns_output_but_preserves_vision_cache(): + model = VisionLanguageEncoder().eval() + namespace = Encoder() + cache = ConditioningCache(4096) + pixels = torch.ones(4) + hooks = [] + model.register_forward_hook(lambda *args: hooks.append(1)) + + def stage(text): + return cached_encoder_call( + model, + (text, pixels), + {}, + lambda: model(text, pixels)[:1], + namespace=namespace, + ) + + with cache.scope(): + first = stage(torch.ones(4)) + # one vision entry and one consumed output, without a full encoder copy + assert cache.bytes == 20 + assert cache.stats()["entries"] == 2 + first.zero_() + assert stage(torch.ones(4)).item() == 3 + assert model.calls == model.vision_calls == len(hooks) == 1 + assert stage(torch.full((4,), 2.0)).item() == 4 + assert model.calls == len(hooks) == 2 + assert model.vision_calls == 1 + with cache.scope(refresh=True): + stage(torch.ones(4)) + assert model.calls == len(hooks) == 3 + assert model.vision_calls == 2 + + def fail(): + raise RuntimeError("encoding failed") + + with pytest.raises(RuntimeError, match="encoding failed"): + cached_encoder_call(model, ("failure",), {}, fail, namespace=namespace) + # the temporary stage owner must not bypass later encoder caching + model(torch.ones(4), pixels) + model(torch.ones(4), pixels) + assert model.calls == 4 + + +@pytest.mark.parametrize("capacity", [0, 1, 4096]) +@torch.no_grad() +def test_group_cache_preserves_mutable_outputs_and_posterior_rng(capacity): + cache = ConditioningCache(capacity) + encoder = Encoder().eval() + vae = VAE().eval() + x = torch.ones(1, 4, 2, 2) + with cache.scope(), cache.group_scope(): + first = encoder(x) + first.last_hidden_state.zero_() + second = encoder(x) + assert encoder.calls == 1 + assert second.hidden_states[0] is second.last_hidden_state + torch.testing.assert_close(second.last_hidden_state, x, rtol=0, atol=0) + second.last_hidden_state.zero_() + torch.testing.assert_close(encoder(x).last_hidden_state, x, rtol=0, atol=0) + for seed in (13, 27): + generator = torch.Generator().manual_seed(seed) + control = torch.Generator().manual_seed(seed) + posterior = vae.encode(x).latent_dist + expected = DiagonalGaussianDistribution(x).sample(control) + torch.testing.assert_close( + posterior.sample(generator), expected, rtol=0, atol=0 + ) + assert torch.equal(generator.get_state(), control.get_state()) + posterior.mean.zero_() + assert vae.calls == 1 + assert cache._group_entries.get() is None + + +@torch.no_grad() +def test_group_hit_preserves_negative_host_entry_under_capacity_pressure(): + model = Encoder().eval() + cache = ConditioningCache(24) + negative = torch.zeros(4) + with cache.scope(), cache.group_scope(): + model(negative) + with prefer_conditioning_cache(): + model(negative) + model(torch.ones(4)) + with cache.scope(): + actual = model(negative) + assert model.calls == 2 + assert cache.hits == 2 + assert cache.group_hits == 1 + assert cache.evictions == 0 + torch.testing.assert_close(actual.last_hidden_state, negative, rtol=0, atol=0) + + +@torch.no_grad() +def test_group_only_scope_ignores_host_entries_and_nested_encoder_caches(): + model = VisionLanguageEncoder().eval() + cache = ConditioningCache(4096) + namespace = Encoder() + x = torch.ones(4) + + def stage(): + return cached_encoder_call( + model, + (x, x), + {}, + lambda: model(x, x), + namespace=namespace, + share_in_group=True, + ) + + with cache.scope(): + expected = stage() + entries = cache.stats()["entries"] + with cache.scope(cross_request=False): + for _ in range(2): + with cache.group_scope(): + torch.testing.assert_close(stage(), expected, rtol=0, atol=0) + stage() + assert cache.stats()["entries"] == entries + assert model.calls == model.vision_calls == 3 + stage() + stage() + assert model.calls == model.vision_calls == 5 + stage() + assert model.calls == 5 + assert cache.group_hits == 2 + + +@torch.no_grad() +def test_warmup_refreshes_existing_group_then_reuses_fresh_results(): + cache = ConditioningCache(4096) + encoder = Encoder().eval() + x = torch.ones(4) + with cache.scope(), cache.group_scope(): + encoder(x) + with cache.scope(refresh=True): + encoder(x) + encoder(x) + assert encoder.calls == 2 + assert cache.group_hits == 1 + + +@torch.no_grad() +def test_group_cache_invalidates_and_releases_after_failure(): + cache = ConditioningCache(0) + model = Encoder().eval() + x = torch.ones(4) + with pytest.raises(RuntimeError, match="stop"), cache.scope(), cache.group_scope(): + model(x) + model(x) + invalidate_conditioning_caches([model]) + model.weight.fill_(2) + torch.testing.assert_close(model(x).last_hidden_state, x * 2) + assert model.calls == 2 + raise RuntimeError("stop") + assert cache._group_entries.get() is None + with cache.scope(), cache.group_scope(): + model(x) + assert model.calls == 3 + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@torch.no_grad() +def test_group_cache_waits_for_producing_cuda_stream(): + cache = ConditioningCache(0) + model = Encoder().cuda().eval() + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with cache.scope(), cache.group_scope(): + with torch.cuda.stream(stream): + torch.cuda._sleep(10_000_000) + x = torch.ones(1024, device="cuda") + cache.run(model, "stream", (), {}, lambda: model.forward(x)) + output = cache.run(model, "stream", (), {}, lambda: pytest.fail("cache miss")) + torch.testing.assert_close( + output.last_hidden_state, torch.ones_like(output.last_hidden_state) + ) + assert model.calls == 1 + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, *sys.argv[1:]])) diff --git a/python/sglang/multimodal_gen/test/unit/test_conditioning_model_boundaries.py b/python/sglang/multimodal_gen/test/unit/test_conditioning_model_boundaries.py new file mode 100644 index 0000000000..d493d1bf77 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_conditioning_model_boundaries.py @@ -0,0 +1,142 @@ +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch + +from sglang.multimodal_gen.runtime.cache.conditioning import ( + ConditioningCache, + invalidate_conditioning_caches, +) +from sglang.multimodal_gen.runtime.models.dits.cosmos3video import Cosmos3LanguageModel +from sglang.multimodal_gen.runtime.models.sensenova_u1.neo_unify.modeling_neo_chat import ( + NEOChatModel, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.realtime.text_encoding import ( + RealtimeTextEncodingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import ( + TextEncodingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.vla import ( + VLAPrefixEncodingStage, +) +from sglang.multimodal_gen.runtime.realtime.session import RealtimeSession + + +class UndLayer(torch.nn.Module): + def __init__(self): + super().__init__() + self.calls = 0 + + def forward(self, hidden, cos_sin, positions): + self.calls += 1 + return hidden, hidden + 2, hidden + 3 + + +@torch.no_grad() +def test_cosmos_understanding_cache_restores_private_kv(): + model = Cosmos3LanguageModel.__new__(Cosmos3LanguageModel) + torch.nn.Module.__init__(model) + model.embed_tokens = torch.nn.Embedding(8, 4) + model.rotary_emb = SimpleNamespace( + build_rope_cache_inputs=lambda positions, cache_dtype: (None, positions) + ) + layer = UndLayer() + model.layers = torch.nn.ModuleList([layer]) + model.eval() + ids = torch.tensor([[1, 2]]) + mask = torch.ones_like(ids) + positions = torch.zeros(3, 1, 2, dtype=torch.long) + with ConditioningCache(4096).scope(): + first = model(ids, mask, positions) + expected = first[0][0].clone() + first[0][0].zero_() + hit = model(ids, mask, positions) + torch.testing.assert_close(hit[0][0], expected, rtol=0, atol=0) + assert layer.calls == 1 + model(ids, mask, positions + 1) + model(ids, mask * 0, positions) + assert layer.calls == 3 + + +class Vision(torch.nn.Module): + def __init__(self): + super().__init__() + self.calls = 0 + + def forward(self, pixel_values, **kwargs): + self.calls += 1 + return SimpleNamespace(last_hidden_state=pixel_values * 2) + + +@torch.no_grad() +def test_sensenova_caches_references_but_runs_noisy_vision_each_step(): + model = NEOChatModel.__new__(NEOChatModel) + torch.nn.Module.__init__(model) + model.vision_model = Vision() + model.fm_modules = torch.nn.ModuleDict({"vision_model_mot_gen": Vision()}) + model.eval() + pixels = torch.ones(2, 4) + with ConditioningCache(4096).scope(): + for _ in range(2): + model.extract_feature(pixels) + model.extract_feature(pixels, gen_model=True) + assert model.vision_model.calls == 1 + assert model.fm_modules["vision_model_mot_gen"].calls == 2 + + +@pytest.mark.parametrize("disabled, capacity", [(False, 512), (True, 512), (False, 0)]) +def test_realtime_text_respects_disable_and_weight_invalidation( + monkeypatch, disabled, capacity +): + calls = [] + + def encode(self, batch, server_args): + calls.append(batch.prompt) + batch.prompt_embeds = [torch.ones(1)] + return batch + + monkeypatch.setattr(TextEncodingStage, "forward", encode) + stage = RealtimeTextEncodingStage([], []) + session = RealtimeSession() + args = SimpleNamespace( + disable_conditioning_cache=disabled, + conditioning_cache_max_size_mb=capacity, + ) + for _ in range(2): + stage.forward(Req(prompt="a cat", session=session), args) + expected = 2 if disabled or capacity == 0 else 1 + assert len(calls) == expected + invalidate_conditioning_caches() + stage.forward(Req(prompt="a cat", session=session), args) + assert len(calls) == expected + 1 + + +@pytest.mark.parametrize("disabled, capacity", [(False, 512), (True, 512), (False, 0)]) +def test_vla_prefix_respects_disable_and_weight_invalidation(disabled, capacity): + stage = VLAPrefixEncodingStage.__new__(VLAPrefixEncodingStage) + stage.policy_model = Mock() + stage.policy_model.build_prefix_cache_key.return_value = "observation" + entries = {} + stage.prefix_cache = entries + args = SimpleNamespace( + disable_conditioning_cache=disabled, + conditioning_cache_max_size_mb=capacity, + pipeline_config=SimpleNamespace(enable_global_prefix_cache=True), + ) + batch = Req(prompt="pick up the cube", extra={"vla": {}}) + key, cached = stage.get_cached_context(batch, args, None) + assert cached is None + if disabled or capacity == 0: + assert key is None + stage.policy_model.build_prefix_cache_key.assert_not_called() + else: + context = object() + entries[key] = context + assert stage.get_cached_context(batch, args, None)[1] is context + invalidate_conditioning_caches() + assert stage.get_cached_context(batch, args, None)[1] is None diff --git a/python/sglang/multimodal_gen/test/unit/test_ideogram4.py b/python/sglang/multimodal_gen/test/unit/test_ideogram4.py index 657c588404..cbb3e989fc 100644 --- a/python/sglang/multimodal_gen/test/unit/test_ideogram4.py +++ b/python/sglang/multimodal_gen/test/unit/test_ideogram4.py @@ -704,6 +704,9 @@ class TestIdeogram4(unittest.TestCase): } stage.copy_deduplicated_outputs(base, same) + self.assertIs(same.prompt_embeds[0], base.prompt_embeds[0]) + self.assertIs(same.prompt_embeds_mask[0], base.prompt_embeds_mask[0]) + self.assertIsNot(same.prompt_embeds, base.prompt_embeds) self.assertIn("ideogram4", same.extra) self.assertTrue( torch.equal( diff --git a/python/sglang/multimodal_gen/test/unit/test_lingbot_video_moe.py b/python/sglang/multimodal_gen/test/unit/test_lingbot_video_moe.py index e82f692843..063ec0c7de 100644 --- a/python/sglang/multimodal_gen/test/unit/test_lingbot_video_moe.py +++ b/python/sglang/multimodal_gen/test/unit/test_lingbot_video_moe.py @@ -235,6 +235,7 @@ class _FakeQwenProcessor: def _text_encoding_stage(processor, encoder): stage = object.__new__(LingBotVideoTextEncodingStage) + stage._component_residency_manager = None stage.text_encoders = [encoder] stage.tokenizers = [processor] stage.token_length = 128 diff --git a/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py b/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py index 813c1b1cd9..29dd9fa573 100644 --- a/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py +++ b/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py @@ -7,6 +7,7 @@ import pytest import torch from prometheus_client import CollectorRegistry +from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningCache from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear from sglang.multimodal_gen.runtime.layers.lora.linear import ( BaseLayerWithLoRA, @@ -88,6 +89,35 @@ def test_worker_metrics_count_individual_adapters_in_multi_lora(): ] +@pytest.mark.parametrize("operation", ["set", "merge", "unmerge", "deactivate"]) +@torch.no_grad() +def test_lora_mutations_preserve_independent_conditioning(operation): + layer = _make_layer() + pipeline = _make_pipeline(layer) + pipeline.modules["transformer"].add_module("linear", layer) + pipeline._temporarily_disable_offload = lambda *args, **kwargs: nullcontext([]) + encoder = torch.nn.Linear(2, 2).eval() + cache = ConditioningCache(1024) + x = torch.ones(1, 2) + with patch(_RANK_PATCH, return_value=0): + pipeline.set_lora("adapter", merge_mode="dynamic") + cache.run(encoder, "forward", (x,), {}, lambda: encoder(x)) + cache.run(layer, "forward", (x,), {}, lambda: x.clone()) + if operation == "set": + pipeline.set_lora("adapter", strength=0.5, merge_mode="dynamic") + elif operation == "merge": + pipeline.merge_lora_weights() + elif operation == "unmerge": + pipeline.unmerge_lora_weights() + else: + pipeline.deactivate_lora_weights() + cached = cache.run(encoder, "forward", (x,), {}, lambda: encoder(x)) + cache.run(layer, "forward", (x,), {}, lambda: x.clone()) + torch.testing.assert_close(cached, encoder(x), rtol=0, atol=0) + assert cache.hits == 1 + assert cache.misses == 3 + + def test_merge_cache_only_accepts_cpu_backed_weights(): pipeline = _make_pipeline(_make_layer()) cpu_cache = pipeline._merge_cache_for( diff --git a/python/sglang/multimodal_gen/test/unit/test_pipeline_executor.py b/python/sglang/multimodal_gen/test/unit/test_pipeline_executor.py index f6a5853b27..345b4d5762 100644 --- a/python/sglang/multimodal_gen/test/unit/test_pipeline_executor.py +++ b/python/sglang/multimodal_gen/test/unit/test_pipeline_executor.py @@ -5,6 +5,7 @@ from unittest.mock import Mock import pytest import torch +from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningEncoderMixin from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency import ( COMPONENT_OFFLOAD, LAYERWISE_OFFLOAD, @@ -14,11 +15,12 @@ from sglang.multimodal_gen.runtime.pipelines_core.executors.pipeline_executor im PipelineExecutor, ) from sglang.multimodal_gen.runtime.platforms.npu import NPUPlatformBase +from sglang.multimodal_gen.runtime.server_args import ServerArgs class _RecordingExecutor(PipelineExecutor): def __init__(self): - super().__init__(server_args=SimpleNamespace()) + super().__init__(server_args=_server_args()) self.single_inference_mode = None self.group_inference_mode = None self.single_grad_enabled = None @@ -47,6 +49,8 @@ class _TestServerArgs(SimpleNamespace): def _server_args(**overrides): values = { "use_fsdp_inference": False, + "disable_conditioning_cache": ServerArgs.disable_conditioning_cache, + "conditioning_cache_max_size_mb": ServerArgs.conditioning_cache_max_size_mb, "component_modes": {}, } values.update(overrides) @@ -115,6 +119,31 @@ def test_execute_group_with_profiling_uses_platform_inference_mode(monkeypatch): assert executor.group_grad_enabled is False +@pytest.mark.parametrize("fsdp", [False, True]) +@torch.no_grad() +def test_warmup_conditioning_is_reused_by_first_serving_request(fsdp): + class Encoder(ConditioningEncoderMixin, torch.nn.Module): + def __init__(self): + super().__init__() + self.calls = 0 + + def forward(self, tokens): + self.calls += 1 + return tokens * 2 + + executor = _RecordingExecutor() + executor.component_residency_manager = Mock() + encoder = Encoder().eval() + tokens = torch.arange(8) + args = _server_args(use_fsdp_inference=fsdp) + for warmup in (True, True, False): + batch = _batch() + batch.is_warmup = warmup + with executor._component_residency_request([], batch, args): + torch.testing.assert_close(encoder(tokens), tokens * 2, rtol=0, atol=0) + assert encoder.calls == (3 if fsdp else 2) + + def test_group_payload_is_forwarded_to_component_residency_manager(): executor = _RecordingExecutor() executor.component_residency_manager = Mock() diff --git a/python/sglang/multimodal_gen/test/unit/test_qwen_image21.py b/python/sglang/multimodal_gen/test/unit/test_qwen_image21.py index 8be23b28a3..291a05b918 100644 --- a/python/sglang/multimodal_gen/test/unit/test_qwen_image21.py +++ b/python/sglang/multimodal_gen/test/unit/test_qwen_image21.py @@ -22,6 +22,7 @@ from sglang.multimodal_gen.configs.pipeline_configs.qwen_image21 import ( ) from sglang.multimodal_gen.configs.sample.qwenimage21 import QwenImage21SamplingParams from sglang.multimodal_gen.registry import _get_config_info +from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningCache from sglang.multimodal_gen.runtime.entrypoints.openai.image_api import ( _resolve_image_output_format, ) @@ -33,6 +34,9 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency_ ) from sglang.multimodal_gen.runtime.managers.scheduler import Scheduler from sglang.multimodal_gen.runtime.models.dits.qwen_image21 import build_layout +from sglang.multimodal_gen.runtime.models.encoders.base import ( + EncoderTensorParallelMixin, +) from sglang.multimodal_gen.runtime.models.encoders.qwen3vl_vision import ( Qwen3VLVisionRotaryEmbedding, ) @@ -120,6 +124,78 @@ def test_prompt_conditioning_uses_training_template_and_pre_norm(prompt, image_c assert image.getpixel((0, 0)) == (12, 34, 56, 0) +class ConditioningTestEncoder(EncoderTensorParallelMixin, torch.nn.Module): + def __init__(self, device): + super().__init__() + self.weight = torch.nn.Parameter(torch.ones((), device=device)) + self.model = SimpleNamespace( + visual=SimpleNamespace(rotary_pos_emb=SimpleNamespace()) + ) + self.calls = 0 + + def forward(self, input_ids, **kwargs): + self.calls += 1 + hidden = input_ids[..., None].float().expand(-1, -1, 4) * self.weight + return SimpleNamespace(hidden_states=tuple(hidden + i for i in range(32))) + + +@pytest.mark.parametrize( + "device", + [ + "cpu", + pytest.param( + "cuda", + marks=pytest.mark.skipif( + not torch.cuda.is_available(), reason="CUDA required" + ), + ), + ], +) +@torch.no_grad() +def test_prompt_cache_skips_processor_and_weight_preparation(device): + def process(text, **kwargs): + image_value = sum(kwargs["images"][0].getpixel((0, 0))) + return BatchFeature( + data={ + "input_ids": torch.tensor([[1, len(text[0]), 99, 99, image_value, 0]]), + "attention_mask": torch.tensor([[1, 1, 1, 1, 1, 0]]), + } + ) + + processor = Mock(side_effect=process) + processor.tokenizer.convert_tokens_to_ids.return_value = 99 + processor.apply_chat_template.return_value = [[1]] + encoder = ConditioningTestEncoder(device).eval() + stage = QwenImage21EncodingStage(encoder, processor, None, None) + stage.use_declared_component = Mock(return_value=nullcontext(encoder)) + image = Image.new("RGBA", (2, 2), (12, 34, 56, 128)) + cache = ConditioningCache(1024) + with cache.scope(): + first, slots = stage.encode_prompt("edit", [image], device) + expected = first.clone() + first.zero_() + slots.zero_() + restored, slots = stage.encode_prompt("edit", [image.copy()], device) + torch.testing.assert_close(restored, expected, atol=0, rtol=0) + assert slots.tolist() == [False, True, False] + assert encoder.calls == processor.call_count == 1 + stage.use_declared_component.assert_called_once() + assert cache.stats()["entries"] == 1 + assert cache.bytes == 3 * 4 * 4 + 3 + assert cache.bypasses == 0 # no oversized duplicate of all hidden states + changed, _ = stage.encode_prompt("another edit", [image], device) + assert not torch.equal(changed, expected) + image.putpixel((0, 0), (12, 34, 56, 255)) + changed_alpha, _ = stage.encode_prompt("edit", [image], device) + assert not torch.equal(changed_alpha, expected) + assert encoder.calls == 3 + with cache.scope(refresh=True): + stage.encode_prompt("edit", [image], device) + with cache.scope(enabled=False): + stage.encode_prompt("edit", [image], device) + assert encoder.calls == processor.call_count == 5 + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") def test_encoder_component_offload_preserves_loaded_dtypes(monkeypatch): monkeypatch.setattr( diff --git a/python/sglang/multimodal_gen/test/unit/test_sensenova_u1.py b/python/sglang/multimodal_gen/test/unit/test_sensenova_u1.py index 917da42be8..9678e8ffb4 100644 --- a/python/sglang/multimodal_gen/test/unit/test_sensenova_u1.py +++ b/python/sglang/multimodal_gen/test/unit/test_sensenova_u1.py @@ -1299,7 +1299,12 @@ def _make_sensenova_u1_sequential_entrypoint(*, fail=False, fail_request_ids=Non output_file_name="sample.png", trace_ctx=trace_ctx, ) - server_args = SimpleNamespace(pipeline_config=SenseNovaU1PipelineConfig()) + server_args = SimpleNamespace( + pipeline_config=SenseNovaU1PipelineConfig(), + disable_conditioning_cache=False, + conditioning_cache_max_size_mb=512, + use_fsdp_inference=False, + ) pipeline = _SequentialTestPipeline( server_args, fail=fail, fail_request_ids=fail_request_ids ) diff --git a/python/sglang/multimodal_gen/test/unit/test_text_encoding_cache.py b/python/sglang/multimodal_gen/test/unit/test_text_encoding_cache.py index a0ed2c93b6..1bc92c02b4 100644 --- a/python/sglang/multimodal_gen/test/unit/test_text_encoding_cache.py +++ b/python/sglang/multimodal_gen/test/unit/test_text_encoding_cache.py @@ -1,17 +1,202 @@ +import os +from contextlib import nullcontext +from datetime import timedelta +from functools import partial from types import SimpleNamespace -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, Mock, patch +import pytest import torch +import torch.distributed as dist +import torch.multiprocessing as mp +from torch.distributed.device_mesh import init_device_mesh +from torch.distributed.fsdp import fully_shard +from transformers import BatchEncoding +from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput +from sglang.multimodal_gen.configs.pipeline_configs.base import TextConditioningOutput +from sglang.multimodal_gen.configs.sample.longlive2 import LongLive2SamplingParams +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams +from sglang.multimodal_gen.runtime.cache import conditioning +from sglang.multimodal_gen.runtime.cache.conditioning import ConditioningCache +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + destroy_distributed_environment, + destroy_model_parallel, + init_distributed_environment, + initialize_model_parallel, +) +from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( + ComponentResidencyManager, +) +from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency_strategies import ( + ComponentOffloadStrategy, +) +from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder +from sglang.multimodal_gen.runtime.pipelines_core.executors.sync_executor import ( + SyncExecutor, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.causal_denoising import ( + CAUSAL_BLOCK_PROMPTS_KEY, + CAUSAL_SCENE_CUT_MASK_KEY, + CAUSAL_SHOT_INDICES_KEY, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.lingbot_video_moe.text_encoding import ( + LingBotVideoTextEncodingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.longlive2 import ( + LongLive2TextEncodingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ming_image import ( + MingImageEncodingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.realtime.text_encoding import ( + RealtimeTextEncodingStage, + RealtimeTextState, +) from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import ( TextEncodingStage, ) +from sglang.multimodal_gen.runtime.realtime.session import RealtimeSession _GLOBAL_ARGS_PATCH = ( "sglang.multimodal_gen.runtime.pipelines_core.stages.base.get_global_server_args" ) +class FullHiddenStateEncoder(TextEncoder): + uses_sglang_forward_context = False + + def __init__(self): + torch.nn.Module.__init__(self) + self.weight = torch.nn.Parameter(torch.ones(())) + self.calls = 0 + + def forward(self, input_ids, **kwargs): + self.calls += 1 + states = tuple(input_ids[..., None].float() + layer for layer in range(32)) + return BaseEncoderOutput( + last_hidden_state=states[-1], + hidden_states=states, + pooler_output=states[-1][:, 0].clone(), + ) + + +class LibraryEncoder(torch.nn.Module): + __init__ = FullHiddenStateEncoder.__init__ + forward = FullHiddenStateEncoder.forward + + +def make_text_config(output_type="tensor"): + def postprocess(output, text_inputs, return_attention_mask=False): + embedding = output.hidden_states[9] + mask = text_inputs["attention_mask"].bool() + if output_type == "tuple": + return embedding, mask + if output_type == "structured": + return TextConditioningOutput(embedding, mask, [2]) + return embedding + + def tokenize(texts, _tokenizer, _kwargs): + return BatchEncoding( + { + "input_ids": torch.tensor([[len(text), 2] for text in texts]), + "attention_mask": torch.ones((len(texts), 2), dtype=torch.long), + } + ) + + return SimpleNamespace( + text_encoder_configs=[SimpleNamespace(tokenizer_kwargs={})], + preprocess_text_funcs=[None], + postprocess_text_funcs=[postprocess], + text_encoder_extra_args=[], + is_flux_v1=lambda: False, + tokenize_prompt=tokenize, + get_text_encoder_attention_mask=lambda inputs, _: inputs["attention_mask"], + get_text_encoder_pooler_output=lambda outputs, _: outputs.pooler_output, + build_text_conditioning_mask=lambda inputs, mask, embeds, _: mask.bool(), + seq_lens_from_text_conditioning_mask=lambda mask: mask.sum(-1).tolist(), + ) + + +@pytest.mark.parametrize("output_type", ["tensor", "tuple", "structured"]) +@pytest.mark.parametrize( + "device", + [ + "cpu", + pytest.param( + "cuda", + marks=pytest.mark.skipif( + not torch.cuda.is_available(), reason="requires CUDA" + ), + ), + ], +) +@torch.no_grad() +def test_cache_stores_only_consumed_text_conditioning(output_type, device, monkeypatch): + encoder = FullHiddenStateEncoder().to(device).eval() + fingerprint = conditioning._fingerprint + + def fingerprint_host_inputs(value): + if isinstance(value, torch.Tensor): + assert value.device.type == "cpu", "token keys must not read GPU inputs" + return fingerprint(value) + + monkeypatch.setattr(conditioning, "_fingerprint", fingerprint_host_inputs) + + config = make_text_config(output_type) + args = make_server_args(pipeline_config=config) + + def make_stage(): + with patch(_GLOBAL_ARGS_PATCH, return_value=MagicMock()): + stage = TextEncodingStage(text_encoders=[encoder], tokenizers=[object()]) + stage._begin_text_encoder_use = MagicMock() + stage._text_encode_dp_group = MagicMock(return_value=None) + return stage + + stage = make_stage() + cache = ConditioningCache(4096) + with cache.scope(): + first = stage.encode_text( + "hello", args, device=device, return_attention_mask=True + ) + expected = first[0][0].clone() + expected_pooled = first[2][0].clone() + first[0][0].zero_() + first[2][0].zero_() + restored = stage.encode_text( + "hello", args, device=device, return_attention_mask=True + ) + torch.testing.assert_close(restored[0][0], expected, rtol=0, atol=0) + torch.testing.assert_close(restored[2][0], expected_pooled, rtol=0, atol=0) + assert restored[4] == [[2]] + assert encoder.calls == 1 + stage._begin_text_encoder_use.assert_called_once_with(0) + assert cache.stats()["entries"] == 1 + assert cache.bytes < 32 # two embeddings, one pooled value, optional mask + stage.encode_text("changed", args, device=device, return_attention_mask=True) + assert encoder.calls == 2 + # The same encoder can serve different pipeline postprocessing contracts. + make_stage().encode_text( + "hello", args, device=device, return_attention_mask=True + ) + assert encoder.calls == 3 + + # exercise the real negative-stage boundary with only one entry of capacity + cache = ConditioningCache(cache.bytes // 3) + stage.encode_text = partial(stage.encode_text, device=device) + calls_before = encoder.calls + with cache.scope(refresh=True): + stage.encode_text("hello", args, return_attention_mask=True) + stage.get_or_compute_negative_text_embedding(make_req(), args, [0]) + with cache.scope(): + stage.encode_text("changed positive", args, return_attention_mask=True) + stage.get_or_compute_negative_text_embedding(make_req(), args, [0]) + assert encoder.calls == calls_before + 3 + assert cache.hits == 1 + assert cache.bytes <= cache.max_bytes + + class DummyTextEncodingStage(TextEncodingStage): def __init__(self): with patch(_GLOBAL_ARGS_PATCH) as mock_global_args: @@ -53,6 +238,341 @@ def make_server_args(**kwargs): return SimpleNamespace(**defaults) +def make_group_executor( + stage_type, device, capacity, *, fsdp=False, native=True, tokenizer=None +): + encoder = ( + (FullHiddenStateEncoder() if native else LibraryEncoder()).to(device).eval() + ) + config = make_text_config() + config.supports_auto_residency = False + config.vae_config = SimpleNamespace(use_temporal_scaling_frames=False) + config.dit_config = SimpleNamespace( + arch_config=SimpleNamespace(num_frames_per_block=1) + ) + args = make_server_args( + pipeline_config=config, + component_precisions={}, + comfyui_mode=True, + enable_layerwise_nvtx_marker=False, + use_fsdp_inference=fsdp, + disable_conditioning_cache=capacity == 0, + conditioning_cache_max_size_mb=capacity / 1024**2, + should_cpu_offload_component=lambda _: False, + ) + with patch(_GLOBAL_ARGS_PATCH, return_value=args): + stage = stage_type( + [encoder], [tokenizer if tokenizer is not None else object()] + ) + stage._text_encode_dp_group = Mock(return_value=None) + stage.encode_text = partial(stage.encode_text, device=device) + pipeline = SimpleNamespace( + modules={"text_encoder": encoder}, + _stage_name_mapping={"text": stage}, + component_residency_strategies={}, + ) + executor = SyncExecutor(args) + manager = ComponentResidencyManager(pipeline, args) + strategy = Mock() + strategy.prefetch_for_use.return_value = False + manager.strategy_for = Mock(return_value=strategy) + executor.component_residency_manager = manager + return executor, stage, encoder, args + + +@pytest.mark.parametrize("capacity", [0, 4096]) +@torch.no_grad() +def test_grouped_realtime_text_updates_each_session(capacity): + executor, stage, encoder, args = make_group_executor( + RealtimeTextEncodingStage, "cpu", capacity + ) + sessions = [RealtimeSession(), RealtimeSession()] + requests = [ + Req( + sampling_params=SamplingParams(prompt="hello", num_inference_steps=4), + do_classifier_free_guidance=False, + session=session, + ) + for session in sessions + ] + + outputs = executor.execute_group([stage], requests, args) + + assert encoder.calls == 1 + for session, output in zip(sessions, outputs): + state = session.get_or_create_state(RealtimeTextState) + if capacity: + assert state.cache_key is not None + assert state.prompt_embeds[0] is output.prompt_embeds[0] + assert state.prompt_seq_lens[0] is not output.prompt_seq_lens[0] + else: + assert state.cache_key is None + + +@pytest.mark.parametrize("fsdp, native", [(True, False), (True, True), (False, False)]) +@torch.no_grad() +def test_one_unique_stage_skips_group_only_cache_lookup(fsdp, native, monkeypatch): + executor, stage, encoder, args = make_group_executor( + TextEncodingStage, "cpu", 4096, fsdp=fsdp, native=native + ) + fingerprint = Mock(wraps=conditioning._fingerprint) + monkeypatch.setattr(conditioning, "_fingerprint", fingerprint) + capture_query = Mock(wraps=torch.cuda.is_current_stream_capturing) + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", capture_query) + stage.forward = Mock(wraps=stage.forward) + requests = [ + Req( + sampling_params=SamplingParams(prompt="hello", num_inference_steps=4), + do_classifier_free_guidance=False, + ) + for _ in range(3) + ] + + outputs = executor.execute_group([stage], requests, args) + + assert stage.forward.call_count == 1 + assert encoder.calls == 1 + assert outputs[0].prompt_embeds[0] is outputs[2].prompt_embeds[0] + fingerprint.assert_not_called() + capture_query.assert_not_called() + + +@pytest.mark.parametrize("capacity", [0, 1, 4096]) +@pytest.mark.parametrize("fsdp", [False, True]) +@pytest.mark.parametrize("native", [False, True]) +@pytest.mark.parametrize("warmup", [False, True]) +@pytest.mark.parametrize( + "device", + [ + "cpu", + pytest.param( + "cuda", + marks=pytest.mark.skipif( + not torch.cuda.is_available(), reason="requires CUDA" + ), + ), + ], +) +@torch.no_grad() +def test_grouped_conditioning_reuses_positive_and_negative_independently( + capacity, device, fsdp, native, warmup +): + executor, stage, encoder, args = make_group_executor( + TextEncodingStage, device, capacity, fsdp=fsdp, native=native + ) + stage.forward = Mock(wraps=stage.forward) + + def requests(): + return [ + Req( + sampling_params=SamplingParams( + prompt=prompt, negative_prompt="bad quality", num_inference_steps=4 + ), + do_classifier_free_guidance=True, + is_warmup=warmup, + ) + for prompt in ("hello", "different", "hello") + ] + + first = executor.execute_group([stage], requests(), args) + assert stage.forward.call_count == 2 + assert encoder.calls == 3 # two positives, one shared negative + assert executor.conditioning_cache.group_hits == 1 + assert first[0].prompt_embeds[0] is first[2].prompt_embeds[0] + assert first[0].negative_prompt_embeds[0] is first[1].negative_prompt_embeds[0] + assert first[0].prompt_seq_lens is not first[2].prompt_seq_lens + assert first[0].prompt_seq_lens[0] is not first[2].prompt_seq_lens[0] + expected = first[0].prompt_embeds[0].clone() + first[0].prompt_embeds[0].zero_() + first[0].prompt_seq_lens[0][0] = 0 + assert executor.conditioning_cache._group_entries.get() is None + + second = executor.execute_group([stage], requests(), args) + assert stage.forward.call_count == 4 + persistent = capacity == 4096 and not fsdp and not warmup + expected_calls = (3 if native else 5) if persistent else 6 + assert encoder.calls == expected_calls + torch.testing.assert_close(second[0].prompt_embeds[0], expected, rtol=0, atol=0) + assert second[0].prompt_seq_lens == [[2]] + assert executor.conditioning_cache.bytes <= capacity + if fsdp: + assert executor.conditioning_cache.bytes == 0 + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("native", [False, True]) +@torch.no_grad() +def test_negative_conditioning_keeps_private_device_snapshot(native): + encoder = (FullHiddenStateEncoder() if native else LibraryEncoder()).cuda().eval() + config = make_text_config("structured") + config.postprocess_text_funcs[0] = Mock(wraps=config.postprocess_text_funcs[0]) + args = make_server_args(pipeline_config=config) + with patch(_GLOBAL_ARGS_PATCH, return_value=args): + stage = TextEncodingStage([encoder], [object()]) + stage._text_encode_dp_group = Mock(return_value=None) + stage._begin_text_encoder_use = Mock() + stage.encode_text = partial(stage.encode_text, device="cuda") + cache = ConditioningCache(4096) + producer, consumer = torch.cuda.Stream(), torch.cuda.Stream() + expected = torch.tensor([[[20.0], [11.0]]]) + + with cache.scope(), torch.cuda.stream(producer): + torch.cuda._sleep(250_000_000) + first = stage.get_or_compute_negative_text_embedding(make_req(), args, [0]) + first[0][0].zero_() + first[1][0].zero_() + first[4][0][0] = 0 + entry = next(iter(cache._entries.values())) + assert entry.device_resident + assert entry.output[0].device.type == "cuda" + assert entry.output[0].data_ptr() != first[0][0].data_ptr() + assert cache.stats()["device_bytes"] == cache.bytes <= 4096 + + with cache.scope(), torch.cuda.stream(consumer): + for _ in range(2): + hit = stage.get_or_compute_negative_text_embedding(make_req(), args, [0]) + torch.testing.assert_close(hit[0][0].cpu(), expected, rtol=0, atol=0) + assert hit[1][0].tolist() == [[1, 1]] + assert hit[4] == [[2]] + hit[0][0].zero_() + assert encoder.calls == config.postprocess_text_funcs[0].call_count == 1 + stage._begin_text_encoder_use.assert_called_once_with(0) + conditioning.invalidate_conditioning_caches([encoder]) + assert cache.bytes == cache.stats()["device_bytes"] == 0 + stage.get_or_compute_negative_text_embedding(make_req(), args, [0]) + assert encoder.calls == 2 + with cache.scope(refresh=True): + stage.get_or_compute_negative_text_embedding(make_req(), args, [0]) + assert encoder.calls == 3 + stage.get_or_compute_negative_text_embedding( + make_req(negative_prompt="changed negative"), args, [0] + ) + assert encoder.calls == 4 + + +@torch.no_grad() +def test_batched_request_reuses_conditioning_without_changing_output_seeds(): + executor, stage, encoder, args = make_group_executor(TextEncodingStage, "cpu", 0) + for _ in range(2): + request = Req( + sampling_params=SamplingParams( + prompt="hello", + negative_prompt="hello", + num_outputs_per_prompt=3, + seed=[7, 8, 9], + ), + do_classifier_free_guidance=True, + ) + output = executor.execute([stage], request, args) + assert output.seed == [7, 8, 9] + assert output.num_outputs_per_prompt == 3 + assert output.prompt_embeds[0] is output.negative_prompt_embeds[0] + assert executor.conditioning_cache._group_entries.get() is None + assert encoder.calls == 2 + assert executor.conditioning_cache.group_hits == 2 + + +@torch.no_grad() +def test_grouped_longlive_conditioning_keeps_per_request_shot_metadata(): + executor, stage, encoder, args = make_group_executor( + LongLive2TextEncodingStage, "cpu", 0 + ) + requests = [ + Req( + sampling_params=LongLive2SamplingParams( + prompt="hello", + shot_prompts=["first", "second"], + shot_durations=[1, 1], + num_frames=2, + ), + do_classifier_free_guidance=False, + ) + for _ in range(2) + ] + outputs = executor.execute_group([stage], requests, args) + assert encoder.calls == 1 + for key in ( + CAUSAL_BLOCK_PROMPTS_KEY, + CAUSAL_SCENE_CUT_MASK_KEY, + CAUSAL_SHOT_INDICES_KEY, + ): + assert key in outputs[0].extra and key in outputs[1].extra + assert outputs[0].extra[key] == outputs[1].extra[key] + assert outputs[0].extra[key] is not outputs[1].extra[key] + + +@pytest.mark.parametrize("fsdp", [False, True]) +@pytest.mark.parametrize( + "stage_type", [MingImageEncodingStage, LingBotVideoTextEncodingStage] +) +@torch.no_grad() +def test_custom_text_stages_preserve_group_reuse_and_request_metadata(fsdp, stage_type): + tokenizer = Mock(name_or_path="fixture", eos_token="") + tokenizer.side_effect = lambda text, **kwargs: {"input_ids": [len(text), 2]} + tokenizer.convert_tokens_to_ids.side_effect = lambda text: ( + 3 if text == "" else 4 + ) + factory = stage_type + if stage_type is LingBotVideoTextEncodingStage: + factory = partial(stage_type, transformer=torch.nn.Linear(1, 1)) + + def tokenize(**kwargs): + texts = kwargs["text"] + ids = ( + [[1]] + if isinstance(texts, str) + else [[1, len(text), 2] for text in texts] + ) + return BatchEncoding( + { + "input_ids": torch.tensor(ids), + "attention_mask": torch.ones_like(torch.tensor(ids)), + } + ) + + tokenizer.side_effect = tokenize + + with patch( + "sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages." + "ming_image.Qwen2VLImageProcessorPil.from_pretrained" + ): + executor, stage, encoder, args = make_group_executor( + factory, "cpu", 0, fsdp=fsdp, tokenizer=tokenizer + ) + encoder.dtype = torch.bfloat16 + encoder.config = SimpleNamespace(projection_config={"img_gen_scales": [1]}) + encoder.image_token = 99 + args.pipeline_config.dit_config.arch_config.alignment_padding_mode = "zero_masked" + args.pipeline_config.dit_config.arch_config.multi_frame_output = False + + def requests(): + return [ + Req( + sampling_params=SamplingParams( + prompt=prompt, negative_prompt="", height=64, width=64 + ), + do_classifier_free_guidance=False, + ) + for prompt in ("hello", "different", "hello") + ] + + with patch( + f"{stage_type.__module__}.get_local_torch_device", + return_value=torch.device("cpu"), + ): + for run in range(2): + outputs = executor.execute_group([stage], requests(), args) + assert encoder.calls == 2 * (run + 1) + assert outputs[0].prompt_embeds[0] is outputs[2].prompt_embeds[0] + assert outputs[0].prompt_embeds is not outputs[2].prompt_embeds + if stage_type is MingImageEncodingStage: + assert all(output.extra["ming_frames"] == 1 for output in outputs) + assert ( + outputs[0].extra["ming_direct"] is outputs[2].extra["ming_direct"] + ) + assert executor.conditioning_cache._group_entries.get() is None + + def get_negative_embedding_twice(stage, server_args, first_req, second_req=None): stage.get_or_compute_negative_text_embedding(first_req, server_args, [0]) stage.get_or_compute_negative_text_embedding( @@ -60,23 +580,62 @@ def get_negative_embedding_twice(stage, server_args, first_req, second_req=None) ) -def test_negative_text_cache_key_tracks_encode_options(): +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("capacity", [0, 1, 4096]) +@torch.no_grad() +def test_lingbot_cache_miss_prepares_offloaded_encoder(capacity, monkeypatch): + device = torch.device("cuda", torch.cuda.current_device()) + monkeypatch.setattr( + "sglang.multimodal_gen.runtime.managers.memory_managers." + "component_residency_strategies.get_local_torch_device", + lambda: device, + ) + executor, stage, encoder, args = make_group_executor( + partial(LingBotVideoTextEncodingStage, transformer=torch.nn.Linear(1, 1)), + "cpu", + capacity, + ) + encoder.embedding = torch.nn.Embedding(8, 2) + + def encode(input_ids, **kwargs): + encoder.calls += 1 + states = encoder.embedding(input_ids) + return BaseEncoderOutput(last_hidden_state=states, hidden_states=(states,)) + + encoder.forward = encode + stage._crop_start = 0 + stage._build_prompt_inputs = lambda prompt: BatchEncoding( + {"input_ids": torch.tensor([[1, 2]]), "attention_mask": torch.ones(1, 2)} + ) + manager = executor.component_residency_manager + strategy = ComponentOffloadStrategy() + strategy.prepare_for_use = Mock(wraps=strategy.prepare_for_use) + manager.strategy_for = Mock(return_value=strategy) + stage.set_component_residency_manager(manager) + cache = ConditioningCache(capacity) + results = [] + for _ in range(2): + batch = make_req() + manager.begin_request([stage], batch, args) + manager.before_stage(stage, 0, batch, args) + manager.begin_stage() + with cache.scope(): + results.append(stage._encode_prompt("hello", device, torch.float32)) + manager.end_stage() + manager.finish_request() + assert encoder.embedding.weight.device.type == "cpu" + assert results[-1][0].device == device + expected_calls = 1 if capacity == 4096 else 2 + assert encoder.calls == strategy.prepare_for_use.call_count == expected_calls + torch.testing.assert_close(results[0][0], results[1][0], rtol=0, atol=0) + + +def test_negative_text_encoding_has_no_separate_gpu_cache(): stage = DummyTextEncodingStage() server_args = make_server_args() - get_negative_embedding_twice(stage, server_args, make_req()) - assert stage.calls == 1 - - stage.get_or_compute_negative_text_embedding( - make_req(max_sequence_length=512), server_args, [0] - ) assert stage.calls == 2 - stage.get_or_compute_negative_text_embedding( - make_req(prompt_template={"template": "negative: {}"}), server_args, [0] - ) - assert stage.calls == 3 - def test_component_uses_exact_encoder_precision(): with patch(_GLOBAL_ARGS_PATCH) as mock_global_args: @@ -97,25 +656,280 @@ def test_component_uses_exact_encoder_precision(): ] -def test_negative_text_cache_skips_warmup(): +def test_negative_text_encoding_warmup_does_not_seed_a_private_cache(): stage = DummyTextEncodingStage() - server_args = make_server_args() - - with patch.object( - stage, "_get_model_default_negative_prompt", return_value="default negative" - ): - get_negative_embedding_twice(stage, server_args, make_req(is_warmup=True)) - + get_negative_embedding_twice(stage, make_server_args(), make_req(is_warmup=True)) assert stage.calls == 2 -def test_negative_text_cache_keeps_default_warmup(): - stage = DummyTextEncodingStage() - server_args = make_server_args() +@pytest.mark.parametrize("encoder_count", [1, 2]) +@torch.no_grad() +def test_cache_hit_skips_encoder_residency_and_input_preparation(encoder_count): + encoders = [FullHiddenStateEncoder().eval() for _ in range(encoder_count)] + config = make_text_config() + config.text_encoder_configs *= encoder_count + config.preprocess_text_funcs *= encoder_count + config.postprocess_text_funcs *= encoder_count + config.supports_auto_residency = False + config.get_text_encoder_attention_mask = Mock( + wraps=config.get_text_encoder_attention_mask + ) + config.seq_lens_from_text_conditioning_mask = Mock( + wraps=config.seq_lens_from_text_conditioning_mask + ) + args = make_server_args( + pipeline_config=config, + component_precisions={}, + enable_layerwise_nvtx_marker=False, + ) + with patch(_GLOBAL_ARGS_PATCH, return_value=args): + stage = TextEncodingStage(encoders, [object()] * encoder_count) + stage._text_encode_dp_group = Mock(return_value=None) + pipeline = SimpleNamespace( + modules={ + "text_encoder" if i == 0 else f"text_encoder_{i + 1}": encoder + for i, encoder in enumerate(encoders) + }, + _stage_name_mapping={"text": stage}, + component_residency_strategies={}, + ) + manager = ComponentResidencyManager(pipeline, args) + strategy = Mock() + strategy.prefetch_for_use.return_value = False + manager.strategy_for = Mock(return_value=strategy) + stage.set_component_residency_manager(manager) + cache = ConditioningCache(4096) - with patch.object( - stage, "_get_model_default_negative_prompt", return_value="bad quality" - ): - get_negative_embedding_twice(stage, server_args, make_req(is_warmup=True)) + def request(prompt, *, refresh=False, enabled=True, dtype=None): + batch = make_req(is_warmup=refresh) + manager.begin_request([stage], batch, args) + manager.before_stage(stage, 0, batch, args) + manager.begin_stage() + with (cache if enabled else ConditioningCache(0)).scope(refresh=refresh): + result = stage.encode_text( + prompt, + args, + encoder_index=list(range(encoder_count)), + device="cpu", + dtype=dtype, + return_attention_mask=True, + ) + manager.end_stage() + manager.finish_request() + return result - assert stage.calls == 1 + first = request("hello") + expected = first[0][0].clone() + first[0][0].zero_() + first[1][0].zero_() + first[3][0].zero_() + first[4][0][0] = 0 + strategy.reset_mock() + restored = request("hello") + strategy.prepare_for_use.assert_not_called() + strategy.wait_for_use.assert_not_called() + assert config.get_text_encoder_attention_mask.call_count == encoder_count + assert config.seq_lens_from_text_conditioning_mask.call_count == encoder_count + torch.testing.assert_close(restored[0][0], expected, rtol=0, atol=0) + assert restored[1][0].tolist() == [[1, 1]] + assert restored[3][0].tolist() == [[True, True]] + assert restored[4][0] == [2] + + # dtype is part of the consumed conditioning contract + cast = request("hello", dtype=torch.float64) + assert cast[0][0].dtype == torch.float64 + assert encoders[0].calls == 2 + request("hello", refresh=True) + request("changed") + request("hello", enabled=False) + assert all(encoder.calls == 5 for encoder in encoders) + assert strategy.prepare_for_use.call_count == 4 * encoder_count + + +class TextEncodingDPGroup: + world_size = 2 + + def __init__(self, rank): + self.rank_in_group = rank + self.gathers = 0 + self.cpu_group = dist.group.WORLD + + def all_reduce(self, tensor): + dist.all_reduce(tensor) + return tensor + + def all_gather(self, tensor, dim): + self.gathers += 1 + outputs = [torch.empty_like(tensor) for _ in range(self.world_size)] + dist.all_gather(outputs, tensor) + return torch.cat(outputs, dim=dim) + + +def run_text_encoding_dp(rank, rendezvous, grouped, negative): + dist.init_process_group( + "gloo", + rank=rank, + world_size=2, + init_method=rendezvous, + timeout=timedelta(seconds=30), + ) + try: + encoder = FullHiddenStateEncoder().eval() + args = make_server_args(pipeline_config=make_text_config()) + with patch(_GLOBAL_ARGS_PATCH, return_value=args): + stage = TextEncodingStage([encoder], [object()]) + group = TextEncodingDPGroup(rank) + stage._text_encode_dp_group = Mock(return_value=group) + cache = ConditioningCache(4096) + stage_module = TextEncodingStage.__module__ + with ( + torch.no_grad(), + cache.scope(), + cache.group_scope(enabled=grouped), + conditioning.prefer_conditioning_cache() if negative else nullcontext(), + patch(f"{stage_module}.model_parallel_is_initialized", return_value=True), + patch(f"{stage_module}.get_replica_group", return_value=group), + ): + for attempt in range(3): + if attempt == 1 and rank == 0: + cache.clear() + before = group.gathers + outputs = stage.encode_text( + ["a", "bb", "ccc"], args, device="cpu", return_attention_mask=True + ) + if (grouped or negative) and attempt == 2: + assert group.gathers == before + else: + assert group.gathers > before + assert outputs[0][0][:, 0, 0].tolist() == [10, 11, 12] + assert outputs[4] == [[2, 2, 2]] + assert encoder.calls == (2 if negative or rank == 0 else 1) + assert cache.hits == (1 if negative or rank == 0 else 2) + finally: + dist.destroy_process_group() + + +@pytest.mark.parametrize("grouped", [False, True]) +@pytest.mark.parametrize("negative", [False, True]) +def test_batch_dp_requires_consensus_to_skip_gather(tmp_path, grouped, negative): + mp.spawn( + run_text_encoding_dp, + args=(f"file://{tmp_path / 'dp-rendezvous'}", grouped, negative), + nprocs=2, + join=True, + ) + + +class ShardedTextEncoder(FullHiddenStateEncoder): + def __init__(self): + torch.nn.Module.__init__(self) + self.projection = torch.nn.Linear(1, 8, bias=False) + torch.nn.init.ones_(self.projection.weight) + self.calls = 0 + + def forward(self, input_ids, **kwargs): + self.calls += 1 + hidden = self.projection(input_ids[..., None].float()) + return BaseEncoderOutput(last_hidden_state=hidden, hidden_states=(hidden,) * 32) + + +def run_fsdp_conditioning(rank, rendezvous): + os.environ["LOCAL_RANK"] = str(rank) + torch.cuda.set_device(rank) + init_distributed_environment( + world_size=2, + rank=rank, + local_rank=rank, + distributed_init_method=rendezvous, + backend="nccl", + timeout=60, + ) + initialize_model_parallel(sequence_parallel_degree=2, ulysses_degree=2) + try: + encoder = ShardedTextEncoder().cuda().eval() + fully_shard( + encoder, mesh=init_device_mesh("cuda", (2,)), reshard_after_forward=True + ) + assert encoder.projection.weight.to_local().numel() == 4 + args = make_server_args(pipeline_config=make_text_config()) + with patch(_GLOBAL_ARGS_PATCH, return_value=args): + stage = TextEncodingStage([encoder], [object()]) + stage._text_encode_dp_group = Mock(return_value=None) + cache = ConditioningCache(4096) + with torch.no_grad(), cache.scope(cross_request=False): + for _ in range(2): + with cache.group_scope(): + for attempt in range(3): + if rank == 0 and attempt == 1: + cache.clear() + output = stage.encode_text("hello", args, device=f"cuda:{rank}") + torch.testing.assert_close( + output[0][0][0, :, 0], + torch.tensor([5.0, 2.0], device=f"cuda:{rank}"), + rtol=0, + atol=0, + ) + assert cache._group_entries.get() is None + assert encoder.calls == 4 + assert cache.group_hits == 2 + assert cache.bytes == 0 + assert encoder.projection.weight.to_local().numel() == 4 + finally: + destroy_model_parallel() + destroy_distributed_environment() + + +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="requires two CUDA GPUs") +def test_fsdp_group_reuse_agrees_across_shards_and_expires(tmp_path): + mp.spawn( + run_fsdp_conditioning, + args=(f"file://{tmp_path / 'fsdp-rendezvous'}",), + nprocs=2, + join=True, + ) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") +@torch.no_grad() +def test_cache_hit_releases_weights_retained_after_warmup(monkeypatch): + monkeypatch.setattr( + "sglang.multimodal_gen.runtime.managers.memory_managers." + "component_residency_strategies.get_local_torch_device", + lambda: torch.device("cuda"), + ) + encoder = FullHiddenStateEncoder().eval() + config = make_text_config() + config.supports_auto_residency = False + args = make_server_args( + pipeline_config=config, + component_precisions={}, + enable_layerwise_nvtx_marker=False, + ) + with patch(_GLOBAL_ARGS_PATCH, return_value=args): + stage = TextEncodingStage([encoder], [object()]) + stage._text_encode_dp_group = Mock(return_value=None) + pipeline = SimpleNamespace( + modules={"text_encoder": encoder}, + _stage_name_mapping={"text": stage}, + component_residency_strategies={}, + ) + manager = ComponentResidencyManager(pipeline, args) + strategy = ComponentOffloadStrategy() + manager.strategy_for = Mock(return_value=strategy) + stage.set_component_residency_manager(manager) + cache = ConditioningCache(4096) + for warmup in [True, False]: + batch = make_req(is_warmup=warmup) + manager.begin_request([stage], batch, args) + manager.before_stage(stage, 0, batch, args) + manager.begin_stage() + with cache.scope(refresh=warmup): + result = stage.encode_text( + "hello", args, device="cuda", return_attention_mask=True + ) + manager.end_stage() + manager.finish_request() + assert encoder.weight.device.type == ("cuda" if warmup else "cpu") + assert result[0][0].device.type == "cuda" + assert encoder.calls == 1 + assert cache.hits == 1