From a954b395e63f67b5fe4c5a4e4ea3bcf39b0f82f2 Mon Sep 17 00:00:00 2001 From: WenhaoZhang <42087078+niehen6174@users.noreply.github.com> Date: Mon, 12 Oct 2026 09:34:35 +0800 Subject: [PATCH] [diffusion] perf: layerwise offload for the LTX-2.5 Gemma-4 text encoder (#43683) --- docs/cookbook/diffusion/LTX/LTX2.5.mdx | 2 + .../component_loaders/text_encoder_loader.py | 21 +++++ .../models/encoders/gemma_4_unified.py | 61 +++++++++++++ .../test_gemma4_unified_layerwise_offload.py | 91 +++++++++++++++++++ 4 files changed, 175 insertions(+) create mode 100644 python/sglang/multimodal_gen/runtime/models/encoders/gemma_4_unified.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_gemma4_unified_layerwise_offload.py diff --git a/docs/cookbook/diffusion/LTX/LTX2.5.mdx b/docs/cookbook/diffusion/LTX/LTX2.5.mdx index 7b760058e9..483030870e 100644 --- a/docs/cookbook/diffusion/LTX/LTX2.5.mdx +++ b/docs/cookbook/diffusion/LTX/LTX2.5.mdx @@ -137,7 +137,9 @@ Selecting weights: | 1 high-VRAM GPU | *(no extra flags)* | 960×544 fits comfortably on an H200. | | 1 tight-VRAM GPU | `--quantization fp8` | Halves the DiT and cuts peak memory ~18 GB at unchanged speed. See [section 3.4](#3-4-fp8-quantization). | | 1 very tight GPU | `--dit-layerwise-offload` | Cuts peak memory by roughly 10 GB, at about 4x the wall clock. | +| 1 32 GB GPU (e.g. RTX 5090) | `--layerwise-offload-components dit text_encoder` | Streams the DiT and the 22 GB Gemma-4 encoder layer by layer and keeps the rest resident. 960×544 / 121 frames / 8 steps: 15.2 s at 17 GB peak, vs 17.0 s at 31 GB with `--cpu-offload-components text_encoder ...`. Pins ~55 GB of host memory per GPU. | | 2 GPUs, long sequences | `--num-gpus 2 --ulysses-degree 2` | Sequence parallel; the memory/long-sequence tool. | +| 2 32 GB GPUs | `--num-gpus 2 --ulysses-degree 2 --layerwise-offload-components dit text_encoder --vae-tiling false --vae-sp false --dit-layerwise-resident-layers 12` | Untiled VAE decode splits the frame height across the ranks (2.2 s to 1.1 s); 12 resident DiT layers cut the per-step host stream. RTX 5090 960×544 / 121 frames: 16.9 s at 26 GB peak per GPU, vs 21.8 s. Pins ~55 GB of host memory per GPU. | | 2 GPUs, large DiT | `--num-gpus 2 --tp-size 2` | Tensor parallel across attention heads. | | 2 GPUs, dev weights | `--num-gpus 2 --enable-cfg-parallel` | Splits the guided and unguided branches across GPUs. Measured 1.77x on denoising (15.1s to 8.5s, 960×544 / 57 frames / 30 steps). | | 2 GPUs, diffusion decoder | `--num-gpus 2 --ulysses-degree 2` | The decoder's tiles are split across the ranks by default. See [section 4.6.1](#4-6-1-memory-and-multi-gpu). | diff --git a/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py b/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py index fd6147e402..5a17f25593 100644 --- a/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py +++ b/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py @@ -79,6 +79,9 @@ from sglang.multimodal_gen.runtime.models.encoders.base import ( finalize_encoder_folding, get_folding_tp_group, ) +from sglang.multimodal_gen.runtime.models.encoders.gemma_4_unified import ( + NATIVE_LAYERWISE_ADAPTERS, +) from sglang.multimodal_gen.runtime.models.registry import ModelRegistry from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.server_args import ServerArgs @@ -473,6 +476,24 @@ class TextEncoderLoader(OnlineQuantizationComponentLoader): return transformers_model_class return transformers.AutoModel + def load_native( + self, + component_model_path: str, + server_args: ServerArgs, + transformers_or_diffusers: str, + component_name: str | None = None, + ): + model = super().load_native( + component_model_path, + server_args, + transformers_or_diffusers, + component_name, + ) + # A native encoder with no SGLang port can still stream its decoder + # stack when its layout is known; see NATIVE_LAYERWISE_ADAPTERS. + adapter = NATIVE_LAYERWISE_ADAPTERS.get(type(model).__name__) + return model if adapter is None else adapter(model) + def _get_all_weights( self, model: EncoderTensorParallelMixin, diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/gemma_4_unified.py b/python/sglang/multimodal_gen/runtime/models/encoders/gemma_4_unified.py new file mode 100644 index 0000000000..b07969877f --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/encoders/gemma_4_unified.py @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Layerwise-offload adapter for the transformers Gemma-4-Unified encoder. + +LTX-2.5's text encoder has no SGLang implementation and loads through +``transformers.AutoModel`` as ``Gemma4UnifiedModel``. Its decoder stack is a +plain ``ModuleList`` at ``language_model.layers``, which is all the layerwise +manager needs, so the loaded module's class is swapped for a subclass that also +carries the mixin. Forward, signature and weights are untouched. +""" + +import torch +from torch import nn + +from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + LayerwiseOffloadableModuleMixin, +) + + +class _Gemma4UnifiedLayerwise(LayerwiseOffloadableModuleMixin): + layerwise_offload_dit_group_enabled = False + # 48 decoder layers of ~454 MB in bf16: 21.8 of the 23.9 GB. + layer_names = ["language_model.layers"] + # The 2.0 GB vocab table is reached only through embed_tokens.__call__; + # AutoModel builds no tied lm_head. + host_resident_table_names = ["language_model.embed_tokens"] + + +class _HostGatherScaledEmbedding: + """Apply Gemma's 0-dim ``embed_scale`` on whichever device ran the gather. + + Upstream multiplies by the buffer in place; under host residency the gather + runs on the host while the buffer follows the module to the device. + """ + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + embeds = nn.Embedding.forward(self, input_ids) + return embeds * self.embed_scale.to(device=embeds.device, dtype=embeds.dtype) + + +def _swap_class(module: nn.Module, *, extra: type, first: bool) -> None: + cls = type(module) + if issubclass(cls, extra): + return + bases = (extra, cls) if first else (cls, extra) + module.__class__ = type(cls.__name__, bases, {"__module__": cls.__module__}) + + +def make_gemma4_unified_layerwise_offloadable(model: nn.Module) -> nn.Module: + _swap_class(model, extra=_Gemma4UnifiedLayerwise, first=False) + _swap_class( + model.language_model.embed_tokens, + extra=_HostGatherScaledEmbedding, + first=True, + ) + return model + + +# transformers class name -> adapter applied after the native load +NATIVE_LAYERWISE_ADAPTERS = { + "Gemma4UnifiedModel": make_gemma4_unified_layerwise_offloadable, +} diff --git a/python/sglang/multimodal_gen/test/unit/test_gemma4_unified_layerwise_offload.py b/python/sglang/multimodal_gen/test/unit/test_gemma4_unified_layerwise_offload.py new file mode 100644 index 0000000000..55e26b23ab --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_gemma4_unified_layerwise_offload.py @@ -0,0 +1,91 @@ +"""The native Gemma-4-Unified encoder (LTX-2.5) is layerwise-offloadable and exact.""" + +from unittest.mock import patch + +import pytest +import torch + +transformers = pytest.importorskip("transformers") +gemma4_unified = pytest.importorskip("transformers.models.gemma4_unified") + +from sglang.multimodal_gen.runtime.managers.memory_managers import layerwise_offload +from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + LayerwiseOffloadableModuleMixin, + _host_resident_tables, + detach_host_resident_tables, + restore_host_resident_tables, +) +from sglang.multimodal_gen.runtime.models.encoders.gemma_4_unified import ( + NATIVE_LAYERWISE_ADAPTERS, +) + + +def _tiny_model() -> torch.nn.Module: + config = gemma4_unified.Gemma4UnifiedConfig( + text_config=dict( + hidden_size=64, + intermediate_size=128, + num_hidden_layers=2, + num_attention_heads=2, + num_key_value_heads=1, + head_dim=32, + global_head_dim=32, + vocab_size=512, + layer_types=["sliding_attention", "full_attention"], + sliding_window=16, + hidden_size_per_layer_input=0, + pad_token_id=0, + ), + vision_config=dict( + hidden_size=32, + intermediate_size=64, + num_hidden_layers=1, + num_attention_heads=2, + num_key_value_heads=2, + head_dim=16, + ), + audio_config=None, + ) + torch.manual_seed(0) + return transformers.AutoModel.from_config(config).eval() + + +def _encode(model: torch.nn.Module, input_ids: torch.Tensor) -> torch.Tensor: + with torch.no_grad(): + out = model( + input_ids=input_ids, + attention_mask=torch.ones_like(input_ids), + output_hidden_states=True, + ) + return torch.stack(out.hidden_states) + + +def test_adapter_keeps_outputs_and_exposes_the_decoder_stack(): + model = _tiny_model() + input_ids = torch.randint(1, 500, (1, 12)) + reference = _encode(model, input_ids) + + adapter = NATIVE_LAYERWISE_ADAPTERS[type(model).__name__] + model = adapter(model) + + assert isinstance(model, LayerwiseOffloadableModuleMixin) + assert type(model).__name__ == "Gemma4UnifiedModel" + assert isinstance( + dict(model.named_modules())[model.layer_names[0]], torch.nn.ModuleList + ) + torch.testing.assert_close(_encode(model, input_ids), reference, rtol=0, atol=0) + # idempotent + assert type(adapter(model)) is type(model) + + +def test_host_resident_embedding_matches_resident_embedding(): + model = NATIVE_LAYERWISE_ADAPTERS["Gemma4UnifiedModel"](_tiny_model()) + input_ids = torch.randint(1, 500, (1, 12)) + reference = _encode(model, input_ids) + + threshold = f"{layerwise_offload.__name__}.HOST_RESIDENT_TABLE_MIN_BYTES" + with patch(threshold, 1024): + assert _host_resident_tables(model) == [model.language_model.embed_tokens] + restore_host_resident_tables(detach_host_resident_tables(model), "cpu") + + torch.testing.assert_close(_encode(model, input_ids), reference, rtol=0, atol=0)