[diffusion] perf: layerwise offload for the LTX-2.5 Gemma-4 text encoder (#43683)

This commit is contained in:
WenhaoZhang
2026-10-12 09:34:35 +08:00
committed by GitHub
parent a67228aa43
commit a954b395e6
4 changed files with 175 additions and 0 deletions
+2
View File
@@ -137,7 +137,9 @@ Selecting weights:
| 1 high-VRAM GPU | *(no extra flags)* | 960×544 fits comfortably on an H200. |
| 1 tight-VRAM GPU | `--quantization fp8` | Halves the DiT and cuts peak memory ~18 GB at unchanged speed. See [section 3.4](#3-4-fp8-quantization). |
| 1 very tight GPU | `--dit-layerwise-offload` | Cuts peak memory by roughly 10 GB, at about 4x the wall clock. |
| 1 32 GB GPU (e.g. RTX 5090) | `--layerwise-offload-components dit text_encoder` | Streams the DiT and the 22 GB Gemma-4 encoder layer by layer and keeps the rest resident. 960×544 / 121 frames / 8 steps: 15.2 s at 17 GB peak, vs 17.0 s at 31 GB with `--cpu-offload-components text_encoder ...`. Pins ~55 GB of host memory per GPU. |
| 2 GPUs, long sequences | `--num-gpus 2 --ulysses-degree 2` | Sequence parallel; the memory/long-sequence tool. |
| 2 32 GB GPUs | `--num-gpus 2 --ulysses-degree 2 --layerwise-offload-components dit text_encoder --vae-tiling false --vae-sp false --dit-layerwise-resident-layers 12` | Untiled VAE decode splits the frame height across the ranks (2.2 s to 1.1 s); 12 resident DiT layers cut the per-step host stream. RTX 5090 960×544 / 121 frames: 16.9 s at 26 GB peak per GPU, vs 21.8 s. Pins ~55 GB of host memory per GPU. |
| 2 GPUs, large DiT | `--num-gpus 2 --tp-size 2` | Tensor parallel across attention heads. |
| 2 GPUs, dev weights | `--num-gpus 2 --enable-cfg-parallel` | Splits the guided and unguided branches across GPUs. Measured 1.77x on denoising (15.1s to 8.5s, 960×544 / 57 frames / 30 steps). |
| 2 GPUs, diffusion decoder | `--num-gpus 2 --ulysses-degree 2` | The decoder's tiles are split across the ranks by default. See [section 4.6.1](#4-6-1-memory-and-multi-gpu). |
@@ -79,6 +79,9 @@ from sglang.multimodal_gen.runtime.models.encoders.base import (
finalize_encoder_folding,
get_folding_tp_group,
)
from sglang.multimodal_gen.runtime.models.encoders.gemma_4_unified import (
NATIVE_LAYERWISE_ADAPTERS,
)
from sglang.multimodal_gen.runtime.models.registry import ModelRegistry
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.server_args import ServerArgs
@@ -473,6 +476,24 @@ class TextEncoderLoader(OnlineQuantizationComponentLoader):
return transformers_model_class
return transformers.AutoModel
def load_native(
self,
component_model_path: str,
server_args: ServerArgs,
transformers_or_diffusers: str,
component_name: str | None = None,
):
model = super().load_native(
component_model_path,
server_args,
transformers_or_diffusers,
component_name,
)
# A native encoder with no SGLang port can still stream its decoder
# stack when its layout is known; see NATIVE_LAYERWISE_ADAPTERS.
adapter = NATIVE_LAYERWISE_ADAPTERS.get(type(model).__name__)
return model if adapter is None else adapter(model)
def _get_all_weights(
self,
model: EncoderTensorParallelMixin,
@@ -0,0 +1,61 @@
# SPDX-License-Identifier: Apache-2.0
"""Layerwise-offload adapter for the transformers Gemma-4-Unified encoder.
LTX-2.5's text encoder has no SGLang implementation and loads through
``transformers.AutoModel`` as ``Gemma4UnifiedModel``. Its decoder stack is a
plain ``ModuleList`` at ``language_model.layers``, which is all the layerwise
manager needs, so the loaded module's class is swapped for a subclass that also
carries the mixin. Forward, signature and weights are untouched.
"""
import torch
from torch import nn
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
)
class _Gemma4UnifiedLayerwise(LayerwiseOffloadableModuleMixin):
layerwise_offload_dit_group_enabled = False
# 48 decoder layers of ~454 MB in bf16: 21.8 of the 23.9 GB.
layer_names = ["language_model.layers"]
# The 2.0 GB vocab table is reached only through embed_tokens.__call__;
# AutoModel builds no tied lm_head.
host_resident_table_names = ["language_model.embed_tokens"]
class _HostGatherScaledEmbedding:
"""Apply Gemma's 0-dim ``embed_scale`` on whichever device ran the gather.
Upstream multiplies by the buffer in place; under host residency the gather
runs on the host while the buffer follows the module to the device.
"""
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
embeds = nn.Embedding.forward(self, input_ids)
return embeds * self.embed_scale.to(device=embeds.device, dtype=embeds.dtype)
def _swap_class(module: nn.Module, *, extra: type, first: bool) -> None:
cls = type(module)
if issubclass(cls, extra):
return
bases = (extra, cls) if first else (cls, extra)
module.__class__ = type(cls.__name__, bases, {"__module__": cls.__module__})
def make_gemma4_unified_layerwise_offloadable(model: nn.Module) -> nn.Module:
_swap_class(model, extra=_Gemma4UnifiedLayerwise, first=False)
_swap_class(
model.language_model.embed_tokens,
extra=_HostGatherScaledEmbedding,
first=True,
)
return model
# transformers class name -> adapter applied after the native load
NATIVE_LAYERWISE_ADAPTERS = {
"Gemma4UnifiedModel": make_gemma4_unified_layerwise_offloadable,
}
@@ -0,0 +1,91 @@
"""The native Gemma-4-Unified encoder (LTX-2.5) is layerwise-offloadable and exact."""
from unittest.mock import patch
import pytest
import torch
transformers = pytest.importorskip("transformers")
gemma4_unified = pytest.importorskip("transformers.models.gemma4_unified")
from sglang.multimodal_gen.runtime.managers.memory_managers import layerwise_offload
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
_host_resident_tables,
detach_host_resident_tables,
restore_host_resident_tables,
)
from sglang.multimodal_gen.runtime.models.encoders.gemma_4_unified import (
NATIVE_LAYERWISE_ADAPTERS,
)
def _tiny_model() -> torch.nn.Module:
config = gemma4_unified.Gemma4UnifiedConfig(
text_config=dict(
hidden_size=64,
intermediate_size=128,
num_hidden_layers=2,
num_attention_heads=2,
num_key_value_heads=1,
head_dim=32,
global_head_dim=32,
vocab_size=512,
layer_types=["sliding_attention", "full_attention"],
sliding_window=16,
hidden_size_per_layer_input=0,
pad_token_id=0,
),
vision_config=dict(
hidden_size=32,
intermediate_size=64,
num_hidden_layers=1,
num_attention_heads=2,
num_key_value_heads=2,
head_dim=16,
),
audio_config=None,
)
torch.manual_seed(0)
return transformers.AutoModel.from_config(config).eval()
def _encode(model: torch.nn.Module, input_ids: torch.Tensor) -> torch.Tensor:
with torch.no_grad():
out = model(
input_ids=input_ids,
attention_mask=torch.ones_like(input_ids),
output_hidden_states=True,
)
return torch.stack(out.hidden_states)
def test_adapter_keeps_outputs_and_exposes_the_decoder_stack():
model = _tiny_model()
input_ids = torch.randint(1, 500, (1, 12))
reference = _encode(model, input_ids)
adapter = NATIVE_LAYERWISE_ADAPTERS[type(model).__name__]
model = adapter(model)
assert isinstance(model, LayerwiseOffloadableModuleMixin)
assert type(model).__name__ == "Gemma4UnifiedModel"
assert isinstance(
dict(model.named_modules())[model.layer_names[0]], torch.nn.ModuleList
)
torch.testing.assert_close(_encode(model, input_ids), reference, rtol=0, atol=0)
# idempotent
assert type(adapter(model)) is type(model)
def test_host_resident_embedding_matches_resident_embedding():
model = NATIVE_LAYERWISE_ADAPTERS["Gemma4UnifiedModel"](_tiny_model())
input_ids = torch.randint(1, 500, (1, 12))
reference = _encode(model, input_ids)
threshold = f"{layerwise_offload.__name__}.HOST_RESIDENT_TABLE_MIN_BYTES"
with patch(threshold, 1024):
assert _host_resident_tables(model) == [model.language_model.embed_tokens]
restore_host_resident_tables(detach_host_resident_tables(model), "cpu")
torch.testing.assert_close(_encode(model, input_ids), reference, rtol=0, atol=0)