[diffusion] perf: layerwise offload for the LTX-2.5 Gemma-4 text encoder (#43683)
This commit is contained in:
@@ -79,6 +79,9 @@ from sglang.multimodal_gen.runtime.models.encoders.base import (
|
||||
finalize_encoder_folding,
|
||||
get_folding_tp_group,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.gemma_4_unified import (
|
||||
NATIVE_LAYERWISE_ADAPTERS,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.registry import ModelRegistry
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
@@ -473,6 +476,24 @@ class TextEncoderLoader(OnlineQuantizationComponentLoader):
|
||||
return transformers_model_class
|
||||
return transformers.AutoModel
|
||||
|
||||
def load_native(
|
||||
self,
|
||||
component_model_path: str,
|
||||
server_args: ServerArgs,
|
||||
transformers_or_diffusers: str,
|
||||
component_name: str | None = None,
|
||||
):
|
||||
model = super().load_native(
|
||||
component_model_path,
|
||||
server_args,
|
||||
transformers_or_diffusers,
|
||||
component_name,
|
||||
)
|
||||
# A native encoder with no SGLang port can still stream its decoder
|
||||
# stack when its layout is known; see NATIVE_LAYERWISE_ADAPTERS.
|
||||
adapter = NATIVE_LAYERWISE_ADAPTERS.get(type(model).__name__)
|
||||
return model if adapter is None else adapter(model)
|
||||
|
||||
def _get_all_weights(
|
||||
self,
|
||||
model: EncoderTensorParallelMixin,
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Layerwise-offload adapter for the transformers Gemma-4-Unified encoder.
|
||||
|
||||
LTX-2.5's text encoder has no SGLang implementation and loads through
|
||||
``transformers.AutoModel`` as ``Gemma4UnifiedModel``. Its decoder stack is a
|
||||
plain ``ModuleList`` at ``language_model.layers``, which is all the layerwise
|
||||
manager needs, so the loaded module's class is swapped for a subclass that also
|
||||
carries the mixin. Forward, signature and weights are untouched.
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
|
||||
|
||||
class _Gemma4UnifiedLayerwise(LayerwiseOffloadableModuleMixin):
|
||||
layerwise_offload_dit_group_enabled = False
|
||||
# 48 decoder layers of ~454 MB in bf16: 21.8 of the 23.9 GB.
|
||||
layer_names = ["language_model.layers"]
|
||||
# The 2.0 GB vocab table is reached only through embed_tokens.__call__;
|
||||
# AutoModel builds no tied lm_head.
|
||||
host_resident_table_names = ["language_model.embed_tokens"]
|
||||
|
||||
|
||||
class _HostGatherScaledEmbedding:
|
||||
"""Apply Gemma's 0-dim ``embed_scale`` on whichever device ran the gather.
|
||||
|
||||
Upstream multiplies by the buffer in place; under host residency the gather
|
||||
runs on the host while the buffer follows the module to the device.
|
||||
"""
|
||||
|
||||
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
embeds = nn.Embedding.forward(self, input_ids)
|
||||
return embeds * self.embed_scale.to(device=embeds.device, dtype=embeds.dtype)
|
||||
|
||||
|
||||
def _swap_class(module: nn.Module, *, extra: type, first: bool) -> None:
|
||||
cls = type(module)
|
||||
if issubclass(cls, extra):
|
||||
return
|
||||
bases = (extra, cls) if first else (cls, extra)
|
||||
module.__class__ = type(cls.__name__, bases, {"__module__": cls.__module__})
|
||||
|
||||
|
||||
def make_gemma4_unified_layerwise_offloadable(model: nn.Module) -> nn.Module:
|
||||
_swap_class(model, extra=_Gemma4UnifiedLayerwise, first=False)
|
||||
_swap_class(
|
||||
model.language_model.embed_tokens,
|
||||
extra=_HostGatherScaledEmbedding,
|
||||
first=True,
|
||||
)
|
||||
return model
|
||||
|
||||
|
||||
# transformers class name -> adapter applied after the native load
|
||||
NATIVE_LAYERWISE_ADAPTERS = {
|
||||
"Gemma4UnifiedModel": make_gemma4_unified_layerwise_offloadable,
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
"""The native Gemma-4-Unified encoder (LTX-2.5) is layerwise-offloadable and exact."""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
transformers = pytest.importorskip("transformers")
|
||||
gemma4_unified = pytest.importorskip("transformers.models.gemma4_unified")
|
||||
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers import layerwise_offload
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
_host_resident_tables,
|
||||
detach_host_resident_tables,
|
||||
restore_host_resident_tables,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.gemma_4_unified import (
|
||||
NATIVE_LAYERWISE_ADAPTERS,
|
||||
)
|
||||
|
||||
|
||||
def _tiny_model() -> torch.nn.Module:
|
||||
config = gemma4_unified.Gemma4UnifiedConfig(
|
||||
text_config=dict(
|
||||
hidden_size=64,
|
||||
intermediate_size=128,
|
||||
num_hidden_layers=2,
|
||||
num_attention_heads=2,
|
||||
num_key_value_heads=1,
|
||||
head_dim=32,
|
||||
global_head_dim=32,
|
||||
vocab_size=512,
|
||||
layer_types=["sliding_attention", "full_attention"],
|
||||
sliding_window=16,
|
||||
hidden_size_per_layer_input=0,
|
||||
pad_token_id=0,
|
||||
),
|
||||
vision_config=dict(
|
||||
hidden_size=32,
|
||||
intermediate_size=64,
|
||||
num_hidden_layers=1,
|
||||
num_attention_heads=2,
|
||||
num_key_value_heads=2,
|
||||
head_dim=16,
|
||||
),
|
||||
audio_config=None,
|
||||
)
|
||||
torch.manual_seed(0)
|
||||
return transformers.AutoModel.from_config(config).eval()
|
||||
|
||||
|
||||
def _encode(model: torch.nn.Module, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
with torch.no_grad():
|
||||
out = model(
|
||||
input_ids=input_ids,
|
||||
attention_mask=torch.ones_like(input_ids),
|
||||
output_hidden_states=True,
|
||||
)
|
||||
return torch.stack(out.hidden_states)
|
||||
|
||||
|
||||
def test_adapter_keeps_outputs_and_exposes_the_decoder_stack():
|
||||
model = _tiny_model()
|
||||
input_ids = torch.randint(1, 500, (1, 12))
|
||||
reference = _encode(model, input_ids)
|
||||
|
||||
adapter = NATIVE_LAYERWISE_ADAPTERS[type(model).__name__]
|
||||
model = adapter(model)
|
||||
|
||||
assert isinstance(model, LayerwiseOffloadableModuleMixin)
|
||||
assert type(model).__name__ == "Gemma4UnifiedModel"
|
||||
assert isinstance(
|
||||
dict(model.named_modules())[model.layer_names[0]], torch.nn.ModuleList
|
||||
)
|
||||
torch.testing.assert_close(_encode(model, input_ids), reference, rtol=0, atol=0)
|
||||
# idempotent
|
||||
assert type(adapter(model)) is type(model)
|
||||
|
||||
|
||||
def test_host_resident_embedding_matches_resident_embedding():
|
||||
model = NATIVE_LAYERWISE_ADAPTERS["Gemma4UnifiedModel"](_tiny_model())
|
||||
input_ids = torch.randint(1, 500, (1, 12))
|
||||
reference = _encode(model, input_ids)
|
||||
|
||||
threshold = f"{layerwise_offload.__name__}.HOST_RESIDENT_TABLE_MIN_BYTES"
|
||||
with patch(threshold, 1024):
|
||||
assert _host_resident_tables(model) == [model.language_model.embed_tokens]
|
||||
restore_host_resident_tables(detach_host_resident_tables(model), "cpu")
|
||||
|
||||
torch.testing.assert_close(_encode(model, input_ids), reference, rtol=0, atol=0)
|
||||
Reference in New Issue
Block a user