[Perf] diffusion: Drop LTX-2's all-valid prompt mask under lossless quality (#46106)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
5b8955e70f
commit
386fa9ecfd
@@ -612,6 +612,11 @@ class LTX2TextConnectors(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
) -> torch.Tensor:
|
||||
return x * math.sqrt(target_dim / source_dim)
|
||||
|
||||
@property
|
||||
def output_mask_is_all_valid(self) -> bool:
|
||||
"""Learned registers replace every padded text token, so the returned mask is all ones"""
|
||||
return self.video_connector.learnable_registers is not None
|
||||
|
||||
def forward(
|
||||
self,
|
||||
text_encoder_hidden_states: torch.Tensor,
|
||||
|
||||
+8
@@ -1,5 +1,6 @@
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import quality_allows
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
@@ -107,4 +108,11 @@ class LTX2TextConnectorStage(PipelineStage):
|
||||
batch.audio_prompt_embeds = [connector_audio_prompt_embeds]
|
||||
batch.prompt_attention_mask = connector_mask
|
||||
|
||||
if self.connectors.output_mask_is_all_valid and quality_allows(
|
||||
batch.quality, "lossless"
|
||||
):
|
||||
# an all-valid mask adds zeros to every score but pins the DiT's prompt
|
||||
# cross-attention to the slower masked SDPA kernels
|
||||
batch.prompt_attention_mask = None
|
||||
batch.negative_attention_mask = None
|
||||
return batch
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.adapter.ltx_2_connector import (
|
||||
LTX2TextConnectors,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.text_connector import (
|
||||
LTX2TextConnectorStage,
|
||||
)
|
||||
|
||||
|
||||
class _Connectors:
|
||||
def __init__(self, *, registers: bool):
|
||||
self.output_mask_is_all_valid = registers
|
||||
|
||||
def __call__(self, embeds, mask, additive_mask):
|
||||
return embeds, embeds * 2, torch.ones(embeds.shape[:2], dtype=torch.int64)
|
||||
|
||||
|
||||
def _run(*, quality: str, registers: bool, cfg: bool = True):
|
||||
stage = object.__new__(LTX2TextConnectorStage)
|
||||
stage.connectors = _Connectors(registers=registers)
|
||||
mask = torch.ones(1, 4, dtype=torch.int64)
|
||||
mask[:, 3:] = 0
|
||||
batch = SimpleNamespace(
|
||||
prompt_embeds=[torch.randn(1, 4, 8)],
|
||||
prompt_attention_mask=mask,
|
||||
negative_prompt_embeds=[torch.randn(1, 4, 8)],
|
||||
negative_attention_mask=mask,
|
||||
do_classifier_free_guidance=cfg,
|
||||
quality=quality,
|
||||
)
|
||||
return stage.forward(batch, server_args=None)
|
||||
|
||||
|
||||
class TestLTX2RegisteredPromptMask(unittest.TestCase):
|
||||
def test_lossless_drops_the_all_valid_mask_of_a_registered_connector(self):
|
||||
for cfg in (True, False):
|
||||
batch = _run(quality="lossless", registers=True, cfg=cfg)
|
||||
self.assertIsNone(batch.prompt_attention_mask)
|
||||
self.assertIsNone(batch.negative_attention_mask)
|
||||
|
||||
def test_exact_keeps_the_reference_mask(self):
|
||||
batch = _run(quality="exact", registers=True)
|
||||
all_valid = torch.ones(1, 4, dtype=torch.int64)
|
||||
self.assertTrue(torch.equal(batch.prompt_attention_mask, all_valid))
|
||||
self.assertTrue(torch.equal(batch.negative_attention_mask, all_valid))
|
||||
|
||||
def test_registers_make_the_connector_mask_all_valid(self):
|
||||
for registers, all_valid in ((torch.zeros(2, 8), True), (None, False)):
|
||||
connectors = SimpleNamespace(
|
||||
video_connector=SimpleNamespace(learnable_registers=registers)
|
||||
)
|
||||
self.assertEqual(
|
||||
LTX2TextConnectors.output_mask_is_all_valid.fget(connectors), all_valid
|
||||
)
|
||||
|
||||
def test_connector_without_registers_keeps_its_mask(self):
|
||||
batch = _run(quality="lossless", registers=False)
|
||||
self.assertIsNotNone(batch.prompt_attention_mask)
|
||||
self.assertIsNotNone(batch.negative_attention_mask)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user