From 386fa9ecfdfeaeb131b119f448ab148b196b18d5 Mon Sep 17 00:00:00 2001 From: Mick Date: Sat, 24 Oct 2026 09:19:12 +0800 Subject: [PATCH] [Perf] diffusion: Drop LTX-2's all-valid prompt mask under lossless quality (#46106) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5.5 --- .../runtime/models/adapter/ltx_2_connector.py | 5 ++ .../ltx_2/text_connector.py | 8 +++ .../unit/test_ltx2_prompt_mask_registers.py | 67 +++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 python/sglang/multimodal_gen/test/unit/test_ltx2_prompt_mask_registers.py diff --git a/python/sglang/multimodal_gen/runtime/models/adapter/ltx_2_connector.py b/python/sglang/multimodal_gen/runtime/models/adapter/ltx_2_connector.py index f33c3ab063..9370dcba65 100644 --- a/python/sglang/multimodal_gen/runtime/models/adapter/ltx_2_connector.py +++ b/python/sglang/multimodal_gen/runtime/models/adapter/ltx_2_connector.py @@ -612,6 +612,11 @@ class LTX2TextConnectors(nn.Module, LayerwiseOffloadableModuleMixin): ) -> torch.Tensor: return x * math.sqrt(target_dim / source_dim) + @property + def output_mask_is_all_valid(self) -> bool: + """Learned registers replace every padded text token, so the returned mask is all ones""" + return self.video_connector.learnable_registers is not None + def forward( self, text_encoder_hidden_states: torch.Tensor, diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/text_connector.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/text_connector.py index 8babe18569..e30ed195dd 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/text_connector.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/text_connector.py @@ -1,5 +1,6 @@ import torch +from sglang.multimodal_gen.configs.sample.sampling_params import quality_allows from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( ComponentUse, @@ -107,4 +108,11 @@ class LTX2TextConnectorStage(PipelineStage): batch.audio_prompt_embeds = [connector_audio_prompt_embeds] batch.prompt_attention_mask = connector_mask + if self.connectors.output_mask_is_all_valid and quality_allows( + batch.quality, "lossless" + ): + # an all-valid mask adds zeros to every score but pins the DiT's prompt + # cross-attention to the slower masked SDPA kernels + batch.prompt_attention_mask = None + batch.negative_attention_mask = None return batch diff --git a/python/sglang/multimodal_gen/test/unit/test_ltx2_prompt_mask_registers.py b/python/sglang/multimodal_gen/test/unit/test_ltx2_prompt_mask_registers.py new file mode 100644 index 0000000000..6163a1ed4f --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_ltx2_prompt_mask_registers.py @@ -0,0 +1,67 @@ +import unittest +from types import SimpleNamespace + +import torch + +from sglang.multimodal_gen.runtime.models.adapter.ltx_2_connector import ( + LTX2TextConnectors, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.text_connector import ( + LTX2TextConnectorStage, +) + + +class _Connectors: + def __init__(self, *, registers: bool): + self.output_mask_is_all_valid = registers + + def __call__(self, embeds, mask, additive_mask): + return embeds, embeds * 2, torch.ones(embeds.shape[:2], dtype=torch.int64) + + +def _run(*, quality: str, registers: bool, cfg: bool = True): + stage = object.__new__(LTX2TextConnectorStage) + stage.connectors = _Connectors(registers=registers) + mask = torch.ones(1, 4, dtype=torch.int64) + mask[:, 3:] = 0 + batch = SimpleNamespace( + prompt_embeds=[torch.randn(1, 4, 8)], + prompt_attention_mask=mask, + negative_prompt_embeds=[torch.randn(1, 4, 8)], + negative_attention_mask=mask, + do_classifier_free_guidance=cfg, + quality=quality, + ) + return stage.forward(batch, server_args=None) + + +class TestLTX2RegisteredPromptMask(unittest.TestCase): + def test_lossless_drops_the_all_valid_mask_of_a_registered_connector(self): + for cfg in (True, False): + batch = _run(quality="lossless", registers=True, cfg=cfg) + self.assertIsNone(batch.prompt_attention_mask) + self.assertIsNone(batch.negative_attention_mask) + + def test_exact_keeps_the_reference_mask(self): + batch = _run(quality="exact", registers=True) + all_valid = torch.ones(1, 4, dtype=torch.int64) + self.assertTrue(torch.equal(batch.prompt_attention_mask, all_valid)) + self.assertTrue(torch.equal(batch.negative_attention_mask, all_valid)) + + def test_registers_make_the_connector_mask_all_valid(self): + for registers, all_valid in ((torch.zeros(2, 8), True), (None, False)): + connectors = SimpleNamespace( + video_connector=SimpleNamespace(learnable_registers=registers) + ) + self.assertEqual( + LTX2TextConnectors.output_mask_is_all_valid.fget(connectors), all_valid + ) + + def test_connector_without_registers_keeps_its_mask(self): + batch = _run(quality="lossless", registers=False) + self.assertIsNotNone(batch.prompt_attention_mask) + self.assertIsNotNone(batch.negative_attention_mask) + + +if __name__ == "__main__": + unittest.main()