From 53fe21cdfd95cec67241e41800199f97d00aebe9 Mon Sep 17 00:00:00 2001 From: Mick Date: Wed, 14 Oct 2026 11:15:28 +0800 Subject: [PATCH] [diffusion] perf: replace padded rows without a host sync for Z-Image (#44069) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5.5 --- .../runtime/models/dits/zimage.py | 33 ++++++------- .../test/unit/test_zimage_padding_no_sync.py | 47 +++++++++++++++++++ 2 files changed, 64 insertions(+), 16 deletions(-) create mode 100644 python/sglang/multimodal_gen/test/unit/test_zimage_padding_no_sync.py diff --git a/python/sglang/multimodal_gen/runtime/models/dits/zimage.py b/python/sglang/multimodal_gen/runtime/models/dits/zimage.py index 8226062724..c68cd3245f 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/zimage.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/zimage.py @@ -1451,23 +1451,24 @@ class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin): valid_lens: list[int] | torch.Tensor, pad_token: torch.Tensor, ) -> torch.Tensor: - """Replace padded token rows after each valid sequence length.""" - if not torch.is_tensor(valid_lens) and all( - int(length) >= tensor.shape[1] for length in valid_lens - ): + """Replace padded token rows after each valid sequence length. + + Selects rather than mask-assigns: a boolean-mask write syncs with the host. + """ + seq_len = tensor.shape[1] + pad_value = pad_token.to(device=tensor.device, dtype=tensor.dtype) + if not torch.is_tensor(valid_lens): + if all(int(length) >= seq_len for length in valid_lens): + return tensor + tensor = tensor.clone() + for row, length in enumerate(valid_lens): + tensor[row, int(length) :] = pad_value return tensor - positions = torch.arange(tensor.shape[1], device=tensor.device).unsqueeze(0) - if torch.is_tensor(valid_lens): - lengths = valid_lens.to(device=tensor.device, dtype=torch.long) - else: - lengths = torch.tensor(valid_lens, device=tensor.device) - if lengths.ndim == 0: - lengths = lengths.reshape(1) - lengths = lengths.unsqueeze(1) - pad_mask = positions >= lengths - tensor = tensor.clone() - tensor[pad_mask] = pad_token.to(device=tensor.device, dtype=tensor.dtype) - return tensor + lengths = valid_lens.to(device=tensor.device, dtype=torch.long).reshape(-1, 1) + positions = torch.arange(seq_len, device=tensor.device).unsqueeze(0) + return torch.where( + (positions >= lengths).unsqueeze(-1), pad_value.view(1, 1, -1), tensor + ) @staticmethod def _replace_padding_with_token_mask( diff --git a/python/sglang/multimodal_gen/test/unit/test_zimage_padding_no_sync.py b/python/sglang/multimodal_gen/test/unit/test_zimage_padding_no_sync.py new file mode 100644 index 0000000000..0c3da5f408 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_zimage_padding_no_sync.py @@ -0,0 +1,47 @@ +import unittest + +import torch + +from sglang.multimodal_gen.runtime.models.dits.zimage import ZImageTransformer2DModel + +_replace = ZImageTransformer2DModel._replace_padding_with_token + + +def _mask_assign_reference(tensor, valid_lens, pad_token): + """The previous implementation: a boolean-mask write.""" + positions = torch.arange(tensor.shape[1], device=tensor.device).unsqueeze(0) + lengths = torch.as_tensor(valid_lens, device=tensor.device).reshape(-1, 1) + out = tensor.clone() + out[positions >= lengths] = pad_token.to(tensor.dtype) + return out + + +@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") +class TestZImagePaddingWithoutSync(unittest.TestCase): + def setUp(self): + torch.manual_seed(0) + self.tensor = torch.randn(3, 40, 64, device="cuda", dtype=torch.bfloat16) + self.pad = torch.randn(64, device="cuda") + + def _check(self, valid_lens, as_tensor): + lens = torch.tensor(valid_lens, device="cuda") if as_tensor else valid_lens + expected = _mask_assign_reference(self.tensor, valid_lens, self.pad) + torch.cuda.set_sync_debug_mode("error") + try: + out = _replace(self.tensor, lens, self.pad) + finally: + torch.cuda.set_sync_debug_mode("default") + self.assertTrue(torch.equal(out, expected)) + + def test_list_lengths(self): + self._check([40, 17, 0], as_tensor=False) + + def test_tensor_lengths(self): + self._check([40, 17, 0], as_tensor=True) + + def test_no_padding_returns_input(self): + self.assertIs(_replace(self.tensor, [40, 40, 40], self.pad), self.tensor) + + +if __name__ == "__main__": + unittest.main()