[diffusion] perf: replace padded rows without a host sync for Z-Image (#44069)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-14 11:15:28 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 7e226b1b71
commit 53fe21cdfd
2 changed files with 64 additions and 16 deletions
@@ -1451,23 +1451,24 @@ class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
valid_lens: list[int] | torch.Tensor,
pad_token: torch.Tensor,
) -> torch.Tensor:
"""Replace padded token rows after each valid sequence length."""
if not torch.is_tensor(valid_lens) and all(
int(length) >= tensor.shape[1] for length in valid_lens
):
"""Replace padded token rows after each valid sequence length.
Selects rather than mask-assigns: a boolean-mask write syncs with the host.
"""
seq_len = tensor.shape[1]
pad_value = pad_token.to(device=tensor.device, dtype=tensor.dtype)
if not torch.is_tensor(valid_lens):
if all(int(length) >= seq_len for length in valid_lens):
return tensor
tensor = tensor.clone()
for row, length in enumerate(valid_lens):
tensor[row, int(length) :] = pad_value
return tensor
positions = torch.arange(tensor.shape[1], device=tensor.device).unsqueeze(0)
if torch.is_tensor(valid_lens):
lengths = valid_lens.to(device=tensor.device, dtype=torch.long)
else:
lengths = torch.tensor(valid_lens, device=tensor.device)
if lengths.ndim == 0:
lengths = lengths.reshape(1)
lengths = lengths.unsqueeze(1)
pad_mask = positions >= lengths
tensor = tensor.clone()
tensor[pad_mask] = pad_token.to(device=tensor.device, dtype=tensor.dtype)
return tensor
lengths = valid_lens.to(device=tensor.device, dtype=torch.long).reshape(-1, 1)
positions = torch.arange(seq_len, device=tensor.device).unsqueeze(0)
return torch.where(
(positions >= lengths).unsqueeze(-1), pad_value.view(1, 1, -1), tensor
)
@staticmethod
def _replace_padding_with_token_mask(
@@ -0,0 +1,47 @@
import unittest
import torch
from sglang.multimodal_gen.runtime.models.dits.zimage import ZImageTransformer2DModel
_replace = ZImageTransformer2DModel._replace_padding_with_token
def _mask_assign_reference(tensor, valid_lens, pad_token):
"""The previous implementation: a boolean-mask write."""
positions = torch.arange(tensor.shape[1], device=tensor.device).unsqueeze(0)
lengths = torch.as_tensor(valid_lens, device=tensor.device).reshape(-1, 1)
out = tensor.clone()
out[positions >= lengths] = pad_token.to(tensor.dtype)
return out
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA")
class TestZImagePaddingWithoutSync(unittest.TestCase):
def setUp(self):
torch.manual_seed(0)
self.tensor = torch.randn(3, 40, 64, device="cuda", dtype=torch.bfloat16)
self.pad = torch.randn(64, device="cuda")
def _check(self, valid_lens, as_tensor):
lens = torch.tensor(valid_lens, device="cuda") if as_tensor else valid_lens
expected = _mask_assign_reference(self.tensor, valid_lens, self.pad)
torch.cuda.set_sync_debug_mode("error")
try:
out = _replace(self.tensor, lens, self.pad)
finally:
torch.cuda.set_sync_debug_mode("default")
self.assertTrue(torch.equal(out, expected))
def test_list_lengths(self):
self._check([40, 17, 0], as_tensor=False)
def test_tensor_lengths(self):
self._check([40, 17, 0], as_tensor=True)
def test_no_padding_returns_input(self):
self.assertIs(_replace(self.tensor, [40, 40, 40], self.pad), self.tensor)
if __name__ == "__main__":
unittest.main()