[diffusion] perf: replace padded rows without a host sync for Z-Image (#44069)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
7e226b1b71
commit
53fe21cdfd
@@ -1451,23 +1451,24 @@ class ZImageTransformer2DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
valid_lens: list[int] | torch.Tensor,
|
||||
pad_token: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Replace padded token rows after each valid sequence length."""
|
||||
if not torch.is_tensor(valid_lens) and all(
|
||||
int(length) >= tensor.shape[1] for length in valid_lens
|
||||
):
|
||||
"""Replace padded token rows after each valid sequence length.
|
||||
|
||||
Selects rather than mask-assigns: a boolean-mask write syncs with the host.
|
||||
"""
|
||||
seq_len = tensor.shape[1]
|
||||
pad_value = pad_token.to(device=tensor.device, dtype=tensor.dtype)
|
||||
if not torch.is_tensor(valid_lens):
|
||||
if all(int(length) >= seq_len for length in valid_lens):
|
||||
return tensor
|
||||
tensor = tensor.clone()
|
||||
for row, length in enumerate(valid_lens):
|
||||
tensor[row, int(length) :] = pad_value
|
||||
return tensor
|
||||
positions = torch.arange(tensor.shape[1], device=tensor.device).unsqueeze(0)
|
||||
if torch.is_tensor(valid_lens):
|
||||
lengths = valid_lens.to(device=tensor.device, dtype=torch.long)
|
||||
else:
|
||||
lengths = torch.tensor(valid_lens, device=tensor.device)
|
||||
if lengths.ndim == 0:
|
||||
lengths = lengths.reshape(1)
|
||||
lengths = lengths.unsqueeze(1)
|
||||
pad_mask = positions >= lengths
|
||||
tensor = tensor.clone()
|
||||
tensor[pad_mask] = pad_token.to(device=tensor.device, dtype=tensor.dtype)
|
||||
return tensor
|
||||
lengths = valid_lens.to(device=tensor.device, dtype=torch.long).reshape(-1, 1)
|
||||
positions = torch.arange(seq_len, device=tensor.device).unsqueeze(0)
|
||||
return torch.where(
|
||||
(positions >= lengths).unsqueeze(-1), pad_value.view(1, 1, -1), tensor
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _replace_padding_with_token_mask(
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.dits.zimage import ZImageTransformer2DModel
|
||||
|
||||
_replace = ZImageTransformer2DModel._replace_padding_with_token
|
||||
|
||||
|
||||
def _mask_assign_reference(tensor, valid_lens, pad_token):
|
||||
"""The previous implementation: a boolean-mask write."""
|
||||
positions = torch.arange(tensor.shape[1], device=tensor.device).unsqueeze(0)
|
||||
lengths = torch.as_tensor(valid_lens, device=tensor.device).reshape(-1, 1)
|
||||
out = tensor.clone()
|
||||
out[positions >= lengths] = pad_token.to(tensor.dtype)
|
||||
return out
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA")
|
||||
class TestZImagePaddingWithoutSync(unittest.TestCase):
|
||||
def setUp(self):
|
||||
torch.manual_seed(0)
|
||||
self.tensor = torch.randn(3, 40, 64, device="cuda", dtype=torch.bfloat16)
|
||||
self.pad = torch.randn(64, device="cuda")
|
||||
|
||||
def _check(self, valid_lens, as_tensor):
|
||||
lens = torch.tensor(valid_lens, device="cuda") if as_tensor else valid_lens
|
||||
expected = _mask_assign_reference(self.tensor, valid_lens, self.pad)
|
||||
torch.cuda.set_sync_debug_mode("error")
|
||||
try:
|
||||
out = _replace(self.tensor, lens, self.pad)
|
||||
finally:
|
||||
torch.cuda.set_sync_debug_mode("default")
|
||||
self.assertTrue(torch.equal(out, expected))
|
||||
|
||||
def test_list_lengths(self):
|
||||
self._check([40, 17, 0], as_tensor=False)
|
||||
|
||||
def test_tensor_lengths(self):
|
||||
self._check([40, 17, 0], as_tensor=True)
|
||||
|
||||
def test_no_padding_returns_input(self):
|
||||
self.assertIs(_replace(self.tensor, [40, 40, 40], self.pad), self.tensor)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user