[Perf] diffusion: Patchify Wan latents straight into tokens (#46148)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
2d5ad3243f
commit
3712b5944d
@@ -78,7 +78,8 @@ class PatchEmbed(nn.Module):
|
||||
)
|
||||
self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
|
||||
|
||||
def forward(self, x):
|
||||
def forward(self, x, tokens: bool = False):
|
||||
"""``tokens`` returns ``[B, T'*H'*W', C]`` even when ``flatten`` is off."""
|
||||
if x.dim() == 5:
|
||||
B, C, T, H, W = x.shape
|
||||
pt, ph, pw = self.patch_size
|
||||
@@ -95,7 +96,7 @@ class PatchEmbed(nn.Module):
|
||||
w = self.proj.weight.reshape(self.proj.weight.shape[0], -1)
|
||||
x = F.linear(x, w, self.proj.bias) # [B, T'*H'*W', embed_dim]
|
||||
|
||||
if not self.flatten:
|
||||
if not (self.flatten or tokens):
|
||||
x = x.reshape(B, T_, H_, W_, -1).permute(0, 4, 1, 2, 3).contiguous()
|
||||
|
||||
x = self.norm(x)
|
||||
@@ -103,7 +104,7 @@ class PatchEmbed(nn.Module):
|
||||
|
||||
# Fallback to Conv3d for non-5D input or indivisible spatial dims.
|
||||
x = self.proj(x)
|
||||
if self.flatten:
|
||||
if self.flatten or tokens:
|
||||
x = x.flatten(2).transpose(1, 2)
|
||||
x = self.norm(x)
|
||||
return x
|
||||
|
||||
@@ -1211,8 +1211,8 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
else None
|
||||
)
|
||||
|
||||
hidden_states = self.patch_embedding(hidden_states)
|
||||
hidden_states = hidden_states.flatten(2).transpose(1, 2).contiguous()
|
||||
# token-major straight from the patchify GEMM, no [B, C, T, H, W] round trip
|
||||
hidden_states = self.patch_embedding(hidden_states, tokens=True).contiguous()
|
||||
|
||||
# shape is [B, T' * H' * W', C]
|
||||
seq_len_orig = hidden_states.shape[1]
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
"""PatchEmbed(tokens=True) is the token-major view of its [B, C, T, H, W] output."""
|
||||
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.visual_embedding import PatchEmbed
|
||||
|
||||
|
||||
class TestPatchEmbedTokens(unittest.TestCase):
|
||||
def test_tokens_match_flattened_channel_first_output(self):
|
||||
torch.manual_seed(0)
|
||||
embed = PatchEmbed(
|
||||
patch_size=(1, 2, 2), in_chans=16, embed_dim=32, flatten=False
|
||||
)
|
||||
# divisible (patchify GEMM) and indivisible (Conv3d fallback) grids
|
||||
for shape in ((1, 16, 3, 8, 6), (2, 16, 2, 5, 6)):
|
||||
with self.subTest(shape=shape), torch.no_grad():
|
||||
x = torch.randn(shape)
|
||||
expected = embed(x).flatten(2).transpose(1, 2)
|
||||
actual = embed(x, tokens=True)
|
||||
self.assertTrue(torch.equal(actual, expected))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user