[Perf] diffusion: Patchify Wan latents straight into tokens (#46148)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-24 09:12:49 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 2d5ad3243f
commit 3712b5944d
3 changed files with 32 additions and 5 deletions
@@ -78,7 +78,8 @@ class PatchEmbed(nn.Module):
)
self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
def forward(self, x):
def forward(self, x, tokens: bool = False):
"""``tokens`` returns ``[B, T'*H'*W', C]`` even when ``flatten`` is off."""
if x.dim() == 5:
B, C, T, H, W = x.shape
pt, ph, pw = self.patch_size
@@ -95,7 +96,7 @@ class PatchEmbed(nn.Module):
w = self.proj.weight.reshape(self.proj.weight.shape[0], -1)
x = F.linear(x, w, self.proj.bias) # [B, T'*H'*W', embed_dim]
if not self.flatten:
if not (self.flatten or tokens):
x = x.reshape(B, T_, H_, W_, -1).permute(0, 4, 1, 2, 3).contiguous()
x = self.norm(x)
@@ -103,7 +104,7 @@ class PatchEmbed(nn.Module):
# Fallback to Conv3d for non-5D input or indivisible spatial dims.
x = self.proj(x)
if self.flatten:
if self.flatten or tokens:
x = x.flatten(2).transpose(1, 2)
x = self.norm(x)
return x
@@ -1211,8 +1211,8 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
else None
)
hidden_states = self.patch_embedding(hidden_states)
hidden_states = hidden_states.flatten(2).transpose(1, 2).contiguous()
# token-major straight from the patchify GEMM, no [B, C, T, H, W] round trip
hidden_states = self.patch_embedding(hidden_states, tokens=True).contiguous()
# shape is [B, T' * H' * W', C]
seq_len_orig = hidden_states.shape[1]
@@ -0,0 +1,26 @@
"""PatchEmbed(tokens=True) is the token-major view of its [B, C, T, H, W] output."""
import unittest
import torch
from sglang.multimodal_gen.runtime.layers.visual_embedding import PatchEmbed
class TestPatchEmbedTokens(unittest.TestCase):
def test_tokens_match_flattened_channel_first_output(self):
torch.manual_seed(0)
embed = PatchEmbed(
patch_size=(1, 2, 2), in_chans=16, embed_dim=32, flatten=False
)
# divisible (patchify GEMM) and indivisible (Conv3d fallback) grids
for shape in ((1, 16, 3, 8, 6), (2, 16, 2, 5, 6)):
with self.subTest(shape=shape), torch.no_grad():
x = torch.randn(shape)
expected = embed(x).flatten(2).transpose(1, 2)
actual = embed(x, tokens=True)
self.assertTrue(torch.equal(actual, expected))
if __name__ == "__main__":
unittest.main()