diff --git a/python/sglang/multimodal_gen/runtime/layers/visual_embedding.py b/python/sglang/multimodal_gen/runtime/layers/visual_embedding.py index 2f3efe73e6..172a3ff10f 100644 --- a/python/sglang/multimodal_gen/runtime/layers/visual_embedding.py +++ b/python/sglang/multimodal_gen/runtime/layers/visual_embedding.py @@ -78,7 +78,8 @@ class PatchEmbed(nn.Module): ) self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() - def forward(self, x): + def forward(self, x, tokens: bool = False): + """``tokens`` returns ``[B, T'*H'*W', C]`` even when ``flatten`` is off.""" if x.dim() == 5: B, C, T, H, W = x.shape pt, ph, pw = self.patch_size @@ -95,7 +96,7 @@ class PatchEmbed(nn.Module): w = self.proj.weight.reshape(self.proj.weight.shape[0], -1) x = F.linear(x, w, self.proj.bias) # [B, T'*H'*W', embed_dim] - if not self.flatten: + if not (self.flatten or tokens): x = x.reshape(B, T_, H_, W_, -1).permute(0, 4, 1, 2, 3).contiguous() x = self.norm(x) @@ -103,7 +104,7 @@ class PatchEmbed(nn.Module): # Fallback to Conv3d for non-5D input or indivisible spatial dims. x = self.proj(x) - if self.flatten: + if self.flatten or tokens: x = x.flatten(2).transpose(1, 2) x = self.norm(x) return x diff --git a/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py b/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py index b569305607..8a919e642e 100755 --- a/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py @@ -1211,8 +1211,8 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin): else None ) - hidden_states = self.patch_embedding(hidden_states) - hidden_states = hidden_states.flatten(2).transpose(1, 2).contiguous() + # token-major straight from the patchify GEMM, no [B, C, T, H, W] round trip + hidden_states = self.patch_embedding(hidden_states, tokens=True).contiguous() # shape is [B, T' * H' * W', C] seq_len_orig = hidden_states.shape[1] diff --git a/python/sglang/multimodal_gen/test/unit/test_patch_embed_tokens.py b/python/sglang/multimodal_gen/test/unit/test_patch_embed_tokens.py new file mode 100644 index 0000000000..0ac383fc51 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_patch_embed_tokens.py @@ -0,0 +1,26 @@ +"""PatchEmbed(tokens=True) is the token-major view of its [B, C, T, H, W] output.""" + +import unittest + +import torch + +from sglang.multimodal_gen.runtime.layers.visual_embedding import PatchEmbed + + +class TestPatchEmbedTokens(unittest.TestCase): + def test_tokens_match_flattened_channel_first_output(self): + torch.manual_seed(0) + embed = PatchEmbed( + patch_size=(1, 2, 2), in_chans=16, embed_dim=32, flatten=False + ) + # divisible (patchify GEMM) and indivisible (Conv3d fallback) grids + for shape in ((1, 16, 3, 8, 6), (2, 16, 2, 5, 6)): + with self.subTest(shape=shape), torch.no_grad(): + x = torch.randn(shape) + expected = embed(x).flatten(2).transpose(1, 2) + actual = embed(x, tokens=True) + self.assertTrue(torch.equal(actual, expected)) + + +if __name__ == "__main__": + unittest.main()