[Diffusion] Apply latent-ids and packing to caller-provided initial latents (#34418)
Co-authored-by: aimicahchen <aimicahchen@tencent.com> Co-authored-by: WenhaoZhang <42087078+niehen6174@users.noreply.github.com>
This commit is contained in:
co-authored by
aimicahchen
WenhaoZhang
parent
f1e62e3a2e
commit
a0ba1968b1
@@ -141,18 +141,24 @@ class LatentPreparationStage(PipelineStage):
|
||||
f" size of {batch_size}. Make sure the batch size matches the length of the generators."
|
||||
)
|
||||
|
||||
# Generate or use provided latents
|
||||
# Apply the same preparation to generated and unpacked provided latents.
|
||||
spec = self.get_latent_preparation_spec(
|
||||
batch, server_args, batch_size, latent_num_frames, device
|
||||
)
|
||||
if latents is None:
|
||||
spec = self.get_latent_preparation_spec(
|
||||
batch, server_args, batch_size, latent_num_frames, device
|
||||
)
|
||||
latents = randn_tensor(
|
||||
spec.shape,
|
||||
generator=generator,
|
||||
device=spec.device,
|
||||
dtype=spec.dtype,
|
||||
)
|
||||
needs_preparation = True
|
||||
else:
|
||||
latents = latents.to(device)
|
||||
# ComfyUI may provide already-packed [B, S, D] latents.
|
||||
needs_preparation = tuple(latents.shape) == tuple(spec.shape)
|
||||
|
||||
if needs_preparation:
|
||||
latent_ids = (
|
||||
server_args.pipeline_config.maybe_prepare_latent_ids(latents)
|
||||
if spec.prepare_latent_ids
|
||||
@@ -166,8 +172,6 @@ class LatentPreparationStage(PipelineStage):
|
||||
latents = server_args.pipeline_config.maybe_pack_latents(
|
||||
latents, batch_size, batch
|
||||
)
|
||||
else:
|
||||
latents = latents.to(device)
|
||||
|
||||
# Scale the initial noise if needed
|
||||
if self.should_scale_initial_noise(batch, server_args) and hasattr(
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.latent_preparation import (
|
||||
LatentPreparationStage,
|
||||
)
|
||||
|
||||
|
||||
class _PackingPipelineConfig:
|
||||
"""Minimal stand-in for a packed model (FLUX-style).
|
||||
|
||||
``maybe_prepare_latent_ids`` / ``maybe_pack_latents`` both expect unpacked
|
||||
``(B, C, H, W)`` latents, matching the real configs. ``maybe_pack_latents``
|
||||
reshapes off the spec dims rather than validating the input layout, which is
|
||||
what makes packing an already-packed tensor silently wrong instead of loud --
|
||||
the real ``_pack_latents`` does a ``view`` for the same reason.
|
||||
"""
|
||||
|
||||
channels = 4
|
||||
height = 8
|
||||
width = 8
|
||||
|
||||
def __init__(self):
|
||||
self.pack_calls = 0
|
||||
|
||||
def prepare_latent_shape(self, batch, batch_size, num_frames):
|
||||
return (batch_size, self.channels, self.height, self.width)
|
||||
|
||||
def get_latent_dtype(self, dtype):
|
||||
return torch.float32
|
||||
|
||||
def maybe_prepare_latent_ids(self, latents):
|
||||
batch = latents.shape[0]
|
||||
return torch.zeros(batch, self.height * self.width, 4)
|
||||
|
||||
def maybe_pack_latents(self, latents, batch_size, batch):
|
||||
self.pack_calls += 1
|
||||
return latents.reshape(
|
||||
batch_size, self.channels, self.height * self.width
|
||||
).permute(0, 2, 1)
|
||||
|
||||
|
||||
def _make_stage(latents):
|
||||
stage = object.__new__(LatentPreparationStage)
|
||||
# No init_noise_sigma -> the scale step is skipped, keeping the assertions on
|
||||
# shape/latent_ids rather than values.
|
||||
stage.scheduler = SimpleNamespace()
|
||||
batch = SimpleNamespace(
|
||||
batch_size=1,
|
||||
generator=None,
|
||||
latents=latents,
|
||||
height=16,
|
||||
width=16,
|
||||
num_frames=1,
|
||||
latent_ids=None,
|
||||
raw_latent_shape=None,
|
||||
num_outputs_per_prompt=1,
|
||||
prompt_embeds=[torch.zeros(1, 1, dtype=torch.float32)],
|
||||
)
|
||||
pipeline_config = _PackingPipelineConfig()
|
||||
server_args = SimpleNamespace(pipeline_config=pipeline_config)
|
||||
return stage, batch, server_args, pipeline_config
|
||||
|
||||
|
||||
class TestProvidedLatentsPreparation(unittest.TestCase):
|
||||
"""Guard packing and latent IDs for caller-supplied initial latents."""
|
||||
|
||||
def test_provided_latents_get_latent_ids_and_packing(self):
|
||||
provided = torch.zeros(1, 4, 8, 8)
|
||||
stage, batch, server_args, pipeline_config = _make_stage(provided)
|
||||
|
||||
stage.get_forward_latent_num_frames = lambda batch, server_args: 1
|
||||
|
||||
result = stage.forward(batch, server_args)
|
||||
|
||||
self.assertIsNotNone(result.latent_ids)
|
||||
self.assertEqual(tuple(result.latent_ids.shape), (1, 64, 4))
|
||||
self.assertEqual(pipeline_config.pack_calls, 1)
|
||||
self.assertEqual(tuple(result.latents.shape), (1, 64, 4))
|
||||
|
||||
def test_already_packed_latents_are_left_untouched(self):
|
||||
"""Do not pack ComfyUI-style flat latents twice."""
|
||||
packed = torch.arange(1 * 16 * 4, dtype=torch.float32).reshape(1, 16, 4)
|
||||
stage, batch, server_args, pipeline_config = _make_stage(packed.clone())
|
||||
|
||||
stage.get_forward_latent_num_frames = lambda batch, server_args: 1
|
||||
|
||||
result = stage.forward(batch, server_args)
|
||||
|
||||
self.assertEqual(pipeline_config.pack_calls, 0)
|
||||
self.assertIsNone(result.latent_ids)
|
||||
# forward moves latents onto the execution device without changing values.
|
||||
self.assertTrue(torch.equal(result.latents.cpu(), packed))
|
||||
|
||||
def test_randn_and_provided_latents_agree_on_shapes(self):
|
||||
"""Leave generated and unpacked provided latents in the same shape."""
|
||||
stage_r, batch_r, args_r, _ = _make_stage(None)
|
||||
stage_r.get_forward_latent_num_frames = lambda batch, server_args: 1
|
||||
drawn = stage_r.forward(batch_r, args_r)
|
||||
|
||||
stage_p, batch_p, args_p, _ = _make_stage(torch.zeros(1, 4, 8, 8))
|
||||
stage_p.get_forward_latent_num_frames = lambda batch, server_args: 1
|
||||
injected = stage_p.forward(batch_p, args_p)
|
||||
|
||||
self.assertEqual(drawn.latents.shape, injected.latents.shape)
|
||||
self.assertEqual(drawn.latent_ids.shape, injected.latent_ids.shape)
|
||||
self.assertEqual(drawn.raw_latent_shape, injected.raw_latent_shape)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user