From a0ba1968b161f1d2989aaacfdbd88a7e522d59a1 Mon Sep 17 00:00:00 2001 From: Jianghai <72591262+CjhHa1@users.noreply.github.com> Date: Sun, 27 Sep 2026 18:32:20 +0800 Subject: [PATCH] [Diffusion] Apply latent-ids and packing to caller-provided initial latents (#34418) Co-authored-by: aimicahchen Co-authored-by: WenhaoZhang <42087078+niehen6174@users.noreply.github.com> --- .../stages/latent_preparation.py | 16 ++- .../unit/test_provided_latents_preparation.py | 113 ++++++++++++++++++ 2 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 python/sglang/multimodal_gen/test/unit/test_provided_latents_preparation.py diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py index ce07e10f88..6664bf01a5 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/latent_preparation.py @@ -141,18 +141,24 @@ class LatentPreparationStage(PipelineStage): f" size of {batch_size}. Make sure the batch size matches the length of the generators." ) - # Generate or use provided latents + # Apply the same preparation to generated and unpacked provided latents. + spec = self.get_latent_preparation_spec( + batch, server_args, batch_size, latent_num_frames, device + ) if latents is None: - spec = self.get_latent_preparation_spec( - batch, server_args, batch_size, latent_num_frames, device - ) latents = randn_tensor( spec.shape, generator=generator, device=spec.device, dtype=spec.dtype, ) + needs_preparation = True + else: + latents = latents.to(device) + # ComfyUI may provide already-packed [B, S, D] latents. + needs_preparation = tuple(latents.shape) == tuple(spec.shape) + if needs_preparation: latent_ids = ( server_args.pipeline_config.maybe_prepare_latent_ids(latents) if spec.prepare_latent_ids @@ -166,8 +172,6 @@ class LatentPreparationStage(PipelineStage): latents = server_args.pipeline_config.maybe_pack_latents( latents, batch_size, batch ) - else: - latents = latents.to(device) # Scale the initial noise if needed if self.should_scale_initial_noise(batch, server_args) and hasattr( diff --git a/python/sglang/multimodal_gen/test/unit/test_provided_latents_preparation.py b/python/sglang/multimodal_gen/test/unit/test_provided_latents_preparation.py new file mode 100644 index 0000000000..24a91e3177 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_provided_latents_preparation.py @@ -0,0 +1,113 @@ +import unittest +from types import SimpleNamespace + +import torch + +from sglang.multimodal_gen.runtime.pipelines_core.stages.latent_preparation import ( + LatentPreparationStage, +) + + +class _PackingPipelineConfig: + """Minimal stand-in for a packed model (FLUX-style). + + ``maybe_prepare_latent_ids`` / ``maybe_pack_latents`` both expect unpacked + ``(B, C, H, W)`` latents, matching the real configs. ``maybe_pack_latents`` + reshapes off the spec dims rather than validating the input layout, which is + what makes packing an already-packed tensor silently wrong instead of loud -- + the real ``_pack_latents`` does a ``view`` for the same reason. + """ + + channels = 4 + height = 8 + width = 8 + + def __init__(self): + self.pack_calls = 0 + + def prepare_latent_shape(self, batch, batch_size, num_frames): + return (batch_size, self.channels, self.height, self.width) + + def get_latent_dtype(self, dtype): + return torch.float32 + + def maybe_prepare_latent_ids(self, latents): + batch = latents.shape[0] + return torch.zeros(batch, self.height * self.width, 4) + + def maybe_pack_latents(self, latents, batch_size, batch): + self.pack_calls += 1 + return latents.reshape( + batch_size, self.channels, self.height * self.width + ).permute(0, 2, 1) + + +def _make_stage(latents): + stage = object.__new__(LatentPreparationStage) + # No init_noise_sigma -> the scale step is skipped, keeping the assertions on + # shape/latent_ids rather than values. + stage.scheduler = SimpleNamespace() + batch = SimpleNamespace( + batch_size=1, + generator=None, + latents=latents, + height=16, + width=16, + num_frames=1, + latent_ids=None, + raw_latent_shape=None, + num_outputs_per_prompt=1, + prompt_embeds=[torch.zeros(1, 1, dtype=torch.float32)], + ) + pipeline_config = _PackingPipelineConfig() + server_args = SimpleNamespace(pipeline_config=pipeline_config) + return stage, batch, server_args, pipeline_config + + +class TestProvidedLatentsPreparation(unittest.TestCase): + """Guard packing and latent IDs for caller-supplied initial latents.""" + + def test_provided_latents_get_latent_ids_and_packing(self): + provided = torch.zeros(1, 4, 8, 8) + stage, batch, server_args, pipeline_config = _make_stage(provided) + + stage.get_forward_latent_num_frames = lambda batch, server_args: 1 + + result = stage.forward(batch, server_args) + + self.assertIsNotNone(result.latent_ids) + self.assertEqual(tuple(result.latent_ids.shape), (1, 64, 4)) + self.assertEqual(pipeline_config.pack_calls, 1) + self.assertEqual(tuple(result.latents.shape), (1, 64, 4)) + + def test_already_packed_latents_are_left_untouched(self): + """Do not pack ComfyUI-style flat latents twice.""" + packed = torch.arange(1 * 16 * 4, dtype=torch.float32).reshape(1, 16, 4) + stage, batch, server_args, pipeline_config = _make_stage(packed.clone()) + + stage.get_forward_latent_num_frames = lambda batch, server_args: 1 + + result = stage.forward(batch, server_args) + + self.assertEqual(pipeline_config.pack_calls, 0) + self.assertIsNone(result.latent_ids) + # forward moves latents onto the execution device without changing values. + self.assertTrue(torch.equal(result.latents.cpu(), packed)) + + def test_randn_and_provided_latents_agree_on_shapes(self): + """Leave generated and unpacked provided latents in the same shape.""" + stage_r, batch_r, args_r, _ = _make_stage(None) + stage_r.get_forward_latent_num_frames = lambda batch, server_args: 1 + drawn = stage_r.forward(batch_r, args_r) + + stage_p, batch_p, args_p, _ = _make_stage(torch.zeros(1, 4, 8, 8)) + stage_p.get_forward_latent_num_frames = lambda batch, server_args: 1 + injected = stage_p.forward(batch_p, args_p) + + self.assertEqual(drawn.latents.shape, injected.latents.shape) + self.assertEqual(drawn.latent_ids.shape, injected.latent_ids.shape) + self.assertEqual(drawn.raw_latent_shape, injected.raw_latent_shape) + + +if __name__ == "__main__": + unittest.main()