From acc15c193b9d8813cb31b66cd19980fe0de9d72d Mon Sep 17 00:00:00 2001 From: Jianghai <72591262+CjhHa1@users.noreply.github.com> Date: Sun, 27 Sep 2026 13:08:45 +0800 Subject: [PATCH] [Diffusion] Preserve per-sample rollout trajectories across multi-output merge (#34416) Co-authored-by: aimicahchen --- .../entrypoints/diffusion_generator.py | 7 +- .../runtime/managers/gpu_worker.py | 17 ++- .../runtime/post_training/rl_dataclasses.py | 108 ++++++++++++++++++ .../test_multi_output_rollout_trajectory.py | 95 +++++++++++++++ 4 files changed, 221 insertions(+), 6 deletions(-) create mode 100644 python/sglang/multimodal_gen/test/unit/test_multi_output_rollout_trajectory.py diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py b/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py index dc6efe5d0f..9fce9bb852 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py @@ -37,6 +37,9 @@ from sglang.multimodal_gen.runtime.entrypoints.utils import ( from sglang.multimodal_gen.runtime.pipelines_core import Req from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch from sglang.multimodal_gen.runtime.platforms.plugins import apply_plugin_hooks +from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import ( + select_output_rollout_trajectory, +) from sglang.multimodal_gen.runtime.scheduler_client import sync_scheduler_client from sglang.multimodal_gen.runtime.server_args import PortArgs, ServerArgs from sglang.multimodal_gen.runtime.server_warmup import ( @@ -566,7 +569,9 @@ class DiffGenerator: action=output_batch.action_pred, trajectory_latents=output_batch.trajectory_latents, trajectory_timesteps=output_batch.trajectory_timesteps, - rollout_trajectory_data=output_batch.rollout_trajectory_data, + rollout_trajectory_data=select_output_rollout_trajectory( + output_batch.rollout_trajectory_data, output_index + ), trajectory_decoded=output_batch.trajectory_decoded, ) diff --git a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py index da4a412c83..0ce719e3ab 100644 --- a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py +++ b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py @@ -85,6 +85,10 @@ from sglang.multimodal_gen.runtime.platforms import ( from sglang.multimodal_gen.runtime.post_training.gpu_worker_post_training_mixin import ( GPUWorkerPostTrainingMixin, ) +from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import ( + RolloutTrajectoryData, + concat_rollout_trajectory_data, +) from sglang.multimodal_gen.runtime.realtime.session import RealtimeSessionCache from sglang.multimodal_gen.runtime.realtime.video import ( RAW_RGB_CONTENT_TYPE, @@ -141,6 +145,9 @@ class _ExpandedOutputParts: output_file_paths: list[str] = field(default_factory=list) metrics_list: list[Any] = field(default_factory=list) trajectory_decoded_parts: list[list[torch.Tensor]] | None = None + rollout_trajectory_data: list[RolloutTrajectoryData | None] = field( + default_factory=list + ) def _worker_cpu_intra_op_threads(num_gpus: int) -> int | None: @@ -1335,11 +1342,6 @@ class GPUWorker(GPUWorkerPostTrainingMixin): and output_batch.trajectory_timesteps is not None ): merged.trajectory_timesteps = output_batch.trajectory_timesteps - if ( - merged.rollout_trajectory_data is None - and output_batch.rollout_trajectory_data is not None - ): - merged.rollout_trajectory_data = output_batch.rollout_trajectory_data @staticmethod def _collect_expanded_parts( @@ -1359,6 +1361,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin): parts.trajectory_latents.append(output_batch.trajectory_latents) if isinstance(output_batch.noise_pred, torch.Tensor): parts.noise_preds.append(output_batch.noise_pred) + parts.rollout_trajectory_data.append(output_batch.rollout_trajectory_data) if output_batch.trajectory_decoded: GPUWorker._collect_trajectory_decoded( parts, output_batch.trajectory_decoded @@ -1408,6 +1411,10 @@ class GPUWorker(GPUWorkerPostTrainingMixin): torch.cat(decoded_step, dim=0) for decoded_step in parts.trajectory_decoded_parts ] + if any(data is not None for data in parts.rollout_trajectory_data): + merged.rollout_trajectory_data = concat_rollout_trajectory_data( + parts.rollout_trajectory_data + ) def get_can_stay_resident_components( self, remaining_gpu_mem_gb: float diff --git a/python/sglang/multimodal_gen/runtime/post_training/rl_dataclasses.py b/python/sglang/multimodal_gen/runtime/post_training/rl_dataclasses.py index 1882776d57..012c69c1fc 100644 --- a/python/sglang/multimodal_gen/runtime/post_training/rl_dataclasses.py +++ b/python/sglang/multimodal_gen/runtime/post_training/rl_dataclasses.py @@ -62,3 +62,111 @@ class RolloutTrajectoryData: rollout_debug_tensors: RolloutDebugTensors | None = None denoising_env: RolloutDenoisingEnv | None = None dit_trajectory: RolloutDitTrajectory | None = None + + +def _cat_per_output(tensors: list[torch.Tensor | None]) -> torch.Tensor | None: + """Concatenate complete per-output tensor fields along the batch dimension.""" + if not tensors or any(tensor is None for tensor in tensors): + return None + return torch.cat(tensors, dim=0) + + +def concat_rollout_trajectory_data( + per_output: list[RolloutTrajectoryData | None], +) -> RolloutTrajectoryData | None: + """Combine aligned per-output trajectories into one batched trajectory.""" + if any(data is None for data in per_output): + return None + + first = per_output[0] + if len(per_output) == 1: + return first + + debug_tensors = None + if all(data.rollout_debug_tensors is not None for data in per_output): + debug = [data.rollout_debug_tensors for data in per_output] + debug_tensors = RolloutDebugTensors( + rollout_variance_noises=_cat_per_output( + [entry.rollout_variance_noises for entry in debug] + ), + rollout_prev_sample_means=_cat_per_output( + [entry.rollout_prev_sample_means for entry in debug] + ), + rollout_noise_std_devs=_cat_per_output( + [entry.rollout_noise_std_devs for entry in debug] + ), + rollout_model_outputs=_cat_per_output( + [entry.rollout_model_outputs for entry in debug] + ), + ) + + dit_trajectory = None + if all(data.dit_trajectory is not None for data in per_output): + dit_trajectory = RolloutDitTrajectory( + latents=_cat_per_output( + [data.dit_trajectory.latents for data in per_output] + ), + timesteps=first.dit_trajectory.timesteps, + sigmas=first.dit_trajectory.sigmas, + ) + + return RolloutTrajectoryData( + rollout_log_probs=_cat_per_output( + [data.rollout_log_probs for data in per_output] + ), + rollout_debug_tensors=debug_tensors, + denoising_env=first.denoising_env, + dit_trajectory=dit_trajectory, + ) + + +def _slice_output_dim( + tensor: torch.Tensor | None, output_index: int +) -> torch.Tensor | None: + """Keep one output row while preserving the batch dimension.""" + if tensor is None or tensor.dim() < 1 or output_index >= tensor.shape[0]: + return tensor + return tensor[output_index : output_index + 1] + + +def select_output_rollout_trajectory( + rollout_trajectory_data: RolloutTrajectoryData | None, + output_index: int | None, +) -> RolloutTrajectoryData | None: + """Select one output row from a batched rollout trajectory.""" + if rollout_trajectory_data is None or output_index is None: + return rollout_trajectory_data + + debug_tensors = rollout_trajectory_data.rollout_debug_tensors + if debug_tensors is not None: + debug_tensors = RolloutDebugTensors( + rollout_variance_noises=_slice_output_dim( + debug_tensors.rollout_variance_noises, output_index + ), + rollout_prev_sample_means=_slice_output_dim( + debug_tensors.rollout_prev_sample_means, output_index + ), + rollout_noise_std_devs=_slice_output_dim( + debug_tensors.rollout_noise_std_devs, output_index + ), + rollout_model_outputs=_slice_output_dim( + debug_tensors.rollout_model_outputs, output_index + ), + ) + + dit_trajectory = rollout_trajectory_data.dit_trajectory + if dit_trajectory is not None: + dit_trajectory = RolloutDitTrajectory( + latents=_slice_output_dim(dit_trajectory.latents, output_index), + timesteps=dit_trajectory.timesteps, + sigmas=dit_trajectory.sigmas, + ) + + return RolloutTrajectoryData( + rollout_log_probs=_slice_output_dim( + rollout_trajectory_data.rollout_log_probs, output_index + ), + rollout_debug_tensors=debug_tensors, + denoising_env=rollout_trajectory_data.denoising_env, + dit_trajectory=dit_trajectory, + ) diff --git a/python/sglang/multimodal_gen/test/unit/test_multi_output_rollout_trajectory.py b/python/sglang/multimodal_gen/test/unit/test_multi_output_rollout_trajectory.py new file mode 100644 index 0000000000..c6b49cd937 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_multi_output_rollout_trajectory.py @@ -0,0 +1,95 @@ +import unittest + +import torch + +from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import ( + RolloutDitTrajectory, + RolloutTrajectoryData, + concat_rollout_trajectory_data, + select_output_rollout_trajectory, +) + +_STEPS = 4 + + +def _per_output_trajectory(output_index: int) -> RolloutTrajectoryData: + """One per-output forward's trajectory: batch dim 1, values tagged by index.""" + tag = float(output_index + 1) + return RolloutTrajectoryData( + rollout_log_probs=torch.full((1, _STEPS), tag), + dit_trajectory=RolloutDitTrajectory( + latents=torch.full((1, _STEPS + 1, 2), tag), + timesteps=torch.arange(_STEPS).float(), + sigmas=torch.linspace(1.0, 0.0, _STEPS + 1), + ), + ) + + +class TestMultiOutputRolloutTrajectory(unittest.TestCase): + """Guard per-sample rollout trajectories across the multi-output merge.""" + + def test_merge_concatenates_per_output_trajectories(self): + merged = concat_rollout_trajectory_data( + [_per_output_trajectory(i) for i in range(3)] + ) + + self.assertEqual(tuple(merged.rollout_log_probs.shape), (3, _STEPS)) + self.assertEqual( + [row[0].item() for row in merged.rollout_log_probs], [1.0, 2.0, 3.0] + ) + self.assertEqual(tuple(merged.dit_trajectory.latents.shape), (3, _STEPS + 1, 2)) + + def test_merge_keeps_group_shared_schedule_from_first_output(self): + merged = concat_rollout_trajectory_data( + [_per_output_trajectory(i) for i in range(3)] + ) + + # timesteps/sigmas describe the shared schedule, so they must not gain a + # batch dim when K outputs are merged. + self.assertEqual(tuple(merged.dit_trajectory.timesteps.shape), (_STEPS,)) + self.assertEqual(tuple(merged.dit_trajectory.sigmas.shape), (_STEPS + 1,)) + + def test_merge_drops_trajectory_when_only_some_outputs_have_one(self): + """Drop a partial trajectory group that cannot stay output-aligned.""" + partial = [_per_output_trajectory(0), None, _per_output_trajectory(2)] + + self.assertIsNone(concat_rollout_trajectory_data(partial)) + + def test_single_output_group_is_passed_through(self): + only = _per_output_trajectory(0) + + merged = concat_rollout_trajectory_data([only]) + + self.assertIs(merged, only) + + def test_each_output_index_selects_its_own_row(self): + merged = concat_rollout_trajectory_data( + [_per_output_trajectory(i) for i in range(3)] + ) + + selected = [ + select_output_rollout_trajectory(merged, index) for index in range(3) + ] + + values = [entry.rollout_log_probs.flatten()[0].item() for entry in selected] + self.assertEqual(values, [1.0, 2.0, 3.0]) + for entry in selected: + # Keep-dim so a grouped result matches an unexpanded one's shape. + self.assertEqual(tuple(entry.rollout_log_probs.shape), (1, _STEPS)) + self.assertEqual( + tuple(entry.dit_trajectory.latents.shape), (1, _STEPS + 1, 2) + ) + + def test_select_without_output_index_returns_group_trajectory(self): + merged = concat_rollout_trajectory_data( + [_per_output_trajectory(i) for i in range(3)] + ) + + self.assertIs(select_output_rollout_trajectory(merged, None), merged) + + def test_select_tolerates_missing_trajectory(self): + self.assertIsNone(select_output_rollout_trajectory(None, 0)) + + +if __name__ == "__main__": + unittest.main()