[Diffusion] Preserve per-sample rollout trajectories across multi-output merge (#34416)
Co-authored-by: aimicahchen <aimicahchen@tencent.com>
This commit is contained in:
@@ -37,6 +37,9 @@ from sglang.multimodal_gen.runtime.entrypoints.utils import (
|
||||
from sglang.multimodal_gen.runtime.pipelines_core import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
|
||||
from sglang.multimodal_gen.runtime.platforms.plugins import apply_plugin_hooks
|
||||
from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import (
|
||||
select_output_rollout_trajectory,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.scheduler_client import sync_scheduler_client
|
||||
from sglang.multimodal_gen.runtime.server_args import PortArgs, ServerArgs
|
||||
from sglang.multimodal_gen.runtime.server_warmup import (
|
||||
@@ -566,7 +569,9 @@ class DiffGenerator:
|
||||
action=output_batch.action_pred,
|
||||
trajectory_latents=output_batch.trajectory_latents,
|
||||
trajectory_timesteps=output_batch.trajectory_timesteps,
|
||||
rollout_trajectory_data=output_batch.rollout_trajectory_data,
|
||||
rollout_trajectory_data=select_output_rollout_trajectory(
|
||||
output_batch.rollout_trajectory_data, output_index
|
||||
),
|
||||
trajectory_decoded=output_batch.trajectory_decoded,
|
||||
)
|
||||
|
||||
|
||||
@@ -85,6 +85,10 @@ from sglang.multimodal_gen.runtime.platforms import (
|
||||
from sglang.multimodal_gen.runtime.post_training.gpu_worker_post_training_mixin import (
|
||||
GPUWorkerPostTrainingMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import (
|
||||
RolloutTrajectoryData,
|
||||
concat_rollout_trajectory_data,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.realtime.session import RealtimeSessionCache
|
||||
from sglang.multimodal_gen.runtime.realtime.video import (
|
||||
RAW_RGB_CONTENT_TYPE,
|
||||
@@ -141,6 +145,9 @@ class _ExpandedOutputParts:
|
||||
output_file_paths: list[str] = field(default_factory=list)
|
||||
metrics_list: list[Any] = field(default_factory=list)
|
||||
trajectory_decoded_parts: list[list[torch.Tensor]] | None = None
|
||||
rollout_trajectory_data: list[RolloutTrajectoryData | None] = field(
|
||||
default_factory=list
|
||||
)
|
||||
|
||||
|
||||
def _worker_cpu_intra_op_threads(num_gpus: int) -> int | None:
|
||||
@@ -1335,11 +1342,6 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
and output_batch.trajectory_timesteps is not None
|
||||
):
|
||||
merged.trajectory_timesteps = output_batch.trajectory_timesteps
|
||||
if (
|
||||
merged.rollout_trajectory_data is None
|
||||
and output_batch.rollout_trajectory_data is not None
|
||||
):
|
||||
merged.rollout_trajectory_data = output_batch.rollout_trajectory_data
|
||||
|
||||
@staticmethod
|
||||
def _collect_expanded_parts(
|
||||
@@ -1359,6 +1361,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
parts.trajectory_latents.append(output_batch.trajectory_latents)
|
||||
if isinstance(output_batch.noise_pred, torch.Tensor):
|
||||
parts.noise_preds.append(output_batch.noise_pred)
|
||||
parts.rollout_trajectory_data.append(output_batch.rollout_trajectory_data)
|
||||
if output_batch.trajectory_decoded:
|
||||
GPUWorker._collect_trajectory_decoded(
|
||||
parts, output_batch.trajectory_decoded
|
||||
@@ -1408,6 +1411,10 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
torch.cat(decoded_step, dim=0)
|
||||
for decoded_step in parts.trajectory_decoded_parts
|
||||
]
|
||||
if any(data is not None for data in parts.rollout_trajectory_data):
|
||||
merged.rollout_trajectory_data = concat_rollout_trajectory_data(
|
||||
parts.rollout_trajectory_data
|
||||
)
|
||||
|
||||
def get_can_stay_resident_components(
|
||||
self, remaining_gpu_mem_gb: float
|
||||
|
||||
@@ -62,3 +62,111 @@ class RolloutTrajectoryData:
|
||||
rollout_debug_tensors: RolloutDebugTensors | None = None
|
||||
denoising_env: RolloutDenoisingEnv | None = None
|
||||
dit_trajectory: RolloutDitTrajectory | None = None
|
||||
|
||||
|
||||
def _cat_per_output(tensors: list[torch.Tensor | None]) -> torch.Tensor | None:
|
||||
"""Concatenate complete per-output tensor fields along the batch dimension."""
|
||||
if not tensors or any(tensor is None for tensor in tensors):
|
||||
return None
|
||||
return torch.cat(tensors, dim=0)
|
||||
|
||||
|
||||
def concat_rollout_trajectory_data(
|
||||
per_output: list[RolloutTrajectoryData | None],
|
||||
) -> RolloutTrajectoryData | None:
|
||||
"""Combine aligned per-output trajectories into one batched trajectory."""
|
||||
if any(data is None for data in per_output):
|
||||
return None
|
||||
|
||||
first = per_output[0]
|
||||
if len(per_output) == 1:
|
||||
return first
|
||||
|
||||
debug_tensors = None
|
||||
if all(data.rollout_debug_tensors is not None for data in per_output):
|
||||
debug = [data.rollout_debug_tensors for data in per_output]
|
||||
debug_tensors = RolloutDebugTensors(
|
||||
rollout_variance_noises=_cat_per_output(
|
||||
[entry.rollout_variance_noises for entry in debug]
|
||||
),
|
||||
rollout_prev_sample_means=_cat_per_output(
|
||||
[entry.rollout_prev_sample_means for entry in debug]
|
||||
),
|
||||
rollout_noise_std_devs=_cat_per_output(
|
||||
[entry.rollout_noise_std_devs for entry in debug]
|
||||
),
|
||||
rollout_model_outputs=_cat_per_output(
|
||||
[entry.rollout_model_outputs for entry in debug]
|
||||
),
|
||||
)
|
||||
|
||||
dit_trajectory = None
|
||||
if all(data.dit_trajectory is not None for data in per_output):
|
||||
dit_trajectory = RolloutDitTrajectory(
|
||||
latents=_cat_per_output(
|
||||
[data.dit_trajectory.latents for data in per_output]
|
||||
),
|
||||
timesteps=first.dit_trajectory.timesteps,
|
||||
sigmas=first.dit_trajectory.sigmas,
|
||||
)
|
||||
|
||||
return RolloutTrajectoryData(
|
||||
rollout_log_probs=_cat_per_output(
|
||||
[data.rollout_log_probs for data in per_output]
|
||||
),
|
||||
rollout_debug_tensors=debug_tensors,
|
||||
denoising_env=first.denoising_env,
|
||||
dit_trajectory=dit_trajectory,
|
||||
)
|
||||
|
||||
|
||||
def _slice_output_dim(
|
||||
tensor: torch.Tensor | None, output_index: int
|
||||
) -> torch.Tensor | None:
|
||||
"""Keep one output row while preserving the batch dimension."""
|
||||
if tensor is None or tensor.dim() < 1 or output_index >= tensor.shape[0]:
|
||||
return tensor
|
||||
return tensor[output_index : output_index + 1]
|
||||
|
||||
|
||||
def select_output_rollout_trajectory(
|
||||
rollout_trajectory_data: RolloutTrajectoryData | None,
|
||||
output_index: int | None,
|
||||
) -> RolloutTrajectoryData | None:
|
||||
"""Select one output row from a batched rollout trajectory."""
|
||||
if rollout_trajectory_data is None or output_index is None:
|
||||
return rollout_trajectory_data
|
||||
|
||||
debug_tensors = rollout_trajectory_data.rollout_debug_tensors
|
||||
if debug_tensors is not None:
|
||||
debug_tensors = RolloutDebugTensors(
|
||||
rollout_variance_noises=_slice_output_dim(
|
||||
debug_tensors.rollout_variance_noises, output_index
|
||||
),
|
||||
rollout_prev_sample_means=_slice_output_dim(
|
||||
debug_tensors.rollout_prev_sample_means, output_index
|
||||
),
|
||||
rollout_noise_std_devs=_slice_output_dim(
|
||||
debug_tensors.rollout_noise_std_devs, output_index
|
||||
),
|
||||
rollout_model_outputs=_slice_output_dim(
|
||||
debug_tensors.rollout_model_outputs, output_index
|
||||
),
|
||||
)
|
||||
|
||||
dit_trajectory = rollout_trajectory_data.dit_trajectory
|
||||
if dit_trajectory is not None:
|
||||
dit_trajectory = RolloutDitTrajectory(
|
||||
latents=_slice_output_dim(dit_trajectory.latents, output_index),
|
||||
timesteps=dit_trajectory.timesteps,
|
||||
sigmas=dit_trajectory.sigmas,
|
||||
)
|
||||
|
||||
return RolloutTrajectoryData(
|
||||
rollout_log_probs=_slice_output_dim(
|
||||
rollout_trajectory_data.rollout_log_probs, output_index
|
||||
),
|
||||
rollout_debug_tensors=debug_tensors,
|
||||
denoising_env=rollout_trajectory_data.denoising_env,
|
||||
dit_trajectory=dit_trajectory,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import (
|
||||
RolloutDitTrajectory,
|
||||
RolloutTrajectoryData,
|
||||
concat_rollout_trajectory_data,
|
||||
select_output_rollout_trajectory,
|
||||
)
|
||||
|
||||
_STEPS = 4
|
||||
|
||||
|
||||
def _per_output_trajectory(output_index: int) -> RolloutTrajectoryData:
|
||||
"""One per-output forward's trajectory: batch dim 1, values tagged by index."""
|
||||
tag = float(output_index + 1)
|
||||
return RolloutTrajectoryData(
|
||||
rollout_log_probs=torch.full((1, _STEPS), tag),
|
||||
dit_trajectory=RolloutDitTrajectory(
|
||||
latents=torch.full((1, _STEPS + 1, 2), tag),
|
||||
timesteps=torch.arange(_STEPS).float(),
|
||||
sigmas=torch.linspace(1.0, 0.0, _STEPS + 1),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class TestMultiOutputRolloutTrajectory(unittest.TestCase):
|
||||
"""Guard per-sample rollout trajectories across the multi-output merge."""
|
||||
|
||||
def test_merge_concatenates_per_output_trajectories(self):
|
||||
merged = concat_rollout_trajectory_data(
|
||||
[_per_output_trajectory(i) for i in range(3)]
|
||||
)
|
||||
|
||||
self.assertEqual(tuple(merged.rollout_log_probs.shape), (3, _STEPS))
|
||||
self.assertEqual(
|
||||
[row[0].item() for row in merged.rollout_log_probs], [1.0, 2.0, 3.0]
|
||||
)
|
||||
self.assertEqual(tuple(merged.dit_trajectory.latents.shape), (3, _STEPS + 1, 2))
|
||||
|
||||
def test_merge_keeps_group_shared_schedule_from_first_output(self):
|
||||
merged = concat_rollout_trajectory_data(
|
||||
[_per_output_trajectory(i) for i in range(3)]
|
||||
)
|
||||
|
||||
# timesteps/sigmas describe the shared schedule, so they must not gain a
|
||||
# batch dim when K outputs are merged.
|
||||
self.assertEqual(tuple(merged.dit_trajectory.timesteps.shape), (_STEPS,))
|
||||
self.assertEqual(tuple(merged.dit_trajectory.sigmas.shape), (_STEPS + 1,))
|
||||
|
||||
def test_merge_drops_trajectory_when_only_some_outputs_have_one(self):
|
||||
"""Drop a partial trajectory group that cannot stay output-aligned."""
|
||||
partial = [_per_output_trajectory(0), None, _per_output_trajectory(2)]
|
||||
|
||||
self.assertIsNone(concat_rollout_trajectory_data(partial))
|
||||
|
||||
def test_single_output_group_is_passed_through(self):
|
||||
only = _per_output_trajectory(0)
|
||||
|
||||
merged = concat_rollout_trajectory_data([only])
|
||||
|
||||
self.assertIs(merged, only)
|
||||
|
||||
def test_each_output_index_selects_its_own_row(self):
|
||||
merged = concat_rollout_trajectory_data(
|
||||
[_per_output_trajectory(i) for i in range(3)]
|
||||
)
|
||||
|
||||
selected = [
|
||||
select_output_rollout_trajectory(merged, index) for index in range(3)
|
||||
]
|
||||
|
||||
values = [entry.rollout_log_probs.flatten()[0].item() for entry in selected]
|
||||
self.assertEqual(values, [1.0, 2.0, 3.0])
|
||||
for entry in selected:
|
||||
# Keep-dim so a grouped result matches an unexpanded one's shape.
|
||||
self.assertEqual(tuple(entry.rollout_log_probs.shape), (1, _STEPS))
|
||||
self.assertEqual(
|
||||
tuple(entry.dit_trajectory.latents.shape), (1, _STEPS + 1, 2)
|
||||
)
|
||||
|
||||
def test_select_without_output_index_returns_group_trajectory(self):
|
||||
merged = concat_rollout_trajectory_data(
|
||||
[_per_output_trajectory(i) for i in range(3)]
|
||||
)
|
||||
|
||||
self.assertIs(select_output_rollout_trajectory(merged, None), merged)
|
||||
|
||||
def test_select_tolerates_missing_trajectory(self):
|
||||
self.assertIsNone(select_output_rollout_trajectory(None, 0))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user