From 0931a72eb2265b136b9d664448c4a79b34d988bd Mon Sep 17 00:00:00 2001 From: WenhaoZhang <42087078+niehen6174@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:23:56 +0800 Subject: [PATCH] [diffusion] feat: support MiniMax H3 RL (#34365) --- .../configs/sample/minimax_h3.py | 13 +- .../minimax_h3/denoise_loop.py | 53 ++++- .../minimax_h3/minimax_h3_rollout.py | 225 ++++++++++++++++++ .../minimax_h3/stages/denoising.py | 43 ++++ 4 files changed, 319 insertions(+), 15 deletions(-) create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/minimax_h3_rollout.py diff --git a/python/sglang/multimodal_gen/configs/sample/minimax_h3.py b/python/sglang/multimodal_gen/configs/sample/minimax_h3.py index a6c158c738..855c04002e 100644 --- a/python/sglang/multimodal_gen/configs/sample/minimax_h3.py +++ b/python/sglang/multimodal_gen/configs/sample/minimax_h3.py @@ -262,14 +262,15 @@ class MiniMaxH3SamplingParams(SamplingParams): self.spectrum_params = SpectrumParams() apply_h3_spectrum_param_defaults(self.spectrum_params) if self.rollout: - raise ValueError( - "MiniMax H3 does not support rollout: its coupled video/audio " - "scheduler has no SchedulerRLMixin contract" - ) + task = str(self.task or "t2va").lower() + if task not in ("t2va",): + raise ValueError( + f"MiniMax H3 rollout currently supports task=t2va only, got {task!r}" + ) if self.return_trajectory_latents or self.return_trajectory_decoded: raise ValueError( - "MiniMax H3 does not support trajectory output for its coupled " - "video/audio denoise state" + "MiniMax H3 does not support return_trajectory_latents/decoded; " + "use rollout=True with rollout_return_dit_trajectory instead" ) seeds = self.seed if isinstance(self.seed, list) else [self.seed] for seed in seeds: diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/denoise_loop.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/denoise_loop.py index 77f389b08f..62599da199 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/denoise_loop.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/denoise_loop.py @@ -462,6 +462,7 @@ def minimax_h3_denoise_loop( attn_metadata: AttentionMetadata | None = None, on_step: Callable[[int, torch.Tensor, torch.Tensor], None] | None = None, step_profiler: Callable[[int], AbstractContextManager] | None = None, + rollout_ctx=None, ) -> tuple[torch.Tensor, torch.Tensor]: """Run the full denoise loop; returns final (video_rows, audio_rows). @@ -557,11 +558,22 @@ def minimax_h3_denoise_loop( audio_one_minus_sigma_ratios = 1.0 - audio_sigma_ratios video_denoised_scratch = torch.empty_like(video_rows[video_target_slice]) audio_denoised_scratch = torch.empty_like(audio_rows[audio_target_slice]) + if rollout_ctx is not None: + from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.minimax_h3_rollout import ( + minimax_h3_rollout_update_video_target, + ) + + rollout_ctx.batch._h3_rollout_sigma_max = float(max(sigmas_video)) + video_target = video_rows[video_target_slice] + rollout_ctx.collector.record_initial(video_target.detach().clone()) for step in range(num_steps): step_cm = step_profiler(step) if step_profiler is not None else nullcontext() with step_cm: s_v = sigmas_video[step] s_a = sigmas_audio[step] + s_n_v = sigmas_video[step + 1] + if rollout_ctx is not None: + rollout_ctx.batch._rollout_loop_step_index = step fk = positive.forward_kwargs( video_rows=video_rows, @@ -592,15 +604,38 @@ def minimax_h3_denoise_loop( mv_audio_t = v_audio[audio_target_slice].float() video_target = video_rows[video_target_slice] - _minimax_h3_update_target_rows_( - video_target, - mv_video_t, - sigma_t=video_sigma_t[step], - sigma_curr=s_v, - sigma_ratio=video_sigma_ratios[step], - one_minus_sigma_ratio=video_one_minus_sigma_ratios[step], - denoised_scratch=video_denoised_scratch, - ) + if rollout_ctx is not None: + ( + updated, + log_sum, + log_count, + rollout_ctx.noise_buffer, + ) = minimax_h3_rollout_update_video_target( + video_target, + mv_video_t, + sigma_curr=s_v, + sigma_next=s_n_v, + batch=rollout_ctx.batch, + generator=rollout_ctx.generator, + loop_step_index=step, + noise_buffer=rollout_ctx.noise_buffer, + ) + video_target.copy_(updated) + rollout_ctx.collector.record_step( + video_target.detach().clone(), + log_sum, + log_count, + ) + else: + _minimax_h3_update_target_rows_( + video_target, + mv_video_t, + sigma_t=video_sigma_t[step], + sigma_curr=s_v, + sigma_ratio=video_sigma_ratios[step], + one_minus_sigma_ratio=video_one_minus_sigma_ratios[step], + denoised_scratch=video_denoised_scratch, + ) audio_target = audio_rows[audio_target_slice] _minimax_h3_update_target_rows_( diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/minimax_h3_rollout.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/minimax_h3_rollout.py new file mode 100644 index 0000000000..e2cd6877b6 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/minimax_h3_rollout.py @@ -0,0 +1,225 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Rollout (RL) helpers for MiniMax H3 video-only denoising.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import Any + +import torch + +from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import ( + RolloutDenoisingEnv, + RolloutDitTrajectory, + RolloutTrajectoryData, +) + +_LOG_SQRT_2PI = math.log(math.sqrt(2 * math.pi)) + + +def _effective_sde_type(batch, loop_step_index: int) -> str: + sde_type = str(getattr(batch, "rollout_sde_type", "sde")) + if sde_type == "ode": + return "ode" + sde_step_indices = getattr(batch, "rollout_sde_step_indices", None) + if sde_step_indices is not None and loop_step_index not in sde_step_indices: + return "ode" + return sde_type + + +def minimax_h3_rollout_update_video_target( + video_target: torch.Tensor, + velocity: torch.Tensor, + *, + sigma_curr: float, + sigma_next: float, + batch, + generator: torch.Generator, + loop_step_index: int, + noise_buffer: torch.Tensor | None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None]: + """Stochastic or deterministic update for video target rows during rollout. + + H3 velocity ``v`` relates to flow-matching model output as ``model_output = -v``. + Returns ``(updated_target, log_prob_sum[B=1], log_prob_count[B=1], noise_buffer)``. + """ + sde_type = _effective_sde_type(batch, loop_step_index) + noise_level = float(getattr(batch, "rollout_noise_level", 0.0)) + log_prob_no_const = bool(getattr(batch, "rollout_log_prob_no_const", False)) + + if sde_type == "ode" or noise_level == 0.0: + from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.denoise_loop import ( + _minimax_h3_update_target_rows_, + ) + + out = video_target.clone() + sigma_t = out.new_tensor(sigma_curr) + ratio = out.new_tensor(sigma_next / sigma_curr if sigma_curr != 0.0 else 1.0) + one_minus_ratio = 1.0 - ratio + scratch = torch.empty_like(out) + _minimax_h3_update_target_rows_( + out, + velocity.float(), + sigma_t=sigma_t, + sigma_curr=sigma_curr, + sigma_ratio=ratio, + one_minus_sigma_ratio=one_minus_ratio, + denoised_scratch=scratch, + ) + elem_count = float(out.numel()) + log_prob_sum = torch.zeros(1, device=out.device, dtype=torch.float32) + log_prob_count = out.new_tensor([elem_count]) + return out, log_prob_sum, log_prob_count, noise_buffer + + model_output = (-velocity).float() + sample = video_target.float() + current_sigma = sample.new_tensor(float(sigma_curr)) + next_sigma = sample.new_tensor(float(sigma_next)) + + if ( + noise_buffer is None + or noise_buffer.shape != sample.shape + or noise_buffer.device != sample.device + ): + noise_buffer = torch.empty_like(sample) + variance_noise = torch.randn( + sample.shape, + generator=generator, + device=sample.device, + dtype=torch.float32, + ) + noise_buffer.copy_(variance_noise) + + if sde_type == "cps": + std_dev_t = next_sigma * math.sin(noise_level * math.pi / 2) + noise_std_dev = std_dev_t + pred_original = sample - current_sigma * model_output + noise_estimate = sample + model_output * (1.0 - current_sigma) + prev_mean = pred_original * (1.0 - next_sigma) + noise_estimate * torch.sqrt( + torch.clamp(next_sigma**2 - std_dev_t**2, min=1e-12) + ) + prev_sample = prev_mean + variance_noise * noise_std_dev + log_prob_no_const_val = -((variance_noise * noise_std_dev) ** 2) + elif sde_type == "sde": + dt = next_sigma - current_sigma + sigma_max = float(getattr(batch, "_h3_rollout_sigma_max", 1.0)) + std_dev_t = ( + torch.sqrt( + current_sigma + / ( + 1.0 + - torch.where( + torch.isclose(current_sigma, current_sigma.new_tensor(1.0)), + current_sigma.new_tensor(sigma_max), + current_sigma, + ) + ) + ) + * noise_level + ) + noise_std_dev = std_dev_t * torch.sqrt(-1.0 * dt) + prev_mean = ( + sample * (1.0 + std_dev_t**2 / (2.0 * current_sigma) * dt) + + model_output + * (1.0 + std_dev_t**2 * (1.0 - current_sigma) / (2.0 * current_sigma)) + * dt + ) + prev_sample = prev_mean + variance_noise * noise_std_dev + log_prob_no_const_val = -((variance_noise * noise_std_dev) ** 2) + else: + raise ValueError(f"Unsupported rollout_sde_type for H3: {sde_type!r}") + + if log_prob_no_const or sde_type == "ode": + log_prob_sum = log_prob_no_const_val.sum().unsqueeze(0) + else: + log_prob_sum = ( + ( + log_prob_no_const_val / (2.0 * (noise_std_dev**2)) + - torch.log(noise_std_dev) + - _LOG_SQRT_2PI + ) + .sum() + .unsqueeze(0) + ) + log_prob_count = sample.new_tensor([float(sample.numel())]) + return ( + prev_sample.to(dtype=video_target.dtype), + log_prob_sum, + log_prob_count, + noise_buffer, + ) + + +@dataclass +class MiniMaxH3RolloutCtx: + """Per-denoise-loop rollout state for H3.""" + + batch: Any + generator: torch.Generator + sigmas_video: list[float] + collector: MiniMaxH3RolloutCollector + noise_buffer: torch.Tensor | None = None + denoising_env_kwargs: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class MiniMaxH3RolloutCollector: + """Accumulates video-target trajectory and per-step log probs.""" + + sigmas_video: list[float] + latent_steps: list[torch.Tensor] = field(default_factory=list) + log_prob_sums: list[torch.Tensor] = field(default_factory=list) + log_prob_counts: list[torch.Tensor] = field(default_factory=list) + pos_cond_kwargs: dict[str, Any] = field(default_factory=dict) + + def record_initial(self, video_target: torch.Tensor) -> None: + self.latent_steps.append(video_target.detach().cpu().clone()) + + def record_step( + self, + video_target: torch.Tensor, + log_prob_sum: torch.Tensor, + log_prob_count: torch.Tensor, + ) -> None: + self.latent_steps.append(video_target.detach().cpu().clone()) + self.log_prob_sums.append(log_prob_sum.detach().cpu()) + self.log_prob_counts.append(log_prob_count.detach().cpu()) + + def build_trajectory_data(self) -> RolloutTrajectoryData: + # latents: [B=1, T+1, num_video_target_rows, width] + stacked = torch.stack(self.latent_steps, dim=0).unsqueeze(0) + divisor = 1000.0 + step_sigmas = torch.tensor( + [float(s) for s in self.sigmas_video[:-1]], + dtype=torch.float32, + ) + timesteps = step_sigmas * divisor + sigmas = torch.tensor( + [float(s) for s in self.sigmas_video], + dtype=torch.float32, + ) + log_probs = None + if self.log_prob_sums: + sums = torch.stack(self.log_prob_sums, dim=0) + counts = torch.stack(self.log_prob_counts, dim=0) + per_step = sums.squeeze(-1) / counts.squeeze(-1).clamp(min=1.0) + log_probs = per_step.unsqueeze(0) + return RolloutTrajectoryData( + rollout_log_probs=log_probs, + denoising_env=RolloutDenoisingEnv( + pos_cond_kwargs=self.pos_cond_kwargs, + ), + dit_trajectory=RolloutDitTrajectory( + latents=stacked, + timesteps=timesteps, + sigmas=sigmas, + ), + ) + + +__all__ = [ + "MiniMaxH3RolloutCollector", + "MiniMaxH3RolloutCtx", + "minimax_h3_rollout_update_video_target", +] diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py index 1b7b65c846..a8b6c0628c 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py @@ -783,6 +783,44 @@ class MiniMaxH3DenoisingStage(DenoisingStage): device=device, ) initial_video, initial_audio = _expand_initial_rows(ctx, positive) + rollout_ctx = None + if getattr(batch, "rollout", False): + from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.minimax_h3_rollout import ( + MiniMaxH3RolloutCollector, + MiniMaxH3RolloutCtx, + ) + from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import ( + RolloutTrajectoryData, + ) + + task = str(getattr(batch.sampling_params, "task", "") or "t2va").lower() + if task not in ("t2va",): + raise ValueError( + f"MiniMax H3 rollout currently supports task=t2va only, got {task!r}" + ) + generator = torch.Generator(device=device) + seed = getattr(batch.sampling_params, "seed", 0) + if isinstance(seed, list): + seed = seed[0] + generator.manual_seed(int(seed)) + collector = MiniMaxH3RolloutCollector(sigmas_video=sigmas_video) + packed_cpu = { + k: (v.detach().cpu() if isinstance(v, torch.Tensor) else v) + for k, v in packed.items() + } + collector.pos_cond_kwargs = { + "encoder_hidden_states": emb["hidden_states"].detach().cpu(), + "h3_packed_layout": packed_cpu, + "h3_token_tags": tags.detach().cpu(), + "h3_video_target_start": positive.video_target_start, + } + rollout_ctx = MiniMaxH3RolloutCtx( + batch=batch, + generator=generator, + sigmas_video=sigmas_video, + collector=collector, + ) + batch.rollout_trajectory_data = RolloutTrajectoryData() with ( maybe_nvtx_range("denoising_loop", self.current_use_nvtx), self.progress_bar( @@ -821,7 +859,12 @@ class MiniMaxH3DenoisingStage(DenoisingStage): self._profile_denoising_step, batch=batch, ), + rollout_ctx=rollout_ctx, ) + if rollout_ctx is not None: + batch.rollout_trajectory_data = ( + rollout_ctx.collector.build_trajectory_data() + ) finally: self._finish_active_component_use() _publish_full_loop_outputs(