[diffusion] feat: support MiniMax H3 RL (#34365)
This commit is contained in:
@@ -262,14 +262,15 @@ class MiniMaxH3SamplingParams(SamplingParams):
|
||||
self.spectrum_params = SpectrumParams()
|
||||
apply_h3_spectrum_param_defaults(self.spectrum_params)
|
||||
if self.rollout:
|
||||
raise ValueError(
|
||||
"MiniMax H3 does not support rollout: its coupled video/audio "
|
||||
"scheduler has no SchedulerRLMixin contract"
|
||||
)
|
||||
task = str(self.task or "t2va").lower()
|
||||
if task not in ("t2va",):
|
||||
raise ValueError(
|
||||
f"MiniMax H3 rollout currently supports task=t2va only, got {task!r}"
|
||||
)
|
||||
if self.return_trajectory_latents or self.return_trajectory_decoded:
|
||||
raise ValueError(
|
||||
"MiniMax H3 does not support trajectory output for its coupled "
|
||||
"video/audio denoise state"
|
||||
"MiniMax H3 does not support return_trajectory_latents/decoded; "
|
||||
"use rollout=True with rollout_return_dit_trajectory instead"
|
||||
)
|
||||
seeds = self.seed if isinstance(self.seed, list) else [self.seed]
|
||||
for seed in seeds:
|
||||
|
||||
+44
-9
@@ -462,6 +462,7 @@ def minimax_h3_denoise_loop(
|
||||
attn_metadata: AttentionMetadata | None = None,
|
||||
on_step: Callable[[int, torch.Tensor, torch.Tensor], None] | None = None,
|
||||
step_profiler: Callable[[int], AbstractContextManager] | None = None,
|
||||
rollout_ctx=None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run the full denoise loop; returns final (video_rows, audio_rows).
|
||||
|
||||
@@ -557,11 +558,22 @@ def minimax_h3_denoise_loop(
|
||||
audio_one_minus_sigma_ratios = 1.0 - audio_sigma_ratios
|
||||
video_denoised_scratch = torch.empty_like(video_rows[video_target_slice])
|
||||
audio_denoised_scratch = torch.empty_like(audio_rows[audio_target_slice])
|
||||
if rollout_ctx is not None:
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.minimax_h3_rollout import (
|
||||
minimax_h3_rollout_update_video_target,
|
||||
)
|
||||
|
||||
rollout_ctx.batch._h3_rollout_sigma_max = float(max(sigmas_video))
|
||||
video_target = video_rows[video_target_slice]
|
||||
rollout_ctx.collector.record_initial(video_target.detach().clone())
|
||||
for step in range(num_steps):
|
||||
step_cm = step_profiler(step) if step_profiler is not None else nullcontext()
|
||||
with step_cm:
|
||||
s_v = sigmas_video[step]
|
||||
s_a = sigmas_audio[step]
|
||||
s_n_v = sigmas_video[step + 1]
|
||||
if rollout_ctx is not None:
|
||||
rollout_ctx.batch._rollout_loop_step_index = step
|
||||
|
||||
fk = positive.forward_kwargs(
|
||||
video_rows=video_rows,
|
||||
@@ -592,15 +604,38 @@ def minimax_h3_denoise_loop(
|
||||
mv_audio_t = v_audio[audio_target_slice].float()
|
||||
|
||||
video_target = video_rows[video_target_slice]
|
||||
_minimax_h3_update_target_rows_(
|
||||
video_target,
|
||||
mv_video_t,
|
||||
sigma_t=video_sigma_t[step],
|
||||
sigma_curr=s_v,
|
||||
sigma_ratio=video_sigma_ratios[step],
|
||||
one_minus_sigma_ratio=video_one_minus_sigma_ratios[step],
|
||||
denoised_scratch=video_denoised_scratch,
|
||||
)
|
||||
if rollout_ctx is not None:
|
||||
(
|
||||
updated,
|
||||
log_sum,
|
||||
log_count,
|
||||
rollout_ctx.noise_buffer,
|
||||
) = minimax_h3_rollout_update_video_target(
|
||||
video_target,
|
||||
mv_video_t,
|
||||
sigma_curr=s_v,
|
||||
sigma_next=s_n_v,
|
||||
batch=rollout_ctx.batch,
|
||||
generator=rollout_ctx.generator,
|
||||
loop_step_index=step,
|
||||
noise_buffer=rollout_ctx.noise_buffer,
|
||||
)
|
||||
video_target.copy_(updated)
|
||||
rollout_ctx.collector.record_step(
|
||||
video_target.detach().clone(),
|
||||
log_sum,
|
||||
log_count,
|
||||
)
|
||||
else:
|
||||
_minimax_h3_update_target_rows_(
|
||||
video_target,
|
||||
mv_video_t,
|
||||
sigma_t=video_sigma_t[step],
|
||||
sigma_curr=s_v,
|
||||
sigma_ratio=video_sigma_ratios[step],
|
||||
one_minus_sigma_ratio=video_one_minus_sigma_ratios[step],
|
||||
denoised_scratch=video_denoised_scratch,
|
||||
)
|
||||
|
||||
audio_target = audio_rows[audio_target_slice]
|
||||
_minimax_h3_update_target_rows_(
|
||||
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Rollout (RL) helpers for MiniMax H3 video-only denoising."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import (
|
||||
RolloutDenoisingEnv,
|
||||
RolloutDitTrajectory,
|
||||
RolloutTrajectoryData,
|
||||
)
|
||||
|
||||
_LOG_SQRT_2PI = math.log(math.sqrt(2 * math.pi))
|
||||
|
||||
|
||||
def _effective_sde_type(batch, loop_step_index: int) -> str:
|
||||
sde_type = str(getattr(batch, "rollout_sde_type", "sde"))
|
||||
if sde_type == "ode":
|
||||
return "ode"
|
||||
sde_step_indices = getattr(batch, "rollout_sde_step_indices", None)
|
||||
if sde_step_indices is not None and loop_step_index not in sde_step_indices:
|
||||
return "ode"
|
||||
return sde_type
|
||||
|
||||
|
||||
def minimax_h3_rollout_update_video_target(
|
||||
video_target: torch.Tensor,
|
||||
velocity: torch.Tensor,
|
||||
*,
|
||||
sigma_curr: float,
|
||||
sigma_next: float,
|
||||
batch,
|
||||
generator: torch.Generator,
|
||||
loop_step_index: int,
|
||||
noise_buffer: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
"""Stochastic or deterministic update for video target rows during rollout.
|
||||
|
||||
H3 velocity ``v`` relates to flow-matching model output as ``model_output = -v``.
|
||||
Returns ``(updated_target, log_prob_sum[B=1], log_prob_count[B=1], noise_buffer)``.
|
||||
"""
|
||||
sde_type = _effective_sde_type(batch, loop_step_index)
|
||||
noise_level = float(getattr(batch, "rollout_noise_level", 0.0))
|
||||
log_prob_no_const = bool(getattr(batch, "rollout_log_prob_no_const", False))
|
||||
|
||||
if sde_type == "ode" or noise_level == 0.0:
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.denoise_loop import (
|
||||
_minimax_h3_update_target_rows_,
|
||||
)
|
||||
|
||||
out = video_target.clone()
|
||||
sigma_t = out.new_tensor(sigma_curr)
|
||||
ratio = out.new_tensor(sigma_next / sigma_curr if sigma_curr != 0.0 else 1.0)
|
||||
one_minus_ratio = 1.0 - ratio
|
||||
scratch = torch.empty_like(out)
|
||||
_minimax_h3_update_target_rows_(
|
||||
out,
|
||||
velocity.float(),
|
||||
sigma_t=sigma_t,
|
||||
sigma_curr=sigma_curr,
|
||||
sigma_ratio=ratio,
|
||||
one_minus_sigma_ratio=one_minus_ratio,
|
||||
denoised_scratch=scratch,
|
||||
)
|
||||
elem_count = float(out.numel())
|
||||
log_prob_sum = torch.zeros(1, device=out.device, dtype=torch.float32)
|
||||
log_prob_count = out.new_tensor([elem_count])
|
||||
return out, log_prob_sum, log_prob_count, noise_buffer
|
||||
|
||||
model_output = (-velocity).float()
|
||||
sample = video_target.float()
|
||||
current_sigma = sample.new_tensor(float(sigma_curr))
|
||||
next_sigma = sample.new_tensor(float(sigma_next))
|
||||
|
||||
if (
|
||||
noise_buffer is None
|
||||
or noise_buffer.shape != sample.shape
|
||||
or noise_buffer.device != sample.device
|
||||
):
|
||||
noise_buffer = torch.empty_like(sample)
|
||||
variance_noise = torch.randn(
|
||||
sample.shape,
|
||||
generator=generator,
|
||||
device=sample.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
noise_buffer.copy_(variance_noise)
|
||||
|
||||
if sde_type == "cps":
|
||||
std_dev_t = next_sigma * math.sin(noise_level * math.pi / 2)
|
||||
noise_std_dev = std_dev_t
|
||||
pred_original = sample - current_sigma * model_output
|
||||
noise_estimate = sample + model_output * (1.0 - current_sigma)
|
||||
prev_mean = pred_original * (1.0 - next_sigma) + noise_estimate * torch.sqrt(
|
||||
torch.clamp(next_sigma**2 - std_dev_t**2, min=1e-12)
|
||||
)
|
||||
prev_sample = prev_mean + variance_noise * noise_std_dev
|
||||
log_prob_no_const_val = -((variance_noise * noise_std_dev) ** 2)
|
||||
elif sde_type == "sde":
|
||||
dt = next_sigma - current_sigma
|
||||
sigma_max = float(getattr(batch, "_h3_rollout_sigma_max", 1.0))
|
||||
std_dev_t = (
|
||||
torch.sqrt(
|
||||
current_sigma
|
||||
/ (
|
||||
1.0
|
||||
- torch.where(
|
||||
torch.isclose(current_sigma, current_sigma.new_tensor(1.0)),
|
||||
current_sigma.new_tensor(sigma_max),
|
||||
current_sigma,
|
||||
)
|
||||
)
|
||||
)
|
||||
* noise_level
|
||||
)
|
||||
noise_std_dev = std_dev_t * torch.sqrt(-1.0 * dt)
|
||||
prev_mean = (
|
||||
sample * (1.0 + std_dev_t**2 / (2.0 * current_sigma) * dt)
|
||||
+ model_output
|
||||
* (1.0 + std_dev_t**2 * (1.0 - current_sigma) / (2.0 * current_sigma))
|
||||
* dt
|
||||
)
|
||||
prev_sample = prev_mean + variance_noise * noise_std_dev
|
||||
log_prob_no_const_val = -((variance_noise * noise_std_dev) ** 2)
|
||||
else:
|
||||
raise ValueError(f"Unsupported rollout_sde_type for H3: {sde_type!r}")
|
||||
|
||||
if log_prob_no_const or sde_type == "ode":
|
||||
log_prob_sum = log_prob_no_const_val.sum().unsqueeze(0)
|
||||
else:
|
||||
log_prob_sum = (
|
||||
(
|
||||
log_prob_no_const_val / (2.0 * (noise_std_dev**2))
|
||||
- torch.log(noise_std_dev)
|
||||
- _LOG_SQRT_2PI
|
||||
)
|
||||
.sum()
|
||||
.unsqueeze(0)
|
||||
)
|
||||
log_prob_count = sample.new_tensor([float(sample.numel())])
|
||||
return (
|
||||
prev_sample.to(dtype=video_target.dtype),
|
||||
log_prob_sum,
|
||||
log_prob_count,
|
||||
noise_buffer,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MiniMaxH3RolloutCtx:
|
||||
"""Per-denoise-loop rollout state for H3."""
|
||||
|
||||
batch: Any
|
||||
generator: torch.Generator
|
||||
sigmas_video: list[float]
|
||||
collector: MiniMaxH3RolloutCollector
|
||||
noise_buffer: torch.Tensor | None = None
|
||||
denoising_env_kwargs: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MiniMaxH3RolloutCollector:
|
||||
"""Accumulates video-target trajectory and per-step log probs."""
|
||||
|
||||
sigmas_video: list[float]
|
||||
latent_steps: list[torch.Tensor] = field(default_factory=list)
|
||||
log_prob_sums: list[torch.Tensor] = field(default_factory=list)
|
||||
log_prob_counts: list[torch.Tensor] = field(default_factory=list)
|
||||
pos_cond_kwargs: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def record_initial(self, video_target: torch.Tensor) -> None:
|
||||
self.latent_steps.append(video_target.detach().cpu().clone())
|
||||
|
||||
def record_step(
|
||||
self,
|
||||
video_target: torch.Tensor,
|
||||
log_prob_sum: torch.Tensor,
|
||||
log_prob_count: torch.Tensor,
|
||||
) -> None:
|
||||
self.latent_steps.append(video_target.detach().cpu().clone())
|
||||
self.log_prob_sums.append(log_prob_sum.detach().cpu())
|
||||
self.log_prob_counts.append(log_prob_count.detach().cpu())
|
||||
|
||||
def build_trajectory_data(self) -> RolloutTrajectoryData:
|
||||
# latents: [B=1, T+1, num_video_target_rows, width]
|
||||
stacked = torch.stack(self.latent_steps, dim=0).unsqueeze(0)
|
||||
divisor = 1000.0
|
||||
step_sigmas = torch.tensor(
|
||||
[float(s) for s in self.sigmas_video[:-1]],
|
||||
dtype=torch.float32,
|
||||
)
|
||||
timesteps = step_sigmas * divisor
|
||||
sigmas = torch.tensor(
|
||||
[float(s) for s in self.sigmas_video],
|
||||
dtype=torch.float32,
|
||||
)
|
||||
log_probs = None
|
||||
if self.log_prob_sums:
|
||||
sums = torch.stack(self.log_prob_sums, dim=0)
|
||||
counts = torch.stack(self.log_prob_counts, dim=0)
|
||||
per_step = sums.squeeze(-1) / counts.squeeze(-1).clamp(min=1.0)
|
||||
log_probs = per_step.unsqueeze(0)
|
||||
return RolloutTrajectoryData(
|
||||
rollout_log_probs=log_probs,
|
||||
denoising_env=RolloutDenoisingEnv(
|
||||
pos_cond_kwargs=self.pos_cond_kwargs,
|
||||
),
|
||||
dit_trajectory=RolloutDitTrajectory(
|
||||
latents=stacked,
|
||||
timesteps=timesteps,
|
||||
sigmas=sigmas,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MiniMaxH3RolloutCollector",
|
||||
"MiniMaxH3RolloutCtx",
|
||||
"minimax_h3_rollout_update_video_target",
|
||||
]
|
||||
+43
@@ -783,6 +783,44 @@ class MiniMaxH3DenoisingStage(DenoisingStage):
|
||||
device=device,
|
||||
)
|
||||
initial_video, initial_audio = _expand_initial_rows(ctx, positive)
|
||||
rollout_ctx = None
|
||||
if getattr(batch, "rollout", False):
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.minimax_h3_rollout import (
|
||||
MiniMaxH3RolloutCollector,
|
||||
MiniMaxH3RolloutCtx,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.post_training.rl_dataclasses import (
|
||||
RolloutTrajectoryData,
|
||||
)
|
||||
|
||||
task = str(getattr(batch.sampling_params, "task", "") or "t2va").lower()
|
||||
if task not in ("t2va",):
|
||||
raise ValueError(
|
||||
f"MiniMax H3 rollout currently supports task=t2va only, got {task!r}"
|
||||
)
|
||||
generator = torch.Generator(device=device)
|
||||
seed = getattr(batch.sampling_params, "seed", 0)
|
||||
if isinstance(seed, list):
|
||||
seed = seed[0]
|
||||
generator.manual_seed(int(seed))
|
||||
collector = MiniMaxH3RolloutCollector(sigmas_video=sigmas_video)
|
||||
packed_cpu = {
|
||||
k: (v.detach().cpu() if isinstance(v, torch.Tensor) else v)
|
||||
for k, v in packed.items()
|
||||
}
|
||||
collector.pos_cond_kwargs = {
|
||||
"encoder_hidden_states": emb["hidden_states"].detach().cpu(),
|
||||
"h3_packed_layout": packed_cpu,
|
||||
"h3_token_tags": tags.detach().cpu(),
|
||||
"h3_video_target_start": positive.video_target_start,
|
||||
}
|
||||
rollout_ctx = MiniMaxH3RolloutCtx(
|
||||
batch=batch,
|
||||
generator=generator,
|
||||
sigmas_video=sigmas_video,
|
||||
collector=collector,
|
||||
)
|
||||
batch.rollout_trajectory_data = RolloutTrajectoryData()
|
||||
with (
|
||||
maybe_nvtx_range("denoising_loop", self.current_use_nvtx),
|
||||
self.progress_bar(
|
||||
@@ -821,7 +859,12 @@ class MiniMaxH3DenoisingStage(DenoisingStage):
|
||||
self._profile_denoising_step,
|
||||
batch=batch,
|
||||
),
|
||||
rollout_ctx=rollout_ctx,
|
||||
)
|
||||
if rollout_ctx is not None:
|
||||
batch.rollout_trajectory_data = (
|
||||
rollout_ctx.collector.build_trajectory_data()
|
||||
)
|
||||
finally:
|
||||
self._finish_active_component_use()
|
||||
_publish_full_loop_outputs(
|
||||
|
||||
Reference in New Issue
Block a user