From 973fb44471189af7c2b0152a27be79065e9c170d Mon Sep 17 00:00:00 2001 From: HuangJi <32611516+IPostYellow@users.noreply.github.com> Date: Wed, 23 Sep 2026 12:57:26 +0800 Subject: [PATCH] [Diffusion] Support MiniMax-H3 PDD(Parallel Decoding Distillation) inference (#40568) Co-authored-by: Forrest Lin Co-authored-by: Claude Opus 5 --- .../cookbook/diffusion/MiniMax/MiniMax-H3.mdx | 66 +++++++ python/sglang/multimodal_gen/envs.py | 7 + .../runtime/models/dits/minimax_h3.py | 66 ++++++- .../minimax_h3/stages/timestep_preparation.py | 47 +++++ .../tools/build_minimax_h3_pdd_weights.py | 167 +++++++++++++++++ .../tools/fuse_minimax_h3_pdd_heads.py | 109 +++++++++++ .../diffusion/test_minimax_h3_pdd_offline.py | 171 ++++++++++++++++++ 7 files changed, 627 insertions(+), 6 deletions(-) create mode 100755 python/sglang/multimodal_gen/tools/build_minimax_h3_pdd_weights.py create mode 100755 python/sglang/multimodal_gen/tools/fuse_minimax_h3_pdd_heads.py create mode 100644 test/registered/unit/diffusion/test_minimax_h3_pdd_offline.py diff --git a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx index 296cc7e815..151f7c0362 100644 --- a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx +++ b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx @@ -736,6 +736,72 @@ SGLang projects those adapter factors onto the pruned coordinates at load time. A structurally modified checkpoint without that metadata still fails closed, and packed GGUF weights remain incompatible with LoRA. +### PDD acceleration LoRAs + +[Alibaba PAI's MiniMax-H3-Acc-LoRAs](https://huggingface.co/alibaba-pai/MiniMax-H3-Acc-LoRAs) +use Parallel Decoding Distillation (PDD) for eight denoiser evaluations. These +checkpoints contain backbone LoRA weights and interval-specific output heads. +Merge the backbone weights offline, then fuse the output heads before serving. +The `--lora-path` recipe above does not perform these preparation steps. + +Match the adapter and original MiniMax-H3 transformer weights to the server's +model variant: + +| Model variant | Adapter file | Tasks | +| --- | --- | --- | +| `fl2va` | `MiniMax-H3-FL2VA-Acc-8Step.safetensors` | `t2va`, `fl2va` | +| `ref2va` | `MiniMax-H3-Ref2VA-Acc-8Step.safetensors` | `ref2va` | + +`BASE_TRANSFORMER_DIR` is the directory containing the original model's +BF16/FP32 transformer weight files, not the model root or the LoRA directory. +If you downloaded `MiniMaxAI/MiniMax-H3` to `/path/to/MiniMax-H3`, use +`/path/to/MiniMax-H3/FL2VA/transformer` for T2VA/FL2VA or +`/path/to/MiniMax-H3/Ref2VA/transformer` for Ref2VA. This directory contains +`model-*.safetensors`, `model.safetensors.index.json`, and `config.json`. +The builder reads these files and writes the merged weights to +`$PDD_DIR/transformer`, leaving the original weights unchanged. + +The following example prepares FL2VA +weights and serves them on eight NVIDIA CUDA GPUs with TP2 + Ulysses4: + +```bash Command +BASE_TRANSFORMER_DIR=/path/to/MiniMax-H3/FL2VA/transformer +LORA_DIR=/path/to/h3-acc-loras +PDD_DIR=/path/to/h3-fl2va-pdd + +hf download alibaba-pai/MiniMax-H3-Acc-LoRAs \ + MiniMax-H3-FL2VA-Acc-8Step.safetensors --local-dir "$LORA_DIR" + +python3 -m sglang.multimodal_gen.tools.build_minimax_h3_pdd_weights \ + "$BASE_TRANSFORMER_DIR" \ + "$LORA_DIR/MiniMax-H3-FL2VA-Acc-8Step.safetensors" \ + "$PDD_DIR" + +python3 -m sglang.multimodal_gen.tools.fuse_minimax_h3_pdd_heads \ + "$PDD_DIR" --video-shift 12.0 --audio-shift 3.0 + +SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS="$PDD_DIR/pdd_fused_heads.safetensors" \ +sglang serve \ + --model-path MiniMaxAI/MiniMax-H3 \ + --model-variant fl2va \ + --transformer-weights-path "$PDD_DIR/transformer" \ + --num-gpus 8 \ + --tp-size 2 \ + --ulysses-degree 4 \ + --performance-mode speed \ + --port 30010 +``` + +Keep `pdd_config.json` beside `pdd_fused_heads.safetensors`, outside the +`transformer/` directory. For Ref2VA, use its adapter and base transformer, +a separate output directory, and `--model-variant ref2va`. + +Use the requests in [Generate video and audio](#4-generate-video-and-audio) +with `num_inference_steps: 9`, `flow_shift: 12.0`, and `audio_flow_shift: 3.0`. +The nine sigma grid points give eight denoiser evaluations. These settings +must match the fused heads; the server rejects a different step count or +shift. Use `quality: "lossless"` in requests. + ## 6. FastH3: 4-step distilled preview [FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree](https://huggingface.co/FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree) diff --git a/python/sglang/multimodal_gen/envs.py b/python/sglang/multimodal_gen/envs.py index 6a429fc19b..67d06c10e5 100644 --- a/python/sglang/multimodal_gen/envs.py +++ b/python/sglang/multimodal_gen/envs.py @@ -47,6 +47,7 @@ if TYPE_CHECKING: SGLANG_DIFFUSION_DISABLE_AUTO_RESIDENCY: bool = False SGLANG_DIFFUSION_MINIMAX_H3_ADALN_GPU_PLANS: int = 64 SGLANG_DIFFUSION_MINIMAX_H3_ADALN_FP32: bool = False + SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS: str | None = None SGLANG_DIFFUSION_CFG_GATE_STEP: float = 1.0 # cache-dit env vars (primary transformer) # on by default; engages only on 2 ranks with peer-to-peer access and falls @@ -327,6 +328,12 @@ environment_variables: dict[str, Callable[[], Any]] = { "SGLANG_DIFFUSION_CFG_GATE_STEP": _lazy_float( "SGLANG_DIFFUSION_CFG_GATE_STEP", 1.0 ), + # Path to a Parallel Decoding Distillation head stack (one fused output head + # per denoise step, produced by fuse_minimax_h3_pdd_heads.py). Set only when serving a + # PDD-distilled checkpoint; an ordinary run leaves the projection alone. + "SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS": _lazy_str( + "SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS" + ), "SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D": _lazy_str( "SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D", "auto" ), diff --git a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py index db1d29a2d5..ec8e0f9ce1 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py @@ -67,7 +67,9 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor ) from sglang.multimodal_gen.runtime.layers.usp import _ring_attention_varlen from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping -from sglang.multimodal_gen.runtime.managers.forward_context import get_forward_context +from sglang.multimodal_gen.runtime.managers.forward_context import ( + get_forward_context, +) from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( LayerwiseOffloadableModuleMixin, is_layerwise_offloaded_module, @@ -1526,6 +1528,58 @@ class MiniMaxH3FinalLayer(nn.Module): quant_config=None, prefix=f"{prefix}.audio_out", ) + # Parallel Decoding Distillation heads, one per denoise step, loaded by + # `load_pdd_fused_heads`. None on every ordinary run. + self._pdd_heads: dict[str, torch.Tensor] | None = None + + def load_pdd_fused_heads(self, path: str) -> None: + """Swap the two output heads for a per-step stack (PDD). + + PDD replicates the final linear layer once per time interval and lets one + backbone evaluation advance a whole block of them; the per-block heads + fuse into one because H3's euler-eta0 step is linear in the predicted + velocity. `tools/fuse_minimax_h3_pdd_heads.py` does that fusion offline, + leaving one head per denoise step. + """ + from safetensors import safe_open + + with safe_open(path, "pt") as f: + heads = {k: f.get_tensor(k) for k in f.keys()} + steps = heads["video_out.weight"].shape[0] + for name in ("video_out", "audio_out"): + projection = getattr(self, name) + for suffix, shape in ( + ("weight", (steps, projection.output_size, projection.input_size)), + ("bias", (steps, projection.output_size)), + ): + key = f"{name}.{suffix}" + if steps == 0 or heads[key].shape != shape: + raise ValueError(f"MiniMax-H3 PDD {key} must have shape {shape}") + width = projection.output_size_per_partition + heads[key] = heads[key].narrow(1, projection.tp_rank * width, width) + self._pdd_heads = heads + logger.info("MiniMax-H3 PDD: %d fused output heads loaded from %s", steps, path) + + def _pdd_project(self, h: torch.Tensor, name: str) -> torch.Tensor: + heads = self._pdd_heads + step = int(get_forward_context().current_timestep) + stack = heads[f"{name}.weight"] + if not 0 <= step < stack.shape[0]: + raise ValueError( + f"MiniMax-H3 PDD has {stack.shape[0]} fused heads but the loop is at " + f"step {step}; run with --num-inference-steps {stack.shape[0] + 1} " + "(H3 counts sigma grid points, so that is one more than the steps)." + ) + weight = stack[step].to(device=h.device, dtype=h.dtype) + bias = heads[f"{name}.bias"][step].to(device=h.device, dtype=h.dtype) + return torch.nn.functional.linear(h, weight, bias) + + def _project(self, h: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + if self._pdd_heads is not None: + return self._pdd_project(h, "video_out"), self._pdd_project(h, "audio_out") + video, _ = self.video_out(h) + audio, _ = self.audio_out(h) + return video, audio def forward( self, @@ -1559,8 +1613,7 @@ class MiniMaxH3FinalLayer(nn.Module): inverse_indices[start:stop], dtype=_BF16_DTYPE, ).to(_FP32_DTYPE) - video_chunk, _ = self.video_out(h) - audio_chunk, _ = self.audio_out(h) + video_chunk, audio_chunk = self._project(h) if video is None: video = torch.empty( (x.shape[0], video_chunk.shape[-1]), @@ -1583,9 +1636,7 @@ class MiniMaxH3FinalLayer(nn.Module): h = _modulate_scale_shift(h, shift, scale, inverse_indices, dtype=_BF16_DTYPE) # Preserve full precision through both final output projections. h = h.to(_FP32_DTYPE) - video, _ = self.video_out(h) - audio, _ = self.audio_out(h) - return video, audio + return self._project(h) def _reject_adaln_lora(names: list[str]) -> None: @@ -1985,6 +2036,9 @@ class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin): prefix="final_layer", use_adaln_cache=self._adaln_precomputed, ) + pdd_heads = envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS + if pdd_heads: + self.final_layer.load_pdd_fused_heads(pdd_heads) self.adaln_cache = ( MiniMaxH3AdalnCache( arch, diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/timestep_preparation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/timestep_preparation.py index bc4712420d..e9da0a80d4 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/timestep_preparation.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/timestep_preparation.py @@ -1,11 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +import json import math from collections.abc import Mapping +from pathlib import Path import torch +from sglang.multimodal_gen import envs from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( @@ -29,6 +32,18 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage): # block, sigma_shift_scales): the schedule constants are a MODEL # serving contract — fl2va and ref2va use video 12 / audio 3 by default. self.sigma_shift_scales = sigma_shift_scales + self._pdd_config = None + pdd_heads = envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS + if pdd_heads: + from safetensors import safe_open + + self._pdd_config = json.loads( + Path(pdd_heads).with_name("pdd_config.json").read_text() + ) + with safe_open(pdd_heads, "pt") as f: + steps = f.get_slice("video_out.weight").get_shape()[0] + if self._pdd_config["num_inference_steps"] != steps + 1: + raise ValueError("MiniMax-H3 PDD config does not match the fused heads") def forward(self, batch: Req, server_args: ServerArgs) -> Req: from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.resolved_plan import ( @@ -42,6 +57,7 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage): "and has no implementation yet." ) self._generate_sigmas_from_plan(batch, plan) + self._apply_pdd_schedule(batch) self._publish_native_timestep_state(batch) return batch @@ -58,6 +74,7 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage): ) return ( batch.num_inference_steps, + batch.is_warmup, plan.flow_shift, plan.audio_flow_shift, plan.default_flow_shift, @@ -65,6 +82,36 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage): self.freeze_for_dedup(self.sigma_shift_scales), ) + def _apply_pdd_schedule(self, batch: Req) -> None: + if self._pdd_config is None: + return + from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import ( + minimax_h3_time_shift_sigmas, + ) + + config = self._pdd_config + sigmas = batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY] + for modality in ("video", "audio"): + expected = minimax_h3_time_shift_sigmas( + num_steps=config["num_inference_steps"], + shift_scale=config[f"{modality}_shift"], + ) + if batch.is_warmup: + # Warmup may run fewer steps, but must use the same intervals + # as serving rather than rescaling a shorter grid to [1, 0]. + sigmas[modality] = expected[: max(2, batch.num_inference_steps)] + continue + actual = sigmas[modality] + if len(actual) != len(expected) or any( + not math.isclose(a, b, rel_tol=1e-6, abs_tol=1e-7) + for a, b in zip(actual, expected) + ): + raise ValueError( + f"MiniMax-H3 PDD requires num_inference_steps=" + f"{config['num_inference_steps']} and {modality} shift=" + f"{config[f'{modality}_shift']} to match the fused heads" + ) + @staticmethod def _publish_native_timestep_state(batch: Req) -> None: sigmas = batch.extra.get(MINIMAX_H3_SIGMAS_EXTRA_KEY) diff --git a/python/sglang/multimodal_gen/tools/build_minimax_h3_pdd_weights.py b/python/sglang/multimodal_gen/tools/build_minimax_h3_pdd_weights.py new file mode 100755 index 0000000000..25675a6d89 --- /dev/null +++ b/python/sglang/multimodal_gen/tools/build_minimax_h3_pdd_weights.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Merge alibaba-pai's PDD LoRA into a native MiniMax-H3 checkpoint. + + python3 -m sglang.multimodal_gen.tools.build_minimax_h3_pdd_weights \ + + +Writes merged weights to `transformer/` and interval-specific output heads to +`pdd_heads.safetensors`, alongside `pdd_config.json`. The heads must stay outside +`transformer/` so the model loader does not treat them as checkpoint shards. +Run `fuse_minimax_h3_pdd_heads.py` on the output directory before serving. + +Serve with `--transformer-weights-path /transformer` and set +`SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS=/pdd_fused_heads.safetensors`. +Keep `pdd_config.json` beside the fused heads for schedule validation. +""" + +from __future__ import annotations + +import argparse +import json +import shutil +from pathlib import Path + +import torch +from safetensors import safe_open +from safetensors.torch import load_file, save_file + +HEAD_DIM = 128 + + +def _interleave_qkv(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor: + """[3 x (heads*head_dim), in] -> the official per-head interleaved layout.""" + in_dim = q.shape[1] + return torch.stack( + [x.reshape(-1, HEAD_DIM, in_dim) for x in (q, k, v)], dim=1 + ).reshape(-1, in_dim) + + +def _target_of(lora_key: str) -> str | None: + """LoRA module name -> official weight name. None means "not handled here".""" + k = lora_key + if k.startswith("token_refiner.refiner_blocks."): + k = "token_refiner.blocks." + k[len("token_refiner.refiner_blocks.") :] + elif k.startswith("transformer_blocks."): + k = "blocks." + k[len("transformer_blocks.") :] + else: + return None + for src, dst in ( + (".attn.to_out.0", ".attn.out_proj"), + (".ff.net.0.proj", ".mlp.fc1"), + (".ff.net.2", ".mlp.fc2"), + ): + if k.endswith(src): + return k[: -len(src)] + dst + return k # attn.to_q/k/v pass through; the caller interleaves them + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("base_dir", type=Path) + ap.add_argument("lora_path") + ap.add_argument("out_dir", type=Path) + ap.add_argument( + "--verify", + action="store_true", + help="print the relative magnitude of each merged weight change", + ) + args = ap.parse_args(argv) + + with safe_open(args.lora_path, "pt") as f: + meta = f.metadata() or {} + lora = {k: f.get_tensor(k) for k in f.keys()} + rank = int(meta.get("lora_rank", 64)) + alpha = float(meta.get("lora_alpha", rank)) + scale = alpha / rank + pdd = { + "num_steps": int(meta.get("pdd_num_steps", 32)), + "block_size": int(meta.get("pdd_block_size", 4)), + } + print(f"LoRA: rank={rank} alpha={alpha} scale={scale} pdd={pdd}") + + # Group lora_down/lora_up per module; q/k/v are held back and interleaved at the end. + pairs: dict[str, dict[str, torch.Tensor]] = {} + for key, value in lora.items(): + for suffix in (".lora_down", ".lora_up"): + if key.endswith(suffix): + pairs.setdefault(key[: -len(suffix)], {})[suffix[1:]] = value + break + + deltas: dict[str, torch.Tensor] = {} # official weight name -> delta W + qkv: dict[str, dict[str, torch.Tensor]] = {} # module prefix -> {q,k,v: delta W} + for module, dv in pairs.items(): + if "lora_down" not in dv or "lora_up" not in dv: + raise SystemExit(f"{module} has only one half of the LoRA pair") + delta = (dv["lora_up"].float() @ dv["lora_down"].float()) * scale + target = _target_of(module) + if target is None: + raise SystemExit(f"no mapping rule for {module}") + for role in ("to_q", "to_k", "to_v"): + if target.endswith(f".attn.{role}"): + qkv.setdefault(target[: -len(f".attn.{role}")], {})[role[-1]] = delta + break + else: + deltas[target + ".weight"] = delta + + for prefix, parts in qkv.items(): + if set(parts) != {"q", "k", "v"}: + raise SystemExit(f"{prefix} is missing part of q/k/v: {sorted(parts)}") + deltas[prefix + ".attn.qkv_proj.weight"] = _interleave_qkv( + parts["q"], parts["k"], parts["v"] + ) + print(f"2D deltas to merge: {len(deltas)} ({len(qkv)} of them qkv)") + + out = args.out_dir + transformer_out = out / "transformer" + transformer_out.mkdir(parents=True, exist_ok=True) + applied = 0 + for shard in sorted(args.base_dir.glob("*.safetensors")): + tensors = load_file(str(shard)) + for key, base in tensors.items(): + if key in deltas: + merged = base.float() + deltas[key] + if args.verify: + rel = ((merged - base.float()).norm() / base.float().norm()).item() + print(f" {key}: relative change {rel:.4f}") + tensors[key] = merged.to(base.dtype) + applied += 1 + save_file(tensors, str(transformer_out / shard.name), metadata={"format": "pt"}) + print(f"merged {applied}/{len(deltas)} tensors") + if applied != len(deltas): + raise SystemExit( + f"{len(deltas) - applied} deltas found no target; do not use this output" + ) + + for extra in ( + "config.json", + "model.safetensors.index.json", + "diffusion_pytorch_model.safetensors.index.json", + ): + src = args.base_dir / extra + if src.exists(): + shutil.copy(src, transformer_out / extra) + + heads = { + k: lora[k] + for k in ( + "proj_out.weight", + "proj_out.bias", + "audio_proj_out.weight", + "audio_proj_out.bias", + ) + } + save_file(heads, str(out / "pdd_heads.safetensors"), metadata={"format": "pt"}) + (out / "pdd_config.json").write_text(json.dumps(pdd, indent=2) + "\n") + print( + f"position-level output heads -> {out / 'pdd_heads.safetensors'} " + f"({pdd['num_steps']} of them, block size {pdd['block_size']}, " + f"i.e. {pdd['num_steps'] // pdd['block_size']} forwards)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/sglang/multimodal_gen/tools/fuse_minimax_h3_pdd_heads.py b/python/sglang/multimodal_gen/tools/fuse_minimax_h3_pdd_heads.py new file mode 100755 index 0000000000..91ad5d96b8 --- /dev/null +++ b/python/sglang/multimodal_gen/tools/fuse_minimax_h3_pdd_heads.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Fuse PDD's 32 position-level output heads into one per block, i.e. 8. + + python3 -m sglang.multimodal_gen.tools.fuse_minimax_h3_pdd_heads \ + [--video-shift 12.0 --audio-shift 3.0] + +## Why the fusion is exact + +MiniMax-H3's scheduler has "ancestral" in its name but takes the eta0 branch, which +injects no noise: + + x_next = r*x + (1-r)*x0, r = sigma_next/sigma_curr, x0 = x - sigma_curr*v + = x - (sigma_curr - sigma_next)*v + +That is plain Euler, and linear in v. The L heads in one PDD block share a single +backbone evaluation, so + + x_{n+L} = x_n - sum_j dsigma_{n+j} * W_{n+j} h + = x_n - (sum_j dsigma_{n+j}) * (sum_j w_j W_{n+j}) h, w_j = dsigma_{n+j} / sum dsigma + +and the bracket is the fused head -- the paper's "fuse the layers into a single +linear layer that directly predicts a step across the full block". Each block then +costs one forward and one scheduler step, and the denoise loop needs no changes. + +This depends on eta0. Under true ancestral sampling (noise injected every step) the +state at the block's second step is no longer the block's first state, the sum +cannot be moved onto W, and the fusion is invalid -- the L sub-steps have to be run. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import torch +from safetensors import safe_open +from safetensors.torch import save_file + + +def sigma_grid(num_points: int, shift: float) -> torch.Tensor: + base = torch.linspace(1.0, 0.0, num_points, dtype=torch.float64) + return shift * base / (1 + (shift - 1) * base) + + +def fuse(heads: torch.Tensor, sigmas: torch.Tensor, block: int) -> torch.Tensor: + """heads: (N, ...) -> (N//block, ...), averaged within a block weighted by dsigma.""" + steps = heads.shape[0] + if block <= 0 or steps == 0 or steps % block: + raise ValueError("PDD head count must be positive and divisible by block size") + if sigmas.numel() != steps + 1: + raise SystemExit( + f"sigma grid has {sigmas.numel()} points but there are {steps} heads" + ) + dsigma = (sigmas[:-1] - sigmas[1:]).clamp(min=0) + out = [] + for b in range(steps // block): + w = dsigma[b * block : (b + 1) * block] + if float(w.sum()) <= 0: + raise SystemExit(f"block {b} has an all-zero dsigma") + w = (w / w.sum()).to(heads.dtype) + chunk = heads[b * block : (b + 1) * block].float() + out.append((chunk * w.float().view(-1, *([1] * (chunk.dim() - 1)))).sum(0)) + return torch.stack(out).to(heads.dtype) + + +def main() -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument("pdd_dir") + ap.add_argument("--video-shift", type=float, default=12.0) + ap.add_argument("--audio-shift", type=float, default=3.0) + args = ap.parse_args() + + d = Path(args.pdd_dir) + cfg = json.loads((d / "pdd_config.json").read_text()) + n, block = cfg["num_steps"], cfg["block_size"] + with safe_open(str(d / "pdd_heads.safetensors"), "pt") as f: + heads = {k: f.get_tensor(k) for k in f.keys()} + + sv = sigma_grid(n + 1, args.video_shift) + sa = sigma_grid(n + 1, args.audio_shift) + fused = { + "video_out.weight": fuse(heads["proj_out.weight"], sv, block), + "video_out.bias": fuse(heads["proj_out.bias"], sv, block), + "audio_out.weight": fuse(heads["audio_proj_out.weight"], sa, block), + "audio_out.bias": fuse(heads["audio_proj_out.bias"], sa, block), + } + save_file(fused, str(d / "pdd_fused_heads.safetensors"), metadata={"format": "pt"}) + cfg["fused_steps"] = n // block + cfg["num_inference_steps"] = n // block + 1 # H3 counts steps as sigma grid points + cfg["video_shift"], cfg["audio_shift"] = args.video_shift, args.audio_shift + (d / "pdd_config.json").write_text(json.dumps(cfg, indent=2) + "\n") + + print(f"fused {n} -> {n // block} heads (block size {block})") + for k, v in fused.items(): + print(f" {k}: {tuple(v.shape)}") + print(f"run with --num-inference-steps {cfg['num_inference_steps']}") + print( + "first 9 points of the video sigma grid:", + [round(float(x), 4) for x in sv[::block]], + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/registered/unit/diffusion/test_minimax_h3_pdd_offline.py b/test/registered/unit/diffusion/test_minimax_h3_pdd_offline.py new file mode 100644 index 0000000000..0f016e9d48 --- /dev/null +++ b/test/registered/unit/diffusion/test_minimax_h3_pdd_offline.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: Apache-2.0 +import json +import tempfile +import unittest +from functools import partial +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import torch +from safetensors.torch import load_file, save_file + +from sglang.multimodal_gen.configs.models.dits.minimax_h3 import MiniMaxH3DiTArchConfig +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.models.dits.minimax_h3 import MiniMaxH3FinalLayer +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.constants import ( + MINIMAX_H3_SIGMAS_EXTRA_KEY, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.timestep_preparation import ( + MiniMaxH3TimestepPreparationStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import ( + minimax_h3_time_shift_sigmas, +) +from sglang.multimodal_gen.tools import build_minimax_h3_pdd_weights as build +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=12, stage="base-b", runner_config="diffusion-unit-1-gpu-h100") + + +def _make_heads() -> dict[str, torch.Tensor]: + heads = {} + for name, width in (("video_out", 6), ("audio_out", 4)): + heads[f"{name}.weight"] = ( + torch.arange(8 * width * 5, dtype=torch.float32).reshape(8, width, 5) / 100 + ) + heads[f"{name}.bias"] = ( + torch.arange(8 * width, dtype=torch.float32).reshape(8, width) / 100 + ) + return heads + + +class TestMiniMaxH3PDDOffline(CustomTestCase): + def test_tp_projection_matches_full_head(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + heads = _make_heads() + path = root / "heads.safetensors" + save_file(heads, str(path)) + arch = MiniMaxH3DiTArchConfig( + hidden_size=5, latents_dim=6, audio_latents_dim=4, patch_size=(1, 1, 1) + ) + layers = [] + for rank in range(2): + with patch( + "sglang.multimodal_gen.runtime.layers.linear.get_tp_group", + return_value=SimpleNamespace(world_size=2, rank_in_group=rank), + ): + layer = MiniMaxH3FinalLayer( + arch, None, prefix="final_layer", use_adaln_cache=True + ) + layer.load_pdd_fused_heads(str(path)) + layers.append(layer) + + h = torch.arange(15, dtype=torch.float32).reshape(3, 5) + for step in (0, 7): + with self.subTest(step=step), set_forward_context(step, None): + outputs = [layer._project(h) for layer in layers] + for i, name in enumerate(("video_out", "audio_out")): + expected = torch.nn.functional.linear( + h, + heads[f"{name}.weight"][step], + heads[f"{name}.bias"][step], + ) + torch.testing.assert_close( + torch.cat([output[i] for output in outputs], dim=-1), + expected, + ) + + def test_schedule_validation_and_warmup(self): + apply_schedule = partial( + MiniMaxH3TimestepPreparationStage._apply_pdd_schedule, + SimpleNamespace( + _pdd_config={ + "num_inference_steps": 9, + "video_shift": 12.0, + "audio_shift": 3.0, + } + ), + ) + + sigmas = { + name: minimax_h3_time_shift_sigmas(num_steps=9, shift_scale=shift) + for name, shift in (("video", 12.0), ("audio", 3.0)) + } + batch = SimpleNamespace( + is_warmup=False, extra={MINIMAX_H3_SIGMAS_EXTRA_KEY: sigmas} + ) + apply_schedule(batch) + wrong_shift = minimax_h3_time_shift_sigmas(num_steps=9, shift_scale=6.0) + for name, invalid in ( + ("video", sigmas["video"][:-1]), + ("video", sigmas["video"] + [0.0]), + ("video", wrong_shift), + ("audio", wrong_shift), + ): + batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY] = {**sigmas, name: invalid} + with ( + self.subTest(modality=name, sigmas=invalid), + self.assertRaisesRegex(ValueError, "match the fused heads"), + ): + apply_schedule(batch) + batch.is_warmup = True + batch.num_inference_steps = 3 + apply_schedule(batch) + self.assertEqual( + batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY], + {name: values[:3] for name, values in sigmas.items()}, + ) + + def test_conversion_keeps_heads_outside_transformer(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + heads = _make_heads() + base = root / "base" + base.mkdir() + weight_name = "blocks.0.attn.out_proj.weight" + save_file({weight_name: torch.zeros(2, 3)}, str(base / "model.safetensors")) + (base / "config.json").write_text("{}") + index = {"weight_map": {weight_name: "model.safetensors"}} + (base / "model.safetensors.index.json").write_text(json.dumps(index)) + lora = { + "transformer_blocks.0.attn.to_out.0.lora_down": torch.ones(1, 3), + "transformer_blocks.0.attn.to_out.0.lora_up": torch.ones(2, 1), + } + for key, value in heads.items(): + lora[ + key.replace("video_out", "proj_out").replace( + "audio_out", "audio_proj_out" + ) + ] = value + lora_path = root / "lora.safetensors" + save_file( + lora, + str(lora_path), + metadata={ + "lora_rank": "1", + "pdd_num_steps": "8", + "pdd_block_size": "2", + }, + ) + out = root / "out" + build.main([str(base), str(lora_path), str(out)]) + transformer = out / "transformer" + self.assertEqual( + list(transformer.glob("*.safetensors")), + [transformer / "model.safetensors"], + ) + torch.testing.assert_close( + load_file(str(transformer / "model.safetensors"))[weight_name], + torch.ones(2, 3), + ) + self.assertEqual( + json.loads((transformer / "model.safetensors.index.json").read_text()), + index, + ) + + +if __name__ == "__main__": + unittest.main(verbosity=3)