diff --git a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx
index 296cc7e815..151f7c0362 100644
--- a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx
+++ b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx
@@ -736,6 +736,72 @@ SGLang projects those adapter factors onto the pruned coordinates at load time.
A structurally modified checkpoint without that metadata still fails closed,
and packed GGUF weights remain incompatible with LoRA.
+### PDD acceleration LoRAs
+
+[Alibaba PAI's MiniMax-H3-Acc-LoRAs](https://huggingface.co/alibaba-pai/MiniMax-H3-Acc-LoRAs)
+use Parallel Decoding Distillation (PDD) for eight denoiser evaluations. These
+checkpoints contain backbone LoRA weights and interval-specific output heads.
+Merge the backbone weights offline, then fuse the output heads before serving.
+The `--lora-path` recipe above does not perform these preparation steps.
+
+Match the adapter and original MiniMax-H3 transformer weights to the server's
+model variant:
+
+| Model variant | Adapter file | Tasks |
+| --- | --- | --- |
+| `fl2va` | `MiniMax-H3-FL2VA-Acc-8Step.safetensors` | `t2va`, `fl2va` |
+| `ref2va` | `MiniMax-H3-Ref2VA-Acc-8Step.safetensors` | `ref2va` |
+
+`BASE_TRANSFORMER_DIR` is the directory containing the original model's
+BF16/FP32 transformer weight files, not the model root or the LoRA directory.
+If you downloaded `MiniMaxAI/MiniMax-H3` to `/path/to/MiniMax-H3`, use
+`/path/to/MiniMax-H3/FL2VA/transformer` for T2VA/FL2VA or
+`/path/to/MiniMax-H3/Ref2VA/transformer` for Ref2VA. This directory contains
+`model-*.safetensors`, `model.safetensors.index.json`, and `config.json`.
+The builder reads these files and writes the merged weights to
+`$PDD_DIR/transformer`, leaving the original weights unchanged.
+
+The following example prepares FL2VA
+weights and serves them on eight NVIDIA CUDA GPUs with TP2 + Ulysses4:
+
+```bash Command
+BASE_TRANSFORMER_DIR=/path/to/MiniMax-H3/FL2VA/transformer
+LORA_DIR=/path/to/h3-acc-loras
+PDD_DIR=/path/to/h3-fl2va-pdd
+
+hf download alibaba-pai/MiniMax-H3-Acc-LoRAs \
+ MiniMax-H3-FL2VA-Acc-8Step.safetensors --local-dir "$LORA_DIR"
+
+python3 -m sglang.multimodal_gen.tools.build_minimax_h3_pdd_weights \
+ "$BASE_TRANSFORMER_DIR" \
+ "$LORA_DIR/MiniMax-H3-FL2VA-Acc-8Step.safetensors" \
+ "$PDD_DIR"
+
+python3 -m sglang.multimodal_gen.tools.fuse_minimax_h3_pdd_heads \
+ "$PDD_DIR" --video-shift 12.0 --audio-shift 3.0
+
+SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS="$PDD_DIR/pdd_fused_heads.safetensors" \
+sglang serve \
+ --model-path MiniMaxAI/MiniMax-H3 \
+ --model-variant fl2va \
+ --transformer-weights-path "$PDD_DIR/transformer" \
+ --num-gpus 8 \
+ --tp-size 2 \
+ --ulysses-degree 4 \
+ --performance-mode speed \
+ --port 30010
+```
+
+Keep `pdd_config.json` beside `pdd_fused_heads.safetensors`, outside the
+`transformer/` directory. For Ref2VA, use its adapter and base transformer,
+a separate output directory, and `--model-variant ref2va`.
+
+Use the requests in [Generate video and audio](#4-generate-video-and-audio)
+with `num_inference_steps: 9`, `flow_shift: 12.0`, and `audio_flow_shift: 3.0`.
+The nine sigma grid points give eight denoiser evaluations. These settings
+must match the fused heads; the server rejects a different step count or
+shift. Use `quality: "lossless"` in requests.
+
## 6. FastH3: 4-step distilled preview
[FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree](https://huggingface.co/FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree)
diff --git a/python/sglang/multimodal_gen/envs.py b/python/sglang/multimodal_gen/envs.py
index 6a429fc19b..67d06c10e5 100644
--- a/python/sglang/multimodal_gen/envs.py
+++ b/python/sglang/multimodal_gen/envs.py
@@ -47,6 +47,7 @@ if TYPE_CHECKING:
SGLANG_DIFFUSION_DISABLE_AUTO_RESIDENCY: bool = False
SGLANG_DIFFUSION_MINIMAX_H3_ADALN_GPU_PLANS: int = 64
SGLANG_DIFFUSION_MINIMAX_H3_ADALN_FP32: bool = False
+ SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS: str | None = None
SGLANG_DIFFUSION_CFG_GATE_STEP: float = 1.0
# cache-dit env vars (primary transformer)
# on by default; engages only on 2 ranks with peer-to-peer access and falls
@@ -327,6 +328,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
"SGLANG_DIFFUSION_CFG_GATE_STEP": _lazy_float(
"SGLANG_DIFFUSION_CFG_GATE_STEP", 1.0
),
+ # Path to a Parallel Decoding Distillation head stack (one fused output head
+ # per denoise step, produced by fuse_minimax_h3_pdd_heads.py). Set only when serving a
+ # PDD-distilled checkpoint; an ordinary run leaves the projection alone.
+ "SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS": _lazy_str(
+ "SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS"
+ ),
"SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D": _lazy_str(
"SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D", "auto"
),
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py
index db1d29a2d5..ec8e0f9ce1 100644
--- a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py
+++ b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py
@@ -67,7 +67,9 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor
)
from sglang.multimodal_gen.runtime.layers.usp import _ring_attention_varlen
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
-from sglang.multimodal_gen.runtime.managers.forward_context import get_forward_context
+from sglang.multimodal_gen.runtime.managers.forward_context import (
+ get_forward_context,
+)
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
is_layerwise_offloaded_module,
@@ -1526,6 +1528,58 @@ class MiniMaxH3FinalLayer(nn.Module):
quant_config=None,
prefix=f"{prefix}.audio_out",
)
+ # Parallel Decoding Distillation heads, one per denoise step, loaded by
+ # `load_pdd_fused_heads`. None on every ordinary run.
+ self._pdd_heads: dict[str, torch.Tensor] | None = None
+
+ def load_pdd_fused_heads(self, path: str) -> None:
+ """Swap the two output heads for a per-step stack (PDD).
+
+ PDD replicates the final linear layer once per time interval and lets one
+ backbone evaluation advance a whole block of them; the per-block heads
+ fuse into one because H3's euler-eta0 step is linear in the predicted
+ velocity. `tools/fuse_minimax_h3_pdd_heads.py` does that fusion offline,
+ leaving one head per denoise step.
+ """
+ from safetensors import safe_open
+
+ with safe_open(path, "pt") as f:
+ heads = {k: f.get_tensor(k) for k in f.keys()}
+ steps = heads["video_out.weight"].shape[0]
+ for name in ("video_out", "audio_out"):
+ projection = getattr(self, name)
+ for suffix, shape in (
+ ("weight", (steps, projection.output_size, projection.input_size)),
+ ("bias", (steps, projection.output_size)),
+ ):
+ key = f"{name}.{suffix}"
+ if steps == 0 or heads[key].shape != shape:
+ raise ValueError(f"MiniMax-H3 PDD {key} must have shape {shape}")
+ width = projection.output_size_per_partition
+ heads[key] = heads[key].narrow(1, projection.tp_rank * width, width)
+ self._pdd_heads = heads
+ logger.info("MiniMax-H3 PDD: %d fused output heads loaded from %s", steps, path)
+
+ def _pdd_project(self, h: torch.Tensor, name: str) -> torch.Tensor:
+ heads = self._pdd_heads
+ step = int(get_forward_context().current_timestep)
+ stack = heads[f"{name}.weight"]
+ if not 0 <= step < stack.shape[0]:
+ raise ValueError(
+ f"MiniMax-H3 PDD has {stack.shape[0]} fused heads but the loop is at "
+ f"step {step}; run with --num-inference-steps {stack.shape[0] + 1} "
+ "(H3 counts sigma grid points, so that is one more than the steps)."
+ )
+ weight = stack[step].to(device=h.device, dtype=h.dtype)
+ bias = heads[f"{name}.bias"][step].to(device=h.device, dtype=h.dtype)
+ return torch.nn.functional.linear(h, weight, bias)
+
+ def _project(self, h: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ if self._pdd_heads is not None:
+ return self._pdd_project(h, "video_out"), self._pdd_project(h, "audio_out")
+ video, _ = self.video_out(h)
+ audio, _ = self.audio_out(h)
+ return video, audio
def forward(
self,
@@ -1559,8 +1613,7 @@ class MiniMaxH3FinalLayer(nn.Module):
inverse_indices[start:stop],
dtype=_BF16_DTYPE,
).to(_FP32_DTYPE)
- video_chunk, _ = self.video_out(h)
- audio_chunk, _ = self.audio_out(h)
+ video_chunk, audio_chunk = self._project(h)
if video is None:
video = torch.empty(
(x.shape[0], video_chunk.shape[-1]),
@@ -1583,9 +1636,7 @@ class MiniMaxH3FinalLayer(nn.Module):
h = _modulate_scale_shift(h, shift, scale, inverse_indices, dtype=_BF16_DTYPE)
# Preserve full precision through both final output projections.
h = h.to(_FP32_DTYPE)
- video, _ = self.video_out(h)
- audio, _ = self.audio_out(h)
- return video, audio
+ return self._project(h)
def _reject_adaln_lora(names: list[str]) -> None:
@@ -1985,6 +2036,9 @@ class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin):
prefix="final_layer",
use_adaln_cache=self._adaln_precomputed,
)
+ pdd_heads = envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS
+ if pdd_heads:
+ self.final_layer.load_pdd_fused_heads(pdd_heads)
self.adaln_cache = (
MiniMaxH3AdalnCache(
arch,
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/timestep_preparation.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/timestep_preparation.py
index bc4712420d..e9da0a80d4 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/timestep_preparation.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/timestep_preparation.py
@@ -1,11 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
+import json
import math
from collections.abc import Mapping
+from pathlib import Path
import torch
+from sglang.multimodal_gen import envs
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
@@ -29,6 +32,18 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
# block, sigma_shift_scales): the schedule constants are a MODEL
# serving contract — fl2va and ref2va use video 12 / audio 3 by default.
self.sigma_shift_scales = sigma_shift_scales
+ self._pdd_config = None
+ pdd_heads = envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS
+ if pdd_heads:
+ from safetensors import safe_open
+
+ self._pdd_config = json.loads(
+ Path(pdd_heads).with_name("pdd_config.json").read_text()
+ )
+ with safe_open(pdd_heads, "pt") as f:
+ steps = f.get_slice("video_out.weight").get_shape()[0]
+ if self._pdd_config["num_inference_steps"] != steps + 1:
+ raise ValueError("MiniMax-H3 PDD config does not match the fused heads")
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.resolved_plan import (
@@ -42,6 +57,7 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
"and has no implementation yet."
)
self._generate_sigmas_from_plan(batch, plan)
+ self._apply_pdd_schedule(batch)
self._publish_native_timestep_state(batch)
return batch
@@ -58,6 +74,7 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
)
return (
batch.num_inference_steps,
+ batch.is_warmup,
plan.flow_shift,
plan.audio_flow_shift,
plan.default_flow_shift,
@@ -65,6 +82,36 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
self.freeze_for_dedup(self.sigma_shift_scales),
)
+ def _apply_pdd_schedule(self, batch: Req) -> None:
+ if self._pdd_config is None:
+ return
+ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import (
+ minimax_h3_time_shift_sigmas,
+ )
+
+ config = self._pdd_config
+ sigmas = batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY]
+ for modality in ("video", "audio"):
+ expected = minimax_h3_time_shift_sigmas(
+ num_steps=config["num_inference_steps"],
+ shift_scale=config[f"{modality}_shift"],
+ )
+ if batch.is_warmup:
+ # Warmup may run fewer steps, but must use the same intervals
+ # as serving rather than rescaling a shorter grid to [1, 0].
+ sigmas[modality] = expected[: max(2, batch.num_inference_steps)]
+ continue
+ actual = sigmas[modality]
+ if len(actual) != len(expected) or any(
+ not math.isclose(a, b, rel_tol=1e-6, abs_tol=1e-7)
+ for a, b in zip(actual, expected)
+ ):
+ raise ValueError(
+ f"MiniMax-H3 PDD requires num_inference_steps="
+ f"{config['num_inference_steps']} and {modality} shift="
+ f"{config[f'{modality}_shift']} to match the fused heads"
+ )
+
@staticmethod
def _publish_native_timestep_state(batch: Req) -> None:
sigmas = batch.extra.get(MINIMAX_H3_SIGMAS_EXTRA_KEY)
diff --git a/python/sglang/multimodal_gen/tools/build_minimax_h3_pdd_weights.py b/python/sglang/multimodal_gen/tools/build_minimax_h3_pdd_weights.py
new file mode 100755
index 0000000000..25675a6d89
--- /dev/null
+++ b/python/sglang/multimodal_gen/tools/build_minimax_h3_pdd_weights.py
@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""Merge alibaba-pai's PDD LoRA into a native MiniMax-H3 checkpoint.
+
+ python3 -m sglang.multimodal_gen.tools.build_minimax_h3_pdd_weights \
+
+
+Writes merged weights to `transformer/` and interval-specific output heads to
+`pdd_heads.safetensors`, alongside `pdd_config.json`. The heads must stay outside
+`transformer/` so the model loader does not treat them as checkpoint shards.
+Run `fuse_minimax_h3_pdd_heads.py` on the output directory before serving.
+
+Serve with `--transformer-weights-path /transformer` and set
+`SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS=/pdd_fused_heads.safetensors`.
+Keep `pdd_config.json` beside the fused heads for schedule validation.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import shutil
+from pathlib import Path
+
+import torch
+from safetensors import safe_open
+from safetensors.torch import load_file, save_file
+
+HEAD_DIM = 128
+
+
+def _interleave_qkv(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
+ """[3 x (heads*head_dim), in] -> the official per-head interleaved layout."""
+ in_dim = q.shape[1]
+ return torch.stack(
+ [x.reshape(-1, HEAD_DIM, in_dim) for x in (q, k, v)], dim=1
+ ).reshape(-1, in_dim)
+
+
+def _target_of(lora_key: str) -> str | None:
+ """LoRA module name -> official weight name. None means "not handled here"."""
+ k = lora_key
+ if k.startswith("token_refiner.refiner_blocks."):
+ k = "token_refiner.blocks." + k[len("token_refiner.refiner_blocks.") :]
+ elif k.startswith("transformer_blocks."):
+ k = "blocks." + k[len("transformer_blocks.") :]
+ else:
+ return None
+ for src, dst in (
+ (".attn.to_out.0", ".attn.out_proj"),
+ (".ff.net.0.proj", ".mlp.fc1"),
+ (".ff.net.2", ".mlp.fc2"),
+ ):
+ if k.endswith(src):
+ return k[: -len(src)] + dst
+ return k # attn.to_q/k/v pass through; the caller interleaves them
+
+
+def main(argv: list[str] | None = None) -> int:
+ ap = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+ )
+ ap.add_argument("base_dir", type=Path)
+ ap.add_argument("lora_path")
+ ap.add_argument("out_dir", type=Path)
+ ap.add_argument(
+ "--verify",
+ action="store_true",
+ help="print the relative magnitude of each merged weight change",
+ )
+ args = ap.parse_args(argv)
+
+ with safe_open(args.lora_path, "pt") as f:
+ meta = f.metadata() or {}
+ lora = {k: f.get_tensor(k) for k in f.keys()}
+ rank = int(meta.get("lora_rank", 64))
+ alpha = float(meta.get("lora_alpha", rank))
+ scale = alpha / rank
+ pdd = {
+ "num_steps": int(meta.get("pdd_num_steps", 32)),
+ "block_size": int(meta.get("pdd_block_size", 4)),
+ }
+ print(f"LoRA: rank={rank} alpha={alpha} scale={scale} pdd={pdd}")
+
+ # Group lora_down/lora_up per module; q/k/v are held back and interleaved at the end.
+ pairs: dict[str, dict[str, torch.Tensor]] = {}
+ for key, value in lora.items():
+ for suffix in (".lora_down", ".lora_up"):
+ if key.endswith(suffix):
+ pairs.setdefault(key[: -len(suffix)], {})[suffix[1:]] = value
+ break
+
+ deltas: dict[str, torch.Tensor] = {} # official weight name -> delta W
+ qkv: dict[str, dict[str, torch.Tensor]] = {} # module prefix -> {q,k,v: delta W}
+ for module, dv in pairs.items():
+ if "lora_down" not in dv or "lora_up" not in dv:
+ raise SystemExit(f"{module} has only one half of the LoRA pair")
+ delta = (dv["lora_up"].float() @ dv["lora_down"].float()) * scale
+ target = _target_of(module)
+ if target is None:
+ raise SystemExit(f"no mapping rule for {module}")
+ for role in ("to_q", "to_k", "to_v"):
+ if target.endswith(f".attn.{role}"):
+ qkv.setdefault(target[: -len(f".attn.{role}")], {})[role[-1]] = delta
+ break
+ else:
+ deltas[target + ".weight"] = delta
+
+ for prefix, parts in qkv.items():
+ if set(parts) != {"q", "k", "v"}:
+ raise SystemExit(f"{prefix} is missing part of q/k/v: {sorted(parts)}")
+ deltas[prefix + ".attn.qkv_proj.weight"] = _interleave_qkv(
+ parts["q"], parts["k"], parts["v"]
+ )
+ print(f"2D deltas to merge: {len(deltas)} ({len(qkv)} of them qkv)")
+
+ out = args.out_dir
+ transformer_out = out / "transformer"
+ transformer_out.mkdir(parents=True, exist_ok=True)
+ applied = 0
+ for shard in sorted(args.base_dir.glob("*.safetensors")):
+ tensors = load_file(str(shard))
+ for key, base in tensors.items():
+ if key in deltas:
+ merged = base.float() + deltas[key]
+ if args.verify:
+ rel = ((merged - base.float()).norm() / base.float().norm()).item()
+ print(f" {key}: relative change {rel:.4f}")
+ tensors[key] = merged.to(base.dtype)
+ applied += 1
+ save_file(tensors, str(transformer_out / shard.name), metadata={"format": "pt"})
+ print(f"merged {applied}/{len(deltas)} tensors")
+ if applied != len(deltas):
+ raise SystemExit(
+ f"{len(deltas) - applied} deltas found no target; do not use this output"
+ )
+
+ for extra in (
+ "config.json",
+ "model.safetensors.index.json",
+ "diffusion_pytorch_model.safetensors.index.json",
+ ):
+ src = args.base_dir / extra
+ if src.exists():
+ shutil.copy(src, transformer_out / extra)
+
+ heads = {
+ k: lora[k]
+ for k in (
+ "proj_out.weight",
+ "proj_out.bias",
+ "audio_proj_out.weight",
+ "audio_proj_out.bias",
+ )
+ }
+ save_file(heads, str(out / "pdd_heads.safetensors"), metadata={"format": "pt"})
+ (out / "pdd_config.json").write_text(json.dumps(pdd, indent=2) + "\n")
+ print(
+ f"position-level output heads -> {out / 'pdd_heads.safetensors'} "
+ f"({pdd['num_steps']} of them, block size {pdd['block_size']}, "
+ f"i.e. {pdd['num_steps'] // pdd['block_size']} forwards)"
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/python/sglang/multimodal_gen/tools/fuse_minimax_h3_pdd_heads.py b/python/sglang/multimodal_gen/tools/fuse_minimax_h3_pdd_heads.py
new file mode 100755
index 0000000000..91ad5d96b8
--- /dev/null
+++ b/python/sglang/multimodal_gen/tools/fuse_minimax_h3_pdd_heads.py
@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""Fuse PDD's 32 position-level output heads into one per block, i.e. 8.
+
+ python3 -m sglang.multimodal_gen.tools.fuse_minimax_h3_pdd_heads \
+ [--video-shift 12.0 --audio-shift 3.0]
+
+## Why the fusion is exact
+
+MiniMax-H3's scheduler has "ancestral" in its name but takes the eta0 branch, which
+injects no noise:
+
+ x_next = r*x + (1-r)*x0, r = sigma_next/sigma_curr, x0 = x - sigma_curr*v
+ = x - (sigma_curr - sigma_next)*v
+
+That is plain Euler, and linear in v. The L heads in one PDD block share a single
+backbone evaluation, so
+
+ x_{n+L} = x_n - sum_j dsigma_{n+j} * W_{n+j} h
+ = x_n - (sum_j dsigma_{n+j}) * (sum_j w_j W_{n+j}) h, w_j = dsigma_{n+j} / sum dsigma
+
+and the bracket is the fused head -- the paper's "fuse the layers into a single
+linear layer that directly predicts a step across the full block". Each block then
+costs one forward and one scheduler step, and the denoise loop needs no changes.
+
+This depends on eta0. Under true ancestral sampling (noise injected every step) the
+state at the block's second step is no longer the block's first state, the sum
+cannot be moved onto W, and the fusion is invalid -- the L sub-steps have to be run.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+import torch
+from safetensors import safe_open
+from safetensors.torch import save_file
+
+
+def sigma_grid(num_points: int, shift: float) -> torch.Tensor:
+ base = torch.linspace(1.0, 0.0, num_points, dtype=torch.float64)
+ return shift * base / (1 + (shift - 1) * base)
+
+
+def fuse(heads: torch.Tensor, sigmas: torch.Tensor, block: int) -> torch.Tensor:
+ """heads: (N, ...) -> (N//block, ...), averaged within a block weighted by dsigma."""
+ steps = heads.shape[0]
+ if block <= 0 or steps == 0 or steps % block:
+ raise ValueError("PDD head count must be positive and divisible by block size")
+ if sigmas.numel() != steps + 1:
+ raise SystemExit(
+ f"sigma grid has {sigmas.numel()} points but there are {steps} heads"
+ )
+ dsigma = (sigmas[:-1] - sigmas[1:]).clamp(min=0)
+ out = []
+ for b in range(steps // block):
+ w = dsigma[b * block : (b + 1) * block]
+ if float(w.sum()) <= 0:
+ raise SystemExit(f"block {b} has an all-zero dsigma")
+ w = (w / w.sum()).to(heads.dtype)
+ chunk = heads[b * block : (b + 1) * block].float()
+ out.append((chunk * w.float().view(-1, *([1] * (chunk.dim() - 1)))).sum(0))
+ return torch.stack(out).to(heads.dtype)
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+ )
+ ap.add_argument("pdd_dir")
+ ap.add_argument("--video-shift", type=float, default=12.0)
+ ap.add_argument("--audio-shift", type=float, default=3.0)
+ args = ap.parse_args()
+
+ d = Path(args.pdd_dir)
+ cfg = json.loads((d / "pdd_config.json").read_text())
+ n, block = cfg["num_steps"], cfg["block_size"]
+ with safe_open(str(d / "pdd_heads.safetensors"), "pt") as f:
+ heads = {k: f.get_tensor(k) for k in f.keys()}
+
+ sv = sigma_grid(n + 1, args.video_shift)
+ sa = sigma_grid(n + 1, args.audio_shift)
+ fused = {
+ "video_out.weight": fuse(heads["proj_out.weight"], sv, block),
+ "video_out.bias": fuse(heads["proj_out.bias"], sv, block),
+ "audio_out.weight": fuse(heads["audio_proj_out.weight"], sa, block),
+ "audio_out.bias": fuse(heads["audio_proj_out.bias"], sa, block),
+ }
+ save_file(fused, str(d / "pdd_fused_heads.safetensors"), metadata={"format": "pt"})
+ cfg["fused_steps"] = n // block
+ cfg["num_inference_steps"] = n // block + 1 # H3 counts steps as sigma grid points
+ cfg["video_shift"], cfg["audio_shift"] = args.video_shift, args.audio_shift
+ (d / "pdd_config.json").write_text(json.dumps(cfg, indent=2) + "\n")
+
+ print(f"fused {n} -> {n // block} heads (block size {block})")
+ for k, v in fused.items():
+ print(f" {k}: {tuple(v.shape)}")
+ print(f"run with --num-inference-steps {cfg['num_inference_steps']}")
+ print(
+ "first 9 points of the video sigma grid:",
+ [round(float(x), 4) for x in sv[::block]],
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/test/registered/unit/diffusion/test_minimax_h3_pdd_offline.py b/test/registered/unit/diffusion/test_minimax_h3_pdd_offline.py
new file mode 100644
index 0000000000..0f016e9d48
--- /dev/null
+++ b/test/registered/unit/diffusion/test_minimax_h3_pdd_offline.py
@@ -0,0 +1,171 @@
+# SPDX-License-Identifier: Apache-2.0
+import json
+import tempfile
+import unittest
+from functools import partial
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import torch
+from safetensors.torch import load_file, save_file
+
+from sglang.multimodal_gen.configs.models.dits.minimax_h3 import MiniMaxH3DiTArchConfig
+from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
+from sglang.multimodal_gen.runtime.models.dits.minimax_h3 import MiniMaxH3FinalLayer
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.constants import (
+ MINIMAX_H3_SIGMAS_EXTRA_KEY,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.timestep_preparation import (
+ MiniMaxH3TimestepPreparationStage,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import (
+ minimax_h3_time_shift_sigmas,
+)
+from sglang.multimodal_gen.tools import build_minimax_h3_pdd_weights as build
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cuda_ci(est_time=12, stage="base-b", runner_config="diffusion-unit-1-gpu-h100")
+
+
+def _make_heads() -> dict[str, torch.Tensor]:
+ heads = {}
+ for name, width in (("video_out", 6), ("audio_out", 4)):
+ heads[f"{name}.weight"] = (
+ torch.arange(8 * width * 5, dtype=torch.float32).reshape(8, width, 5) / 100
+ )
+ heads[f"{name}.bias"] = (
+ torch.arange(8 * width, dtype=torch.float32).reshape(8, width) / 100
+ )
+ return heads
+
+
+class TestMiniMaxH3PDDOffline(CustomTestCase):
+ def test_tp_projection_matches_full_head(self):
+ with tempfile.TemporaryDirectory() as temp:
+ root = Path(temp)
+ heads = _make_heads()
+ path = root / "heads.safetensors"
+ save_file(heads, str(path))
+ arch = MiniMaxH3DiTArchConfig(
+ hidden_size=5, latents_dim=6, audio_latents_dim=4, patch_size=(1, 1, 1)
+ )
+ layers = []
+ for rank in range(2):
+ with patch(
+ "sglang.multimodal_gen.runtime.layers.linear.get_tp_group",
+ return_value=SimpleNamespace(world_size=2, rank_in_group=rank),
+ ):
+ layer = MiniMaxH3FinalLayer(
+ arch, None, prefix="final_layer", use_adaln_cache=True
+ )
+ layer.load_pdd_fused_heads(str(path))
+ layers.append(layer)
+
+ h = torch.arange(15, dtype=torch.float32).reshape(3, 5)
+ for step in (0, 7):
+ with self.subTest(step=step), set_forward_context(step, None):
+ outputs = [layer._project(h) for layer in layers]
+ for i, name in enumerate(("video_out", "audio_out")):
+ expected = torch.nn.functional.linear(
+ h,
+ heads[f"{name}.weight"][step],
+ heads[f"{name}.bias"][step],
+ )
+ torch.testing.assert_close(
+ torch.cat([output[i] for output in outputs], dim=-1),
+ expected,
+ )
+
+ def test_schedule_validation_and_warmup(self):
+ apply_schedule = partial(
+ MiniMaxH3TimestepPreparationStage._apply_pdd_schedule,
+ SimpleNamespace(
+ _pdd_config={
+ "num_inference_steps": 9,
+ "video_shift": 12.0,
+ "audio_shift": 3.0,
+ }
+ ),
+ )
+
+ sigmas = {
+ name: minimax_h3_time_shift_sigmas(num_steps=9, shift_scale=shift)
+ for name, shift in (("video", 12.0), ("audio", 3.0))
+ }
+ batch = SimpleNamespace(
+ is_warmup=False, extra={MINIMAX_H3_SIGMAS_EXTRA_KEY: sigmas}
+ )
+ apply_schedule(batch)
+ wrong_shift = minimax_h3_time_shift_sigmas(num_steps=9, shift_scale=6.0)
+ for name, invalid in (
+ ("video", sigmas["video"][:-1]),
+ ("video", sigmas["video"] + [0.0]),
+ ("video", wrong_shift),
+ ("audio", wrong_shift),
+ ):
+ batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY] = {**sigmas, name: invalid}
+ with (
+ self.subTest(modality=name, sigmas=invalid),
+ self.assertRaisesRegex(ValueError, "match the fused heads"),
+ ):
+ apply_schedule(batch)
+ batch.is_warmup = True
+ batch.num_inference_steps = 3
+ apply_schedule(batch)
+ self.assertEqual(
+ batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY],
+ {name: values[:3] for name, values in sigmas.items()},
+ )
+
+ def test_conversion_keeps_heads_outside_transformer(self):
+ with tempfile.TemporaryDirectory() as temp:
+ root = Path(temp)
+ heads = _make_heads()
+ base = root / "base"
+ base.mkdir()
+ weight_name = "blocks.0.attn.out_proj.weight"
+ save_file({weight_name: torch.zeros(2, 3)}, str(base / "model.safetensors"))
+ (base / "config.json").write_text("{}")
+ index = {"weight_map": {weight_name: "model.safetensors"}}
+ (base / "model.safetensors.index.json").write_text(json.dumps(index))
+ lora = {
+ "transformer_blocks.0.attn.to_out.0.lora_down": torch.ones(1, 3),
+ "transformer_blocks.0.attn.to_out.0.lora_up": torch.ones(2, 1),
+ }
+ for key, value in heads.items():
+ lora[
+ key.replace("video_out", "proj_out").replace(
+ "audio_out", "audio_proj_out"
+ )
+ ] = value
+ lora_path = root / "lora.safetensors"
+ save_file(
+ lora,
+ str(lora_path),
+ metadata={
+ "lora_rank": "1",
+ "pdd_num_steps": "8",
+ "pdd_block_size": "2",
+ },
+ )
+ out = root / "out"
+ build.main([str(base), str(lora_path), str(out)])
+ transformer = out / "transformer"
+ self.assertEqual(
+ list(transformer.glob("*.safetensors")),
+ [transformer / "model.safetensors"],
+ )
+ torch.testing.assert_close(
+ load_file(str(transformer / "model.safetensors"))[weight_name],
+ torch.ones(2, 3),
+ )
+ self.assertEqual(
+ json.loads((transformer / "model.safetensors.index.json").read_text()),
+ index,
+ )
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=3)