[Diffusion] Support MiniMax-H3 PDD(Parallel Decoding Distillation) inference (#40568)

Co-authored-by: Forrest Lin <forrestl@nvidia.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
HuangJi
2026-09-23 12:57:26 +08:00
committed by GitHub
co-authored by Forrest Lin Claude Opus 5
parent 1d59ce7c90
commit 973fb44471
7 changed files with 627 additions and 6 deletions
+7
View File
@@ -47,6 +47,7 @@ if TYPE_CHECKING:
SGLANG_DIFFUSION_DISABLE_AUTO_RESIDENCY: bool = False
SGLANG_DIFFUSION_MINIMAX_H3_ADALN_GPU_PLANS: int = 64
SGLANG_DIFFUSION_MINIMAX_H3_ADALN_FP32: bool = False
SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS: str | None = None
SGLANG_DIFFUSION_CFG_GATE_STEP: float = 1.0
# cache-dit env vars (primary transformer)
# on by default; engages only on 2 ranks with peer-to-peer access and falls
@@ -327,6 +328,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
"SGLANG_DIFFUSION_CFG_GATE_STEP": _lazy_float(
"SGLANG_DIFFUSION_CFG_GATE_STEP", 1.0
),
# Path to a Parallel Decoding Distillation head stack (one fused output head
# per denoise step, produced by fuse_minimax_h3_pdd_heads.py). Set only when serving a
# PDD-distilled checkpoint; an ordinary run leaves the projection alone.
"SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS": _lazy_str(
"SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS"
),
"SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D": _lazy_str(
"SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D", "auto"
),
@@ -67,7 +67,9 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor
)
from sglang.multimodal_gen.runtime.layers.usp import _ring_attention_varlen
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
from sglang.multimodal_gen.runtime.managers.forward_context import get_forward_context
from sglang.multimodal_gen.runtime.managers.forward_context import (
get_forward_context,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
is_layerwise_offloaded_module,
@@ -1526,6 +1528,58 @@ class MiniMaxH3FinalLayer(nn.Module):
quant_config=None,
prefix=f"{prefix}.audio_out",
)
# Parallel Decoding Distillation heads, one per denoise step, loaded by
# `load_pdd_fused_heads`. None on every ordinary run.
self._pdd_heads: dict[str, torch.Tensor] | None = None
def load_pdd_fused_heads(self, path: str) -> None:
"""Swap the two output heads for a per-step stack (PDD).
PDD replicates the final linear layer once per time interval and lets one
backbone evaluation advance a whole block of them; the per-block heads
fuse into one because H3's euler-eta0 step is linear in the predicted
velocity. `tools/fuse_minimax_h3_pdd_heads.py` does that fusion offline,
leaving one head per denoise step.
"""
from safetensors import safe_open
with safe_open(path, "pt") as f:
heads = {k: f.get_tensor(k) for k in f.keys()}
steps = heads["video_out.weight"].shape[0]
for name in ("video_out", "audio_out"):
projection = getattr(self, name)
for suffix, shape in (
("weight", (steps, projection.output_size, projection.input_size)),
("bias", (steps, projection.output_size)),
):
key = f"{name}.{suffix}"
if steps == 0 or heads[key].shape != shape:
raise ValueError(f"MiniMax-H3 PDD {key} must have shape {shape}")
width = projection.output_size_per_partition
heads[key] = heads[key].narrow(1, projection.tp_rank * width, width)
self._pdd_heads = heads
logger.info("MiniMax-H3 PDD: %d fused output heads loaded from %s", steps, path)
def _pdd_project(self, h: torch.Tensor, name: str) -> torch.Tensor:
heads = self._pdd_heads
step = int(get_forward_context().current_timestep)
stack = heads[f"{name}.weight"]
if not 0 <= step < stack.shape[0]:
raise ValueError(
f"MiniMax-H3 PDD has {stack.shape[0]} fused heads but the loop is at "
f"step {step}; run with --num-inference-steps {stack.shape[0] + 1} "
"(H3 counts sigma grid points, so that is one more than the steps)."
)
weight = stack[step].to(device=h.device, dtype=h.dtype)
bias = heads[f"{name}.bias"][step].to(device=h.device, dtype=h.dtype)
return torch.nn.functional.linear(h, weight, bias)
def _project(self, h: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
if self._pdd_heads is not None:
return self._pdd_project(h, "video_out"), self._pdd_project(h, "audio_out")
video, _ = self.video_out(h)
audio, _ = self.audio_out(h)
return video, audio
def forward(
self,
@@ -1559,8 +1613,7 @@ class MiniMaxH3FinalLayer(nn.Module):
inverse_indices[start:stop],
dtype=_BF16_DTYPE,
).to(_FP32_DTYPE)
video_chunk, _ = self.video_out(h)
audio_chunk, _ = self.audio_out(h)
video_chunk, audio_chunk = self._project(h)
if video is None:
video = torch.empty(
(x.shape[0], video_chunk.shape[-1]),
@@ -1583,9 +1636,7 @@ class MiniMaxH3FinalLayer(nn.Module):
h = _modulate_scale_shift(h, shift, scale, inverse_indices, dtype=_BF16_DTYPE)
# Preserve full precision through both final output projections.
h = h.to(_FP32_DTYPE)
video, _ = self.video_out(h)
audio, _ = self.audio_out(h)
return video, audio
return self._project(h)
def _reject_adaln_lora(names: list[str]) -> None:
@@ -1985,6 +2036,9 @@ class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin):
prefix="final_layer",
use_adaln_cache=self._adaln_precomputed,
)
pdd_heads = envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS
if pdd_heads:
self.final_layer.load_pdd_fused_heads(pdd_heads)
self.adaln_cache = (
MiniMaxH3AdalnCache(
arch,
@@ -1,11 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import json
import math
from collections.abc import Mapping
from pathlib import Path
import torch
from sglang.multimodal_gen import envs
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
@@ -29,6 +32,18 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
# block, sigma_shift_scales): the schedule constants are a MODEL
# serving contract — fl2va and ref2va use video 12 / audio 3 by default.
self.sigma_shift_scales = sigma_shift_scales
self._pdd_config = None
pdd_heads = envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS
if pdd_heads:
from safetensors import safe_open
self._pdd_config = json.loads(
Path(pdd_heads).with_name("pdd_config.json").read_text()
)
with safe_open(pdd_heads, "pt") as f:
steps = f.get_slice("video_out.weight").get_shape()[0]
if self._pdd_config["num_inference_steps"] != steps + 1:
raise ValueError("MiniMax-H3 PDD config does not match the fused heads")
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.resolved_plan import (
@@ -42,6 +57,7 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
"and has no implementation yet."
)
self._generate_sigmas_from_plan(batch, plan)
self._apply_pdd_schedule(batch)
self._publish_native_timestep_state(batch)
return batch
@@ -58,6 +74,7 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
)
return (
batch.num_inference_steps,
batch.is_warmup,
plan.flow_shift,
plan.audio_flow_shift,
plan.default_flow_shift,
@@ -65,6 +82,36 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
self.freeze_for_dedup(self.sigma_shift_scales),
)
def _apply_pdd_schedule(self, batch: Req) -> None:
if self._pdd_config is None:
return
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import (
minimax_h3_time_shift_sigmas,
)
config = self._pdd_config
sigmas = batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY]
for modality in ("video", "audio"):
expected = minimax_h3_time_shift_sigmas(
num_steps=config["num_inference_steps"],
shift_scale=config[f"{modality}_shift"],
)
if batch.is_warmup:
# Warmup may run fewer steps, but must use the same intervals
# as serving rather than rescaling a shorter grid to [1, 0].
sigmas[modality] = expected[: max(2, batch.num_inference_steps)]
continue
actual = sigmas[modality]
if len(actual) != len(expected) or any(
not math.isclose(a, b, rel_tol=1e-6, abs_tol=1e-7)
for a, b in zip(actual, expected)
):
raise ValueError(
f"MiniMax-H3 PDD requires num_inference_steps="
f"{config['num_inference_steps']} and {modality} shift="
f"{config[f'{modality}_shift']} to match the fused heads"
)
@staticmethod
def _publish_native_timestep_state(batch: Req) -> None:
sigmas = batch.extra.get(MINIMAX_H3_SIGMAS_EXTRA_KEY)
@@ -0,0 +1,167 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
"""Merge alibaba-pai's PDD LoRA into a native MiniMax-H3 checkpoint.
python3 -m sglang.multimodal_gen.tools.build_minimax_h3_pdd_weights \
<base transformer dir> <lora.safetensors> <out dir>
Writes merged weights to `transformer/` and interval-specific output heads to
`pdd_heads.safetensors`, alongside `pdd_config.json`. The heads must stay outside
`transformer/` so the model loader does not treat them as checkpoint shards.
Run `fuse_minimax_h3_pdd_heads.py` on the output directory before serving.
Serve with `--transformer-weights-path <out dir>/transformer` and set
`SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS=<out dir>/pdd_fused_heads.safetensors`.
Keep `pdd_config.json` beside the fused heads for schedule validation.
"""
from __future__ import annotations
import argparse
import json
import shutil
from pathlib import Path
import torch
from safetensors import safe_open
from safetensors.torch import load_file, save_file
HEAD_DIM = 128
def _interleave_qkv(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
"""[3 x (heads*head_dim), in] -> the official per-head interleaved layout."""
in_dim = q.shape[1]
return torch.stack(
[x.reshape(-1, HEAD_DIM, in_dim) for x in (q, k, v)], dim=1
).reshape(-1, in_dim)
def _target_of(lora_key: str) -> str | None:
"""LoRA module name -> official weight name. None means "not handled here"."""
k = lora_key
if k.startswith("token_refiner.refiner_blocks."):
k = "token_refiner.blocks." + k[len("token_refiner.refiner_blocks.") :]
elif k.startswith("transformer_blocks."):
k = "blocks." + k[len("transformer_blocks.") :]
else:
return None
for src, dst in (
(".attn.to_out.0", ".attn.out_proj"),
(".ff.net.0.proj", ".mlp.fc1"),
(".ff.net.2", ".mlp.fc2"),
):
if k.endswith(src):
return k[: -len(src)] + dst
return k # attn.to_q/k/v pass through; the caller interleaves them
def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
ap.add_argument("base_dir", type=Path)
ap.add_argument("lora_path")
ap.add_argument("out_dir", type=Path)
ap.add_argument(
"--verify",
action="store_true",
help="print the relative magnitude of each merged weight change",
)
args = ap.parse_args(argv)
with safe_open(args.lora_path, "pt") as f:
meta = f.metadata() or {}
lora = {k: f.get_tensor(k) for k in f.keys()}
rank = int(meta.get("lora_rank", 64))
alpha = float(meta.get("lora_alpha", rank))
scale = alpha / rank
pdd = {
"num_steps": int(meta.get("pdd_num_steps", 32)),
"block_size": int(meta.get("pdd_block_size", 4)),
}
print(f"LoRA: rank={rank} alpha={alpha} scale={scale} pdd={pdd}")
# Group lora_down/lora_up per module; q/k/v are held back and interleaved at the end.
pairs: dict[str, dict[str, torch.Tensor]] = {}
for key, value in lora.items():
for suffix in (".lora_down", ".lora_up"):
if key.endswith(suffix):
pairs.setdefault(key[: -len(suffix)], {})[suffix[1:]] = value
break
deltas: dict[str, torch.Tensor] = {} # official weight name -> delta W
qkv: dict[str, dict[str, torch.Tensor]] = {} # module prefix -> {q,k,v: delta W}
for module, dv in pairs.items():
if "lora_down" not in dv or "lora_up" not in dv:
raise SystemExit(f"{module} has only one half of the LoRA pair")
delta = (dv["lora_up"].float() @ dv["lora_down"].float()) * scale
target = _target_of(module)
if target is None:
raise SystemExit(f"no mapping rule for {module}")
for role in ("to_q", "to_k", "to_v"):
if target.endswith(f".attn.{role}"):
qkv.setdefault(target[: -len(f".attn.{role}")], {})[role[-1]] = delta
break
else:
deltas[target + ".weight"] = delta
for prefix, parts in qkv.items():
if set(parts) != {"q", "k", "v"}:
raise SystemExit(f"{prefix} is missing part of q/k/v: {sorted(parts)}")
deltas[prefix + ".attn.qkv_proj.weight"] = _interleave_qkv(
parts["q"], parts["k"], parts["v"]
)
print(f"2D deltas to merge: {len(deltas)} ({len(qkv)} of them qkv)")
out = args.out_dir
transformer_out = out / "transformer"
transformer_out.mkdir(parents=True, exist_ok=True)
applied = 0
for shard in sorted(args.base_dir.glob("*.safetensors")):
tensors = load_file(str(shard))
for key, base in tensors.items():
if key in deltas:
merged = base.float() + deltas[key]
if args.verify:
rel = ((merged - base.float()).norm() / base.float().norm()).item()
print(f" {key}: relative change {rel:.4f}")
tensors[key] = merged.to(base.dtype)
applied += 1
save_file(tensors, str(transformer_out / shard.name), metadata={"format": "pt"})
print(f"merged {applied}/{len(deltas)} tensors")
if applied != len(deltas):
raise SystemExit(
f"{len(deltas) - applied} deltas found no target; do not use this output"
)
for extra in (
"config.json",
"model.safetensors.index.json",
"diffusion_pytorch_model.safetensors.index.json",
):
src = args.base_dir / extra
if src.exists():
shutil.copy(src, transformer_out / extra)
heads = {
k: lora[k]
for k in (
"proj_out.weight",
"proj_out.bias",
"audio_proj_out.weight",
"audio_proj_out.bias",
)
}
save_file(heads, str(out / "pdd_heads.safetensors"), metadata={"format": "pt"})
(out / "pdd_config.json").write_text(json.dumps(pdd, indent=2) + "\n")
print(
f"position-level output heads -> {out / 'pdd_heads.safetensors'} "
f"({pdd['num_steps']} of them, block size {pdd['block_size']}, "
f"i.e. {pdd['num_steps'] // pdd['block_size']} forwards)"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,109 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
"""Fuse PDD's 32 position-level output heads into one per block, i.e. 8.
python3 -m sglang.multimodal_gen.tools.fuse_minimax_h3_pdd_heads <pdd dir> \
[--video-shift 12.0 --audio-shift 3.0]
## Why the fusion is exact
MiniMax-H3's scheduler has "ancestral" in its name but takes the eta0 branch, which
injects no noise:
x_next = r*x + (1-r)*x0, r = sigma_next/sigma_curr, x0 = x - sigma_curr*v
= x - (sigma_curr - sigma_next)*v
That is plain Euler, and linear in v. The L heads in one PDD block share a single
backbone evaluation, so
x_{n+L} = x_n - sum_j dsigma_{n+j} * W_{n+j} h
= x_n - (sum_j dsigma_{n+j}) * (sum_j w_j W_{n+j}) h, w_j = dsigma_{n+j} / sum dsigma
and the bracket is the fused head -- the paper's "fuse the layers into a single
linear layer that directly predicts a step across the full block". Each block then
costs one forward and one scheduler step, and the denoise loop needs no changes.
This depends on eta0. Under true ancestral sampling (noise injected every step) the
state at the block's second step is no longer the block's first state, the sum
cannot be moved onto W, and the fusion is invalid -- the L sub-steps have to be run.
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import torch
from safetensors import safe_open
from safetensors.torch import save_file
def sigma_grid(num_points: int, shift: float) -> torch.Tensor:
base = torch.linspace(1.0, 0.0, num_points, dtype=torch.float64)
return shift * base / (1 + (shift - 1) * base)
def fuse(heads: torch.Tensor, sigmas: torch.Tensor, block: int) -> torch.Tensor:
"""heads: (N, ...) -> (N//block, ...), averaged within a block weighted by dsigma."""
steps = heads.shape[0]
if block <= 0 or steps == 0 or steps % block:
raise ValueError("PDD head count must be positive and divisible by block size")
if sigmas.numel() != steps + 1:
raise SystemExit(
f"sigma grid has {sigmas.numel()} points but there are {steps} heads"
)
dsigma = (sigmas[:-1] - sigmas[1:]).clamp(min=0)
out = []
for b in range(steps // block):
w = dsigma[b * block : (b + 1) * block]
if float(w.sum()) <= 0:
raise SystemExit(f"block {b} has an all-zero dsigma")
w = (w / w.sum()).to(heads.dtype)
chunk = heads[b * block : (b + 1) * block].float()
out.append((chunk * w.float().view(-1, *([1] * (chunk.dim() - 1)))).sum(0))
return torch.stack(out).to(heads.dtype)
def main() -> int:
ap = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
ap.add_argument("pdd_dir")
ap.add_argument("--video-shift", type=float, default=12.0)
ap.add_argument("--audio-shift", type=float, default=3.0)
args = ap.parse_args()
d = Path(args.pdd_dir)
cfg = json.loads((d / "pdd_config.json").read_text())
n, block = cfg["num_steps"], cfg["block_size"]
with safe_open(str(d / "pdd_heads.safetensors"), "pt") as f:
heads = {k: f.get_tensor(k) for k in f.keys()}
sv = sigma_grid(n + 1, args.video_shift)
sa = sigma_grid(n + 1, args.audio_shift)
fused = {
"video_out.weight": fuse(heads["proj_out.weight"], sv, block),
"video_out.bias": fuse(heads["proj_out.bias"], sv, block),
"audio_out.weight": fuse(heads["audio_proj_out.weight"], sa, block),
"audio_out.bias": fuse(heads["audio_proj_out.bias"], sa, block),
}
save_file(fused, str(d / "pdd_fused_heads.safetensors"), metadata={"format": "pt"})
cfg["fused_steps"] = n // block
cfg["num_inference_steps"] = n // block + 1 # H3 counts steps as sigma grid points
cfg["video_shift"], cfg["audio_shift"] = args.video_shift, args.audio_shift
(d / "pdd_config.json").write_text(json.dumps(cfg, indent=2) + "\n")
print(f"fused {n} -> {n // block} heads (block size {block})")
for k, v in fused.items():
print(f" {k}: {tuple(v.shape)}")
print(f"run with --num-inference-steps {cfg['num_inference_steps']}")
print(
"first 9 points of the video sigma grid:",
[round(float(x), 4) for x in sv[::block]],
)
return 0
if __name__ == "__main__":
raise SystemExit(main())