[Diffusion] Support MiniMax-H3 PDD(Parallel Decoding Distillation) inference (#40568)
Co-authored-by: Forrest Lin <forrestl@nvidia.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Forrest Lin
Claude Opus 5
parent
1d59ce7c90
commit
973fb44471
@@ -47,6 +47,7 @@ if TYPE_CHECKING:
|
||||
SGLANG_DIFFUSION_DISABLE_AUTO_RESIDENCY: bool = False
|
||||
SGLANG_DIFFUSION_MINIMAX_H3_ADALN_GPU_PLANS: int = 64
|
||||
SGLANG_DIFFUSION_MINIMAX_H3_ADALN_FP32: bool = False
|
||||
SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS: str | None = None
|
||||
SGLANG_DIFFUSION_CFG_GATE_STEP: float = 1.0
|
||||
# cache-dit env vars (primary transformer)
|
||||
# on by default; engages only on 2 ranks with peer-to-peer access and falls
|
||||
@@ -327,6 +328,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"SGLANG_DIFFUSION_CFG_GATE_STEP": _lazy_float(
|
||||
"SGLANG_DIFFUSION_CFG_GATE_STEP", 1.0
|
||||
),
|
||||
# Path to a Parallel Decoding Distillation head stack (one fused output head
|
||||
# per denoise step, produced by fuse_minimax_h3_pdd_heads.py). Set only when serving a
|
||||
# PDD-distilled checkpoint; an ordinary run leaves the projection alone.
|
||||
"SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS": _lazy_str(
|
||||
"SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS"
|
||||
),
|
||||
"SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D": _lazy_str(
|
||||
"SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D", "auto"
|
||||
),
|
||||
|
||||
@@ -67,7 +67,9 @@ from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config impor
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.usp import _ring_attention_varlen
|
||||
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import get_forward_context
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import (
|
||||
get_forward_context,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
is_layerwise_offloaded_module,
|
||||
@@ -1526,6 +1528,58 @@ class MiniMaxH3FinalLayer(nn.Module):
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.audio_out",
|
||||
)
|
||||
# Parallel Decoding Distillation heads, one per denoise step, loaded by
|
||||
# `load_pdd_fused_heads`. None on every ordinary run.
|
||||
self._pdd_heads: dict[str, torch.Tensor] | None = None
|
||||
|
||||
def load_pdd_fused_heads(self, path: str) -> None:
|
||||
"""Swap the two output heads for a per-step stack (PDD).
|
||||
|
||||
PDD replicates the final linear layer once per time interval and lets one
|
||||
backbone evaluation advance a whole block of them; the per-block heads
|
||||
fuse into one because H3's euler-eta0 step is linear in the predicted
|
||||
velocity. `tools/fuse_minimax_h3_pdd_heads.py` does that fusion offline,
|
||||
leaving one head per denoise step.
|
||||
"""
|
||||
from safetensors import safe_open
|
||||
|
||||
with safe_open(path, "pt") as f:
|
||||
heads = {k: f.get_tensor(k) for k in f.keys()}
|
||||
steps = heads["video_out.weight"].shape[0]
|
||||
for name in ("video_out", "audio_out"):
|
||||
projection = getattr(self, name)
|
||||
for suffix, shape in (
|
||||
("weight", (steps, projection.output_size, projection.input_size)),
|
||||
("bias", (steps, projection.output_size)),
|
||||
):
|
||||
key = f"{name}.{suffix}"
|
||||
if steps == 0 or heads[key].shape != shape:
|
||||
raise ValueError(f"MiniMax-H3 PDD {key} must have shape {shape}")
|
||||
width = projection.output_size_per_partition
|
||||
heads[key] = heads[key].narrow(1, projection.tp_rank * width, width)
|
||||
self._pdd_heads = heads
|
||||
logger.info("MiniMax-H3 PDD: %d fused output heads loaded from %s", steps, path)
|
||||
|
||||
def _pdd_project(self, h: torch.Tensor, name: str) -> torch.Tensor:
|
||||
heads = self._pdd_heads
|
||||
step = int(get_forward_context().current_timestep)
|
||||
stack = heads[f"{name}.weight"]
|
||||
if not 0 <= step < stack.shape[0]:
|
||||
raise ValueError(
|
||||
f"MiniMax-H3 PDD has {stack.shape[0]} fused heads but the loop is at "
|
||||
f"step {step}; run with --num-inference-steps {stack.shape[0] + 1} "
|
||||
"(H3 counts sigma grid points, so that is one more than the steps)."
|
||||
)
|
||||
weight = stack[step].to(device=h.device, dtype=h.dtype)
|
||||
bias = heads[f"{name}.bias"][step].to(device=h.device, dtype=h.dtype)
|
||||
return torch.nn.functional.linear(h, weight, bias)
|
||||
|
||||
def _project(self, h: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if self._pdd_heads is not None:
|
||||
return self._pdd_project(h, "video_out"), self._pdd_project(h, "audio_out")
|
||||
video, _ = self.video_out(h)
|
||||
audio, _ = self.audio_out(h)
|
||||
return video, audio
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -1559,8 +1613,7 @@ class MiniMaxH3FinalLayer(nn.Module):
|
||||
inverse_indices[start:stop],
|
||||
dtype=_BF16_DTYPE,
|
||||
).to(_FP32_DTYPE)
|
||||
video_chunk, _ = self.video_out(h)
|
||||
audio_chunk, _ = self.audio_out(h)
|
||||
video_chunk, audio_chunk = self._project(h)
|
||||
if video is None:
|
||||
video = torch.empty(
|
||||
(x.shape[0], video_chunk.shape[-1]),
|
||||
@@ -1583,9 +1636,7 @@ class MiniMaxH3FinalLayer(nn.Module):
|
||||
h = _modulate_scale_shift(h, shift, scale, inverse_indices, dtype=_BF16_DTYPE)
|
||||
# Preserve full precision through both final output projections.
|
||||
h = h.to(_FP32_DTYPE)
|
||||
video, _ = self.video_out(h)
|
||||
audio, _ = self.audio_out(h)
|
||||
return video, audio
|
||||
return self._project(h)
|
||||
|
||||
|
||||
def _reject_adaln_lora(names: list[str]) -> None:
|
||||
@@ -1985,6 +2036,9 @@ class MiniMaxH3DiTModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
prefix="final_layer",
|
||||
use_adaln_cache=self._adaln_precomputed,
|
||||
)
|
||||
pdd_heads = envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS
|
||||
if pdd_heads:
|
||||
self.final_layer.load_pdd_fused_heads(pdd_heads)
|
||||
self.adaln_cache = (
|
||||
MiniMaxH3AdalnCache(
|
||||
arch,
|
||||
|
||||
+47
@@ -1,11 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen import envs
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
@@ -29,6 +32,18 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
|
||||
# block, sigma_shift_scales): the schedule constants are a MODEL
|
||||
# serving contract — fl2va and ref2va use video 12 / audio 3 by default.
|
||||
self.sigma_shift_scales = sigma_shift_scales
|
||||
self._pdd_config = None
|
||||
pdd_heads = envs.SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS
|
||||
if pdd_heads:
|
||||
from safetensors import safe_open
|
||||
|
||||
self._pdd_config = json.loads(
|
||||
Path(pdd_heads).with_name("pdd_config.json").read_text()
|
||||
)
|
||||
with safe_open(pdd_heads, "pt") as f:
|
||||
steps = f.get_slice("video_out.weight").get_shape()[0]
|
||||
if self._pdd_config["num_inference_steps"] != steps + 1:
|
||||
raise ValueError("MiniMax-H3 PDD config does not match the fused heads")
|
||||
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.resolved_plan import (
|
||||
@@ -42,6 +57,7 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
|
||||
"and has no implementation yet."
|
||||
)
|
||||
self._generate_sigmas_from_plan(batch, plan)
|
||||
self._apply_pdd_schedule(batch)
|
||||
self._publish_native_timestep_state(batch)
|
||||
return batch
|
||||
|
||||
@@ -58,6 +74,7 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
|
||||
)
|
||||
return (
|
||||
batch.num_inference_steps,
|
||||
batch.is_warmup,
|
||||
plan.flow_shift,
|
||||
plan.audio_flow_shift,
|
||||
plan.default_flow_shift,
|
||||
@@ -65,6 +82,36 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
|
||||
self.freeze_for_dedup(self.sigma_shift_scales),
|
||||
)
|
||||
|
||||
def _apply_pdd_schedule(self, batch: Req) -> None:
|
||||
if self._pdd_config is None:
|
||||
return
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import (
|
||||
minimax_h3_time_shift_sigmas,
|
||||
)
|
||||
|
||||
config = self._pdd_config
|
||||
sigmas = batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY]
|
||||
for modality in ("video", "audio"):
|
||||
expected = minimax_h3_time_shift_sigmas(
|
||||
num_steps=config["num_inference_steps"],
|
||||
shift_scale=config[f"{modality}_shift"],
|
||||
)
|
||||
if batch.is_warmup:
|
||||
# Warmup may run fewer steps, but must use the same intervals
|
||||
# as serving rather than rescaling a shorter grid to [1, 0].
|
||||
sigmas[modality] = expected[: max(2, batch.num_inference_steps)]
|
||||
continue
|
||||
actual = sigmas[modality]
|
||||
if len(actual) != len(expected) or any(
|
||||
not math.isclose(a, b, rel_tol=1e-6, abs_tol=1e-7)
|
||||
for a, b in zip(actual, expected)
|
||||
):
|
||||
raise ValueError(
|
||||
f"MiniMax-H3 PDD requires num_inference_steps="
|
||||
f"{config['num_inference_steps']} and {modality} shift="
|
||||
f"{config[f'{modality}_shift']} to match the fused heads"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _publish_native_timestep_state(batch: Req) -> None:
|
||||
sigmas = batch.extra.get(MINIMAX_H3_SIGMAS_EXTRA_KEY)
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Merge alibaba-pai's PDD LoRA into a native MiniMax-H3 checkpoint.
|
||||
|
||||
python3 -m sglang.multimodal_gen.tools.build_minimax_h3_pdd_weights \
|
||||
<base transformer dir> <lora.safetensors> <out dir>
|
||||
|
||||
Writes merged weights to `transformer/` and interval-specific output heads to
|
||||
`pdd_heads.safetensors`, alongside `pdd_config.json`. The heads must stay outside
|
||||
`transformer/` so the model loader does not treat them as checkpoint shards.
|
||||
Run `fuse_minimax_h3_pdd_heads.py` on the output directory before serving.
|
||||
|
||||
Serve with `--transformer-weights-path <out dir>/transformer` and set
|
||||
`SGLANG_DIFFUSION_MINIMAX_H3_PDD_HEADS=<out dir>/pdd_fused_heads.safetensors`.
|
||||
Keep `pdd_config.json` beside the fused heads for schedule validation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from safetensors import safe_open
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
HEAD_DIM = 128
|
||||
|
||||
|
||||
def _interleave_qkv(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
|
||||
"""[3 x (heads*head_dim), in] -> the official per-head interleaved layout."""
|
||||
in_dim = q.shape[1]
|
||||
return torch.stack(
|
||||
[x.reshape(-1, HEAD_DIM, in_dim) for x in (q, k, v)], dim=1
|
||||
).reshape(-1, in_dim)
|
||||
|
||||
|
||||
def _target_of(lora_key: str) -> str | None:
|
||||
"""LoRA module name -> official weight name. None means "not handled here"."""
|
||||
k = lora_key
|
||||
if k.startswith("token_refiner.refiner_blocks."):
|
||||
k = "token_refiner.blocks." + k[len("token_refiner.refiner_blocks.") :]
|
||||
elif k.startswith("transformer_blocks."):
|
||||
k = "blocks." + k[len("transformer_blocks.") :]
|
||||
else:
|
||||
return None
|
||||
for src, dst in (
|
||||
(".attn.to_out.0", ".attn.out_proj"),
|
||||
(".ff.net.0.proj", ".mlp.fc1"),
|
||||
(".ff.net.2", ".mlp.fc2"),
|
||||
):
|
||||
if k.endswith(src):
|
||||
return k[: -len(src)] + dst
|
||||
return k # attn.to_q/k/v pass through; the caller interleaves them
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
ap = argparse.ArgumentParser(
|
||||
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
||||
)
|
||||
ap.add_argument("base_dir", type=Path)
|
||||
ap.add_argument("lora_path")
|
||||
ap.add_argument("out_dir", type=Path)
|
||||
ap.add_argument(
|
||||
"--verify",
|
||||
action="store_true",
|
||||
help="print the relative magnitude of each merged weight change",
|
||||
)
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
with safe_open(args.lora_path, "pt") as f:
|
||||
meta = f.metadata() or {}
|
||||
lora = {k: f.get_tensor(k) for k in f.keys()}
|
||||
rank = int(meta.get("lora_rank", 64))
|
||||
alpha = float(meta.get("lora_alpha", rank))
|
||||
scale = alpha / rank
|
||||
pdd = {
|
||||
"num_steps": int(meta.get("pdd_num_steps", 32)),
|
||||
"block_size": int(meta.get("pdd_block_size", 4)),
|
||||
}
|
||||
print(f"LoRA: rank={rank} alpha={alpha} scale={scale} pdd={pdd}")
|
||||
|
||||
# Group lora_down/lora_up per module; q/k/v are held back and interleaved at the end.
|
||||
pairs: dict[str, dict[str, torch.Tensor]] = {}
|
||||
for key, value in lora.items():
|
||||
for suffix in (".lora_down", ".lora_up"):
|
||||
if key.endswith(suffix):
|
||||
pairs.setdefault(key[: -len(suffix)], {})[suffix[1:]] = value
|
||||
break
|
||||
|
||||
deltas: dict[str, torch.Tensor] = {} # official weight name -> delta W
|
||||
qkv: dict[str, dict[str, torch.Tensor]] = {} # module prefix -> {q,k,v: delta W}
|
||||
for module, dv in pairs.items():
|
||||
if "lora_down" not in dv or "lora_up" not in dv:
|
||||
raise SystemExit(f"{module} has only one half of the LoRA pair")
|
||||
delta = (dv["lora_up"].float() @ dv["lora_down"].float()) * scale
|
||||
target = _target_of(module)
|
||||
if target is None:
|
||||
raise SystemExit(f"no mapping rule for {module}")
|
||||
for role in ("to_q", "to_k", "to_v"):
|
||||
if target.endswith(f".attn.{role}"):
|
||||
qkv.setdefault(target[: -len(f".attn.{role}")], {})[role[-1]] = delta
|
||||
break
|
||||
else:
|
||||
deltas[target + ".weight"] = delta
|
||||
|
||||
for prefix, parts in qkv.items():
|
||||
if set(parts) != {"q", "k", "v"}:
|
||||
raise SystemExit(f"{prefix} is missing part of q/k/v: {sorted(parts)}")
|
||||
deltas[prefix + ".attn.qkv_proj.weight"] = _interleave_qkv(
|
||||
parts["q"], parts["k"], parts["v"]
|
||||
)
|
||||
print(f"2D deltas to merge: {len(deltas)} ({len(qkv)} of them qkv)")
|
||||
|
||||
out = args.out_dir
|
||||
transformer_out = out / "transformer"
|
||||
transformer_out.mkdir(parents=True, exist_ok=True)
|
||||
applied = 0
|
||||
for shard in sorted(args.base_dir.glob("*.safetensors")):
|
||||
tensors = load_file(str(shard))
|
||||
for key, base in tensors.items():
|
||||
if key in deltas:
|
||||
merged = base.float() + deltas[key]
|
||||
if args.verify:
|
||||
rel = ((merged - base.float()).norm() / base.float().norm()).item()
|
||||
print(f" {key}: relative change {rel:.4f}")
|
||||
tensors[key] = merged.to(base.dtype)
|
||||
applied += 1
|
||||
save_file(tensors, str(transformer_out / shard.name), metadata={"format": "pt"})
|
||||
print(f"merged {applied}/{len(deltas)} tensors")
|
||||
if applied != len(deltas):
|
||||
raise SystemExit(
|
||||
f"{len(deltas) - applied} deltas found no target; do not use this output"
|
||||
)
|
||||
|
||||
for extra in (
|
||||
"config.json",
|
||||
"model.safetensors.index.json",
|
||||
"diffusion_pytorch_model.safetensors.index.json",
|
||||
):
|
||||
src = args.base_dir / extra
|
||||
if src.exists():
|
||||
shutil.copy(src, transformer_out / extra)
|
||||
|
||||
heads = {
|
||||
k: lora[k]
|
||||
for k in (
|
||||
"proj_out.weight",
|
||||
"proj_out.bias",
|
||||
"audio_proj_out.weight",
|
||||
"audio_proj_out.bias",
|
||||
)
|
||||
}
|
||||
save_file(heads, str(out / "pdd_heads.safetensors"), metadata={"format": "pt"})
|
||||
(out / "pdd_config.json").write_text(json.dumps(pdd, indent=2) + "\n")
|
||||
print(
|
||||
f"position-level output heads -> {out / 'pdd_heads.safetensors'} "
|
||||
f"({pdd['num_steps']} of them, block size {pdd['block_size']}, "
|
||||
f"i.e. {pdd['num_steps'] // pdd['block_size']} forwards)"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Fuse PDD's 32 position-level output heads into one per block, i.e. 8.
|
||||
|
||||
python3 -m sglang.multimodal_gen.tools.fuse_minimax_h3_pdd_heads <pdd dir> \
|
||||
[--video-shift 12.0 --audio-shift 3.0]
|
||||
|
||||
## Why the fusion is exact
|
||||
|
||||
MiniMax-H3's scheduler has "ancestral" in its name but takes the eta0 branch, which
|
||||
injects no noise:
|
||||
|
||||
x_next = r*x + (1-r)*x0, r = sigma_next/sigma_curr, x0 = x - sigma_curr*v
|
||||
= x - (sigma_curr - sigma_next)*v
|
||||
|
||||
That is plain Euler, and linear in v. The L heads in one PDD block share a single
|
||||
backbone evaluation, so
|
||||
|
||||
x_{n+L} = x_n - sum_j dsigma_{n+j} * W_{n+j} h
|
||||
= x_n - (sum_j dsigma_{n+j}) * (sum_j w_j W_{n+j}) h, w_j = dsigma_{n+j} / sum dsigma
|
||||
|
||||
and the bracket is the fused head -- the paper's "fuse the layers into a single
|
||||
linear layer that directly predicts a step across the full block". Each block then
|
||||
costs one forward and one scheduler step, and the denoise loop needs no changes.
|
||||
|
||||
This depends on eta0. Under true ancestral sampling (noise injected every step) the
|
||||
state at the block's second step is no longer the block's first state, the sum
|
||||
cannot be moved onto W, and the fusion is invalid -- the L sub-steps have to be run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from safetensors import safe_open
|
||||
from safetensors.torch import save_file
|
||||
|
||||
|
||||
def sigma_grid(num_points: int, shift: float) -> torch.Tensor:
|
||||
base = torch.linspace(1.0, 0.0, num_points, dtype=torch.float64)
|
||||
return shift * base / (1 + (shift - 1) * base)
|
||||
|
||||
|
||||
def fuse(heads: torch.Tensor, sigmas: torch.Tensor, block: int) -> torch.Tensor:
|
||||
"""heads: (N, ...) -> (N//block, ...), averaged within a block weighted by dsigma."""
|
||||
steps = heads.shape[0]
|
||||
if block <= 0 or steps == 0 or steps % block:
|
||||
raise ValueError("PDD head count must be positive and divisible by block size")
|
||||
if sigmas.numel() != steps + 1:
|
||||
raise SystemExit(
|
||||
f"sigma grid has {sigmas.numel()} points but there are {steps} heads"
|
||||
)
|
||||
dsigma = (sigmas[:-1] - sigmas[1:]).clamp(min=0)
|
||||
out = []
|
||||
for b in range(steps // block):
|
||||
w = dsigma[b * block : (b + 1) * block]
|
||||
if float(w.sum()) <= 0:
|
||||
raise SystemExit(f"block {b} has an all-zero dsigma")
|
||||
w = (w / w.sum()).to(heads.dtype)
|
||||
chunk = heads[b * block : (b + 1) * block].float()
|
||||
out.append((chunk * w.float().view(-1, *([1] * (chunk.dim() - 1)))).sum(0))
|
||||
return torch.stack(out).to(heads.dtype)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(
|
||||
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
||||
)
|
||||
ap.add_argument("pdd_dir")
|
||||
ap.add_argument("--video-shift", type=float, default=12.0)
|
||||
ap.add_argument("--audio-shift", type=float, default=3.0)
|
||||
args = ap.parse_args()
|
||||
|
||||
d = Path(args.pdd_dir)
|
||||
cfg = json.loads((d / "pdd_config.json").read_text())
|
||||
n, block = cfg["num_steps"], cfg["block_size"]
|
||||
with safe_open(str(d / "pdd_heads.safetensors"), "pt") as f:
|
||||
heads = {k: f.get_tensor(k) for k in f.keys()}
|
||||
|
||||
sv = sigma_grid(n + 1, args.video_shift)
|
||||
sa = sigma_grid(n + 1, args.audio_shift)
|
||||
fused = {
|
||||
"video_out.weight": fuse(heads["proj_out.weight"], sv, block),
|
||||
"video_out.bias": fuse(heads["proj_out.bias"], sv, block),
|
||||
"audio_out.weight": fuse(heads["audio_proj_out.weight"], sa, block),
|
||||
"audio_out.bias": fuse(heads["audio_proj_out.bias"], sa, block),
|
||||
}
|
||||
save_file(fused, str(d / "pdd_fused_heads.safetensors"), metadata={"format": "pt"})
|
||||
cfg["fused_steps"] = n // block
|
||||
cfg["num_inference_steps"] = n // block + 1 # H3 counts steps as sigma grid points
|
||||
cfg["video_shift"], cfg["audio_shift"] = args.video_shift, args.audio_shift
|
||||
(d / "pdd_config.json").write_text(json.dumps(cfg, indent=2) + "\n")
|
||||
|
||||
print(f"fused {n} -> {n // block} heads (block size {block})")
|
||||
for k, v in fused.items():
|
||||
print(f" {k}: {tuple(v.shape)}")
|
||||
print(f"run with --num-inference-steps {cfg['num_inference_steps']}")
|
||||
print(
|
||||
"first 9 points of the video sigma grid:",
|
||||
[round(float(x), 4) for x in sv[::block]],
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user