[diffusion] feat: add community LoRA recipes and Kohya mapping (#35857)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -704,10 +704,11 @@ refer to files visible inside the SGLang server environment.
|
||||
|
||||
## 5. LoRA recipes
|
||||
|
||||
H3 accepts both native fused adapters and standard Diffusers/PEFT adapters.
|
||||
Native adapters target modules such as `blocks.*.attn.qkv_proj`; PEFT adapters
|
||||
may instead provide separate `to_q`, `to_k`, and `to_v` projections and the
|
||||
`default` adapter namespace. SGLang normalizes both layouts.
|
||||
H3 accepts native fused adapters, standard Diffusers/PEFT adapters, and Kohya
|
||||
`lora_unet_blocks_*` exports from `networks.lora_minimax_h3`. Native adapters
|
||||
target modules such as `blocks.*.attn.qkv_proj`; PEFT adapters may instead
|
||||
provide separate `to_q`, `to_k`, and `to_v` projections and the `default`
|
||||
adapter namespace. SGLang normalizes all three layouts.
|
||||
|
||||
The following pinned FL2VA adapters have distinct purposes:
|
||||
|
||||
@@ -716,13 +717,18 @@ The following pinned FL2VA adapters have distinct purposes:
|
||||
| Recommended speed/quality balance | [`larryvrh/MiniMax-H3-Turbo-Lora`](https://huggingface.co/larryvrh/MiniMax-H3-Turbo-Lora), `minimax_h3_turbo_v4_step600_ema.safetensors` | `num_inference_steps: 9` (8 denoiser evaluations), `lora_scale: 1.0` | None |
|
||||
| Most aggressive speed preset (standard PEFT layout) | [`lightx2v/Minimax-h3-Turbo`](https://huggingface.co/lightx2v/Minimax-h3-Turbo), `minimax_h3_fl2v_turbo_4step_v0.1.safetensors` | `num_inference_steps: 5` (4 denoiser evaluations), `lora_scale: 1.0`, `lora_alpha: 8` | None |
|
||||
| Realistic people style | [`fal/MiniMax-H3-Realism-People-LoRA`](https://huggingface.co/fal/MiniMax-H3-Realism-People-LoRA), `h3-realism-people-t2v-i2v-r2v.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 0.7` | Include `r34l1sm` in the prompt |
|
||||
| Spatial and physics | [`Jojocodex/minimax-h3-spatial-physics-lora`](https://huggingface.co/Jojocodex/minimax-h3-spatial-physics-lora), `wushu_spatial_physics_clean_3000_pruned.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 0.4` | None. Describe collisions, stacking, and gravity-consistent motion in the prompt |
|
||||
| Wushu action | [`Jojocodex/minimax-h3-wushu-action-lora`](https://huggingface.co/Jojocodex/minimax-h3-wushu-action-lora), `wushu_action_h3_lora_v5_3000_pruned.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 0.8` | Put `wushu_action` at the start of the prompt |
|
||||
| Camera motion | [`Jojocodex/minimax-h3-yunjing-lora`](https://huggingface.co/Jojocodex/minimax-h3-yunjing-lora), `camera_motion_h3_lora_v1_3000_pruned.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 0.8` | Put `camera motion` at the start of the prompt |
|
||||
| Looping sketch anime | [`Inner-Reflections/MiniMax-H3-Looping-Sketch-Anime`](https://huggingface.co/Inner-Reflections/MiniMax-H3-Looping-Sketch-Anime), `minimax_h3_looping_sketch_anime_v1.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 1.0` | Include the author's hand-drawn sketch style phrase in the prompt |
|
||||
| 3D-to-real detail slider | [`siraxe/3d_to_real_detail_slider_H3`](https://huggingface.co/siraxe/3d_to_real_detail_slider_H3), `3d_to_real_H3_detail_slider_bf16.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 1.0` | None. Sweep the scale; this is a slider, not a fixed style |
|
||||
|
||||
The H3 request field controls the number of sigma grid points, including the
|
||||
terminal zero; the denoising loop therefore runs one fewer model evaluation.
|
||||
This is why an adapter described as 8-step uses `9`, and a 4-step adapter uses
|
||||
`5`, in the request.
|
||||
|
||||
All three use the same launch shape. Pinning the filename is required for
|
||||
Every recipe uses the same launch shape. Pinning the filename is required for
|
||||
repositories that publish multiple revisions, and is also recommended for a
|
||||
reproducible single-file recipe:
|
||||
|
||||
|
||||
@@ -83,6 +83,26 @@ class MiniMaxH3DiTArchConfig(DiTArchConfig):
|
||||
}
|
||||
)
|
||||
|
||||
# Kohya `networks.lora_minimax_h3` uses lora_unet_blocks_* + lora_down/up.
|
||||
# The format adapter rewrites these first; keep the aliases here in case a
|
||||
# raw key still reaches the loader.
|
||||
lora_param_names_mapping: dict = field(
|
||||
default_factory=lambda: {
|
||||
r"^lora_unet_blocks_(\d+)_attn_qkv_proj\.lora_down$": r"blocks.\1.attn.qkv_proj.lora_A",
|
||||
r"^lora_unet_blocks_(\d+)_attn_qkv_proj\.lora_up$": r"blocks.\1.attn.qkv_proj.lora_B",
|
||||
r"^lora_unet_blocks_(\d+)_attn_qkv_proj\.(lora_[AB]|alpha)$": r"blocks.\1.attn.qkv_proj.\2",
|
||||
r"^lora_unet_blocks_(\d+)_attn_out_proj\.lora_down$": r"blocks.\1.attn.out_proj.lora_A",
|
||||
r"^lora_unet_blocks_(\d+)_attn_out_proj\.lora_up$": r"blocks.\1.attn.out_proj.lora_B",
|
||||
r"^lora_unet_blocks_(\d+)_attn_out_proj\.(lora_[AB]|alpha)$": r"blocks.\1.attn.out_proj.\2",
|
||||
r"^lora_unet_blocks_(\d+)_mlp_fc1\.lora_down$": r"blocks.\1.mlp.fc1.lora_A",
|
||||
r"^lora_unet_blocks_(\d+)_mlp_fc1\.lora_up$": r"blocks.\1.mlp.fc1.lora_B",
|
||||
r"^lora_unet_blocks_(\d+)_mlp_fc1\.(lora_[AB]|alpha)$": r"blocks.\1.mlp.fc1.\2",
|
||||
r"^lora_unet_blocks_(\d+)_mlp_fc2\.lora_down$": r"blocks.\1.mlp.fc2.lora_A",
|
||||
r"^lora_unet_blocks_(\d+)_mlp_fc2\.lora_up$": r"blocks.\1.mlp.fc2.lora_B",
|
||||
r"^lora_unet_blocks_(\d+)_mlp_fc2\.(lora_[AB]|alpha)$": r"blocks.\1.mlp.fc2.\2",
|
||||
}
|
||||
)
|
||||
|
||||
num_layers: int = 50
|
||||
token_refiner_num_layers: int = 2
|
||||
hidden_size: int = 5376
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, Iterable, Mapping, Optional
|
||||
|
||||
@@ -23,6 +24,7 @@ class LoRAFormat(str, Enum):
|
||||
QWEN_IMAGE_STANDARD = "qwen-image-standard"
|
||||
XLABS_FLUX = "xlabs-ai"
|
||||
KOHYA_FLUX = "kohya-flux"
|
||||
KOHYA_MINIMAX_H3 = "kohya-minimax-h3"
|
||||
WAN = "wan"
|
||||
AI_TOOLKIT_FLUX = "ai-toolkit-flux"
|
||||
|
||||
@@ -62,6 +64,68 @@ def _looks_like_xlabs_flux_key(k: str) -> bool:
|
||||
return ".processor." in k or ".proj_lora" in k or ".qkv_lora" in k
|
||||
|
||||
|
||||
_KOHYA_H3_KEY = re.compile(
|
||||
r"^lora_unet_blocks_(\d+)_(attn_qkv_proj|attn_out_proj|mlp_fc1|mlp_fc2)"
|
||||
r"\.(lora_down|lora_up|alpha)(?:\.weight)?$"
|
||||
)
|
||||
_KOHYA_H3_MODULES = {
|
||||
"attn_qkv_proj": "attn.qkv_proj",
|
||||
"attn_out_proj": "attn.out_proj",
|
||||
"mlp_fc1": "mlp.fc1",
|
||||
"mlp_fc2": "mlp.fc2",
|
||||
}
|
||||
_KOHYA_H3_SUFFIXES = {
|
||||
"lora_down": "lora_A",
|
||||
"lora_up": "lora_B",
|
||||
"alpha": "alpha",
|
||||
}
|
||||
|
||||
|
||||
def _looks_like_kohya_minimax_h3(state_dict: Mapping[str, torch.Tensor]) -> bool:
|
||||
"""Kohya `networks.lora_minimax_h3` export: lora_unet_blocks_* fused names."""
|
||||
if not state_dict:
|
||||
return False
|
||||
return any(
|
||||
k.startswith("lora_unet_blocks_") and "_attn_qkv_proj." in k for k in state_dict
|
||||
)
|
||||
|
||||
|
||||
def _convert_kohya_minimax_h3(
|
||||
state_dict: Mapping[str, torch.Tensor],
|
||||
log: logging.Logger,
|
||||
) -> Dict[str, torch.Tensor]:
|
||||
"""lora_unet_blocks_N_* + lora_down/up -> native H3 blocks.N.*.lora_A/B."""
|
||||
out: Dict[str, torch.Tensor] = {}
|
||||
unmatched: list[str] = []
|
||||
|
||||
for name, tensor in state_dict.items():
|
||||
match = _KOHYA_H3_KEY.match(name)
|
||||
if match is None:
|
||||
unmatched.append(name)
|
||||
out[name] = tensor
|
||||
continue
|
||||
layer, module, suffix = match.group(1), match.group(2), match.group(3)
|
||||
target = (
|
||||
f"blocks.{layer}.{_KOHYA_H3_MODULES[module]}.{_KOHYA_H3_SUFFIXES[suffix]}"
|
||||
)
|
||||
out[target] = tensor
|
||||
|
||||
sample = _sample_keys(out.keys(), 20)
|
||||
log.info(
|
||||
"[LoRAFormatAdapter] after KOHYA_MINIMAX_H3 conversion, "
|
||||
"unmatched=%d sample keys (<=20): %s",
|
||||
len(unmatched),
|
||||
", ".join(sample),
|
||||
)
|
||||
if unmatched:
|
||||
log.warning(
|
||||
"[LoRAFormatAdapter] KOHYA_MINIMAX_H3 left %d unmatched keys: %s",
|
||||
len(unmatched),
|
||||
", ".join(_sample_keys(unmatched, 8)),
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _looks_like_kohya_flux(state_dict: Mapping[str, torch.Tensor]) -> bool:
|
||||
"""Kohya FLUX LoRA (flux_lora.py) under lora_unet_double/single_blocks_ prefixes."""
|
||||
if not state_dict:
|
||||
@@ -152,6 +216,8 @@ def detect_lora_format_from_state_dict(
|
||||
|
||||
if any(_looks_like_xlabs_flux_key(k) for k in keys):
|
||||
return LoRAFormat.XLABS_FLUX
|
||||
if _looks_like_kohya_minimax_h3(state_dict):
|
||||
return LoRAFormat.KOHYA_MINIMAX_H3
|
||||
if _looks_like_kohya_flux(state_dict):
|
||||
return LoRAFormat.KOHYA_FLUX
|
||||
|
||||
@@ -506,6 +572,9 @@ def convert_lora_state_dict_by_format(
|
||||
converted = _convert_xlabs_ai_via_diffusers(state_dict, log)
|
||||
return _convert_non_diffusers_sd_simple(converted, log)
|
||||
|
||||
if fmt == LoRAFormat.KOHYA_MINIMAX_H3:
|
||||
return _convert_kohya_minimax_h3(state_dict, log)
|
||||
|
||||
if fmt == LoRAFormat.KOHYA_FLUX:
|
||||
converted = _convert_kohya_flux_via_diffusers(state_dict, log)
|
||||
return _convert_non_diffusers_sd_simple(converted, log)
|
||||
|
||||
Reference in New Issue
Block a user