[diffusion] feat: add community LoRA recipes and Kohya mapping (#35857)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
WenhaoZhang
2026-10-05 18:22:56 +08:00
committed by GitHub
co-authored by Mick Qian
parent 08fa37e48d
commit f048d5aa4b
3 changed files with 100 additions and 5 deletions
+11 -5
View File
@@ -704,10 +704,11 @@ refer to files visible inside the SGLang server environment.
## 5. LoRA recipes
H3 accepts both native fused adapters and standard Diffusers/PEFT adapters.
Native adapters target modules such as `blocks.*.attn.qkv_proj`; PEFT adapters
may instead provide separate `to_q`, `to_k`, and `to_v` projections and the
`default` adapter namespace. SGLang normalizes both layouts.
H3 accepts native fused adapters, standard Diffusers/PEFT adapters, and Kohya
`lora_unet_blocks_*` exports from `networks.lora_minimax_h3`. Native adapters
target modules such as `blocks.*.attn.qkv_proj`; PEFT adapters may instead
provide separate `to_q`, `to_k`, and `to_v` projections and the `default`
adapter namespace. SGLang normalizes all three layouts.
The following pinned FL2VA adapters have distinct purposes:
@@ -716,13 +717,18 @@ The following pinned FL2VA adapters have distinct purposes:
| Recommended speed/quality balance | [`larryvrh/MiniMax-H3-Turbo-Lora`](https://huggingface.co/larryvrh/MiniMax-H3-Turbo-Lora), `minimax_h3_turbo_v4_step600_ema.safetensors` | `num_inference_steps: 9` (8 denoiser evaluations), `lora_scale: 1.0` | None |
| Most aggressive speed preset (standard PEFT layout) | [`lightx2v/Minimax-h3-Turbo`](https://huggingface.co/lightx2v/Minimax-h3-Turbo), `minimax_h3_fl2v_turbo_4step_v0.1.safetensors` | `num_inference_steps: 5` (4 denoiser evaluations), `lora_scale: 1.0`, `lora_alpha: 8` | None |
| Realistic people style | [`fal/MiniMax-H3-Realism-People-LoRA`](https://huggingface.co/fal/MiniMax-H3-Realism-People-LoRA), `h3-realism-people-t2v-i2v-r2v.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 0.7` | Include `r34l1sm` in the prompt |
| Spatial and physics | [`Jojocodex/minimax-h3-spatial-physics-lora`](https://huggingface.co/Jojocodex/minimax-h3-spatial-physics-lora), `wushu_spatial_physics_clean_3000_pruned.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 0.4` | None. Describe collisions, stacking, and gravity-consistent motion in the prompt |
| Wushu action | [`Jojocodex/minimax-h3-wushu-action-lora`](https://huggingface.co/Jojocodex/minimax-h3-wushu-action-lora), `wushu_action_h3_lora_v5_3000_pruned.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 0.8` | Put `wushu_action` at the start of the prompt |
| Camera motion | [`Jojocodex/minimax-h3-yunjing-lora`](https://huggingface.co/Jojocodex/minimax-h3-yunjing-lora), `camera_motion_h3_lora_v1_3000_pruned.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 0.8` | Put `camera motion` at the start of the prompt |
| Looping sketch anime | [`Inner-Reflections/MiniMax-H3-Looping-Sketch-Anime`](https://huggingface.co/Inner-Reflections/MiniMax-H3-Looping-Sketch-Anime), `minimax_h3_looping_sketch_anime_v1.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 1.0` | Include the author's hand-drawn sketch style phrase in the prompt |
| 3D-to-real detail slider | [`siraxe/3d_to_real_detail_slider_H3`](https://huggingface.co/siraxe/3d_to_real_detail_slider_H3), `3d_to_real_H3_detail_slider_bf16.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 1.0` | None. Sweep the scale; this is a slider, not a fixed style |
The H3 request field controls the number of sigma grid points, including the
terminal zero; the denoising loop therefore runs one fewer model evaluation.
This is why an adapter described as 8-step uses `9`, and a 4-step adapter uses
`5`, in the request.
All three use the same launch shape. Pinning the filename is required for
Every recipe uses the same launch shape. Pinning the filename is required for
repositories that publish multiple revisions, and is also recommended for a
reproducible single-file recipe:
@@ -83,6 +83,26 @@ class MiniMaxH3DiTArchConfig(DiTArchConfig):
}
)
# Kohya `networks.lora_minimax_h3` uses lora_unet_blocks_* + lora_down/up.
# The format adapter rewrites these first; keep the aliases here in case a
# raw key still reaches the loader.
lora_param_names_mapping: dict = field(
default_factory=lambda: {
r"^lora_unet_blocks_(\d+)_attn_qkv_proj\.lora_down$": r"blocks.\1.attn.qkv_proj.lora_A",
r"^lora_unet_blocks_(\d+)_attn_qkv_proj\.lora_up$": r"blocks.\1.attn.qkv_proj.lora_B",
r"^lora_unet_blocks_(\d+)_attn_qkv_proj\.(lora_[AB]|alpha)$": r"blocks.\1.attn.qkv_proj.\2",
r"^lora_unet_blocks_(\d+)_attn_out_proj\.lora_down$": r"blocks.\1.attn.out_proj.lora_A",
r"^lora_unet_blocks_(\d+)_attn_out_proj\.lora_up$": r"blocks.\1.attn.out_proj.lora_B",
r"^lora_unet_blocks_(\d+)_attn_out_proj\.(lora_[AB]|alpha)$": r"blocks.\1.attn.out_proj.\2",
r"^lora_unet_blocks_(\d+)_mlp_fc1\.lora_down$": r"blocks.\1.mlp.fc1.lora_A",
r"^lora_unet_blocks_(\d+)_mlp_fc1\.lora_up$": r"blocks.\1.mlp.fc1.lora_B",
r"^lora_unet_blocks_(\d+)_mlp_fc1\.(lora_[AB]|alpha)$": r"blocks.\1.mlp.fc1.\2",
r"^lora_unet_blocks_(\d+)_mlp_fc2\.lora_down$": r"blocks.\1.mlp.fc2.lora_A",
r"^lora_unet_blocks_(\d+)_mlp_fc2\.lora_up$": r"blocks.\1.mlp.fc2.lora_B",
r"^lora_unet_blocks_(\d+)_mlp_fc2\.(lora_[AB]|alpha)$": r"blocks.\1.mlp.fc2.\2",
}
)
num_layers: int = 50
token_refiner_num_layers: int = 2
hidden_size: int = 5376
@@ -1,6 +1,7 @@
from __future__ import annotations
import logging
import re
from enum import Enum
from typing import Any, Dict, Iterable, Mapping, Optional
@@ -23,6 +24,7 @@ class LoRAFormat(str, Enum):
QWEN_IMAGE_STANDARD = "qwen-image-standard"
XLABS_FLUX = "xlabs-ai"
KOHYA_FLUX = "kohya-flux"
KOHYA_MINIMAX_H3 = "kohya-minimax-h3"
WAN = "wan"
AI_TOOLKIT_FLUX = "ai-toolkit-flux"
@@ -62,6 +64,68 @@ def _looks_like_xlabs_flux_key(k: str) -> bool:
return ".processor." in k or ".proj_lora" in k or ".qkv_lora" in k
_KOHYA_H3_KEY = re.compile(
r"^lora_unet_blocks_(\d+)_(attn_qkv_proj|attn_out_proj|mlp_fc1|mlp_fc2)"
r"\.(lora_down|lora_up|alpha)(?:\.weight)?$"
)
_KOHYA_H3_MODULES = {
"attn_qkv_proj": "attn.qkv_proj",
"attn_out_proj": "attn.out_proj",
"mlp_fc1": "mlp.fc1",
"mlp_fc2": "mlp.fc2",
}
_KOHYA_H3_SUFFIXES = {
"lora_down": "lora_A",
"lora_up": "lora_B",
"alpha": "alpha",
}
def _looks_like_kohya_minimax_h3(state_dict: Mapping[str, torch.Tensor]) -> bool:
"""Kohya `networks.lora_minimax_h3` export: lora_unet_blocks_* fused names."""
if not state_dict:
return False
return any(
k.startswith("lora_unet_blocks_") and "_attn_qkv_proj." in k for k in state_dict
)
def _convert_kohya_minimax_h3(
state_dict: Mapping[str, torch.Tensor],
log: logging.Logger,
) -> Dict[str, torch.Tensor]:
"""lora_unet_blocks_N_* + lora_down/up -> native H3 blocks.N.*.lora_A/B."""
out: Dict[str, torch.Tensor] = {}
unmatched: list[str] = []
for name, tensor in state_dict.items():
match = _KOHYA_H3_KEY.match(name)
if match is None:
unmatched.append(name)
out[name] = tensor
continue
layer, module, suffix = match.group(1), match.group(2), match.group(3)
target = (
f"blocks.{layer}.{_KOHYA_H3_MODULES[module]}.{_KOHYA_H3_SUFFIXES[suffix]}"
)
out[target] = tensor
sample = _sample_keys(out.keys(), 20)
log.info(
"[LoRAFormatAdapter] after KOHYA_MINIMAX_H3 conversion, "
"unmatched=%d sample keys (<=20): %s",
len(unmatched),
", ".join(sample),
)
if unmatched:
log.warning(
"[LoRAFormatAdapter] KOHYA_MINIMAX_H3 left %d unmatched keys: %s",
len(unmatched),
", ".join(_sample_keys(unmatched, 8)),
)
return out
def _looks_like_kohya_flux(state_dict: Mapping[str, torch.Tensor]) -> bool:
"""Kohya FLUX LoRA (flux_lora.py) under lora_unet_double/single_blocks_ prefixes."""
if not state_dict:
@@ -152,6 +216,8 @@ def detect_lora_format_from_state_dict(
if any(_looks_like_xlabs_flux_key(k) for k in keys):
return LoRAFormat.XLABS_FLUX
if _looks_like_kohya_minimax_h3(state_dict):
return LoRAFormat.KOHYA_MINIMAX_H3
if _looks_like_kohya_flux(state_dict):
return LoRAFormat.KOHYA_FLUX
@@ -506,6 +572,9 @@ def convert_lora_state_dict_by_format(
converted = _convert_xlabs_ai_via_diffusers(state_dict, log)
return _convert_non_diffusers_sd_simple(converted, log)
if fmt == LoRAFormat.KOHYA_MINIMAX_H3:
return _convert_kohya_minimax_h3(state_dict, log)
if fmt == LoRAFormat.KOHYA_FLUX:
converted = _convert_kohya_flux_via_diffusers(state_dict, log)
return _convert_non_diffusers_sd_simple(converted, log)