[diffusion] feat: add community LoRA recipes and Kohya mapping (#35857)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -83,6 +83,26 @@ class MiniMaxH3DiTArchConfig(DiTArchConfig):
|
||||
}
|
||||
)
|
||||
|
||||
# Kohya `networks.lora_minimax_h3` uses lora_unet_blocks_* + lora_down/up.
|
||||
# The format adapter rewrites these first; keep the aliases here in case a
|
||||
# raw key still reaches the loader.
|
||||
lora_param_names_mapping: dict = field(
|
||||
default_factory=lambda: {
|
||||
r"^lora_unet_blocks_(\d+)_attn_qkv_proj\.lora_down$": r"blocks.\1.attn.qkv_proj.lora_A",
|
||||
r"^lora_unet_blocks_(\d+)_attn_qkv_proj\.lora_up$": r"blocks.\1.attn.qkv_proj.lora_B",
|
||||
r"^lora_unet_blocks_(\d+)_attn_qkv_proj\.(lora_[AB]|alpha)$": r"blocks.\1.attn.qkv_proj.\2",
|
||||
r"^lora_unet_blocks_(\d+)_attn_out_proj\.lora_down$": r"blocks.\1.attn.out_proj.lora_A",
|
||||
r"^lora_unet_blocks_(\d+)_attn_out_proj\.lora_up$": r"blocks.\1.attn.out_proj.lora_B",
|
||||
r"^lora_unet_blocks_(\d+)_attn_out_proj\.(lora_[AB]|alpha)$": r"blocks.\1.attn.out_proj.\2",
|
||||
r"^lora_unet_blocks_(\d+)_mlp_fc1\.lora_down$": r"blocks.\1.mlp.fc1.lora_A",
|
||||
r"^lora_unet_blocks_(\d+)_mlp_fc1\.lora_up$": r"blocks.\1.mlp.fc1.lora_B",
|
||||
r"^lora_unet_blocks_(\d+)_mlp_fc1\.(lora_[AB]|alpha)$": r"blocks.\1.mlp.fc1.\2",
|
||||
r"^lora_unet_blocks_(\d+)_mlp_fc2\.lora_down$": r"blocks.\1.mlp.fc2.lora_A",
|
||||
r"^lora_unet_blocks_(\d+)_mlp_fc2\.lora_up$": r"blocks.\1.mlp.fc2.lora_B",
|
||||
r"^lora_unet_blocks_(\d+)_mlp_fc2\.(lora_[AB]|alpha)$": r"blocks.\1.mlp.fc2.\2",
|
||||
}
|
||||
)
|
||||
|
||||
num_layers: int = 50
|
||||
token_refiner_num_layers: int = 2
|
||||
hidden_size: int = 5376
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, Iterable, Mapping, Optional
|
||||
|
||||
@@ -23,6 +24,7 @@ class LoRAFormat(str, Enum):
|
||||
QWEN_IMAGE_STANDARD = "qwen-image-standard"
|
||||
XLABS_FLUX = "xlabs-ai"
|
||||
KOHYA_FLUX = "kohya-flux"
|
||||
KOHYA_MINIMAX_H3 = "kohya-minimax-h3"
|
||||
WAN = "wan"
|
||||
AI_TOOLKIT_FLUX = "ai-toolkit-flux"
|
||||
|
||||
@@ -62,6 +64,68 @@ def _looks_like_xlabs_flux_key(k: str) -> bool:
|
||||
return ".processor." in k or ".proj_lora" in k or ".qkv_lora" in k
|
||||
|
||||
|
||||
_KOHYA_H3_KEY = re.compile(
|
||||
r"^lora_unet_blocks_(\d+)_(attn_qkv_proj|attn_out_proj|mlp_fc1|mlp_fc2)"
|
||||
r"\.(lora_down|lora_up|alpha)(?:\.weight)?$"
|
||||
)
|
||||
_KOHYA_H3_MODULES = {
|
||||
"attn_qkv_proj": "attn.qkv_proj",
|
||||
"attn_out_proj": "attn.out_proj",
|
||||
"mlp_fc1": "mlp.fc1",
|
||||
"mlp_fc2": "mlp.fc2",
|
||||
}
|
||||
_KOHYA_H3_SUFFIXES = {
|
||||
"lora_down": "lora_A",
|
||||
"lora_up": "lora_B",
|
||||
"alpha": "alpha",
|
||||
}
|
||||
|
||||
|
||||
def _looks_like_kohya_minimax_h3(state_dict: Mapping[str, torch.Tensor]) -> bool:
|
||||
"""Kohya `networks.lora_minimax_h3` export: lora_unet_blocks_* fused names."""
|
||||
if not state_dict:
|
||||
return False
|
||||
return any(
|
||||
k.startswith("lora_unet_blocks_") and "_attn_qkv_proj." in k for k in state_dict
|
||||
)
|
||||
|
||||
|
||||
def _convert_kohya_minimax_h3(
|
||||
state_dict: Mapping[str, torch.Tensor],
|
||||
log: logging.Logger,
|
||||
) -> Dict[str, torch.Tensor]:
|
||||
"""lora_unet_blocks_N_* + lora_down/up -> native H3 blocks.N.*.lora_A/B."""
|
||||
out: Dict[str, torch.Tensor] = {}
|
||||
unmatched: list[str] = []
|
||||
|
||||
for name, tensor in state_dict.items():
|
||||
match = _KOHYA_H3_KEY.match(name)
|
||||
if match is None:
|
||||
unmatched.append(name)
|
||||
out[name] = tensor
|
||||
continue
|
||||
layer, module, suffix = match.group(1), match.group(2), match.group(3)
|
||||
target = (
|
||||
f"blocks.{layer}.{_KOHYA_H3_MODULES[module]}.{_KOHYA_H3_SUFFIXES[suffix]}"
|
||||
)
|
||||
out[target] = tensor
|
||||
|
||||
sample = _sample_keys(out.keys(), 20)
|
||||
log.info(
|
||||
"[LoRAFormatAdapter] after KOHYA_MINIMAX_H3 conversion, "
|
||||
"unmatched=%d sample keys (<=20): %s",
|
||||
len(unmatched),
|
||||
", ".join(sample),
|
||||
)
|
||||
if unmatched:
|
||||
log.warning(
|
||||
"[LoRAFormatAdapter] KOHYA_MINIMAX_H3 left %d unmatched keys: %s",
|
||||
len(unmatched),
|
||||
", ".join(_sample_keys(unmatched, 8)),
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _looks_like_kohya_flux(state_dict: Mapping[str, torch.Tensor]) -> bool:
|
||||
"""Kohya FLUX LoRA (flux_lora.py) under lora_unet_double/single_blocks_ prefixes."""
|
||||
if not state_dict:
|
||||
@@ -152,6 +216,8 @@ def detect_lora_format_from_state_dict(
|
||||
|
||||
if any(_looks_like_xlabs_flux_key(k) for k in keys):
|
||||
return LoRAFormat.XLABS_FLUX
|
||||
if _looks_like_kohya_minimax_h3(state_dict):
|
||||
return LoRAFormat.KOHYA_MINIMAX_H3
|
||||
if _looks_like_kohya_flux(state_dict):
|
||||
return LoRAFormat.KOHYA_FLUX
|
||||
|
||||
@@ -506,6 +572,9 @@ def convert_lora_state_dict_by_format(
|
||||
converted = _convert_xlabs_ai_via_diffusers(state_dict, log)
|
||||
return _convert_non_diffusers_sd_simple(converted, log)
|
||||
|
||||
if fmt == LoRAFormat.KOHYA_MINIMAX_H3:
|
||||
return _convert_kohya_minimax_h3(state_dict, log)
|
||||
|
||||
if fmt == LoRAFormat.KOHYA_FLUX:
|
||||
converted = _convert_kohya_flux_via_diffusers(state_dict, log)
|
||||
return _convert_non_diffusers_sd_simple(converted, log)
|
||||
|
||||
Reference in New Issue
Block a user