diff --git a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx index ff72d9d4d0..87f8e86406 100644 --- a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx +++ b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx @@ -704,10 +704,11 @@ refer to files visible inside the SGLang server environment. ## 5. LoRA recipes -H3 accepts both native fused adapters and standard Diffusers/PEFT adapters. -Native adapters target modules such as `blocks.*.attn.qkv_proj`; PEFT adapters -may instead provide separate `to_q`, `to_k`, and `to_v` projections and the -`default` adapter namespace. SGLang normalizes both layouts. +H3 accepts native fused adapters, standard Diffusers/PEFT adapters, and Kohya +`lora_unet_blocks_*` exports from `networks.lora_minimax_h3`. Native adapters +target modules such as `blocks.*.attn.qkv_proj`; PEFT adapters may instead +provide separate `to_q`, `to_k`, and `to_v` projections and the `default` +adapter namespace. SGLang normalizes all three layouts. The following pinned FL2VA adapters have distinct purposes: @@ -716,13 +717,18 @@ The following pinned FL2VA adapters have distinct purposes: | Recommended speed/quality balance | [`larryvrh/MiniMax-H3-Turbo-Lora`](https://huggingface.co/larryvrh/MiniMax-H3-Turbo-Lora), `minimax_h3_turbo_v4_step600_ema.safetensors` | `num_inference_steps: 9` (8 denoiser evaluations), `lora_scale: 1.0` | None | | Most aggressive speed preset (standard PEFT layout) | [`lightx2v/Minimax-h3-Turbo`](https://huggingface.co/lightx2v/Minimax-h3-Turbo), `minimax_h3_fl2v_turbo_4step_v0.1.safetensors` | `num_inference_steps: 5` (4 denoiser evaluations), `lora_scale: 1.0`, `lora_alpha: 8` | None | | Realistic people style | [`fal/MiniMax-H3-Realism-People-LoRA`](https://huggingface.co/fal/MiniMax-H3-Realism-People-LoRA), `h3-realism-people-t2v-i2v-r2v.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 0.7` | Include `r34l1sm` in the prompt | +| Spatial and physics | [`Jojocodex/minimax-h3-spatial-physics-lora`](https://huggingface.co/Jojocodex/minimax-h3-spatial-physics-lora), `wushu_spatial_physics_clean_3000_pruned.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 0.4` | None. Describe collisions, stacking, and gravity-consistent motion in the prompt | +| Wushu action | [`Jojocodex/minimax-h3-wushu-action-lora`](https://huggingface.co/Jojocodex/minimax-h3-wushu-action-lora), `wushu_action_h3_lora_v5_3000_pruned.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 0.8` | Put `wushu_action` at the start of the prompt | +| Camera motion | [`Jojocodex/minimax-h3-yunjing-lora`](https://huggingface.co/Jojocodex/minimax-h3-yunjing-lora), `camera_motion_h3_lora_v1_3000_pruned.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 0.8` | Put `camera motion` at the start of the prompt | +| Looping sketch anime | [`Inner-Reflections/MiniMax-H3-Looping-Sketch-Anime`](https://huggingface.co/Inner-Reflections/MiniMax-H3-Looping-Sketch-Anime), `minimax_h3_looping_sketch_anime_v1.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 1.0` | Include the author's hand-drawn sketch style phrase in the prompt | +| 3D-to-real detail slider | [`siraxe/3d_to_real_detail_slider_H3`](https://huggingface.co/siraxe/3d_to_real_detail_slider_H3), `3d_to_real_H3_detail_slider_bf16.safetensors` | Keep the normal `num_inference_steps: 50` schedule; start with `lora_scale: 1.0` | None. Sweep the scale; this is a slider, not a fixed style | The H3 request field controls the number of sigma grid points, including the terminal zero; the denoising loop therefore runs one fewer model evaluation. This is why an adapter described as 8-step uses `9`, and a 4-step adapter uses `5`, in the request. -All three use the same launch shape. Pinning the filename is required for +Every recipe uses the same launch shape. Pinning the filename is required for repositories that publish multiple revisions, and is also recommended for a reproducible single-file recipe: diff --git a/python/sglang/multimodal_gen/configs/models/dits/minimax_h3.py b/python/sglang/multimodal_gen/configs/models/dits/minimax_h3.py index 9cd41febb2..934fda1f72 100644 --- a/python/sglang/multimodal_gen/configs/models/dits/minimax_h3.py +++ b/python/sglang/multimodal_gen/configs/models/dits/minimax_h3.py @@ -83,6 +83,26 @@ class MiniMaxH3DiTArchConfig(DiTArchConfig): } ) + # Kohya `networks.lora_minimax_h3` uses lora_unet_blocks_* + lora_down/up. + # The format adapter rewrites these first; keep the aliases here in case a + # raw key still reaches the loader. + lora_param_names_mapping: dict = field( + default_factory=lambda: { + r"^lora_unet_blocks_(\d+)_attn_qkv_proj\.lora_down$": r"blocks.\1.attn.qkv_proj.lora_A", + r"^lora_unet_blocks_(\d+)_attn_qkv_proj\.lora_up$": r"blocks.\1.attn.qkv_proj.lora_B", + r"^lora_unet_blocks_(\d+)_attn_qkv_proj\.(lora_[AB]|alpha)$": r"blocks.\1.attn.qkv_proj.\2", + r"^lora_unet_blocks_(\d+)_attn_out_proj\.lora_down$": r"blocks.\1.attn.out_proj.lora_A", + r"^lora_unet_blocks_(\d+)_attn_out_proj\.lora_up$": r"blocks.\1.attn.out_proj.lora_B", + r"^lora_unet_blocks_(\d+)_attn_out_proj\.(lora_[AB]|alpha)$": r"blocks.\1.attn.out_proj.\2", + r"^lora_unet_blocks_(\d+)_mlp_fc1\.lora_down$": r"blocks.\1.mlp.fc1.lora_A", + r"^lora_unet_blocks_(\d+)_mlp_fc1\.lora_up$": r"blocks.\1.mlp.fc1.lora_B", + r"^lora_unet_blocks_(\d+)_mlp_fc1\.(lora_[AB]|alpha)$": r"blocks.\1.mlp.fc1.\2", + r"^lora_unet_blocks_(\d+)_mlp_fc2\.lora_down$": r"blocks.\1.mlp.fc2.lora_A", + r"^lora_unet_blocks_(\d+)_mlp_fc2\.lora_up$": r"blocks.\1.mlp.fc2.lora_B", + r"^lora_unet_blocks_(\d+)_mlp_fc2\.(lora_[AB]|alpha)$": r"blocks.\1.mlp.fc2.\2", + } + ) + num_layers: int = 50 token_refiner_num_layers: int = 2 hidden_size: int = 5376 diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/lora/format_adapter.py b/python/sglang/multimodal_gen/runtime/pipelines_core/lora/format_adapter.py index 557756d05f..919c6bde79 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/lora/format_adapter.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/lora/format_adapter.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import re from enum import Enum from typing import Any, Dict, Iterable, Mapping, Optional @@ -23,6 +24,7 @@ class LoRAFormat(str, Enum): QWEN_IMAGE_STANDARD = "qwen-image-standard" XLABS_FLUX = "xlabs-ai" KOHYA_FLUX = "kohya-flux" + KOHYA_MINIMAX_H3 = "kohya-minimax-h3" WAN = "wan" AI_TOOLKIT_FLUX = "ai-toolkit-flux" @@ -62,6 +64,68 @@ def _looks_like_xlabs_flux_key(k: str) -> bool: return ".processor." in k or ".proj_lora" in k or ".qkv_lora" in k +_KOHYA_H3_KEY = re.compile( + r"^lora_unet_blocks_(\d+)_(attn_qkv_proj|attn_out_proj|mlp_fc1|mlp_fc2)" + r"\.(lora_down|lora_up|alpha)(?:\.weight)?$" +) +_KOHYA_H3_MODULES = { + "attn_qkv_proj": "attn.qkv_proj", + "attn_out_proj": "attn.out_proj", + "mlp_fc1": "mlp.fc1", + "mlp_fc2": "mlp.fc2", +} +_KOHYA_H3_SUFFIXES = { + "lora_down": "lora_A", + "lora_up": "lora_B", + "alpha": "alpha", +} + + +def _looks_like_kohya_minimax_h3(state_dict: Mapping[str, torch.Tensor]) -> bool: + """Kohya `networks.lora_minimax_h3` export: lora_unet_blocks_* fused names.""" + if not state_dict: + return False + return any( + k.startswith("lora_unet_blocks_") and "_attn_qkv_proj." in k for k in state_dict + ) + + +def _convert_kohya_minimax_h3( + state_dict: Mapping[str, torch.Tensor], + log: logging.Logger, +) -> Dict[str, torch.Tensor]: + """lora_unet_blocks_N_* + lora_down/up -> native H3 blocks.N.*.lora_A/B.""" + out: Dict[str, torch.Tensor] = {} + unmatched: list[str] = [] + + for name, tensor in state_dict.items(): + match = _KOHYA_H3_KEY.match(name) + if match is None: + unmatched.append(name) + out[name] = tensor + continue + layer, module, suffix = match.group(1), match.group(2), match.group(3) + target = ( + f"blocks.{layer}.{_KOHYA_H3_MODULES[module]}.{_KOHYA_H3_SUFFIXES[suffix]}" + ) + out[target] = tensor + + sample = _sample_keys(out.keys(), 20) + log.info( + "[LoRAFormatAdapter] after KOHYA_MINIMAX_H3 conversion, " + "unmatched=%d sample keys (<=20): %s", + len(unmatched), + ", ".join(sample), + ) + if unmatched: + log.warning( + "[LoRAFormatAdapter] KOHYA_MINIMAX_H3 left %d unmatched keys: %s", + len(unmatched), + ", ".join(_sample_keys(unmatched, 8)), + ) + return out + + def _looks_like_kohya_flux(state_dict: Mapping[str, torch.Tensor]) -> bool: """Kohya FLUX LoRA (flux_lora.py) under lora_unet_double/single_blocks_ prefixes.""" if not state_dict: @@ -152,6 +216,8 @@ def detect_lora_format_from_state_dict( if any(_looks_like_xlabs_flux_key(k) for k in keys): return LoRAFormat.XLABS_FLUX + if _looks_like_kohya_minimax_h3(state_dict): + return LoRAFormat.KOHYA_MINIMAX_H3 if _looks_like_kohya_flux(state_dict): return LoRAFormat.KOHYA_FLUX @@ -506,6 +572,9 @@ def convert_lora_state_dict_by_format( converted = _convert_xlabs_ai_via_diffusers(state_dict, log) return _convert_non_diffusers_sd_simple(converted, log) + if fmt == LoRAFormat.KOHYA_MINIMAX_H3: + return _convert_kohya_minimax_h3(state_dict, log) + if fmt == LoRAFormat.KOHYA_FLUX: converted = _convert_kohya_flux_via_diffusers(state_dict, log) return _convert_non_diffusers_sd_simple(converted, log)