[diffusion] refactor: share ModelOpt checkpoint conversion I/O (#42856)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-08 09:26:39 +08:00
committed by GitHub
co-authored by Mick Qian
parent 35608e48d1
commit 2489bd1d2e
4 changed files with 273 additions and 227 deletions
@@ -0,0 +1,130 @@
# SPDX-License-Identifier: Apache-2.0
import json
import pytest
import torch
from safetensors import safe_open
from safetensors.torch import save_file
from sglang.multimodal_gen.tools.build_modelopt_fp8_transformer import (
build_modelopt_fp8_transformer,
)
from sglang.multimodal_gen.tools.build_modelopt_nvfp4_transformer import (
build_modelopt_nvfp4_transformer,
)
from sglang.multimodal_gen.tools.modelopt_checkpoint import load_weight_map
def _write_checkpoint(path, config, shards, index_name):
path.mkdir(parents=True)
(path / "config.json").write_text(json.dumps(config))
weight_map = {}
for i, tensors in enumerate(shards):
filename = f"weights-{i}.safetensors"
save_file(tensors, path / filename, metadata={"source": "preserved"})
weight_map.update(dict.fromkeys(tensors, filename))
if index_name:
(path / index_name).write_text(json.dumps({"weight_map": weight_map}))
@pytest.mark.parametrize("kind", ["fp8", "nvfp4"])
@pytest.mark.parametrize("keep_bf16", [False, True])
@pytest.mark.parametrize(
"index_name",
[
None,
"diffusion_pytorch_model.safetensors.index.json",
"custom.safetensors.index.json",
],
)
def test_modelopt_checkpoint_conversion(tmp_path, kind, keep_bf16, index_name):
source, base, output = (tmp_path / name for name in ("source", "base", "output"))
quant = {"quant_method": "modelopt", "quant_algo": kind.upper(), "ignore": []}
weights = torch.arange(32).reshape(4, 8).to(torch.bfloat16)
serialized = weights if kind == "fp8" else weights.to(torch.uint8)
shards = [
{
f"{name}.weight": serialized.clone(),
f"{name}.weight_scale": torch.tensor(1.0),
}
for name in ("quantized", "fallback")
]
shards[0]["quantized.weight_quantizer._amax"] = torch.tensor(448.0)
if index_name is None:
shards = [shards[0] | shards[1]]
_write_checkpoint(
source / "transformer", {"quantization_config": quant}, shards, index_name
)
_write_checkpoint(base, {}, [{"fallback.weight": weights + 17}], None)
(source / "transformer" / "assets").mkdir()
(source / "transformer" / "assets" / "notes.txt").write_text("preserved")
kwargs = dict(
modelopt_hf_dir=str(source),
base_transformer_dir=str(base),
output_dir=str(output),
keep_bf16_patterns=["fallback"] if keep_bf16 else [],
)
if kind == "fp8":
backbone = tmp_path / "backbone.pt"
torch.save(
{
"model_state_dict": {
f"{name}.{quantizer}_quantizer._amax": torch.tensor(448.0)
for name in ("quantized", "fallback")
for quantizer in ("weight", "input")
}
},
backbone,
)
kwargs.update(modelopt_backbone_ckpt=str(backbone), model_type="none")
build = build_modelopt_fp8_transformer
else:
build = build_modelopt_nvfp4_transformer
stats = build(**kwargs)
with pytest.raises(FileExistsError, match="--overwrite"):
build(**kwargs)
(output / "stale.txt").write_text("remove on explicit overwrite")
assert build(**kwargs, overwrite=True) == stats
assert not (output / "stale.txt").exists()
assert (output / "assets" / "notes.txt").read_text() == "preserved"
weight_map, output_index = load_weight_map(str(output))
assert output_index == (index_name or "weights-0.safetensors.index.json")
assert not any("_quantizer." in key for key in weight_map)
tensors = {}
for filename in set(weight_map.values()):
with safe_open(output / filename, framework="pt") as shard:
assert shard.metadata()["source"] == "preserved"
assert (
json.loads(shard.metadata()["quantization_config"])["quant_algo"]
== kind.upper()
)
tensors.update({key: shard.get_tensor(key) for key in shard.keys()})
assert set(tensors) == set(weight_map)
quantized = weights.to(torch.float8_e4m3fn) if kind == "fp8" else serialized
for name, expected in (
("quantized", quantized),
("fallback", weights + 17 if keep_bf16 else quantized),
):
actual = tensors[f"{name}.weight"]
assert actual.dtype == expected.dtype
torch.testing.assert_close(actual.float(), expected.float(), rtol=0, atol=0)
assert ("fallback.weight_scale" in tensors) is not keep_bf16
index = json.loads((output / output_index).read_text())
assert index["metadata"]["total_size"] == sum(
t.numel() * t.element_size() for t in tensors.values()
)
def test_modelopt_index_selection(tmp_path):
with pytest.raises(ValueError, match="found 0 shard"):
load_weight_map(str(tmp_path))
for name in ("a", "b"):
save_file({name: torch.ones(1)}, tmp_path / f"{name}.safetensors")
with pytest.raises(ValueError, match="found 2 shard"):
load_weight_map(str(tmp_path))
for name in ("z", "a", "diffusion_pytorch_model", "model"):
filename = f"{name}.safetensors.index.json"
mapping = {name: "a.safetensors"}
(tmp_path / filename).write_text(json.dumps({"weight_map": mapping}))
assert load_weight_map(str(tmp_path)) == (mapping, filename)
@@ -26,10 +26,9 @@ import gc
import json
import os
import re
import shutil
from collections import defaultdict
from pathlib import Path
from typing import Callable, Iterable, Mapping, Sequence
from typing import Callable, Mapping, Sequence
import torch
from safetensors import safe_open
@@ -38,11 +37,14 @@ from safetensors.torch import load_file, save_file
from sglang.multimodal_gen.runtime.utils.quantization_utils import (
normalize_flat_modelopt_quant_config,
)
from sglang.multimodal_gen.tools.modelopt_checkpoint import (
load_config,
load_selected_tensors,
load_weight_map,
prepare_output_dir,
resolve_transformer_dir,
)
INDEX_FILENAMES = [
"model.safetensors.index.json",
"diffusion_pytorch_model.safetensors.index.json",
]
FP8_E4M3_MAXBOUND = 448.0
DEFAULT_FLUX2_KEEP_BF16_PATTERNS = [
r"^time_guidance_embed\.(timestep_embedder|guidance_embedder)\.linear_[12]$",
@@ -209,16 +211,6 @@ DEFAULT_QWEN_IMAGE_KEEP_BF16_PATTERNS = [
]
def _resolve_transformer_dir(path: str) -> str:
candidate = Path(path).expanduser().resolve()
if (candidate / "config.json").is_file():
return str(candidate)
transformer_dir = candidate / "transformer"
if (transformer_dir / "config.json").is_file():
return str(transformer_dir)
raise FileNotFoundError(f"Could not resolve a transformer directory from: {path}")
def _resolve_backbone_ckpt(path: str) -> str:
candidate = Path(path).expanduser().resolve()
if candidate.is_file():
@@ -229,53 +221,6 @@ def _resolve_backbone_ckpt(path: str) -> str:
raise FileNotFoundError(f"Could not resolve backbone.pt from: {path}")
def _find_index_file(model_dir: str) -> str | None:
for filename in INDEX_FILENAMES:
candidate = os.path.join(model_dir, filename)
if os.path.isfile(candidate):
return filename
matches = sorted(
filename
for filename in os.listdir(model_dir)
if filename.endswith(".safetensors.index.json")
)
return matches[0] if matches else None
def _load_weight_map(model_dir: str) -> tuple[dict[str, str], str | None]:
index_filename = _find_index_file(model_dir)
if index_filename is not None:
with open(os.path.join(model_dir, index_filename), encoding="utf-8") as f:
index_data = json.load(f)
return dict(index_data["weight_map"]), index_filename
safetensors_files = sorted(
filename
for filename in os.listdir(model_dir)
if filename.endswith(".safetensors")
)
if len(safetensors_files) != 1:
raise ValueError(
f"Expected an index file or a single safetensors shard in {model_dir}, "
f"found {len(safetensors_files)} shard(s)."
)
shard_name = safetensors_files[0]
with safe_open(
os.path.join(model_dir, shard_name), framework="pt", device="cpu"
) as f:
weight_map = {key: shard_name for key in f.keys()}
index_filename = f"{Path(shard_name).stem}.safetensors.index.json"
return weight_map, index_filename
def _load_config(model_dir: str) -> dict:
config_path = os.path.join(model_dir, "config.json")
with open(config_path, encoding="utf-8") as f:
return json.load(f)
def _load_first_shard_metadata(
model_dir: str, weight_map: Mapping[str, str]
) -> dict[str, str]:
@@ -512,37 +457,6 @@ def quantize_fp8_weight(
return quantized.cpu().contiguous()
def _copy_non_shard_files(source_dir: str, output_dir: str) -> None:
ignored = set(INDEX_FILENAMES)
for entry in os.listdir(source_dir):
if entry.endswith(".safetensors") or entry in ignored:
continue
source_path = os.path.join(source_dir, entry)
output_path = os.path.join(output_dir, entry)
if os.path.isdir(source_path):
shutil.copytree(source_path, output_path, dirs_exist_ok=True)
else:
shutil.copy2(source_path, output_path)
def _load_selected_tensors(
model_dir: str,
weight_map: Mapping[str, str],
tensor_names: Iterable[str],
) -> dict[str, torch.Tensor]:
tensors: dict[str, torch.Tensor] = {}
names_by_file: dict[str, list[str]] = defaultdict(list)
for name in tensor_names:
names_by_file[weight_map[name]].append(name)
for filename, names in names_by_file.items():
shard_path = os.path.join(model_dir, filename)
with safe_open(shard_path, framework="pt", device="cpu") as f:
for name in names:
tensors[name] = f.get_tensor(name).contiguous()
return tensors
def build_modelopt_fp8_transformer(
*,
modelopt_hf_dir: str,
@@ -554,13 +468,13 @@ def build_modelopt_fp8_transformer(
maxbound: float = FP8_E4M3_MAXBOUND,
overwrite: bool = False,
) -> dict[str, int]:
source_dir = _resolve_transformer_dir(modelopt_hf_dir)
source_dir = resolve_transformer_dir(modelopt_hf_dir)
backbone_ckpt_path = _resolve_backbone_ckpt(modelopt_backbone_ckpt)
base_dir = (
_resolve_transformer_dir(base_transformer_dir) if base_transformer_dir else None
resolve_transformer_dir(base_transformer_dir) if base_transformer_dir else None
)
config = _load_config(source_dir)
config = load_config(source_dir)
quant_config = config.get("quantization_config")
if not isinstance(quant_config, dict):
raise ValueError(
@@ -572,7 +486,7 @@ def build_modelopt_fp8_transformer(
"(quant_method=modelopt)."
)
source_weight_map_all, index_filename = _load_weight_map(source_dir)
source_weight_map_all, index_filename = load_weight_map(source_dir)
source_metadata = _load_first_shard_metadata(source_dir, source_weight_map_all)
is_ltx2_export = _is_ltx2_x0_export(
config=config,
@@ -596,17 +510,7 @@ def build_modelopt_fp8_transformer(
"BF16 fallback patterns are enabled, but --base-transformer-dir was not provided."
)
output_path = Path(output_dir).expanduser().resolve()
if output_path.exists():
if not overwrite:
raise FileExistsError(
f"Output directory already exists: {output_path}. "
"Use --overwrite to replace it."
)
shutil.rmtree(output_path)
output_path.mkdir(parents=True, exist_ok=True)
_copy_non_shard_files(source_dir, str(output_path))
output_path = prepare_output_dir(source_dir, output_dir, overwrite=overwrite)
if is_ltx2_export:
source_weight_map = {
@@ -618,7 +522,7 @@ def build_modelopt_fp8_transformer(
source_weight_map = source_weight_map_all
base_weight_map: dict[str, str] = {}
if base_dir is not None:
base_weight_map, _ = _load_weight_map(base_dir)
base_weight_map, _ = load_weight_map(base_dir)
fallback_weight_names = sorted(
weight_name
for weight_name in source_weight_map
@@ -681,7 +585,7 @@ def build_modelopt_fp8_transformer(
)
fallback_tensors = (
_load_selected_tensors(base_dir, base_weight_map, fallback_weight_names)
load_selected_tensors(base_dir, base_weight_map, fallback_weight_names)
if fallback_weight_names and base_dir is not None
else {}
)
@@ -16,18 +16,20 @@ import argparse
import json
import os
import re
import shutil
from collections import defaultdict
from pathlib import Path
from typing import Iterable, Mapping, Sequence
from typing import Mapping, Sequence
from safetensors import safe_open
from safetensors.torch import load_file, save_file
INDEX_FILENAMES = [
"model.safetensors.index.json",
"diffusion_pytorch_model.safetensors.index.json",
]
from sglang.multimodal_gen.tools.modelopt_checkpoint import (
load_config,
load_selected_tensors,
load_weight_map,
prepare_output_dir,
resolve_transformer_dir,
)
DEFAULT_FLUX1_NVFP4_FALLBACK_PATTERNS = [
"transformer_blocks.*.norm1.linear*",
@@ -49,100 +51,12 @@ _TENSOR_MODULE_SUFFIXES = (
)
def _resolve_transformer_dir(path: str) -> str:
candidate = Path(path).expanduser().resolve()
if (candidate / "config.json").is_file():
return str(candidate)
transformer_dir = candidate / "transformer"
if (transformer_dir / "config.json").is_file():
return str(transformer_dir)
raise FileNotFoundError(f"Could not resolve a transformer directory from: {path}")
def _find_index_file(model_dir: str) -> str | None:
for filename in INDEX_FILENAMES:
candidate = os.path.join(model_dir, filename)
if os.path.isfile(candidate):
return filename
matches = sorted(
filename
for filename in os.listdir(model_dir)
if filename.endswith(".safetensors.index.json")
)
return matches[0] if matches else None
def _load_weight_map(model_dir: str) -> tuple[dict[str, str], str | None]:
index_filename = _find_index_file(model_dir)
if index_filename is not None:
with open(os.path.join(model_dir, index_filename), encoding="utf-8") as f:
index_data = json.load(f)
return dict(index_data["weight_map"]), index_filename
safetensors_files = sorted(
filename
for filename in os.listdir(model_dir)
if filename.endswith(".safetensors")
)
if len(safetensors_files) != 1:
raise ValueError(
f"Expected an index file or a single safetensors shard in {model_dir}, "
f"found {len(safetensors_files)} shard(s)."
)
shard_name = safetensors_files[0]
with safe_open(
os.path.join(model_dir, shard_name), framework="pt", device="cpu"
) as f:
weight_map = {key: shard_name for key in f.keys()}
index_filename = f"{Path(shard_name).stem}.safetensors.index.json"
return weight_map, index_filename
def _load_config(model_dir: str) -> dict:
config_path = os.path.join(model_dir, "config.json")
with open(config_path, encoding="utf-8") as f:
return json.load(f)
def _write_config(model_dir: Path, config: Mapping[str, object]) -> None:
with open(model_dir / "config.json", "w", encoding="utf-8") as f:
json.dump(config, f, indent=2, sort_keys=True)
f.write("\n")
def _copy_non_shard_files(source_dir: str, output_dir: str) -> None:
ignored = set(INDEX_FILENAMES)
for entry in os.listdir(source_dir):
if entry.endswith(".safetensors") or entry in ignored:
continue
source_path = os.path.join(source_dir, entry)
output_path = os.path.join(output_dir, entry)
if os.path.isdir(source_path):
shutil.copytree(source_path, output_path, dirs_exist_ok=True)
else:
shutil.copy2(source_path, output_path)
def _load_selected_tensors(
model_dir: str,
weight_map: Mapping[str, str],
tensor_names: Iterable[str],
):
tensors = {}
names_by_file: dict[str, list[str]] = defaultdict(list)
for name in tensor_names:
names_by_file[weight_map[name]].append(name)
for filename, names in names_by_file.items():
shard_path = os.path.join(model_dir, filename)
with safe_open(shard_path, framework="pt", device="cpu") as f:
for name in names:
tensors[name] = f.get_tensor(name).contiguous()
return tensors
def _module_name_for_tensor(tensor_name: str) -> str:
for suffix in _TENSOR_MODULE_SUFFIXES:
if tensor_name.endswith(suffix):
@@ -210,8 +124,8 @@ def build_modelopt_nvfp4_transformer(
swap_weight_nibbles: bool | None = None,
overwrite: bool = False,
) -> dict[str, int | bool]:
source_dir = _resolve_transformer_dir(modelopt_hf_dir)
base_dir = _resolve_transformer_dir(base_transformer_dir)
source_dir = resolve_transformer_dir(modelopt_hf_dir)
base_dir = resolve_transformer_dir(base_transformer_dir)
patterns = _preset_patterns(pattern_preset)
if keep_bf16_patterns:
@@ -221,28 +135,18 @@ def build_modelopt_nvfp4_transformer(
swap_weight_nibbles if swap_weight_nibbles is not None else False
)
output_config = _updated_quant_config(
_load_config(source_dir),
load_config(source_dir),
fallback_patterns=patterns,
swap_weight_nibbles=resolved_swap_weight_nibbles,
)
quant_config = output_config["quantization_config"]
serialized_quant_config = json.dumps(quant_config, sort_keys=True)
output_path = Path(output_dir).expanduser().resolve()
if output_path.exists():
if not overwrite:
raise FileExistsError(
f"Output directory already exists: {output_path}. "
"Use --overwrite to replace it."
)
shutil.rmtree(output_path)
output_path.mkdir(parents=True, exist_ok=True)
_copy_non_shard_files(source_dir, str(output_path))
output_path = prepare_output_dir(source_dir, output_dir, overwrite=overwrite)
_write_config(output_path, output_config)
source_weight_map, index_filename = _load_weight_map(source_dir)
base_weight_map, _ = _load_weight_map(base_dir)
source_weight_map, index_filename = load_weight_map(source_dir)
base_weight_map, _ = load_weight_map(base_dir)
fallback_tensor_names = sorted(
name
@@ -250,7 +154,7 @@ def build_modelopt_nvfp4_transformer(
if name in source_weight_map
and _matches_any_pattern(_module_name_for_tensor(name), patterns)
)
fallback_tensors = _load_selected_tensors(
fallback_tensors = load_selected_tensors(
base_dir,
base_weight_map,
fallback_tensor_names,
@@ -302,11 +206,6 @@ def build_modelopt_nvfp4_transformer(
updated_weight_map[name] = filename
total_size += tensor.element_size() * tensor.numel()
if index_filename is None:
raise ValueError(
"Expected a sharded or indexed ModelOpt HF export, but no index file was found."
)
with open(output_path / index_filename, "w", encoding="utf-8") as f:
json.dump(
{
@@ -0,0 +1,113 @@
# SPDX-License-Identifier: Apache-2.0
"""Checkpoint I/O shared by the ModelOpt transformer conversion tools."""
import json
import os
import shutil
from collections import defaultdict
from pathlib import Path
from typing import Iterable, Mapping
import torch
from safetensors import safe_open
_INDEX_FILENAMES = (
"model.safetensors.index.json",
"diffusion_pytorch_model.safetensors.index.json",
)
def resolve_transformer_dir(path: str) -> str:
candidate = Path(path).expanduser().resolve()
if (candidate / "config.json").is_file():
return str(candidate)
transformer_dir = candidate / "transformer"
if (transformer_dir / "config.json").is_file():
return str(transformer_dir)
raise FileNotFoundError(f"Could not resolve a transformer directory from: {path}")
def load_weight_map(model_dir: str) -> tuple[dict[str, str], str]:
index_filename = next(
(
name
for name in _INDEX_FILENAMES
if os.path.isfile(os.path.join(model_dir, name))
),
None,
)
if index_filename is None:
matches = sorted(
name
for name in os.listdir(model_dir)
if name.endswith(".safetensors.index.json")
)
index_filename = matches[0] if matches else None
if index_filename is not None:
with open(os.path.join(model_dir, index_filename), encoding="utf-8") as f:
index_data = json.load(f)
return dict(index_data["weight_map"]), index_filename
safetensors_files = sorted(
filename
for filename in os.listdir(model_dir)
if filename.endswith(".safetensors")
)
if len(safetensors_files) != 1:
raise ValueError(
f"Expected an index file or a single safetensors shard in {model_dir}, "
f"found {len(safetensors_files)} shard(s)."
)
shard_name = safetensors_files[0]
with safe_open(
os.path.join(model_dir, shard_name), framework="pt", device="cpu"
) as f:
weight_map = {key: shard_name for key in f.keys()}
return weight_map, f"{Path(shard_name).stem}.safetensors.index.json"
def load_config(model_dir: str) -> dict:
with open(os.path.join(model_dir, "config.json"), encoding="utf-8") as f:
return json.load(f)
def prepare_output_dir(source_dir: str, output_dir: str, *, overwrite: bool) -> Path:
output_path = Path(output_dir).expanduser().resolve()
if output_path.exists():
if not overwrite:
raise FileExistsError(
f"Output directory already exists: {output_path}. "
"Use --overwrite to replace it."
)
shutil.rmtree(output_path)
output_path.mkdir(parents=True, exist_ok=True)
for entry in os.listdir(source_dir):
if entry.endswith(".safetensors") or entry in _INDEX_FILENAMES:
continue
source_path = os.path.join(source_dir, entry)
destination = output_path / entry
if os.path.isdir(source_path):
shutil.copytree(source_path, destination, dirs_exist_ok=True)
else:
shutil.copy2(source_path, destination)
return output_path
def load_selected_tensors(
model_dir: str,
weight_map: Mapping[str, str],
tensor_names: Iterable[str],
) -> dict[str, torch.Tensor]:
tensors: dict[str, torch.Tensor] = {}
names_by_file: dict[str, list[str]] = defaultdict(list)
for name in tensor_names:
names_by_file[weight_map[name]].append(name)
for filename, names in names_by_file.items():
shard_path = os.path.join(model_dir, filename)
with safe_open(shard_path, framework="pt", device="cpu") as f:
for name in names:
tensors[name] = f.get_tensor(name).contiguous()
return tensors