[diffusion] refactor: share ModelOpt checkpoint conversion I/O (#42856)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,130 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors import safe_open
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from sglang.multimodal_gen.tools.build_modelopt_fp8_transformer import (
|
||||
build_modelopt_fp8_transformer,
|
||||
)
|
||||
from sglang.multimodal_gen.tools.build_modelopt_nvfp4_transformer import (
|
||||
build_modelopt_nvfp4_transformer,
|
||||
)
|
||||
from sglang.multimodal_gen.tools.modelopt_checkpoint import load_weight_map
|
||||
|
||||
|
||||
def _write_checkpoint(path, config, shards, index_name):
|
||||
path.mkdir(parents=True)
|
||||
(path / "config.json").write_text(json.dumps(config))
|
||||
weight_map = {}
|
||||
for i, tensors in enumerate(shards):
|
||||
filename = f"weights-{i}.safetensors"
|
||||
save_file(tensors, path / filename, metadata={"source": "preserved"})
|
||||
weight_map.update(dict.fromkeys(tensors, filename))
|
||||
if index_name:
|
||||
(path / index_name).write_text(json.dumps({"weight_map": weight_map}))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["fp8", "nvfp4"])
|
||||
@pytest.mark.parametrize("keep_bf16", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"index_name",
|
||||
[
|
||||
None,
|
||||
"diffusion_pytorch_model.safetensors.index.json",
|
||||
"custom.safetensors.index.json",
|
||||
],
|
||||
)
|
||||
def test_modelopt_checkpoint_conversion(tmp_path, kind, keep_bf16, index_name):
|
||||
source, base, output = (tmp_path / name for name in ("source", "base", "output"))
|
||||
quant = {"quant_method": "modelopt", "quant_algo": kind.upper(), "ignore": []}
|
||||
weights = torch.arange(32).reshape(4, 8).to(torch.bfloat16)
|
||||
serialized = weights if kind == "fp8" else weights.to(torch.uint8)
|
||||
shards = [
|
||||
{
|
||||
f"{name}.weight": serialized.clone(),
|
||||
f"{name}.weight_scale": torch.tensor(1.0),
|
||||
}
|
||||
for name in ("quantized", "fallback")
|
||||
]
|
||||
shards[0]["quantized.weight_quantizer._amax"] = torch.tensor(448.0)
|
||||
if index_name is None:
|
||||
shards = [shards[0] | shards[1]]
|
||||
_write_checkpoint(
|
||||
source / "transformer", {"quantization_config": quant}, shards, index_name
|
||||
)
|
||||
_write_checkpoint(base, {}, [{"fallback.weight": weights + 17}], None)
|
||||
(source / "transformer" / "assets").mkdir()
|
||||
(source / "transformer" / "assets" / "notes.txt").write_text("preserved")
|
||||
kwargs = dict(
|
||||
modelopt_hf_dir=str(source),
|
||||
base_transformer_dir=str(base),
|
||||
output_dir=str(output),
|
||||
keep_bf16_patterns=["fallback"] if keep_bf16 else [],
|
||||
)
|
||||
if kind == "fp8":
|
||||
backbone = tmp_path / "backbone.pt"
|
||||
torch.save(
|
||||
{
|
||||
"model_state_dict": {
|
||||
f"{name}.{quantizer}_quantizer._amax": torch.tensor(448.0)
|
||||
for name in ("quantized", "fallback")
|
||||
for quantizer in ("weight", "input")
|
||||
}
|
||||
},
|
||||
backbone,
|
||||
)
|
||||
kwargs.update(modelopt_backbone_ckpt=str(backbone), model_type="none")
|
||||
build = build_modelopt_fp8_transformer
|
||||
else:
|
||||
build = build_modelopt_nvfp4_transformer
|
||||
stats = build(**kwargs)
|
||||
with pytest.raises(FileExistsError, match="--overwrite"):
|
||||
build(**kwargs)
|
||||
(output / "stale.txt").write_text("remove on explicit overwrite")
|
||||
assert build(**kwargs, overwrite=True) == stats
|
||||
assert not (output / "stale.txt").exists()
|
||||
assert (output / "assets" / "notes.txt").read_text() == "preserved"
|
||||
weight_map, output_index = load_weight_map(str(output))
|
||||
assert output_index == (index_name or "weights-0.safetensors.index.json")
|
||||
assert not any("_quantizer." in key for key in weight_map)
|
||||
tensors = {}
|
||||
for filename in set(weight_map.values()):
|
||||
with safe_open(output / filename, framework="pt") as shard:
|
||||
assert shard.metadata()["source"] == "preserved"
|
||||
assert (
|
||||
json.loads(shard.metadata()["quantization_config"])["quant_algo"]
|
||||
== kind.upper()
|
||||
)
|
||||
tensors.update({key: shard.get_tensor(key) for key in shard.keys()})
|
||||
assert set(tensors) == set(weight_map)
|
||||
quantized = weights.to(torch.float8_e4m3fn) if kind == "fp8" else serialized
|
||||
for name, expected in (
|
||||
("quantized", quantized),
|
||||
("fallback", weights + 17 if keep_bf16 else quantized),
|
||||
):
|
||||
actual = tensors[f"{name}.weight"]
|
||||
assert actual.dtype == expected.dtype
|
||||
torch.testing.assert_close(actual.float(), expected.float(), rtol=0, atol=0)
|
||||
assert ("fallback.weight_scale" in tensors) is not keep_bf16
|
||||
index = json.loads((output / output_index).read_text())
|
||||
assert index["metadata"]["total_size"] == sum(
|
||||
t.numel() * t.element_size() for t in tensors.values()
|
||||
)
|
||||
|
||||
|
||||
def test_modelopt_index_selection(tmp_path):
|
||||
with pytest.raises(ValueError, match="found 0 shard"):
|
||||
load_weight_map(str(tmp_path))
|
||||
for name in ("a", "b"):
|
||||
save_file({name: torch.ones(1)}, tmp_path / f"{name}.safetensors")
|
||||
with pytest.raises(ValueError, match="found 2 shard"):
|
||||
load_weight_map(str(tmp_path))
|
||||
for name in ("z", "a", "diffusion_pytorch_model", "model"):
|
||||
filename = f"{name}.safetensors.index.json"
|
||||
mapping = {name: "a.safetensors"}
|
||||
(tmp_path / filename).write_text(json.dumps({"weight_map": mapping}))
|
||||
assert load_weight_map(str(tmp_path)) == (mapping, filename)
|
||||
@@ -26,10 +26,9 @@ import gc
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Callable, Iterable, Mapping, Sequence
|
||||
from typing import Callable, Mapping, Sequence
|
||||
|
||||
import torch
|
||||
from safetensors import safe_open
|
||||
@@ -38,11 +37,14 @@ from safetensors.torch import load_file, save_file
|
||||
from sglang.multimodal_gen.runtime.utils.quantization_utils import (
|
||||
normalize_flat_modelopt_quant_config,
|
||||
)
|
||||
from sglang.multimodal_gen.tools.modelopt_checkpoint import (
|
||||
load_config,
|
||||
load_selected_tensors,
|
||||
load_weight_map,
|
||||
prepare_output_dir,
|
||||
resolve_transformer_dir,
|
||||
)
|
||||
|
||||
INDEX_FILENAMES = [
|
||||
"model.safetensors.index.json",
|
||||
"diffusion_pytorch_model.safetensors.index.json",
|
||||
]
|
||||
FP8_E4M3_MAXBOUND = 448.0
|
||||
DEFAULT_FLUX2_KEEP_BF16_PATTERNS = [
|
||||
r"^time_guidance_embed\.(timestep_embedder|guidance_embedder)\.linear_[12]$",
|
||||
@@ -209,16 +211,6 @@ DEFAULT_QWEN_IMAGE_KEEP_BF16_PATTERNS = [
|
||||
]
|
||||
|
||||
|
||||
def _resolve_transformer_dir(path: str) -> str:
|
||||
candidate = Path(path).expanduser().resolve()
|
||||
if (candidate / "config.json").is_file():
|
||||
return str(candidate)
|
||||
transformer_dir = candidate / "transformer"
|
||||
if (transformer_dir / "config.json").is_file():
|
||||
return str(transformer_dir)
|
||||
raise FileNotFoundError(f"Could not resolve a transformer directory from: {path}")
|
||||
|
||||
|
||||
def _resolve_backbone_ckpt(path: str) -> str:
|
||||
candidate = Path(path).expanduser().resolve()
|
||||
if candidate.is_file():
|
||||
@@ -229,53 +221,6 @@ def _resolve_backbone_ckpt(path: str) -> str:
|
||||
raise FileNotFoundError(f"Could not resolve backbone.pt from: {path}")
|
||||
|
||||
|
||||
def _find_index_file(model_dir: str) -> str | None:
|
||||
for filename in INDEX_FILENAMES:
|
||||
candidate = os.path.join(model_dir, filename)
|
||||
if os.path.isfile(candidate):
|
||||
return filename
|
||||
|
||||
matches = sorted(
|
||||
filename
|
||||
for filename in os.listdir(model_dir)
|
||||
if filename.endswith(".safetensors.index.json")
|
||||
)
|
||||
return matches[0] if matches else None
|
||||
|
||||
|
||||
def _load_weight_map(model_dir: str) -> tuple[dict[str, str], str | None]:
|
||||
index_filename = _find_index_file(model_dir)
|
||||
if index_filename is not None:
|
||||
with open(os.path.join(model_dir, index_filename), encoding="utf-8") as f:
|
||||
index_data = json.load(f)
|
||||
return dict(index_data["weight_map"]), index_filename
|
||||
|
||||
safetensors_files = sorted(
|
||||
filename
|
||||
for filename in os.listdir(model_dir)
|
||||
if filename.endswith(".safetensors")
|
||||
)
|
||||
if len(safetensors_files) != 1:
|
||||
raise ValueError(
|
||||
f"Expected an index file or a single safetensors shard in {model_dir}, "
|
||||
f"found {len(safetensors_files)} shard(s)."
|
||||
)
|
||||
|
||||
shard_name = safetensors_files[0]
|
||||
with safe_open(
|
||||
os.path.join(model_dir, shard_name), framework="pt", device="cpu"
|
||||
) as f:
|
||||
weight_map = {key: shard_name for key in f.keys()}
|
||||
index_filename = f"{Path(shard_name).stem}.safetensors.index.json"
|
||||
return weight_map, index_filename
|
||||
|
||||
|
||||
def _load_config(model_dir: str) -> dict:
|
||||
config_path = os.path.join(model_dir, "config.json")
|
||||
with open(config_path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def _load_first_shard_metadata(
|
||||
model_dir: str, weight_map: Mapping[str, str]
|
||||
) -> dict[str, str]:
|
||||
@@ -512,37 +457,6 @@ def quantize_fp8_weight(
|
||||
return quantized.cpu().contiguous()
|
||||
|
||||
|
||||
def _copy_non_shard_files(source_dir: str, output_dir: str) -> None:
|
||||
ignored = set(INDEX_FILENAMES)
|
||||
for entry in os.listdir(source_dir):
|
||||
if entry.endswith(".safetensors") or entry in ignored:
|
||||
continue
|
||||
source_path = os.path.join(source_dir, entry)
|
||||
output_path = os.path.join(output_dir, entry)
|
||||
if os.path.isdir(source_path):
|
||||
shutil.copytree(source_path, output_path, dirs_exist_ok=True)
|
||||
else:
|
||||
shutil.copy2(source_path, output_path)
|
||||
|
||||
|
||||
def _load_selected_tensors(
|
||||
model_dir: str,
|
||||
weight_map: Mapping[str, str],
|
||||
tensor_names: Iterable[str],
|
||||
) -> dict[str, torch.Tensor]:
|
||||
tensors: dict[str, torch.Tensor] = {}
|
||||
names_by_file: dict[str, list[str]] = defaultdict(list)
|
||||
for name in tensor_names:
|
||||
names_by_file[weight_map[name]].append(name)
|
||||
|
||||
for filename, names in names_by_file.items():
|
||||
shard_path = os.path.join(model_dir, filename)
|
||||
with safe_open(shard_path, framework="pt", device="cpu") as f:
|
||||
for name in names:
|
||||
tensors[name] = f.get_tensor(name).contiguous()
|
||||
return tensors
|
||||
|
||||
|
||||
def build_modelopt_fp8_transformer(
|
||||
*,
|
||||
modelopt_hf_dir: str,
|
||||
@@ -554,13 +468,13 @@ def build_modelopt_fp8_transformer(
|
||||
maxbound: float = FP8_E4M3_MAXBOUND,
|
||||
overwrite: bool = False,
|
||||
) -> dict[str, int]:
|
||||
source_dir = _resolve_transformer_dir(modelopt_hf_dir)
|
||||
source_dir = resolve_transformer_dir(modelopt_hf_dir)
|
||||
backbone_ckpt_path = _resolve_backbone_ckpt(modelopt_backbone_ckpt)
|
||||
base_dir = (
|
||||
_resolve_transformer_dir(base_transformer_dir) if base_transformer_dir else None
|
||||
resolve_transformer_dir(base_transformer_dir) if base_transformer_dir else None
|
||||
)
|
||||
|
||||
config = _load_config(source_dir)
|
||||
config = load_config(source_dir)
|
||||
quant_config = config.get("quantization_config")
|
||||
if not isinstance(quant_config, dict):
|
||||
raise ValueError(
|
||||
@@ -572,7 +486,7 @@ def build_modelopt_fp8_transformer(
|
||||
"(quant_method=modelopt)."
|
||||
)
|
||||
|
||||
source_weight_map_all, index_filename = _load_weight_map(source_dir)
|
||||
source_weight_map_all, index_filename = load_weight_map(source_dir)
|
||||
source_metadata = _load_first_shard_metadata(source_dir, source_weight_map_all)
|
||||
is_ltx2_export = _is_ltx2_x0_export(
|
||||
config=config,
|
||||
@@ -596,17 +510,7 @@ def build_modelopt_fp8_transformer(
|
||||
"BF16 fallback patterns are enabled, but --base-transformer-dir was not provided."
|
||||
)
|
||||
|
||||
output_path = Path(output_dir).expanduser().resolve()
|
||||
if output_path.exists():
|
||||
if not overwrite:
|
||||
raise FileExistsError(
|
||||
f"Output directory already exists: {output_path}. "
|
||||
"Use --overwrite to replace it."
|
||||
)
|
||||
shutil.rmtree(output_path)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
_copy_non_shard_files(source_dir, str(output_path))
|
||||
output_path = prepare_output_dir(source_dir, output_dir, overwrite=overwrite)
|
||||
|
||||
if is_ltx2_export:
|
||||
source_weight_map = {
|
||||
@@ -618,7 +522,7 @@ def build_modelopt_fp8_transformer(
|
||||
source_weight_map = source_weight_map_all
|
||||
base_weight_map: dict[str, str] = {}
|
||||
if base_dir is not None:
|
||||
base_weight_map, _ = _load_weight_map(base_dir)
|
||||
base_weight_map, _ = load_weight_map(base_dir)
|
||||
fallback_weight_names = sorted(
|
||||
weight_name
|
||||
for weight_name in source_weight_map
|
||||
@@ -681,7 +585,7 @@ def build_modelopt_fp8_transformer(
|
||||
)
|
||||
|
||||
fallback_tensors = (
|
||||
_load_selected_tensors(base_dir, base_weight_map, fallback_weight_names)
|
||||
load_selected_tensors(base_dir, base_weight_map, fallback_weight_names)
|
||||
if fallback_weight_names and base_dir is not None
|
||||
else {}
|
||||
)
|
||||
|
||||
@@ -16,18 +16,20 @@ import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Mapping, Sequence
|
||||
from typing import Mapping, Sequence
|
||||
|
||||
from safetensors import safe_open
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
INDEX_FILENAMES = [
|
||||
"model.safetensors.index.json",
|
||||
"diffusion_pytorch_model.safetensors.index.json",
|
||||
]
|
||||
from sglang.multimodal_gen.tools.modelopt_checkpoint import (
|
||||
load_config,
|
||||
load_selected_tensors,
|
||||
load_weight_map,
|
||||
prepare_output_dir,
|
||||
resolve_transformer_dir,
|
||||
)
|
||||
|
||||
DEFAULT_FLUX1_NVFP4_FALLBACK_PATTERNS = [
|
||||
"transformer_blocks.*.norm1.linear*",
|
||||
@@ -49,100 +51,12 @@ _TENSOR_MODULE_SUFFIXES = (
|
||||
)
|
||||
|
||||
|
||||
def _resolve_transformer_dir(path: str) -> str:
|
||||
candidate = Path(path).expanduser().resolve()
|
||||
if (candidate / "config.json").is_file():
|
||||
return str(candidate)
|
||||
transformer_dir = candidate / "transformer"
|
||||
if (transformer_dir / "config.json").is_file():
|
||||
return str(transformer_dir)
|
||||
raise FileNotFoundError(f"Could not resolve a transformer directory from: {path}")
|
||||
|
||||
|
||||
def _find_index_file(model_dir: str) -> str | None:
|
||||
for filename in INDEX_FILENAMES:
|
||||
candidate = os.path.join(model_dir, filename)
|
||||
if os.path.isfile(candidate):
|
||||
return filename
|
||||
|
||||
matches = sorted(
|
||||
filename
|
||||
for filename in os.listdir(model_dir)
|
||||
if filename.endswith(".safetensors.index.json")
|
||||
)
|
||||
return matches[0] if matches else None
|
||||
|
||||
|
||||
def _load_weight_map(model_dir: str) -> tuple[dict[str, str], str | None]:
|
||||
index_filename = _find_index_file(model_dir)
|
||||
if index_filename is not None:
|
||||
with open(os.path.join(model_dir, index_filename), encoding="utf-8") as f:
|
||||
index_data = json.load(f)
|
||||
return dict(index_data["weight_map"]), index_filename
|
||||
|
||||
safetensors_files = sorted(
|
||||
filename
|
||||
for filename in os.listdir(model_dir)
|
||||
if filename.endswith(".safetensors")
|
||||
)
|
||||
if len(safetensors_files) != 1:
|
||||
raise ValueError(
|
||||
f"Expected an index file or a single safetensors shard in {model_dir}, "
|
||||
f"found {len(safetensors_files)} shard(s)."
|
||||
)
|
||||
|
||||
shard_name = safetensors_files[0]
|
||||
with safe_open(
|
||||
os.path.join(model_dir, shard_name), framework="pt", device="cpu"
|
||||
) as f:
|
||||
weight_map = {key: shard_name for key in f.keys()}
|
||||
index_filename = f"{Path(shard_name).stem}.safetensors.index.json"
|
||||
return weight_map, index_filename
|
||||
|
||||
|
||||
def _load_config(model_dir: str) -> dict:
|
||||
config_path = os.path.join(model_dir, "config.json")
|
||||
with open(config_path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def _write_config(model_dir: Path, config: Mapping[str, object]) -> None:
|
||||
with open(model_dir / "config.json", "w", encoding="utf-8") as f:
|
||||
json.dump(config, f, indent=2, sort_keys=True)
|
||||
f.write("\n")
|
||||
|
||||
|
||||
def _copy_non_shard_files(source_dir: str, output_dir: str) -> None:
|
||||
ignored = set(INDEX_FILENAMES)
|
||||
for entry in os.listdir(source_dir):
|
||||
if entry.endswith(".safetensors") or entry in ignored:
|
||||
continue
|
||||
source_path = os.path.join(source_dir, entry)
|
||||
output_path = os.path.join(output_dir, entry)
|
||||
if os.path.isdir(source_path):
|
||||
shutil.copytree(source_path, output_path, dirs_exist_ok=True)
|
||||
else:
|
||||
shutil.copy2(source_path, output_path)
|
||||
|
||||
|
||||
def _load_selected_tensors(
|
||||
model_dir: str,
|
||||
weight_map: Mapping[str, str],
|
||||
tensor_names: Iterable[str],
|
||||
):
|
||||
tensors = {}
|
||||
names_by_file: dict[str, list[str]] = defaultdict(list)
|
||||
for name in tensor_names:
|
||||
names_by_file[weight_map[name]].append(name)
|
||||
|
||||
for filename, names in names_by_file.items():
|
||||
shard_path = os.path.join(model_dir, filename)
|
||||
with safe_open(shard_path, framework="pt", device="cpu") as f:
|
||||
for name in names:
|
||||
tensors[name] = f.get_tensor(name).contiguous()
|
||||
return tensors
|
||||
|
||||
|
||||
def _module_name_for_tensor(tensor_name: str) -> str:
|
||||
for suffix in _TENSOR_MODULE_SUFFIXES:
|
||||
if tensor_name.endswith(suffix):
|
||||
@@ -210,8 +124,8 @@ def build_modelopt_nvfp4_transformer(
|
||||
swap_weight_nibbles: bool | None = None,
|
||||
overwrite: bool = False,
|
||||
) -> dict[str, int | bool]:
|
||||
source_dir = _resolve_transformer_dir(modelopt_hf_dir)
|
||||
base_dir = _resolve_transformer_dir(base_transformer_dir)
|
||||
source_dir = resolve_transformer_dir(modelopt_hf_dir)
|
||||
base_dir = resolve_transformer_dir(base_transformer_dir)
|
||||
|
||||
patterns = _preset_patterns(pattern_preset)
|
||||
if keep_bf16_patterns:
|
||||
@@ -221,28 +135,18 @@ def build_modelopt_nvfp4_transformer(
|
||||
swap_weight_nibbles if swap_weight_nibbles is not None else False
|
||||
)
|
||||
output_config = _updated_quant_config(
|
||||
_load_config(source_dir),
|
||||
load_config(source_dir),
|
||||
fallback_patterns=patterns,
|
||||
swap_weight_nibbles=resolved_swap_weight_nibbles,
|
||||
)
|
||||
quant_config = output_config["quantization_config"]
|
||||
serialized_quant_config = json.dumps(quant_config, sort_keys=True)
|
||||
|
||||
output_path = Path(output_dir).expanduser().resolve()
|
||||
if output_path.exists():
|
||||
if not overwrite:
|
||||
raise FileExistsError(
|
||||
f"Output directory already exists: {output_path}. "
|
||||
"Use --overwrite to replace it."
|
||||
)
|
||||
shutil.rmtree(output_path)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
_copy_non_shard_files(source_dir, str(output_path))
|
||||
output_path = prepare_output_dir(source_dir, output_dir, overwrite=overwrite)
|
||||
_write_config(output_path, output_config)
|
||||
|
||||
source_weight_map, index_filename = _load_weight_map(source_dir)
|
||||
base_weight_map, _ = _load_weight_map(base_dir)
|
||||
source_weight_map, index_filename = load_weight_map(source_dir)
|
||||
base_weight_map, _ = load_weight_map(base_dir)
|
||||
|
||||
fallback_tensor_names = sorted(
|
||||
name
|
||||
@@ -250,7 +154,7 @@ def build_modelopt_nvfp4_transformer(
|
||||
if name in source_weight_map
|
||||
and _matches_any_pattern(_module_name_for_tensor(name), patterns)
|
||||
)
|
||||
fallback_tensors = _load_selected_tensors(
|
||||
fallback_tensors = load_selected_tensors(
|
||||
base_dir,
|
||||
base_weight_map,
|
||||
fallback_tensor_names,
|
||||
@@ -302,11 +206,6 @@ def build_modelopt_nvfp4_transformer(
|
||||
updated_weight_map[name] = filename
|
||||
total_size += tensor.element_size() * tensor.numel()
|
||||
|
||||
if index_filename is None:
|
||||
raise ValueError(
|
||||
"Expected a sharded or indexed ModelOpt HF export, but no index file was found."
|
||||
)
|
||||
|
||||
with open(output_path / index_filename, "w", encoding="utf-8") as f:
|
||||
json.dump(
|
||||
{
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Checkpoint I/O shared by the ModelOpt transformer conversion tools."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Mapping
|
||||
|
||||
import torch
|
||||
from safetensors import safe_open
|
||||
|
||||
_INDEX_FILENAMES = (
|
||||
"model.safetensors.index.json",
|
||||
"diffusion_pytorch_model.safetensors.index.json",
|
||||
)
|
||||
|
||||
|
||||
def resolve_transformer_dir(path: str) -> str:
|
||||
candidate = Path(path).expanduser().resolve()
|
||||
if (candidate / "config.json").is_file():
|
||||
return str(candidate)
|
||||
transformer_dir = candidate / "transformer"
|
||||
if (transformer_dir / "config.json").is_file():
|
||||
return str(transformer_dir)
|
||||
raise FileNotFoundError(f"Could not resolve a transformer directory from: {path}")
|
||||
|
||||
|
||||
def load_weight_map(model_dir: str) -> tuple[dict[str, str], str]:
|
||||
index_filename = next(
|
||||
(
|
||||
name
|
||||
for name in _INDEX_FILENAMES
|
||||
if os.path.isfile(os.path.join(model_dir, name))
|
||||
),
|
||||
None,
|
||||
)
|
||||
if index_filename is None:
|
||||
matches = sorted(
|
||||
name
|
||||
for name in os.listdir(model_dir)
|
||||
if name.endswith(".safetensors.index.json")
|
||||
)
|
||||
index_filename = matches[0] if matches else None
|
||||
if index_filename is not None:
|
||||
with open(os.path.join(model_dir, index_filename), encoding="utf-8") as f:
|
||||
index_data = json.load(f)
|
||||
return dict(index_data["weight_map"]), index_filename
|
||||
|
||||
safetensors_files = sorted(
|
||||
filename
|
||||
for filename in os.listdir(model_dir)
|
||||
if filename.endswith(".safetensors")
|
||||
)
|
||||
if len(safetensors_files) != 1:
|
||||
raise ValueError(
|
||||
f"Expected an index file or a single safetensors shard in {model_dir}, "
|
||||
f"found {len(safetensors_files)} shard(s)."
|
||||
)
|
||||
|
||||
shard_name = safetensors_files[0]
|
||||
with safe_open(
|
||||
os.path.join(model_dir, shard_name), framework="pt", device="cpu"
|
||||
) as f:
|
||||
weight_map = {key: shard_name for key in f.keys()}
|
||||
return weight_map, f"{Path(shard_name).stem}.safetensors.index.json"
|
||||
|
||||
|
||||
def load_config(model_dir: str) -> dict:
|
||||
with open(os.path.join(model_dir, "config.json"), encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def prepare_output_dir(source_dir: str, output_dir: str, *, overwrite: bool) -> Path:
|
||||
output_path = Path(output_dir).expanduser().resolve()
|
||||
if output_path.exists():
|
||||
if not overwrite:
|
||||
raise FileExistsError(
|
||||
f"Output directory already exists: {output_path}. "
|
||||
"Use --overwrite to replace it."
|
||||
)
|
||||
shutil.rmtree(output_path)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for entry in os.listdir(source_dir):
|
||||
if entry.endswith(".safetensors") or entry in _INDEX_FILENAMES:
|
||||
continue
|
||||
source_path = os.path.join(source_dir, entry)
|
||||
destination = output_path / entry
|
||||
if os.path.isdir(source_path):
|
||||
shutil.copytree(source_path, destination, dirs_exist_ok=True)
|
||||
else:
|
||||
shutil.copy2(source_path, destination)
|
||||
return output_path
|
||||
|
||||
|
||||
def load_selected_tensors(
|
||||
model_dir: str,
|
||||
weight_map: Mapping[str, str],
|
||||
tensor_names: Iterable[str],
|
||||
) -> dict[str, torch.Tensor]:
|
||||
tensors: dict[str, torch.Tensor] = {}
|
||||
names_by_file: dict[str, list[str]] = defaultdict(list)
|
||||
for name in tensor_names:
|
||||
names_by_file[weight_map[name]].append(name)
|
||||
|
||||
for filename, names in names_by_file.items():
|
||||
shard_path = os.path.join(model_dir, filename)
|
||||
with safe_open(shard_path, framework="pt", device="cpu") as f:
|
||||
for name in names:
|
||||
tensors[name] = f.get_tensor(name).contiguous()
|
||||
return tensors
|
||||
Reference in New Issue
Block a user