diff --git a/python/sglang/multimodal_gen/test/unit/test_modelopt_checkpoint_tools.py b/python/sglang/multimodal_gen/test/unit/test_modelopt_checkpoint_tools.py new file mode 100644 index 0000000000..7394c3f132 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_modelopt_checkpoint_tools.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: Apache-2.0 + +import json + +import pytest +import torch +from safetensors import safe_open +from safetensors.torch import save_file + +from sglang.multimodal_gen.tools.build_modelopt_fp8_transformer import ( + build_modelopt_fp8_transformer, +) +from sglang.multimodal_gen.tools.build_modelopt_nvfp4_transformer import ( + build_modelopt_nvfp4_transformer, +) +from sglang.multimodal_gen.tools.modelopt_checkpoint import load_weight_map + + +def _write_checkpoint(path, config, shards, index_name): + path.mkdir(parents=True) + (path / "config.json").write_text(json.dumps(config)) + weight_map = {} + for i, tensors in enumerate(shards): + filename = f"weights-{i}.safetensors" + save_file(tensors, path / filename, metadata={"source": "preserved"}) + weight_map.update(dict.fromkeys(tensors, filename)) + if index_name: + (path / index_name).write_text(json.dumps({"weight_map": weight_map})) + + +@pytest.mark.parametrize("kind", ["fp8", "nvfp4"]) +@pytest.mark.parametrize("keep_bf16", [False, True]) +@pytest.mark.parametrize( + "index_name", + [ + None, + "diffusion_pytorch_model.safetensors.index.json", + "custom.safetensors.index.json", + ], +) +def test_modelopt_checkpoint_conversion(tmp_path, kind, keep_bf16, index_name): + source, base, output = (tmp_path / name for name in ("source", "base", "output")) + quant = {"quant_method": "modelopt", "quant_algo": kind.upper(), "ignore": []} + weights = torch.arange(32).reshape(4, 8).to(torch.bfloat16) + serialized = weights if kind == "fp8" else weights.to(torch.uint8) + shards = [ + { + f"{name}.weight": serialized.clone(), + f"{name}.weight_scale": torch.tensor(1.0), + } + for name in ("quantized", "fallback") + ] + shards[0]["quantized.weight_quantizer._amax"] = torch.tensor(448.0) + if index_name is None: + shards = [shards[0] | shards[1]] + _write_checkpoint( + source / "transformer", {"quantization_config": quant}, shards, index_name + ) + _write_checkpoint(base, {}, [{"fallback.weight": weights + 17}], None) + (source / "transformer" / "assets").mkdir() + (source / "transformer" / "assets" / "notes.txt").write_text("preserved") + kwargs = dict( + modelopt_hf_dir=str(source), + base_transformer_dir=str(base), + output_dir=str(output), + keep_bf16_patterns=["fallback"] if keep_bf16 else [], + ) + if kind == "fp8": + backbone = tmp_path / "backbone.pt" + torch.save( + { + "model_state_dict": { + f"{name}.{quantizer}_quantizer._amax": torch.tensor(448.0) + for name in ("quantized", "fallback") + for quantizer in ("weight", "input") + } + }, + backbone, + ) + kwargs.update(modelopt_backbone_ckpt=str(backbone), model_type="none") + build = build_modelopt_fp8_transformer + else: + build = build_modelopt_nvfp4_transformer + stats = build(**kwargs) + with pytest.raises(FileExistsError, match="--overwrite"): + build(**kwargs) + (output / "stale.txt").write_text("remove on explicit overwrite") + assert build(**kwargs, overwrite=True) == stats + assert not (output / "stale.txt").exists() + assert (output / "assets" / "notes.txt").read_text() == "preserved" + weight_map, output_index = load_weight_map(str(output)) + assert output_index == (index_name or "weights-0.safetensors.index.json") + assert not any("_quantizer." in key for key in weight_map) + tensors = {} + for filename in set(weight_map.values()): + with safe_open(output / filename, framework="pt") as shard: + assert shard.metadata()["source"] == "preserved" + assert ( + json.loads(shard.metadata()["quantization_config"])["quant_algo"] + == kind.upper() + ) + tensors.update({key: shard.get_tensor(key) for key in shard.keys()}) + assert set(tensors) == set(weight_map) + quantized = weights.to(torch.float8_e4m3fn) if kind == "fp8" else serialized + for name, expected in ( + ("quantized", quantized), + ("fallback", weights + 17 if keep_bf16 else quantized), + ): + actual = tensors[f"{name}.weight"] + assert actual.dtype == expected.dtype + torch.testing.assert_close(actual.float(), expected.float(), rtol=0, atol=0) + assert ("fallback.weight_scale" in tensors) is not keep_bf16 + index = json.loads((output / output_index).read_text()) + assert index["metadata"]["total_size"] == sum( + t.numel() * t.element_size() for t in tensors.values() + ) + + +def test_modelopt_index_selection(tmp_path): + with pytest.raises(ValueError, match="found 0 shard"): + load_weight_map(str(tmp_path)) + for name in ("a", "b"): + save_file({name: torch.ones(1)}, tmp_path / f"{name}.safetensors") + with pytest.raises(ValueError, match="found 2 shard"): + load_weight_map(str(tmp_path)) + for name in ("z", "a", "diffusion_pytorch_model", "model"): + filename = f"{name}.safetensors.index.json" + mapping = {name: "a.safetensors"} + (tmp_path / filename).write_text(json.dumps({"weight_map": mapping})) + assert load_weight_map(str(tmp_path)) == (mapping, filename) diff --git a/python/sglang/multimodal_gen/tools/build_modelopt_fp8_transformer.py b/python/sglang/multimodal_gen/tools/build_modelopt_fp8_transformer.py index a579e2035d..b99c14f0d4 100644 --- a/python/sglang/multimodal_gen/tools/build_modelopt_fp8_transformer.py +++ b/python/sglang/multimodal_gen/tools/build_modelopt_fp8_transformer.py @@ -26,10 +26,9 @@ import gc import json import os import re -import shutil from collections import defaultdict from pathlib import Path -from typing import Callable, Iterable, Mapping, Sequence +from typing import Callable, Mapping, Sequence import torch from safetensors import safe_open @@ -38,11 +37,14 @@ from safetensors.torch import load_file, save_file from sglang.multimodal_gen.runtime.utils.quantization_utils import ( normalize_flat_modelopt_quant_config, ) +from sglang.multimodal_gen.tools.modelopt_checkpoint import ( + load_config, + load_selected_tensors, + load_weight_map, + prepare_output_dir, + resolve_transformer_dir, +) -INDEX_FILENAMES = [ - "model.safetensors.index.json", - "diffusion_pytorch_model.safetensors.index.json", -] FP8_E4M3_MAXBOUND = 448.0 DEFAULT_FLUX2_KEEP_BF16_PATTERNS = [ r"^time_guidance_embed\.(timestep_embedder|guidance_embedder)\.linear_[12]$", @@ -209,16 +211,6 @@ DEFAULT_QWEN_IMAGE_KEEP_BF16_PATTERNS = [ ] -def _resolve_transformer_dir(path: str) -> str: - candidate = Path(path).expanduser().resolve() - if (candidate / "config.json").is_file(): - return str(candidate) - transformer_dir = candidate / "transformer" - if (transformer_dir / "config.json").is_file(): - return str(transformer_dir) - raise FileNotFoundError(f"Could not resolve a transformer directory from: {path}") - - def _resolve_backbone_ckpt(path: str) -> str: candidate = Path(path).expanduser().resolve() if candidate.is_file(): @@ -229,53 +221,6 @@ def _resolve_backbone_ckpt(path: str) -> str: raise FileNotFoundError(f"Could not resolve backbone.pt from: {path}") -def _find_index_file(model_dir: str) -> str | None: - for filename in INDEX_FILENAMES: - candidate = os.path.join(model_dir, filename) - if os.path.isfile(candidate): - return filename - - matches = sorted( - filename - for filename in os.listdir(model_dir) - if filename.endswith(".safetensors.index.json") - ) - return matches[0] if matches else None - - -def _load_weight_map(model_dir: str) -> tuple[dict[str, str], str | None]: - index_filename = _find_index_file(model_dir) - if index_filename is not None: - with open(os.path.join(model_dir, index_filename), encoding="utf-8") as f: - index_data = json.load(f) - return dict(index_data["weight_map"]), index_filename - - safetensors_files = sorted( - filename - for filename in os.listdir(model_dir) - if filename.endswith(".safetensors") - ) - if len(safetensors_files) != 1: - raise ValueError( - f"Expected an index file or a single safetensors shard in {model_dir}, " - f"found {len(safetensors_files)} shard(s)." - ) - - shard_name = safetensors_files[0] - with safe_open( - os.path.join(model_dir, shard_name), framework="pt", device="cpu" - ) as f: - weight_map = {key: shard_name for key in f.keys()} - index_filename = f"{Path(shard_name).stem}.safetensors.index.json" - return weight_map, index_filename - - -def _load_config(model_dir: str) -> dict: - config_path = os.path.join(model_dir, "config.json") - with open(config_path, encoding="utf-8") as f: - return json.load(f) - - def _load_first_shard_metadata( model_dir: str, weight_map: Mapping[str, str] ) -> dict[str, str]: @@ -512,37 +457,6 @@ def quantize_fp8_weight( return quantized.cpu().contiguous() -def _copy_non_shard_files(source_dir: str, output_dir: str) -> None: - ignored = set(INDEX_FILENAMES) - for entry in os.listdir(source_dir): - if entry.endswith(".safetensors") or entry in ignored: - continue - source_path = os.path.join(source_dir, entry) - output_path = os.path.join(output_dir, entry) - if os.path.isdir(source_path): - shutil.copytree(source_path, output_path, dirs_exist_ok=True) - else: - shutil.copy2(source_path, output_path) - - -def _load_selected_tensors( - model_dir: str, - weight_map: Mapping[str, str], - tensor_names: Iterable[str], -) -> dict[str, torch.Tensor]: - tensors: dict[str, torch.Tensor] = {} - names_by_file: dict[str, list[str]] = defaultdict(list) - for name in tensor_names: - names_by_file[weight_map[name]].append(name) - - for filename, names in names_by_file.items(): - shard_path = os.path.join(model_dir, filename) - with safe_open(shard_path, framework="pt", device="cpu") as f: - for name in names: - tensors[name] = f.get_tensor(name).contiguous() - return tensors - - def build_modelopt_fp8_transformer( *, modelopt_hf_dir: str, @@ -554,13 +468,13 @@ def build_modelopt_fp8_transformer( maxbound: float = FP8_E4M3_MAXBOUND, overwrite: bool = False, ) -> dict[str, int]: - source_dir = _resolve_transformer_dir(modelopt_hf_dir) + source_dir = resolve_transformer_dir(modelopt_hf_dir) backbone_ckpt_path = _resolve_backbone_ckpt(modelopt_backbone_ckpt) base_dir = ( - _resolve_transformer_dir(base_transformer_dir) if base_transformer_dir else None + resolve_transformer_dir(base_transformer_dir) if base_transformer_dir else None ) - config = _load_config(source_dir) + config = load_config(source_dir) quant_config = config.get("quantization_config") if not isinstance(quant_config, dict): raise ValueError( @@ -572,7 +486,7 @@ def build_modelopt_fp8_transformer( "(quant_method=modelopt)." ) - source_weight_map_all, index_filename = _load_weight_map(source_dir) + source_weight_map_all, index_filename = load_weight_map(source_dir) source_metadata = _load_first_shard_metadata(source_dir, source_weight_map_all) is_ltx2_export = _is_ltx2_x0_export( config=config, @@ -596,17 +510,7 @@ def build_modelopt_fp8_transformer( "BF16 fallback patterns are enabled, but --base-transformer-dir was not provided." ) - output_path = Path(output_dir).expanduser().resolve() - if output_path.exists(): - if not overwrite: - raise FileExistsError( - f"Output directory already exists: {output_path}. " - "Use --overwrite to replace it." - ) - shutil.rmtree(output_path) - output_path.mkdir(parents=True, exist_ok=True) - - _copy_non_shard_files(source_dir, str(output_path)) + output_path = prepare_output_dir(source_dir, output_dir, overwrite=overwrite) if is_ltx2_export: source_weight_map = { @@ -618,7 +522,7 @@ def build_modelopt_fp8_transformer( source_weight_map = source_weight_map_all base_weight_map: dict[str, str] = {} if base_dir is not None: - base_weight_map, _ = _load_weight_map(base_dir) + base_weight_map, _ = load_weight_map(base_dir) fallback_weight_names = sorted( weight_name for weight_name in source_weight_map @@ -681,7 +585,7 @@ def build_modelopt_fp8_transformer( ) fallback_tensors = ( - _load_selected_tensors(base_dir, base_weight_map, fallback_weight_names) + load_selected_tensors(base_dir, base_weight_map, fallback_weight_names) if fallback_weight_names and base_dir is not None else {} ) diff --git a/python/sglang/multimodal_gen/tools/build_modelopt_nvfp4_transformer.py b/python/sglang/multimodal_gen/tools/build_modelopt_nvfp4_transformer.py index 7bd7b2c81d..7c0670634d 100644 --- a/python/sglang/multimodal_gen/tools/build_modelopt_nvfp4_transformer.py +++ b/python/sglang/multimodal_gen/tools/build_modelopt_nvfp4_transformer.py @@ -16,18 +16,20 @@ import argparse import json import os import re -import shutil from collections import defaultdict from pathlib import Path -from typing import Iterable, Mapping, Sequence +from typing import Mapping, Sequence from safetensors import safe_open from safetensors.torch import load_file, save_file -INDEX_FILENAMES = [ - "model.safetensors.index.json", - "diffusion_pytorch_model.safetensors.index.json", -] +from sglang.multimodal_gen.tools.modelopt_checkpoint import ( + load_config, + load_selected_tensors, + load_weight_map, + prepare_output_dir, + resolve_transformer_dir, +) DEFAULT_FLUX1_NVFP4_FALLBACK_PATTERNS = [ "transformer_blocks.*.norm1.linear*", @@ -49,100 +51,12 @@ _TENSOR_MODULE_SUFFIXES = ( ) -def _resolve_transformer_dir(path: str) -> str: - candidate = Path(path).expanduser().resolve() - if (candidate / "config.json").is_file(): - return str(candidate) - transformer_dir = candidate / "transformer" - if (transformer_dir / "config.json").is_file(): - return str(transformer_dir) - raise FileNotFoundError(f"Could not resolve a transformer directory from: {path}") - - -def _find_index_file(model_dir: str) -> str | None: - for filename in INDEX_FILENAMES: - candidate = os.path.join(model_dir, filename) - if os.path.isfile(candidate): - return filename - - matches = sorted( - filename - for filename in os.listdir(model_dir) - if filename.endswith(".safetensors.index.json") - ) - return matches[0] if matches else None - - -def _load_weight_map(model_dir: str) -> tuple[dict[str, str], str | None]: - index_filename = _find_index_file(model_dir) - if index_filename is not None: - with open(os.path.join(model_dir, index_filename), encoding="utf-8") as f: - index_data = json.load(f) - return dict(index_data["weight_map"]), index_filename - - safetensors_files = sorted( - filename - for filename in os.listdir(model_dir) - if filename.endswith(".safetensors") - ) - if len(safetensors_files) != 1: - raise ValueError( - f"Expected an index file or a single safetensors shard in {model_dir}, " - f"found {len(safetensors_files)} shard(s)." - ) - - shard_name = safetensors_files[0] - with safe_open( - os.path.join(model_dir, shard_name), framework="pt", device="cpu" - ) as f: - weight_map = {key: shard_name for key in f.keys()} - index_filename = f"{Path(shard_name).stem}.safetensors.index.json" - return weight_map, index_filename - - -def _load_config(model_dir: str) -> dict: - config_path = os.path.join(model_dir, "config.json") - with open(config_path, encoding="utf-8") as f: - return json.load(f) - - def _write_config(model_dir: Path, config: Mapping[str, object]) -> None: with open(model_dir / "config.json", "w", encoding="utf-8") as f: json.dump(config, f, indent=2, sort_keys=True) f.write("\n") -def _copy_non_shard_files(source_dir: str, output_dir: str) -> None: - ignored = set(INDEX_FILENAMES) - for entry in os.listdir(source_dir): - if entry.endswith(".safetensors") or entry in ignored: - continue - source_path = os.path.join(source_dir, entry) - output_path = os.path.join(output_dir, entry) - if os.path.isdir(source_path): - shutil.copytree(source_path, output_path, dirs_exist_ok=True) - else: - shutil.copy2(source_path, output_path) - - -def _load_selected_tensors( - model_dir: str, - weight_map: Mapping[str, str], - tensor_names: Iterable[str], -): - tensors = {} - names_by_file: dict[str, list[str]] = defaultdict(list) - for name in tensor_names: - names_by_file[weight_map[name]].append(name) - - for filename, names in names_by_file.items(): - shard_path = os.path.join(model_dir, filename) - with safe_open(shard_path, framework="pt", device="cpu") as f: - for name in names: - tensors[name] = f.get_tensor(name).contiguous() - return tensors - - def _module_name_for_tensor(tensor_name: str) -> str: for suffix in _TENSOR_MODULE_SUFFIXES: if tensor_name.endswith(suffix): @@ -210,8 +124,8 @@ def build_modelopt_nvfp4_transformer( swap_weight_nibbles: bool | None = None, overwrite: bool = False, ) -> dict[str, int | bool]: - source_dir = _resolve_transformer_dir(modelopt_hf_dir) - base_dir = _resolve_transformer_dir(base_transformer_dir) + source_dir = resolve_transformer_dir(modelopt_hf_dir) + base_dir = resolve_transformer_dir(base_transformer_dir) patterns = _preset_patterns(pattern_preset) if keep_bf16_patterns: @@ -221,28 +135,18 @@ def build_modelopt_nvfp4_transformer( swap_weight_nibbles if swap_weight_nibbles is not None else False ) output_config = _updated_quant_config( - _load_config(source_dir), + load_config(source_dir), fallback_patterns=patterns, swap_weight_nibbles=resolved_swap_weight_nibbles, ) quant_config = output_config["quantization_config"] serialized_quant_config = json.dumps(quant_config, sort_keys=True) - output_path = Path(output_dir).expanduser().resolve() - if output_path.exists(): - if not overwrite: - raise FileExistsError( - f"Output directory already exists: {output_path}. " - "Use --overwrite to replace it." - ) - shutil.rmtree(output_path) - output_path.mkdir(parents=True, exist_ok=True) - - _copy_non_shard_files(source_dir, str(output_path)) + output_path = prepare_output_dir(source_dir, output_dir, overwrite=overwrite) _write_config(output_path, output_config) - source_weight_map, index_filename = _load_weight_map(source_dir) - base_weight_map, _ = _load_weight_map(base_dir) + source_weight_map, index_filename = load_weight_map(source_dir) + base_weight_map, _ = load_weight_map(base_dir) fallback_tensor_names = sorted( name @@ -250,7 +154,7 @@ def build_modelopt_nvfp4_transformer( if name in source_weight_map and _matches_any_pattern(_module_name_for_tensor(name), patterns) ) - fallback_tensors = _load_selected_tensors( + fallback_tensors = load_selected_tensors( base_dir, base_weight_map, fallback_tensor_names, @@ -302,11 +206,6 @@ def build_modelopt_nvfp4_transformer( updated_weight_map[name] = filename total_size += tensor.element_size() * tensor.numel() - if index_filename is None: - raise ValueError( - "Expected a sharded or indexed ModelOpt HF export, but no index file was found." - ) - with open(output_path / index_filename, "w", encoding="utf-8") as f: json.dump( { diff --git a/python/sglang/multimodal_gen/tools/modelopt_checkpoint.py b/python/sglang/multimodal_gen/tools/modelopt_checkpoint.py new file mode 100644 index 0000000000..049370031c --- /dev/null +++ b/python/sglang/multimodal_gen/tools/modelopt_checkpoint.py @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Checkpoint I/O shared by the ModelOpt transformer conversion tools.""" + +import json +import os +import shutil +from collections import defaultdict +from pathlib import Path +from typing import Iterable, Mapping + +import torch +from safetensors import safe_open + +_INDEX_FILENAMES = ( + "model.safetensors.index.json", + "diffusion_pytorch_model.safetensors.index.json", +) + + +def resolve_transformer_dir(path: str) -> str: + candidate = Path(path).expanduser().resolve() + if (candidate / "config.json").is_file(): + return str(candidate) + transformer_dir = candidate / "transformer" + if (transformer_dir / "config.json").is_file(): + return str(transformer_dir) + raise FileNotFoundError(f"Could not resolve a transformer directory from: {path}") + + +def load_weight_map(model_dir: str) -> tuple[dict[str, str], str]: + index_filename = next( + ( + name + for name in _INDEX_FILENAMES + if os.path.isfile(os.path.join(model_dir, name)) + ), + None, + ) + if index_filename is None: + matches = sorted( + name + for name in os.listdir(model_dir) + if name.endswith(".safetensors.index.json") + ) + index_filename = matches[0] if matches else None + if index_filename is not None: + with open(os.path.join(model_dir, index_filename), encoding="utf-8") as f: + index_data = json.load(f) + return dict(index_data["weight_map"]), index_filename + + safetensors_files = sorted( + filename + for filename in os.listdir(model_dir) + if filename.endswith(".safetensors") + ) + if len(safetensors_files) != 1: + raise ValueError( + f"Expected an index file or a single safetensors shard in {model_dir}, " + f"found {len(safetensors_files)} shard(s)." + ) + + shard_name = safetensors_files[0] + with safe_open( + os.path.join(model_dir, shard_name), framework="pt", device="cpu" + ) as f: + weight_map = {key: shard_name for key in f.keys()} + return weight_map, f"{Path(shard_name).stem}.safetensors.index.json" + + +def load_config(model_dir: str) -> dict: + with open(os.path.join(model_dir, "config.json"), encoding="utf-8") as f: + return json.load(f) + + +def prepare_output_dir(source_dir: str, output_dir: str, *, overwrite: bool) -> Path: + output_path = Path(output_dir).expanduser().resolve() + if output_path.exists(): + if not overwrite: + raise FileExistsError( + f"Output directory already exists: {output_path}. " + "Use --overwrite to replace it." + ) + shutil.rmtree(output_path) + output_path.mkdir(parents=True, exist_ok=True) + + for entry in os.listdir(source_dir): + if entry.endswith(".safetensors") or entry in _INDEX_FILENAMES: + continue + source_path = os.path.join(source_dir, entry) + destination = output_path / entry + if os.path.isdir(source_path): + shutil.copytree(source_path, destination, dirs_exist_ok=True) + else: + shutil.copy2(source_path, destination) + return output_path + + +def load_selected_tensors( + model_dir: str, + weight_map: Mapping[str, str], + tensor_names: Iterable[str], +) -> dict[str, torch.Tensor]: + tensors: dict[str, torch.Tensor] = {} + names_by_file: dict[str, list[str]] = defaultdict(list) + for name in tensor_names: + names_by_file[weight_map[name]].append(name) + + for filename, names in names_by_file.items(): + shard_path = os.path.join(model_dir, filename) + with safe_open(shard_path, framework="pt", device="cpu") as f: + for name in names: + tensors[name] = f.get_tensor(name).contiguous() + return tensors