[diffusion] fix: pin the JoyAI-Echo overlay's source revision (#42471)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
ba68306256
commit
1e490772e5
@@ -33,6 +33,9 @@ BUILTIN_MODEL_OVERLAY_REGISTRY: dict[str, dict[str, Any]] = {
|
||||
"jdopensource/JoyAI-Echo": {
|
||||
"overlay_repo_id": "Niehen6174/JoyAI-Echo-overlay",
|
||||
"overlay_revision": "0a19f315c96532b7a5f61bcd765d1fefdd83dc7d",
|
||||
# Last source commit with JoyAI-Echo-release.safetensors, which the
|
||||
# overlay splits; the Echo 1.5 release (a7d585b1) removed it.
|
||||
"source_revision": "4187f9a53c6eff3a76c51e79bd27f70d10f7591b",
|
||||
},
|
||||
"Efficient-Large-Model/SANA-WM_bidirectional": {
|
||||
"overlay_repo_id": "sjmshsh/SANA-WM_bidirectional-overlay",
|
||||
@@ -323,6 +326,7 @@ def ensure_overlay_source_dir_complete(
|
||||
allow_patterns: list[str] | None,
|
||||
download: bool,
|
||||
snapshot_download_fn: Callable[..., str],
|
||||
revision: str | None = None,
|
||||
) -> str:
|
||||
required_source_files = cast(
|
||||
list[str], list(manifest.get("required_source_files", []))
|
||||
@@ -362,6 +366,7 @@ def ensure_overlay_source_dir_complete(
|
||||
local_dir=local_dir,
|
||||
max_workers=8,
|
||||
force_download=True,
|
||||
revision=revision,
|
||||
)
|
||||
missing_after_redownload = _find_missing_required_paths(
|
||||
source_dir, required_source_files
|
||||
@@ -426,6 +431,15 @@ def resolve_direct_overlay_repo(
|
||||
return overlay_spec, overlay_dir, manifest
|
||||
|
||||
|
||||
def _overlay_source_revision(
|
||||
overlay_spec: dict[str, Any], manifest: dict[str, Any]
|
||||
) -> str | None:
|
||||
# A registry entry pins the source repo the way it pins the overlay repo;
|
||||
# the manifest field covers overlay repos that are loaded directly.
|
||||
revision = overlay_spec.get("source_revision") or manifest.get("source_revision")
|
||||
return str(revision) if revision else None
|
||||
|
||||
|
||||
def download_overlay_metadata(
|
||||
source_model_id: str,
|
||||
overlay_spec: dict[str, Any],
|
||||
@@ -545,6 +559,7 @@ def materialize_overlay_model(
|
||||
overlay_dir: str,
|
||||
source_dir: str,
|
||||
verify_diffusers_model_complete_fn: Callable[[str], bool],
|
||||
source_revision: str | None = None,
|
||||
) -> str:
|
||||
overlay_manifest_path = os.path.join(
|
||||
overlay_dir, "_overlay", "overlay_manifest.json"
|
||||
@@ -644,6 +659,7 @@ def materialize_overlay_model(
|
||||
{
|
||||
"source_model_id": source_model_id,
|
||||
"source_dir": source_dir,
|
||||
"source_revision": source_revision,
|
||||
"overlay_repo_id": overlay_repo_id,
|
||||
"overlay_revision": overlay_revision,
|
||||
"materializer_version": materializer_version,
|
||||
@@ -725,6 +741,7 @@ def maybe_resolve_overlay_model_path(
|
||||
source_allow_patterns = cast(
|
||||
list[str] | None, manifest.get("source_allow_patterns")
|
||||
)
|
||||
source_revision = _overlay_source_revision(overlay_spec, manifest)
|
||||
# For local source paths, reuse the directory directly instead of
|
||||
# round-tripping through snapshot_download.
|
||||
source_dir = (
|
||||
@@ -736,6 +753,7 @@ def maybe_resolve_overlay_model_path(
|
||||
download=download,
|
||||
allow_patterns=source_allow_patterns or allow_patterns,
|
||||
force_diffusers_model=False,
|
||||
revision=source_revision,
|
||||
skip_overlay_resolution=True,
|
||||
)
|
||||
)
|
||||
@@ -747,6 +765,7 @@ def maybe_resolve_overlay_model_path(
|
||||
allow_patterns=allow_patterns,
|
||||
download=download,
|
||||
snapshot_download_fn=snapshot_download_fn,
|
||||
revision=source_revision,
|
||||
)
|
||||
return materialize_overlay_model(
|
||||
source_model_id=source_model_id,
|
||||
@@ -754,6 +773,7 @@ def maybe_resolve_overlay_model_path(
|
||||
overlay_dir=overlay_dir,
|
||||
source_dir=source_dir,
|
||||
verify_diffusers_model_complete_fn=verify_diffusers_model_complete_fn,
|
||||
source_revision=source_revision,
|
||||
)
|
||||
|
||||
direct_overlay = resolve_direct_overlay_repo(
|
||||
@@ -769,12 +789,14 @@ def maybe_resolve_overlay_model_path(
|
||||
source_allow_patterns = cast(
|
||||
list[str] | None, manifest.get("source_allow_patterns")
|
||||
)
|
||||
source_revision = _overlay_source_revision(overlay_spec, manifest)
|
||||
source_dir = base_model_download_fn(
|
||||
source_model_id,
|
||||
local_dir=local_dir,
|
||||
download=download,
|
||||
allow_patterns=source_allow_patterns or allow_patterns,
|
||||
force_diffusers_model=False,
|
||||
revision=source_revision,
|
||||
skip_overlay_resolution=True,
|
||||
)
|
||||
source_dir = ensure_overlay_source_dir_complete(
|
||||
@@ -785,6 +807,7 @@ def maybe_resolve_overlay_model_path(
|
||||
allow_patterns=allow_patterns,
|
||||
download=download,
|
||||
snapshot_download_fn=snapshot_download_fn,
|
||||
revision=source_revision,
|
||||
)
|
||||
return materialize_overlay_model(
|
||||
source_model_id=source_model_id,
|
||||
@@ -792,4 +815,5 @@ def maybe_resolve_overlay_model_path(
|
||||
overlay_dir=overlay_dir,
|
||||
source_dir=source_dir,
|
||||
verify_diffusers_model_complete_fn=verify_diffusers_model_complete_fn,
|
||||
source_revision=source_revision,
|
||||
)
|
||||
|
||||
@@ -63,3 +63,58 @@ def test_large_files_are_still_shared_with_the_cache(tmp_path, monkeypatch):
|
||||
_copytree_link_or_copy(str(snapshot), str(out))
|
||||
|
||||
assert os.path.samefile(out / "model.safetensors", weights)
|
||||
|
||||
|
||||
def test_registry_source_revision_survives_upstream_removing_required_file(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
"""An overlay whose source repo later deleted a required file still resolves
|
||||
on a host with an empty cache, because both source downloads use the pin."""
|
||||
pinned = "4187f9a53c6eff3a76c51e79bd27f70d10f7591b"
|
||||
monkeypatch.setenv("SGLANG_DIFFUSION_CACHE_ROOT", str(tmp_path / "cache"))
|
||||
monkeypatch.setattr(
|
||||
model_overlay,
|
||||
"_load_model_overlay_registry",
|
||||
lambda: {
|
||||
"org/source": {
|
||||
"overlay_repo_id": "org/source-overlay",
|
||||
"overlay_revision": "overlay-sha",
|
||||
"source_revision": pinned,
|
||||
}
|
||||
},
|
||||
)
|
||||
overlay_dir = tmp_path / "overlay"
|
||||
(overlay_dir / "_overlay").mkdir(parents=True)
|
||||
(overlay_dir / "_overlay" / "overlay_manifest.json").write_text(
|
||||
'{"source_model_id": "org/source",'
|
||||
' "required_source_files": ["release.safetensors"]}'
|
||||
)
|
||||
|
||||
def source_snapshot(revision):
|
||||
# Upstream main no longer ships the file; the pinned commit does.
|
||||
snapshot = tmp_path / f"source-{revision}"
|
||||
snapshot.mkdir(exist_ok=True)
|
||||
if revision == pinned:
|
||||
(snapshot / "release.safetensors").write_bytes(b"w")
|
||||
return str(snapshot)
|
||||
|
||||
def snapshot_download_fn(*, repo_id, revision=None, **_):
|
||||
if repo_id == "org/source-overlay":
|
||||
return str(overlay_dir)
|
||||
return source_snapshot(revision)
|
||||
|
||||
def base_model_download_fn(model_id, *, revision=None, **_):
|
||||
return source_snapshot(revision)
|
||||
|
||||
resolved = model_overlay.maybe_resolve_overlay_model_path(
|
||||
"org/source",
|
||||
local_dir=None,
|
||||
download=True,
|
||||
allow_patterns=None,
|
||||
snapshot_download_fn=snapshot_download_fn,
|
||||
hf_hub_download_fn=lambda **_: None,
|
||||
verify_diffusers_model_complete_fn=lambda _: True,
|
||||
base_model_download_fn=base_model_download_fn,
|
||||
)
|
||||
|
||||
assert os.path.isdir(resolved)
|
||||
|
||||
Reference in New Issue
Block a user