[diffusion] fix: pin the JoyAI-Echo overlay's source revision (#42471)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WenhaoZhang
2026-10-04 17:19:09 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent ba68306256
commit 1e490772e5
2 changed files with 79 additions and 0 deletions
@@ -33,6 +33,9 @@ BUILTIN_MODEL_OVERLAY_REGISTRY: dict[str, dict[str, Any]] = {
"jdopensource/JoyAI-Echo": {
"overlay_repo_id": "Niehen6174/JoyAI-Echo-overlay",
"overlay_revision": "0a19f315c96532b7a5f61bcd765d1fefdd83dc7d",
# Last source commit with JoyAI-Echo-release.safetensors, which the
# overlay splits; the Echo 1.5 release (a7d585b1) removed it.
"source_revision": "4187f9a53c6eff3a76c51e79bd27f70d10f7591b",
},
"Efficient-Large-Model/SANA-WM_bidirectional": {
"overlay_repo_id": "sjmshsh/SANA-WM_bidirectional-overlay",
@@ -323,6 +326,7 @@ def ensure_overlay_source_dir_complete(
allow_patterns: list[str] | None,
download: bool,
snapshot_download_fn: Callable[..., str],
revision: str | None = None,
) -> str:
required_source_files = cast(
list[str], list(manifest.get("required_source_files", []))
@@ -362,6 +366,7 @@ def ensure_overlay_source_dir_complete(
local_dir=local_dir,
max_workers=8,
force_download=True,
revision=revision,
)
missing_after_redownload = _find_missing_required_paths(
source_dir, required_source_files
@@ -426,6 +431,15 @@ def resolve_direct_overlay_repo(
return overlay_spec, overlay_dir, manifest
def _overlay_source_revision(
overlay_spec: dict[str, Any], manifest: dict[str, Any]
) -> str | None:
# A registry entry pins the source repo the way it pins the overlay repo;
# the manifest field covers overlay repos that are loaded directly.
revision = overlay_spec.get("source_revision") or manifest.get("source_revision")
return str(revision) if revision else None
def download_overlay_metadata(
source_model_id: str,
overlay_spec: dict[str, Any],
@@ -545,6 +559,7 @@ def materialize_overlay_model(
overlay_dir: str,
source_dir: str,
verify_diffusers_model_complete_fn: Callable[[str], bool],
source_revision: str | None = None,
) -> str:
overlay_manifest_path = os.path.join(
overlay_dir, "_overlay", "overlay_manifest.json"
@@ -644,6 +659,7 @@ def materialize_overlay_model(
{
"source_model_id": source_model_id,
"source_dir": source_dir,
"source_revision": source_revision,
"overlay_repo_id": overlay_repo_id,
"overlay_revision": overlay_revision,
"materializer_version": materializer_version,
@@ -725,6 +741,7 @@ def maybe_resolve_overlay_model_path(
source_allow_patterns = cast(
list[str] | None, manifest.get("source_allow_patterns")
)
source_revision = _overlay_source_revision(overlay_spec, manifest)
# For local source paths, reuse the directory directly instead of
# round-tripping through snapshot_download.
source_dir = (
@@ -736,6 +753,7 @@ def maybe_resolve_overlay_model_path(
download=download,
allow_patterns=source_allow_patterns or allow_patterns,
force_diffusers_model=False,
revision=source_revision,
skip_overlay_resolution=True,
)
)
@@ -747,6 +765,7 @@ def maybe_resolve_overlay_model_path(
allow_patterns=allow_patterns,
download=download,
snapshot_download_fn=snapshot_download_fn,
revision=source_revision,
)
return materialize_overlay_model(
source_model_id=source_model_id,
@@ -754,6 +773,7 @@ def maybe_resolve_overlay_model_path(
overlay_dir=overlay_dir,
source_dir=source_dir,
verify_diffusers_model_complete_fn=verify_diffusers_model_complete_fn,
source_revision=source_revision,
)
direct_overlay = resolve_direct_overlay_repo(
@@ -769,12 +789,14 @@ def maybe_resolve_overlay_model_path(
source_allow_patterns = cast(
list[str] | None, manifest.get("source_allow_patterns")
)
source_revision = _overlay_source_revision(overlay_spec, manifest)
source_dir = base_model_download_fn(
source_model_id,
local_dir=local_dir,
download=download,
allow_patterns=source_allow_patterns or allow_patterns,
force_diffusers_model=False,
revision=source_revision,
skip_overlay_resolution=True,
)
source_dir = ensure_overlay_source_dir_complete(
@@ -785,6 +807,7 @@ def maybe_resolve_overlay_model_path(
allow_patterns=allow_patterns,
download=download,
snapshot_download_fn=snapshot_download_fn,
revision=source_revision,
)
return materialize_overlay_model(
source_model_id=source_model_id,
@@ -792,4 +815,5 @@ def maybe_resolve_overlay_model_path(
overlay_dir=overlay_dir,
source_dir=source_dir,
verify_diffusers_model_complete_fn=verify_diffusers_model_complete_fn,
source_revision=source_revision,
)
@@ -63,3 +63,58 @@ def test_large_files_are_still_shared_with_the_cache(tmp_path, monkeypatch):
_copytree_link_or_copy(str(snapshot), str(out))
assert os.path.samefile(out / "model.safetensors", weights)
def test_registry_source_revision_survives_upstream_removing_required_file(
tmp_path, monkeypatch
):
"""An overlay whose source repo later deleted a required file still resolves
on a host with an empty cache, because both source downloads use the pin."""
pinned = "4187f9a53c6eff3a76c51e79bd27f70d10f7591b"
monkeypatch.setenv("SGLANG_DIFFUSION_CACHE_ROOT", str(tmp_path / "cache"))
monkeypatch.setattr(
model_overlay,
"_load_model_overlay_registry",
lambda: {
"org/source": {
"overlay_repo_id": "org/source-overlay",
"overlay_revision": "overlay-sha",
"source_revision": pinned,
}
},
)
overlay_dir = tmp_path / "overlay"
(overlay_dir / "_overlay").mkdir(parents=True)
(overlay_dir / "_overlay" / "overlay_manifest.json").write_text(
'{"source_model_id": "org/source",'
' "required_source_files": ["release.safetensors"]}'
)
def source_snapshot(revision):
# Upstream main no longer ships the file; the pinned commit does.
snapshot = tmp_path / f"source-{revision}"
snapshot.mkdir(exist_ok=True)
if revision == pinned:
(snapshot / "release.safetensors").write_bytes(b"w")
return str(snapshot)
def snapshot_download_fn(*, repo_id, revision=None, **_):
if repo_id == "org/source-overlay":
return str(overlay_dir)
return source_snapshot(revision)
def base_model_download_fn(model_id, *, revision=None, **_):
return source_snapshot(revision)
resolved = model_overlay.maybe_resolve_overlay_model_path(
"org/source",
local_dir=None,
download=True,
allow_patterns=None,
snapshot_download_fn=snapshot_download_fn,
hf_hub_download_fn=lambda **_: None,
verify_diffusers_model_complete_fn=lambda _: True,
base_model_download_fn=base_model_download_fn,
)
assert os.path.isdir(resolved)