mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-08-16 10:43:21 -03:00
feat(recipes): add filename fallback tier to recipe rematch
This commit is contained in:
+225
-46
@@ -36,6 +36,11 @@ logger = logging.getLogger(__name__)
|
|||||||
# explicitly to "diffusion_model" (mirrors Oracle R2-F1).
|
# explicitly to "diffusion_model" (mirrors Oracle R2-F1).
|
||||||
_CHECKPOINT_MODEL_TYPE_ALIASES = {"diffusionmodel": "diffusion_model"}
|
_CHECKPOINT_MODEL_TYPE_ALIASES = {"diffusionmodel": "diffusion_model"}
|
||||||
|
|
||||||
|
# Known weight-file extensions stripped by _normalize_filename_key. Names are
|
||||||
|
# stored extensionless on both sides, so splitext would misread dotted stems
|
||||||
|
# ("my.mix" -> "my") and silently collide distinct models.
|
||||||
|
_WEIGHT_FILE_EXTS = (".safetensors", ".ckpt", ".pt", ".pth", ".gguf", ".bin", ".safebin", ".sft")
|
||||||
|
|
||||||
|
|
||||||
class RecipeScanner:
|
class RecipeScanner:
|
||||||
"""Service for scanning and managing recipe images"""
|
"""Service for scanning and managing recipe images"""
|
||||||
@@ -116,6 +121,12 @@ class RecipeScanner:
|
|||||||
self._rematch_autov3_cache: dict[str, dict[str, Any]] | None = None
|
self._rematch_autov3_cache: dict[str, dict[str, Any]] | None = None
|
||||||
self._rematch_autov3_versions: tuple[int, int] | None = None
|
self._rematch_autov3_versions: tuple[int, int] | None = None
|
||||||
self._rematch_autov3_lock = asyncio.Lock()
|
self._rematch_autov3_lock = asyncio.Lock()
|
||||||
|
# Normalized filename -> [items] map for the L4 rematch fallback,
|
||||||
|
# rebuilt only when either model scanner's cache_version changes.
|
||||||
|
# Mirrors the build_local_hash_cache version pattern.
|
||||||
|
self._local_filename_cache: dict[str, list[dict[str, Any]]] | None = None
|
||||||
|
self._local_filename_cache_versions: tuple[int, int] | None = None
|
||||||
|
self._local_filename_cache_lock = asyncio.Lock()
|
||||||
self._initialized = True
|
self._initialized = True
|
||||||
|
|
||||||
async def build_local_hash_cache(self) -> dict[str, dict[str, Any]]:
|
async def build_local_hash_cache(self) -> dict[str, dict[str, Any]]:
|
||||||
@@ -162,6 +173,70 @@ class RecipeScanner:
|
|||||||
self._local_hash_cache_versions = versions
|
self._local_hash_cache_versions = versions
|
||||||
return cache
|
return cache
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _normalize_filename_key(name: str) -> str:
|
||||||
|
"""Normalize a file name to a lookup key (basename, lowercase).
|
||||||
|
|
||||||
|
Only known weight-file extensions are stripped — names are stored
|
||||||
|
extensionless on both sides, so splitext would misread dotted stems
|
||||||
|
("my.mix" -> "my") and collide distinct models.
|
||||||
|
"""
|
||||||
|
if not name:
|
||||||
|
return ""
|
||||||
|
basename = os.path.basename(name.replace("\\", "/"))
|
||||||
|
lower = basename.lower()
|
||||||
|
for ext in _WEIGHT_FILE_EXTS:
|
||||||
|
if lower.endswith(ext):
|
||||||
|
basename = basename[: -len(ext)]
|
||||||
|
break
|
||||||
|
return basename.strip().lower()
|
||||||
|
|
||||||
|
async def _build_local_filename_cache(self) -> dict[str, list[dict[str, Any]]]:
|
||||||
|
"""Build a version-cached map of normalized file names to local items.
|
||||||
|
|
||||||
|
Keys are lowercase basenames without extension. Values are lists of
|
||||||
|
items (lora + checkpoint, type-blind) sharing that name. Only items
|
||||||
|
with a sha256 are indexed — matching a pending or failed download
|
||||||
|
(empty sha256) would leave the entry without a usable hash. The dict
|
||||||
|
is reused while both scanners' cache_version values are unchanged;
|
||||||
|
concurrent callers share a single build via the lock.
|
||||||
|
"""
|
||||||
|
async with self._local_filename_cache_lock:
|
||||||
|
lora_scanner = self._lora_scanner
|
||||||
|
checkpoint_scanner = self._checkpoint_scanner
|
||||||
|
versions = (
|
||||||
|
lora_scanner.cache_version if lora_scanner is not None else 0,
|
||||||
|
checkpoint_scanner.cache_version
|
||||||
|
if checkpoint_scanner is not None
|
||||||
|
else 0,
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
self._local_filename_cache is not None
|
||||||
|
and self._local_filename_cache_versions == versions
|
||||||
|
):
|
||||||
|
return self._local_filename_cache
|
||||||
|
|
||||||
|
cache: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for scanner in (lora_scanner, checkpoint_scanner):
|
||||||
|
if scanner is None:
|
||||||
|
continue
|
||||||
|
data = await scanner.get_cached_data()
|
||||||
|
for item in data.raw_data:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
if not (item.get("sha256") or "").lower():
|
||||||
|
continue
|
||||||
|
file_path = item.get("file_path") or ""
|
||||||
|
file_name = item.get("file_name") or ""
|
||||||
|
key = self._normalize_filename_key(file_name or file_path)
|
||||||
|
if not key:
|
||||||
|
continue
|
||||||
|
cache.setdefault(key, []).append(item)
|
||||||
|
|
||||||
|
self._local_filename_cache = cache
|
||||||
|
self._local_filename_cache_versions = versions
|
||||||
|
return cache
|
||||||
|
|
||||||
def _is_rematch_candidate(self, entry: dict[str, Any]) -> bool:
|
def _is_rematch_candidate(self, entry: dict[str, Any]) -> bool:
|
||||||
"""Return True when a recipe entry is eligible for local re-matching."""
|
"""Return True when a recipe entry is eligible for local re-matching."""
|
||||||
if not isinstance(entry, dict):
|
if not isinstance(entry, dict):
|
||||||
@@ -170,7 +245,10 @@ class RecipeScanner:
|
|||||||
entry.get("isDeleted") or not entry.get("hash") or not entry.get("file_name")
|
entry.get("isDeleted") or not entry.get("hash") or not entry.get("file_name")
|
||||||
)
|
)
|
||||||
has_identifier = (
|
has_identifier = (
|
||||||
entry.get("hash") or entry.get("modelVersionId") or entry.get("id")
|
entry.get("hash")
|
||||||
|
or entry.get("modelVersionId")
|
||||||
|
or entry.get("id")
|
||||||
|
or entry.get("file_name")
|
||||||
)
|
)
|
||||||
return bool(unresolved and has_identifier)
|
return bool(unresolved and has_identifier)
|
||||||
|
|
||||||
@@ -221,6 +299,97 @@ class RecipeScanner:
|
|||||||
self._rematch_autov3_versions = versions
|
self._rematch_autov3_versions = versions
|
||||||
return cache
|
return cache
|
||||||
|
|
||||||
|
def _is_type_compatible(self, item: dict[str, Any], *, is_checkpoint: bool) -> bool:
|
||||||
|
"""Return True when a local item's type matches the entry kind.
|
||||||
|
|
||||||
|
The L1 hash cache and the L4 filename cache merge lora and checkpoint
|
||||||
|
items and are type-blind, so a match must be verified against the
|
||||||
|
entry kind before it is accepted.
|
||||||
|
"""
|
||||||
|
sub_type = (item.get("sub_type") or "").lower()
|
||||||
|
if sub_type:
|
||||||
|
valid = (
|
||||||
|
VALID_CHECKPOINT_SUB_TYPES if is_checkpoint else VALID_LORA_TYPES
|
||||||
|
)
|
||||||
|
return sub_type in valid
|
||||||
|
|
||||||
|
civitai_type = (
|
||||||
|
(item.get("civitai") or {}).get("model", {}) or {}
|
||||||
|
).get("type", "")
|
||||||
|
if civitai_type:
|
||||||
|
normalized = civitai_type.lower()
|
||||||
|
if is_checkpoint:
|
||||||
|
normalized = _CHECKPOINT_MODEL_TYPE_ALIASES.get(
|
||||||
|
normalized, normalized
|
||||||
|
)
|
||||||
|
valid = VALID_CHECKPOINT_SUB_TYPES
|
||||||
|
else:
|
||||||
|
valid = VALID_LORA_TYPES
|
||||||
|
return normalized in valid
|
||||||
|
return True
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _has_positive_type_evidence(item: dict[str, Any]) -> bool:
|
||||||
|
"""Return True when the item carries an explicit type marker.
|
||||||
|
|
||||||
|
Lora raw items rarely carry ``sub_type`` (it is only written when
|
||||||
|
metadata provides it), while checkpoint items always do — so for
|
||||||
|
checkpoint slots a type-less candidate is a red flag, not the norm.
|
||||||
|
"""
|
||||||
|
if (item.get("sub_type") or "").lower():
|
||||||
|
return True
|
||||||
|
civitai_type = (
|
||||||
|
(item.get("civitai") or {}).get("model", {}) or {}
|
||||||
|
).get("type", "")
|
||||||
|
return bool(civitai_type)
|
||||||
|
|
||||||
|
def _match_rematch_entry_filename(
|
||||||
|
self,
|
||||||
|
entry: dict[str, Any],
|
||||||
|
recipe_base_model: Optional[str],
|
||||||
|
filename_cache: dict[str, list[dict[str, Any]]],
|
||||||
|
*,
|
||||||
|
is_checkpoint: bool,
|
||||||
|
) -> Tuple[Optional[dict[str, Any]], Optional[str]]:
|
||||||
|
"""Match a recipe entry against local models by file name (L4).
|
||||||
|
|
||||||
|
Conservative fallback used only after the hash (L1), version-index
|
||||||
|
(L2) and computed-autov3 (L3) tiers all failed. Candidates share the
|
||||||
|
entry's normalized file name; a candidate is accepted only when BOTH
|
||||||
|
the recipe base model and the candidate's base model are known and
|
||||||
|
equal (unknown on either side rejects — never guess on missing
|
||||||
|
metadata), the type gate passes, and exactly one candidate survives
|
||||||
|
(ambiguity is a miss). Checkpoint slots additionally require positive
|
||||||
|
type evidence: lora raw items often lack ``sub_type`` while
|
||||||
|
checkpoints always carry it, so a type-less candidate is a red flag
|
||||||
|
there — an unknown-type lora must not be bound into a checkpoint
|
||||||
|
slot.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (matched item, "L4") — or ``(None, None)``.
|
||||||
|
"""
|
||||||
|
entry_name = self._normalize_filename_key(entry.get("file_name") or "")
|
||||||
|
if not entry_name:
|
||||||
|
return (None, None)
|
||||||
|
|
||||||
|
recipe_base = (recipe_base_model or "").strip().lower()
|
||||||
|
matched: list[dict[str, Any]] = []
|
||||||
|
for candidate in filename_cache.get(entry_name, []):
|
||||||
|
candidate_base = (candidate.get("base_model") or "").strip().lower()
|
||||||
|
if not recipe_base or not candidate_base:
|
||||||
|
continue
|
||||||
|
if recipe_base != candidate_base:
|
||||||
|
continue
|
||||||
|
if is_checkpoint and not self._has_positive_type_evidence(candidate):
|
||||||
|
continue
|
||||||
|
if not self._is_type_compatible(candidate, is_checkpoint=is_checkpoint):
|
||||||
|
continue
|
||||||
|
matched.append(candidate)
|
||||||
|
|
||||||
|
if len(matched) != 1:
|
||||||
|
return (None, None)
|
||||||
|
return (matched[0], "L4")
|
||||||
|
|
||||||
async def _match_rematch_entry(
|
async def _match_rematch_entry(
|
||||||
self,
|
self,
|
||||||
entry: dict[str, Any],
|
entry: dict[str, Any],
|
||||||
@@ -247,19 +416,23 @@ class RecipeScanner:
|
|||||||
autov3_cache: dict[str, Any],
|
autov3_cache: dict[str, Any],
|
||||||
*,
|
*,
|
||||||
is_checkpoint: bool,
|
is_checkpoint: bool,
|
||||||
|
filename_cache: Optional[dict[str, list[dict[str, Any]]]] = None,
|
||||||
|
recipe_base_model: Optional[str] = None,
|
||||||
) -> Tuple[Optional[dict[str, Any]], Optional[str]]:
|
) -> Tuple[Optional[dict[str, Any]], Optional[str]]:
|
||||||
"""Match a recipe entry against local models across three levels.
|
"""Match a recipe entry against local models across four levels.
|
||||||
|
|
||||||
L1 looks the stored hash up in the type-blind local hash cache; L2
|
L1 looks the stored hash up in the type-blind local hash cache; L2
|
||||||
falls back to the version index via ``modelVersionId`` or ``id``; L3
|
falls back to the version index via ``modelVersionId`` or ``id``; L3
|
||||||
resolves 12-char hashes through the computed AutoV3 cache. Matched
|
resolves 12-char hashes through the computed AutoV3 cache; L4
|
||||||
items are type-verified against the entry kind before being returned.
|
(conservative) falls back to the file name when a filename cache is
|
||||||
|
provided. Matched items are type-verified against the entry kind
|
||||||
|
before being returned.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (matched item, match level) where level is "L1", "L2" or
|
Tuple of (matched item, match level) where level is "L1", "L2",
|
||||||
"L3" — or ``(None, None)`` when no usable match exists. A missing
|
"L3" or "L4" — or ``(None, None)`` when no usable match exists. A
|
||||||
local match is an expected outcome (the model may simply not be
|
missing local match is an expected outcome (the model may simply
|
||||||
present locally), not an error.
|
not be present locally), not an error.
|
||||||
"""
|
"""
|
||||||
entry_hash = (entry.get("hash") or "").lower()
|
entry_hash = (entry.get("hash") or "").lower()
|
||||||
|
|
||||||
@@ -279,33 +452,20 @@ class RecipeScanner:
|
|||||||
item = autov3_cache.get(entry_hash)
|
item = autov3_cache.get(entry_hash)
|
||||||
level = "L3" if item is not None else None
|
level = "L3" if item is not None else None
|
||||||
|
|
||||||
|
if item is None and filename_cache is not None:
|
||||||
|
item, level = self._match_rematch_entry_filename(
|
||||||
|
entry,
|
||||||
|
recipe_base_model,
|
||||||
|
filename_cache,
|
||||||
|
is_checkpoint=is_checkpoint,
|
||||||
|
)
|
||||||
|
level = "L4" if item is not None else None
|
||||||
|
|
||||||
if item is None:
|
if item is None:
|
||||||
return (None, None)
|
return (None, None)
|
||||||
|
|
||||||
# Type gate: the L1 cache merges lora and checkpoint items and is
|
if not self._is_type_compatible(item, is_checkpoint=is_checkpoint):
|
||||||
# type-blind, so a match must be verified against the entry kind.
|
return (None, None)
|
||||||
sub_type = (item.get("sub_type") or "").lower()
|
|
||||||
if sub_type:
|
|
||||||
valid = (
|
|
||||||
VALID_CHECKPOINT_SUB_TYPES if is_checkpoint else VALID_LORA_TYPES
|
|
||||||
)
|
|
||||||
if sub_type not in valid:
|
|
||||||
return (None, None)
|
|
||||||
else:
|
|
||||||
civitai_type = (
|
|
||||||
(item.get("civitai") or {}).get("model", {}) or {}
|
|
||||||
).get("type", "")
|
|
||||||
if civitai_type:
|
|
||||||
normalized = civitai_type.lower()
|
|
||||||
if is_checkpoint:
|
|
||||||
normalized = _CHECKPOINT_MODEL_TYPE_ALIASES.get(
|
|
||||||
normalized, normalized
|
|
||||||
)
|
|
||||||
valid = VALID_CHECKPOINT_SUB_TYPES
|
|
||||||
else:
|
|
||||||
valid = VALID_LORA_TYPES
|
|
||||||
if normalized not in valid:
|
|
||||||
return (None, None)
|
|
||||||
|
|
||||||
return (item, level)
|
return (item, level)
|
||||||
|
|
||||||
@@ -617,10 +777,11 @@ class RecipeScanner:
|
|||||||
async def _rematch_recipe_by_id(self, recipe_id: str) -> Dict[str, Any]:
|
async def _rematch_recipe_by_id(self, recipe_id: str) -> Dict[str, Any]:
|
||||||
"""Rematch a single recipe's deleted lora/checkpoint entries locally.
|
"""Rematch a single recipe's deleted lora/checkpoint entries locally.
|
||||||
|
|
||||||
Match snapshots (local hash cache + computed autov3 cache) are built
|
Match snapshots (local hash cache, computed autov3 cache, filename
|
||||||
BEFORE acquiring the mutation lock — both are read-only snapshots and
|
cache) are built BEFORE acquiring the mutation lock — all three are
|
||||||
the version-cached hash dict would otherwise rebuild mid-run if a scan
|
read-only snapshots and the version-cached dicts would otherwise
|
||||||
bumps a scanner's cache_version while we hold the lock.
|
rebuild mid-run if a scan bumps a scanner's cache_version while we
|
||||||
|
hold the lock.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
recipe_id: ID of the recipe to rematch
|
recipe_id: ID of the recipe to rematch
|
||||||
@@ -636,6 +797,7 @@ class RecipeScanner:
|
|||||||
"""
|
"""
|
||||||
local_cache = await self.build_local_hash_cache()
|
local_cache = await self.build_local_hash_cache()
|
||||||
autov3_cache = await self._build_rematch_autov3_cache()
|
autov3_cache = await self._build_rematch_autov3_cache()
|
||||||
|
filename_cache = await self._build_local_filename_cache()
|
||||||
|
|
||||||
async with self._mutation_lock:
|
async with self._mutation_lock:
|
||||||
# Get raw recipe from cache directly to avoid formatted fields
|
# Get raw recipe from cache directly to avoid formatted fields
|
||||||
@@ -649,7 +811,7 @@ class RecipeScanner:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
rematched, _errors, details = await self._rematch_single_recipe(
|
rematched, _errors, details = await self._rematch_single_recipe(
|
||||||
recipe, local_cache, autov3_cache
|
recipe, local_cache, autov3_cache, filename_cache
|
||||||
)
|
)
|
||||||
except RecipePersistenceError as exc:
|
except RecipePersistenceError as exc:
|
||||||
logger.error(
|
logger.error(
|
||||||
@@ -706,6 +868,7 @@ class RecipeScanner:
|
|||||||
recipe: Dict[str, Any],
|
recipe: Dict[str, Any],
|
||||||
local_cache: dict[str, dict[str, Any]],
|
local_cache: dict[str, dict[str, Any]],
|
||||||
autov3_cache: dict[str, dict[str, Any]],
|
autov3_cache: dict[str, dict[str, Any]],
|
||||||
|
filename_cache: Optional[dict[str, list[dict[str, Any]]]] = None,
|
||||||
) -> Tuple[int, int, Dict[str, Any]]:
|
) -> Tuple[int, int, Dict[str, Any]]:
|
||||||
"""Rematch a single recipe's lora/checkpoint entries against local models.
|
"""Rematch a single recipe's lora/checkpoint entries against local models.
|
||||||
|
|
||||||
@@ -719,6 +882,8 @@ class RecipeScanner:
|
|||||||
recipe: The recipe dictionary to rematch (modified in-place)
|
recipe: The recipe dictionary to rematch (modified in-place)
|
||||||
local_cache: L1 hash cache snapshot (build_local_hash_cache)
|
local_cache: L1 hash cache snapshot (build_local_hash_cache)
|
||||||
autov3_cache: L3 computed-autov3 cache snapshot
|
autov3_cache: L3 computed-autov3 cache snapshot
|
||||||
|
filename_cache: L4 filename cache snapshot, or None to disable
|
||||||
|
the filename fallback
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (rematched_entries, errors, details). The errors element
|
Tuple of (rematched_entries, errors, details). The errors element
|
||||||
@@ -744,7 +909,13 @@ class RecipeScanner:
|
|||||||
if not self._is_rematch_candidate(entry):
|
if not self._is_rematch_candidate(entry):
|
||||||
continue
|
continue
|
||||||
item, level = await self._match_rematch_entry_with_level(
|
item, level = await self._match_rematch_entry_with_level(
|
||||||
entry, local_cache, autov3_cache, is_checkpoint=False
|
entry,
|
||||||
|
local_cache,
|
||||||
|
autov3_cache,
|
||||||
|
is_checkpoint=False,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model=entry.get("baseModel")
|
||||||
|
or recipe.get("base_model"),
|
||||||
)
|
)
|
||||||
if item is None:
|
if item is None:
|
||||||
details["unresolved"].append(
|
details["unresolved"].append(
|
||||||
@@ -770,7 +941,13 @@ class RecipeScanner:
|
|||||||
if isinstance(checkpoint, dict):
|
if isinstance(checkpoint, dict):
|
||||||
if self._is_rematch_candidate(checkpoint):
|
if self._is_rematch_candidate(checkpoint):
|
||||||
item, level = await self._match_rematch_entry_with_level(
|
item, level = await self._match_rematch_entry_with_level(
|
||||||
checkpoint, local_cache, autov3_cache, is_checkpoint=True
|
checkpoint,
|
||||||
|
local_cache,
|
||||||
|
autov3_cache,
|
||||||
|
is_checkpoint=True,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model=checkpoint.get("baseModel")
|
||||||
|
or recipe.get("base_model"),
|
||||||
)
|
)
|
||||||
if item is None:
|
if item is None:
|
||||||
details["unresolved"].append(
|
details["unresolved"].append(
|
||||||
@@ -832,12 +1009,13 @@ class RecipeScanner:
|
|||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
"""Rematch every recipe's deleted lora/checkpoint entries locally.
|
"""Rematch every recipe's deleted lora/checkpoint entries locally.
|
||||||
|
|
||||||
Match snapshots (local hash cache + computed autov3 cache) are built
|
Match snapshots (local hash cache, computed autov3 cache, filename
|
||||||
ONCE before the loop — both are read-only and the version-cached hash
|
cache) are built ONCE before the loop — all three are read-only and
|
||||||
dict would otherwise rebuild mid-run if a scan bumps a scanner's
|
the version-cached dicts would otherwise rebuild mid-run if a scan
|
||||||
cache_version while the mutation lock is held. ``_schedule_resort`` is
|
bumps a scanner's cache_version while the mutation lock is held.
|
||||||
called exactly once after the loop: it spawns an asyncio task per call,
|
``_schedule_resort`` is called exactly once after the loop: it spawns
|
||||||
so per-recipe calls would race one resort task per recipe.
|
an asyncio task per call, so per-recipe calls would race one resort
|
||||||
|
task per recipe.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
progress_callback: Optional callback for progress updates
|
progress_callback: Optional callback for progress updates
|
||||||
@@ -858,6 +1036,7 @@ class RecipeScanner:
|
|||||||
# Match snapshots built once and shared by every recipe in the loop.
|
# Match snapshots built once and shared by every recipe in the loop.
|
||||||
local_cache = await self.build_local_hash_cache()
|
local_cache = await self.build_local_hash_cache()
|
||||||
autov3_cache = await self._build_rematch_autov3_cache()
|
autov3_cache = await self._build_rematch_autov3_cache()
|
||||||
|
filename_cache = await self._build_local_filename_cache()
|
||||||
|
|
||||||
async with self._mutation_lock:
|
async with self._mutation_lock:
|
||||||
cache = await self.get_cached_data()
|
cache = await self.get_cached_data()
|
||||||
@@ -925,7 +1104,7 @@ class RecipeScanner:
|
|||||||
)
|
)
|
||||||
|
|
||||||
rematched, _errors, details = await self._rematch_single_recipe(
|
rematched, _errors, details = await self._rematch_single_recipe(
|
||||||
recipe, local_cache, autov3_cache
|
recipe, local_cache, autov3_cache, filename_cache
|
||||||
)
|
)
|
||||||
if rematched > 0:
|
if rematched > 0:
|
||||||
matched_recipes += 1
|
matched_recipes += 1
|
||||||
|
|||||||
@@ -1883,9 +1883,10 @@ async def test_is_rematch_candidate_rejects_healthy_entry(tmp_path: Path):
|
|||||||
assert not scanner._is_rematch_candidate({"hash": "abc", "file_name": "m.safetensors"})
|
assert not scanner._is_rematch_candidate({"hash": "abc", "file_name": "m.safetensors"})
|
||||||
|
|
||||||
|
|
||||||
async def test_is_rematch_candidate_rejects_no_identifier(tmp_path: Path):
|
async def test_is_rematch_candidate_file_name_only_is_identifier(tmp_path: Path):
|
||||||
scanner, _, _ = _make_rematch_scanner([], [], tmp_path)
|
scanner, _, _ = _make_rematch_scanner([], [], tmp_path)
|
||||||
assert not scanner._is_rematch_candidate({"isDeleted": True, "file_name": "m.safetensors"})
|
# file_name alone is now an identifier (enables the L4 filename fallback)
|
||||||
|
assert scanner._is_rematch_candidate({"isDeleted": True, "file_name": "m.safetensors"})
|
||||||
assert not scanner._is_rematch_candidate({"isDeleted": True})
|
assert not scanner._is_rematch_candidate({"isDeleted": True})
|
||||||
|
|
||||||
|
|
||||||
@@ -2220,6 +2221,481 @@ async def test_match_rematch_type_gate_lora_accepts_lora_typed_item(tmp_path: Pa
|
|||||||
assert matched is not None
|
assert matched is not None
|
||||||
|
|
||||||
|
|
||||||
|
# _match_rematch_entry — L4 filename fallback (conservative)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_match_rematch_entry_l4_filename_hit(tmp_path: Path):
|
||||||
|
item = _rematch_item(
|
||||||
|
sha256=("T1" * 32).lower(),
|
||||||
|
sub_type="lora",
|
||||||
|
base_model="SD 1.5",
|
||||||
|
file_name="detail.safetensors",
|
||||||
|
)
|
||||||
|
scanner, lora, _ = _make_rematch_scanner([item], [], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
matched, level = await scanner._match_rematch_entry_with_level(
|
||||||
|
{"file_name": "detail.safetensors", "isDeleted": True},
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
is_checkpoint=False,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model="SD 1.5",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert matched is lora._cache.raw_data[0]
|
||||||
|
assert level == "L4"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_match_rematch_entry_l4_filename_normalized_key(tmp_path: Path):
|
||||||
|
# case, path and extension differences are normalized on both sides
|
||||||
|
item = _rematch_item(
|
||||||
|
sha256=("T2" * 32).lower(),
|
||||||
|
sub_type="lora",
|
||||||
|
base_model="SDXL",
|
||||||
|
file_name="My_Mix.safetensors",
|
||||||
|
)
|
||||||
|
scanner, lora, _ = _make_rematch_scanner([item], [], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
matched, level = await scanner._match_rematch_entry_with_level(
|
||||||
|
{"file_name": "subdir/my_mix", "isDeleted": True},
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
is_checkpoint=False,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model="sdxl",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert matched is lora._cache.raw_data[0]
|
||||||
|
assert level == "L4"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_match_rematch_entry_l4_dotted_stem_no_collision(tmp_path: Path):
|
||||||
|
# "my.mix" (dotted stem) and "my" are distinct names — splitext-style
|
||||||
|
# stripping would collapse both to "my" and bind the wrong model as a
|
||||||
|
# unique candidate.
|
||||||
|
item = _rematch_item(
|
||||||
|
sha256=("T2A" * 32).lower(),
|
||||||
|
sub_type="lora",
|
||||||
|
base_model="SD 1.5",
|
||||||
|
file_name="my.mix",
|
||||||
|
)
|
||||||
|
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
matched, level = await scanner._match_rematch_entry_with_level(
|
||||||
|
{"file_name": "my", "isDeleted": True},
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
is_checkpoint=False,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model="SD 1.5",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (matched, level) == (None, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_match_rematch_entry_l4_extension_bearing_entry_reconciled(tmp_path: Path):
|
||||||
|
# extension-bearing entry names reconcile with extensionless items
|
||||||
|
item = _rematch_item(
|
||||||
|
sha256=("T2B" * 32).lower(),
|
||||||
|
sub_type="lora",
|
||||||
|
base_model="SD 1.5",
|
||||||
|
file_name="my.mix.v1",
|
||||||
|
)
|
||||||
|
scanner, lora, _ = _make_rematch_scanner([item], [], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
matched, level = await scanner._match_rematch_entry_with_level(
|
||||||
|
{"file_name": "my.mix.v1.safetensors", "isDeleted": True},
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
is_checkpoint=False,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model="SD 1.5",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert matched is lora._cache.raw_data[0]
|
||||||
|
assert level == "L4"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_match_rematch_entry_l4_base_model_mismatch_rejects(tmp_path: Path):
|
||||||
|
item = _rematch_item(
|
||||||
|
sha256=("T3" * 32).lower(),
|
||||||
|
sub_type="lora",
|
||||||
|
base_model="SD 1.5",
|
||||||
|
file_name="detail.safetensors",
|
||||||
|
)
|
||||||
|
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
matched, level = await scanner._match_rematch_entry_with_level(
|
||||||
|
{"file_name": "detail.safetensors", "isDeleted": True},
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
is_checkpoint=False,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model="SDXL",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (matched, level) == (None, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_match_rematch_entry_l4_recipe_base_model_unknown_rejects(tmp_path: Path):
|
||||||
|
item = _rematch_item(
|
||||||
|
sha256=("T4" * 32).lower(),
|
||||||
|
sub_type="lora",
|
||||||
|
base_model="SD 1.5",
|
||||||
|
file_name="detail.safetensors",
|
||||||
|
)
|
||||||
|
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
matched, level = await scanner._match_rematch_entry_with_level(
|
||||||
|
{"file_name": "detail.safetensors", "isDeleted": True},
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
is_checkpoint=False,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model="",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (matched, level) == (None, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_match_rematch_entry_l4_item_base_model_unknown_rejects(tmp_path: Path):
|
||||||
|
item = _rematch_item(
|
||||||
|
sha256=("T5" * 32).lower(), sub_type="lora", file_name="detail.safetensors"
|
||||||
|
)
|
||||||
|
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
matched, level = await scanner._match_rematch_entry_with_level(
|
||||||
|
{"file_name": "detail.safetensors", "isDeleted": True},
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
is_checkpoint=False,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model="SD 1.5",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (matched, level) == (None, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_match_rematch_entry_l4_ambiguous_same_base_model_rejects(tmp_path: Path):
|
||||||
|
items = [
|
||||||
|
_rematch_item(
|
||||||
|
sha256=("T6" * 32).lower(),
|
||||||
|
sub_type="lora",
|
||||||
|
base_model="SD 1.5",
|
||||||
|
file_name="detail.safetensors",
|
||||||
|
),
|
||||||
|
_rematch_item(
|
||||||
|
sha256=("T7" * 32).lower(),
|
||||||
|
sub_type="lora",
|
||||||
|
base_model="SD 1.5",
|
||||||
|
file_name="detail.safetensors",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
scanner, _, _ = _make_rematch_scanner(items, [], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
matched, level = await scanner._match_rematch_entry_with_level(
|
||||||
|
{"file_name": "detail.safetensors", "isDeleted": True},
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
is_checkpoint=False,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model="SD 1.5",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (matched, level) == (None, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_match_rematch_entry_l4_ambiguity_resolved_by_base_model(tmp_path: Path):
|
||||||
|
sdxl_item = _rematch_item(
|
||||||
|
sha256=("T8" * 32).lower(),
|
||||||
|
sub_type="lora",
|
||||||
|
base_model="SDXL",
|
||||||
|
file_name="detail.safetensors",
|
||||||
|
)
|
||||||
|
sd15_item = _rematch_item(
|
||||||
|
sha256=("T9" * 32).lower(),
|
||||||
|
sub_type="lora",
|
||||||
|
base_model="SD 1.5",
|
||||||
|
file_name="detail.safetensors",
|
||||||
|
)
|
||||||
|
scanner, lora, _ = _make_rematch_scanner([sdxl_item, sd15_item], [], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
matched, level = await scanner._match_rematch_entry_with_level(
|
||||||
|
{"file_name": "detail.safetensors", "isDeleted": True},
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
is_checkpoint=False,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model="SDXL",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert matched is lora._cache.raw_data[0]
|
||||||
|
assert level == "L4"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_match_rematch_entry_l4_type_gate_rejects(tmp_path: Path):
|
||||||
|
# a checkpoint-typed item with a matching name must not satisfy a lora entry
|
||||||
|
item = _rematch_item(
|
||||||
|
sha256=("TA" * 32).lower(),
|
||||||
|
sub_type="checkpoint",
|
||||||
|
base_model="SD 1.5",
|
||||||
|
file_name="detail.safetensors",
|
||||||
|
)
|
||||||
|
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
matched, level = await scanner._match_rematch_entry_with_level(
|
||||||
|
{"file_name": "detail.safetensors", "isDeleted": True},
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
is_checkpoint=False,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model="SD 1.5",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (matched, level) == (None, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_match_rematch_entry_l4_checkpoint_slot_rejects_type_less_candidate(
|
||||||
|
tmp_path: Path,
|
||||||
|
):
|
||||||
|
# lora raw items often carry no sub_type; an unknown-type candidate must
|
||||||
|
# not be bound into a checkpoint slot
|
||||||
|
item = _rematch_item(
|
||||||
|
sha256=("TA1" * 32).lower(),
|
||||||
|
base_model="SD 1.5",
|
||||||
|
file_name="realistic.safetensors",
|
||||||
|
)
|
||||||
|
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
matched, level = await scanner._match_rematch_entry_with_level(
|
||||||
|
{"file_name": "realistic.safetensors", "isDeleted": True},
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
is_checkpoint=True,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model="SD 1.5",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (matched, level) == (None, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_match_rematch_entry_l4_checkpoint_slot_accepts_typed_candidate(
|
||||||
|
tmp_path: Path,
|
||||||
|
):
|
||||||
|
item = _rematch_item(
|
||||||
|
sha256=("TA2" * 32).lower(),
|
||||||
|
sub_type="checkpoint",
|
||||||
|
base_model="SD 1.5",
|
||||||
|
file_name="realistic.safetensors",
|
||||||
|
)
|
||||||
|
scanner, _, checkpoint = _make_rematch_scanner([], [item], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
matched, level = await scanner._match_rematch_entry_with_level(
|
||||||
|
{"file_name": "realistic.safetensors", "isDeleted": True},
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
is_checkpoint=True,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model="SD 1.5",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert matched is checkpoint._cache.raw_data[0]
|
||||||
|
assert level == "L4"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_match_rematch_entry_l4_lora_slot_accepts_type_less_candidate(tmp_path: Path):
|
||||||
|
# asymmetry: lora slots still accept type-less candidates (the norm for
|
||||||
|
# lora raw items); checkpoint items always carry sub_type, so the type
|
||||||
|
# gate alone protects the reverse direction
|
||||||
|
item = _rematch_item(
|
||||||
|
sha256=("TA3" * 32).lower(), base_model="SD 1.5", file_name="detail.safetensors"
|
||||||
|
)
|
||||||
|
scanner, lora, _ = _make_rematch_scanner([item], [], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
matched, level = await scanner._match_rematch_entry_with_level(
|
||||||
|
{"file_name": "detail.safetensors", "isDeleted": True},
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
is_checkpoint=False,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model="SD 1.5",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert matched is lora._cache.raw_data[0]
|
||||||
|
assert level == "L4"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_rematch_l4_entry_base_model_preferred_over_recipe(tmp_path: Path, monkeypatch):
|
||||||
|
# a Pony lora inside an SD 1.5 recipe matches via its own baseModel
|
||||||
|
item = _rematch_item(
|
||||||
|
sha256=("TB1" * 32).lower(),
|
||||||
|
sub_type="lora",
|
||||||
|
base_model="Pony",
|
||||||
|
file_name="pony.safetensors",
|
||||||
|
)
|
||||||
|
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
saved, _ = await _spy_rematch_persistence(scanner, monkeypatch)
|
||||||
|
await _spy_fts(scanner, monkeypatch)
|
||||||
|
|
||||||
|
recipe: Dict[str, Any] = {
|
||||||
|
"id": "r1",
|
||||||
|
"base_model": "SD 1.5",
|
||||||
|
"loras": [
|
||||||
|
{"file_name": "pony.safetensors", "isDeleted": True, "baseModel": "Pony"}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
rematched, _errors, details = await scanner._rematch_single_recipe(
|
||||||
|
recipe, {}, {}, filename_cache
|
||||||
|
)
|
||||||
|
|
||||||
|
assert rematched == 1
|
||||||
|
assert details["matched"][0]["match_level"] == "L4"
|
||||||
|
assert recipe["loras"][0]["hash"] == ("TB1" * 32).lower()
|
||||||
|
assert saved == [recipe]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_rematch_l4_entry_base_model_missing_falls_back_to_recipe(
|
||||||
|
tmp_path: Path, monkeypatch
|
||||||
|
):
|
||||||
|
# without entry-level baseModel the recipe-level gate governs: a Pony
|
||||||
|
# candidate must not match an SD 1.5 recipe
|
||||||
|
item = _rematch_item(
|
||||||
|
sha256=("TB2" * 32).lower(),
|
||||||
|
sub_type="lora",
|
||||||
|
base_model="Pony",
|
||||||
|
file_name="pony.safetensors",
|
||||||
|
)
|
||||||
|
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
await _spy_rematch_persistence(scanner, monkeypatch)
|
||||||
|
await _spy_fts(scanner, monkeypatch)
|
||||||
|
|
||||||
|
recipe: Dict[str, Any] = {
|
||||||
|
"id": "r1",
|
||||||
|
"base_model": "SD 1.5",
|
||||||
|
"loras": [{"file_name": "pony.safetensors", "isDeleted": True}],
|
||||||
|
}
|
||||||
|
rematched, _errors, details = await scanner._rematch_single_recipe(
|
||||||
|
recipe, {}, {}, filename_cache
|
||||||
|
)
|
||||||
|
|
||||||
|
assert rematched == 0
|
||||||
|
assert details["unresolved"] == [{"type": "lora", "entry": "pony.safetensors"}]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_match_rematch_entry_l4_no_filename_hit(tmp_path: Path):
|
||||||
|
item = _rematch_item(
|
||||||
|
sha256=("TB" * 32).lower(),
|
||||||
|
sub_type="lora",
|
||||||
|
base_model="SD 1.5",
|
||||||
|
file_name="other.safetensors",
|
||||||
|
)
|
||||||
|
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
matched, level = await scanner._match_rematch_entry_with_level(
|
||||||
|
{"file_name": "missing.safetensors", "isDeleted": True},
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
is_checkpoint=False,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model="SD 1.5",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (matched, level) == (None, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_match_rematch_entry_l4_entry_without_file_name_skipped(tmp_path: Path):
|
||||||
|
item = _rematch_item(
|
||||||
|
sha256=("TC" * 32).lower(),
|
||||||
|
sub_type="lora",
|
||||||
|
base_model="SD 1.5",
|
||||||
|
file_name="detail.safetensors",
|
||||||
|
)
|
||||||
|
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
matched, level = await scanner._match_rematch_entry_with_level(
|
||||||
|
{"isDeleted": True, "hash": ""},
|
||||||
|
{},
|
||||||
|
{},
|
||||||
|
is_checkpoint=False,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model="SD 1.5",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (matched, level) == (None, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_match_rematch_entry_l1_wins_over_l4_filename(tmp_path: Path):
|
||||||
|
# a valid stored hash resolves via L1 even when the filename would match
|
||||||
|
sha256 = ("TD" * 32).lower()
|
||||||
|
l1_item = _rematch_item(
|
||||||
|
sha256=sha256, sub_type="lora", base_model="SD 1.5", file_name="l1-item.safetensors"
|
||||||
|
)
|
||||||
|
l4_item = _rematch_item(
|
||||||
|
sha256=("TE" * 32).lower(),
|
||||||
|
sub_type="lora",
|
||||||
|
base_model="SD 1.5",
|
||||||
|
file_name="detail.safetensors",
|
||||||
|
)
|
||||||
|
scanner, lora, _ = _make_rematch_scanner([l1_item, l4_item], [], tmp_path)
|
||||||
|
local_cache = await scanner.build_local_hash_cache()
|
||||||
|
filename_cache = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
matched, level = await scanner._match_rematch_entry_with_level(
|
||||||
|
{"hash": sha256, "file_name": "detail.safetensors", "isDeleted": True},
|
||||||
|
local_cache,
|
||||||
|
{},
|
||||||
|
is_checkpoint=False,
|
||||||
|
filename_cache=filename_cache,
|
||||||
|
recipe_base_model="SD 1.5",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert matched is lora._cache.raw_data[0]
|
||||||
|
assert level == "L1"
|
||||||
|
|
||||||
|
|
||||||
|
# _build_local_filename_cache
|
||||||
|
|
||||||
|
|
||||||
|
async def test_build_local_filename_cache_normalized_keys_sha256_only(tmp_path: Path):
|
||||||
|
lora_items = [
|
||||||
|
_rematch_item(sha256=("TF" * 32).lower(), file_name="Case.Mix.safetensors"),
|
||||||
|
_rematch_item(sha256="", file_name="no-hash.safetensors"), # skipped
|
||||||
|
]
|
||||||
|
checkpoint_items = [
|
||||||
|
_rematch_item(
|
||||||
|
sha256=("TG" * 32).lower(), sub_type="checkpoint", file_name="Base.safetensors"
|
||||||
|
)
|
||||||
|
]
|
||||||
|
scanner, lora, checkpoint = _make_rematch_scanner(
|
||||||
|
lora_items, checkpoint_items, tmp_path
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await scanner._build_local_filename_cache()
|
||||||
|
|
||||||
|
assert set(result) == {"case.mix", "base"}
|
||||||
|
assert len(result["case.mix"]) == 1
|
||||||
|
assert result["case.mix"][0] is lora._cache.raw_data[0]
|
||||||
|
# checkpoint items are indexed too (type-blind cache)
|
||||||
|
assert result["base"][0] is checkpoint._cache.raw_data[0]
|
||||||
|
|
||||||
|
|
||||||
# _build_rematch_autov3_cache
|
# _build_rematch_autov3_cache
|
||||||
|
|
||||||
|
|
||||||
@@ -3089,6 +3565,7 @@ async def test_rematch_all_recipes_per_recipe_error_continues_loop(
|
|||||||
recipe: Dict[str, Any],
|
recipe: Dict[str, Any],
|
||||||
local_cache: dict[str, Any],
|
local_cache: dict[str, Any],
|
||||||
autov3_cache: dict[str, Any],
|
autov3_cache: dict[str, Any],
|
||||||
|
filename_cache=None,
|
||||||
) -> tuple[int, int, dict[str, Any]]:
|
) -> tuple[int, int, dict[str, Any]]:
|
||||||
if recipe.get("id") == "boom":
|
if recipe.get("id") == "boom":
|
||||||
raise RuntimeError("kaboom")
|
raise RuntimeError("kaboom")
|
||||||
@@ -3146,12 +3623,13 @@ async def test_rematch_all_recipes_holds_mutation_lock(tmp_path: Path, monkeypat
|
|||||||
recipe: Dict[str, Any],
|
recipe: Dict[str, Any],
|
||||||
local_cache: dict[str, Any],
|
local_cache: dict[str, Any],
|
||||||
autov3_cache: dict[str, Any],
|
autov3_cache: dict[str, Any],
|
||||||
) -> tuple[int, int]:
|
filename_cache=None,
|
||||||
|
) -> tuple[int, int, dict[str, Any]]:
|
||||||
nonlocal entered
|
nonlocal entered
|
||||||
if recipe.get("id") == "r0":
|
if recipe.get("id") == "r0":
|
||||||
entered = True
|
entered = True
|
||||||
await release.wait()
|
await release.wait()
|
||||||
return await original(recipe, local_cache, autov3_cache)
|
return await original(recipe, local_cache, autov3_cache, filename_cache)
|
||||||
|
|
||||||
monkeypatch.setattr(scanner, "_rematch_single_recipe", blocking_single)
|
monkeypatch.setattr(scanner, "_rematch_single_recipe", blocking_single)
|
||||||
|
|
||||||
@@ -3271,6 +3749,8 @@ async def test_rematch_bulk_generic_exception_continues(tmp_path: Path, monkeypa
|
|||||||
autov3_cache: dict[str, Any],
|
autov3_cache: dict[str, Any],
|
||||||
*,
|
*,
|
||||||
is_checkpoint: bool,
|
is_checkpoint: bool,
|
||||||
|
filename_cache=None,
|
||||||
|
recipe_base_model=None,
|
||||||
) -> Any:
|
) -> Any:
|
||||||
nonlocal calls
|
nonlocal calls
|
||||||
calls += 1
|
calls += 1
|
||||||
|
|||||||
Reference in New Issue
Block a user