feat(recipes): add filename fallback tier to recipe rematch

This commit is contained in:
Will Miao
2026-08-16 09:17:59 +08:00
parent 395682509c
commit 38809a9d1b
2 changed files with 709 additions and 50 deletions
+225 -46
View File
@@ -36,6 +36,11 @@ logger = logging.getLogger(__name__)
# explicitly to "diffusion_model" (mirrors Oracle R2-F1). # explicitly to "diffusion_model" (mirrors Oracle R2-F1).
_CHECKPOINT_MODEL_TYPE_ALIASES = {"diffusionmodel": "diffusion_model"} _CHECKPOINT_MODEL_TYPE_ALIASES = {"diffusionmodel": "diffusion_model"}
# Known weight-file extensions stripped by _normalize_filename_key. Names are
# stored extensionless on both sides, so splitext would misread dotted stems
# ("my.mix" -> "my") and silently collide distinct models.
_WEIGHT_FILE_EXTS = (".safetensors", ".ckpt", ".pt", ".pth", ".gguf", ".bin", ".safebin", ".sft")
class RecipeScanner: class RecipeScanner:
"""Service for scanning and managing recipe images""" """Service for scanning and managing recipe images"""
@@ -116,6 +121,12 @@ class RecipeScanner:
self._rematch_autov3_cache: dict[str, dict[str, Any]] | None = None self._rematch_autov3_cache: dict[str, dict[str, Any]] | None = None
self._rematch_autov3_versions: tuple[int, int] | None = None self._rematch_autov3_versions: tuple[int, int] | None = None
self._rematch_autov3_lock = asyncio.Lock() self._rematch_autov3_lock = asyncio.Lock()
# Normalized filename -> [items] map for the L4 rematch fallback,
# rebuilt only when either model scanner's cache_version changes.
# Mirrors the build_local_hash_cache version pattern.
self._local_filename_cache: dict[str, list[dict[str, Any]]] | None = None
self._local_filename_cache_versions: tuple[int, int] | None = None
self._local_filename_cache_lock = asyncio.Lock()
self._initialized = True self._initialized = True
async def build_local_hash_cache(self) -> dict[str, dict[str, Any]]: async def build_local_hash_cache(self) -> dict[str, dict[str, Any]]:
@@ -162,6 +173,70 @@ class RecipeScanner:
self._local_hash_cache_versions = versions self._local_hash_cache_versions = versions
return cache return cache
@staticmethod
def _normalize_filename_key(name: str) -> str:
"""Normalize a file name to a lookup key (basename, lowercase).
Only known weight-file extensions are stripped — names are stored
extensionless on both sides, so splitext would misread dotted stems
("my.mix" -> "my") and collide distinct models.
"""
if not name:
return ""
basename = os.path.basename(name.replace("\\", "/"))
lower = basename.lower()
for ext in _WEIGHT_FILE_EXTS:
if lower.endswith(ext):
basename = basename[: -len(ext)]
break
return basename.strip().lower()
async def _build_local_filename_cache(self) -> dict[str, list[dict[str, Any]]]:
"""Build a version-cached map of normalized file names to local items.
Keys are lowercase basenames without extension. Values are lists of
items (lora + checkpoint, type-blind) sharing that name. Only items
with a sha256 are indexed — matching a pending or failed download
(empty sha256) would leave the entry without a usable hash. The dict
is reused while both scanners' cache_version values are unchanged;
concurrent callers share a single build via the lock.
"""
async with self._local_filename_cache_lock:
lora_scanner = self._lora_scanner
checkpoint_scanner = self._checkpoint_scanner
versions = (
lora_scanner.cache_version if lora_scanner is not None else 0,
checkpoint_scanner.cache_version
if checkpoint_scanner is not None
else 0,
)
if (
self._local_filename_cache is not None
and self._local_filename_cache_versions == versions
):
return self._local_filename_cache
cache: dict[str, list[dict[str, Any]]] = {}
for scanner in (lora_scanner, checkpoint_scanner):
if scanner is None:
continue
data = await scanner.get_cached_data()
for item in data.raw_data:
if not isinstance(item, dict):
continue
if not (item.get("sha256") or "").lower():
continue
file_path = item.get("file_path") or ""
file_name = item.get("file_name") or ""
key = self._normalize_filename_key(file_name or file_path)
if not key:
continue
cache.setdefault(key, []).append(item)
self._local_filename_cache = cache
self._local_filename_cache_versions = versions
return cache
def _is_rematch_candidate(self, entry: dict[str, Any]) -> bool: def _is_rematch_candidate(self, entry: dict[str, Any]) -> bool:
"""Return True when a recipe entry is eligible for local re-matching.""" """Return True when a recipe entry is eligible for local re-matching."""
if not isinstance(entry, dict): if not isinstance(entry, dict):
@@ -170,7 +245,10 @@ class RecipeScanner:
entry.get("isDeleted") or not entry.get("hash") or not entry.get("file_name") entry.get("isDeleted") or not entry.get("hash") or not entry.get("file_name")
) )
has_identifier = ( has_identifier = (
entry.get("hash") or entry.get("modelVersionId") or entry.get("id") entry.get("hash")
or entry.get("modelVersionId")
or entry.get("id")
or entry.get("file_name")
) )
return bool(unresolved and has_identifier) return bool(unresolved and has_identifier)
@@ -221,6 +299,97 @@ class RecipeScanner:
self._rematch_autov3_versions = versions self._rematch_autov3_versions = versions
return cache return cache
def _is_type_compatible(self, item: dict[str, Any], *, is_checkpoint: bool) -> bool:
"""Return True when a local item's type matches the entry kind.
The L1 hash cache and the L4 filename cache merge lora and checkpoint
items and are type-blind, so a match must be verified against the
entry kind before it is accepted.
"""
sub_type = (item.get("sub_type") or "").lower()
if sub_type:
valid = (
VALID_CHECKPOINT_SUB_TYPES if is_checkpoint else VALID_LORA_TYPES
)
return sub_type in valid
civitai_type = (
(item.get("civitai") or {}).get("model", {}) or {}
).get("type", "")
if civitai_type:
normalized = civitai_type.lower()
if is_checkpoint:
normalized = _CHECKPOINT_MODEL_TYPE_ALIASES.get(
normalized, normalized
)
valid = VALID_CHECKPOINT_SUB_TYPES
else:
valid = VALID_LORA_TYPES
return normalized in valid
return True
@staticmethod
def _has_positive_type_evidence(item: dict[str, Any]) -> bool:
"""Return True when the item carries an explicit type marker.
Lora raw items rarely carry ``sub_type`` (it is only written when
metadata provides it), while checkpoint items always do — so for
checkpoint slots a type-less candidate is a red flag, not the norm.
"""
if (item.get("sub_type") or "").lower():
return True
civitai_type = (
(item.get("civitai") or {}).get("model", {}) or {}
).get("type", "")
return bool(civitai_type)
def _match_rematch_entry_filename(
self,
entry: dict[str, Any],
recipe_base_model: Optional[str],
filename_cache: dict[str, list[dict[str, Any]]],
*,
is_checkpoint: bool,
) -> Tuple[Optional[dict[str, Any]], Optional[str]]:
"""Match a recipe entry against local models by file name (L4).
Conservative fallback used only after the hash (L1), version-index
(L2) and computed-autov3 (L3) tiers all failed. Candidates share the
entry's normalized file name; a candidate is accepted only when BOTH
the recipe base model and the candidate's base model are known and
equal (unknown on either side rejects — never guess on missing
metadata), the type gate passes, and exactly one candidate survives
(ambiguity is a miss). Checkpoint slots additionally require positive
type evidence: lora raw items often lack ``sub_type`` while
checkpoints always carry it, so a type-less candidate is a red flag
there — an unknown-type lora must not be bound into a checkpoint
slot.
Returns:
Tuple of (matched item, "L4") — or ``(None, None)``.
"""
entry_name = self._normalize_filename_key(entry.get("file_name") or "")
if not entry_name:
return (None, None)
recipe_base = (recipe_base_model or "").strip().lower()
matched: list[dict[str, Any]] = []
for candidate in filename_cache.get(entry_name, []):
candidate_base = (candidate.get("base_model") or "").strip().lower()
if not recipe_base or not candidate_base:
continue
if recipe_base != candidate_base:
continue
if is_checkpoint and not self._has_positive_type_evidence(candidate):
continue
if not self._is_type_compatible(candidate, is_checkpoint=is_checkpoint):
continue
matched.append(candidate)
if len(matched) != 1:
return (None, None)
return (matched[0], "L4")
async def _match_rematch_entry( async def _match_rematch_entry(
self, self,
entry: dict[str, Any], entry: dict[str, Any],
@@ -247,19 +416,23 @@ class RecipeScanner:
autov3_cache: dict[str, Any], autov3_cache: dict[str, Any],
*, *,
is_checkpoint: bool, is_checkpoint: bool,
filename_cache: Optional[dict[str, list[dict[str, Any]]]] = None,
recipe_base_model: Optional[str] = None,
) -> Tuple[Optional[dict[str, Any]], Optional[str]]: ) -> Tuple[Optional[dict[str, Any]], Optional[str]]:
"""Match a recipe entry against local models across three levels. """Match a recipe entry against local models across four levels.
L1 looks the stored hash up in the type-blind local hash cache; L2 L1 looks the stored hash up in the type-blind local hash cache; L2
falls back to the version index via ``modelVersionId`` or ``id``; L3 falls back to the version index via ``modelVersionId`` or ``id``; L3
resolves 12-char hashes through the computed AutoV3 cache. Matched resolves 12-char hashes through the computed AutoV3 cache; L4
items are type-verified against the entry kind before being returned. (conservative) falls back to the file name when a filename cache is
provided. Matched items are type-verified against the entry kind
before being returned.
Returns: Returns:
Tuple of (matched item, match level) where level is "L1", "L2" or Tuple of (matched item, match level) where level is "L1", "L2",
"L3" — or ``(None, None)`` when no usable match exists. A missing "L3" or "L4" — or ``(None, None)`` when no usable match exists. A
local match is an expected outcome (the model may simply not be missing local match is an expected outcome (the model may simply
present locally), not an error. not be present locally), not an error.
""" """
entry_hash = (entry.get("hash") or "").lower() entry_hash = (entry.get("hash") or "").lower()
@@ -279,33 +452,20 @@ class RecipeScanner:
item = autov3_cache.get(entry_hash) item = autov3_cache.get(entry_hash)
level = "L3" if item is not None else None level = "L3" if item is not None else None
if item is None and filename_cache is not None:
item, level = self._match_rematch_entry_filename(
entry,
recipe_base_model,
filename_cache,
is_checkpoint=is_checkpoint,
)
level = "L4" if item is not None else None
if item is None: if item is None:
return (None, None) return (None, None)
# Type gate: the L1 cache merges lora and checkpoint items and is if not self._is_type_compatible(item, is_checkpoint=is_checkpoint):
# type-blind, so a match must be verified against the entry kind. return (None, None)
sub_type = (item.get("sub_type") or "").lower()
if sub_type:
valid = (
VALID_CHECKPOINT_SUB_TYPES if is_checkpoint else VALID_LORA_TYPES
)
if sub_type not in valid:
return (None, None)
else:
civitai_type = (
(item.get("civitai") or {}).get("model", {}) or {}
).get("type", "")
if civitai_type:
normalized = civitai_type.lower()
if is_checkpoint:
normalized = _CHECKPOINT_MODEL_TYPE_ALIASES.get(
normalized, normalized
)
valid = VALID_CHECKPOINT_SUB_TYPES
else:
valid = VALID_LORA_TYPES
if normalized not in valid:
return (None, None)
return (item, level) return (item, level)
@@ -617,10 +777,11 @@ class RecipeScanner:
async def _rematch_recipe_by_id(self, recipe_id: str) -> Dict[str, Any]: async def _rematch_recipe_by_id(self, recipe_id: str) -> Dict[str, Any]:
"""Rematch a single recipe's deleted lora/checkpoint entries locally. """Rematch a single recipe's deleted lora/checkpoint entries locally.
Match snapshots (local hash cache + computed autov3 cache) are built Match snapshots (local hash cache, computed autov3 cache, filename
BEFORE acquiring the mutation lock — both are read-only snapshots and cache) are built BEFORE acquiring the mutation lock — all three are
the version-cached hash dict would otherwise rebuild mid-run if a scan read-only snapshots and the version-cached dicts would otherwise
bumps a scanner's cache_version while we hold the lock. rebuild mid-run if a scan bumps a scanner's cache_version while we
hold the lock.
Args: Args:
recipe_id: ID of the recipe to rematch recipe_id: ID of the recipe to rematch
@@ -636,6 +797,7 @@ class RecipeScanner:
""" """
local_cache = await self.build_local_hash_cache() local_cache = await self.build_local_hash_cache()
autov3_cache = await self._build_rematch_autov3_cache() autov3_cache = await self._build_rematch_autov3_cache()
filename_cache = await self._build_local_filename_cache()
async with self._mutation_lock: async with self._mutation_lock:
# Get raw recipe from cache directly to avoid formatted fields # Get raw recipe from cache directly to avoid formatted fields
@@ -649,7 +811,7 @@ class RecipeScanner:
try: try:
rematched, _errors, details = await self._rematch_single_recipe( rematched, _errors, details = await self._rematch_single_recipe(
recipe, local_cache, autov3_cache recipe, local_cache, autov3_cache, filename_cache
) )
except RecipePersistenceError as exc: except RecipePersistenceError as exc:
logger.error( logger.error(
@@ -706,6 +868,7 @@ class RecipeScanner:
recipe: Dict[str, Any], recipe: Dict[str, Any],
local_cache: dict[str, dict[str, Any]], local_cache: dict[str, dict[str, Any]],
autov3_cache: dict[str, dict[str, Any]], autov3_cache: dict[str, dict[str, Any]],
filename_cache: Optional[dict[str, list[dict[str, Any]]]] = None,
) -> Tuple[int, int, Dict[str, Any]]: ) -> Tuple[int, int, Dict[str, Any]]:
"""Rematch a single recipe's lora/checkpoint entries against local models. """Rematch a single recipe's lora/checkpoint entries against local models.
@@ -719,6 +882,8 @@ class RecipeScanner:
recipe: The recipe dictionary to rematch (modified in-place) recipe: The recipe dictionary to rematch (modified in-place)
local_cache: L1 hash cache snapshot (build_local_hash_cache) local_cache: L1 hash cache snapshot (build_local_hash_cache)
autov3_cache: L3 computed-autov3 cache snapshot autov3_cache: L3 computed-autov3 cache snapshot
filename_cache: L4 filename cache snapshot, or None to disable
the filename fallback
Returns: Returns:
Tuple of (rematched_entries, errors, details). The errors element Tuple of (rematched_entries, errors, details). The errors element
@@ -744,7 +909,13 @@ class RecipeScanner:
if not self._is_rematch_candidate(entry): if not self._is_rematch_candidate(entry):
continue continue
item, level = await self._match_rematch_entry_with_level( item, level = await self._match_rematch_entry_with_level(
entry, local_cache, autov3_cache, is_checkpoint=False entry,
local_cache,
autov3_cache,
is_checkpoint=False,
filename_cache=filename_cache,
recipe_base_model=entry.get("baseModel")
or recipe.get("base_model"),
) )
if item is None: if item is None:
details["unresolved"].append( details["unresolved"].append(
@@ -770,7 +941,13 @@ class RecipeScanner:
if isinstance(checkpoint, dict): if isinstance(checkpoint, dict):
if self._is_rematch_candidate(checkpoint): if self._is_rematch_candidate(checkpoint):
item, level = await self._match_rematch_entry_with_level( item, level = await self._match_rematch_entry_with_level(
checkpoint, local_cache, autov3_cache, is_checkpoint=True checkpoint,
local_cache,
autov3_cache,
is_checkpoint=True,
filename_cache=filename_cache,
recipe_base_model=checkpoint.get("baseModel")
or recipe.get("base_model"),
) )
if item is None: if item is None:
details["unresolved"].append( details["unresolved"].append(
@@ -832,12 +1009,13 @@ class RecipeScanner:
) -> Dict[str, Any]: ) -> Dict[str, Any]:
"""Rematch every recipe's deleted lora/checkpoint entries locally. """Rematch every recipe's deleted lora/checkpoint entries locally.
Match snapshots (local hash cache + computed autov3 cache) are built Match snapshots (local hash cache, computed autov3 cache, filename
ONCE before the loop — both are read-only and the version-cached hash cache) are built ONCE before the loop — all three are read-only and
dict would otherwise rebuild mid-run if a scan bumps a scanner's the version-cached dicts would otherwise rebuild mid-run if a scan
cache_version while the mutation lock is held. ``_schedule_resort`` is bumps a scanner's cache_version while the mutation lock is held.
called exactly once after the loop: it spawns an asyncio task per call, ``_schedule_resort`` is called exactly once after the loop: it spawns
so per-recipe calls would race one resort task per recipe. an asyncio task per call, so per-recipe calls would race one resort
task per recipe.
Args: Args:
progress_callback: Optional callback for progress updates progress_callback: Optional callback for progress updates
@@ -858,6 +1036,7 @@ class RecipeScanner:
# Match snapshots built once and shared by every recipe in the loop. # Match snapshots built once and shared by every recipe in the loop.
local_cache = await self.build_local_hash_cache() local_cache = await self.build_local_hash_cache()
autov3_cache = await self._build_rematch_autov3_cache() autov3_cache = await self._build_rematch_autov3_cache()
filename_cache = await self._build_local_filename_cache()
async with self._mutation_lock: async with self._mutation_lock:
cache = await self.get_cached_data() cache = await self.get_cached_data()
@@ -925,7 +1104,7 @@ class RecipeScanner:
) )
rematched, _errors, details = await self._rematch_single_recipe( rematched, _errors, details = await self._rematch_single_recipe(
recipe, local_cache, autov3_cache recipe, local_cache, autov3_cache, filename_cache
) )
if rematched > 0: if rematched > 0:
matched_recipes += 1 matched_recipes += 1
+484 -4
View File
@@ -1883,9 +1883,10 @@ async def test_is_rematch_candidate_rejects_healthy_entry(tmp_path: Path):
assert not scanner._is_rematch_candidate({"hash": "abc", "file_name": "m.safetensors"}) assert not scanner._is_rematch_candidate({"hash": "abc", "file_name": "m.safetensors"})
async def test_is_rematch_candidate_rejects_no_identifier(tmp_path: Path): async def test_is_rematch_candidate_file_name_only_is_identifier(tmp_path: Path):
scanner, _, _ = _make_rematch_scanner([], [], tmp_path) scanner, _, _ = _make_rematch_scanner([], [], tmp_path)
assert not scanner._is_rematch_candidate({"isDeleted": True, "file_name": "m.safetensors"}) # file_name alone is now an identifier (enables the L4 filename fallback)
assert scanner._is_rematch_candidate({"isDeleted": True, "file_name": "m.safetensors"})
assert not scanner._is_rematch_candidate({"isDeleted": True}) assert not scanner._is_rematch_candidate({"isDeleted": True})
@@ -2220,6 +2221,481 @@ async def test_match_rematch_type_gate_lora_accepts_lora_typed_item(tmp_path: Pa
assert matched is not None assert matched is not None
# _match_rematch_entry — L4 filename fallback (conservative)
async def test_match_rematch_entry_l4_filename_hit(tmp_path: Path):
item = _rematch_item(
sha256=("T1" * 32).lower(),
sub_type="lora",
base_model="SD 1.5",
file_name="detail.safetensors",
)
scanner, lora, _ = _make_rematch_scanner([item], [], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
matched, level = await scanner._match_rematch_entry_with_level(
{"file_name": "detail.safetensors", "isDeleted": True},
{},
{},
is_checkpoint=False,
filename_cache=filename_cache,
recipe_base_model="SD 1.5",
)
assert matched is lora._cache.raw_data[0]
assert level == "L4"
async def test_match_rematch_entry_l4_filename_normalized_key(tmp_path: Path):
# case, path and extension differences are normalized on both sides
item = _rematch_item(
sha256=("T2" * 32).lower(),
sub_type="lora",
base_model="SDXL",
file_name="My_Mix.safetensors",
)
scanner, lora, _ = _make_rematch_scanner([item], [], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
matched, level = await scanner._match_rematch_entry_with_level(
{"file_name": "subdir/my_mix", "isDeleted": True},
{},
{},
is_checkpoint=False,
filename_cache=filename_cache,
recipe_base_model="sdxl",
)
assert matched is lora._cache.raw_data[0]
assert level == "L4"
async def test_match_rematch_entry_l4_dotted_stem_no_collision(tmp_path: Path):
# "my.mix" (dotted stem) and "my" are distinct names — splitext-style
# stripping would collapse both to "my" and bind the wrong model as a
# unique candidate.
item = _rematch_item(
sha256=("T2A" * 32).lower(),
sub_type="lora",
base_model="SD 1.5",
file_name="my.mix",
)
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
matched, level = await scanner._match_rematch_entry_with_level(
{"file_name": "my", "isDeleted": True},
{},
{},
is_checkpoint=False,
filename_cache=filename_cache,
recipe_base_model="SD 1.5",
)
assert (matched, level) == (None, None)
async def test_match_rematch_entry_l4_extension_bearing_entry_reconciled(tmp_path: Path):
# extension-bearing entry names reconcile with extensionless items
item = _rematch_item(
sha256=("T2B" * 32).lower(),
sub_type="lora",
base_model="SD 1.5",
file_name="my.mix.v1",
)
scanner, lora, _ = _make_rematch_scanner([item], [], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
matched, level = await scanner._match_rematch_entry_with_level(
{"file_name": "my.mix.v1.safetensors", "isDeleted": True},
{},
{},
is_checkpoint=False,
filename_cache=filename_cache,
recipe_base_model="SD 1.5",
)
assert matched is lora._cache.raw_data[0]
assert level == "L4"
async def test_match_rematch_entry_l4_base_model_mismatch_rejects(tmp_path: Path):
item = _rematch_item(
sha256=("T3" * 32).lower(),
sub_type="lora",
base_model="SD 1.5",
file_name="detail.safetensors",
)
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
matched, level = await scanner._match_rematch_entry_with_level(
{"file_name": "detail.safetensors", "isDeleted": True},
{},
{},
is_checkpoint=False,
filename_cache=filename_cache,
recipe_base_model="SDXL",
)
assert (matched, level) == (None, None)
async def test_match_rematch_entry_l4_recipe_base_model_unknown_rejects(tmp_path: Path):
item = _rematch_item(
sha256=("T4" * 32).lower(),
sub_type="lora",
base_model="SD 1.5",
file_name="detail.safetensors",
)
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
matched, level = await scanner._match_rematch_entry_with_level(
{"file_name": "detail.safetensors", "isDeleted": True},
{},
{},
is_checkpoint=False,
filename_cache=filename_cache,
recipe_base_model="",
)
assert (matched, level) == (None, None)
async def test_match_rematch_entry_l4_item_base_model_unknown_rejects(tmp_path: Path):
item = _rematch_item(
sha256=("T5" * 32).lower(), sub_type="lora", file_name="detail.safetensors"
)
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
matched, level = await scanner._match_rematch_entry_with_level(
{"file_name": "detail.safetensors", "isDeleted": True},
{},
{},
is_checkpoint=False,
filename_cache=filename_cache,
recipe_base_model="SD 1.5",
)
assert (matched, level) == (None, None)
async def test_match_rematch_entry_l4_ambiguous_same_base_model_rejects(tmp_path: Path):
items = [
_rematch_item(
sha256=("T6" * 32).lower(),
sub_type="lora",
base_model="SD 1.5",
file_name="detail.safetensors",
),
_rematch_item(
sha256=("T7" * 32).lower(),
sub_type="lora",
base_model="SD 1.5",
file_name="detail.safetensors",
),
]
scanner, _, _ = _make_rematch_scanner(items, [], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
matched, level = await scanner._match_rematch_entry_with_level(
{"file_name": "detail.safetensors", "isDeleted": True},
{},
{},
is_checkpoint=False,
filename_cache=filename_cache,
recipe_base_model="SD 1.5",
)
assert (matched, level) == (None, None)
async def test_match_rematch_entry_l4_ambiguity_resolved_by_base_model(tmp_path: Path):
sdxl_item = _rematch_item(
sha256=("T8" * 32).lower(),
sub_type="lora",
base_model="SDXL",
file_name="detail.safetensors",
)
sd15_item = _rematch_item(
sha256=("T9" * 32).lower(),
sub_type="lora",
base_model="SD 1.5",
file_name="detail.safetensors",
)
scanner, lora, _ = _make_rematch_scanner([sdxl_item, sd15_item], [], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
matched, level = await scanner._match_rematch_entry_with_level(
{"file_name": "detail.safetensors", "isDeleted": True},
{},
{},
is_checkpoint=False,
filename_cache=filename_cache,
recipe_base_model="SDXL",
)
assert matched is lora._cache.raw_data[0]
assert level == "L4"
async def test_match_rematch_entry_l4_type_gate_rejects(tmp_path: Path):
# a checkpoint-typed item with a matching name must not satisfy a lora entry
item = _rematch_item(
sha256=("TA" * 32).lower(),
sub_type="checkpoint",
base_model="SD 1.5",
file_name="detail.safetensors",
)
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
matched, level = await scanner._match_rematch_entry_with_level(
{"file_name": "detail.safetensors", "isDeleted": True},
{},
{},
is_checkpoint=False,
filename_cache=filename_cache,
recipe_base_model="SD 1.5",
)
assert (matched, level) == (None, None)
async def test_match_rematch_entry_l4_checkpoint_slot_rejects_type_less_candidate(
tmp_path: Path,
):
# lora raw items often carry no sub_type; an unknown-type candidate must
# not be bound into a checkpoint slot
item = _rematch_item(
sha256=("TA1" * 32).lower(),
base_model="SD 1.5",
file_name="realistic.safetensors",
)
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
matched, level = await scanner._match_rematch_entry_with_level(
{"file_name": "realistic.safetensors", "isDeleted": True},
{},
{},
is_checkpoint=True,
filename_cache=filename_cache,
recipe_base_model="SD 1.5",
)
assert (matched, level) == (None, None)
async def test_match_rematch_entry_l4_checkpoint_slot_accepts_typed_candidate(
tmp_path: Path,
):
item = _rematch_item(
sha256=("TA2" * 32).lower(),
sub_type="checkpoint",
base_model="SD 1.5",
file_name="realistic.safetensors",
)
scanner, _, checkpoint = _make_rematch_scanner([], [item], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
matched, level = await scanner._match_rematch_entry_with_level(
{"file_name": "realistic.safetensors", "isDeleted": True},
{},
{},
is_checkpoint=True,
filename_cache=filename_cache,
recipe_base_model="SD 1.5",
)
assert matched is checkpoint._cache.raw_data[0]
assert level == "L4"
async def test_match_rematch_entry_l4_lora_slot_accepts_type_less_candidate(tmp_path: Path):
# asymmetry: lora slots still accept type-less candidates (the norm for
# lora raw items); checkpoint items always carry sub_type, so the type
# gate alone protects the reverse direction
item = _rematch_item(
sha256=("TA3" * 32).lower(), base_model="SD 1.5", file_name="detail.safetensors"
)
scanner, lora, _ = _make_rematch_scanner([item], [], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
matched, level = await scanner._match_rematch_entry_with_level(
{"file_name": "detail.safetensors", "isDeleted": True},
{},
{},
is_checkpoint=False,
filename_cache=filename_cache,
recipe_base_model="SD 1.5",
)
assert matched is lora._cache.raw_data[0]
assert level == "L4"
async def test_rematch_l4_entry_base_model_preferred_over_recipe(tmp_path: Path, monkeypatch):
# a Pony lora inside an SD 1.5 recipe matches via its own baseModel
item = _rematch_item(
sha256=("TB1" * 32).lower(),
sub_type="lora",
base_model="Pony",
file_name="pony.safetensors",
)
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
saved, _ = await _spy_rematch_persistence(scanner, monkeypatch)
await _spy_fts(scanner, monkeypatch)
recipe: Dict[str, Any] = {
"id": "r1",
"base_model": "SD 1.5",
"loras": [
{"file_name": "pony.safetensors", "isDeleted": True, "baseModel": "Pony"}
],
}
rematched, _errors, details = await scanner._rematch_single_recipe(
recipe, {}, {}, filename_cache
)
assert rematched == 1
assert details["matched"][0]["match_level"] == "L4"
assert recipe["loras"][0]["hash"] == ("TB1" * 32).lower()
assert saved == [recipe]
async def test_rematch_l4_entry_base_model_missing_falls_back_to_recipe(
tmp_path: Path, monkeypatch
):
# without entry-level baseModel the recipe-level gate governs: a Pony
# candidate must not match an SD 1.5 recipe
item = _rematch_item(
sha256=("TB2" * 32).lower(),
sub_type="lora",
base_model="Pony",
file_name="pony.safetensors",
)
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
await _spy_rematch_persistence(scanner, monkeypatch)
await _spy_fts(scanner, monkeypatch)
recipe: Dict[str, Any] = {
"id": "r1",
"base_model": "SD 1.5",
"loras": [{"file_name": "pony.safetensors", "isDeleted": True}],
}
rematched, _errors, details = await scanner._rematch_single_recipe(
recipe, {}, {}, filename_cache
)
assert rematched == 0
assert details["unresolved"] == [{"type": "lora", "entry": "pony.safetensors"}]
async def test_match_rematch_entry_l4_no_filename_hit(tmp_path: Path):
item = _rematch_item(
sha256=("TB" * 32).lower(),
sub_type="lora",
base_model="SD 1.5",
file_name="other.safetensors",
)
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
matched, level = await scanner._match_rematch_entry_with_level(
{"file_name": "missing.safetensors", "isDeleted": True},
{},
{},
is_checkpoint=False,
filename_cache=filename_cache,
recipe_base_model="SD 1.5",
)
assert (matched, level) == (None, None)
async def test_match_rematch_entry_l4_entry_without_file_name_skipped(tmp_path: Path):
item = _rematch_item(
sha256=("TC" * 32).lower(),
sub_type="lora",
base_model="SD 1.5",
file_name="detail.safetensors",
)
scanner, _, _ = _make_rematch_scanner([item], [], tmp_path)
filename_cache = await scanner._build_local_filename_cache()
matched, level = await scanner._match_rematch_entry_with_level(
{"isDeleted": True, "hash": ""},
{},
{},
is_checkpoint=False,
filename_cache=filename_cache,
recipe_base_model="SD 1.5",
)
assert (matched, level) == (None, None)
async def test_match_rematch_entry_l1_wins_over_l4_filename(tmp_path: Path):
# a valid stored hash resolves via L1 even when the filename would match
sha256 = ("TD" * 32).lower()
l1_item = _rematch_item(
sha256=sha256, sub_type="lora", base_model="SD 1.5", file_name="l1-item.safetensors"
)
l4_item = _rematch_item(
sha256=("TE" * 32).lower(),
sub_type="lora",
base_model="SD 1.5",
file_name="detail.safetensors",
)
scanner, lora, _ = _make_rematch_scanner([l1_item, l4_item], [], tmp_path)
local_cache = await scanner.build_local_hash_cache()
filename_cache = await scanner._build_local_filename_cache()
matched, level = await scanner._match_rematch_entry_with_level(
{"hash": sha256, "file_name": "detail.safetensors", "isDeleted": True},
local_cache,
{},
is_checkpoint=False,
filename_cache=filename_cache,
recipe_base_model="SD 1.5",
)
assert matched is lora._cache.raw_data[0]
assert level == "L1"
# _build_local_filename_cache
async def test_build_local_filename_cache_normalized_keys_sha256_only(tmp_path: Path):
lora_items = [
_rematch_item(sha256=("TF" * 32).lower(), file_name="Case.Mix.safetensors"),
_rematch_item(sha256="", file_name="no-hash.safetensors"), # skipped
]
checkpoint_items = [
_rematch_item(
sha256=("TG" * 32).lower(), sub_type="checkpoint", file_name="Base.safetensors"
)
]
scanner, lora, checkpoint = _make_rematch_scanner(
lora_items, checkpoint_items, tmp_path
)
result = await scanner._build_local_filename_cache()
assert set(result) == {"case.mix", "base"}
assert len(result["case.mix"]) == 1
assert result["case.mix"][0] is lora._cache.raw_data[0]
# checkpoint items are indexed too (type-blind cache)
assert result["base"][0] is checkpoint._cache.raw_data[0]
# _build_rematch_autov3_cache # _build_rematch_autov3_cache
@@ -3089,6 +3565,7 @@ async def test_rematch_all_recipes_per_recipe_error_continues_loop(
recipe: Dict[str, Any], recipe: Dict[str, Any],
local_cache: dict[str, Any], local_cache: dict[str, Any],
autov3_cache: dict[str, Any], autov3_cache: dict[str, Any],
filename_cache=None,
) -> tuple[int, int, dict[str, Any]]: ) -> tuple[int, int, dict[str, Any]]:
if recipe.get("id") == "boom": if recipe.get("id") == "boom":
raise RuntimeError("kaboom") raise RuntimeError("kaboom")
@@ -3146,12 +3623,13 @@ async def test_rematch_all_recipes_holds_mutation_lock(tmp_path: Path, monkeypat
recipe: Dict[str, Any], recipe: Dict[str, Any],
local_cache: dict[str, Any], local_cache: dict[str, Any],
autov3_cache: dict[str, Any], autov3_cache: dict[str, Any],
) -> tuple[int, int]: filename_cache=None,
) -> tuple[int, int, dict[str, Any]]:
nonlocal entered nonlocal entered
if recipe.get("id") == "r0": if recipe.get("id") == "r0":
entered = True entered = True
await release.wait() await release.wait()
return await original(recipe, local_cache, autov3_cache) return await original(recipe, local_cache, autov3_cache, filename_cache)
monkeypatch.setattr(scanner, "_rematch_single_recipe", blocking_single) monkeypatch.setattr(scanner, "_rematch_single_recipe", blocking_single)
@@ -3271,6 +3749,8 @@ async def test_rematch_bulk_generic_exception_continues(tmp_path: Path, monkeypa
autov3_cache: dict[str, Any], autov3_cache: dict[str, Any],
*, *,
is_checkpoint: bool, is_checkpoint: bool,
filename_cache=None,
recipe_base_model=None,
) -> Any: ) -> Any:
nonlocal calls nonlocal calls
calls += 1 calls += 1