feat(recipes): add manual checkpoint reconnect for broken recipe entries

Checkpoint entries that cannot be restored by download (deleted,
unresolvable hash, or name-only remnants with no CivitAI identifiers)
now get the same remediation chain LoRAs already had:

- scanner: parameterized reconnect-suggestion ranking, update/restore/
  set-hash-invalid for the checkpoint entry, and clear hashInvalid on
  rematch write-back (was only done for LoRAs)
- persistence/handlers/routes: reconnect/restore/reconnect-suggestions/
  mark-hash-invalid endpoints under /api/lm/recipe/checkpoint/*
- modal: checkpoint reconnect UI (deleted/hash-invalid badges, inline
  form with suggestions, undo for reconnected entries); download
  failures mark the hash invalid only on explicit unresolvable signals
  (not found/deleted/404/410), matching the LoRA rule
- css: checkpoint undo button shares the LoRA undo styles
- i18n: the 14 new keys translated in all 9 locales
This commit is contained in:
Will Miao
2026-08-30 18:02:15 +08:00
parent bce7d1d30c
commit c8b9db5bf4
21 changed files with 2031 additions and 38 deletions
+114
View File
@@ -116,6 +116,10 @@ class RecipeHandlerSet:
"restore_lora": self.management.restore_lora,
"get_reconnect_suggestions": self.management.get_reconnect_suggestions,
"mark_lora_hash_invalid": self.management.mark_lora_hash_invalid,
"reconnect_checkpoint": self.management.reconnect_checkpoint,
"restore_checkpoint": self.management.restore_checkpoint,
"get_checkpoint_reconnect_suggestions": self.management.get_checkpoint_reconnect_suggestions,
"mark_checkpoint_hash_invalid": self.management.mark_checkpoint_hash_invalid,
"find_duplicates": self.query.find_duplicates,
"move_recipes_bulk": self.management.move_recipes_bulk,
"bulk_delete": self.management.bulk_delete,
@@ -1683,6 +1687,116 @@ class RecipeManagementHandler:
)
return web.json_response({"error": str(exc)}, status=500)
async def reconnect_checkpoint(self, request: web.Request) -> web.Response:
try:
await self._ensure_dependencies_ready()
recipe_scanner = self._recipe_scanner_getter()
if recipe_scanner is None:
raise RuntimeError("Recipe scanner unavailable")
data = await request.json()
for field in ("recipe_id", "target_name"):
if field not in data:
raise RecipeValidationError(f"Missing required field: {field}")
result = await self._persistence_service.reconnect_checkpoint(
recipe_scanner=recipe_scanner,
recipe_id=data["recipe_id"],
target_name=data["target_name"],
)
return web.json_response(result.payload, status=result.status)
except RecipeValidationError as exc:
return web.json_response({"error": str(exc)}, status=400)
except RecipeNotFoundError as exc:
return web.json_response({"error": str(exc)}, status=404)
except Exception as exc:
self._logger.error(
"Error reconnecting checkpoint: %s", exc, exc_info=True
)
return web.json_response({"error": str(exc)}, status=500)
async def restore_checkpoint(self, request: web.Request) -> web.Response:
try:
await self._ensure_dependencies_ready()
recipe_scanner = self._recipe_scanner_getter()
if recipe_scanner is None:
raise RuntimeError("Recipe scanner unavailable")
data = await request.json()
if "recipe_id" not in data:
raise RecipeValidationError("Missing required field: recipe_id")
result = await self._persistence_service.restore_checkpoint(
recipe_scanner=recipe_scanner,
recipe_id=data["recipe_id"],
)
return web.json_response(result.payload, status=result.status)
except RecipeValidationError as exc:
return web.json_response({"error": str(exc)}, status=400)
except RecipeNotFoundError as exc:
return web.json_response({"error": str(exc)}, status=404)
except Exception as exc:
self._logger.error("Error restoring checkpoint: %s", exc, exc_info=True)
return web.json_response({"error": str(exc)}, status=500)
async def get_checkpoint_reconnect_suggestions(
self, request: web.Request
) -> web.Response:
try:
await self._ensure_dependencies_ready()
recipe_scanner = self._recipe_scanner_getter()
if recipe_scanner is None:
raise RuntimeError("Recipe scanner unavailable")
recipe_id = request.match_info.get("recipe_id")
if not recipe_id:
raise RecipeValidationError("recipe_id is required")
result = await self._persistence_service.get_checkpoint_reconnect_suggestions(
recipe_scanner=recipe_scanner,
recipe_id=recipe_id,
query=request.query.get("query") or None,
)
return web.json_response(result.payload, status=result.status)
except RecipeValidationError as exc:
return web.json_response({"error": str(exc)}, status=400)
except RecipeNotFoundError as exc:
return web.json_response({"error": str(exc)}, status=404)
except Exception as exc:
self._logger.error(
"Error suggesting checkpoint reconnect candidates: %s",
exc,
exc_info=True,
)
return web.json_response({"error": str(exc)}, status=500)
async def mark_checkpoint_hash_invalid(self, request: web.Request) -> web.Response:
try:
await self._ensure_dependencies_ready()
recipe_scanner = self._recipe_scanner_getter()
if recipe_scanner is None:
raise RuntimeError("Recipe scanner unavailable")
data = await request.json()
if "recipe_id" not in data:
raise RecipeValidationError("Missing required field: recipe_id")
result = await self._persistence_service.mark_checkpoint_hash_invalid(
recipe_scanner=recipe_scanner,
recipe_id=data["recipe_id"],
hash_invalid=bool(data.get("hash_invalid", True)),
)
return web.json_response(result.payload, status=result.status)
except RecipeValidationError as exc:
return web.json_response({"error": str(exc)}, status=400)
except RecipeNotFoundError as exc:
return web.json_response({"error": str(exc)}, status=404)
except Exception as exc:
self._logger.error(
"Error marking checkpoint hash invalid: %s", exc, exc_info=True
)
return web.json_response({"error": str(exc)}, status=500)
async def bulk_delete(self, request: web.Request) -> web.Response:
try:
await self._ensure_dependencies_ready()
+16
View File
@@ -58,6 +58,22 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
RouteDefinition(
"POST", "/api/lm/recipe/lora/mark-hash-invalid", "mark_lora_hash_invalid"
),
RouteDefinition(
"POST", "/api/lm/recipe/checkpoint/reconnect", "reconnect_checkpoint"
),
RouteDefinition(
"POST", "/api/lm/recipe/checkpoint/restore", "restore_checkpoint"
),
RouteDefinition(
"GET",
"/api/lm/recipe/{recipe_id}/checkpoint/reconnect-suggestions",
"get_checkpoint_reconnect_suggestions",
),
RouteDefinition(
"POST",
"/api/lm/recipe/checkpoint/mark-hash-invalid",
"mark_checkpoint_hash_invalid",
),
RouteDefinition("GET", "/api/lm/recipes/find-duplicates", "find_duplicates"),
RouteDefinition("POST", "/api/lm/recipes/bulk-delete", "bulk_delete"),
RouteDefinition(
+280 -7
View File
@@ -265,6 +265,49 @@ class RecipeScanner:
) -> list[dict[str, Any]]:
"""Rank local LoRAs as reconnect candidates for a broken recipe entry.
Thin wrapper over ``_suggest_reconnect_candidates`` scoped to the
LoRA library (see it for the ranking contract).
"""
return await self._suggest_reconnect_candidates(
entry=entry,
recipe_base_model=recipe_base_model,
query=query,
limit=limit,
is_checkpoint=False,
)
async def suggest_checkpoint_reconnect_candidates(
self,
*,
entry: dict[str, Any],
recipe_base_model: Optional[str],
query: Optional[str] = None,
limit: int = 5,
) -> list[dict[str, Any]]:
"""Rank local checkpoints as reconnect candidates for a broken entry.
Thin wrapper over ``_suggest_reconnect_candidates`` scoped to the
checkpoint library (see it for the ranking contract).
"""
return await self._suggest_reconnect_candidates(
entry=entry,
recipe_base_model=recipe_base_model,
query=query,
limit=limit,
is_checkpoint=True,
)
async def _suggest_reconnect_candidates(
self,
*,
entry: dict[str, Any],
recipe_base_model: Optional[str],
query: Optional[str] = None,
limit: int = 5,
is_checkpoint: bool,
) -> list[dict[str, Any]]:
"""Rank local models as reconnect candidates for a broken recipe entry.
Identity signals (same hash / same CivitAI model version) outrank
similarity signals (filename / model name fuzzy match). A confident
base-model mismatch (both sides known and different) is a hard
@@ -286,11 +329,11 @@ class RecipeScanner:
if limit <= 0 or not isinstance(entry, dict):
return []
lora_scanner = self._lora_scanner
if lora_scanner is None:
scanner = self._checkpoint_scanner if is_checkpoint else self._lora_scanner
if scanner is None:
return []
data = await lora_scanner.get_cached_data()
data = await scanner.get_cached_data()
recipe_bm = (recipe_base_model or "").strip().casefold()
def _base_model_known_mismatch(item: dict[str, Any]) -> bool:
@@ -315,7 +358,7 @@ class RecipeScanner:
# entry without a usable hash — same rule as the filename cache.
if not (item.get("sha256") or "").strip():
continue
if not self._is_type_compatible(item, is_checkpoint=False):
if not self._is_type_compatible(item, is_checkpoint=is_checkpoint):
continue
if _base_model_known_mismatch(item):
continue
@@ -351,14 +394,17 @@ class RecipeScanner:
if (
isinstance(hit, dict)
and (hit.get("sha256") or "").strip()
and self._is_type_compatible(hit, is_checkpoint=False)
and self._is_type_compatible(hit, is_checkpoint=is_checkpoint)
and not _base_model_known_mismatch(hit)
):
_consider(hit, 1.0 + _base_model_adjustment(hit), "same_hash")
version_id = entry.get("modelVersionId") or entry.get("id")
if version_id is not None:
hit = self._get_lora_from_version_index(str(version_id))
if is_checkpoint:
hit = self._get_checkpoint_from_version_index(str(version_id))
else:
hit = self._get_lora_from_version_index(str(version_id))
if (
isinstance(hit, dict)
and (hit.get("sha256") or "").strip()
@@ -367,7 +413,12 @@ class RecipeScanner:
_consider(hit, 0.95 + _base_model_adjustment(hit), "same_version")
filename_source = query_text or (entry.get("file_name") or "")
name_source = query_text or (entry.get("modelName") or "")
# Parser-style checkpoint entries carry the model name under ``name``,
# widget-style ones under ``modelName`` — try both for checkpoints.
if is_checkpoint:
name_source = query_text or (entry.get("name") or entry.get("modelName") or "")
else:
name_source = query_text or (entry.get("modelName") or "")
norm_filename_source = self._normalize_filename_key(filename_source)
name_source_cf = name_source.casefold()
# Substring hits floor the similarity ratio, but only for meaningful
@@ -1496,6 +1547,7 @@ class RecipeScanner:
identifier key when neither identifier form exists).
"""
entry["isDeleted"] = False
entry["hashInvalid"] = False
new_hash = (item.get("sha256") or "").lower()
if new_hash:
@@ -3290,6 +3342,19 @@ class RecipeScanner:
return await self._lora_scanner.find_models_by_name(name, base_model=base_model)
async def find_local_checkpoints_by_name(
self, name: str, base_model: Optional[str] = None
) -> List[Dict[str, Any]]:
"""Return every local checkpoint matching ``name`` (used to explain lookup misses)."""
checkpoint_scanner = getattr(self, "_checkpoint_scanner", None)
if not checkpoint_scanner or not name:
return []
return await checkpoint_scanner.find_models_by_name(
name, base_model=base_model
)
async def get_local_lora_by_hash(self, hash_value: str) -> Optional[Dict[str, Any]]:
"""Lookup a local LoRA through the scanner's hash index."""
@@ -4036,6 +4101,214 @@ class RecipeScanner:
updated_lora = self._enrich_lora_entry(dict(lora_entry))
return recipe_data, updated_lora
async def update_checkpoint_entry(
self,
recipe_id: str,
*,
target_name: str,
target_checkpoint: Optional[Dict[str, Any]] = None,
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
"""Update the checkpoint entry within a recipe (manual reconnect).
Mirrors :meth:`update_lora_entry`: the pre-update entry is snapshotted
under ``reconnectSnapshot`` so the association can be restored later,
then the matched local checkpoint is written back following the same
pinned key set as ``_write_rematch_checkpoint_entry``. ``file_name``
keeps the user-entered ``target_name`` (the same convention as the
LoRA reconnect), while hash/name/version/baseModel/identifier are
refreshed from the local item. The fingerprint is untouched it is
computed over LoRAs only.
Returns:
The updated recipe data and the refreshed checkpoint metadata.
"""
if target_name is None:
raise ValueError("target_name must be provided")
recipe_json_path = await self.get_recipe_json_path(recipe_id)
if not recipe_json_path or not os.path.exists(recipe_json_path):
raise RecipeNotFoundError("Recipe not found")
async with self._mutation_lock:
with open(recipe_json_path, "r", encoding="utf-8") as file_obj:
recipe_data = json.load(file_obj)
checkpoint = recipe_data.get("checkpoint")
if not isinstance(checkpoint, dict):
raise RecipeValidationError(
"Recipe has no checkpoint entry to reconnect"
)
# Snapshot the pre-update state so the association can be restored
# later (undo reconnect). Never nest snapshots.
snapshot = {
key: copy.deepcopy(value)
for key, value in checkpoint.items()
if key != "reconnectSnapshot"
}
checkpoint["isDeleted"] = False
checkpoint["hashInvalid"] = False
checkpoint["file_name"] = target_name
if target_checkpoint is not None:
sha_value = target_checkpoint.get("sha256") or target_checkpoint.get(
"sha"
)
if sha_value:
checkpoint["hash"] = sha_value.lower()
self._write_rematch_checkpoint_entry(checkpoint, target_checkpoint)
# The write-back only refreshes keys the entry already has;
# a manual reconnect must also backfill the display keys so a
# sparse parser-style entry renders properly after the swap.
if not checkpoint.get("name") and target_checkpoint.get("model_name"):
checkpoint["name"] = target_checkpoint["model_name"]
civitai = target_checkpoint.get("civitai") or {}
civ_name = civitai.get("name")
if not checkpoint.get("version") and civ_name:
checkpoint["version"] = civ_name
if (
not checkpoint.get("baseModel")
and target_checkpoint.get("base_model")
):
checkpoint["baseModel"] = target_checkpoint["base_model"]
checkpoint["reconnectSnapshot"] = snapshot
recipe_data["modified"] = time.time()
with open(recipe_json_path, "w", encoding="utf-8") as file_obj:
json.dump(recipe_data, file_obj, indent=4, ensure_ascii=False)
cache = await self.get_cached_data()
replaced = await cache.replace_recipe(recipe_id, recipe_data, resort=False)
if not replaced:
await cache.add_recipe(recipe_data, resort=False)
self._schedule_resort()
# Update FTS index
self._update_fts_index_for_recipe(recipe_data, "update")
# Update persistent SQLite cache
if self._persistent_cache:
self._persistent_cache.update_recipe(recipe_data, recipe_json_path)
self._json_path_map[recipe_id] = recipe_json_path
updated_checkpoint = dict(checkpoint)
if target_checkpoint is not None:
preview_url = target_checkpoint.get("preview_url")
if preview_url:
updated_checkpoint["preview_url"] = config.get_preview_static_url(
preview_url
)
if target_checkpoint.get("file_path"):
updated_checkpoint["localPath"] = target_checkpoint["file_path"]
updated_checkpoint = self._enrich_checkpoint_entry(updated_checkpoint)
return recipe_data, updated_checkpoint
async def restore_checkpoint_entry(
self,
recipe_id: str,
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
"""Restore the checkpoint entry to its pre-reconnect snapshot.
Reverses :meth:`update_checkpoint_entry`: the entry saved under
``reconnectSnapshot`` becomes the checkpoint again and the snapshot is
dropped. Returns the updated recipe data and the restored checkpoint
metadata.
"""
recipe_json_path = await self.get_recipe_json_path(recipe_id)
if not recipe_json_path or not os.path.exists(recipe_json_path):
raise RecipeNotFoundError("Recipe not found")
async with self._mutation_lock:
with open(recipe_json_path, "r", encoding="utf-8") as file_obj:
recipe_data = json.load(file_obj)
checkpoint = recipe_data.get("checkpoint")
if not isinstance(checkpoint, dict):
raise RecipeValidationError(
"Recipe has no checkpoint entry to restore"
)
snapshot = checkpoint.get("reconnectSnapshot")
if not isinstance(snapshot, dict):
raise RecipeValidationError(
"Checkpoint entry has no reconnect snapshot to restore"
)
restored_entry = copy.deepcopy(snapshot)
restored_entry.pop("reconnectSnapshot", None)
recipe_data["checkpoint"] = restored_entry
recipe_data["modified"] = time.time()
with open(recipe_json_path, "w", encoding="utf-8") as file_obj:
json.dump(recipe_data, file_obj, indent=4, ensure_ascii=False)
cache = await self.get_cached_data()
replaced = await cache.replace_recipe(recipe_id, recipe_data, resort=False)
if not replaced:
await cache.add_recipe(recipe_data, resort=False)
self._schedule_resort()
# Update FTS index
self._update_fts_index_for_recipe(recipe_data, "update")
# Update persistent SQLite cache
if self._persistent_cache:
self._persistent_cache.update_recipe(recipe_data, recipe_json_path)
self._json_path_map[recipe_id] = recipe_json_path
restored_checkpoint = self._enrich_checkpoint_entry(dict(restored_entry))
return recipe_data, restored_checkpoint
async def set_checkpoint_entry_hash_invalid(
self,
recipe_id: str,
hash_invalid: bool,
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
"""Set the ``hashInvalid`` flag on the recipe's checkpoint entry.
``hashInvalid`` records that the entry's hash could not be resolved
on CivitAI (e.g. a download attempt returned "Model not found").
Marking it makes the entry an unresolved rematch candidate without
touching its stored hash/file_name.
Returns:
The updated recipe data and the refreshed checkpoint metadata.
"""
recipe_json_path = await self.get_recipe_json_path(recipe_id)
if not recipe_json_path or not os.path.exists(recipe_json_path):
raise RecipeNotFoundError("Recipe not found")
async with self._mutation_lock:
with open(recipe_json_path, "r", encoding="utf-8") as file_obj:
recipe_data = json.load(file_obj)
checkpoint = recipe_data.get("checkpoint")
if not isinstance(checkpoint, dict):
raise RecipeValidationError("Checkpoint entry is not a dict")
checkpoint["hashInvalid"] = bool(hash_invalid)
recipe_data["modified"] = time.time()
with open(recipe_json_path, "w", encoding="utf-8") as file_obj:
json.dump(recipe_data, file_obj, indent=4, ensure_ascii=False)
cache = await self.get_cached_data()
replaced = await cache.replace_recipe(recipe_id, recipe_data, resort=False)
if not replaced:
await cache.add_recipe(recipe_data, resort=False)
self._schedule_resort()
if self._persistent_cache:
self._persistent_cache.update_recipe(recipe_data, recipe_json_path)
self._json_path_map[recipe_id] = recipe_json_path
updated_checkpoint = self._enrich_checkpoint_entry(dict(checkpoint))
return recipe_data, updated_checkpoint
async def get_recipes_for_lora(self, lora_hash: str) -> List[Dict[str, Any]]:
"""Return recipes that reference a given LoRA hash."""
+166
View File
@@ -599,6 +599,172 @@ class RecipePersistenceService:
}
)
async def reconnect_checkpoint(
self,
*,
recipe_scanner,
recipe_id: str,
target_name: str,
) -> PersistenceResult:
"""Reconnect the checkpoint entry within an existing recipe."""
recipe_path = await recipe_scanner.get_recipe_json_path(recipe_id)
if not recipe_path or not os.path.exists(recipe_path):
raise RecipeNotFoundError("Recipe not found")
with open(recipe_path, "r", encoding="utf-8") as file_obj:
recipe_base_model = json.load(file_obj).get("base_model", "")
matches = await recipe_scanner.find_local_checkpoints_by_name(target_name)
if not matches:
raise RecipeNotFoundError(
f"Local checkpoint not found with name: {target_name}"
)
# Same three-tier base-model guard as reconnect_lora: exact/unknown
# labels pass silently; same-architecture-family labels pass but are
# reported so the UI can warn; confident mismatches stay hard-rejected.
eligible: list[tuple[dict, str]] = []
for match in matches:
relation = base_model_relation(recipe_base_model, match.get("base_model"))
if relation != RELATION_INCOMPATIBLE:
eligible.append((match, relation))
if not eligible:
raise RecipeValidationError(
f"Local checkpoint '{target_name}' has a different base model "
"than the recipe"
)
if len(eligible) > 1:
raise RecipeValidationError(
f"Multiple local checkpoints match '{target_name}'; "
"include the folder path to disambiguate"
)
target_checkpoint, target_relation = eligible[0]
recipe_data, updated_checkpoint = await recipe_scanner.update_checkpoint_entry(
recipe_id,
target_name=target_name,
target_checkpoint=target_checkpoint,
)
image_path = recipe_data.get("file_path")
if image_path and os.path.exists(image_path):
self._exif_utils.append_recipe_metadata(image_path, recipe_data)
matching_recipes = []
if "fingerprint" in recipe_data:
matching_recipes = await recipe_scanner.find_recipes_by_fingerprint(
recipe_data["fingerprint"]
)
if recipe_id in matching_recipes:
matching_recipes.remove(recipe_id)
payload: dict[str, Any] = {
"success": True,
"recipe_id": recipe_id,
"updated_checkpoint": updated_checkpoint,
"matching_recipes": matching_recipes,
}
if target_relation == RELATION_COMPATIBLE:
# Structured data, not prose — the frontend localizes the warning.
payload["base_model_mismatch"] = {
"recipe_base_model": recipe_base_model,
"checkpoint_base_model": target_checkpoint.get("base_model") or "",
}
return PersistenceResult(payload)
async def restore_checkpoint(
self,
*,
recipe_scanner,
recipe_id: str,
) -> PersistenceResult:
"""Restore the checkpoint entry to the state captured before its reconnect."""
recipe_data, updated_checkpoint = await recipe_scanner.restore_checkpoint_entry(
recipe_id
)
image_path = recipe_data.get("file_path")
if image_path and os.path.exists(image_path):
self._exif_utils.append_recipe_metadata(image_path, recipe_data)
matching_recipes = []
if "fingerprint" in recipe_data:
matching_recipes = await recipe_scanner.find_recipes_by_fingerprint(
recipe_data["fingerprint"]
)
if recipe_id in matching_recipes:
matching_recipes.remove(recipe_id)
return PersistenceResult(
{
"success": True,
"recipe_id": recipe_id,
"updated_checkpoint": updated_checkpoint,
"matching_recipes": matching_recipes,
}
)
async def get_checkpoint_reconnect_suggestions(
self,
*,
recipe_scanner,
recipe_id: str,
query: str | None = None,
) -> PersistenceResult:
"""Return ranked local checkpoint candidates for reconnecting a recipe entry."""
recipe_path = await recipe_scanner.get_recipe_json_path(recipe_id)
if not recipe_path or not os.path.exists(recipe_path):
raise RecipeNotFoundError("Recipe not found")
with open(recipe_path, "r", encoding="utf-8") as file_obj:
recipe_data = json.load(file_obj)
checkpoint = recipe_data.get("checkpoint")
if not isinstance(checkpoint, dict):
raise RecipeValidationError("Recipe has no checkpoint entry")
suggestions = await recipe_scanner.suggest_checkpoint_reconnect_candidates(
entry=checkpoint,
recipe_base_model=recipe_data.get("base_model"),
query=query,
)
return PersistenceResult({"success": True, "suggestions": suggestions})
async def mark_checkpoint_hash_invalid(
self,
*,
recipe_scanner,
recipe_id: str,
hash_invalid: bool = True,
) -> PersistenceResult:
"""Mark the recipe checkpoint entry's hash as unresolvable on CivitAI.
Called when a download attempt by hash returned "Model not found".
The flag makes the entry an unresolved rematch candidate without
altering its stored hash/file_name.
"""
recipe_data, updated_checkpoint = (
await recipe_scanner.set_checkpoint_entry_hash_invalid(
recipe_id,
hash_invalid=hash_invalid,
)
)
return PersistenceResult(
{
"success": True,
"recipe_id": recipe_id,
"hash_invalid": bool(hash_invalid),
"updated_checkpoint": updated_checkpoint,
}
)
async def bulk_delete(
self,
*,