diff --git a/locales/de.json b/locales/de.json index 003d85e6..04ab208e 100644 --- a/locales/de.json +++ b/locales/de.json @@ -159,6 +159,12 @@ "success": "Updated license metadata for {count} {typePlural}", "none": "All {typePlural} already have license metadata", "error": "Failed to refresh license metadata for {typePlural}: {message}" + }, + "repairRecipes": { + "label": "Recipe-Daten reparieren", + "loading": "Recipe-Daten werden repariert...", + "success": "{count} Rezepte erfolgreich repariert.", + "error": "Recipe-Reparatur fehlgeschlagen: {message}" } }, "header": { @@ -520,6 +526,7 @@ "replacePreview": "Vorschau ersetzen", "setContentRating": "Inhaltsbewertung festlegen", "moveToFolder": "In Ordner verschieben", + "repairMetadata": "[TODO: Translate] Repair metadata", "excludeModel": "Modell ausschließen", "deleteModel": "Modell löschen", "shareRecipe": "Rezept teilen", @@ -635,6 +642,13 @@ "noMissingLoras": "Keine fehlenden LoRAs zum Herunterladen", "getInfoFailed": "Fehler beim Abrufen der Informationen für fehlende LoRAs", "prepareError": "Fehler beim Vorbereiten der LoRAs für den Download: {message}" + }, + "repair": { + "starting": "[TODO: Translate] Repairing recipe metadata...", + "success": "[TODO: Translate] Recipe metadata repaired successfully", + "skipped": "[TODO: Translate] Recipe already at latest version, no repair needed", + "failed": "[TODO: Translate] Failed to repair recipe: {message}", + "missingId": "[TODO: Translate] Cannot repair recipe: Missing recipe ID" } } }, @@ -1498,4 +1512,4 @@ "learnMore": "LM Civitai Extension Tutorial" } } -} \ No newline at end of file +} diff --git a/locales/en.json b/locales/en.json index 8df93044..8f4bbf4c 100644 --- a/locales/en.json +++ b/locales/en.json @@ -159,6 +159,12 @@ "success": "Updated license metadata for {count} {typePlural}", "none": "All {typePlural} already have license metadata", "error": "Failed to refresh license metadata for {typePlural}: {message}" + }, + "repairRecipes": { + "label": "Repair recipes data", + "loading": "Repairing recipe data...", + "success": "Successfully repaired {count} recipes.", + "error": "Recipe repair failed: {message}" } }, "header": { @@ -520,6 +526,7 @@ "replacePreview": "Replace Preview", "setContentRating": "Set Content Rating", "moveToFolder": "Move to Folder", + "repairMetadata": "Repair metadata", "excludeModel": "Exclude Model", "deleteModel": "Delete Model", "shareRecipe": "Share Recipe", @@ -635,6 +642,13 @@ "noMissingLoras": "No missing LoRAs to download", "getInfoFailed": "Failed to get information for missing LoRAs", "prepareError": "Error preparing LoRAs for download: {message}" + }, + "repair": { + "starting": "Repairing recipe metadata...", + "success": "Recipe metadata repaired successfully", + "skipped": "Recipe already at latest version, no repair needed", + "failed": "Failed to repair recipe: {message}", + "missingId": "Cannot repair recipe: Missing recipe ID" } } }, diff --git a/locales/es.json b/locales/es.json index 9ad9eb66..60535da5 100644 --- a/locales/es.json +++ b/locales/es.json @@ -159,6 +159,12 @@ "success": "Updated license metadata for {count} {typePlural}", "none": "All {typePlural} already have license metadata", "error": "Failed to refresh license metadata for {typePlural}: {message}" + }, + "repairRecipes": { + "label": "Reparar datos de recetas", + "loading": "Reparando datos de recetas...", + "success": "Se repararon con éxito {count} recetas.", + "error": "Error al reparar recetas: {message}" } }, "header": { @@ -520,6 +526,7 @@ "replacePreview": "Reemplazar vista previa", "setContentRating": "Establecer clasificación de contenido", "moveToFolder": "Mover a carpeta", + "repairMetadata": "[TODO: Translate] Repair metadata", "excludeModel": "Excluir modelo", "deleteModel": "Eliminar modelo", "shareRecipe": "Compartir receta", @@ -635,6 +642,13 @@ "noMissingLoras": "No hay LoRAs faltantes para descargar", "getInfoFailed": "Error al obtener información de LoRAs faltantes", "prepareError": "Error preparando LoRAs para descarga: {message}" + }, + "repair": { + "starting": "[TODO: Translate] Repairing recipe metadata...", + "success": "[TODO: Translate] Recipe metadata repaired successfully", + "skipped": "[TODO: Translate] Recipe already at latest version, no repair needed", + "failed": "[TODO: Translate] Failed to repair recipe: {message}", + "missingId": "[TODO: Translate] Cannot repair recipe: Missing recipe ID" } } }, @@ -1498,4 +1512,4 @@ "learnMore": "LM Civitai Extension Tutorial" } } -} \ No newline at end of file +} diff --git a/locales/fr.json b/locales/fr.json index 5488742f..796a81ad 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -159,6 +159,12 @@ "success": "Updated license metadata for {count} {typePlural}", "none": "All {typePlural} already have license metadata", "error": "Failed to refresh license metadata for {typePlural}: {message}" + }, + "repairRecipes": { + "label": "Réparer les données de recettes", + "loading": "Réparation des données de recettes...", + "success": "{count} recettes réparées avec succès.", + "error": "Échec de la réparation des recettes : {message}" } }, "header": { @@ -520,6 +526,7 @@ "replacePreview": "Remplacer l'aperçu", "setContentRating": "Définir la classification du contenu", "moveToFolder": "Déplacer vers un dossier", + "repairMetadata": "[TODO: Translate] Repair metadata", "excludeModel": "Exclure le modèle", "deleteModel": "Supprimer le modèle", "shareRecipe": "Partager la recipe", @@ -635,6 +642,13 @@ "noMissingLoras": "Aucun LoRA manquant à télécharger", "getInfoFailed": "Échec de l'obtention des informations pour les LoRAs manquants", "prepareError": "Erreur lors de la préparation des LoRAs pour le téléchargement : {message}" + }, + "repair": { + "starting": "[TODO: Translate] Repairing recipe metadata...", + "success": "[TODO: Translate] Recipe metadata repaired successfully", + "skipped": "[TODO: Translate] Recipe already at latest version, no repair needed", + "failed": "[TODO: Translate] Failed to repair recipe: {message}", + "missingId": "[TODO: Translate] Cannot repair recipe: Missing recipe ID" } } }, @@ -1498,4 +1512,4 @@ "learnMore": "LM Civitai Extension Tutorial" } } -} \ No newline at end of file +} diff --git a/locales/he.json b/locales/he.json index 8a000812..97694c57 100644 --- a/locales/he.json +++ b/locales/he.json @@ -159,6 +159,12 @@ "success": "Updated license metadata for {count} {typePlural}", "none": "All {typePlural} already have license metadata", "error": "Failed to refresh license metadata for {typePlural}: {message}" + }, + "repairRecipes": { + "label": "תיקון נתוני מתכונים", + "loading": "מתקן נתוני מתכונים...", + "success": "תוקנו בהצלחה {count} מתכונים.", + "error": "תיקון המתכונים נכשל: {message}" } }, "header": { @@ -520,6 +526,7 @@ "replacePreview": "החלף תצוגה מקדימה", "setContentRating": "הגדר דירוג תוכן", "moveToFolder": "העבר לתיקייה", + "repairMetadata": "[TODO: Translate] Repair metadata", "excludeModel": "החרג מודל", "deleteModel": "מחק מודל", "shareRecipe": "שתף מתכון", @@ -635,6 +642,13 @@ "noMissingLoras": "אין LoRAs חסרים להורדה", "getInfoFailed": "קבלת מידע עבור LoRAs חסרים נכשלה", "prepareError": "שגיאה בהכנת LoRAs להורדה: {message}" + }, + "repair": { + "starting": "[TODO: Translate] Repairing recipe metadata...", + "success": "[TODO: Translate] Recipe metadata repaired successfully", + "skipped": "[TODO: Translate] Recipe already at latest version, no repair needed", + "failed": "[TODO: Translate] Failed to repair recipe: {message}", + "missingId": "[TODO: Translate] Cannot repair recipe: Missing recipe ID" } } }, @@ -1498,4 +1512,4 @@ "learnMore": "LM Civitai Extension Tutorial" } } -} \ No newline at end of file +} diff --git a/locales/ja.json b/locales/ja.json index 961f3c9c..4b5bdfb9 100644 --- a/locales/ja.json +++ b/locales/ja.json @@ -159,6 +159,12 @@ "success": "Updated license metadata for {count} {typePlural}", "none": "All {typePlural} already have license metadata", "error": "Failed to refresh license metadata for {typePlural}: {message}" + }, + "repairRecipes": { + "label": "レシピデータの修復", + "loading": "レシピデータを修復中...", + "success": "{count} 件のレシピを正常に修復しました。", + "error": "レシピの修復に失敗しました: {message}" } }, "header": { @@ -520,6 +526,7 @@ "replacePreview": "プレビューを置換", "setContentRating": "コンテンツレーティングを設定", "moveToFolder": "フォルダに移動", + "repairMetadata": "[TODO: Translate] Repair metadata", "excludeModel": "モデルを除外", "deleteModel": "モデルを削除", "shareRecipe": "レシピを共有", @@ -635,6 +642,13 @@ "noMissingLoras": "ダウンロードする不足LoRAがありません", "getInfoFailed": "不足LoRAの情報取得に失敗しました", "prepareError": "ダウンロード用LoRAの準備中にエラー：{message}" + }, + "repair": { + "starting": "[TODO: Translate] Repairing recipe metadata...", + "success": "[TODO: Translate] Recipe metadata repaired successfully", + "skipped": "[TODO: Translate] Recipe already at latest version, no repair needed", + "failed": "[TODO: Translate] Failed to repair recipe: {message}", + "missingId": "[TODO: Translate] Cannot repair recipe: Missing recipe ID" } } }, @@ -1498,4 +1512,4 @@ "learnMore": "LM Civitai Extension Tutorial" } } -} \ No newline at end of file +} diff --git a/locales/ko.json b/locales/ko.json index 5d674689..5a95fca9 100644 --- a/locales/ko.json +++ b/locales/ko.json @@ -159,6 +159,12 @@ "success": "Updated license metadata for {count} {typePlural}", "none": "All {typePlural} already have license metadata", "error": "Failed to refresh license metadata for {typePlural}: {message}" + }, + "repairRecipes": { + "label": "레시피 데이터 복구", + "loading": "레시피 데이터 복구 중...", + "success": "{count}개의 레시피가 성공적으로 복구되었습니다.", + "error": "레시피 복구 실패: {message}" } }, "header": { @@ -520,6 +526,7 @@ "replacePreview": "미리보기 교체", "setContentRating": "콘텐츠 등급 설정", "moveToFolder": "폴더로 이동", + "repairMetadata": "[TODO: Translate] Repair metadata", "excludeModel": "모델 제외", "deleteModel": "모델 삭제", "shareRecipe": "레시피 공유", @@ -635,6 +642,13 @@ "noMissingLoras": "다운로드할 누락된 LoRA가 없습니다", "getInfoFailed": "누락된 LoRA 정보를 가져오는데 실패했습니다", "prepareError": "LoRA 다운로드 준비 중 오류: {message}" + }, + "repair": { + "starting": "[TODO: Translate] Repairing recipe metadata...", + "success": "[TODO: Translate] Recipe metadata repaired successfully", + "skipped": "[TODO: Translate] Recipe already at latest version, no repair needed", + "failed": "[TODO: Translate] Failed to repair recipe: {message}", + "missingId": "[TODO: Translate] Cannot repair recipe: Missing recipe ID" } } }, @@ -1498,4 +1512,4 @@ "learnMore": "LM Civitai Extension Tutorial" } } -} \ No newline at end of file +} diff --git a/locales/ru.json b/locales/ru.json index 4eff37ef..1c580136 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -159,6 +159,12 @@ "success": "Updated license metadata for {count} {typePlural}", "none": "All {typePlural} already have license metadata", "error": "Failed to refresh license metadata for {typePlural}: {message}" + }, + "repairRecipes": { + "label": "Восстановить данные рецептов", + "loading": "Восстановление данных рецептов...", + "success": "Успешно восстановлено {count} рецептов.", + "error": "Ошибка восстановления рецептов: {message}" } }, "header": { @@ -520,6 +526,7 @@ "replacePreview": "Заменить превью", "setContentRating": "Установить рейтинг контента", "moveToFolder": "Переместить в папку", + "repairMetadata": "[TODO: Translate] Repair metadata", "excludeModel": "Исключить модель", "deleteModel": "Удалить модель", "shareRecipe": "Поделиться рецептом", @@ -635,6 +642,13 @@ "noMissingLoras": "Нет отсутствующих LoRAs для загрузки", "getInfoFailed": "Не удалось получить информацию для отсутствующих LoRAs", "prepareError": "Ошибка подготовки LoRAs для загрузки: {message}" + }, + "repair": { + "starting": "[TODO: Translate] Repairing recipe metadata...", + "success": "[TODO: Translate] Recipe metadata repaired successfully", + "skipped": "[TODO: Translate] Recipe already at latest version, no repair needed", + "failed": "[TODO: Translate] Failed to repair recipe: {message}", + "missingId": "[TODO: Translate] Cannot repair recipe: Missing recipe ID" } } }, @@ -1498,4 +1512,4 @@ "learnMore": "LM Civitai Extension Tutorial" } } -} \ No newline at end of file +} diff --git a/locales/zh-CN.json b/locales/zh-CN.json index 53a16b16..c3c5b6f6 100644 --- a/locales/zh-CN.json +++ b/locales/zh-CN.json @@ -159,6 +159,12 @@ "success": "Updated license metadata for {count} {typePlural}", "none": "All {typePlural} already have license metadata", "error": "Failed to refresh license metadata for {typePlural}: {message}" + }, + "repairRecipes": { + "label": "修复配方数据", + "loading": "正在修复配方数据...", + "success": "成功修复了 {count} 个配方。", + "error": "配方修复失败：{message}" } }, "header": { @@ -520,6 +526,7 @@ "replacePreview": "替换预览", "setContentRating": "设置内容评级", "moveToFolder": "移动到文件夹", + "repairMetadata": "[TODO: Translate] Repair metadata", "excludeModel": "排除模型", "deleteModel": "删除模型", "shareRecipe": "分享配方", @@ -635,6 +642,13 @@ "noMissingLoras": "没有缺失的 LoRA 可下载", "getInfoFailed": "获取缺失 LoRA 信息失败", "prepareError": "准备下载 LoRA 时出错：{message}" + }, + "repair": { + "starting": "[TODO: Translate] Repairing recipe metadata...", + "success": "[TODO: Translate] Recipe metadata repaired successfully", + "skipped": "[TODO: Translate] Recipe already at latest version, no repair needed", + "failed": "[TODO: Translate] Failed to repair recipe: {message}", + "missingId": "[TODO: Translate] Cannot repair recipe: Missing recipe ID" } } }, @@ -1498,4 +1512,4 @@ "learnMore": "浏览器插件教程" } } -} \ No newline at end of file +} diff --git a/locales/zh-TW.json b/locales/zh-TW.json index 3a8375df..2c089775 100644 --- a/locales/zh-TW.json +++ b/locales/zh-TW.json @@ -154,11 +154,17 @@ "error": "清理範例圖片資料夾失敗：{message}" }, "fetchMissingLicenses": { - "label": "Refresh license metadata", - "loading": "Refreshing license metadata for {typePlural}...", - "success": "Updated license metadata for {count} {typePlural}", - "none": "All {typePlural} already have license metadata", - "error": "Failed to refresh license metadata for {typePlural}: {message}" + "label": "重新整理授權中繼資料", + "loading": "正在重新整理 {typePlural} 的授權中繼資料...", + "success": "已更新 {count} 個 {typePlural} 的授權中繼資料", + "none": "所有 {typePlural} 已具備授權中繼資料", + "error": "重新整理 {typePlural} 授權中繼資料失敗：{message}" + }, + "repairRecipes": { + "label": "修復配方資料", + "loading": "正在修復配方資料...", + "success": "成功修復 {count} 個配方。", + "error": "配方修復失敗：{message}" } }, "header": { @@ -520,6 +526,7 @@ "replacePreview": "更換預覽圖", "setContentRating": "設定內容分級", "moveToFolder": "移動到資料夾", + "repairMetadata": "[TODO: Translate] Repair metadata", "excludeModel": "排除模型", "deleteModel": "刪除模型", "shareRecipe": "分享配方", @@ -635,6 +642,13 @@ "noMissingLoras": "無缺少的 LoRA 可下載", "getInfoFailed": "取得缺少 LoRA 資訊失敗", "prepareError": "準備下載 LoRA 時發生錯誤：{message}" + }, + "repair": { + "starting": "[TODO: Translate] Repairing recipe metadata...", + "success": "[TODO: Translate] Recipe metadata repaired successfully", + "skipped": "[TODO: Translate] Recipe already at latest version, no repair needed", + "failed": "[TODO: Translate] Failed to repair recipe: {message}", + "missingId": "[TODO: Translate] Cannot repair recipe: Missing recipe ID" } } }, @@ -1498,4 +1512,4 @@ "learnMore": "LM Civitai Extension Tutorial" } } -} \ No newline at end of file +} diff --git a/py/routes/handlers/recipe_handlers.py b/py/routes/handlers/recipe_handlers.py index c18155dc..a125fa1f 100644 --- a/py/routes/handlers/recipe_handlers.py +++ b/py/routes/handlers/recipe_handlers.py @@ -5,6 +5,7 @@ import json import logging import os import re +import asyncio import tempfile from dataclasses import dataclass from typing import Any, Awaitable, Callable, Dict, List, Mapping, Optional @@ -26,6 +27,7 @@ from ...services.metadata_service import get_default_metadata_provider from ...utils.civitai_utils import rewrite_preview_url from ...utils.exif_utils import ExifUtils from ...recipes.merger import GenParamsMerger +from ...services.websocket_manager import ws_manager as default_ws_manager Logger = logging.Logger EnsureDependenciesCallable = Callable[[], Awaitable[None]] @@ -74,6 +76,9 @@ class RecipeHandlerSet: "get_recipes_for_lora": self.query.get_recipes_for_lora, "scan_recipes": self.query.scan_recipes, "move_recipe": self.management.move_recipe, + "repair_recipes": self.management.repair_recipes, + "repair_recipe": self.management.repair_recipe, + "get_repair_progress": self.management.get_repair_progress, } @@ -479,6 +484,7 @@ class RecipeManagementHandler: analysis_service: RecipeAnalysisService, downloader_factory, civitai_client_getter: CivitaiClientGetter, + ws_manager=default_ws_manager, ) -> None: self._ensure_dependencies_ready = ensure_dependencies_ready self._recipe_scanner_getter = recipe_scanner_getter @@ -487,6 +493,7 @@ class RecipeManagementHandler: self._analysis_service = analysis_service self._downloader_factory = downloader_factory self._civitai_client_getter = civitai_client_getter + self._ws_manager = ws_manager async def save_recipe(self, request: web.Request) -> web.Response: try: @@ -514,6 +521,70 @@ class RecipeManagementHandler: self._logger.error("Error saving recipe: %s", exc, exc_info=True) return web.json_response({"error": str(exc)}, status=500) + async def repair_recipes(self, request: web.Request) -> web.Response: + try: + await self._ensure_dependencies_ready() + recipe_scanner = self._recipe_scanner_getter() + if recipe_scanner is None: + return web.json_response({"success": False, "error": "Recipe scanner unavailable"}, status=503) + + # Check if already running + if self._ws_manager.get_recipe_repair_progress(): + return web.json_response({"success": False, "error": "Recipe repair already in progress"}, status=409) + + async def progress_callback(data): + await self._ws_manager.broadcast_recipe_repair_progress(data) + + # Run in background to avoid timeout + async def run_repair(): + try: + await recipe_scanner.repair_all_recipes( + progress_callback=progress_callback + ) + except Exception as e: + self._logger.error(f"Error in recipe repair task: {e}", exc_info=True) + await self._ws_manager.broadcast_recipe_repair_progress({ + "status": "error", + "error": str(e) + }) + finally: + # Keep the final status for a while so the UI can see it + await asyncio.sleep(5) + self._ws_manager.cleanup_recipe_repair_progress() + + asyncio.create_task(run_repair()) + + return web.json_response({"success": True, "message": "Recipe repair started"}) + except Exception as exc: + self._logger.error("Error starting recipe repair: %s", exc, exc_info=True) + return web.json_response({"success": False, "error": str(exc)}, status=500) + + async def repair_recipe(self, request: web.Request) -> web.Response: + try: + await self._ensure_dependencies_ready() + recipe_scanner = self._recipe_scanner_getter() + if recipe_scanner is None: + return web.json_response({"success": False, "error": "Recipe scanner unavailable"}, status=503) + + recipe_id = request.match_info["recipe_id"] + result = await recipe_scanner.repair_recipe_by_id(recipe_id) + return web.json_response(result) + except RecipeNotFoundError as exc: + return web.json_response({"success": False, "error": str(exc)}, status=404) + except Exception as exc: + self._logger.error("Error repairing single recipe: %s", exc, exc_info=True) + return web.json_response({"success": False, "error": str(exc)}, status=500) + + async def get_repair_progress(self, request: web.Request) -> web.Response: + try: + progress = self._ws_manager.get_recipe_repair_progress() + if progress: + return web.json_response({"success": True, "progress": progress}) + return web.json_response({"success": False, "message": "No repair in progress"}, status=404) + except Exception as exc: + self._logger.error("Error getting repair progress: %s", exc, exc_info=True) + return web.json_response({"success": False, "error": str(exc)}, status=500) + async def import_remote_recipe(self, request: web.Request) -> web.Response: try: await self._ensure_dependencies_ready() diff --git a/py/routes/recipe_route_registrar.py b/py/routes/recipe_route_registrar.py index b91693c4..b07327b4 100644 --- a/py/routes/recipe_route_registrar.py +++ b/py/routes/recipe_route_registrar.py @@ -33,7 +33,7 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = ( RouteDefinition("GET", "/api/lm/recipes/unified-folder-tree", "get_unified_folder_tree"), RouteDefinition("GET", "/api/lm/recipe/{recipe_id}/share", "share_recipe"), RouteDefinition("GET", "/api/lm/recipe/{recipe_id}/share/download", "download_shared_recipe"), - RouteDefinition("GET", "/api/lm/recipe/{recipe_id}/syntax", "get_recipe_syntax"), + RouteDefinition("GET", "/api/lm/recipes/syntax", "get_recipe_syntax"), RouteDefinition("PUT", "/api/lm/recipe/{recipe_id}/update", "update_recipe"), RouteDefinition("POST", "/api/lm/recipe/move", "move_recipe"), RouteDefinition("POST", "/api/lm/recipes/move-bulk", "move_recipes_bulk"), @@ -43,6 +43,9 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = ( RouteDefinition("POST", "/api/lm/recipes/save-from-widget", "save_recipe_from_widget"), RouteDefinition("GET", "/api/lm/recipes/for-lora", "get_recipes_for_lora"), RouteDefinition("GET", "/api/lm/recipes/scan", "scan_recipes"), + RouteDefinition("POST", "/api/lm/recipes/repair", "repair_recipes"), + RouteDefinition("POST", "/api/lm/recipe/{recipe_id}/repair", "repair_recipe"), + RouteDefinition("GET", "/api/lm/recipes/repair-progress", "get_repair_progress"), ) diff --git a/py/services/recipe_scanner.py b/py/services/recipe_scanner.py index d5afa324..683b683e 100644 --- a/py/services/recipe_scanner.py +++ b/py/services/recipe_scanner.py @@ -16,6 +16,8 @@ from .recipes.errors import RecipeNotFoundError from ..utils.utils import calculate_recipe_fingerprint, fuzzy_match from natsort import natsorted import sys +import re +from ..recipes.merger import GenParamsMerger logger = logging.getLogger(__name__) @@ -54,6 +56,8 @@ class RecipeScanner: cls._instance._civitai_client = None # Will be lazily initialized return cls._instance + REPAIR_VERSION = 2 + def __init__( self, lora_scanner: Optional[LoraScanner] = None, @@ -109,6 +113,283 @@ class RecipeScanner: self._civitai_client = await ServiceRegistry.get_civitai_client() return self._civitai_client + async def repair_all_recipes( + self, + progress_callback: Optional[Callable[[Dict], Any]] = None + ) -> Dict[str, Any]: + """Repair all recipes by enrichment with Civitai and embedded metadata. + + Args: + persistence_service: Service for saving updated recipes + progress_callback: Optional callback for progress updates + + Returns: + Dict summary of repair results + """ + async with self._mutation_lock: + cache = await self.get_cached_data() + all_recipes = list(cache.raw_data) + total = len(all_recipes) + repaired_count = 0 + skipped_count = 0 + errors_count = 0 + + civitai_client = await self._get_civitai_client() + + for i, recipe in enumerate(all_recipes): + try: + # Report progress + if progress_callback: + await progress_callback({ + "status": "processing", + "current": i + 1, + "total": total, + "recipe_name": recipe.get("name", "Unknown") + }) + + if await self._repair_single_recipe(recipe, civitai_client): + repaired_count += 1 + else: + skipped_count += 1 + + except Exception as e: + logger.error(f"Error repairing recipe {recipe.get('file_path')}: {e}") + errors_count += 1 + + # Final progress update + if progress_callback: + await progress_callback({ + "status": "completed", + "repaired": repaired_count, + "skipped": skipped_count, + "errors": errors_count, + "total": total + }) + + return { + "success": True, + "repaired": repaired_count, + "skipped": skipped_count, + "errors": errors_count, + "total": total + } + + async def repair_recipe_by_id(self, recipe_id: str) -> Dict[str, Any]: + """Repair a single recipe by its ID. + + Args: + recipe_id: ID of the recipe to repair + + Returns: + Dict summary of repair result + """ + async with self._mutation_lock: + recipe = await self.get_recipe_by_id(recipe_id) + if not recipe: + raise RecipeNotFoundError(f"Recipe {recipe_id} not found") + + civitai_client = await self._get_civitai_client() + success = await self._repair_single_recipe(recipe, civitai_client) + + return { + "success": True, + "repaired": 1 if success else 0, + "skipped": 0 if success else 1, + "recipe": recipe + } + + async def _repair_single_recipe(self, recipe: Dict[str, Any], civitai_client: Any) -> bool: + """Internal helper to repair a single recipe object. + + Args: + recipe: The recipe dictionary to repair (modified in-place) + civitai_client: Authenticated Civitai client + + Returns: + bool: True if recipe was repaired or updated, False if skipped + """ + # 1. Skip if already at latest repair version + if recipe.get("repair_version", 0) >= self.REPAIR_VERSION: + return False + + # 2. Identification: Is repair needed? + has_checkpoint = "checkpoint" in recipe and recipe["checkpoint"] and recipe["checkpoint"].get("name") + gen_params = recipe.get("gen_params", {}) + has_prompt = bool(gen_params.get("prompt")) + + needs_repair = not has_checkpoint or not has_prompt + + if not needs_repair: + # Even if no repair needed, we mark it with version if it was processed + if "repair_version" not in recipe: + recipe["repair_version"] = self.REPAIR_VERSION + await self._save_recipe_persistently(recipe) + return True + return False + + # 3. Data Fetching & Merging + source_url = recipe.get("source_url", "") + civitai_meta = None + model_version_id = None + + # Check if it's a Civitai image URL + image_id_match = re.search(r'civitai\.com/images/(\d+)', source_url) + if image_id_match: + image_id = image_id_match.group(1) + image_info = await civitai_client.get_image_info(image_id) + if image_info: + if "meta" in image_info: + civitai_meta = image_info["meta"] + model_version_id = image_info.get("modelVersionId") + + # Merge with existing data + new_gen_params = GenParamsMerger.merge( + civitai_meta=civitai_meta, + embedded_metadata=gen_params + ) + + updated = False + if new_gen_params != gen_params: + recipe["gen_params"] = new_gen_params + updated = True + + # 4. Update checkpoint if missing or repairable + if not has_checkpoint: + metadata_provider = await get_default_metadata_provider() + + target_version_id = model_version_id or new_gen_params.get("modelVersionId") + target_hash = new_gen_params.get("Model hash") + + civitai_info = None + if target_version_id: + civitai_info = await metadata_provider.get_model_version_info(str(target_version_id)) + elif target_hash: + civitai_info = await metadata_provider.get_model_by_hash(target_hash) + + if civitai_info and not (isinstance(civitai_info, tuple) and civitai_info[1] == "Model not found"): + recipe["checkpoint"] = await self._populate_checkpoint(civitai_info) + updated = True + else: + # Fallback to name extraction + cp_name = new_gen_params.get("Checkpoint") or new_gen_params.get("checkpoint") + if cp_name: + recipe["checkpoint"] = { + "name": cp_name, + "file_name": os.path.splitext(cp_name)[0] + } + updated = True + + # 5. Mark version and save + recipe["repair_version"] = self.REPAIR_VERSION + await self._save_recipe_persistently(recipe) + return True + + async def _save_recipe_persistently(self, recipe: Dict[str, Any]) -> bool: + """Helper to save a recipe to both JSON and EXIF metadata.""" + recipe_id = recipe.get("id") + if not recipe_id: + return False + + recipe_json_path = await self.get_recipe_json_path(recipe_id) + if not recipe_json_path: + return False + + try: + # 1. Sanitize for storage (remove runtime convenience fields) + clean_recipe = self._sanitize_recipe_for_storage(recipe) + + # 2. Update the original dictionary so that we persist the clean version + # globally if needed, effectively overwriting it in-place. + recipe.clear() + recipe.update(clean_recipe) + + # 3. Save JSON + with open(recipe_json_path, 'w', encoding='utf-8') as f: + json.dump(recipe, f, indent=4, ensure_ascii=False) + + # 4. Update EXIF if image exists + image_path = recipe.get('file_path') + if image_path and os.path.exists(image_path): + from ..utils.exif_utils import ExifUtils + ExifUtils.append_recipe_metadata(image_path, recipe) + + return True + except Exception as e: + logger.error(f"Error persisting recipe {recipe_id}: {e}") + return False + + async def _populate_checkpoint(self, civitai_info_tuple: Any) -> Dict[str, Any]: + """Helper to populate checkpoint info using common logic.""" + civitai_data, error_msg = civitai_info_tuple if isinstance(civitai_info_tuple, tuple) else (civitai_info_tuple, None) + + checkpoint = { + "name": "", + "file_name": "", + "isDeleted": False, + "hash": "" + } + + if not civitai_data or error_msg == "Model not found": + checkpoint["isDeleted"] = True + return checkpoint + + try: + if "model" in civitai_data and "name" in civitai_data["model"]: + checkpoint["name"] = civitai_data["model"]["name"] + + if "name" in civitai_data: + checkpoint["version"] = civitai_data.get("name", "") + + if "images" in civitai_data and civitai_data["images"]: + from ..utils.civitai_utils import rewrite_preview_url + image_url = civitai_data["images"][0].get("url") + if image_url: + rewritten_url, _ = rewrite_preview_url(image_url, media_type="image") + checkpoint["thumbnailUrl"] = rewritten_url or image_url + + checkpoint["baseModel"] = civitai_data.get("baseModel", "") + checkpoint["modelId"] = civitai_data.get("modelId", 0) + checkpoint["id"] = civitai_data.get("id", 0) + + if "files" in civitai_data: + model_file = next((f for f in civitai_data.get("files", []) if f.get("type") == "Model"), None) + if model_file: + sha256 = model_file.get("hashes", {}).get("SHA256") + if sha256: + checkpoint["hash"] = sha256.lower() + f_name = model_file.get("name", "") + if f_name: + checkpoint["file_name"] = os.path.splitext(f_name)[0] + except Exception as e: + logger.error(f"Error populating checkpoint: {e}") + + return checkpoint + + def _sanitize_recipe_for_storage(self, recipe: Dict[str, Any]) -> Dict[str, Any]: + """Create a clean copy of the recipe without runtime convenience fields.""" + import copy + clean = copy.deepcopy(recipe) + + # 1. Clean LORAs + if "loras" in clean and isinstance(clean["loras"], list): + for lora in clean["loras"]: + # Fields to remove (runtime only) + for key in ("inLibrary", "preview_url", "localPath"): + lora.pop(key, None) + + # Normalize weight/strength if mapping is desired (standard in persistence_service) + if "weight" in lora and "strength" not in lora: + lora["strength"] = float(lora.pop("weight")) + + # 2. Clean Checkpoint + if "checkpoint" in clean and isinstance(clean["checkpoint"], dict): + cp = clean["checkpoint"] + # Fields to remove (runtime only) + for key in ("inLibrary", "localPath", "preview_url", "thumbnailUrl", "size", "downloadUrl"): + cp.pop(key, None) + + return clean + async def initialize_in_background(self) -> None: """Initialize cache in background using thread pool""" try: diff --git a/py/services/websocket_manager.py b/py/services/websocket_manager.py index 4c786853..1108f72c 100644 --- a/py/services/websocket_manager.py +++ b/py/services/websocket_manager.py @@ -20,6 +20,8 @@ class WebSocketManager: self._last_init_progress: Dict[str, Dict] = {} # Add auto-organize progress tracking self._auto_organize_progress: Optional[Dict] = None + # Add recipe repair progress tracking + self._recipe_repair_progress: Optional[Dict] = None self._auto_organize_lock = asyncio.Lock() async def handle_connection(self, request: web.Request) -> web.WebSocketResponse: @@ -189,6 +191,14 @@ class WebSocketManager: # Broadcast via WebSocket await self.broadcast(data) + async def broadcast_recipe_repair_progress(self, data: Dict): + """Broadcast recipe repair progress to connected clients""" + # Store progress data in memory + self._recipe_repair_progress = data + + # Broadcast via WebSocket + await self.broadcast(data) + def get_auto_organize_progress(self) -> Optional[Dict]: """Get current auto-organize progress""" return self._auto_organize_progress @@ -197,6 +207,14 @@ class WebSocketManager: """Clear auto-organize progress data""" self._auto_organize_progress = None + def get_recipe_repair_progress(self) -> Optional[Dict]: + """Get current recipe repair progress""" + return self._recipe_repair_progress + + def cleanup_recipe_repair_progress(self): + """Clear recipe repair progress data""" + self._recipe_repair_progress = None + def is_auto_organize_running(self) -> bool: """Check if auto-organize is currently running""" if not self._auto_organize_progress: diff --git a/static/css/base.css b/static/css/base.css index 25be44ce..8d6130a9 100644 --- a/static/css/base.css +++ b/static/css/base.css @@ -1,8 +1,10 @@ -html, body { +html, +body { margin: 0; padding: 0; height: 100%; - overflow: hidden; /* Disable default scrolling */ + overflow: hidden; + /* Disable default scrolling */ } /* 针对Firefox */ @@ -58,12 +60,12 @@ html, body { --badge-update-bg: oklch(72% 0.2 220); --badge-update-text: oklch(28% 0.03 220); --badge-update-glow: oklch(72% 0.2 220 / 0.28); - + /* Spacing Scale */ --space-1: calc(8px * 1); --space-2: calc(8px * 2); --space-3: calc(8px * 3); - + /* Z-index Scale */ --z-base: 10; --z-header: 100; @@ -75,8 +77,9 @@ html, body { --border-radius-sm: 8px; --border-radius-xs: 4px; - --scrollbar-width: 8px; /* 添加滚动条宽度变量 */ - + --scrollbar-width: 8px; + /* 添加滚动条宽度变量 */ + /* Shortcut styles */ --shortcut-bg: oklch(var(--lora-accent-l) var(--lora-accent-c) var(--lora-accent-h) / 0.12); --shortcut-border: oklch(var(--lora-accent-l) var(--lora-accent-c) var(--lora-accent-h) / 0.25); @@ -104,7 +107,8 @@ html[data-theme="light"] { --lora-surface: oklch(25% 0.02 256 / 0.98); --lora-border: oklch(90% 0.02 256 / 0.15); --lora-text: oklch(98% 0.02 256); - --lora-warning: oklch(75% 0.25 80); /* Modified to be used with oklch() */ + --lora-warning: oklch(75% 0.25 80); + /* Modified to be used with oklch() */ --lora-error-bg: color-mix(in oklch, var(--lora-error) 15%, transparent); --lora-error-border: color-mix(in oklch, var(--lora-error) 40%, transparent); --badge-update-bg: oklch(62% 0.18 220); @@ -118,5 +122,10 @@ body { color: var(--text-color); display: flex; flex-direction: column; - padding-top: 0; /* Remove the padding-top */ + padding-top: 0; + /* Remove the padding-top */ } + +.hidden { + display: none !important; +} \ No newline at end of file diff --git a/static/js/components/ContextMenu/GlobalContextMenu.js b/static/js/components/ContextMenu/GlobalContextMenu.js index ec4f794a..1102ea83 100644 --- a/static/js/components/ContextMenu/GlobalContextMenu.js +++ b/static/js/components/ContextMenu/GlobalContextMenu.js @@ -15,6 +15,29 @@ export class GlobalContextMenu extends BaseContextMenu { showMenu(x, y, origin = null) { const contextOrigin = origin || { type: 'global' }; + + // Conditional visibility for recipes page + const isRecipesPage = state.currentPageType === 'recipes'; + const modelUpdateItem = this.menu.querySelector('[data-action="check-model-updates"]'); + const licenseRefreshItem = this.menu.querySelector('[data-action="fetch-missing-licenses"]'); + const downloadExamplesItem = this.menu.querySelector('[data-action="download-example-images"]'); + const cleanupExamplesItem = this.menu.querySelector('[data-action="cleanup-example-images-folders"]'); + const repairRecipesItem = this.menu.querySelector('[data-action="repair-recipes"]'); + + if (isRecipesPage) { + modelUpdateItem?.classList.add('hidden'); + licenseRefreshItem?.classList.add('hidden'); + downloadExamplesItem?.classList.add('hidden'); + cleanupExamplesItem?.classList.add('hidden'); + repairRecipesItem?.classList.remove('hidden'); + } else { + modelUpdateItem?.classList.remove('hidden'); + licenseRefreshItem?.classList.remove('hidden'); + downloadExamplesItem?.classList.remove('hidden'); + cleanupExamplesItem?.classList.remove('hidden'); + repairRecipesItem?.classList.add('hidden'); + } + super.showMenu(x, y, contextOrigin); } @@ -40,6 +63,11 @@ export class GlobalContextMenu extends BaseContextMenu { console.error('Failed to refresh missing license metadata:', error); }); break; + case 'repair-recipes': + this.repairRecipes(menuItem).catch((error) => { + console.error('Failed to repair recipes:', error); + }); + break; default: console.warn(`Unhandled global context menu action: ${action}`); break; @@ -235,4 +263,78 @@ export class GlobalContextMenu extends BaseContextMenu { return `${displayName}s`; } + + async repairRecipes(menuItem) { + if (this._repairInProgress) { + return; + } + + this._repairInProgress = true; + menuItem?.classList.add('disabled'); + + const loadingMessage = translate( + 'globalContextMenu.repairRecipes.loading', + {}, + 'Repairing recipe data...' + ); + + const progressUI = state.loadingManager?.showEnhancedProgress(loadingMessage); + + try { + const response = await fetch('/api/lm/recipes/repair', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }); + + const result = await response.json(); + if (!response.ok || !result.success) { + throw new Error(result.error || 'Failed to start repair'); + } + + // Poll for progress (or wait for WebSocket if preferred, but polling is simpler for this implementation) + let isComplete = false; + while (!isComplete && this._repairInProgress) { + const progressResponse = await fetch('/api/lm/recipes/repair-progress'); + if (progressResponse.ok) { + const progressResult = await progressResponse.json(); + if (progressResult.success && progressResult.progress) { + const p = progressResult.progress; + if (p.status === 'processing') { + const percent = (p.current / p.total) * 100; + progressUI?.updateProgress(percent, p.recipe_name, `${loadingMessage} (${p.current}/${p.total})`); + } else if (p.status === 'completed') { + isComplete = true; + progressUI?.complete(translate( + 'globalContextMenu.repairRecipes.success', + { count: p.repaired }, + `Repaired ${p.repaired} recipes.` + )); + showToast('globalContextMenu.repairRecipes.success', { count: p.repaired }, 'success'); + // Refresh recipes page if active + if (window.recipesPage) { + window.recipesPage.refresh(); + } + } else if (p.status === 'error') { + throw new Error(p.error || 'Repair failed'); + } + } else if (progressResponse.status === 404) { + // Progress might have finished quickly and been cleaned up + isComplete = true; + progressUI?.complete(); + } + } + + if (!isComplete) { + await new Promise(resolve => setTimeout(resolve, 1000)); + } + } + } catch (error) { + console.error('Recipe repair failed:', error); + progressUI?.complete(translate('globalContextMenu.repairRecipes.error', { message: error.message }, 'Repair failed: {message}')); + showToast('globalContextMenu.repairRecipes.error', { message: error.message }, 'error'); + } finally { + this._repairInProgress = false; + menuItem?.classList.remove('disabled'); + } + } } diff --git a/static/js/components/ContextMenu/RecipeContextMenu.js b/static/js/components/ContextMenu/RecipeContextMenu.js index 6dcb709b..a3a2eb32 100644 --- a/static/js/components/ContextMenu/RecipeContextMenu.js +++ b/static/js/components/ContextMenu/RecipeContextMenu.js @@ -11,7 +11,7 @@ export class RecipeContextMenu extends BaseContextMenu { super('recipeContextMenu', '.model-card'); this.nsfwSelector = document.getElementById('nsfwLevelSelector'); this.modelType = 'recipe'; - + this.initNSFWSelector(); } @@ -25,20 +25,20 @@ export class RecipeContextMenu extends BaseContextMenu { const { resetAndReload } = await import('../../api/recipeApi.js'); return resetAndReload(); } - + showMenu(x, y, card) { // Call the parent method first to handle basic positioning super.showMenu(x, y, card); - + // Get recipe data to check for missing LoRAs const recipeId = card.dataset.id; const missingLorasItem = this.menu.querySelector('.download-missing-item'); - + if (recipeId && missingLorasItem) { // Check if this card has missing LoRAs const loraCountElement = card.querySelector('.lora-count'); const hasMissingLoras = loraCountElement && loraCountElement.classList.contains('missing'); - + // Show/hide the download missing LoRAs option based on missing status if (hasMissingLoras) { missingLorasItem.style.display = 'flex'; @@ -47,7 +47,7 @@ export class RecipeContextMenu extends BaseContextMenu { } } } - + handleMenuAction(action) { // First try to handle with common actions from ModelContextMenuMixin if (ModelContextMenuMixin.handleCommonMenuActions.call(this, action)) { @@ -56,8 +56,8 @@ export class RecipeContextMenu extends BaseContextMenu { // Handle recipe-specific actions const recipeId = this.currentCard.dataset.id; - - switch(action) { + + switch (action) { case 'details': // Show recipe details this.currentCard.click(); @@ -93,9 +93,13 @@ export class RecipeContextMenu extends BaseContextMenu { // Download missing LoRAs this.downloadMissingLoRAs(recipeId); break; + case 'repair': + // Repair recipe metadata + this.repairRecipe(recipeId); + break; } } - + // New method to copy recipe syntax to clipboard copyRecipeSyntax() { const recipeId = this.currentCard.dataset.id; @@ -118,7 +122,7 @@ export class RecipeContextMenu extends BaseContextMenu { showToast('recipes.contextMenu.copyRecipe.failed', {}, 'error'); }); } - + // New method to send recipe to workflow sendRecipeToWorkflow(replaceMode) { const recipeId = this.currentCard.dataset.id; @@ -141,14 +145,14 @@ export class RecipeContextMenu extends BaseContextMenu { showToast('recipes.contextMenu.sendRecipe.failed', {}, 'error'); }); } - + // View all LoRAs in the recipe viewRecipeLoRAs(recipeId) { if (!recipeId) { showToast('recipes.contextMenu.viewLoras.missingId', {}, 'error'); return; } - + // First get the recipe details to access its LoRAs fetch(`/api/lm/recipe/${recipeId}`) .then(response => response.json()) @@ -158,17 +162,17 @@ export class RecipeContextMenu extends BaseContextMenu { removeSessionItem('recipe_to_lora_filterLoraHashes'); removeSessionItem('filterRecipeName'); removeSessionItem('viewLoraDetail'); - + // Collect all hashes from the recipe's LoRAs const loraHashes = recipe.loras .filter(lora => lora.hash) .map(lora => lora.hash.toLowerCase()); - + if (loraHashes.length > 0) { // Store the LoRA hashes and recipe name in session storage setSessionItem('recipe_to_lora_filterLoraHashes', JSON.stringify(loraHashes)); setSessionItem('filterRecipeName', recipe.title); - + // Navigate to the LoRAs page window.location.href = '/loras'; } else { @@ -180,34 +184,34 @@ export class RecipeContextMenu extends BaseContextMenu { showToast('recipes.contextMenu.viewLoras.loadError', { message: error.message }, 'error'); }); } - + // Download missing LoRAs async downloadMissingLoRAs(recipeId) { if (!recipeId) { showToast('recipes.contextMenu.downloadMissing.missingId', {}, 'error'); return; } - + try { // First get the recipe details const response = await fetch(`/api/lm/recipe/${recipeId}`); const recipe = await response.json(); - + // Get missing LoRAs const missingLoras = recipe.loras.filter(lora => !lora.inLibrary && !lora.isDeleted); - + if (missingLoras.length === 0) { showToast('recipes.contextMenu.downloadMissing.noMissingLoras', {}, 'info'); return; } - + // Show loading toast state.loadingManager.showSimpleLoading('Getting version info for missing LoRAs...'); - + // Get version info for each missing LoRA const missingLorasWithVersionInfoPromises = missingLoras.map(async lora => { let endpoint; - + // Determine which endpoint to use based on available data if (lora.modelVersionId) { endpoint = `/api/lm/loras/civitai/model/version/${lora.modelVersionId}`; @@ -217,52 +221,52 @@ export class RecipeContextMenu extends BaseContextMenu { console.error("Missing both hash and modelVersionId for lora:", lora); return null; } - + const versionResponse = await fetch(endpoint); const versionInfo = await versionResponse.json(); - + // Return original lora data combined with version info return { ...lora, civitaiInfo: versionInfo }; }); - + // Wait for all API calls to complete const lorasWithVersionInfo = await Promise.all(missingLorasWithVersionInfoPromises); - + // Filter out null values (failed requests) const validLoras = lorasWithVersionInfo.filter(lora => lora !== null); - + if (validLoras.length === 0) { showToast('recipes.contextMenu.downloadMissing.getInfoFailed', {}, 'error'); return; } - + // Prepare data for import manager using the retrieved information const recipeData = { loras: validLoras.map(lora => { const civitaiInfo = lora.civitaiInfo; - const modelFile = civitaiInfo.files ? + const modelFile = civitaiInfo.files ? civitaiInfo.files.find(file => file.type === 'Model') : null; - + return { // Basic lora info name: civitaiInfo.model?.name || lora.name, version: civitaiInfo.name || '', strength: lora.strength || 1.0, - + // Model identifiers hash: modelFile?.hashes?.SHA256?.toLowerCase() || lora.hash, modelVersionId: civitaiInfo.id || lora.modelVersionId, - + // Metadata thumbnailUrl: civitaiInfo.images?.[0]?.url || '', baseModel: civitaiInfo.baseModel || '', downloadUrl: civitaiInfo.downloadUrl || '', size: modelFile ? (modelFile.sizeKB * 1024) : 0, file_name: modelFile ? modelFile.name.split('.')[0] : '', - + // Status flags existsLocally: false, isDeleted: civitaiInfo.error === "Model not found", @@ -271,7 +275,7 @@ export class RecipeContextMenu extends BaseContextMenu { }; }) }; - + // Call ImportManager's download missing LoRAs method window.importManager.downloadMissingLoras(recipeData, recipeId); } catch (error) { @@ -283,6 +287,38 @@ export class RecipeContextMenu extends BaseContextMenu { } } } + + // Repair recipe metadata + async repairRecipe(recipeId) { + if (!recipeId) { + showToast('recipes.contextMenu.repair.missingId', {}, 'error'); + return; + } + + try { + showToast('recipes.contextMenu.repair.starting', {}, 'info'); + + const response = await fetch(`/api/lm/recipe/${recipeId}/repair`, { + method: 'POST' + }); + const result = await response.json(); + + if (result.success) { + if (result.repaired > 0) { + showToast('recipes.contextMenu.repair.success', {}, 'success'); + // Refresh the current card or reload + this.resetAndReload(); + } else { + showToast('recipes.contextMenu.repair.skipped', {}, 'info'); + } + } else { + throw new Error(result.error || 'Repair failed'); + } + } catch (error) { + console.error('Error repairing recipe:', error); + showToast('recipes.contextMenu.repair.failed', { message: error.message }, 'error'); + } + } } // Mix in shared methods from ModelContextMenuMixin diff --git a/templates/components/context_menu.html b/templates/components/context_menu.html index 525f0008..b020ea12 100644 --- a/templates/components/context_menu.html +++ b/templates/components/context_menu.html @@ -102,6 +102,9 @@
{{ t('globalContextMenu.cleanupExampleImages.label') }}
+
+ {{ t('globalContextMenu.repairRecipes.label') }} +
@@ -110,7 +113,8 @@
-
{{ t('modals.contentRating.current') }}: {{ t('common.status.unknown') }}
+
{{ t('modals.contentRating.current') }}: {{ + t('common.status.unknown') }}
@@ -123,4 +127,4 @@
-
+
\ No newline at end of file diff --git a/templates/recipes.html b/templates/recipes.html index 2d7392d2..2820a6a1 100644 --- a/templates/recipes.html +++ b/templates/recipes.html @@ -30,9 +30,12 @@
{{ t('loras.contextMenu.setContentRating') }}
+
+ {{ t('loras.contextMenu.repairMetadata') }} +
+
{{ t('loras.contextMenu.moveToFolder') }}
-
{{ t('loras.contextMenu.deleteRecipe') }}
diff --git a/tests/services/test_recipe_repair.py b/tests/services/test_recipe_repair.py new file mode 100644 index 00000000..7334d6eb --- /dev/null +++ b/tests/services/test_recipe_repair.py @@ -0,0 +1,278 @@ +import pytest +import asyncio +from unittest.mock import AsyncMock, MagicMock +from py.services.recipe_scanner import RecipeScanner +from types import SimpleNamespace + +# We define these here to help with spec= if needed +class MockCivitaiClient: + async def get_image_info(self, image_id): + pass + +class MockPersistenceService: + async def save_recipe(self, recipe): + pass + +@pytest.fixture +def mock_civitai_client(): + client = MagicMock(spec=MockCivitaiClient) + client.get_image_info = AsyncMock() + return client + +@pytest.fixture +def mock_metadata_provider(): + provider = MagicMock() + provider.get_model_version_info = AsyncMock(return_value=(None, None)) + provider.get_model_by_hash = AsyncMock(return_value=(None, None)) + return provider + +@pytest.fixture +def recipe_scanner(): + lora_scanner = MagicMock() + lora_scanner.get_cached_data = AsyncMock(return_value=SimpleNamespace(raw_data=[])) + + scanner = RecipeScanner(lora_scanner=lora_scanner) + return scanner + +@pytest.fixture +def setup_scanner(recipe_scanner, mock_civitai_client, mock_metadata_provider, monkeypatch): + monkeypatch.setattr(recipe_scanner, "_get_civitai_client", AsyncMock(return_value=mock_civitai_client)) + + # Wrap the real method with a mock so we can check calls but still execute it + real_save = recipe_scanner._save_recipe_persistently + mock_save = AsyncMock(side_effect=real_save) + monkeypatch.setattr(recipe_scanner, "_save_recipe_persistently", mock_save) + + monkeypatch.setattr("py.services.recipe_scanner.get_default_metadata_provider", AsyncMock(return_value=mock_metadata_provider)) + + # Mock get_recipe_json_path to avoid file system issues in tests + recipe_scanner.get_recipe_json_path = AsyncMock(return_value="/tmp/test_recipe.json") + # Mock open to avoid actual file writing + monkeypatch.setattr("builtins.open", MagicMock()) + monkeypatch.setattr("json.dump", MagicMock()) + monkeypatch.setattr("os.path.exists", MagicMock(return_value=False)) # avoid EXIF logic + + return recipe_scanner, mock_civitai_client, mock_metadata_provider + +@pytest.mark.asyncio +async def test_repair_all_recipes_skip_up_to_date(setup_scanner): + recipe_scanner, _, _ = setup_scanner + + recipe_scanner._cache = SimpleNamespace(raw_data=[ + {"id": "r1", "repair_version": RecipeScanner.REPAIR_VERSION, "title": "Up to date"} + ]) + + # Run + results = await recipe_scanner.repair_all_recipes() + + # Verify + assert results["repaired"] == 0 + assert results["skipped"] == 1 + recipe_scanner._save_recipe_persistently.assert_not_called() + +@pytest.mark.asyncio +async def test_repair_all_recipes_with_enriched_checkpoint_id(setup_scanner): + recipe_scanner, mock_civitai_client, mock_metadata_provider = setup_scanner + + recipe = { + "id": "r1", + "title": "Old Recipe", + "source_url": "https://civitai.com/images/12345", + "checkpoint": None, + "gen_params": {"prompt": ""} + } + recipe_scanner._cache = SimpleNamespace(raw_data=[recipe]) + + # Mock image info returning modelVersionId + mock_civitai_client.get_image_info.return_value = { + "modelVersionId": 5678, + "meta": {"prompt": "a beautiful forest", "Checkpoint": "basic_name.safetensors"} + } + + # Mock metadata provider returning full info + mock_metadata_provider.get_model_version_info.return_value = ({ + "id": 5678, + "modelId": 1234, + "name": "v1.0", + "model": {"name": "Full Model Name"}, + "baseModel": "SDXL 1.0", + "images": [{"url": "https://image.url/thumb.jpg"}], + "files": [{"type": "Model", "hashes": {"SHA256": "ABCDEF"}, "name": "full_filename.safetensors"}] + }, None) + + # Run + results = await recipe_scanner.repair_all_recipes() + + # Verify + assert results["repaired"] == 1 + mock_metadata_provider.get_model_version_info.assert_called_with("5678") + + saved_recipe = recipe_scanner._save_recipe_persistently.call_args[0][0] + checkpoint = saved_recipe["checkpoint"] + assert checkpoint["name"] == "Full Model Name" + assert checkpoint["version"] == "v1.0" + assert checkpoint["modelId"] == 1234 + assert checkpoint["id"] == 5678 + assert checkpoint["hash"] == "abcdef" + assert checkpoint["file_name"] == "full_filename" + assert "thumbnailUrl" not in checkpoint # Stripped during sanitation + +@pytest.mark.asyncio +async def test_repair_all_recipes_with_enriched_checkpoint_hash(setup_scanner): + recipe_scanner, mock_civitai_client, mock_metadata_provider = setup_scanner + + recipe = { + "id": "r1", + "title": "Embedded Only", + "checkpoint": None, + "gen_params": { + "prompt": "", + "Model hash": "hash123" + } + } + recipe_scanner._cache = SimpleNamespace(raw_data=[recipe]) + + # Mock metadata provider lookup by hash + mock_metadata_provider.get_model_by_hash.return_value = ({ + "id": 999, + "modelId": 888, + "name": "v2.0", + "model": {"name": "Hashed Model"}, + "baseModel": "SD 1.5", + "files": [{"type": "Model", "hashes": {"SHA256": "hash123"}, "name": "hashed.safetensors"}] + }, None) + + # Run + results = await recipe_scanner.repair_all_recipes() + + # Verify + assert results["repaired"] == 1 + mock_metadata_provider.get_model_by_hash.assert_called_with("hash123") + + saved_recipe = recipe_scanner._save_recipe_persistently.call_args[0][0] + checkpoint = saved_recipe["checkpoint"] + assert checkpoint["name"] == "Hashed Model" + assert checkpoint["version"] == "v2.0" + assert checkpoint["modelId"] == 888 + assert checkpoint["hash"] == "hash123" + +@pytest.mark.asyncio +async def test_repair_all_recipes_fallback_to_basic(setup_scanner): + recipe_scanner, mock_civitai_client, mock_metadata_provider = setup_scanner + + recipe = { + "id": "r1", + "title": "No Meta Lookup", + "checkpoint": None, + "gen_params": { + "prompt": "", + "Checkpoint": "just_a_name.safetensors" + } + } + recipe_scanner._cache = SimpleNamespace(raw_data=[recipe]) + + # Mock metadata provider returning nothing + mock_metadata_provider.get_model_by_hash.return_value = (None, "Model not found") + + # Run + results = await recipe_scanner.repair_all_recipes() + + # Verify + assert results["repaired"] == 1 + saved_recipe = recipe_scanner._save_recipe_persistently.call_args[0][0] + assert saved_recipe["checkpoint"]["name"] == "just_a_name.safetensors" + assert "modelId" not in saved_recipe["checkpoint"] + +@pytest.mark.asyncio +async def test_repair_all_recipes_progress_callback(setup_scanner): + recipe_scanner, _, _ = setup_scanner + + recipe_scanner._cache = SimpleNamespace(raw_data=[ + {"id": "r1", "title": "R1", "checkpoint": None}, + {"id": "r2", "title": "R2", "checkpoint": None} + ]) + + progress_calls = [] + async def progress_callback(data): + progress_calls.append(data) + + # Run + await recipe_scanner.repair_all_recipes( + progress_callback=progress_callback + ) + + # Verify + assert len(progress_calls) >= 2 + assert progress_calls[-1]["status"] == "completed" + assert progress_calls[-1]["total"] == 2 + assert progress_calls[-1]["repaired"] == 2 + +@pytest.mark.asyncio +async def test_repair_all_recipes_strips_runtime_fields(setup_scanner): + recipe_scanner, mock_civitai_client, mock_metadata_provider = setup_scanner + + # Recipe with runtime fields + recipe = { + "id": "r1", + "title": "Cleanup Test", + "checkpoint": { + "name": "CP", + "inLibrary": True, + "localPath": "/path/to/cp", + "thumbnailUrl": "thumb.jpg" + }, + "loras": [ + { + "name": "L1", + "weight": 0.8, + "inLibrary": True, + "localPath": "/path/to/l1", + "preview_url": "p.jpg" + } + ], + "gen_params": {"prompt": ""} + } + recipe_scanner._cache = SimpleNamespace(raw_data=[recipe]) + # Set high version to trigger repair if needed (or just ensure it processes) + recipe["repair_version"] = 0 + + # Run + await recipe_scanner.repair_all_recipes() + + # Verify sanitation + assert recipe_scanner._save_recipe_persistently.called + saved_recipe = recipe_scanner._save_recipe_persistently.call_args[0][0] + + # 1. Check LORA + lora = saved_recipe["loras"][0] + assert "inLibrary" not in lora + assert "localPath" not in lora + assert "preview_url" not in lora + assert "strength" in lora # weight renamed to strength + assert lora["strength"] == 0.8 + + # 2. Check Checkpoint + cp = saved_recipe["checkpoint"] + assert "inLibrary" not in cp + assert "localPath" not in cp + assert "thumbnailUrl" not in cp + +@pytest.mark.asyncio +async def test_sanitize_recipe_for_storage(recipe_scanner): + import sys + import py.services.recipe_scanner + print(f"\nDEBUG_ENV: sys.path: {sys.path}") + print(f"DEBUG_ENV: recipe_scanner file: {py.services.recipe_scanner.__file__}") + + recipe = { + "loras": [{"name": "L1", "inLibrary": True, "weight": 0.5}], + "checkpoint": {"name": "CP", "localPath": "/tmp/cp"} + } + + clean = recipe_scanner._sanitize_recipe_for_storage(recipe) + + assert "inLibrary" not in clean["loras"][0] + assert "strength" in clean["loras"][0] + assert clean["loras"][0]["strength"] == 0.5 + assert "localPath" not in clean["checkpoint"] + assert clean["checkpoint"]["name"] == "CP"