mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-06-25 04:21:17 -03:00
Compare commits
20 Commits
v1.1.0
...
a9e5ee7e79
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a9e5ee7e79 | ||
|
|
a17b0e9901 | ||
|
|
8f23d966bf | ||
|
|
7a76fc72d0 | ||
|
|
518a4dd5ee | ||
|
|
2b6d4e5d8b | ||
|
|
1f4edbeb9d | ||
|
|
a256558a0e | ||
|
|
818b9113f0 | ||
|
|
6a4fd020dc | ||
|
|
7a23040452 | ||
|
|
138024aefe | ||
|
|
a19ddc14f6 | ||
|
|
7001ced694 | ||
|
|
a5c861646c | ||
|
|
3e0bb73793 | ||
|
|
ac51f6a2f6 | ||
|
|
bef222c77d | ||
|
|
7cd6a53447 | ||
|
|
6850b35770 |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -12,12 +12,14 @@ coverage/
|
||||
.coverage
|
||||
model_cache/
|
||||
|
||||
# agent
|
||||
# agent / dev tooling
|
||||
.opencode/
|
||||
.claude/
|
||||
.sisyphus/
|
||||
.codex
|
||||
.omo
|
||||
reasonix.toml
|
||||
.codegraph/
|
||||
|
||||
# Vue widgets development cache (but keep build output)
|
||||
vue-widgets/node_modules/
|
||||
|
||||
@@ -448,7 +448,9 @@
|
||||
"modelName": "Modellname",
|
||||
"fileName": "Dateiname"
|
||||
},
|
||||
"modelNameDisplayHelp": "Wählen Sie aus, was in der Fußzeile der Modellkarte angezeigt werden soll"
|
||||
"modelNameDisplayHelp": "Wählen Sie aus, was in der Fußzeile der Modellkarte angezeigt werden soll",
|
||||
"cardBlurAmount": "Karten-Overlay-Unschärfe",
|
||||
"cardBlurAmountHelp": "Passen Sie die Unschärfeintensität der Kopf- und Fußzeilen-Overlays auf Modell- und Rezeptkarten an (0 = keine Unschärfe, 20 = maximale Unschärfe)."
|
||||
},
|
||||
"folderSettings": {
|
||||
"activeLibrary": "Aktive Bibliothek",
|
||||
|
||||
@@ -448,7 +448,9 @@
|
||||
"modelName": "Model Name",
|
||||
"fileName": "File Name"
|
||||
},
|
||||
"modelNameDisplayHelp": "Choose what to display in the model card footer"
|
||||
"modelNameDisplayHelp": "Choose what to display in the model card footer",
|
||||
"cardBlurAmount": "Card Overlay Blur",
|
||||
"cardBlurAmountHelp": "Adjust the blur intensity of the header and footer overlays on model and recipe cards (0 = no blur, 20 = maximum blur)."
|
||||
},
|
||||
"folderSettings": {
|
||||
"activeLibrary": "Active Library",
|
||||
|
||||
@@ -448,7 +448,9 @@
|
||||
"modelName": "Nombre del modelo",
|
||||
"fileName": "Nombre del archivo"
|
||||
},
|
||||
"modelNameDisplayHelp": "Elige qué mostrar en el pie de la tarjeta del modelo"
|
||||
"modelNameDisplayHelp": "Elige qué mostrar en el pie de la tarjeta del modelo",
|
||||
"cardBlurAmount": "Desenfoque de superposición de tarjetas",
|
||||
"cardBlurAmountHelp": "Ajuste la intensidad de desenfoque de las superposiciones del encabezado y pie de página en las tarjetas de modelos y recetas (0 = sin desenfoque, 20 = desenfoque máximo)."
|
||||
},
|
||||
"folderSettings": {
|
||||
"activeLibrary": "Biblioteca activa",
|
||||
|
||||
@@ -448,7 +448,9 @@
|
||||
"modelName": "Nom du modèle",
|
||||
"fileName": "Nom du fichier"
|
||||
},
|
||||
"modelNameDisplayHelp": "Choisissez ce qui doit être affiché dans le pied de page de la carte du modèle"
|
||||
"modelNameDisplayHelp": "Choisissez ce qui doit être affiché dans le pied de page de la carte du modèle",
|
||||
"cardBlurAmount": "Flou de superposition des cartes",
|
||||
"cardBlurAmountHelp": "Ajustez l'intensité du flou des superpositions d'en-tête et de pied de page sur les cartes de modèles et de recettes (0 = aucun flou, 20 = flou maximal)."
|
||||
},
|
||||
"folderSettings": {
|
||||
"activeLibrary": "Bibliothèque active",
|
||||
|
||||
@@ -448,7 +448,9 @@
|
||||
"modelName": "שם מודל",
|
||||
"fileName": "שם קובץ"
|
||||
},
|
||||
"modelNameDisplayHelp": "בחר מה להציג בכותרת התחתונה של כרטיס המודל"
|
||||
"modelNameDisplayHelp": "בחר מה להציג בכותרת התחתונה של כרטיס המודל",
|
||||
"cardBlurAmount": "עוצמת טשטוש שכבת-על בכרטיס",
|
||||
"cardBlurAmountHelp": "כוונן את עוצמת הטשטוש של שכבת-העל בכותרת ובכותרות תחתונה בכרטיסי מודל ומתכונים (0 = ללא טשטוש, 20 = טשטוש מקסימלי)."
|
||||
},
|
||||
"folderSettings": {
|
||||
"activeLibrary": "ספרייה פעילה",
|
||||
|
||||
@@ -448,7 +448,9 @@
|
||||
"modelName": "モデル名",
|
||||
"fileName": "ファイル名"
|
||||
},
|
||||
"modelNameDisplayHelp": "モデルカードのフッターに表示する内容を選択"
|
||||
"modelNameDisplayHelp": "モデルカードのフッターに表示する内容を選択",
|
||||
"cardBlurAmount": "カードオーバーレイのぼかし",
|
||||
"cardBlurAmountHelp": "モデルカードとレシピカードのヘッダー・フッターオーバーレイのぼかし強度を調整します(0 = ぼかしなし、20 = 最大ぼかし)。"
|
||||
},
|
||||
"folderSettings": {
|
||||
"activeLibrary": "アクティブライブラリ",
|
||||
|
||||
@@ -448,7 +448,9 @@
|
||||
"modelName": "모델명",
|
||||
"fileName": "파일명"
|
||||
},
|
||||
"modelNameDisplayHelp": "모델 카드 하단에 표시할 내용을 선택하세요"
|
||||
"modelNameDisplayHelp": "모델 카드 하단에 표시할 내용을 선택하세요",
|
||||
"cardBlurAmount": "카드 오버레이 흐림 강도",
|
||||
"cardBlurAmountHelp": "모델 및 레시피 카드의 헤더와 푸터 오버레이 흐림 강도를 조정합니다 (0 = 흐림 없음, 20 = 최대 흐림)."
|
||||
},
|
||||
"folderSettings": {
|
||||
"activeLibrary": "활성 라이브러리",
|
||||
|
||||
@@ -448,7 +448,9 @@
|
||||
"modelName": "Название модели",
|
||||
"fileName": "Имя файла"
|
||||
},
|
||||
"modelNameDisplayHelp": "Выберите, что отображать в нижней части карточки модели"
|
||||
"modelNameDisplayHelp": "Выберите, что отображать в нижней части карточки модели",
|
||||
"cardBlurAmount": "Размытие наложения карточек",
|
||||
"cardBlurAmountHelp": "Настройте интенсивность размытия наложений верхнего и нижнего колонтитулов на карточках моделей и рецептов (0 = без размытия, 20 = максимальное размытие)."
|
||||
},
|
||||
"folderSettings": {
|
||||
"activeLibrary": "Активная библиотека",
|
||||
|
||||
@@ -448,7 +448,9 @@
|
||||
"modelName": "模型名称",
|
||||
"fileName": "文件名"
|
||||
},
|
||||
"modelNameDisplayHelp": "选择在模型卡片底部显示的内容"
|
||||
"modelNameDisplayHelp": "选择在模型卡片底部显示的内容",
|
||||
"cardBlurAmount": "卡片叠加模糊强度",
|
||||
"cardBlurAmountHelp": "调整模型和配方卡片上页眉和页脚叠加层的模糊强度(0 = 无模糊,20 = 最大模糊)。"
|
||||
},
|
||||
"folderSettings": {
|
||||
"activeLibrary": "活动库",
|
||||
|
||||
@@ -448,7 +448,9 @@
|
||||
"modelName": "模型名稱",
|
||||
"fileName": "檔案名稱"
|
||||
},
|
||||
"modelNameDisplayHelp": "選擇在模型卡片底部顯示的內容"
|
||||
"modelNameDisplayHelp": "選擇在模型卡片底部顯示的內容",
|
||||
"cardBlurAmount": "卡片疊加模糊強度",
|
||||
"cardBlurAmountHelp": "調整模型和配方卡片上頁首和頁尾疊加層的模糊強度(0 = 無模糊,20 = 最大模糊)。"
|
||||
},
|
||||
"folderSettings": {
|
||||
"activeLibrary": "使用中的資料庫",
|
||||
|
||||
@@ -33,6 +33,7 @@ from .utils.example_images_migration import ExampleImagesMigration
|
||||
from .services.websocket_manager import ws_manager
|
||||
from .services.example_images_cleanup_service import ExampleImagesCleanupService
|
||||
from .middleware.csp_middleware import relax_csp_for_remote_media
|
||||
from .middleware.error_middleware import api_json_error
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -76,6 +77,11 @@ class LoraManager:
|
||||
"""Initialize and register all routes using the new refactored architecture"""
|
||||
app = PromptServer.instance.app
|
||||
|
||||
# Register JSON error middleware for /api/* routes as the outermost
|
||||
# middleware so it catches errors from all other middlewares.
|
||||
if api_json_error not in app.middlewares:
|
||||
app.middlewares.insert(0, api_json_error)
|
||||
|
||||
if relax_csp_for_remote_media not in app.middlewares:
|
||||
# Ensure CSP relaxer executes after ComfyUI's block_external_middleware so it can
|
||||
# see and extend the restrictive header instead of being overwritten by it.
|
||||
|
||||
@@ -16,6 +16,8 @@ IMG_EXTENSIONS = (
|
||||
".tif",
|
||||
".tiff",
|
||||
".webp",
|
||||
".avif",
|
||||
".jxl",
|
||||
".mp4"
|
||||
)
|
||||
|
||||
|
||||
71
py/middleware/error_middleware.py
Normal file
71
py/middleware/error_middleware.py
Normal file
@@ -0,0 +1,71 @@
|
||||
"""JSON error middleware for API routes.
|
||||
|
||||
Ensures all responses to /api/* requests return valid JSON that the
|
||||
browser-extension frontend can JSON.parse() without crashing, even when
|
||||
the route does not exist (404) or the handler raises an exception (500).
|
||||
|
||||
Extension consumers call response.json() unconditionally — an HTML error
|
||||
page causes ``SyntaxError: unexpected end of data`` that leaks into the
|
||||
popup UI as a toast notification.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Awaitable, Callable
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@web.middleware
|
||||
async def api_json_error(
|
||||
request: web.Request,
|
||||
handler: Callable[[web.Request], Awaitable[web.Response]],
|
||||
) -> web.Response:
|
||||
"""Return JSON ``{"success": false, "error": "..."}`` for API errors.
|
||||
|
||||
Only intercepts paths starting with ``/api/`` — all other routes
|
||||
(frontend pages, static files, WebSocket upgrades) pass through
|
||||
unchanged.
|
||||
"""
|
||||
if not request.path.startswith("/api/"):
|
||||
return await handler(request)
|
||||
|
||||
try:
|
||||
response = await handler(request)
|
||||
return response
|
||||
except web.HTTPException as exc:
|
||||
# Let redirects (301, 302, 307, 308) propagate — they are not errors.
|
||||
if exc.status < 400:
|
||||
raise
|
||||
|
||||
logger.warning(
|
||||
"API %s %s returned HTTP %d: %s",
|
||||
request.method,
|
||||
request.path,
|
||||
exc.status,
|
||||
exc.reason,
|
||||
)
|
||||
|
||||
return web.json_response(
|
||||
{"success": False, "error": f"{exc.status}: {exc.reason}"},
|
||||
status=exc.status,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"API %s %s raised unhandled exception: %s",
|
||||
request.method,
|
||||
request.path,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": f"500: Internal Server Error ({type(exc).__name__})",
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
@@ -11,7 +11,7 @@ from ..metadata_collector.metadata_processor import MetadataProcessor
|
||||
from ..metadata_collector import get_metadata
|
||||
from ..utils.constants import CARD_PREVIEW_WIDTH
|
||||
from ..utils.exif_utils import ExifUtils
|
||||
from ..utils.utils import calculate_recipe_fingerprint
|
||||
from ..utils.utils import calculate_recipe_fingerprint, sanitize_folder_name
|
||||
from PIL import Image, PngImagePlugin
|
||||
import piexif
|
||||
import logging
|
||||
@@ -309,12 +309,14 @@ class SaveImageLM:
|
||||
filename = filename.replace(segment, str(h))
|
||||
elif key == "pprompt" and "prompt" in metadata_dict:
|
||||
prompt = metadata_dict.get("prompt", "").replace("\n", " ")
|
||||
prompt = sanitize_folder_name(prompt)
|
||||
if len(parts) >= 2:
|
||||
length = int(parts[1])
|
||||
prompt = prompt[:length]
|
||||
filename = filename.replace(segment, prompt.strip())
|
||||
elif key == "nprompt" and "negative_prompt" in metadata_dict:
|
||||
prompt = metadata_dict.get("negative_prompt", "").replace("\n", " ")
|
||||
prompt = sanitize_folder_name(prompt)
|
||||
if len(parts) >= 2:
|
||||
length = int(parts[1])
|
||||
prompt = prompt[:length]
|
||||
@@ -328,6 +330,7 @@ class SaveImageLM:
|
||||
model = "model_unavailable"
|
||||
else:
|
||||
model = os.path.splitext(os.path.basename(model_value))[0]
|
||||
model = sanitize_folder_name(model)
|
||||
if len(parts) >= 2:
|
||||
length = int(parts[1])
|
||||
model = model[:length]
|
||||
|
||||
@@ -1785,6 +1785,8 @@ class ModelDownloadHandler:
|
||||
bytes_downloaded = 0
|
||||
total_bytes_raw = request.query.get("total_bytes")
|
||||
total_bytes = int(total_bytes_raw) if total_bytes_raw else None
|
||||
completed_at_raw = request.query.get("completed_at")
|
||||
completed_at = float(completed_at_raw) if completed_at_raw else None
|
||||
|
||||
service = await DownloadQueueService.get_instance()
|
||||
item = await service.complete_download(
|
||||
@@ -1794,6 +1796,7 @@ class ModelDownloadHandler:
|
||||
file_path=file_path,
|
||||
bytes_downloaded=bytes_downloaded,
|
||||
total_bytes=total_bytes,
|
||||
completed_at=completed_at,
|
||||
)
|
||||
if item is None:
|
||||
return web.json_response(
|
||||
|
||||
@@ -13,7 +13,7 @@ from ...config import config as global_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CHUNK_SIZE = 256 * 1024 # 256 KB
|
||||
_CHUNK_SIZE = 1024 * 1024 # 1 MB — balance between streaming iteration overhead and per-chunk memory
|
||||
|
||||
# Video file extensions that bypass native sendfile on Windows
|
||||
# to avoid IOCP/ProactorEventLoop crashes during client disconnect.
|
||||
@@ -55,16 +55,19 @@ class PreviewHandler:
|
||||
logger.debug("Preview file not found at %s", str(resolved))
|
||||
raise web.HTTPNotFound(text="Preview file not found")
|
||||
|
||||
# Video files: stream manually to avoid Windows native sendfile crash.
|
||||
# aiohttp's FileResponse uses _sendfile_native on Windows (IOCP-based),
|
||||
# which breaks when the client disconnects mid-transfer — this happens
|
||||
# constantly when users scroll through a gallery of animated previews.
|
||||
suffix = resolved.suffix.lower()
|
||||
if suffix in _VIDEO_EXTENSIONS:
|
||||
return await self._stream_file(request, resolved)
|
||||
|
||||
# aiohttp's FileResponse handles range requests and content headers for us.
|
||||
return web.FileResponse(path=resolved, chunk_size=_CHUNK_SIZE)
|
||||
# aiohttp's FileResponse handles range requests, content headers, and
|
||||
# uses kernel sendfile (zero-copy DMA) on Linux/macOS. On Windows it
|
||||
# uses IOCP-based _sendfile_native which can crash when the client
|
||||
# disconnects mid-transfer during fast scrolling. The _stream_file()
|
||||
# fallback is kept for a future compat toggle.
|
||||
#
|
||||
# Set explicit Cache-Control so the browser can cache video (and image)
|
||||
# previews across VirtualScroller recycling cycles. Without this,
|
||||
# Chrome does not cache 206 Partial Content responses for <video>
|
||||
# elements, causing the same video to be re-downloaded on every scroll.
|
||||
resp = web.FileResponse(path=resolved, chunk_size=_CHUNK_SIZE)
|
||||
resp.headers["Cache-Control"] = "public, max-age=86400"
|
||||
return resp
|
||||
|
||||
async def _stream_file(
|
||||
self, request: web.Request, path: Path
|
||||
@@ -83,6 +86,10 @@ class PreviewHandler:
|
||||
resp.content_type = content_type
|
||||
resp.content_length = file_size
|
||||
|
||||
# Allow browser caching: video previews rarely change during a session.
|
||||
# The frontend already appends ?t={version} to bust cache on update.
|
||||
resp.headers["Cache-Control"] = "public, max-age=86400"
|
||||
|
||||
await resp.prepare(request)
|
||||
|
||||
try:
|
||||
|
||||
@@ -1597,15 +1597,8 @@ class RecipeManagementHandler:
|
||||
|
||||
cache = await recipe_scanner.get_cached_data()
|
||||
|
||||
# Build lookup: image_id -> recipe_id from stored source_path
|
||||
image_to_recipe = {}
|
||||
for recipe in getattr(cache, "raw_data", []):
|
||||
source = recipe.get("source_path")
|
||||
if not source:
|
||||
continue
|
||||
image_id = extract_civitai_image_id(source)
|
||||
if image_id and image_id not in image_to_recipe:
|
||||
image_to_recipe[image_id] = recipe.get("id")
|
||||
# Use precomputed image_id_map (built once at cache init)
|
||||
image_to_recipe = getattr(cache, "image_id_map", {})
|
||||
|
||||
results = {}
|
||||
for img_id in requested_ids:
|
||||
@@ -1641,20 +1634,22 @@ class RecipeManagementHandler:
|
||||
"Could not extract Civitai image ID from URL"
|
||||
)
|
||||
|
||||
# Check for duplicate (fast, before acquiring semaphore), unless force
|
||||
if not force:
|
||||
cache = await recipe_scanner.get_cached_data()
|
||||
for recipe in getattr(cache, "raw_data", []):
|
||||
source = recipe.get("source_path")
|
||||
if source:
|
||||
existing_id = extract_civitai_image_id(source)
|
||||
if existing_id == image_id:
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"recipe_id": recipe.get("id"),
|
||||
"name": recipe.get("title", ""),
|
||||
"already_exists": True,
|
||||
})
|
||||
image_to_recipe = getattr(cache, "image_id_map", {})
|
||||
existing_recipe_id = image_to_recipe.get(image_id)
|
||||
if existing_recipe_id:
|
||||
recipe_name = ""
|
||||
for recipe in getattr(cache, "raw_data", []):
|
||||
if str(recipe.get("id", "")) == existing_recipe_id:
|
||||
recipe_name = recipe.get("title", "") or ""
|
||||
break
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"recipe_id": existing_recipe_id,
|
||||
"name": recipe_name,
|
||||
"already_exists": True,
|
||||
})
|
||||
|
||||
async with self._import_semaphore:
|
||||
return await self._do_import_from_url(image_url, recipe_scanner)
|
||||
|
||||
@@ -82,6 +82,7 @@ class DownloadQueueService:
|
||||
async with cls._class_lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = cls()
|
||||
await cls._instance.deduplicate()
|
||||
return cls._instance
|
||||
|
||||
def __init__(self, db_path: Optional[str] = None) -> None:
|
||||
@@ -349,6 +350,7 @@ class DownloadQueueService:
|
||||
file_path: Optional[str] = None,
|
||||
bytes_downloaded: int = 0,
|
||||
total_bytes: Optional[int] = None,
|
||||
completed_at: Optional[float] = None,
|
||||
) -> Optional[dict[str, Any]]:
|
||||
"""Atomically move a download from the queue into the history table.
|
||||
|
||||
@@ -356,6 +358,9 @@ class DownloadQueueService:
|
||||
queue, and inserts a corresponding history entry with the given
|
||||
terminal status (``completed``, ``failed``, or ``canceled``).
|
||||
|
||||
When *completed_at* is provided it is used as the completion
|
||||
timestamp; otherwise ``time.time()`` is used.
|
||||
|
||||
Returns the original queue record (before deletion) on success,
|
||||
or ``None`` if the download was not found in the queue.
|
||||
"""
|
||||
@@ -368,7 +373,7 @@ class DownloadQueueService:
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
now = time.time()
|
||||
now = completed_at if completed_at is not None else time.time()
|
||||
conn.execute(
|
||||
"DELETE FROM download_queue WHERE download_id = ?",
|
||||
(download_id,),
|
||||
@@ -604,7 +609,9 @@ class DownloadQueueService:
|
||||
|
||||
Looks up the history record by its primary key. If the status is
|
||||
``failed`` or ``canceled`` a new queue entry is created with the
|
||||
same model metadata and a fresh download id.
|
||||
same model metadata and a fresh download id, and the original
|
||||
history entry is **deleted** to prevent exponential growth when
|
||||
the retried item is later canceled or fails again and re-retried.
|
||||
"""
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
@@ -641,6 +648,10 @@ class DownloadQueueService:
|
||||
now,
|
||||
),
|
||||
)
|
||||
conn.execute(
|
||||
"DELETE FROM download_history WHERE id = ?",
|
||||
(item_id,),
|
||||
)
|
||||
conn.commit()
|
||||
queued = conn.execute(
|
||||
"SELECT * FROM download_queue WHERE download_id = ?",
|
||||
@@ -652,6 +663,9 @@ class DownloadQueueService:
|
||||
async def retry_all_failed(self) -> int:
|
||||
"""Re-queue all failed and canceled downloads from history.
|
||||
|
||||
Each history entry is **deleted** after being re-queued so that
|
||||
repeated retry-all calls do not cause exponential growth.
|
||||
|
||||
Returns the number of items that were re-queued.
|
||||
"""
|
||||
async with self._lock:
|
||||
@@ -687,6 +701,10 @@ class DownloadQueueService:
|
||||
now,
|
||||
),
|
||||
)
|
||||
conn.execute(
|
||||
"DELETE FROM download_history WHERE id = ?",
|
||||
(row["id"],),
|
||||
)
|
||||
count += 1
|
||||
conn.commit()
|
||||
|
||||
@@ -728,3 +746,126 @@ class DownloadQueueService:
|
||||
"failed": history_stats.get("failed", 0),
|
||||
"canceled": history_stats.get("canceled", 0),
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Deduplication (one-time cleanup for bug #980)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def deduplicate(self) -> dict[str, int]:
|
||||
"""Remove duplicate entries caused by the retry-amplification bug.
|
||||
|
||||
The bug (issue #980) caused the same download to appear N times in
|
||||
both the queue and history tables when ``retry_all_failed`` was
|
||||
called repeatedly without deleting the original history rows.
|
||||
|
||||
This method is called **once** when the singleton is first created.
|
||||
It is idempotent — after the first run there will be no duplicates
|
||||
to remove, so subsequent calls are a no-op.
|
||||
|
||||
Returns a dict with the count of removed rows per table.
|
||||
"""
|
||||
result: dict[str, int] = {
|
||||
"removed_history": 0,
|
||||
"removed_queue": 0,
|
||||
"removed_orphan_queue": 0,
|
||||
}
|
||||
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
|
||||
# 1. History: for each (model_id, model_version_id, status) triplet
|
||||
# keep only the row with the highest id (most recently inserted).
|
||||
conn.execute("""
|
||||
DELETE FROM download_history
|
||||
WHERE id NOT IN (
|
||||
SELECT MAX(id)
|
||||
FROM download_history
|
||||
GROUP BY model_id, model_version_id, status
|
||||
)
|
||||
""")
|
||||
result["removed_history"] = conn.execute(
|
||||
"SELECT changes()"
|
||||
).fetchone()[0]
|
||||
|
||||
# 2. Cross-status dedup: for each (model_id, model_version_id),
|
||||
# keep only the entry with the highest-priority terminal status.
|
||||
# Priority: completed (3) > failed (2) > canceled (1).
|
||||
# This prevents the same model version from having both a
|
||||
# 'failed' and a 'canceled' entry (or a 'completed' alongside
|
||||
# either) after the bug-created duplicates are removed.
|
||||
conn.execute("""
|
||||
DELETE FROM download_history
|
||||
WHERE id NOT IN (
|
||||
SELECT dh.id
|
||||
FROM download_history dh
|
||||
INNER JOIN (
|
||||
SELECT model_id, model_version_id,
|
||||
MAX(CASE status
|
||||
WHEN 'completed' THEN 3
|
||||
WHEN 'failed' THEN 2
|
||||
WHEN 'canceled' THEN 1
|
||||
ELSE 0
|
||||
END) AS best_prio
|
||||
FROM download_history
|
||||
GROUP BY model_id, model_version_id
|
||||
) best
|
||||
ON dh.model_id = best.model_id
|
||||
AND dh.model_version_id = best.model_version_id
|
||||
AND CASE dh.status
|
||||
WHEN 'completed' THEN 3
|
||||
WHEN 'failed' THEN 2
|
||||
WHEN 'canceled' THEN 1
|
||||
ELSE 0
|
||||
END = best.best_prio
|
||||
GROUP BY dh.model_id, dh.model_version_id
|
||||
HAVING dh.id = MAX(dh.id)
|
||||
)
|
||||
""")
|
||||
result["removed_history"] += conn.execute(
|
||||
"SELECT changes()"
|
||||
).fetchone()[0]
|
||||
|
||||
# 3. Queue: for each (model_id, model_version_id) keep only the
|
||||
# row with the latest added_at (most recently enqueued).
|
||||
conn.execute("""
|
||||
DELETE FROM download_queue
|
||||
WHERE rowid NOT IN (
|
||||
SELECT MAX(rowid)
|
||||
FROM download_queue
|
||||
WHERE status IN ('queued', 'downloading', 'paused', 'waiting')
|
||||
GROUP BY model_id, model_version_id
|
||||
)
|
||||
AND status IN ('queued', 'downloading', 'paused', 'waiting')
|
||||
""")
|
||||
result["removed_queue"] = conn.execute(
|
||||
"SELECT changes()"
|
||||
).fetchone()[0]
|
||||
|
||||
# 4. Remove orphaned queue entries — items that were re-queued
|
||||
# (source='retry') but whose model version already has a
|
||||
# terminal history entry. These are artifacts of the buggy
|
||||
# retry cycle that were never cleaned up.
|
||||
conn.execute("""
|
||||
DELETE FROM download_queue
|
||||
WHERE source = 'retry'
|
||||
AND (model_id, model_version_id) IN (
|
||||
SELECT model_id, model_version_id
|
||||
FROM download_history
|
||||
WHERE status IN ('failed', 'canceled')
|
||||
)
|
||||
AND status IN ('queued', 'waiting')
|
||||
""")
|
||||
result["removed_orphan_queue"] = conn.execute(
|
||||
"SELECT changes()"
|
||||
).fetchone()[0]
|
||||
|
||||
conn.commit()
|
||||
|
||||
logger.info(
|
||||
"Deduplicate: removed %s history rows, %s queue rows, "
|
||||
"%s orphaned queue rows",
|
||||
result["removed_history"],
|
||||
result["removed_queue"],
|
||||
result["removed_orphan_queue"],
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -256,7 +256,9 @@ class Downloader:
|
||||
self._session = None
|
||||
|
||||
# Check for app-level proxy settings
|
||||
proxy_url = None
|
||||
proxy_url = None # http(s) proxy, passed via the per-request `proxy=` kwarg
|
||||
socks_proxy_url = None # SOCKS proxy, handled via aiohttp-socks connector
|
||||
app_proxy_active = False
|
||||
settings_manager = get_settings_manager()
|
||||
if settings_manager.get("proxy_enabled", False):
|
||||
proxy_host = settings_manager.get("proxy_host", "").strip()
|
||||
@@ -268,9 +270,19 @@ class Downloader:
|
||||
if proxy_host and proxy_port:
|
||||
# Build proxy URL
|
||||
if proxy_username and proxy_password:
|
||||
proxy_url = f"{proxy_type}://{proxy_username}:{proxy_password}@{proxy_host}:{proxy_port}"
|
||||
full_proxy_url = f"{proxy_type}://{proxy_username}:{proxy_password}@{proxy_host}:{proxy_port}"
|
||||
else:
|
||||
proxy_url = f"{proxy_type}://{proxy_host}:{proxy_port}"
|
||||
full_proxy_url = f"{proxy_type}://{proxy_host}:{proxy_port}"
|
||||
|
||||
app_proxy_active = True
|
||||
# aiohttp cannot tunnel SOCKS via the per-request `proxy=` kwarg
|
||||
# (it would send HTTP to the SOCKS port and fail parsing the
|
||||
# SOCKS handshake reply). SOCKS must be handled by an
|
||||
# aiohttp-socks ProxyConnector instead.
|
||||
if proxy_type.startswith("socks"):
|
||||
socks_proxy_url = full_proxy_url
|
||||
else:
|
||||
proxy_url = full_proxy_url
|
||||
|
||||
logger.debug(
|
||||
f"Using app-level proxy: {proxy_type}://{proxy_host}:{proxy_port}"
|
||||
@@ -294,13 +306,27 @@ class Downloader:
|
||||
logger.debug("SSL: certifi unavailable; using system default CA bundle")
|
||||
|
||||
# Optimize TCP connection parameters
|
||||
connector = aiohttp.TCPConnector(
|
||||
connector_kwargs = dict(
|
||||
ssl=ssl_context,
|
||||
limit=8, # Concurrent connections
|
||||
ttl_dns_cache=300, # DNS cache timeout
|
||||
force_close=False, # Keep connections for reuse
|
||||
enable_cleanup_closed=True,
|
||||
)
|
||||
if socks_proxy_url:
|
||||
# Route all traffic through the SOCKS proxy via aiohttp-socks. The
|
||||
# connector tunnels every connection, so no per-request `proxy=` is
|
||||
# used (and must not be — see self._proxy_url below).
|
||||
try:
|
||||
from aiohttp_socks import ProxyConnector
|
||||
except ImportError as e: # pragma: no cover
|
||||
raise RuntimeError(
|
||||
"A SOCKS proxy is configured but the 'aiohttp-socks' package "
|
||||
"is not installed. Install it with: pip install aiohttp-socks"
|
||||
) from e
|
||||
connector = ProxyConnector.from_url(socks_proxy_url, **connector_kwargs)
|
||||
else:
|
||||
connector = aiohttp.TCPConnector(**connector_kwargs)
|
||||
|
||||
# Configure timeout parameters
|
||||
timeout = aiohttp.ClientTimeout(
|
||||
@@ -311,12 +337,14 @@ class Downloader:
|
||||
|
||||
self._session = aiohttp.ClientSession(
|
||||
connector=connector,
|
||||
trust_env=proxy_url
|
||||
is None, # Only use system proxy if no app-level proxy is set
|
||||
# Only fall back to system/env proxy when no app-level proxy is active
|
||||
trust_env=not app_proxy_active,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
# Store proxy URL for use in requests
|
||||
# Store proxy URL for per-request use. Stays None for SOCKS because the
|
||||
# ProxyConnector already tunnels everything; passing proxy= for SOCKS
|
||||
# would re-trigger the original aiohttp parse error.
|
||||
self._proxy_url = proxy_url
|
||||
self._session_created_at = datetime.now()
|
||||
|
||||
|
||||
@@ -216,13 +216,19 @@ class MetadataSyncService:
|
||||
provider_used: Optional[str] = None
|
||||
last_error: Optional[str] = None
|
||||
civitai_api_not_found = False
|
||||
any_rate_limited = False
|
||||
|
||||
for provider_name, provider in provider_attempts:
|
||||
try:
|
||||
civitai_metadata_candidate, error = await provider.get_model_by_hash(sha256)
|
||||
except RateLimitError as exc:
|
||||
exc.provider = exc.provider or (provider_name or provider.__class__.__name__)
|
||||
raise
|
||||
logger.warning(
|
||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||
provider_name or provider.__class__.__name__,
|
||||
exc.retry_after or 0,
|
||||
)
|
||||
any_rate_limited = True
|
||||
continue
|
||||
except Exception as exc: # pragma: no cover - defensive logging
|
||||
logger.error("Provider %s failed for hash %s: %s", provider_name, sha256, exc)
|
||||
civitai_metadata_candidate, error = None, str(exc)
|
||||
@@ -276,6 +282,8 @@ class MetadataSyncService:
|
||||
)
|
||||
|
||||
resolved_error = last_error or default_error
|
||||
if any_rate_limited and "Rate limited" not in resolved_error:
|
||||
resolved_error = "Rate limited"
|
||||
if is_expected_offline_error(resolved_error):
|
||||
resolved_error = OFFLINE_FRIENDLY_MESSAGE
|
||||
|
||||
|
||||
@@ -65,7 +65,14 @@ class _RateLimitRetryHelper:
|
||||
return await func(*args, **kwargs)
|
||||
except RateLimitError as exc:
|
||||
attempt += 1
|
||||
if attempt >= self._retry_limit:
|
||||
|
||||
# Determine effective retry limit based on rate-limit magnitude
|
||||
effective_retry_limit = self._retry_limit # default: 3
|
||||
if exc.retry_after is not None and exc.retry_after >= 120.0:
|
||||
# Long rate-limit window (>=2 min) — retries are futile
|
||||
effective_retry_limit = 1 # total 1 attempt = 0 retries
|
||||
|
||||
if attempt >= effective_retry_limit:
|
||||
exc.provider = exc.provider or label
|
||||
raise
|
||||
|
||||
@@ -81,7 +88,11 @@ class _RateLimitRetryHelper:
|
||||
|
||||
def _calculate_delay(self, retry_after: Optional[float], attempt: int) -> float:
|
||||
if retry_after is not None:
|
||||
return min(self._max_delay, max(0.0, retry_after))
|
||||
# Cap at 1800s (30 min) as a safety ceiling. The old 30s cap was
|
||||
# too low — CivArchive can return retry_after ~1500s, causing all
|
||||
# retries to fail. A generous ceiling protects against pathological
|
||||
# server values while still respecting the server's guidance.
|
||||
return min(1800.0, max(0.0, retry_after))
|
||||
|
||||
base_delay = self._base_delay * (2 ** max(0, attempt - 1))
|
||||
jitter_span = base_delay * self._jitter_ratio
|
||||
@@ -474,8 +485,12 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
if result:
|
||||
return result, error
|
||||
except RateLimitError as exc:
|
||||
exc.provider = exc.provider or label
|
||||
raise exc
|
||||
logger.warning(
|
||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||
label,
|
||||
exc.retry_after or 0,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.debug("Provider %s failed for get_model_by_hash: %s", label, e)
|
||||
continue
|
||||
@@ -493,16 +508,12 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
if result:
|
||||
return result
|
||||
except RateLimitError as exc:
|
||||
if not_found_confirmed:
|
||||
logger.debug(
|
||||
"Suppressing rate limit from %s for model %s: "
|
||||
"already confirmed as not found by another provider",
|
||||
label,
|
||||
model_id,
|
||||
)
|
||||
return None
|
||||
exc.provider = exc.provider or label
|
||||
raise exc
|
||||
logger.warning(
|
||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||
label,
|
||||
exc.retry_after or 0,
|
||||
)
|
||||
continue
|
||||
except ResourceNotFoundError:
|
||||
not_found_confirmed = True
|
||||
logger.debug(
|
||||
@@ -528,8 +539,12 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
if result:
|
||||
return result
|
||||
except RateLimitError as exc:
|
||||
exc.provider = exc.provider or label
|
||||
raise exc
|
||||
logger.warning(
|
||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||
label,
|
||||
exc.retry_after or 0,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.debug("Provider %s failed for get_model_version: %s", label, e)
|
||||
continue
|
||||
@@ -546,8 +561,12 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
if result:
|
||||
return result, error
|
||||
except RateLimitError as exc:
|
||||
exc.provider = exc.provider or label
|
||||
raise exc
|
||||
logger.warning(
|
||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||
label,
|
||||
exc.retry_after or 0,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.debug("Provider %s failed for get_model_version_info: %s", label, e)
|
||||
continue
|
||||
@@ -568,8 +587,12 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
except NotImplementedError:
|
||||
continue
|
||||
except RateLimitError as exc:
|
||||
exc.provider = exc.provider or label
|
||||
raise exc
|
||||
logger.warning(
|
||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||
label,
|
||||
exc.retry_after or 0,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
"Provider %s failed for get_model_versions_by_hashes: %s",
|
||||
@@ -590,8 +613,12 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
if result is not None:
|
||||
return result
|
||||
except RateLimitError as exc:
|
||||
exc.provider = exc.provider or label
|
||||
raise exc
|
||||
logger.warning(
|
||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||
label,
|
||||
exc.retry_after or 0,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.debug("Provider %s failed for get_user_models: %s", label, e)
|
||||
continue
|
||||
|
||||
@@ -12,7 +12,7 @@ import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional, Set, Tuple
|
||||
|
||||
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
||||
@@ -26,6 +26,8 @@ class PersistedRecipeData:
|
||||
|
||||
raw_data: List[Dict]
|
||||
file_stats: Dict[str, Tuple[float, int]] # json_path -> (mtime, size)
|
||||
image_id_map: Dict[str, str] = field(default_factory=dict)
|
||||
"""Precomputed mapping of civitai image_id → recipe_id."""
|
||||
|
||||
|
||||
class PersistentRecipeCache:
|
||||
@@ -116,6 +118,20 @@ class PersistentRecipeCache:
|
||||
if not rows:
|
||||
return None
|
||||
|
||||
# Restore precomputed image_id_map if available
|
||||
image_id_map: Dict[str, str] = {}
|
||||
try:
|
||||
meta_row = conn.execute(
|
||||
"SELECT value FROM cache_metadata WHERE key = ?",
|
||||
("image_id_map",),
|
||||
).fetchone()
|
||||
if meta_row:
|
||||
parsed = json.loads(meta_row["value"])
|
||||
if isinstance(parsed, dict):
|
||||
image_id_map = parsed
|
||||
except Exception:
|
||||
pass # missing or corrupt — rebuilt on next cache refresh
|
||||
|
||||
finally:
|
||||
conn.close()
|
||||
except FileNotFoundError:
|
||||
@@ -138,14 +154,24 @@ class PersistentRecipeCache:
|
||||
row["file_size"] or 0,
|
||||
)
|
||||
|
||||
return PersistedRecipeData(raw_data=raw_data, file_stats=file_stats)
|
||||
return PersistedRecipeData(
|
||||
raw_data=raw_data,
|
||||
file_stats=file_stats,
|
||||
image_id_map=image_id_map,
|
||||
)
|
||||
|
||||
def save_cache(self, recipes: List[Dict], json_paths: Optional[Dict[str, str]] = None) -> None:
|
||||
def save_cache(
|
||||
self,
|
||||
recipes: List[Dict],
|
||||
json_paths: Optional[Dict[str, str]] = None,
|
||||
image_id_map: Optional[Dict[str, str]] = None,
|
||||
) -> None:
|
||||
"""Save all recipes to SQLite cache.
|
||||
|
||||
Args:
|
||||
recipes: List of recipe dictionaries to persist.
|
||||
json_paths: Optional mapping of recipe_id -> json_path for file stats.
|
||||
image_id_map: Optional precomputed civitai image_id → recipe_id mapping.
|
||||
"""
|
||||
if not self.is_enabled():
|
||||
return
|
||||
@@ -186,6 +212,12 @@ class PersistentRecipeCache:
|
||||
recipe_rows,
|
||||
)
|
||||
|
||||
# Persist image_id_map for O(1) lookups on cache load
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO cache_metadata (key, value) VALUES (?, ?)",
|
||||
("image_id_map", json.dumps(image_id_map or {})),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
logger.debug("Persisted %d recipes to cache", len(recipe_rows))
|
||||
finally:
|
||||
@@ -273,6 +305,29 @@ class PersistentRecipeCache:
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to remove recipe %s from cache: %s", recipe_id, exc)
|
||||
|
||||
def save_image_id_map(self, image_id_map: Dict[str, str]) -> None:
|
||||
"""Persist the image_id_map to cache_metadata without rewriting the full cache.
|
||||
|
||||
This is called after ``add_recipe`` / ``remove_recipe`` mutations so
|
||||
the persistent copy does not go stale between full ``save_cache`` calls.
|
||||
"""
|
||||
if not self.is_enabled() or not self._schema_initialized:
|
||||
return
|
||||
|
||||
try:
|
||||
with self._db_lock:
|
||||
conn = self._connect()
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO cache_metadata (key, value) VALUES (?, ?)",
|
||||
("image_id_map", json.dumps(image_id_map)),
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to persist image_id_map: %s", exc)
|
||||
|
||||
def get_indexed_recipe_ids(self) -> Set[str]:
|
||||
"""Return all recipe IDs in the cache.
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import asyncio
|
||||
from typing import Iterable, List, Dict, Optional
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from operator import itemgetter
|
||||
from natsort import natsorted
|
||||
|
||||
@@ -14,6 +14,15 @@ class RecipeCache:
|
||||
sorted_by_date: List[Dict]
|
||||
folders: List[str] | None = None
|
||||
folder_tree: Dict | None = None
|
||||
image_id_map: Dict[str, str] = field(default_factory=dict)
|
||||
"""Mapping of civitai image_id → recipe_id, precomputed at cache build time.
|
||||
|
||||
Built once during cache initialization (O(n)) so that
|
||||
``check_image_exists`` and ``import_from_url`` duplicate checks
|
||||
can look up image_id in O(1) instead of scanning all recipes.
|
||||
Recipes imported from local files have no valid civitai image_id
|
||||
and are naturally excluded from this map.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
@@ -20,6 +20,7 @@ from .metadata_service import get_default_metadata_provider
|
||||
from .checkpoint_scanner import CheckpointScanner
|
||||
from .settings_manager import get_settings_manager
|
||||
from .recipes.errors import RecipeNotFoundError
|
||||
from ..utils.civitai_utils import extract_civitai_image_id
|
||||
from ..utils.utils import calculate_recipe_fingerprint, fuzzy_match
|
||||
from natsort import natsorted
|
||||
import sys
|
||||
@@ -532,7 +533,21 @@ class RecipeScanner:
|
||||
self._sort_cache_sync()
|
||||
# Backfill source_path from JSON files if missing (schema migration)
|
||||
if self._backfill_source_path_if_needed(recipes, json_paths):
|
||||
self._persistent_cache.save_cache(recipes, json_paths)
|
||||
self._cache.image_id_map = self._build_image_id_map()
|
||||
self._persistent_cache.save_cache(
|
||||
recipes, json_paths, self._cache.image_id_map
|
||||
)
|
||||
else:
|
||||
# Use persisted map, or rebuild if empty (e.g. first startup
|
||||
# after deploying the image_id_map feature).
|
||||
if persisted.image_id_map:
|
||||
self._cache.image_id_map = dict(persisted.image_id_map)
|
||||
else:
|
||||
self._cache.image_id_map = self._build_image_id_map()
|
||||
if self._cache.image_id_map:
|
||||
self._persistent_cache.save_image_id_map(
|
||||
self._cache.image_id_map
|
||||
)
|
||||
return self._cache
|
||||
else:
|
||||
# Partial update: some files changed
|
||||
@@ -545,8 +560,11 @@ class RecipeScanner:
|
||||
self._sort_cache_sync()
|
||||
# Backfill source_path from JSON files if missing (schema migration)
|
||||
self._backfill_source_path_if_needed(recipes, json_paths)
|
||||
self._cache.image_id_map = self._build_image_id_map()
|
||||
# Persist updated cache
|
||||
self._persistent_cache.save_cache(recipes, json_paths)
|
||||
self._persistent_cache.save_cache(
|
||||
recipes, json_paths, self._cache.image_id_map
|
||||
)
|
||||
return self._cache
|
||||
|
||||
# Fall back to full directory scan
|
||||
@@ -558,9 +576,12 @@ class RecipeScanner:
|
||||
self._cache.raw_data = recipes
|
||||
self._update_folder_metadata(self._cache)
|
||||
self._sort_cache_sync()
|
||||
self._cache.image_id_map = self._build_image_id_map()
|
||||
|
||||
# Persist for next startup
|
||||
self._persistent_cache.save_cache(recipes, json_paths)
|
||||
self._persistent_cache.save_cache(
|
||||
recipes, json_paths, self._cache.image_id_map
|
||||
)
|
||||
|
||||
return self._cache
|
||||
except Exception as e:
|
||||
@@ -832,6 +853,28 @@ class RecipeScanner:
|
||||
except Exception as e:
|
||||
logger.error(f"Error sorting recipe cache: {e}")
|
||||
|
||||
def _build_image_id_map(self) -> Dict[str, str]:
|
||||
"""Build civitai image_id → recipe_id mapping from cached recipes.
|
||||
|
||||
Only recipes with a valid CivitAI image URL source_path produce an
|
||||
entry. Recipes imported from local files are naturally excluded.
|
||||
"""
|
||||
mapping: Dict[str, str] = {}
|
||||
if not self._cache:
|
||||
return mapping
|
||||
for recipe in getattr(self._cache, "raw_data", []):
|
||||
if not isinstance(recipe, dict):
|
||||
continue
|
||||
source = recipe.get("source_path")
|
||||
if not source:
|
||||
continue
|
||||
image_id = extract_civitai_image_id(source)
|
||||
if image_id and image_id not in mapping:
|
||||
recipe_id = recipe.get("id")
|
||||
if recipe_id is not None:
|
||||
mapping[image_id] = str(recipe_id)
|
||||
return mapping
|
||||
|
||||
async def _wait_for_lora_scanner(self) -> None:
|
||||
"""Ensure the LoRA scanner has initialized before recipe enrichment."""
|
||||
|
||||
@@ -1296,11 +1339,20 @@ class RecipeScanner:
|
||||
# Update FTS index
|
||||
self._update_fts_index_for_recipe(recipe_data, "add")
|
||||
|
||||
source = recipe_data.get("source_path")
|
||||
if source:
|
||||
image_id = extract_civitai_image_id(source)
|
||||
if image_id:
|
||||
recipe_id_value = recipe_data.get("id")
|
||||
if recipe_id_value is not None:
|
||||
cache.image_id_map[image_id] = str(recipe_id_value)
|
||||
|
||||
# Persist to SQLite cache
|
||||
if self._persistent_cache:
|
||||
recipe_id = str(recipe_data.get("id", ""))
|
||||
json_path = self._json_path_map.get(recipe_id, "")
|
||||
self._persistent_cache.update_recipe(recipe_data, json_path)
|
||||
self._persistent_cache.save_image_id_map(cache.image_id_map)
|
||||
|
||||
async def remove_recipe(self, recipe_id: str) -> bool:
|
||||
"""Remove a recipe from the cache by ID."""
|
||||
@@ -1319,9 +1371,15 @@ class RecipeScanner:
|
||||
# Update FTS index
|
||||
self._update_fts_index_for_recipe(recipe_id, "remove")
|
||||
|
||||
# Remove any image_id entry pointing to this recipe
|
||||
stale = [k for k, v in cache.image_id_map.items() if v == recipe_id]
|
||||
for k in stale:
|
||||
del cache.image_id_map[k]
|
||||
|
||||
# Remove from SQLite cache
|
||||
if self._persistent_cache:
|
||||
self._persistent_cache.remove_recipe(recipe_id)
|
||||
self._persistent_cache.save_image_id_map(cache.image_id_map)
|
||||
self._json_path_map.pop(recipe_id, None)
|
||||
|
||||
return True
|
||||
@@ -1332,14 +1390,21 @@ class RecipeScanner:
|
||||
cache = await self.get_cached_data()
|
||||
removed = await cache.bulk_remove(recipe_ids, resort=False)
|
||||
if removed:
|
||||
removed_ids = {str(r.get("id", "")) for r in removed}
|
||||
stale = [k for k, v in cache.image_id_map.items() if v in removed_ids]
|
||||
for k in stale:
|
||||
del cache.image_id_map[k]
|
||||
|
||||
self._schedule_resort()
|
||||
# Update FTS index and persistent cache for each removed recipe
|
||||
for recipe in removed:
|
||||
recipe_id = str(recipe.get("id", ""))
|
||||
self._update_fts_index_for_recipe(recipe_id, "remove")
|
||||
if self._persistent_cache:
|
||||
self._persistent_cache.remove_recipe(recipe_id)
|
||||
self._json_path_map.pop(recipe_id, None)
|
||||
|
||||
if self._persistent_cache:
|
||||
self._persistent_cache.save_image_id_map(cache.image_id_map)
|
||||
return len(removed)
|
||||
|
||||
async def scan_all_recipes(self) -> List[Dict]:
|
||||
|
||||
@@ -77,6 +77,9 @@ class BulkMetadataRefreshUseCase:
|
||||
|
||||
await emit("started")
|
||||
|
||||
RATE_LIMIT_ABORT_THRESHOLD = 3
|
||||
consecutive_rate_limits = 0
|
||||
|
||||
for model in to_process:
|
||||
if self._service.scanner.is_cancelled():
|
||||
self._logger.info("Bulk metadata refresh cancelled by user")
|
||||
@@ -115,12 +118,39 @@ class BulkMetadataRefreshUseCase:
|
||||
continue
|
||||
|
||||
await MetadataManager.hydrate_model_data(model)
|
||||
result, _ = await self._metadata_sync.fetch_and_update_model(
|
||||
result, error_msg = await self._metadata_sync.fetch_and_update_model(
|
||||
sha256=model["sha256"],
|
||||
file_path=model["file_path"],
|
||||
model_data=model,
|
||||
update_cache_func=self._service.scanner.update_single_model_cache,
|
||||
)
|
||||
|
||||
if not result and error_msg and "Rate limited" in error_msg:
|
||||
consecutive_rate_limits += 1
|
||||
else:
|
||||
consecutive_rate_limits = 0
|
||||
|
||||
if consecutive_rate_limits >= RATE_LIMIT_ABORT_THRESHOLD:
|
||||
self._logger.warning(
|
||||
"Bulk metadata refresh aborted: %d consecutive rate limits detected. "
|
||||
"Processed %d/%d models.",
|
||||
consecutive_rate_limits,
|
||||
processed,
|
||||
total_to_process,
|
||||
)
|
||||
await emit(
|
||||
"rate_limited",
|
||||
processed=processed,
|
||||
success=success,
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
"message": f"Rate limit detected; {total_to_process - processed} models skipped",
|
||||
"processed": processed,
|
||||
"updated": success,
|
||||
"total": total_models,
|
||||
}
|
||||
|
||||
if result:
|
||||
success += 1
|
||||
if original_name != model.get("model_name"):
|
||||
|
||||
@@ -31,6 +31,8 @@ PREVIEW_EXTENSIONS = [
|
||||
".mp4",
|
||||
".gif",
|
||||
".webm",
|
||||
".avif",
|
||||
".jxl",
|
||||
]
|
||||
|
||||
# Card preview image width
|
||||
@@ -41,7 +43,7 @@ EXAMPLE_IMAGE_WIDTH = 832
|
||||
|
||||
# Supported media extensions for example downloads
|
||||
SUPPORTED_MEDIA_EXTENSIONS = {
|
||||
"images": [".jpg", ".jpeg", ".png", ".webp", ".gif"],
|
||||
"images": [".jpg", ".jpeg", ".png", ".webp", ".gif", ".avif", ".jxl"],
|
||||
"videos": [".mp4", ".webm"],
|
||||
}
|
||||
|
||||
|
||||
@@ -62,6 +62,10 @@ class ExampleImagesProcessor:
|
||||
return '.gif'
|
||||
elif content.startswith(b'RIFF') and b'WEBP' in content[:12]:
|
||||
return '.webp'
|
||||
elif len(content) >= 12 and content[4:8] == b'ftyp' and b'avif' in content[8:24]:
|
||||
return '.avif'
|
||||
elif content.startswith(b'\x00\x00\x00\x0cJXL \x0d\x0a\x87\x0a'):
|
||||
return '.jxl'
|
||||
elif content.startswith(b'\x00\x00\x00\x18ftypmp4') or content.startswith(b'\x00\x00\x00\x20ftypmp4'):
|
||||
return '.mp4'
|
||||
elif content.startswith(b'\x1A\x45\xDF\xA3'):
|
||||
@@ -75,6 +79,8 @@ class ExampleImagesProcessor:
|
||||
'image/png': '.png',
|
||||
'image/gif': '.gif',
|
||||
'image/webp': '.webp',
|
||||
'image/avif': '.avif',
|
||||
'image/jxl': '.jxl',
|
||||
'video/mp4': '.mp4',
|
||||
'video/webm': '.webm',
|
||||
'video/quicktime': '.mov'
|
||||
|
||||
@@ -1,17 +1,125 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import struct
|
||||
from io import BytesIO
|
||||
from typing import Any, Optional
|
||||
|
||||
import piexif
|
||||
from PIL import Image, PngImagePlugin
|
||||
|
||||
try:
|
||||
import brotli
|
||||
_BROTLI_AVAILABLE = True
|
||||
except ImportError:
|
||||
brotli = None
|
||||
_BROTLI_AVAILABLE = False
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class ExifUtils:
|
||||
"""Utility functions for working with EXIF data in images"""
|
||||
|
||||
@staticmethod
|
||||
def _parse_isobmff_boxes(data: bytes, offset: int = 0) -> list[dict]:
|
||||
boxes = []
|
||||
while offset + 8 <= len(data):
|
||||
size = struct.unpack('>I', data[offset:offset + 4])[0]
|
||||
box_type = data[offset + 4:offset + 8]
|
||||
if size == 0:
|
||||
break
|
||||
if size < 8 or offset + size > len(data):
|
||||
break
|
||||
box_data = data[offset + 8:offset + size]
|
||||
boxes.append({'type': box_type, 'data': box_data, 'size': size})
|
||||
offset += size
|
||||
return boxes
|
||||
|
||||
@staticmethod
|
||||
def _is_jxl_container(data: bytes) -> bool:
|
||||
if len(data) < 32:
|
||||
return False
|
||||
return (
|
||||
struct.unpack('>I', data[:4])[0] == 12
|
||||
and data[4:8] == b'JXL '
|
||||
and data[8:12] == bytes([0x0d, 0x0a, 0x87, 0x0a])
|
||||
and struct.unpack('>I', data[12:16])[0] >= 16
|
||||
and data[16:20] == b'ftyp'
|
||||
and data[20:24] == b'jxl '
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_avif_container(data: bytes) -> bool:
|
||||
if len(data) < 16:
|
||||
return False
|
||||
for box in ExifUtils._parse_isobmff_boxes(data):
|
||||
if box['type'] == b'ftyp' and b'avif' in box['data']:
|
||||
return True
|
||||
return False
|
||||
|
||||
# Max decompressed size for brotli metadata (2 MB)
|
||||
_BROTLI_MAX_DECOMPRESSED = 2 * 1024 * 1024
|
||||
|
||||
@staticmethod
|
||||
def _extract_isobmff_brotli(image_path: str) -> Optional[dict]:
|
||||
try:
|
||||
with open(image_path, 'rb') as f:
|
||||
data = f.read()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
if ExifUtils._is_jxl_container(data):
|
||||
boxes = ExifUtils._parse_isobmff_boxes(data, offset=12)
|
||||
elif ExifUtils._is_avif_container(data):
|
||||
boxes = ExifUtils._parse_isobmff_boxes(data)
|
||||
else:
|
||||
return None
|
||||
|
||||
brob = None
|
||||
for box in boxes:
|
||||
if box['type'] == b'brob':
|
||||
brob = box
|
||||
break
|
||||
if brob is None:
|
||||
return None
|
||||
|
||||
payload = brob['data']
|
||||
if payload[:4] != b'comf':
|
||||
return None
|
||||
compressed = payload[4:]
|
||||
|
||||
if _BROTLI_AVAILABLE:
|
||||
try:
|
||||
decompressed = brotli.decompress(compressed)
|
||||
if len(decompressed) > ExifUtils._BROTLI_MAX_DECOMPRESSED:
|
||||
logger.warning(
|
||||
"Brotli metadata too large (%d bytes, max %d), ignoring",
|
||||
len(decompressed),
|
||||
ExifUtils._BROTLI_MAX_DECOMPRESSED,
|
||||
)
|
||||
decompressed = None
|
||||
except Exception:
|
||||
decompressed = None
|
||||
else:
|
||||
decompressed = None
|
||||
|
||||
raw = decompressed if decompressed is not None else compressed
|
||||
try:
|
||||
meta = json.loads(raw.decode('utf-8'))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
result = {"parameters": None, "prompt": None, "workflow": None, "comment": None}
|
||||
if isinstance(meta.get("prompt"), (dict, list)):
|
||||
result["prompt"] = json.dumps(meta["prompt"])
|
||||
elif isinstance(meta.get("prompt"), str):
|
||||
result["prompt"] = meta["prompt"]
|
||||
if isinstance(meta.get("workflow"), (dict, list)):
|
||||
result["workflow"] = json.dumps(meta["workflow"])
|
||||
elif isinstance(meta.get("workflow"), str):
|
||||
result["workflow"] = meta["workflow"]
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _decode_user_comment(user_comment: Any) -> Optional[str]:
|
||||
if user_comment is None:
|
||||
@@ -43,6 +151,12 @@ class ExifUtils:
|
||||
"comment": None,
|
||||
}
|
||||
|
||||
ext = os.path.splitext(image_path)[1].lower()
|
||||
if ext in ('.avif', '.jxl'):
|
||||
brotli_meta = ExifUtils._extract_isobmff_brotli(image_path)
|
||||
if brotli_meta:
|
||||
return brotli_meta
|
||||
|
||||
with Image.open(image_path) as img:
|
||||
info = getattr(img, "info", {}) or {}
|
||||
|
||||
@@ -149,7 +263,6 @@ class ExifUtils:
|
||||
Optional[str]: Extracted metadata or None if not found
|
||||
"""
|
||||
try:
|
||||
# Skip for video files
|
||||
if image_path:
|
||||
ext = os.path.splitext(image_path)[1].lower()
|
||||
if ext in ['.mp4', '.webm']:
|
||||
@@ -177,10 +290,9 @@ class ExifUtils:
|
||||
str: Path to the updated image
|
||||
"""
|
||||
try:
|
||||
# Skip for video files
|
||||
if image_path:
|
||||
ext = os.path.splitext(image_path)[1].lower()
|
||||
if ext in ['.mp4', '.webm']:
|
||||
if ext in ['.mp4', '.webm', '.avif', '.jxl']:
|
||||
return image_path
|
||||
|
||||
metadata_fields = ExifUtils._load_structured_metadata(image_path)
|
||||
@@ -212,10 +324,9 @@ class ExifUtils:
|
||||
def append_recipe_metadata(image_path, recipe_data) -> str:
|
||||
"""Append recipe metadata to an image's EXIF data"""
|
||||
try:
|
||||
# Skip for video files
|
||||
if image_path:
|
||||
ext = os.path.splitext(image_path)[1].lower()
|
||||
if ext in ['.mp4', '.webm']:
|
||||
if ext in ['.mp4', '.webm', '.avif', '.jxl']:
|
||||
return image_path
|
||||
|
||||
# First, extract existing metadata
|
||||
@@ -327,10 +438,9 @@ class ExifUtils:
|
||||
Tuple of (optimized_image_data, extension)
|
||||
"""
|
||||
try:
|
||||
# Skip for video files early if it's a file path
|
||||
if isinstance(image_data, str) and os.path.exists(image_data):
|
||||
ext = os.path.splitext(image_data)[1].lower()
|
||||
if ext in ['.mp4', '.webm']:
|
||||
if ext in ['.mp4', '.webm', '.avif', '.jxl']:
|
||||
try:
|
||||
with open(image_data, 'rb') as f:
|
||||
return f.read(), ext
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "comfyui-lora-manager"
|
||||
description = "Revolutionize your workflow with the ultimate LoRA companion for ComfyUI!"
|
||||
version = "1.1.0"
|
||||
version = "1.1.1"
|
||||
license = {file = "LICENSE"}
|
||||
dependencies = [
|
||||
"aiohttp",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
aiohttp
|
||||
aiohttp-socks
|
||||
jinja2
|
||||
safetensors
|
||||
piexif
|
||||
@@ -12,3 +13,5 @@ aiosqlite
|
||||
beautifulsoup4
|
||||
platformdirs
|
||||
pyyaml
|
||||
# brotli — ISOBMFF (AVIF/JXL) metadata decompression
|
||||
brotli>=1.2.0
|
||||
|
||||
@@ -2,6 +2,7 @@ import os
|
||||
import sys
|
||||
import json
|
||||
from py.middleware.cache_middleware import cache_control
|
||||
from py.middleware.error_middleware import api_json_error
|
||||
from py.utils.settings_paths import ensure_settings_file
|
||||
|
||||
# Set environment variable to indicate standalone mode
|
||||
@@ -157,7 +158,7 @@ class StandaloneServer:
|
||||
def __init__(self):
|
||||
self.app = web.Application(
|
||||
logger=logger,
|
||||
middlewares=[cache_control],
|
||||
middlewares=[api_json_error, cache_control],
|
||||
client_max_size=256 * 1024 * 1024,
|
||||
handler_args={
|
||||
"max_field_size": HEADER_SIZE_LIMIT,
|
||||
|
||||
@@ -278,7 +278,7 @@
|
||||
left: 0;
|
||||
right: 0;
|
||||
background: linear-gradient(transparent 15%, oklch(0% 0 0 / 0.75));
|
||||
backdrop-filter: blur(8px);
|
||||
backdrop-filter: blur(var(--card-blur-amount, 8px));
|
||||
color: white;
|
||||
padding: var(--space-1);
|
||||
display: flex;
|
||||
@@ -294,7 +294,7 @@
|
||||
left: 0;
|
||||
right: 0;
|
||||
background: linear-gradient(oklch(0% 0 0 / 0.75), transparent 85%);
|
||||
backdrop-filter: blur(8px);
|
||||
backdrop-filter: blur(var(--card-blur-amount, 8px));
|
||||
color: white;
|
||||
padding: var(--space-1);
|
||||
display: flex;
|
||||
|
||||
@@ -813,6 +813,67 @@
|
||||
outline: none;
|
||||
}
|
||||
|
||||
/* Range Slider Control */
|
||||
.range-control {
|
||||
width: 100%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
justify-content: flex-end;
|
||||
}
|
||||
|
||||
.range-control input[type="range"] {
|
||||
width: 120px;
|
||||
height: 4px;
|
||||
-webkit-appearance: none;
|
||||
appearance: none;
|
||||
background: var(--border-color);
|
||||
border-radius: 2px;
|
||||
outline: none;
|
||||
cursor: pointer;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.range-control input[type="range"]::-webkit-slider-thumb {
|
||||
-webkit-appearance: none;
|
||||
appearance: none;
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
border-radius: 50%;
|
||||
background: var(--lora-accent);
|
||||
cursor: pointer;
|
||||
border: 2px solid var(--lora-surface);
|
||||
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.3);
|
||||
transition: transform 0.15s ease;
|
||||
}
|
||||
|
||||
.range-control input[type="range"]::-webkit-slider-thumb:hover {
|
||||
transform: scale(1.15);
|
||||
}
|
||||
|
||||
.range-control input[type="range"]::-moz-range-thumb {
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
border-radius: 50%;
|
||||
background: var(--lora-accent);
|
||||
cursor: pointer;
|
||||
border: 2px solid var(--lora-surface);
|
||||
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
.range-control .range-value {
|
||||
min-width: 36px;
|
||||
text-align: center;
|
||||
font-size: 0.9em;
|
||||
font-weight: 600;
|
||||
color: var(--text-color);
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
[data-theme="dark"] .range-control input[type="range"] {
|
||||
background: rgba(255, 255, 255, 0.15);
|
||||
}
|
||||
|
||||
/* Toggle Switch */
|
||||
.toggle-switch {
|
||||
position: relative;
|
||||
|
||||
@@ -355,9 +355,9 @@ function renderImportInterface(isEmpty) {
|
||||
<button class="select-files-btn" id="selectExampleFilesBtn">
|
||||
<i class="fas fa-folder-open"></i> Select Files
|
||||
</button>
|
||||
<p class="import-formats">Supported formats: jpg, png, gif, webp, mp4, webm</p>
|
||||
<p class="import-formats">Supported formats: jpg, png, gif, webp, avif, jxl, mp4, webm</p>
|
||||
</div>
|
||||
<input type="file" id="exampleFilesInput" multiple accept="image/*,video/mp4,video/webm" style="display: none;">
|
||||
<input type="file" id="exampleFilesInput" multiple accept="image/*,image/avif,image/jxl,video/mp4,video/webm" style="display: none;">
|
||||
<div class="import-progress-container" style="display: none;">
|
||||
<div class="import-progress">
|
||||
<div class="progress-bar"></div>
|
||||
@@ -473,7 +473,7 @@ export function initExampleImport(modelHash, container) {
|
||||
*/
|
||||
async function handleImportFiles(files, modelHash, importContainer) {
|
||||
// Filter for supported file types
|
||||
const supportedImages = ['.jpg', '.jpeg', '.png', '.gif', '.webp'];
|
||||
const supportedImages = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.avif', '.jxl'];
|
||||
const supportedVideos = ['.mp4', '.webm'];
|
||||
const supportedExtensions = [...supportedImages, ...supportedVideos];
|
||||
|
||||
|
||||
@@ -804,6 +804,16 @@ export class SettingsManager {
|
||||
);
|
||||
}
|
||||
|
||||
// Set card blur amount slider
|
||||
const cardBlurAmountInput = document.getElementById('cardBlurAmount');
|
||||
if (cardBlurAmountInput) {
|
||||
cardBlurAmountInput.value = state.global.settings.card_blur_amount ?? 8;
|
||||
}
|
||||
const cardBlurAmountValue = document.getElementById('cardBlurAmountValue');
|
||||
if (cardBlurAmountValue) {
|
||||
cardBlurAmountValue.textContent = `${state.global.settings.card_blur_amount ?? 8}px`;
|
||||
}
|
||||
|
||||
const usePortableCheckbox = document.getElementById('usePortableSettings');
|
||||
if (usePortableCheckbox) {
|
||||
usePortableCheckbox.checked = !!state.global.settings.use_portable_settings;
|
||||
@@ -2051,6 +2061,28 @@ export class SettingsManager {
|
||||
}
|
||||
}
|
||||
|
||||
async saveRangeSetting(elementId, displayId, settingKey) {
|
||||
const element = document.getElementById(elementId);
|
||||
if (!element) return;
|
||||
|
||||
const value = parseInt(element.value, 10);
|
||||
|
||||
try {
|
||||
await this.saveSetting(settingKey, value);
|
||||
this.applyFrontendSettings();
|
||||
|
||||
// Update the displayed value next to the slider
|
||||
const displayEl = document.getElementById(displayId);
|
||||
if (displayEl) {
|
||||
displayEl.textContent = `${value}px`;
|
||||
}
|
||||
|
||||
showToast('toast.settings.settingsUpdated', { setting: settingKey.replace(/_/g, ' ') }, 'success');
|
||||
} catch (error) {
|
||||
showToast('toast.settings.settingSaveFailed', { message: error.message }, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
updateExampleImagesOpenSettingsVisibility() {
|
||||
const openMode = state.global.settings.example_images_open_mode || 'system';
|
||||
const localRootSetting = document.getElementById('exampleImagesLocalRootSetting');
|
||||
@@ -2887,6 +2919,10 @@ export class SettingsManager {
|
||||
}
|
||||
|
||||
applyFrontendSettings() {
|
||||
// Apply card blur amount to CSS custom property
|
||||
const cardBlurAmount = state.global.settings.card_blur_amount ?? 8;
|
||||
document.documentElement.style.setProperty('--card-blur-amount', `${cardBlurAmount}px`);
|
||||
|
||||
// Apply autoplay setting to existing videos in card previews
|
||||
const autoplayOnHover = state.global.settings.autoplay_on_hover;
|
||||
document.querySelectorAll('.card-preview video').forEach(video => {
|
||||
|
||||
@@ -32,6 +32,7 @@ const DEFAULT_SETTINGS_BASE = Object.freeze({
|
||||
auto_download_example_images: false,
|
||||
blur_mature_content: true,
|
||||
mature_blur_level: 'R',
|
||||
card_blur_amount: 8,
|
||||
autoplay_on_hover: false,
|
||||
display_density: 'default',
|
||||
card_info_display: 'always',
|
||||
|
||||
@@ -448,6 +448,7 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<!-- Video Settings -->
|
||||
@@ -556,6 +557,23 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="setting-item">
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label for="cardBlurAmount">
|
||||
{{ t('settings.layoutSettings.cardBlurAmount') }}
|
||||
<i class="fas fa-info-circle info-icon" data-tooltip="{{ t('settings.layoutSettings.cardBlurAmountHelp') }}"></i>
|
||||
</label>
|
||||
</div>
|
||||
<div class="setting-control range-control">
|
||||
<input type="range" id="cardBlurAmount" min="0" max="20" value="8" step="1"
|
||||
oninput="document.getElementById('cardBlurAmountValue').textContent = this.value + 'px'"
|
||||
onchange="settingsManager.saveRangeSetting('cardBlurAmount', 'cardBlurAmountValue', 'card_blur_amount')">
|
||||
<span id="cardBlurAmountValue" class="range-value">8px</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="setting-item">
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
|
||||
@@ -46,6 +46,7 @@ class StubRecipeScanner:
|
||||
self.last_paginated_params: Dict[str, Any] | None = None
|
||||
self.lora_lookup: Dict[str, List[Dict[str, Any]]] = {}
|
||||
self.checkpoint_lookup: Dict[str, List[Dict[str, Any]]] = {}
|
||||
self.image_id_map_override: Dict[str, str] = {}
|
||||
|
||||
async def _noop_get_cached_data(force_refresh: bool = False) -> None: # noqa: ARG001 - signature mirrors real scanner
|
||||
return None
|
||||
@@ -56,7 +57,10 @@ class StubRecipeScanner:
|
||||
)
|
||||
|
||||
async def get_cached_data(self, force_refresh: bool = False) -> SimpleNamespace: # noqa: ARG002 - flag unused by stub
|
||||
return SimpleNamespace(raw_data=list(self.cached_raw))
|
||||
return SimpleNamespace(
|
||||
raw_data=list(self.cached_raw),
|
||||
image_id_map=dict(getattr(self, "image_id_map_override", {})),
|
||||
)
|
||||
|
||||
async def get_paginated_data(self, **params: Any) -> Dict[str, Any]:
|
||||
self.last_paginated_params = params
|
||||
@@ -999,3 +1003,95 @@ async def test_batch_import_cancel_missing_id(monkeypatch, tmp_path: Path) -> No
|
||||
payload = await response.json()
|
||||
assert response.status == 400
|
||||
assert payload["success"] is False
|
||||
|
||||
|
||||
async def test_check_image_exists_uses_image_id_map(monkeypatch, tmp_path: Path) -> None:
|
||||
"""check_image_exists must use precomputed image_id_map instead of scanning raw_data."""
|
||||
async with recipe_harness(monkeypatch, tmp_path) as harness:
|
||||
harness.scanner.image_id_map_override = {
|
||||
"123": "recipe-alpha",
|
||||
"789": "recipe-gamma",
|
||||
}
|
||||
|
||||
response = await harness.client.get(
|
||||
"/api/lm/recipes/check-image-exists",
|
||||
params={"image_ids": "123,456,789"},
|
||||
)
|
||||
payload = await response.json()
|
||||
|
||||
assert response.status == 200
|
||||
assert payload["success"] is True
|
||||
assert payload["results"]["123"] == {
|
||||
"in_library": True,
|
||||
"recipe_id": "recipe-alpha",
|
||||
}
|
||||
assert payload["results"]["456"] == {
|
||||
"in_library": False,
|
||||
"recipe_id": None,
|
||||
}
|
||||
assert payload["results"]["789"] == {
|
||||
"in_library": True,
|
||||
"recipe_id": "recipe-gamma",
|
||||
}
|
||||
|
||||
|
||||
async def test_check_image_exists_handles_empty_input(monkeypatch, tmp_path: Path) -> None:
|
||||
"""Empty or non-numeric image_ids must return an empty results dict."""
|
||||
async with recipe_harness(monkeypatch, tmp_path) as harness:
|
||||
response = await harness.client.get(
|
||||
"/api/lm/recipes/check-image-exists",
|
||||
params={"image_ids": ""},
|
||||
)
|
||||
payload = await response.json()
|
||||
assert response.status == 200
|
||||
assert payload["results"] == {}
|
||||
|
||||
|
||||
async def test_import_from_url_detects_duplicate_via_image_id_map(
|
||||
monkeypatch, tmp_path: Path,
|
||||
) -> None:
|
||||
"""import_from_url must return already_exists when image_id is in image_id_map."""
|
||||
async with recipe_harness(monkeypatch, tmp_path) as harness:
|
||||
harness.scanner.cached_raw = [
|
||||
{"id": "existing-recipe", "title": "My Recipe"},
|
||||
]
|
||||
harness.scanner.image_id_map_override = {
|
||||
"99999": "existing-recipe",
|
||||
}
|
||||
|
||||
response = await harness.client.get(
|
||||
"/api/lm/recipes/import-from-url",
|
||||
params={"image_url": "https://civitai.com/images/99999"},
|
||||
)
|
||||
payload = await response.json()
|
||||
|
||||
assert response.status == 200
|
||||
assert payload["already_exists"] is True
|
||||
assert payload["recipe_id"] == "existing-recipe"
|
||||
assert payload["name"] == "My Recipe"
|
||||
|
||||
|
||||
async def test_import_from_url_proceeds_when_image_id_not_in_map(
|
||||
monkeypatch, tmp_path: Path,
|
||||
) -> None:
|
||||
"""When image_id is absent from image_id_map, import_from_url must proceed to import."""
|
||||
async with recipe_harness(monkeypatch, tmp_path) as harness:
|
||||
harness.scanner.image_id_map_override = {
|
||||
"111": "some-other-recipe",
|
||||
}
|
||||
harness.civitai.image_info["99999"] = {
|
||||
"id": 99999,
|
||||
"url": "https://image.civitai.com/x/y/original=true/sample.jpeg",
|
||||
"type": "image",
|
||||
"meta": {"prompt": "test"},
|
||||
}
|
||||
|
||||
response = await harness.client.get(
|
||||
"/api/lm/recipes/import-from-url",
|
||||
params={"image_url": "https://civitai.com/images/99999"},
|
||||
)
|
||||
|
||||
# The import may succeed or fail depending on downstream stubs,
|
||||
# but it must NOT return already_exists
|
||||
payload = await response.json()
|
||||
assert payload.get("already_exists") is not True
|
||||
|
||||
@@ -441,7 +441,6 @@ async def test_fetch_and_update_model_returns_rate_limit_error(tmp_path):
|
||||
|
||||
assert ok is False
|
||||
assert error is not None and "Rate limited" in error
|
||||
assert "7" in error
|
||||
helpers.metadata_manager.save_metadata.assert_not_awaited()
|
||||
update_cache.assert_not_awaited()
|
||||
helpers.provider_selector.assert_not_awaited()
|
||||
|
||||
@@ -63,7 +63,8 @@ async def test_fallback_retries_same_provider_on_rate_limit(monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_respects_retry_limit(monkeypatch):
|
||||
async def test_fallback_continues_to_next_provider_on_rate_limit(monkeypatch):
|
||||
"""After exhausting retries on primary, fallback should continue to secondary."""
|
||||
sleep_mock = AsyncMock()
|
||||
monkeypatch.setattr(provider_module.asyncio, "sleep", sleep_mock)
|
||||
monkeypatch.setattr(provider_module.random, "uniform", lambda *_: 0.0)
|
||||
@@ -76,13 +77,13 @@ async def test_fallback_respects_retry_limit(monkeypatch):
|
||||
rate_limit_retry_limit=2,
|
||||
)
|
||||
|
||||
with pytest.raises(RateLimitError) as exc_info:
|
||||
await fallback.get_model_by_hash("abc")
|
||||
# After Change A: no longer raises; falls through to secondary
|
||||
result, error = await fallback.get_model_by_hash("abc")
|
||||
|
||||
assert exc_info.value.provider == "primary"
|
||||
assert primary.calls == 2
|
||||
assert secondary.calls == 0
|
||||
sleep_mock.assert_awaited_once()
|
||||
assert error is None
|
||||
assert result == {"id": "secondary"}
|
||||
assert primary.calls == 2 # retry_limit exhausted on primary
|
||||
assert secondary.calls == 1 # secondary IS called now
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -117,3 +118,40 @@ async def test_rate_limit_retrying_provider_respects_limit(monkeypatch):
|
||||
assert exc_info.value.provider == "inner"
|
||||
assert inner.calls == 2
|
||||
sleep_mock.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_helper_limits_retries_for_large_retry_after():
|
||||
"""With retry_after >= 120s, _RateLimitRetryHelper should only attempt once (no retries)."""
|
||||
calls = 0
|
||||
|
||||
async def failing():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise RateLimitError("limited", retry_after=1500.0)
|
||||
|
||||
helper = provider_module._RateLimitRetryHelper(retry_limit=3)
|
||||
with pytest.raises(RateLimitError):
|
||||
await helper.run("test", failing)
|
||||
assert calls == 1 # No retries for large retry_after
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_helper_retries_normally_for_small_retry_after(monkeypatch):
|
||||
"""With retry_after < 120s, _RateLimitRetryHelper should retry normally (up to limit)."""
|
||||
sleep_mock = AsyncMock()
|
||||
monkeypatch.setattr(provider_module.asyncio, "sleep", sleep_mock)
|
||||
|
||||
calls = 0
|
||||
|
||||
async def succeeding():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
raise RateLimitError("limited", retry_after=30.0)
|
||||
return {"ok": True}, None
|
||||
|
||||
helper = provider_module._RateLimitRetryHelper(retry_limit=3)
|
||||
result, _ = await helper.run("test", succeeding)
|
||||
assert result == {"ok": True}
|
||||
assert calls == 2 # Retried once (small retry_after)
|
||||
|
||||
@@ -1015,3 +1015,85 @@ async def test_get_paginated_data_sorting(recipe_scanner):
|
||||
# Test Date ASC: Gamma (5), Alpha (10), Beta (20)
|
||||
res = await scanner.get_paginated_data(page=1, page_size=10, sort_by="date:asc")
|
||||
assert [i["id"] for i in res["items"]] == ["C", "A", "B"]
|
||||
|
||||
|
||||
async def test_build_image_id_map_filters_correctly(recipe_scanner):
|
||||
"""Only recipes with valid CivitAI source_path appear in image_id_map.
|
||||
|
||||
Recipes imported from local files or with empty/missing source_path
|
||||
must be naturally excluded.
|
||||
"""
|
||||
scanner, _ = recipe_scanner
|
||||
from py.services.recipe_cache import RecipeCache
|
||||
|
||||
scanner._cache = RecipeCache(
|
||||
raw_data=[
|
||||
{"id": "r1", "source_path": "https://civitai.com/images/12345"},
|
||||
{"id": "r2", "source_path": "https://civitai.com/images/67890"},
|
||||
{"id": "r3", "source_path": "/home/user/local_image.png"},
|
||||
{"id": "r4", "source_path": ""},
|
||||
{"id": "r5"},
|
||||
],
|
||||
sorted_by_name=[],
|
||||
sorted_by_date=[],
|
||||
)
|
||||
|
||||
result = scanner._build_image_id_map()
|
||||
|
||||
assert result == {
|
||||
"12345": "r1",
|
||||
"67890": "r2",
|
||||
}
|
||||
# r3 = local file path, r4 = empty string, r5 = no key → all excluded
|
||||
for rid in ("r3", "r4", "r5"):
|
||||
assert rid not in result.values()
|
||||
|
||||
|
||||
async def test_add_recipe_updates_image_id_map(recipe_scanner):
|
||||
"""Adding a recipe with a CivitAI URL must update image_id_map.
|
||||
|
||||
A recipe with a local file path must NOT produce an entry.
|
||||
"""
|
||||
scanner, _ = recipe_scanner
|
||||
|
||||
await scanner.add_recipe({
|
||||
"id": "civitai-recipe",
|
||||
"title": "CivitAI",
|
||||
"source_path": "https://civitai.com/images/55555",
|
||||
})
|
||||
|
||||
cache = await scanner.get_cached_data()
|
||||
assert cache.image_id_map.get("55555") == "civitai-recipe"
|
||||
|
||||
await scanner.add_recipe({
|
||||
"id": "local-recipe",
|
||||
"title": "Local",
|
||||
"source_path": "/path/to/local.png",
|
||||
})
|
||||
|
||||
assert "local-recipe" not in cache.image_id_map.values()
|
||||
|
||||
|
||||
async def test_remove_recipe_clears_image_id_map(recipe_scanner):
|
||||
"""Removing a recipe that has a CivitAI image_id must clean up the map."""
|
||||
scanner, _ = recipe_scanner
|
||||
|
||||
await scanner.add_recipe({
|
||||
"id": "recipe-a",
|
||||
"title": "A",
|
||||
"source_path": "https://civitai.com/images/111",
|
||||
})
|
||||
await scanner.add_recipe({
|
||||
"id": "recipe-b",
|
||||
"title": "B",
|
||||
"source_path": "https://civitai.com/images/222",
|
||||
})
|
||||
|
||||
cache = await scanner.get_cached_data()
|
||||
assert "111" in cache.image_id_map
|
||||
assert cache.image_id_map["222"] == "recipe-b"
|
||||
|
||||
await scanner.remove_recipe("recipe-a")
|
||||
|
||||
assert "111" not in cache.image_id_map
|
||||
assert cache.image_id_map["222"] == "recipe-b"
|
||||
|
||||
@@ -465,3 +465,81 @@ class TestPersistentRecipeCache:
|
||||
# Operations should complete
|
||||
assert operation_counts["saves"] == 5
|
||||
assert operation_counts["removes"] == 5
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# image_id_map persistence (Phase 1 improvement)
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
def test_save_and_load_image_id_map_roundtrip(self, temp_db_path, sample_recipes):
|
||||
"""Save image_id_map via save_cache() and verify it round-trips through load_cache()."""
|
||||
cache = PersistentRecipeCache(db_path=temp_db_path)
|
||||
|
||||
image_id_map = {
|
||||
"12345": "recipe-alpha",
|
||||
"67890": "recipe-beta",
|
||||
}
|
||||
cache.save_cache(sample_recipes, image_id_map=image_id_map)
|
||||
|
||||
loaded = cache.load_cache()
|
||||
assert loaded is not None
|
||||
assert loaded.image_id_map == image_id_map
|
||||
|
||||
def test_load_without_image_id_map_returns_empty_dict(self, temp_db_path, sample_recipes):
|
||||
"""Loading from a cache that has no image_id_map metadata must yield {}."""
|
||||
cache = PersistentRecipeCache(db_path=temp_db_path)
|
||||
|
||||
# Save without image_id_map
|
||||
cache.save_cache(sample_recipes)
|
||||
|
||||
loaded = cache.load_cache()
|
||||
assert loaded is not None
|
||||
assert loaded.image_id_map == {}
|
||||
|
||||
def test_save_cache_without_image_id_map_does_not_corrupt_existing(
|
||||
self, temp_db_path, sample_recipes,
|
||||
):
|
||||
"""Overwriting cache without passing image_id_map must not leave stale data.
|
||||
|
||||
The previous image_id_map entry in cache_metadata should be replaced with {}.
|
||||
"""
|
||||
cache = PersistentRecipeCache(db_path=temp_db_path)
|
||||
|
||||
cache.save_cache(sample_recipes, image_id_map={"123": "old-recipe"})
|
||||
# Overwrite without image_id_map
|
||||
cache.save_cache(sample_recipes)
|
||||
|
||||
loaded = cache.load_cache()
|
||||
assert loaded.image_id_map == {}
|
||||
|
||||
def test_image_id_map_survives_recipe_update(self, temp_db_path, sample_recipes):
|
||||
"""Updating a single recipe must not drop the image_id_map metadata."""
|
||||
cache = PersistentRecipeCache(db_path=temp_db_path)
|
||||
|
||||
cache.save_cache(sample_recipes, image_id_map={"123": "recipe-alpha"})
|
||||
|
||||
updated = dict(sample_recipes[0])
|
||||
updated["title"] = "Updated"
|
||||
cache.update_recipe(updated)
|
||||
|
||||
loaded = cache.load_cache()
|
||||
assert loaded.image_id_map == {"123": "recipe-alpha"}
|
||||
|
||||
def test_save_image_id_map_persists_without_full_save(self, temp_db_path, sample_recipes):
|
||||
"""save_image_id_map must update cache_metadata without rewriting all recipes."""
|
||||
cache = PersistentRecipeCache(db_path=temp_db_path)
|
||||
cache.save_cache(sample_recipes)
|
||||
|
||||
cache.save_image_id_map({"555": "new-recipe", "666": "another-recipe"})
|
||||
|
||||
loaded = cache.load_cache()
|
||||
assert loaded.image_id_map == {"555": "new-recipe", "666": "another-recipe"}
|
||||
|
||||
def test_save_image_id_map_overwrites_previous(self, temp_db_path, sample_recipes):
|
||||
"""Calling save_image_id_map twice must replace, not merge."""
|
||||
cache = PersistentRecipeCache(db_path=temp_db_path)
|
||||
cache.save_cache(sample_recipes, image_id_map={"111": "old"})
|
||||
|
||||
cache.save_image_id_map({"222": "new-only"})
|
||||
|
||||
loaded = cache.load_cache()
|
||||
assert loaded.image_id_map == {"222": "new-only"}
|
||||
|
||||
@@ -141,3 +141,150 @@ def test_update_image_metadata_preserves_png_workflow(tmp_path):
|
||||
img.info["parameters"]
|
||||
== 'prompt text\nRecipe metadata: {"title":"recipe"}'
|
||||
)
|
||||
|
||||
|
||||
# --- ISOBMFF / brotli extraction tests ---
|
||||
|
||||
import struct
|
||||
|
||||
import brotli
|
||||
|
||||
|
||||
def _build_jxl_with_brob(payload_json: dict) -> bytes:
|
||||
"""Build a minimal JXL container with a brob box containing brotli-compressed JSON."""
|
||||
# ISOBMFF box 1: JXL signature box (size=12, type='JXL ', signature)
|
||||
box1 = struct.pack(">I", 12) + b"JXL " + bytes([0x0d, 0x0a, 0x87, 0x0a])
|
||||
# ISOBMFF box 2: ftyp (size=16, type='ftyp', major='jxl ', minor=0)
|
||||
box2 = struct.pack(">I", 16) + b"ftyp" + b"jxl " + struct.pack(">I", 0)
|
||||
# ISOBMFF box 3: brob — payload is b'comf' + brotli(json)
|
||||
compressed = brotli.compress(json.dumps(payload_json).encode("utf-8"))
|
||||
brob_payload = b"comf" + compressed
|
||||
box3 = struct.pack(">I", 8 + len(brob_payload)) + b"brob" + brob_payload
|
||||
return box1 + box2 + box3
|
||||
|
||||
|
||||
def _build_avif_with_brob(payload_json: dict) -> bytes:
|
||||
"""Build a minimal AVIF container with a brob box containing brotli-compressed JSON."""
|
||||
compressed = brotli.compress(json.dumps(payload_json).encode("utf-8"))
|
||||
brob_payload = b"comf" + compressed
|
||||
ftyp_box = struct.pack(">I", 20) + b"ftyp" + b"avif" + struct.pack(">I", 0) + b"avif"
|
||||
brob_box = struct.pack(">I", 8 + len(brob_payload)) + b"brob" + brob_payload
|
||||
return ftyp_box + brob_box
|
||||
|
||||
|
||||
class TestIsobmffBrotliExtraction:
|
||||
"""Tests for ISOBMFF brotli metadata extraction in ExifUtils."""
|
||||
|
||||
def test_extract_jxl_brotli_happy_path(self, tmp_path):
|
||||
"""JXL container with valid brob box extracts prompt and workflow."""
|
||||
payload = {"prompt": "a cute cat", "workflow": {"nodes": [{"id": 1}]}}
|
||||
data = _build_jxl_with_brob(payload)
|
||||
path = tmp_path / "test.jxl"
|
||||
path.write_bytes(data)
|
||||
|
||||
result = ExifUtils._load_structured_metadata(str(path))
|
||||
|
||||
assert result["prompt"] == "a cute cat"
|
||||
assert result["workflow"] == '{"nodes": [{"id": 1}]}'
|
||||
assert result["parameters"] is None
|
||||
assert result["comment"] is None
|
||||
|
||||
def test_extract_avif_brotli_happy_path(self, tmp_path):
|
||||
"""AVIF container with valid brob box extracts prompt and workflow."""
|
||||
payload = {"prompt": "landscape", "workflow": {"nodes": []}}
|
||||
data = _build_avif_with_brob(payload)
|
||||
path = tmp_path / "test.avif"
|
||||
path.write_bytes(data)
|
||||
|
||||
result = ExifUtils._load_structured_metadata(str(path))
|
||||
|
||||
assert result["prompt"] == "landscape"
|
||||
assert result["workflow"] == '{"nodes": []}'
|
||||
|
||||
def test_extract_no_brob_box_returns_none(self, tmp_path):
|
||||
"""JXL container without a brob box returns None from _extract_isobmff_brotli."""
|
||||
# Only JXL signature + ftyp, no brob
|
||||
box1 = struct.pack(">I", 12) + b"JXL " + bytes([0x0d, 0x0a, 0x87, 0x0a])
|
||||
box2 = struct.pack(">I", 16) + b"ftyp" + b"jxl " + struct.pack(">I", 0)
|
||||
path = tmp_path / "test.jxl"
|
||||
path.write_bytes(box1 + box2)
|
||||
|
||||
# The low-level extraction should return None (no brob box)
|
||||
result = ExifUtils._extract_isobmff_brotli(str(path))
|
||||
assert result is None
|
||||
|
||||
def test_extract_corrupt_brob_returns_none(self, tmp_path):
|
||||
"""Broken brob box payload gracefully returns None."""
|
||||
box1 = struct.pack(">I", 12) + b"JXL " + bytes([0x0d, 0x0a, 0x87, 0x0a])
|
||||
box2 = struct.pack(">I", 16) + b"ftyp" + b"jxl " + struct.pack(">I", 0)
|
||||
# brob with garbage payload that doesn't start with b'comf'
|
||||
garbage = b"\xff\xff\xff\xff" * 32
|
||||
box3 = struct.pack(">I", 8 + len(garbage)) + b"brob" + garbage
|
||||
path = tmp_path / "test.jxl"
|
||||
path.write_bytes(box1 + box2 + box3)
|
||||
|
||||
result = ExifUtils._extract_isobmff_brotli(str(path))
|
||||
assert result is None
|
||||
|
||||
def test_extract_non_isobmff_file_falls_through(self, tmp_path):
|
||||
"""A regular PNG file is not processed as ISOBMFF and returns PIL metadata."""
|
||||
png_info = PngImagePlugin.PngInfo()
|
||||
png_info.add_text("prompt", "from png")
|
||||
path = tmp_path / "test.png"
|
||||
Image.new("RGB", (4, 4), color="red").save(path, pnginfo=png_info)
|
||||
|
||||
result = ExifUtils._load_structured_metadata(str(path))
|
||||
assert result["prompt"] == "from png"
|
||||
|
||||
def test_extract_skip_on_update_and_optimize(self, tmp_path):
|
||||
"""AVIF/JXL files are skipped for write operations (update/append/optimize)."""
|
||||
path = tmp_path / "test.avif"
|
||||
path.write_bytes(b"fake avif data")
|
||||
|
||||
# update_image_metadata should return the path unchanged
|
||||
result = ExifUtils.update_image_metadata(str(path), "some metadata")
|
||||
assert result == str(path)
|
||||
|
||||
# append_recipe_metadata should also skip
|
||||
result = ExifUtils.append_recipe_metadata(str(path), {"title": "test"})
|
||||
assert result == str(path)
|
||||
|
||||
# optimize_image should passthrough for AVIF/JXL paths
|
||||
result_data, ext = ExifUtils.optimize_image(str(path))
|
||||
assert ext == ".avif"
|
||||
assert result_data == b"fake avif data"
|
||||
|
||||
def test_extract_prompt_as_dict(self, tmp_path):
|
||||
"""prompt field as dict is JSON-serialized."""
|
||||
payload = {"prompt": {"text": "hello", "negative": "bad"}}
|
||||
data = _build_jxl_with_brob(payload)
|
||||
path = tmp_path / "test.jxl"
|
||||
path.write_bytes(data)
|
||||
|
||||
result = ExifUtils._load_structured_metadata(str(path))
|
||||
assert json.loads(result["prompt"]) == {"text": "hello", "negative": "bad"}
|
||||
|
||||
def test_extract_workflow_as_list(self, tmp_path):
|
||||
"""workflow field as list is JSON-serialized."""
|
||||
payload = {"workflow": [{"id": 1}, {"id": 2}]}
|
||||
data = _build_avif_with_brob(payload)
|
||||
path = tmp_path / "test.avif"
|
||||
path.write_bytes(data)
|
||||
|
||||
result = ExifUtils._load_structured_metadata(str(path))
|
||||
assert json.loads(result["workflow"]) == [{"id": 1}, {"id": 2}]
|
||||
|
||||
def test_over_decompressed_size_limit(self, tmp_path, monkeypatch):
|
||||
"""Decompressed data exceeding _BROTLI_MAX_DECOMPRESSED is rejected."""
|
||||
# Monkey-patch the limit to a small value to avoid large test data
|
||||
monkeypatch.setattr(ExifUtils, "_BROTLI_MAX_DECOMPRESSED", 100)
|
||||
|
||||
large_content = "x" * 200
|
||||
payload = {"prompt": large_content}
|
||||
data = _build_jxl_with_brob(payload)
|
||||
path = tmp_path / "test.jxl"
|
||||
path.write_bytes(data)
|
||||
|
||||
# Direct extraction should return None because decompressed size exceeds limit
|
||||
result = ExifUtils._extract_isobmff_brotli(str(path))
|
||||
assert result is None
|
||||
|
||||
Reference in New Issue
Block a user