mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-06-26 12:51:16 -03:00
Compare commits
20 Commits
v1.1.0
...
a9e5ee7e79
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a9e5ee7e79 | ||
|
|
a17b0e9901 | ||
|
|
8f23d966bf | ||
|
|
7a76fc72d0 | ||
|
|
518a4dd5ee | ||
|
|
2b6d4e5d8b | ||
|
|
1f4edbeb9d | ||
|
|
a256558a0e | ||
|
|
818b9113f0 | ||
|
|
6a4fd020dc | ||
|
|
7a23040452 | ||
|
|
138024aefe | ||
|
|
a19ddc14f6 | ||
|
|
7001ced694 | ||
|
|
a5c861646c | ||
|
|
3e0bb73793 | ||
|
|
ac51f6a2f6 | ||
|
|
bef222c77d | ||
|
|
7cd6a53447 | ||
|
|
6850b35770 |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -12,12 +12,14 @@ coverage/
|
|||||||
.coverage
|
.coverage
|
||||||
model_cache/
|
model_cache/
|
||||||
|
|
||||||
# agent
|
# agent / dev tooling
|
||||||
.opencode/
|
.opencode/
|
||||||
.claude/
|
.claude/
|
||||||
.sisyphus/
|
.sisyphus/
|
||||||
.codex
|
.codex
|
||||||
.omo
|
.omo
|
||||||
|
reasonix.toml
|
||||||
|
.codegraph/
|
||||||
|
|
||||||
# Vue widgets development cache (but keep build output)
|
# Vue widgets development cache (but keep build output)
|
||||||
vue-widgets/node_modules/
|
vue-widgets/node_modules/
|
||||||
|
|||||||
@@ -448,7 +448,9 @@
|
|||||||
"modelName": "Modellname",
|
"modelName": "Modellname",
|
||||||
"fileName": "Dateiname"
|
"fileName": "Dateiname"
|
||||||
},
|
},
|
||||||
"modelNameDisplayHelp": "Wählen Sie aus, was in der Fußzeile der Modellkarte angezeigt werden soll"
|
"modelNameDisplayHelp": "Wählen Sie aus, was in der Fußzeile der Modellkarte angezeigt werden soll",
|
||||||
|
"cardBlurAmount": "Karten-Overlay-Unschärfe",
|
||||||
|
"cardBlurAmountHelp": "Passen Sie die Unschärfeintensität der Kopf- und Fußzeilen-Overlays auf Modell- und Rezeptkarten an (0 = keine Unschärfe, 20 = maximale Unschärfe)."
|
||||||
},
|
},
|
||||||
"folderSettings": {
|
"folderSettings": {
|
||||||
"activeLibrary": "Aktive Bibliothek",
|
"activeLibrary": "Aktive Bibliothek",
|
||||||
|
|||||||
@@ -448,7 +448,9 @@
|
|||||||
"modelName": "Model Name",
|
"modelName": "Model Name",
|
||||||
"fileName": "File Name"
|
"fileName": "File Name"
|
||||||
},
|
},
|
||||||
"modelNameDisplayHelp": "Choose what to display in the model card footer"
|
"modelNameDisplayHelp": "Choose what to display in the model card footer",
|
||||||
|
"cardBlurAmount": "Card Overlay Blur",
|
||||||
|
"cardBlurAmountHelp": "Adjust the blur intensity of the header and footer overlays on model and recipe cards (0 = no blur, 20 = maximum blur)."
|
||||||
},
|
},
|
||||||
"folderSettings": {
|
"folderSettings": {
|
||||||
"activeLibrary": "Active Library",
|
"activeLibrary": "Active Library",
|
||||||
|
|||||||
@@ -448,7 +448,9 @@
|
|||||||
"modelName": "Nombre del modelo",
|
"modelName": "Nombre del modelo",
|
||||||
"fileName": "Nombre del archivo"
|
"fileName": "Nombre del archivo"
|
||||||
},
|
},
|
||||||
"modelNameDisplayHelp": "Elige qué mostrar en el pie de la tarjeta del modelo"
|
"modelNameDisplayHelp": "Elige qué mostrar en el pie de la tarjeta del modelo",
|
||||||
|
"cardBlurAmount": "Desenfoque de superposición de tarjetas",
|
||||||
|
"cardBlurAmountHelp": "Ajuste la intensidad de desenfoque de las superposiciones del encabezado y pie de página en las tarjetas de modelos y recetas (0 = sin desenfoque, 20 = desenfoque máximo)."
|
||||||
},
|
},
|
||||||
"folderSettings": {
|
"folderSettings": {
|
||||||
"activeLibrary": "Biblioteca activa",
|
"activeLibrary": "Biblioteca activa",
|
||||||
|
|||||||
@@ -448,7 +448,9 @@
|
|||||||
"modelName": "Nom du modèle",
|
"modelName": "Nom du modèle",
|
||||||
"fileName": "Nom du fichier"
|
"fileName": "Nom du fichier"
|
||||||
},
|
},
|
||||||
"modelNameDisplayHelp": "Choisissez ce qui doit être affiché dans le pied de page de la carte du modèle"
|
"modelNameDisplayHelp": "Choisissez ce qui doit être affiché dans le pied de page de la carte du modèle",
|
||||||
|
"cardBlurAmount": "Flou de superposition des cartes",
|
||||||
|
"cardBlurAmountHelp": "Ajustez l'intensité du flou des superpositions d'en-tête et de pied de page sur les cartes de modèles et de recettes (0 = aucun flou, 20 = flou maximal)."
|
||||||
},
|
},
|
||||||
"folderSettings": {
|
"folderSettings": {
|
||||||
"activeLibrary": "Bibliothèque active",
|
"activeLibrary": "Bibliothèque active",
|
||||||
|
|||||||
@@ -448,7 +448,9 @@
|
|||||||
"modelName": "שם מודל",
|
"modelName": "שם מודל",
|
||||||
"fileName": "שם קובץ"
|
"fileName": "שם קובץ"
|
||||||
},
|
},
|
||||||
"modelNameDisplayHelp": "בחר מה להציג בכותרת התחתונה של כרטיס המודל"
|
"modelNameDisplayHelp": "בחר מה להציג בכותרת התחתונה של כרטיס המודל",
|
||||||
|
"cardBlurAmount": "עוצמת טשטוש שכבת-על בכרטיס",
|
||||||
|
"cardBlurAmountHelp": "כוונן את עוצמת הטשטוש של שכבת-העל בכותרת ובכותרות תחתונה בכרטיסי מודל ומתכונים (0 = ללא טשטוש, 20 = טשטוש מקסימלי)."
|
||||||
},
|
},
|
||||||
"folderSettings": {
|
"folderSettings": {
|
||||||
"activeLibrary": "ספרייה פעילה",
|
"activeLibrary": "ספרייה פעילה",
|
||||||
|
|||||||
@@ -448,7 +448,9 @@
|
|||||||
"modelName": "モデル名",
|
"modelName": "モデル名",
|
||||||
"fileName": "ファイル名"
|
"fileName": "ファイル名"
|
||||||
},
|
},
|
||||||
"modelNameDisplayHelp": "モデルカードのフッターに表示する内容を選択"
|
"modelNameDisplayHelp": "モデルカードのフッターに表示する内容を選択",
|
||||||
|
"cardBlurAmount": "カードオーバーレイのぼかし",
|
||||||
|
"cardBlurAmountHelp": "モデルカードとレシピカードのヘッダー・フッターオーバーレイのぼかし強度を調整します(0 = ぼかしなし、20 = 最大ぼかし)。"
|
||||||
},
|
},
|
||||||
"folderSettings": {
|
"folderSettings": {
|
||||||
"activeLibrary": "アクティブライブラリ",
|
"activeLibrary": "アクティブライブラリ",
|
||||||
|
|||||||
@@ -448,7 +448,9 @@
|
|||||||
"modelName": "모델명",
|
"modelName": "모델명",
|
||||||
"fileName": "파일명"
|
"fileName": "파일명"
|
||||||
},
|
},
|
||||||
"modelNameDisplayHelp": "모델 카드 하단에 표시할 내용을 선택하세요"
|
"modelNameDisplayHelp": "모델 카드 하단에 표시할 내용을 선택하세요",
|
||||||
|
"cardBlurAmount": "카드 오버레이 흐림 강도",
|
||||||
|
"cardBlurAmountHelp": "모델 및 레시피 카드의 헤더와 푸터 오버레이 흐림 강도를 조정합니다 (0 = 흐림 없음, 20 = 최대 흐림)."
|
||||||
},
|
},
|
||||||
"folderSettings": {
|
"folderSettings": {
|
||||||
"activeLibrary": "활성 라이브러리",
|
"activeLibrary": "활성 라이브러리",
|
||||||
|
|||||||
@@ -448,7 +448,9 @@
|
|||||||
"modelName": "Название модели",
|
"modelName": "Название модели",
|
||||||
"fileName": "Имя файла"
|
"fileName": "Имя файла"
|
||||||
},
|
},
|
||||||
"modelNameDisplayHelp": "Выберите, что отображать в нижней части карточки модели"
|
"modelNameDisplayHelp": "Выберите, что отображать в нижней части карточки модели",
|
||||||
|
"cardBlurAmount": "Размытие наложения карточек",
|
||||||
|
"cardBlurAmountHelp": "Настройте интенсивность размытия наложений верхнего и нижнего колонтитулов на карточках моделей и рецептов (0 = без размытия, 20 = максимальное размытие)."
|
||||||
},
|
},
|
||||||
"folderSettings": {
|
"folderSettings": {
|
||||||
"activeLibrary": "Активная библиотека",
|
"activeLibrary": "Активная библиотека",
|
||||||
|
|||||||
@@ -448,7 +448,9 @@
|
|||||||
"modelName": "模型名称",
|
"modelName": "模型名称",
|
||||||
"fileName": "文件名"
|
"fileName": "文件名"
|
||||||
},
|
},
|
||||||
"modelNameDisplayHelp": "选择在模型卡片底部显示的内容"
|
"modelNameDisplayHelp": "选择在模型卡片底部显示的内容",
|
||||||
|
"cardBlurAmount": "卡片叠加模糊强度",
|
||||||
|
"cardBlurAmountHelp": "调整模型和配方卡片上页眉和页脚叠加层的模糊强度(0 = 无模糊,20 = 最大模糊)。"
|
||||||
},
|
},
|
||||||
"folderSettings": {
|
"folderSettings": {
|
||||||
"activeLibrary": "活动库",
|
"activeLibrary": "活动库",
|
||||||
|
|||||||
@@ -448,7 +448,9 @@
|
|||||||
"modelName": "模型名稱",
|
"modelName": "模型名稱",
|
||||||
"fileName": "檔案名稱"
|
"fileName": "檔案名稱"
|
||||||
},
|
},
|
||||||
"modelNameDisplayHelp": "選擇在模型卡片底部顯示的內容"
|
"modelNameDisplayHelp": "選擇在模型卡片底部顯示的內容",
|
||||||
|
"cardBlurAmount": "卡片疊加模糊強度",
|
||||||
|
"cardBlurAmountHelp": "調整模型和配方卡片上頁首和頁尾疊加層的模糊強度(0 = 無模糊,20 = 最大模糊)。"
|
||||||
},
|
},
|
||||||
"folderSettings": {
|
"folderSettings": {
|
||||||
"activeLibrary": "使用中的資料庫",
|
"activeLibrary": "使用中的資料庫",
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ from .utils.example_images_migration import ExampleImagesMigration
|
|||||||
from .services.websocket_manager import ws_manager
|
from .services.websocket_manager import ws_manager
|
||||||
from .services.example_images_cleanup_service import ExampleImagesCleanupService
|
from .services.example_images_cleanup_service import ExampleImagesCleanupService
|
||||||
from .middleware.csp_middleware import relax_csp_for_remote_media
|
from .middleware.csp_middleware import relax_csp_for_remote_media
|
||||||
|
from .middleware.error_middleware import api_json_error
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -76,6 +77,11 @@ class LoraManager:
|
|||||||
"""Initialize and register all routes using the new refactored architecture"""
|
"""Initialize and register all routes using the new refactored architecture"""
|
||||||
app = PromptServer.instance.app
|
app = PromptServer.instance.app
|
||||||
|
|
||||||
|
# Register JSON error middleware for /api/* routes as the outermost
|
||||||
|
# middleware so it catches errors from all other middlewares.
|
||||||
|
if api_json_error not in app.middlewares:
|
||||||
|
app.middlewares.insert(0, api_json_error)
|
||||||
|
|
||||||
if relax_csp_for_remote_media not in app.middlewares:
|
if relax_csp_for_remote_media not in app.middlewares:
|
||||||
# Ensure CSP relaxer executes after ComfyUI's block_external_middleware so it can
|
# Ensure CSP relaxer executes after ComfyUI's block_external_middleware so it can
|
||||||
# see and extend the restrictive header instead of being overwritten by it.
|
# see and extend the restrictive header instead of being overwritten by it.
|
||||||
|
|||||||
@@ -16,6 +16,8 @@ IMG_EXTENSIONS = (
|
|||||||
".tif",
|
".tif",
|
||||||
".tiff",
|
".tiff",
|
||||||
".webp",
|
".webp",
|
||||||
|
".avif",
|
||||||
|
".jxl",
|
||||||
".mp4"
|
".mp4"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
71
py/middleware/error_middleware.py
Normal file
71
py/middleware/error_middleware.py
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
"""JSON error middleware for API routes.
|
||||||
|
|
||||||
|
Ensures all responses to /api/* requests return valid JSON that the
|
||||||
|
browser-extension frontend can JSON.parse() without crashing, even when
|
||||||
|
the route does not exist (404) or the handler raises an exception (500).
|
||||||
|
|
||||||
|
Extension consumers call response.json() unconditionally — an HTML error
|
||||||
|
page causes ``SyntaxError: unexpected end of data`` that leaks into the
|
||||||
|
popup UI as a toast notification.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Awaitable, Callable
|
||||||
|
|
||||||
|
from aiohttp import web
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@web.middleware
|
||||||
|
async def api_json_error(
|
||||||
|
request: web.Request,
|
||||||
|
handler: Callable[[web.Request], Awaitable[web.Response]],
|
||||||
|
) -> web.Response:
|
||||||
|
"""Return JSON ``{"success": false, "error": "..."}`` for API errors.
|
||||||
|
|
||||||
|
Only intercepts paths starting with ``/api/`` — all other routes
|
||||||
|
(frontend pages, static files, WebSocket upgrades) pass through
|
||||||
|
unchanged.
|
||||||
|
"""
|
||||||
|
if not request.path.startswith("/api/"):
|
||||||
|
return await handler(request)
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = await handler(request)
|
||||||
|
return response
|
||||||
|
except web.HTTPException as exc:
|
||||||
|
# Let redirects (301, 302, 307, 308) propagate — they are not errors.
|
||||||
|
if exc.status < 400:
|
||||||
|
raise
|
||||||
|
|
||||||
|
logger.warning(
|
||||||
|
"API %s %s returned HTTP %d: %s",
|
||||||
|
request.method,
|
||||||
|
request.path,
|
||||||
|
exc.status,
|
||||||
|
exc.reason,
|
||||||
|
)
|
||||||
|
|
||||||
|
return web.json_response(
|
||||||
|
{"success": False, "error": f"{exc.status}: {exc.reason}"},
|
||||||
|
status=exc.status,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error(
|
||||||
|
"API %s %s raised unhandled exception: %s",
|
||||||
|
request.method,
|
||||||
|
request.path,
|
||||||
|
exc,
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
return web.json_response(
|
||||||
|
{
|
||||||
|
"success": False,
|
||||||
|
"error": f"500: Internal Server Error ({type(exc).__name__})",
|
||||||
|
},
|
||||||
|
status=500,
|
||||||
|
)
|
||||||
@@ -11,7 +11,7 @@ from ..metadata_collector.metadata_processor import MetadataProcessor
|
|||||||
from ..metadata_collector import get_metadata
|
from ..metadata_collector import get_metadata
|
||||||
from ..utils.constants import CARD_PREVIEW_WIDTH
|
from ..utils.constants import CARD_PREVIEW_WIDTH
|
||||||
from ..utils.exif_utils import ExifUtils
|
from ..utils.exif_utils import ExifUtils
|
||||||
from ..utils.utils import calculate_recipe_fingerprint
|
from ..utils.utils import calculate_recipe_fingerprint, sanitize_folder_name
|
||||||
from PIL import Image, PngImagePlugin
|
from PIL import Image, PngImagePlugin
|
||||||
import piexif
|
import piexif
|
||||||
import logging
|
import logging
|
||||||
@@ -309,12 +309,14 @@ class SaveImageLM:
|
|||||||
filename = filename.replace(segment, str(h))
|
filename = filename.replace(segment, str(h))
|
||||||
elif key == "pprompt" and "prompt" in metadata_dict:
|
elif key == "pprompt" and "prompt" in metadata_dict:
|
||||||
prompt = metadata_dict.get("prompt", "").replace("\n", " ")
|
prompt = metadata_dict.get("prompt", "").replace("\n", " ")
|
||||||
|
prompt = sanitize_folder_name(prompt)
|
||||||
if len(parts) >= 2:
|
if len(parts) >= 2:
|
||||||
length = int(parts[1])
|
length = int(parts[1])
|
||||||
prompt = prompt[:length]
|
prompt = prompt[:length]
|
||||||
filename = filename.replace(segment, prompt.strip())
|
filename = filename.replace(segment, prompt.strip())
|
||||||
elif key == "nprompt" and "negative_prompt" in metadata_dict:
|
elif key == "nprompt" and "negative_prompt" in metadata_dict:
|
||||||
prompt = metadata_dict.get("negative_prompt", "").replace("\n", " ")
|
prompt = metadata_dict.get("negative_prompt", "").replace("\n", " ")
|
||||||
|
prompt = sanitize_folder_name(prompt)
|
||||||
if len(parts) >= 2:
|
if len(parts) >= 2:
|
||||||
length = int(parts[1])
|
length = int(parts[1])
|
||||||
prompt = prompt[:length]
|
prompt = prompt[:length]
|
||||||
@@ -328,6 +330,7 @@ class SaveImageLM:
|
|||||||
model = "model_unavailable"
|
model = "model_unavailable"
|
||||||
else:
|
else:
|
||||||
model = os.path.splitext(os.path.basename(model_value))[0]
|
model = os.path.splitext(os.path.basename(model_value))[0]
|
||||||
|
model = sanitize_folder_name(model)
|
||||||
if len(parts) >= 2:
|
if len(parts) >= 2:
|
||||||
length = int(parts[1])
|
length = int(parts[1])
|
||||||
model = model[:length]
|
model = model[:length]
|
||||||
|
|||||||
@@ -1785,6 +1785,8 @@ class ModelDownloadHandler:
|
|||||||
bytes_downloaded = 0
|
bytes_downloaded = 0
|
||||||
total_bytes_raw = request.query.get("total_bytes")
|
total_bytes_raw = request.query.get("total_bytes")
|
||||||
total_bytes = int(total_bytes_raw) if total_bytes_raw else None
|
total_bytes = int(total_bytes_raw) if total_bytes_raw else None
|
||||||
|
completed_at_raw = request.query.get("completed_at")
|
||||||
|
completed_at = float(completed_at_raw) if completed_at_raw else None
|
||||||
|
|
||||||
service = await DownloadQueueService.get_instance()
|
service = await DownloadQueueService.get_instance()
|
||||||
item = await service.complete_download(
|
item = await service.complete_download(
|
||||||
@@ -1794,6 +1796,7 @@ class ModelDownloadHandler:
|
|||||||
file_path=file_path,
|
file_path=file_path,
|
||||||
bytes_downloaded=bytes_downloaded,
|
bytes_downloaded=bytes_downloaded,
|
||||||
total_bytes=total_bytes,
|
total_bytes=total_bytes,
|
||||||
|
completed_at=completed_at,
|
||||||
)
|
)
|
||||||
if item is None:
|
if item is None:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ from ...config import config as global_config
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
_CHUNK_SIZE = 256 * 1024 # 256 KB
|
_CHUNK_SIZE = 1024 * 1024 # 1 MB — balance between streaming iteration overhead and per-chunk memory
|
||||||
|
|
||||||
# Video file extensions that bypass native sendfile on Windows
|
# Video file extensions that bypass native sendfile on Windows
|
||||||
# to avoid IOCP/ProactorEventLoop crashes during client disconnect.
|
# to avoid IOCP/ProactorEventLoop crashes during client disconnect.
|
||||||
@@ -55,16 +55,19 @@ class PreviewHandler:
|
|||||||
logger.debug("Preview file not found at %s", str(resolved))
|
logger.debug("Preview file not found at %s", str(resolved))
|
||||||
raise web.HTTPNotFound(text="Preview file not found")
|
raise web.HTTPNotFound(text="Preview file not found")
|
||||||
|
|
||||||
# Video files: stream manually to avoid Windows native sendfile crash.
|
# aiohttp's FileResponse handles range requests, content headers, and
|
||||||
# aiohttp's FileResponse uses _sendfile_native on Windows (IOCP-based),
|
# uses kernel sendfile (zero-copy DMA) on Linux/macOS. On Windows it
|
||||||
# which breaks when the client disconnects mid-transfer — this happens
|
# uses IOCP-based _sendfile_native which can crash when the client
|
||||||
# constantly when users scroll through a gallery of animated previews.
|
# disconnects mid-transfer during fast scrolling. The _stream_file()
|
||||||
suffix = resolved.suffix.lower()
|
# fallback is kept for a future compat toggle.
|
||||||
if suffix in _VIDEO_EXTENSIONS:
|
#
|
||||||
return await self._stream_file(request, resolved)
|
# Set explicit Cache-Control so the browser can cache video (and image)
|
||||||
|
# previews across VirtualScroller recycling cycles. Without this,
|
||||||
# aiohttp's FileResponse handles range requests and content headers for us.
|
# Chrome does not cache 206 Partial Content responses for <video>
|
||||||
return web.FileResponse(path=resolved, chunk_size=_CHUNK_SIZE)
|
# elements, causing the same video to be re-downloaded on every scroll.
|
||||||
|
resp = web.FileResponse(path=resolved, chunk_size=_CHUNK_SIZE)
|
||||||
|
resp.headers["Cache-Control"] = "public, max-age=86400"
|
||||||
|
return resp
|
||||||
|
|
||||||
async def _stream_file(
|
async def _stream_file(
|
||||||
self, request: web.Request, path: Path
|
self, request: web.Request, path: Path
|
||||||
@@ -83,6 +86,10 @@ class PreviewHandler:
|
|||||||
resp.content_type = content_type
|
resp.content_type = content_type
|
||||||
resp.content_length = file_size
|
resp.content_length = file_size
|
||||||
|
|
||||||
|
# Allow browser caching: video previews rarely change during a session.
|
||||||
|
# The frontend already appends ?t={version} to bust cache on update.
|
||||||
|
resp.headers["Cache-Control"] = "public, max-age=86400"
|
||||||
|
|
||||||
await resp.prepare(request)
|
await resp.prepare(request)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -1597,15 +1597,8 @@ class RecipeManagementHandler:
|
|||||||
|
|
||||||
cache = await recipe_scanner.get_cached_data()
|
cache = await recipe_scanner.get_cached_data()
|
||||||
|
|
||||||
# Build lookup: image_id -> recipe_id from stored source_path
|
# Use precomputed image_id_map (built once at cache init)
|
||||||
image_to_recipe = {}
|
image_to_recipe = getattr(cache, "image_id_map", {})
|
||||||
for recipe in getattr(cache, "raw_data", []):
|
|
||||||
source = recipe.get("source_path")
|
|
||||||
if not source:
|
|
||||||
continue
|
|
||||||
image_id = extract_civitai_image_id(source)
|
|
||||||
if image_id and image_id not in image_to_recipe:
|
|
||||||
image_to_recipe[image_id] = recipe.get("id")
|
|
||||||
|
|
||||||
results = {}
|
results = {}
|
||||||
for img_id in requested_ids:
|
for img_id in requested_ids:
|
||||||
@@ -1641,18 +1634,20 @@ class RecipeManagementHandler:
|
|||||||
"Could not extract Civitai image ID from URL"
|
"Could not extract Civitai image ID from URL"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Check for duplicate (fast, before acquiring semaphore), unless force
|
|
||||||
if not force:
|
if not force:
|
||||||
cache = await recipe_scanner.get_cached_data()
|
cache = await recipe_scanner.get_cached_data()
|
||||||
|
image_to_recipe = getattr(cache, "image_id_map", {})
|
||||||
|
existing_recipe_id = image_to_recipe.get(image_id)
|
||||||
|
if existing_recipe_id:
|
||||||
|
recipe_name = ""
|
||||||
for recipe in getattr(cache, "raw_data", []):
|
for recipe in getattr(cache, "raw_data", []):
|
||||||
source = recipe.get("source_path")
|
if str(recipe.get("id", "")) == existing_recipe_id:
|
||||||
if source:
|
recipe_name = recipe.get("title", "") or ""
|
||||||
existing_id = extract_civitai_image_id(source)
|
break
|
||||||
if existing_id == image_id:
|
|
||||||
return web.json_response({
|
return web.json_response({
|
||||||
"success": True,
|
"success": True,
|
||||||
"recipe_id": recipe.get("id"),
|
"recipe_id": existing_recipe_id,
|
||||||
"name": recipe.get("title", ""),
|
"name": recipe_name,
|
||||||
"already_exists": True,
|
"already_exists": True,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -82,6 +82,7 @@ class DownloadQueueService:
|
|||||||
async with cls._class_lock:
|
async with cls._class_lock:
|
||||||
if cls._instance is None:
|
if cls._instance is None:
|
||||||
cls._instance = cls()
|
cls._instance = cls()
|
||||||
|
await cls._instance.deduplicate()
|
||||||
return cls._instance
|
return cls._instance
|
||||||
|
|
||||||
def __init__(self, db_path: Optional[str] = None) -> None:
|
def __init__(self, db_path: Optional[str] = None) -> None:
|
||||||
@@ -349,6 +350,7 @@ class DownloadQueueService:
|
|||||||
file_path: Optional[str] = None,
|
file_path: Optional[str] = None,
|
||||||
bytes_downloaded: int = 0,
|
bytes_downloaded: int = 0,
|
||||||
total_bytes: Optional[int] = None,
|
total_bytes: Optional[int] = None,
|
||||||
|
completed_at: Optional[float] = None,
|
||||||
) -> Optional[dict[str, Any]]:
|
) -> Optional[dict[str, Any]]:
|
||||||
"""Atomically move a download from the queue into the history table.
|
"""Atomically move a download from the queue into the history table.
|
||||||
|
|
||||||
@@ -356,6 +358,9 @@ class DownloadQueueService:
|
|||||||
queue, and inserts a corresponding history entry with the given
|
queue, and inserts a corresponding history entry with the given
|
||||||
terminal status (``completed``, ``failed``, or ``canceled``).
|
terminal status (``completed``, ``failed``, or ``canceled``).
|
||||||
|
|
||||||
|
When *completed_at* is provided it is used as the completion
|
||||||
|
timestamp; otherwise ``time.time()`` is used.
|
||||||
|
|
||||||
Returns the original queue record (before deletion) on success,
|
Returns the original queue record (before deletion) on success,
|
||||||
or ``None`` if the download was not found in the queue.
|
or ``None`` if the download was not found in the queue.
|
||||||
"""
|
"""
|
||||||
@@ -368,7 +373,7 @@ class DownloadQueueService:
|
|||||||
if row is None:
|
if row is None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
now = time.time()
|
now = completed_at if completed_at is not None else time.time()
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"DELETE FROM download_queue WHERE download_id = ?",
|
"DELETE FROM download_queue WHERE download_id = ?",
|
||||||
(download_id,),
|
(download_id,),
|
||||||
@@ -604,7 +609,9 @@ class DownloadQueueService:
|
|||||||
|
|
||||||
Looks up the history record by its primary key. If the status is
|
Looks up the history record by its primary key. If the status is
|
||||||
``failed`` or ``canceled`` a new queue entry is created with the
|
``failed`` or ``canceled`` a new queue entry is created with the
|
||||||
same model metadata and a fresh download id.
|
same model metadata and a fresh download id, and the original
|
||||||
|
history entry is **deleted** to prevent exponential growth when
|
||||||
|
the retried item is later canceled or fails again and re-retried.
|
||||||
"""
|
"""
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
conn = self._get_conn()
|
conn = self._get_conn()
|
||||||
@@ -641,6 +648,10 @@ class DownloadQueueService:
|
|||||||
now,
|
now,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM download_history WHERE id = ?",
|
||||||
|
(item_id,),
|
||||||
|
)
|
||||||
conn.commit()
|
conn.commit()
|
||||||
queued = conn.execute(
|
queued = conn.execute(
|
||||||
"SELECT * FROM download_queue WHERE download_id = ?",
|
"SELECT * FROM download_queue WHERE download_id = ?",
|
||||||
@@ -652,6 +663,9 @@ class DownloadQueueService:
|
|||||||
async def retry_all_failed(self) -> int:
|
async def retry_all_failed(self) -> int:
|
||||||
"""Re-queue all failed and canceled downloads from history.
|
"""Re-queue all failed and canceled downloads from history.
|
||||||
|
|
||||||
|
Each history entry is **deleted** after being re-queued so that
|
||||||
|
repeated retry-all calls do not cause exponential growth.
|
||||||
|
|
||||||
Returns the number of items that were re-queued.
|
Returns the number of items that were re-queued.
|
||||||
"""
|
"""
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
@@ -687,6 +701,10 @@ class DownloadQueueService:
|
|||||||
now,
|
now,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM download_history WHERE id = ?",
|
||||||
|
(row["id"],),
|
||||||
|
)
|
||||||
count += 1
|
count += 1
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|
||||||
@@ -728,3 +746,126 @@ class DownloadQueueService:
|
|||||||
"failed": history_stats.get("failed", 0),
|
"failed": history_stats.get("failed", 0),
|
||||||
"canceled": history_stats.get("canceled", 0),
|
"canceled": history_stats.get("canceled", 0),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Deduplication (one-time cleanup for bug #980)
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
async def deduplicate(self) -> dict[str, int]:
|
||||||
|
"""Remove duplicate entries caused by the retry-amplification bug.
|
||||||
|
|
||||||
|
The bug (issue #980) caused the same download to appear N times in
|
||||||
|
both the queue and history tables when ``retry_all_failed`` was
|
||||||
|
called repeatedly without deleting the original history rows.
|
||||||
|
|
||||||
|
This method is called **once** when the singleton is first created.
|
||||||
|
It is idempotent — after the first run there will be no duplicates
|
||||||
|
to remove, so subsequent calls are a no-op.
|
||||||
|
|
||||||
|
Returns a dict with the count of removed rows per table.
|
||||||
|
"""
|
||||||
|
result: dict[str, int] = {
|
||||||
|
"removed_history": 0,
|
||||||
|
"removed_queue": 0,
|
||||||
|
"removed_orphan_queue": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
async with self._lock:
|
||||||
|
conn = self._get_conn()
|
||||||
|
|
||||||
|
# 1. History: for each (model_id, model_version_id, status) triplet
|
||||||
|
# keep only the row with the highest id (most recently inserted).
|
||||||
|
conn.execute("""
|
||||||
|
DELETE FROM download_history
|
||||||
|
WHERE id NOT IN (
|
||||||
|
SELECT MAX(id)
|
||||||
|
FROM download_history
|
||||||
|
GROUP BY model_id, model_version_id, status
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
result["removed_history"] = conn.execute(
|
||||||
|
"SELECT changes()"
|
||||||
|
).fetchone()[0]
|
||||||
|
|
||||||
|
# 2. Cross-status dedup: for each (model_id, model_version_id),
|
||||||
|
# keep only the entry with the highest-priority terminal status.
|
||||||
|
# Priority: completed (3) > failed (2) > canceled (1).
|
||||||
|
# This prevents the same model version from having both a
|
||||||
|
# 'failed' and a 'canceled' entry (or a 'completed' alongside
|
||||||
|
# either) after the bug-created duplicates are removed.
|
||||||
|
conn.execute("""
|
||||||
|
DELETE FROM download_history
|
||||||
|
WHERE id NOT IN (
|
||||||
|
SELECT dh.id
|
||||||
|
FROM download_history dh
|
||||||
|
INNER JOIN (
|
||||||
|
SELECT model_id, model_version_id,
|
||||||
|
MAX(CASE status
|
||||||
|
WHEN 'completed' THEN 3
|
||||||
|
WHEN 'failed' THEN 2
|
||||||
|
WHEN 'canceled' THEN 1
|
||||||
|
ELSE 0
|
||||||
|
END) AS best_prio
|
||||||
|
FROM download_history
|
||||||
|
GROUP BY model_id, model_version_id
|
||||||
|
) best
|
||||||
|
ON dh.model_id = best.model_id
|
||||||
|
AND dh.model_version_id = best.model_version_id
|
||||||
|
AND CASE dh.status
|
||||||
|
WHEN 'completed' THEN 3
|
||||||
|
WHEN 'failed' THEN 2
|
||||||
|
WHEN 'canceled' THEN 1
|
||||||
|
ELSE 0
|
||||||
|
END = best.best_prio
|
||||||
|
GROUP BY dh.model_id, dh.model_version_id
|
||||||
|
HAVING dh.id = MAX(dh.id)
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
result["removed_history"] += conn.execute(
|
||||||
|
"SELECT changes()"
|
||||||
|
).fetchone()[0]
|
||||||
|
|
||||||
|
# 3. Queue: for each (model_id, model_version_id) keep only the
|
||||||
|
# row with the latest added_at (most recently enqueued).
|
||||||
|
conn.execute("""
|
||||||
|
DELETE FROM download_queue
|
||||||
|
WHERE rowid NOT IN (
|
||||||
|
SELECT MAX(rowid)
|
||||||
|
FROM download_queue
|
||||||
|
WHERE status IN ('queued', 'downloading', 'paused', 'waiting')
|
||||||
|
GROUP BY model_id, model_version_id
|
||||||
|
)
|
||||||
|
AND status IN ('queued', 'downloading', 'paused', 'waiting')
|
||||||
|
""")
|
||||||
|
result["removed_queue"] = conn.execute(
|
||||||
|
"SELECT changes()"
|
||||||
|
).fetchone()[0]
|
||||||
|
|
||||||
|
# 4. Remove orphaned queue entries — items that were re-queued
|
||||||
|
# (source='retry') but whose model version already has a
|
||||||
|
# terminal history entry. These are artifacts of the buggy
|
||||||
|
# retry cycle that were never cleaned up.
|
||||||
|
conn.execute("""
|
||||||
|
DELETE FROM download_queue
|
||||||
|
WHERE source = 'retry'
|
||||||
|
AND (model_id, model_version_id) IN (
|
||||||
|
SELECT model_id, model_version_id
|
||||||
|
FROM download_history
|
||||||
|
WHERE status IN ('failed', 'canceled')
|
||||||
|
)
|
||||||
|
AND status IN ('queued', 'waiting')
|
||||||
|
""")
|
||||||
|
result["removed_orphan_queue"] = conn.execute(
|
||||||
|
"SELECT changes()"
|
||||||
|
).fetchone()[0]
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Deduplicate: removed %s history rows, %s queue rows, "
|
||||||
|
"%s orphaned queue rows",
|
||||||
|
result["removed_history"],
|
||||||
|
result["removed_queue"],
|
||||||
|
result["removed_orphan_queue"],
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|||||||
@@ -256,7 +256,9 @@ class Downloader:
|
|||||||
self._session = None
|
self._session = None
|
||||||
|
|
||||||
# Check for app-level proxy settings
|
# Check for app-level proxy settings
|
||||||
proxy_url = None
|
proxy_url = None # http(s) proxy, passed via the per-request `proxy=` kwarg
|
||||||
|
socks_proxy_url = None # SOCKS proxy, handled via aiohttp-socks connector
|
||||||
|
app_proxy_active = False
|
||||||
settings_manager = get_settings_manager()
|
settings_manager = get_settings_manager()
|
||||||
if settings_manager.get("proxy_enabled", False):
|
if settings_manager.get("proxy_enabled", False):
|
||||||
proxy_host = settings_manager.get("proxy_host", "").strip()
|
proxy_host = settings_manager.get("proxy_host", "").strip()
|
||||||
@@ -268,9 +270,19 @@ class Downloader:
|
|||||||
if proxy_host and proxy_port:
|
if proxy_host and proxy_port:
|
||||||
# Build proxy URL
|
# Build proxy URL
|
||||||
if proxy_username and proxy_password:
|
if proxy_username and proxy_password:
|
||||||
proxy_url = f"{proxy_type}://{proxy_username}:{proxy_password}@{proxy_host}:{proxy_port}"
|
full_proxy_url = f"{proxy_type}://{proxy_username}:{proxy_password}@{proxy_host}:{proxy_port}"
|
||||||
else:
|
else:
|
||||||
proxy_url = f"{proxy_type}://{proxy_host}:{proxy_port}"
|
full_proxy_url = f"{proxy_type}://{proxy_host}:{proxy_port}"
|
||||||
|
|
||||||
|
app_proxy_active = True
|
||||||
|
# aiohttp cannot tunnel SOCKS via the per-request `proxy=` kwarg
|
||||||
|
# (it would send HTTP to the SOCKS port and fail parsing the
|
||||||
|
# SOCKS handshake reply). SOCKS must be handled by an
|
||||||
|
# aiohttp-socks ProxyConnector instead.
|
||||||
|
if proxy_type.startswith("socks"):
|
||||||
|
socks_proxy_url = full_proxy_url
|
||||||
|
else:
|
||||||
|
proxy_url = full_proxy_url
|
||||||
|
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f"Using app-level proxy: {proxy_type}://{proxy_host}:{proxy_port}"
|
f"Using app-level proxy: {proxy_type}://{proxy_host}:{proxy_port}"
|
||||||
@@ -294,13 +306,27 @@ class Downloader:
|
|||||||
logger.debug("SSL: certifi unavailable; using system default CA bundle")
|
logger.debug("SSL: certifi unavailable; using system default CA bundle")
|
||||||
|
|
||||||
# Optimize TCP connection parameters
|
# Optimize TCP connection parameters
|
||||||
connector = aiohttp.TCPConnector(
|
connector_kwargs = dict(
|
||||||
ssl=ssl_context,
|
ssl=ssl_context,
|
||||||
limit=8, # Concurrent connections
|
limit=8, # Concurrent connections
|
||||||
ttl_dns_cache=300, # DNS cache timeout
|
ttl_dns_cache=300, # DNS cache timeout
|
||||||
force_close=False, # Keep connections for reuse
|
force_close=False, # Keep connections for reuse
|
||||||
enable_cleanup_closed=True,
|
enable_cleanup_closed=True,
|
||||||
)
|
)
|
||||||
|
if socks_proxy_url:
|
||||||
|
# Route all traffic through the SOCKS proxy via aiohttp-socks. The
|
||||||
|
# connector tunnels every connection, so no per-request `proxy=` is
|
||||||
|
# used (and must not be — see self._proxy_url below).
|
||||||
|
try:
|
||||||
|
from aiohttp_socks import ProxyConnector
|
||||||
|
except ImportError as e: # pragma: no cover
|
||||||
|
raise RuntimeError(
|
||||||
|
"A SOCKS proxy is configured but the 'aiohttp-socks' package "
|
||||||
|
"is not installed. Install it with: pip install aiohttp-socks"
|
||||||
|
) from e
|
||||||
|
connector = ProxyConnector.from_url(socks_proxy_url, **connector_kwargs)
|
||||||
|
else:
|
||||||
|
connector = aiohttp.TCPConnector(**connector_kwargs)
|
||||||
|
|
||||||
# Configure timeout parameters
|
# Configure timeout parameters
|
||||||
timeout = aiohttp.ClientTimeout(
|
timeout = aiohttp.ClientTimeout(
|
||||||
@@ -311,12 +337,14 @@ class Downloader:
|
|||||||
|
|
||||||
self._session = aiohttp.ClientSession(
|
self._session = aiohttp.ClientSession(
|
||||||
connector=connector,
|
connector=connector,
|
||||||
trust_env=proxy_url
|
# Only fall back to system/env proxy when no app-level proxy is active
|
||||||
is None, # Only use system proxy if no app-level proxy is set
|
trust_env=not app_proxy_active,
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Store proxy URL for use in requests
|
# Store proxy URL for per-request use. Stays None for SOCKS because the
|
||||||
|
# ProxyConnector already tunnels everything; passing proxy= for SOCKS
|
||||||
|
# would re-trigger the original aiohttp parse error.
|
||||||
self._proxy_url = proxy_url
|
self._proxy_url = proxy_url
|
||||||
self._session_created_at = datetime.now()
|
self._session_created_at = datetime.now()
|
||||||
|
|
||||||
|
|||||||
@@ -216,13 +216,19 @@ class MetadataSyncService:
|
|||||||
provider_used: Optional[str] = None
|
provider_used: Optional[str] = None
|
||||||
last_error: Optional[str] = None
|
last_error: Optional[str] = None
|
||||||
civitai_api_not_found = False
|
civitai_api_not_found = False
|
||||||
|
any_rate_limited = False
|
||||||
|
|
||||||
for provider_name, provider in provider_attempts:
|
for provider_name, provider in provider_attempts:
|
||||||
try:
|
try:
|
||||||
civitai_metadata_candidate, error = await provider.get_model_by_hash(sha256)
|
civitai_metadata_candidate, error = await provider.get_model_by_hash(sha256)
|
||||||
except RateLimitError as exc:
|
except RateLimitError as exc:
|
||||||
exc.provider = exc.provider or (provider_name or provider.__class__.__name__)
|
logger.warning(
|
||||||
raise
|
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||||
|
provider_name or provider.__class__.__name__,
|
||||||
|
exc.retry_after or 0,
|
||||||
|
)
|
||||||
|
any_rate_limited = True
|
||||||
|
continue
|
||||||
except Exception as exc: # pragma: no cover - defensive logging
|
except Exception as exc: # pragma: no cover - defensive logging
|
||||||
logger.error("Provider %s failed for hash %s: %s", provider_name, sha256, exc)
|
logger.error("Provider %s failed for hash %s: %s", provider_name, sha256, exc)
|
||||||
civitai_metadata_candidate, error = None, str(exc)
|
civitai_metadata_candidate, error = None, str(exc)
|
||||||
@@ -276,6 +282,8 @@ class MetadataSyncService:
|
|||||||
)
|
)
|
||||||
|
|
||||||
resolved_error = last_error or default_error
|
resolved_error = last_error or default_error
|
||||||
|
if any_rate_limited and "Rate limited" not in resolved_error:
|
||||||
|
resolved_error = "Rate limited"
|
||||||
if is_expected_offline_error(resolved_error):
|
if is_expected_offline_error(resolved_error):
|
||||||
resolved_error = OFFLINE_FRIENDLY_MESSAGE
|
resolved_error = OFFLINE_FRIENDLY_MESSAGE
|
||||||
|
|
||||||
|
|||||||
@@ -65,7 +65,14 @@ class _RateLimitRetryHelper:
|
|||||||
return await func(*args, **kwargs)
|
return await func(*args, **kwargs)
|
||||||
except RateLimitError as exc:
|
except RateLimitError as exc:
|
||||||
attempt += 1
|
attempt += 1
|
||||||
if attempt >= self._retry_limit:
|
|
||||||
|
# Determine effective retry limit based on rate-limit magnitude
|
||||||
|
effective_retry_limit = self._retry_limit # default: 3
|
||||||
|
if exc.retry_after is not None and exc.retry_after >= 120.0:
|
||||||
|
# Long rate-limit window (>=2 min) — retries are futile
|
||||||
|
effective_retry_limit = 1 # total 1 attempt = 0 retries
|
||||||
|
|
||||||
|
if attempt >= effective_retry_limit:
|
||||||
exc.provider = exc.provider or label
|
exc.provider = exc.provider or label
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@@ -81,7 +88,11 @@ class _RateLimitRetryHelper:
|
|||||||
|
|
||||||
def _calculate_delay(self, retry_after: Optional[float], attempt: int) -> float:
|
def _calculate_delay(self, retry_after: Optional[float], attempt: int) -> float:
|
||||||
if retry_after is not None:
|
if retry_after is not None:
|
||||||
return min(self._max_delay, max(0.0, retry_after))
|
# Cap at 1800s (30 min) as a safety ceiling. The old 30s cap was
|
||||||
|
# too low — CivArchive can return retry_after ~1500s, causing all
|
||||||
|
# retries to fail. A generous ceiling protects against pathological
|
||||||
|
# server values while still respecting the server's guidance.
|
||||||
|
return min(1800.0, max(0.0, retry_after))
|
||||||
|
|
||||||
base_delay = self._base_delay * (2 ** max(0, attempt - 1))
|
base_delay = self._base_delay * (2 ** max(0, attempt - 1))
|
||||||
jitter_span = base_delay * self._jitter_ratio
|
jitter_span = base_delay * self._jitter_ratio
|
||||||
@@ -474,8 +485,12 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
if result:
|
if result:
|
||||||
return result, error
|
return result, error
|
||||||
except RateLimitError as exc:
|
except RateLimitError as exc:
|
||||||
exc.provider = exc.provider or label
|
logger.warning(
|
||||||
raise exc
|
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||||
|
label,
|
||||||
|
exc.retry_after or 0,
|
||||||
|
)
|
||||||
|
continue
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug("Provider %s failed for get_model_by_hash: %s", label, e)
|
logger.debug("Provider %s failed for get_model_by_hash: %s", label, e)
|
||||||
continue
|
continue
|
||||||
@@ -493,16 +508,12 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
if result:
|
if result:
|
||||||
return result
|
return result
|
||||||
except RateLimitError as exc:
|
except RateLimitError as exc:
|
||||||
if not_found_confirmed:
|
logger.warning(
|
||||||
logger.debug(
|
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||||
"Suppressing rate limit from %s for model %s: "
|
|
||||||
"already confirmed as not found by another provider",
|
|
||||||
label,
|
label,
|
||||||
model_id,
|
exc.retry_after or 0,
|
||||||
)
|
)
|
||||||
return None
|
continue
|
||||||
exc.provider = exc.provider or label
|
|
||||||
raise exc
|
|
||||||
except ResourceNotFoundError:
|
except ResourceNotFoundError:
|
||||||
not_found_confirmed = True
|
not_found_confirmed = True
|
||||||
logger.debug(
|
logger.debug(
|
||||||
@@ -528,8 +539,12 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
if result:
|
if result:
|
||||||
return result
|
return result
|
||||||
except RateLimitError as exc:
|
except RateLimitError as exc:
|
||||||
exc.provider = exc.provider or label
|
logger.warning(
|
||||||
raise exc
|
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||||
|
label,
|
||||||
|
exc.retry_after or 0,
|
||||||
|
)
|
||||||
|
continue
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug("Provider %s failed for get_model_version: %s", label, e)
|
logger.debug("Provider %s failed for get_model_version: %s", label, e)
|
||||||
continue
|
continue
|
||||||
@@ -546,8 +561,12 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
if result:
|
if result:
|
||||||
return result, error
|
return result, error
|
||||||
except RateLimitError as exc:
|
except RateLimitError as exc:
|
||||||
exc.provider = exc.provider or label
|
logger.warning(
|
||||||
raise exc
|
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||||
|
label,
|
||||||
|
exc.retry_after or 0,
|
||||||
|
)
|
||||||
|
continue
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug("Provider %s failed for get_model_version_info: %s", label, e)
|
logger.debug("Provider %s failed for get_model_version_info: %s", label, e)
|
||||||
continue
|
continue
|
||||||
@@ -568,8 +587,12 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
except NotImplementedError:
|
except NotImplementedError:
|
||||||
continue
|
continue
|
||||||
except RateLimitError as exc:
|
except RateLimitError as exc:
|
||||||
exc.provider = exc.provider or label
|
logger.warning(
|
||||||
raise exc
|
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||||
|
label,
|
||||||
|
exc.retry_after or 0,
|
||||||
|
)
|
||||||
|
continue
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Provider %s failed for get_model_versions_by_hashes: %s",
|
"Provider %s failed for get_model_versions_by_hashes: %s",
|
||||||
@@ -590,8 +613,12 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
if result is not None:
|
if result is not None:
|
||||||
return result
|
return result
|
||||||
except RateLimitError as exc:
|
except RateLimitError as exc:
|
||||||
exc.provider = exc.provider or label
|
logger.warning(
|
||||||
raise exc
|
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||||
|
label,
|
||||||
|
exc.retry_after or 0,
|
||||||
|
)
|
||||||
|
continue
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug("Provider %s failed for get_user_models: %s", label, e)
|
logger.debug("Provider %s failed for get_user_models: %s", label, e)
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import threading
|
import threading
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass, field
|
||||||
from typing import Dict, List, Optional, Set, Tuple
|
from typing import Dict, List, Optional, Set, Tuple
|
||||||
|
|
||||||
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
||||||
@@ -26,6 +26,8 @@ class PersistedRecipeData:
|
|||||||
|
|
||||||
raw_data: List[Dict]
|
raw_data: List[Dict]
|
||||||
file_stats: Dict[str, Tuple[float, int]] # json_path -> (mtime, size)
|
file_stats: Dict[str, Tuple[float, int]] # json_path -> (mtime, size)
|
||||||
|
image_id_map: Dict[str, str] = field(default_factory=dict)
|
||||||
|
"""Precomputed mapping of civitai image_id → recipe_id."""
|
||||||
|
|
||||||
|
|
||||||
class PersistentRecipeCache:
|
class PersistentRecipeCache:
|
||||||
@@ -116,6 +118,20 @@ class PersistentRecipeCache:
|
|||||||
if not rows:
|
if not rows:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
# Restore precomputed image_id_map if available
|
||||||
|
image_id_map: Dict[str, str] = {}
|
||||||
|
try:
|
||||||
|
meta_row = conn.execute(
|
||||||
|
"SELECT value FROM cache_metadata WHERE key = ?",
|
||||||
|
("image_id_map",),
|
||||||
|
).fetchone()
|
||||||
|
if meta_row:
|
||||||
|
parsed = json.loads(meta_row["value"])
|
||||||
|
if isinstance(parsed, dict):
|
||||||
|
image_id_map = parsed
|
||||||
|
except Exception:
|
||||||
|
pass # missing or corrupt — rebuilt on next cache refresh
|
||||||
|
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
@@ -138,14 +154,24 @@ class PersistentRecipeCache:
|
|||||||
row["file_size"] or 0,
|
row["file_size"] or 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
return PersistedRecipeData(raw_data=raw_data, file_stats=file_stats)
|
return PersistedRecipeData(
|
||||||
|
raw_data=raw_data,
|
||||||
|
file_stats=file_stats,
|
||||||
|
image_id_map=image_id_map,
|
||||||
|
)
|
||||||
|
|
||||||
def save_cache(self, recipes: List[Dict], json_paths: Optional[Dict[str, str]] = None) -> None:
|
def save_cache(
|
||||||
|
self,
|
||||||
|
recipes: List[Dict],
|
||||||
|
json_paths: Optional[Dict[str, str]] = None,
|
||||||
|
image_id_map: Optional[Dict[str, str]] = None,
|
||||||
|
) -> None:
|
||||||
"""Save all recipes to SQLite cache.
|
"""Save all recipes to SQLite cache.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
recipes: List of recipe dictionaries to persist.
|
recipes: List of recipe dictionaries to persist.
|
||||||
json_paths: Optional mapping of recipe_id -> json_path for file stats.
|
json_paths: Optional mapping of recipe_id -> json_path for file stats.
|
||||||
|
image_id_map: Optional precomputed civitai image_id → recipe_id mapping.
|
||||||
"""
|
"""
|
||||||
if not self.is_enabled():
|
if not self.is_enabled():
|
||||||
return
|
return
|
||||||
@@ -186,6 +212,12 @@ class PersistentRecipeCache:
|
|||||||
recipe_rows,
|
recipe_rows,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Persist image_id_map for O(1) lookups on cache load
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR REPLACE INTO cache_metadata (key, value) VALUES (?, ?)",
|
||||||
|
("image_id_map", json.dumps(image_id_map or {})),
|
||||||
|
)
|
||||||
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
logger.debug("Persisted %d recipes to cache", len(recipe_rows))
|
logger.debug("Persisted %d recipes to cache", len(recipe_rows))
|
||||||
finally:
|
finally:
|
||||||
@@ -273,6 +305,29 @@ class PersistentRecipeCache:
|
|||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.debug("Failed to remove recipe %s from cache: %s", recipe_id, exc)
|
logger.debug("Failed to remove recipe %s from cache: %s", recipe_id, exc)
|
||||||
|
|
||||||
|
def save_image_id_map(self, image_id_map: Dict[str, str]) -> None:
|
||||||
|
"""Persist the image_id_map to cache_metadata without rewriting the full cache.
|
||||||
|
|
||||||
|
This is called after ``add_recipe`` / ``remove_recipe`` mutations so
|
||||||
|
the persistent copy does not go stale between full ``save_cache`` calls.
|
||||||
|
"""
|
||||||
|
if not self.is_enabled() or not self._schema_initialized:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self._db_lock:
|
||||||
|
conn = self._connect()
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR REPLACE INTO cache_metadata (key, value) VALUES (?, ?)",
|
||||||
|
("image_id_map", json.dumps(image_id_map)),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("Failed to persist image_id_map: %s", exc)
|
||||||
|
|
||||||
def get_indexed_recipe_ids(self) -> Set[str]:
|
def get_indexed_recipe_ids(self) -> Set[str]:
|
||||||
"""Return all recipe IDs in the cache.
|
"""Return all recipe IDs in the cache.
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
from typing import Iterable, List, Dict, Optional
|
from typing import Iterable, List, Dict, Optional
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass, field
|
||||||
from operator import itemgetter
|
from operator import itemgetter
|
||||||
from natsort import natsorted
|
from natsort import natsorted
|
||||||
|
|
||||||
@@ -14,6 +14,15 @@ class RecipeCache:
|
|||||||
sorted_by_date: List[Dict]
|
sorted_by_date: List[Dict]
|
||||||
folders: List[str] | None = None
|
folders: List[str] | None = None
|
||||||
folder_tree: Dict | None = None
|
folder_tree: Dict | None = None
|
||||||
|
image_id_map: Dict[str, str] = field(default_factory=dict)
|
||||||
|
"""Mapping of civitai image_id → recipe_id, precomputed at cache build time.
|
||||||
|
|
||||||
|
Built once during cache initialization (O(n)) so that
|
||||||
|
``check_image_exists`` and ``import_from_url`` duplicate checks
|
||||||
|
can look up image_id in O(1) instead of scanning all recipes.
|
||||||
|
Recipes imported from local files have no valid civitai image_id
|
||||||
|
and are naturally excluded from this map.
|
||||||
|
"""
|
||||||
|
|
||||||
def __post_init__(self):
|
def __post_init__(self):
|
||||||
self._lock = asyncio.Lock()
|
self._lock = asyncio.Lock()
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from .metadata_service import get_default_metadata_provider
|
|||||||
from .checkpoint_scanner import CheckpointScanner
|
from .checkpoint_scanner import CheckpointScanner
|
||||||
from .settings_manager import get_settings_manager
|
from .settings_manager import get_settings_manager
|
||||||
from .recipes.errors import RecipeNotFoundError
|
from .recipes.errors import RecipeNotFoundError
|
||||||
|
from ..utils.civitai_utils import extract_civitai_image_id
|
||||||
from ..utils.utils import calculate_recipe_fingerprint, fuzzy_match
|
from ..utils.utils import calculate_recipe_fingerprint, fuzzy_match
|
||||||
from natsort import natsorted
|
from natsort import natsorted
|
||||||
import sys
|
import sys
|
||||||
@@ -532,7 +533,21 @@ class RecipeScanner:
|
|||||||
self._sort_cache_sync()
|
self._sort_cache_sync()
|
||||||
# Backfill source_path from JSON files if missing (schema migration)
|
# Backfill source_path from JSON files if missing (schema migration)
|
||||||
if self._backfill_source_path_if_needed(recipes, json_paths):
|
if self._backfill_source_path_if_needed(recipes, json_paths):
|
||||||
self._persistent_cache.save_cache(recipes, json_paths)
|
self._cache.image_id_map = self._build_image_id_map()
|
||||||
|
self._persistent_cache.save_cache(
|
||||||
|
recipes, json_paths, self._cache.image_id_map
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# Use persisted map, or rebuild if empty (e.g. first startup
|
||||||
|
# after deploying the image_id_map feature).
|
||||||
|
if persisted.image_id_map:
|
||||||
|
self._cache.image_id_map = dict(persisted.image_id_map)
|
||||||
|
else:
|
||||||
|
self._cache.image_id_map = self._build_image_id_map()
|
||||||
|
if self._cache.image_id_map:
|
||||||
|
self._persistent_cache.save_image_id_map(
|
||||||
|
self._cache.image_id_map
|
||||||
|
)
|
||||||
return self._cache
|
return self._cache
|
||||||
else:
|
else:
|
||||||
# Partial update: some files changed
|
# Partial update: some files changed
|
||||||
@@ -545,8 +560,11 @@ class RecipeScanner:
|
|||||||
self._sort_cache_sync()
|
self._sort_cache_sync()
|
||||||
# Backfill source_path from JSON files if missing (schema migration)
|
# Backfill source_path from JSON files if missing (schema migration)
|
||||||
self._backfill_source_path_if_needed(recipes, json_paths)
|
self._backfill_source_path_if_needed(recipes, json_paths)
|
||||||
|
self._cache.image_id_map = self._build_image_id_map()
|
||||||
# Persist updated cache
|
# Persist updated cache
|
||||||
self._persistent_cache.save_cache(recipes, json_paths)
|
self._persistent_cache.save_cache(
|
||||||
|
recipes, json_paths, self._cache.image_id_map
|
||||||
|
)
|
||||||
return self._cache
|
return self._cache
|
||||||
|
|
||||||
# Fall back to full directory scan
|
# Fall back to full directory scan
|
||||||
@@ -558,9 +576,12 @@ class RecipeScanner:
|
|||||||
self._cache.raw_data = recipes
|
self._cache.raw_data = recipes
|
||||||
self._update_folder_metadata(self._cache)
|
self._update_folder_metadata(self._cache)
|
||||||
self._sort_cache_sync()
|
self._sort_cache_sync()
|
||||||
|
self._cache.image_id_map = self._build_image_id_map()
|
||||||
|
|
||||||
# Persist for next startup
|
# Persist for next startup
|
||||||
self._persistent_cache.save_cache(recipes, json_paths)
|
self._persistent_cache.save_cache(
|
||||||
|
recipes, json_paths, self._cache.image_id_map
|
||||||
|
)
|
||||||
|
|
||||||
return self._cache
|
return self._cache
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -832,6 +853,28 @@ class RecipeScanner:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error sorting recipe cache: {e}")
|
logger.error(f"Error sorting recipe cache: {e}")
|
||||||
|
|
||||||
|
def _build_image_id_map(self) -> Dict[str, str]:
|
||||||
|
"""Build civitai image_id → recipe_id mapping from cached recipes.
|
||||||
|
|
||||||
|
Only recipes with a valid CivitAI image URL source_path produce an
|
||||||
|
entry. Recipes imported from local files are naturally excluded.
|
||||||
|
"""
|
||||||
|
mapping: Dict[str, str] = {}
|
||||||
|
if not self._cache:
|
||||||
|
return mapping
|
||||||
|
for recipe in getattr(self._cache, "raw_data", []):
|
||||||
|
if not isinstance(recipe, dict):
|
||||||
|
continue
|
||||||
|
source = recipe.get("source_path")
|
||||||
|
if not source:
|
||||||
|
continue
|
||||||
|
image_id = extract_civitai_image_id(source)
|
||||||
|
if image_id and image_id not in mapping:
|
||||||
|
recipe_id = recipe.get("id")
|
||||||
|
if recipe_id is not None:
|
||||||
|
mapping[image_id] = str(recipe_id)
|
||||||
|
return mapping
|
||||||
|
|
||||||
async def _wait_for_lora_scanner(self) -> None:
|
async def _wait_for_lora_scanner(self) -> None:
|
||||||
"""Ensure the LoRA scanner has initialized before recipe enrichment."""
|
"""Ensure the LoRA scanner has initialized before recipe enrichment."""
|
||||||
|
|
||||||
@@ -1296,11 +1339,20 @@ class RecipeScanner:
|
|||||||
# Update FTS index
|
# Update FTS index
|
||||||
self._update_fts_index_for_recipe(recipe_data, "add")
|
self._update_fts_index_for_recipe(recipe_data, "add")
|
||||||
|
|
||||||
|
source = recipe_data.get("source_path")
|
||||||
|
if source:
|
||||||
|
image_id = extract_civitai_image_id(source)
|
||||||
|
if image_id:
|
||||||
|
recipe_id_value = recipe_data.get("id")
|
||||||
|
if recipe_id_value is not None:
|
||||||
|
cache.image_id_map[image_id] = str(recipe_id_value)
|
||||||
|
|
||||||
# Persist to SQLite cache
|
# Persist to SQLite cache
|
||||||
if self._persistent_cache:
|
if self._persistent_cache:
|
||||||
recipe_id = str(recipe_data.get("id", ""))
|
recipe_id = str(recipe_data.get("id", ""))
|
||||||
json_path = self._json_path_map.get(recipe_id, "")
|
json_path = self._json_path_map.get(recipe_id, "")
|
||||||
self._persistent_cache.update_recipe(recipe_data, json_path)
|
self._persistent_cache.update_recipe(recipe_data, json_path)
|
||||||
|
self._persistent_cache.save_image_id_map(cache.image_id_map)
|
||||||
|
|
||||||
async def remove_recipe(self, recipe_id: str) -> bool:
|
async def remove_recipe(self, recipe_id: str) -> bool:
|
||||||
"""Remove a recipe from the cache by ID."""
|
"""Remove a recipe from the cache by ID."""
|
||||||
@@ -1319,9 +1371,15 @@ class RecipeScanner:
|
|||||||
# Update FTS index
|
# Update FTS index
|
||||||
self._update_fts_index_for_recipe(recipe_id, "remove")
|
self._update_fts_index_for_recipe(recipe_id, "remove")
|
||||||
|
|
||||||
|
# Remove any image_id entry pointing to this recipe
|
||||||
|
stale = [k for k, v in cache.image_id_map.items() if v == recipe_id]
|
||||||
|
for k in stale:
|
||||||
|
del cache.image_id_map[k]
|
||||||
|
|
||||||
# Remove from SQLite cache
|
# Remove from SQLite cache
|
||||||
if self._persistent_cache:
|
if self._persistent_cache:
|
||||||
self._persistent_cache.remove_recipe(recipe_id)
|
self._persistent_cache.remove_recipe(recipe_id)
|
||||||
|
self._persistent_cache.save_image_id_map(cache.image_id_map)
|
||||||
self._json_path_map.pop(recipe_id, None)
|
self._json_path_map.pop(recipe_id, None)
|
||||||
|
|
||||||
return True
|
return True
|
||||||
@@ -1332,14 +1390,21 @@ class RecipeScanner:
|
|||||||
cache = await self.get_cached_data()
|
cache = await self.get_cached_data()
|
||||||
removed = await cache.bulk_remove(recipe_ids, resort=False)
|
removed = await cache.bulk_remove(recipe_ids, resort=False)
|
||||||
if removed:
|
if removed:
|
||||||
|
removed_ids = {str(r.get("id", "")) for r in removed}
|
||||||
|
stale = [k for k, v in cache.image_id_map.items() if v in removed_ids]
|
||||||
|
for k in stale:
|
||||||
|
del cache.image_id_map[k]
|
||||||
|
|
||||||
self._schedule_resort()
|
self._schedule_resort()
|
||||||
# Update FTS index and persistent cache for each removed recipe
|
|
||||||
for recipe in removed:
|
for recipe in removed:
|
||||||
recipe_id = str(recipe.get("id", ""))
|
recipe_id = str(recipe.get("id", ""))
|
||||||
self._update_fts_index_for_recipe(recipe_id, "remove")
|
self._update_fts_index_for_recipe(recipe_id, "remove")
|
||||||
if self._persistent_cache:
|
if self._persistent_cache:
|
||||||
self._persistent_cache.remove_recipe(recipe_id)
|
self._persistent_cache.remove_recipe(recipe_id)
|
||||||
self._json_path_map.pop(recipe_id, None)
|
self._json_path_map.pop(recipe_id, None)
|
||||||
|
|
||||||
|
if self._persistent_cache:
|
||||||
|
self._persistent_cache.save_image_id_map(cache.image_id_map)
|
||||||
return len(removed)
|
return len(removed)
|
||||||
|
|
||||||
async def scan_all_recipes(self) -> List[Dict]:
|
async def scan_all_recipes(self) -> List[Dict]:
|
||||||
|
|||||||
@@ -77,6 +77,9 @@ class BulkMetadataRefreshUseCase:
|
|||||||
|
|
||||||
await emit("started")
|
await emit("started")
|
||||||
|
|
||||||
|
RATE_LIMIT_ABORT_THRESHOLD = 3
|
||||||
|
consecutive_rate_limits = 0
|
||||||
|
|
||||||
for model in to_process:
|
for model in to_process:
|
||||||
if self._service.scanner.is_cancelled():
|
if self._service.scanner.is_cancelled():
|
||||||
self._logger.info("Bulk metadata refresh cancelled by user")
|
self._logger.info("Bulk metadata refresh cancelled by user")
|
||||||
@@ -115,12 +118,39 @@ class BulkMetadataRefreshUseCase:
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
await MetadataManager.hydrate_model_data(model)
|
await MetadataManager.hydrate_model_data(model)
|
||||||
result, _ = await self._metadata_sync.fetch_and_update_model(
|
result, error_msg = await self._metadata_sync.fetch_and_update_model(
|
||||||
sha256=model["sha256"],
|
sha256=model["sha256"],
|
||||||
file_path=model["file_path"],
|
file_path=model["file_path"],
|
||||||
model_data=model,
|
model_data=model,
|
||||||
update_cache_func=self._service.scanner.update_single_model_cache,
|
update_cache_func=self._service.scanner.update_single_model_cache,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if not result and error_msg and "Rate limited" in error_msg:
|
||||||
|
consecutive_rate_limits += 1
|
||||||
|
else:
|
||||||
|
consecutive_rate_limits = 0
|
||||||
|
|
||||||
|
if consecutive_rate_limits >= RATE_LIMIT_ABORT_THRESHOLD:
|
||||||
|
self._logger.warning(
|
||||||
|
"Bulk metadata refresh aborted: %d consecutive rate limits detected. "
|
||||||
|
"Processed %d/%d models.",
|
||||||
|
consecutive_rate_limits,
|
||||||
|
processed,
|
||||||
|
total_to_process,
|
||||||
|
)
|
||||||
|
await emit(
|
||||||
|
"rate_limited",
|
||||||
|
processed=processed,
|
||||||
|
success=success,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"success": False,
|
||||||
|
"message": f"Rate limit detected; {total_to_process - processed} models skipped",
|
||||||
|
"processed": processed,
|
||||||
|
"updated": success,
|
||||||
|
"total": total_models,
|
||||||
|
}
|
||||||
|
|
||||||
if result:
|
if result:
|
||||||
success += 1
|
success += 1
|
||||||
if original_name != model.get("model_name"):
|
if original_name != model.get("model_name"):
|
||||||
|
|||||||
@@ -31,6 +31,8 @@ PREVIEW_EXTENSIONS = [
|
|||||||
".mp4",
|
".mp4",
|
||||||
".gif",
|
".gif",
|
||||||
".webm",
|
".webm",
|
||||||
|
".avif",
|
||||||
|
".jxl",
|
||||||
]
|
]
|
||||||
|
|
||||||
# Card preview image width
|
# Card preview image width
|
||||||
@@ -41,7 +43,7 @@ EXAMPLE_IMAGE_WIDTH = 832
|
|||||||
|
|
||||||
# Supported media extensions for example downloads
|
# Supported media extensions for example downloads
|
||||||
SUPPORTED_MEDIA_EXTENSIONS = {
|
SUPPORTED_MEDIA_EXTENSIONS = {
|
||||||
"images": [".jpg", ".jpeg", ".png", ".webp", ".gif"],
|
"images": [".jpg", ".jpeg", ".png", ".webp", ".gif", ".avif", ".jxl"],
|
||||||
"videos": [".mp4", ".webm"],
|
"videos": [".mp4", ".webm"],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -62,6 +62,10 @@ class ExampleImagesProcessor:
|
|||||||
return '.gif'
|
return '.gif'
|
||||||
elif content.startswith(b'RIFF') and b'WEBP' in content[:12]:
|
elif content.startswith(b'RIFF') and b'WEBP' in content[:12]:
|
||||||
return '.webp'
|
return '.webp'
|
||||||
|
elif len(content) >= 12 and content[4:8] == b'ftyp' and b'avif' in content[8:24]:
|
||||||
|
return '.avif'
|
||||||
|
elif content.startswith(b'\x00\x00\x00\x0cJXL \x0d\x0a\x87\x0a'):
|
||||||
|
return '.jxl'
|
||||||
elif content.startswith(b'\x00\x00\x00\x18ftypmp4') or content.startswith(b'\x00\x00\x00\x20ftypmp4'):
|
elif content.startswith(b'\x00\x00\x00\x18ftypmp4') or content.startswith(b'\x00\x00\x00\x20ftypmp4'):
|
||||||
return '.mp4'
|
return '.mp4'
|
||||||
elif content.startswith(b'\x1A\x45\xDF\xA3'):
|
elif content.startswith(b'\x1A\x45\xDF\xA3'):
|
||||||
@@ -75,6 +79,8 @@ class ExampleImagesProcessor:
|
|||||||
'image/png': '.png',
|
'image/png': '.png',
|
||||||
'image/gif': '.gif',
|
'image/gif': '.gif',
|
||||||
'image/webp': '.webp',
|
'image/webp': '.webp',
|
||||||
|
'image/avif': '.avif',
|
||||||
|
'image/jxl': '.jxl',
|
||||||
'video/mp4': '.mp4',
|
'video/mp4': '.mp4',
|
||||||
'video/webm': '.webm',
|
'video/webm': '.webm',
|
||||||
'video/quicktime': '.mov'
|
'video/quicktime': '.mov'
|
||||||
|
|||||||
@@ -1,17 +1,125 @@
|
|||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import struct
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
|
|
||||||
import piexif
|
import piexif
|
||||||
from PIL import Image, PngImagePlugin
|
from PIL import Image, PngImagePlugin
|
||||||
|
|
||||||
|
try:
|
||||||
|
import brotli
|
||||||
|
_BROTLI_AVAILABLE = True
|
||||||
|
except ImportError:
|
||||||
|
brotli = None
|
||||||
|
_BROTLI_AVAILABLE = False
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
class ExifUtils:
|
class ExifUtils:
|
||||||
"""Utility functions for working with EXIF data in images"""
|
"""Utility functions for working with EXIF data in images"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _parse_isobmff_boxes(data: bytes, offset: int = 0) -> list[dict]:
|
||||||
|
boxes = []
|
||||||
|
while offset + 8 <= len(data):
|
||||||
|
size = struct.unpack('>I', data[offset:offset + 4])[0]
|
||||||
|
box_type = data[offset + 4:offset + 8]
|
||||||
|
if size == 0:
|
||||||
|
break
|
||||||
|
if size < 8 or offset + size > len(data):
|
||||||
|
break
|
||||||
|
box_data = data[offset + 8:offset + size]
|
||||||
|
boxes.append({'type': box_type, 'data': box_data, 'size': size})
|
||||||
|
offset += size
|
||||||
|
return boxes
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_jxl_container(data: bytes) -> bool:
|
||||||
|
if len(data) < 32:
|
||||||
|
return False
|
||||||
|
return (
|
||||||
|
struct.unpack('>I', data[:4])[0] == 12
|
||||||
|
and data[4:8] == b'JXL '
|
||||||
|
and data[8:12] == bytes([0x0d, 0x0a, 0x87, 0x0a])
|
||||||
|
and struct.unpack('>I', data[12:16])[0] >= 16
|
||||||
|
and data[16:20] == b'ftyp'
|
||||||
|
and data[20:24] == b'jxl '
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_avif_container(data: bytes) -> bool:
|
||||||
|
if len(data) < 16:
|
||||||
|
return False
|
||||||
|
for box in ExifUtils._parse_isobmff_boxes(data):
|
||||||
|
if box['type'] == b'ftyp' and b'avif' in box['data']:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Max decompressed size for brotli metadata (2 MB)
|
||||||
|
_BROTLI_MAX_DECOMPRESSED = 2 * 1024 * 1024
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _extract_isobmff_brotli(image_path: str) -> Optional[dict]:
|
||||||
|
try:
|
||||||
|
with open(image_path, 'rb') as f:
|
||||||
|
data = f.read()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if ExifUtils._is_jxl_container(data):
|
||||||
|
boxes = ExifUtils._parse_isobmff_boxes(data, offset=12)
|
||||||
|
elif ExifUtils._is_avif_container(data):
|
||||||
|
boxes = ExifUtils._parse_isobmff_boxes(data)
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
brob = None
|
||||||
|
for box in boxes:
|
||||||
|
if box['type'] == b'brob':
|
||||||
|
brob = box
|
||||||
|
break
|
||||||
|
if brob is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
payload = brob['data']
|
||||||
|
if payload[:4] != b'comf':
|
||||||
|
return None
|
||||||
|
compressed = payload[4:]
|
||||||
|
|
||||||
|
if _BROTLI_AVAILABLE:
|
||||||
|
try:
|
||||||
|
decompressed = brotli.decompress(compressed)
|
||||||
|
if len(decompressed) > ExifUtils._BROTLI_MAX_DECOMPRESSED:
|
||||||
|
logger.warning(
|
||||||
|
"Brotli metadata too large (%d bytes, max %d), ignoring",
|
||||||
|
len(decompressed),
|
||||||
|
ExifUtils._BROTLI_MAX_DECOMPRESSED,
|
||||||
|
)
|
||||||
|
decompressed = None
|
||||||
|
except Exception:
|
||||||
|
decompressed = None
|
||||||
|
else:
|
||||||
|
decompressed = None
|
||||||
|
|
||||||
|
raw = decompressed if decompressed is not None else compressed
|
||||||
|
try:
|
||||||
|
meta = json.loads(raw.decode('utf-8'))
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
result = {"parameters": None, "prompt": None, "workflow": None, "comment": None}
|
||||||
|
if isinstance(meta.get("prompt"), (dict, list)):
|
||||||
|
result["prompt"] = json.dumps(meta["prompt"])
|
||||||
|
elif isinstance(meta.get("prompt"), str):
|
||||||
|
result["prompt"] = meta["prompt"]
|
||||||
|
if isinstance(meta.get("workflow"), (dict, list)):
|
||||||
|
result["workflow"] = json.dumps(meta["workflow"])
|
||||||
|
elif isinstance(meta.get("workflow"), str):
|
||||||
|
result["workflow"] = meta["workflow"]
|
||||||
|
return result
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _decode_user_comment(user_comment: Any) -> Optional[str]:
|
def _decode_user_comment(user_comment: Any) -> Optional[str]:
|
||||||
if user_comment is None:
|
if user_comment is None:
|
||||||
@@ -43,6 +151,12 @@ class ExifUtils:
|
|||||||
"comment": None,
|
"comment": None,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ext = os.path.splitext(image_path)[1].lower()
|
||||||
|
if ext in ('.avif', '.jxl'):
|
||||||
|
brotli_meta = ExifUtils._extract_isobmff_brotli(image_path)
|
||||||
|
if brotli_meta:
|
||||||
|
return brotli_meta
|
||||||
|
|
||||||
with Image.open(image_path) as img:
|
with Image.open(image_path) as img:
|
||||||
info = getattr(img, "info", {}) or {}
|
info = getattr(img, "info", {}) or {}
|
||||||
|
|
||||||
@@ -149,7 +263,6 @@ class ExifUtils:
|
|||||||
Optional[str]: Extracted metadata or None if not found
|
Optional[str]: Extracted metadata or None if not found
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
# Skip for video files
|
|
||||||
if image_path:
|
if image_path:
|
||||||
ext = os.path.splitext(image_path)[1].lower()
|
ext = os.path.splitext(image_path)[1].lower()
|
||||||
if ext in ['.mp4', '.webm']:
|
if ext in ['.mp4', '.webm']:
|
||||||
@@ -177,10 +290,9 @@ class ExifUtils:
|
|||||||
str: Path to the updated image
|
str: Path to the updated image
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
# Skip for video files
|
|
||||||
if image_path:
|
if image_path:
|
||||||
ext = os.path.splitext(image_path)[1].lower()
|
ext = os.path.splitext(image_path)[1].lower()
|
||||||
if ext in ['.mp4', '.webm']:
|
if ext in ['.mp4', '.webm', '.avif', '.jxl']:
|
||||||
return image_path
|
return image_path
|
||||||
|
|
||||||
metadata_fields = ExifUtils._load_structured_metadata(image_path)
|
metadata_fields = ExifUtils._load_structured_metadata(image_path)
|
||||||
@@ -212,10 +324,9 @@ class ExifUtils:
|
|||||||
def append_recipe_metadata(image_path, recipe_data) -> str:
|
def append_recipe_metadata(image_path, recipe_data) -> str:
|
||||||
"""Append recipe metadata to an image's EXIF data"""
|
"""Append recipe metadata to an image's EXIF data"""
|
||||||
try:
|
try:
|
||||||
# Skip for video files
|
|
||||||
if image_path:
|
if image_path:
|
||||||
ext = os.path.splitext(image_path)[1].lower()
|
ext = os.path.splitext(image_path)[1].lower()
|
||||||
if ext in ['.mp4', '.webm']:
|
if ext in ['.mp4', '.webm', '.avif', '.jxl']:
|
||||||
return image_path
|
return image_path
|
||||||
|
|
||||||
# First, extract existing metadata
|
# First, extract existing metadata
|
||||||
@@ -327,10 +438,9 @@ class ExifUtils:
|
|||||||
Tuple of (optimized_image_data, extension)
|
Tuple of (optimized_image_data, extension)
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
# Skip for video files early if it's a file path
|
|
||||||
if isinstance(image_data, str) and os.path.exists(image_data):
|
if isinstance(image_data, str) and os.path.exists(image_data):
|
||||||
ext = os.path.splitext(image_data)[1].lower()
|
ext = os.path.splitext(image_data)[1].lower()
|
||||||
if ext in ['.mp4', '.webm']:
|
if ext in ['.mp4', '.webm', '.avif', '.jxl']:
|
||||||
try:
|
try:
|
||||||
with open(image_data, 'rb') as f:
|
with open(image_data, 'rb') as f:
|
||||||
return f.read(), ext
|
return f.read(), ext
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "comfyui-lora-manager"
|
name = "comfyui-lora-manager"
|
||||||
description = "Revolutionize your workflow with the ultimate LoRA companion for ComfyUI!"
|
description = "Revolutionize your workflow with the ultimate LoRA companion for ComfyUI!"
|
||||||
version = "1.1.0"
|
version = "1.1.1"
|
||||||
license = {file = "LICENSE"}
|
license = {file = "LICENSE"}
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aiohttp",
|
"aiohttp",
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
aiohttp
|
aiohttp
|
||||||
|
aiohttp-socks
|
||||||
jinja2
|
jinja2
|
||||||
safetensors
|
safetensors
|
||||||
piexif
|
piexif
|
||||||
@@ -12,3 +13,5 @@ aiosqlite
|
|||||||
beautifulsoup4
|
beautifulsoup4
|
||||||
platformdirs
|
platformdirs
|
||||||
pyyaml
|
pyyaml
|
||||||
|
# brotli — ISOBMFF (AVIF/JXL) metadata decompression
|
||||||
|
brotli>=1.2.0
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import os
|
|||||||
import sys
|
import sys
|
||||||
import json
|
import json
|
||||||
from py.middleware.cache_middleware import cache_control
|
from py.middleware.cache_middleware import cache_control
|
||||||
|
from py.middleware.error_middleware import api_json_error
|
||||||
from py.utils.settings_paths import ensure_settings_file
|
from py.utils.settings_paths import ensure_settings_file
|
||||||
|
|
||||||
# Set environment variable to indicate standalone mode
|
# Set environment variable to indicate standalone mode
|
||||||
@@ -157,7 +158,7 @@ class StandaloneServer:
|
|||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.app = web.Application(
|
self.app = web.Application(
|
||||||
logger=logger,
|
logger=logger,
|
||||||
middlewares=[cache_control],
|
middlewares=[api_json_error, cache_control],
|
||||||
client_max_size=256 * 1024 * 1024,
|
client_max_size=256 * 1024 * 1024,
|
||||||
handler_args={
|
handler_args={
|
||||||
"max_field_size": HEADER_SIZE_LIMIT,
|
"max_field_size": HEADER_SIZE_LIMIT,
|
||||||
|
|||||||
@@ -278,7 +278,7 @@
|
|||||||
left: 0;
|
left: 0;
|
||||||
right: 0;
|
right: 0;
|
||||||
background: linear-gradient(transparent 15%, oklch(0% 0 0 / 0.75));
|
background: linear-gradient(transparent 15%, oklch(0% 0 0 / 0.75));
|
||||||
backdrop-filter: blur(8px);
|
backdrop-filter: blur(var(--card-blur-amount, 8px));
|
||||||
color: white;
|
color: white;
|
||||||
padding: var(--space-1);
|
padding: var(--space-1);
|
||||||
display: flex;
|
display: flex;
|
||||||
@@ -294,7 +294,7 @@
|
|||||||
left: 0;
|
left: 0;
|
||||||
right: 0;
|
right: 0;
|
||||||
background: linear-gradient(oklch(0% 0 0 / 0.75), transparent 85%);
|
background: linear-gradient(oklch(0% 0 0 / 0.75), transparent 85%);
|
||||||
backdrop-filter: blur(8px);
|
backdrop-filter: blur(var(--card-blur-amount, 8px));
|
||||||
color: white;
|
color: white;
|
||||||
padding: var(--space-1);
|
padding: var(--space-1);
|
||||||
display: flex;
|
display: flex;
|
||||||
|
|||||||
@@ -813,6 +813,67 @@
|
|||||||
outline: none;
|
outline: none;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Range Slider Control */
|
||||||
|
.range-control {
|
||||||
|
width: 100%;
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 10px;
|
||||||
|
justify-content: flex-end;
|
||||||
|
}
|
||||||
|
|
||||||
|
.range-control input[type="range"] {
|
||||||
|
width: 120px;
|
||||||
|
height: 4px;
|
||||||
|
-webkit-appearance: none;
|
||||||
|
appearance: none;
|
||||||
|
background: var(--border-color);
|
||||||
|
border-radius: 2px;
|
||||||
|
outline: none;
|
||||||
|
cursor: pointer;
|
||||||
|
flex-shrink: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.range-control input[type="range"]::-webkit-slider-thumb {
|
||||||
|
-webkit-appearance: none;
|
||||||
|
appearance: none;
|
||||||
|
width: 16px;
|
||||||
|
height: 16px;
|
||||||
|
border-radius: 50%;
|
||||||
|
background: var(--lora-accent);
|
||||||
|
cursor: pointer;
|
||||||
|
border: 2px solid var(--lora-surface);
|
||||||
|
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.3);
|
||||||
|
transition: transform 0.15s ease;
|
||||||
|
}
|
||||||
|
|
||||||
|
.range-control input[type="range"]::-webkit-slider-thumb:hover {
|
||||||
|
transform: scale(1.15);
|
||||||
|
}
|
||||||
|
|
||||||
|
.range-control input[type="range"]::-moz-range-thumb {
|
||||||
|
width: 16px;
|
||||||
|
height: 16px;
|
||||||
|
border-radius: 50%;
|
||||||
|
background: var(--lora-accent);
|
||||||
|
cursor: pointer;
|
||||||
|
border: 2px solid var(--lora-surface);
|
||||||
|
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.3);
|
||||||
|
}
|
||||||
|
|
||||||
|
.range-control .range-value {
|
||||||
|
min-width: 36px;
|
||||||
|
text-align: center;
|
||||||
|
font-size: 0.9em;
|
||||||
|
font-weight: 600;
|
||||||
|
color: var(--text-color);
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
}
|
||||||
|
|
||||||
|
[data-theme="dark"] .range-control input[type="range"] {
|
||||||
|
background: rgba(255, 255, 255, 0.15);
|
||||||
|
}
|
||||||
|
|
||||||
/* Toggle Switch */
|
/* Toggle Switch */
|
||||||
.toggle-switch {
|
.toggle-switch {
|
||||||
position: relative;
|
position: relative;
|
||||||
|
|||||||
@@ -355,9 +355,9 @@ function renderImportInterface(isEmpty) {
|
|||||||
<button class="select-files-btn" id="selectExampleFilesBtn">
|
<button class="select-files-btn" id="selectExampleFilesBtn">
|
||||||
<i class="fas fa-folder-open"></i> Select Files
|
<i class="fas fa-folder-open"></i> Select Files
|
||||||
</button>
|
</button>
|
||||||
<p class="import-formats">Supported formats: jpg, png, gif, webp, mp4, webm</p>
|
<p class="import-formats">Supported formats: jpg, png, gif, webp, avif, jxl, mp4, webm</p>
|
||||||
</div>
|
</div>
|
||||||
<input type="file" id="exampleFilesInput" multiple accept="image/*,video/mp4,video/webm" style="display: none;">
|
<input type="file" id="exampleFilesInput" multiple accept="image/*,image/avif,image/jxl,video/mp4,video/webm" style="display: none;">
|
||||||
<div class="import-progress-container" style="display: none;">
|
<div class="import-progress-container" style="display: none;">
|
||||||
<div class="import-progress">
|
<div class="import-progress">
|
||||||
<div class="progress-bar"></div>
|
<div class="progress-bar"></div>
|
||||||
@@ -473,7 +473,7 @@ export function initExampleImport(modelHash, container) {
|
|||||||
*/
|
*/
|
||||||
async function handleImportFiles(files, modelHash, importContainer) {
|
async function handleImportFiles(files, modelHash, importContainer) {
|
||||||
// Filter for supported file types
|
// Filter for supported file types
|
||||||
const supportedImages = ['.jpg', '.jpeg', '.png', '.gif', '.webp'];
|
const supportedImages = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.avif', '.jxl'];
|
||||||
const supportedVideos = ['.mp4', '.webm'];
|
const supportedVideos = ['.mp4', '.webm'];
|
||||||
const supportedExtensions = [...supportedImages, ...supportedVideos];
|
const supportedExtensions = [...supportedImages, ...supportedVideos];
|
||||||
|
|
||||||
|
|||||||
@@ -804,6 +804,16 @@ export class SettingsManager {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Set card blur amount slider
|
||||||
|
const cardBlurAmountInput = document.getElementById('cardBlurAmount');
|
||||||
|
if (cardBlurAmountInput) {
|
||||||
|
cardBlurAmountInput.value = state.global.settings.card_blur_amount ?? 8;
|
||||||
|
}
|
||||||
|
const cardBlurAmountValue = document.getElementById('cardBlurAmountValue');
|
||||||
|
if (cardBlurAmountValue) {
|
||||||
|
cardBlurAmountValue.textContent = `${state.global.settings.card_blur_amount ?? 8}px`;
|
||||||
|
}
|
||||||
|
|
||||||
const usePortableCheckbox = document.getElementById('usePortableSettings');
|
const usePortableCheckbox = document.getElementById('usePortableSettings');
|
||||||
if (usePortableCheckbox) {
|
if (usePortableCheckbox) {
|
||||||
usePortableCheckbox.checked = !!state.global.settings.use_portable_settings;
|
usePortableCheckbox.checked = !!state.global.settings.use_portable_settings;
|
||||||
@@ -2051,6 +2061,28 @@ export class SettingsManager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async saveRangeSetting(elementId, displayId, settingKey) {
|
||||||
|
const element = document.getElementById(elementId);
|
||||||
|
if (!element) return;
|
||||||
|
|
||||||
|
const value = parseInt(element.value, 10);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.saveSetting(settingKey, value);
|
||||||
|
this.applyFrontendSettings();
|
||||||
|
|
||||||
|
// Update the displayed value next to the slider
|
||||||
|
const displayEl = document.getElementById(displayId);
|
||||||
|
if (displayEl) {
|
||||||
|
displayEl.textContent = `${value}px`;
|
||||||
|
}
|
||||||
|
|
||||||
|
showToast('toast.settings.settingsUpdated', { setting: settingKey.replace(/_/g, ' ') }, 'success');
|
||||||
|
} catch (error) {
|
||||||
|
showToast('toast.settings.settingSaveFailed', { message: error.message }, 'error');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
updateExampleImagesOpenSettingsVisibility() {
|
updateExampleImagesOpenSettingsVisibility() {
|
||||||
const openMode = state.global.settings.example_images_open_mode || 'system';
|
const openMode = state.global.settings.example_images_open_mode || 'system';
|
||||||
const localRootSetting = document.getElementById('exampleImagesLocalRootSetting');
|
const localRootSetting = document.getElementById('exampleImagesLocalRootSetting');
|
||||||
@@ -2887,6 +2919,10 @@ export class SettingsManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
applyFrontendSettings() {
|
applyFrontendSettings() {
|
||||||
|
// Apply card blur amount to CSS custom property
|
||||||
|
const cardBlurAmount = state.global.settings.card_blur_amount ?? 8;
|
||||||
|
document.documentElement.style.setProperty('--card-blur-amount', `${cardBlurAmount}px`);
|
||||||
|
|
||||||
// Apply autoplay setting to existing videos in card previews
|
// Apply autoplay setting to existing videos in card previews
|
||||||
const autoplayOnHover = state.global.settings.autoplay_on_hover;
|
const autoplayOnHover = state.global.settings.autoplay_on_hover;
|
||||||
document.querySelectorAll('.card-preview video').forEach(video => {
|
document.querySelectorAll('.card-preview video').forEach(video => {
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ const DEFAULT_SETTINGS_BASE = Object.freeze({
|
|||||||
auto_download_example_images: false,
|
auto_download_example_images: false,
|
||||||
blur_mature_content: true,
|
blur_mature_content: true,
|
||||||
mature_blur_level: 'R',
|
mature_blur_level: 'R',
|
||||||
|
card_blur_amount: 8,
|
||||||
autoplay_on_hover: false,
|
autoplay_on_hover: false,
|
||||||
display_density: 'default',
|
display_density: 'default',
|
||||||
card_info_display: 'always',
|
card_info_display: 'always',
|
||||||
|
|||||||
@@ -448,6 +448,7 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Video Settings -->
|
<!-- Video Settings -->
|
||||||
@@ -556,6 +557,23 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="setting-item">
|
||||||
|
<div class="setting-row">
|
||||||
|
<div class="setting-info">
|
||||||
|
<label for="cardBlurAmount">
|
||||||
|
{{ t('settings.layoutSettings.cardBlurAmount') }}
|
||||||
|
<i class="fas fa-info-circle info-icon" data-tooltip="{{ t('settings.layoutSettings.cardBlurAmountHelp') }}"></i>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
<div class="setting-control range-control">
|
||||||
|
<input type="range" id="cardBlurAmount" min="0" max="20" value="8" step="1"
|
||||||
|
oninput="document.getElementById('cardBlurAmountValue').textContent = this.value + 'px'"
|
||||||
|
onchange="settingsManager.saveRangeSetting('cardBlurAmount', 'cardBlurAmountValue', 'card_blur_amount')">
|
||||||
|
<span id="cardBlurAmountValue" class="range-value">8px</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="setting-item">
|
<div class="setting-item">
|
||||||
<div class="setting-row">
|
<div class="setting-row">
|
||||||
<div class="setting-info">
|
<div class="setting-info">
|
||||||
|
|||||||
@@ -46,6 +46,7 @@ class StubRecipeScanner:
|
|||||||
self.last_paginated_params: Dict[str, Any] | None = None
|
self.last_paginated_params: Dict[str, Any] | None = None
|
||||||
self.lora_lookup: Dict[str, List[Dict[str, Any]]] = {}
|
self.lora_lookup: Dict[str, List[Dict[str, Any]]] = {}
|
||||||
self.checkpoint_lookup: Dict[str, List[Dict[str, Any]]] = {}
|
self.checkpoint_lookup: Dict[str, List[Dict[str, Any]]] = {}
|
||||||
|
self.image_id_map_override: Dict[str, str] = {}
|
||||||
|
|
||||||
async def _noop_get_cached_data(force_refresh: bool = False) -> None: # noqa: ARG001 - signature mirrors real scanner
|
async def _noop_get_cached_data(force_refresh: bool = False) -> None: # noqa: ARG001 - signature mirrors real scanner
|
||||||
return None
|
return None
|
||||||
@@ -56,7 +57,10 @@ class StubRecipeScanner:
|
|||||||
)
|
)
|
||||||
|
|
||||||
async def get_cached_data(self, force_refresh: bool = False) -> SimpleNamespace: # noqa: ARG002 - flag unused by stub
|
async def get_cached_data(self, force_refresh: bool = False) -> SimpleNamespace: # noqa: ARG002 - flag unused by stub
|
||||||
return SimpleNamespace(raw_data=list(self.cached_raw))
|
return SimpleNamespace(
|
||||||
|
raw_data=list(self.cached_raw),
|
||||||
|
image_id_map=dict(getattr(self, "image_id_map_override", {})),
|
||||||
|
)
|
||||||
|
|
||||||
async def get_paginated_data(self, **params: Any) -> Dict[str, Any]:
|
async def get_paginated_data(self, **params: Any) -> Dict[str, Any]:
|
||||||
self.last_paginated_params = params
|
self.last_paginated_params = params
|
||||||
@@ -999,3 +1003,95 @@ async def test_batch_import_cancel_missing_id(monkeypatch, tmp_path: Path) -> No
|
|||||||
payload = await response.json()
|
payload = await response.json()
|
||||||
assert response.status == 400
|
assert response.status == 400
|
||||||
assert payload["success"] is False
|
assert payload["success"] is False
|
||||||
|
|
||||||
|
|
||||||
|
async def test_check_image_exists_uses_image_id_map(monkeypatch, tmp_path: Path) -> None:
|
||||||
|
"""check_image_exists must use precomputed image_id_map instead of scanning raw_data."""
|
||||||
|
async with recipe_harness(monkeypatch, tmp_path) as harness:
|
||||||
|
harness.scanner.image_id_map_override = {
|
||||||
|
"123": "recipe-alpha",
|
||||||
|
"789": "recipe-gamma",
|
||||||
|
}
|
||||||
|
|
||||||
|
response = await harness.client.get(
|
||||||
|
"/api/lm/recipes/check-image-exists",
|
||||||
|
params={"image_ids": "123,456,789"},
|
||||||
|
)
|
||||||
|
payload = await response.json()
|
||||||
|
|
||||||
|
assert response.status == 200
|
||||||
|
assert payload["success"] is True
|
||||||
|
assert payload["results"]["123"] == {
|
||||||
|
"in_library": True,
|
||||||
|
"recipe_id": "recipe-alpha",
|
||||||
|
}
|
||||||
|
assert payload["results"]["456"] == {
|
||||||
|
"in_library": False,
|
||||||
|
"recipe_id": None,
|
||||||
|
}
|
||||||
|
assert payload["results"]["789"] == {
|
||||||
|
"in_library": True,
|
||||||
|
"recipe_id": "recipe-gamma",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def test_check_image_exists_handles_empty_input(monkeypatch, tmp_path: Path) -> None:
|
||||||
|
"""Empty or non-numeric image_ids must return an empty results dict."""
|
||||||
|
async with recipe_harness(monkeypatch, tmp_path) as harness:
|
||||||
|
response = await harness.client.get(
|
||||||
|
"/api/lm/recipes/check-image-exists",
|
||||||
|
params={"image_ids": ""},
|
||||||
|
)
|
||||||
|
payload = await response.json()
|
||||||
|
assert response.status == 200
|
||||||
|
assert payload["results"] == {}
|
||||||
|
|
||||||
|
|
||||||
|
async def test_import_from_url_detects_duplicate_via_image_id_map(
|
||||||
|
monkeypatch, tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""import_from_url must return already_exists when image_id is in image_id_map."""
|
||||||
|
async with recipe_harness(monkeypatch, tmp_path) as harness:
|
||||||
|
harness.scanner.cached_raw = [
|
||||||
|
{"id": "existing-recipe", "title": "My Recipe"},
|
||||||
|
]
|
||||||
|
harness.scanner.image_id_map_override = {
|
||||||
|
"99999": "existing-recipe",
|
||||||
|
}
|
||||||
|
|
||||||
|
response = await harness.client.get(
|
||||||
|
"/api/lm/recipes/import-from-url",
|
||||||
|
params={"image_url": "https://civitai.com/images/99999"},
|
||||||
|
)
|
||||||
|
payload = await response.json()
|
||||||
|
|
||||||
|
assert response.status == 200
|
||||||
|
assert payload["already_exists"] is True
|
||||||
|
assert payload["recipe_id"] == "existing-recipe"
|
||||||
|
assert payload["name"] == "My Recipe"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_import_from_url_proceeds_when_image_id_not_in_map(
|
||||||
|
monkeypatch, tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""When image_id is absent from image_id_map, import_from_url must proceed to import."""
|
||||||
|
async with recipe_harness(monkeypatch, tmp_path) as harness:
|
||||||
|
harness.scanner.image_id_map_override = {
|
||||||
|
"111": "some-other-recipe",
|
||||||
|
}
|
||||||
|
harness.civitai.image_info["99999"] = {
|
||||||
|
"id": 99999,
|
||||||
|
"url": "https://image.civitai.com/x/y/original=true/sample.jpeg",
|
||||||
|
"type": "image",
|
||||||
|
"meta": {"prompt": "test"},
|
||||||
|
}
|
||||||
|
|
||||||
|
response = await harness.client.get(
|
||||||
|
"/api/lm/recipes/import-from-url",
|
||||||
|
params={"image_url": "https://civitai.com/images/99999"},
|
||||||
|
)
|
||||||
|
|
||||||
|
# The import may succeed or fail depending on downstream stubs,
|
||||||
|
# but it must NOT return already_exists
|
||||||
|
payload = await response.json()
|
||||||
|
assert payload.get("already_exists") is not True
|
||||||
|
|||||||
@@ -441,7 +441,6 @@ async def test_fetch_and_update_model_returns_rate_limit_error(tmp_path):
|
|||||||
|
|
||||||
assert ok is False
|
assert ok is False
|
||||||
assert error is not None and "Rate limited" in error
|
assert error is not None and "Rate limited" in error
|
||||||
assert "7" in error
|
|
||||||
helpers.metadata_manager.save_metadata.assert_not_awaited()
|
helpers.metadata_manager.save_metadata.assert_not_awaited()
|
||||||
update_cache.assert_not_awaited()
|
update_cache.assert_not_awaited()
|
||||||
helpers.provider_selector.assert_not_awaited()
|
helpers.provider_selector.assert_not_awaited()
|
||||||
|
|||||||
@@ -63,7 +63,8 @@ async def test_fallback_retries_same_provider_on_rate_limit(monkeypatch):
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_fallback_respects_retry_limit(monkeypatch):
|
async def test_fallback_continues_to_next_provider_on_rate_limit(monkeypatch):
|
||||||
|
"""After exhausting retries on primary, fallback should continue to secondary."""
|
||||||
sleep_mock = AsyncMock()
|
sleep_mock = AsyncMock()
|
||||||
monkeypatch.setattr(provider_module.asyncio, "sleep", sleep_mock)
|
monkeypatch.setattr(provider_module.asyncio, "sleep", sleep_mock)
|
||||||
monkeypatch.setattr(provider_module.random, "uniform", lambda *_: 0.0)
|
monkeypatch.setattr(provider_module.random, "uniform", lambda *_: 0.0)
|
||||||
@@ -76,13 +77,13 @@ async def test_fallback_respects_retry_limit(monkeypatch):
|
|||||||
rate_limit_retry_limit=2,
|
rate_limit_retry_limit=2,
|
||||||
)
|
)
|
||||||
|
|
||||||
with pytest.raises(RateLimitError) as exc_info:
|
# After Change A: no longer raises; falls through to secondary
|
||||||
await fallback.get_model_by_hash("abc")
|
result, error = await fallback.get_model_by_hash("abc")
|
||||||
|
|
||||||
assert exc_info.value.provider == "primary"
|
assert error is None
|
||||||
assert primary.calls == 2
|
assert result == {"id": "secondary"}
|
||||||
assert secondary.calls == 0
|
assert primary.calls == 2 # retry_limit exhausted on primary
|
||||||
sleep_mock.assert_awaited_once()
|
assert secondary.calls == 1 # secondary IS called now
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@@ -117,3 +118,40 @@ async def test_rate_limit_retrying_provider_respects_limit(monkeypatch):
|
|||||||
assert exc_info.value.provider == "inner"
|
assert exc_info.value.provider == "inner"
|
||||||
assert inner.calls == 2
|
assert inner.calls == 2
|
||||||
sleep_mock.assert_awaited_once()
|
sleep_mock.assert_awaited_once()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_retry_helper_limits_retries_for_large_retry_after():
|
||||||
|
"""With retry_after >= 120s, _RateLimitRetryHelper should only attempt once (no retries)."""
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
async def failing():
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
raise RateLimitError("limited", retry_after=1500.0)
|
||||||
|
|
||||||
|
helper = provider_module._RateLimitRetryHelper(retry_limit=3)
|
||||||
|
with pytest.raises(RateLimitError):
|
||||||
|
await helper.run("test", failing)
|
||||||
|
assert calls == 1 # No retries for large retry_after
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_retry_helper_retries_normally_for_small_retry_after(monkeypatch):
|
||||||
|
"""With retry_after < 120s, _RateLimitRetryHelper should retry normally (up to limit)."""
|
||||||
|
sleep_mock = AsyncMock()
|
||||||
|
monkeypatch.setattr(provider_module.asyncio, "sleep", sleep_mock)
|
||||||
|
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
async def succeeding():
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
if calls == 1:
|
||||||
|
raise RateLimitError("limited", retry_after=30.0)
|
||||||
|
return {"ok": True}, None
|
||||||
|
|
||||||
|
helper = provider_module._RateLimitRetryHelper(retry_limit=3)
|
||||||
|
result, _ = await helper.run("test", succeeding)
|
||||||
|
assert result == {"ok": True}
|
||||||
|
assert calls == 2 # Retried once (small retry_after)
|
||||||
|
|||||||
@@ -1015,3 +1015,85 @@ async def test_get_paginated_data_sorting(recipe_scanner):
|
|||||||
# Test Date ASC: Gamma (5), Alpha (10), Beta (20)
|
# Test Date ASC: Gamma (5), Alpha (10), Beta (20)
|
||||||
res = await scanner.get_paginated_data(page=1, page_size=10, sort_by="date:asc")
|
res = await scanner.get_paginated_data(page=1, page_size=10, sort_by="date:asc")
|
||||||
assert [i["id"] for i in res["items"]] == ["C", "A", "B"]
|
assert [i["id"] for i in res["items"]] == ["C", "A", "B"]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_build_image_id_map_filters_correctly(recipe_scanner):
|
||||||
|
"""Only recipes with valid CivitAI source_path appear in image_id_map.
|
||||||
|
|
||||||
|
Recipes imported from local files or with empty/missing source_path
|
||||||
|
must be naturally excluded.
|
||||||
|
"""
|
||||||
|
scanner, _ = recipe_scanner
|
||||||
|
from py.services.recipe_cache import RecipeCache
|
||||||
|
|
||||||
|
scanner._cache = RecipeCache(
|
||||||
|
raw_data=[
|
||||||
|
{"id": "r1", "source_path": "https://civitai.com/images/12345"},
|
||||||
|
{"id": "r2", "source_path": "https://civitai.com/images/67890"},
|
||||||
|
{"id": "r3", "source_path": "/home/user/local_image.png"},
|
||||||
|
{"id": "r4", "source_path": ""},
|
||||||
|
{"id": "r5"},
|
||||||
|
],
|
||||||
|
sorted_by_name=[],
|
||||||
|
sorted_by_date=[],
|
||||||
|
)
|
||||||
|
|
||||||
|
result = scanner._build_image_id_map()
|
||||||
|
|
||||||
|
assert result == {
|
||||||
|
"12345": "r1",
|
||||||
|
"67890": "r2",
|
||||||
|
}
|
||||||
|
# r3 = local file path, r4 = empty string, r5 = no key → all excluded
|
||||||
|
for rid in ("r3", "r4", "r5"):
|
||||||
|
assert rid not in result.values()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_add_recipe_updates_image_id_map(recipe_scanner):
|
||||||
|
"""Adding a recipe with a CivitAI URL must update image_id_map.
|
||||||
|
|
||||||
|
A recipe with a local file path must NOT produce an entry.
|
||||||
|
"""
|
||||||
|
scanner, _ = recipe_scanner
|
||||||
|
|
||||||
|
await scanner.add_recipe({
|
||||||
|
"id": "civitai-recipe",
|
||||||
|
"title": "CivitAI",
|
||||||
|
"source_path": "https://civitai.com/images/55555",
|
||||||
|
})
|
||||||
|
|
||||||
|
cache = await scanner.get_cached_data()
|
||||||
|
assert cache.image_id_map.get("55555") == "civitai-recipe"
|
||||||
|
|
||||||
|
await scanner.add_recipe({
|
||||||
|
"id": "local-recipe",
|
||||||
|
"title": "Local",
|
||||||
|
"source_path": "/path/to/local.png",
|
||||||
|
})
|
||||||
|
|
||||||
|
assert "local-recipe" not in cache.image_id_map.values()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_remove_recipe_clears_image_id_map(recipe_scanner):
|
||||||
|
"""Removing a recipe that has a CivitAI image_id must clean up the map."""
|
||||||
|
scanner, _ = recipe_scanner
|
||||||
|
|
||||||
|
await scanner.add_recipe({
|
||||||
|
"id": "recipe-a",
|
||||||
|
"title": "A",
|
||||||
|
"source_path": "https://civitai.com/images/111",
|
||||||
|
})
|
||||||
|
await scanner.add_recipe({
|
||||||
|
"id": "recipe-b",
|
||||||
|
"title": "B",
|
||||||
|
"source_path": "https://civitai.com/images/222",
|
||||||
|
})
|
||||||
|
|
||||||
|
cache = await scanner.get_cached_data()
|
||||||
|
assert "111" in cache.image_id_map
|
||||||
|
assert cache.image_id_map["222"] == "recipe-b"
|
||||||
|
|
||||||
|
await scanner.remove_recipe("recipe-a")
|
||||||
|
|
||||||
|
assert "111" not in cache.image_id_map
|
||||||
|
assert cache.image_id_map["222"] == "recipe-b"
|
||||||
|
|||||||
@@ -465,3 +465,81 @@ class TestPersistentRecipeCache:
|
|||||||
# Operations should complete
|
# Operations should complete
|
||||||
assert operation_counts["saves"] == 5
|
assert operation_counts["saves"] == 5
|
||||||
assert operation_counts["removes"] == 5
|
assert operation_counts["removes"] == 5
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------
|
||||||
|
# image_id_map persistence (Phase 1 improvement)
|
||||||
|
# -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_save_and_load_image_id_map_roundtrip(self, temp_db_path, sample_recipes):
|
||||||
|
"""Save image_id_map via save_cache() and verify it round-trips through load_cache()."""
|
||||||
|
cache = PersistentRecipeCache(db_path=temp_db_path)
|
||||||
|
|
||||||
|
image_id_map = {
|
||||||
|
"12345": "recipe-alpha",
|
||||||
|
"67890": "recipe-beta",
|
||||||
|
}
|
||||||
|
cache.save_cache(sample_recipes, image_id_map=image_id_map)
|
||||||
|
|
||||||
|
loaded = cache.load_cache()
|
||||||
|
assert loaded is not None
|
||||||
|
assert loaded.image_id_map == image_id_map
|
||||||
|
|
||||||
|
def test_load_without_image_id_map_returns_empty_dict(self, temp_db_path, sample_recipes):
|
||||||
|
"""Loading from a cache that has no image_id_map metadata must yield {}."""
|
||||||
|
cache = PersistentRecipeCache(db_path=temp_db_path)
|
||||||
|
|
||||||
|
# Save without image_id_map
|
||||||
|
cache.save_cache(sample_recipes)
|
||||||
|
|
||||||
|
loaded = cache.load_cache()
|
||||||
|
assert loaded is not None
|
||||||
|
assert loaded.image_id_map == {}
|
||||||
|
|
||||||
|
def test_save_cache_without_image_id_map_does_not_corrupt_existing(
|
||||||
|
self, temp_db_path, sample_recipes,
|
||||||
|
):
|
||||||
|
"""Overwriting cache without passing image_id_map must not leave stale data.
|
||||||
|
|
||||||
|
The previous image_id_map entry in cache_metadata should be replaced with {}.
|
||||||
|
"""
|
||||||
|
cache = PersistentRecipeCache(db_path=temp_db_path)
|
||||||
|
|
||||||
|
cache.save_cache(sample_recipes, image_id_map={"123": "old-recipe"})
|
||||||
|
# Overwrite without image_id_map
|
||||||
|
cache.save_cache(sample_recipes)
|
||||||
|
|
||||||
|
loaded = cache.load_cache()
|
||||||
|
assert loaded.image_id_map == {}
|
||||||
|
|
||||||
|
def test_image_id_map_survives_recipe_update(self, temp_db_path, sample_recipes):
|
||||||
|
"""Updating a single recipe must not drop the image_id_map metadata."""
|
||||||
|
cache = PersistentRecipeCache(db_path=temp_db_path)
|
||||||
|
|
||||||
|
cache.save_cache(sample_recipes, image_id_map={"123": "recipe-alpha"})
|
||||||
|
|
||||||
|
updated = dict(sample_recipes[0])
|
||||||
|
updated["title"] = "Updated"
|
||||||
|
cache.update_recipe(updated)
|
||||||
|
|
||||||
|
loaded = cache.load_cache()
|
||||||
|
assert loaded.image_id_map == {"123": "recipe-alpha"}
|
||||||
|
|
||||||
|
def test_save_image_id_map_persists_without_full_save(self, temp_db_path, sample_recipes):
|
||||||
|
"""save_image_id_map must update cache_metadata without rewriting all recipes."""
|
||||||
|
cache = PersistentRecipeCache(db_path=temp_db_path)
|
||||||
|
cache.save_cache(sample_recipes)
|
||||||
|
|
||||||
|
cache.save_image_id_map({"555": "new-recipe", "666": "another-recipe"})
|
||||||
|
|
||||||
|
loaded = cache.load_cache()
|
||||||
|
assert loaded.image_id_map == {"555": "new-recipe", "666": "another-recipe"}
|
||||||
|
|
||||||
|
def test_save_image_id_map_overwrites_previous(self, temp_db_path, sample_recipes):
|
||||||
|
"""Calling save_image_id_map twice must replace, not merge."""
|
||||||
|
cache = PersistentRecipeCache(db_path=temp_db_path)
|
||||||
|
cache.save_cache(sample_recipes, image_id_map={"111": "old"})
|
||||||
|
|
||||||
|
cache.save_image_id_map({"222": "new-only"})
|
||||||
|
|
||||||
|
loaded = cache.load_cache()
|
||||||
|
assert loaded.image_id_map == {"222": "new-only"}
|
||||||
|
|||||||
@@ -141,3 +141,150 @@ def test_update_image_metadata_preserves_png_workflow(tmp_path):
|
|||||||
img.info["parameters"]
|
img.info["parameters"]
|
||||||
== 'prompt text\nRecipe metadata: {"title":"recipe"}'
|
== 'prompt text\nRecipe metadata: {"title":"recipe"}'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# --- ISOBMFF / brotli extraction tests ---
|
||||||
|
|
||||||
|
import struct
|
||||||
|
|
||||||
|
import brotli
|
||||||
|
|
||||||
|
|
||||||
|
def _build_jxl_with_brob(payload_json: dict) -> bytes:
|
||||||
|
"""Build a minimal JXL container with a brob box containing brotli-compressed JSON."""
|
||||||
|
# ISOBMFF box 1: JXL signature box (size=12, type='JXL ', signature)
|
||||||
|
box1 = struct.pack(">I", 12) + b"JXL " + bytes([0x0d, 0x0a, 0x87, 0x0a])
|
||||||
|
# ISOBMFF box 2: ftyp (size=16, type='ftyp', major='jxl ', minor=0)
|
||||||
|
box2 = struct.pack(">I", 16) + b"ftyp" + b"jxl " + struct.pack(">I", 0)
|
||||||
|
# ISOBMFF box 3: brob — payload is b'comf' + brotli(json)
|
||||||
|
compressed = brotli.compress(json.dumps(payload_json).encode("utf-8"))
|
||||||
|
brob_payload = b"comf" + compressed
|
||||||
|
box3 = struct.pack(">I", 8 + len(brob_payload)) + b"brob" + brob_payload
|
||||||
|
return box1 + box2 + box3
|
||||||
|
|
||||||
|
|
||||||
|
def _build_avif_with_brob(payload_json: dict) -> bytes:
|
||||||
|
"""Build a minimal AVIF container with a brob box containing brotli-compressed JSON."""
|
||||||
|
compressed = brotli.compress(json.dumps(payload_json).encode("utf-8"))
|
||||||
|
brob_payload = b"comf" + compressed
|
||||||
|
ftyp_box = struct.pack(">I", 20) + b"ftyp" + b"avif" + struct.pack(">I", 0) + b"avif"
|
||||||
|
brob_box = struct.pack(">I", 8 + len(brob_payload)) + b"brob" + brob_payload
|
||||||
|
return ftyp_box + brob_box
|
||||||
|
|
||||||
|
|
||||||
|
class TestIsobmffBrotliExtraction:
|
||||||
|
"""Tests for ISOBMFF brotli metadata extraction in ExifUtils."""
|
||||||
|
|
||||||
|
def test_extract_jxl_brotli_happy_path(self, tmp_path):
|
||||||
|
"""JXL container with valid brob box extracts prompt and workflow."""
|
||||||
|
payload = {"prompt": "a cute cat", "workflow": {"nodes": [{"id": 1}]}}
|
||||||
|
data = _build_jxl_with_brob(payload)
|
||||||
|
path = tmp_path / "test.jxl"
|
||||||
|
path.write_bytes(data)
|
||||||
|
|
||||||
|
result = ExifUtils._load_structured_metadata(str(path))
|
||||||
|
|
||||||
|
assert result["prompt"] == "a cute cat"
|
||||||
|
assert result["workflow"] == '{"nodes": [{"id": 1}]}'
|
||||||
|
assert result["parameters"] is None
|
||||||
|
assert result["comment"] is None
|
||||||
|
|
||||||
|
def test_extract_avif_brotli_happy_path(self, tmp_path):
|
||||||
|
"""AVIF container with valid brob box extracts prompt and workflow."""
|
||||||
|
payload = {"prompt": "landscape", "workflow": {"nodes": []}}
|
||||||
|
data = _build_avif_with_brob(payload)
|
||||||
|
path = tmp_path / "test.avif"
|
||||||
|
path.write_bytes(data)
|
||||||
|
|
||||||
|
result = ExifUtils._load_structured_metadata(str(path))
|
||||||
|
|
||||||
|
assert result["prompt"] == "landscape"
|
||||||
|
assert result["workflow"] == '{"nodes": []}'
|
||||||
|
|
||||||
|
def test_extract_no_brob_box_returns_none(self, tmp_path):
|
||||||
|
"""JXL container without a brob box returns None from _extract_isobmff_brotli."""
|
||||||
|
# Only JXL signature + ftyp, no brob
|
||||||
|
box1 = struct.pack(">I", 12) + b"JXL " + bytes([0x0d, 0x0a, 0x87, 0x0a])
|
||||||
|
box2 = struct.pack(">I", 16) + b"ftyp" + b"jxl " + struct.pack(">I", 0)
|
||||||
|
path = tmp_path / "test.jxl"
|
||||||
|
path.write_bytes(box1 + box2)
|
||||||
|
|
||||||
|
# The low-level extraction should return None (no brob box)
|
||||||
|
result = ExifUtils._extract_isobmff_brotli(str(path))
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_extract_corrupt_brob_returns_none(self, tmp_path):
|
||||||
|
"""Broken brob box payload gracefully returns None."""
|
||||||
|
box1 = struct.pack(">I", 12) + b"JXL " + bytes([0x0d, 0x0a, 0x87, 0x0a])
|
||||||
|
box2 = struct.pack(">I", 16) + b"ftyp" + b"jxl " + struct.pack(">I", 0)
|
||||||
|
# brob with garbage payload that doesn't start with b'comf'
|
||||||
|
garbage = b"\xff\xff\xff\xff" * 32
|
||||||
|
box3 = struct.pack(">I", 8 + len(garbage)) + b"brob" + garbage
|
||||||
|
path = tmp_path / "test.jxl"
|
||||||
|
path.write_bytes(box1 + box2 + box3)
|
||||||
|
|
||||||
|
result = ExifUtils._extract_isobmff_brotli(str(path))
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_extract_non_isobmff_file_falls_through(self, tmp_path):
|
||||||
|
"""A regular PNG file is not processed as ISOBMFF and returns PIL metadata."""
|
||||||
|
png_info = PngImagePlugin.PngInfo()
|
||||||
|
png_info.add_text("prompt", "from png")
|
||||||
|
path = tmp_path / "test.png"
|
||||||
|
Image.new("RGB", (4, 4), color="red").save(path, pnginfo=png_info)
|
||||||
|
|
||||||
|
result = ExifUtils._load_structured_metadata(str(path))
|
||||||
|
assert result["prompt"] == "from png"
|
||||||
|
|
||||||
|
def test_extract_skip_on_update_and_optimize(self, tmp_path):
|
||||||
|
"""AVIF/JXL files are skipped for write operations (update/append/optimize)."""
|
||||||
|
path = tmp_path / "test.avif"
|
||||||
|
path.write_bytes(b"fake avif data")
|
||||||
|
|
||||||
|
# update_image_metadata should return the path unchanged
|
||||||
|
result = ExifUtils.update_image_metadata(str(path), "some metadata")
|
||||||
|
assert result == str(path)
|
||||||
|
|
||||||
|
# append_recipe_metadata should also skip
|
||||||
|
result = ExifUtils.append_recipe_metadata(str(path), {"title": "test"})
|
||||||
|
assert result == str(path)
|
||||||
|
|
||||||
|
# optimize_image should passthrough for AVIF/JXL paths
|
||||||
|
result_data, ext = ExifUtils.optimize_image(str(path))
|
||||||
|
assert ext == ".avif"
|
||||||
|
assert result_data == b"fake avif data"
|
||||||
|
|
||||||
|
def test_extract_prompt_as_dict(self, tmp_path):
|
||||||
|
"""prompt field as dict is JSON-serialized."""
|
||||||
|
payload = {"prompt": {"text": "hello", "negative": "bad"}}
|
||||||
|
data = _build_jxl_with_brob(payload)
|
||||||
|
path = tmp_path / "test.jxl"
|
||||||
|
path.write_bytes(data)
|
||||||
|
|
||||||
|
result = ExifUtils._load_structured_metadata(str(path))
|
||||||
|
assert json.loads(result["prompt"]) == {"text": "hello", "negative": "bad"}
|
||||||
|
|
||||||
|
def test_extract_workflow_as_list(self, tmp_path):
|
||||||
|
"""workflow field as list is JSON-serialized."""
|
||||||
|
payload = {"workflow": [{"id": 1}, {"id": 2}]}
|
||||||
|
data = _build_avif_with_brob(payload)
|
||||||
|
path = tmp_path / "test.avif"
|
||||||
|
path.write_bytes(data)
|
||||||
|
|
||||||
|
result = ExifUtils._load_structured_metadata(str(path))
|
||||||
|
assert json.loads(result["workflow"]) == [{"id": 1}, {"id": 2}]
|
||||||
|
|
||||||
|
def test_over_decompressed_size_limit(self, tmp_path, monkeypatch):
|
||||||
|
"""Decompressed data exceeding _BROTLI_MAX_DECOMPRESSED is rejected."""
|
||||||
|
# Monkey-patch the limit to a small value to avoid large test data
|
||||||
|
monkeypatch.setattr(ExifUtils, "_BROTLI_MAX_DECOMPRESSED", 100)
|
||||||
|
|
||||||
|
large_content = "x" * 200
|
||||||
|
payload = {"prompt": large_content}
|
||||||
|
data = _build_jxl_with_brob(payload)
|
||||||
|
path = tmp_path / "test.jxl"
|
||||||
|
path.write_bytes(data)
|
||||||
|
|
||||||
|
# Direct extraction should return None because decompressed size exceeds limit
|
||||||
|
result = ExifUtils._extract_isobmff_brotli(str(path))
|
||||||
|
assert result is None
|
||||||
|
|||||||
Reference in New Issue
Block a user