mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-09-21 11:11:26 -03:00
Compare commits
12 Commits
v1.2.3
...
d4b82d98b2
| Author | SHA1 | Date | |
|---|---|---|---|
| d4b82d98b2 | |||
| 8c1c1691e3 | |||
| e14a084f0d | |||
| c55c6f0a41 | |||
| 7d963b27b5 | |||
| eba03800b9 | |||
| bf497d5144 | |||
| 369613f811 | |||
| 9eeebac40b | |||
| b9a516c9f8 | |||
| ef7fa7d3dd | |||
| 9c67dbbf15 |
@@ -15,6 +15,7 @@ node_modules/
|
||||
coverage/
|
||||
.coverage
|
||||
model_cache/
|
||||
recipe_cache/
|
||||
|
||||
# agent / dev tooling
|
||||
.opencode/
|
||||
|
||||
@@ -57,6 +57,15 @@ Locales: `en`, `zh-CN`, `zh-TW`, `ja`, `ko`, `fr`, `de`, `es`, `ru`, `he` (RTL).
|
||||
> prototyped and removed because it collided with the browser's Alt + Arrow handling and the
|
||||
> modal's arrow-key navigation.
|
||||
|
||||
> **Status (2026-09, standalone no-paths guidance):** the standalone branch of the
|
||||
> `other.noPaths` empty state now shows the real `settings.json` path plus an
|
||||
> `other.noPaths.openSettingsFolder` button (each locale reuses its
|
||||
> `settings.openSettingsFileLocation.label` rendering), and `descriptionStandalone` was
|
||||
> reworded in `en.json` — from "none of the configured folders exist on disk" to "no
|
||||
> other-model folders were found; add the folder keys you need to the `folder_paths`
|
||||
> section" — and re-translated in all 9 locales. The `on disk` phrase now survives only in
|
||||
> the ComfyUI variant (`descriptionComfyUI`).
|
||||
|
||||
---
|
||||
|
||||
## 1. Hard rules (do not violate)
|
||||
|
||||
+3
-2
@@ -1241,11 +1241,12 @@
|
||||
},
|
||||
"noPaths": {
|
||||
"title": "Keine Ordner für weitere Modelle gefunden",
|
||||
"descriptionStandalone": "Die Verwaltung weiterer Modelle ist aktiviert, aber keiner der konfigurierten Modellordner existiert auf dem Datenträger. Fügen Sie die unten stehenden Ordnerpfade zu settings.json hinzu und starten Sie LoRA Manager neu.",
|
||||
"descriptionStandalone": "Die Verwaltung weiterer Modelle ist aktiviert, aber es wurden keine Ordner für weitere Modelle gefunden. Fügen Sie die benötigten Ordnerschlüssel zum Abschnitt folder_paths Ihrer settings.json hinzu und starten Sie LoRA Manager neu.",
|
||||
"hintStandalone": "Nur die oben aufgeführten Ordnerschlüssel werden gescannt; nicht benötigte Schlüssel können weggelassen werden.",
|
||||
"descriptionComfyUI": "Die Verwaltung weiterer Modelle ist aktiviert, aber keiner der konfigurierten Modellordner existiert auf dem Datenträger. Fügen Sie die entsprechenden Modellordner zu Ihren ComfyUI-Modellpfaden hinzu und laden Sie diese Seite neu.",
|
||||
"hintComfyUI": "Weitere Modelle werden aus den Ordnern vae, upscale_models, text_encoders, clip_vision und controlnet von ComfyUI gelesen.",
|
||||
"openSettings": "Einstellungen öffnen"
|
||||
"openSettings": "Einstellungen öffnen",
|
||||
"openSettingsFolder": "Einstellungsordner öffnen"
|
||||
}
|
||||
},
|
||||
"sidebar": {
|
||||
|
||||
+3
-2
@@ -1241,11 +1241,12 @@
|
||||
},
|
||||
"noPaths": {
|
||||
"title": "No other-model folders found",
|
||||
"descriptionStandalone": "Other Models management is on, but none of the configured model folders exist on disk. Add the folder paths below to settings.json and restart LoRA Manager.",
|
||||
"descriptionStandalone": "Other Models management is on, but no other-model folders were found. Add the folder keys you need to the folder_paths section of your settings.json, then restart LoRA Manager.",
|
||||
"hintStandalone": "Only the folder keys listed above are scanned; keys you do not need can be omitted.",
|
||||
"descriptionComfyUI": "Other Models management is on, but none of the configured model folders exist on disk. Add the matching model folders to your ComfyUI model paths, then reload this page.",
|
||||
"hintComfyUI": "Other models are read from ComfyUI's vae, upscale_models, text_encoders, clip_vision and controlnet folders.",
|
||||
"openSettings": "Open Settings"
|
||||
"openSettings": "Open Settings",
|
||||
"openSettingsFolder": "Open Settings Folder"
|
||||
}
|
||||
},
|
||||
"sidebar": {
|
||||
|
||||
+3
-2
@@ -1241,11 +1241,12 @@
|
||||
},
|
||||
"noPaths": {
|
||||
"title": "No se encontraron carpetas de otros modelos",
|
||||
"descriptionStandalone": "La gestión de otros modelos está activada, pero ninguna de las carpetas de modelos configuradas existe en el disco. Añade las rutas de carpetas de abajo a settings.json y reinicia LoRA Manager.",
|
||||
"descriptionStandalone": "La gestión de otros modelos está activada, pero no se encontraron carpetas de otros modelos. Añade las claves de carpeta que necesites a la sección folder_paths de tu settings.json y reinicia LoRA Manager.",
|
||||
"hintStandalone": "Solo se escanean las claves de carpeta listadas arriba; las claves que no necesites puedes omitirlas.",
|
||||
"descriptionComfyUI": "La gestión de otros modelos está activada, pero ninguna de las carpetas de modelos configuradas existe en el disco. Añade las carpetas de modelos correspondientes a tus rutas de modelos de ComfyUI y recarga esta página.",
|
||||
"hintComfyUI": "Los otros modelos se leen de las carpetas vae, upscale_models, text_encoders, clip_vision y controlnet de ComfyUI.",
|
||||
"openSettings": "Abrir configuración"
|
||||
"openSettings": "Abrir configuración",
|
||||
"openSettingsFolder": "Abrir carpeta de ajustes"
|
||||
}
|
||||
},
|
||||
"sidebar": {
|
||||
|
||||
+3
-2
@@ -1241,11 +1241,12 @@
|
||||
},
|
||||
"noPaths": {
|
||||
"title": "Aucun dossier d’autres modèles trouvé",
|
||||
"descriptionStandalone": "La gestion des autres modèles est activée, mais aucun des dossiers de modèles configurés n’existe sur le disque. Ajoutez les chemins de dossiers ci-dessous à settings.json, puis redémarrez LoRA Manager.",
|
||||
"descriptionStandalone": "La gestion des autres modèles est activée, mais aucun dossier d’autres modèles n’a été trouvé. Ajoutez les clés de dossiers dont vous avez besoin à la section folder_paths de votre settings.json, puis redémarrez LoRA Manager.",
|
||||
"hintStandalone": "Seules les clés de dossiers listées ci-dessus sont analysées ; les clés inutiles peuvent être omises.",
|
||||
"descriptionComfyUI": "La gestion des autres modèles est activée, mais aucun des dossiers de modèles configurés n’existe sur le disque. Ajoutez les dossiers de modèles correspondants à vos chemins de modèles ComfyUI, puis rechargez cette page.",
|
||||
"hintComfyUI": "Les autres modèles sont lus depuis les dossiers vae, upscale_models, text_encoders, clip_vision et controlnet de ComfyUI.",
|
||||
"openSettings": "Ouvrir les paramètres"
|
||||
"openSettings": "Ouvrir les paramètres",
|
||||
"openSettingsFolder": "Ouvrir le dossier des paramètres"
|
||||
}
|
||||
},
|
||||
"sidebar": {
|
||||
|
||||
+3
-2
@@ -1241,11 +1241,12 @@
|
||||
},
|
||||
"noPaths": {
|
||||
"title": "לא נמצאו תיקיות של מודלים אחרים",
|
||||
"descriptionStandalone": "ניהול המודלים האחרים פועל, אך אף אחת מתיקיות המודלים המוגדרות אינה קיימת בדיסק. הוסף את נתיבי התיקיות שלמטה ל-settings.json והפעל מחדש את LoRA Manager.",
|
||||
"descriptionStandalone": "ניהול המודלים האחרים פועל, אך לא נמצאו תיקיות של מודלים אחרים. הוסף את מפתחות התיקיות הדרושים למקטע folder_paths ב-settings.json והפעל מחדש את LoRA Manager.",
|
||||
"hintStandalone": "רק מפתחות התיקיות המפורטים למעלה נסרקים; ניתן להשמיט מפתחות שאינך צריך.",
|
||||
"descriptionComfyUI": "ניהול המודלים האחרים פועל, אך אף אחת מתיקיות המודלים המוגדרות אינה קיימת בדיסק. הוסף את תיקיות המודלים המתאימות לנתיבי המודלים של ComfyUI וטען מחדש עמוד זה.",
|
||||
"hintComfyUI": "מודלים אחרים נקראים מתיקיות vae, upscale_models, text_encoders, clip_vision ו-controlnet של ComfyUI.",
|
||||
"openSettings": "פתח הגדרות"
|
||||
"openSettings": "פתח הגדרות",
|
||||
"openSettingsFolder": "פתח תיקיית הגדרות"
|
||||
}
|
||||
},
|
||||
"sidebar": {
|
||||
|
||||
+3
-2
@@ -1241,11 +1241,12 @@
|
||||
},
|
||||
"noPaths": {
|
||||
"title": "その他のモデルのフォルダーが見つかりません",
|
||||
"descriptionStandalone": "その他のモデル管理はオンですが、設定されたモデルフォルダーがディスク上に存在しません。以下のフォルダーパスをsettings.jsonに追加し、LoRA Managerを再起動してください。",
|
||||
"descriptionStandalone": "その他のモデル管理はオンですが、その他のモデルのフォルダーが見つかりません。必要なフォルダーキーをsettings.jsonのfolder_pathsセクションに追加し、LoRA Managerを再起動してください。",
|
||||
"hintStandalone": "スキャンされるのは上記のフォルダーキーのみです。不要なキーは省略できます。",
|
||||
"descriptionComfyUI": "その他のモデル管理はオンですが、設定されたモデルフォルダーがディスク上に存在しません。該当するモデルフォルダーをComfyUIのモデルパスに追加し、このページを再読み込みしてください。",
|
||||
"hintComfyUI": "その他のモデルは、ComfyUIのvae、upscale_models、text_encoders、clip_vision、controlnetフォルダーから読み込まれます。",
|
||||
"openSettings": "設定を開く"
|
||||
"openSettings": "設定を開く",
|
||||
"openSettingsFolder": "設定フォルダーを開く"
|
||||
}
|
||||
},
|
||||
"sidebar": {
|
||||
|
||||
+3
-2
@@ -1241,11 +1241,12 @@
|
||||
},
|
||||
"noPaths": {
|
||||
"title": "기타 모델 폴더를 찾을 수 없습니다",
|
||||
"descriptionStandalone": "기타 모델 관리가 켜져 있지만, 설정된 모델 폴더가 디스크에 존재하지 않습니다. 아래 폴더 경로를 settings.json에 추가한 뒤 LoRA Manager를 재시작하세요.",
|
||||
"descriptionStandalone": "기타 모델 관리가 켜져 있지만, 기타 모델 폴더를 찾을 수 없습니다. 필요한 폴더 키를 settings.json의 folder_paths 섹션에 추가한 뒤 LoRA Manager를 재시작하세요.",
|
||||
"hintStandalone": "위에 나열된 폴더 키만 스캔됩니다. 필요 없는 키는 생략할 수 있습니다.",
|
||||
"descriptionComfyUI": "기타 모델 관리가 켜져 있지만, 설정된 모델 폴더가 디스크에 존재하지 않습니다. 해당 모델 폴더를 ComfyUI 모델 경로에 추가한 뒤 이 페이지를 새로 고침하세요.",
|
||||
"hintComfyUI": "기타 모델은 ComfyUI의 vae, upscale_models, text_encoders, clip_vision, controlnet 폴더에서 읽어옵니다.",
|
||||
"openSettings": "설정 열기"
|
||||
"openSettings": "설정 열기",
|
||||
"openSettingsFolder": "설정 폴더 열기"
|
||||
}
|
||||
},
|
||||
"sidebar": {
|
||||
|
||||
+3
-2
@@ -1241,11 +1241,12 @@
|
||||
},
|
||||
"noPaths": {
|
||||
"title": "Папки других моделей не найдены",
|
||||
"descriptionStandalone": "Управление другими моделями включено, но ни одна из настроенных папок моделей не существует на диске. Добавьте указанные ниже пути к папкам в settings.json и перезапустите LoRA Manager.",
|
||||
"descriptionStandalone": "Управление другими моделями включено, но папки других моделей не найдены. Добавьте нужные ключи папок в раздел folder_paths файла settings.json и перезапустите LoRA Manager.",
|
||||
"hintStandalone": "Сканируются только перечисленные выше ключи папок; ненужные ключи можно опустить.",
|
||||
"descriptionComfyUI": "Управление другими моделями включено, но ни одна из настроенных папок моделей не существует на диске. Добавьте соответствующие папки моделей в пути к моделям ComfyUI и перезагрузите эту страницу.",
|
||||
"hintComfyUI": "Другие модели читаются из папок vae, upscale_models, text_encoders, clip_vision и controlnet в ComfyUI.",
|
||||
"openSettings": "Открыть настройки"
|
||||
"openSettings": "Открыть настройки",
|
||||
"openSettingsFolder": "Открыть папку настроек"
|
||||
}
|
||||
},
|
||||
"sidebar": {
|
||||
|
||||
+3
-2
@@ -1241,11 +1241,12 @@
|
||||
},
|
||||
"noPaths": {
|
||||
"title": "未找到其他模型文件夹",
|
||||
"descriptionStandalone": "其他模型管理已开启,但配置的模型文件夹在磁盘上都不存在。请将下面的文件夹路径添加到 settings.json,然后重启 LoRA Manager。",
|
||||
"descriptionStandalone": "其他模型管理已开启,但未找到其他模型文件夹。请将你需要的文件夹键添加到 settings.json 的 folder_paths 部分,然后重启 LoRA Manager。",
|
||||
"hintStandalone": "只会扫描上面列出的文件夹键;不需要的键可以省略。",
|
||||
"descriptionComfyUI": "其他模型管理已开启,但配置的模型文件夹在磁盘上都不存在。请将对应的模型文件夹添加到 ComfyUI 的模型路径,然后重新加载此页面。",
|
||||
"hintComfyUI": "其他模型从 ComfyUI 的 vae、upscale_models、text_encoders、clip_vision 和 controlnet 文件夹中读取。",
|
||||
"openSettings": "打开设置"
|
||||
"openSettings": "打开设置",
|
||||
"openSettingsFolder": "打开设置文件夹"
|
||||
}
|
||||
},
|
||||
"sidebar": {
|
||||
|
||||
+3
-2
@@ -1241,11 +1241,12 @@
|
||||
},
|
||||
"noPaths": {
|
||||
"title": "找不到其他模型資料夾",
|
||||
"descriptionStandalone": "其他模型管理已開啟,但設定的模型資料夾在磁碟上都不存在。請將下方的資料夾路徑加入 settings.json,然後重新啟動 LoRA Manager。",
|
||||
"descriptionStandalone": "其他模型管理已開啟,但找不到其他模型資料夾。請將您需要的資料夾鍵加入 settings.json 的 folder_paths 區段,然後重新啟動 LoRA Manager。",
|
||||
"hintStandalone": "只會掃描上方列出的資料夾鍵;不需要的鍵可以省略。",
|
||||
"descriptionComfyUI": "其他模型管理已開啟,但設定的模型資料夾在磁碟上都不存在。請將對應的模型資料夾加入 ComfyUI 的模型路徑,然後重新載入此頁面。",
|
||||
"hintComfyUI": "其他模型會從 ComfyUI 的 vae、upscale_models、text_encoders、clip_vision 和 controlnet 資料夾讀取。",
|
||||
"openSettings": "開啟設定"
|
||||
"openSettings": "開啟設定",
|
||||
"openSettingsFolder": "開啟設定資料夾"
|
||||
}
|
||||
},
|
||||
"sidebar": {
|
||||
|
||||
@@ -421,6 +421,11 @@ def _wsl_to_windows_path(wsl_path: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _has_gui_display() -> bool:
|
||||
"""Check whether a GUI session is reachable for xdg-open."""
|
||||
return bool(os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"))
|
||||
|
||||
|
||||
class PromptServerProtocol(Protocol):
|
||||
"""Subset of PromptServer used by the handlers."""
|
||||
|
||||
@@ -2759,12 +2764,40 @@ class ModelLibraryHandler:
|
||||
|
||||
normalized_type, scanner = await self._get_scanner_for_type(model_type)
|
||||
if not normalized_type:
|
||||
# The lookup cannot be served as a fully interactive list. Two
|
||||
# cases share this branch: a CivitAI type with no scanner at all
|
||||
# (Wildcards, Workflows, Hypernetwork, Poses, AestheticGradient)
|
||||
# and an Other-model type while the opt-in master switch is off.
|
||||
# Answer 200 with the CivitAI list marked read-only plus a
|
||||
# machine-readable reason, so clients can still show the
|
||||
# versions and explain why the actions are missing. Legacy
|
||||
# clients keep working: they only read `success`/`versions`.
|
||||
reason = (
|
||||
"other_models_disabled"
|
||||
if self._normalize_model_type(model_type) == "other"
|
||||
else "model_type_unsupported"
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": f'Model type "{model_type}" is not supported',
|
||||
},
|
||||
status=400,
|
||||
"success": True,
|
||||
"modelId": model_id,
|
||||
"modelName": model_name,
|
||||
"modelType": model_type,
|
||||
"supported": False,
|
||||
"reason": reason,
|
||||
"versions": [
|
||||
{
|
||||
"id": version.get("id"),
|
||||
"name": version.get("name", ""),
|
||||
"thumbnailUrl": version.get("images")[0]["url"]
|
||||
if version.get("images")
|
||||
else None,
|
||||
"inLibrary": False,
|
||||
"hasBeenDownloaded": False,
|
||||
}
|
||||
for version in versions
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
if not scanner:
|
||||
@@ -2806,6 +2839,7 @@ class ModelLibraryHandler:
|
||||
"modelId": model_id,
|
||||
"modelName": model_name,
|
||||
"modelType": model_type,
|
||||
"supported": True,
|
||||
"versions": enriched_versions,
|
||||
}
|
||||
)
|
||||
@@ -3393,6 +3427,18 @@ class FileSystemHandler:
|
||||
subprocess.Popen(["open", "-R", settings_file])
|
||||
else:
|
||||
folder = os.path.dirname(settings_file)
|
||||
if not _has_gui_display():
|
||||
# Headless/SSH session: xdg-open cannot open a file
|
||||
# manager, so hand the path to the browser for copying
|
||||
# instead of reporting a success that never happened.
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"message": "Headless session: path available for copying",
|
||||
"path": settings_file,
|
||||
"mode": "clipboard",
|
||||
}
|
||||
)
|
||||
subprocess.Popen(["xdg-open", folder])
|
||||
|
||||
return web.json_response(
|
||||
|
||||
@@ -83,11 +83,16 @@ class OtherRoutes(BaseModelRoutes):
|
||||
# resolved to no existing folder. Render an actionable empty state
|
||||
# instead of an apparently broken empty grid.
|
||||
standalone_mode = os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"
|
||||
return {
|
||||
context = {
|
||||
"other_disabled": False,
|
||||
"other_no_paths": not bool(config.other_roots),
|
||||
"standalone_mode": standalone_mode,
|
||||
}
|
||||
if standalone_mode:
|
||||
# The settings UI cannot edit primary folder_paths, so the empty
|
||||
# state must point at the actual file the user has to edit.
|
||||
context["settings_file"] = getattr(self._settings, "settings_file", "") or ""
|
||||
return context
|
||||
|
||||
def _get_expected_model_types(self) -> str:
|
||||
"""Get expected model types string for error messages"""
|
||||
|
||||
@@ -21,7 +21,20 @@ NETWORK_EXCEPTIONS = (ClientError, OSError, asyncio.TimeoutError)
|
||||
# otherwise delete them because they are untracked and, in released tags,
|
||||
# not listed in ``.gitignore``. ``-e`` excludes a path from cleaning
|
||||
# regardless of whether it is ignored.
|
||||
_PRESERVE_DIRS = ('settings.json', 'civitai', 'wildcards', 'backups', 'stats', 'logs', 'cache', 'model_cache')
|
||||
# ``cache`` covers the resolved cache tree (cache/model, cache/recipe,
|
||||
# cache/fts, ...); the legacy ``recipe_cache`` / ``model_cache`` directories
|
||||
# are listed too because a portable install can predate the cache/ move.
|
||||
_PRESERVE_DIRS = (
|
||||
'settings.json',
|
||||
'civitai',
|
||||
'wildcards',
|
||||
'backups',
|
||||
'stats',
|
||||
'logs',
|
||||
'cache',
|
||||
'model_cache',
|
||||
'recipe_cache',
|
||||
)
|
||||
|
||||
|
||||
def _clean_excludes() -> List[str]:
|
||||
|
||||
@@ -6,7 +6,9 @@ import threading
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple
|
||||
|
||||
from ..utils.cache_db import connect_cache_db
|
||||
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
||||
from ..utils.file_lock import exclusive_lock
|
||||
from .model_sources import normalize_metadata_source
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -257,267 +259,271 @@ class PersistentModelCache:
|
||||
return
|
||||
try:
|
||||
with self._db_lock:
|
||||
conn = self._connect()
|
||||
try:
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("BEGIN")
|
||||
# Cross-process serialization: another LoRA Manager instance may
|
||||
# share this settings directory, and the read-merge-write below
|
||||
# spans several statements.
|
||||
with exclusive_lock(self._db_path):
|
||||
conn = self._connect()
|
||||
try:
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("BEGIN")
|
||||
|
||||
model_rows = [self._prepare_model_row(model_type, item) for item in raw_data]
|
||||
model_map: Dict[str, Tuple[Any, ...]] = {
|
||||
row[1]: row for row in model_rows if row[1] # row[1] is file_path
|
||||
}
|
||||
model_rows = [self._prepare_model_row(model_type, item) for item in raw_data]
|
||||
model_map: Dict[str, Tuple[Any, ...]] = {
|
||||
row[1]: row for row in model_rows if row[1] # row[1] is file_path
|
||||
}
|
||||
|
||||
existing_models = conn.execute(
|
||||
"SELECT "
|
||||
+ ", ".join(self._MODEL_COLUMNS[1:])
|
||||
+ " FROM models WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_model_map: Dict[str, sqlite3.Row] = {
|
||||
row["file_path"]: row for row in existing_models
|
||||
}
|
||||
|
||||
to_remove_models = [
|
||||
(model_type, path)
|
||||
for path in existing_model_map.keys()
|
||||
if path not in model_map
|
||||
]
|
||||
if to_remove_models:
|
||||
conn.executemany(
|
||||
"DELETE FROM models WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
conn.executemany(
|
||||
"DELETE FROM model_tags WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
conn.executemany(
|
||||
"DELETE FROM hash_index WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
conn.executemany(
|
||||
"DELETE FROM autov3_index WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
conn.executemany(
|
||||
"DELETE FROM excluded_models WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
|
||||
insert_rows: List[Tuple[Any, ...]] = []
|
||||
update_rows: List[Tuple[Any, ...]] = []
|
||||
|
||||
for file_path, row in model_map.items():
|
||||
existing = existing_model_map.get(file_path)
|
||||
if existing is None:
|
||||
insert_rows.append(row)
|
||||
continue
|
||||
|
||||
existing_values = tuple(
|
||||
existing[column] for column in self._MODEL_COLUMNS[1:]
|
||||
)
|
||||
current_values = row[1:]
|
||||
if existing_values != current_values:
|
||||
update_rows.append(row[2:] + (model_type, file_path))
|
||||
|
||||
if insert_rows:
|
||||
conn.executemany(self._insert_model_sql(), insert_rows)
|
||||
|
||||
if update_rows:
|
||||
set_clause = ", ".join(
|
||||
f"{column} = ?"
|
||||
for column in self._MODEL_UPDATE_COLUMNS
|
||||
)
|
||||
update_sql = (
|
||||
f"UPDATE models SET {set_clause} WHERE model_type = ? AND file_path = ?"
|
||||
)
|
||||
conn.executemany(update_sql, update_rows)
|
||||
|
||||
existing_tags_rows = conn.execute(
|
||||
"SELECT file_path, tag FROM model_tags WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_tags: Dict[str, set[str]] = {}
|
||||
for row in existing_tags_rows:
|
||||
existing_tags.setdefault(row["file_path"], set()).add(row["tag"])
|
||||
|
||||
new_tags: Dict[str, set[str]] = {}
|
||||
for item in raw_data:
|
||||
file_path = item.get("file_path")
|
||||
if not file_path:
|
||||
continue
|
||||
tags = set(item.get("tags") or [])
|
||||
if tags:
|
||||
new_tags[file_path] = tags
|
||||
|
||||
tag_inserts: List[Tuple[str, str, str]] = []
|
||||
tag_deletes: List[Tuple[str, str, str]] = []
|
||||
|
||||
all_tag_paths = set(existing_tags.keys()) | set(new_tags.keys())
|
||||
for path in all_tag_paths:
|
||||
existing_set = existing_tags.get(path, set())
|
||||
new_set = new_tags.get(path, set())
|
||||
to_add = new_set - existing_set
|
||||
to_remove = existing_set - new_set
|
||||
|
||||
for tag in to_add:
|
||||
tag_inserts.append((model_type, path, tag))
|
||||
for tag in to_remove:
|
||||
tag_deletes.append((model_type, path, tag))
|
||||
|
||||
if tag_deletes:
|
||||
conn.executemany(
|
||||
"DELETE FROM model_tags WHERE model_type = ? AND file_path = ? AND tag = ?",
|
||||
tag_deletes,
|
||||
)
|
||||
if tag_inserts:
|
||||
conn.executemany(
|
||||
"INSERT INTO model_tags (model_type, file_path, tag) VALUES (?, ?, ?)",
|
||||
tag_inserts,
|
||||
)
|
||||
|
||||
existing_hash_rows = conn.execute(
|
||||
"SELECT sha256, file_path FROM hash_index WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_hash_map: Dict[str, set[str]] = {}
|
||||
for row in existing_hash_rows:
|
||||
sha_value = (row["sha256"] or "").lower()
|
||||
if not sha_value:
|
||||
continue
|
||||
existing_hash_map.setdefault(sha_value, set()).add(row["file_path"])
|
||||
|
||||
new_hash_map: Dict[str, set[str]] = {}
|
||||
for sha_value, paths in hash_index.items():
|
||||
normalized_sha = (sha_value or "").lower()
|
||||
if not normalized_sha:
|
||||
continue
|
||||
bucket = new_hash_map.setdefault(normalized_sha, set())
|
||||
for path in paths:
|
||||
if path:
|
||||
bucket.add(path)
|
||||
|
||||
hash_inserts: List[Tuple[str, str, str]] = []
|
||||
hash_deletes: List[Tuple[str, str, str]] = []
|
||||
|
||||
all_shas = set(existing_hash_map.keys()) | set(new_hash_map.keys())
|
||||
for sha_value in all_shas:
|
||||
existing_paths = existing_hash_map.get(sha_value, set())
|
||||
new_paths = new_hash_map.get(sha_value, set())
|
||||
|
||||
for path in existing_paths - new_paths:
|
||||
hash_deletes.append((model_type, sha_value, path))
|
||||
for path in new_paths - existing_paths:
|
||||
hash_inserts.append((model_type, sha_value, path))
|
||||
|
||||
if hash_deletes:
|
||||
conn.executemany(
|
||||
"DELETE FROM hash_index WHERE model_type = ? AND sha256 = ? AND file_path = ?",
|
||||
hash_deletes,
|
||||
)
|
||||
if hash_inserts:
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO hash_index (model_type, sha256, file_path) VALUES (?, ?, ?)",
|
||||
hash_inserts,
|
||||
)
|
||||
|
||||
if autov3_hash_index is not None:
|
||||
existing_autov3_rows = conn.execute(
|
||||
"SELECT autov3, file_path FROM autov3_index WHERE model_type = ?",
|
||||
existing_models = conn.execute(
|
||||
"SELECT "
|
||||
+ ", ".join(self._MODEL_COLUMNS[1:])
|
||||
+ " FROM models WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_autov3_map: Dict[str, set[str]] = {}
|
||||
for row in existing_autov3_rows:
|
||||
autov3_value = (row["autov3"] or "").lower()
|
||||
if not autov3_value:
|
||||
continue
|
||||
existing_autov3_map.setdefault(autov3_value, set()).add(row["file_path"])
|
||||
existing_model_map: Dict[str, sqlite3.Row] = {
|
||||
row["file_path"]: row for row in existing_models
|
||||
}
|
||||
|
||||
new_autov3_map: Dict[str, set[str]] = {}
|
||||
for autov3_value, paths in autov3_hash_index.items():
|
||||
normalized_autov3 = (autov3_value or "").lower()
|
||||
if not normalized_autov3:
|
||||
to_remove_models = [
|
||||
(model_type, path)
|
||||
for path in existing_model_map.keys()
|
||||
if path not in model_map
|
||||
]
|
||||
if to_remove_models:
|
||||
conn.executemany(
|
||||
"DELETE FROM models WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
conn.executemany(
|
||||
"DELETE FROM model_tags WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
conn.executemany(
|
||||
"DELETE FROM hash_index WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
conn.executemany(
|
||||
"DELETE FROM autov3_index WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
conn.executemany(
|
||||
"DELETE FROM excluded_models WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
|
||||
insert_rows: List[Tuple[Any, ...]] = []
|
||||
update_rows: List[Tuple[Any, ...]] = []
|
||||
|
||||
for file_path, row in model_map.items():
|
||||
existing = existing_model_map.get(file_path)
|
||||
if existing is None:
|
||||
insert_rows.append(row)
|
||||
continue
|
||||
bucket = new_autov3_map.setdefault(normalized_autov3, set())
|
||||
|
||||
existing_values = tuple(
|
||||
existing[column] for column in self._MODEL_COLUMNS[1:]
|
||||
)
|
||||
current_values = row[1:]
|
||||
if existing_values != current_values:
|
||||
update_rows.append(row[2:] + (model_type, file_path))
|
||||
|
||||
if insert_rows:
|
||||
conn.executemany(self._insert_model_sql(), insert_rows)
|
||||
|
||||
if update_rows:
|
||||
set_clause = ", ".join(
|
||||
f"{column} = ?"
|
||||
for column in self._MODEL_UPDATE_COLUMNS
|
||||
)
|
||||
update_sql = (
|
||||
f"UPDATE models SET {set_clause} WHERE model_type = ? AND file_path = ?"
|
||||
)
|
||||
conn.executemany(update_sql, update_rows)
|
||||
|
||||
existing_tags_rows = conn.execute(
|
||||
"SELECT file_path, tag FROM model_tags WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_tags: Dict[str, set[str]] = {}
|
||||
for row in existing_tags_rows:
|
||||
existing_tags.setdefault(row["file_path"], set()).add(row["tag"])
|
||||
|
||||
new_tags: Dict[str, set[str]] = {}
|
||||
for item in raw_data:
|
||||
file_path = item.get("file_path")
|
||||
if not file_path:
|
||||
continue
|
||||
tags = set(item.get("tags") or [])
|
||||
if tags:
|
||||
new_tags[file_path] = tags
|
||||
|
||||
tag_inserts: List[Tuple[str, str, str]] = []
|
||||
tag_deletes: List[Tuple[str, str, str]] = []
|
||||
|
||||
all_tag_paths = set(existing_tags.keys()) | set(new_tags.keys())
|
||||
for path in all_tag_paths:
|
||||
existing_set = existing_tags.get(path, set())
|
||||
new_set = new_tags.get(path, set())
|
||||
to_add = new_set - existing_set
|
||||
to_remove = existing_set - new_set
|
||||
|
||||
for tag in to_add:
|
||||
tag_inserts.append((model_type, path, tag))
|
||||
for tag in to_remove:
|
||||
tag_deletes.append((model_type, path, tag))
|
||||
|
||||
if tag_deletes:
|
||||
conn.executemany(
|
||||
"DELETE FROM model_tags WHERE model_type = ? AND file_path = ? AND tag = ?",
|
||||
tag_deletes,
|
||||
)
|
||||
if tag_inserts:
|
||||
conn.executemany(
|
||||
"INSERT INTO model_tags (model_type, file_path, tag) VALUES (?, ?, ?)",
|
||||
tag_inserts,
|
||||
)
|
||||
|
||||
existing_hash_rows = conn.execute(
|
||||
"SELECT sha256, file_path FROM hash_index WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_hash_map: Dict[str, set[str]] = {}
|
||||
for row in existing_hash_rows:
|
||||
sha_value = (row["sha256"] or "").lower()
|
||||
if not sha_value:
|
||||
continue
|
||||
existing_hash_map.setdefault(sha_value, set()).add(row["file_path"])
|
||||
|
||||
new_hash_map: Dict[str, set[str]] = {}
|
||||
for sha_value, paths in hash_index.items():
|
||||
normalized_sha = (sha_value or "").lower()
|
||||
if not normalized_sha:
|
||||
continue
|
||||
bucket = new_hash_map.setdefault(normalized_sha, set())
|
||||
for path in paths:
|
||||
if path:
|
||||
bucket.add(path)
|
||||
|
||||
autov3_inserts: List[Tuple[str, str, str]] = []
|
||||
autov3_deletes: List[Tuple[str, str, str]] = []
|
||||
hash_inserts: List[Tuple[str, str, str]] = []
|
||||
hash_deletes: List[Tuple[str, str, str]] = []
|
||||
|
||||
all_autov3 = set(existing_autov3_map.keys()) | set(new_autov3_map.keys())
|
||||
for autov3_value in all_autov3:
|
||||
existing_paths = existing_autov3_map.get(autov3_value, set())
|
||||
new_paths = new_autov3_map.get(autov3_value, set())
|
||||
all_shas = set(existing_hash_map.keys()) | set(new_hash_map.keys())
|
||||
for sha_value in all_shas:
|
||||
existing_paths = existing_hash_map.get(sha_value, set())
|
||||
new_paths = new_hash_map.get(sha_value, set())
|
||||
|
||||
for path in existing_paths - new_paths:
|
||||
autov3_deletes.append((model_type, autov3_value, path))
|
||||
hash_deletes.append((model_type, sha_value, path))
|
||||
for path in new_paths - existing_paths:
|
||||
autov3_inserts.append((model_type, autov3_value, path))
|
||||
hash_inserts.append((model_type, sha_value, path))
|
||||
|
||||
if autov3_deletes:
|
||||
if hash_deletes:
|
||||
conn.executemany(
|
||||
"DELETE FROM autov3_index WHERE model_type = ? AND autov3 = ? AND file_path = ?",
|
||||
autov3_deletes,
|
||||
"DELETE FROM hash_index WHERE model_type = ? AND sha256 = ? AND file_path = ?",
|
||||
hash_deletes,
|
||||
)
|
||||
if autov3_inserts:
|
||||
if hash_inserts:
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO autov3_index (model_type, autov3, file_path) VALUES (?, ?, ?)",
|
||||
autov3_inserts,
|
||||
"INSERT OR IGNORE INTO hash_index (model_type, sha256, file_path) VALUES (?, ?, ?)",
|
||||
hash_inserts,
|
||||
)
|
||||
|
||||
existing_excluded_rows = conn.execute(
|
||||
"SELECT file_path FROM excluded_models WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_excluded = {row["file_path"] for row in existing_excluded_rows}
|
||||
new_excluded = {path for path in excluded_models if path}
|
||||
if autov3_hash_index is not None:
|
||||
existing_autov3_rows = conn.execute(
|
||||
"SELECT autov3, file_path FROM autov3_index WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_autov3_map: Dict[str, set[str]] = {}
|
||||
for row in existing_autov3_rows:
|
||||
autov3_value = (row["autov3"] or "").lower()
|
||||
if not autov3_value:
|
||||
continue
|
||||
existing_autov3_map.setdefault(autov3_value, set()).add(row["file_path"])
|
||||
|
||||
excluded_deletes = [
|
||||
(model_type, path)
|
||||
for path in existing_excluded - new_excluded
|
||||
]
|
||||
excluded_inserts = [
|
||||
(model_type, path)
|
||||
for path in new_excluded - existing_excluded
|
||||
]
|
||||
new_autov3_map: Dict[str, set[str]] = {}
|
||||
for autov3_value, paths in autov3_hash_index.items():
|
||||
normalized_autov3 = (autov3_value or "").lower()
|
||||
if not normalized_autov3:
|
||||
continue
|
||||
bucket = new_autov3_map.setdefault(normalized_autov3, set())
|
||||
for path in paths:
|
||||
if path:
|
||||
bucket.add(path)
|
||||
|
||||
if excluded_deletes:
|
||||
conn.executemany(
|
||||
"DELETE FROM excluded_models WHERE model_type = ? AND file_path = ?",
|
||||
excluded_deletes,
|
||||
)
|
||||
if excluded_inserts:
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO excluded_models (model_type, file_path) VALUES (?, ?)",
|
||||
excluded_inserts,
|
||||
)
|
||||
autov3_inserts: List[Tuple[str, str, str]] = []
|
||||
autov3_deletes: List[Tuple[str, str, str]] = []
|
||||
|
||||
if all_folders is not None:
|
||||
conn.execute(
|
||||
"DELETE FROM folders WHERE model_type = ?",
|
||||
all_autov3 = set(existing_autov3_map.keys()) | set(new_autov3_map.keys())
|
||||
for autov3_value in all_autov3:
|
||||
existing_paths = existing_autov3_map.get(autov3_value, set())
|
||||
new_paths = new_autov3_map.get(autov3_value, set())
|
||||
|
||||
for path in existing_paths - new_paths:
|
||||
autov3_deletes.append((model_type, autov3_value, path))
|
||||
for path in new_paths - existing_paths:
|
||||
autov3_inserts.append((model_type, autov3_value, path))
|
||||
|
||||
if autov3_deletes:
|
||||
conn.executemany(
|
||||
"DELETE FROM autov3_index WHERE model_type = ? AND autov3 = ? AND file_path = ?",
|
||||
autov3_deletes,
|
||||
)
|
||||
if autov3_inserts:
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO autov3_index (model_type, autov3, file_path) VALUES (?, ?, ?)",
|
||||
autov3_inserts,
|
||||
)
|
||||
|
||||
existing_excluded_rows = conn.execute(
|
||||
"SELECT file_path FROM excluded_models WHERE model_type = ?",
|
||||
(model_type,),
|
||||
)
|
||||
folder_inserts = [
|
||||
(model_type, path) for path in all_folders if path
|
||||
]
|
||||
if folder_inserts:
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO folders (model_type, path) VALUES (?, ?)",
|
||||
folder_inserts,
|
||||
)
|
||||
# Mark the snapshot as having folder data even when the
|
||||
# library has no subfolders, so an empty list is not
|
||||
# mistaken for "never recorded" on load.
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO cache_meta (key, value) VALUES (?, ?)",
|
||||
(f"folders_recorded:{model_type}", "1"),
|
||||
)
|
||||
).fetchall()
|
||||
existing_excluded = {row["file_path"] for row in existing_excluded_rows}
|
||||
new_excluded = {path for path in excluded_models if path}
|
||||
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
excluded_deletes = [
|
||||
(model_type, path)
|
||||
for path in existing_excluded - new_excluded
|
||||
]
|
||||
excluded_inserts = [
|
||||
(model_type, path)
|
||||
for path in new_excluded - existing_excluded
|
||||
]
|
||||
|
||||
if excluded_deletes:
|
||||
conn.executemany(
|
||||
"DELETE FROM excluded_models WHERE model_type = ? AND file_path = ?",
|
||||
excluded_deletes,
|
||||
)
|
||||
if excluded_inserts:
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO excluded_models (model_type, file_path) VALUES (?, ?)",
|
||||
excluded_inserts,
|
||||
)
|
||||
|
||||
if all_folders is not None:
|
||||
conn.execute(
|
||||
"DELETE FROM folders WHERE model_type = ?",
|
||||
(model_type,),
|
||||
)
|
||||
folder_inserts = [
|
||||
(model_type, path) for path in all_folders if path
|
||||
]
|
||||
if folder_inserts:
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO folders (model_type, path) VALUES (?, ?)",
|
||||
folder_inserts,
|
||||
)
|
||||
# Mark the snapshot as having folder data even when the
|
||||
# library has no subfolders, so an empty list is not
|
||||
# mistaken for "never recorded" on load.
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO cache_meta (key, value) VALUES (?, ?)",
|
||||
(f"folders_recorded:{model_type}", "1"),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to persist cache for %s: %s", model_type, exc)
|
||||
|
||||
@@ -650,16 +656,14 @@ class PersistentModelCache:
|
||||
conn.execute(f"ALTER TABLE models ADD COLUMN {column} {definition}")
|
||||
|
||||
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
||||
uri = False
|
||||
path = self._db_path
|
||||
if readonly:
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(path)
|
||||
path = f"file:{path}?mode=ro"
|
||||
uri = True
|
||||
conn = sqlite3.connect(path, check_same_thread=False, uri=uri, detect_types=sqlite3.PARSE_DECLTYPES)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
if readonly and not os.path.exists(self._db_path):
|
||||
raise FileNotFoundError(self._db_path)
|
||||
return connect_cache_db(
|
||||
self._db_path,
|
||||
readonly=readonly,
|
||||
detect_types=sqlite3.PARSE_DECLTYPES,
|
||||
row_factory=sqlite3.Row,
|
||||
)
|
||||
|
||||
def _prepare_model_row(self, model_type: str, item: Dict[str, Any]) -> Tuple[Any, ...]:
|
||||
# Keep `source_*` and the legacy `hf_url` alias consistent no matter
|
||||
|
||||
@@ -19,7 +19,9 @@ import threading
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||
|
||||
from ..utils.cache_db import connect_cache_db
|
||||
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
||||
from ..utils.file_lock import exclusive_lock
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -170,65 +172,98 @@ class PersistentRecipeCache:
|
||||
recipes: List[Dict[str, Any]],
|
||||
json_paths: Optional[Dict[str, str]] = None,
|
||||
image_id_map: Optional[Dict[str, str]] = None,
|
||||
) -> None:
|
||||
skip_if_empty: bool = False,
|
||||
) -> bool:
|
||||
"""Save all recipes to SQLite cache.
|
||||
|
||||
Args:
|
||||
recipes: List of recipe dictionaries to persist.
|
||||
json_paths: Optional mapping of recipe_id -> json_path for file stats.
|
||||
image_id_map: Optional precomputed civitai image_id → recipe_id mapping.
|
||||
skip_if_empty: When True, refuse to replace a non-empty cache with an
|
||||
empty one. This is the storage-level backstop against a scan that
|
||||
silently loses every recipe (unavailable drive / mis-resolved
|
||||
recipes directory): overwriting both deletes the user's data and
|
||||
destroys their only record of it. Intentional full clears (manual
|
||||
rebuild) must pass ``skip_if_empty=False``.
|
||||
|
||||
Returns:
|
||||
``True`` when the write happened, ``False`` when it was skipped.
|
||||
"""
|
||||
if not self.is_enabled():
|
||||
return
|
||||
return False
|
||||
if not self._schema_initialized:
|
||||
self._initialize_schema()
|
||||
if not self._schema_initialized:
|
||||
return
|
||||
return False
|
||||
|
||||
try:
|
||||
with self._db_lock:
|
||||
conn = self._connect()
|
||||
try:
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("BEGIN")
|
||||
# Cross-process serialization: another LoRA Manager instance may
|
||||
# share this settings directory, and a full-table replace is a
|
||||
# read-modify-write that SQLite alone cannot make atomic.
|
||||
with exclusive_lock(self._db_path):
|
||||
conn = self._connect()
|
||||
try:
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("BEGIN")
|
||||
|
||||
# Clear existing data
|
||||
conn.execute("DELETE FROM recipes")
|
||||
if skip_if_empty and not recipes:
|
||||
existing = conn.execute(
|
||||
"SELECT COUNT(*) FROM recipes"
|
||||
).fetchone()
|
||||
if existing and existing[0]:
|
||||
conn.rollback()
|
||||
logger.warning(
|
||||
"Refusing to persist an empty recipe cache: the "
|
||||
"stored cache still holds %d recipe(s). The scan "
|
||||
"found nothing, which usually means the recipes "
|
||||
"path was unavailable or resolved elsewhere; "
|
||||
"keeping the stored cache so the data stays "
|
||||
"recoverable.",
|
||||
existing[0],
|
||||
)
|
||||
return False
|
||||
|
||||
# Prepare and insert all rows
|
||||
recipe_rows = []
|
||||
for recipe in recipes:
|
||||
recipe_id = str(recipe.get("id", ""))
|
||||
if not recipe_id:
|
||||
continue
|
||||
# Clear existing data
|
||||
conn.execute("DELETE FROM recipes")
|
||||
|
||||
json_path = ""
|
||||
if json_paths:
|
||||
json_path = json_paths.get(recipe_id, "")
|
||||
# Prepare and insert all rows
|
||||
recipe_rows = []
|
||||
for recipe in recipes:
|
||||
recipe_id = str(recipe.get("id", ""))
|
||||
if not recipe_id:
|
||||
continue
|
||||
|
||||
row = self._prepare_recipe_row(recipe, json_path)
|
||||
recipe_rows.append(row)
|
||||
json_path = ""
|
||||
if json_paths:
|
||||
json_path = json_paths.get(recipe_id, "")
|
||||
|
||||
if recipe_rows:
|
||||
placeholders = ", ".join(["?"] * len(self._RECIPE_COLUMNS))
|
||||
columns = ", ".join(self._RECIPE_COLUMNS)
|
||||
conn.executemany(
|
||||
f"INSERT INTO recipes ({columns}) VALUES ({placeholders})",
|
||||
recipe_rows,
|
||||
row = self._prepare_recipe_row(recipe, json_path)
|
||||
recipe_rows.append(row)
|
||||
|
||||
if recipe_rows:
|
||||
placeholders = ", ".join(["?"] * len(self._RECIPE_COLUMNS))
|
||||
columns = ", ".join(self._RECIPE_COLUMNS)
|
||||
conn.executemany(
|
||||
f"INSERT INTO recipes ({columns}) VALUES ({placeholders})",
|
||||
recipe_rows,
|
||||
)
|
||||
|
||||
# Persist image_id_map for O(1) lookups on cache load
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO cache_metadata (key, value) VALUES (?, ?)",
|
||||
("image_id_map", json.dumps(image_id_map or {})),
|
||||
)
|
||||
|
||||
# Persist image_id_map for O(1) lookups on cache load
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO cache_metadata (key, value) VALUES (?, ?)",
|
||||
("image_id_map", json.dumps(image_id_map or {})),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
logger.debug("Persisted %d recipes to cache", len(recipe_rows))
|
||||
finally:
|
||||
conn.close()
|
||||
conn.commit()
|
||||
logger.debug("Persisted %d recipes to cache", len(recipe_rows))
|
||||
return True
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to persist recipe cache: %s", exc)
|
||||
return False
|
||||
|
||||
def get_file_stats(self) -> Dict[str, Tuple[float, int]]:
|
||||
"""Return stored file stats for all cached recipes.
|
||||
@@ -486,16 +521,14 @@ class PersistentRecipeCache:
|
||||
logger.warning("Failed to initialize persistent recipe cache schema: %s", exc)
|
||||
|
||||
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
||||
uri = False
|
||||
path = self._db_path
|
||||
if readonly:
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(path)
|
||||
path = f"file:{path}?mode=ro"
|
||||
uri = True
|
||||
conn = sqlite3.connect(path, check_same_thread=False, uri=uri, detect_types=sqlite3.PARSE_DECLTYPES)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
if readonly and not os.path.exists(self._db_path):
|
||||
raise FileNotFoundError(self._db_path)
|
||||
return connect_cache_db(
|
||||
self._db_path,
|
||||
readonly=readonly,
|
||||
detect_types=sqlite3.PARSE_DECLTYPES,
|
||||
row_factory=sqlite3.Row,
|
||||
)
|
||||
|
||||
def _prepare_recipe_row(self, recipe: Dict[str, Any], json_path: str) -> Tuple[Any, ...]:
|
||||
"""Convert a recipe dict to a row tuple for SQLite insertion."""
|
||||
|
||||
@@ -16,6 +16,7 @@ import threading
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||
|
||||
from ..utils.cache_db import connect_cache_db
|
||||
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -633,16 +634,13 @@ class RecipeFTSIndex:
|
||||
|
||||
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
||||
"""Create a database connection."""
|
||||
uri = False
|
||||
path = self._db_path
|
||||
if readonly:
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(path)
|
||||
path = f"file:{path}?mode=ro"
|
||||
uri = True
|
||||
conn = sqlite3.connect(path, check_same_thread=False, uri=uri)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
if readonly and not os.path.exists(self._db_path):
|
||||
raise FileNotFoundError(self._db_path)
|
||||
return connect_cache_db(
|
||||
self._db_path,
|
||||
readonly=readonly,
|
||||
row_factory=sqlite3.Row,
|
||||
)
|
||||
|
||||
def _remove_recipe_locked(self, conn: sqlite3.Connection, recipe_id: str) -> None:
|
||||
"""Remove a recipe entry. Caller must hold the lock."""
|
||||
|
||||
+124
-16
@@ -116,6 +116,12 @@ class RecipeScanner:
|
||||
self._persistent_cache: Optional[PersistentRecipeCache] = None
|
||||
self._civitai_client: Any = None # Lazily initialized from registry
|
||||
self._json_path_map: Dict[str, str] = {} # recipe_id -> json_path
|
||||
# True when the last scan refused to prune the stored cache because
|
||||
# every recorded recipe file was missing (see
|
||||
# :meth:`_initialize_recipe_cache_sync`). Keeps dependent background
|
||||
# work (FTS index) aligned with the stored rows instead of the
|
||||
# intentionally out-of-sync in-memory view.
|
||||
self._prune_skipped: bool = False
|
||||
if lora_scanner:
|
||||
self._lora_scanner = lora_scanner
|
||||
if checkpoint_scanner:
|
||||
@@ -1651,8 +1657,12 @@ class RecipeScanner:
|
||||
'pageType': 'recipes',
|
||||
})
|
||||
self._schedule_post_scan_enrichment()
|
||||
# Schedule FTS index build in background (non-blocking)
|
||||
self._schedule_fts_index_build()
|
||||
# Schedule FTS index build in background (non-blocking). When the
|
||||
# prune was skipped the in-memory cache is intentionally out of sync
|
||||
# with the stored rows, so leave the existing index alone instead of
|
||||
# rebuilding it from the empty view.
|
||||
if not self._prune_skipped:
|
||||
self._schedule_fts_index_build()
|
||||
except Exception as e:
|
||||
logger.error(f"Recipe Scanner: Error initializing cache in background: {e}")
|
||||
# Ensure the cache is never None so the page stops showing the
|
||||
@@ -1723,6 +1733,7 @@ class RecipeScanner:
|
||||
"""
|
||||
loop = None
|
||||
scan_start_time: Optional[float] = None
|
||||
self._prune_skipped = False
|
||||
try:
|
||||
# Ensure cache exists to avoid None reference errors
|
||||
if self._cache is None:
|
||||
@@ -1749,14 +1760,38 @@ class RecipeScanner:
|
||||
logger.warning(f"Recipes directory not found: {recipes_dir}")
|
||||
return self._cache
|
||||
|
||||
# Record which directory the scan actually used. When the Recipes
|
||||
# Storage Path is empty this falls back to the first LoRA root, and
|
||||
# a support reader needs that path to tell a real wipe apart from a
|
||||
# scan that looked somewhere else (see the prune guard below).
|
||||
logger.info(f"Recipe scan directory: {recipes_dir}")
|
||||
|
||||
# Try to load from persistent cache first
|
||||
persisted = self._persistent_cache.load_cache()
|
||||
if persisted:
|
||||
recipes, changed, json_paths = self._reconcile_recipe_cache(
|
||||
persisted, recipes_dir
|
||||
)
|
||||
(
|
||||
recipes,
|
||||
changed,
|
||||
json_paths,
|
||||
skipped_prune_reason,
|
||||
) = self._reconcile_recipe_cache(persisted, recipes_dir)
|
||||
self._json_path_map = json_paths
|
||||
|
||||
if skipped_prune_reason:
|
||||
# Every persisted recipe file vanished at once. That is not a
|
||||
# reliable deletion signal: a drive that did not mount, a
|
||||
# recipes_path that silently fell back to another root, or a
|
||||
# shared cache touched by a second instance all look exactly
|
||||
# like this. Keep the stored cache and skip the prune, so the
|
||||
# only copy of the user's recipes is not destroyed.
|
||||
logger.warning(
|
||||
f"Recipe cache prune skipped: {skipped_prune_reason}. "
|
||||
f"Keeping {len(persisted.raw_data)} stored recipe(s); this "
|
||||
"session reports no recipes until the files are found again."
|
||||
)
|
||||
self._prune_skipped = True
|
||||
return self._cache
|
||||
|
||||
if not changed:
|
||||
# Fast path: use cached data directly
|
||||
logger.info(
|
||||
@@ -1770,7 +1805,10 @@ class RecipeScanner:
|
||||
if self._backfill_source_path_if_needed(recipes, json_paths):
|
||||
self._cache.image_id_map = self._build_image_id_map()
|
||||
self._persistent_cache.save_cache(
|
||||
recipes, json_paths, self._cache.image_id_map
|
||||
recipes,
|
||||
json_paths,
|
||||
self._cache.image_id_map,
|
||||
skip_if_empty=True,
|
||||
)
|
||||
else:
|
||||
# Use persisted map, or rebuild if empty (e.g. first startup
|
||||
@@ -1798,7 +1836,10 @@ class RecipeScanner:
|
||||
self._cache.image_id_map = self._build_image_id_map()
|
||||
# Persist updated cache
|
||||
self._persistent_cache.save_cache(
|
||||
recipes, json_paths, self._cache.image_id_map
|
||||
recipes,
|
||||
json_paths,
|
||||
self._cache.image_id_map,
|
||||
skip_if_empty=True,
|
||||
)
|
||||
return self._cache
|
||||
|
||||
@@ -1825,7 +1866,10 @@ class RecipeScanner:
|
||||
|
||||
# Persist for next startup
|
||||
self._persistent_cache.save_cache(
|
||||
recipes, json_paths, self._cache.image_id_map
|
||||
recipes,
|
||||
json_paths,
|
||||
self._cache.image_id_map,
|
||||
skip_if_empty=True,
|
||||
)
|
||||
|
||||
if report_progress:
|
||||
@@ -1862,7 +1906,7 @@ class RecipeScanner:
|
||||
self,
|
||||
persisted: PersistedRecipeData,
|
||||
recipes_dir: str,
|
||||
) -> Tuple[List[Dict[str, Any]], bool, Dict[str, str]]:
|
||||
) -> Tuple[List[Dict[str, Any]], bool, Dict[str, str], Optional[str]]:
|
||||
"""Reconcile persisted cache with current filesystem state.
|
||||
|
||||
Args:
|
||||
@@ -1870,7 +1914,11 @@ class RecipeScanner:
|
||||
recipes_dir: Path to the recipes directory.
|
||||
|
||||
Returns:
|
||||
Tuple of (recipes list, changed flag, json_paths dict).
|
||||
Tuple of (recipes list, changed flag, json_paths dict,
|
||||
skipped_prune_reason). The last element is ``None`` on a normal
|
||||
reconcile. When it is a string, the scan saw every persisted recipe
|
||||
file disappear at once; the caller must then keep the persisted
|
||||
cache instead of overwriting it. The reason text is user-facing.
|
||||
"""
|
||||
recipes: List[Dict[str, Any]] = []
|
||||
json_paths: Dict[str, str] = {}
|
||||
@@ -1951,12 +1999,67 @@ class RecipeScanner:
|
||||
time.sleep(0)
|
||||
|
||||
# Check for deleted files
|
||||
for json_path in persisted.file_stats.keys():
|
||||
if json_path not in current_files:
|
||||
changed = True
|
||||
logger.debug("Recipe file deleted: %s", json_path)
|
||||
orphaned_stats = [
|
||||
json_path
|
||||
for json_path in persisted.file_stats.keys()
|
||||
if json_path not in current_files
|
||||
]
|
||||
if orphaned_stats:
|
||||
changed = True
|
||||
# This single line plus the resolved scan directory logged by the
|
||||
# caller are the evidence a support reader gets for a recipes path
|
||||
# that moved; the per-file lines stay at debug to avoid flooding.
|
||||
if len(orphaned_stats) > 10:
|
||||
logger.info(
|
||||
f"Recipe reconcile: {len(orphaned_stats)} of "
|
||||
f"{len(persisted.file_stats)} cached recipe file(s) are not in "
|
||||
f"{recipes_dir} (first: {orphaned_stats[0]}, "
|
||||
f"last: {orphaned_stats[-1]})"
|
||||
)
|
||||
else:
|
||||
for json_path in orphaned_stats:
|
||||
logger.debug("Recipe file deleted: %s", json_path)
|
||||
|
||||
return recipes, changed, json_paths
|
||||
skipped_prune_reason: Optional[str] = None
|
||||
if not current_files and persisted.file_stats:
|
||||
metadata_is_coherent = self._persisted_metadata_is_coherent(persisted)
|
||||
if metadata_is_coherent:
|
||||
skipped_prune_reason = (
|
||||
f"every recipe file recorded in the cache "
|
||||
f"({len(persisted.file_stats)}) is missing from {recipes_dir}"
|
||||
)
|
||||
else:
|
||||
# The stored row set and its recorded file stats disagree, so
|
||||
# this cache is stale rather than a faithful record of recipes
|
||||
# that have just gone missing. Pruning it is safe.
|
||||
logger.info(
|
||||
f"Recipe reconcile: stored cache is inconsistent "
|
||||
f"({len(persisted.raw_data)} row(s) vs "
|
||||
f"{len(persisted.file_stats)} file record(s)); falling back "
|
||||
"to a normal prune."
|
||||
)
|
||||
|
||||
return recipes, changed, json_paths, skipped_prune_reason
|
||||
|
||||
@staticmethod
|
||||
def _persisted_metadata_is_coherent(persisted: PersistedRecipeData) -> bool:
|
||||
"""Return True when the stored rows and their file stats describe one set.
|
||||
|
||||
The prune guard treats "no recipe files found" as a signal that the
|
||||
directory moved out from under us, which is only meaningful when the
|
||||
stored cache is a faithful record of recipes that exist on disk. A cache
|
||||
whose row set and file-stat set have diverged (left behind by an older
|
||||
reconcile) carries recipes that were already orphaned, so it is not
|
||||
evidence of a fresh disappearance.
|
||||
"""
|
||||
stats_ids = {
|
||||
os.path.basename(json_path)[: -len(".recipe.json")]
|
||||
for json_path in persisted.file_stats
|
||||
if os.path.basename(json_path).lower().endswith(".recipe.json")
|
||||
}
|
||||
rows_ids = {str(recipe.get("id", "")) for recipe in persisted.raw_data}
|
||||
rows_ids.discard("")
|
||||
return bool(rows_ids) and rows_ids == stats_ids
|
||||
|
||||
# Metadata key recording that the one-shot source_path backfill has run.
|
||||
_SOURCE_PATH_BACKFILL_MARKER = "source_path_backfilled"
|
||||
@@ -2626,6 +2729,10 @@ class RecipeScanner:
|
||||
try:
|
||||
# Invalidate persistent cache so the sync path does a
|
||||
# full directory scan instead of reconciling stale data.
|
||||
# This is the deliberate escape hatch from the
|
||||
# all-missing prune guard: an explicit user rebuild is
|
||||
# allowed to clear the stored cache, while an implicit
|
||||
# startup scan is not.
|
||||
if self._persistent_cache:
|
||||
self._persistent_cache.save_cache([], {})
|
||||
self._json_path_map = {}
|
||||
@@ -2656,7 +2763,8 @@ class RecipeScanner:
|
||||
|
||||
# Schedule non-blocking background work
|
||||
self._schedule_post_scan_enrichment()
|
||||
self._schedule_fts_index_build()
|
||||
if not self._prune_skipped:
|
||||
self._schedule_fts_index_build()
|
||||
|
||||
return cast(RecipeCache, self._cache)
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ from ..utils.constants import (
|
||||
from ..utils.preview_selection import VALID_MATURE_BLUR_LEVELS
|
||||
from ..utils.settings_paths import (
|
||||
APP_NAME,
|
||||
_portable_env_override,
|
||||
ensure_settings_file,
|
||||
get_legacy_settings_path,
|
||||
get_settings_dir_override,
|
||||
@@ -172,13 +173,23 @@ class SettingsManager:
|
||||
self._check_environment_variables()
|
||||
self._collect_configuration_warnings()
|
||||
|
||||
if (
|
||||
os.environ.get("LORA_MANAGER_PORTABLE", "0") == "1"
|
||||
and not is_settings_dir_pinned()
|
||||
):
|
||||
portable_override = _portable_env_override()
|
||||
if portable_override is True and not is_settings_dir_pinned():
|
||||
if not self.settings.get("use_portable_settings"):
|
||||
self.settings["use_portable_settings"] = True
|
||||
self._save_settings()
|
||||
elif portable_override is False and self.settings.get(
|
||||
"use_portable_settings"
|
||||
):
|
||||
# Explicit opt-out from a persisted portable mode: clear the flag so
|
||||
# later runs go back to the shared settings directory instead of
|
||||
# requiring a manual edit of settings.json.
|
||||
logger.info(
|
||||
"Clearing the persisted portable-mode flag because %s=0",
|
||||
"LORA_MANAGER_PORTABLE",
|
||||
)
|
||||
self.settings["use_portable_settings"] = False
|
||||
self._save_settings()
|
||||
|
||||
if self._needs_initial_save:
|
||||
self._save_settings()
|
||||
|
||||
@@ -20,6 +20,7 @@ import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
from ..utils.cache_db import connect_cache_db
|
||||
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -677,16 +678,13 @@ class TagFTSIndex:
|
||||
|
||||
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
||||
"""Create a database connection."""
|
||||
uri = False
|
||||
path = self._db_path
|
||||
if readonly:
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(path)
|
||||
path = f"file:{path}?mode=ro"
|
||||
uri = True
|
||||
conn = sqlite3.connect(path, check_same_thread=False, uri=uri)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
if readonly and not os.path.exists(self._db_path):
|
||||
raise FileNotFoundError(self._db_path)
|
||||
return connect_cache_db(
|
||||
self._db_path,
|
||||
readonly=readonly,
|
||||
row_factory=sqlite3.Row,
|
||||
)
|
||||
|
||||
def _build_fts_query(self, query: str) -> str:
|
||||
"""Build an FTS5 query string with prefix matching.
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Shared SQLite connection setup for LoRA Manager cache databases.
|
||||
|
||||
Cache databases live under the settings directory (``cache/model/<library>.sqlite``,
|
||||
``cache/recipe/<library>.sqlite``, ``cache/fts/*.sqlite``). With portable mode or a
|
||||
pinned ``LORA_MANAGER_SETTINGS_DIR`` off, that directory is shared by every ComfyUI
|
||||
instance on the machine, so two processes can open the same cache file at once.
|
||||
|
||||
SQLite serializes writers, but the default ``timeout`` is 5 seconds: a second
|
||||
instance that writes while the first is mid-transaction fails with "database is
|
||||
locked". These settings make concurrent access wait instead of failing, and keep
|
||||
the write path in WAL so readers are never blocked by a writer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from typing import Any
|
||||
|
||||
# How long a connection waits for a competing writer before raising.
|
||||
CONCURRENT_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
# PRAGMAs applied to every cache connection.
|
||||
#
|
||||
# ``busy_timeout`` mirrors the connection timeout so a busy database is retried
|
||||
# inside SQLite rather than surfacing as an immediate error. ``synchronous=NORMAL``
|
||||
# is the documented companion of WAL: still crash-safe, far fewer fsyncs.
|
||||
_TUNING_PRAGMAS = (
|
||||
"PRAGMA busy_timeout = 30000",
|
||||
"PRAGMA synchronous = NORMAL",
|
||||
)
|
||||
|
||||
|
||||
def connect_cache_db(
|
||||
path: str,
|
||||
*,
|
||||
readonly: bool = False,
|
||||
uri: bool = False,
|
||||
detect_types: int = 0,
|
||||
row_factory: Any = None,
|
||||
) -> sqlite3.Connection:
|
||||
"""Open a cache database with multi-instance-friendly settings.
|
||||
|
||||
Args:
|
||||
path: Database path, or a ``file:`` URI when *uri* is True.
|
||||
readonly: Open through a read-only URI. Callers still pass the
|
||||
plain path; the ``mode=ro`` suffix is added here. The
|
||||
write-oriented tuning pragmas are skipped in that case so a
|
||||
read-only connection never attempts to change the file.
|
||||
uri: Treat *path* as a SQLite URI.
|
||||
detect_types: Forwarded to :func:`sqlite3.connect`.
|
||||
row_factory: Optional ``row_factory`` for the connection.
|
||||
|
||||
Returns:
|
||||
A configured :class:`sqlite3.Connection`.
|
||||
"""
|
||||
if readonly:
|
||||
if not uri and not path.startswith("file:"):
|
||||
path = f"file:{path}?mode=ro"
|
||||
uri = True
|
||||
|
||||
conn = sqlite3.connect(
|
||||
path,
|
||||
check_same_thread=False,
|
||||
uri=uri,
|
||||
detect_types=detect_types,
|
||||
timeout=CONCURRENT_TIMEOUT_SECONDS,
|
||||
)
|
||||
if row_factory is not None:
|
||||
conn.row_factory = row_factory
|
||||
|
||||
try:
|
||||
for pragma in _TUNING_PRAGMAS:
|
||||
# A read-only connection may reject write PRAGMAs; they are not
|
||||
# needed there anyway.
|
||||
conn.execute(pragma)
|
||||
except sqlite3.Error:
|
||||
# Tuning is best-effort: a connection that cannot set pragmas still
|
||||
# works, just without the concurrency headroom.
|
||||
pass
|
||||
|
||||
return conn
|
||||
@@ -2,7 +2,7 @@ import inspect
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional
|
||||
from typing import TYPE_CHECKING, Any, Dict, Mapping, MutableMapping, Optional
|
||||
|
||||
from ..recipes.constants import GEN_PARAM_KEYS
|
||||
from ..services.metadata_service import get_default_metadata_provider, get_metadata_provider
|
||||
@@ -13,9 +13,20 @@ from ..services.downloader import get_downloader
|
||||
from ..utils.constants import SUPPORTED_MEDIA_EXTENSIONS
|
||||
from ..utils.exif_utils import ExifUtils
|
||||
from ..utils.metadata_manager import MetadataManager
|
||||
from ..utils.video_metadata import get_video_dimensions
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Placeholder dimensions written when the real ones cannot be determined.
|
||||
# Kept for backwards compatibility with pre-existing metadata entries.
|
||||
_DEFAULT_MEDIA_WIDTH = 720
|
||||
_DEFAULT_MEDIA_HEIGHT = 1280
|
||||
|
||||
# Example metadata entries carry a marker: ``customImages`` use their ``id``
|
||||
# while ``images`` use the positional index. Either way the marker must be a
|
||||
# plain filename-safe token, never a path fragment.
|
||||
_ENTRY_MARKER_PATTERN = re.compile(r"^(?:custom_|image_)?([^./\\]+)$")
|
||||
|
||||
_preview_service = PreviewAssetService(
|
||||
metadata_manager=MetadataManager,
|
||||
downloader_factory=get_downloader,
|
||||
@@ -66,6 +77,141 @@ def _build_metadata_sync_service(settings_manager: "SettingsManager") -> Metadat
|
||||
)
|
||||
|
||||
|
||||
def _read_media_dimensions(path: str, is_video: bool) -> tuple[int, int]:
|
||||
"""Return ``(width, height)`` for an example image or video file.
|
||||
|
||||
Videos are read from their container headers (PIL cannot open them) so the
|
||||
showcase viewer sizes the gallery to the real aspect ratio. Falls back to
|
||||
the legacy ``720x1280`` placeholder when the dimensions cannot be
|
||||
determined — e.g. an unreadable file or an exotic codec — which only
|
||||
affects the displayed aspect ratio, never the file itself.
|
||||
"""
|
||||
|
||||
dimensions = None
|
||||
|
||||
if is_video:
|
||||
dimensions = get_video_dimensions(path)
|
||||
else:
|
||||
try:
|
||||
from PIL import Image
|
||||
|
||||
if os.path.exists(path):
|
||||
with Image.open(path) as img:
|
||||
dimensions = img.size
|
||||
except Exception:
|
||||
dimensions = None
|
||||
|
||||
if dimensions:
|
||||
width, height = dimensions
|
||||
if width > 0 and height > 0:
|
||||
return int(width), int(height)
|
||||
|
||||
return _DEFAULT_MEDIA_WIDTH, _DEFAULT_MEDIA_HEIGHT
|
||||
|
||||
|
||||
def _is_video_entry(file_path: Optional[str], entry: Mapping[str, Any]) -> bool:
|
||||
"""Return True when an example entry points at a video file.
|
||||
|
||||
The local file extension wins over the recorded ``type`` because files in
|
||||
the wild are frequently mislabelled (animated WebP saved as ``.mp4``);
|
||||
``_read_media_dimensions`` handles that correctly either way.
|
||||
"""
|
||||
|
||||
if file_path:
|
||||
ext = os.path.splitext(file_path)[1].lower()
|
||||
if ext in SUPPORTED_MEDIA_EXTENSIONS["videos"]:
|
||||
return True
|
||||
if ext in SUPPORTED_MEDIA_EXTENSIONS["images"]:
|
||||
return False
|
||||
return str(entry.get("type", "")).lower() == "video"
|
||||
|
||||
|
||||
def _resolve_local_file(
|
||||
entry: Mapping[str, Any],
|
||||
index: int,
|
||||
local_files: Mapping[str, str],
|
||||
) -> Optional[str]:
|
||||
"""Map a metadata entry onto its example file inside the model folder.
|
||||
|
||||
Reads the entry's own marker (``id`` for ``customImages``, positional
|
||||
``index`` for ``images``) with an anchored regex, so the identifier can
|
||||
never bleed into a neighbouring filename the way a prefix comparison can.
|
||||
"""
|
||||
|
||||
marker = entry.get("id")
|
||||
if not isinstance(marker, str) or not marker:
|
||||
marker = str(index)
|
||||
|
||||
match = _ENTRY_MARKER_PATTERN.fullmatch(marker)
|
||||
if not match:
|
||||
return None
|
||||
|
||||
return local_files.get(match.group(1))
|
||||
|
||||
|
||||
def repair_local_video_dimensions(
|
||||
metadata: MutableMapping[str, Any],
|
||||
local_files: Mapping[str, str],
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
) -> int:
|
||||
"""Backfill real video dimensions for an entry that has local files.
|
||||
|
||||
Only entries with an empty ``url`` are considered: those have no remote
|
||||
source, so the local file is the single source of truth for their size and
|
||||
rewriting them cannot discard API-supplied data. Entries whose dimensions
|
||||
already match the file are left byte-identical.
|
||||
|
||||
Args:
|
||||
metadata: Raw metadata payload (mutated in place unless ``dry_run``).
|
||||
local_files: ``{identifier: path}`` for files present in the model's
|
||||
example folder, where the identifier is the entry's ``id`` (for
|
||||
``customImages``) or its positional index (for ``images``).
|
||||
dry_run: Count the fixes without mutating ``metadata``.
|
||||
|
||||
Returns:
|
||||
The number of entries that were (or would be) repaired.
|
||||
"""
|
||||
|
||||
civitai = metadata.get("civitai")
|
||||
if not isinstance(civitai, dict):
|
||||
return 0
|
||||
|
||||
repaired = 0
|
||||
|
||||
for key in ("customImages", "images"):
|
||||
entries = civitai.get(key)
|
||||
if not isinstance(entries, list) or not entries:
|
||||
continue
|
||||
|
||||
for index, entry in enumerate(entries):
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
if entry.get("url", "") != "":
|
||||
# Remote-backed entry: never rebuilt from local state.
|
||||
continue
|
||||
|
||||
file_path = _resolve_local_file(entry, index, local_files)
|
||||
if not file_path or not os.path.isfile(file_path):
|
||||
continue
|
||||
|
||||
dimensions = _read_media_dimensions(
|
||||
file_path, _is_video_entry(file_path, entry)
|
||||
)
|
||||
width, height = dimensions
|
||||
if width <= 0 or height <= 0:
|
||||
continue
|
||||
if entry.get("width") == width and entry.get("height") == height:
|
||||
continue
|
||||
|
||||
if not dry_run:
|
||||
entry["width"] = width
|
||||
entry["height"] = height
|
||||
repaired += 1
|
||||
|
||||
return repaired
|
||||
|
||||
|
||||
def _get_metadata_sync_service() -> MetadataSyncService:
|
||||
"""Return the shared metadata sync service, initialising it lazily."""
|
||||
|
||||
@@ -231,28 +377,20 @@ class MetadataUpdater:
|
||||
file_ext = os.path.splitext(path)[1].lower()
|
||||
is_video = file_ext in SUPPORTED_MEDIA_EXTENSIONS['videos']
|
||||
|
||||
width, height = _read_media_dimensions(path, is_video)
|
||||
|
||||
# Create image metadata entry
|
||||
image_entry = {
|
||||
"url": "", # Empty URL as required
|
||||
"nsfwLevel": 0,
|
||||
"width": 720, # Default dimensions
|
||||
"height": 1280,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"type": "video" if is_video else "image",
|
||||
"meta": None,
|
||||
"hasMeta": False,
|
||||
"hasPositivePrompt": False
|
||||
}
|
||||
|
||||
# If it's an image, try to get actual dimensions (optional enhancement)
|
||||
try:
|
||||
from PIL import Image
|
||||
if not is_video and os.path.exists(path):
|
||||
with Image.open(path) as img:
|
||||
image_entry["width"], image_entry["height"] = img.size
|
||||
except:
|
||||
# If PIL fails or is unavailable, use default dimensions
|
||||
pass
|
||||
|
||||
images.append(image_entry)
|
||||
|
||||
# Update the model's civitai.images field
|
||||
@@ -322,13 +460,15 @@ class MetadataUpdater:
|
||||
file_ext = os.path.splitext(path)[1].lower()
|
||||
is_video = file_ext in SUPPORTED_MEDIA_EXTENSIONS['videos']
|
||||
|
||||
width, height = _read_media_dimensions(path, is_video)
|
||||
|
||||
# Create image metadata entry
|
||||
image_entry = {
|
||||
"url": "", # Empty URL as requested
|
||||
"id": short_id,
|
||||
"nsfwLevel": 0,
|
||||
"width": 720, # Default dimensions
|
||||
"height": 1280,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"type": "video" if is_video else "image",
|
||||
"meta": None,
|
||||
"hasMeta": False,
|
||||
@@ -353,16 +493,6 @@ class MetadataUpdater:
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to extract metadata from {os.path.basename(path)}: {e}")
|
||||
|
||||
# If it's an image, try to get actual dimensions
|
||||
try:
|
||||
from PIL import Image
|
||||
if not is_video and os.path.exists(path):
|
||||
with Image.open(path) as img:
|
||||
image_entry["width"], image_entry["height"] = img.size
|
||||
except:
|
||||
# If PIL fails or is unavailable, use default dimensions
|
||||
pass
|
||||
|
||||
# Append to existing customImages array
|
||||
custom_images.append(image_entry)
|
||||
|
||||
|
||||
@@ -15,12 +15,20 @@ from ..utils.example_images_paths import (
|
||||
)
|
||||
from ..utils.metadata_manager import MetadataManager
|
||||
from ..utils.example_images_processor import ExampleImagesProcessor
|
||||
from ..utils.example_images_metadata import update_cache_from_metadata
|
||||
from ..utils.example_images_metadata import (
|
||||
repair_local_video_dimensions,
|
||||
update_cache_from_metadata,
|
||||
)
|
||||
from ..utils.constants import SUPPORTED_MEDIA_EXTENSIONS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CURRENT_NAMING_VERSION = 2 # Increment this when naming conventions change
|
||||
CURRENT_NAMING_VERSION = 3 # Increment this when naming conventions change
|
||||
|
||||
# Example files worth inspecting during the dimension repair.
|
||||
_REPAIRABLE_EXTENSIONS = frozenset(
|
||||
SUPPORTED_MEDIA_EXTENSIONS["images"] + SUPPORTED_MEDIA_EXTENSIONS["videos"]
|
||||
)
|
||||
|
||||
|
||||
class _SettingsProxy:
|
||||
@@ -185,6 +193,9 @@ class ExampleImagesMigration:
|
||||
if from_version < 2 and to_version >= 2:
|
||||
await ExampleImagesMigration._migrate_to_v2(model_folders)
|
||||
|
||||
if from_version < 3 and to_version >= 3:
|
||||
await ExampleImagesMigration._migrate_to_v3(example_images_path, model_folders)
|
||||
|
||||
# Update version in progress file
|
||||
progress_file = os.path.join(example_images_path, '.download_progress.json')
|
||||
try:
|
||||
@@ -438,3 +449,136 @@ class ExampleImagesMigration:
|
||||
migration_errors += 1
|
||||
|
||||
logger.info(f"Migration to v2 complete: migrated {count} custom examples across {updated_models} models with {migration_errors} errors")
|
||||
|
||||
@staticmethod
|
||||
def _build_local_file_map(folder):
|
||||
"""Map entry markers to their files inside a model's example folder.
|
||||
|
||||
Keys are the marker alone (``custom_<id>`` → ``<id>``,
|
||||
``image_<index>`` → ``<index>``) so they line up with the metadata
|
||||
entries' ``id``/positional index without any prefix ambiguity.
|
||||
"""
|
||||
|
||||
local_files = {}
|
||||
try:
|
||||
entries = os.listdir(folder)
|
||||
except OSError as exc:
|
||||
logger.debug("Could not list example folder %s: %s", folder, exc)
|
||||
return local_files
|
||||
|
||||
for name in entries:
|
||||
stem, ext = os.path.splitext(name)
|
||||
if ext.lower() not in _REPAIRABLE_EXTENSIONS:
|
||||
continue
|
||||
if stem.startswith("custom_"):
|
||||
local_files[stem[len("custom_"):]] = os.path.join(folder, name)
|
||||
elif stem.startswith("image_"):
|
||||
local_files[stem[len("image_"):]] = os.path.join(folder, name)
|
||||
|
||||
return local_files
|
||||
|
||||
@staticmethod
|
||||
async def _find_scanner_for_hash(model_hash):
|
||||
"""Return the scanner owning ``model_hash``, or ``None``."""
|
||||
|
||||
lora_scanner = await ServiceRegistry.get_lora_scanner()
|
||||
checkpoint_scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||
embedding_scanner = await ServiceRegistry.get_embedding_scanner()
|
||||
|
||||
for scanner in (lora_scanner, checkpoint_scanner, embedding_scanner):
|
||||
if scanner is None:
|
||||
continue
|
||||
try:
|
||||
if scanner.has_hash(model_hash):
|
||||
return scanner
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug("has_hash check failed for %s: %s", type(scanner).__name__, exc)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
async def _migrate_to_v3(example_images_path, model_folders):
|
||||
"""Backfill real dimensions for locally imported example videos.
|
||||
|
||||
Imported videos were stored with a hardcoded ``720x1280`` placeholder
|
||||
(issue #1115), so landscape clips were rendered inside a portrait
|
||||
container. Only entries with an empty ``url`` are touched — those have
|
||||
no remote source, which makes the local file authoritative and the
|
||||
rewrite lossless. Entries already carrying the right size are left
|
||||
untouched, so re-running this migration is a no-op.
|
||||
|
||||
This runs once per library via the ``naming_version`` gate in
|
||||
``run_migrations``; it is deliberately not wired into any request path.
|
||||
"""
|
||||
|
||||
repaired_entries = 0
|
||||
updated_models = 0
|
||||
migration_errors = 0
|
||||
|
||||
logger.info(
|
||||
"Starting v3 migration (local example video dimensions) for %d model folders",
|
||||
len(model_folders),
|
||||
)
|
||||
|
||||
for folder in model_folders:
|
||||
try:
|
||||
model_hash = os.path.basename(folder)
|
||||
if not model_hash or len(model_hash) != 64:
|
||||
continue
|
||||
|
||||
local_files = ExampleImagesMigration._build_local_file_map(folder)
|
||||
if not local_files:
|
||||
continue
|
||||
|
||||
scanner = await ExampleImagesMigration._find_scanner_for_hash(model_hash)
|
||||
if scanner is None:
|
||||
logger.debug(
|
||||
"Model %s not found in any scanner cache, skipping dimension repair",
|
||||
model_hash,
|
||||
)
|
||||
continue
|
||||
|
||||
cache = await scanner.get_cached_data()
|
||||
model_data = None
|
||||
for item in cache.raw_data:
|
||||
if item.get("sha256") == model_hash:
|
||||
model_data = item
|
||||
break
|
||||
|
||||
if not model_data:
|
||||
continue
|
||||
|
||||
file_path = model_data.get("file_path")
|
||||
if not file_path:
|
||||
continue
|
||||
|
||||
payload = await MetadataManager.load_metadata_payload(file_path)
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
|
||||
repaired = repair_local_video_dimensions(payload, local_files)
|
||||
if repaired <= 0:
|
||||
continue
|
||||
|
||||
# The model cache shape differs from the on-disk payload, so
|
||||
# persist the file first and let the cache sync re-read it.
|
||||
await MetadataManager.save_metadata(file_path, payload)
|
||||
await update_cache_from_metadata(scanner, file_path, payload)
|
||||
|
||||
repaired_entries += repaired
|
||||
updated_models += 1
|
||||
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"Failed to repair example video dimensions for %s: %s",
|
||||
folder,
|
||||
exc,
|
||||
)
|
||||
migration_errors += 1
|
||||
|
||||
logger.info(
|
||||
"Migration to v3 complete: repaired %d example entr(ies) across %d model(s) "
|
||||
"with %d error(s)",
|
||||
repaired_entries,
|
||||
updated_models,
|
||||
migration_errors,
|
||||
)
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Cross-process advisory locking for shared LoRA Manager state.
|
||||
|
||||
Two LoRA Manager processes (the ComfyUI plugin and a standalone server, or two
|
||||
ComfyUI installs pointed at the same settings directory) can open the same cache
|
||||
database. SQLite serializes individual statements, but it cannot make a
|
||||
read-modify-write *sequence* atomic across processes: two full-table cache
|
||||
replacements can interleave so that one process's snapshot overwrites the
|
||||
other's.
|
||||
|
||||
This module provides a small advisory file lock for those sequences. It is
|
||||
deliberately non-fatal: if locking is unavailable or the wait times out, callers
|
||||
keep working with SQLite's own ``busy_timeout`` as the fallback.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# How long to wait for another process to release the lock before giving up.
|
||||
DEFAULT_LOCK_TIMEOUT_SECONDS = 30.0
|
||||
_POLL_INTERVAL_SECONDS = 0.05
|
||||
|
||||
# Windows byte-range locks; fcntl.flock on POSIX.
|
||||
try: # pragma: no cover - platform dependent
|
||||
import fcntl
|
||||
except ImportError: # pragma: no cover - Windows
|
||||
fcntl = None # type: ignore[assignment]
|
||||
|
||||
try: # pragma: no cover - Windows only
|
||||
import msvcrt
|
||||
except ImportError: # pragma: no cover - POSIX
|
||||
msvcrt = None # type: ignore[assignment]
|
||||
|
||||
|
||||
class FileLockUnavailable(RuntimeError):
|
||||
"""Raised when the lock could not be acquired within the timeout."""
|
||||
|
||||
|
||||
def lock_path_for(db_path: str) -> str:
|
||||
"""Return the sibling lock file path used for *db_path*."""
|
||||
absolute = os.path.abspath(db_path)
|
||||
directory = os.path.dirname(absolute)
|
||||
if not directory:
|
||||
raise ValueError(f"Cannot derive a lock directory from {db_path!r}")
|
||||
return os.path.join(directory, f".{os.path.basename(absolute)}.lock")
|
||||
|
||||
|
||||
class CrossProcessLock:
|
||||
"""A best-effort advisory lock backed by a lock file.
|
||||
|
||||
The lock file is a sibling of the guarded resource and is never deleted:
|
||||
unlinking it would let a second process create a fresh inode and lock that
|
||||
instead, defeating mutual exclusion.
|
||||
"""
|
||||
|
||||
def __init__(self, path: str, timeout: float = DEFAULT_LOCK_TIMEOUT_SECONDS):
|
||||
self.path = path
|
||||
self.timeout = timeout
|
||||
self._handle = None
|
||||
|
||||
def acquire(self) -> bool:
|
||||
"""Try to take the lock, waiting up to ``timeout`` seconds.
|
||||
|
||||
Returns:
|
||||
True when the lock is held (including when another lock is already
|
||||
held by *this* process — the calls are not reentrant, so callers must
|
||||
not nest them). False when locking is unsupported or timed out; the
|
||||
caller should proceed and rely on the SQLite busy timeout instead.
|
||||
"""
|
||||
if fcntl is None and msvcrt is None: # pragma: no cover - exotic platform
|
||||
return False
|
||||
|
||||
os.makedirs(os.path.dirname(self.path), exist_ok=True)
|
||||
try:
|
||||
handle = open(self.path, "a+b")
|
||||
except OSError as exc:
|
||||
logger.debug("Could not open lock file %s: %s", self.path, exc)
|
||||
return False
|
||||
|
||||
deadline = time.monotonic() + max(0.0, self.timeout)
|
||||
while True:
|
||||
if self._try_lock(handle):
|
||||
self._handle = handle
|
||||
return True
|
||||
if time.monotonic() >= deadline:
|
||||
handle.close()
|
||||
return False
|
||||
time.sleep(_POLL_INTERVAL_SECONDS)
|
||||
|
||||
def release(self) -> None:
|
||||
"""Release the lock if held. Safe to call more than once."""
|
||||
handle = self._handle
|
||||
if handle is None:
|
||||
return
|
||||
self._handle = None
|
||||
try:
|
||||
self._unlock(handle)
|
||||
except OSError as exc: # pragma: no cover - defensive
|
||||
logger.debug("Failed to release lock %s: %s", self.path, exc)
|
||||
finally:
|
||||
try:
|
||||
handle.close()
|
||||
except OSError: # pragma: no cover - defensive
|
||||
pass
|
||||
|
||||
def __enter__(self) -> "CrossProcessLock":
|
||||
self.acquire()
|
||||
return self
|
||||
|
||||
def __exit__(self, *_exc_info: object) -> None:
|
||||
self.release()
|
||||
|
||||
# -- platform primitives -------------------------------------------------
|
||||
|
||||
def _try_lock(self, handle) -> bool:
|
||||
if fcntl is not None:
|
||||
try:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
if msvcrt is not None: # pragma: no cover - Windows
|
||||
try:
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
return False
|
||||
|
||||
def _unlock(self, handle) -> None:
|
||||
if fcntl is not None:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||
return
|
||||
if msvcrt is not None: # pragma: no cover - Windows
|
||||
handle.seek(0)
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
|
||||
|
||||
def exclusive_lock(db_path: str, timeout: float = DEFAULT_LOCK_TIMEOUT_SECONDS):
|
||||
"""Return a :class:`CrossProcessLock` for the database at *db_path*."""
|
||||
return CrossProcessLock(lock_path_for(db_path), timeout=timeout)
|
||||
@@ -174,12 +174,42 @@ def ensure_settings_file(logger: Optional[logging.Logger] = None) -> str:
|
||||
return target_path
|
||||
|
||||
|
||||
def _portable_env_override() -> Optional[bool]:
|
||||
"""Return the portable mode forced by ``LORA_MANAGER_PORTABLE``, if any.
|
||||
|
||||
Returns:
|
||||
``True`` when the variable enables portable mode, ``False`` when it is
|
||||
explicitly set to ``"0"``, and ``None`` when it is unset or holds some
|
||||
other value (in which case the persisted settings flag decides).
|
||||
"""
|
||||
|
||||
raw = os.environ.get(_LM_PORTABLE_ENV)
|
||||
if raw is None:
|
||||
return None
|
||||
if raw == "1":
|
||||
return True
|
||||
if raw == "0":
|
||||
return False
|
||||
return None
|
||||
|
||||
|
||||
def _should_use_portable_settings(path: str, logger: logging.Logger) -> bool:
|
||||
"""Return ``True`` when the env var forces it or the settings file enables it."""
|
||||
|
||||
if os.environ.get(_LM_PORTABLE_ENV, "0") == "1":
|
||||
override = _portable_env_override()
|
||||
if override is True:
|
||||
logger.debug("Portable mode enabled via %s", _LM_PORTABLE_ENV)
|
||||
return True
|
||||
if override is False:
|
||||
# Explicit opt-out. Without this, a single `LORA_MANAGER_PORTABLE=1`
|
||||
# run would pin the shared plugin settings.json to portable mode
|
||||
# forever, with no way back except editing that file by hand.
|
||||
logger.info(
|
||||
"Portable mode disabled via %s=%s",
|
||||
_LM_PORTABLE_ENV,
|
||||
os.environ.get(_LM_PORTABLE_ENV, ""),
|
||||
)
|
||||
return False
|
||||
|
||||
if not os.path.exists(path):
|
||||
return False
|
||||
|
||||
@@ -0,0 +1,623 @@
|
||||
"""Read intrinsic dimensions from video containers without external tooling.
|
||||
|
||||
PIL cannot open ``.mp4``/``.webm`` files, so example videos imported through
|
||||
the "Add examples" flow used to fall back to a hardcoded ``720x1280`` (portrait)
|
||||
entry, which forced the showcase viewer to letterbox landscape videos.
|
||||
|
||||
This module reads the dimensions out of the container headers themselves:
|
||||
|
||||
* ISO base media files (``.mp4``/``.mov``/``.m4v``) — ``moov/trak/tkhd``,
|
||||
falling back to the sample description of the video track.
|
||||
* WebM/Matroska (``.webm``/``.mkv``) — ``Segment/Tracks/TrackEntry/Video``
|
||||
``PixelWidth``/``PixelHeight``.
|
||||
* Animated WebP (``RIFF``/``WEBP``) — handled because users routinely save
|
||||
animated examples with a video extension.
|
||||
|
||||
The container signature decides which reader runs, so a mislabelled file
|
||||
(a ``.mp4`` that is really WebM) still reports the right dimensions.
|
||||
|
||||
Both readers stream over the file: only container headers are read, so a
|
||||
multi-gigabyte ``mdat`` is never pulled into memory (it is seeked past).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import struct
|
||||
from typing import BinaryIO, Iterator, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ISO_MEDIA_EXTENSIONS = frozenset({".mp4", ".m4v", ".mov"})
|
||||
EBML_MEDIA_EXTENSIONS = frozenset({".webm", ".mkv"})
|
||||
|
||||
_EBML_MAGIC = b"\x1a\x45\xdf\xa3"
|
||||
|
||||
# Cap recursion into nesting containers so a crafted/corrupt file cannot blow
|
||||
# the Python stack.
|
||||
_MAX_BOX_DEPTH = 12
|
||||
_MAX_EBML_DEPTH = 12
|
||||
|
||||
# Header structs (``tkhd``, sample entries) are tiny; guard against a bogus
|
||||
# size claiming the whole file.
|
||||
_MAX_HEADER_PAYLOAD = 1024 * 1024
|
||||
|
||||
_WIDTH_HEIGHT_UNSET = (0, 0)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=4096)
|
||||
def _get_video_dimensions_cached(
|
||||
path: str, _mtime_ns: int, _size: int
|
||||
) -> Optional[Tuple[int, int]]:
|
||||
"""Return ``(width, height)`` for ``path``, or ``None`` on any failure.
|
||||
|
||||
``_mtime_ns`` and ``_size`` participate in the cache key only so a replaced
|
||||
file is re-probed; they are never read by the parser.
|
||||
"""
|
||||
try:
|
||||
return _read_video_dimensions(path)
|
||||
except Exception:
|
||||
logger.debug("Failed to read video dimensions for %s", path, exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def _read_video_dimensions(path: str) -> Optional[Tuple[int, int]]:
|
||||
"""Dispatch to the ISO or EBML reader based on the container's magic bytes.
|
||||
|
||||
Real libraries contain files whose extension lies about their container
|
||||
(a ``.mp4`` that is really WebM, typically), so the sniffed signature wins
|
||||
and the extension is only a fallback.
|
||||
"""
|
||||
|
||||
ext = os.path.splitext(path)[1].lower()
|
||||
file_size = os.path.getsize(path)
|
||||
|
||||
with open(path, "rb") as stream:
|
||||
magic = stream.read(12)
|
||||
|
||||
if _looks_like_iso_media(magic):
|
||||
return _read_iso_media_dimensions(stream, file_size)
|
||||
if magic[:4] == _EBML_MAGIC:
|
||||
return _read_ebml_dimensions(stream, file_size)
|
||||
if magic[:4] == b"RIFF" and magic[8:12] == b"WEBP":
|
||||
return _read_riff_webp_dimensions(stream, file_size)
|
||||
|
||||
# Signature is inconclusive (truncated or unusual file): fall back to
|
||||
# the extension.
|
||||
if ext in EBML_MEDIA_EXTENSIONS:
|
||||
return _read_ebml_dimensions(stream, file_size)
|
||||
if ext in ISO_MEDIA_EXTENSIONS:
|
||||
return _read_iso_media_dimensions(stream, file_size)
|
||||
return None
|
||||
|
||||
|
||||
def _looks_like_iso_media(magic: bytes) -> bool:
|
||||
"""Return True when the leading bytes are an ISO base media box header."""
|
||||
|
||||
return len(magic) >= 8 and magic[4:8] in {
|
||||
b"ftyp",
|
||||
b"moov",
|
||||
b"mdat",
|
||||
b"free",
|
||||
b"skip",
|
||||
b"wide",
|
||||
}
|
||||
|
||||
|
||||
def get_video_dimensions(path: str) -> Optional[Tuple[int, int]]:
|
||||
"""Return the intrinsic ``(width, height)`` of a local video file.
|
||||
|
||||
Returns ``None`` when the extension is unsupported, the file is missing or
|
||||
corrupt, or the dimensions cannot be determined. Never raises.
|
||||
"""
|
||||
if not path:
|
||||
return None
|
||||
try:
|
||||
stat = os.stat(path)
|
||||
except OSError:
|
||||
return None
|
||||
return _get_video_dimensions_cached(path, stat.st_mtime_ns, stat.st_size)
|
||||
|
||||
|
||||
def _clear_video_dimensions_cache() -> None:
|
||||
"""Drop the dimension cache (used by tests)."""
|
||||
|
||||
_get_video_dimensions_cached.cache_clear()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# ISO base media (MP4 / MOV)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _iter_boxes(
|
||||
stream: BinaryIO, end: int, depth: int = 0
|
||||
) -> Iterator[Tuple[bytes, int, int]]:
|
||||
"""Yield ``(type, payload_start, box_end)`` for boxes in ``[tell, end)``.
|
||||
|
||||
The stream is left at the next box boundary after each yielded box.
|
||||
"""
|
||||
if depth > _MAX_BOX_DEPTH:
|
||||
return
|
||||
|
||||
while True:
|
||||
start = stream.tell()
|
||||
if start + 8 > end:
|
||||
return
|
||||
|
||||
header = stream.read(8)
|
||||
if len(header) < 8:
|
||||
return
|
||||
|
||||
size, box_type = struct.unpack(">I4s", header)
|
||||
header_size = 8
|
||||
|
||||
if size == 1:
|
||||
# 64-bit ``largesize`` follows the type.
|
||||
extended = stream.read(8)
|
||||
if len(extended) < 8:
|
||||
return
|
||||
size = struct.unpack(">Q", extended)[0]
|
||||
header_size = 16
|
||||
elif size == 0:
|
||||
# Box extends to the end of the enclosing container.
|
||||
size = end - start
|
||||
|
||||
if size < header_size or start + size > end:
|
||||
return
|
||||
|
||||
yield box_type, start + header_size, start + size
|
||||
stream.seek(start + size)
|
||||
|
||||
|
||||
def _read_iso_media_dimensions(
|
||||
stream: BinaryIO, file_size: int
|
||||
) -> Optional[Tuple[int, int]]:
|
||||
"""Walk ``moov`` looking for the video track's dimensions."""
|
||||
|
||||
stream.seek(0)
|
||||
moov: Optional[Tuple[int, int]] = None
|
||||
for box_type, payload_start, box_end in _iter_boxes(stream, file_size):
|
||||
if box_type == b"moov":
|
||||
moov = (payload_start, box_end)
|
||||
break
|
||||
|
||||
if moov is None:
|
||||
return None
|
||||
|
||||
stream.seek(moov[0])
|
||||
for box_type, payload_start, box_end in _iter_boxes(stream, moov[1], depth=1):
|
||||
if box_type != b"trak":
|
||||
continue
|
||||
dimensions = _read_trak_dimensions(stream, payload_start, box_end)
|
||||
if dimensions is not None:
|
||||
return dimensions
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _read_trak_dimensions(
|
||||
stream: BinaryIO, trak_start: int, trak_end: int
|
||||
) -> Optional[Tuple[int, int]]:
|
||||
"""Return the dimensions of a ``trak`` when it describes a video track."""
|
||||
|
||||
stream.seek(trak_start)
|
||||
|
||||
is_video = False
|
||||
tkhd_dimensions = _WIDTH_HEIGHT_UNSET
|
||||
stsd_dimensions = _WIDTH_HEIGHT_UNSET
|
||||
|
||||
for box_type, payload_start, box_end in _iter_boxes(stream, trak_end, depth=2):
|
||||
if box_type == b"tkhd":
|
||||
tkhd_dimensions = _parse_tkhd(stream, payload_start, box_end)
|
||||
elif box_type == b"mdia":
|
||||
stream.seek(payload_start)
|
||||
media = _read_mdia_dimensions(stream, payload_start, box_end)
|
||||
if media is not None:
|
||||
is_video, stsd_dimensions = media
|
||||
|
||||
if not is_video:
|
||||
return None
|
||||
|
||||
# ``tkhd`` is preferred: it is display space, and its 16.16 fixed point
|
||||
# encoding keeps non-integer dimensions (odd crops produce those).
|
||||
for width, height in (tkhd_dimensions, stsd_dimensions):
|
||||
if width > 0 and height > 0:
|
||||
return int(round(width)), int(round(height))
|
||||
return None
|
||||
|
||||
|
||||
def _read_mdia_dimensions(
|
||||
stream: BinaryIO, mdia_start: int, mdia_end: int
|
||||
) -> Optional[Tuple[bool, Tuple[float, float]]]:
|
||||
"""Return ``(is_video, dimensions)`` for a ``mdia`` box."""
|
||||
|
||||
handler_type = b""
|
||||
stsd_dimensions = _WIDTH_HEIGHT_UNSET
|
||||
|
||||
for box_type, payload_start, box_end in _iter_boxes(stream, mdia_end, depth=3):
|
||||
if box_type == b"hdlr":
|
||||
handler_type = _parse_handler_type(stream, payload_start, box_end)
|
||||
elif box_type == b"minf":
|
||||
stream.seek(payload_start)
|
||||
stsd_dimensions = _read_minf_dimensions(stream, payload_start, box_end)
|
||||
|
||||
return handler_type == b"vide", stsd_dimensions
|
||||
|
||||
|
||||
def _read_minf_dimensions(
|
||||
stream: BinaryIO, minf_start: int, minf_end: int
|
||||
) -> Tuple[float, float]:
|
||||
"""Return the sample-entry dimensions declared under ``minf/stbl/stsd``."""
|
||||
|
||||
for box_type, payload_start, box_end in _iter_boxes(stream, minf_end, depth=4):
|
||||
if box_type != b"stbl":
|
||||
continue
|
||||
stream.seek(payload_start)
|
||||
for inner_type, inner_start, inner_end in _iter_boxes(
|
||||
stream, box_end, depth=5
|
||||
):
|
||||
if inner_type == b"stsd":
|
||||
return _parse_stsd(stream, inner_start, inner_end)
|
||||
return _WIDTH_HEIGHT_UNSET
|
||||
|
||||
|
||||
def _parse_tkhd(
|
||||
stream: BinaryIO, payload_start: int, box_end: int
|
||||
) -> Tuple[float, float]:
|
||||
"""Parse the 16.16 fixed point width/height trailer of a ``tkhd`` box."""
|
||||
|
||||
size = box_end - payload_start
|
||||
if size < 8 or size > _MAX_HEADER_PAYLOAD:
|
||||
return _WIDTH_HEIGHT_UNSET
|
||||
|
||||
stream.seek(box_end - 8)
|
||||
trailer = stream.read(8)
|
||||
if len(trailer) < 8:
|
||||
return _WIDTH_HEIGHT_UNSET
|
||||
|
||||
width, height = struct.unpack(">II", trailer)
|
||||
return width / 65536.0, height / 65536.0
|
||||
|
||||
|
||||
def _parse_handler_type(
|
||||
stream: BinaryIO, payload_start: int, box_end: int
|
||||
) -> bytes:
|
||||
"""Parse the handler type from an ``hdlr`` box.
|
||||
|
||||
Layout: version/flags (4) + pre_defined (4) + handler_type (4).
|
||||
"""
|
||||
|
||||
if box_end - payload_start < 12:
|
||||
return b""
|
||||
stream.seek(payload_start)
|
||||
data = stream.read(12)
|
||||
if len(data) < 12:
|
||||
return b""
|
||||
return data[8:12]
|
||||
|
||||
|
||||
def _parse_stsd(
|
||||
stream: BinaryIO, payload_start: int, box_end: int
|
||||
) -> Tuple[float, float]:
|
||||
"""Parse the visual sample entry dimensions from an ``stsd`` box.
|
||||
|
||||
Only the first entry is inspected: video tracks are single-entry in every
|
||||
container we import from.
|
||||
"""
|
||||
|
||||
if box_end - payload_start < 16:
|
||||
return _WIDTH_HEIGHT_UNSET
|
||||
|
||||
stream.seek(payload_start)
|
||||
header = stream.read(8) # version/flags + entry_count
|
||||
if len(header) < 8:
|
||||
return _WIDTH_HEIGHT_UNSET
|
||||
|
||||
entry_start = payload_start + 8
|
||||
if entry_start + 8 > box_end:
|
||||
return _WIDTH_HEIGHT_UNSET
|
||||
|
||||
stream.seek(entry_start)
|
||||
entry_header = stream.read(8)
|
||||
if len(entry_header) < 8:
|
||||
return _WIDTH_HEIGHT_UNSET
|
||||
|
||||
entry_size = struct.unpack(">I", entry_header[:4])[0]
|
||||
header_size = 8
|
||||
|
||||
if entry_size == 1:
|
||||
extended = stream.read(8)
|
||||
if len(extended) < 8:
|
||||
return _WIDTH_HEIGHT_UNSET
|
||||
entry_size = struct.unpack(">Q", extended)[0]
|
||||
header_size = 16
|
||||
elif entry_size == 0:
|
||||
entry_size = box_end - entry_start
|
||||
|
||||
if entry_size < header_size + 8 or entry_start + entry_size > box_end:
|
||||
return _WIDTH_HEIGHT_UNSET
|
||||
|
||||
# Visual sample entries: 6 bytes reserved + 2 bytes data_reference_index,
|
||||
# then width (2) and height (2).
|
||||
stream.seek(entry_start + header_size + 6 + 2)
|
||||
dimensions = stream.read(4)
|
||||
if len(dimensions) < 4:
|
||||
return _WIDTH_HEIGHT_UNSET
|
||||
|
||||
width, height = struct.unpack(">HH", dimensions)
|
||||
return float(width), float(height)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# WebM / Matroska (EBML)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
# EBML element IDs (stored with their length marker, as they appear on disk).
|
||||
_ID_SEGMENT = 0x18538067
|
||||
_ID_TRACKS = 0x1654AE6B
|
||||
_ID_TRACK_ENTRY = 0xAE
|
||||
_ID_TRACK_TYPE = 0x83
|
||||
_ID_VIDEO = 0xE0
|
||||
_ID_PIXEL_WIDTH = 0xB0
|
||||
_ID_PIXEL_HEIGHT = 0xBA
|
||||
|
||||
# Nested containers we descend into while hunting for video dimensions.
|
||||
_EBML_CONTAINER_IDS = frozenset({_ID_SEGMENT, _ID_TRACKS, _ID_TRACK_ENTRY})
|
||||
|
||||
|
||||
def _read_ebml_vint(stream: BinaryIO, *, keep_marker: bool) -> Optional[Tuple[int, int]]:
|
||||
"""Read an EBML variable-length integer.
|
||||
|
||||
Returns ``(value, byte_length)``. For element IDs the marker bit is kept
|
||||
(``keep_marker=True``) because IDs are compared in their on-disk form; for
|
||||
sizes the marker is stripped to yield the actual payload length.
|
||||
"""
|
||||
|
||||
first = stream.read(1)
|
||||
if not first:
|
||||
return None
|
||||
|
||||
first_byte = first[0]
|
||||
if first_byte == 0:
|
||||
return None
|
||||
|
||||
length = 1
|
||||
mask = 0x80
|
||||
while not first_byte & mask:
|
||||
mask >>= 1
|
||||
length += 1
|
||||
if length > 8:
|
||||
return None
|
||||
|
||||
value = first_byte if keep_marker else first_byte & (mask - 1)
|
||||
remaining = length - 1
|
||||
|
||||
if remaining:
|
||||
extra = stream.read(remaining)
|
||||
if len(extra) < remaining:
|
||||
return None
|
||||
for byte in extra:
|
||||
value = (value << 8) | byte
|
||||
|
||||
return value, length
|
||||
|
||||
|
||||
def _read_ebml_dimensions(
|
||||
stream: BinaryIO, file_size: int
|
||||
) -> Optional[Tuple[int, int]]:
|
||||
"""Parse ``Segment/Tracks`` for the first video ``TrackEntry``."""
|
||||
|
||||
stream.seek(0)
|
||||
header = stream.read(4)
|
||||
if header != _EBML_MAGIC:
|
||||
return None
|
||||
|
||||
return _walk_ebml(stream, 0, file_size, depth=0)
|
||||
|
||||
|
||||
def _walk_ebml(
|
||||
stream: BinaryIO, start: int, end: int, *, depth: int
|
||||
) -> Optional[Tuple[int, int]]:
|
||||
"""Recursively scan EBML elements in ``[start, end)`` for video dimensions."""
|
||||
|
||||
if depth > _MAX_EBML_DEPTH:
|
||||
return None
|
||||
|
||||
stream.seek(start)
|
||||
|
||||
while stream.tell() < end:
|
||||
element_start = stream.tell()
|
||||
|
||||
element_id = _read_ebml_vint(stream, keep_marker=True)
|
||||
if element_id is None:
|
||||
return None
|
||||
element_id_value = element_id[0]
|
||||
|
||||
size_field = _read_ebml_vint(stream, keep_marker=False)
|
||||
if size_field is None:
|
||||
return None
|
||||
payload_size, size_length = size_field
|
||||
|
||||
payload_start = element_start + element_id[1] + size_length
|
||||
|
||||
# A size field of all-ones marks an unknown-size element, which is
|
||||
# legal for Segment/Tracks; treat it as "until the parent ends".
|
||||
unknown_size = payload_size == (1 << (7 * size_length)) - 1
|
||||
payload_end = end if unknown_size else payload_start + payload_size
|
||||
|
||||
if payload_end > end:
|
||||
return None
|
||||
|
||||
if element_id_value == _ID_VIDEO:
|
||||
dimensions = _read_ebml_video(stream, payload_start, min(payload_end, end))
|
||||
if dimensions is not None:
|
||||
return dimensions
|
||||
elif element_id_value == _ID_TRACK_ENTRY:
|
||||
track = _read_ebml_track_entry(
|
||||
stream, payload_start, min(payload_end, end)
|
||||
)
|
||||
if track is not None:
|
||||
return track
|
||||
elif element_id_value in _EBML_CONTAINER_IDS:
|
||||
found = _walk_ebml(
|
||||
stream, payload_start, min(payload_end, end), depth=depth + 1
|
||||
)
|
||||
if found is not None:
|
||||
return found
|
||||
|
||||
if unknown_size:
|
||||
# Cannot resume after an unknown-size element; its siblings cannot
|
||||
# be located reliably, so stop scanning this level.
|
||||
return None
|
||||
|
||||
stream.seek(payload_end)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _read_ebml_track_entry(
|
||||
stream: BinaryIO, start: int, end: int
|
||||
) -> Optional[Tuple[int, int]]:
|
||||
"""Return dimensions when a ``TrackEntry`` is a video track."""
|
||||
|
||||
track_type: Optional[int] = None
|
||||
dimensions: Optional[Tuple[int, int]] = None
|
||||
|
||||
stream.seek(start)
|
||||
while stream.tell() < end:
|
||||
element_start = stream.tell()
|
||||
|
||||
element_id = _read_ebml_vint(stream, keep_marker=True)
|
||||
if element_id is None:
|
||||
return None
|
||||
|
||||
size_field = _read_ebml_vint(stream, keep_marker=False)
|
||||
if size_field is None:
|
||||
return None
|
||||
payload_size, size_length = size_field
|
||||
|
||||
payload_start = element_start + element_id[1] + size_length
|
||||
payload_end = min(payload_start + payload_size, end)
|
||||
|
||||
if element_id[0] == _ID_TRACK_TYPE:
|
||||
track_type = _read_ebml_uint(stream, payload_start, payload_end)
|
||||
elif element_id[0] == _ID_VIDEO:
|
||||
dimensions = _read_ebml_video(stream, payload_start, payload_end)
|
||||
|
||||
stream.seek(payload_end)
|
||||
|
||||
# Track type 1 is video.
|
||||
if track_type == 1 and dimensions is not None:
|
||||
return dimensions
|
||||
return None
|
||||
|
||||
|
||||
def _read_ebml_video(
|
||||
stream: BinaryIO, start: int, end: int
|
||||
) -> Optional[Tuple[int, int]]:
|
||||
"""Return ``PixelWidth``/``PixelHeight`` from a ``Video`` element."""
|
||||
|
||||
width: Optional[int] = None
|
||||
height: Optional[int] = None
|
||||
|
||||
stream.seek(start)
|
||||
while stream.tell() < end:
|
||||
element_start = stream.tell()
|
||||
|
||||
element_id = _read_ebml_vint(stream, keep_marker=True)
|
||||
if element_id is None:
|
||||
return None
|
||||
|
||||
size_field = _read_ebml_vint(stream, keep_marker=False)
|
||||
if size_field is None:
|
||||
return None
|
||||
payload_size, size_length = size_field
|
||||
|
||||
payload_start = element_start + element_id[1] + size_length
|
||||
payload_end = min(payload_start + payload_size, end)
|
||||
|
||||
if element_id[0] == _ID_PIXEL_WIDTH:
|
||||
width = _read_ebml_uint(stream, payload_start, payload_end)
|
||||
elif element_id[0] == _ID_PIXEL_HEIGHT:
|
||||
height = _read_ebml_uint(stream, payload_start, payload_end)
|
||||
|
||||
stream.seek(payload_end)
|
||||
|
||||
if width and height and width > 0 and height > 0:
|
||||
return width, height
|
||||
return None
|
||||
|
||||
|
||||
def _read_ebml_uint(stream: BinaryIO, start: int, end: int) -> Optional[int]:
|
||||
"""Read an unsigned big-endian integer element payload."""
|
||||
|
||||
length = end - start
|
||||
if length <= 0 or length > 8:
|
||||
return None
|
||||
|
||||
stream.seek(start)
|
||||
raw = stream.read(length)
|
||||
if len(raw) < length:
|
||||
return None
|
||||
|
||||
value = 0
|
||||
for byte in raw:
|
||||
value = (value << 8) | byte
|
||||
return value
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# RIFF / WebP (animated examples are often renamed to ``.mp4``)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _read_riff_webp_dimensions(
|
||||
stream: BinaryIO, file_size: int
|
||||
) -> Optional[Tuple[int, int]]:
|
||||
"""Return dimensions from a WebP file's first dimension-bearing chunk."""
|
||||
|
||||
stream.seek(12)
|
||||
|
||||
while stream.tell() + 8 <= file_size:
|
||||
header = stream.read(8)
|
||||
if len(header) < 8:
|
||||
return None
|
||||
|
||||
fourcc, chunk_size = struct.unpack("<4sI", header)
|
||||
payload_start = stream.tell()
|
||||
|
||||
if fourcc == b"VP8X":
|
||||
payload = stream.read(10)
|
||||
if len(payload) < 10:
|
||||
return None
|
||||
# Canvas size is stored minus one, as 24-bit little endian values.
|
||||
width = int.from_bytes(payload[4:7], "little") + 1
|
||||
height = int.from_bytes(payload[7:10], "little") + 1
|
||||
return width, height
|
||||
|
||||
if fourcc == b"VP8 ":
|
||||
# Frame tag (3 bytes, bit 0 = key frame) then the key frame start
|
||||
# code 0x9d 0x01 0x2a and the 16-bit dimensions.
|
||||
payload = stream.read(10)
|
||||
if len(payload) < 10:
|
||||
return None
|
||||
start = payload.find(b"\x9d\x01\x2a")
|
||||
if start < 0 or start + 7 > len(payload):
|
||||
return None
|
||||
width, height = struct.unpack("<HH", payload[start + 3 : start + 7])
|
||||
return width & 0x3FFF, height & 0x3FFF
|
||||
|
||||
if fourcc == b"VP8L":
|
||||
payload = stream.read(5)
|
||||
if len(payload) < 5 or payload[0] != 0x2F:
|
||||
return None
|
||||
bits = int.from_bytes(payload[1:5], "little")
|
||||
return (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1
|
||||
|
||||
# Skip this chunk (payloads are padded to an even byte boundary).
|
||||
stream.seek(payload_start + chunk_size + (chunk_size & 1))
|
||||
|
||||
return None
|
||||
@@ -225,10 +225,9 @@ def main() -> int:
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Get project root (parent of .agents directory)
|
||||
# Get project root: this script lives in <project_root>/scripts/e2e/.
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
skill_dir = os.path.dirname(script_dir)
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.dirname(skill_dir)))
|
||||
project_root = os.path.dirname(os.path.dirname(script_dir))
|
||||
|
||||
managed_pids = read_managed_pids(args.port)
|
||||
|
||||
|
||||
@@ -2432,6 +2432,11 @@ export class SettingsManager {
|
||||
|| ['vae', 'upscaler', 'text_encoder']
|
||||
);
|
||||
|
||||
const masterToggle = document.getElementById('enableOtherModels');
|
||||
if (masterToggle) {
|
||||
masterToggle.checked = enableOtherModels;
|
||||
}
|
||||
|
||||
document.querySelectorAll('[data-other-subtype-toggle]').forEach((input) => {
|
||||
input.checked = enabledSubTypes.has(input.value);
|
||||
input.disabled = !enableOtherModels;
|
||||
|
||||
@@ -7,8 +7,10 @@ import { enableOtherModels, openOtherModelsSettings } from './utils/otherModels.
|
||||
* empty state whose button turns the feature on; the backend then rebuilds the
|
||||
* other-model roots and starts scanning, so a reload lands on the real page.
|
||||
*
|
||||
* The same module backs the "enabled but no folders found" state, where the
|
||||
* only useful action is jumping to Settings instead of enabling anything.
|
||||
* The same module backs the "enabled but no folders found" state: ComfyUI
|
||||
* mode points to the Settings page's Library section, while standalone mode
|
||||
* (where the settings UI cannot edit primary folder paths) reveals the
|
||||
* settings.json file the user must edit instead.
|
||||
*/
|
||||
async function handleEnableClick() {
|
||||
const button = document.getElementById('enableOtherModelsBtn');
|
||||
@@ -32,6 +34,41 @@ function handleOpenSettingsClick(event) {
|
||||
openOtherModelsSettings();
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the settings.json location from the standalone no-folders state.
|
||||
* The settings UI cannot edit primary folder_paths, so the only useful
|
||||
* action is revealing the file itself (or copying its path in Docker).
|
||||
*/
|
||||
async function handleOpenSettingsFolderClick() {
|
||||
const button = document.getElementById('openSettingsFolderBtn');
|
||||
if (!button || button.disabled) return;
|
||||
|
||||
button.disabled = true;
|
||||
try {
|
||||
const response = await fetch('/api/lm/settings/open-location', { method: 'POST' });
|
||||
const data = await response.json().catch(() => ({}));
|
||||
if (!response.ok || data.success === false) {
|
||||
throw new Error(data.error || `HTTP ${response.status}`);
|
||||
}
|
||||
if (data.mode === 'clipboard' && data.path) {
|
||||
try {
|
||||
await navigator.clipboard.writeText(data.path);
|
||||
showToast('settings.openSettingsFileLocation.copied', { path: data.path }, 'success');
|
||||
} catch (clipboardError) {
|
||||
console.warn('Clipboard API not available:', clipboardError);
|
||||
showToast('settings.openSettingsFileLocation.clipboardFallback', { path: data.path }, 'info');
|
||||
}
|
||||
} else {
|
||||
showToast('settings.openSettingsFileLocation.success', {}, 'success');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to open settings location:', error);
|
||||
showToast('settings.openSettingsFileLocation.failed', {}, 'error');
|
||||
} finally {
|
||||
button.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function initializeOtherDisabledPage() {
|
||||
// appCore.initialize() wires the shared header (theme, settings modal,
|
||||
// language) so this page is not a dead end.
|
||||
@@ -46,8 +83,13 @@ async function initializeOtherDisabledPage() {
|
||||
if (settingsButton) {
|
||||
settingsButton.addEventListener('click', handleOpenSettingsClick);
|
||||
}
|
||||
|
||||
const settingsFolderButton = document.getElementById('openSettingsFolderBtn');
|
||||
if (settingsFolderButton) {
|
||||
settingsFolderButton.addEventListener('click', handleOpenSettingsFolderClick);
|
||||
}
|
||||
}
|
||||
|
||||
document.addEventListener('DOMContentLoaded', initializeOtherDisabledPage);
|
||||
|
||||
export { handleEnableClick as enableOtherModels, initializeOtherDisabledPage };
|
||||
export { handleEnableClick as enableOtherModels, handleOpenSettingsFolderClick, initializeOtherDisabledPage };
|
||||
|
||||
+20
-1
@@ -63,6 +63,19 @@
|
||||
background: rgba(127, 127, 127, 0.15);
|
||||
border: 1px solid rgba(127, 127, 127, 0.25);
|
||||
}
|
||||
.other-settings-file {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
font-size: 13px;
|
||||
}
|
||||
.other-settings-file code {
|
||||
padding: 4px 8px;
|
||||
border-radius: 4px;
|
||||
background: rgba(127, 127, 127, 0.15);
|
||||
border: 1px solid rgba(127, 127, 127, 0.25);
|
||||
word-break: break-all;
|
||||
}
|
||||
</style>
|
||||
{% endblock %}
|
||||
|
||||
@@ -127,6 +140,9 @@
|
||||
<h2>{{ t('other.noPaths.title') }}</h2>
|
||||
{% if standalone_mode %}
|
||||
<p>{{ t('other.noPaths.descriptionStandalone') }}</p>
|
||||
{% if settings_file %}
|
||||
<p class="other-settings-file"><i class="fas fa-file-alt"></i> <code>{{ settings_file }}</code></p>
|
||||
{% endif %}
|
||||
<pre class="other-no-paths-config"><code>"folder_paths": {
|
||||
"vae": ["/path/to/vae"],
|
||||
"upscale_models": ["/path/to/upscale_models"],
|
||||
@@ -135,13 +151,16 @@
|
||||
"controlnet": ["/path/to/controlnet"]
|
||||
}</code></pre>
|
||||
<p class="other-disabled-hint">{{ t('other.noPaths.hintStandalone') }}</p>
|
||||
<button id="openSettingsFolderBtn" type="button">
|
||||
<i class="fas fa-folder-open"></i> {{ t('other.noPaths.openSettingsFolder') }}
|
||||
</button>
|
||||
{% else %}
|
||||
<p>{{ t('other.noPaths.descriptionComfyUI') }}</p>
|
||||
<p class="other-disabled-hint">{{ t('other.noPaths.hintComfyUI') }}</p>
|
||||
{% endif %}
|
||||
<button id="openOtherModelsSettingsBtn" type="button">
|
||||
<i class="fas fa-cog"></i> {{ t('other.noPaths.openSettings') }}
|
||||
</button>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="sticky-topbar">
|
||||
|
||||
@@ -333,6 +333,21 @@ def mock_websocket_manager():
|
||||
return RecordingWebSocketManager()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_media_dimension_caches():
|
||||
"""Clear path-keyed dimension caches so files reused across tests re-probe."""
|
||||
from py.utils.exif_utils import _get_image_dimensions_cached
|
||||
from py.utils.video_metadata import _clear_video_dimensions_cache
|
||||
|
||||
_get_image_dimensions_cached.cache_clear()
|
||||
_clear_video_dimensions_cache()
|
||||
|
||||
yield
|
||||
|
||||
_get_image_dimensions_cached.cache_clear()
|
||||
_clear_video_dimensions_cache()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_singletons():
|
||||
"""Reset all singletons before each test to ensure isolation."""
|
||||
|
||||
@@ -665,6 +665,22 @@ describe('SettingsManager other-model root selects', () => {
|
||||
expect(container.classList.contains('is-disabled')).toBe(false);
|
||||
});
|
||||
|
||||
it('restores the master toggle checked state from settings', () => {
|
||||
const manager = createManager();
|
||||
const masterToggle = document.createElement('input');
|
||||
masterToggle.type = 'checkbox';
|
||||
masterToggle.id = 'enableOtherModels';
|
||||
document.body.appendChild(masterToggle);
|
||||
|
||||
state.global.settings = { enable_other_models: true };
|
||||
manager.updateOtherModelsControls();
|
||||
expect(masterToggle.checked).toBe(true);
|
||||
|
||||
state.global.settings = { enable_other_models: false };
|
||||
manager.updateOtherModelsControls();
|
||||
expect(masterToggle.checked).toBe(false);
|
||||
});
|
||||
|
||||
it('persists the checked sub_types as the whole allow-list', async () => {
|
||||
const manager = createManager();
|
||||
appendToggles('vae', 'upscaler', 'controlnet');
|
||||
|
||||
@@ -25,6 +25,7 @@ describe('Other Models disabled page', () => {
|
||||
document.body.innerHTML = [
|
||||
'<button id="enableOtherModelsBtn"></button>',
|
||||
'<button id="openOtherModelsSettingsBtn"></button>',
|
||||
'<button id="openSettingsFolderBtn"></button>',
|
||||
].join('');
|
||||
|
||||
Object.defineProperty(window, 'location', {
|
||||
@@ -64,6 +65,52 @@ describe('Other Models disabled page', () => {
|
||||
expect(showModal).toHaveBeenCalledWith('settingsModal');
|
||||
});
|
||||
|
||||
it('reveals the settings.json location from the standalone no-folders state', async () => {
|
||||
global.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ success: true, message: 'Opened settings folder' }),
|
||||
});
|
||||
|
||||
const button = document.getElementById('openSettingsFolderBtn');
|
||||
button.dispatchEvent(new MouseEvent('click', { bubbles: true }));
|
||||
await vi.waitFor(() => expect(showToastMock).toHaveBeenCalled());
|
||||
|
||||
expect(global.fetch).toHaveBeenCalledWith(
|
||||
'/api/lm/settings/open-location',
|
||||
expect.objectContaining({ method: 'POST' }),
|
||||
);
|
||||
expect(showToastMock).toHaveBeenCalledWith(
|
||||
'settings.openSettingsFileLocation.success',
|
||||
{},
|
||||
'success',
|
||||
);
|
||||
expect(button.disabled).toBe(false);
|
||||
});
|
||||
|
||||
it('copies the settings path to the clipboard in Docker mode', async () => {
|
||||
const writeText = vi.fn().mockResolvedValue(undefined);
|
||||
Object.defineProperty(navigator, 'clipboard', {
|
||||
value: { writeText },
|
||||
configurable: true,
|
||||
});
|
||||
global.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ success: true, mode: 'clipboard', path: '/data/settings.json' }),
|
||||
});
|
||||
|
||||
document.getElementById('openSettingsFolderBtn').dispatchEvent(
|
||||
new MouseEvent('click', { bubbles: true }),
|
||||
);
|
||||
await vi.waitFor(() => expect(showToastMock).toHaveBeenCalled());
|
||||
|
||||
expect(writeText).toHaveBeenCalledWith('/data/settings.json');
|
||||
expect(showToastMock).toHaveBeenCalledWith(
|
||||
'settings.openSettingsFileLocation.copied',
|
||||
{ path: '/data/settings.json' },
|
||||
'success',
|
||||
);
|
||||
});
|
||||
|
||||
it('enables Other Models through the settings API and reloads', async () => {
|
||||
global.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
|
||||
@@ -532,6 +532,62 @@ async def test_open_backup_location_uses_settings_directory(tmp_path, monkeypatc
|
||||
assert calls == [["xdg-open", str(backup_dir)]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_settings_location_headless_returns_clipboard_mode(tmp_path, monkeypatch):
|
||||
"""Without a GUI session xdg-open cannot work; the handler must hand the
|
||||
path to the browser instead of reporting a success that never happened."""
|
||||
settings_file = tmp_path / "settings" / "settings.json"
|
||||
settings_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
settings_file.write_text("{}", encoding="utf-8")
|
||||
|
||||
handler = FileSystemHandler(settings_service=SimpleNamespace(settings_file=str(settings_file)))
|
||||
|
||||
monkeypatch.delenv("DISPLAY", raising=False)
|
||||
monkeypatch.delenv("WAYLAND_DISPLAY", raising=False)
|
||||
monkeypatch.setattr("py.routes.handlers.misc_handlers._is_docker", lambda: False)
|
||||
monkeypatch.setattr("py.routes.handlers.misc_handlers._is_wsl", lambda: False)
|
||||
|
||||
popen_calls = []
|
||||
monkeypatch.setattr(subprocess, "Popen", lambda *args, **kwargs: popen_calls.append(args))
|
||||
|
||||
response = await handler.open_settings_location(FakeRequest()) # pyright: ignore[reportArgumentType]
|
||||
payload = _json_payload(response)
|
||||
|
||||
assert response.status == 200
|
||||
assert payload["success"] is True
|
||||
assert payload["mode"] == "clipboard"
|
||||
assert payload["path"] == str(settings_file)
|
||||
assert popen_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_settings_location_with_display_opens_folder(tmp_path, monkeypatch):
|
||||
settings_file = tmp_path / "settings" / "settings.json"
|
||||
settings_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
settings_file.write_text("{}", encoding="utf-8")
|
||||
|
||||
handler = FileSystemHandler(settings_service=SimpleNamespace(settings_file=str(settings_file)))
|
||||
|
||||
monkeypatch.setenv("DISPLAY", ":0")
|
||||
monkeypatch.setattr("py.routes.handlers.misc_handlers._is_docker", lambda: False)
|
||||
monkeypatch.setattr("py.routes.handlers.misc_handlers._is_wsl", lambda: False)
|
||||
|
||||
calls = []
|
||||
|
||||
def fake_popen(args):
|
||||
calls.append(args)
|
||||
return MagicMock()
|
||||
|
||||
monkeypatch.setattr(subprocess, "Popen", fake_popen)
|
||||
|
||||
response = await handler.open_settings_location(FakeRequest()) # pyright: ignore[reportArgumentType]
|
||||
payload = _json_payload(response)
|
||||
|
||||
assert response.status == 200
|
||||
assert payload["success"] is True
|
||||
assert calls == [["xdg-open", str(settings_file.parent)]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_wildcards_location_creates_and_opens_directory(tmp_path, monkeypatch):
|
||||
wildcards_dir = tmp_path / "settings" / "wildcards"
|
||||
@@ -2369,3 +2425,135 @@ async def test_get_init_status_reports_pending_scanners():
|
||||
assert "embedding" in payload["details"]
|
||||
assert "recipe" in payload["details"]
|
||||
assert "lora" not in payload["details"]
|
||||
|
||||
|
||||
class StaticMetadataProvider:
|
||||
"""Metadata provider returning one fixed CivitAI model payload."""
|
||||
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
|
||||
async def get_model_versions(self, _model_id):
|
||||
return self.payload
|
||||
|
||||
async def get_user_models(self, _username, cursor=None):
|
||||
return {"items": [], "nextCursor": None}
|
||||
|
||||
async def get_creator_model_count(self, _username):
|
||||
return None
|
||||
|
||||
|
||||
def _versions_status_handler(payload, *, other_scanner=None):
|
||||
async def metadata_factory():
|
||||
return StaticMetadataProvider(payload)
|
||||
|
||||
async def other_factory():
|
||||
return other_scanner
|
||||
|
||||
return ModelLibraryHandler(
|
||||
ServiceRegistryAdapter(
|
||||
get_lora_scanner=fake_scanner_factory,
|
||||
get_checkpoint_scanner=fake_scanner_factory,
|
||||
get_embedding_scanner=fake_scanner_factory,
|
||||
get_other_scanner=other_factory,
|
||||
get_downloaded_version_history_service=fake_download_history_service_factory,
|
||||
),
|
||||
metadata_provider_factory=metadata_factory,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_model_versions_status_unsupported_type_is_read_only():
|
||||
"""A type with no scanner answers 200 with a read-only list + reason."""
|
||||
handler = _versions_status_handler(
|
||||
{
|
||||
"name": "Wildcards pack",
|
||||
"type": "Wildcards",
|
||||
"modelVersions": [
|
||||
{"id": 11, "name": "v1", "images": [{"url": "https://img/1.png"}]},
|
||||
{"id": 12, "name": "v2", "images": []},
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
response = await handler.get_model_versions_status(
|
||||
FakeRequest(query={"modelId": "45448"}) # pyright: ignore[reportArgumentType]
|
||||
)
|
||||
payload = _json_payload(response)
|
||||
|
||||
assert response.status == 200
|
||||
assert payload["success"] is True
|
||||
assert payload["supported"] is False
|
||||
assert payload["reason"] == "model_type_unsupported"
|
||||
assert payload["modelType"] == "wildcards"
|
||||
assert payload["versions"] == [
|
||||
{
|
||||
"id": 11,
|
||||
"name": "v1",
|
||||
"thumbnailUrl": "https://img/1.png",
|
||||
"inLibrary": False,
|
||||
"hasBeenDownloaded": False,
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"name": "v2",
|
||||
"thumbnailUrl": None,
|
||||
"inLibrary": False,
|
||||
"hasBeenDownloaded": False,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_model_versions_status_other_disabled_is_read_only():
|
||||
"""The opt-in gate keeps its own reason instead of the permanent one."""
|
||||
_set_other_models_enabled(False)
|
||||
handler = _versions_status_handler(
|
||||
{
|
||||
"name": "SDXL VAE",
|
||||
"type": "VAE",
|
||||
"modelVersions": [{"id": 333245, "name": "SDXL-VAE", "images": []}],
|
||||
}
|
||||
)
|
||||
|
||||
response = await handler.get_model_versions_status(
|
||||
FakeRequest(query={"modelId": "296576"}) # pyright: ignore[reportArgumentType]
|
||||
)
|
||||
payload = _json_payload(response)
|
||||
|
||||
assert response.status == 200
|
||||
assert payload["success"] is True
|
||||
assert payload["supported"] is False
|
||||
assert payload["reason"] == "other_models_disabled"
|
||||
assert payload["modelType"] == "vae"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_model_versions_status_supported_type_stays_interactive():
|
||||
"""A managed type keeps the existing enriched, fully interactive payload."""
|
||||
handler = _versions_status_handler(
|
||||
{
|
||||
"name": "Some LoRA",
|
||||
"type": "LORA",
|
||||
"modelVersions": [{"id": 1, "name": "v1", "images": []}],
|
||||
}
|
||||
)
|
||||
|
||||
response = await handler.get_model_versions_status(
|
||||
FakeRequest(query={"modelId": "5"}) # pyright: ignore[reportArgumentType]
|
||||
)
|
||||
payload = _json_payload(response)
|
||||
|
||||
assert response.status == 200
|
||||
assert payload["success"] is True
|
||||
assert payload["supported"] is True
|
||||
assert "reason" not in payload
|
||||
assert payload["versions"] == [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "v1",
|
||||
"thumbnailUrl": None,
|
||||
"inLibrary": False,
|
||||
"hasBeenDownloaded": False,
|
||||
}
|
||||
]
|
||||
|
||||
@@ -121,6 +121,25 @@ def test_page_context_reports_feature_state(monkeypatch):
|
||||
assert provider(None) == {"other_disabled": True, "other_no_paths": False}
|
||||
|
||||
|
||||
def test_page_context_exposes_settings_file_in_standalone(monkeypatch):
|
||||
"""Standalone users must edit settings.json by hand; the empty state
|
||||
needs the real file path to point them at."""
|
||||
from py.config import config
|
||||
from py.services.settings_manager import get_settings_manager
|
||||
|
||||
manager = get_settings_manager()
|
||||
handler = OtherRoutes()
|
||||
provider = handler._get_page_context_provider()
|
||||
|
||||
monkeypatch.setattr(config, "other_roots", [], raising=False)
|
||||
monkeypatch.setenv("LORA_MANAGER_STANDALONE", "1")
|
||||
|
||||
context = provider(None)
|
||||
assert context["other_no_paths"] is True
|
||||
assert context["standalone_mode"] is True
|
||||
assert context["settings_file"] == manager.settings_file
|
||||
|
||||
|
||||
def test_get_expected_model_types_mentions_supported_types():
|
||||
expected = OtherRoutes()._get_expected_model_types()
|
||||
for name in ("VAE", "Upscaler", "TextEncoder", "CLIPVision", "Controlnet"):
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Tests for the portable-mode flag lifecycle (issue #1114 follow-up).
|
||||
|
||||
``LORA_MANAGER_PORTABLE=1`` persists ``use_portable_settings: true`` into the
|
||||
plugin''s own settings.json. That is convenient for repeat runs, but it used to
|
||||
be a one-way trip: the flag made every instance sharing that plugin folder read
|
||||
(and write) the portable settings directory, and the only way back was editing
|
||||
settings.json by hand. ``LORA_MANAGER_PORTABLE=0`` is now the explicit exit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from py.services import settings_manager as settings_manager_module
|
||||
from py.services.settings_manager import SettingsManager
|
||||
|
||||
|
||||
def _write_settings(path, **extra):
|
||||
payload = {
|
||||
"folder_paths": {"loras": ["/loras"]},
|
||||
}
|
||||
payload.update(extra)
|
||||
path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
return payload
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated_settings_path(tmp_path, monkeypatch):
|
||||
"""Point SettingsManager at a settings.json we control."""
|
||||
settings_path = tmp_path / "settings.json"
|
||||
monkeypatch.setattr(
|
||||
"py.services.settings_manager.ensure_settings_file",
|
||||
lambda logger=None: str(settings_path),
|
||||
)
|
||||
settings_manager_module.reset_settings_manager()
|
||||
yield settings_path
|
||||
settings_manager_module.reset_settings_manager()
|
||||
|
||||
|
||||
def test_portable_env_enables_and_persists_the_flag(
|
||||
isolated_settings_path, monkeypatch
|
||||
):
|
||||
_write_settings(isolated_settings_path)
|
||||
monkeypatch.setenv("LORA_MANAGER_PORTABLE", "1")
|
||||
|
||||
manager = SettingsManager()
|
||||
|
||||
assert manager.get("use_portable_settings") is True
|
||||
persisted = json.loads(isolated_settings_path.read_text(encoding="utf-8"))
|
||||
assert persisted["use_portable_settings"] is True
|
||||
|
||||
|
||||
def test_explicit_zero_clears_the_persisted_flag(
|
||||
isolated_settings_path, monkeypatch
|
||||
):
|
||||
"""`=0` must undo a previous `=1`, without hand-editing settings.json."""
|
||||
_write_settings(isolated_settings_path, use_portable_settings=True)
|
||||
monkeypatch.setenv("LORA_MANAGER_PORTABLE", "0")
|
||||
|
||||
manager = SettingsManager()
|
||||
|
||||
assert manager.get("use_portable_settings") is False
|
||||
persisted = json.loads(isolated_settings_path.read_text(encoding="utf-8"))
|
||||
# A default value is omitted from disk, so the key is gone entirely.
|
||||
assert persisted.get("use_portable_settings") is None
|
||||
|
||||
|
||||
def test_unset_env_keeps_the_persisted_flag(
|
||||
isolated_settings_path, monkeypatch
|
||||
):
|
||||
"""Portable mode must persist across runs when the variable is unset."""
|
||||
_write_settings(isolated_settings_path, use_portable_settings=True)
|
||||
monkeypatch.delenv("LORA_MANAGER_PORTABLE", raising=False)
|
||||
|
||||
manager = SettingsManager()
|
||||
|
||||
assert manager.get("use_portable_settings") is True
|
||||
|
||||
|
||||
def test_zero_is_a_noop_when_portable_was_never_enabled(
|
||||
isolated_settings_path, monkeypatch
|
||||
):
|
||||
_write_settings(isolated_settings_path)
|
||||
monkeypatch.setenv("LORA_MANAGER_PORTABLE", "0")
|
||||
|
||||
manager = SettingsManager()
|
||||
|
||||
assert manager.get("use_portable_settings") in (False, None)
|
||||
|
||||
|
||||
def test_pinned_settings_dir_wins_over_portable_env(
|
||||
isolated_settings_path, monkeypatch
|
||||
):
|
||||
"""LORA_MANAGER_SETTINGS_DIR still takes precedence, as documented."""
|
||||
_write_settings(isolated_settings_path)
|
||||
monkeypatch.setenv("LORA_MANAGER_PORTABLE", "1")
|
||||
monkeypatch.setenv("LORA_MANAGER_SETTINGS_DIR", str(isolated_settings_path.parent))
|
||||
|
||||
manager = SettingsManager()
|
||||
|
||||
# The pinned directory already decides the location, so the portable flag
|
||||
# is deliberately left alone.
|
||||
assert not manager.get("use_portable_settings")
|
||||
@@ -0,0 +1,300 @@
|
||||
"""Regression tests for the recipe empty-prune guard (issue #1116).
|
||||
|
||||
A scan that finds no recipe files at all is not a trustworthy deletion signal:
|
||||
an unmounted drive, a ``recipes_path`` that silently fell back to another LoRA
|
||||
root, or a cache shared with a second instance all look identical to a real
|
||||
wipe. Before this guard, such a scan overwrote the persistent cache with an
|
||||
empty one, destroying the user's only record of their recipes.
|
||||
|
||||
Covered contracts:
|
||||
|
||||
1. ``_reconcile_recipe_cache`` reports the "every persisted file vanished"
|
||||
condition and does not treat an empty directory as a trustworthy prune.
|
||||
2. ``_initialize_recipe_cache_sync`` keeps the stored cache in that case
|
||||
instead of persisting the empty result.
|
||||
3. A partial orphan (some files still present) still prunes normally, so
|
||||
ordinary manual deletions keep working.
|
||||
4. ``PersistentRecipeCache.save_cache(skip_if_empty=True)`` is the
|
||||
storage-level backstop and a manual rebuild can still clear the cache.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from py.config import config
|
||||
from py.services import recipe_scanner as recipe_scanner_module
|
||||
from py.services import settings_manager as settings_manager_module
|
||||
from py.services.persistent_recipe_cache import (
|
||||
PersistedRecipeData,
|
||||
PersistentRecipeCache,
|
||||
)
|
||||
from py.services.recipe_cache import RecipeCache
|
||||
from py.services.recipe_scanner import RecipeScanner
|
||||
|
||||
|
||||
def _write_recipe_json(path: Path, recipe_id: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"id": recipe_id,
|
||||
"file_path": str(path.with_suffix(".png")),
|
||||
"title": f"Recipe {recipe_id}",
|
||||
"modified": 0.0,
|
||||
"created_date": 0.0,
|
||||
"loras": [],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _persisted_for(paths: list[Path]) -> PersistedRecipeData:
|
||||
"""Build persisted cache state describing *paths* as known recipe files."""
|
||||
raw_data = []
|
||||
file_stats = {}
|
||||
for path in paths:
|
||||
recipe_id = path.name[: -len(".recipe.json")]
|
||||
raw_data.append({"id": recipe_id, "title": f"Recipe {recipe_id}"})
|
||||
stat = path.stat()
|
||||
file_stats[str(path)] = (stat.st_mtime, stat.st_size)
|
||||
return PersistedRecipeData(
|
||||
raw_data=raw_data, file_stats=file_stats, image_id_map={}
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def guard_scanner(tmp_path: Path, monkeypatch):
|
||||
"""RecipeScanner wired to a real persistent cache, without a ComfyUI app."""
|
||||
RecipeScanner._instance = None
|
||||
settings_manager_module.reset_settings_manager()
|
||||
monkeypatch.setattr(config, "loras_roots", [str(tmp_path / "loras-root")])
|
||||
|
||||
scanner = RecipeScanner.__new__(RecipeScanner)
|
||||
scanner._persistent_cache = PersistentRecipeCache(
|
||||
db_path=str(tmp_path / "recipe_cache.sqlite")
|
||||
)
|
||||
scanner._cache = None
|
||||
scanner._json_path_map = {}
|
||||
scanner._lora_scanner = SimpleNamespace()
|
||||
|
||||
yield scanner, scanner._persistent_cache
|
||||
|
||||
RecipeScanner._instance = None
|
||||
settings_manager_module.reset_settings_manager()
|
||||
|
||||
|
||||
def test_reconcile_flags_prune_when_every_persisted_file_is_gone(
|
||||
guard_scanner, tmp_path: Path
|
||||
):
|
||||
"""An empty recipes dir must not be reported as a trustworthy prune."""
|
||||
scanner, _cache = guard_scanner
|
||||
recipes_dir = tmp_path / "recipes"
|
||||
recipes_dir.mkdir()
|
||||
|
||||
# The files used to live at another root (a changed recipes_path) and are
|
||||
# all gone from the directory the scanner resolved this time.
|
||||
old_files = [tmp_path / "elsewhere" / f"r{idx}.recipe.json" for idx in range(3)]
|
||||
for path in old_files:
|
||||
_write_recipe_json(path, path.name[: -len(".recipe.json")])
|
||||
persisted = _persisted_for(old_files)
|
||||
for path in old_files:
|
||||
path.unlink()
|
||||
|
||||
recipes, changed, json_paths, skipped_prune_reason = (
|
||||
scanner._reconcile_recipe_cache(persisted, str(recipes_dir))
|
||||
)
|
||||
|
||||
assert recipes == []
|
||||
assert json_paths == {}
|
||||
assert changed is True
|
||||
assert skipped_prune_reason is not None
|
||||
assert str(recipes_dir) in skipped_prune_reason
|
||||
assert "3" in skipped_prune_reason
|
||||
|
||||
|
||||
def test_reconcile_prunes_normally_when_only_some_files_disappear(
|
||||
guard_scanner, tmp_path: Path
|
||||
):
|
||||
"""A partial orphan is an ordinary deletion and keeps its old behaviour."""
|
||||
scanner, _cache = guard_scanner
|
||||
recipes_dir = tmp_path / "recipes"
|
||||
|
||||
survivor = recipes_dir / "survivor.recipe.json"
|
||||
_write_recipe_json(survivor, "survivor")
|
||||
vanished = recipes_dir / "vanished.recipe.json"
|
||||
_write_recipe_json(vanished, "vanished")
|
||||
persisted = _persisted_for([survivor, vanished])
|
||||
vanished.unlink()
|
||||
|
||||
recipes, changed, _json_paths, skipped_prune_reason = (
|
||||
scanner._reconcile_recipe_cache(persisted, str(recipes_dir))
|
||||
)
|
||||
|
||||
assert skipped_prune_reason is None
|
||||
assert changed is True
|
||||
assert [recipe["id"] for recipe in recipes] == ["survivor"]
|
||||
|
||||
|
||||
def test_reconcile_ignores_empty_persisted_cache(guard_scanner, tmp_path: Path):
|
||||
"""A genuinely empty cache has nothing to lose and must not be guarded."""
|
||||
scanner, _cache = guard_scanner
|
||||
recipes_dir = tmp_path / "recipes"
|
||||
recipes_dir.mkdir()
|
||||
|
||||
persisted = PersistedRecipeData(raw_data=[], file_stats={}, image_id_map={})
|
||||
|
||||
_recipes, changed, _json_paths, skipped_prune_reason = (
|
||||
scanner._reconcile_recipe_cache(persisted, str(recipes_dir))
|
||||
)
|
||||
|
||||
assert changed is False
|
||||
assert skipped_prune_reason is None
|
||||
|
||||
|
||||
def test_reconcile_prunes_when_stored_metadata_is_inconsistent(
|
||||
guard_scanner, tmp_path: Path
|
||||
):
|
||||
"""A stale row set must not masquerade as a fresh mass disappearance.
|
||||
|
||||
Leftover rows (rows without a recorded file stat) mean the stored cache is
|
||||
already out of date; guarding them would preserve orphans forever.
|
||||
"""
|
||||
scanner, _cache = guard_scanner
|
||||
recipes_dir = tmp_path / "recipes"
|
||||
recipes_dir.mkdir()
|
||||
|
||||
gone = tmp_path / "old-location" / "kept.recipe.json"
|
||||
_write_recipe_json(gone, "kept")
|
||||
persisted = _persisted_for([gone])
|
||||
gone.unlink()
|
||||
# A row with no matching file record: the cache diverged at some point.
|
||||
persisted.raw_data.append({"id": "orphan-row", "title": "Orphan"})
|
||||
|
||||
recipes, changed, _json_paths, skipped_prune_reason = (
|
||||
scanner._reconcile_recipe_cache(persisted, str(recipes_dir))
|
||||
)
|
||||
|
||||
assert recipes == []
|
||||
assert changed is True
|
||||
assert skipped_prune_reason is None
|
||||
|
||||
|
||||
def test_sync_init_keeps_stored_cache_when_scan_finds_nothing(
|
||||
guard_scanner, tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||||
):
|
||||
"""The startup path must not overwrite the stored cache with an empty one."""
|
||||
scanner, cache = guard_scanner
|
||||
recipes_dir = Path(config.loras_roots[0]) / "recipes"
|
||||
gone = tmp_path / "old-location" / "kept.recipe.json"
|
||||
_write_recipe_json(gone, "kept")
|
||||
|
||||
assert cache.save_cache(
|
||||
[{"id": "kept", "title": "Recipe kept"}], {"kept": str(gone)}
|
||||
)
|
||||
gone.unlink()
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger=recipe_scanner_module.__name__):
|
||||
scanner._initialize_recipe_cache_sync()
|
||||
|
||||
assert "Recipe cache prune skipped" in caplog.text
|
||||
assert scanner._prune_skipped is True
|
||||
# The stored cache survived, so the recipes remain recoverable.
|
||||
persisted = cache.load_cache()
|
||||
assert persisted is not None
|
||||
assert [recipe["id"] for recipe in persisted.raw_data] == ["kept"]
|
||||
|
||||
|
||||
def test_skipped_prune_leaves_fts_index_untouched(guard_scanner, tmp_path: Path):
|
||||
"""A skipped prune must not rebuild the FTS index from the empty view."""
|
||||
scanner, cache = guard_scanner
|
||||
gone = tmp_path / "old-location" / "kept.recipe.json"
|
||||
_write_recipe_json(gone, "kept")
|
||||
assert cache.save_cache(
|
||||
[{"id": "kept", "title": "Recipe kept"}], {"kept": str(gone)}
|
||||
)
|
||||
gone.unlink()
|
||||
|
||||
schedule_calls = []
|
||||
scanner._schedule_fts_index_build = lambda: schedule_calls.append(True)
|
||||
|
||||
scanner._initialize_recipe_cache_sync()
|
||||
|
||||
assert scanner._prune_skipped is True
|
||||
assert schedule_calls == []
|
||||
|
||||
|
||||
def test_sync_init_persists_when_recipes_are_found(guard_scanner, tmp_path: Path):
|
||||
"""The guard must not block a normal successful scan."""
|
||||
scanner, cache = guard_scanner
|
||||
recipes_dir = Path(config.loras_roots[0]) / "recipes"
|
||||
_write_recipe_json(recipes_dir / "fresh.recipe.json", "fresh")
|
||||
|
||||
scanner._initialize_recipe_cache_sync()
|
||||
|
||||
persisted = cache.load_cache()
|
||||
assert persisted is not None
|
||||
assert [recipe["id"] for recipe in persisted.raw_data] == ["fresh"]
|
||||
|
||||
|
||||
def test_force_refresh_scan_persists_an_empty_result(guard_scanner, tmp_path: Path):
|
||||
"""A manual rebuild stays the escape hatch from a skipped prune.
|
||||
|
||||
The startup guard deliberately keeps a stale cache, which leaves the in-memory
|
||||
view empty until the files come back. An explicit rebuild must be able to land
|
||||
on the real (empty) filesystem state instead, otherwise there is no way out.
|
||||
The route to it is `refresh_cache(force=True)`, which clears the stored cache
|
||||
first and then does a full directory scan.
|
||||
"""
|
||||
scanner, cache = guard_scanner
|
||||
gone = tmp_path / "old-location" / "kept.recipe.json"
|
||||
_write_recipe_json(gone, "kept")
|
||||
assert cache.save_cache(
|
||||
[{"id": "kept", "title": "Recipe kept"}], {"kept": str(gone)}
|
||||
)
|
||||
gone.unlink()
|
||||
|
||||
# Simulate the explicit rebuild: clear the stored cache, then full scan.
|
||||
assert cache.save_cache([], {}) is True
|
||||
scanner._initialize_recipe_cache_sync()
|
||||
|
||||
assert scanner._prune_skipped is False
|
||||
persisted = cache.load_cache()
|
||||
assert persisted is None or persisted.raw_data == []
|
||||
|
||||
|
||||
def test_save_cache_skip_if_empty_preserves_existing_rows(tmp_path: Path):
|
||||
"""The storage-level backstop refuses to empty a populated cache."""
|
||||
cache = PersistentRecipeCache(db_path=str(tmp_path / "recipe_cache.sqlite"))
|
||||
assert cache.save_cache([{"id": "r1", "title": "One"}], {"r1": "/tmp/r1.json"})
|
||||
|
||||
written = cache.save_cache([], {}, skip_if_empty=True)
|
||||
|
||||
assert written is False
|
||||
persisted = cache.load_cache()
|
||||
assert persisted is not None
|
||||
assert [recipe["id"] for recipe in persisted.raw_data] == ["r1"]
|
||||
|
||||
|
||||
def test_save_cache_skip_if_empty_allows_clearing_an_empty_cache(tmp_path: Path):
|
||||
"""Nothing to protect: an already-empty cache still returns success."""
|
||||
cache = PersistentRecipeCache(db_path=str(tmp_path / "recipe_cache.sqlite"))
|
||||
|
||||
assert cache.save_cache([], {}, skip_if_empty=True) is True
|
||||
|
||||
|
||||
def test_save_cache_default_still_allows_intentional_full_clear(tmp_path: Path):
|
||||
"""A manual rebuild passes skip_if_empty=False and must clear the cache."""
|
||||
cache = PersistentRecipeCache(db_path=str(tmp_path / "recipe_cache.sqlite"))
|
||||
assert cache.save_cache([{"id": "r1", "title": "One"}], {"r1": "/tmp/r1.json"})
|
||||
|
||||
assert cache.save_cache([], {}) is True
|
||||
|
||||
persisted = cache.load_cache()
|
||||
assert persisted is None or persisted.raw_data == []
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Tests for the shared cache SQLite connection settings (:mod:`py.utils.cache_db`).
|
||||
|
||||
Two LoRA Manager processes can share one settings directory, so cache
|
||||
connections must tolerate a competing writer instead of failing immediately
|
||||
with "database is locked".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
|
||||
from py.utils.cache_db import CONCURRENT_TIMEOUT_SECONDS, connect_cache_db
|
||||
|
||||
|
||||
def test_busy_timeout_pragma_is_applied(tmp_path):
|
||||
"""The connection must retry inside SQLite, not just at connect() time."""
|
||||
conn = connect_cache_db(str(tmp_path / "cache.sqlite"))
|
||||
try:
|
||||
value = conn.execute("PRAGMA busy_timeout").fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
assert value == int(CONCURRENT_TIMEOUT_SECONDS * 1000)
|
||||
|
||||
|
||||
def test_waiting_writer_succeeds_after_competing_writer_commits(tmp_path):
|
||||
"""A blocked writer waits for the lock instead of raising."""
|
||||
db_path = str(tmp_path / "cache.sqlite")
|
||||
|
||||
holder = connect_cache_db(db_path)
|
||||
holder.execute("CREATE TABLE t (v INTEGER)")
|
||||
holder.commit()
|
||||
holder.execute("BEGIN IMMEDIATE")
|
||||
|
||||
def release_after_delay() -> None:
|
||||
time.sleep(0.5)
|
||||
holder.commit()
|
||||
|
||||
releaser = threading.Thread(target=release_after_delay)
|
||||
releaser.start()
|
||||
try:
|
||||
waiter = connect_cache_db(db_path)
|
||||
try:
|
||||
# Under the old 5s default this still worked, but an immediate
|
||||
# failure is what low-timeout connections produced; assert the
|
||||
# write lands rather than propagating "database is locked".
|
||||
waiter.execute("INSERT INTO t VALUES (1)")
|
||||
waiter.commit()
|
||||
finally:
|
||||
waiter.close()
|
||||
finally:
|
||||
releaser.join()
|
||||
holder.close()
|
||||
|
||||
check = connect_cache_db(db_path)
|
||||
try:
|
||||
assert check.execute("SELECT COUNT(*) FROM t").fetchone()[0] == 1
|
||||
finally:
|
||||
check.close()
|
||||
|
||||
|
||||
def test_readwrite_connection_uses_row_factory(tmp_path):
|
||||
conn = connect_cache_db(str(tmp_path / "cache.sqlite"), row_factory=sqlite3.Row)
|
||||
try:
|
||||
conn.execute("CREATE TABLE t (v INTEGER)")
|
||||
conn.execute("INSERT INTO t VALUES (7)")
|
||||
conn.commit()
|
||||
row = conn.execute("SELECT v FROM t").fetchone()
|
||||
assert row["v"] == 7
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_readonly_connection_reads_without_writing(tmp_path):
|
||||
db_path = str(tmp_path / "cache.sqlite")
|
||||
writer = connect_cache_db(db_path)
|
||||
writer.execute("CREATE TABLE t (v INTEGER)")
|
||||
writer.execute("INSERT INTO t VALUES (1)")
|
||||
writer.commit()
|
||||
writer.close()
|
||||
|
||||
conn = connect_cache_db(db_path, readonly=True)
|
||||
try:
|
||||
assert conn.execute("SELECT v FROM t").fetchone()[0] == 1
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_readonly_connection_rejects_writes(tmp_path):
|
||||
db_path = str(tmp_path / "cache.sqlite")
|
||||
writer = connect_cache_db(db_path)
|
||||
writer.execute("CREATE TABLE t (v INTEGER)")
|
||||
writer.commit()
|
||||
writer.close()
|
||||
|
||||
conn = connect_cache_db(db_path, readonly=True)
|
||||
try:
|
||||
try:
|
||||
conn.execute("INSERT INTO t VALUES (1)")
|
||||
conn.commit()
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
else: # pragma: no cover - would mean mode=ro was not applied
|
||||
raise AssertionError("read-only connection accepted a write")
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -9,6 +9,7 @@ from typing import Any, Dict, List, Tuple
|
||||
import pytest
|
||||
|
||||
from py.utils import example_images_metadata as metadata_module
|
||||
from tests.utils.test_video_dimension_probe import build_mp4, build_webm
|
||||
|
||||
|
||||
class StubScanner:
|
||||
@@ -217,3 +218,127 @@ async def test_update_metadata_from_local_examples_generates_entries(monkeypatch
|
||||
)
|
||||
assert success is True
|
||||
assert model_data["civitai"]["images"]
|
||||
|
||||
|
||||
async def test_update_metadata_after_import_uses_real_video_dimensions(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path, patch_metadata_manager
|
||||
):
|
||||
"""Regression: imported videos must not fall back to the 720x1280 default.
|
||||
|
||||
See issue #1115 — landscape videos were stored as portrait, so the showcase
|
||||
viewer letterboxed them into a 9:16 container.
|
||||
"""
|
||||
|
||||
model_hash = "d" * 64
|
||||
model_file = tmp_path / "video-model.safetensors"
|
||||
model_file.write_text("content", encoding="utf-8")
|
||||
model_data = {
|
||||
"model_name": "VideoExample",
|
||||
"file_path": str(model_file),
|
||||
"civitai": {},
|
||||
}
|
||||
scanner = StubScanner([model_data])
|
||||
|
||||
video_path = tmp_path / "custom_abc.mp4"
|
||||
video_path.write_bytes(build_mp4(1280, 720))
|
||||
|
||||
monkeypatch.setattr(metadata_module.ExifUtils, "extract_image_metadata", staticmethod(lambda _path: None))
|
||||
|
||||
_regular, custom = await metadata_module.MetadataUpdater.update_metadata_after_import(
|
||||
model_hash,
|
||||
model_data,
|
||||
scanner,
|
||||
[(str(video_path), "abc")],
|
||||
)
|
||||
|
||||
assert custom[0]["type"] == "video"
|
||||
assert (custom[0]["width"], custom[0]["height"]) == (1280, 720)
|
||||
assert patch_metadata_manager[-1][1]["civitai"]["customImages"][0]["width"] == 1280
|
||||
|
||||
|
||||
async def test_update_metadata_after_import_uses_real_webm_dimensions(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path, patch_metadata_manager
|
||||
):
|
||||
model_hash = "e" * 64
|
||||
model_file = tmp_path / "webm-model.safetensors"
|
||||
model_file.write_text("content", encoding="utf-8")
|
||||
model_data = {
|
||||
"model_name": "WebmExample",
|
||||
"file_path": str(model_file),
|
||||
"civitai": {},
|
||||
}
|
||||
|
||||
video_path = tmp_path / "custom_def.webm"
|
||||
video_path.write_bytes(build_webm(480, 832))
|
||||
|
||||
monkeypatch.setattr(metadata_module.ExifUtils, "extract_image_metadata", staticmethod(lambda _path: None))
|
||||
|
||||
_regular, custom = await metadata_module.MetadataUpdater.update_metadata_after_import(
|
||||
model_hash,
|
||||
model_data,
|
||||
StubScanner([model_data]),
|
||||
[(str(video_path), "def")],
|
||||
)
|
||||
|
||||
assert (custom[0]["width"], custom[0]["height"]) == (480, 832)
|
||||
|
||||
|
||||
async def test_update_metadata_after_import_falls_back_for_unreadable_video(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path, patch_metadata_manager
|
||||
):
|
||||
"""An unparsable video keeps the legacy placeholder rather than failing."""
|
||||
|
||||
model_hash = "f" * 64
|
||||
model_file = tmp_path / "broken-model.safetensors"
|
||||
model_file.write_text("content", encoding="utf-8")
|
||||
model_data = {
|
||||
"model_name": "BrokenExample",
|
||||
"file_path": str(model_file),
|
||||
"civitai": {},
|
||||
}
|
||||
|
||||
video_path = tmp_path / "custom_ghi.mp4"
|
||||
video_path.write_bytes(b"\x00\x00\x00\x20ftypisom" + b"\xff" * 32)
|
||||
|
||||
monkeypatch.setattr(metadata_module.ExifUtils, "extract_image_metadata", staticmethod(lambda _path: None))
|
||||
|
||||
_regular, custom = await metadata_module.MetadataUpdater.update_metadata_after_import(
|
||||
model_hash,
|
||||
model_data,
|
||||
StubScanner([model_data]),
|
||||
[(str(video_path), "ghi")],
|
||||
)
|
||||
|
||||
assert (custom[0]["width"], custom[0]["height"]) == (720, 1280)
|
||||
|
||||
|
||||
async def test_update_metadata_from_local_examples_uses_real_video_dimensions(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path
|
||||
):
|
||||
model_hash = "1" * 64
|
||||
model_dir = tmp_path / model_hash
|
||||
model_dir.mkdir()
|
||||
(model_dir / "clip.mp4").write_bytes(build_mp4(1920, 1080))
|
||||
model_data: Dict[str, Any] = {
|
||||
"model_name": "LocalVideo",
|
||||
"civitai": {},
|
||||
"file_path": str(tmp_path / "model.safetensors"),
|
||||
}
|
||||
|
||||
async def fake_save(path, metadata):
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(metadata_module.MetadataManager, "save_metadata", staticmethod(fake_save))
|
||||
|
||||
success = await metadata_module.MetadataUpdater.update_metadata_from_local_examples(
|
||||
model_hash,
|
||||
model_data,
|
||||
"lora",
|
||||
StubScanner([model_data]),
|
||||
str(model_dir),
|
||||
)
|
||||
|
||||
assert success is True
|
||||
entry = model_data["civitai"]["images"][0]
|
||||
assert entry["type"] == "video"
|
||||
assert (entry["width"], entry["height"]) == (1920, 1080)
|
||||
@@ -177,3 +177,156 @@ async def test_migrations_run_and_update_progress(tmp_path, monkeypatch):
|
||||
update_args = lora_scanner.update_calls[0]
|
||||
assert update_args[0] == str(metadata_path)
|
||||
assert update_args[2]["civitai"]["customImages"][0]["id"] == "short1234"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v2_to_v3_migration_repairs_video_dimensions(tmp_path, monkeypatch):
|
||||
"""Upgrading a library already at v2 backfills local video dimensions once.
|
||||
|
||||
This mirrors the real upgrade path for issue #1115: the naming migration is
|
||||
already done, but imported videos still carry the 720x1280 placeholder.
|
||||
"""
|
||||
|
||||
from tests.utils.test_video_dimension_probe import build_mp4
|
||||
|
||||
example_root = tmp_path / "example_images"
|
||||
library_root = example_root / "main"
|
||||
library_root.mkdir(parents=True)
|
||||
|
||||
progress_path = library_root / ".download_progress.json"
|
||||
progress_path.write_text(json.dumps({"naming_version": 2}))
|
||||
|
||||
model_hash = "d" * 64
|
||||
model_folder = library_root / model_hash
|
||||
model_folder.mkdir()
|
||||
# Landscape clip stored during the buggy import path.
|
||||
(model_folder / "custom_land1.mp4").write_bytes(build_mp4(1280, 720))
|
||||
|
||||
model_file = tmp_path / "models" / "video.safetensors"
|
||||
model_file.parent.mkdir()
|
||||
model_file.write_text("weights", encoding="utf-8")
|
||||
|
||||
scanner = FakeScanner(
|
||||
{
|
||||
model_hash: {
|
||||
"sha256": model_hash,
|
||||
"file_path": str(model_file),
|
||||
"civitai": {
|
||||
"images": [
|
||||
{"url": "https://example.com/remote.jpg", "type": "image", "width": 512, "height": 512}
|
||||
],
|
||||
"customImages": [
|
||||
{"url": "", "id": "land1", "type": "video", "width": 720, "height": 1280}
|
||||
],
|
||||
},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
async def fake_get_lora_scanner(cls):
|
||||
return scanner
|
||||
|
||||
async def fake_get_checkpoint_scanner(cls):
|
||||
return FakeScanner({})
|
||||
|
||||
monkeypatch.setattr(
|
||||
migration_module.ServiceRegistry, "get_lora_scanner", classmethod(fake_get_lora_scanner)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
migration_module.ServiceRegistry,
|
||||
"get_checkpoint_scanner",
|
||||
classmethod(fake_get_checkpoint_scanner),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
migration_module.settings,
|
||||
"get",
|
||||
lambda key, default=None: str(example_root) if key == "example_images_path" else default,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
migration_module,
|
||||
"iter_library_roots",
|
||||
lambda: [("main", str(library_root))],
|
||||
)
|
||||
|
||||
saved_metadata = []
|
||||
|
||||
async def fake_save_metadata(path, metadata):
|
||||
saved_metadata.append((path, metadata))
|
||||
return True
|
||||
|
||||
async def fake_load_payload(path):
|
||||
return {
|
||||
"model_name": "Video",
|
||||
"civitai": {
|
||||
"images": [
|
||||
{"url": "https://example.com/remote.jpg", "type": "image", "width": 512, "height": 512}
|
||||
],
|
||||
"customImages": [
|
||||
{"url": "", "id": "land1", "type": "video", "width": 720, "height": 1280}
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
migration_module.MetadataManager, "save_metadata", staticmethod(fake_save_metadata)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
migration_module.MetadataManager, "load_metadata_payload", staticmethod(fake_load_payload)
|
||||
)
|
||||
|
||||
scheduled = []
|
||||
original_create_task = asyncio.create_task
|
||||
|
||||
def capture_create_task(coro, *args, **kwargs):
|
||||
task = original_create_task(coro, *args, **kwargs)
|
||||
scheduled.append(task)
|
||||
return task
|
||||
|
||||
monkeypatch.setattr(migration_module.asyncio, "create_task", capture_create_task)
|
||||
|
||||
await migration_module.ExampleImagesMigration.check_and_run_migrations()
|
||||
await asyncio.gather(*scheduled)
|
||||
|
||||
assert len(saved_metadata) == 1
|
||||
_path, payload = saved_metadata[0]
|
||||
entry = payload["civitai"]["customImages"][0]
|
||||
assert (entry["width"], entry["height"]) == (1280, 720)
|
||||
# Remote-backed entry is untouched.
|
||||
assert payload["civitai"]["images"][0]["width"] == 512
|
||||
|
||||
assert json.loads(progress_path.read_text())["naming_version"] == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v3_migration_does_not_run_twice(tmp_path, monkeypatch):
|
||||
"""The version gate keeps the repair off the startup path after one run."""
|
||||
|
||||
example_root = tmp_path / "example_images"
|
||||
library_root = example_root / "main"
|
||||
library_root.mkdir(parents=True)
|
||||
(library_root / ".download_progress.json").write_text(json.dumps({"naming_version": 3}))
|
||||
|
||||
monkeypatch.setattr(
|
||||
migration_module.settings,
|
||||
"get",
|
||||
lambda key, default=None: str(example_root) if key == "example_images_path" else default,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
migration_module,
|
||||
"iter_library_roots",
|
||||
lambda: [("main", str(library_root))],
|
||||
)
|
||||
|
||||
called = []
|
||||
|
||||
async def spy_run_migrations(*args, **kwargs):
|
||||
called.append(args)
|
||||
|
||||
monkeypatch.setattr(
|
||||
migration_module.ExampleImagesMigration, "run_migrations", staticmethod(spy_run_migrations)
|
||||
)
|
||||
|
||||
await migration_module.ExampleImagesMigration.check_and_run_migrations()
|
||||
|
||||
assert called == []
|
||||
|
||||
@@ -0,0 +1,299 @@
|
||||
"""Tests for the one-shot repair of locally imported video dimensions (issue #1115)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
import pytest
|
||||
|
||||
from py.utils import example_images_migration as migration_module
|
||||
from py.utils import example_images_metadata as metadata_module
|
||||
from tests.utils.test_video_dimension_probe import build_mp4
|
||||
|
||||
|
||||
def _metadata_payload(**civitai: Any) -> Dict[str, Any]:
|
||||
return {"model_name": "Example", "civitai": civitai}
|
||||
|
||||
|
||||
def test_repair_backfills_landscape_video_dimensions(tmp_path: Path):
|
||||
video = tmp_path / "custom_abc123.mp4"
|
||||
video.write_bytes(build_mp4(1280, 720))
|
||||
|
||||
payload = _metadata_payload(
|
||||
customImages=[
|
||||
{
|
||||
"url": "",
|
||||
"id": "abc123",
|
||||
"type": "video",
|
||||
"width": 720,
|
||||
"height": 1280,
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
repaired = metadata_module.repair_local_video_dimensions(
|
||||
payload, {"abc123": str(video)}
|
||||
)
|
||||
|
||||
assert repaired == 1
|
||||
entry = payload["civitai"]["customImages"][0]
|
||||
assert (entry["width"], entry["height"]) == (1280, 720)
|
||||
|
||||
|
||||
def test_repair_handles_index_marked_images_array(tmp_path: Path):
|
||||
video = tmp_path / "image_3.mp4"
|
||||
video.write_bytes(build_mp4(1920, 1080))
|
||||
|
||||
payload = _metadata_payload(
|
||||
images=[
|
||||
{"url": "https://example.com/remote.png", "type": "image"},
|
||||
{"url": "", "type": "video", "width": 720, "height": 1280},
|
||||
{"url": "", "type": "video", "width": 720, "height": 1280},
|
||||
{"url": "", "type": "video", "width": 720, "height": 1280},
|
||||
]
|
||||
)
|
||||
|
||||
repaired = metadata_module.repair_local_video_dimensions(
|
||||
payload, {"3": str(video)}
|
||||
)
|
||||
|
||||
assert repaired == 1
|
||||
# Position 3 (index 3) is the one carrying the local file.
|
||||
assert payload["civitai"]["images"][3]["width"] == 1920
|
||||
assert payload["civitai"]["images"][3]["height"] == 1080
|
||||
# The remote entry keeps its API-provided shape.
|
||||
assert payload["civitai"]["images"][0].get("width") is None
|
||||
|
||||
|
||||
def test_repair_never_touches_remote_entries(tmp_path: Path):
|
||||
"""Remote entries keep API-provided dimensions even if a file exists."""
|
||||
|
||||
video = tmp_path / "custom_remote.mp4"
|
||||
video.write_bytes(build_mp4(1280, 720))
|
||||
|
||||
payload = _metadata_payload(
|
||||
customImages=[
|
||||
{
|
||||
"url": "https://civitai.com/1234.mp4",
|
||||
"id": "remote",
|
||||
"type": "video",
|
||||
"width": 720,
|
||||
"height": 1280,
|
||||
}
|
||||
]
|
||||
)
|
||||
before = json.dumps(payload, sort_keys=True)
|
||||
|
||||
repaired = metadata_module.repair_local_video_dimensions(
|
||||
payload, {"remote": str(video)}
|
||||
)
|
||||
|
||||
assert repaired == 0
|
||||
assert json.dumps(payload, sort_keys=True) == before
|
||||
|
||||
|
||||
def test_repair_is_idempotent(tmp_path: Path):
|
||||
video = tmp_path / "custom_abc.mp4"
|
||||
video.write_bytes(build_mp4(1280, 720))
|
||||
|
||||
payload = _metadata_payload(
|
||||
customImages=[{"url": "", "id": "abc", "type": "video", "width": 720, "height": 1280}]
|
||||
)
|
||||
files = {"abc": str(video)}
|
||||
|
||||
assert metadata_module.repair_local_video_dimensions(payload, files) == 1
|
||||
# Second run finds nothing to do and leaves the entry byte-identical.
|
||||
snapshot = json.dumps(payload, sort_keys=True)
|
||||
assert metadata_module.repair_local_video_dimensions(payload, files) == 0
|
||||
assert json.dumps(payload, sort_keys=True) == snapshot
|
||||
|
||||
|
||||
def test_repair_dry_run_does_not_mutate(tmp_path: Path):
|
||||
video = tmp_path / "custom_abc.mp4"
|
||||
video.write_bytes(build_mp4(1280, 720))
|
||||
|
||||
payload = _metadata_payload(
|
||||
customImages=[{"url": "", "id": "abc", "type": "video", "width": 720, "height": 1280}]
|
||||
)
|
||||
before = json.dumps(payload, sort_keys=True)
|
||||
|
||||
repaired = metadata_module.repair_local_video_dimensions(
|
||||
payload, {"abc": str(video)}, dry_run=True
|
||||
)
|
||||
|
||||
assert repaired == 1
|
||||
assert json.dumps(payload, sort_keys=True) == before
|
||||
|
||||
|
||||
def test_repair_skips_missing_file(tmp_path: Path):
|
||||
payload = _metadata_payload(
|
||||
customImages=[{"url": "", "id": "gone", "type": "video", "width": 720, "height": 1280}]
|
||||
)
|
||||
|
||||
repaired = metadata_module.repair_local_video_dimensions(
|
||||
payload, {"gone": str(tmp_path / "does-not-exist.mp4")}
|
||||
)
|
||||
|
||||
assert repaired == 0
|
||||
assert payload["civitai"]["customImages"][0]["width"] == 720
|
||||
|
||||
|
||||
def test_repair_leaves_correct_entries_untouched(tmp_path: Path):
|
||||
video = tmp_path / "custom_ok.mp4"
|
||||
video.write_bytes(build_mp4(1280, 720))
|
||||
|
||||
payload = _metadata_payload(
|
||||
customImages=[{"url": "", "id": "ok", "type": "video", "width": 1280, "height": 720}]
|
||||
)
|
||||
|
||||
assert metadata_module.repair_local_video_dimensions(payload, {"ok": str(video)}) == 0
|
||||
|
||||
|
||||
def test_local_file_map_keys_strip_naming_prefix(tmp_path: Path):
|
||||
(tmp_path / "custom_abc.mp4").write_bytes(build_mp4(1280, 720))
|
||||
(tmp_path / "image_2.png").write_bytes(b"not-a-real-image")
|
||||
(tmp_path / "notes.txt").write_text("ignore me", encoding="utf-8")
|
||||
|
||||
mapping = migration_module.ExampleImagesMigration._build_local_file_map(str(tmp_path))
|
||||
|
||||
assert set(mapping) == {"abc", "2"}
|
||||
|
||||
|
||||
async def test_migrate_to_v3_repairs_and_syncs_cache(tmp_path: Path, monkeypatch):
|
||||
model_hash = "a" * 64
|
||||
folder = tmp_path / model_hash
|
||||
folder.mkdir()
|
||||
(folder / "custom_xyz.mp4").write_bytes(build_mp4(1080, 1920))
|
||||
|
||||
model_file = tmp_path / "model.safetensors"
|
||||
model_file.write_text("weights", encoding="utf-8")
|
||||
|
||||
payload = _metadata_payload(
|
||||
customImages=[{"url": "", "id": "xyz", "type": "video", "width": 720, "height": 1280}]
|
||||
)
|
||||
saved: list[tuple[str, Dict[str, Any]]] = []
|
||||
|
||||
async def fake_load(file_path):
|
||||
return dict(payload, civitai=dict(payload["civitai"]))
|
||||
|
||||
async def fake_save(file_path, data):
|
||||
saved.append((file_path, data))
|
||||
return True
|
||||
|
||||
synced: list[tuple[str, Dict[str, Any]]] = []
|
||||
|
||||
async def fake_sync(scanner, file_path, data):
|
||||
synced.append((file_path, data))
|
||||
return True
|
||||
|
||||
class StubScanner:
|
||||
def has_hash(self, _hash):
|
||||
return True
|
||||
|
||||
async def get_cached_data(self):
|
||||
from types import SimpleNamespace
|
||||
|
||||
return SimpleNamespace(raw_data=[{"sha256": model_hash, "file_path": str(model_file)}])
|
||||
|
||||
monkeypatch.setattr(migration_module.MetadataManager, "load_metadata_payload", fake_load)
|
||||
monkeypatch.setattr(migration_module.MetadataManager, "save_metadata", fake_save)
|
||||
monkeypatch.setattr(migration_module, "update_cache_from_metadata", fake_sync)
|
||||
|
||||
async def fake_lora():
|
||||
return StubScanner()
|
||||
|
||||
async def fake_none():
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(migration_module.ServiceRegistry, "get_lora_scanner", fake_lora)
|
||||
monkeypatch.setattr(migration_module.ServiceRegistry, "get_checkpoint_scanner", fake_none)
|
||||
monkeypatch.setattr(migration_module.ServiceRegistry, "get_embedding_scanner", fake_none)
|
||||
|
||||
await migration_module.ExampleImagesMigration._migrate_to_v3(
|
||||
str(tmp_path), [str(folder)]
|
||||
)
|
||||
|
||||
assert len(saved) == 1
|
||||
saved_entry = saved[0][1]["civitai"]["customImages"][0]
|
||||
assert (saved_entry["width"], saved_entry["height"]) == (1080, 1920)
|
||||
assert len(synced) == 1
|
||||
assert synced[0][1]["civitai"]["customImages"][0]["width"] == 1080
|
||||
|
||||
|
||||
async def test_migrate_to_v3_skips_when_nothing_to_repair(tmp_path: Path, monkeypatch):
|
||||
model_hash = "b" * 64
|
||||
folder = tmp_path / model_hash
|
||||
folder.mkdir()
|
||||
(folder / "custom_ok.mp4").write_bytes(build_mp4(1080, 1920))
|
||||
|
||||
model_file = tmp_path / "model.safetensors"
|
||||
model_file.write_text("weights", encoding="utf-8")
|
||||
|
||||
payload = _metadata_payload(
|
||||
customImages=[{"url": "", "id": "ok", "type": "video", "width": 1080, "height": 1920}]
|
||||
)
|
||||
saved: list[Any] = []
|
||||
|
||||
async def fake_load(file_path):
|
||||
return dict(payload, civitai=dict(payload["civitai"]))
|
||||
|
||||
async def fake_save(file_path, data):
|
||||
saved.append(data)
|
||||
return True
|
||||
|
||||
class StubScanner:
|
||||
def has_hash(self, _hash):
|
||||
return True
|
||||
|
||||
async def get_cached_data(self):
|
||||
from types import SimpleNamespace
|
||||
|
||||
return SimpleNamespace(raw_data=[{"sha256": model_hash, "file_path": str(model_file)}])
|
||||
|
||||
monkeypatch.setattr(migration_module.MetadataManager, "load_metadata_payload", fake_load)
|
||||
monkeypatch.setattr(migration_module.MetadataManager, "save_metadata", fake_save)
|
||||
|
||||
async def fake_lora():
|
||||
return StubScanner()
|
||||
|
||||
async def fake_none():
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(migration_module.ServiceRegistry, "get_lora_scanner", fake_lora)
|
||||
monkeypatch.setattr(migration_module.ServiceRegistry, "get_checkpoint_scanner", fake_none)
|
||||
monkeypatch.setattr(migration_module.ServiceRegistry, "get_embedding_scanner", fake_none)
|
||||
|
||||
await migration_module.ExampleImagesMigration._migrate_to_v3(str(tmp_path), [str(folder)])
|
||||
|
||||
# Correctly-sized entries are never rewritten.
|
||||
assert saved == []
|
||||
|
||||
|
||||
async def test_migrate_to_v3_skips_unindexed_model(tmp_path: Path, monkeypatch):
|
||||
"""A folder whose model is absent from every scanner cache is skipped, not fatal."""
|
||||
|
||||
model_hash = "c" * 64
|
||||
folder = tmp_path / model_hash
|
||||
folder.mkdir()
|
||||
(folder / "custom_zzz.mp4").write_bytes(build_mp4(1080, 1920))
|
||||
|
||||
class EmptyScanner:
|
||||
def has_hash(self, _hash):
|
||||
return False
|
||||
|
||||
async def get_cached_data(self):
|
||||
from types import SimpleNamespace
|
||||
|
||||
return SimpleNamespace(raw_data=[])
|
||||
|
||||
async def fake_scanner():
|
||||
return EmptyScanner()
|
||||
|
||||
monkeypatch.setattr(migration_module.ServiceRegistry, "get_lora_scanner", fake_scanner)
|
||||
monkeypatch.setattr(migration_module.ServiceRegistry, "get_checkpoint_scanner", fake_scanner)
|
||||
monkeypatch.setattr(migration_module.ServiceRegistry, "get_embedding_scanner", fake_scanner)
|
||||
|
||||
# Must not raise.
|
||||
await migration_module.ExampleImagesMigration._migrate_to_v3(str(tmp_path), [str(folder)])
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Tests for the cross-process advisory lock (:mod:`py.utils.file_lock`)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from py.utils.file_lock import (
|
||||
CrossProcessLock,
|
||||
FileLockUnavailable,
|
||||
exclusive_lock,
|
||||
lock_path_for,
|
||||
)
|
||||
|
||||
|
||||
def test_lock_path_is_a_sibling_of_the_resource(tmp_path):
|
||||
db_path = str(tmp_path / "recipe" / "default.sqlite")
|
||||
lock_path = lock_path_for(db_path)
|
||||
|
||||
assert os.path.dirname(lock_path) == os.path.dirname(db_path)
|
||||
assert os.path.basename(lock_path) == ".default.sqlite.lock"
|
||||
|
||||
|
||||
def test_acquire_and_release_round_trip(tmp_path):
|
||||
lock = exclusive_lock(str(tmp_path / "cache.sqlite"))
|
||||
|
||||
assert lock.acquire() is True
|
||||
lock.release()
|
||||
# Releasing twice must be safe.
|
||||
lock.release()
|
||||
# ...and the lock is reusable afterwards.
|
||||
assert lock.acquire() is True
|
||||
lock.release()
|
||||
|
||||
|
||||
def test_second_lock_holder_waits_until_release(tmp_path):
|
||||
"""A held lock blocks a competing holder for the same resource."""
|
||||
db_path = str(tmp_path / "cache.sqlite")
|
||||
first = exclusive_lock(db_path)
|
||||
second = CrossProcessLock(lock_path_for(db_path), timeout=0.2)
|
||||
|
||||
assert first.acquire() is True
|
||||
try:
|
||||
started = time.monotonic()
|
||||
assert second.acquire() is False
|
||||
# It must have waited for the timeout rather than failing instantly.
|
||||
assert time.monotonic() - started >= 0.15
|
||||
finally:
|
||||
first.release()
|
||||
|
||||
# Once released, the contender gets the lock.
|
||||
assert second.acquire() is True
|
||||
second.release()
|
||||
|
||||
|
||||
def test_context_manager_releases_on_exception(tmp_path):
|
||||
lock = exclusive_lock(str(tmp_path / "cache.sqlite"))
|
||||
contender = CrossProcessLock(lock.path, timeout=0.2)
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
with lock:
|
||||
raise RuntimeError("boom")
|
||||
|
||||
assert contender.acquire() is True
|
||||
contender.release()
|
||||
|
||||
|
||||
def test_lock_file_is_not_deleted(tmp_path):
|
||||
"""Deleting the lock file would let a second process lock a fresh inode."""
|
||||
lock = exclusive_lock(str(tmp_path / "cache.sqlite"))
|
||||
assert lock.acquire() is True
|
||||
lock.release()
|
||||
|
||||
assert os.path.exists(lock.path)
|
||||
|
||||
|
||||
def test_unsupported_platform_degrades_gracefully(tmp_path, monkeypatch):
|
||||
"""Without a platform primitive the lock reports failure instead of raising."""
|
||||
import py.utils.file_lock as file_lock_module
|
||||
|
||||
monkeypatch.setattr(file_lock_module, "fcntl", None)
|
||||
monkeypatch.setattr(file_lock_module, "msvcrt", None)
|
||||
|
||||
lock = exclusive_lock(str(tmp_path / "cache.sqlite"))
|
||||
assert lock.acquire() is False
|
||||
# Callers use it as a context manager and continue without the lock.
|
||||
with exclusive_lock(str(tmp_path / "cache.sqlite")):
|
||||
pass
|
||||
|
||||
|
||||
def test_file_lock_unavailable_is_exported():
|
||||
assert issubclass(FileLockUnavailable, RuntimeError)
|
||||
|
||||
|
||||
def test_save_cache_creates_lock_next_to_database(tmp_path):
|
||||
"""The recipe cache write path actually takes the cross-process lock."""
|
||||
from py.services.persistent_recipe_cache import PersistentRecipeCache
|
||||
|
||||
db_path = tmp_path / "recipe_cache.sqlite"
|
||||
cache = PersistentRecipeCache(db_path=str(db_path))
|
||||
assert cache.save_cache([{"id": "r1", "title": "One"}], {"r1": "/tmp/r1.json"})
|
||||
|
||||
assert os.path.exists(lock_path_for(str(db_path)))
|
||||
|
||||
|
||||
def test_save_cache_releases_lock_after_write(tmp_path):
|
||||
"""A second writer must not be blocked once the first has finished."""
|
||||
from py.services.persistent_recipe_cache import PersistentRecipeCache
|
||||
|
||||
db_path = tmp_path / "recipe_cache.sqlite"
|
||||
cache = PersistentRecipeCache(db_path=str(db_path))
|
||||
cache.save_cache([{"id": "r1", "title": "One"}], {"r1": "/tmp/r1.json"})
|
||||
|
||||
contender = CrossProcessLock(lock_path_for(str(db_path)), timeout=0.2)
|
||||
assert contender.acquire() is True
|
||||
contender.release()
|
||||
@@ -34,10 +34,12 @@ class TestShouldUsePortableSettings:
|
||||
@pytest.mark.parametrize(
|
||||
"env_value, settings_flag, expected",
|
||||
[
|
||||
("1", False, True), # env = 1 overrides settings.json false
|
||||
("1", False, True), # env = 1 forces portable on
|
||||
("1", True, True), # env = 1 matches settings.json true
|
||||
("0", False, False), # env = 0 → rely on settings.json
|
||||
("0", True, True), # env = 0 → rely on settings.json
|
||||
("0", False, False), # env = 0 forces portable off
|
||||
("0", True, False), # env = 0 overrides a persisted true
|
||||
("yes", False, False), # unrecognised value → rely on settings.json
|
||||
("yes", True, True), # unrecognised value → rely on settings.json
|
||||
("", False, False), # unset → rely on settings.json
|
||||
("", True, True), # unset → rely on settings.json
|
||||
],
|
||||
@@ -58,6 +60,21 @@ class TestShouldUsePortableSettings:
|
||||
result = _should_use_portable_settings(str(settings_file), logging.getLogger())
|
||||
assert result == expected
|
||||
|
||||
def test_explicit_zero_is_the_documented_opt_out(self, tmp_path, caplog):
|
||||
"""`=0` must be honoured even against a persisted true flag."""
|
||||
settings_file = tmp_path / "settings.json"
|
||||
settings_file.write_text(json.dumps({"use_portable_settings": True}))
|
||||
|
||||
with pytest.MonkeyPatch.context() as mp:
|
||||
mp.setenv("LORA_MANAGER_PORTABLE", "0")
|
||||
with caplog.at_level(logging.INFO):
|
||||
result = _should_use_portable_settings(
|
||||
str(settings_file), logging.getLogger()
|
||||
)
|
||||
|
||||
assert result is False
|
||||
assert "Portable mode disabled" in caplog.text
|
||||
|
||||
def test_missing_file_without_env(self, tmp_path):
|
||||
"""Without env var, missing settings file returns False."""
|
||||
missing = tmp_path / "nonexistent.json"
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Tests for the container-level video dimension probe."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
|
||||
from py.utils.video_metadata import get_video_dimensions
|
||||
|
||||
|
||||
def _box(box_type: bytes, payload: bytes) -> bytes:
|
||||
return struct.pack(">I", len(payload) + 8) + box_type + payload
|
||||
|
||||
|
||||
def _full_box(box_type: bytes, payload: bytes) -> bytes:
|
||||
"""Build a box with a 4-byte version/flags header."""
|
||||
|
||||
return _box(box_type, b"\x00\x00\x00\x00" + payload)
|
||||
|
||||
|
||||
def build_mp4(width: int, height: int, *, with_stsd: bool = False) -> bytes:
|
||||
"""Build a minimal but structurally valid MP4 holding one video track."""
|
||||
|
||||
mvhd = _full_box(b"mvhd", b"\x00" * 96)
|
||||
|
||||
hdlr = _full_box(b"hdlr", b"\x00" * 4 + b"vide" + b"\x00" * 12)
|
||||
|
||||
tkhd_payload = struct.pack(">IIII", 0, 0, 0, 0) + b"\x00" * 52
|
||||
tkhd_payload += struct.pack(">II", width << 16, height << 16)
|
||||
tkhd = _full_box(b"tkhd", tkhd_payload)
|
||||
|
||||
stbl_children = b""
|
||||
if with_stsd:
|
||||
sample_entry = (
|
||||
b"\x00" * 6 + struct.pack(">H", 1) + struct.pack(">HH", width, height)
|
||||
)
|
||||
stsd = _full_box(b"stsd", struct.pack(">I", 1) + _box(b"avc1", sample_entry))
|
||||
stbl_children = stsd
|
||||
|
||||
minf = _box(b"minf", _box(b"stbl", stbl_children))
|
||||
mdia = _box(b"mdia", hdlr + minf)
|
||||
trak = _box(b"trak", tkhd + mdia)
|
||||
moov = _box(b"moov", mvhd + trak)
|
||||
ftyp = _box(b"ftyp", b"isom" + b"\x00\x00\x02\x00" + b"isomiso2avc1mp41")
|
||||
|
||||
return ftyp + moov
|
||||
|
||||
|
||||
def _ebml_vint(value: int) -> bytes:
|
||||
"""Encode a value as a minimal-length EBML variable length integer."""
|
||||
|
||||
for length in range(1, 9):
|
||||
if value < (1 << (7 * length)):
|
||||
encoded = value | (1 << (7 * length))
|
||||
return encoded.to_bytes(length, "big")
|
||||
raise ValueError("value too large for an EBML vint")
|
||||
|
||||
|
||||
def _ebml_element(element_id: bytes, payload: bytes) -> bytes:
|
||||
return element_id + _ebml_vint(len(payload)) + payload
|
||||
|
||||
|
||||
def _uint_element(element_id: int, value: int) -> bytes:
|
||||
length = max(1, (value.bit_length() + 7) // 8)
|
||||
return _ebml_element(
|
||||
element_id.to_bytes(2, "big") if element_id > 0xFF else element_id.to_bytes(1, "big"),
|
||||
value.to_bytes(length, "big"),
|
||||
)
|
||||
|
||||
|
||||
def build_webm(width: int, height: int, *, track_type: int = 1) -> bytes:
|
||||
"""Build a minimal WebM file holding one TrackEntry."""
|
||||
|
||||
video = _ebml_element(b"\xe0", _uint_element(0xB0, width) + _uint_element(0xBA, height))
|
||||
track_entry = _ebml_element(
|
||||
b"\xae", _uint_element(0x83, track_type) + video
|
||||
)
|
||||
tracks = _ebml_element(b"\x16\x54\xae\x6b", track_entry)
|
||||
|
||||
segment = _ebml_element(b"\x18\x53\x80\x67", tracks)
|
||||
ebml_header = _ebml_element(
|
||||
b"\x1a\x45\xdf\xa3",
|
||||
_uint_element(0x4286, 1) + _ebml_element(b"\x42\x82", b"webm"),
|
||||
)
|
||||
|
||||
return ebml_header + segment
|
||||
|
||||
|
||||
def test_mp4_dimensions_come_from_tkhd(tmp_path):
|
||||
video = tmp_path / "landscape.mp4"
|
||||
video.write_bytes(build_mp4(1280, 720))
|
||||
|
||||
assert get_video_dimensions(str(video)) == (1280, 720)
|
||||
|
||||
|
||||
def test_mp4_uses_stsd_when_tkhd_is_empty(tmp_path):
|
||||
video = tmp_path / "stsd-only.mp4"
|
||||
video.write_bytes(build_mp4(640, 480, with_stsd=True))
|
||||
|
||||
assert get_video_dimensions(str(video)) == (640, 480)
|
||||
|
||||
|
||||
def test_mp4_without_video_track_returns_none(tmp_path):
|
||||
# A moov whose only trak has no mdia box at all.
|
||||
tkhd = _full_box(b"tkhd", b"\x00" * 60)
|
||||
moov = _box(b"moov", _box(b"trak", tkhd))
|
||||
video = tmp_path / "audio-only.mp4"
|
||||
video.write_bytes(moov)
|
||||
|
||||
assert get_video_dimensions(str(video)) is None
|
||||
|
||||
|
||||
def test_webm_dimensions(tmp_path):
|
||||
video = tmp_path / "portrait.webm"
|
||||
video.write_bytes(build_webm(720, 1280))
|
||||
|
||||
assert get_video_dimensions(str(video)) == (720, 1280)
|
||||
|
||||
|
||||
def test_webm_non_video_track_is_ignored(tmp_path):
|
||||
video = tmp_path / "audio.webm"
|
||||
video.write_bytes(build_webm(720, 1280, track_type=2))
|
||||
|
||||
assert get_video_dimensions(str(video)) is None
|
||||
|
||||
|
||||
def test_container_signature_wins_over_extension(tmp_path):
|
||||
"""A WebM file named ``.mp4`` is still parsed as WebM."""
|
||||
|
||||
video = tmp_path / "actually-webm.mp4"
|
||||
video.write_bytes(build_webm(480, 832))
|
||||
|
||||
assert get_video_dimensions(str(video)) == (480, 832)
|
||||
|
||||
|
||||
def test_webp_renamed_to_mp4_is_read(tmp_path):
|
||||
"""Animated WebP examples are frequently saved with a video extension."""
|
||||
|
||||
vp8_payload = b"\x30\x36\x02" + b"\x9d\x01\x2a" + struct.pack("<HH", 450, 800)
|
||||
chunk = b"VP8 " + struct.pack("<I", len(vp8_payload)) + vp8_payload
|
||||
body = b"WEBP" + chunk
|
||||
riff = b"RIFF" + struct.pack("<I", len(body)) + body
|
||||
|
||||
video = tmp_path / "animated.mp4"
|
||||
video.write_bytes(riff)
|
||||
|
||||
assert get_video_dimensions(str(video)) == (450, 800)
|
||||
|
||||
|
||||
def test_webp_vp8x_canvas_dimensions(tmp_path):
|
||||
vp8x_payload = b"\x00" * 4 + (449).to_bytes(3, "little") + (799).to_bytes(3, "little")
|
||||
chunk = b"VP8X" + struct.pack("<I", len(vp8x_payload)) + vp8x_payload
|
||||
body = b"WEBP" + chunk
|
||||
riff = b"RIFF" + struct.pack("<I", len(body)) + body
|
||||
|
||||
video = tmp_path / "canvas.mp4"
|
||||
video.write_bytes(riff)
|
||||
|
||||
assert get_video_dimensions(str(video)) == (450, 800)
|
||||
|
||||
|
||||
def test_missing_file_returns_none(tmp_path):
|
||||
assert get_video_dimensions(str(tmp_path / "nope.mp4")) is None
|
||||
|
||||
|
||||
def test_corrupt_file_returns_none(tmp_path):
|
||||
video = tmp_path / "corrupt.mp4"
|
||||
video.write_bytes(b"\x00\x00\x00\x20ftypisom" + b"\xff" * 64)
|
||||
|
||||
assert get_video_dimensions(str(video)) is None
|
||||
|
||||
|
||||
def test_unsupported_extension_without_video_signature_returns_none(tmp_path):
|
||||
"""A non-video file is not probed just because of a video-like name."""
|
||||
|
||||
video = tmp_path / "clip.avi"
|
||||
video.write_bytes(b"RIFF\x00\x00\x00\x00AVI LIST\x00\x00\x00\x00")
|
||||
|
||||
assert get_video_dimensions(str(video)) is None
|
||||
Reference in New Issue
Block a user