mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-09-21 03:01:27 -03:00
Compare commits
12 Commits
v1.2.3
...
d4b82d98b2
| Author | SHA1 | Date | |
|---|---|---|---|
| d4b82d98b2 | |||
| 8c1c1691e3 | |||
| e14a084f0d | |||
| c55c6f0a41 | |||
| 7d963b27b5 | |||
| eba03800b9 | |||
| bf497d5144 | |||
| 369613f811 | |||
| 9eeebac40b | |||
| b9a516c9f8 | |||
| ef7fa7d3dd | |||
| 9c67dbbf15 |
@@ -15,6 +15,7 @@ node_modules/
|
|||||||
coverage/
|
coverage/
|
||||||
.coverage
|
.coverage
|
||||||
model_cache/
|
model_cache/
|
||||||
|
recipe_cache/
|
||||||
|
|
||||||
# agent / dev tooling
|
# agent / dev tooling
|
||||||
.opencode/
|
.opencode/
|
||||||
|
|||||||
@@ -57,6 +57,15 @@ Locales: `en`, `zh-CN`, `zh-TW`, `ja`, `ko`, `fr`, `de`, `es`, `ru`, `he` (RTL).
|
|||||||
> prototyped and removed because it collided with the browser's Alt + Arrow handling and the
|
> prototyped and removed because it collided with the browser's Alt + Arrow handling and the
|
||||||
> modal's arrow-key navigation.
|
> modal's arrow-key navigation.
|
||||||
|
|
||||||
|
> **Status (2026-09, standalone no-paths guidance):** the standalone branch of the
|
||||||
|
> `other.noPaths` empty state now shows the real `settings.json` path plus an
|
||||||
|
> `other.noPaths.openSettingsFolder` button (each locale reuses its
|
||||||
|
> `settings.openSettingsFileLocation.label` rendering), and `descriptionStandalone` was
|
||||||
|
> reworded in `en.json` — from "none of the configured folders exist on disk" to "no
|
||||||
|
> other-model folders were found; add the folder keys you need to the `folder_paths`
|
||||||
|
> section" — and re-translated in all 9 locales. The `on disk` phrase now survives only in
|
||||||
|
> the ComfyUI variant (`descriptionComfyUI`).
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 1. Hard rules (do not violate)
|
## 1. Hard rules (do not violate)
|
||||||
|
|||||||
+3
-2
@@ -1241,11 +1241,12 @@
|
|||||||
},
|
},
|
||||||
"noPaths": {
|
"noPaths": {
|
||||||
"title": "Keine Ordner für weitere Modelle gefunden",
|
"title": "Keine Ordner für weitere Modelle gefunden",
|
||||||
"descriptionStandalone": "Die Verwaltung weiterer Modelle ist aktiviert, aber keiner der konfigurierten Modellordner existiert auf dem Datenträger. Fügen Sie die unten stehenden Ordnerpfade zu settings.json hinzu und starten Sie LoRA Manager neu.",
|
"descriptionStandalone": "Die Verwaltung weiterer Modelle ist aktiviert, aber es wurden keine Ordner für weitere Modelle gefunden. Fügen Sie die benötigten Ordnerschlüssel zum Abschnitt folder_paths Ihrer settings.json hinzu und starten Sie LoRA Manager neu.",
|
||||||
"hintStandalone": "Nur die oben aufgeführten Ordnerschlüssel werden gescannt; nicht benötigte Schlüssel können weggelassen werden.",
|
"hintStandalone": "Nur die oben aufgeführten Ordnerschlüssel werden gescannt; nicht benötigte Schlüssel können weggelassen werden.",
|
||||||
"descriptionComfyUI": "Die Verwaltung weiterer Modelle ist aktiviert, aber keiner der konfigurierten Modellordner existiert auf dem Datenträger. Fügen Sie die entsprechenden Modellordner zu Ihren ComfyUI-Modellpfaden hinzu und laden Sie diese Seite neu.",
|
"descriptionComfyUI": "Die Verwaltung weiterer Modelle ist aktiviert, aber keiner der konfigurierten Modellordner existiert auf dem Datenträger. Fügen Sie die entsprechenden Modellordner zu Ihren ComfyUI-Modellpfaden hinzu und laden Sie diese Seite neu.",
|
||||||
"hintComfyUI": "Weitere Modelle werden aus den Ordnern vae, upscale_models, text_encoders, clip_vision und controlnet von ComfyUI gelesen.",
|
"hintComfyUI": "Weitere Modelle werden aus den Ordnern vae, upscale_models, text_encoders, clip_vision und controlnet von ComfyUI gelesen.",
|
||||||
"openSettings": "Einstellungen öffnen"
|
"openSettings": "Einstellungen öffnen",
|
||||||
|
"openSettingsFolder": "Einstellungsordner öffnen"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"sidebar": {
|
"sidebar": {
|
||||||
|
|||||||
+3
-2
@@ -1241,11 +1241,12 @@
|
|||||||
},
|
},
|
||||||
"noPaths": {
|
"noPaths": {
|
||||||
"title": "No other-model folders found",
|
"title": "No other-model folders found",
|
||||||
"descriptionStandalone": "Other Models management is on, but none of the configured model folders exist on disk. Add the folder paths below to settings.json and restart LoRA Manager.",
|
"descriptionStandalone": "Other Models management is on, but no other-model folders were found. Add the folder keys you need to the folder_paths section of your settings.json, then restart LoRA Manager.",
|
||||||
"hintStandalone": "Only the folder keys listed above are scanned; keys you do not need can be omitted.",
|
"hintStandalone": "Only the folder keys listed above are scanned; keys you do not need can be omitted.",
|
||||||
"descriptionComfyUI": "Other Models management is on, but none of the configured model folders exist on disk. Add the matching model folders to your ComfyUI model paths, then reload this page.",
|
"descriptionComfyUI": "Other Models management is on, but none of the configured model folders exist on disk. Add the matching model folders to your ComfyUI model paths, then reload this page.",
|
||||||
"hintComfyUI": "Other models are read from ComfyUI's vae, upscale_models, text_encoders, clip_vision and controlnet folders.",
|
"hintComfyUI": "Other models are read from ComfyUI's vae, upscale_models, text_encoders, clip_vision and controlnet folders.",
|
||||||
"openSettings": "Open Settings"
|
"openSettings": "Open Settings",
|
||||||
|
"openSettingsFolder": "Open Settings Folder"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"sidebar": {
|
"sidebar": {
|
||||||
|
|||||||
+3
-2
@@ -1241,11 +1241,12 @@
|
|||||||
},
|
},
|
||||||
"noPaths": {
|
"noPaths": {
|
||||||
"title": "No se encontraron carpetas de otros modelos",
|
"title": "No se encontraron carpetas de otros modelos",
|
||||||
"descriptionStandalone": "La gestión de otros modelos está activada, pero ninguna de las carpetas de modelos configuradas existe en el disco. Añade las rutas de carpetas de abajo a settings.json y reinicia LoRA Manager.",
|
"descriptionStandalone": "La gestión de otros modelos está activada, pero no se encontraron carpetas de otros modelos. Añade las claves de carpeta que necesites a la sección folder_paths de tu settings.json y reinicia LoRA Manager.",
|
||||||
"hintStandalone": "Solo se escanean las claves de carpeta listadas arriba; las claves que no necesites puedes omitirlas.",
|
"hintStandalone": "Solo se escanean las claves de carpeta listadas arriba; las claves que no necesites puedes omitirlas.",
|
||||||
"descriptionComfyUI": "La gestión de otros modelos está activada, pero ninguna de las carpetas de modelos configuradas existe en el disco. Añade las carpetas de modelos correspondientes a tus rutas de modelos de ComfyUI y recarga esta página.",
|
"descriptionComfyUI": "La gestión de otros modelos está activada, pero ninguna de las carpetas de modelos configuradas existe en el disco. Añade las carpetas de modelos correspondientes a tus rutas de modelos de ComfyUI y recarga esta página.",
|
||||||
"hintComfyUI": "Los otros modelos se leen de las carpetas vae, upscale_models, text_encoders, clip_vision y controlnet de ComfyUI.",
|
"hintComfyUI": "Los otros modelos se leen de las carpetas vae, upscale_models, text_encoders, clip_vision y controlnet de ComfyUI.",
|
||||||
"openSettings": "Abrir configuración"
|
"openSettings": "Abrir configuración",
|
||||||
|
"openSettingsFolder": "Abrir carpeta de ajustes"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"sidebar": {
|
"sidebar": {
|
||||||
|
|||||||
+3
-2
@@ -1241,11 +1241,12 @@
|
|||||||
},
|
},
|
||||||
"noPaths": {
|
"noPaths": {
|
||||||
"title": "Aucun dossier d’autres modèles trouvé",
|
"title": "Aucun dossier d’autres modèles trouvé",
|
||||||
"descriptionStandalone": "La gestion des autres modèles est activée, mais aucun des dossiers de modèles configurés n’existe sur le disque. Ajoutez les chemins de dossiers ci-dessous à settings.json, puis redémarrez LoRA Manager.",
|
"descriptionStandalone": "La gestion des autres modèles est activée, mais aucun dossier d’autres modèles n’a été trouvé. Ajoutez les clés de dossiers dont vous avez besoin à la section folder_paths de votre settings.json, puis redémarrez LoRA Manager.",
|
||||||
"hintStandalone": "Seules les clés de dossiers listées ci-dessus sont analysées ; les clés inutiles peuvent être omises.",
|
"hintStandalone": "Seules les clés de dossiers listées ci-dessus sont analysées ; les clés inutiles peuvent être omises.",
|
||||||
"descriptionComfyUI": "La gestion des autres modèles est activée, mais aucun des dossiers de modèles configurés n’existe sur le disque. Ajoutez les dossiers de modèles correspondants à vos chemins de modèles ComfyUI, puis rechargez cette page.",
|
"descriptionComfyUI": "La gestion des autres modèles est activée, mais aucun des dossiers de modèles configurés n’existe sur le disque. Ajoutez les dossiers de modèles correspondants à vos chemins de modèles ComfyUI, puis rechargez cette page.",
|
||||||
"hintComfyUI": "Les autres modèles sont lus depuis les dossiers vae, upscale_models, text_encoders, clip_vision et controlnet de ComfyUI.",
|
"hintComfyUI": "Les autres modèles sont lus depuis les dossiers vae, upscale_models, text_encoders, clip_vision et controlnet de ComfyUI.",
|
||||||
"openSettings": "Ouvrir les paramètres"
|
"openSettings": "Ouvrir les paramètres",
|
||||||
|
"openSettingsFolder": "Ouvrir le dossier des paramètres"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"sidebar": {
|
"sidebar": {
|
||||||
|
|||||||
+3
-2
@@ -1241,11 +1241,12 @@
|
|||||||
},
|
},
|
||||||
"noPaths": {
|
"noPaths": {
|
||||||
"title": "לא נמצאו תיקיות של מודלים אחרים",
|
"title": "לא נמצאו תיקיות של מודלים אחרים",
|
||||||
"descriptionStandalone": "ניהול המודלים האחרים פועל, אך אף אחת מתיקיות המודלים המוגדרות אינה קיימת בדיסק. הוסף את נתיבי התיקיות שלמטה ל-settings.json והפעל מחדש את LoRA Manager.",
|
"descriptionStandalone": "ניהול המודלים האחרים פועל, אך לא נמצאו תיקיות של מודלים אחרים. הוסף את מפתחות התיקיות הדרושים למקטע folder_paths ב-settings.json והפעל מחדש את LoRA Manager.",
|
||||||
"hintStandalone": "רק מפתחות התיקיות המפורטים למעלה נסרקים; ניתן להשמיט מפתחות שאינך צריך.",
|
"hintStandalone": "רק מפתחות התיקיות המפורטים למעלה נסרקים; ניתן להשמיט מפתחות שאינך צריך.",
|
||||||
"descriptionComfyUI": "ניהול המודלים האחרים פועל, אך אף אחת מתיקיות המודלים המוגדרות אינה קיימת בדיסק. הוסף את תיקיות המודלים המתאימות לנתיבי המודלים של ComfyUI וטען מחדש עמוד זה.",
|
"descriptionComfyUI": "ניהול המודלים האחרים פועל, אך אף אחת מתיקיות המודלים המוגדרות אינה קיימת בדיסק. הוסף את תיקיות המודלים המתאימות לנתיבי המודלים של ComfyUI וטען מחדש עמוד זה.",
|
||||||
"hintComfyUI": "מודלים אחרים נקראים מתיקיות vae, upscale_models, text_encoders, clip_vision ו-controlnet של ComfyUI.",
|
"hintComfyUI": "מודלים אחרים נקראים מתיקיות vae, upscale_models, text_encoders, clip_vision ו-controlnet של ComfyUI.",
|
||||||
"openSettings": "פתח הגדרות"
|
"openSettings": "פתח הגדרות",
|
||||||
|
"openSettingsFolder": "פתח תיקיית הגדרות"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"sidebar": {
|
"sidebar": {
|
||||||
|
|||||||
+3
-2
@@ -1241,11 +1241,12 @@
|
|||||||
},
|
},
|
||||||
"noPaths": {
|
"noPaths": {
|
||||||
"title": "その他のモデルのフォルダーが見つかりません",
|
"title": "その他のモデルのフォルダーが見つかりません",
|
||||||
"descriptionStandalone": "その他のモデル管理はオンですが、設定されたモデルフォルダーがディスク上に存在しません。以下のフォルダーパスをsettings.jsonに追加し、LoRA Managerを再起動してください。",
|
"descriptionStandalone": "その他のモデル管理はオンですが、その他のモデルのフォルダーが見つかりません。必要なフォルダーキーをsettings.jsonのfolder_pathsセクションに追加し、LoRA Managerを再起動してください。",
|
||||||
"hintStandalone": "スキャンされるのは上記のフォルダーキーのみです。不要なキーは省略できます。",
|
"hintStandalone": "スキャンされるのは上記のフォルダーキーのみです。不要なキーは省略できます。",
|
||||||
"descriptionComfyUI": "その他のモデル管理はオンですが、設定されたモデルフォルダーがディスク上に存在しません。該当するモデルフォルダーをComfyUIのモデルパスに追加し、このページを再読み込みしてください。",
|
"descriptionComfyUI": "その他のモデル管理はオンですが、設定されたモデルフォルダーがディスク上に存在しません。該当するモデルフォルダーをComfyUIのモデルパスに追加し、このページを再読み込みしてください。",
|
||||||
"hintComfyUI": "その他のモデルは、ComfyUIのvae、upscale_models、text_encoders、clip_vision、controlnetフォルダーから読み込まれます。",
|
"hintComfyUI": "その他のモデルは、ComfyUIのvae、upscale_models、text_encoders、clip_vision、controlnetフォルダーから読み込まれます。",
|
||||||
"openSettings": "設定を開く"
|
"openSettings": "設定を開く",
|
||||||
|
"openSettingsFolder": "設定フォルダーを開く"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"sidebar": {
|
"sidebar": {
|
||||||
|
|||||||
+3
-2
@@ -1241,11 +1241,12 @@
|
|||||||
},
|
},
|
||||||
"noPaths": {
|
"noPaths": {
|
||||||
"title": "기타 모델 폴더를 찾을 수 없습니다",
|
"title": "기타 모델 폴더를 찾을 수 없습니다",
|
||||||
"descriptionStandalone": "기타 모델 관리가 켜져 있지만, 설정된 모델 폴더가 디스크에 존재하지 않습니다. 아래 폴더 경로를 settings.json에 추가한 뒤 LoRA Manager를 재시작하세요.",
|
"descriptionStandalone": "기타 모델 관리가 켜져 있지만, 기타 모델 폴더를 찾을 수 없습니다. 필요한 폴더 키를 settings.json의 folder_paths 섹션에 추가한 뒤 LoRA Manager를 재시작하세요.",
|
||||||
"hintStandalone": "위에 나열된 폴더 키만 스캔됩니다. 필요 없는 키는 생략할 수 있습니다.",
|
"hintStandalone": "위에 나열된 폴더 키만 스캔됩니다. 필요 없는 키는 생략할 수 있습니다.",
|
||||||
"descriptionComfyUI": "기타 모델 관리가 켜져 있지만, 설정된 모델 폴더가 디스크에 존재하지 않습니다. 해당 모델 폴더를 ComfyUI 모델 경로에 추가한 뒤 이 페이지를 새로 고침하세요.",
|
"descriptionComfyUI": "기타 모델 관리가 켜져 있지만, 설정된 모델 폴더가 디스크에 존재하지 않습니다. 해당 모델 폴더를 ComfyUI 모델 경로에 추가한 뒤 이 페이지를 새로 고침하세요.",
|
||||||
"hintComfyUI": "기타 모델은 ComfyUI의 vae, upscale_models, text_encoders, clip_vision, controlnet 폴더에서 읽어옵니다.",
|
"hintComfyUI": "기타 모델은 ComfyUI의 vae, upscale_models, text_encoders, clip_vision, controlnet 폴더에서 읽어옵니다.",
|
||||||
"openSettings": "설정 열기"
|
"openSettings": "설정 열기",
|
||||||
|
"openSettingsFolder": "설정 폴더 열기"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"sidebar": {
|
"sidebar": {
|
||||||
|
|||||||
+3
-2
@@ -1241,11 +1241,12 @@
|
|||||||
},
|
},
|
||||||
"noPaths": {
|
"noPaths": {
|
||||||
"title": "Папки других моделей не найдены",
|
"title": "Папки других моделей не найдены",
|
||||||
"descriptionStandalone": "Управление другими моделями включено, но ни одна из настроенных папок моделей не существует на диске. Добавьте указанные ниже пути к папкам в settings.json и перезапустите LoRA Manager.",
|
"descriptionStandalone": "Управление другими моделями включено, но папки других моделей не найдены. Добавьте нужные ключи папок в раздел folder_paths файла settings.json и перезапустите LoRA Manager.",
|
||||||
"hintStandalone": "Сканируются только перечисленные выше ключи папок; ненужные ключи можно опустить.",
|
"hintStandalone": "Сканируются только перечисленные выше ключи папок; ненужные ключи можно опустить.",
|
||||||
"descriptionComfyUI": "Управление другими моделями включено, но ни одна из настроенных папок моделей не существует на диске. Добавьте соответствующие папки моделей в пути к моделям ComfyUI и перезагрузите эту страницу.",
|
"descriptionComfyUI": "Управление другими моделями включено, но ни одна из настроенных папок моделей не существует на диске. Добавьте соответствующие папки моделей в пути к моделям ComfyUI и перезагрузите эту страницу.",
|
||||||
"hintComfyUI": "Другие модели читаются из папок vae, upscale_models, text_encoders, clip_vision и controlnet в ComfyUI.",
|
"hintComfyUI": "Другие модели читаются из папок vae, upscale_models, text_encoders, clip_vision и controlnet в ComfyUI.",
|
||||||
"openSettings": "Открыть настройки"
|
"openSettings": "Открыть настройки",
|
||||||
|
"openSettingsFolder": "Открыть папку настроек"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"sidebar": {
|
"sidebar": {
|
||||||
|
|||||||
+3
-2
@@ -1241,11 +1241,12 @@
|
|||||||
},
|
},
|
||||||
"noPaths": {
|
"noPaths": {
|
||||||
"title": "未找到其他模型文件夹",
|
"title": "未找到其他模型文件夹",
|
||||||
"descriptionStandalone": "其他模型管理已开启,但配置的模型文件夹在磁盘上都不存在。请将下面的文件夹路径添加到 settings.json,然后重启 LoRA Manager。",
|
"descriptionStandalone": "其他模型管理已开启,但未找到其他模型文件夹。请将你需要的文件夹键添加到 settings.json 的 folder_paths 部分,然后重启 LoRA Manager。",
|
||||||
"hintStandalone": "只会扫描上面列出的文件夹键;不需要的键可以省略。",
|
"hintStandalone": "只会扫描上面列出的文件夹键;不需要的键可以省略。",
|
||||||
"descriptionComfyUI": "其他模型管理已开启,但配置的模型文件夹在磁盘上都不存在。请将对应的模型文件夹添加到 ComfyUI 的模型路径,然后重新加载此页面。",
|
"descriptionComfyUI": "其他模型管理已开启,但配置的模型文件夹在磁盘上都不存在。请将对应的模型文件夹添加到 ComfyUI 的模型路径,然后重新加载此页面。",
|
||||||
"hintComfyUI": "其他模型从 ComfyUI 的 vae、upscale_models、text_encoders、clip_vision 和 controlnet 文件夹中读取。",
|
"hintComfyUI": "其他模型从 ComfyUI 的 vae、upscale_models、text_encoders、clip_vision 和 controlnet 文件夹中读取。",
|
||||||
"openSettings": "打开设置"
|
"openSettings": "打开设置",
|
||||||
|
"openSettingsFolder": "打开设置文件夹"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"sidebar": {
|
"sidebar": {
|
||||||
|
|||||||
+3
-2
@@ -1241,11 +1241,12 @@
|
|||||||
},
|
},
|
||||||
"noPaths": {
|
"noPaths": {
|
||||||
"title": "找不到其他模型資料夾",
|
"title": "找不到其他模型資料夾",
|
||||||
"descriptionStandalone": "其他模型管理已開啟,但設定的模型資料夾在磁碟上都不存在。請將下方的資料夾路徑加入 settings.json,然後重新啟動 LoRA Manager。",
|
"descriptionStandalone": "其他模型管理已開啟,但找不到其他模型資料夾。請將您需要的資料夾鍵加入 settings.json 的 folder_paths 區段,然後重新啟動 LoRA Manager。",
|
||||||
"hintStandalone": "只會掃描上方列出的資料夾鍵;不需要的鍵可以省略。",
|
"hintStandalone": "只會掃描上方列出的資料夾鍵;不需要的鍵可以省略。",
|
||||||
"descriptionComfyUI": "其他模型管理已開啟,但設定的模型資料夾在磁碟上都不存在。請將對應的模型資料夾加入 ComfyUI 的模型路徑,然後重新載入此頁面。",
|
"descriptionComfyUI": "其他模型管理已開啟,但設定的模型資料夾在磁碟上都不存在。請將對應的模型資料夾加入 ComfyUI 的模型路徑,然後重新載入此頁面。",
|
||||||
"hintComfyUI": "其他模型會從 ComfyUI 的 vae、upscale_models、text_encoders、clip_vision 和 controlnet 資料夾讀取。",
|
"hintComfyUI": "其他模型會從 ComfyUI 的 vae、upscale_models、text_encoders、clip_vision 和 controlnet 資料夾讀取。",
|
||||||
"openSettings": "開啟設定"
|
"openSettings": "開啟設定",
|
||||||
|
"openSettingsFolder": "開啟設定資料夾"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"sidebar": {
|
"sidebar": {
|
||||||
|
|||||||
@@ -421,6 +421,11 @@ def _wsl_to_windows_path(wsl_path: str) -> str | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _has_gui_display() -> bool:
|
||||||
|
"""Check whether a GUI session is reachable for xdg-open."""
|
||||||
|
return bool(os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"))
|
||||||
|
|
||||||
|
|
||||||
class PromptServerProtocol(Protocol):
|
class PromptServerProtocol(Protocol):
|
||||||
"""Subset of PromptServer used by the handlers."""
|
"""Subset of PromptServer used by the handlers."""
|
||||||
|
|
||||||
@@ -2759,12 +2764,40 @@ class ModelLibraryHandler:
|
|||||||
|
|
||||||
normalized_type, scanner = await self._get_scanner_for_type(model_type)
|
normalized_type, scanner = await self._get_scanner_for_type(model_type)
|
||||||
if not normalized_type:
|
if not normalized_type:
|
||||||
|
# The lookup cannot be served as a fully interactive list. Two
|
||||||
|
# cases share this branch: a CivitAI type with no scanner at all
|
||||||
|
# (Wildcards, Workflows, Hypernetwork, Poses, AestheticGradient)
|
||||||
|
# and an Other-model type while the opt-in master switch is off.
|
||||||
|
# Answer 200 with the CivitAI list marked read-only plus a
|
||||||
|
# machine-readable reason, so clients can still show the
|
||||||
|
# versions and explain why the actions are missing. Legacy
|
||||||
|
# clients keep working: they only read `success`/`versions`.
|
||||||
|
reason = (
|
||||||
|
"other_models_disabled"
|
||||||
|
if self._normalize_model_type(model_type) == "other"
|
||||||
|
else "model_type_unsupported"
|
||||||
|
)
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{
|
{
|
||||||
"success": False,
|
"success": True,
|
||||||
"error": f'Model type "{model_type}" is not supported',
|
"modelId": model_id,
|
||||||
},
|
"modelName": model_name,
|
||||||
status=400,
|
"modelType": model_type,
|
||||||
|
"supported": False,
|
||||||
|
"reason": reason,
|
||||||
|
"versions": [
|
||||||
|
{
|
||||||
|
"id": version.get("id"),
|
||||||
|
"name": version.get("name", ""),
|
||||||
|
"thumbnailUrl": version.get("images")[0]["url"]
|
||||||
|
if version.get("images")
|
||||||
|
else None,
|
||||||
|
"inLibrary": False,
|
||||||
|
"hasBeenDownloaded": False,
|
||||||
|
}
|
||||||
|
for version in versions
|
||||||
|
],
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
if not scanner:
|
if not scanner:
|
||||||
@@ -2806,6 +2839,7 @@ class ModelLibraryHandler:
|
|||||||
"modelId": model_id,
|
"modelId": model_id,
|
||||||
"modelName": model_name,
|
"modelName": model_name,
|
||||||
"modelType": model_type,
|
"modelType": model_type,
|
||||||
|
"supported": True,
|
||||||
"versions": enriched_versions,
|
"versions": enriched_versions,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
@@ -3393,6 +3427,18 @@ class FileSystemHandler:
|
|||||||
subprocess.Popen(["open", "-R", settings_file])
|
subprocess.Popen(["open", "-R", settings_file])
|
||||||
else:
|
else:
|
||||||
folder = os.path.dirname(settings_file)
|
folder = os.path.dirname(settings_file)
|
||||||
|
if not _has_gui_display():
|
||||||
|
# Headless/SSH session: xdg-open cannot open a file
|
||||||
|
# manager, so hand the path to the browser for copying
|
||||||
|
# instead of reporting a success that never happened.
|
||||||
|
return web.json_response(
|
||||||
|
{
|
||||||
|
"success": True,
|
||||||
|
"message": "Headless session: path available for copying",
|
||||||
|
"path": settings_file,
|
||||||
|
"mode": "clipboard",
|
||||||
|
}
|
||||||
|
)
|
||||||
subprocess.Popen(["xdg-open", folder])
|
subprocess.Popen(["xdg-open", folder])
|
||||||
|
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
|
|||||||
@@ -83,11 +83,16 @@ class OtherRoutes(BaseModelRoutes):
|
|||||||
# resolved to no existing folder. Render an actionable empty state
|
# resolved to no existing folder. Render an actionable empty state
|
||||||
# instead of an apparently broken empty grid.
|
# instead of an apparently broken empty grid.
|
||||||
standalone_mode = os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"
|
standalone_mode = os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"
|
||||||
return {
|
context = {
|
||||||
"other_disabled": False,
|
"other_disabled": False,
|
||||||
"other_no_paths": not bool(config.other_roots),
|
"other_no_paths": not bool(config.other_roots),
|
||||||
"standalone_mode": standalone_mode,
|
"standalone_mode": standalone_mode,
|
||||||
}
|
}
|
||||||
|
if standalone_mode:
|
||||||
|
# The settings UI cannot edit primary folder_paths, so the empty
|
||||||
|
# state must point at the actual file the user has to edit.
|
||||||
|
context["settings_file"] = getattr(self._settings, "settings_file", "") or ""
|
||||||
|
return context
|
||||||
|
|
||||||
def _get_expected_model_types(self) -> str:
|
def _get_expected_model_types(self) -> str:
|
||||||
"""Get expected model types string for error messages"""
|
"""Get expected model types string for error messages"""
|
||||||
|
|||||||
@@ -21,7 +21,20 @@ NETWORK_EXCEPTIONS = (ClientError, OSError, asyncio.TimeoutError)
|
|||||||
# otherwise delete them because they are untracked and, in released tags,
|
# otherwise delete them because they are untracked and, in released tags,
|
||||||
# not listed in ``.gitignore``. ``-e`` excludes a path from cleaning
|
# not listed in ``.gitignore``. ``-e`` excludes a path from cleaning
|
||||||
# regardless of whether it is ignored.
|
# regardless of whether it is ignored.
|
||||||
_PRESERVE_DIRS = ('settings.json', 'civitai', 'wildcards', 'backups', 'stats', 'logs', 'cache', 'model_cache')
|
# ``cache`` covers the resolved cache tree (cache/model, cache/recipe,
|
||||||
|
# cache/fts, ...); the legacy ``recipe_cache`` / ``model_cache`` directories
|
||||||
|
# are listed too because a portable install can predate the cache/ move.
|
||||||
|
_PRESERVE_DIRS = (
|
||||||
|
'settings.json',
|
||||||
|
'civitai',
|
||||||
|
'wildcards',
|
||||||
|
'backups',
|
||||||
|
'stats',
|
||||||
|
'logs',
|
||||||
|
'cache',
|
||||||
|
'model_cache',
|
||||||
|
'recipe_cache',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _clean_excludes() -> List[str]:
|
def _clean_excludes() -> List[str]:
|
||||||
|
|||||||
@@ -6,7 +6,9 @@ import threading
|
|||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple
|
from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple
|
||||||
|
|
||||||
|
from ..utils.cache_db import connect_cache_db
|
||||||
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
||||||
|
from ..utils.file_lock import exclusive_lock
|
||||||
from .model_sources import normalize_metadata_source
|
from .model_sources import normalize_metadata_source
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -257,6 +259,10 @@ class PersistentModelCache:
|
|||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
with self._db_lock:
|
with self._db_lock:
|
||||||
|
# Cross-process serialization: another LoRA Manager instance may
|
||||||
|
# share this settings directory, and the read-merge-write below
|
||||||
|
# spans several statements.
|
||||||
|
with exclusive_lock(self._db_path):
|
||||||
conn = self._connect()
|
conn = self._connect()
|
||||||
try:
|
try:
|
||||||
conn.execute("PRAGMA foreign_keys = ON")
|
conn.execute("PRAGMA foreign_keys = ON")
|
||||||
@@ -650,16 +656,14 @@ class PersistentModelCache:
|
|||||||
conn.execute(f"ALTER TABLE models ADD COLUMN {column} {definition}")
|
conn.execute(f"ALTER TABLE models ADD COLUMN {column} {definition}")
|
||||||
|
|
||||||
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
||||||
uri = False
|
if readonly and not os.path.exists(self._db_path):
|
||||||
path = self._db_path
|
raise FileNotFoundError(self._db_path)
|
||||||
if readonly:
|
return connect_cache_db(
|
||||||
if not os.path.exists(path):
|
self._db_path,
|
||||||
raise FileNotFoundError(path)
|
readonly=readonly,
|
||||||
path = f"file:{path}?mode=ro"
|
detect_types=sqlite3.PARSE_DECLTYPES,
|
||||||
uri = True
|
row_factory=sqlite3.Row,
|
||||||
conn = sqlite3.connect(path, check_same_thread=False, uri=uri, detect_types=sqlite3.PARSE_DECLTYPES)
|
)
|
||||||
conn.row_factory = sqlite3.Row
|
|
||||||
return conn
|
|
||||||
|
|
||||||
def _prepare_model_row(self, model_type: str, item: Dict[str, Any]) -> Tuple[Any, ...]:
|
def _prepare_model_row(self, model_type: str, item: Dict[str, Any]) -> Tuple[Any, ...]:
|
||||||
# Keep `source_*` and the legacy `hf_url` alias consistent no matter
|
# Keep `source_*` and the legacy `hf_url` alias consistent no matter
|
||||||
|
|||||||
@@ -19,7 +19,9 @@ import threading
|
|||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import Any, Dict, List, Optional, Set, Tuple
|
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||||
|
|
||||||
|
from ..utils.cache_db import connect_cache_db
|
||||||
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
||||||
|
from ..utils.file_lock import exclusive_lock
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -170,28 +172,59 @@ class PersistentRecipeCache:
|
|||||||
recipes: List[Dict[str, Any]],
|
recipes: List[Dict[str, Any]],
|
||||||
json_paths: Optional[Dict[str, str]] = None,
|
json_paths: Optional[Dict[str, str]] = None,
|
||||||
image_id_map: Optional[Dict[str, str]] = None,
|
image_id_map: Optional[Dict[str, str]] = None,
|
||||||
) -> None:
|
skip_if_empty: bool = False,
|
||||||
|
) -> bool:
|
||||||
"""Save all recipes to SQLite cache.
|
"""Save all recipes to SQLite cache.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
recipes: List of recipe dictionaries to persist.
|
recipes: List of recipe dictionaries to persist.
|
||||||
json_paths: Optional mapping of recipe_id -> json_path for file stats.
|
json_paths: Optional mapping of recipe_id -> json_path for file stats.
|
||||||
image_id_map: Optional precomputed civitai image_id → recipe_id mapping.
|
image_id_map: Optional precomputed civitai image_id → recipe_id mapping.
|
||||||
|
skip_if_empty: When True, refuse to replace a non-empty cache with an
|
||||||
|
empty one. This is the storage-level backstop against a scan that
|
||||||
|
silently loses every recipe (unavailable drive / mis-resolved
|
||||||
|
recipes directory): overwriting both deletes the user's data and
|
||||||
|
destroys their only record of it. Intentional full clears (manual
|
||||||
|
rebuild) must pass ``skip_if_empty=False``.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
``True`` when the write happened, ``False`` when it was skipped.
|
||||||
"""
|
"""
|
||||||
if not self.is_enabled():
|
if not self.is_enabled():
|
||||||
return
|
return False
|
||||||
if not self._schema_initialized:
|
if not self._schema_initialized:
|
||||||
self._initialize_schema()
|
self._initialize_schema()
|
||||||
if not self._schema_initialized:
|
if not self._schema_initialized:
|
||||||
return
|
return False
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with self._db_lock:
|
with self._db_lock:
|
||||||
|
# Cross-process serialization: another LoRA Manager instance may
|
||||||
|
# share this settings directory, and a full-table replace is a
|
||||||
|
# read-modify-write that SQLite alone cannot make atomic.
|
||||||
|
with exclusive_lock(self._db_path):
|
||||||
conn = self._connect()
|
conn = self._connect()
|
||||||
try:
|
try:
|
||||||
conn.execute("PRAGMA foreign_keys = ON")
|
conn.execute("PRAGMA foreign_keys = ON")
|
||||||
conn.execute("BEGIN")
|
conn.execute("BEGIN")
|
||||||
|
|
||||||
|
if skip_if_empty and not recipes:
|
||||||
|
existing = conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM recipes"
|
||||||
|
).fetchone()
|
||||||
|
if existing and existing[0]:
|
||||||
|
conn.rollback()
|
||||||
|
logger.warning(
|
||||||
|
"Refusing to persist an empty recipe cache: the "
|
||||||
|
"stored cache still holds %d recipe(s). The scan "
|
||||||
|
"found nothing, which usually means the recipes "
|
||||||
|
"path was unavailable or resolved elsewhere; "
|
||||||
|
"keeping the stored cache so the data stays "
|
||||||
|
"recoverable.",
|
||||||
|
existing[0],
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
# Clear existing data
|
# Clear existing data
|
||||||
conn.execute("DELETE FROM recipes")
|
conn.execute("DELETE FROM recipes")
|
||||||
|
|
||||||
@@ -225,10 +258,12 @@ class PersistentRecipeCache:
|
|||||||
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
logger.debug("Persisted %d recipes to cache", len(recipe_rows))
|
logger.debug("Persisted %d recipes to cache", len(recipe_rows))
|
||||||
|
return True
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.warning("Failed to persist recipe cache: %s", exc)
|
logger.warning("Failed to persist recipe cache: %s", exc)
|
||||||
|
return False
|
||||||
|
|
||||||
def get_file_stats(self) -> Dict[str, Tuple[float, int]]:
|
def get_file_stats(self) -> Dict[str, Tuple[float, int]]:
|
||||||
"""Return stored file stats for all cached recipes.
|
"""Return stored file stats for all cached recipes.
|
||||||
@@ -486,16 +521,14 @@ class PersistentRecipeCache:
|
|||||||
logger.warning("Failed to initialize persistent recipe cache schema: %s", exc)
|
logger.warning("Failed to initialize persistent recipe cache schema: %s", exc)
|
||||||
|
|
||||||
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
||||||
uri = False
|
if readonly and not os.path.exists(self._db_path):
|
||||||
path = self._db_path
|
raise FileNotFoundError(self._db_path)
|
||||||
if readonly:
|
return connect_cache_db(
|
||||||
if not os.path.exists(path):
|
self._db_path,
|
||||||
raise FileNotFoundError(path)
|
readonly=readonly,
|
||||||
path = f"file:{path}?mode=ro"
|
detect_types=sqlite3.PARSE_DECLTYPES,
|
||||||
uri = True
|
row_factory=sqlite3.Row,
|
||||||
conn = sqlite3.connect(path, check_same_thread=False, uri=uri, detect_types=sqlite3.PARSE_DECLTYPES)
|
)
|
||||||
conn.row_factory = sqlite3.Row
|
|
||||||
return conn
|
|
||||||
|
|
||||||
def _prepare_recipe_row(self, recipe: Dict[str, Any], json_path: str) -> Tuple[Any, ...]:
|
def _prepare_recipe_row(self, recipe: Dict[str, Any], json_path: str) -> Tuple[Any, ...]:
|
||||||
"""Convert a recipe dict to a row tuple for SQLite insertion."""
|
"""Convert a recipe dict to a row tuple for SQLite insertion."""
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import threading
|
|||||||
import time
|
import time
|
||||||
from typing import Any, Dict, List, Optional, Set, Tuple
|
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||||
|
|
||||||
|
from ..utils.cache_db import connect_cache_db
|
||||||
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -633,16 +634,13 @@ class RecipeFTSIndex:
|
|||||||
|
|
||||||
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
||||||
"""Create a database connection."""
|
"""Create a database connection."""
|
||||||
uri = False
|
if readonly and not os.path.exists(self._db_path):
|
||||||
path = self._db_path
|
raise FileNotFoundError(self._db_path)
|
||||||
if readonly:
|
return connect_cache_db(
|
||||||
if not os.path.exists(path):
|
self._db_path,
|
||||||
raise FileNotFoundError(path)
|
readonly=readonly,
|
||||||
path = f"file:{path}?mode=ro"
|
row_factory=sqlite3.Row,
|
||||||
uri = True
|
)
|
||||||
conn = sqlite3.connect(path, check_same_thread=False, uri=uri)
|
|
||||||
conn.row_factory = sqlite3.Row
|
|
||||||
return conn
|
|
||||||
|
|
||||||
def _remove_recipe_locked(self, conn: sqlite3.Connection, recipe_id: str) -> None:
|
def _remove_recipe_locked(self, conn: sqlite3.Connection, recipe_id: str) -> None:
|
||||||
"""Remove a recipe entry. Caller must hold the lock."""
|
"""Remove a recipe entry. Caller must hold the lock."""
|
||||||
|
|||||||
+120
-12
@@ -116,6 +116,12 @@ class RecipeScanner:
|
|||||||
self._persistent_cache: Optional[PersistentRecipeCache] = None
|
self._persistent_cache: Optional[PersistentRecipeCache] = None
|
||||||
self._civitai_client: Any = None # Lazily initialized from registry
|
self._civitai_client: Any = None # Lazily initialized from registry
|
||||||
self._json_path_map: Dict[str, str] = {} # recipe_id -> json_path
|
self._json_path_map: Dict[str, str] = {} # recipe_id -> json_path
|
||||||
|
# True when the last scan refused to prune the stored cache because
|
||||||
|
# every recorded recipe file was missing (see
|
||||||
|
# :meth:`_initialize_recipe_cache_sync`). Keeps dependent background
|
||||||
|
# work (FTS index) aligned with the stored rows instead of the
|
||||||
|
# intentionally out-of-sync in-memory view.
|
||||||
|
self._prune_skipped: bool = False
|
||||||
if lora_scanner:
|
if lora_scanner:
|
||||||
self._lora_scanner = lora_scanner
|
self._lora_scanner = lora_scanner
|
||||||
if checkpoint_scanner:
|
if checkpoint_scanner:
|
||||||
@@ -1651,7 +1657,11 @@ class RecipeScanner:
|
|||||||
'pageType': 'recipes',
|
'pageType': 'recipes',
|
||||||
})
|
})
|
||||||
self._schedule_post_scan_enrichment()
|
self._schedule_post_scan_enrichment()
|
||||||
# Schedule FTS index build in background (non-blocking)
|
# Schedule FTS index build in background (non-blocking). When the
|
||||||
|
# prune was skipped the in-memory cache is intentionally out of sync
|
||||||
|
# with the stored rows, so leave the existing index alone instead of
|
||||||
|
# rebuilding it from the empty view.
|
||||||
|
if not self._prune_skipped:
|
||||||
self._schedule_fts_index_build()
|
self._schedule_fts_index_build()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Recipe Scanner: Error initializing cache in background: {e}")
|
logger.error(f"Recipe Scanner: Error initializing cache in background: {e}")
|
||||||
@@ -1723,6 +1733,7 @@ class RecipeScanner:
|
|||||||
"""
|
"""
|
||||||
loop = None
|
loop = None
|
||||||
scan_start_time: Optional[float] = None
|
scan_start_time: Optional[float] = None
|
||||||
|
self._prune_skipped = False
|
||||||
try:
|
try:
|
||||||
# Ensure cache exists to avoid None reference errors
|
# Ensure cache exists to avoid None reference errors
|
||||||
if self._cache is None:
|
if self._cache is None:
|
||||||
@@ -1749,14 +1760,38 @@ class RecipeScanner:
|
|||||||
logger.warning(f"Recipes directory not found: {recipes_dir}")
|
logger.warning(f"Recipes directory not found: {recipes_dir}")
|
||||||
return self._cache
|
return self._cache
|
||||||
|
|
||||||
|
# Record which directory the scan actually used. When the Recipes
|
||||||
|
# Storage Path is empty this falls back to the first LoRA root, and
|
||||||
|
# a support reader needs that path to tell a real wipe apart from a
|
||||||
|
# scan that looked somewhere else (see the prune guard below).
|
||||||
|
logger.info(f"Recipe scan directory: {recipes_dir}")
|
||||||
|
|
||||||
# Try to load from persistent cache first
|
# Try to load from persistent cache first
|
||||||
persisted = self._persistent_cache.load_cache()
|
persisted = self._persistent_cache.load_cache()
|
||||||
if persisted:
|
if persisted:
|
||||||
recipes, changed, json_paths = self._reconcile_recipe_cache(
|
(
|
||||||
persisted, recipes_dir
|
recipes,
|
||||||
)
|
changed,
|
||||||
|
json_paths,
|
||||||
|
skipped_prune_reason,
|
||||||
|
) = self._reconcile_recipe_cache(persisted, recipes_dir)
|
||||||
self._json_path_map = json_paths
|
self._json_path_map = json_paths
|
||||||
|
|
||||||
|
if skipped_prune_reason:
|
||||||
|
# Every persisted recipe file vanished at once. That is not a
|
||||||
|
# reliable deletion signal: a drive that did not mount, a
|
||||||
|
# recipes_path that silently fell back to another root, or a
|
||||||
|
# shared cache touched by a second instance all look exactly
|
||||||
|
# like this. Keep the stored cache and skip the prune, so the
|
||||||
|
# only copy of the user's recipes is not destroyed.
|
||||||
|
logger.warning(
|
||||||
|
f"Recipe cache prune skipped: {skipped_prune_reason}. "
|
||||||
|
f"Keeping {len(persisted.raw_data)} stored recipe(s); this "
|
||||||
|
"session reports no recipes until the files are found again."
|
||||||
|
)
|
||||||
|
self._prune_skipped = True
|
||||||
|
return self._cache
|
||||||
|
|
||||||
if not changed:
|
if not changed:
|
||||||
# Fast path: use cached data directly
|
# Fast path: use cached data directly
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -1770,7 +1805,10 @@ class RecipeScanner:
|
|||||||
if self._backfill_source_path_if_needed(recipes, json_paths):
|
if self._backfill_source_path_if_needed(recipes, json_paths):
|
||||||
self._cache.image_id_map = self._build_image_id_map()
|
self._cache.image_id_map = self._build_image_id_map()
|
||||||
self._persistent_cache.save_cache(
|
self._persistent_cache.save_cache(
|
||||||
recipes, json_paths, self._cache.image_id_map
|
recipes,
|
||||||
|
json_paths,
|
||||||
|
self._cache.image_id_map,
|
||||||
|
skip_if_empty=True,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
# Use persisted map, or rebuild if empty (e.g. first startup
|
# Use persisted map, or rebuild if empty (e.g. first startup
|
||||||
@@ -1798,7 +1836,10 @@ class RecipeScanner:
|
|||||||
self._cache.image_id_map = self._build_image_id_map()
|
self._cache.image_id_map = self._build_image_id_map()
|
||||||
# Persist updated cache
|
# Persist updated cache
|
||||||
self._persistent_cache.save_cache(
|
self._persistent_cache.save_cache(
|
||||||
recipes, json_paths, self._cache.image_id_map
|
recipes,
|
||||||
|
json_paths,
|
||||||
|
self._cache.image_id_map,
|
||||||
|
skip_if_empty=True,
|
||||||
)
|
)
|
||||||
return self._cache
|
return self._cache
|
||||||
|
|
||||||
@@ -1825,7 +1866,10 @@ class RecipeScanner:
|
|||||||
|
|
||||||
# Persist for next startup
|
# Persist for next startup
|
||||||
self._persistent_cache.save_cache(
|
self._persistent_cache.save_cache(
|
||||||
recipes, json_paths, self._cache.image_id_map
|
recipes,
|
||||||
|
json_paths,
|
||||||
|
self._cache.image_id_map,
|
||||||
|
skip_if_empty=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
if report_progress:
|
if report_progress:
|
||||||
@@ -1862,7 +1906,7 @@ class RecipeScanner:
|
|||||||
self,
|
self,
|
||||||
persisted: PersistedRecipeData,
|
persisted: PersistedRecipeData,
|
||||||
recipes_dir: str,
|
recipes_dir: str,
|
||||||
) -> Tuple[List[Dict[str, Any]], bool, Dict[str, str]]:
|
) -> Tuple[List[Dict[str, Any]], bool, Dict[str, str], Optional[str]]:
|
||||||
"""Reconcile persisted cache with current filesystem state.
|
"""Reconcile persisted cache with current filesystem state.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -1870,7 +1914,11 @@ class RecipeScanner:
|
|||||||
recipes_dir: Path to the recipes directory.
|
recipes_dir: Path to the recipes directory.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (recipes list, changed flag, json_paths dict).
|
Tuple of (recipes list, changed flag, json_paths dict,
|
||||||
|
skipped_prune_reason). The last element is ``None`` on a normal
|
||||||
|
reconcile. When it is a string, the scan saw every persisted recipe
|
||||||
|
file disappear at once; the caller must then keep the persisted
|
||||||
|
cache instead of overwriting it. The reason text is user-facing.
|
||||||
"""
|
"""
|
||||||
recipes: List[Dict[str, Any]] = []
|
recipes: List[Dict[str, Any]] = []
|
||||||
json_paths: Dict[str, str] = {}
|
json_paths: Dict[str, str] = {}
|
||||||
@@ -1951,12 +1999,67 @@ class RecipeScanner:
|
|||||||
time.sleep(0)
|
time.sleep(0)
|
||||||
|
|
||||||
# Check for deleted files
|
# Check for deleted files
|
||||||
for json_path in persisted.file_stats.keys():
|
orphaned_stats = [
|
||||||
if json_path not in current_files:
|
json_path
|
||||||
|
for json_path in persisted.file_stats.keys()
|
||||||
|
if json_path not in current_files
|
||||||
|
]
|
||||||
|
if orphaned_stats:
|
||||||
changed = True
|
changed = True
|
||||||
|
# This single line plus the resolved scan directory logged by the
|
||||||
|
# caller are the evidence a support reader gets for a recipes path
|
||||||
|
# that moved; the per-file lines stay at debug to avoid flooding.
|
||||||
|
if len(orphaned_stats) > 10:
|
||||||
|
logger.info(
|
||||||
|
f"Recipe reconcile: {len(orphaned_stats)} of "
|
||||||
|
f"{len(persisted.file_stats)} cached recipe file(s) are not in "
|
||||||
|
f"{recipes_dir} (first: {orphaned_stats[0]}, "
|
||||||
|
f"last: {orphaned_stats[-1]})"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
for json_path in orphaned_stats:
|
||||||
logger.debug("Recipe file deleted: %s", json_path)
|
logger.debug("Recipe file deleted: %s", json_path)
|
||||||
|
|
||||||
return recipes, changed, json_paths
|
skipped_prune_reason: Optional[str] = None
|
||||||
|
if not current_files and persisted.file_stats:
|
||||||
|
metadata_is_coherent = self._persisted_metadata_is_coherent(persisted)
|
||||||
|
if metadata_is_coherent:
|
||||||
|
skipped_prune_reason = (
|
||||||
|
f"every recipe file recorded in the cache "
|
||||||
|
f"({len(persisted.file_stats)}) is missing from {recipes_dir}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# The stored row set and its recorded file stats disagree, so
|
||||||
|
# this cache is stale rather than a faithful record of recipes
|
||||||
|
# that have just gone missing. Pruning it is safe.
|
||||||
|
logger.info(
|
||||||
|
f"Recipe reconcile: stored cache is inconsistent "
|
||||||
|
f"({len(persisted.raw_data)} row(s) vs "
|
||||||
|
f"{len(persisted.file_stats)} file record(s)); falling back "
|
||||||
|
"to a normal prune."
|
||||||
|
)
|
||||||
|
|
||||||
|
return recipes, changed, json_paths, skipped_prune_reason
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _persisted_metadata_is_coherent(persisted: PersistedRecipeData) -> bool:
|
||||||
|
"""Return True when the stored rows and their file stats describe one set.
|
||||||
|
|
||||||
|
The prune guard treats "no recipe files found" as a signal that the
|
||||||
|
directory moved out from under us, which is only meaningful when the
|
||||||
|
stored cache is a faithful record of recipes that exist on disk. A cache
|
||||||
|
whose row set and file-stat set have diverged (left behind by an older
|
||||||
|
reconcile) carries recipes that were already orphaned, so it is not
|
||||||
|
evidence of a fresh disappearance.
|
||||||
|
"""
|
||||||
|
stats_ids = {
|
||||||
|
os.path.basename(json_path)[: -len(".recipe.json")]
|
||||||
|
for json_path in persisted.file_stats
|
||||||
|
if os.path.basename(json_path).lower().endswith(".recipe.json")
|
||||||
|
}
|
||||||
|
rows_ids = {str(recipe.get("id", "")) for recipe in persisted.raw_data}
|
||||||
|
rows_ids.discard("")
|
||||||
|
return bool(rows_ids) and rows_ids == stats_ids
|
||||||
|
|
||||||
# Metadata key recording that the one-shot source_path backfill has run.
|
# Metadata key recording that the one-shot source_path backfill has run.
|
||||||
_SOURCE_PATH_BACKFILL_MARKER = "source_path_backfilled"
|
_SOURCE_PATH_BACKFILL_MARKER = "source_path_backfilled"
|
||||||
@@ -2626,6 +2729,10 @@ class RecipeScanner:
|
|||||||
try:
|
try:
|
||||||
# Invalidate persistent cache so the sync path does a
|
# Invalidate persistent cache so the sync path does a
|
||||||
# full directory scan instead of reconciling stale data.
|
# full directory scan instead of reconciling stale data.
|
||||||
|
# This is the deliberate escape hatch from the
|
||||||
|
# all-missing prune guard: an explicit user rebuild is
|
||||||
|
# allowed to clear the stored cache, while an implicit
|
||||||
|
# startup scan is not.
|
||||||
if self._persistent_cache:
|
if self._persistent_cache:
|
||||||
self._persistent_cache.save_cache([], {})
|
self._persistent_cache.save_cache([], {})
|
||||||
self._json_path_map = {}
|
self._json_path_map = {}
|
||||||
@@ -2656,6 +2763,7 @@ class RecipeScanner:
|
|||||||
|
|
||||||
# Schedule non-blocking background work
|
# Schedule non-blocking background work
|
||||||
self._schedule_post_scan_enrichment()
|
self._schedule_post_scan_enrichment()
|
||||||
|
if not self._prune_skipped:
|
||||||
self._schedule_fts_index_build()
|
self._schedule_fts_index_build()
|
||||||
|
|
||||||
return cast(RecipeCache, self._cache)
|
return cast(RecipeCache, self._cache)
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ from ..utils.constants import (
|
|||||||
from ..utils.preview_selection import VALID_MATURE_BLUR_LEVELS
|
from ..utils.preview_selection import VALID_MATURE_BLUR_LEVELS
|
||||||
from ..utils.settings_paths import (
|
from ..utils.settings_paths import (
|
||||||
APP_NAME,
|
APP_NAME,
|
||||||
|
_portable_env_override,
|
||||||
ensure_settings_file,
|
ensure_settings_file,
|
||||||
get_legacy_settings_path,
|
get_legacy_settings_path,
|
||||||
get_settings_dir_override,
|
get_settings_dir_override,
|
||||||
@@ -172,13 +173,23 @@ class SettingsManager:
|
|||||||
self._check_environment_variables()
|
self._check_environment_variables()
|
||||||
self._collect_configuration_warnings()
|
self._collect_configuration_warnings()
|
||||||
|
|
||||||
if (
|
portable_override = _portable_env_override()
|
||||||
os.environ.get("LORA_MANAGER_PORTABLE", "0") == "1"
|
if portable_override is True and not is_settings_dir_pinned():
|
||||||
and not is_settings_dir_pinned()
|
|
||||||
):
|
|
||||||
if not self.settings.get("use_portable_settings"):
|
if not self.settings.get("use_portable_settings"):
|
||||||
self.settings["use_portable_settings"] = True
|
self.settings["use_portable_settings"] = True
|
||||||
self._save_settings()
|
self._save_settings()
|
||||||
|
elif portable_override is False and self.settings.get(
|
||||||
|
"use_portable_settings"
|
||||||
|
):
|
||||||
|
# Explicit opt-out from a persisted portable mode: clear the flag so
|
||||||
|
# later runs go back to the shared settings directory instead of
|
||||||
|
# requiring a manual edit of settings.json.
|
||||||
|
logger.info(
|
||||||
|
"Clearing the persisted portable-mode flag because %s=0",
|
||||||
|
"LORA_MANAGER_PORTABLE",
|
||||||
|
)
|
||||||
|
self.settings["use_portable_settings"] = False
|
||||||
|
self._save_settings()
|
||||||
|
|
||||||
if self._needs_initial_save:
|
if self._needs_initial_save:
|
||||||
self._save_settings()
|
self._save_settings()
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ import time
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, List, Optional, Set
|
from typing import Any, Dict, List, Optional, Set
|
||||||
|
|
||||||
|
from ..utils.cache_db import connect_cache_db
|
||||||
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -677,16 +678,13 @@ class TagFTSIndex:
|
|||||||
|
|
||||||
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
||||||
"""Create a database connection."""
|
"""Create a database connection."""
|
||||||
uri = False
|
if readonly and not os.path.exists(self._db_path):
|
||||||
path = self._db_path
|
raise FileNotFoundError(self._db_path)
|
||||||
if readonly:
|
return connect_cache_db(
|
||||||
if not os.path.exists(path):
|
self._db_path,
|
||||||
raise FileNotFoundError(path)
|
readonly=readonly,
|
||||||
path = f"file:{path}?mode=ro"
|
row_factory=sqlite3.Row,
|
||||||
uri = True
|
)
|
||||||
conn = sqlite3.connect(path, check_same_thread=False, uri=uri)
|
|
||||||
conn.row_factory = sqlite3.Row
|
|
||||||
return conn
|
|
||||||
|
|
||||||
def _build_fts_query(self, query: str) -> str:
|
def _build_fts_query(self, query: str) -> str:
|
||||||
"""Build an FTS5 query string with prefix matching.
|
"""Build an FTS5 query string with prefix matching.
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
"""Shared SQLite connection setup for LoRA Manager cache databases.
|
||||||
|
|
||||||
|
Cache databases live under the settings directory (``cache/model/<library>.sqlite``,
|
||||||
|
``cache/recipe/<library>.sqlite``, ``cache/fts/*.sqlite``). With portable mode or a
|
||||||
|
pinned ``LORA_MANAGER_SETTINGS_DIR`` off, that directory is shared by every ComfyUI
|
||||||
|
instance on the machine, so two processes can open the same cache file at once.
|
||||||
|
|
||||||
|
SQLite serializes writers, but the default ``timeout`` is 5 seconds: a second
|
||||||
|
instance that writes while the first is mid-transaction fails with "database is
|
||||||
|
locked". These settings make concurrent access wait instead of failing, and keep
|
||||||
|
the write path in WAL so readers are never blocked by a writer.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlite3
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
# How long a connection waits for a competing writer before raising.
|
||||||
|
CONCURRENT_TIMEOUT_SECONDS = 30.0
|
||||||
|
|
||||||
|
# PRAGMAs applied to every cache connection.
|
||||||
|
#
|
||||||
|
# ``busy_timeout`` mirrors the connection timeout so a busy database is retried
|
||||||
|
# inside SQLite rather than surfacing as an immediate error. ``synchronous=NORMAL``
|
||||||
|
# is the documented companion of WAL: still crash-safe, far fewer fsyncs.
|
||||||
|
_TUNING_PRAGMAS = (
|
||||||
|
"PRAGMA busy_timeout = 30000",
|
||||||
|
"PRAGMA synchronous = NORMAL",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def connect_cache_db(
|
||||||
|
path: str,
|
||||||
|
*,
|
||||||
|
readonly: bool = False,
|
||||||
|
uri: bool = False,
|
||||||
|
detect_types: int = 0,
|
||||||
|
row_factory: Any = None,
|
||||||
|
) -> sqlite3.Connection:
|
||||||
|
"""Open a cache database with multi-instance-friendly settings.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
path: Database path, or a ``file:`` URI when *uri* is True.
|
||||||
|
readonly: Open through a read-only URI. Callers still pass the
|
||||||
|
plain path; the ``mode=ro`` suffix is added here. The
|
||||||
|
write-oriented tuning pragmas are skipped in that case so a
|
||||||
|
read-only connection never attempts to change the file.
|
||||||
|
uri: Treat *path* as a SQLite URI.
|
||||||
|
detect_types: Forwarded to :func:`sqlite3.connect`.
|
||||||
|
row_factory: Optional ``row_factory`` for the connection.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
A configured :class:`sqlite3.Connection`.
|
||||||
|
"""
|
||||||
|
if readonly:
|
||||||
|
if not uri and not path.startswith("file:"):
|
||||||
|
path = f"file:{path}?mode=ro"
|
||||||
|
uri = True
|
||||||
|
|
||||||
|
conn = sqlite3.connect(
|
||||||
|
path,
|
||||||
|
check_same_thread=False,
|
||||||
|
uri=uri,
|
||||||
|
detect_types=detect_types,
|
||||||
|
timeout=CONCURRENT_TIMEOUT_SECONDS,
|
||||||
|
)
|
||||||
|
if row_factory is not None:
|
||||||
|
conn.row_factory = row_factory
|
||||||
|
|
||||||
|
try:
|
||||||
|
for pragma in _TUNING_PRAGMAS:
|
||||||
|
# A read-only connection may reject write PRAGMAs; they are not
|
||||||
|
# needed there anyway.
|
||||||
|
conn.execute(pragma)
|
||||||
|
except sqlite3.Error:
|
||||||
|
# Tuning is best-effort: a connection that cannot set pragmas still
|
||||||
|
# works, just without the concurrency headroom.
|
||||||
|
pass
|
||||||
|
|
||||||
|
return conn
|
||||||
@@ -2,7 +2,7 @@ import inspect
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
from typing import TYPE_CHECKING, Any, Dict, Optional
|
from typing import TYPE_CHECKING, Any, Dict, Mapping, MutableMapping, Optional
|
||||||
|
|
||||||
from ..recipes.constants import GEN_PARAM_KEYS
|
from ..recipes.constants import GEN_PARAM_KEYS
|
||||||
from ..services.metadata_service import get_default_metadata_provider, get_metadata_provider
|
from ..services.metadata_service import get_default_metadata_provider, get_metadata_provider
|
||||||
@@ -13,9 +13,20 @@ from ..services.downloader import get_downloader
|
|||||||
from ..utils.constants import SUPPORTED_MEDIA_EXTENSIONS
|
from ..utils.constants import SUPPORTED_MEDIA_EXTENSIONS
|
||||||
from ..utils.exif_utils import ExifUtils
|
from ..utils.exif_utils import ExifUtils
|
||||||
from ..utils.metadata_manager import MetadataManager
|
from ..utils.metadata_manager import MetadataManager
|
||||||
|
from ..utils.video_metadata import get_video_dimensions
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Placeholder dimensions written when the real ones cannot be determined.
|
||||||
|
# Kept for backwards compatibility with pre-existing metadata entries.
|
||||||
|
_DEFAULT_MEDIA_WIDTH = 720
|
||||||
|
_DEFAULT_MEDIA_HEIGHT = 1280
|
||||||
|
|
||||||
|
# Example metadata entries carry a marker: ``customImages`` use their ``id``
|
||||||
|
# while ``images`` use the positional index. Either way the marker must be a
|
||||||
|
# plain filename-safe token, never a path fragment.
|
||||||
|
_ENTRY_MARKER_PATTERN = re.compile(r"^(?:custom_|image_)?([^./\\]+)$")
|
||||||
|
|
||||||
_preview_service = PreviewAssetService(
|
_preview_service = PreviewAssetService(
|
||||||
metadata_manager=MetadataManager,
|
metadata_manager=MetadataManager,
|
||||||
downloader_factory=get_downloader,
|
downloader_factory=get_downloader,
|
||||||
@@ -66,6 +77,141 @@ def _build_metadata_sync_service(settings_manager: "SettingsManager") -> Metadat
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _read_media_dimensions(path: str, is_video: bool) -> tuple[int, int]:
|
||||||
|
"""Return ``(width, height)`` for an example image or video file.
|
||||||
|
|
||||||
|
Videos are read from their container headers (PIL cannot open them) so the
|
||||||
|
showcase viewer sizes the gallery to the real aspect ratio. Falls back to
|
||||||
|
the legacy ``720x1280`` placeholder when the dimensions cannot be
|
||||||
|
determined — e.g. an unreadable file or an exotic codec — which only
|
||||||
|
affects the displayed aspect ratio, never the file itself.
|
||||||
|
"""
|
||||||
|
|
||||||
|
dimensions = None
|
||||||
|
|
||||||
|
if is_video:
|
||||||
|
dimensions = get_video_dimensions(path)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
if os.path.exists(path):
|
||||||
|
with Image.open(path) as img:
|
||||||
|
dimensions = img.size
|
||||||
|
except Exception:
|
||||||
|
dimensions = None
|
||||||
|
|
||||||
|
if dimensions:
|
||||||
|
width, height = dimensions
|
||||||
|
if width > 0 and height > 0:
|
||||||
|
return int(width), int(height)
|
||||||
|
|
||||||
|
return _DEFAULT_MEDIA_WIDTH, _DEFAULT_MEDIA_HEIGHT
|
||||||
|
|
||||||
|
|
||||||
|
def _is_video_entry(file_path: Optional[str], entry: Mapping[str, Any]) -> bool:
|
||||||
|
"""Return True when an example entry points at a video file.
|
||||||
|
|
||||||
|
The local file extension wins over the recorded ``type`` because files in
|
||||||
|
the wild are frequently mislabelled (animated WebP saved as ``.mp4``);
|
||||||
|
``_read_media_dimensions`` handles that correctly either way.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if file_path:
|
||||||
|
ext = os.path.splitext(file_path)[1].lower()
|
||||||
|
if ext in SUPPORTED_MEDIA_EXTENSIONS["videos"]:
|
||||||
|
return True
|
||||||
|
if ext in SUPPORTED_MEDIA_EXTENSIONS["images"]:
|
||||||
|
return False
|
||||||
|
return str(entry.get("type", "")).lower() == "video"
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_local_file(
|
||||||
|
entry: Mapping[str, Any],
|
||||||
|
index: int,
|
||||||
|
local_files: Mapping[str, str],
|
||||||
|
) -> Optional[str]:
|
||||||
|
"""Map a metadata entry onto its example file inside the model folder.
|
||||||
|
|
||||||
|
Reads the entry's own marker (``id`` for ``customImages``, positional
|
||||||
|
``index`` for ``images``) with an anchored regex, so the identifier can
|
||||||
|
never bleed into a neighbouring filename the way a prefix comparison can.
|
||||||
|
"""
|
||||||
|
|
||||||
|
marker = entry.get("id")
|
||||||
|
if not isinstance(marker, str) or not marker:
|
||||||
|
marker = str(index)
|
||||||
|
|
||||||
|
match = _ENTRY_MARKER_PATTERN.fullmatch(marker)
|
||||||
|
if not match:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return local_files.get(match.group(1))
|
||||||
|
|
||||||
|
|
||||||
|
def repair_local_video_dimensions(
|
||||||
|
metadata: MutableMapping[str, Any],
|
||||||
|
local_files: Mapping[str, str],
|
||||||
|
*,
|
||||||
|
dry_run: bool = False,
|
||||||
|
) -> int:
|
||||||
|
"""Backfill real video dimensions for an entry that has local files.
|
||||||
|
|
||||||
|
Only entries with an empty ``url`` are considered: those have no remote
|
||||||
|
source, so the local file is the single source of truth for their size and
|
||||||
|
rewriting them cannot discard API-supplied data. Entries whose dimensions
|
||||||
|
already match the file are left byte-identical.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
metadata: Raw metadata payload (mutated in place unless ``dry_run``).
|
||||||
|
local_files: ``{identifier: path}`` for files present in the model's
|
||||||
|
example folder, where the identifier is the entry's ``id`` (for
|
||||||
|
``customImages``) or its positional index (for ``images``).
|
||||||
|
dry_run: Count the fixes without mutating ``metadata``.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The number of entries that were (or would be) repaired.
|
||||||
|
"""
|
||||||
|
|
||||||
|
civitai = metadata.get("civitai")
|
||||||
|
if not isinstance(civitai, dict):
|
||||||
|
return 0
|
||||||
|
|
||||||
|
repaired = 0
|
||||||
|
|
||||||
|
for key in ("customImages", "images"):
|
||||||
|
entries = civitai.get(key)
|
||||||
|
if not isinstance(entries, list) or not entries:
|
||||||
|
continue
|
||||||
|
|
||||||
|
for index, entry in enumerate(entries):
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
continue
|
||||||
|
if entry.get("url", "") != "":
|
||||||
|
# Remote-backed entry: never rebuilt from local state.
|
||||||
|
continue
|
||||||
|
|
||||||
|
file_path = _resolve_local_file(entry, index, local_files)
|
||||||
|
if not file_path or not os.path.isfile(file_path):
|
||||||
|
continue
|
||||||
|
|
||||||
|
dimensions = _read_media_dimensions(
|
||||||
|
file_path, _is_video_entry(file_path, entry)
|
||||||
|
)
|
||||||
|
width, height = dimensions
|
||||||
|
if width <= 0 or height <= 0:
|
||||||
|
continue
|
||||||
|
if entry.get("width") == width and entry.get("height") == height:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not dry_run:
|
||||||
|
entry["width"] = width
|
||||||
|
entry["height"] = height
|
||||||
|
repaired += 1
|
||||||
|
|
||||||
|
return repaired
|
||||||
|
|
||||||
|
|
||||||
def _get_metadata_sync_service() -> MetadataSyncService:
|
def _get_metadata_sync_service() -> MetadataSyncService:
|
||||||
"""Return the shared metadata sync service, initialising it lazily."""
|
"""Return the shared metadata sync service, initialising it lazily."""
|
||||||
|
|
||||||
@@ -231,28 +377,20 @@ class MetadataUpdater:
|
|||||||
file_ext = os.path.splitext(path)[1].lower()
|
file_ext = os.path.splitext(path)[1].lower()
|
||||||
is_video = file_ext in SUPPORTED_MEDIA_EXTENSIONS['videos']
|
is_video = file_ext in SUPPORTED_MEDIA_EXTENSIONS['videos']
|
||||||
|
|
||||||
|
width, height = _read_media_dimensions(path, is_video)
|
||||||
|
|
||||||
# Create image metadata entry
|
# Create image metadata entry
|
||||||
image_entry = {
|
image_entry = {
|
||||||
"url": "", # Empty URL as required
|
"url": "", # Empty URL as required
|
||||||
"nsfwLevel": 0,
|
"nsfwLevel": 0,
|
||||||
"width": 720, # Default dimensions
|
"width": width,
|
||||||
"height": 1280,
|
"height": height,
|
||||||
"type": "video" if is_video else "image",
|
"type": "video" if is_video else "image",
|
||||||
"meta": None,
|
"meta": None,
|
||||||
"hasMeta": False,
|
"hasMeta": False,
|
||||||
"hasPositivePrompt": False
|
"hasPositivePrompt": False
|
||||||
}
|
}
|
||||||
|
|
||||||
# If it's an image, try to get actual dimensions (optional enhancement)
|
|
||||||
try:
|
|
||||||
from PIL import Image
|
|
||||||
if not is_video and os.path.exists(path):
|
|
||||||
with Image.open(path) as img:
|
|
||||||
image_entry["width"], image_entry["height"] = img.size
|
|
||||||
except:
|
|
||||||
# If PIL fails or is unavailable, use default dimensions
|
|
||||||
pass
|
|
||||||
|
|
||||||
images.append(image_entry)
|
images.append(image_entry)
|
||||||
|
|
||||||
# Update the model's civitai.images field
|
# Update the model's civitai.images field
|
||||||
@@ -322,13 +460,15 @@ class MetadataUpdater:
|
|||||||
file_ext = os.path.splitext(path)[1].lower()
|
file_ext = os.path.splitext(path)[1].lower()
|
||||||
is_video = file_ext in SUPPORTED_MEDIA_EXTENSIONS['videos']
|
is_video = file_ext in SUPPORTED_MEDIA_EXTENSIONS['videos']
|
||||||
|
|
||||||
|
width, height = _read_media_dimensions(path, is_video)
|
||||||
|
|
||||||
# Create image metadata entry
|
# Create image metadata entry
|
||||||
image_entry = {
|
image_entry = {
|
||||||
"url": "", # Empty URL as requested
|
"url": "", # Empty URL as requested
|
||||||
"id": short_id,
|
"id": short_id,
|
||||||
"nsfwLevel": 0,
|
"nsfwLevel": 0,
|
||||||
"width": 720, # Default dimensions
|
"width": width,
|
||||||
"height": 1280,
|
"height": height,
|
||||||
"type": "video" if is_video else "image",
|
"type": "video" if is_video else "image",
|
||||||
"meta": None,
|
"meta": None,
|
||||||
"hasMeta": False,
|
"hasMeta": False,
|
||||||
@@ -353,16 +493,6 @@ class MetadataUpdater:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Failed to extract metadata from {os.path.basename(path)}: {e}")
|
logger.warning(f"Failed to extract metadata from {os.path.basename(path)}: {e}")
|
||||||
|
|
||||||
# If it's an image, try to get actual dimensions
|
|
||||||
try:
|
|
||||||
from PIL import Image
|
|
||||||
if not is_video and os.path.exists(path):
|
|
||||||
with Image.open(path) as img:
|
|
||||||
image_entry["width"], image_entry["height"] = img.size
|
|
||||||
except:
|
|
||||||
# If PIL fails or is unavailable, use default dimensions
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Append to existing customImages array
|
# Append to existing customImages array
|
||||||
custom_images.append(image_entry)
|
custom_images.append(image_entry)
|
||||||
|
|
||||||
|
|||||||
@@ -15,12 +15,20 @@ from ..utils.example_images_paths import (
|
|||||||
)
|
)
|
||||||
from ..utils.metadata_manager import MetadataManager
|
from ..utils.metadata_manager import MetadataManager
|
||||||
from ..utils.example_images_processor import ExampleImagesProcessor
|
from ..utils.example_images_processor import ExampleImagesProcessor
|
||||||
from ..utils.example_images_metadata import update_cache_from_metadata
|
from ..utils.example_images_metadata import (
|
||||||
|
repair_local_video_dimensions,
|
||||||
|
update_cache_from_metadata,
|
||||||
|
)
|
||||||
from ..utils.constants import SUPPORTED_MEDIA_EXTENSIONS
|
from ..utils.constants import SUPPORTED_MEDIA_EXTENSIONS
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
CURRENT_NAMING_VERSION = 2 # Increment this when naming conventions change
|
CURRENT_NAMING_VERSION = 3 # Increment this when naming conventions change
|
||||||
|
|
||||||
|
# Example files worth inspecting during the dimension repair.
|
||||||
|
_REPAIRABLE_EXTENSIONS = frozenset(
|
||||||
|
SUPPORTED_MEDIA_EXTENSIONS["images"] + SUPPORTED_MEDIA_EXTENSIONS["videos"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class _SettingsProxy:
|
class _SettingsProxy:
|
||||||
@@ -185,6 +193,9 @@ class ExampleImagesMigration:
|
|||||||
if from_version < 2 and to_version >= 2:
|
if from_version < 2 and to_version >= 2:
|
||||||
await ExampleImagesMigration._migrate_to_v2(model_folders)
|
await ExampleImagesMigration._migrate_to_v2(model_folders)
|
||||||
|
|
||||||
|
if from_version < 3 and to_version >= 3:
|
||||||
|
await ExampleImagesMigration._migrate_to_v3(example_images_path, model_folders)
|
||||||
|
|
||||||
# Update version in progress file
|
# Update version in progress file
|
||||||
progress_file = os.path.join(example_images_path, '.download_progress.json')
|
progress_file = os.path.join(example_images_path, '.download_progress.json')
|
||||||
try:
|
try:
|
||||||
@@ -438,3 +449,136 @@ class ExampleImagesMigration:
|
|||||||
migration_errors += 1
|
migration_errors += 1
|
||||||
|
|
||||||
logger.info(f"Migration to v2 complete: migrated {count} custom examples across {updated_models} models with {migration_errors} errors")
|
logger.info(f"Migration to v2 complete: migrated {count} custom examples across {updated_models} models with {migration_errors} errors")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _build_local_file_map(folder):
|
||||||
|
"""Map entry markers to their files inside a model's example folder.
|
||||||
|
|
||||||
|
Keys are the marker alone (``custom_<id>`` → ``<id>``,
|
||||||
|
``image_<index>`` → ``<index>``) so they line up with the metadata
|
||||||
|
entries' ``id``/positional index without any prefix ambiguity.
|
||||||
|
"""
|
||||||
|
|
||||||
|
local_files = {}
|
||||||
|
try:
|
||||||
|
entries = os.listdir(folder)
|
||||||
|
except OSError as exc:
|
||||||
|
logger.debug("Could not list example folder %s: %s", folder, exc)
|
||||||
|
return local_files
|
||||||
|
|
||||||
|
for name in entries:
|
||||||
|
stem, ext = os.path.splitext(name)
|
||||||
|
if ext.lower() not in _REPAIRABLE_EXTENSIONS:
|
||||||
|
continue
|
||||||
|
if stem.startswith("custom_"):
|
||||||
|
local_files[stem[len("custom_"):]] = os.path.join(folder, name)
|
||||||
|
elif stem.startswith("image_"):
|
||||||
|
local_files[stem[len("image_"):]] = os.path.join(folder, name)
|
||||||
|
|
||||||
|
return local_files
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def _find_scanner_for_hash(model_hash):
|
||||||
|
"""Return the scanner owning ``model_hash``, or ``None``."""
|
||||||
|
|
||||||
|
lora_scanner = await ServiceRegistry.get_lora_scanner()
|
||||||
|
checkpoint_scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||||
|
embedding_scanner = await ServiceRegistry.get_embedding_scanner()
|
||||||
|
|
||||||
|
for scanner in (lora_scanner, checkpoint_scanner, embedding_scanner):
|
||||||
|
if scanner is None:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
if scanner.has_hash(model_hash):
|
||||||
|
return scanner
|
||||||
|
except Exception as exc: # pragma: no cover - defensive
|
||||||
|
logger.debug("has_hash check failed for %s: %s", type(scanner).__name__, exc)
|
||||||
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def _migrate_to_v3(example_images_path, model_folders):
|
||||||
|
"""Backfill real dimensions for locally imported example videos.
|
||||||
|
|
||||||
|
Imported videos were stored with a hardcoded ``720x1280`` placeholder
|
||||||
|
(issue #1115), so landscape clips were rendered inside a portrait
|
||||||
|
container. Only entries with an empty ``url`` are touched — those have
|
||||||
|
no remote source, which makes the local file authoritative and the
|
||||||
|
rewrite lossless. Entries already carrying the right size are left
|
||||||
|
untouched, so re-running this migration is a no-op.
|
||||||
|
|
||||||
|
This runs once per library via the ``naming_version`` gate in
|
||||||
|
``run_migrations``; it is deliberately not wired into any request path.
|
||||||
|
"""
|
||||||
|
|
||||||
|
repaired_entries = 0
|
||||||
|
updated_models = 0
|
||||||
|
migration_errors = 0
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Starting v3 migration (local example video dimensions) for %d model folders",
|
||||||
|
len(model_folders),
|
||||||
|
)
|
||||||
|
|
||||||
|
for folder in model_folders:
|
||||||
|
try:
|
||||||
|
model_hash = os.path.basename(folder)
|
||||||
|
if not model_hash or len(model_hash) != 64:
|
||||||
|
continue
|
||||||
|
|
||||||
|
local_files = ExampleImagesMigration._build_local_file_map(folder)
|
||||||
|
if not local_files:
|
||||||
|
continue
|
||||||
|
|
||||||
|
scanner = await ExampleImagesMigration._find_scanner_for_hash(model_hash)
|
||||||
|
if scanner is None:
|
||||||
|
logger.debug(
|
||||||
|
"Model %s not found in any scanner cache, skipping dimension repair",
|
||||||
|
model_hash,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
cache = await scanner.get_cached_data()
|
||||||
|
model_data = None
|
||||||
|
for item in cache.raw_data:
|
||||||
|
if item.get("sha256") == model_hash:
|
||||||
|
model_data = item
|
||||||
|
break
|
||||||
|
|
||||||
|
if not model_data:
|
||||||
|
continue
|
||||||
|
|
||||||
|
file_path = model_data.get("file_path")
|
||||||
|
if not file_path:
|
||||||
|
continue
|
||||||
|
|
||||||
|
payload = await MetadataManager.load_metadata_payload(file_path)
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
continue
|
||||||
|
|
||||||
|
repaired = repair_local_video_dimensions(payload, local_files)
|
||||||
|
if repaired <= 0:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# The model cache shape differs from the on-disk payload, so
|
||||||
|
# persist the file first and let the cache sync re-read it.
|
||||||
|
await MetadataManager.save_metadata(file_path, payload)
|
||||||
|
await update_cache_from_metadata(scanner, file_path, payload)
|
||||||
|
|
||||||
|
repaired_entries += repaired
|
||||||
|
updated_models += 1
|
||||||
|
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error(
|
||||||
|
"Failed to repair example video dimensions for %s: %s",
|
||||||
|
folder,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
migration_errors += 1
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Migration to v3 complete: repaired %d example entr(ies) across %d model(s) "
|
||||||
|
"with %d error(s)",
|
||||||
|
repaired_entries,
|
||||||
|
updated_models,
|
||||||
|
migration_errors,
|
||||||
|
)
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
"""Cross-process advisory locking for shared LoRA Manager state.
|
||||||
|
|
||||||
|
Two LoRA Manager processes (the ComfyUI plugin and a standalone server, or two
|
||||||
|
ComfyUI installs pointed at the same settings directory) can open the same cache
|
||||||
|
database. SQLite serializes individual statements, but it cannot make a
|
||||||
|
read-modify-write *sequence* atomic across processes: two full-table cache
|
||||||
|
replacements can interleave so that one process's snapshot overwrites the
|
||||||
|
other's.
|
||||||
|
|
||||||
|
This module provides a small advisory file lock for those sequences. It is
|
||||||
|
deliberately non-fatal: if locking is unavailable or the wait times out, callers
|
||||||
|
keep working with SQLite's own ``busy_timeout`` as the fallback.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# How long to wait for another process to release the lock before giving up.
|
||||||
|
DEFAULT_LOCK_TIMEOUT_SECONDS = 30.0
|
||||||
|
_POLL_INTERVAL_SECONDS = 0.05
|
||||||
|
|
||||||
|
# Windows byte-range locks; fcntl.flock on POSIX.
|
||||||
|
try: # pragma: no cover - platform dependent
|
||||||
|
import fcntl
|
||||||
|
except ImportError: # pragma: no cover - Windows
|
||||||
|
fcntl = None # type: ignore[assignment]
|
||||||
|
|
||||||
|
try: # pragma: no cover - Windows only
|
||||||
|
import msvcrt
|
||||||
|
except ImportError: # pragma: no cover - POSIX
|
||||||
|
msvcrt = None # type: ignore[assignment]
|
||||||
|
|
||||||
|
|
||||||
|
class FileLockUnavailable(RuntimeError):
|
||||||
|
"""Raised when the lock could not be acquired within the timeout."""
|
||||||
|
|
||||||
|
|
||||||
|
def lock_path_for(db_path: str) -> str:
|
||||||
|
"""Return the sibling lock file path used for *db_path*."""
|
||||||
|
absolute = os.path.abspath(db_path)
|
||||||
|
directory = os.path.dirname(absolute)
|
||||||
|
if not directory:
|
||||||
|
raise ValueError(f"Cannot derive a lock directory from {db_path!r}")
|
||||||
|
return os.path.join(directory, f".{os.path.basename(absolute)}.lock")
|
||||||
|
|
||||||
|
|
||||||
|
class CrossProcessLock:
|
||||||
|
"""A best-effort advisory lock backed by a lock file.
|
||||||
|
|
||||||
|
The lock file is a sibling of the guarded resource and is never deleted:
|
||||||
|
unlinking it would let a second process create a fresh inode and lock that
|
||||||
|
instead, defeating mutual exclusion.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, path: str, timeout: float = DEFAULT_LOCK_TIMEOUT_SECONDS):
|
||||||
|
self.path = path
|
||||||
|
self.timeout = timeout
|
||||||
|
self._handle = None
|
||||||
|
|
||||||
|
def acquire(self) -> bool:
|
||||||
|
"""Try to take the lock, waiting up to ``timeout`` seconds.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True when the lock is held (including when another lock is already
|
||||||
|
held by *this* process — the calls are not reentrant, so callers must
|
||||||
|
not nest them). False when locking is unsupported or timed out; the
|
||||||
|
caller should proceed and rely on the SQLite busy timeout instead.
|
||||||
|
"""
|
||||||
|
if fcntl is None and msvcrt is None: # pragma: no cover - exotic platform
|
||||||
|
return False
|
||||||
|
|
||||||
|
os.makedirs(os.path.dirname(self.path), exist_ok=True)
|
||||||
|
try:
|
||||||
|
handle = open(self.path, "a+b")
|
||||||
|
except OSError as exc:
|
||||||
|
logger.debug("Could not open lock file %s: %s", self.path, exc)
|
||||||
|
return False
|
||||||
|
|
||||||
|
deadline = time.monotonic() + max(0.0, self.timeout)
|
||||||
|
while True:
|
||||||
|
if self._try_lock(handle):
|
||||||
|
self._handle = handle
|
||||||
|
return True
|
||||||
|
if time.monotonic() >= deadline:
|
||||||
|
handle.close()
|
||||||
|
return False
|
||||||
|
time.sleep(_POLL_INTERVAL_SECONDS)
|
||||||
|
|
||||||
|
def release(self) -> None:
|
||||||
|
"""Release the lock if held. Safe to call more than once."""
|
||||||
|
handle = self._handle
|
||||||
|
if handle is None:
|
||||||
|
return
|
||||||
|
self._handle = None
|
||||||
|
try:
|
||||||
|
self._unlock(handle)
|
||||||
|
except OSError as exc: # pragma: no cover - defensive
|
||||||
|
logger.debug("Failed to release lock %s: %s", self.path, exc)
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
handle.close()
|
||||||
|
except OSError: # pragma: no cover - defensive
|
||||||
|
pass
|
||||||
|
|
||||||
|
def __enter__(self) -> "CrossProcessLock":
|
||||||
|
self.acquire()
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_exc_info: object) -> None:
|
||||||
|
self.release()
|
||||||
|
|
||||||
|
# -- platform primitives -------------------------------------------------
|
||||||
|
|
||||||
|
def _try_lock(self, handle) -> bool:
|
||||||
|
if fcntl is not None:
|
||||||
|
try:
|
||||||
|
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
return True
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
if msvcrt is not None: # pragma: no cover - Windows
|
||||||
|
try:
|
||||||
|
handle.seek(0)
|
||||||
|
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
|
||||||
|
return True
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _unlock(self, handle) -> None:
|
||||||
|
if fcntl is not None:
|
||||||
|
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||||
|
return
|
||||||
|
if msvcrt is not None: # pragma: no cover - Windows
|
||||||
|
handle.seek(0)
|
||||||
|
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
||||||
|
|
||||||
|
|
||||||
|
def exclusive_lock(db_path: str, timeout: float = DEFAULT_LOCK_TIMEOUT_SECONDS):
|
||||||
|
"""Return a :class:`CrossProcessLock` for the database at *db_path*."""
|
||||||
|
return CrossProcessLock(lock_path_for(db_path), timeout=timeout)
|
||||||
@@ -174,12 +174,42 @@ def ensure_settings_file(logger: Optional[logging.Logger] = None) -> str:
|
|||||||
return target_path
|
return target_path
|
||||||
|
|
||||||
|
|
||||||
|
def _portable_env_override() -> Optional[bool]:
|
||||||
|
"""Return the portable mode forced by ``LORA_MANAGER_PORTABLE``, if any.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
``True`` when the variable enables portable mode, ``False`` when it is
|
||||||
|
explicitly set to ``"0"``, and ``None`` when it is unset or holds some
|
||||||
|
other value (in which case the persisted settings flag decides).
|
||||||
|
"""
|
||||||
|
|
||||||
|
raw = os.environ.get(_LM_PORTABLE_ENV)
|
||||||
|
if raw is None:
|
||||||
|
return None
|
||||||
|
if raw == "1":
|
||||||
|
return True
|
||||||
|
if raw == "0":
|
||||||
|
return False
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _should_use_portable_settings(path: str, logger: logging.Logger) -> bool:
|
def _should_use_portable_settings(path: str, logger: logging.Logger) -> bool:
|
||||||
"""Return ``True`` when the env var forces it or the settings file enables it."""
|
"""Return ``True`` when the env var forces it or the settings file enables it."""
|
||||||
|
|
||||||
if os.environ.get(_LM_PORTABLE_ENV, "0") == "1":
|
override = _portable_env_override()
|
||||||
|
if override is True:
|
||||||
logger.debug("Portable mode enabled via %s", _LM_PORTABLE_ENV)
|
logger.debug("Portable mode enabled via %s", _LM_PORTABLE_ENV)
|
||||||
return True
|
return True
|
||||||
|
if override is False:
|
||||||
|
# Explicit opt-out. Without this, a single `LORA_MANAGER_PORTABLE=1`
|
||||||
|
# run would pin the shared plugin settings.json to portable mode
|
||||||
|
# forever, with no way back except editing that file by hand.
|
||||||
|
logger.info(
|
||||||
|
"Portable mode disabled via %s=%s",
|
||||||
|
_LM_PORTABLE_ENV,
|
||||||
|
os.environ.get(_LM_PORTABLE_ENV, ""),
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
if not os.path.exists(path):
|
if not os.path.exists(path):
|
||||||
return False
|
return False
|
||||||
|
|||||||
@@ -0,0 +1,623 @@
|
|||||||
|
"""Read intrinsic dimensions from video containers without external tooling.
|
||||||
|
|
||||||
|
PIL cannot open ``.mp4``/``.webm`` files, so example videos imported through
|
||||||
|
the "Add examples" flow used to fall back to a hardcoded ``720x1280`` (portrait)
|
||||||
|
entry, which forced the showcase viewer to letterbox landscape videos.
|
||||||
|
|
||||||
|
This module reads the dimensions out of the container headers themselves:
|
||||||
|
|
||||||
|
* ISO base media files (``.mp4``/``.mov``/``.m4v``) — ``moov/trak/tkhd``,
|
||||||
|
falling back to the sample description of the video track.
|
||||||
|
* WebM/Matroska (``.webm``/``.mkv``) — ``Segment/Tracks/TrackEntry/Video``
|
||||||
|
``PixelWidth``/``PixelHeight``.
|
||||||
|
* Animated WebP (``RIFF``/``WEBP``) — handled because users routinely save
|
||||||
|
animated examples with a video extension.
|
||||||
|
|
||||||
|
The container signature decides which reader runs, so a mislabelled file
|
||||||
|
(a ``.mp4`` that is really WebM) still reports the right dimensions.
|
||||||
|
|
||||||
|
Both readers stream over the file: only container headers are read, so a
|
||||||
|
multi-gigabyte ``mdat`` is never pulled into memory (it is seeked past).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import functools
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import struct
|
||||||
|
from typing import BinaryIO, Iterator, Optional, Tuple
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
ISO_MEDIA_EXTENSIONS = frozenset({".mp4", ".m4v", ".mov"})
|
||||||
|
EBML_MEDIA_EXTENSIONS = frozenset({".webm", ".mkv"})
|
||||||
|
|
||||||
|
_EBML_MAGIC = b"\x1a\x45\xdf\xa3"
|
||||||
|
|
||||||
|
# Cap recursion into nesting containers so a crafted/corrupt file cannot blow
|
||||||
|
# the Python stack.
|
||||||
|
_MAX_BOX_DEPTH = 12
|
||||||
|
_MAX_EBML_DEPTH = 12
|
||||||
|
|
||||||
|
# Header structs (``tkhd``, sample entries) are tiny; guard against a bogus
|
||||||
|
# size claiming the whole file.
|
||||||
|
_MAX_HEADER_PAYLOAD = 1024 * 1024
|
||||||
|
|
||||||
|
_WIDTH_HEIGHT_UNSET = (0, 0)
|
||||||
|
|
||||||
|
|
||||||
|
@functools.lru_cache(maxsize=4096)
|
||||||
|
def _get_video_dimensions_cached(
|
||||||
|
path: str, _mtime_ns: int, _size: int
|
||||||
|
) -> Optional[Tuple[int, int]]:
|
||||||
|
"""Return ``(width, height)`` for ``path``, or ``None`` on any failure.
|
||||||
|
|
||||||
|
``_mtime_ns`` and ``_size`` participate in the cache key only so a replaced
|
||||||
|
file is re-probed; they are never read by the parser.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return _read_video_dimensions(path)
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Failed to read video dimensions for %s", path, exc_info=True)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _read_video_dimensions(path: str) -> Optional[Tuple[int, int]]:
|
||||||
|
"""Dispatch to the ISO or EBML reader based on the container's magic bytes.
|
||||||
|
|
||||||
|
Real libraries contain files whose extension lies about their container
|
||||||
|
(a ``.mp4`` that is really WebM, typically), so the sniffed signature wins
|
||||||
|
and the extension is only a fallback.
|
||||||
|
"""
|
||||||
|
|
||||||
|
ext = os.path.splitext(path)[1].lower()
|
||||||
|
file_size = os.path.getsize(path)
|
||||||
|
|
||||||
|
with open(path, "rb") as stream:
|
||||||
|
magic = stream.read(12)
|
||||||
|
|
||||||
|
if _looks_like_iso_media(magic):
|
||||||
|
return _read_iso_media_dimensions(stream, file_size)
|
||||||
|
if magic[:4] == _EBML_MAGIC:
|
||||||
|
return _read_ebml_dimensions(stream, file_size)
|
||||||
|
if magic[:4] == b"RIFF" and magic[8:12] == b"WEBP":
|
||||||
|
return _read_riff_webp_dimensions(stream, file_size)
|
||||||
|
|
||||||
|
# Signature is inconclusive (truncated or unusual file): fall back to
|
||||||
|
# the extension.
|
||||||
|
if ext in EBML_MEDIA_EXTENSIONS:
|
||||||
|
return _read_ebml_dimensions(stream, file_size)
|
||||||
|
if ext in ISO_MEDIA_EXTENSIONS:
|
||||||
|
return _read_iso_media_dimensions(stream, file_size)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _looks_like_iso_media(magic: bytes) -> bool:
|
||||||
|
"""Return True when the leading bytes are an ISO base media box header."""
|
||||||
|
|
||||||
|
return len(magic) >= 8 and magic[4:8] in {
|
||||||
|
b"ftyp",
|
||||||
|
b"moov",
|
||||||
|
b"mdat",
|
||||||
|
b"free",
|
||||||
|
b"skip",
|
||||||
|
b"wide",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_video_dimensions(path: str) -> Optional[Tuple[int, int]]:
|
||||||
|
"""Return the intrinsic ``(width, height)`` of a local video file.
|
||||||
|
|
||||||
|
Returns ``None`` when the extension is unsupported, the file is missing or
|
||||||
|
corrupt, or the dimensions cannot be determined. Never raises.
|
||||||
|
"""
|
||||||
|
if not path:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
stat = os.stat(path)
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
return _get_video_dimensions_cached(path, stat.st_mtime_ns, stat.st_size)
|
||||||
|
|
||||||
|
|
||||||
|
def _clear_video_dimensions_cache() -> None:
|
||||||
|
"""Drop the dimension cache (used by tests)."""
|
||||||
|
|
||||||
|
_get_video_dimensions_cached.cache_clear()
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# ISO base media (MP4 / MOV)
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_boxes(
|
||||||
|
stream: BinaryIO, end: int, depth: int = 0
|
||||||
|
) -> Iterator[Tuple[bytes, int, int]]:
|
||||||
|
"""Yield ``(type, payload_start, box_end)`` for boxes in ``[tell, end)``.
|
||||||
|
|
||||||
|
The stream is left at the next box boundary after each yielded box.
|
||||||
|
"""
|
||||||
|
if depth > _MAX_BOX_DEPTH:
|
||||||
|
return
|
||||||
|
|
||||||
|
while True:
|
||||||
|
start = stream.tell()
|
||||||
|
if start + 8 > end:
|
||||||
|
return
|
||||||
|
|
||||||
|
header = stream.read(8)
|
||||||
|
if len(header) < 8:
|
||||||
|
return
|
||||||
|
|
||||||
|
size, box_type = struct.unpack(">I4s", header)
|
||||||
|
header_size = 8
|
||||||
|
|
||||||
|
if size == 1:
|
||||||
|
# 64-bit ``largesize`` follows the type.
|
||||||
|
extended = stream.read(8)
|
||||||
|
if len(extended) < 8:
|
||||||
|
return
|
||||||
|
size = struct.unpack(">Q", extended)[0]
|
||||||
|
header_size = 16
|
||||||
|
elif size == 0:
|
||||||
|
# Box extends to the end of the enclosing container.
|
||||||
|
size = end - start
|
||||||
|
|
||||||
|
if size < header_size or start + size > end:
|
||||||
|
return
|
||||||
|
|
||||||
|
yield box_type, start + header_size, start + size
|
||||||
|
stream.seek(start + size)
|
||||||
|
|
||||||
|
|
||||||
|
def _read_iso_media_dimensions(
|
||||||
|
stream: BinaryIO, file_size: int
|
||||||
|
) -> Optional[Tuple[int, int]]:
|
||||||
|
"""Walk ``moov`` looking for the video track's dimensions."""
|
||||||
|
|
||||||
|
stream.seek(0)
|
||||||
|
moov: Optional[Tuple[int, int]] = None
|
||||||
|
for box_type, payload_start, box_end in _iter_boxes(stream, file_size):
|
||||||
|
if box_type == b"moov":
|
||||||
|
moov = (payload_start, box_end)
|
||||||
|
break
|
||||||
|
|
||||||
|
if moov is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
stream.seek(moov[0])
|
||||||
|
for box_type, payload_start, box_end in _iter_boxes(stream, moov[1], depth=1):
|
||||||
|
if box_type != b"trak":
|
||||||
|
continue
|
||||||
|
dimensions = _read_trak_dimensions(stream, payload_start, box_end)
|
||||||
|
if dimensions is not None:
|
||||||
|
return dimensions
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _read_trak_dimensions(
|
||||||
|
stream: BinaryIO, trak_start: int, trak_end: int
|
||||||
|
) -> Optional[Tuple[int, int]]:
|
||||||
|
"""Return the dimensions of a ``trak`` when it describes a video track."""
|
||||||
|
|
||||||
|
stream.seek(trak_start)
|
||||||
|
|
||||||
|
is_video = False
|
||||||
|
tkhd_dimensions = _WIDTH_HEIGHT_UNSET
|
||||||
|
stsd_dimensions = _WIDTH_HEIGHT_UNSET
|
||||||
|
|
||||||
|
for box_type, payload_start, box_end in _iter_boxes(stream, trak_end, depth=2):
|
||||||
|
if box_type == b"tkhd":
|
||||||
|
tkhd_dimensions = _parse_tkhd(stream, payload_start, box_end)
|
||||||
|
elif box_type == b"mdia":
|
||||||
|
stream.seek(payload_start)
|
||||||
|
media = _read_mdia_dimensions(stream, payload_start, box_end)
|
||||||
|
if media is not None:
|
||||||
|
is_video, stsd_dimensions = media
|
||||||
|
|
||||||
|
if not is_video:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# ``tkhd`` is preferred: it is display space, and its 16.16 fixed point
|
||||||
|
# encoding keeps non-integer dimensions (odd crops produce those).
|
||||||
|
for width, height in (tkhd_dimensions, stsd_dimensions):
|
||||||
|
if width > 0 and height > 0:
|
||||||
|
return int(round(width)), int(round(height))
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _read_mdia_dimensions(
|
||||||
|
stream: BinaryIO, mdia_start: int, mdia_end: int
|
||||||
|
) -> Optional[Tuple[bool, Tuple[float, float]]]:
|
||||||
|
"""Return ``(is_video, dimensions)`` for a ``mdia`` box."""
|
||||||
|
|
||||||
|
handler_type = b""
|
||||||
|
stsd_dimensions = _WIDTH_HEIGHT_UNSET
|
||||||
|
|
||||||
|
for box_type, payload_start, box_end in _iter_boxes(stream, mdia_end, depth=3):
|
||||||
|
if box_type == b"hdlr":
|
||||||
|
handler_type = _parse_handler_type(stream, payload_start, box_end)
|
||||||
|
elif box_type == b"minf":
|
||||||
|
stream.seek(payload_start)
|
||||||
|
stsd_dimensions = _read_minf_dimensions(stream, payload_start, box_end)
|
||||||
|
|
||||||
|
return handler_type == b"vide", stsd_dimensions
|
||||||
|
|
||||||
|
|
||||||
|
def _read_minf_dimensions(
|
||||||
|
stream: BinaryIO, minf_start: int, minf_end: int
|
||||||
|
) -> Tuple[float, float]:
|
||||||
|
"""Return the sample-entry dimensions declared under ``minf/stbl/stsd``."""
|
||||||
|
|
||||||
|
for box_type, payload_start, box_end in _iter_boxes(stream, minf_end, depth=4):
|
||||||
|
if box_type != b"stbl":
|
||||||
|
continue
|
||||||
|
stream.seek(payload_start)
|
||||||
|
for inner_type, inner_start, inner_end in _iter_boxes(
|
||||||
|
stream, box_end, depth=5
|
||||||
|
):
|
||||||
|
if inner_type == b"stsd":
|
||||||
|
return _parse_stsd(stream, inner_start, inner_end)
|
||||||
|
return _WIDTH_HEIGHT_UNSET
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_tkhd(
|
||||||
|
stream: BinaryIO, payload_start: int, box_end: int
|
||||||
|
) -> Tuple[float, float]:
|
||||||
|
"""Parse the 16.16 fixed point width/height trailer of a ``tkhd`` box."""
|
||||||
|
|
||||||
|
size = box_end - payload_start
|
||||||
|
if size < 8 or size > _MAX_HEADER_PAYLOAD:
|
||||||
|
return _WIDTH_HEIGHT_UNSET
|
||||||
|
|
||||||
|
stream.seek(box_end - 8)
|
||||||
|
trailer = stream.read(8)
|
||||||
|
if len(trailer) < 8:
|
||||||
|
return _WIDTH_HEIGHT_UNSET
|
||||||
|
|
||||||
|
width, height = struct.unpack(">II", trailer)
|
||||||
|
return width / 65536.0, height / 65536.0
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_handler_type(
|
||||||
|
stream: BinaryIO, payload_start: int, box_end: int
|
||||||
|
) -> bytes:
|
||||||
|
"""Parse the handler type from an ``hdlr`` box.
|
||||||
|
|
||||||
|
Layout: version/flags (4) + pre_defined (4) + handler_type (4).
|
||||||
|
"""
|
||||||
|
|
||||||
|
if box_end - payload_start < 12:
|
||||||
|
return b""
|
||||||
|
stream.seek(payload_start)
|
||||||
|
data = stream.read(12)
|
||||||
|
if len(data) < 12:
|
||||||
|
return b""
|
||||||
|
return data[8:12]
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_stsd(
|
||||||
|
stream: BinaryIO, payload_start: int, box_end: int
|
||||||
|
) -> Tuple[float, float]:
|
||||||
|
"""Parse the visual sample entry dimensions from an ``stsd`` box.
|
||||||
|
|
||||||
|
Only the first entry is inspected: video tracks are single-entry in every
|
||||||
|
container we import from.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if box_end - payload_start < 16:
|
||||||
|
return _WIDTH_HEIGHT_UNSET
|
||||||
|
|
||||||
|
stream.seek(payload_start)
|
||||||
|
header = stream.read(8) # version/flags + entry_count
|
||||||
|
if len(header) < 8:
|
||||||
|
return _WIDTH_HEIGHT_UNSET
|
||||||
|
|
||||||
|
entry_start = payload_start + 8
|
||||||
|
if entry_start + 8 > box_end:
|
||||||
|
return _WIDTH_HEIGHT_UNSET
|
||||||
|
|
||||||
|
stream.seek(entry_start)
|
||||||
|
entry_header = stream.read(8)
|
||||||
|
if len(entry_header) < 8:
|
||||||
|
return _WIDTH_HEIGHT_UNSET
|
||||||
|
|
||||||
|
entry_size = struct.unpack(">I", entry_header[:4])[0]
|
||||||
|
header_size = 8
|
||||||
|
|
||||||
|
if entry_size == 1:
|
||||||
|
extended = stream.read(8)
|
||||||
|
if len(extended) < 8:
|
||||||
|
return _WIDTH_HEIGHT_UNSET
|
||||||
|
entry_size = struct.unpack(">Q", extended)[0]
|
||||||
|
header_size = 16
|
||||||
|
elif entry_size == 0:
|
||||||
|
entry_size = box_end - entry_start
|
||||||
|
|
||||||
|
if entry_size < header_size + 8 or entry_start + entry_size > box_end:
|
||||||
|
return _WIDTH_HEIGHT_UNSET
|
||||||
|
|
||||||
|
# Visual sample entries: 6 bytes reserved + 2 bytes data_reference_index,
|
||||||
|
# then width (2) and height (2).
|
||||||
|
stream.seek(entry_start + header_size + 6 + 2)
|
||||||
|
dimensions = stream.read(4)
|
||||||
|
if len(dimensions) < 4:
|
||||||
|
return _WIDTH_HEIGHT_UNSET
|
||||||
|
|
||||||
|
width, height = struct.unpack(">HH", dimensions)
|
||||||
|
return float(width), float(height)
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# WebM / Matroska (EBML)
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
# EBML element IDs (stored with their length marker, as they appear on disk).
|
||||||
|
_ID_SEGMENT = 0x18538067
|
||||||
|
_ID_TRACKS = 0x1654AE6B
|
||||||
|
_ID_TRACK_ENTRY = 0xAE
|
||||||
|
_ID_TRACK_TYPE = 0x83
|
||||||
|
_ID_VIDEO = 0xE0
|
||||||
|
_ID_PIXEL_WIDTH = 0xB0
|
||||||
|
_ID_PIXEL_HEIGHT = 0xBA
|
||||||
|
|
||||||
|
# Nested containers we descend into while hunting for video dimensions.
|
||||||
|
_EBML_CONTAINER_IDS = frozenset({_ID_SEGMENT, _ID_TRACKS, _ID_TRACK_ENTRY})
|
||||||
|
|
||||||
|
|
||||||
|
def _read_ebml_vint(stream: BinaryIO, *, keep_marker: bool) -> Optional[Tuple[int, int]]:
|
||||||
|
"""Read an EBML variable-length integer.
|
||||||
|
|
||||||
|
Returns ``(value, byte_length)``. For element IDs the marker bit is kept
|
||||||
|
(``keep_marker=True``) because IDs are compared in their on-disk form; for
|
||||||
|
sizes the marker is stripped to yield the actual payload length.
|
||||||
|
"""
|
||||||
|
|
||||||
|
first = stream.read(1)
|
||||||
|
if not first:
|
||||||
|
return None
|
||||||
|
|
||||||
|
first_byte = first[0]
|
||||||
|
if first_byte == 0:
|
||||||
|
return None
|
||||||
|
|
||||||
|
length = 1
|
||||||
|
mask = 0x80
|
||||||
|
while not first_byte & mask:
|
||||||
|
mask >>= 1
|
||||||
|
length += 1
|
||||||
|
if length > 8:
|
||||||
|
return None
|
||||||
|
|
||||||
|
value = first_byte if keep_marker else first_byte & (mask - 1)
|
||||||
|
remaining = length - 1
|
||||||
|
|
||||||
|
if remaining:
|
||||||
|
extra = stream.read(remaining)
|
||||||
|
if len(extra) < remaining:
|
||||||
|
return None
|
||||||
|
for byte in extra:
|
||||||
|
value = (value << 8) | byte
|
||||||
|
|
||||||
|
return value, length
|
||||||
|
|
||||||
|
|
||||||
|
def _read_ebml_dimensions(
|
||||||
|
stream: BinaryIO, file_size: int
|
||||||
|
) -> Optional[Tuple[int, int]]:
|
||||||
|
"""Parse ``Segment/Tracks`` for the first video ``TrackEntry``."""
|
||||||
|
|
||||||
|
stream.seek(0)
|
||||||
|
header = stream.read(4)
|
||||||
|
if header != _EBML_MAGIC:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return _walk_ebml(stream, 0, file_size, depth=0)
|
||||||
|
|
||||||
|
|
||||||
|
def _walk_ebml(
|
||||||
|
stream: BinaryIO, start: int, end: int, *, depth: int
|
||||||
|
) -> Optional[Tuple[int, int]]:
|
||||||
|
"""Recursively scan EBML elements in ``[start, end)`` for video dimensions."""
|
||||||
|
|
||||||
|
if depth > _MAX_EBML_DEPTH:
|
||||||
|
return None
|
||||||
|
|
||||||
|
stream.seek(start)
|
||||||
|
|
||||||
|
while stream.tell() < end:
|
||||||
|
element_start = stream.tell()
|
||||||
|
|
||||||
|
element_id = _read_ebml_vint(stream, keep_marker=True)
|
||||||
|
if element_id is None:
|
||||||
|
return None
|
||||||
|
element_id_value = element_id[0]
|
||||||
|
|
||||||
|
size_field = _read_ebml_vint(stream, keep_marker=False)
|
||||||
|
if size_field is None:
|
||||||
|
return None
|
||||||
|
payload_size, size_length = size_field
|
||||||
|
|
||||||
|
payload_start = element_start + element_id[1] + size_length
|
||||||
|
|
||||||
|
# A size field of all-ones marks an unknown-size element, which is
|
||||||
|
# legal for Segment/Tracks; treat it as "until the parent ends".
|
||||||
|
unknown_size = payload_size == (1 << (7 * size_length)) - 1
|
||||||
|
payload_end = end if unknown_size else payload_start + payload_size
|
||||||
|
|
||||||
|
if payload_end > end:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if element_id_value == _ID_VIDEO:
|
||||||
|
dimensions = _read_ebml_video(stream, payload_start, min(payload_end, end))
|
||||||
|
if dimensions is not None:
|
||||||
|
return dimensions
|
||||||
|
elif element_id_value == _ID_TRACK_ENTRY:
|
||||||
|
track = _read_ebml_track_entry(
|
||||||
|
stream, payload_start, min(payload_end, end)
|
||||||
|
)
|
||||||
|
if track is not None:
|
||||||
|
return track
|
||||||
|
elif element_id_value in _EBML_CONTAINER_IDS:
|
||||||
|
found = _walk_ebml(
|
||||||
|
stream, payload_start, min(payload_end, end), depth=depth + 1
|
||||||
|
)
|
||||||
|
if found is not None:
|
||||||
|
return found
|
||||||
|
|
||||||
|
if unknown_size:
|
||||||
|
# Cannot resume after an unknown-size element; its siblings cannot
|
||||||
|
# be located reliably, so stop scanning this level.
|
||||||
|
return None
|
||||||
|
|
||||||
|
stream.seek(payload_end)
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _read_ebml_track_entry(
|
||||||
|
stream: BinaryIO, start: int, end: int
|
||||||
|
) -> Optional[Tuple[int, int]]:
|
||||||
|
"""Return dimensions when a ``TrackEntry`` is a video track."""
|
||||||
|
|
||||||
|
track_type: Optional[int] = None
|
||||||
|
dimensions: Optional[Tuple[int, int]] = None
|
||||||
|
|
||||||
|
stream.seek(start)
|
||||||
|
while stream.tell() < end:
|
||||||
|
element_start = stream.tell()
|
||||||
|
|
||||||
|
element_id = _read_ebml_vint(stream, keep_marker=True)
|
||||||
|
if element_id is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
size_field = _read_ebml_vint(stream, keep_marker=False)
|
||||||
|
if size_field is None:
|
||||||
|
return None
|
||||||
|
payload_size, size_length = size_field
|
||||||
|
|
||||||
|
payload_start = element_start + element_id[1] + size_length
|
||||||
|
payload_end = min(payload_start + payload_size, end)
|
||||||
|
|
||||||
|
if element_id[0] == _ID_TRACK_TYPE:
|
||||||
|
track_type = _read_ebml_uint(stream, payload_start, payload_end)
|
||||||
|
elif element_id[0] == _ID_VIDEO:
|
||||||
|
dimensions = _read_ebml_video(stream, payload_start, payload_end)
|
||||||
|
|
||||||
|
stream.seek(payload_end)
|
||||||
|
|
||||||
|
# Track type 1 is video.
|
||||||
|
if track_type == 1 and dimensions is not None:
|
||||||
|
return dimensions
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _read_ebml_video(
|
||||||
|
stream: BinaryIO, start: int, end: int
|
||||||
|
) -> Optional[Tuple[int, int]]:
|
||||||
|
"""Return ``PixelWidth``/``PixelHeight`` from a ``Video`` element."""
|
||||||
|
|
||||||
|
width: Optional[int] = None
|
||||||
|
height: Optional[int] = None
|
||||||
|
|
||||||
|
stream.seek(start)
|
||||||
|
while stream.tell() < end:
|
||||||
|
element_start = stream.tell()
|
||||||
|
|
||||||
|
element_id = _read_ebml_vint(stream, keep_marker=True)
|
||||||
|
if element_id is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
size_field = _read_ebml_vint(stream, keep_marker=False)
|
||||||
|
if size_field is None:
|
||||||
|
return None
|
||||||
|
payload_size, size_length = size_field
|
||||||
|
|
||||||
|
payload_start = element_start + element_id[1] + size_length
|
||||||
|
payload_end = min(payload_start + payload_size, end)
|
||||||
|
|
||||||
|
if element_id[0] == _ID_PIXEL_WIDTH:
|
||||||
|
width = _read_ebml_uint(stream, payload_start, payload_end)
|
||||||
|
elif element_id[0] == _ID_PIXEL_HEIGHT:
|
||||||
|
height = _read_ebml_uint(stream, payload_start, payload_end)
|
||||||
|
|
||||||
|
stream.seek(payload_end)
|
||||||
|
|
||||||
|
if width and height and width > 0 and height > 0:
|
||||||
|
return width, height
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _read_ebml_uint(stream: BinaryIO, start: int, end: int) -> Optional[int]:
|
||||||
|
"""Read an unsigned big-endian integer element payload."""
|
||||||
|
|
||||||
|
length = end - start
|
||||||
|
if length <= 0 or length > 8:
|
||||||
|
return None
|
||||||
|
|
||||||
|
stream.seek(start)
|
||||||
|
raw = stream.read(length)
|
||||||
|
if len(raw) < length:
|
||||||
|
return None
|
||||||
|
|
||||||
|
value = 0
|
||||||
|
for byte in raw:
|
||||||
|
value = (value << 8) | byte
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# RIFF / WebP (animated examples are often renamed to ``.mp4``)
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
|
||||||
|
def _read_riff_webp_dimensions(
|
||||||
|
stream: BinaryIO, file_size: int
|
||||||
|
) -> Optional[Tuple[int, int]]:
|
||||||
|
"""Return dimensions from a WebP file's first dimension-bearing chunk."""
|
||||||
|
|
||||||
|
stream.seek(12)
|
||||||
|
|
||||||
|
while stream.tell() + 8 <= file_size:
|
||||||
|
header = stream.read(8)
|
||||||
|
if len(header) < 8:
|
||||||
|
return None
|
||||||
|
|
||||||
|
fourcc, chunk_size = struct.unpack("<4sI", header)
|
||||||
|
payload_start = stream.tell()
|
||||||
|
|
||||||
|
if fourcc == b"VP8X":
|
||||||
|
payload = stream.read(10)
|
||||||
|
if len(payload) < 10:
|
||||||
|
return None
|
||||||
|
# Canvas size is stored minus one, as 24-bit little endian values.
|
||||||
|
width = int.from_bytes(payload[4:7], "little") + 1
|
||||||
|
height = int.from_bytes(payload[7:10], "little") + 1
|
||||||
|
return width, height
|
||||||
|
|
||||||
|
if fourcc == b"VP8 ":
|
||||||
|
# Frame tag (3 bytes, bit 0 = key frame) then the key frame start
|
||||||
|
# code 0x9d 0x01 0x2a and the 16-bit dimensions.
|
||||||
|
payload = stream.read(10)
|
||||||
|
if len(payload) < 10:
|
||||||
|
return None
|
||||||
|
start = payload.find(b"\x9d\x01\x2a")
|
||||||
|
if start < 0 or start + 7 > len(payload):
|
||||||
|
return None
|
||||||
|
width, height = struct.unpack("<HH", payload[start + 3 : start + 7])
|
||||||
|
return width & 0x3FFF, height & 0x3FFF
|
||||||
|
|
||||||
|
if fourcc == b"VP8L":
|
||||||
|
payload = stream.read(5)
|
||||||
|
if len(payload) < 5 or payload[0] != 0x2F:
|
||||||
|
return None
|
||||||
|
bits = int.from_bytes(payload[1:5], "little")
|
||||||
|
return (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1
|
||||||
|
|
||||||
|
# Skip this chunk (payloads are padded to an even byte boundary).
|
||||||
|
stream.seek(payload_start + chunk_size + (chunk_size & 1))
|
||||||
|
|
||||||
|
return None
|
||||||
@@ -225,10 +225,9 @@ def main() -> int:
|
|||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
# Get project root (parent of .agents directory)
|
# Get project root: this script lives in <project_root>/scripts/e2e/.
|
||||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||||
skill_dir = os.path.dirname(script_dir)
|
project_root = os.path.dirname(os.path.dirname(script_dir))
|
||||||
project_root = os.path.dirname(os.path.dirname(os.path.dirname(skill_dir)))
|
|
||||||
|
|
||||||
managed_pids = read_managed_pids(args.port)
|
managed_pids = read_managed_pids(args.port)
|
||||||
|
|
||||||
|
|||||||
@@ -2432,6 +2432,11 @@ export class SettingsManager {
|
|||||||
|| ['vae', 'upscaler', 'text_encoder']
|
|| ['vae', 'upscaler', 'text_encoder']
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const masterToggle = document.getElementById('enableOtherModels');
|
||||||
|
if (masterToggle) {
|
||||||
|
masterToggle.checked = enableOtherModels;
|
||||||
|
}
|
||||||
|
|
||||||
document.querySelectorAll('[data-other-subtype-toggle]').forEach((input) => {
|
document.querySelectorAll('[data-other-subtype-toggle]').forEach((input) => {
|
||||||
input.checked = enabledSubTypes.has(input.value);
|
input.checked = enabledSubTypes.has(input.value);
|
||||||
input.disabled = !enableOtherModels;
|
input.disabled = !enableOtherModels;
|
||||||
|
|||||||
@@ -7,8 +7,10 @@ import { enableOtherModels, openOtherModelsSettings } from './utils/otherModels.
|
|||||||
* empty state whose button turns the feature on; the backend then rebuilds the
|
* empty state whose button turns the feature on; the backend then rebuilds the
|
||||||
* other-model roots and starts scanning, so a reload lands on the real page.
|
* other-model roots and starts scanning, so a reload lands on the real page.
|
||||||
*
|
*
|
||||||
* The same module backs the "enabled but no folders found" state, where the
|
* The same module backs the "enabled but no folders found" state: ComfyUI
|
||||||
* only useful action is jumping to Settings instead of enabling anything.
|
* mode points to the Settings page's Library section, while standalone mode
|
||||||
|
* (where the settings UI cannot edit primary folder paths) reveals the
|
||||||
|
* settings.json file the user must edit instead.
|
||||||
*/
|
*/
|
||||||
async function handleEnableClick() {
|
async function handleEnableClick() {
|
||||||
const button = document.getElementById('enableOtherModelsBtn');
|
const button = document.getElementById('enableOtherModelsBtn');
|
||||||
@@ -32,6 +34,41 @@ function handleOpenSettingsClick(event) {
|
|||||||
openOtherModelsSettings();
|
openOtherModelsSettings();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Open the settings.json location from the standalone no-folders state.
|
||||||
|
* The settings UI cannot edit primary folder_paths, so the only useful
|
||||||
|
* action is revealing the file itself (or copying its path in Docker).
|
||||||
|
*/
|
||||||
|
async function handleOpenSettingsFolderClick() {
|
||||||
|
const button = document.getElementById('openSettingsFolderBtn');
|
||||||
|
if (!button || button.disabled) return;
|
||||||
|
|
||||||
|
button.disabled = true;
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/lm/settings/open-location', { method: 'POST' });
|
||||||
|
const data = await response.json().catch(() => ({}));
|
||||||
|
if (!response.ok || data.success === false) {
|
||||||
|
throw new Error(data.error || `HTTP ${response.status}`);
|
||||||
|
}
|
||||||
|
if (data.mode === 'clipboard' && data.path) {
|
||||||
|
try {
|
||||||
|
await navigator.clipboard.writeText(data.path);
|
||||||
|
showToast('settings.openSettingsFileLocation.copied', { path: data.path }, 'success');
|
||||||
|
} catch (clipboardError) {
|
||||||
|
console.warn('Clipboard API not available:', clipboardError);
|
||||||
|
showToast('settings.openSettingsFileLocation.clipboardFallback', { path: data.path }, 'info');
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
showToast('settings.openSettingsFileLocation.success', {}, 'success');
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to open settings location:', error);
|
||||||
|
showToast('settings.openSettingsFileLocation.failed', {}, 'error');
|
||||||
|
} finally {
|
||||||
|
button.disabled = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function initializeOtherDisabledPage() {
|
async function initializeOtherDisabledPage() {
|
||||||
// appCore.initialize() wires the shared header (theme, settings modal,
|
// appCore.initialize() wires the shared header (theme, settings modal,
|
||||||
// language) so this page is not a dead end.
|
// language) so this page is not a dead end.
|
||||||
@@ -46,8 +83,13 @@ async function initializeOtherDisabledPage() {
|
|||||||
if (settingsButton) {
|
if (settingsButton) {
|
||||||
settingsButton.addEventListener('click', handleOpenSettingsClick);
|
settingsButton.addEventListener('click', handleOpenSettingsClick);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const settingsFolderButton = document.getElementById('openSettingsFolderBtn');
|
||||||
|
if (settingsFolderButton) {
|
||||||
|
settingsFolderButton.addEventListener('click', handleOpenSettingsFolderClick);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
document.addEventListener('DOMContentLoaded', initializeOtherDisabledPage);
|
document.addEventListener('DOMContentLoaded', initializeOtherDisabledPage);
|
||||||
|
|
||||||
export { handleEnableClick as enableOtherModels, initializeOtherDisabledPage };
|
export { handleEnableClick as enableOtherModels, handleOpenSettingsFolderClick, initializeOtherDisabledPage };
|
||||||
|
|||||||
+20
-1
@@ -63,6 +63,19 @@
|
|||||||
background: rgba(127, 127, 127, 0.15);
|
background: rgba(127, 127, 127, 0.15);
|
||||||
border: 1px solid rgba(127, 127, 127, 0.25);
|
border: 1px solid rgba(127, 127, 127, 0.25);
|
||||||
}
|
}
|
||||||
|
.other-settings-file {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
.other-settings-file code {
|
||||||
|
padding: 4px 8px;
|
||||||
|
border-radius: 4px;
|
||||||
|
background: rgba(127, 127, 127, 0.15);
|
||||||
|
border: 1px solid rgba(127, 127, 127, 0.25);
|
||||||
|
word-break: break-all;
|
||||||
|
}
|
||||||
</style>
|
</style>
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|
||||||
@@ -127,6 +140,9 @@
|
|||||||
<h2>{{ t('other.noPaths.title') }}</h2>
|
<h2>{{ t('other.noPaths.title') }}</h2>
|
||||||
{% if standalone_mode %}
|
{% if standalone_mode %}
|
||||||
<p>{{ t('other.noPaths.descriptionStandalone') }}</p>
|
<p>{{ t('other.noPaths.descriptionStandalone') }}</p>
|
||||||
|
{% if settings_file %}
|
||||||
|
<p class="other-settings-file"><i class="fas fa-file-alt"></i> <code>{{ settings_file }}</code></p>
|
||||||
|
{% endif %}
|
||||||
<pre class="other-no-paths-config"><code>"folder_paths": {
|
<pre class="other-no-paths-config"><code>"folder_paths": {
|
||||||
"vae": ["/path/to/vae"],
|
"vae": ["/path/to/vae"],
|
||||||
"upscale_models": ["/path/to/upscale_models"],
|
"upscale_models": ["/path/to/upscale_models"],
|
||||||
@@ -135,13 +151,16 @@
|
|||||||
"controlnet": ["/path/to/controlnet"]
|
"controlnet": ["/path/to/controlnet"]
|
||||||
}</code></pre>
|
}</code></pre>
|
||||||
<p class="other-disabled-hint">{{ t('other.noPaths.hintStandalone') }}</p>
|
<p class="other-disabled-hint">{{ t('other.noPaths.hintStandalone') }}</p>
|
||||||
|
<button id="openSettingsFolderBtn" type="button">
|
||||||
|
<i class="fas fa-folder-open"></i> {{ t('other.noPaths.openSettingsFolder') }}
|
||||||
|
</button>
|
||||||
{% else %}
|
{% else %}
|
||||||
<p>{{ t('other.noPaths.descriptionComfyUI') }}</p>
|
<p>{{ t('other.noPaths.descriptionComfyUI') }}</p>
|
||||||
<p class="other-disabled-hint">{{ t('other.noPaths.hintComfyUI') }}</p>
|
<p class="other-disabled-hint">{{ t('other.noPaths.hintComfyUI') }}</p>
|
||||||
{% endif %}
|
|
||||||
<button id="openOtherModelsSettingsBtn" type="button">
|
<button id="openOtherModelsSettingsBtn" type="button">
|
||||||
<i class="fas fa-cog"></i> {{ t('other.noPaths.openSettings') }}
|
<i class="fas fa-cog"></i> {{ t('other.noPaths.openSettings') }}
|
||||||
</button>
|
</button>
|
||||||
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
{% else %}
|
{% else %}
|
||||||
<div class="sticky-topbar">
|
<div class="sticky-topbar">
|
||||||
|
|||||||
@@ -333,6 +333,21 @@ def mock_websocket_manager():
|
|||||||
return RecordingWebSocketManager()
|
return RecordingWebSocketManager()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def reset_media_dimension_caches():
|
||||||
|
"""Clear path-keyed dimension caches so files reused across tests re-probe."""
|
||||||
|
from py.utils.exif_utils import _get_image_dimensions_cached
|
||||||
|
from py.utils.video_metadata import _clear_video_dimensions_cache
|
||||||
|
|
||||||
|
_get_image_dimensions_cached.cache_clear()
|
||||||
|
_clear_video_dimensions_cache()
|
||||||
|
|
||||||
|
yield
|
||||||
|
|
||||||
|
_get_image_dimensions_cached.cache_clear()
|
||||||
|
_clear_video_dimensions_cache()
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(autouse=True)
|
@pytest.fixture(autouse=True)
|
||||||
def reset_singletons():
|
def reset_singletons():
|
||||||
"""Reset all singletons before each test to ensure isolation."""
|
"""Reset all singletons before each test to ensure isolation."""
|
||||||
|
|||||||
@@ -665,6 +665,22 @@ describe('SettingsManager other-model root selects', () => {
|
|||||||
expect(container.classList.contains('is-disabled')).toBe(false);
|
expect(container.classList.contains('is-disabled')).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('restores the master toggle checked state from settings', () => {
|
||||||
|
const manager = createManager();
|
||||||
|
const masterToggle = document.createElement('input');
|
||||||
|
masterToggle.type = 'checkbox';
|
||||||
|
masterToggle.id = 'enableOtherModels';
|
||||||
|
document.body.appendChild(masterToggle);
|
||||||
|
|
||||||
|
state.global.settings = { enable_other_models: true };
|
||||||
|
manager.updateOtherModelsControls();
|
||||||
|
expect(masterToggle.checked).toBe(true);
|
||||||
|
|
||||||
|
state.global.settings = { enable_other_models: false };
|
||||||
|
manager.updateOtherModelsControls();
|
||||||
|
expect(masterToggle.checked).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
it('persists the checked sub_types as the whole allow-list', async () => {
|
it('persists the checked sub_types as the whole allow-list', async () => {
|
||||||
const manager = createManager();
|
const manager = createManager();
|
||||||
appendToggles('vae', 'upscaler', 'controlnet');
|
appendToggles('vae', 'upscaler', 'controlnet');
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ describe('Other Models disabled page', () => {
|
|||||||
document.body.innerHTML = [
|
document.body.innerHTML = [
|
||||||
'<button id="enableOtherModelsBtn"></button>',
|
'<button id="enableOtherModelsBtn"></button>',
|
||||||
'<button id="openOtherModelsSettingsBtn"></button>',
|
'<button id="openOtherModelsSettingsBtn"></button>',
|
||||||
|
'<button id="openSettingsFolderBtn"></button>',
|
||||||
].join('');
|
].join('');
|
||||||
|
|
||||||
Object.defineProperty(window, 'location', {
|
Object.defineProperty(window, 'location', {
|
||||||
@@ -64,6 +65,52 @@ describe('Other Models disabled page', () => {
|
|||||||
expect(showModal).toHaveBeenCalledWith('settingsModal');
|
expect(showModal).toHaveBeenCalledWith('settingsModal');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('reveals the settings.json location from the standalone no-folders state', async () => {
|
||||||
|
global.fetch = vi.fn().mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({ success: true, message: 'Opened settings folder' }),
|
||||||
|
});
|
||||||
|
|
||||||
|
const button = document.getElementById('openSettingsFolderBtn');
|
||||||
|
button.dispatchEvent(new MouseEvent('click', { bubbles: true }));
|
||||||
|
await vi.waitFor(() => expect(showToastMock).toHaveBeenCalled());
|
||||||
|
|
||||||
|
expect(global.fetch).toHaveBeenCalledWith(
|
||||||
|
'/api/lm/settings/open-location',
|
||||||
|
expect.objectContaining({ method: 'POST' }),
|
||||||
|
);
|
||||||
|
expect(showToastMock).toHaveBeenCalledWith(
|
||||||
|
'settings.openSettingsFileLocation.success',
|
||||||
|
{},
|
||||||
|
'success',
|
||||||
|
);
|
||||||
|
expect(button.disabled).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('copies the settings path to the clipboard in Docker mode', async () => {
|
||||||
|
const writeText = vi.fn().mockResolvedValue(undefined);
|
||||||
|
Object.defineProperty(navigator, 'clipboard', {
|
||||||
|
value: { writeText },
|
||||||
|
configurable: true,
|
||||||
|
});
|
||||||
|
global.fetch = vi.fn().mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({ success: true, mode: 'clipboard', path: '/data/settings.json' }),
|
||||||
|
});
|
||||||
|
|
||||||
|
document.getElementById('openSettingsFolderBtn').dispatchEvent(
|
||||||
|
new MouseEvent('click', { bubbles: true }),
|
||||||
|
);
|
||||||
|
await vi.waitFor(() => expect(showToastMock).toHaveBeenCalled());
|
||||||
|
|
||||||
|
expect(writeText).toHaveBeenCalledWith('/data/settings.json');
|
||||||
|
expect(showToastMock).toHaveBeenCalledWith(
|
||||||
|
'settings.openSettingsFileLocation.copied',
|
||||||
|
{ path: '/data/settings.json' },
|
||||||
|
'success',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it('enables Other Models through the settings API and reloads', async () => {
|
it('enables Other Models through the settings API and reloads', async () => {
|
||||||
global.fetch = vi.fn().mockResolvedValue({
|
global.fetch = vi.fn().mockResolvedValue({
|
||||||
ok: true,
|
ok: true,
|
||||||
|
|||||||
@@ -532,6 +532,62 @@ async def test_open_backup_location_uses_settings_directory(tmp_path, monkeypatc
|
|||||||
assert calls == [["xdg-open", str(backup_dir)]]
|
assert calls == [["xdg-open", str(backup_dir)]]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_open_settings_location_headless_returns_clipboard_mode(tmp_path, monkeypatch):
|
||||||
|
"""Without a GUI session xdg-open cannot work; the handler must hand the
|
||||||
|
path to the browser instead of reporting a success that never happened."""
|
||||||
|
settings_file = tmp_path / "settings" / "settings.json"
|
||||||
|
settings_file.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
settings_file.write_text("{}", encoding="utf-8")
|
||||||
|
|
||||||
|
handler = FileSystemHandler(settings_service=SimpleNamespace(settings_file=str(settings_file)))
|
||||||
|
|
||||||
|
monkeypatch.delenv("DISPLAY", raising=False)
|
||||||
|
monkeypatch.delenv("WAYLAND_DISPLAY", raising=False)
|
||||||
|
monkeypatch.setattr("py.routes.handlers.misc_handlers._is_docker", lambda: False)
|
||||||
|
monkeypatch.setattr("py.routes.handlers.misc_handlers._is_wsl", lambda: False)
|
||||||
|
|
||||||
|
popen_calls = []
|
||||||
|
monkeypatch.setattr(subprocess, "Popen", lambda *args, **kwargs: popen_calls.append(args))
|
||||||
|
|
||||||
|
response = await handler.open_settings_location(FakeRequest()) # pyright: ignore[reportArgumentType]
|
||||||
|
payload = _json_payload(response)
|
||||||
|
|
||||||
|
assert response.status == 200
|
||||||
|
assert payload["success"] is True
|
||||||
|
assert payload["mode"] == "clipboard"
|
||||||
|
assert payload["path"] == str(settings_file)
|
||||||
|
assert popen_calls == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_open_settings_location_with_display_opens_folder(tmp_path, monkeypatch):
|
||||||
|
settings_file = tmp_path / "settings" / "settings.json"
|
||||||
|
settings_file.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
settings_file.write_text("{}", encoding="utf-8")
|
||||||
|
|
||||||
|
handler = FileSystemHandler(settings_service=SimpleNamespace(settings_file=str(settings_file)))
|
||||||
|
|
||||||
|
monkeypatch.setenv("DISPLAY", ":0")
|
||||||
|
monkeypatch.setattr("py.routes.handlers.misc_handlers._is_docker", lambda: False)
|
||||||
|
monkeypatch.setattr("py.routes.handlers.misc_handlers._is_wsl", lambda: False)
|
||||||
|
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def fake_popen(args):
|
||||||
|
calls.append(args)
|
||||||
|
return MagicMock()
|
||||||
|
|
||||||
|
monkeypatch.setattr(subprocess, "Popen", fake_popen)
|
||||||
|
|
||||||
|
response = await handler.open_settings_location(FakeRequest()) # pyright: ignore[reportArgumentType]
|
||||||
|
payload = _json_payload(response)
|
||||||
|
|
||||||
|
assert response.status == 200
|
||||||
|
assert payload["success"] is True
|
||||||
|
assert calls == [["xdg-open", str(settings_file.parent)]]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_open_wildcards_location_creates_and_opens_directory(tmp_path, monkeypatch):
|
async def test_open_wildcards_location_creates_and_opens_directory(tmp_path, monkeypatch):
|
||||||
wildcards_dir = tmp_path / "settings" / "wildcards"
|
wildcards_dir = tmp_path / "settings" / "wildcards"
|
||||||
@@ -2369,3 +2425,135 @@ async def test_get_init_status_reports_pending_scanners():
|
|||||||
assert "embedding" in payload["details"]
|
assert "embedding" in payload["details"]
|
||||||
assert "recipe" in payload["details"]
|
assert "recipe" in payload["details"]
|
||||||
assert "lora" not in payload["details"]
|
assert "lora" not in payload["details"]
|
||||||
|
|
||||||
|
|
||||||
|
class StaticMetadataProvider:
|
||||||
|
"""Metadata provider returning one fixed CivitAI model payload."""
|
||||||
|
|
||||||
|
def __init__(self, payload):
|
||||||
|
self.payload = payload
|
||||||
|
|
||||||
|
async def get_model_versions(self, _model_id):
|
||||||
|
return self.payload
|
||||||
|
|
||||||
|
async def get_user_models(self, _username, cursor=None):
|
||||||
|
return {"items": [], "nextCursor": None}
|
||||||
|
|
||||||
|
async def get_creator_model_count(self, _username):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _versions_status_handler(payload, *, other_scanner=None):
|
||||||
|
async def metadata_factory():
|
||||||
|
return StaticMetadataProvider(payload)
|
||||||
|
|
||||||
|
async def other_factory():
|
||||||
|
return other_scanner
|
||||||
|
|
||||||
|
return ModelLibraryHandler(
|
||||||
|
ServiceRegistryAdapter(
|
||||||
|
get_lora_scanner=fake_scanner_factory,
|
||||||
|
get_checkpoint_scanner=fake_scanner_factory,
|
||||||
|
get_embedding_scanner=fake_scanner_factory,
|
||||||
|
get_other_scanner=other_factory,
|
||||||
|
get_downloaded_version_history_service=fake_download_history_service_factory,
|
||||||
|
),
|
||||||
|
metadata_provider_factory=metadata_factory,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_get_model_versions_status_unsupported_type_is_read_only():
|
||||||
|
"""A type with no scanner answers 200 with a read-only list + reason."""
|
||||||
|
handler = _versions_status_handler(
|
||||||
|
{
|
||||||
|
"name": "Wildcards pack",
|
||||||
|
"type": "Wildcards",
|
||||||
|
"modelVersions": [
|
||||||
|
{"id": 11, "name": "v1", "images": [{"url": "https://img/1.png"}]},
|
||||||
|
{"id": 12, "name": "v2", "images": []},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
response = await handler.get_model_versions_status(
|
||||||
|
FakeRequest(query={"modelId": "45448"}) # pyright: ignore[reportArgumentType]
|
||||||
|
)
|
||||||
|
payload = _json_payload(response)
|
||||||
|
|
||||||
|
assert response.status == 200
|
||||||
|
assert payload["success"] is True
|
||||||
|
assert payload["supported"] is False
|
||||||
|
assert payload["reason"] == "model_type_unsupported"
|
||||||
|
assert payload["modelType"] == "wildcards"
|
||||||
|
assert payload["versions"] == [
|
||||||
|
{
|
||||||
|
"id": 11,
|
||||||
|
"name": "v1",
|
||||||
|
"thumbnailUrl": "https://img/1.png",
|
||||||
|
"inLibrary": False,
|
||||||
|
"hasBeenDownloaded": False,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 12,
|
||||||
|
"name": "v2",
|
||||||
|
"thumbnailUrl": None,
|
||||||
|
"inLibrary": False,
|
||||||
|
"hasBeenDownloaded": False,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_get_model_versions_status_other_disabled_is_read_only():
|
||||||
|
"""The opt-in gate keeps its own reason instead of the permanent one."""
|
||||||
|
_set_other_models_enabled(False)
|
||||||
|
handler = _versions_status_handler(
|
||||||
|
{
|
||||||
|
"name": "SDXL VAE",
|
||||||
|
"type": "VAE",
|
||||||
|
"modelVersions": [{"id": 333245, "name": "SDXL-VAE", "images": []}],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
response = await handler.get_model_versions_status(
|
||||||
|
FakeRequest(query={"modelId": "296576"}) # pyright: ignore[reportArgumentType]
|
||||||
|
)
|
||||||
|
payload = _json_payload(response)
|
||||||
|
|
||||||
|
assert response.status == 200
|
||||||
|
assert payload["success"] is True
|
||||||
|
assert payload["supported"] is False
|
||||||
|
assert payload["reason"] == "other_models_disabled"
|
||||||
|
assert payload["modelType"] == "vae"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_get_model_versions_status_supported_type_stays_interactive():
|
||||||
|
"""A managed type keeps the existing enriched, fully interactive payload."""
|
||||||
|
handler = _versions_status_handler(
|
||||||
|
{
|
||||||
|
"name": "Some LoRA",
|
||||||
|
"type": "LORA",
|
||||||
|
"modelVersions": [{"id": 1, "name": "v1", "images": []}],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
response = await handler.get_model_versions_status(
|
||||||
|
FakeRequest(query={"modelId": "5"}) # pyright: ignore[reportArgumentType]
|
||||||
|
)
|
||||||
|
payload = _json_payload(response)
|
||||||
|
|
||||||
|
assert response.status == 200
|
||||||
|
assert payload["success"] is True
|
||||||
|
assert payload["supported"] is True
|
||||||
|
assert "reason" not in payload
|
||||||
|
assert payload["versions"] == [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"name": "v1",
|
||||||
|
"thumbnailUrl": None,
|
||||||
|
"inLibrary": False,
|
||||||
|
"hasBeenDownloaded": False,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|||||||
@@ -121,6 +121,25 @@ def test_page_context_reports_feature_state(monkeypatch):
|
|||||||
assert provider(None) == {"other_disabled": True, "other_no_paths": False}
|
assert provider(None) == {"other_disabled": True, "other_no_paths": False}
|
||||||
|
|
||||||
|
|
||||||
|
def test_page_context_exposes_settings_file_in_standalone(monkeypatch):
|
||||||
|
"""Standalone users must edit settings.json by hand; the empty state
|
||||||
|
needs the real file path to point them at."""
|
||||||
|
from py.config import config
|
||||||
|
from py.services.settings_manager import get_settings_manager
|
||||||
|
|
||||||
|
manager = get_settings_manager()
|
||||||
|
handler = OtherRoutes()
|
||||||
|
provider = handler._get_page_context_provider()
|
||||||
|
|
||||||
|
monkeypatch.setattr(config, "other_roots", [], raising=False)
|
||||||
|
monkeypatch.setenv("LORA_MANAGER_STANDALONE", "1")
|
||||||
|
|
||||||
|
context = provider(None)
|
||||||
|
assert context["other_no_paths"] is True
|
||||||
|
assert context["standalone_mode"] is True
|
||||||
|
assert context["settings_file"] == manager.settings_file
|
||||||
|
|
||||||
|
|
||||||
def test_get_expected_model_types_mentions_supported_types():
|
def test_get_expected_model_types_mentions_supported_types():
|
||||||
expected = OtherRoutes()._get_expected_model_types()
|
expected = OtherRoutes()._get_expected_model_types()
|
||||||
for name in ("VAE", "Upscaler", "TextEncoder", "CLIPVision", "Controlnet"):
|
for name in ("VAE", "Upscaler", "TextEncoder", "CLIPVision", "Controlnet"):
|
||||||
|
|||||||
@@ -0,0 +1,105 @@
|
|||||||
|
"""Tests for the portable-mode flag lifecycle (issue #1114 follow-up).
|
||||||
|
|
||||||
|
``LORA_MANAGER_PORTABLE=1`` persists ``use_portable_settings: true`` into the
|
||||||
|
plugin''s own settings.json. That is convenient for repeat runs, but it used to
|
||||||
|
be a one-way trip: the flag made every instance sharing that plugin folder read
|
||||||
|
(and write) the portable settings directory, and the only way back was editing
|
||||||
|
settings.json by hand. ``LORA_MANAGER_PORTABLE=0`` is now the explicit exit.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from py.services import settings_manager as settings_manager_module
|
||||||
|
from py.services.settings_manager import SettingsManager
|
||||||
|
|
||||||
|
|
||||||
|
def _write_settings(path, **extra):
|
||||||
|
payload = {
|
||||||
|
"folder_paths": {"loras": ["/loras"]},
|
||||||
|
}
|
||||||
|
payload.update(extra)
|
||||||
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def isolated_settings_path(tmp_path, monkeypatch):
|
||||||
|
"""Point SettingsManager at a settings.json we control."""
|
||||||
|
settings_path = tmp_path / "settings.json"
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"py.services.settings_manager.ensure_settings_file",
|
||||||
|
lambda logger=None: str(settings_path),
|
||||||
|
)
|
||||||
|
settings_manager_module.reset_settings_manager()
|
||||||
|
yield settings_path
|
||||||
|
settings_manager_module.reset_settings_manager()
|
||||||
|
|
||||||
|
|
||||||
|
def test_portable_env_enables_and_persists_the_flag(
|
||||||
|
isolated_settings_path, monkeypatch
|
||||||
|
):
|
||||||
|
_write_settings(isolated_settings_path)
|
||||||
|
monkeypatch.setenv("LORA_MANAGER_PORTABLE", "1")
|
||||||
|
|
||||||
|
manager = SettingsManager()
|
||||||
|
|
||||||
|
assert manager.get("use_portable_settings") is True
|
||||||
|
persisted = json.loads(isolated_settings_path.read_text(encoding="utf-8"))
|
||||||
|
assert persisted["use_portable_settings"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_explicit_zero_clears_the_persisted_flag(
|
||||||
|
isolated_settings_path, monkeypatch
|
||||||
|
):
|
||||||
|
"""`=0` must undo a previous `=1`, without hand-editing settings.json."""
|
||||||
|
_write_settings(isolated_settings_path, use_portable_settings=True)
|
||||||
|
monkeypatch.setenv("LORA_MANAGER_PORTABLE", "0")
|
||||||
|
|
||||||
|
manager = SettingsManager()
|
||||||
|
|
||||||
|
assert manager.get("use_portable_settings") is False
|
||||||
|
persisted = json.loads(isolated_settings_path.read_text(encoding="utf-8"))
|
||||||
|
# A default value is omitted from disk, so the key is gone entirely.
|
||||||
|
assert persisted.get("use_portable_settings") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_unset_env_keeps_the_persisted_flag(
|
||||||
|
isolated_settings_path, monkeypatch
|
||||||
|
):
|
||||||
|
"""Portable mode must persist across runs when the variable is unset."""
|
||||||
|
_write_settings(isolated_settings_path, use_portable_settings=True)
|
||||||
|
monkeypatch.delenv("LORA_MANAGER_PORTABLE", raising=False)
|
||||||
|
|
||||||
|
manager = SettingsManager()
|
||||||
|
|
||||||
|
assert manager.get("use_portable_settings") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_zero_is_a_noop_when_portable_was_never_enabled(
|
||||||
|
isolated_settings_path, monkeypatch
|
||||||
|
):
|
||||||
|
_write_settings(isolated_settings_path)
|
||||||
|
monkeypatch.setenv("LORA_MANAGER_PORTABLE", "0")
|
||||||
|
|
||||||
|
manager = SettingsManager()
|
||||||
|
|
||||||
|
assert manager.get("use_portable_settings") in (False, None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_pinned_settings_dir_wins_over_portable_env(
|
||||||
|
isolated_settings_path, monkeypatch
|
||||||
|
):
|
||||||
|
"""LORA_MANAGER_SETTINGS_DIR still takes precedence, as documented."""
|
||||||
|
_write_settings(isolated_settings_path)
|
||||||
|
monkeypatch.setenv("LORA_MANAGER_PORTABLE", "1")
|
||||||
|
monkeypatch.setenv("LORA_MANAGER_SETTINGS_DIR", str(isolated_settings_path.parent))
|
||||||
|
|
||||||
|
manager = SettingsManager()
|
||||||
|
|
||||||
|
# The pinned directory already decides the location, so the portable flag
|
||||||
|
# is deliberately left alone.
|
||||||
|
assert not manager.get("use_portable_settings")
|
||||||
@@ -0,0 +1,300 @@
|
|||||||
|
"""Regression tests for the recipe empty-prune guard (issue #1116).
|
||||||
|
|
||||||
|
A scan that finds no recipe files at all is not a trustworthy deletion signal:
|
||||||
|
an unmounted drive, a ``recipes_path`` that silently fell back to another LoRA
|
||||||
|
root, or a cache shared with a second instance all look identical to a real
|
||||||
|
wipe. Before this guard, such a scan overwrote the persistent cache with an
|
||||||
|
empty one, destroying the user's only record of their recipes.
|
||||||
|
|
||||||
|
Covered contracts:
|
||||||
|
|
||||||
|
1. ``_reconcile_recipe_cache`` reports the "every persisted file vanished"
|
||||||
|
condition and does not treat an empty directory as a trustworthy prune.
|
||||||
|
2. ``_initialize_recipe_cache_sync`` keeps the stored cache in that case
|
||||||
|
instead of persisting the empty result.
|
||||||
|
3. A partial orphan (some files still present) still prunes normally, so
|
||||||
|
ordinary manual deletions keep working.
|
||||||
|
4. ``PersistentRecipeCache.save_cache(skip_if_empty=True)`` is the
|
||||||
|
storage-level backstop and a manual rebuild can still clear the cache.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from py.config import config
|
||||||
|
from py.services import recipe_scanner as recipe_scanner_module
|
||||||
|
from py.services import settings_manager as settings_manager_module
|
||||||
|
from py.services.persistent_recipe_cache import (
|
||||||
|
PersistedRecipeData,
|
||||||
|
PersistentRecipeCache,
|
||||||
|
)
|
||||||
|
from py.services.recipe_cache import RecipeCache
|
||||||
|
from py.services.recipe_scanner import RecipeScanner
|
||||||
|
|
||||||
|
|
||||||
|
def _write_recipe_json(path: Path, recipe_id: str) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"id": recipe_id,
|
||||||
|
"file_path": str(path.with_suffix(".png")),
|
||||||
|
"title": f"Recipe {recipe_id}",
|
||||||
|
"modified": 0.0,
|
||||||
|
"created_date": 0.0,
|
||||||
|
"loras": [],
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _persisted_for(paths: list[Path]) -> PersistedRecipeData:
|
||||||
|
"""Build persisted cache state describing *paths* as known recipe files."""
|
||||||
|
raw_data = []
|
||||||
|
file_stats = {}
|
||||||
|
for path in paths:
|
||||||
|
recipe_id = path.name[: -len(".recipe.json")]
|
||||||
|
raw_data.append({"id": recipe_id, "title": f"Recipe {recipe_id}"})
|
||||||
|
stat = path.stat()
|
||||||
|
file_stats[str(path)] = (stat.st_mtime, stat.st_size)
|
||||||
|
return PersistedRecipeData(
|
||||||
|
raw_data=raw_data, file_stats=file_stats, image_id_map={}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def guard_scanner(tmp_path: Path, monkeypatch):
|
||||||
|
"""RecipeScanner wired to a real persistent cache, without a ComfyUI app."""
|
||||||
|
RecipeScanner._instance = None
|
||||||
|
settings_manager_module.reset_settings_manager()
|
||||||
|
monkeypatch.setattr(config, "loras_roots", [str(tmp_path / "loras-root")])
|
||||||
|
|
||||||
|
scanner = RecipeScanner.__new__(RecipeScanner)
|
||||||
|
scanner._persistent_cache = PersistentRecipeCache(
|
||||||
|
db_path=str(tmp_path / "recipe_cache.sqlite")
|
||||||
|
)
|
||||||
|
scanner._cache = None
|
||||||
|
scanner._json_path_map = {}
|
||||||
|
scanner._lora_scanner = SimpleNamespace()
|
||||||
|
|
||||||
|
yield scanner, scanner._persistent_cache
|
||||||
|
|
||||||
|
RecipeScanner._instance = None
|
||||||
|
settings_manager_module.reset_settings_manager()
|
||||||
|
|
||||||
|
|
||||||
|
def test_reconcile_flags_prune_when_every_persisted_file_is_gone(
|
||||||
|
guard_scanner, tmp_path: Path
|
||||||
|
):
|
||||||
|
"""An empty recipes dir must not be reported as a trustworthy prune."""
|
||||||
|
scanner, _cache = guard_scanner
|
||||||
|
recipes_dir = tmp_path / "recipes"
|
||||||
|
recipes_dir.mkdir()
|
||||||
|
|
||||||
|
# The files used to live at another root (a changed recipes_path) and are
|
||||||
|
# all gone from the directory the scanner resolved this time.
|
||||||
|
old_files = [tmp_path / "elsewhere" / f"r{idx}.recipe.json" for idx in range(3)]
|
||||||
|
for path in old_files:
|
||||||
|
_write_recipe_json(path, path.name[: -len(".recipe.json")])
|
||||||
|
persisted = _persisted_for(old_files)
|
||||||
|
for path in old_files:
|
||||||
|
path.unlink()
|
||||||
|
|
||||||
|
recipes, changed, json_paths, skipped_prune_reason = (
|
||||||
|
scanner._reconcile_recipe_cache(persisted, str(recipes_dir))
|
||||||
|
)
|
||||||
|
|
||||||
|
assert recipes == []
|
||||||
|
assert json_paths == {}
|
||||||
|
assert changed is True
|
||||||
|
assert skipped_prune_reason is not None
|
||||||
|
assert str(recipes_dir) in skipped_prune_reason
|
||||||
|
assert "3" in skipped_prune_reason
|
||||||
|
|
||||||
|
|
||||||
|
def test_reconcile_prunes_normally_when_only_some_files_disappear(
|
||||||
|
guard_scanner, tmp_path: Path
|
||||||
|
):
|
||||||
|
"""A partial orphan is an ordinary deletion and keeps its old behaviour."""
|
||||||
|
scanner, _cache = guard_scanner
|
||||||
|
recipes_dir = tmp_path / "recipes"
|
||||||
|
|
||||||
|
survivor = recipes_dir / "survivor.recipe.json"
|
||||||
|
_write_recipe_json(survivor, "survivor")
|
||||||
|
vanished = recipes_dir / "vanished.recipe.json"
|
||||||
|
_write_recipe_json(vanished, "vanished")
|
||||||
|
persisted = _persisted_for([survivor, vanished])
|
||||||
|
vanished.unlink()
|
||||||
|
|
||||||
|
recipes, changed, _json_paths, skipped_prune_reason = (
|
||||||
|
scanner._reconcile_recipe_cache(persisted, str(recipes_dir))
|
||||||
|
)
|
||||||
|
|
||||||
|
assert skipped_prune_reason is None
|
||||||
|
assert changed is True
|
||||||
|
assert [recipe["id"] for recipe in recipes] == ["survivor"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_reconcile_ignores_empty_persisted_cache(guard_scanner, tmp_path: Path):
|
||||||
|
"""A genuinely empty cache has nothing to lose and must not be guarded."""
|
||||||
|
scanner, _cache = guard_scanner
|
||||||
|
recipes_dir = tmp_path / "recipes"
|
||||||
|
recipes_dir.mkdir()
|
||||||
|
|
||||||
|
persisted = PersistedRecipeData(raw_data=[], file_stats={}, image_id_map={})
|
||||||
|
|
||||||
|
_recipes, changed, _json_paths, skipped_prune_reason = (
|
||||||
|
scanner._reconcile_recipe_cache(persisted, str(recipes_dir))
|
||||||
|
)
|
||||||
|
|
||||||
|
assert changed is False
|
||||||
|
assert skipped_prune_reason is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_reconcile_prunes_when_stored_metadata_is_inconsistent(
|
||||||
|
guard_scanner, tmp_path: Path
|
||||||
|
):
|
||||||
|
"""A stale row set must not masquerade as a fresh mass disappearance.
|
||||||
|
|
||||||
|
Leftover rows (rows without a recorded file stat) mean the stored cache is
|
||||||
|
already out of date; guarding them would preserve orphans forever.
|
||||||
|
"""
|
||||||
|
scanner, _cache = guard_scanner
|
||||||
|
recipes_dir = tmp_path / "recipes"
|
||||||
|
recipes_dir.mkdir()
|
||||||
|
|
||||||
|
gone = tmp_path / "old-location" / "kept.recipe.json"
|
||||||
|
_write_recipe_json(gone, "kept")
|
||||||
|
persisted = _persisted_for([gone])
|
||||||
|
gone.unlink()
|
||||||
|
# A row with no matching file record: the cache diverged at some point.
|
||||||
|
persisted.raw_data.append({"id": "orphan-row", "title": "Orphan"})
|
||||||
|
|
||||||
|
recipes, changed, _json_paths, skipped_prune_reason = (
|
||||||
|
scanner._reconcile_recipe_cache(persisted, str(recipes_dir))
|
||||||
|
)
|
||||||
|
|
||||||
|
assert recipes == []
|
||||||
|
assert changed is True
|
||||||
|
assert skipped_prune_reason is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_sync_init_keeps_stored_cache_when_scan_finds_nothing(
|
||||||
|
guard_scanner, tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||||||
|
):
|
||||||
|
"""The startup path must not overwrite the stored cache with an empty one."""
|
||||||
|
scanner, cache = guard_scanner
|
||||||
|
recipes_dir = Path(config.loras_roots[0]) / "recipes"
|
||||||
|
gone = tmp_path / "old-location" / "kept.recipe.json"
|
||||||
|
_write_recipe_json(gone, "kept")
|
||||||
|
|
||||||
|
assert cache.save_cache(
|
||||||
|
[{"id": "kept", "title": "Recipe kept"}], {"kept": str(gone)}
|
||||||
|
)
|
||||||
|
gone.unlink()
|
||||||
|
|
||||||
|
with caplog.at_level(logging.WARNING, logger=recipe_scanner_module.__name__):
|
||||||
|
scanner._initialize_recipe_cache_sync()
|
||||||
|
|
||||||
|
assert "Recipe cache prune skipped" in caplog.text
|
||||||
|
assert scanner._prune_skipped is True
|
||||||
|
# The stored cache survived, so the recipes remain recoverable.
|
||||||
|
persisted = cache.load_cache()
|
||||||
|
assert persisted is not None
|
||||||
|
assert [recipe["id"] for recipe in persisted.raw_data] == ["kept"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_skipped_prune_leaves_fts_index_untouched(guard_scanner, tmp_path: Path):
|
||||||
|
"""A skipped prune must not rebuild the FTS index from the empty view."""
|
||||||
|
scanner, cache = guard_scanner
|
||||||
|
gone = tmp_path / "old-location" / "kept.recipe.json"
|
||||||
|
_write_recipe_json(gone, "kept")
|
||||||
|
assert cache.save_cache(
|
||||||
|
[{"id": "kept", "title": "Recipe kept"}], {"kept": str(gone)}
|
||||||
|
)
|
||||||
|
gone.unlink()
|
||||||
|
|
||||||
|
schedule_calls = []
|
||||||
|
scanner._schedule_fts_index_build = lambda: schedule_calls.append(True)
|
||||||
|
|
||||||
|
scanner._initialize_recipe_cache_sync()
|
||||||
|
|
||||||
|
assert scanner._prune_skipped is True
|
||||||
|
assert schedule_calls == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_sync_init_persists_when_recipes_are_found(guard_scanner, tmp_path: Path):
|
||||||
|
"""The guard must not block a normal successful scan."""
|
||||||
|
scanner, cache = guard_scanner
|
||||||
|
recipes_dir = Path(config.loras_roots[0]) / "recipes"
|
||||||
|
_write_recipe_json(recipes_dir / "fresh.recipe.json", "fresh")
|
||||||
|
|
||||||
|
scanner._initialize_recipe_cache_sync()
|
||||||
|
|
||||||
|
persisted = cache.load_cache()
|
||||||
|
assert persisted is not None
|
||||||
|
assert [recipe["id"] for recipe in persisted.raw_data] == ["fresh"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_force_refresh_scan_persists_an_empty_result(guard_scanner, tmp_path: Path):
|
||||||
|
"""A manual rebuild stays the escape hatch from a skipped prune.
|
||||||
|
|
||||||
|
The startup guard deliberately keeps a stale cache, which leaves the in-memory
|
||||||
|
view empty until the files come back. An explicit rebuild must be able to land
|
||||||
|
on the real (empty) filesystem state instead, otherwise there is no way out.
|
||||||
|
The route to it is `refresh_cache(force=True)`, which clears the stored cache
|
||||||
|
first and then does a full directory scan.
|
||||||
|
"""
|
||||||
|
scanner, cache = guard_scanner
|
||||||
|
gone = tmp_path / "old-location" / "kept.recipe.json"
|
||||||
|
_write_recipe_json(gone, "kept")
|
||||||
|
assert cache.save_cache(
|
||||||
|
[{"id": "kept", "title": "Recipe kept"}], {"kept": str(gone)}
|
||||||
|
)
|
||||||
|
gone.unlink()
|
||||||
|
|
||||||
|
# Simulate the explicit rebuild: clear the stored cache, then full scan.
|
||||||
|
assert cache.save_cache([], {}) is True
|
||||||
|
scanner._initialize_recipe_cache_sync()
|
||||||
|
|
||||||
|
assert scanner._prune_skipped is False
|
||||||
|
persisted = cache.load_cache()
|
||||||
|
assert persisted is None or persisted.raw_data == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_cache_skip_if_empty_preserves_existing_rows(tmp_path: Path):
|
||||||
|
"""The storage-level backstop refuses to empty a populated cache."""
|
||||||
|
cache = PersistentRecipeCache(db_path=str(tmp_path / "recipe_cache.sqlite"))
|
||||||
|
assert cache.save_cache([{"id": "r1", "title": "One"}], {"r1": "/tmp/r1.json"})
|
||||||
|
|
||||||
|
written = cache.save_cache([], {}, skip_if_empty=True)
|
||||||
|
|
||||||
|
assert written is False
|
||||||
|
persisted = cache.load_cache()
|
||||||
|
assert persisted is not None
|
||||||
|
assert [recipe["id"] for recipe in persisted.raw_data] == ["r1"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_cache_skip_if_empty_allows_clearing_an_empty_cache(tmp_path: Path):
|
||||||
|
"""Nothing to protect: an already-empty cache still returns success."""
|
||||||
|
cache = PersistentRecipeCache(db_path=str(tmp_path / "recipe_cache.sqlite"))
|
||||||
|
|
||||||
|
assert cache.save_cache([], {}, skip_if_empty=True) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_cache_default_still_allows_intentional_full_clear(tmp_path: Path):
|
||||||
|
"""A manual rebuild passes skip_if_empty=False and must clear the cache."""
|
||||||
|
cache = PersistentRecipeCache(db_path=str(tmp_path / "recipe_cache.sqlite"))
|
||||||
|
assert cache.save_cache([{"id": "r1", "title": "One"}], {"r1": "/tmp/r1.json"})
|
||||||
|
|
||||||
|
assert cache.save_cache([], {}) is True
|
||||||
|
|
||||||
|
persisted = cache.load_cache()
|
||||||
|
assert persisted is None or persisted.raw_data == []
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
"""Tests for the shared cache SQLite connection settings (:mod:`py.utils.cache_db`).
|
||||||
|
|
||||||
|
Two LoRA Manager processes can share one settings directory, so cache
|
||||||
|
connections must tolerate a competing writer instead of failing immediately
|
||||||
|
with "database is locked".
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlite3
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
from py.utils.cache_db import CONCURRENT_TIMEOUT_SECONDS, connect_cache_db
|
||||||
|
|
||||||
|
|
||||||
|
def test_busy_timeout_pragma_is_applied(tmp_path):
|
||||||
|
"""The connection must retry inside SQLite, not just at connect() time."""
|
||||||
|
conn = connect_cache_db(str(tmp_path / "cache.sqlite"))
|
||||||
|
try:
|
||||||
|
value = conn.execute("PRAGMA busy_timeout").fetchone()[0]
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
assert value == int(CONCURRENT_TIMEOUT_SECONDS * 1000)
|
||||||
|
|
||||||
|
|
||||||
|
def test_waiting_writer_succeeds_after_competing_writer_commits(tmp_path):
|
||||||
|
"""A blocked writer waits for the lock instead of raising."""
|
||||||
|
db_path = str(tmp_path / "cache.sqlite")
|
||||||
|
|
||||||
|
holder = connect_cache_db(db_path)
|
||||||
|
holder.execute("CREATE TABLE t (v INTEGER)")
|
||||||
|
holder.commit()
|
||||||
|
holder.execute("BEGIN IMMEDIATE")
|
||||||
|
|
||||||
|
def release_after_delay() -> None:
|
||||||
|
time.sleep(0.5)
|
||||||
|
holder.commit()
|
||||||
|
|
||||||
|
releaser = threading.Thread(target=release_after_delay)
|
||||||
|
releaser.start()
|
||||||
|
try:
|
||||||
|
waiter = connect_cache_db(db_path)
|
||||||
|
try:
|
||||||
|
# Under the old 5s default this still worked, but an immediate
|
||||||
|
# failure is what low-timeout connections produced; assert the
|
||||||
|
# write lands rather than propagating "database is locked".
|
||||||
|
waiter.execute("INSERT INTO t VALUES (1)")
|
||||||
|
waiter.commit()
|
||||||
|
finally:
|
||||||
|
waiter.close()
|
||||||
|
finally:
|
||||||
|
releaser.join()
|
||||||
|
holder.close()
|
||||||
|
|
||||||
|
check = connect_cache_db(db_path)
|
||||||
|
try:
|
||||||
|
assert check.execute("SELECT COUNT(*) FROM t").fetchone()[0] == 1
|
||||||
|
finally:
|
||||||
|
check.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_readwrite_connection_uses_row_factory(tmp_path):
|
||||||
|
conn = connect_cache_db(str(tmp_path / "cache.sqlite"), row_factory=sqlite3.Row)
|
||||||
|
try:
|
||||||
|
conn.execute("CREATE TABLE t (v INTEGER)")
|
||||||
|
conn.execute("INSERT INTO t VALUES (7)")
|
||||||
|
conn.commit()
|
||||||
|
row = conn.execute("SELECT v FROM t").fetchone()
|
||||||
|
assert row["v"] == 7
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_readonly_connection_reads_without_writing(tmp_path):
|
||||||
|
db_path = str(tmp_path / "cache.sqlite")
|
||||||
|
writer = connect_cache_db(db_path)
|
||||||
|
writer.execute("CREATE TABLE t (v INTEGER)")
|
||||||
|
writer.execute("INSERT INTO t VALUES (1)")
|
||||||
|
writer.commit()
|
||||||
|
writer.close()
|
||||||
|
|
||||||
|
conn = connect_cache_db(db_path, readonly=True)
|
||||||
|
try:
|
||||||
|
assert conn.execute("SELECT v FROM t").fetchone()[0] == 1
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_readonly_connection_rejects_writes(tmp_path):
|
||||||
|
db_path = str(tmp_path / "cache.sqlite")
|
||||||
|
writer = connect_cache_db(db_path)
|
||||||
|
writer.execute("CREATE TABLE t (v INTEGER)")
|
||||||
|
writer.commit()
|
||||||
|
writer.close()
|
||||||
|
|
||||||
|
conn = connect_cache_db(db_path, readonly=True)
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
conn.execute("INSERT INTO t VALUES (1)")
|
||||||
|
conn.commit()
|
||||||
|
except sqlite3.OperationalError:
|
||||||
|
pass
|
||||||
|
else: # pragma: no cover - would mean mode=ro was not applied
|
||||||
|
raise AssertionError("read-only connection accepted a write")
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
@@ -9,6 +9,7 @@ from typing import Any, Dict, List, Tuple
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from py.utils import example_images_metadata as metadata_module
|
from py.utils import example_images_metadata as metadata_module
|
||||||
|
from tests.utils.test_video_dimension_probe import build_mp4, build_webm
|
||||||
|
|
||||||
|
|
||||||
class StubScanner:
|
class StubScanner:
|
||||||
@@ -217,3 +218,127 @@ async def test_update_metadata_from_local_examples_generates_entries(monkeypatch
|
|||||||
)
|
)
|
||||||
assert success is True
|
assert success is True
|
||||||
assert model_data["civitai"]["images"]
|
assert model_data["civitai"]["images"]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_update_metadata_after_import_uses_real_video_dimensions(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path, patch_metadata_manager
|
||||||
|
):
|
||||||
|
"""Regression: imported videos must not fall back to the 720x1280 default.
|
||||||
|
|
||||||
|
See issue #1115 — landscape videos were stored as portrait, so the showcase
|
||||||
|
viewer letterboxed them into a 9:16 container.
|
||||||
|
"""
|
||||||
|
|
||||||
|
model_hash = "d" * 64
|
||||||
|
model_file = tmp_path / "video-model.safetensors"
|
||||||
|
model_file.write_text("content", encoding="utf-8")
|
||||||
|
model_data = {
|
||||||
|
"model_name": "VideoExample",
|
||||||
|
"file_path": str(model_file),
|
||||||
|
"civitai": {},
|
||||||
|
}
|
||||||
|
scanner = StubScanner([model_data])
|
||||||
|
|
||||||
|
video_path = tmp_path / "custom_abc.mp4"
|
||||||
|
video_path.write_bytes(build_mp4(1280, 720))
|
||||||
|
|
||||||
|
monkeypatch.setattr(metadata_module.ExifUtils, "extract_image_metadata", staticmethod(lambda _path: None))
|
||||||
|
|
||||||
|
_regular, custom = await metadata_module.MetadataUpdater.update_metadata_after_import(
|
||||||
|
model_hash,
|
||||||
|
model_data,
|
||||||
|
scanner,
|
||||||
|
[(str(video_path), "abc")],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert custom[0]["type"] == "video"
|
||||||
|
assert (custom[0]["width"], custom[0]["height"]) == (1280, 720)
|
||||||
|
assert patch_metadata_manager[-1][1]["civitai"]["customImages"][0]["width"] == 1280
|
||||||
|
|
||||||
|
|
||||||
|
async def test_update_metadata_after_import_uses_real_webm_dimensions(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path, patch_metadata_manager
|
||||||
|
):
|
||||||
|
model_hash = "e" * 64
|
||||||
|
model_file = tmp_path / "webm-model.safetensors"
|
||||||
|
model_file.write_text("content", encoding="utf-8")
|
||||||
|
model_data = {
|
||||||
|
"model_name": "WebmExample",
|
||||||
|
"file_path": str(model_file),
|
||||||
|
"civitai": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
video_path = tmp_path / "custom_def.webm"
|
||||||
|
video_path.write_bytes(build_webm(480, 832))
|
||||||
|
|
||||||
|
monkeypatch.setattr(metadata_module.ExifUtils, "extract_image_metadata", staticmethod(lambda _path: None))
|
||||||
|
|
||||||
|
_regular, custom = await metadata_module.MetadataUpdater.update_metadata_after_import(
|
||||||
|
model_hash,
|
||||||
|
model_data,
|
||||||
|
StubScanner([model_data]),
|
||||||
|
[(str(video_path), "def")],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (custom[0]["width"], custom[0]["height"]) == (480, 832)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_update_metadata_after_import_falls_back_for_unreadable_video(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path, patch_metadata_manager
|
||||||
|
):
|
||||||
|
"""An unparsable video keeps the legacy placeholder rather than failing."""
|
||||||
|
|
||||||
|
model_hash = "f" * 64
|
||||||
|
model_file = tmp_path / "broken-model.safetensors"
|
||||||
|
model_file.write_text("content", encoding="utf-8")
|
||||||
|
model_data = {
|
||||||
|
"model_name": "BrokenExample",
|
||||||
|
"file_path": str(model_file),
|
||||||
|
"civitai": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
video_path = tmp_path / "custom_ghi.mp4"
|
||||||
|
video_path.write_bytes(b"\x00\x00\x00\x20ftypisom" + b"\xff" * 32)
|
||||||
|
|
||||||
|
monkeypatch.setattr(metadata_module.ExifUtils, "extract_image_metadata", staticmethod(lambda _path: None))
|
||||||
|
|
||||||
|
_regular, custom = await metadata_module.MetadataUpdater.update_metadata_after_import(
|
||||||
|
model_hash,
|
||||||
|
model_data,
|
||||||
|
StubScanner([model_data]),
|
||||||
|
[(str(video_path), "ghi")],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (custom[0]["width"], custom[0]["height"]) == (720, 1280)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_update_metadata_from_local_examples_uses_real_video_dimensions(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path
|
||||||
|
):
|
||||||
|
model_hash = "1" * 64
|
||||||
|
model_dir = tmp_path / model_hash
|
||||||
|
model_dir.mkdir()
|
||||||
|
(model_dir / "clip.mp4").write_bytes(build_mp4(1920, 1080))
|
||||||
|
model_data: Dict[str, Any] = {
|
||||||
|
"model_name": "LocalVideo",
|
||||||
|
"civitai": {},
|
||||||
|
"file_path": str(tmp_path / "model.safetensors"),
|
||||||
|
}
|
||||||
|
|
||||||
|
async def fake_save(path, metadata):
|
||||||
|
return True
|
||||||
|
|
||||||
|
monkeypatch.setattr(metadata_module.MetadataManager, "save_metadata", staticmethod(fake_save))
|
||||||
|
|
||||||
|
success = await metadata_module.MetadataUpdater.update_metadata_from_local_examples(
|
||||||
|
model_hash,
|
||||||
|
model_data,
|
||||||
|
"lora",
|
||||||
|
StubScanner([model_data]),
|
||||||
|
str(model_dir),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert success is True
|
||||||
|
entry = model_data["civitai"]["images"][0]
|
||||||
|
assert entry["type"] == "video"
|
||||||
|
assert (entry["width"], entry["height"]) == (1920, 1080)
|
||||||
@@ -177,3 +177,156 @@ async def test_migrations_run_and_update_progress(tmp_path, monkeypatch):
|
|||||||
update_args = lora_scanner.update_calls[0]
|
update_args = lora_scanner.update_calls[0]
|
||||||
assert update_args[0] == str(metadata_path)
|
assert update_args[0] == str(metadata_path)
|
||||||
assert update_args[2]["civitai"]["customImages"][0]["id"] == "short1234"
|
assert update_args[2]["civitai"]["customImages"][0]["id"] == "short1234"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_v2_to_v3_migration_repairs_video_dimensions(tmp_path, monkeypatch):
|
||||||
|
"""Upgrading a library already at v2 backfills local video dimensions once.
|
||||||
|
|
||||||
|
This mirrors the real upgrade path for issue #1115: the naming migration is
|
||||||
|
already done, but imported videos still carry the 720x1280 placeholder.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from tests.utils.test_video_dimension_probe import build_mp4
|
||||||
|
|
||||||
|
example_root = tmp_path / "example_images"
|
||||||
|
library_root = example_root / "main"
|
||||||
|
library_root.mkdir(parents=True)
|
||||||
|
|
||||||
|
progress_path = library_root / ".download_progress.json"
|
||||||
|
progress_path.write_text(json.dumps({"naming_version": 2}))
|
||||||
|
|
||||||
|
model_hash = "d" * 64
|
||||||
|
model_folder = library_root / model_hash
|
||||||
|
model_folder.mkdir()
|
||||||
|
# Landscape clip stored during the buggy import path.
|
||||||
|
(model_folder / "custom_land1.mp4").write_bytes(build_mp4(1280, 720))
|
||||||
|
|
||||||
|
model_file = tmp_path / "models" / "video.safetensors"
|
||||||
|
model_file.parent.mkdir()
|
||||||
|
model_file.write_text("weights", encoding="utf-8")
|
||||||
|
|
||||||
|
scanner = FakeScanner(
|
||||||
|
{
|
||||||
|
model_hash: {
|
||||||
|
"sha256": model_hash,
|
||||||
|
"file_path": str(model_file),
|
||||||
|
"civitai": {
|
||||||
|
"images": [
|
||||||
|
{"url": "https://example.com/remote.jpg", "type": "image", "width": 512, "height": 512}
|
||||||
|
],
|
||||||
|
"customImages": [
|
||||||
|
{"url": "", "id": "land1", "type": "video", "width": 720, "height": 1280}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
async def fake_get_lora_scanner(cls):
|
||||||
|
return scanner
|
||||||
|
|
||||||
|
async def fake_get_checkpoint_scanner(cls):
|
||||||
|
return FakeScanner({})
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
migration_module.ServiceRegistry, "get_lora_scanner", classmethod(fake_get_lora_scanner)
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
migration_module.ServiceRegistry,
|
||||||
|
"get_checkpoint_scanner",
|
||||||
|
classmethod(fake_get_checkpoint_scanner),
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
migration_module.settings,
|
||||||
|
"get",
|
||||||
|
lambda key, default=None: str(example_root) if key == "example_images_path" else default,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
migration_module,
|
||||||
|
"iter_library_roots",
|
||||||
|
lambda: [("main", str(library_root))],
|
||||||
|
)
|
||||||
|
|
||||||
|
saved_metadata = []
|
||||||
|
|
||||||
|
async def fake_save_metadata(path, metadata):
|
||||||
|
saved_metadata.append((path, metadata))
|
||||||
|
return True
|
||||||
|
|
||||||
|
async def fake_load_payload(path):
|
||||||
|
return {
|
||||||
|
"model_name": "Video",
|
||||||
|
"civitai": {
|
||||||
|
"images": [
|
||||||
|
{"url": "https://example.com/remote.jpg", "type": "image", "width": 512, "height": 512}
|
||||||
|
],
|
||||||
|
"customImages": [
|
||||||
|
{"url": "", "id": "land1", "type": "video", "width": 720, "height": 1280}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
migration_module.MetadataManager, "save_metadata", staticmethod(fake_save_metadata)
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
migration_module.MetadataManager, "load_metadata_payload", staticmethod(fake_load_payload)
|
||||||
|
)
|
||||||
|
|
||||||
|
scheduled = []
|
||||||
|
original_create_task = asyncio.create_task
|
||||||
|
|
||||||
|
def capture_create_task(coro, *args, **kwargs):
|
||||||
|
task = original_create_task(coro, *args, **kwargs)
|
||||||
|
scheduled.append(task)
|
||||||
|
return task
|
||||||
|
|
||||||
|
monkeypatch.setattr(migration_module.asyncio, "create_task", capture_create_task)
|
||||||
|
|
||||||
|
await migration_module.ExampleImagesMigration.check_and_run_migrations()
|
||||||
|
await asyncio.gather(*scheduled)
|
||||||
|
|
||||||
|
assert len(saved_metadata) == 1
|
||||||
|
_path, payload = saved_metadata[0]
|
||||||
|
entry = payload["civitai"]["customImages"][0]
|
||||||
|
assert (entry["width"], entry["height"]) == (1280, 720)
|
||||||
|
# Remote-backed entry is untouched.
|
||||||
|
assert payload["civitai"]["images"][0]["width"] == 512
|
||||||
|
|
||||||
|
assert json.loads(progress_path.read_text())["naming_version"] == 3
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_v3_migration_does_not_run_twice(tmp_path, monkeypatch):
|
||||||
|
"""The version gate keeps the repair off the startup path after one run."""
|
||||||
|
|
||||||
|
example_root = tmp_path / "example_images"
|
||||||
|
library_root = example_root / "main"
|
||||||
|
library_root.mkdir(parents=True)
|
||||||
|
(library_root / ".download_progress.json").write_text(json.dumps({"naming_version": 3}))
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
migration_module.settings,
|
||||||
|
"get",
|
||||||
|
lambda key, default=None: str(example_root) if key == "example_images_path" else default,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
migration_module,
|
||||||
|
"iter_library_roots",
|
||||||
|
lambda: [("main", str(library_root))],
|
||||||
|
)
|
||||||
|
|
||||||
|
called = []
|
||||||
|
|
||||||
|
async def spy_run_migrations(*args, **kwargs):
|
||||||
|
called.append(args)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
migration_module.ExampleImagesMigration, "run_migrations", staticmethod(spy_run_migrations)
|
||||||
|
)
|
||||||
|
|
||||||
|
await migration_module.ExampleImagesMigration.check_and_run_migrations()
|
||||||
|
|
||||||
|
assert called == []
|
||||||
|
|||||||
@@ -0,0 +1,299 @@
|
|||||||
|
"""Tests for the one-shot repair of locally imported video dimensions (issue #1115)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from py.utils import example_images_migration as migration_module
|
||||||
|
from py.utils import example_images_metadata as metadata_module
|
||||||
|
from tests.utils.test_video_dimension_probe import build_mp4
|
||||||
|
|
||||||
|
|
||||||
|
def _metadata_payload(**civitai: Any) -> Dict[str, Any]:
|
||||||
|
return {"model_name": "Example", "civitai": civitai}
|
||||||
|
|
||||||
|
|
||||||
|
def test_repair_backfills_landscape_video_dimensions(tmp_path: Path):
|
||||||
|
video = tmp_path / "custom_abc123.mp4"
|
||||||
|
video.write_bytes(build_mp4(1280, 720))
|
||||||
|
|
||||||
|
payload = _metadata_payload(
|
||||||
|
customImages=[
|
||||||
|
{
|
||||||
|
"url": "",
|
||||||
|
"id": "abc123",
|
||||||
|
"type": "video",
|
||||||
|
"width": 720,
|
||||||
|
"height": 1280,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
repaired = metadata_module.repair_local_video_dimensions(
|
||||||
|
payload, {"abc123": str(video)}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert repaired == 1
|
||||||
|
entry = payload["civitai"]["customImages"][0]
|
||||||
|
assert (entry["width"], entry["height"]) == (1280, 720)
|
||||||
|
|
||||||
|
|
||||||
|
def test_repair_handles_index_marked_images_array(tmp_path: Path):
|
||||||
|
video = tmp_path / "image_3.mp4"
|
||||||
|
video.write_bytes(build_mp4(1920, 1080))
|
||||||
|
|
||||||
|
payload = _metadata_payload(
|
||||||
|
images=[
|
||||||
|
{"url": "https://example.com/remote.png", "type": "image"},
|
||||||
|
{"url": "", "type": "video", "width": 720, "height": 1280},
|
||||||
|
{"url": "", "type": "video", "width": 720, "height": 1280},
|
||||||
|
{"url": "", "type": "video", "width": 720, "height": 1280},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
repaired = metadata_module.repair_local_video_dimensions(
|
||||||
|
payload, {"3": str(video)}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert repaired == 1
|
||||||
|
# Position 3 (index 3) is the one carrying the local file.
|
||||||
|
assert payload["civitai"]["images"][3]["width"] == 1920
|
||||||
|
assert payload["civitai"]["images"][3]["height"] == 1080
|
||||||
|
# The remote entry keeps its API-provided shape.
|
||||||
|
assert payload["civitai"]["images"][0].get("width") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_repair_never_touches_remote_entries(tmp_path: Path):
|
||||||
|
"""Remote entries keep API-provided dimensions even if a file exists."""
|
||||||
|
|
||||||
|
video = tmp_path / "custom_remote.mp4"
|
||||||
|
video.write_bytes(build_mp4(1280, 720))
|
||||||
|
|
||||||
|
payload = _metadata_payload(
|
||||||
|
customImages=[
|
||||||
|
{
|
||||||
|
"url": "https://civitai.com/1234.mp4",
|
||||||
|
"id": "remote",
|
||||||
|
"type": "video",
|
||||||
|
"width": 720,
|
||||||
|
"height": 1280,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
before = json.dumps(payload, sort_keys=True)
|
||||||
|
|
||||||
|
repaired = metadata_module.repair_local_video_dimensions(
|
||||||
|
payload, {"remote": str(video)}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert repaired == 0
|
||||||
|
assert json.dumps(payload, sort_keys=True) == before
|
||||||
|
|
||||||
|
|
||||||
|
def test_repair_is_idempotent(tmp_path: Path):
|
||||||
|
video = tmp_path / "custom_abc.mp4"
|
||||||
|
video.write_bytes(build_mp4(1280, 720))
|
||||||
|
|
||||||
|
payload = _metadata_payload(
|
||||||
|
customImages=[{"url": "", "id": "abc", "type": "video", "width": 720, "height": 1280}]
|
||||||
|
)
|
||||||
|
files = {"abc": str(video)}
|
||||||
|
|
||||||
|
assert metadata_module.repair_local_video_dimensions(payload, files) == 1
|
||||||
|
# Second run finds nothing to do and leaves the entry byte-identical.
|
||||||
|
snapshot = json.dumps(payload, sort_keys=True)
|
||||||
|
assert metadata_module.repair_local_video_dimensions(payload, files) == 0
|
||||||
|
assert json.dumps(payload, sort_keys=True) == snapshot
|
||||||
|
|
||||||
|
|
||||||
|
def test_repair_dry_run_does_not_mutate(tmp_path: Path):
|
||||||
|
video = tmp_path / "custom_abc.mp4"
|
||||||
|
video.write_bytes(build_mp4(1280, 720))
|
||||||
|
|
||||||
|
payload = _metadata_payload(
|
||||||
|
customImages=[{"url": "", "id": "abc", "type": "video", "width": 720, "height": 1280}]
|
||||||
|
)
|
||||||
|
before = json.dumps(payload, sort_keys=True)
|
||||||
|
|
||||||
|
repaired = metadata_module.repair_local_video_dimensions(
|
||||||
|
payload, {"abc": str(video)}, dry_run=True
|
||||||
|
)
|
||||||
|
|
||||||
|
assert repaired == 1
|
||||||
|
assert json.dumps(payload, sort_keys=True) == before
|
||||||
|
|
||||||
|
|
||||||
|
def test_repair_skips_missing_file(tmp_path: Path):
|
||||||
|
payload = _metadata_payload(
|
||||||
|
customImages=[{"url": "", "id": "gone", "type": "video", "width": 720, "height": 1280}]
|
||||||
|
)
|
||||||
|
|
||||||
|
repaired = metadata_module.repair_local_video_dimensions(
|
||||||
|
payload, {"gone": str(tmp_path / "does-not-exist.mp4")}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert repaired == 0
|
||||||
|
assert payload["civitai"]["customImages"][0]["width"] == 720
|
||||||
|
|
||||||
|
|
||||||
|
def test_repair_leaves_correct_entries_untouched(tmp_path: Path):
|
||||||
|
video = tmp_path / "custom_ok.mp4"
|
||||||
|
video.write_bytes(build_mp4(1280, 720))
|
||||||
|
|
||||||
|
payload = _metadata_payload(
|
||||||
|
customImages=[{"url": "", "id": "ok", "type": "video", "width": 1280, "height": 720}]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert metadata_module.repair_local_video_dimensions(payload, {"ok": str(video)}) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_local_file_map_keys_strip_naming_prefix(tmp_path: Path):
|
||||||
|
(tmp_path / "custom_abc.mp4").write_bytes(build_mp4(1280, 720))
|
||||||
|
(tmp_path / "image_2.png").write_bytes(b"not-a-real-image")
|
||||||
|
(tmp_path / "notes.txt").write_text("ignore me", encoding="utf-8")
|
||||||
|
|
||||||
|
mapping = migration_module.ExampleImagesMigration._build_local_file_map(str(tmp_path))
|
||||||
|
|
||||||
|
assert set(mapping) == {"abc", "2"}
|
||||||
|
|
||||||
|
|
||||||
|
async def test_migrate_to_v3_repairs_and_syncs_cache(tmp_path: Path, monkeypatch):
|
||||||
|
model_hash = "a" * 64
|
||||||
|
folder = tmp_path / model_hash
|
||||||
|
folder.mkdir()
|
||||||
|
(folder / "custom_xyz.mp4").write_bytes(build_mp4(1080, 1920))
|
||||||
|
|
||||||
|
model_file = tmp_path / "model.safetensors"
|
||||||
|
model_file.write_text("weights", encoding="utf-8")
|
||||||
|
|
||||||
|
payload = _metadata_payload(
|
||||||
|
customImages=[{"url": "", "id": "xyz", "type": "video", "width": 720, "height": 1280}]
|
||||||
|
)
|
||||||
|
saved: list[tuple[str, Dict[str, Any]]] = []
|
||||||
|
|
||||||
|
async def fake_load(file_path):
|
||||||
|
return dict(payload, civitai=dict(payload["civitai"]))
|
||||||
|
|
||||||
|
async def fake_save(file_path, data):
|
||||||
|
saved.append((file_path, data))
|
||||||
|
return True
|
||||||
|
|
||||||
|
synced: list[tuple[str, Dict[str, Any]]] = []
|
||||||
|
|
||||||
|
async def fake_sync(scanner, file_path, data):
|
||||||
|
synced.append((file_path, data))
|
||||||
|
return True
|
||||||
|
|
||||||
|
class StubScanner:
|
||||||
|
def has_hash(self, _hash):
|
||||||
|
return True
|
||||||
|
|
||||||
|
async def get_cached_data(self):
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
return SimpleNamespace(raw_data=[{"sha256": model_hash, "file_path": str(model_file)}])
|
||||||
|
|
||||||
|
monkeypatch.setattr(migration_module.MetadataManager, "load_metadata_payload", fake_load)
|
||||||
|
monkeypatch.setattr(migration_module.MetadataManager, "save_metadata", fake_save)
|
||||||
|
monkeypatch.setattr(migration_module, "update_cache_from_metadata", fake_sync)
|
||||||
|
|
||||||
|
async def fake_lora():
|
||||||
|
return StubScanner()
|
||||||
|
|
||||||
|
async def fake_none():
|
||||||
|
return None
|
||||||
|
|
||||||
|
monkeypatch.setattr(migration_module.ServiceRegistry, "get_lora_scanner", fake_lora)
|
||||||
|
monkeypatch.setattr(migration_module.ServiceRegistry, "get_checkpoint_scanner", fake_none)
|
||||||
|
monkeypatch.setattr(migration_module.ServiceRegistry, "get_embedding_scanner", fake_none)
|
||||||
|
|
||||||
|
await migration_module.ExampleImagesMigration._migrate_to_v3(
|
||||||
|
str(tmp_path), [str(folder)]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(saved) == 1
|
||||||
|
saved_entry = saved[0][1]["civitai"]["customImages"][0]
|
||||||
|
assert (saved_entry["width"], saved_entry["height"]) == (1080, 1920)
|
||||||
|
assert len(synced) == 1
|
||||||
|
assert synced[0][1]["civitai"]["customImages"][0]["width"] == 1080
|
||||||
|
|
||||||
|
|
||||||
|
async def test_migrate_to_v3_skips_when_nothing_to_repair(tmp_path: Path, monkeypatch):
|
||||||
|
model_hash = "b" * 64
|
||||||
|
folder = tmp_path / model_hash
|
||||||
|
folder.mkdir()
|
||||||
|
(folder / "custom_ok.mp4").write_bytes(build_mp4(1080, 1920))
|
||||||
|
|
||||||
|
model_file = tmp_path / "model.safetensors"
|
||||||
|
model_file.write_text("weights", encoding="utf-8")
|
||||||
|
|
||||||
|
payload = _metadata_payload(
|
||||||
|
customImages=[{"url": "", "id": "ok", "type": "video", "width": 1080, "height": 1920}]
|
||||||
|
)
|
||||||
|
saved: list[Any] = []
|
||||||
|
|
||||||
|
async def fake_load(file_path):
|
||||||
|
return dict(payload, civitai=dict(payload["civitai"]))
|
||||||
|
|
||||||
|
async def fake_save(file_path, data):
|
||||||
|
saved.append(data)
|
||||||
|
return True
|
||||||
|
|
||||||
|
class StubScanner:
|
||||||
|
def has_hash(self, _hash):
|
||||||
|
return True
|
||||||
|
|
||||||
|
async def get_cached_data(self):
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
return SimpleNamespace(raw_data=[{"sha256": model_hash, "file_path": str(model_file)}])
|
||||||
|
|
||||||
|
monkeypatch.setattr(migration_module.MetadataManager, "load_metadata_payload", fake_load)
|
||||||
|
monkeypatch.setattr(migration_module.MetadataManager, "save_metadata", fake_save)
|
||||||
|
|
||||||
|
async def fake_lora():
|
||||||
|
return StubScanner()
|
||||||
|
|
||||||
|
async def fake_none():
|
||||||
|
return None
|
||||||
|
|
||||||
|
monkeypatch.setattr(migration_module.ServiceRegistry, "get_lora_scanner", fake_lora)
|
||||||
|
monkeypatch.setattr(migration_module.ServiceRegistry, "get_checkpoint_scanner", fake_none)
|
||||||
|
monkeypatch.setattr(migration_module.ServiceRegistry, "get_embedding_scanner", fake_none)
|
||||||
|
|
||||||
|
await migration_module.ExampleImagesMigration._migrate_to_v3(str(tmp_path), [str(folder)])
|
||||||
|
|
||||||
|
# Correctly-sized entries are never rewritten.
|
||||||
|
assert saved == []
|
||||||
|
|
||||||
|
|
||||||
|
async def test_migrate_to_v3_skips_unindexed_model(tmp_path: Path, monkeypatch):
|
||||||
|
"""A folder whose model is absent from every scanner cache is skipped, not fatal."""
|
||||||
|
|
||||||
|
model_hash = "c" * 64
|
||||||
|
folder = tmp_path / model_hash
|
||||||
|
folder.mkdir()
|
||||||
|
(folder / "custom_zzz.mp4").write_bytes(build_mp4(1080, 1920))
|
||||||
|
|
||||||
|
class EmptyScanner:
|
||||||
|
def has_hash(self, _hash):
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def get_cached_data(self):
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
return SimpleNamespace(raw_data=[])
|
||||||
|
|
||||||
|
async def fake_scanner():
|
||||||
|
return EmptyScanner()
|
||||||
|
|
||||||
|
monkeypatch.setattr(migration_module.ServiceRegistry, "get_lora_scanner", fake_scanner)
|
||||||
|
monkeypatch.setattr(migration_module.ServiceRegistry, "get_checkpoint_scanner", fake_scanner)
|
||||||
|
monkeypatch.setattr(migration_module.ServiceRegistry, "get_embedding_scanner", fake_scanner)
|
||||||
|
|
||||||
|
# Must not raise.
|
||||||
|
await migration_module.ExampleImagesMigration._migrate_to_v3(str(tmp_path), [str(folder)])
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
"""Tests for the cross-process advisory lock (:mod:`py.utils.file_lock`)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from py.utils.file_lock import (
|
||||||
|
CrossProcessLock,
|
||||||
|
FileLockUnavailable,
|
||||||
|
exclusive_lock,
|
||||||
|
lock_path_for,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_lock_path_is_a_sibling_of_the_resource(tmp_path):
|
||||||
|
db_path = str(tmp_path / "recipe" / "default.sqlite")
|
||||||
|
lock_path = lock_path_for(db_path)
|
||||||
|
|
||||||
|
assert os.path.dirname(lock_path) == os.path.dirname(db_path)
|
||||||
|
assert os.path.basename(lock_path) == ".default.sqlite.lock"
|
||||||
|
|
||||||
|
|
||||||
|
def test_acquire_and_release_round_trip(tmp_path):
|
||||||
|
lock = exclusive_lock(str(tmp_path / "cache.sqlite"))
|
||||||
|
|
||||||
|
assert lock.acquire() is True
|
||||||
|
lock.release()
|
||||||
|
# Releasing twice must be safe.
|
||||||
|
lock.release()
|
||||||
|
# ...and the lock is reusable afterwards.
|
||||||
|
assert lock.acquire() is True
|
||||||
|
lock.release()
|
||||||
|
|
||||||
|
|
||||||
|
def test_second_lock_holder_waits_until_release(tmp_path):
|
||||||
|
"""A held lock blocks a competing holder for the same resource."""
|
||||||
|
db_path = str(tmp_path / "cache.sqlite")
|
||||||
|
first = exclusive_lock(db_path)
|
||||||
|
second = CrossProcessLock(lock_path_for(db_path), timeout=0.2)
|
||||||
|
|
||||||
|
assert first.acquire() is True
|
||||||
|
try:
|
||||||
|
started = time.monotonic()
|
||||||
|
assert second.acquire() is False
|
||||||
|
# It must have waited for the timeout rather than failing instantly.
|
||||||
|
assert time.monotonic() - started >= 0.15
|
||||||
|
finally:
|
||||||
|
first.release()
|
||||||
|
|
||||||
|
# Once released, the contender gets the lock.
|
||||||
|
assert second.acquire() is True
|
||||||
|
second.release()
|
||||||
|
|
||||||
|
|
||||||
|
def test_context_manager_releases_on_exception(tmp_path):
|
||||||
|
lock = exclusive_lock(str(tmp_path / "cache.sqlite"))
|
||||||
|
contender = CrossProcessLock(lock.path, timeout=0.2)
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError):
|
||||||
|
with lock:
|
||||||
|
raise RuntimeError("boom")
|
||||||
|
|
||||||
|
assert contender.acquire() is True
|
||||||
|
contender.release()
|
||||||
|
|
||||||
|
|
||||||
|
def test_lock_file_is_not_deleted(tmp_path):
|
||||||
|
"""Deleting the lock file would let a second process lock a fresh inode."""
|
||||||
|
lock = exclusive_lock(str(tmp_path / "cache.sqlite"))
|
||||||
|
assert lock.acquire() is True
|
||||||
|
lock.release()
|
||||||
|
|
||||||
|
assert os.path.exists(lock.path)
|
||||||
|
|
||||||
|
|
||||||
|
def test_unsupported_platform_degrades_gracefully(tmp_path, monkeypatch):
|
||||||
|
"""Without a platform primitive the lock reports failure instead of raising."""
|
||||||
|
import py.utils.file_lock as file_lock_module
|
||||||
|
|
||||||
|
monkeypatch.setattr(file_lock_module, "fcntl", None)
|
||||||
|
monkeypatch.setattr(file_lock_module, "msvcrt", None)
|
||||||
|
|
||||||
|
lock = exclusive_lock(str(tmp_path / "cache.sqlite"))
|
||||||
|
assert lock.acquire() is False
|
||||||
|
# Callers use it as a context manager and continue without the lock.
|
||||||
|
with exclusive_lock(str(tmp_path / "cache.sqlite")):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def test_file_lock_unavailable_is_exported():
|
||||||
|
assert issubclass(FileLockUnavailable, RuntimeError)
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_cache_creates_lock_next_to_database(tmp_path):
|
||||||
|
"""The recipe cache write path actually takes the cross-process lock."""
|
||||||
|
from py.services.persistent_recipe_cache import PersistentRecipeCache
|
||||||
|
|
||||||
|
db_path = tmp_path / "recipe_cache.sqlite"
|
||||||
|
cache = PersistentRecipeCache(db_path=str(db_path))
|
||||||
|
assert cache.save_cache([{"id": "r1", "title": "One"}], {"r1": "/tmp/r1.json"})
|
||||||
|
|
||||||
|
assert os.path.exists(lock_path_for(str(db_path)))
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_cache_releases_lock_after_write(tmp_path):
|
||||||
|
"""A second writer must not be blocked once the first has finished."""
|
||||||
|
from py.services.persistent_recipe_cache import PersistentRecipeCache
|
||||||
|
|
||||||
|
db_path = tmp_path / "recipe_cache.sqlite"
|
||||||
|
cache = PersistentRecipeCache(db_path=str(db_path))
|
||||||
|
cache.save_cache([{"id": "r1", "title": "One"}], {"r1": "/tmp/r1.json"})
|
||||||
|
|
||||||
|
contender = CrossProcessLock(lock_path_for(str(db_path)), timeout=0.2)
|
||||||
|
assert contender.acquire() is True
|
||||||
|
contender.release()
|
||||||
@@ -34,10 +34,12 @@ class TestShouldUsePortableSettings:
|
|||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"env_value, settings_flag, expected",
|
"env_value, settings_flag, expected",
|
||||||
[
|
[
|
||||||
("1", False, True), # env = 1 overrides settings.json false
|
("1", False, True), # env = 1 forces portable on
|
||||||
("1", True, True), # env = 1 matches settings.json true
|
("1", True, True), # env = 1 matches settings.json true
|
||||||
("0", False, False), # env = 0 → rely on settings.json
|
("0", False, False), # env = 0 forces portable off
|
||||||
("0", True, True), # env = 0 → rely on settings.json
|
("0", True, False), # env = 0 overrides a persisted true
|
||||||
|
("yes", False, False), # unrecognised value → rely on settings.json
|
||||||
|
("yes", True, True), # unrecognised value → rely on settings.json
|
||||||
("", False, False), # unset → rely on settings.json
|
("", False, False), # unset → rely on settings.json
|
||||||
("", True, True), # unset → rely on settings.json
|
("", True, True), # unset → rely on settings.json
|
||||||
],
|
],
|
||||||
@@ -58,6 +60,21 @@ class TestShouldUsePortableSettings:
|
|||||||
result = _should_use_portable_settings(str(settings_file), logging.getLogger())
|
result = _should_use_portable_settings(str(settings_file), logging.getLogger())
|
||||||
assert result == expected
|
assert result == expected
|
||||||
|
|
||||||
|
def test_explicit_zero_is_the_documented_opt_out(self, tmp_path, caplog):
|
||||||
|
"""`=0` must be honoured even against a persisted true flag."""
|
||||||
|
settings_file = tmp_path / "settings.json"
|
||||||
|
settings_file.write_text(json.dumps({"use_portable_settings": True}))
|
||||||
|
|
||||||
|
with pytest.MonkeyPatch.context() as mp:
|
||||||
|
mp.setenv("LORA_MANAGER_PORTABLE", "0")
|
||||||
|
with caplog.at_level(logging.INFO):
|
||||||
|
result = _should_use_portable_settings(
|
||||||
|
str(settings_file), logging.getLogger()
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is False
|
||||||
|
assert "Portable mode disabled" in caplog.text
|
||||||
|
|
||||||
def test_missing_file_without_env(self, tmp_path):
|
def test_missing_file_without_env(self, tmp_path):
|
||||||
"""Without env var, missing settings file returns False."""
|
"""Without env var, missing settings file returns False."""
|
||||||
missing = tmp_path / "nonexistent.json"
|
missing = tmp_path / "nonexistent.json"
|
||||||
|
|||||||
@@ -0,0 +1,178 @@
|
|||||||
|
"""Tests for the container-level video dimension probe."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import struct
|
||||||
|
|
||||||
|
from py.utils.video_metadata import get_video_dimensions
|
||||||
|
|
||||||
|
|
||||||
|
def _box(box_type: bytes, payload: bytes) -> bytes:
|
||||||
|
return struct.pack(">I", len(payload) + 8) + box_type + payload
|
||||||
|
|
||||||
|
|
||||||
|
def _full_box(box_type: bytes, payload: bytes) -> bytes:
|
||||||
|
"""Build a box with a 4-byte version/flags header."""
|
||||||
|
|
||||||
|
return _box(box_type, b"\x00\x00\x00\x00" + payload)
|
||||||
|
|
||||||
|
|
||||||
|
def build_mp4(width: int, height: int, *, with_stsd: bool = False) -> bytes:
|
||||||
|
"""Build a minimal but structurally valid MP4 holding one video track."""
|
||||||
|
|
||||||
|
mvhd = _full_box(b"mvhd", b"\x00" * 96)
|
||||||
|
|
||||||
|
hdlr = _full_box(b"hdlr", b"\x00" * 4 + b"vide" + b"\x00" * 12)
|
||||||
|
|
||||||
|
tkhd_payload = struct.pack(">IIII", 0, 0, 0, 0) + b"\x00" * 52
|
||||||
|
tkhd_payload += struct.pack(">II", width << 16, height << 16)
|
||||||
|
tkhd = _full_box(b"tkhd", tkhd_payload)
|
||||||
|
|
||||||
|
stbl_children = b""
|
||||||
|
if with_stsd:
|
||||||
|
sample_entry = (
|
||||||
|
b"\x00" * 6 + struct.pack(">H", 1) + struct.pack(">HH", width, height)
|
||||||
|
)
|
||||||
|
stsd = _full_box(b"stsd", struct.pack(">I", 1) + _box(b"avc1", sample_entry))
|
||||||
|
stbl_children = stsd
|
||||||
|
|
||||||
|
minf = _box(b"minf", _box(b"stbl", stbl_children))
|
||||||
|
mdia = _box(b"mdia", hdlr + minf)
|
||||||
|
trak = _box(b"trak", tkhd + mdia)
|
||||||
|
moov = _box(b"moov", mvhd + trak)
|
||||||
|
ftyp = _box(b"ftyp", b"isom" + b"\x00\x00\x02\x00" + b"isomiso2avc1mp41")
|
||||||
|
|
||||||
|
return ftyp + moov
|
||||||
|
|
||||||
|
|
||||||
|
def _ebml_vint(value: int) -> bytes:
|
||||||
|
"""Encode a value as a minimal-length EBML variable length integer."""
|
||||||
|
|
||||||
|
for length in range(1, 9):
|
||||||
|
if value < (1 << (7 * length)):
|
||||||
|
encoded = value | (1 << (7 * length))
|
||||||
|
return encoded.to_bytes(length, "big")
|
||||||
|
raise ValueError("value too large for an EBML vint")
|
||||||
|
|
||||||
|
|
||||||
|
def _ebml_element(element_id: bytes, payload: bytes) -> bytes:
|
||||||
|
return element_id + _ebml_vint(len(payload)) + payload
|
||||||
|
|
||||||
|
|
||||||
|
def _uint_element(element_id: int, value: int) -> bytes:
|
||||||
|
length = max(1, (value.bit_length() + 7) // 8)
|
||||||
|
return _ebml_element(
|
||||||
|
element_id.to_bytes(2, "big") if element_id > 0xFF else element_id.to_bytes(1, "big"),
|
||||||
|
value.to_bytes(length, "big"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_webm(width: int, height: int, *, track_type: int = 1) -> bytes:
|
||||||
|
"""Build a minimal WebM file holding one TrackEntry."""
|
||||||
|
|
||||||
|
video = _ebml_element(b"\xe0", _uint_element(0xB0, width) + _uint_element(0xBA, height))
|
||||||
|
track_entry = _ebml_element(
|
||||||
|
b"\xae", _uint_element(0x83, track_type) + video
|
||||||
|
)
|
||||||
|
tracks = _ebml_element(b"\x16\x54\xae\x6b", track_entry)
|
||||||
|
|
||||||
|
segment = _ebml_element(b"\x18\x53\x80\x67", tracks)
|
||||||
|
ebml_header = _ebml_element(
|
||||||
|
b"\x1a\x45\xdf\xa3",
|
||||||
|
_uint_element(0x4286, 1) + _ebml_element(b"\x42\x82", b"webm"),
|
||||||
|
)
|
||||||
|
|
||||||
|
return ebml_header + segment
|
||||||
|
|
||||||
|
|
||||||
|
def test_mp4_dimensions_come_from_tkhd(tmp_path):
|
||||||
|
video = tmp_path / "landscape.mp4"
|
||||||
|
video.write_bytes(build_mp4(1280, 720))
|
||||||
|
|
||||||
|
assert get_video_dimensions(str(video)) == (1280, 720)
|
||||||
|
|
||||||
|
|
||||||
|
def test_mp4_uses_stsd_when_tkhd_is_empty(tmp_path):
|
||||||
|
video = tmp_path / "stsd-only.mp4"
|
||||||
|
video.write_bytes(build_mp4(640, 480, with_stsd=True))
|
||||||
|
|
||||||
|
assert get_video_dimensions(str(video)) == (640, 480)
|
||||||
|
|
||||||
|
|
||||||
|
def test_mp4_without_video_track_returns_none(tmp_path):
|
||||||
|
# A moov whose only trak has no mdia box at all.
|
||||||
|
tkhd = _full_box(b"tkhd", b"\x00" * 60)
|
||||||
|
moov = _box(b"moov", _box(b"trak", tkhd))
|
||||||
|
video = tmp_path / "audio-only.mp4"
|
||||||
|
video.write_bytes(moov)
|
||||||
|
|
||||||
|
assert get_video_dimensions(str(video)) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_webm_dimensions(tmp_path):
|
||||||
|
video = tmp_path / "portrait.webm"
|
||||||
|
video.write_bytes(build_webm(720, 1280))
|
||||||
|
|
||||||
|
assert get_video_dimensions(str(video)) == (720, 1280)
|
||||||
|
|
||||||
|
|
||||||
|
def test_webm_non_video_track_is_ignored(tmp_path):
|
||||||
|
video = tmp_path / "audio.webm"
|
||||||
|
video.write_bytes(build_webm(720, 1280, track_type=2))
|
||||||
|
|
||||||
|
assert get_video_dimensions(str(video)) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_container_signature_wins_over_extension(tmp_path):
|
||||||
|
"""A WebM file named ``.mp4`` is still parsed as WebM."""
|
||||||
|
|
||||||
|
video = tmp_path / "actually-webm.mp4"
|
||||||
|
video.write_bytes(build_webm(480, 832))
|
||||||
|
|
||||||
|
assert get_video_dimensions(str(video)) == (480, 832)
|
||||||
|
|
||||||
|
|
||||||
|
def test_webp_renamed_to_mp4_is_read(tmp_path):
|
||||||
|
"""Animated WebP examples are frequently saved with a video extension."""
|
||||||
|
|
||||||
|
vp8_payload = b"\x30\x36\x02" + b"\x9d\x01\x2a" + struct.pack("<HH", 450, 800)
|
||||||
|
chunk = b"VP8 " + struct.pack("<I", len(vp8_payload)) + vp8_payload
|
||||||
|
body = b"WEBP" + chunk
|
||||||
|
riff = b"RIFF" + struct.pack("<I", len(body)) + body
|
||||||
|
|
||||||
|
video = tmp_path / "animated.mp4"
|
||||||
|
video.write_bytes(riff)
|
||||||
|
|
||||||
|
assert get_video_dimensions(str(video)) == (450, 800)
|
||||||
|
|
||||||
|
|
||||||
|
def test_webp_vp8x_canvas_dimensions(tmp_path):
|
||||||
|
vp8x_payload = b"\x00" * 4 + (449).to_bytes(3, "little") + (799).to_bytes(3, "little")
|
||||||
|
chunk = b"VP8X" + struct.pack("<I", len(vp8x_payload)) + vp8x_payload
|
||||||
|
body = b"WEBP" + chunk
|
||||||
|
riff = b"RIFF" + struct.pack("<I", len(body)) + body
|
||||||
|
|
||||||
|
video = tmp_path / "canvas.mp4"
|
||||||
|
video.write_bytes(riff)
|
||||||
|
|
||||||
|
assert get_video_dimensions(str(video)) == (450, 800)
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_file_returns_none(tmp_path):
|
||||||
|
assert get_video_dimensions(str(tmp_path / "nope.mp4")) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_corrupt_file_returns_none(tmp_path):
|
||||||
|
video = tmp_path / "corrupt.mp4"
|
||||||
|
video.write_bytes(b"\x00\x00\x00\x20ftypisom" + b"\xff" * 64)
|
||||||
|
|
||||||
|
assert get_video_dimensions(str(video)) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_unsupported_extension_without_video_signature_returns_none(tmp_path):
|
||||||
|
"""A non-video file is not probed just because of a video-like name."""
|
||||||
|
|
||||||
|
video = tmp_path / "clip.avi"
|
||||||
|
video.write_bytes(b"RIFF\x00\x00\x00\x00AVI LIST\x00\x00\x00\x00")
|
||||||
|
|
||||||
|
assert get_video_dimensions(str(video)) is None
|
||||||
Reference in New Issue
Block a user