mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-06-24 20:11:17 -03:00
Compare commits
404 Commits
b5a0725d2c
...
v1.1.5
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f48de05102 | ||
|
|
93ad81ed87 | ||
|
|
ea14d211be | ||
|
|
8052cefd46 | ||
|
|
845815b9b7 | ||
|
|
609dc5d783 | ||
|
|
7a71b34b54 | ||
|
|
71a459422f | ||
|
|
cd2628a0ee | ||
|
|
85da7175bc | ||
|
|
d3bf0a164b | ||
|
|
afb6ca1b8d | ||
|
|
94f43426d7 | ||
|
|
2b361f4f5d | ||
|
|
7438072f8c | ||
|
|
26c54fd358 | ||
|
|
7cb6b04c63 | ||
|
|
fc29cde82a | ||
|
|
559ca946dc | ||
|
|
2b8e7c7504 | ||
|
|
6816d75933 | ||
|
|
b58abbad7c | ||
|
|
999814ca87 | ||
|
|
3c2760a803 | ||
|
|
0edbd7bcca | ||
|
|
21e89fa7de | ||
|
|
968d6d1d1f | ||
|
|
cf0fd0e0ad | ||
|
|
16e5dcf7b2 | ||
|
|
ab6bb25d46 | ||
|
|
07f49559be | ||
|
|
b24b1a7e57 | ||
|
|
faf64f8986 | ||
|
|
a617487a43 | ||
|
|
3012a7aef3 | ||
|
|
499e19de34 | ||
|
|
9161762ca9 | ||
|
|
9bbd26efe6 | ||
|
|
258b2622d5 | ||
|
|
80ec9085dd | ||
|
|
c5c7373e10 | ||
|
|
b7721866e5 | ||
|
|
8314b9bedb | ||
|
|
75298a402f | ||
|
|
92b5efd414 | ||
|
|
33ee392b7b | ||
|
|
5237f8b7dc | ||
|
|
5107313fd1 | ||
|
|
95bbc66919 | ||
|
|
e268e59419 | ||
|
|
547e1f9498 | ||
|
|
bf32d8b6fd | ||
|
|
8299881024 | ||
|
|
da02268196 | ||
|
|
8c4b9a1e70 | ||
|
|
0906c484e9 | ||
|
|
4199c30fec | ||
|
|
4a8084cdbc | ||
|
|
6263e6848c | ||
|
|
58c266ad07 | ||
|
|
2939813e1a | ||
|
|
a9e5ee7e79 | ||
|
|
a17b0e9901 | ||
|
|
8f23d966bf | ||
|
|
7a76fc72d0 | ||
|
|
518a4dd5ee | ||
|
|
2b6d4e5d8b | ||
|
|
1f4edbeb9d | ||
|
|
a256558a0e | ||
|
|
818b9113f0 | ||
|
|
6a4fd020dc | ||
|
|
7a23040452 | ||
|
|
138024aefe | ||
|
|
a19ddc14f6 | ||
|
|
7001ced694 | ||
|
|
a5c861646c | ||
|
|
3e0bb73793 | ||
|
|
ac51f6a2f6 | ||
|
|
bef222c77d | ||
|
|
7cd6a53447 | ||
|
|
6850b35770 | ||
|
|
237a015cde | ||
|
|
1ae2778baa | ||
|
|
84fcdb5f20 | ||
|
|
8a0b368b44 | ||
|
|
3990535505 | ||
|
|
3e961a9860 | ||
|
|
d6669f1d04 | ||
|
|
519bafebc8 | ||
|
|
d87863b423 | ||
|
|
84e9fe2dfb | ||
|
|
46cbcf94c8 | ||
|
|
05f3018495 | ||
|
|
f565cc35ca | ||
|
|
dd1cdce16d | ||
|
|
a9e0e7dc8d | ||
|
|
b302d1db7d | ||
|
|
7cbddd9cf7 | ||
|
|
cb8c699224 | ||
|
|
451f74b874 | ||
|
|
a1d248baa6 | ||
|
|
18577fa336 | ||
|
|
5797ce9408 | ||
|
|
826f06255a | ||
|
|
84e16b5c5b | ||
|
|
eb22054580 | ||
|
|
08afb05ece | ||
|
|
f51f125cf1 | ||
|
|
24b2078f21 | ||
|
|
130fb5d2d5 | ||
|
|
23c6863a3a | ||
|
|
c0e2578640 | ||
|
|
e3c812367e | ||
|
|
4d239008a6 | ||
|
|
00177a06d0 | ||
|
|
568daa351e | ||
|
|
5a4664fa12 | ||
|
|
dd5b213adc | ||
|
|
d9ee9b3155 | ||
|
|
01dac57c35 | ||
|
|
7f92d09239 | ||
|
|
62f9e3f44a | ||
|
|
e55895786d | ||
|
|
82b77bf593 | ||
|
|
1beef5dea9 | ||
|
|
c8beaa64e1 | ||
|
|
fb443ed6ae | ||
|
|
151a467598 | ||
|
|
98e1d168b0 | ||
|
|
716f18e0ed | ||
|
|
b060dc99fc | ||
|
|
54bcdfab38 | ||
|
|
2e7532eecc | ||
|
|
7e5e3b1ec7 | ||
|
|
df67bd396a | ||
|
|
dd5d9cfcb2 | ||
|
|
d9fd60bec1 | ||
|
|
b633b22779 | ||
|
|
1ffa543160 | ||
|
|
cdc940586e | ||
|
|
ccf1c6f2ae | ||
|
|
bfe7b5e1c7 | ||
|
|
85c020cd12 | ||
|
|
1b202f8ec7 | ||
|
|
d02a0611d3 | ||
|
|
92166a161a | ||
|
|
b509f27cb7 | ||
|
|
5c2ef48917 | ||
|
|
ad2bd82c67 | ||
|
|
17ba350153 | ||
|
|
60175334b5 | ||
|
|
f65a01df00 | ||
|
|
430e24d70b | ||
|
|
14f0c48fdd | ||
|
|
34791c2ad7 | ||
|
|
3f6824eef6 | ||
|
|
3919dfa3f4 | ||
|
|
7124b5293f | ||
|
|
d2a04f8993 | ||
|
|
7027a7c270 | ||
|
|
0a1d7dfd4c | ||
|
|
3962b1a96d | ||
|
|
8b856276bf | ||
|
|
c97c802956 | ||
|
|
24e2909627 | ||
|
|
b768f1368f | ||
|
|
37ccd29fc0 | ||
|
|
7416080cfb | ||
|
|
26be187d42 | ||
|
|
d7caa1fa47 | ||
|
|
2629fcce23 | ||
|
|
438e7d07b9 | ||
|
|
e9932ea870 | ||
|
|
5dd8b96422 | ||
|
|
5e1cf68bbd | ||
|
|
1044fa3c83 | ||
|
|
397892bb7f | ||
|
|
f105500740 | ||
|
|
806555cf06 | ||
|
|
5cd7204101 | ||
|
|
3b602a3698 | ||
|
|
15dfaed462 | ||
|
|
0e51851025 | ||
|
|
0d0f4defca | ||
|
|
818fa34a48 | ||
|
|
78303b2a5e | ||
|
|
9ce56dd40c | ||
|
|
4e3ede23b7 | ||
|
|
33e5f3d85d | ||
|
|
031d5e4f40 | ||
|
|
4ff5774e34 | ||
|
|
94e1a8ac7b | ||
|
|
cc20d3b992 | ||
|
|
a74cbe7aa2 | ||
|
|
94edfaa190 | ||
|
|
31c54ff068 | ||
|
|
21872a8e9e | ||
|
|
612612f1c7 | ||
|
|
ff240db5b1 | ||
|
|
bcfed4b874 | ||
|
|
1352c6ecbe | ||
|
|
30b01b8a92 | ||
|
|
a105cb322b | ||
|
|
3bf396d003 | ||
|
|
60cfb3b8e0 | ||
|
|
6763abb83c | ||
|
|
5c53968caa | ||
|
|
b4f7dd75af | ||
|
|
86118d0654 | ||
|
|
df1410535e | ||
|
|
75f74d54d8 | ||
|
|
ab6100f596 | ||
|
|
5d3ab3bbf8 | ||
|
|
d9dc0dba8d | ||
|
|
3631c5eb10 | ||
|
|
6d5b4b7312 | ||
|
|
7803bd542d | ||
|
|
f0a86dbbc0 | ||
|
|
682e964f89 | ||
|
|
908464bc0a | ||
|
|
0ffee3a854 | ||
|
|
8aa9739c44 | ||
|
|
50739bbb43 | ||
|
|
e849303763 | ||
|
|
241b2e15d2 | ||
|
|
88da754504 | ||
|
|
b4a706651f | ||
|
|
ff7cc6d9bb | ||
|
|
454210a47c | ||
|
|
2d7c404ebb | ||
|
|
e23d803ecf | ||
|
|
0cc640cfaa | ||
|
|
2ac0eb0f9d | ||
|
|
f028625ce9 | ||
|
|
06acc7f576 | ||
|
|
d324b57274 | ||
|
|
502b7eab31 | ||
|
|
be75ad930e | ||
|
|
763c4f4dad | ||
|
|
d32c492bdb | ||
|
|
5dcfde36ea | ||
|
|
1d035361a4 | ||
|
|
25605c5e78 | ||
|
|
f3268a6179 | ||
|
|
055e94d77b | ||
|
|
47fcd530a0 | ||
|
|
3c32b9e088 | ||
|
|
ffe0670a27 | ||
|
|
cc147a1795 | ||
|
|
e81409bea4 | ||
|
|
b31fae4e51 | ||
|
|
c6e5467907 | ||
|
|
df0e5797d0 | ||
|
|
ebdbb36271 | ||
|
|
2eef629821 | ||
|
|
658a04736d | ||
|
|
ef7f677933 | ||
|
|
63f0942452 | ||
|
|
a1dff6dd47 | ||
|
|
7fa40023b0 | ||
|
|
3c8acdb65e | ||
|
|
1e9a7812d6 | ||
|
|
37f0e8f213 | ||
|
|
ecf7ea21e4 | ||
|
|
79dd9a1b29 | ||
|
|
ef4923fd94 | ||
|
|
1eeba666f5 | ||
|
|
89e26d9292 | ||
|
|
fc19a145ff | ||
|
|
34f03d6495 | ||
|
|
9443175abc | ||
|
|
dc5072628f | ||
|
|
ff4b8ec849 | ||
|
|
7ab271c752 | ||
|
|
5a7f4dc88b | ||
|
|
761108bfd1 | ||
|
|
24dd3a777c | ||
|
|
1c530ea013 | ||
|
|
0ced53c059 | ||
|
|
67ad68a23f | ||
|
|
d9ec9c512e | ||
|
|
0bcd8e09a9 | ||
|
|
fa049a28c8 | ||
|
|
89fd2b43d6 | ||
|
|
c53f44e7ef | ||
|
|
ae7bfdb517 | ||
|
|
68bf8442eb | ||
|
|
605fbf4117 | ||
|
|
406d5fea6a | ||
|
|
af2146f96c | ||
|
|
bdc8dec860 | ||
|
|
c4fa1631ee | ||
|
|
506d763dc2 | ||
|
|
a2cd09b619 | ||
|
|
cdd77029b6 | ||
|
|
439679e15f | ||
|
|
2640258902 | ||
|
|
b910388d54 | ||
|
|
083de395b1 | ||
|
|
4514ca94b7 | ||
|
|
62247bdd87 | ||
|
|
6d0d9600a7 | ||
|
|
70cd3f4e1b | ||
|
|
a95c518b30 | ||
|
|
ba1800095e | ||
|
|
39c083db79 | ||
|
|
55e9e4bb6f | ||
|
|
0253d001e6 | ||
|
|
9998da3241 | ||
|
|
6666a72775 | ||
|
|
5f1bd894b9 | ||
|
|
1817142a7b | ||
|
|
25fa175aa2 | ||
|
|
39643eb2bc | ||
|
|
4ac78f8aa8 | ||
|
|
0bcca0ba68 | ||
|
|
72f8e0d1be | ||
|
|
85b6c91192 | ||
|
|
908016cbd6 | ||
|
|
a5ac9cf81b | ||
|
|
32875042bd | ||
|
|
51fe7aa07e | ||
|
|
db4726a961 | ||
|
|
e13d70248a | ||
|
|
1c4919a3e8 | ||
|
|
18ddadc9ec | ||
|
|
b6dd6938b0 | ||
|
|
b711ac468a | ||
|
|
727d0ef043 | ||
|
|
9344d86332 | ||
|
|
d36b16c213 | ||
|
|
33a7f07558 | ||
|
|
4f599aeced | ||
|
|
30db8c3d1d | ||
|
|
05636712f0 | ||
|
|
d8e5fe1247 | ||
|
|
3e9210394a | ||
|
|
4dd2c0526f | ||
|
|
9bdb337962 | ||
|
|
f93baf5fc0 | ||
|
|
14cb7fec47 | ||
|
|
f3b3e0adad | ||
|
|
ba3f15dbc6 | ||
|
|
8dc2a2f76b | ||
|
|
316f17dd46 | ||
|
|
3dc10b1404 | ||
|
|
331889d872 | ||
|
|
06f1a82d4c | ||
|
|
267082c712 | ||
|
|
a4cb51e96c | ||
|
|
ca44c367b3 | ||
|
|
301ab14781 | ||
|
|
2626dbab8e | ||
|
|
12bbb0572d | ||
|
|
00f5c1e887 | ||
|
|
89b1675ec7 | ||
|
|
dcc7bd33b5 | ||
|
|
e5152108ba | ||
|
|
1ed5eef985 | ||
|
|
a82f89d14a | ||
|
|
16e30ea689 | ||
|
|
ad3bdddb72 | ||
|
|
9121306b06 | ||
|
|
ca0baf9462 | ||
|
|
20e50156a2 | ||
|
|
0b66bf5479 | ||
|
|
1e8aca4787 | ||
|
|
76ee59cdb9 | ||
|
|
a5191414cc | ||
|
|
5b065b47d4 | ||
|
|
ceeab0c998 | ||
|
|
3b001a6cd8 | ||
|
|
95e5bc26d1 | ||
|
|
de3d0571f8 | ||
|
|
6f2a01dc86 | ||
|
|
c5c1b8fd2a | ||
|
|
e97648c70b | ||
|
|
8b85e083e2 | ||
|
|
9112cd3b62 | ||
|
|
7df4e8d037 | ||
|
|
4000b7f7e7 | ||
|
|
76c15105e6 | ||
|
|
b11c90e19b | ||
|
|
9f5d2d0c18 | ||
|
|
a0dc5229f4 | ||
|
|
61c31ecbd0 | ||
|
|
1ae1b0d607 | ||
|
|
8dd849892d | ||
|
|
03e1fa75c5 | ||
|
|
fefcaa4a45 | ||
|
|
701a6a6c44 | ||
|
|
0ef414d17e | ||
|
|
75dccaef87 | ||
|
|
7e87ec9521 | ||
|
|
46522edb1b | ||
|
|
2dae4c1291 | ||
|
|
a32325402e | ||
|
|
70c150bd80 | ||
|
|
9e81c33f8a | ||
|
|
22c0dbd734 | ||
|
|
d0c58472be | ||
|
|
b3c530bf36 | ||
|
|
05ebd7493d | ||
|
|
90986bd795 |
69
.agents/skills/lora-manager-runtime-context/SKILL.md
Normal file
69
.agents/skills/lora-manager-runtime-context/SKILL.md
Normal file
@@ -0,0 +1,69 @@
|
||||
---
|
||||
name: lora-manager-runtime-context
|
||||
description: Inspect ComfyUI LoRA Manager runtime configuration and local diagnostic state. Use when debugging LoRA Manager issues that require locating or reading settings.json, active library paths, model metadata JSON sidecars, recipe metadata JSON files, example image folders, SQLite caches, symlink maps, download history, aria2 state, or other cache files under the LoRA Manager user config directory.
|
||||
---
|
||||
|
||||
# LoRA Manager Runtime Context
|
||||
|
||||
## Core Rules
|
||||
|
||||
- Treat runtime state as local user data. Prefer read-only inspection unless the user explicitly asks for mutation.
|
||||
- Never print secret-like settings values. Redact keys containing `key`, `token`, `secret`, `password`, `auth`, or `credential`, including `civitai_api_key`.
|
||||
- Resolve paths from the runtime configuration before guessing. In this environment the settings file is normally `/home/miao/.config/ComfyUI-LoRA-Manager/settings.json`, but portable settings can override this through the repository `settings.json`.
|
||||
- Use the active library when selecting per-library caches and paths. Read `active_library` from settings; fall back to `default` if missing.
|
||||
- Normalize and expand `~` before comparing paths. Symlinks are common in this repo.
|
||||
|
||||
## Quick Start
|
||||
|
||||
Use the bundled helper for a safe first pass:
|
||||
|
||||
```bash
|
||||
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py summary
|
||||
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py caches
|
||||
```
|
||||
|
||||
The script redacts sensitive settings, opens SQLite databases read-only, and reports inaccessible or locked databases as warnings.
|
||||
|
||||
For focused checks:
|
||||
|
||||
```bash
|
||||
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py recipes
|
||||
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py model --path /path/to/model.safetensors
|
||||
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py sqlite --db /path/to/cache.sqlite --limit 3
|
||||
```
|
||||
|
||||
## Runtime Path Rules
|
||||
|
||||
- Settings directory: use `py/utils/settings_paths.py`. Default platform path is `platformdirs.user_config_dir("ComfyUI-LoRA-Manager", appauthor=False)`.
|
||||
- Settings file: `<settings_dir>/settings.json`.
|
||||
- Cache root: `<settings_dir>/cache`.
|
||||
- Canonical cache files:
|
||||
- Model cache: `cache/model/<active_library>.sqlite`.
|
||||
- Recipe cache: `cache/recipe/<active_library>.sqlite`.
|
||||
- Model update cache: `cache/model_update/<active_library>.sqlite`.
|
||||
- Recipe FTS: `cache/fts/recipe_fts.sqlite`.
|
||||
- Tag FTS: `cache/fts/tag_fts.sqlite`.
|
||||
- Symlink map: `cache/symlink/symlink_map.json`.
|
||||
- Download history: `cache/download_history/downloaded_versions.sqlite`.
|
||||
- aria2 state: `cache/aria2/downloads.json`.
|
||||
- Legacy cache locations may exist; prefer canonical paths unless diagnosing migrations.
|
||||
|
||||
## Data Location Rules
|
||||
|
||||
- Model roots come from `settings.folder_paths` and the active library payload under `settings.libraries[active_library]`.
|
||||
- Model metadata JSON sidecars live next to the model file as `<model basename>.metadata.json`.
|
||||
- Recipes root is `settings.recipes_path` when it is a non-empty string. If empty, use the first configured LoRA root plus `/recipes`.
|
||||
- Recipe JSON files are named `*.recipe.json` under the recipes root and may be nested in folders.
|
||||
- Example image root is `settings.example_images_path`.
|
||||
- If multiple libraries are configured, example images are stored under `<example_images_path>/<sanitized_library>/<sha256>/`; otherwise they are under `<example_images_path>/<sha256>/`.
|
||||
|
||||
## Useful Cache Tables
|
||||
|
||||
- Model cache: `models`, `model_tags`, `hash_index`, `excluded_models`.
|
||||
- Recipe cache: `recipes`, `cache_metadata`.
|
||||
- Model update cache: `model_update_status`, `model_update_versions`.
|
||||
- Tag FTS cache: `tags`, `fts_metadata`, plus FTS internal tables.
|
||||
- Recipe FTS cache: `recipe_rowid`, `fts_metadata`, plus FTS internal tables.
|
||||
- Download history: `downloaded_model_versions`.
|
||||
|
||||
Prefer querying only counts, schema, and a few sample rows unless the user asks for full output.
|
||||
@@ -0,0 +1,4 @@
|
||||
interface:
|
||||
display_name: "LoRA Manager Runtime Context"
|
||||
short_description: "Inspect LoRA Manager runtime state"
|
||||
default_prompt: "Use $lora-manager-runtime-context to inspect LoRA Manager settings, metadata paths, and caches for debugging."
|
||||
381
.agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py
Executable file
381
.agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py
Executable file
@@ -0,0 +1,381 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
SECRET_PATTERN = re.compile(r"(key|token|secret|password|auth|credential)", re.IGNORECASE)
|
||||
APP_NAME = "ComfyUI-LoRA-Manager"
|
||||
CACHE_SQLITE = {
|
||||
"model": ("model", "{library}.sqlite"),
|
||||
"recipe": ("recipe", "{library}.sqlite"),
|
||||
"model_update": ("model_update", "{library}.sqlite"),
|
||||
"recipe_fts": ("fts", "recipe_fts.sqlite"),
|
||||
"tag_fts": ("fts", "tag_fts.sqlite"),
|
||||
"download_history": ("download_history", "downloaded_versions.sqlite"),
|
||||
}
|
||||
CACHE_JSON = {
|
||||
"symlink": ("symlink", "symlink_map.json"),
|
||||
"aria2": ("aria2", "downloads.json"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Inspect LoRA Manager runtime state read-only.")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
subparsers.add_parser("summary", help="Print redacted settings and resolved paths.")
|
||||
subparsers.add_parser("caches", help="Print cache paths and SQLite table summaries.")
|
||||
subparsers.add_parser("recipes", help="Print resolved recipes root and recipe JSON count.")
|
||||
|
||||
model_parser = subparsers.add_parser("model", help="Inspect a model metadata sidecar path.")
|
||||
model_parser.add_argument("--path", required=True, help="Path to a model file or metadata JSON file.")
|
||||
|
||||
sqlite_parser = subparsers.add_parser("sqlite", help="Inspect a SQLite database read-only.")
|
||||
sqlite_parser.add_argument("--db", required=True, help="Path to the SQLite database.")
|
||||
sqlite_parser.add_argument("--limit", type=int, default=3, help="Rows to sample from each user table.")
|
||||
|
||||
args = parser.parse_args()
|
||||
context = build_context()
|
||||
|
||||
if args.command == "summary":
|
||||
print_json(summary_payload(context))
|
||||
elif args.command == "caches":
|
||||
print_json(caches_payload(context))
|
||||
elif args.command == "recipes":
|
||||
print_json(recipes_payload(context))
|
||||
elif args.command == "model":
|
||||
print_json(model_payload(args.path))
|
||||
elif args.command == "sqlite":
|
||||
print_json(sqlite_payload(Path(args.db).expanduser(), args.limit))
|
||||
return 0
|
||||
|
||||
|
||||
def build_context() -> dict[str, Any]:
|
||||
settings_path = resolve_settings_path()
|
||||
settings = load_json(settings_path)
|
||||
settings_dir = settings_path.parent
|
||||
active_library = settings.get("active_library") or "default"
|
||||
safe_library = sanitize_library_name(str(active_library))
|
||||
cache_root = settings_dir / "cache"
|
||||
return {
|
||||
"settings_path": str(settings_path),
|
||||
"settings_dir": str(settings_dir),
|
||||
"settings": settings,
|
||||
"active_library": active_library,
|
||||
"safe_library": safe_library,
|
||||
"cache_root": str(cache_root),
|
||||
"cache_paths": resolve_cache_paths(cache_root, safe_library),
|
||||
}
|
||||
|
||||
|
||||
def resolve_settings_path() -> Path:
|
||||
repo_root = find_repo_root()
|
||||
portable = repo_root / "settings.json"
|
||||
if portable.exists():
|
||||
payload = load_json(portable)
|
||||
if isinstance(payload, dict) and payload.get("use_portable_settings") is True:
|
||||
return portable
|
||||
|
||||
config_home = os.environ.get("XDG_CONFIG_HOME")
|
||||
if config_home:
|
||||
return Path(config_home).expanduser() / APP_NAME / "settings.json"
|
||||
return Path.home() / ".config" / APP_NAME / "settings.json"
|
||||
|
||||
|
||||
def find_repo_root() -> Path:
|
||||
current = Path(__file__).resolve()
|
||||
for parent in current.parents:
|
||||
if (parent / "py").is_dir() and (parent / "standalone.py").exists():
|
||||
return parent
|
||||
return Path.cwd()
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
except json.JSONDecodeError as exc:
|
||||
return {"_error": f"invalid JSON: {exc}"}
|
||||
except OSError as exc:
|
||||
return {"_error": f"unreadable: {exc}"}
|
||||
return payload if isinstance(payload, dict) else {"_error": "JSON root is not an object"}
|
||||
|
||||
|
||||
def resolve_cache_paths(cache_root: Path, library: str) -> dict[str, str]:
|
||||
paths: dict[str, str] = {}
|
||||
for name, (subdir, filename) in CACHE_SQLITE.items():
|
||||
paths[name] = str(cache_root / subdir / filename.format(library=library))
|
||||
for name, (subdir, filename) in CACHE_JSON.items():
|
||||
paths[name] = str(cache_root / subdir / filename)
|
||||
return paths
|
||||
|
||||
|
||||
def summary_payload(context: dict[str, Any]) -> dict[str, Any]:
|
||||
settings = context["settings"]
|
||||
return {
|
||||
"settings_path": context["settings_path"],
|
||||
"settings_dir": context["settings_dir"],
|
||||
"active_library": context["active_library"],
|
||||
"settings": redact(settings),
|
||||
"model_roots": model_roots(settings, context["active_library"]),
|
||||
"recipes_root": str(resolve_recipes_root(settings, context["active_library"]) or ""),
|
||||
"example_images": example_images_payload(settings, context["active_library"]),
|
||||
"cache_root": context["cache_root"],
|
||||
"cache_paths": context["cache_paths"],
|
||||
}
|
||||
|
||||
|
||||
def caches_payload(context: dict[str, Any]) -> dict[str, Any]:
|
||||
caches: dict[str, Any] = {}
|
||||
for name, path_string in context["cache_paths"].items():
|
||||
path = Path(path_string)
|
||||
item: dict[str, Any] = {
|
||||
"path": str(path),
|
||||
"exists": path.exists(),
|
||||
"size": path.stat().st_size if path.exists() else None,
|
||||
}
|
||||
if path.suffix == ".sqlite":
|
||||
item["sqlite"] = sqlite_payload(path, limit=0)
|
||||
elif path.suffix == ".json":
|
||||
item["json"] = json_file_summary(path)
|
||||
caches[name] = item
|
||||
return {"active_library": context["active_library"], "caches": caches}
|
||||
|
||||
|
||||
def recipes_payload(context: dict[str, Any]) -> dict[str, Any]:
|
||||
root = resolve_recipes_root(context["settings"], context["active_library"])
|
||||
files: list[str] = []
|
||||
if root and root.exists():
|
||||
files = [str(path) for path in sorted(root.rglob("*.recipe.json"))[:20]]
|
||||
return {
|
||||
"recipes_root": str(root or ""),
|
||||
"exists": bool(root and root.exists()),
|
||||
"recipe_json_count": count_recipe_files(root),
|
||||
"sample_recipe_json": files,
|
||||
"recipe_cache": context["cache_paths"].get("recipe"),
|
||||
}
|
||||
|
||||
|
||||
def model_payload(raw_path: str) -> dict[str, Any]:
|
||||
path = Path(raw_path).expanduser()
|
||||
metadata_path = path if path.name.endswith(".metadata.json") else path.with_suffix(".metadata.json")
|
||||
payload = {
|
||||
"input_path": str(path),
|
||||
"metadata_path": str(metadata_path),
|
||||
"model_exists": path.exists(),
|
||||
"metadata_exists": metadata_path.exists(),
|
||||
}
|
||||
if metadata_path.exists():
|
||||
data = load_json(metadata_path)
|
||||
payload["metadata_summary"] = redact(summarize_value(data))
|
||||
return payload
|
||||
|
||||
|
||||
def sqlite_payload(path: Path, limit: int = 3, allow_copy: bool = True) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {"path": str(path), "exists": path.exists(), "tables": {}}
|
||||
if not path.exists():
|
||||
return result
|
||||
try:
|
||||
conn = connect_sqlite_readonly(path)
|
||||
except sqlite3.Error as exc:
|
||||
result["error"] = str(exc)
|
||||
return result
|
||||
try:
|
||||
table_rows = conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
|
||||
).fetchall()
|
||||
for table_row in table_rows:
|
||||
table = table_row["name"]
|
||||
columns = [
|
||||
row["name"]
|
||||
for row in conn.execute(f"PRAGMA table_info({quote_identifier(table)})").fetchall()
|
||||
]
|
||||
table_info: dict[str, Any] = {"columns": columns}
|
||||
try:
|
||||
table_info["count"] = conn.execute(
|
||||
f"SELECT COUNT(*) FROM {quote_identifier(table)}"
|
||||
).fetchone()[0]
|
||||
except sqlite3.Error as exc:
|
||||
table_info["count_error"] = str(exc)
|
||||
if limit > 0 and columns and not is_internal_sqlite_table(table):
|
||||
try:
|
||||
rows = conn.execute(
|
||||
f"SELECT * FROM {quote_identifier(table)} LIMIT ?", (limit,)
|
||||
).fetchall()
|
||||
table_info["sample"] = [redact(dict(row)) for row in rows]
|
||||
except sqlite3.Error as exc:
|
||||
table_info["sample_error"] = str(exc)
|
||||
result["tables"][table] = table_info
|
||||
except sqlite3.Error as exc:
|
||||
fallback = sqlite_copy_payload(path, limit, str(exc)) if allow_copy else None
|
||||
if fallback is not None:
|
||||
result.update(fallback)
|
||||
else:
|
||||
result["error"] = str(exc)
|
||||
finally:
|
||||
conn.close()
|
||||
return result
|
||||
|
||||
|
||||
def connect_sqlite_readonly(path: Path) -> sqlite3.Connection:
|
||||
errors: list[str] = []
|
||||
for query in ("mode=ro", "mode=ro&immutable=1"):
|
||||
try:
|
||||
conn = sqlite3.connect(f"file:{path}?{query}", uri=True)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
except sqlite3.Error as exc:
|
||||
errors.append(f"{query}: {exc}")
|
||||
raise sqlite3.OperationalError("; ".join(errors))
|
||||
|
||||
|
||||
def sqlite_copy_payload(path: Path, limit: int, original_error: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
with tempfile.TemporaryDirectory(prefix="lm-cache-inspect-") as temp_dir:
|
||||
copy_path = Path(temp_dir) / path.name
|
||||
shutil.copy2(path, copy_path)
|
||||
payload = sqlite_payload(copy_path, limit, allow_copy=False)
|
||||
payload["path"] = str(path)
|
||||
payload["inspected_copy"] = True
|
||||
payload["original_error"] = original_error
|
||||
return payload
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def json_file_summary(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {"exists": False}
|
||||
data = load_json(path)
|
||||
return {"exists": True, "summary": redact(summarize_value(data))}
|
||||
|
||||
|
||||
def model_roots(settings: dict[str, Any], active_library: str) -> dict[str, list[str]]:
|
||||
roots: dict[str, list[str]] = {}
|
||||
sources = [settings]
|
||||
library = settings.get("libraries", {}).get(active_library)
|
||||
if isinstance(library, dict):
|
||||
sources.insert(0, library)
|
||||
for source in sources:
|
||||
folder_paths = source.get("folder_paths")
|
||||
if isinstance(folder_paths, dict):
|
||||
for key, value in folder_paths.items():
|
||||
roots.setdefault(key, []).extend(normalize_path_list(value))
|
||||
for default_key, folder_key in (
|
||||
("default_lora_root", "loras"),
|
||||
("default_checkpoint_root", "checkpoints"),
|
||||
("default_embedding_root", "embeddings"),
|
||||
("default_unet_root", "unet"),
|
||||
):
|
||||
value = settings.get(default_key)
|
||||
if isinstance(value, str) and value:
|
||||
roots.setdefault(folder_key, []).append(expand_path(value))
|
||||
return {key: dedupe(values) for key, values in roots.items()}
|
||||
|
||||
|
||||
def resolve_recipes_root(settings: dict[str, Any], active_library: str) -> Path | None:
|
||||
recipes_path = settings.get("recipes_path")
|
||||
library = settings.get("libraries", {}).get(active_library)
|
||||
if isinstance(library, dict) and isinstance(library.get("recipes_path"), str):
|
||||
recipes_path = library["recipes_path"] or recipes_path
|
||||
if isinstance(recipes_path, str) and recipes_path.strip():
|
||||
return Path(expand_path(recipes_path.strip()))
|
||||
lora_roots = model_roots(settings, active_library).get("loras") or []
|
||||
return Path(lora_roots[0]) / "recipes" if lora_roots else None
|
||||
|
||||
|
||||
def example_images_payload(settings: dict[str, Any], active_library: str) -> dict[str, Any]:
|
||||
root = settings.get("example_images_path") or ""
|
||||
libraries = settings.get("libraries")
|
||||
library_count = len(libraries) if isinstance(libraries, dict) else 0
|
||||
scoped = library_count > 1
|
||||
root_path = Path(expand_path(root)) if isinstance(root, str) and root else None
|
||||
library_root = root_path / sanitize_library_name(active_library) if root_path and scoped else root_path
|
||||
return {
|
||||
"root": str(root_path or ""),
|
||||
"uses_library_scoped_folders": scoped,
|
||||
"library_root": str(library_root or ""),
|
||||
}
|
||||
|
||||
|
||||
def count_recipe_files(root: Path | None) -> int:
|
||||
if not root or not root.exists():
|
||||
return 0
|
||||
return sum(1 for _ in root.rglob("*.recipe.json"))
|
||||
|
||||
|
||||
def normalize_path_list(value: Any) -> list[str]:
|
||||
if isinstance(value, str):
|
||||
return [expand_path(value)] if value else []
|
||||
if isinstance(value, list):
|
||||
return [expand_path(item) for item in value if isinstance(item, str) and item]
|
||||
return []
|
||||
|
||||
|
||||
def expand_path(value: str) -> str:
|
||||
return str(Path(value).expanduser().resolve(strict=False))
|
||||
|
||||
|
||||
def sanitize_library_name(name: str) -> str:
|
||||
safe = re.sub(r"[^A-Za-z0-9_.-]", "_", name or "default")
|
||||
return safe or "default"
|
||||
|
||||
|
||||
def dedupe(values: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
for value in values:
|
||||
if value not in seen:
|
||||
result.append(value)
|
||||
seen.add(value)
|
||||
return result
|
||||
|
||||
|
||||
def redact(value: Any, key: str = "") -> Any:
|
||||
if key and SECRET_PATTERN.search(key):
|
||||
return "<redacted>"
|
||||
if isinstance(value, dict):
|
||||
return {str(k): redact(v, str(k)) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [redact(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def summarize_value(value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
return {key: summarize_value(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return {
|
||||
"type": "array",
|
||||
"length": len(value),
|
||||
"first": summarize_value(value[0]) if value else None,
|
||||
}
|
||||
return value
|
||||
|
||||
|
||||
def quote_identifier(identifier: str) -> str:
|
||||
return '"' + identifier.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def is_internal_sqlite_table(table: str) -> bool:
|
||||
return table.startswith("sqlite_") or table.endswith(("_data", "_idx", "_docsize", "_config", "_content"))
|
||||
|
||||
|
||||
def print_json(payload: Any) -> None:
|
||||
json.dump(payload, sys.stdout, indent=2, ensure_ascii=False)
|
||||
sys.stdout.write("\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,153 +0,0 @@
|
||||
# Recipe Batch Import Feature Design
|
||||
|
||||
## Overview
|
||||
Enable users to import multiple images as recipes in a single operation, rather than processing them individually. This feature addresses the need for efficient bulk recipe creation from existing image collections.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Frontend │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ BatchImportManager.js │
|
||||
│ ├── InputCollector (收集URL列表/目录路径) │
|
||||
│ ├── ConcurrencyController (自适应并发控制) │
|
||||
│ ├── ProgressTracker (进度追踪) │
|
||||
│ └── ResultAggregator (结果汇总) │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ batch_import_modal.html │
|
||||
│ └── 批量导入UI组件 │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ batch_import_progress.css │
|
||||
│ └── 进度显示样式 │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Backend │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ py/routes/handlers/recipe_handlers.py │
|
||||
│ ├── start_batch_import() - 启动批量导入 │
|
||||
│ ├── get_batch_import_progress() - 查询进度 │
|
||||
│ └── cancel_batch_import() - 取消导入 │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ py/services/batch_import_service.py │
|
||||
│ ├── 自适应并发执行 │
|
||||
│ ├── 结果汇总 │
|
||||
│ └── WebSocket进度广播 │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## API Endpoints
|
||||
|
||||
| 端点 | 方法 | 说明 |
|
||||
|------|------|------|
|
||||
| `/api/lm/recipes/batch-import/start` | POST | 启动批量导入,返回 operation_id |
|
||||
| `/api/lm/recipes/batch-import/progress` | GET | 查询进度状态 |
|
||||
| `/api/lm/recipes/batch-import/cancel` | POST | 取消导入 |
|
||||
|
||||
## Backend Implementation Details
|
||||
|
||||
### BatchImportService
|
||||
|
||||
Location: `py/services/batch_import_service.py`
|
||||
|
||||
Key classes:
|
||||
- `BatchImportItem`: Dataclass for individual import item
|
||||
- `BatchImportProgress`: Dataclass for tracking progress
|
||||
- `BatchImportService`: Main service class
|
||||
|
||||
Features:
|
||||
- Adaptive concurrency control (adjusts based on success/failure rate)
|
||||
- WebSocket progress broadcasting
|
||||
- Graceful error handling (individual failures don't stop the batch)
|
||||
- Result aggregation
|
||||
|
||||
### WebSocket Message Format
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "batch_import_progress",
|
||||
"operation_id": "xxx",
|
||||
"total": 50,
|
||||
"completed": 23,
|
||||
"success": 21,
|
||||
"failed": 2,
|
||||
"skipped": 0,
|
||||
"current_item": "image_024.png",
|
||||
"status": "running"
|
||||
}
|
||||
```
|
||||
|
||||
### Input Types
|
||||
|
||||
1. **URL List**: Array of URLs (http/https)
|
||||
2. **Local Paths**: Array of local file paths
|
||||
3. **Directory**: Path to directory with optional recursive flag
|
||||
|
||||
### Error Handling
|
||||
|
||||
- Invalid URLs/paths: Skip and record error
|
||||
- Download failures: Record error, continue
|
||||
- Metadata extraction failures: Mark as "no metadata"
|
||||
- Duplicate detection: Option to skip duplicates
|
||||
|
||||
## Frontend Implementation Details (TODO)
|
||||
|
||||
### UI Components
|
||||
|
||||
1. **BatchImportModal**: Main modal with tabs for URLs/Directory input
|
||||
2. **ProgressDisplay**: Real-time progress bar and status
|
||||
3. **ResultsSummary**: Final results with success/failure breakdown
|
||||
|
||||
### Adaptive Concurrency Controller
|
||||
|
||||
```javascript
|
||||
class AdaptiveConcurrencyController {
|
||||
constructor(options = {}) {
|
||||
this.minConcurrency = options.minConcurrency || 1;
|
||||
this.maxConcurrency = options.maxConcurrency || 5;
|
||||
this.currentConcurrency = options.initialConcurrency || 3;
|
||||
}
|
||||
|
||||
adjustConcurrency(taskDuration, success) {
|
||||
if (success && taskDuration < 1000 && this.currentConcurrency < this.maxConcurrency) {
|
||||
this.currentConcurrency = Math.min(this.currentConcurrency + 1, this.maxConcurrency);
|
||||
}
|
||||
if (!success || taskDuration > 10000) {
|
||||
this.currentConcurrency = Math.max(this.currentConcurrency - 1, this.minConcurrency);
|
||||
}
|
||||
return this.currentConcurrency;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
Backend (implemented):
|
||||
├── py/services/batch_import_service.py # 后端服务
|
||||
├── py/routes/handlers/batch_import_handler.py # API处理器 (added to recipe_handlers.py)
|
||||
├── tests/services/test_batch_import_service.py # 单元测试
|
||||
└── tests/routes/test_batch_import_routes.py # API集成测试
|
||||
|
||||
Frontend (TODO):
|
||||
├── static/js/managers/BatchImportManager.js # 主管理器
|
||||
├── static/js/managers/batch/ # 子模块
|
||||
│ ├── ConcurrencyController.js # 并发控制
|
||||
│ ├── ProgressTracker.js # 进度追踪
|
||||
│ └── ResultAggregator.js # 结果汇总
|
||||
├── static/css/components/batch-import-modal.css # 样式
|
||||
└── templates/components/batch_import_modal.html # Modal模板
|
||||
```
|
||||
|
||||
## Implementation Status
|
||||
|
||||
- [x] Backend BatchImportService
|
||||
- [x] Backend API handlers
|
||||
- [x] WebSocket progress broadcasting
|
||||
- [x] Unit tests
|
||||
- [x] Integration tests
|
||||
- [ ] Frontend BatchImportManager
|
||||
- [ ] Frontend UI components
|
||||
- [ ] E2E tests
|
||||
3
.github/ISSUE_TEMPLATE/feature_request.md
vendored
3
.github/ISSUE_TEMPLATE/feature_request.md
vendored
@@ -13,8 +13,5 @@ A clear and concise description of what the problem is. Ex. I'm always frustrate
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
||||
|
||||
12
.gitignore
vendored
12
.gitignore
vendored
@@ -12,8 +12,15 @@ coverage/
|
||||
.coverage
|
||||
model_cache/
|
||||
|
||||
# agent
|
||||
# agent / dev tooling
|
||||
.opencode/
|
||||
.claude/
|
||||
.sisyphus/
|
||||
.codex
|
||||
.omo
|
||||
reasonix.toml
|
||||
.reasonix/
|
||||
.codegraph/
|
||||
|
||||
# Vue widgets development cache (but keep build output)
|
||||
vue-widgets/node_modules/
|
||||
@@ -22,3 +29,6 @@ vue-widgets/dist/
|
||||
|
||||
# Hypothesis test cache
|
||||
.hypothesis/
|
||||
|
||||
# Working/research notes (not committed)
|
||||
.docs/
|
||||
|
||||
181
.omo/plans/embeddings-hybrid-approach.md
Normal file
181
.omo/plans/embeddings-hybrid-approach.md
Normal file
@@ -0,0 +1,181 @@
|
||||
# Embeddings Usage Tracking — Hybrid Approach (Plan C)
|
||||
|
||||
> **Status**: Reference document for future implementation
|
||||
> **Current implementation**: Plan A (prompt text parsing only, see `usage_stats.py:_process_embeddings`)
|
||||
> **Next step**: Add Plan B as a supplement when edge-case coverage is needed
|
||||
|
||||
## Problem
|
||||
|
||||
Embeddings in ComfyUI are not loaded through dedicated ComfyUI nodes like LoRAs or
|
||||
Checkpoints. They are resolved during CLIP tokenization when the prompt text contains
|
||||
`embedding:<name>` syntax (see `comfy/sd1_clip.py:SDTokenizer.tokenize_with_weights`).
|
||||
|
||||
This means the existing metadata_collector hook (which intercepts node execution via
|
||||
`_map_node_over_list`) cannot capture embeddings the same way it captures LoRAs and
|
||||
checkpoints — there is no "EmbeddingLoader" node to intercept.
|
||||
|
||||
## Solution Architecture
|
||||
|
||||
The hybrid approach combines **two complementary mechanisms** to capture embedding
|
||||
usage from all possible paths.
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ Plan A (已实现) │
|
||||
│ │
|
||||
│ MetadataRegistry.prompt_metadata["prompts"] │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ _process_embeddings() │
|
||||
│ │ │
|
||||
│ ├─ Iterate all prompt node texts │
|
||||
│ ├─ regex extract "embedding:<name>" │
|
||||
│ ├─ resolve name → sha256 via EmbeddingScanner │
|
||||
│ └─ UsageStats.stats["embeddings"][sha256]++ │
|
||||
│ │
|
||||
│ Coverage: ~95% — all CLIPTextEncode/Flux/etc nodes │
|
||||
│ │
|
||||
│ Gap: Custom nodes that load embeddings programmatically │
|
||||
│ without putting embedding:name in prompt text │
|
||||
└─────────────────────────────────────────────────────────┘
|
||||
|
||||
+
|
||||
↓ (future: enable Plan B when needed)
|
||||
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ Plan B (未来 — monkey-patch) │
|
||||
│ │
|
||||
│ comfy/sd1_clip.py:load_embed() │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ Monkey-patch intercepts EVERY embedding file load │
|
||||
│ │ │
|
||||
│ ├─ Records embedding_name + success/failure │
|
||||
│ ├─ Associates with current prompt_id (via registry)│
|
||||
│ └─ Feeds into UsageStats same as Plan A │
|
||||
│ │
|
||||
│ Coverage: 100% — catches ALL embedding loads │
|
||||
│ │
|
||||
│ Cost: Requires patching into ComfyUI internals │
|
||||
│ (sd1_clip.py, sdxl_clip.py, some text_encoders) │
|
||||
└─────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Plan B Detail — Monkey-patch `load_embed`
|
||||
|
||||
### Target Function
|
||||
|
||||
**`comfy.sd1_clip.load_embed(embedding_name, embedding_directory, embedding_size, embed_key=None)`**
|
||||
at line 415 of `sd1_clip.py`.
|
||||
|
||||
This is the **single choke point** for all embedding file loads in ComfyUI. Every
|
||||
CLIP variant (SD1, SDXL, SD3, Flux) calls this same function.
|
||||
|
||||
### Implementation Sketch
|
||||
|
||||
```python
|
||||
# In metadata_collector/metadata_hook.py (or a new module)
|
||||
import comfy.sd1_clip as sd1_clip
|
||||
|
||||
_original_load_embed = sd1_clip.load_embed
|
||||
|
||||
def _patched_load_embed(embedding_name, embedding_directory, embedding_size, embed_key=None):
|
||||
result = _original_load_embed(
|
||||
embedding_name, embedding_directory, embedding_size, embed_key
|
||||
)
|
||||
if result is not None:
|
||||
_record_embedding_usage(embedding_name)
|
||||
return result
|
||||
|
||||
sd1_clip.load_embed = _patched_load_embed
|
||||
```
|
||||
|
||||
### Prompt ID Association
|
||||
|
||||
The challenge is associating the `load_embed` call with the current `prompt_id`.
|
||||
Options:
|
||||
|
||||
1. **Thread-local / contextvar**: Store current `prompt_id` in a `contextvars.ContextVar`
|
||||
that the metadata_collector sets at the start of each prompt execution.
|
||||
|
||||
2. **MetadataRegistry singleton**: The MetadataRegistry already has `current_prompt_id`.
|
||||
The patch can read it directly since both run in the same thread.
|
||||
|
||||
3. **Lazy aggregation**: Instead of associating with prompt_id at load time, collect
|
||||
all loaded embedding names in a global set during execution, then flush to
|
||||
UsageStats after the prompt completes.
|
||||
|
||||
### Files to Patch
|
||||
|
||||
| File | Function | Coverage |
|
||||
|------|----------|----------|
|
||||
| `comfy/sd1_clip.py:415` | `load_embed()` | Primary — SD1.x, SDXL, SD3, Flux |
|
||||
| `comfy/sdxl_clip.py` | Not needed (calls `sd1_clip.SDTokenizer`) | — |
|
||||
| `comfy/text_encoders/sd3_clip.py` | Not needed (calls `sd1_clip.SDTokenizer`) | — |
|
||||
| `comfy/text_encoders/flux.py` | Not needed (calls `sd1_clip.SDTokenizer`) | — |
|
||||
|
||||
The SD1 tokenizer is the base class for all CLIP variants' tokenizers, so patching
|
||||
`load_embed` covers them all.
|
||||
|
||||
### Edge Cases
|
||||
|
||||
| Edge Case | Plan A | Plan B |
|
||||
|-----------|--------|--------|
|
||||
| `embedding:name` in CLIPTextEncode | ✅ | ✅ |
|
||||
| `embedding:name` in CLIPTextEncodeFlux | ✅ | ✅ |
|
||||
| `embedding:name` in PromptLM (LoRA Manager) | ✅ | ✅ |
|
||||
| `embedding:name` in WAS_Text_to_Conditioning | ✅ | ✅ |
|
||||
| Custom node that loads embedding programmatically | ❌ | ✅ |
|
||||
| Embedding loaded multiple times in same prompt | ✅ (dedup via set) | ✅ (dedup via set) |
|
||||
| Embedding file not found | N/A | ✅ (can log) |
|
||||
| Embedding dimension mismatch | N/A | ✅ (can log) |
|
||||
| Text encoder with non-standard tokenizer (LLaMA, T5...) | Partial | ✅ (if it calls load_embed) |
|
||||
|
||||
## Migration Path: Standalone → Hybrid
|
||||
|
||||
### Phase 1 — Plan A (当前状态)
|
||||
- Prompt text parsing only
|
||||
- No monkey-patching required
|
||||
- Covers all standard workflows
|
||||
|
||||
### Phase 2 — Enable Plan B (未来工作)
|
||||
1. Add monkey-patch of `load_embed` in `metadata_collector/metadata_hook.py` (alongside
|
||||
the existing `_map_node_over_list` hook)
|
||||
2. Collect loaded embedding names in a `set()` on the registry
|
||||
3. In `UsageStats._process_embeddings()`, merge the Plan A results (from prompt text)
|
||||
with the Plan B results (from the patch)
|
||||
4. Add `prompt_data` field on MetadataRegistry to store loaded embeddings per prompt
|
||||
|
||||
### Deduplication
|
||||
|
||||
```python
|
||||
# Merge Plan A + Plan B results in _process_embeddings
|
||||
plan_a_names = extract_from_prompt_texts(prompts_data)
|
||||
plan_b_names = registry.get_loaded_embeddings(prompt_id)
|
||||
|
||||
all_names = plan_a_names | plan_b_names
|
||||
```
|
||||
|
||||
## Testing the Hybrid
|
||||
|
||||
| Scenario | What to verify |
|
||||
|----------|---------------|
|
||||
| Standard `embedding:name` in prompt | Plan A captures it |
|
||||
| Embedding loaded by custom node script | Plan B captures it |
|
||||
| Both paths fire for same embedding | No double-counting (dedup) |
|
||||
| Embedding name resolves to hash | EmbeddingScanner.get_hash_by_filename works |
|
||||
| No embedding scanner available | Graceful skip, no crash |
|
||||
| Missing embedding file | Plan B logs warning, Plan A skips gracefully |
|
||||
| Empty prompt | No crash, no entries |
|
||||
| Standalone mode | Both plans disabled gracefully |
|
||||
|
||||
## Key Files Reference
|
||||
|
||||
| File | Role |
|
||||
|------|------|
|
||||
| `py/utils/usage_stats.py` | Core — `_process_embeddings()` for Plan A |
|
||||
| `py/metadata_collector/constants.py` | `EMBEDDINGS` category constant |
|
||||
| `py/metadata_collector/metadata_hook.py` | Future — monkey-patch for Plan B |
|
||||
| `py/services/embedding_scanner.py` | Hash resolution service |
|
||||
| `py/routes/stats_routes.py` | Already handles `usage_data.get('embeddings', {})` |
|
||||
| `comfy/sd1_clip.py` (ComfyUI) | `load_embed()` — Plan B target |
|
||||
@@ -138,6 +138,13 @@ npm run test:coverage # Generate coverage report
|
||||
- Run `python scripts/sync_translation_keys.py` after adding UI strings to `locales/en.json`
|
||||
- Symlinks require normalized paths
|
||||
|
||||
## Git / Commit Messages
|
||||
|
||||
- Follow the style of recent repository commits when writing commit messages
|
||||
- Prefer the repo's existing `feat(...)`, `fix(...)`, `chore:` style where applicable
|
||||
- If the user has provided a GitHub issue link or issue ID for the task, mention that issue in the commit message, for example `(#871)`
|
||||
- When unrelated local changes exist, stage and commit only the files relevant to the requested task
|
||||
|
||||
## Frontend UI Architecture
|
||||
|
||||
### 1. Standalone Web UI
|
||||
|
||||
25
__init__.py
25
__init__.py
@@ -1,10 +1,13 @@
|
||||
try: # pragma: no cover - import fallback for pytest collection
|
||||
from .py.lora_manager import LoraManager
|
||||
from .py.nodes.lora_loader import LoraLoaderLM, LoraTextLoaderLM
|
||||
from .py.nodes.checkpoint_loader import CheckpointLoaderLM
|
||||
from .py.nodes.unet_loader import UNETLoaderLM
|
||||
from .py.nodes.trigger_word_toggle import TriggerWordToggleLM
|
||||
from .py.nodes.prompt import PromptLM
|
||||
from .py.nodes.text import TextLM
|
||||
from .py.nodes.lora_stacker import LoraStackerLM
|
||||
from .py.nodes.lora_stack_combiner import LoraStackCombinerLM
|
||||
from .py.nodes.save_image import SaveImageLM
|
||||
from .py.nodes.debug_metadata import DebugMetadataLM
|
||||
from .py.nodes.wanvideo_lora_select import WanVideoLoraSelectLM
|
||||
@@ -27,16 +30,19 @@ except (
|
||||
PromptLM = importlib.import_module("py.nodes.prompt").PromptLM
|
||||
TextLM = importlib.import_module("py.nodes.text").TextLM
|
||||
LoraManager = importlib.import_module("py.lora_manager").LoraManager
|
||||
LoraLoaderLM = importlib.import_module(
|
||||
"py.nodes.lora_loader"
|
||||
).LoraLoaderLM
|
||||
LoraTextLoaderLM = importlib.import_module(
|
||||
"py.nodes.lora_loader"
|
||||
).LoraTextLoaderLM
|
||||
LoraLoaderLM = importlib.import_module("py.nodes.lora_loader").LoraLoaderLM
|
||||
LoraTextLoaderLM = importlib.import_module("py.nodes.lora_loader").LoraTextLoaderLM
|
||||
CheckpointLoaderLM = importlib.import_module(
|
||||
"py.nodes.checkpoint_loader"
|
||||
).CheckpointLoaderLM
|
||||
UNETLoaderLM = importlib.import_module("py.nodes.unet_loader").UNETLoaderLM
|
||||
TriggerWordToggleLM = importlib.import_module(
|
||||
"py.nodes.trigger_word_toggle"
|
||||
).TriggerWordToggleLM
|
||||
LoraStackerLM = importlib.import_module("py.nodes.lora_stacker").LoraStackerLM
|
||||
LoraStackCombinerLM = importlib.import_module(
|
||||
"py.nodes.lora_stack_combiner"
|
||||
).LoraStackCombinerLM
|
||||
SaveImageLM = importlib.import_module("py.nodes.save_image").SaveImageLM
|
||||
DebugMetadataLM = importlib.import_module("py.nodes.debug_metadata").DebugMetadataLM
|
||||
WanVideoLoraSelectLM = importlib.import_module(
|
||||
@@ -49,9 +55,7 @@ except (
|
||||
LoraRandomizerLM = importlib.import_module(
|
||||
"py.nodes.lora_randomizer"
|
||||
).LoraRandomizerLM
|
||||
LoraCyclerLM = importlib.import_module(
|
||||
"py.nodes.lora_cycler"
|
||||
).LoraCyclerLM
|
||||
LoraCyclerLM = importlib.import_module("py.nodes.lora_cycler").LoraCyclerLM
|
||||
init_metadata_collector = importlib.import_module("py.metadata_collector").init
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
@@ -59,8 +63,11 @@ NODE_CLASS_MAPPINGS = {
|
||||
TextLM.NAME: TextLM,
|
||||
LoraLoaderLM.NAME: LoraLoaderLM,
|
||||
LoraTextLoaderLM.NAME: LoraTextLoaderLM,
|
||||
CheckpointLoaderLM.NAME: CheckpointLoaderLM,
|
||||
UNETLoaderLM.NAME: UNETLoaderLM,
|
||||
TriggerWordToggleLM.NAME: TriggerWordToggleLM,
|
||||
LoraStackerLM.NAME: LoraStackerLM,
|
||||
LoraStackCombinerLM.NAME: LoraStackCombinerLM,
|
||||
SaveImageLM.NAME: SaveImageLM,
|
||||
DebugMetadataLM.NAME: DebugMetadataLM,
|
||||
WanVideoLoraSelectLM.NAME: WanVideoLoraSelectLM,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,183 +0,0 @@
|
||||
## Overview
|
||||
|
||||
The **LoRA Manager Civitai Extension** is a Browser extension designed to work seamlessly with [LoRA Manager](https://github.com/willmiao/ComfyUI-Lora-Manager) to significantly enhance your browsing experience on [Civitai](https://civitai.com). With this extension, you can:
|
||||
|
||||
✅ Instantly see which models are already present in your local library
|
||||
✅ Download new models with a single click
|
||||
✅ Manage downloads efficiently with queue and parallel download support
|
||||
✅ Keep your downloaded models automatically organized according to your custom settings
|
||||
|
||||

|
||||
|
||||
**Update:** It now also supports browsing on [CivArchive](https://civarchive.com/) (formerly CivitaiArchive).
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
## Why Supporter Access?
|
||||
|
||||
LoRA Manager is built with love for the Stable Diffusion and ComfyUI communities. Your support makes it possible for me to keep improving and maintaining the tool full-time.
|
||||
|
||||
Supporter-exclusive features help ensure the long-term sustainability of LoRA Manager, allowing continuous updates, new features, and better performance for everyone.
|
||||
|
||||
Every contribution directly fuels development and keeps the core LoRA Manager free and open-source. In addition to monthly supporters, one-time donation supporters will also receive a license key, with the duration scaling according to the contribution amount. Thank you for helping keep this project alive and growing. ❤️
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
|
||||
### Supported Browsers & Installation Methods
|
||||
|
||||
| Browser | Installation Method |
|
||||
|--------------------|-------------------------------------------------------------------------------------|
|
||||
| **Google Chrome** | [Chrome Web Store link](https://chromewebstore.google.com/detail/capigligggeijgmocnaflanlbghnamgm?utm_source=item-share-cb) |
|
||||
| **Microsoft Edge** | Install via Chrome Web Store (compatible) |
|
||||
| **Brave Browser** | Install via Chrome Web Store (compatible) |
|
||||
| **Opera** | Install via Chrome Web Store (compatible) |
|
||||
| **Firefox** | <div id="firefox-install" class="install-ok"><a href="https://github.com/willmiao/lm-civitai-extension-firefox/releases/latest/download/extension.xpi">📦 Install Firefox Extension (reviewed and verified by Mozilla)</a></div> |
|
||||
|
||||
For non-Chrome browsers (e.g., Microsoft Edge), you can typically install extensions from the Chrome Web Store by following these steps: open the extension’s Chrome Web Store page, click 'Get extension', then click 'Allow' when prompted to enable installations from other stores, and finally click 'Add extension' to complete the installation.
|
||||
|
||||
---
|
||||
|
||||
## Privacy & Security
|
||||
|
||||
I understand concerns around browser extensions and privacy, and I want to be fully transparent about how the **LM Civitai Extension** works:
|
||||
|
||||
- **Reviewed and Verified**
|
||||
This extension has been **manually reviewed and approved by the Chrome Web Store**. The Firefox version uses the **exact same code** (only the packaging format differs) and has passed **Mozilla’s Add-on review**.
|
||||
|
||||
- **Minimal Network Access**
|
||||
The only external server this extension connects to is:
|
||||
**`https://willmiao.shop`** — used solely for **license validation**.
|
||||
|
||||
It does **not collect, transmit, or store any personal or usage data**.
|
||||
No browsing history, no user IDs, no analytics, no hidden trackers.
|
||||
|
||||
- **Local-Only Model Detection**
|
||||
Model detection and LoRA Manager communication all happen **locally** within your browser, directly interacting with your local LoRA Manager backend.
|
||||
|
||||
I value your trust and are committed to keeping your local setup private and secure. If you have any questions, feel free to reach out!
|
||||
|
||||
---
|
||||
|
||||
## How to Use
|
||||
|
||||
After installing the extension, you'll automatically receive a **7-day trial** to explore all features.
|
||||
|
||||
When the extension is correctly installed and your license is valid:
|
||||
|
||||
- Open **Civitai**, and you'll see visual indicators added by the extension on model cards, showing:
|
||||
- ✅ Models already present in your local library
|
||||
- ⬇️ A download button for models not in your library
|
||||
|
||||
Clicking the download button adds the corresponding model version to the download queue, waiting to be downloaded. You can set up to **5 models to download simultaneously**.
|
||||
|
||||
### Visual Indicators Appear On:
|
||||
|
||||
- **Home Page** — Featured models
|
||||
- **Models Page**
|
||||
- **Creator Profiles** — If the creator has set their models to be visible
|
||||
- **Recommended Resources** — On individual model pages
|
||||
|
||||
### Version Buttons on Model Pages
|
||||
|
||||
On a specific model page, visual indicators also appear on version buttons, showing which versions are already in your local library.
|
||||
|
||||
**Starting from v0.4.8**, model pages use a dedicated download button for better compatibility. When switching to a specific version by clicking a version button:
|
||||
|
||||
- The new **dedicated download button** directly triggers download via **LoRA Manager**
|
||||
- The **original download button** remains unchanged for standard browser downloads
|
||||
|
||||

|
||||
|
||||
### Hide Models Already in Library (Beta)
|
||||
|
||||
**New in v0.4.8**: A new **Hide models already in library (Beta)** option makes it easier to focus on models you haven't added yet. It can be enabled from Settings, or toggled quickly using **Ctrl + Shift + H** (macOS: **Command + Shift + H**).
|
||||
|
||||
### Resources on Image Pages — now shows in-library indicators for image resources plus one-click recipe import
|
||||
|
||||
- **One-Click Import Civitai Image as Recipe** — Import any Civitai image as a recipe with a single click in the Resources Used panel.
|
||||
- **Auto-Queue Missing Assets** — In Settings you can decide if LoRAs or checkpoints referenced by that image should automatically be added to your download queue.
|
||||
- **More Accurate Metadata** — Importing directly from the page is faster than copying inside LM and keeps on-site tags and other metadata perfectly aligned.
|
||||
|
||||

|
||||
|
||||
[](https://github.com/user-attachments/assets/41fd4240-c949-4f83-bde7-8f3124c09494)
|
||||
|
||||
---
|
||||
|
||||
## Model Download Location & LoRA Manager Settings
|
||||
|
||||
To use the **one-click download function**, you must first set:
|
||||
|
||||
- Your **Default LoRAs Root**
|
||||
- Your **Default Checkpoints Root**
|
||||
|
||||
These are set within LoRA Manager's settings.
|
||||
|
||||
When everything is configured, downloaded model files will be placed in:
|
||||
|
||||
`<Default_Models_Root>/<Base_Model_of_the_Model>/<First_Tag_of_the_Model>`
|
||||
|
||||
|
||||
### Update: Default Path Customization (2025-07-21)
|
||||
|
||||
A new setting to customize the default download path has been added in the nightly version. You can now personalize where models are saved when downloading via the LM Civitai Extension.
|
||||
|
||||

|
||||
|
||||
The previous YAML path mapping file will be deprecated—settings will now be unified in settings.json to simplify configuration.
|
||||
|
||||
---
|
||||
|
||||
## Backend Port Configuration
|
||||
|
||||
If your **ComfyUI** or **LoRA Manager** backend is running on a port **other than the default 8188**, you must configure the backend port in the extension's settings.
|
||||
|
||||
After correctly setting and saving the port, you'll see in the extension's header area:
|
||||
- A **Healthy** status with the tooltip: `Connected to LoRA Manager on port xxxx`
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Connecting to a Remote LoRA Manager
|
||||
|
||||
If your LoRA Manager is running on another computer, you can still connect from your browser using port forwarding.
|
||||
|
||||
> **Why can't you set a remote IP directly?**
|
||||
>
|
||||
> For privacy and security, the extension only requests access to `http://127.0.0.1/*`. Supporting remote IPs would require much broader permissions, which may be rejected by browser stores and could raise user concerns.
|
||||
|
||||
**Solution: Port Forwarding with `socat`**
|
||||
|
||||
On your browser computer, run:
|
||||
|
||||
`socat TCP-LISTEN:8188,bind=127.0.0.1,fork TCP:REMOTE.IP.ADDRESS.HERE:8188`
|
||||
|
||||
- Replace `REMOTE.IP.ADDRESS.HERE` with the IP of the machine running LoRA Manager.
|
||||
- Adjust the port if needed.
|
||||
|
||||
This lets the extension connect to `127.0.0.1:8188` as usual, with traffic forwarded to your remote server.
|
||||
|
||||
_Thanks to user **Temikus** for sharing this solution!_
|
||||
|
||||
---
|
||||
|
||||
## Roadmap
|
||||
|
||||
The extension will evolve alongside **LoRA Manager** improvements. Planned features include:
|
||||
|
||||
- [x] Support for **additional model types** (e.g., embeddings)
|
||||
- [x] One-click **Recipe Import**
|
||||
- [x] Display of in-library status for all resources in the **Resources Used** section of the image page
|
||||
- [x] One-click **Auto-organize Models**
|
||||
- [x] **Hide models already in library (Beta)** - Focus on models you haven't added yet
|
||||
|
||||
**Stay tuned — and thank you for your support!**
|
||||
|
||||
---
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
640
locales/de.json
640
locales/de.json
File diff suppressed because it is too large
Load Diff
542
locales/en.json
542
locales/en.json
@@ -14,10 +14,15 @@
|
||||
"backToTop": "Back to top",
|
||||
"settings": "Settings",
|
||||
"help": "Help",
|
||||
"add": "Add"
|
||||
"add": "Add",
|
||||
"close": "Close",
|
||||
"menu": "Menu",
|
||||
"remove": "Remove",
|
||||
"change": "Change"
|
||||
},
|
||||
"status": {
|
||||
"loading": "Loading...",
|
||||
"cancelling": "Cancelling...",
|
||||
"unknown": "Unknown",
|
||||
"date": "Date",
|
||||
"version": "Version",
|
||||
@@ -109,6 +114,7 @@
|
||||
"replacePreview": "Replace Preview",
|
||||
"copyCheckpointName": "Copy checkpoint name",
|
||||
"copyEmbeddingName": "Copy embedding name",
|
||||
"embeddingNameCopied": "Embedding syntax copied",
|
||||
"sendCheckpointToWorkflow": "Send to ComfyUI",
|
||||
"sendEmbeddingToWorkflow": "Send to ComfyUI"
|
||||
},
|
||||
@@ -139,6 +145,10 @@
|
||||
},
|
||||
"usage": {
|
||||
"timesUsed": "Times used"
|
||||
},
|
||||
"footer": {
|
||||
"versionCount": "{count} versions",
|
||||
"viewAllVersions": "View all local versions"
|
||||
}
|
||||
},
|
||||
"globalContextMenu": {
|
||||
@@ -174,6 +184,12 @@
|
||||
"success": "Successfully repaired {count} recipes.",
|
||||
"cancelled": "Repair cancelled. {count} recipes were repaired.",
|
||||
"error": "Recipe repair failed: {message}"
|
||||
},
|
||||
"manageExcludedModels": {
|
||||
"label": "Manage Excluded Models"
|
||||
},
|
||||
"groupByModel": {
|
||||
"label": "Group by Model"
|
||||
}
|
||||
},
|
||||
"header": {
|
||||
@@ -186,13 +202,7 @@
|
||||
"statistics": "Stats"
|
||||
},
|
||||
"search": {
|
||||
"placeholder": "Search...",
|
||||
"placeholders": {
|
||||
"loras": "Search LoRAs...",
|
||||
"recipes": "Search recipes...",
|
||||
"checkpoints": "Search checkpoints...",
|
||||
"embeddings": "Search embeddings..."
|
||||
},
|
||||
"placeholder": "Search",
|
||||
"options": "Search Options",
|
||||
"searchIn": "Search In:",
|
||||
"notAvailable": "Search not available on statistics page",
|
||||
@@ -221,12 +231,17 @@
|
||||
"presetOverwriteConfirm": "Preset \"{name}\" already exists. Overwrite?",
|
||||
"presetNamePlaceholder": "Preset name...",
|
||||
"baseModel": "Base Model",
|
||||
"baseModelSearchPlaceholder": "Search base models...",
|
||||
"modelTags": "Tags (Top 20)",
|
||||
"modelTypes": "Model Types",
|
||||
"license": "License",
|
||||
"noCreditRequired": "No Credit Required",
|
||||
"allowSellingGeneratedContent": "Allow Selling",
|
||||
"allowSellingGeneratedContentTooltip": "Allow selling generated images",
|
||||
"noCreditRequiredTooltip": "Use the model without crediting the creator",
|
||||
"noTags": "No tags",
|
||||
"autoTags": "Auto Tags",
|
||||
"noBaseModelMatches": "No base models match the current search.",
|
||||
"clearAll": "Clear All Filters",
|
||||
"any": "Any",
|
||||
"all": "All",
|
||||
@@ -237,7 +252,18 @@
|
||||
"toggle": "Toggle theme",
|
||||
"switchToLight": "Switch to light theme",
|
||||
"switchToDark": "Switch to dark theme",
|
||||
"switchToAuto": "Switch to auto theme"
|
||||
"switchToAuto": "Switch to auto theme",
|
||||
"presets": "Theme Presets",
|
||||
"default": "Default",
|
||||
"nord": "Nord",
|
||||
"midnight": "Midnight",
|
||||
"monokai": "Monokai",
|
||||
"dracula": "Dracula",
|
||||
"solarized": "Solarized",
|
||||
"mode": "Mode",
|
||||
"light": "Light",
|
||||
"dark": "Dark",
|
||||
"auto": "Auto"
|
||||
},
|
||||
"actions": {
|
||||
"checkUpdates": "Check Updates",
|
||||
@@ -249,6 +275,36 @@
|
||||
"civitaiApiKey": "Civitai API Key",
|
||||
"civitaiApiKeyPlaceholder": "Enter your Civitai API key",
|
||||
"civitaiApiKeyHelp": "Used for authentication when downloading models from Civitai",
|
||||
"civitaiApiKeyConfigured": "Configured",
|
||||
"civitaiApiKeyNotConfigured": "Not configured",
|
||||
"civitaiApiKeySet": "Set up",
|
||||
"civitaiHost": {
|
||||
"label": "Civitai host",
|
||||
"help": "Choose which Civitai site opens when using View on Civitai links.",
|
||||
"options": {
|
||||
"com": "civitai.com (SFW)",
|
||||
"red": "civitai.red (unrestricted)"
|
||||
}
|
||||
},
|
||||
"downloadBackend": {
|
||||
"label": "Download backend",
|
||||
"help": "Choose how model files are downloaded. Python uses the built-in downloader. aria2 uses the recommended external downloader process.",
|
||||
"options": {
|
||||
"python": "Python (built-in)",
|
||||
"aria2": "aria2 (recommended)"
|
||||
}
|
||||
},
|
||||
"aria2cPath": {
|
||||
"label": "aria2c path",
|
||||
"help": "Optional path to the aria2c executable. Leave empty to use aria2c from your system PATH.",
|
||||
"placeholder": "Leave empty to use aria2c from PATH"
|
||||
},
|
||||
"aria2HelpLink": "Learn how to set up the aria2 download backend",
|
||||
"civitaiHostBanner": {
|
||||
"title": "Civitai host preference available",
|
||||
"content": "Civitai now uses civitai.com for SFW content and civitai.red for unrestricted content. You can change which site opens by default in Settings.",
|
||||
"openSettings": "Open Settings"
|
||||
},
|
||||
"openSettingsFileLocation": {
|
||||
"label": "Open settings folder",
|
||||
"tooltip": "Open folder containing settings.json",
|
||||
@@ -259,14 +315,18 @@
|
||||
},
|
||||
"sections": {
|
||||
"contentFiltering": "Content Filtering",
|
||||
"downloads": "Downloads",
|
||||
"videoSettings": "Video Settings",
|
||||
"layoutSettings": "Layout Settings",
|
||||
"licenseIcons": "License Icons",
|
||||
"misc": "Miscellaneous",
|
||||
"backup": "Backups",
|
||||
"folderSettings": "Default Roots",
|
||||
"recipeSettings": "Recipes",
|
||||
"extraFolderPaths": "Extra Folder Paths",
|
||||
"downloadPathTemplates": "Download Path Templates",
|
||||
"priorityTags": "Priority Tags",
|
||||
"updateFlags": "Update Flags",
|
||||
"versionScope": "Version Scope",
|
||||
"exampleImages": "Example Images",
|
||||
"autoOrganize": "Auto-organize",
|
||||
"metadata": "Metadata",
|
||||
@@ -290,7 +350,15 @@
|
||||
"blurNsfwContent": "Blur NSFW Content",
|
||||
"blurNsfwContentHelp": "Blur mature (NSFW) content preview images",
|
||||
"showOnlySfw": "Show Only SFW Results",
|
||||
"showOnlySfwHelp": "Filter out all NSFW content when browsing and searching"
|
||||
"showOnlySfwHelp": "Filter out all NSFW content when browsing and searching",
|
||||
"matureBlurThreshold": "Mature Blur Threshold",
|
||||
"matureBlurThresholdHelp": "Set which rating level starts blur filtering when NSFW blur is enabled.",
|
||||
"matureBlurThresholdOptions": {
|
||||
"pg13": "PG13 and above",
|
||||
"r": "R and above (default)",
|
||||
"x": "X and above",
|
||||
"xxx": "XXX only"
|
||||
}
|
||||
},
|
||||
"videoSettings": {
|
||||
"autoplayOnHover": "Autoplay Videos on Hover",
|
||||
@@ -314,7 +382,57 @@
|
||||
"saveFailed": "Unable to save skip paths: {message}"
|
||||
}
|
||||
},
|
||||
"backup": {
|
||||
"autoEnabled": "Automatic backups",
|
||||
"autoEnabledHelp": "Create a local snapshot once per day and keep the latest snapshots according to the retention policy.",
|
||||
"retention": "Retention count",
|
||||
"retentionHelp": "How many automatic snapshots to keep before older ones are pruned.",
|
||||
"management": "Backup management",
|
||||
"managementHelp": "Export your current user state or restore it from a backup archive.",
|
||||
"scopeHelp": "Backs up your settings, download history, and model update state. It does not include model files or rebuildable caches.",
|
||||
"locationSummary": "Current backup location",
|
||||
"openFolderButton": "Open backup folder",
|
||||
"openFolderSuccess": "Opened backup folder",
|
||||
"openFolderFailed": "Failed to open backup folder",
|
||||
"locationCopied": "Backup path copied to clipboard: {{path}}",
|
||||
"locationClipboardFallback": "Backup path: {{path}}",
|
||||
"exportButton": "Export backup",
|
||||
"exportSuccess": "Backup exported successfully.",
|
||||
"exportFailed": "Failed to export backup: {message}",
|
||||
"importButton": "Import backup",
|
||||
"importConfirm": "Import this backup and overwrite local user state?",
|
||||
"importSuccess": "Backup imported successfully.",
|
||||
"importFailed": "Failed to import backup: {message}",
|
||||
"latestSnapshot": "Latest snapshot",
|
||||
"latestAutoSnapshot": "Latest automatic snapshot",
|
||||
"snapshotCount": "Saved snapshots",
|
||||
"noneAvailable": "No snapshots yet"
|
||||
},
|
||||
"downloadSkipBaseModels": {
|
||||
"label": "Skip downloads for base models",
|
||||
"help": "When enabled, versions using the selected base models will be skipped.",
|
||||
"searchPlaceholder": "Filter base models...",
|
||||
"empty": "No base models match the current search.",
|
||||
"summary": {
|
||||
"none": "None selected",
|
||||
"count": "{count} selected"
|
||||
},
|
||||
"actions": {
|
||||
"edit": "Edit",
|
||||
"collapse": "Collapse",
|
||||
"clear": "Clear"
|
||||
},
|
||||
"validation": {
|
||||
"saveFailed": "Unable to save excluded base models: {message}"
|
||||
}
|
||||
},
|
||||
"skipPreviouslyDownloadedModelVersions": {
|
||||
"label": "Skip previously downloaded model versions",
|
||||
"help": "When enabled, versions downloaded before will be skipped."
|
||||
},
|
||||
"layoutSettings": {
|
||||
"groupByModel": "Group by Model",
|
||||
"groupByModelHelp": "When enabled, only the latest version of each Civitai model is shown as a single card. Older versions are hidden.",
|
||||
"displayDensity": "Display Density",
|
||||
"displayDensityOptions": {
|
||||
"default": "Default",
|
||||
@@ -336,6 +454,8 @@
|
||||
"hover": "Reveal on Hover"
|
||||
},
|
||||
"cardInfoDisplayHelp": "Choose when to display model information and action buttons",
|
||||
"showVersionOnCard": "Show Version on Card",
|
||||
"showVersionOnCardHelp": "Show or hide the version name on model cards",
|
||||
"modelCardFooterAction": "Model Card Button Action",
|
||||
"modelCardFooterActionOptions": {
|
||||
"exampleImages": "Open Example Images",
|
||||
@@ -347,7 +467,9 @@
|
||||
"modelName": "Model Name",
|
||||
"fileName": "File Name"
|
||||
},
|
||||
"modelNameDisplayHelp": "Choose what to display in the model card footer"
|
||||
"modelNameDisplayHelp": "Choose what to display in the model card footer",
|
||||
"cardBlurAmount": "Card Overlay Blur",
|
||||
"cardBlurAmountHelp": "Adjust the blur intensity of the header and footer overlays on model and recipe cards (0 = no blur, 20 = maximum blur)."
|
||||
},
|
||||
"folderSettings": {
|
||||
"activeLibrary": "Active Library",
|
||||
@@ -362,12 +484,16 @@
|
||||
"defaultUnetRootHelp": "Set default diffusion model (UNET) root directory for downloads, imports and moves",
|
||||
"defaultEmbeddingRoot": "Embedding Root",
|
||||
"defaultEmbeddingRootHelp": "Set default embedding root directory for downloads, imports and moves",
|
||||
"recipesPath": "Recipes Storage Path",
|
||||
"recipesPathHelp": "Optional custom directory for stored recipes. Leave empty to use the first LoRA root's recipes folder.",
|
||||
"recipesPathPlaceholder": "/path/to/recipes",
|
||||
"recipesPathMigrating": "Migrating recipes storage...",
|
||||
"noDefault": "No Default"
|
||||
},
|
||||
"extraFolderPaths": {
|
||||
"title": "Extra Folder Paths",
|
||||
"help": "Add additional model folders outside of ComfyUI's standard paths. These paths are stored separately and scanned alongside the default folders.",
|
||||
"description": "Configure additional folders to scan for models. These paths are specific to LoRA Manager and will be merged with ComfyUI's default paths.",
|
||||
"description": "Additional model root paths exclusive to LoRA Manager. Load models from locations outside ComfyUI's standard folders—ideal for large libraries that would otherwise slow down ComfyUI.",
|
||||
"restartRequired": "Requires restart to take effect",
|
||||
"modelTypes": {
|
||||
"lora": "LoRA Paths",
|
||||
"checkpoint": "Checkpoint Paths",
|
||||
@@ -375,7 +501,7 @@
|
||||
"embedding": "Embedding Paths"
|
||||
},
|
||||
"pathPlaceholder": "/path/to/extra/models",
|
||||
"saveSuccess": "Extra folder paths updated.",
|
||||
"saveSuccess": "Extra folder paths updated. Restart required to apply changes.",
|
||||
"saveError": "Failed to update extra folder paths: {message}",
|
||||
"validation": {
|
||||
"duplicatePath": "This path is already configured"
|
||||
@@ -443,26 +569,51 @@
|
||||
"downloadLocationHelp": "Enter the folder path where example images from Civitai will be saved",
|
||||
"autoDownload": "Auto Download Example Images",
|
||||
"autoDownloadHelp": "Automatically download example images for models that don't have them (requires download location to be set)",
|
||||
"openMode": "Open Example Images Action",
|
||||
"openModeHelp": "Choose whether the action opens on the server, copies a mapped local path, or launches a custom URI.",
|
||||
"openModeOptions": {
|
||||
"system": "Open on server",
|
||||
"clipboard": "Copy local path",
|
||||
"uriTemplate": "Open custom URI"
|
||||
},
|
||||
"localRoot": "Local Example Images Root",
|
||||
"localRootHelp": "Optional local or mounted root that mirrors the server example images directory. If blank, the server path is reused.",
|
||||
"localRootPlaceholder": "Example: /Volumes/ComfyUI/example_images",
|
||||
"uriTemplate": "Open URI Template",
|
||||
"uriTemplateHelp": "Use a custom deep link such as a file URI or a Shortcuts link.",
|
||||
"uriTemplatePlaceholder": "Example: shortcuts://run-shortcut?name=Open%20Finder&input=text&text={{encoded_local_path}}",
|
||||
"uriTemplatePlaceholders": "Available placeholders: {{local_path}}, {{encoded_local_path}}, {{relative_path}}, {{encoded_relative_path}}, {{file_uri}}, {{encoded_file_uri}}",
|
||||
"openModeWikiLink": "Learn more about remote open modes",
|
||||
"optimizeImages": "Optimize Downloaded Images",
|
||||
"optimizeImagesHelp": "Optimize example images to reduce file size and improve loading speed (metadata will be preserved)",
|
||||
"download": "Download",
|
||||
"restartRequired": "Requires restart"
|
||||
},
|
||||
"updateFlagStrategy": {
|
||||
"label": "Update Flag Strategy",
|
||||
"help": "Decide whether update badges should only appear when a new release shares the same base model as your local files or whenever any newer version exists for that model.",
|
||||
"versionGrouping": {
|
||||
"label": "Version Grouping",
|
||||
"help": "Decide how versions are grouped for display: by base model or all together. Also controls update badge logic and the VLM version list filtering.",
|
||||
"options": {
|
||||
"sameBase": "Match updates by base model",
|
||||
"any": "Flag any available update"
|
||||
"sameBase": "Group by base model (same_base)",
|
||||
"any": "Show all versions (any)"
|
||||
}
|
||||
},
|
||||
"hideEarlyAccessUpdates": {
|
||||
"label": "Hide Early Access Updates",
|
||||
"help": "When enabled, models with only early access updates will not show 'Update available' badge"
|
||||
},
|
||||
"licenseIcons": {
|
||||
"useNewStyle": "Use updated license icons",
|
||||
"useNewStyleHelp": "Display license permissions with colored indicators (new style) or restriction-only icons (classic style). Mirroring the current CivitAI design."
|
||||
},
|
||||
"misc": {
|
||||
"includeTriggerWords": "Include Trigger Words in LoRA Syntax",
|
||||
"includeTriggerWordsHelp": "Include trained trigger words when copying LoRA syntax to clipboard"
|
||||
"includeTriggerWordsHelp": "Include trained trigger words when copying LoRA syntax to clipboard",
|
||||
"loraSyntaxFormat": "LoRA Syntax Format",
|
||||
"loraSyntaxFormatHelp": "LoRA syntax format. Full includes subfolder path (<lora:style/anime/x:1.0>) for lossless model resolution. Legacy uses filename only (<lora:x:1.0>) — A1111 convention, may be ambiguous with duplicate filenames across folders.",
|
||||
"loraSyntaxFormatOptions": {
|
||||
"full": "Full path (subfolder/name)",
|
||||
"legacy": "Legacy A1111 (name only)"
|
||||
}
|
||||
},
|
||||
"metadataArchive": {
|
||||
"enableArchiveDb": "Enable Metadata Archive Database",
|
||||
@@ -522,12 +673,14 @@
|
||||
"sizeAsc": "Smallest",
|
||||
"usage": "Use Count",
|
||||
"usageDesc": "Most",
|
||||
"usageAsc": "Least"
|
||||
"usageAsc": "Least",
|
||||
"versionsCount": "Local Versions",
|
||||
"versionsCountDesc": "Most versions first",
|
||||
"versionsCountAsc": "Fewest versions first",
|
||||
"versionIdDesc": "Newest version first"
|
||||
},
|
||||
"refresh": {
|
||||
"title": "Refresh model list",
|
||||
"quick": "Sync Changes",
|
||||
"quickTooltip": "Scan for new or missing model files so the list stays current.",
|
||||
"full": "Rebuild Cache",
|
||||
"fullTooltip": "Reload all model details from metadata files—use if the library looks out of date or after manual edits."
|
||||
},
|
||||
@@ -568,15 +721,30 @@
|
||||
"setContentRating": "Set Content Rating for Selected",
|
||||
"copyAll": "Copy Selected Syntax",
|
||||
"refreshAll": "Refresh Selected Metadata",
|
||||
"repairMetadata": "Repair Metadata for Selected",
|
||||
"reimportMetadata": "Re-import from Source",
|
||||
"checkUpdates": "Check Updates for Selected",
|
||||
"moveAll": "Move Selected to Folder",
|
||||
"autoOrganize": "Auto-Organize Selected",
|
||||
"skipMetadataRefresh": "Skip Metadata Refresh for Selected",
|
||||
"resumeMetadataRefresh": "Resume Metadata Refresh for Selected",
|
||||
"deleteAll": "Delete Selected Models",
|
||||
"setFavorite": "Set as Favorite",
|
||||
"setFavoriteCount": "Set as Favorite ({favorited}/{total})",
|
||||
"unfavorite": "Remove from Favorites",
|
||||
"deleteAll": "Delete Selected",
|
||||
"downloadMissingLoras": "Download Missing LoRAs",
|
||||
"downloadExamples": "Download Example Images",
|
||||
"clear": "Clear Selection",
|
||||
"skipMetadataRefreshCount": "Skip ({count} models)",
|
||||
"resumeMetadataRefreshCount": "Resume ({count} models)",
|
||||
"sendToWorkflow": "Send to Workflow",
|
||||
"sections": {
|
||||
"workflow": "Workflow",
|
||||
"metadata": "Metadata",
|
||||
"attributes": "Attributes",
|
||||
"organize": "Organize",
|
||||
"download": "Download"
|
||||
},
|
||||
"autoOrganizeProgress": {
|
||||
"initializing": "Initializing auto-organize...",
|
||||
"starting": "Starting auto-organize for {type}...",
|
||||
@@ -602,7 +770,9 @@
|
||||
"setContentRating": "Set Content Rating",
|
||||
"moveToFolder": "Move to Folder",
|
||||
"repairMetadata": "Repair metadata",
|
||||
"reimportMetadata": "Re-import from Source",
|
||||
"excludeModel": "Exclude Model",
|
||||
"restoreModel": "Restore Model",
|
||||
"deleteModel": "Delete Model",
|
||||
"shareRecipe": "Share Recipe",
|
||||
"viewAllLoras": "View All LoRAs",
|
||||
@@ -621,9 +791,9 @@
|
||||
"title": "Import a recipe from image or URL",
|
||||
"urlLocalPath": "URL / Local Path",
|
||||
"uploadImage": "Upload Image",
|
||||
"urlSectionDescription": "Input a Civitai image URL or local file path to import as a recipe.",
|
||||
"urlSectionDescription": "Input a Civitai image URL from civitai.com or civitai.red, or a local file path, to import as a recipe.",
|
||||
"imageUrlOrPath": "Image URL or File Path:",
|
||||
"urlPlaceholder": "https://civitai.com/images/... or C:/path/to/image.png",
|
||||
"urlPlaceholder": "https://civitai.com/images/... or https://civitai.red/images/... or C:/path/to/image.png",
|
||||
"fetchImage": "Fetch Image",
|
||||
"uploadSectionDescription": "Upload an image with LoRA metadata to import as a recipe.",
|
||||
"selectImage": "Select Image",
|
||||
@@ -644,6 +814,8 @@
|
||||
"root": "Root",
|
||||
"browseFolders": "Browse Folders:",
|
||||
"downloadAndSaveRecipe": "Download & Save Recipe",
|
||||
"importRecipeOnly": "Import Recipe Only",
|
||||
"importAndDownload": "Import & Download",
|
||||
"downloadMissingLoras": "Download Missing LoRAs",
|
||||
"saveRecipe": "Save Recipe",
|
||||
"loraCountInfo": "({existing}/{total} in library)",
|
||||
@@ -686,8 +858,6 @@
|
||||
},
|
||||
"refresh": {
|
||||
"title": "Refresh recipe list",
|
||||
"quick": "Sync Changes",
|
||||
"quickTooltip": "Sync changes - quick refresh without rebuilding cache",
|
||||
"full": "Rebuild Cache",
|
||||
"fullTooltip": "Rebuild cache - full rescan of all recipe files"
|
||||
},
|
||||
@@ -728,6 +898,13 @@
|
||||
"skipped": "Recipe already at latest version, no repair needed",
|
||||
"failed": "Failed to repair recipe: {message}",
|
||||
"missingId": "Cannot repair recipe: Missing recipe ID"
|
||||
},
|
||||
"reimport": {
|
||||
"starting": "Re-importing recipe from source...",
|
||||
"success": "Recipe re-imported successfully",
|
||||
"noSourceUrl": "Recipe has no source URL, cannot re-import",
|
||||
"failed": "Failed to re-import recipe: {message}",
|
||||
"missingId": "Cannot re-import recipe: Missing recipe ID"
|
||||
}
|
||||
},
|
||||
"batchImport": {
|
||||
@@ -796,7 +973,8 @@
|
||||
"diffusion_model": "Diffusion Model"
|
||||
},
|
||||
"contextMenu": {
|
||||
"moveToOtherTypeFolder": "Move to {otherType} Folder"
|
||||
"moveToOtherTypeFolder": "Move to {otherType} Folder",
|
||||
"sendToWorkflow": "Send to Workflow"
|
||||
}
|
||||
},
|
||||
"embeddings": {
|
||||
@@ -805,12 +983,13 @@
|
||||
"sidebar": {
|
||||
"modelRoot": "Root",
|
||||
"collapseAll": "Collapse All Folders",
|
||||
"pinSidebar": "Pin Sidebar",
|
||||
"unpinSidebar": "Unpin Sidebar",
|
||||
"hideOnThisPage": "Hide sidebar on this page",
|
||||
"showSidebar": "Show sidebar",
|
||||
"sidebarHiddenNotification": "Folder sidebar hidden on {page} page",
|
||||
"switchToListView": "Switch to List View",
|
||||
"switchToTreeView": "Switch to Tree View",
|
||||
"recursiveOn": "Search subfolders",
|
||||
"recursiveOff": "Search current folder only",
|
||||
"recursiveOn": "Include subfolders",
|
||||
"recursiveOff": "Current folder only",
|
||||
"recursiveUnavailable": "Recursive search is available in tree view only",
|
||||
"collapseAllDisabled": "Not available in list view",
|
||||
"dragDrop": {
|
||||
@@ -826,6 +1005,13 @@
|
||||
"empty": {
|
||||
"noFolders": "No folders found",
|
||||
"dragHint": "Drag items here to create folders"
|
||||
},
|
||||
"folderUpdateCheck": {
|
||||
"label": "Check for updates in this folder",
|
||||
"loading": "Checking {type} updates for this folder...",
|
||||
"success": "Found {count} update(s) for {type}s in this folder",
|
||||
"none": "All {type}s in this folder are up to date",
|
||||
"error": "Failed to check folder for {type} updates: {message}"
|
||||
}
|
||||
},
|
||||
"statistics": {
|
||||
@@ -837,6 +1023,18 @@
|
||||
"storage": "Storage",
|
||||
"insights": "Insights"
|
||||
},
|
||||
"metrics": {
|
||||
"totalModels": "Total Models",
|
||||
"totalStorage": "Total Storage",
|
||||
"totalGenerations": "Total Generations",
|
||||
"usageRate": "Usage Rate",
|
||||
"loras": "LoRAs",
|
||||
"checkpoints": "Checkpoints",
|
||||
"embeddings": "Embeddings",
|
||||
"uniqueTags": "Unique Tags",
|
||||
"unusedModels": "Unused Models",
|
||||
"avgUsesPerModel": "Avg. Uses/Model"
|
||||
},
|
||||
"usage": {
|
||||
"mostUsedLoras": "Most Used LoRAs",
|
||||
"mostUsedCheckpoints": "Most Used Checkpoints",
|
||||
@@ -854,13 +1052,77 @@
|
||||
},
|
||||
"insights": {
|
||||
"smartInsights": "Smart Insights",
|
||||
"recommendations": "Recommendations"
|
||||
"recommendations": "Recommendations",
|
||||
"noInsights": "No insights available",
|
||||
"unusedLoras": {
|
||||
"high": {
|
||||
"title": "High Number of Unused LoRAs",
|
||||
"description": "{percent}% of your LoRAs ({count}/{total}) have never been used.",
|
||||
"suggestion": "Consider organizing or archiving unused models to free up storage space."
|
||||
}
|
||||
},
|
||||
"unusedCheckpoints": {
|
||||
"detected": {
|
||||
"title": "Unused Checkpoints Detected",
|
||||
"description": "{percent}% of your checkpoints ({count}/{total}) have never been used.",
|
||||
"suggestion": "Review and consider removing checkpoints you no longer need."
|
||||
}
|
||||
},
|
||||
"unusedEmbeddings": {
|
||||
"high": {
|
||||
"title": "High Number of Unused Embeddings",
|
||||
"description": "{percent}% of your embeddings ({count}/{total}) have never been used.",
|
||||
"suggestion": "Consider organizing or archiving unused embeddings to optimize your collection."
|
||||
}
|
||||
},
|
||||
"collection": {
|
||||
"large": {
|
||||
"title": "Large Collection Detected",
|
||||
"description": "Your model collection is using {size} of storage.",
|
||||
"suggestion": "Consider using external storage or cloud solutions for better organization."
|
||||
}
|
||||
},
|
||||
"activity": {
|
||||
"active": {
|
||||
"title": "Active User",
|
||||
"description": "You've completed {count} generations so far!",
|
||||
"suggestion": "Keep exploring and creating amazing content with your models."
|
||||
}
|
||||
}
|
||||
},
|
||||
"charts": {
|
||||
"collectionOverview": "Collection Overview",
|
||||
"baseModelDistribution": "Base Model Distribution",
|
||||
"usageTrends": "Usage Trends (Last 30 Days)",
|
||||
"usageDistribution": "Usage Distribution"
|
||||
"usageDistribution": "Usage Distribution",
|
||||
"date": "Date",
|
||||
"usageCount": "Usage Count",
|
||||
"fileSizeBytes": "File Size (bytes)",
|
||||
"models": "Models",
|
||||
"loraUsage": "LoRA Usage",
|
||||
"checkpointUsage": "Checkpoint Usage",
|
||||
"embeddingUsage": "Embedding Usage"
|
||||
},
|
||||
"modelTypes": {
|
||||
"lora": "LoRA",
|
||||
"locon": "LyCORIS",
|
||||
"dora": "DoRA",
|
||||
"checkpoint": "Checkpoint",
|
||||
"diffusion_model": "Diffusion Model",
|
||||
"embedding": "Embeddings"
|
||||
},
|
||||
"placeholders": {
|
||||
"loading": "Loading...",
|
||||
"noModels": "No models found",
|
||||
"errorLoading": "Error loading data",
|
||||
"noStorageData": "No storage data available",
|
||||
"rootFolder": "Root",
|
||||
"chartLibraryMissing": "Chart requires Chart.js library"
|
||||
},
|
||||
"tooltips": {
|
||||
"tagCount": "{tag}: {count} models",
|
||||
"chartUsage": "{name}: {size}, {count} uses",
|
||||
"chartPercentage": "{label}: {value} ({pct}%)"
|
||||
}
|
||||
},
|
||||
"modals": {
|
||||
@@ -870,9 +1132,9 @@
|
||||
"download": {
|
||||
"title": "Download Model from URL",
|
||||
"titleWithType": "Download {type} from URL",
|
||||
"url": "Civitai URL",
|
||||
"civitaiUrl": "Civitai URL:",
|
||||
"civitaiUrl": "Civitai URL(s):",
|
||||
"placeholder": "https://civitai.com/models/...",
|
||||
"urlHint": "Enter one CivitAI or CivArchive URL per line. Supports multiple URLs for batch download.",
|
||||
"locationPreview": "Download Location Preview",
|
||||
"useDefaultPath": "Use Default Path",
|
||||
"useDefaultPathTooltip": "When enabled, files are automatically organized using configured path templates",
|
||||
@@ -890,8 +1152,15 @@
|
||||
"earlyAccess": "Early Access",
|
||||
"earlyAccessTooltip": "Early access required",
|
||||
"inLibrary": "In Library",
|
||||
"downloaded": "Downloaded",
|
||||
"downloadedTooltip": "Previously downloaded, but it is not currently in your library.",
|
||||
"alreadyInLibrary": "Already in Library",
|
||||
"autoOrganizedPath": "[Auto-organized by path template]",
|
||||
"fileSelection": {
|
||||
"title": "Select File Format",
|
||||
"files": "files",
|
||||
"select": "Select File"
|
||||
},
|
||||
"errors": {
|
||||
"invalidUrl": "Invalid Civitai URL format",
|
||||
"noVersions": "No versions available for this model"
|
||||
@@ -956,6 +1225,12 @@
|
||||
"countMessage": "models will be permanently deleted.",
|
||||
"action": "Delete All"
|
||||
},
|
||||
"bulkDeleteRecipes": {
|
||||
"title": "Delete Multiple Recipes",
|
||||
"message": "Are you sure you want to delete all selected recipes and their associated files?",
|
||||
"countMessage": "recipes will be permanently deleted.",
|
||||
"action": "Delete All"
|
||||
},
|
||||
"checkUpdates": {
|
||||
"title": "Check updates for all {typePlural}?",
|
||||
"message": "This checks every {typePlural} in your library for updates. Large collections may take a little longer.",
|
||||
@@ -980,6 +1255,14 @@
|
||||
"save": "Update Base Model",
|
||||
"cancel": "Cancel"
|
||||
},
|
||||
"bulkDownloadMissingLoras": {
|
||||
"title": "Download Missing LoRAs",
|
||||
"message": "Found {uniqueCount} unique missing LoRAs (from {totalCount} total across selected recipes).",
|
||||
"previewTitle": "LoRAs to download:",
|
||||
"moreItems": "...and {count} more",
|
||||
"note": "Files will be downloaded using default path templates. This may take a while depending on the number of LoRAs.",
|
||||
"downloadButton": "Download {count} LoRA(s)"
|
||||
},
|
||||
"exampleAccess": {
|
||||
"title": "Local Example Images",
|
||||
"message": "No local example images found for this model. View options:",
|
||||
@@ -1013,9 +1296,9 @@
|
||||
},
|
||||
"proceedText": "Only proceed if you're sure this is what you want.",
|
||||
"urlLabel": "Civitai Model URL:",
|
||||
"urlPlaceholder": "https://civitai.com/models/649516/model-name?modelVersionId=726676",
|
||||
"urlPlaceholder": "https://civitai.com/models/649516/model-name?modelVersionId=726676 or https://civitai.red/models/649516/model-name?modelVersionId=726676",
|
||||
"helpText": {
|
||||
"title": "Paste any Civitai model URL. Supported formats:",
|
||||
"title": "Paste any Civitai model URL from civitai.com or civitai.red. Supported formats:",
|
||||
"format1": "https://civitai.com/models/649516",
|
||||
"format2": "https://civitai.com/models/649516?modelVersionId=726676",
|
||||
"format3": "https://civitai.com/models/649516/model-name?modelVersionId=726676",
|
||||
@@ -1028,10 +1311,13 @@
|
||||
"editModelName": "Edit model name",
|
||||
"editFileName": "Edit file name",
|
||||
"editBaseModel": "Edit base model",
|
||||
"editVersionName": "Edit version name",
|
||||
"viewOnCivitai": "View on Civitai",
|
||||
"viewOnCivitaiText": "View on Civitai",
|
||||
"viewCreatorProfile": "View Creator Profile",
|
||||
"openFileLocation": "Open File Location"
|
||||
"openFileLocation": "Open File Location",
|
||||
"sendToWorkflow": "Send to ComfyUI",
|
||||
"sendToWorkflowText": "Send to ComfyUI"
|
||||
},
|
||||
"openFileLocation": {
|
||||
"success": "File location opened successfully",
|
||||
@@ -1039,6 +1325,9 @@
|
||||
"copied": "Path copied to clipboard: {{path}}",
|
||||
"clipboardFallback": "Path: {{path}}"
|
||||
},
|
||||
"sendToWorkflow": {
|
||||
"noFilePath": "Unable to send to ComfyUI: No file path available"
|
||||
},
|
||||
"metadata": {
|
||||
"version": "Version",
|
||||
"fileName": "File Name",
|
||||
@@ -1054,7 +1343,9 @@
|
||||
},
|
||||
"notes": {
|
||||
"saved": "Notes saved successfully",
|
||||
"saveFailed": "Failed to save notes"
|
||||
"saveFailed": "Failed to save notes",
|
||||
"showMore": "Show more",
|
||||
"showLess": "Show less"
|
||||
},
|
||||
"usageTips": {
|
||||
"addPresetParameter": "Add preset parameter...",
|
||||
@@ -1075,6 +1366,8 @@
|
||||
"cancel": "Cancel editing",
|
||||
"save": "Save changes",
|
||||
"addPlaceholder": "Type to add or click suggestions below",
|
||||
"editWord": "Edit trigger word",
|
||||
"editPlaceholder": "Edit trigger word",
|
||||
"copyWord": "Copy trigger word",
|
||||
"deleteWord": "Delete trigger word",
|
||||
"suggestions": {
|
||||
@@ -1146,22 +1439,38 @@
|
||||
"days": "in {count}d"
|
||||
},
|
||||
"badges": {
|
||||
"current": "Current Version",
|
||||
"current": "Opened Version",
|
||||
"currentTooltip": "This is the version you opened this modal from",
|
||||
"inLibrary": "In Library",
|
||||
"inLibraryTooltip": "This version exists in your local library",
|
||||
"downloaded": "Downloaded",
|
||||
"downloadedTooltip": "This version was downloaded before, but is not currently in your library",
|
||||
"newer": "Newer Version",
|
||||
"newerTooltip": "This version is newer than your latest local version",
|
||||
"earlyAccess": "Early Access",
|
||||
"ignored": "Ignored"
|
||||
"earlyAccessTooltip": "This version currently requires Civitai early access",
|
||||
"ignored": "Ignored",
|
||||
"ignoredTooltip": "Update notifications are disabled for this version",
|
||||
"onSiteOnly": "On-Site Only",
|
||||
"onSiteOnlyTooltip": "This version is only available for on-site generation on Civitai"
|
||||
},
|
||||
"actions": {
|
||||
"download": "Download",
|
||||
"downloadTooltip": "Download this version",
|
||||
"downloadEarlyAccessTooltip": "Download this early access version from Civitai",
|
||||
"downloadNotAllowedTooltip": "This version is only available for on-site generation on Civitai",
|
||||
"delete": "Delete",
|
||||
"deleteTooltip": "Delete this local version",
|
||||
"ignore": "Ignore",
|
||||
"unignore": "Unignore",
|
||||
"ignoreTooltip": "Ignore update notifications for this version",
|
||||
"unignoreTooltip": "Resume update notifications for this version",
|
||||
"viewVersionOnCivitai": "View version on Civitai",
|
||||
"earlyAccessTooltip": "Requires early access purchase",
|
||||
"resumeModelUpdates": "Resume updates for this model",
|
||||
"ignoreModelUpdates": "Ignore updates for this model",
|
||||
"viewLocalVersions": "View all local versions",
|
||||
"viewLocalTooltip": "Coming soon"
|
||||
"viewLocalTooltip": "Show all local versions of this model on the main page"
|
||||
},
|
||||
"filters": {
|
||||
"label": "Base filter",
|
||||
@@ -1189,6 +1498,21 @@
|
||||
"versionDeleted": "Version deleted"
|
||||
}
|
||||
}
|
||||
},
|
||||
"metadataFetchSummary": {
|
||||
"title": "Metadata Fetch Summary",
|
||||
"statSuccess": "Success",
|
||||
"statFailed": "Failed",
|
||||
"statSkipped": "Skipped",
|
||||
"statTotal": "Total Scanned",
|
||||
"statDuration": "Duration",
|
||||
"successMessage": "All {count} {type}s updated successfully!",
|
||||
"failedItems": "Failed Items ({count})",
|
||||
"close": "Close",
|
||||
"copyReport": "Copy Report",
|
||||
"downloadCsv": "Download CSV",
|
||||
"columnModelName": "Model Name",
|
||||
"columnError": "Error"
|
||||
}
|
||||
},
|
||||
"modelTags": {
|
||||
@@ -1202,15 +1526,6 @@
|
||||
"duplicate": "This tag already exists"
|
||||
}
|
||||
},
|
||||
"keyboard": {
|
||||
"navigation": "Keyboard Navigation:",
|
||||
"shortcuts": {
|
||||
"pageUp": "Scroll up one page",
|
||||
"pageDown": "Scroll down one page",
|
||||
"home": "Jump to top",
|
||||
"end": "Jump to bottom"
|
||||
}
|
||||
},
|
||||
"initialization": {
|
||||
"title": "Initializing",
|
||||
"message": "Preparing your workspace...",
|
||||
@@ -1296,11 +1611,19 @@
|
||||
"recipeReplaced": "Recipe replaced in workflow",
|
||||
"recipeFailedToSend": "Failed to send recipe to workflow",
|
||||
"noMatchingNodes": "No compatible nodes available in the current workflow",
|
||||
"noTargetNodeSelected": "No target node selected"
|
||||
"noTargetNodeSelected": "No target node selected",
|
||||
"modelUpdated": "Model updated in workflow",
|
||||
"modelFailed": "Failed to update model node",
|
||||
"embeddingAdded": "Embedding added to workflow",
|
||||
"embeddingFailed": "Failed to add embedding",
|
||||
"promptSent": "Prompt sent to workflow",
|
||||
"promptFailed": "Failed to send prompt"
|
||||
},
|
||||
"nodeSelector": {
|
||||
"recipe": "Recipe",
|
||||
"lora": "LoRA",
|
||||
"embedding": "Embedding",
|
||||
"prompt": "Prompt",
|
||||
"replace": "Replace",
|
||||
"append": "Append",
|
||||
"selectTargetNode": "Select target node",
|
||||
@@ -1310,6 +1633,10 @@
|
||||
"opened": "Example images folder opened",
|
||||
"openingFolder": "Opening example images folder",
|
||||
"failedToOpen": "Failed to open example images folder",
|
||||
"copiedPath": "Path copied to clipboard: {{path}}",
|
||||
"clipboardFallback": "Path: {{path}}",
|
||||
"copiedUri": "Link copied to clipboard: {{uri}}",
|
||||
"uriClipboardFallback": "Link: {{uri}}",
|
||||
"setupRequired": "Example Images Storage",
|
||||
"setupDescription": "To add custom example images, you need to set a download location first.",
|
||||
"setupUsage": "This path is used for both downloaded and custom example images.",
|
||||
@@ -1447,6 +1774,7 @@
|
||||
"pleaseSelectVersion": "Please select a version",
|
||||
"versionExists": "This version already exists in your library",
|
||||
"downloadCompleted": "Download completed successfully",
|
||||
"downloadSkippedByBaseModel": "Skipped download because base model {baseModel} is excluded",
|
||||
"autoOrganizeSuccess": "Auto-organize completed successfully for {count} {type}",
|
||||
"autoOrganizePartialSuccess": "Auto-organize completed with {success} moved, {failures} failed out of {total} models",
|
||||
"autoOrganizeFailed": "Auto-organize failed: {error}",
|
||||
@@ -1466,14 +1794,23 @@
|
||||
"nameUpdated": "Recipe name updated successfully",
|
||||
"tagsUpdated": "Recipe tags updated successfully",
|
||||
"sourceUrlUpdated": "Source URL updated successfully",
|
||||
"promptUpdated": "Prompt updated successfully",
|
||||
"negativePromptUpdated": "Negative prompt updated successfully",
|
||||
"promptEditorHint": "Press Enter to save, Shift+Enter for new line",
|
||||
"noRecipeId": "No recipe ID available",
|
||||
"sendToWorkflowFailed": "Failed to send recipe to workflow: {message}",
|
||||
"copyFailed": "Error copying recipe syntax: {message}",
|
||||
"createError": "Error creating recipe: {message}",
|
||||
"createFailed": "Failed to create recipe: {error}",
|
||||
"createMissingData": "Missing required data to create recipe",
|
||||
"created": "Recipe created successfully",
|
||||
"noMissingLoras": "No missing LoRAs to download",
|
||||
"missingLorasInfoFailed": "Failed to get information for missing LoRAs",
|
||||
"preparingForDownloadFailed": "Error preparing LoRAs for download",
|
||||
"enterLoraName": "Please enter a LoRA name or syntax",
|
||||
"reconnectedSuccessfully": "LoRA reconnected successfully",
|
||||
"reconnectFailed": "Error reconnecting LoRA: {message}",
|
||||
"noPromptToSend": "No prompt to send",
|
||||
"cannotSend": "Cannot send recipe: Missing recipe ID",
|
||||
"sendFailed": "Failed to send recipe to workflow",
|
||||
"sendError": "Error sending recipe to workflow",
|
||||
@@ -1494,6 +1831,7 @@
|
||||
"processingError": "Processing error: {message}",
|
||||
"folderBrowserError": "Error loading folder browser: {message}",
|
||||
"recipeSaveFailed": "Failed to save recipe: {error}",
|
||||
"recipeSaved": "Recipe saved successfully",
|
||||
"importFailed": "Import failed: {message}",
|
||||
"folderTreeFailed": "Failed to load folder tree",
|
||||
"folderTreeError": "Error loading folder tree",
|
||||
@@ -1503,7 +1841,17 @@
|
||||
"batchImportNoUrls": "Please enter at least one URL or file path",
|
||||
"batchImportNoDirectory": "Please enter a directory path",
|
||||
"batchImportBrowseFailed": "Failed to browse directory: {message}",
|
||||
"batchImportDirectorySelected": "Directory selected: {path}"
|
||||
"batchImportDirectorySelected": "Directory selected: {path}",
|
||||
"noRecipesSelected": "No recipes selected",
|
||||
"repairBulkComplete": "Repair complete: {repaired} repaired, {skipped} skipped (of {total})",
|
||||
"repairBulkSkipped": "No repair needed for any of the {total} selected recipes",
|
||||
"repairBulkFailed": "Failed to repair selected recipes: {message}",
|
||||
"reimporting": "Re-importing recipe from source...",
|
||||
"reimportSuccess": "Recipe re-imported successfully",
|
||||
"reimportBulkComplete": "Re-import complete: {completed} re-imported, {failed} failed (of {total})",
|
||||
"reimportBulkFailed": "Failed to re-import some recipes",
|
||||
"noMissingLorasInSelection": "No missing LoRAs found in selected recipes",
|
||||
"noLoraRootConfigured": "No LoRA root directory configured. Please set a default LoRA root in settings."
|
||||
},
|
||||
"models": {
|
||||
"noModelsSelected": "No models selected",
|
||||
@@ -1532,6 +1880,11 @@
|
||||
"bulkContentRatingSet": "Set content rating to {level} for {count} model(s)",
|
||||
"bulkContentRatingPartial": "Set content rating to {level} for {success} model(s), {failed} failed",
|
||||
"bulkContentRatingFailed": "Failed to update content rating for selected models",
|
||||
"bulkFavoriteUpdating": "Adding {count} model(s) to favorites...",
|
||||
"bulkUnfavoriteUpdating": "Removing {count} model(s) from favorites...",
|
||||
"bulkFavoritePartialAdded": "Added {success} model(s) to favorites, {failed} failed",
|
||||
"bulkFavoritePartialRemoved": "Removed {success} model(s) from favorites, {failed} failed",
|
||||
"bulkFavoriteFailed": "Failed to update favorite status for selected models",
|
||||
"bulkUpdatesChecking": "Checking selected {type}(s) for updates...",
|
||||
"bulkUpdatesSuccess": "Updates available for {count} selected {type}(s)",
|
||||
"bulkUpdatesNone": "No updates found for selected {type}(s)",
|
||||
@@ -1570,6 +1923,8 @@
|
||||
"mappingSaveFailed": "Failed to save base model mappings: {message}",
|
||||
"downloadTemplatesUpdated": "Download path templates updated",
|
||||
"downloadTemplatesFailed": "Failed to save download path templates: {message}",
|
||||
"recipesPathUpdated": "Recipes storage path updated",
|
||||
"recipesPathSaveFailed": "Failed to update recipes storage path: {message}",
|
||||
"settingsUpdated": "Settings updated: {setting}",
|
||||
"compactModeToggled": "Compact Mode {state}",
|
||||
"settingSaveFailed": "Failed to save setting: {message}",
|
||||
@@ -1620,8 +1975,8 @@
|
||||
},
|
||||
"triggerWords": {
|
||||
"loadFailed": "Could not load trained words",
|
||||
"tooLong": "Trigger word should not exceed 100 words",
|
||||
"tooMany": "Maximum 30 trigger words allowed",
|
||||
"tooLong": "Trigger word should not exceed 500 words",
|
||||
"tooMany": "Maximum 100 trigger words allowed",
|
||||
"alreadyExists": "This trigger word already exists",
|
||||
"updateSuccess": "Trigger words updated successfully",
|
||||
"updateFailed": "Failed to update trigger words",
|
||||
@@ -1682,6 +2037,8 @@
|
||||
"deleteFailed": "Failed to delete {type}: {message}",
|
||||
"excludeSuccess": "{type} excluded successfully",
|
||||
"excludeFailed": "Failed to exclude {type}: {message}",
|
||||
"restoreSuccess": "{type} restored successfully",
|
||||
"restoreFailed": "Failed to restore {type}: {message}",
|
||||
"fileNameUpdated": "File name updated successfully",
|
||||
"fileRenameFailed": "Failed to rename file: {error}",
|
||||
"previewUpdated": "Preview updated successfully",
|
||||
@@ -1710,9 +2067,74 @@
|
||||
"bulkMoveSuccess": "Successfully moved {successCount} {type}s",
|
||||
"exampleImagesDownloadSuccess": "Successfully downloaded example images!",
|
||||
"exampleImagesDownloadFailed": "Failed to download example images: {message}",
|
||||
"moveFailed": "Failed to move item: {message}"
|
||||
"moveFailed": "Failed to move item: {message}",
|
||||
"copiedToClipboard": "Copied to clipboard",
|
||||
"downloadStarted": "Download started"
|
||||
}
|
||||
},
|
||||
"doctor": {
|
||||
"kicker": "System diagnostics",
|
||||
"title": "Doctor",
|
||||
"buttonTitle": "Run diagnostics and common fixes",
|
||||
"loading": "Checking environment...",
|
||||
"footer": "Export a diagnostics bundle if the issue still persists after repair.",
|
||||
"summary": {
|
||||
"idle": "Run a health check for settings, cache integrity, and UI consistency.",
|
||||
"ok": "No active issues were found in the current environment.",
|
||||
"warning": "{count} issue(s) were found. Most can be fixed directly from this panel.",
|
||||
"error": "{count} issue(s) need attention before the app is fully healthy."
|
||||
},
|
||||
"status": {
|
||||
"ok": "Healthy",
|
||||
"warning": "Needs Attention",
|
||||
"error": "Action Required"
|
||||
},
|
||||
"issues": {
|
||||
"civitai_api_key": {
|
||||
"title": "Civitai API Key"
|
||||
},
|
||||
"cache_health": {
|
||||
"title": "Model Cache Health"
|
||||
},
|
||||
"filename_conflicts": {
|
||||
"title": "Duplicate Filename Conflicts"
|
||||
},
|
||||
"ui_version": {
|
||||
"title": "UI Version"
|
||||
}
|
||||
},
|
||||
"actions": {
|
||||
"runAgain": "Run Again",
|
||||
"exportBundle": "Export Bundle",
|
||||
"open-settings": "Open Settings",
|
||||
"open-settings-syntax-format": "Switch to Full Path Syntax",
|
||||
"repair-cache": "Rebuild Cache",
|
||||
"resolve-filename-conflicts": "Resolve Conflicts",
|
||||
"reload-page": "Reload UI"
|
||||
},
|
||||
"labels": {
|
||||
"conflicts": "Conflicts",
|
||||
"version": "Version"
|
||||
},
|
||||
"toast": {
|
||||
"loadFailed": "Failed to load diagnostics: {message}",
|
||||
"repairSuccess": "Cache rebuild completed.",
|
||||
"repairFailed": "Cache rebuild failed: {message}",
|
||||
"exportSuccess": "Diagnostics bundle exported.",
|
||||
"exportFailed": "Failed to export diagnostics bundle: {message}",
|
||||
"conflictsResolved": "{count} filename conflict(s) resolved.",
|
||||
"conflictsResolveFailed": "Failed to resolve filename conflicts: {message}"
|
||||
}
|
||||
},
|
||||
"conflictConfirm": {
|
||||
"title": "Resolve Filename Conflicts",
|
||||
"message": "Renaming by appending a 4-character hash to each duplicate filename.",
|
||||
"note": "This operation renames files on disk. Model references in existing workflows may need updating if you use the A1111 syntax format.",
|
||||
"detail": "Example: <code>filename_v1.2</code> → <code>filename_v1.2-ab3c</code>",
|
||||
"impact": "Will rename <strong>{count}</strong> file(s) across <strong>{groups}</strong> duplicate group(s).",
|
||||
"confirm": "Rename Files",
|
||||
"cancel": "Cancel"
|
||||
},
|
||||
"banners": {
|
||||
"versionMismatch": {
|
||||
"title": "Application Update Detected",
|
||||
@@ -1742,4 +2164,4 @@
|
||||
"retry": "Retry"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
640
locales/es.json
640
locales/es.json
File diff suppressed because it is too large
Load Diff
640
locales/fr.json
640
locales/fr.json
File diff suppressed because it is too large
Load Diff
640
locales/he.json
640
locales/he.json
File diff suppressed because it is too large
Load Diff
640
locales/ja.json
640
locales/ja.json
File diff suppressed because it is too large
Load Diff
640
locales/ko.json
640
locales/ko.json
File diff suppressed because it is too large
Load Diff
640
locales/ru.json
640
locales/ru.json
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
3
package-lock.json
generated
3
package-lock.json
generated
@@ -114,7 +114,6 @@
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
@@ -138,7 +137,6 @@
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
@@ -1613,7 +1611,6 @@
|
||||
"integrity": "sha512-MyL55p3Ut3cXbeBEG7Hcv0mVM8pp8PBNWxRqchZnSfAiES1v1mRnMeFfaHWIPULpwsYfvO+ZmMZz5tGCnjzDUQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"cssstyle": "^4.0.1",
|
||||
"data-urls": "^5.0.0",
|
||||
|
||||
410
py/config.py
410
py/config.py
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
import platform
|
||||
import posixpath
|
||||
import threading
|
||||
from pathlib import Path
|
||||
import folder_paths # type: ignore
|
||||
@@ -25,6 +26,67 @@ standalone_mode = (
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _normalize_root_identity(path: str) -> str:
|
||||
"""Normalize a root path for comparisons across slash styles."""
|
||||
|
||||
normalized = posixpath.normpath(path.strip().replace("\\", "/"))
|
||||
if len(normalized) >= 2 and normalized[1] == ":":
|
||||
return normalized.lower()
|
||||
return normalized
|
||||
|
||||
|
||||
def _resolve_valid_default_root(
|
||||
current: str, primary_paths: List[str], allowed_paths: List[str], name: str
|
||||
) -> str:
|
||||
"""Return a valid default root from the current primary/extra path set."""
|
||||
|
||||
valid_paths = [path for path in primary_paths if isinstance(path, str) and path.strip()]
|
||||
fallback_paths: List[str] = []
|
||||
seen: Set[str] = set()
|
||||
for path in allowed_paths:
|
||||
if not isinstance(path, str):
|
||||
continue
|
||||
stripped = path.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
identity = _normalize_root_identity(stripped)
|
||||
if identity in seen:
|
||||
continue
|
||||
seen.add(identity)
|
||||
fallback_paths.append(stripped)
|
||||
|
||||
allowed = {_normalize_root_identity(path) for path in fallback_paths}
|
||||
|
||||
if current and _normalize_root_identity(current) in allowed:
|
||||
return current
|
||||
|
||||
if not valid_paths:
|
||||
if not fallback_paths:
|
||||
return ""
|
||||
if current:
|
||||
logger.info(
|
||||
"Repaired stale %s from '%s' to '%s' because it is not present in primary or extra roots",
|
||||
name,
|
||||
current,
|
||||
fallback_paths[0],
|
||||
)
|
||||
else:
|
||||
logger.info("Auto-setting %s to '%s'", name, fallback_paths[0])
|
||||
return fallback_paths[0]
|
||||
|
||||
if current:
|
||||
logger.info(
|
||||
"Repaired stale %s from '%s' to '%s' because it is not present in primary or extra roots",
|
||||
name,
|
||||
current,
|
||||
valid_paths[0],
|
||||
)
|
||||
else:
|
||||
logger.info("Auto-setting %s to '%s'", name, valid_paths[0])
|
||||
|
||||
return valid_paths[0]
|
||||
|
||||
|
||||
def _normalize_folder_paths_for_comparison(
|
||||
folder_paths: Mapping[str, Iterable[str]],
|
||||
) -> Dict[str, Set[str]]:
|
||||
@@ -109,6 +171,13 @@ class Config:
|
||||
self.extra_checkpoints_roots: List[str] = []
|
||||
self.extra_unet_roots: List[str] = []
|
||||
self.extra_embeddings_roots: List[str] = []
|
||||
self.recipes_path: str = ""
|
||||
|
||||
# Load extra folder paths from active library settings before symlink scan
|
||||
# so both primary and extra paths are discovered in a single pass.
|
||||
if not standalone_mode:
|
||||
self._load_extra_paths_from_settings()
|
||||
|
||||
# Scan symbolic links during initialization
|
||||
self._initialize_symlink_mappings()
|
||||
|
||||
@@ -116,6 +185,96 @@ class Config:
|
||||
# Save the paths to settings.json when running in ComfyUI mode
|
||||
self.save_folder_paths_to_settings()
|
||||
|
||||
def _load_extra_paths_from_settings(self) -> None:
|
||||
"""Read extra folder paths from the active library and apply them.
|
||||
|
||||
Called during ``Config.__init__`` before the symlink scan so both primary and
|
||||
extra paths are discovered in a single pass. Mirrors the extra-path
|
||||
portion of ``_apply_library_paths`` without replacing the primary roots
|
||||
that were already resolved from ComfyUI's ``folder_paths``.
|
||||
"""
|
||||
try:
|
||||
from .services.settings_manager import get_settings_manager
|
||||
|
||||
settings_manager = get_settings_manager()
|
||||
library_name = settings_manager.get_active_library_name()
|
||||
libraries = settings_manager.get_libraries()
|
||||
|
||||
if not library_name or library_name not in libraries:
|
||||
return
|
||||
|
||||
library_config = libraries[library_name]
|
||||
if not isinstance(library_config, dict):
|
||||
return
|
||||
|
||||
extra_folder_paths = library_config.get("extra_folder_paths")
|
||||
if not isinstance(extra_folder_paths, dict):
|
||||
return
|
||||
|
||||
extra_lora = extra_folder_paths.get("loras", []) or []
|
||||
extra_checkpoint = extra_folder_paths.get("checkpoints", []) or []
|
||||
extra_unet = extra_folder_paths.get("unet", []) or []
|
||||
extra_embedding = extra_folder_paths.get("embeddings", []) or []
|
||||
|
||||
if not any([extra_lora, extra_checkpoint, extra_unet, extra_embedding]):
|
||||
return
|
||||
|
||||
filtered_extra_lora = self._filter_overlapping_extra_lora_paths(
|
||||
self.loras_roots, extra_lora
|
||||
)
|
||||
self.extra_loras_roots = self._prepare_lora_paths(filtered_extra_lora)
|
||||
(
|
||||
_,
|
||||
self.extra_checkpoints_roots,
|
||||
self.extra_unet_roots,
|
||||
) = self._prepare_checkpoint_paths(extra_checkpoint, extra_unet)
|
||||
self.extra_embeddings_roots = self._prepare_embedding_paths(
|
||||
extra_embedding
|
||||
)
|
||||
|
||||
recipes_path = library_config.get("recipes_path", "")
|
||||
if isinstance(recipes_path, str) and recipes_path:
|
||||
self.recipes_path = recipes_path
|
||||
|
||||
if self.extra_loras_roots:
|
||||
logger.info(
|
||||
"Found extra LoRA roots:"
|
||||
+ "\n - "
|
||||
+ "\n - ".join(self.extra_loras_roots)
|
||||
)
|
||||
if self.extra_checkpoints_roots:
|
||||
logger.info(
|
||||
"Found extra checkpoint roots:"
|
||||
+ "\n - "
|
||||
+ "\n - ".join(self.extra_checkpoints_roots)
|
||||
)
|
||||
if self.extra_unet_roots:
|
||||
logger.info(
|
||||
"Found extra diffusion model roots:"
|
||||
+ "\n - "
|
||||
+ "\n - ".join(self.extra_unet_roots)
|
||||
)
|
||||
if self.extra_embeddings_roots:
|
||||
logger.info(
|
||||
"Found extra embedding roots:"
|
||||
+ "\n - "
|
||||
+ "\n - ".join(self.extra_embeddings_roots)
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Applied library settings for '%s' with extra paths: loras=%s, "
|
||||
"checkpoints=%s, embeddings=%s",
|
||||
library_name,
|
||||
extra_lora,
|
||||
extra_checkpoint,
|
||||
extra_embedding,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Could not load extra paths from library settings: %s", exc
|
||||
)
|
||||
|
||||
def save_folder_paths_to_settings(self):
|
||||
"""Persist ComfyUI-derived folder paths to the multi-library settings."""
|
||||
try:
|
||||
@@ -197,44 +356,79 @@ class Config:
|
||||
"Failed to rename legacy 'default' library: %s", rename_error
|
||||
)
|
||||
|
||||
default_lora_root = comfy_library.get("default_lora_root", "")
|
||||
if not default_lora_root and len(self.loras_roots) == 1:
|
||||
default_lora_root = self.loras_roots[0]
|
||||
default_lora_root = _resolve_valid_default_root(
|
||||
comfy_library.get("default_lora_root", ""),
|
||||
list(self.loras_roots or []),
|
||||
list(self.loras_roots or [])
|
||||
+ list(comfy_library.get("extra_folder_paths", {}).get("loras", []) or []),
|
||||
"default_lora_root",
|
||||
)
|
||||
|
||||
default_checkpoint_root = comfy_library.get("default_checkpoint_root", "")
|
||||
if (
|
||||
not default_checkpoint_root
|
||||
and self.checkpoints_roots
|
||||
and len(self.checkpoints_roots) == 1
|
||||
):
|
||||
default_checkpoint_root = self.checkpoints_roots[0]
|
||||
default_checkpoint_root = _resolve_valid_default_root(
|
||||
comfy_library.get("default_checkpoint_root", ""),
|
||||
list(self.checkpoints_roots or []),
|
||||
list(self.checkpoints_roots or [])
|
||||
+ list(comfy_library.get("extra_folder_paths", {}).get("checkpoints", []) or []),
|
||||
"default_checkpoint_root",
|
||||
)
|
||||
|
||||
default_embedding_root = comfy_library.get("default_embedding_root", "")
|
||||
if (
|
||||
not default_embedding_root
|
||||
and self.embeddings_roots
|
||||
and len(self.embeddings_roots) == 1
|
||||
):
|
||||
default_embedding_root = self.embeddings_roots[0]
|
||||
default_embedding_root = _resolve_valid_default_root(
|
||||
comfy_library.get("default_embedding_root", ""),
|
||||
list(self.embeddings_roots or []),
|
||||
list(self.embeddings_roots or [])
|
||||
+ list(comfy_library.get("extra_folder_paths", {}).get("embeddings", []) or []),
|
||||
"default_embedding_root",
|
||||
)
|
||||
|
||||
metadata = dict(comfy_library.get("metadata", {}))
|
||||
metadata.setdefault("display_name", "ComfyUI")
|
||||
metadata["source"] = "comfyui"
|
||||
extra_folder_paths = {}
|
||||
if isinstance(comfy_library, Mapping):
|
||||
existing_extra_paths = comfy_library.get("extra_folder_paths", {})
|
||||
if isinstance(existing_extra_paths, Mapping):
|
||||
extra_folder_paths = {
|
||||
key: list(value) if isinstance(value, list) else []
|
||||
for key, value in existing_extra_paths.items()
|
||||
}
|
||||
|
||||
active_library_name = settings_service.get_active_library_name()
|
||||
should_activate = (
|
||||
active_library_name == "comfyui"
|
||||
or self._should_activate_comfy_library(libraries, libraries_changed)
|
||||
)
|
||||
|
||||
settings_service.upsert_library(
|
||||
"comfyui",
|
||||
folder_paths=target_folder_paths,
|
||||
extra_folder_paths=extra_folder_paths,
|
||||
default_lora_root=default_lora_root,
|
||||
default_checkpoint_root=default_checkpoint_root,
|
||||
default_embedding_root=default_embedding_root,
|
||||
metadata=metadata,
|
||||
activate=True,
|
||||
activate=should_activate,
|
||||
)
|
||||
|
||||
logger.info("Updated 'comfyui' library with current folder paths")
|
||||
if should_activate:
|
||||
logger.info("Updated 'comfyui' library with current folder paths")
|
||||
else:
|
||||
logger.info(
|
||||
"Updated 'comfyui' library with current folder paths without activating it"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save folder paths: {e}")
|
||||
|
||||
def _should_activate_comfy_library(
|
||||
self, libraries: Mapping[str, Any], libraries_changed: bool
|
||||
) -> bool:
|
||||
"""Return whether startup sync should make the ComfyUI library active."""
|
||||
|
||||
if libraries_changed:
|
||||
return True
|
||||
if not libraries:
|
||||
return True
|
||||
return "comfyui" in libraries and len(libraries) == 1
|
||||
|
||||
def _is_link(self, path: str) -> bool:
|
||||
try:
|
||||
if os.path.islink(path):
|
||||
@@ -629,6 +823,8 @@ class Config:
|
||||
preview_roots.update(self._expand_preview_root(root))
|
||||
for root in self.extra_embeddings_roots or []:
|
||||
preview_roots.update(self._expand_preview_root(root))
|
||||
if self.recipes_path:
|
||||
preview_roots.update(self._expand_preview_root(self.recipes_path))
|
||||
|
||||
for target, link in self._path_mappings.items():
|
||||
preview_roots.update(self._expand_preview_root(target))
|
||||
@@ -705,9 +901,131 @@ class Config:
|
||||
|
||||
return unique_paths
|
||||
|
||||
@staticmethod
|
||||
def _normalize_path_for_comparison(
|
||||
path: str, *, resolve_realpath: bool = False
|
||||
) -> str:
|
||||
"""Normalize a path for equality checks across platforms."""
|
||||
candidate = os.path.realpath(path) if resolve_realpath else path
|
||||
return os.path.normcase(os.path.normpath(candidate)).replace(os.sep, "/")
|
||||
|
||||
def _filter_overlapping_extra_lora_paths(
|
||||
self,
|
||||
primary_paths: Iterable[str],
|
||||
extra_paths: Iterable[str],
|
||||
) -> List[str]:
|
||||
"""Drop extra LoRA paths that resolve to the same physical location as primary roots."""
|
||||
|
||||
primary_map = {
|
||||
self._normalize_path_for_comparison(path, resolve_realpath=True): path
|
||||
for path in primary_paths
|
||||
if isinstance(path, str) and path.strip() and os.path.exists(path)
|
||||
}
|
||||
primary_symlink_map = self._collect_first_level_symlink_targets(primary_paths)
|
||||
filtered: List[str] = []
|
||||
|
||||
for original_path in extra_paths:
|
||||
if not isinstance(original_path, str):
|
||||
continue
|
||||
|
||||
stripped = original_path.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
if not os.path.exists(stripped):
|
||||
continue
|
||||
|
||||
real_path = self._normalize_path_for_comparison(
|
||||
stripped,
|
||||
resolve_realpath=True,
|
||||
)
|
||||
normalized_path = os.path.normpath(stripped).replace(os.sep, "/")
|
||||
primary_path = primary_map.get(real_path)
|
||||
if primary_path:
|
||||
# Config loading should stay tolerant of existing invalid state and warn.
|
||||
logger.warning(
|
||||
"Detected the same LoRA folder in both ComfyUI model paths and "
|
||||
"LoRA Manager Extra Folder Paths. This can cause duplicate items or "
|
||||
"other unexpected behavior, and it usually means the path setup is "
|
||||
"not doing what you intended. LoRA Manager will keep the ComfyUI "
|
||||
"path and ignore this Extra Folder Paths entry: '%s'. Please review "
|
||||
"your path settings and remove the duplicate entry.",
|
||||
normalized_path,
|
||||
)
|
||||
continue
|
||||
|
||||
symlink_path = primary_symlink_map.get(real_path)
|
||||
if symlink_path:
|
||||
# Config loading should stay tolerant of existing invalid state and warn.
|
||||
logger.warning(
|
||||
"Detected the same LoRA folder in both ComfyUI model paths and "
|
||||
"LoRA Manager Extra Folder Paths. This can cause duplicate items or "
|
||||
"other unexpected behavior, and it usually means the path setup is "
|
||||
"not doing what you intended. LoRA Manager will keep the ComfyUI "
|
||||
"path and ignore this Extra Folder Paths entry: '%s'. Please review "
|
||||
"your path settings and remove the duplicate entry.",
|
||||
normalized_path,
|
||||
)
|
||||
continue
|
||||
|
||||
filtered.append(stripped)
|
||||
|
||||
return filtered
|
||||
|
||||
def _collect_first_level_symlink_targets(
|
||||
self, roots: Iterable[str]
|
||||
) -> Dict[str, str]:
|
||||
"""Return real-path -> link-path mappings for first-level symlinks under the given roots."""
|
||||
|
||||
targets: Dict[str, str] = {}
|
||||
for root in roots:
|
||||
if not isinstance(root, str):
|
||||
continue
|
||||
stripped_root = root.strip()
|
||||
if not stripped_root or not os.path.isdir(stripped_root):
|
||||
continue
|
||||
|
||||
try:
|
||||
with os.scandir(stripped_root) as iterator:
|
||||
for entry in iterator:
|
||||
try:
|
||||
if not self._entry_is_symlink(entry):
|
||||
continue
|
||||
target_path = os.path.realpath(entry.path)
|
||||
if not os.path.isdir(target_path):
|
||||
continue
|
||||
|
||||
normalized_target = self._normalize_path_for_comparison(
|
||||
target_path,
|
||||
resolve_realpath=True,
|
||||
)
|
||||
normalized_link = os.path.normpath(entry.path).replace(
|
||||
os.sep, "/"
|
||||
)
|
||||
targets.setdefault(normalized_target, normalized_link)
|
||||
except Exception as inner_exc:
|
||||
logger.debug(
|
||||
"Error collecting LoRA symlink target for %s: %s",
|
||||
entry.path,
|
||||
inner_exc,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Error scanning first-level LoRA symlinks in %s: %s",
|
||||
stripped_root,
|
||||
exc,
|
||||
)
|
||||
|
||||
return targets
|
||||
|
||||
def _prepare_checkpoint_paths(
|
||||
self, checkpoint_paths: Iterable[str], unet_paths: Iterable[str]
|
||||
) -> List[str]:
|
||||
) -> Tuple[List[str], List[str], List[str]]:
|
||||
"""Prepare checkpoint paths and return (all_roots, checkpoint_roots, unet_roots).
|
||||
|
||||
Returns:
|
||||
Tuple of (all_unique_paths, checkpoint_only_paths, unet_only_paths)
|
||||
This method does NOT modify instance variables - callers must set them.
|
||||
"""
|
||||
checkpoint_map = self._dedupe_existing_paths(checkpoint_paths)
|
||||
unet_map = self._dedupe_existing_paths(unet_paths)
|
||||
|
||||
@@ -737,8 +1055,8 @@ class Config:
|
||||
|
||||
checkpoint_values = set(checkpoint_map.values())
|
||||
unet_values = set(unet_map.values())
|
||||
self.checkpoints_roots = [p for p in unique_paths if p in checkpoint_values]
|
||||
self.unet_roots = [p for p in unique_paths if p in unet_values]
|
||||
checkpoint_roots = [p for p in unique_paths if p in checkpoint_values]
|
||||
unet_roots = [p for p in unique_paths if p in unet_values]
|
||||
|
||||
for original_path in unique_paths:
|
||||
real_path = os.path.normpath(os.path.realpath(original_path)).replace(
|
||||
@@ -747,7 +1065,7 @@ class Config:
|
||||
if real_path != original_path:
|
||||
self.add_path_mapping(original_path, real_path)
|
||||
|
||||
return unique_paths
|
||||
return unique_paths, checkpoint_roots, unet_roots
|
||||
|
||||
def _prepare_embedding_paths(self, raw_paths: Iterable[str]) -> List[str]:
|
||||
path_map = self._dedupe_existing_paths(raw_paths)
|
||||
@@ -766,9 +1084,11 @@ class Config:
|
||||
self,
|
||||
folder_paths: Mapping[str, Iterable[str]],
|
||||
extra_folder_paths: Optional[Mapping[str, Iterable[str]]] = None,
|
||||
recipes_path: str = "",
|
||||
) -> None:
|
||||
self._path_mappings.clear()
|
||||
self._preview_root_paths = set()
|
||||
self.recipes_path = recipes_path if isinstance(recipes_path, str) else ""
|
||||
|
||||
lora_paths = folder_paths.get("loras", []) or []
|
||||
checkpoint_paths = folder_paths.get("checkpoints", []) or []
|
||||
@@ -776,9 +1096,11 @@ class Config:
|
||||
embedding_paths = folder_paths.get("embeddings", []) or []
|
||||
|
||||
self.loras_roots = self._prepare_lora_paths(lora_paths)
|
||||
self.base_models_roots = self._prepare_checkpoint_paths(
|
||||
checkpoint_paths, unet_paths
|
||||
)
|
||||
(
|
||||
self.base_models_roots,
|
||||
self.checkpoints_roots,
|
||||
self.unet_roots,
|
||||
) = self._prepare_checkpoint_paths(checkpoint_paths, unet_paths)
|
||||
self.embeddings_roots = self._prepare_embedding_paths(embedding_paths)
|
||||
|
||||
# Process extra paths (only for LoRA Manager, not shared with ComfyUI)
|
||||
@@ -788,19 +1110,16 @@ class Config:
|
||||
extra_unet_paths = extra_paths.get("unet", []) or []
|
||||
extra_embedding_paths = extra_paths.get("embeddings", []) or []
|
||||
|
||||
self.extra_loras_roots = self._prepare_lora_paths(extra_lora_paths)
|
||||
# Save main paths before processing extra paths ( _prepare_checkpoint_paths overwrites them)
|
||||
saved_checkpoints_roots = self.checkpoints_roots
|
||||
saved_unet_roots = self.unet_roots
|
||||
self.extra_checkpoints_roots = self._prepare_checkpoint_paths(
|
||||
extra_checkpoint_paths, extra_unet_paths
|
||||
filtered_extra_lora_paths = self._filter_overlapping_extra_lora_paths(
|
||||
self.loras_roots,
|
||||
extra_lora_paths,
|
||||
)
|
||||
self.extra_unet_roots = (
|
||||
self.unet_roots if self.unet_roots is not None else []
|
||||
) # unet_roots was set by _prepare_checkpoint_paths
|
||||
# Restore main paths
|
||||
self.checkpoints_roots = saved_checkpoints_roots
|
||||
self.unet_roots = saved_unet_roots
|
||||
self.extra_loras_roots = self._prepare_lora_paths(filtered_extra_lora_paths)
|
||||
(
|
||||
_,
|
||||
self.extra_checkpoints_roots,
|
||||
self.extra_unet_roots,
|
||||
) = self._prepare_checkpoint_paths(extra_checkpoint_paths, extra_unet_paths)
|
||||
self.extra_embeddings_roots = self._prepare_embedding_paths(
|
||||
extra_embedding_paths
|
||||
)
|
||||
@@ -857,9 +1176,11 @@ class Config:
|
||||
try:
|
||||
raw_checkpoint_paths = folder_paths.get_folder_paths("checkpoints")
|
||||
raw_unet_paths = folder_paths.get_folder_paths("unet")
|
||||
unique_paths = self._prepare_checkpoint_paths(
|
||||
raw_checkpoint_paths, raw_unet_paths
|
||||
)
|
||||
(
|
||||
unique_paths,
|
||||
self.checkpoints_roots,
|
||||
self.unet_roots,
|
||||
) = self._prepare_checkpoint_paths(raw_checkpoint_paths, raw_unet_paths)
|
||||
|
||||
logger.info(
|
||||
"Found checkpoint roots:"
|
||||
@@ -1023,7 +1344,12 @@ class Config:
|
||||
if not isinstance(extra_folder_paths, Mapping):
|
||||
extra_folder_paths = None
|
||||
|
||||
self._apply_library_paths(folder_paths, extra_folder_paths)
|
||||
recipes_path = (
|
||||
str(library_config.get("recipes_path", ""))
|
||||
if isinstance(library_config, Mapping)
|
||||
else ""
|
||||
)
|
||||
self._apply_library_paths(folder_paths, extra_folder_paths, recipes_path)
|
||||
|
||||
logger.info(
|
||||
"Applied library settings with %d lora roots (%d extra), %d checkpoint roots (%d extra), and %d embedding roots (%d extra)",
|
||||
|
||||
@@ -33,6 +33,7 @@ from .utils.example_images_migration import ExampleImagesMigration
|
||||
from .services.websocket_manager import ws_manager
|
||||
from .services.example_images_cleanup_service import ExampleImagesCleanupService
|
||||
from .middleware.csp_middleware import relax_csp_for_remote_media
|
||||
from .middleware.error_middleware import api_json_error
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -76,6 +77,11 @@ class LoraManager:
|
||||
"""Initialize and register all routes using the new refactored architecture"""
|
||||
app = PromptServer.instance.app
|
||||
|
||||
# Register JSON error middleware for /api/* routes as the outermost
|
||||
# middleware so it catches errors from all other middlewares.
|
||||
if api_json_error not in app.middlewares:
|
||||
app.middlewares.insert(0, api_json_error)
|
||||
|
||||
if relax_csp_for_remote_media not in app.middlewares:
|
||||
# Ensure CSP relaxer executes after ComfyUI's block_external_middleware so it can
|
||||
# see and extend the restrictive header instead of being overwritten by it.
|
||||
@@ -184,45 +190,17 @@ class LoraManager:
|
||||
async def _initialize_services(cls):
|
||||
"""Initialize all services using the ServiceRegistry"""
|
||||
try:
|
||||
# Apply library settings to load extra folder paths before scanning
|
||||
# Only apply if extra paths haven't been loaded yet (preserves test mocks)
|
||||
try:
|
||||
from .services.settings_manager import get_settings_manager
|
||||
|
||||
settings_manager = get_settings_manager()
|
||||
library_name = settings_manager.get_active_library_name()
|
||||
libraries = settings_manager.get_libraries()
|
||||
if library_name and library_name in libraries:
|
||||
library_config = libraries[library_name]
|
||||
# Only apply settings if extra paths are not already configured
|
||||
# This preserves values set by tests via monkeypatch
|
||||
extra_paths = library_config.get("extra_folder_paths", {})
|
||||
has_extra_paths = (
|
||||
config.extra_loras_roots
|
||||
or config.extra_checkpoints_roots
|
||||
or config.extra_unet_roots
|
||||
or config.extra_embeddings_roots
|
||||
)
|
||||
if not has_extra_paths and any(extra_paths.values()):
|
||||
config.apply_library_settings(library_config)
|
||||
logger.info(
|
||||
"Applied library settings for '%s' with extra paths: loras=%s, checkpoints=%s, embeddings=%s",
|
||||
library_name,
|
||||
extra_paths.get("loras", []),
|
||||
extra_paths.get("checkpoints", []),
|
||||
extra_paths.get("embeddings", []),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to apply library settings during initialization: %s", exc
|
||||
)
|
||||
|
||||
# Initialize CivitaiClient first to ensure it's ready for other services
|
||||
await ServiceRegistry.get_civitai_client()
|
||||
|
||||
# Register DownloadManager with ServiceRegistry
|
||||
await ServiceRegistry.get_download_manager()
|
||||
|
||||
# Initialize DownloadQueueService for persistent queue/history
|
||||
await ServiceRegistry.get_download_queue_service()
|
||||
|
||||
await ServiceRegistry.get_backup_service()
|
||||
|
||||
from .services.metadata_service import initialize_metadata_providers
|
||||
|
||||
await initialize_metadata_providers()
|
||||
@@ -458,5 +436,14 @@ class LoraManager:
|
||||
try:
|
||||
logger.info("LoRA Manager: Cleaning up services")
|
||||
|
||||
# Cancel any in-flight scanner initialization tasks so thread-pool
|
||||
# workers (e.g. _initialize_cache_sync) can break out of their loops
|
||||
# when the server shuts down (e.g. Ctrl+C on WSL).
|
||||
for name in ("lora_scanner", "checkpoint_scanner", "embedding_scanner"):
|
||||
scanner = ServiceRegistry.get_service_sync(name)
|
||||
if scanner is not None and hasattr(scanner, "cancel_task"):
|
||||
scanner.cancel_task()
|
||||
logger.debug("LoRA Manager: Cancelled %s", name)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during cleanup: {e}", exc_info=True)
|
||||
|
||||
@@ -5,9 +5,10 @@ MODELS = "models"
|
||||
PROMPTS = "prompts"
|
||||
SAMPLING = "sampling"
|
||||
LORAS = "loras"
|
||||
EMBEDDINGS = "embeddings"
|
||||
SIZE = "size"
|
||||
IMAGES = "images"
|
||||
IS_SAMPLER = "is_sampler" # New constant to mark sampler nodes
|
||||
|
||||
# Complete list of categories to track
|
||||
METADATA_CATEGORIES = [MODELS, PROMPTS, SAMPLING, LORAS, SIZE, IMAGES]
|
||||
METADATA_CATEGORIES = [MODELS, PROMPTS, SAMPLING, LORAS, EMBEDDINGS, SIZE, IMAGES]
|
||||
|
||||
@@ -148,10 +148,13 @@ class MetadataHook:
|
||||
"""Install hooks for asynchronous execution model"""
|
||||
# Store the original _async_map_node_over_list function
|
||||
original_map_node_over_list = getattr(execution, map_node_func_name)
|
||||
|
||||
# Wrapped async function, compatible with both stable and nightly
|
||||
async def async_map_node_over_list_with_metadata(prompt_id, unique_id, obj, input_data_all, func, allow_interrupt=False, execution_block_cb=None, pre_execute_cb=None, *args, **kwargs):
|
||||
hidden_inputs = kwargs.get('hidden_inputs', None)
|
||||
|
||||
# Wrapped async function - signature must exactly match _async_map_node_over_list
|
||||
async def async_map_node_over_list_with_metadata(
|
||||
prompt_id, unique_id, obj, input_data_all, func,
|
||||
allow_interrupt=False, execution_block_cb=None,
|
||||
pre_execute_cb=None, v3_data=None
|
||||
):
|
||||
# Only collect metadata when calling the main function of nodes
|
||||
if func == obj.FUNCTION and hasattr(obj, '__class__'):
|
||||
try:
|
||||
@@ -163,13 +166,13 @@ class MetadataHook:
|
||||
registry.record_node_execution(node_id, class_type, input_data_all, None)
|
||||
except Exception as e:
|
||||
logger.error(f"Error collecting metadata (pre-execution): {str(e)}")
|
||||
|
||||
# Call original function with all args/kwargs
|
||||
|
||||
# Call original function with exact parameters
|
||||
results = await original_map_node_over_list(
|
||||
prompt_id, unique_id, obj, input_data_all, func,
|
||||
allow_interrupt, execution_block_cb, pre_execute_cb, *args, **kwargs
|
||||
allow_interrupt, execution_block_cb, pre_execute_cb, v3_data=v3_data
|
||||
)
|
||||
|
||||
|
||||
if func == obj.FUNCTION and hasattr(obj, '__class__'):
|
||||
try:
|
||||
registry = MetadataRegistry()
|
||||
@@ -180,28 +183,28 @@ class MetadataHook:
|
||||
registry.update_node_execution(node_id, class_type, results)
|
||||
except Exception as e:
|
||||
logger.error(f"Error collecting metadata (post-execution): {str(e)}")
|
||||
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# Also hook the execute function to track the current prompt_id
|
||||
original_execute = execution.execute
|
||||
|
||||
|
||||
async def async_execute_with_prompt_tracking(*args, **kwargs):
|
||||
if len(args) >= 7: # Check if we have enough arguments
|
||||
server, prompt, caches, node_id, extra_data, executed, prompt_id = args[:7]
|
||||
registry = MetadataRegistry()
|
||||
|
||||
|
||||
# Start collection if this is a new prompt
|
||||
if not registry.current_prompt_id or registry.current_prompt_id != prompt_id:
|
||||
registry.start_collection(prompt_id)
|
||||
|
||||
|
||||
# Store the dynprompt reference for node lookups
|
||||
if hasattr(prompt, 'original_prompt'):
|
||||
registry.set_current_prompt(prompt)
|
||||
|
||||
|
||||
# Execute the original function
|
||||
return await original_execute(*args, **kwargs)
|
||||
|
||||
|
||||
# Replace the functions with async versions
|
||||
setattr(execution, map_node_func_name, async_map_node_over_list_with_metadata)
|
||||
execution.execute = async_execute_with_prompt_tracking
|
||||
|
||||
@@ -352,50 +352,101 @@ class MetadataProcessor:
|
||||
|
||||
# Check if we have stored conditioning objects for this sampler
|
||||
if sampler_id in metadata.get(PROMPTS, {}) and (
|
||||
"pos_conditioning" in metadata[PROMPTS][sampler_id] or
|
||||
"neg_conditioning" in metadata[PROMPTS][sampler_id]):
|
||||
|
||||
"pos_conditioning" in metadata[PROMPTS][sampler_id] or
|
||||
"neg_conditioning" in metadata[PROMPTS][sampler_id]
|
||||
):
|
||||
pos_conditioning = metadata[PROMPTS][sampler_id].get("pos_conditioning")
|
||||
neg_conditioning = metadata[PROMPTS][sampler_id].get("neg_conditioning")
|
||||
|
||||
# Helper function to recursively find prompt text for a conditioning object
|
||||
def find_prompt_text_for_conditioning(conditioning_obj, is_positive=True):
|
||||
|
||||
def extend_unique(target, values):
|
||||
for value in values:
|
||||
if value and value not in target:
|
||||
target.append(value)
|
||||
|
||||
# Helper function to recursively find prompt texts for a conditioning object.
|
||||
# Transform nodes can map one output conditioning to multiple source conditionings.
|
||||
def find_prompt_texts_for_conditioning(
|
||||
conditioning_obj, is_positive=True, visited=None
|
||||
):
|
||||
if conditioning_obj is None:
|
||||
return ""
|
||||
|
||||
return []
|
||||
|
||||
if visited is None:
|
||||
visited = set()
|
||||
|
||||
conditioning_id = id(conditioning_obj)
|
||||
if conditioning_id in visited:
|
||||
return []
|
||||
visited.add(conditioning_id)
|
||||
|
||||
prompt_texts = []
|
||||
|
||||
# Try to match conditioning objects with those stored by extractors
|
||||
for prompt_node_id, prompt_data in metadata[PROMPTS].items():
|
||||
# For nodes with single conditioning output
|
||||
if "conditioning" in prompt_data:
|
||||
if id(prompt_data["conditioning"]) == id(conditioning_obj):
|
||||
return prompt_data.get("text", "")
|
||||
|
||||
# For nodes with separate pos_conditioning and neg_conditioning outputs (like TSC_EfficientLoader)
|
||||
if is_positive and "positive_encoded" in prompt_data:
|
||||
if id(prompt_data["positive_encoded"]) == id(conditioning_obj):
|
||||
if "positive_text" in prompt_data:
|
||||
return prompt_data["positive_text"]
|
||||
else:
|
||||
orig_conditioning = prompt_data.get("orig_pos_cond", None)
|
||||
if orig_conditioning is not None:
|
||||
# Recursively find the prompt text for the original conditioning
|
||||
return find_prompt_text_for_conditioning(orig_conditioning, is_positive=True)
|
||||
|
||||
if not is_positive and "negative_encoded" in prompt_data:
|
||||
if id(prompt_data["negative_encoded"]) == id(conditioning_obj):
|
||||
if "negative_text" in prompt_data:
|
||||
return prompt_data["negative_text"]
|
||||
else:
|
||||
orig_conditioning = prompt_data.get("orig_neg_cond", None)
|
||||
if orig_conditioning is not None:
|
||||
# Recursively find the prompt text for the original conditioning
|
||||
return find_prompt_text_for_conditioning(orig_conditioning, is_positive=False)
|
||||
|
||||
return ""
|
||||
|
||||
if not isinstance(prompt_data, dict):
|
||||
continue
|
||||
|
||||
# For CLIP text nodes with a single conditioning output.
|
||||
if id(prompt_data.get("conditioning")) == conditioning_id:
|
||||
text = prompt_data.get("text", "")
|
||||
if text:
|
||||
extend_unique(prompt_texts, [text])
|
||||
|
||||
# Generic provenance for passthrough/transform/combine nodes.
|
||||
for source in prompt_data.get("conditioning_sources", []):
|
||||
if id(source.get("output")) != conditioning_id:
|
||||
continue
|
||||
for input_conditioning in source.get("inputs", []):
|
||||
extend_unique(
|
||||
prompt_texts,
|
||||
find_prompt_texts_for_conditioning(
|
||||
input_conditioning, is_positive, visited
|
||||
),
|
||||
)
|
||||
|
||||
# For nodes with separate pos_conditioning and neg_conditioning outputs
|
||||
# like TSC_EfficientLoader and existing ControlNet-style metadata.
|
||||
if (
|
||||
is_positive
|
||||
and id(prompt_data.get("positive_encoded")) == conditioning_id
|
||||
):
|
||||
if prompt_data.get("positive_text"):
|
||||
extend_unique(prompt_texts, [prompt_data["positive_text"]])
|
||||
else:
|
||||
extend_unique(
|
||||
prompt_texts,
|
||||
find_prompt_texts_for_conditioning(
|
||||
prompt_data.get("orig_pos_cond"),
|
||||
is_positive=True,
|
||||
visited=visited,
|
||||
),
|
||||
)
|
||||
|
||||
if (
|
||||
not is_positive
|
||||
and id(prompt_data.get("negative_encoded")) == conditioning_id
|
||||
):
|
||||
if prompt_data.get("negative_text"):
|
||||
extend_unique(prompt_texts, [prompt_data["negative_text"]])
|
||||
else:
|
||||
extend_unique(
|
||||
prompt_texts,
|
||||
find_prompt_texts_for_conditioning(
|
||||
prompt_data.get("orig_neg_cond"),
|
||||
is_positive=False,
|
||||
visited=visited,
|
||||
),
|
||||
)
|
||||
|
||||
return prompt_texts
|
||||
|
||||
# Find prompt texts using the helper function
|
||||
result["prompt"] = find_prompt_text_for_conditioning(pos_conditioning, is_positive=True)
|
||||
result["negative_prompt"] = find_prompt_text_for_conditioning(neg_conditioning, is_positive=False)
|
||||
result["prompt"] = ", ".join(
|
||||
find_prompt_texts_for_conditioning(pos_conditioning, is_positive=True)
|
||||
)
|
||||
result["negative_prompt"] = ", ".join(
|
||||
find_prompt_texts_for_conditioning(neg_conditioning, is_positive=False)
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
@@ -509,8 +560,14 @@ class MetadataProcessor:
|
||||
|
||||
params["loras"] = " ".join(lora_parts)
|
||||
|
||||
# Set default clip_skip value
|
||||
params["clip_skip"] = "1" # Common default
|
||||
# Extract clip_skip from any SAMPLING node that provides it
|
||||
for sampler_info in metadata.get(SAMPLING, {}).values():
|
||||
clip_skip = sampler_info.get("parameters", {}).get("clip_skip")
|
||||
if clip_skip is not None:
|
||||
params["clip_skip"] = clip_skip
|
||||
break
|
||||
if params["clip_skip"] is None:
|
||||
params["clip_skip"] = "1"
|
||||
|
||||
return params
|
||||
|
||||
@@ -595,6 +652,15 @@ class MetadataProcessor:
|
||||
if negative_node_id and negative_node_id in metadata.get(PROMPTS, {}):
|
||||
params["negative_prompt"] = metadata[PROMPTS][negative_node_id].get("text", "")
|
||||
else:
|
||||
positive_node_id = MetadataProcessor.trace_node_input(prompt, guider_node_id, "conditioning", max_depth=10)
|
||||
# Generic guider nodes often expose separate positive/negative inputs.
|
||||
positive_node_id = MetadataProcessor.trace_node_input(prompt, guider_node_id, "positive", max_depth=10)
|
||||
if not positive_node_id:
|
||||
positive_node_id = MetadataProcessor.trace_node_input(prompt, guider_node_id, "conditioning", max_depth=10)
|
||||
if positive_node_id and positive_node_id in metadata.get(PROMPTS, {}):
|
||||
params["prompt"] = metadata[PROMPTS][positive_node_id].get("text", "")
|
||||
|
||||
negative_node_id = MetadataProcessor.trace_node_input(prompt, guider_node_id, "negative", max_depth=10)
|
||||
if not negative_node_id:
|
||||
negative_node_id = MetadataProcessor.trace_node_input(prompt, guider_node_id, "conditioning", max_depth=10)
|
||||
if negative_node_id and negative_node_id in metadata.get(PROMPTS, {}):
|
||||
params["negative_prompt"] = metadata[PROMPTS][negative_node_id].get("text", "")
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
from .constants import MODELS, PROMPTS, SAMPLING, LORAS, SIZE, IMAGES, IS_SAMPLER
|
||||
|
||||
@@ -142,6 +144,118 @@ class TSCCheckpointLoaderExtractor(NodeMetadataExtractor):
|
||||
metadata[PROMPTS][node_id]["positive_encoded"] = positive_conditioning
|
||||
metadata[PROMPTS][node_id]["negative_encoded"] = negative_conditioning
|
||||
|
||||
|
||||
class EasyComfyLoaderExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
if "ckpt_name" in inputs:
|
||||
_store_checkpoint_metadata(metadata, node_id, inputs["ckpt_name"])
|
||||
|
||||
# Only extract from optional_lora_stack — skip the single lora_name to
|
||||
# avoid double-counting LoRAs that come through the LORA_STACK path.
|
||||
active_loras = []
|
||||
optional_lora_stack = inputs.get("optional_lora_stack")
|
||||
if optional_lora_stack is not None and isinstance(optional_lora_stack, (list, tuple)):
|
||||
for item in optional_lora_stack:
|
||||
if isinstance(item, (list, tuple)) and len(item) >= 2:
|
||||
lora_path = item[0]
|
||||
model_strength = item[1]
|
||||
lora_name = os.path.splitext(os.path.basename(lora_path))[0]
|
||||
active_loras.append({
|
||||
"name": lora_name,
|
||||
"strength": model_strength
|
||||
})
|
||||
|
||||
if active_loras:
|
||||
metadata[LORAS][node_id] = {
|
||||
"lora_list": active_loras,
|
||||
"node_id": node_id
|
||||
}
|
||||
|
||||
positive_text = inputs.get("positive", "")
|
||||
negative_text = inputs.get("negative", "")
|
||||
|
||||
if positive_text or negative_text:
|
||||
if node_id not in metadata[PROMPTS]:
|
||||
metadata[PROMPTS][node_id] = {"node_id": node_id}
|
||||
metadata[PROMPTS][node_id]["positive_text"] = positive_text
|
||||
metadata[PROMPTS][node_id]["negative_text"] = negative_text
|
||||
|
||||
if "clip_skip" in inputs:
|
||||
clip_skip = inputs["clip_skip"]
|
||||
if node_id not in metadata[SAMPLING]:
|
||||
metadata[SAMPLING][node_id] = {"parameters": {}, "node_id": node_id}
|
||||
metadata[SAMPLING][node_id]["parameters"]["clip_skip"] = clip_skip
|
||||
|
||||
width = inputs.get("empty_latent_width")
|
||||
height = inputs.get("empty_latent_height")
|
||||
if width is not None and height is not None:
|
||||
if SIZE not in metadata:
|
||||
metadata[SIZE] = {}
|
||||
metadata[SIZE][node_id] = {
|
||||
"width": int(width),
|
||||
"height": int(height),
|
||||
"node_id": node_id
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata):
|
||||
# outputs: [(pipe_dict, model, vae), ...]
|
||||
if not outputs or not isinstance(outputs, list) or len(outputs) == 0:
|
||||
return
|
||||
first_output = outputs[0]
|
||||
if not isinstance(first_output, tuple) or len(first_output) < 1:
|
||||
return
|
||||
pipe = first_output[0]
|
||||
if not isinstance(pipe, dict):
|
||||
return
|
||||
|
||||
positive_conditioning = pipe.get("positive")
|
||||
negative_conditioning = pipe.get("negative")
|
||||
|
||||
if positive_conditioning is not None or negative_conditioning is not None:
|
||||
if node_id not in metadata[PROMPTS]:
|
||||
metadata[PROMPTS][node_id] = {"node_id": node_id}
|
||||
if positive_conditioning is not None:
|
||||
metadata[PROMPTS][node_id]["positive_encoded"] = positive_conditioning
|
||||
if negative_conditioning is not None:
|
||||
metadata[PROMPTS][node_id]["negative_encoded"] = negative_conditioning
|
||||
|
||||
|
||||
class EasyPreSamplingExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
sampling_params = {}
|
||||
for key in ("steps", "cfg", "sampler_name", "scheduler", "denoise", "seed"):
|
||||
if key in inputs:
|
||||
sampling_params[key] = inputs[key]
|
||||
|
||||
metadata[SAMPLING][node_id] = {
|
||||
"parameters": sampling_params,
|
||||
"node_id": node_id,
|
||||
IS_SAMPLER: True
|
||||
}
|
||||
|
||||
|
||||
class EasySeedExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs or "seed" not in inputs:
|
||||
return
|
||||
|
||||
metadata[SAMPLING][node_id] = {
|
||||
"parameters": {"seed": inputs["seed"]},
|
||||
"node_id": node_id,
|
||||
IS_SAMPLER: False
|
||||
}
|
||||
|
||||
|
||||
class CLIPTextEncodeExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
@@ -161,6 +275,251 @@ class CLIPTextEncodeExtractor(NodeMetadataExtractor):
|
||||
conditioning = outputs[0][0]
|
||||
metadata[PROMPTS][node_id]["conditioning"] = conditioning
|
||||
|
||||
|
||||
class MyOriginalWaifuTextExtractor(NodeMetadataExtractor):
|
||||
"""Extractor for ComfyUI-MyOriginalWaifu TextProvider nodes."""
|
||||
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
positive_text = inputs.get("positive", "")
|
||||
negative_text = inputs.get("negative", "")
|
||||
|
||||
if positive_text or negative_text:
|
||||
metadata[PROMPTS][node_id] = {
|
||||
"positive_text": positive_text,
|
||||
"negative_text": negative_text,
|
||||
"node_id": node_id,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata):
|
||||
output_tuple = _first_output_tuple(outputs)
|
||||
if not output_tuple or len(output_tuple) < 2:
|
||||
return
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
prompt_metadata["positive_text"] = output_tuple[0]
|
||||
prompt_metadata["negative_text"] = output_tuple[1]
|
||||
|
||||
|
||||
class MyOriginalWaifuClipExtractor(NodeMetadataExtractor):
|
||||
"""Extractor for ComfyUI-MyOriginalWaifu ClipProvider nodes."""
|
||||
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
positive_text = inputs.get("positive", "")
|
||||
negative_text = inputs.get("negative", "")
|
||||
|
||||
if positive_text or negative_text:
|
||||
metadata[PROMPTS][node_id] = {
|
||||
"positive_text": positive_text,
|
||||
"negative_text": negative_text,
|
||||
"node_id": node_id,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata):
|
||||
output_tuple = _first_output_tuple(outputs)
|
||||
if not output_tuple or len(output_tuple) < 2:
|
||||
return
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
prompt_metadata["positive_encoded"] = output_tuple[0]
|
||||
prompt_metadata["negative_encoded"] = output_tuple[1]
|
||||
|
||||
|
||||
def _ensure_prompt_metadata(metadata, node_id):
|
||||
if node_id not in metadata[PROMPTS]:
|
||||
metadata[PROMPTS][node_id] = {"node_id": node_id}
|
||||
return metadata[PROMPTS][node_id]
|
||||
|
||||
|
||||
def _first_output_tuple(outputs):
|
||||
if not outputs or not isinstance(outputs, list) or len(outputs) == 0:
|
||||
return None
|
||||
first_output = outputs[0]
|
||||
if isinstance(first_output, tuple):
|
||||
return first_output
|
||||
return None
|
||||
|
||||
|
||||
def _record_conditioning_source(
|
||||
metadata, node_id, output_conditioning, input_conditionings
|
||||
):
|
||||
if output_conditioning is None:
|
||||
return
|
||||
|
||||
sources = [
|
||||
conditioning for conditioning in input_conditionings if conditioning is not None
|
||||
]
|
||||
if not sources:
|
||||
return
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
prompt_metadata.setdefault("conditioning_sources", []).append(
|
||||
{
|
||||
"output": output_conditioning,
|
||||
"inputs": sources,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _get_variable_name(inputs):
|
||||
for key in ("key", "name", "variable_name", "tag", "text"):
|
||||
value = inputs.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _get_node_variable_name(metadata, node_id, inputs):
|
||||
variable_name = _get_variable_name(inputs)
|
||||
if variable_name:
|
||||
return variable_name
|
||||
|
||||
prompt = metadata.get("current_prompt")
|
||||
original_prompt = getattr(prompt, "original_prompt", None)
|
||||
if not original_prompt or node_id not in original_prompt:
|
||||
return None
|
||||
|
||||
node_data = original_prompt[node_id]
|
||||
variable_name = _get_variable_name(node_data.get("inputs", {}))
|
||||
if variable_name:
|
||||
return variable_name
|
||||
|
||||
widgets_values = node_data.get("widgets_values", [])
|
||||
if widgets_values and isinstance(widgets_values[0], str):
|
||||
return widgets_values[0]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class ControlNetApplyAdvancedExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
if inputs.get("positive") is not None:
|
||||
prompt_metadata["orig_pos_cond"] = inputs["positive"]
|
||||
if inputs.get("negative") is not None:
|
||||
prompt_metadata["orig_neg_cond"] = inputs["negative"]
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata):
|
||||
output_tuple = _first_output_tuple(outputs)
|
||||
if not output_tuple:
|
||||
return
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
positive_input = prompt_metadata.get("orig_pos_cond")
|
||||
negative_input = prompt_metadata.get("orig_neg_cond")
|
||||
|
||||
if len(output_tuple) >= 1:
|
||||
prompt_metadata["positive_encoded"] = output_tuple[0]
|
||||
_record_conditioning_source(
|
||||
metadata, node_id, output_tuple[0], [positive_input]
|
||||
)
|
||||
if len(output_tuple) >= 2:
|
||||
prompt_metadata["negative_encoded"] = output_tuple[1]
|
||||
_record_conditioning_source(
|
||||
metadata, node_id, output_tuple[1], [negative_input]
|
||||
)
|
||||
|
||||
|
||||
class ConditioningCombineExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
input_conditionings = []
|
||||
for input_name in inputs:
|
||||
if (
|
||||
input_name.startswith("conditioning")
|
||||
and inputs[input_name] is not None
|
||||
):
|
||||
input_conditionings.append(inputs[input_name])
|
||||
|
||||
if input_conditionings:
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
prompt_metadata["orig_conditionings"] = input_conditionings
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata):
|
||||
output_tuple = _first_output_tuple(outputs)
|
||||
if not output_tuple or len(output_tuple) < 1:
|
||||
return
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
output_conditioning = output_tuple[0]
|
||||
prompt_metadata["conditioning"] = output_conditioning
|
||||
_record_conditioning_source(
|
||||
metadata,
|
||||
node_id,
|
||||
output_conditioning,
|
||||
prompt_metadata.get("orig_conditionings", []),
|
||||
)
|
||||
|
||||
|
||||
class SetNodeExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
variable_name = _get_node_variable_name(metadata, node_id, inputs)
|
||||
conditioning = inputs.get("CONDITIONING")
|
||||
if conditioning is None:
|
||||
conditioning = inputs.get("conditioning")
|
||||
if conditioning is None:
|
||||
return
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
prompt_metadata["conditioning"] = conditioning
|
||||
if variable_name:
|
||||
prompt_metadata["variable_name"] = variable_name
|
||||
metadata[PROMPTS].setdefault("__conditioning_variables__", {})[
|
||||
variable_name
|
||||
] = conditioning
|
||||
|
||||
|
||||
class GetNodeExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
variable_name = _get_node_variable_name(metadata, node_id, inputs or {})
|
||||
if variable_name:
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
prompt_metadata["variable_name"] = variable_name
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata):
|
||||
output_tuple = _first_output_tuple(outputs)
|
||||
if not output_tuple or len(output_tuple) < 1:
|
||||
return
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
output_conditioning = output_tuple[0]
|
||||
prompt_metadata["conditioning"] = output_conditioning
|
||||
|
||||
variable_name = prompt_metadata.get("variable_name")
|
||||
if not variable_name:
|
||||
return
|
||||
|
||||
input_conditioning = metadata[PROMPTS].get("__conditioning_variables__", {}).get(
|
||||
variable_name
|
||||
)
|
||||
_record_conditioning_source(
|
||||
metadata, node_id, output_conditioning, [input_conditioning]
|
||||
)
|
||||
|
||||
# Base Sampler Extractor to reduce code redundancy
|
||||
class BaseSamplerExtractor(NodeMetadataExtractor):
|
||||
"""Base extractor for sampler nodes with common functionality"""
|
||||
@@ -427,6 +786,75 @@ class ImageSizeExtractor(NodeMetadataExtractor):
|
||||
"node_id": node_id
|
||||
}
|
||||
|
||||
class RgthreePowerLoraLoaderExtractor(NodeMetadataExtractor):
|
||||
"""Extract LoRA metadata from rgthree Power Lora Loader.
|
||||
|
||||
The node passes LoRAs as dynamic kwargs: LORA_1, LORA_2, ... each containing
|
||||
{'on': bool, 'lora': filename, 'strength': float, 'strengthTwo': float}.
|
||||
"""
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
active_loras = []
|
||||
for key, value in inputs.items():
|
||||
if not key.upper().startswith('LORA_'):
|
||||
continue
|
||||
if not isinstance(value, dict):
|
||||
continue
|
||||
if not value.get('on') or not value.get('lora'):
|
||||
continue
|
||||
lora_name = os.path.splitext(os.path.basename(value['lora']))[0]
|
||||
active_loras.append({
|
||||
"name": lora_name,
|
||||
"strength": round(float(value.get('strength', 1.0)), 2)
|
||||
})
|
||||
|
||||
if active_loras:
|
||||
metadata[LORAS][node_id] = {
|
||||
"lora_list": active_loras,
|
||||
"node_id": node_id
|
||||
}
|
||||
|
||||
|
||||
class TensorRTLoaderExtractor(NodeMetadataExtractor):
|
||||
"""Extract checkpoint metadata from TensorRT Loader.
|
||||
|
||||
extract() parses the engine filename from 'unet_name' as a best-effort
|
||||
fallback (strips profile suffix after '_$' and counter suffix).
|
||||
|
||||
update() checks if the output MODEL has attachments["source_model"]
|
||||
set by the node (NubeBuster fork) and overrides with the real name.
|
||||
Vanilla TRT doesn't set this — the filename parse stands.
|
||||
"""
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs or "unet_name" not in inputs:
|
||||
return
|
||||
unet_name = inputs.get("unet_name")
|
||||
# Strip path and extension, then drop the $_profile suffix
|
||||
model_name = os.path.splitext(os.path.basename(unet_name))[0]
|
||||
if "_$" in model_name:
|
||||
model_name = model_name[:model_name.index("_$")]
|
||||
# Strip counter suffix (e.g. _00001_) left by ComfyUI's save path
|
||||
model_name = re.sub(r'_\d+_?$', '', model_name)
|
||||
_store_checkpoint_metadata(metadata, node_id, model_name)
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata):
|
||||
if not outputs or not isinstance(outputs, list) or len(outputs) == 0:
|
||||
return
|
||||
first_output = outputs[0]
|
||||
if not isinstance(first_output, tuple) or len(first_output) < 1:
|
||||
return
|
||||
model = first_output[0]
|
||||
# NubeBuster fork sets attachments["source_model"] on the ModelPatcher
|
||||
source_model = getattr(model, 'attachments', {}).get("source_model")
|
||||
if source_model:
|
||||
_store_checkpoint_metadata(metadata, node_id, source_model)
|
||||
|
||||
|
||||
class LoraLoaderManagerExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
@@ -473,6 +901,55 @@ class LoraLoaderManagerExtractor(NodeMetadataExtractor):
|
||||
"node_id": node_id
|
||||
}
|
||||
|
||||
class LoraTextLoaderManagerExtractor(NodeMetadataExtractor):
|
||||
"""Extract LoRA metadata from LoraTextLoaderLM (LoRA Text Loader).
|
||||
|
||||
The node accepts a `lora_syntax` STRING containing <lora:name:strength> tags
|
||||
(same format as the ComfyUI prompt), plus an optional `lora_stack`.
|
||||
This extractor parses the syntax string using the same regex as the node.
|
||||
"""
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
active_loras = []
|
||||
|
||||
# Process lora_stack if available (optional input)
|
||||
if "lora_stack" in inputs:
|
||||
lora_stack = inputs.get("lora_stack", [])
|
||||
for item in lora_stack:
|
||||
# lora_stack entries are (path, model_strength, clip_strength) tuples
|
||||
if isinstance(item, (list, tuple)) and len(item) >= 2:
|
||||
lora_path = item[0]
|
||||
model_strength = item[1]
|
||||
lora_name = os.path.splitext(os.path.basename(lora_path))[0]
|
||||
active_loras.append({
|
||||
"name": lora_name,
|
||||
"strength": round(float(model_strength), 2)
|
||||
})
|
||||
|
||||
# Process lora_syntax string input
|
||||
if "lora_syntax" in inputs:
|
||||
lora_syntax = inputs.get("lora_syntax", "")
|
||||
if lora_syntax and isinstance(lora_syntax, str):
|
||||
pattern = r"<lora:([^:>]+):([^:>]+)(?::([^:>]+))?>"
|
||||
matches = re.findall(pattern, lora_syntax, re.IGNORECASE)
|
||||
for match in matches:
|
||||
lora_name = match[0]
|
||||
model_strength = float(match[1])
|
||||
active_loras.append({
|
||||
"name": lora_name,
|
||||
"strength": round(model_strength, 2)
|
||||
})
|
||||
|
||||
if active_loras:
|
||||
metadata[LORAS][node_id] = {
|
||||
"lora_list": active_loras,
|
||||
"node_id": node_id
|
||||
}
|
||||
|
||||
|
||||
class FluxGuidanceExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
@@ -577,8 +1054,6 @@ class SamplerCustomAdvancedExtractor(BaseSamplerExtractor):
|
||||
# Extract latent dimensions
|
||||
BaseSamplerExtractor.extract_latent_dimensions(node_id, inputs, metadata)
|
||||
|
||||
import json
|
||||
|
||||
class CLIPTextEncodeFluxExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
@@ -699,9 +1174,12 @@ NODE_EXTRACTORS = {
|
||||
"KSamplerSelect": KSamplerSelectExtractor, # Add KSamplerSelect
|
||||
"BasicScheduler": BasicSchedulerExtractor, # Add BasicScheduler
|
||||
"AlignYourStepsScheduler": BasicSchedulerExtractor, # Add AlignYourStepsScheduler
|
||||
# ComfyUI-Easy-Use pre-sampling / seed
|
||||
"samplerSettings": EasyPreSamplingExtractor, # easy preSampling
|
||||
"easySeed": EasySeedExtractor, # easy seed
|
||||
# Loaders
|
||||
"CheckpointLoaderSimple": CheckpointLoaderExtractor,
|
||||
"comfyLoader": CheckpointLoaderExtractor, # easy comfyLoader
|
||||
"comfyLoader": EasyComfyLoaderExtractor, # ComfyUI-Easy-Use easy comfyLoader
|
||||
"CheckpointLoaderSimpleWithImages": CheckpointLoaderExtractor, # CheckpointLoader|pysssss
|
||||
"TSC_EfficientLoader": TSCCheckpointLoaderExtractor, # Efficient Nodes
|
||||
"NunchakuFluxDiTLoader": NunchakuFluxDiTLoaderExtractor, # ComfyUI-Nunchaku
|
||||
@@ -711,12 +1189,18 @@ NODE_EXTRACTORS = {
|
||||
"GGUFLoaderKJ": KJNodesModelLoaderExtractor, # KJNodes
|
||||
"DiffusionModelLoaderKJ": KJNodesModelLoaderExtractor, # KJNodes
|
||||
"CheckpointLoaderKJ": CheckpointLoaderExtractor, # KJNodes
|
||||
"CheckpointLoaderLM": CheckpointLoaderExtractor, # LoRA Manager
|
||||
"UNETLoader": UNETLoaderExtractor, # Updated to use dedicated extractor
|
||||
"UnetLoaderGGUF": UNETLoaderExtractor, # Updated to use dedicated extractor
|
||||
"UNETLoaderLM": UNETLoaderExtractor, # LoRA Manager
|
||||
"LoraLoader": LoraLoaderExtractor,
|
||||
"LoraLoaderLM": LoraLoaderManagerExtractor,
|
||||
"LoraTextLoaderLM": LoraTextLoaderManagerExtractor,
|
||||
"RgthreePowerLoraLoader": RgthreePowerLoraLoaderExtractor,
|
||||
"TensorRTLoader": TensorRTLoaderExtractor,
|
||||
# Conditioning
|
||||
"CLIPTextEncode": CLIPTextEncodeExtractor,
|
||||
"CLIPTextEncodeAttentionBias": CLIPTextEncodeExtractor, # From https://github.com/silveroxides/ComfyUI_PromptAttention
|
||||
"PromptLM": CLIPTextEncodeExtractor,
|
||||
"CLIPTextEncodeFlux": CLIPTextEncodeFluxExtractor, # Add CLIPTextEncodeFlux
|
||||
"WAS_Text_to_Conditioning": CLIPTextEncodeExtractor,
|
||||
@@ -724,6 +1208,12 @@ NODE_EXTRACTORS = {
|
||||
"smZ_CLIPTextEncode": CLIPTextEncodeExtractor, # From https://github.com/shiimizu/ComfyUI_smZNodes
|
||||
"CR_ApplyControlNetStack": CR_ApplyControlNetStackExtractor, # Add CR_ApplyControlNetStack
|
||||
"PCTextEncode": CLIPTextEncodeExtractor, # From https://github.com/asagi4/comfyui-prompt-control
|
||||
"TextProvider": MyOriginalWaifuTextExtractor, # ComfyUI-MyOriginalWaifu
|
||||
"ClipProvider": MyOriginalWaifuClipExtractor, # ComfyUI-MyOriginalWaifu
|
||||
"ControlNetApplyAdvanced": ControlNetApplyAdvancedExtractor,
|
||||
"ConditioningCombine": ConditioningCombineExtractor,
|
||||
"SetNode": SetNodeExtractor,
|
||||
"GetNode": GetNodeExtractor,
|
||||
# Latent
|
||||
"EmptyLatentImage": ImageSizeExtractor,
|
||||
# Flux
|
||||
|
||||
@@ -16,6 +16,8 @@ IMG_EXTENSIONS = (
|
||||
".tif",
|
||||
".tiff",
|
||||
".webp",
|
||||
".avif",
|
||||
".jxl",
|
||||
".mp4"
|
||||
)
|
||||
|
||||
|
||||
@@ -4,15 +4,21 @@ from typing import Awaitable, Callable, Dict, List
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
# Use wildcard for CivitAI to support their CDN subdomains (e.g., image-b2.civitai.com)
|
||||
# Security note: This is acceptable because:
|
||||
# 1. CSP img-src only controls image/video loading, not script execution
|
||||
# 2. All *.civitai.com subdomains are controlled by Civitai
|
||||
# 3. Explicit domain list would require constant updates as Civitai adds CDN nodes
|
||||
REMOTE_MEDIA_SOURCES = (
|
||||
"https://image.civitai.com",
|
||||
"https://*.civitai.com",
|
||||
"https://img.genur.art",
|
||||
)
|
||||
|
||||
|
||||
@web.middleware
|
||||
async def relax_csp_for_remote_media(
|
||||
request: web.Request, handler: Callable[[web.Request], Awaitable[web.StreamResponse]]
|
||||
request: web.Request,
|
||||
handler: Callable[[web.Request], Awaitable[web.StreamResponse]],
|
||||
) -> web.StreamResponse:
|
||||
"""Allow LoRA Manager media previews to load from trusted remote domains.
|
||||
|
||||
@@ -43,7 +49,9 @@ async def relax_csp_for_remote_media(
|
||||
directive_order.append(name)
|
||||
directives[name] = values
|
||||
|
||||
def merge_sources(name: str, sources: List[str], defaults: List[str] | None = None) -> None:
|
||||
def merge_sources(
|
||||
name: str, sources: List[str], defaults: List[str] | None = None
|
||||
) -> None:
|
||||
existing = directives.get(name, list(defaults or []))
|
||||
|
||||
for source in sources:
|
||||
|
||||
71
py/middleware/error_middleware.py
Normal file
71
py/middleware/error_middleware.py
Normal file
@@ -0,0 +1,71 @@
|
||||
"""JSON error middleware for API routes.
|
||||
|
||||
Ensures all responses to /api/* requests return valid JSON that the
|
||||
browser-extension frontend can JSON.parse() without crashing, even when
|
||||
the route does not exist (404) or the handler raises an exception (500).
|
||||
|
||||
Extension consumers call response.json() unconditionally — an HTML error
|
||||
page causes ``SyntaxError: unexpected end of data`` that leaks into the
|
||||
popup UI as a toast notification.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Awaitable, Callable
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@web.middleware
|
||||
async def api_json_error(
|
||||
request: web.Request,
|
||||
handler: Callable[[web.Request], Awaitable[web.Response]],
|
||||
) -> web.Response:
|
||||
"""Return JSON ``{"success": false, "error": "..."}`` for API errors.
|
||||
|
||||
Only intercepts paths starting with ``/api/`` — all other routes
|
||||
(frontend pages, static files, WebSocket upgrades) pass through
|
||||
unchanged.
|
||||
"""
|
||||
if not request.path.startswith("/api/"):
|
||||
return await handler(request)
|
||||
|
||||
try:
|
||||
response = await handler(request)
|
||||
return response
|
||||
except web.HTTPException as exc:
|
||||
# Let redirects (301, 302, 307, 308) propagate — they are not errors.
|
||||
if exc.status < 400:
|
||||
raise
|
||||
|
||||
logger.warning(
|
||||
"API %s %s returned HTTP %d: %s",
|
||||
request.method,
|
||||
request.path,
|
||||
exc.status,
|
||||
exc.reason,
|
||||
)
|
||||
|
||||
return web.json_response(
|
||||
{"success": False, "error": f"{exc.status}: {exc.reason}"},
|
||||
status=exc.status,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"API %s %s raised unhandled exception: %s",
|
||||
request.method,
|
||||
request.path,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": f"500: Internal Server Error ({type(exc).__name__})",
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
118
py/nodes/checkpoint_loader.py
Normal file
118
py/nodes/checkpoint_loader.py
Normal file
@@ -0,0 +1,118 @@
|
||||
import logging
|
||||
from typing import List, Tuple
|
||||
import comfy.sd # type: ignore
|
||||
import folder_paths # type: ignore
|
||||
from ..utils.utils import get_checkpoint_info_absolute, _format_model_name_for_comfyui
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CheckpointLoaderLM:
|
||||
"""Checkpoint Loader with support for extra folder paths
|
||||
|
||||
Loads checkpoints from both standard ComfyUI folders and LoRA Manager's
|
||||
extra folder paths, providing a unified interface for checkpoint loading.
|
||||
"""
|
||||
|
||||
NAME = "Checkpoint Loader (LoraManager)"
|
||||
CATEGORY = "Lora Manager/loaders"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
# Get list of checkpoint names from scanner (includes extra folder paths)
|
||||
checkpoint_names = s._get_checkpoint_names()
|
||||
return {
|
||||
"required": {
|
||||
"ckpt_name": (
|
||||
checkpoint_names,
|
||||
{"tooltip": "The name of the checkpoint (model) to load."},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("MODEL", "CLIP", "VAE")
|
||||
RETURN_NAMES = ("MODEL", "CLIP", "VAE")
|
||||
OUTPUT_TOOLTIPS = (
|
||||
"The model used for denoising latents.",
|
||||
"The CLIP model used for encoding text prompts.",
|
||||
"The VAE model used for encoding and decoding images to and from latent space.",
|
||||
)
|
||||
FUNCTION = "load_checkpoint"
|
||||
|
||||
@classmethod
|
||||
def _get_checkpoint_names(cls) -> List[str]:
|
||||
"""Get list of checkpoint names from scanner cache in ComfyUI format (relative path with extension)"""
|
||||
try:
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
import asyncio
|
||||
|
||||
async def _get_names():
|
||||
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||
cache = await scanner.get_cached_data()
|
||||
|
||||
# Get all model roots for calculating relative paths
|
||||
model_roots = scanner.get_model_roots()
|
||||
|
||||
# Filter only checkpoint type (not diffusion_model) and format names
|
||||
names = []
|
||||
for item in cache.raw_data:
|
||||
if item.get("sub_type") == "checkpoint":
|
||||
file_path = item.get("file_path", "")
|
||||
if file_path:
|
||||
# Format using relative path with OS-native separator
|
||||
formatted_name = _format_model_name_for_comfyui(
|
||||
file_path, model_roots
|
||||
)
|
||||
if formatted_name:
|
||||
names.append(formatted_name)
|
||||
|
||||
return sorted(names)
|
||||
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
import concurrent.futures
|
||||
|
||||
def run_in_thread():
|
||||
new_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(new_loop)
|
||||
try:
|
||||
return new_loop.run_until_complete(_get_names())
|
||||
finally:
|
||||
new_loop.close()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
future = executor.submit(run_in_thread)
|
||||
return future.result()
|
||||
except RuntimeError:
|
||||
return asyncio.run(_get_names())
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting checkpoint names: {e}")
|
||||
return []
|
||||
|
||||
def load_checkpoint(self, ckpt_name: str) -> Tuple:
|
||||
"""Load a checkpoint by name, supporting extra folder paths
|
||||
|
||||
Args:
|
||||
ckpt_name: The name of the checkpoint to load (relative path with extension)
|
||||
|
||||
Returns:
|
||||
Tuple of (MODEL, CLIP, VAE)
|
||||
"""
|
||||
# Get absolute path from cache using ComfyUI-style name
|
||||
ckpt_path, metadata = get_checkpoint_info_absolute(ckpt_name)
|
||||
|
||||
if metadata is None:
|
||||
raise FileNotFoundError(
|
||||
f"Checkpoint '{ckpt_name}' not found in LoRA Manager cache. "
|
||||
"Make sure the checkpoint is indexed and try again."
|
||||
)
|
||||
|
||||
# Load regular checkpoint using ComfyUI's API
|
||||
logger.info(f"Loading checkpoint from: {ckpt_path}")
|
||||
out = comfy.sd.load_checkpoint_guess_config(
|
||||
ckpt_path,
|
||||
output_vae=True,
|
||||
output_clip=True,
|
||||
embedding_directory=folder_paths.get_folder_paths("embeddings"),
|
||||
)
|
||||
return out[:3]
|
||||
161
py/nodes/gguf_import_helper.py
Normal file
161
py/nodes/gguf_import_helper.py
Normal file
@@ -0,0 +1,161 @@
|
||||
"""
|
||||
Helper module to safely import ComfyUI-GGUF modules.
|
||||
|
||||
This module provides a robust way to import ComfyUI-GGUF functionality
|
||||
regardless of how ComfyUI loaded it.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import importlib.util
|
||||
import logging
|
||||
from typing import Optional, Tuple, Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_gguf_path() -> str:
|
||||
"""Get the path to ComfyUI-GGUF based on this file's location.
|
||||
|
||||
Since ComfyUI-Lora-Manager and ComfyUI-GGUF are both in custom_nodes/,
|
||||
we can derive the GGUF path from our own location.
|
||||
"""
|
||||
# This file is at: custom_nodes/ComfyUI-Lora-Manager/py/nodes/gguf_import_helper.py
|
||||
# ComfyUI-GGUF is at: custom_nodes/ComfyUI-GGUF
|
||||
current_file = os.path.abspath(__file__)
|
||||
# Go up 4 levels: nodes -> py -> ComfyUI-Lora-Manager -> custom_nodes
|
||||
custom_nodes_dir = os.path.dirname(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(current_file)))
|
||||
)
|
||||
return os.path.join(custom_nodes_dir, "ComfyUI-GGUF")
|
||||
|
||||
|
||||
def _find_gguf_module() -> Optional[Any]:
|
||||
"""Find ComfyUI-GGUF module in sys.modules.
|
||||
|
||||
ComfyUI registers modules using the full path with dots replaced by _x_.
|
||||
"""
|
||||
gguf_path = _get_gguf_path()
|
||||
sys_module_name = gguf_path.replace(".", "_x_")
|
||||
|
||||
logger.debug(f"[GGUF Import] Looking for module '{sys_module_name}' in sys.modules")
|
||||
if sys_module_name in sys.modules:
|
||||
logger.info(f"[GGUF Import] Found module: '{sys_module_name}'")
|
||||
return sys.modules[sys_module_name]
|
||||
|
||||
logger.debug(f"[GGUF Import] Module not found: '{sys_module_name}'")
|
||||
return None
|
||||
|
||||
|
||||
def _load_gguf_modules_directly() -> Optional[Any]:
|
||||
"""Load ComfyUI-GGUF modules directly from file paths."""
|
||||
gguf_path = _get_gguf_path()
|
||||
|
||||
logger.info(f"[GGUF Import] Direct Load: Attempting to load from '{gguf_path}'")
|
||||
|
||||
if not os.path.exists(gguf_path):
|
||||
logger.warning(f"[GGUF Import] Path does not exist: {gguf_path}")
|
||||
return None
|
||||
|
||||
try:
|
||||
namespace = "ComfyUI_GGUF_Dynamic"
|
||||
init_path = os.path.join(gguf_path, "__init__.py")
|
||||
|
||||
if not os.path.exists(init_path):
|
||||
logger.warning(f"[GGUF Import] __init__.py not found at '{init_path}'")
|
||||
return None
|
||||
|
||||
logger.debug(f"[GGUF Import] Loading from '{init_path}'")
|
||||
spec = importlib.util.spec_from_file_location(namespace, init_path)
|
||||
if not spec or not spec.loader:
|
||||
logger.error(f"[GGUF Import] Failed to create spec for '{init_path}'")
|
||||
return None
|
||||
|
||||
package = importlib.util.module_from_spec(spec)
|
||||
package.__path__ = [gguf_path]
|
||||
sys.modules[namespace] = package
|
||||
spec.loader.exec_module(package)
|
||||
logger.debug(f"[GGUF Import] Loaded main package '{namespace}'")
|
||||
|
||||
# Load submodules
|
||||
loaded = []
|
||||
for submod_name in ["loader", "ops", "nodes"]:
|
||||
submod_path = os.path.join(gguf_path, f"{submod_name}.py")
|
||||
if os.path.exists(submod_path):
|
||||
submod_spec = importlib.util.spec_from_file_location(
|
||||
f"{namespace}.{submod_name}", submod_path
|
||||
)
|
||||
if submod_spec and submod_spec.loader:
|
||||
submod = importlib.util.module_from_spec(submod_spec)
|
||||
submod.__package__ = namespace
|
||||
sys.modules[f"{namespace}.{submod_name}"] = submod
|
||||
submod_spec.loader.exec_module(submod)
|
||||
setattr(package, submod_name, submod)
|
||||
loaded.append(submod_name)
|
||||
logger.debug(f"[GGUF Import] Loaded submodule '{submod_name}'")
|
||||
|
||||
logger.info(f"[GGUF Import] Direct Load success: {loaded}")
|
||||
return package
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[GGUF Import] Direct Load failed: {e}", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def get_gguf_modules() -> Tuple[Any, Any, Any]:
|
||||
"""Get ComfyUI-GGUF modules (loader, ops, nodes).
|
||||
|
||||
Returns:
|
||||
Tuple of (loader_module, ops_module, nodes_module)
|
||||
|
||||
Raises:
|
||||
RuntimeError: If ComfyUI-GGUF cannot be found or loaded.
|
||||
"""
|
||||
logger.debug("[GGUF Import] Starting module search...")
|
||||
|
||||
# Try to find already loaded module first
|
||||
gguf_module = _find_gguf_module()
|
||||
|
||||
if gguf_module is None:
|
||||
logger.info("[GGUF Import] Not found in sys.modules, trying direct load...")
|
||||
gguf_module = _load_gguf_modules_directly()
|
||||
|
||||
if gguf_module is None:
|
||||
raise RuntimeError(
|
||||
"ComfyUI-GGUF is not installed. "
|
||||
"Please install from https://github.com/city96/ComfyUI-GGUF"
|
||||
)
|
||||
|
||||
# Extract submodules
|
||||
loader = getattr(gguf_module, "loader", None)
|
||||
ops = getattr(gguf_module, "ops", None)
|
||||
nodes = getattr(gguf_module, "nodes", None)
|
||||
|
||||
if loader is None or ops is None or nodes is None:
|
||||
missing = [
|
||||
name
|
||||
for name, mod in [("loader", loader), ("ops", ops), ("nodes", nodes)]
|
||||
if mod is None
|
||||
]
|
||||
raise RuntimeError(f"ComfyUI-GGUF missing submodules: {missing}")
|
||||
|
||||
logger.debug("[GGUF Import] All modules loaded successfully")
|
||||
return loader, ops, nodes
|
||||
|
||||
|
||||
def get_gguf_sd_loader():
|
||||
"""Get the gguf_sd_loader function from ComfyUI-GGUF."""
|
||||
loader, _, _ = get_gguf_modules()
|
||||
return getattr(loader, "gguf_sd_loader")
|
||||
|
||||
|
||||
def get_ggml_ops():
|
||||
"""Get the GGMLOps class from ComfyUI-GGUF."""
|
||||
_, ops, _ = get_gguf_modules()
|
||||
return getattr(ops, "GGMLOps")
|
||||
|
||||
|
||||
def get_gguf_model_patcher():
|
||||
"""Get the GGUFModelPatcher class from ComfyUI-GGUF."""
|
||||
_, _, nodes = get_gguf_modules()
|
||||
return getattr(nodes, "GGUFModelPatcher")
|
||||
@@ -8,6 +8,7 @@ and tracks the cycle progress which persists across workflow save/load.
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from ..utils.utils import get_lora_info
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -54,8 +55,14 @@ class LoraCyclerLM:
|
||||
current_index = cycler_config.get("current_index", 1) # 1-based
|
||||
model_strength = float(cycler_config.get("model_strength", 1.0))
|
||||
clip_strength = float(cycler_config.get("clip_strength", 1.0))
|
||||
use_same_clip_strength = cycler_config.get("use_same_clip_strength", True)
|
||||
use_preset_strength = cycler_config.get("use_preset_strength", False)
|
||||
preset_strength_scale = float(cycler_config.get("preset_strength_scale", 1.0))
|
||||
sort_by = "filename"
|
||||
|
||||
# Include "no lora" option
|
||||
include_no_lora = cycler_config.get("include_no_lora", False)
|
||||
|
||||
# Dual-index mechanism for batch queue synchronization
|
||||
execution_index = cycler_config.get("execution_index") # Can be None
|
||||
# next_index_from_config = cycler_config.get("next_index") # Not used on backend
|
||||
@@ -71,7 +78,10 @@ class LoraCyclerLM:
|
||||
|
||||
total_count = len(lora_list)
|
||||
|
||||
if total_count == 0:
|
||||
# Calculate effective total count (includes no lora option if enabled)
|
||||
effective_total_count = total_count + 1 if include_no_lora else total_count
|
||||
|
||||
if total_count == 0 and not include_no_lora:
|
||||
logger.warning("[LoraCyclerLM] No LoRAs available in pool")
|
||||
return {
|
||||
"result": ([],),
|
||||
@@ -93,42 +103,99 @@ class LoraCyclerLM:
|
||||
else:
|
||||
actual_index = current_index
|
||||
|
||||
# Clamp index to valid range (1-based)
|
||||
clamped_index = max(1, min(actual_index, total_count))
|
||||
# Clamp index to valid range (1-based, includes no lora if enabled)
|
||||
clamped_index = max(1, min(actual_index, effective_total_count))
|
||||
|
||||
# Get LoRA at current index (convert to 0-based for list access)
|
||||
current_lora = lora_list[clamped_index - 1]
|
||||
# Check if current index is the "no lora" option (last position when include_no_lora is True)
|
||||
is_no_lora = include_no_lora and clamped_index == effective_total_count
|
||||
|
||||
# Build LORA_STACK with single LoRA
|
||||
lora_path, _ = get_lora_info(current_lora["file_name"])
|
||||
if not lora_path:
|
||||
logger.warning(
|
||||
f"[LoraCyclerLM] Could not find path for LoRA: {current_lora['file_name']}"
|
||||
)
|
||||
if is_no_lora:
|
||||
# "No LoRA" option - return empty stack
|
||||
lora_stack = []
|
||||
current_lora_name = "No LoRA"
|
||||
current_lora_filename = "No LoRA"
|
||||
else:
|
||||
# Normalize path separators
|
||||
lora_path = lora_path.replace("/", os.sep)
|
||||
lora_stack = [(lora_path, model_strength, clip_strength)]
|
||||
# Get LoRA at current index (convert to 0-based for list access)
|
||||
current_lora = lora_list[clamped_index - 1]
|
||||
current_lora_name = current_lora["file_name"]
|
||||
current_lora_filename = current_lora["file_name"]
|
||||
|
||||
# Build LORA_STACK with single LoRA
|
||||
if current_lora["file_name"] == "None":
|
||||
lora_path = None
|
||||
else:
|
||||
lora_path, _ = get_lora_info(current_lora["file_name"])
|
||||
|
||||
if not lora_path:
|
||||
if current_lora["file_name"] != "None":
|
||||
logger.warning(
|
||||
f"[LoraCyclerLM] Could not find path for LoRA: {current_lora['file_name']}"
|
||||
)
|
||||
lora_stack = []
|
||||
else:
|
||||
# Normalize path separators
|
||||
lora_path = lora_path.replace("/", os.sep)
|
||||
|
||||
if use_preset_strength:
|
||||
lora_metadata = await lora_service.get_lora_metadata_by_filename(
|
||||
current_lora["file_name"]
|
||||
)
|
||||
if lora_metadata:
|
||||
recommended_strength = (
|
||||
lora_service.get_recommended_strength_from_lora_data(
|
||||
lora_metadata
|
||||
)
|
||||
)
|
||||
if recommended_strength is not None:
|
||||
model_strength = round(
|
||||
recommended_strength * preset_strength_scale, 2
|
||||
)
|
||||
|
||||
if use_same_clip_strength:
|
||||
clip_strength = model_strength
|
||||
else:
|
||||
recommended_clip_strength = (
|
||||
lora_service.get_recommended_clip_strength_from_lora_data(
|
||||
lora_metadata
|
||||
)
|
||||
)
|
||||
if recommended_clip_strength is not None:
|
||||
clip_strength = round(
|
||||
recommended_clip_strength * preset_strength_scale, 2
|
||||
)
|
||||
elif use_same_clip_strength:
|
||||
clip_strength = model_strength
|
||||
elif use_same_clip_strength:
|
||||
clip_strength = model_strength
|
||||
|
||||
lora_stack = [(lora_path, model_strength, clip_strength)]
|
||||
|
||||
# Calculate next index (wrap to 1 if at end)
|
||||
next_index = clamped_index + 1
|
||||
if next_index > total_count:
|
||||
if next_index > effective_total_count:
|
||||
next_index = 1
|
||||
|
||||
# Get next LoRA for UI display (what will be used next generation)
|
||||
next_lora = lora_list[next_index - 1]
|
||||
next_display_name = next_lora["file_name"]
|
||||
is_next_no_lora = include_no_lora and next_index == effective_total_count
|
||||
if is_next_no_lora:
|
||||
next_display_name = "No LoRA"
|
||||
next_lora_filename = "No LoRA"
|
||||
else:
|
||||
next_lora = lora_list[next_index - 1]
|
||||
next_display_name = next_lora["file_name"]
|
||||
next_lora_filename = next_lora["file_name"]
|
||||
|
||||
return {
|
||||
"result": (lora_stack,),
|
||||
"ui": {
|
||||
"current_index": [clamped_index],
|
||||
"next_index": [next_index],
|
||||
"total_count": [total_count],
|
||||
"current_lora_name": [current_lora["file_name"]],
|
||||
"current_lora_filename": [current_lora["file_name"]],
|
||||
"total_count": [
|
||||
total_count
|
||||
], # Return actual LoRA count, not effective_total_count
|
||||
"current_lora_name": [current_lora_name],
|
||||
"current_lora_filename": [current_lora_filename],
|
||||
"next_lora_name": [next_display_name],
|
||||
"next_lora_filename": [next_lora["file_name"]],
|
||||
"next_lora_filename": [next_lora_filename],
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1,22 +1,139 @@
|
||||
import importlib
|
||||
import logging
|
||||
import re
|
||||
import comfy.utils # type: ignore
|
||||
import comfy.sd # type: ignore
|
||||
|
||||
import comfy.sd # type: ignore
|
||||
import comfy.utils # type: ignore
|
||||
|
||||
from ..utils.utils import get_lora_info_absolute
|
||||
from .utils import FlexibleOptionalInputType, any_type, extract_lora_name, get_loras_list, nunchaku_load_lora
|
||||
from .utils import (
|
||||
FlexibleOptionalInputType,
|
||||
any_type,
|
||||
apply_lora_syntax_format,
|
||||
detect_nunchaku_model_kind,
|
||||
extract_lora_name,
|
||||
get_loras_list,
|
||||
nunchaku_load_lora,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_nunchaku_load_qwen_loras():
|
||||
try:
|
||||
module = importlib.import_module(".nunchaku_qwen", __package__)
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"Qwen-Image LoRA loading requires the ComfyUI runtime with its torch dependency available."
|
||||
) from exc
|
||||
return module.nunchaku_load_qwen_loras
|
||||
|
||||
|
||||
def _collect_stack_entries(lora_stack):
|
||||
entries = []
|
||||
if not lora_stack:
|
||||
return entries
|
||||
|
||||
for lora_path, model_strength, clip_strength in lora_stack:
|
||||
lora_name = extract_lora_name(lora_path)
|
||||
absolute_lora_path, trigger_words = get_lora_info_absolute(lora_name)
|
||||
entries.append({
|
||||
"name": lora_name,
|
||||
"absolute_path": absolute_lora_path,
|
||||
"input_path": lora_path,
|
||||
"model_strength": float(model_strength),
|
||||
"clip_strength": float(clip_strength),
|
||||
"trigger_words": trigger_words,
|
||||
})
|
||||
return entries
|
||||
|
||||
|
||||
def _collect_widget_entries(kwargs):
|
||||
entries = []
|
||||
for lora in get_loras_list(kwargs):
|
||||
if not lora.get("active", False):
|
||||
continue
|
||||
lora_name = apply_lora_syntax_format(lora["name"])
|
||||
model_strength = float(lora["strength"])
|
||||
clip_strength = float(lora.get("clipStrength", model_strength))
|
||||
lora_path, trigger_words = get_lora_info_absolute(lora_name)
|
||||
entries.append({
|
||||
"name": lora_name,
|
||||
"absolute_path": lora_path,
|
||||
"input_path": lora_path,
|
||||
"model_strength": model_strength,
|
||||
"clip_strength": clip_strength,
|
||||
"trigger_words": trigger_words,
|
||||
})
|
||||
return entries
|
||||
|
||||
|
||||
def _format_loaded_loras(loaded_loras):
|
||||
formatted_loras = []
|
||||
for item in loaded_loras:
|
||||
if item["include_clip_strength"]:
|
||||
formatted_loras.append(
|
||||
f"<lora:{item['name']}:{item['model_strength']}:{item['clip_strength']}>"
|
||||
)
|
||||
else:
|
||||
formatted_loras.append(f"<lora:{item['name']}:{item['model_strength']}>")
|
||||
return " ".join(formatted_loras)
|
||||
|
||||
|
||||
def _apply_entries(model, clip, lora_entries, nunchaku_model_kind):
|
||||
loaded_loras = []
|
||||
all_trigger_words = []
|
||||
|
||||
if nunchaku_model_kind == "qwen_image":
|
||||
nunchaku_load_qwen_loras = _get_nunchaku_load_qwen_loras()
|
||||
qwen_lora_configs = []
|
||||
for entry in lora_entries:
|
||||
qwen_lora_configs.append((entry["absolute_path"], entry["model_strength"]))
|
||||
loaded_loras.append({
|
||||
"name": entry["name"],
|
||||
"model_strength": entry["model_strength"],
|
||||
"clip_strength": entry["model_strength"],
|
||||
"include_clip_strength": False,
|
||||
})
|
||||
all_trigger_words.extend(entry["trigger_words"])
|
||||
if qwen_lora_configs:
|
||||
model = nunchaku_load_qwen_loras(model, qwen_lora_configs)
|
||||
return model, clip, loaded_loras, all_trigger_words
|
||||
|
||||
for entry in lora_entries:
|
||||
if nunchaku_model_kind == "flux":
|
||||
model = nunchaku_load_lora(model, entry["input_path"], entry["model_strength"])
|
||||
else:
|
||||
lora = comfy.utils.load_torch_file(entry["absolute_path"], safe_load=True)
|
||||
model, clip = comfy.sd.load_lora_for_models(
|
||||
model,
|
||||
clip,
|
||||
lora,
|
||||
entry["model_strength"],
|
||||
entry["clip_strength"],
|
||||
)
|
||||
|
||||
include_clip_strength = nunchaku_model_kind is None and abs(entry["model_strength"] - entry["clip_strength"]) > 0.001
|
||||
loaded_loras.append({
|
||||
"name": entry["name"],
|
||||
"model_strength": entry["model_strength"],
|
||||
"clip_strength": entry["clip_strength"],
|
||||
"include_clip_strength": include_clip_strength,
|
||||
})
|
||||
all_trigger_words.extend(entry["trigger_words"])
|
||||
|
||||
return model, clip, loaded_loras, all_trigger_words
|
||||
|
||||
|
||||
class LoraLoaderLM:
|
||||
NAME = "Lora Loader (LoraManager)"
|
||||
CATEGORY = "Lora Manager/loaders"
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("MODEL",),
|
||||
# "clip": ("CLIP",),
|
||||
"text": ("AUTOCOMPLETE_TEXT_LORAS", {
|
||||
"placeholder": "Search LoRAs to add...",
|
||||
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
||||
@@ -28,114 +145,30 @@ class LoraLoaderLM:
|
||||
RETURN_TYPES = ("MODEL", "CLIP", "STRING", "STRING")
|
||||
RETURN_NAMES = ("MODEL", "CLIP", "trigger_words", "loaded_loras")
|
||||
FUNCTION = "load_loras"
|
||||
|
||||
|
||||
def load_loras(self, model, text, **kwargs):
|
||||
"""Loads multiple LoRAs based on the kwargs input and lora_stack."""
|
||||
loaded_loras = []
|
||||
all_trigger_words = []
|
||||
|
||||
clip = kwargs.get('clip', None)
|
||||
lora_stack = kwargs.get('lora_stack', None)
|
||||
|
||||
# Check if model is a Nunchaku Flux model - simplified approach
|
||||
is_nunchaku_model = False
|
||||
|
||||
try:
|
||||
model_wrapper = model.model.diffusion_model
|
||||
# Check if model is a Nunchaku Flux model using only class name
|
||||
if model_wrapper.__class__.__name__ == "ComfyFluxWrapper":
|
||||
is_nunchaku_model = True
|
||||
logger.info("Detected Nunchaku Flux model")
|
||||
except (AttributeError, TypeError):
|
||||
# Not a model with the expected structure
|
||||
pass
|
||||
|
||||
# First process lora_stack if available
|
||||
if lora_stack:
|
||||
for lora_path, model_strength, clip_strength in lora_stack:
|
||||
# Extract lora name and convert to absolute path
|
||||
# lora_stack stores relative paths, but load_torch_file needs absolute paths
|
||||
lora_name = extract_lora_name(lora_path)
|
||||
absolute_lora_path, trigger_words = get_lora_info_absolute(lora_name)
|
||||
|
||||
# Apply the LoRA using the appropriate loader
|
||||
if is_nunchaku_model:
|
||||
# Use our custom function for Flux models
|
||||
model = nunchaku_load_lora(model, lora_path, model_strength)
|
||||
# clip remains unchanged for Nunchaku models
|
||||
else:
|
||||
# Use lower-level API to load LoRA directly without folder_paths validation
|
||||
lora = comfy.utils.load_torch_file(absolute_lora_path, safe_load=True)
|
||||
model, clip = comfy.sd.load_lora_for_models(model, clip, lora, model_strength, clip_strength)
|
||||
|
||||
all_trigger_words.extend(trigger_words)
|
||||
# Add clip strength to output if different from model strength (except for Nunchaku models)
|
||||
if not is_nunchaku_model and abs(model_strength - clip_strength) > 0.001:
|
||||
loaded_loras.append(f"{lora_name}: {model_strength},{clip_strength}")
|
||||
else:
|
||||
loaded_loras.append(f"{lora_name}: {model_strength}")
|
||||
|
||||
# Then process loras from kwargs with support for both old and new formats
|
||||
loras_list = get_loras_list(kwargs)
|
||||
for lora in loras_list:
|
||||
if not lora.get('active', False):
|
||||
continue
|
||||
|
||||
lora_name = lora['name']
|
||||
model_strength = float(lora['strength'])
|
||||
# Get clip strength - use model strength as default if not specified
|
||||
clip_strength = float(lora.get('clipStrength', model_strength))
|
||||
|
||||
# Get lora path and trigger words
|
||||
lora_path, trigger_words = get_lora_info_absolute(lora_name)
|
||||
|
||||
# Apply the LoRA using the appropriate loader
|
||||
if is_nunchaku_model:
|
||||
# For Nunchaku models, use our custom function
|
||||
model = nunchaku_load_lora(model, lora_path, model_strength)
|
||||
# clip remains unchanged
|
||||
else:
|
||||
# Use lower-level API to load LoRA directly without folder_paths validation
|
||||
lora = comfy.utils.load_torch_file(lora_path, safe_load=True)
|
||||
model, clip = comfy.sd.load_lora_for_models(model, clip, lora, model_strength, clip_strength)
|
||||
|
||||
# Include clip strength in output if different from model strength and not a Nunchaku model
|
||||
if not is_nunchaku_model and abs(model_strength - clip_strength) > 0.001:
|
||||
loaded_loras.append(f"{lora_name}: {model_strength},{clip_strength}")
|
||||
else:
|
||||
loaded_loras.append(f"{lora_name}: {model_strength}")
|
||||
|
||||
# Add trigger words to collection
|
||||
all_trigger_words.extend(trigger_words)
|
||||
|
||||
# use ',, ' to separate trigger words for group mode
|
||||
trigger_words_text = ",, ".join(all_trigger_words) if all_trigger_words else ""
|
||||
|
||||
# Format loaded_loras with support for both formats
|
||||
formatted_loras = []
|
||||
for item in loaded_loras:
|
||||
parts = item.split(":")
|
||||
lora_name = parts[0]
|
||||
strength_parts = parts[1].strip().split(",")
|
||||
|
||||
if len(strength_parts) > 1:
|
||||
# Different model and clip strengths
|
||||
model_str = strength_parts[0].strip()
|
||||
clip_str = strength_parts[1].strip()
|
||||
formatted_loras.append(f"<lora:{lora_name}:{model_str}:{clip_str}>")
|
||||
else:
|
||||
# Same strength for both
|
||||
model_str = strength_parts[0].strip()
|
||||
formatted_loras.append(f"<lora:{lora_name}:{model_str}>")
|
||||
|
||||
formatted_loras_text = " ".join(formatted_loras)
|
||||
del text
|
||||
clip = kwargs.get("clip", None)
|
||||
lora_entries = _collect_stack_entries(kwargs.get("lora_stack", None))
|
||||
lora_entries.extend(_collect_widget_entries(kwargs))
|
||||
|
||||
nunchaku_model_kind = detect_nunchaku_model_kind(model)
|
||||
if nunchaku_model_kind == "flux":
|
||||
logger.info("Detected Nunchaku Flux model")
|
||||
elif nunchaku_model_kind == "qwen_image":
|
||||
logger.info("Detected Nunchaku Qwen-Image model")
|
||||
|
||||
model, clip, loaded_loras, all_trigger_words = _apply_entries(model, clip, lora_entries, nunchaku_model_kind)
|
||||
trigger_words_text = ",, ".join(all_trigger_words) if all_trigger_words else ""
|
||||
formatted_loras_text = _format_loaded_loras(loaded_loras)
|
||||
return (model, clip, trigger_words_text, formatted_loras_text)
|
||||
|
||||
|
||||
class LoraTextLoaderLM:
|
||||
NAME = "LoRA Text Loader (LoraManager)"
|
||||
CATEGORY = "Lora Manager/loaders"
|
||||
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
@@ -143,131 +176,55 @@ class LoraTextLoaderLM:
|
||||
"model": ("MODEL",),
|
||||
"lora_syntax": ("STRING", {
|
||||
"forceInput": True,
|
||||
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation"
|
||||
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
||||
}),
|
||||
},
|
||||
"optional": {
|
||||
"clip": ("CLIP",),
|
||||
"lora_stack": ("LORA_STACK",),
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("MODEL", "CLIP", "STRING", "STRING")
|
||||
RETURN_NAMES = ("MODEL", "CLIP", "trigger_words", "loaded_loras")
|
||||
FUNCTION = "load_loras_from_text"
|
||||
|
||||
|
||||
def parse_lora_syntax(self, text):
|
||||
"""Parse LoRA syntax from text input."""
|
||||
# Pattern to match <lora:name:strength> or <lora:name:model_strength:clip_strength>
|
||||
pattern = r'<lora:([^:>]+):([^:>]+)(?::([^:>]+))?>'
|
||||
pattern = r"<lora:([^:>]+):([^:>]+)(?::([^:>]+))?>"
|
||||
matches = re.findall(pattern, text, re.IGNORECASE)
|
||||
|
||||
|
||||
loras = []
|
||||
for match in matches:
|
||||
lora_name = match[0]
|
||||
model_strength = float(match[1])
|
||||
clip_strength = float(match[2]) if match[2] else model_strength
|
||||
|
||||
loras.append({
|
||||
'name': lora_name,
|
||||
'model_strength': model_strength,
|
||||
'clip_strength': clip_strength
|
||||
"name": match[0],
|
||||
"model_strength": model_strength,
|
||||
"clip_strength": float(match[2]) if match[2] else model_strength,
|
||||
})
|
||||
|
||||
return loras
|
||||
|
||||
|
||||
def load_loras_from_text(self, model, lora_syntax, clip=None, lora_stack=None):
|
||||
"""Load LoRAs based on text syntax input."""
|
||||
loaded_loras = []
|
||||
all_trigger_words = []
|
||||
|
||||
# Check if model is a Nunchaku Flux model - simplified approach
|
||||
is_nunchaku_model = False
|
||||
|
||||
try:
|
||||
model_wrapper = model.model.diffusion_model
|
||||
# Check if model is a Nunchaku Flux model using only class name
|
||||
if model_wrapper.__class__.__name__ == "ComfyFluxWrapper":
|
||||
is_nunchaku_model = True
|
||||
logger.info("Detected Nunchaku Flux model")
|
||||
except (AttributeError, TypeError):
|
||||
# Not a model with the expected structure
|
||||
pass
|
||||
|
||||
# First process lora_stack if available
|
||||
if lora_stack:
|
||||
for lora_path, model_strength, clip_strength in lora_stack:
|
||||
# Extract lora name and convert to absolute path
|
||||
# lora_stack stores relative paths, but load_torch_file needs absolute paths
|
||||
lora_name = extract_lora_name(lora_path)
|
||||
absolute_lora_path, trigger_words = get_lora_info_absolute(lora_name)
|
||||
|
||||
# Apply the LoRA using the appropriate loader
|
||||
if is_nunchaku_model:
|
||||
# Use our custom function for Flux models
|
||||
model = nunchaku_load_lora(model, lora_path, model_strength)
|
||||
# clip remains unchanged for Nunchaku models
|
||||
else:
|
||||
# Use lower-level API to load LoRA directly without folder_paths validation
|
||||
lora = comfy.utils.load_torch_file(absolute_lora_path, safe_load=True)
|
||||
model, clip = comfy.sd.load_lora_for_models(model, clip, lora, model_strength, clip_strength)
|
||||
|
||||
all_trigger_words.extend(trigger_words)
|
||||
# Add clip strength to output if different from model strength (except for Nunchaku models)
|
||||
if not is_nunchaku_model and abs(model_strength - clip_strength) > 0.001:
|
||||
loaded_loras.append(f"{lora_name}: {model_strength},{clip_strength}")
|
||||
else:
|
||||
loaded_loras.append(f"{lora_name}: {model_strength}")
|
||||
|
||||
# Parse and process LoRAs from text syntax
|
||||
parsed_loras = self.parse_lora_syntax(lora_syntax)
|
||||
for lora in parsed_loras:
|
||||
lora_name = lora['name']
|
||||
model_strength = lora['model_strength']
|
||||
clip_strength = lora['clip_strength']
|
||||
|
||||
# Get lora path and trigger words
|
||||
lora_path, trigger_words = get_lora_info_absolute(lora_name)
|
||||
|
||||
# Apply the LoRA using the appropriate loader
|
||||
if is_nunchaku_model:
|
||||
# For Nunchaku models, use our custom function
|
||||
model = nunchaku_load_lora(model, lora_path, model_strength)
|
||||
# clip remains unchanged
|
||||
else:
|
||||
# Use lower-level API to load LoRA directly without folder_paths validation
|
||||
lora = comfy.utils.load_torch_file(lora_path, safe_load=True)
|
||||
model, clip = comfy.sd.load_lora_for_models(model, clip, lora, model_strength, clip_strength)
|
||||
|
||||
# Include clip strength in output if different from model strength and not a Nunchaku model
|
||||
if not is_nunchaku_model and abs(model_strength - clip_strength) > 0.001:
|
||||
loaded_loras.append(f"{lora_name}: {model_strength},{clip_strength}")
|
||||
else:
|
||||
loaded_loras.append(f"{lora_name}: {model_strength}")
|
||||
|
||||
# Add trigger words to collection
|
||||
all_trigger_words.extend(trigger_words)
|
||||
|
||||
# use ',, ' to separate trigger words for group mode
|
||||
lora_entries = _collect_stack_entries(lora_stack)
|
||||
for lora in self.parse_lora_syntax(lora_syntax):
|
||||
lora_path, trigger_words = get_lora_info_absolute(lora["name"])
|
||||
lora_entries.append({
|
||||
"name": lora["name"],
|
||||
"absolute_path": lora_path,
|
||||
"input_path": lora_path,
|
||||
"model_strength": lora["model_strength"],
|
||||
"clip_strength": lora["clip_strength"],
|
||||
"trigger_words": trigger_words,
|
||||
})
|
||||
|
||||
nunchaku_model_kind = detect_nunchaku_model_kind(model)
|
||||
if nunchaku_model_kind == "flux":
|
||||
logger.info("Detected Nunchaku Flux model")
|
||||
elif nunchaku_model_kind == "qwen_image":
|
||||
logger.info("Detected Nunchaku Qwen-Image model")
|
||||
|
||||
model, clip, loaded_loras, all_trigger_words = _apply_entries(model, clip, lora_entries, nunchaku_model_kind)
|
||||
trigger_words_text = ",, ".join(all_trigger_words) if all_trigger_words else ""
|
||||
|
||||
# Format loaded_loras with support for both formats
|
||||
formatted_loras = []
|
||||
for item in loaded_loras:
|
||||
parts = item.split(":")
|
||||
lora_name = parts[0].strip()
|
||||
strength_parts = parts[1].strip().split(",")
|
||||
|
||||
if len(strength_parts) > 1:
|
||||
# Different model and clip strengths
|
||||
model_str = strength_parts[0].strip()
|
||||
clip_str = strength_parts[1].strip()
|
||||
formatted_loras.append(f"<lora:{lora_name}:{model_str}:{clip_str}>")
|
||||
else:
|
||||
# Same strength for both
|
||||
model_str = strength_parts[0].strip()
|
||||
formatted_loras.append(f"<lora:{lora_name}:{model_str}>")
|
||||
|
||||
formatted_loras_text = " ".join(formatted_loras)
|
||||
|
||||
return (model, clip, trigger_words_text, formatted_loras_text)
|
||||
formatted_loras_text = _format_loaded_loras(loaded_loras)
|
||||
return (model, clip, trigger_words_text, formatted_loras_text)
|
||||
|
||||
@@ -82,6 +82,7 @@ class LoraPoolLM:
|
||||
"folders": {"include": [], "exclude": []},
|
||||
"favoritesOnly": False,
|
||||
"license": {"noCreditRequired": False, "allowSelling": False},
|
||||
"namePatterns": {"include": [], "exclude": [], "useRegex": False},
|
||||
},
|
||||
"preview": {"matchCount": 0, "lastUpdated": 0},
|
||||
}
|
||||
|
||||
@@ -7,10 +7,8 @@ and tracks the last used combination for reuse.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import random
|
||||
import os
|
||||
from ..utils.utils import get_lora_info
|
||||
from .utils import extract_lora_name
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
26
py/nodes/lora_stack_combiner.py
Normal file
26
py/nodes/lora_stack_combiner.py
Normal file
@@ -0,0 +1,26 @@
|
||||
class LoraStackCombinerLM:
|
||||
NAME = "Lora Stack Combiner (LoraManager)"
|
||||
CATEGORY = "Lora Manager/stackers"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"lora_stack_a": ("LORA_STACK",),
|
||||
"lora_stack_b": ("LORA_STACK",),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("LORA_STACK",)
|
||||
RETURN_NAMES = ("LORA_STACK",)
|
||||
FUNCTION = "combine_stacks"
|
||||
|
||||
def combine_stacks(self, lora_stack_a, lora_stack_b):
|
||||
combined_stack = []
|
||||
|
||||
if lora_stack_a:
|
||||
combined_stack.extend(lora_stack_a)
|
||||
if lora_stack_b:
|
||||
combined_stack.extend(lora_stack_b)
|
||||
|
||||
return (combined_stack,)
|
||||
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
from ..utils.utils import get_lora_info
|
||||
from .utils import FlexibleOptionalInputType, any_type, extract_lora_name, get_loras_list
|
||||
from .utils import FlexibleOptionalInputType, any_type, apply_lora_syntax_format, extract_lora_name, get_loras_list
|
||||
|
||||
import logging
|
||||
|
||||
@@ -48,7 +48,7 @@ class LoraStackerLM:
|
||||
if not lora.get('active', False):
|
||||
continue
|
||||
|
||||
lora_name = lora['name']
|
||||
lora_name = apply_lora_syntax_format(lora['name'])
|
||||
model_strength = float(lora['strength'])
|
||||
# Get clip strength - use model strength as default if not specified
|
||||
clip_strength = float(lora.get('clipStrength', model_strength))
|
||||
|
||||
570
py/nodes/nunchaku_qwen.py
Normal file
570
py/nodes/nunchaku_qwen.py
Normal file
@@ -0,0 +1,570 @@
|
||||
from __future__ import annotations
|
||||
|
||||
"""Qwen-Image LoRA support for Nunchaku models.
|
||||
|
||||
Portions of the LoRA mapping/application logic in this file are adapted from
|
||||
ComfyUI-QwenImageLoraLoader by GitHub user ussoewwin:
|
||||
https://github.com/ussoewwin/ComfyUI-QwenImageLoraLoader
|
||||
|
||||
The upstream project is licensed under Apache License 2.0.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple, Union
|
||||
|
||||
import comfy.utils # type: ignore
|
||||
import folder_paths # type: ignore
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from safetensors import safe_open
|
||||
|
||||
from nunchaku.lora.flux.nunchaku_converter import (
|
||||
pack_lowrank_weight,
|
||||
unpack_lowrank_weight,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
KEY_MAPPING = [
|
||||
(re.compile(r"^(layers)[._](\d+)[._]attention[._]to[._]([qkv])$"), r"\1.\2.attention.to_qkv", "qkv", lambda m: m.group(3).upper()),
|
||||
(re.compile(r"^(layers)[._](\d+)[._]feed_forward[._](w1|w3)$"), r"\1.\2.feed_forward.net.0.proj", "glu", lambda m: m.group(3)),
|
||||
(re.compile(r"^(layers)[._](\d+)[._]feed_forward[._]w2$"), r"\1.\2.feed_forward.net.2", "regular", None),
|
||||
(re.compile(r"^(layers)[._](\d+)[._](.*)$"), r"\1.\2.\3", "regular", None),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._]attn[._]to[._]([qkv])$"), r"\1.\2.attn.to_qkv", "qkv", lambda m: m.group(3).upper()),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._]attn[._](q|k|v)[._]proj$"), r"\1.\2.attn.to_qkv", "qkv", lambda m: m.group(3).upper()),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._]attn[._]add[._](q|k|v)[._]proj$"), r"\1.\2.attn.add_qkv_proj", "add_qkv", lambda m: m.group(3).upper()),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._]out[._]proj[._]context$"), r"\1.\2.attn.to_add_out", "regular", None),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._]out[._]proj$"), r"\1.\2.attn.to_out.0", "regular", None),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._]attn[._]to[._]out$"), r"\1.\2.attn.to_out.0", "regular", None),
|
||||
(re.compile(r"^(single_transformer_blocks)[._](\d+)[._]attn[._]to[._]([qkv])$"), r"\1.\2.attn.to_qkv", "qkv", lambda m: m.group(3).upper()),
|
||||
(re.compile(r"^(single_transformer_blocks)[._](\d+)[._]attn[._]to[._]out$"), r"\1.\2.attn.to_out", "regular", None),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._]ff[._]net[._]0(?:[._]proj)?$"), r"\1.\2.mlp_fc1", "regular", None),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._]ff[._]net[._]2$"), r"\1.\2.mlp_fc2", "regular", None),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._]ff_context[._]net[._]0(?:[._]proj)?$"), r"\1.\2.mlp_context_fc1", "regular", None),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._]ff_context[._]net[._]2$"), r"\1.\2.mlp_context_fc2", "regular", None),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._](img_mlp)[._](net)[._](0)[._](proj)$"), r"\1.\2.\3.\4.\5.\6", "regular", None),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._](img_mlp)[._](net)[._](2)$"), r"\1.\2.\3.\4.\5", "regular", None),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._](txt_mlp)[._](net)[._](0)[._](proj)$"), r"\1.\2.\3.\4.\5.\6", "regular", None),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._](txt_mlp)[._](net)[._](2)$"), r"\1.\2.\3.\4.\5", "regular", None),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._](img_mod)[._](1)$"), r"\1.\2.\3.\4", "regular", None),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._](txt_mod)[._](1)$"), r"\1.\2.\3.\4", "regular", None),
|
||||
(re.compile(r"^(single_transformer_blocks)[._](\d+)[._]proj[._]out$"), r"\1.\2.proj_out", "single_proj_out", None),
|
||||
(re.compile(r"^(single_transformer_blocks)[._](\d+)[._]proj[._]mlp$"), r"\1.\2.mlp_fc1", "regular", None),
|
||||
(re.compile(r"^(single_transformer_blocks)[._](\d+)[._]norm[._]linear$"), r"\1.\2.norm.linear", "regular", None),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._]norm1[._]linear$"), r"\1.\2.norm1.linear", "regular", None),
|
||||
(re.compile(r"^(transformer_blocks)[._](\d+)[._]norm1_context[._]linear$"), r"\1.\2.norm1_context.linear", "regular", None),
|
||||
(re.compile(r"^(img_in)$"), r"\1", "regular", None),
|
||||
(re.compile(r"^(txt_in)$"), r"\1", "regular", None),
|
||||
(re.compile(r"^(proj_out)$"), r"\1", "regular", None),
|
||||
(re.compile(r"^(norm_out)[._](linear)$"), r"\1.\2", "regular", None),
|
||||
(re.compile(r"^(time_text_embed)[._](timestep_embedder)[._](linear_1)$"), r"\1.\2.\3", "regular", None),
|
||||
(re.compile(r"^(time_text_embed)[._](timestep_embedder)[._](linear_2)$"), r"\1.\2.\3", "regular", None),
|
||||
]
|
||||
|
||||
_RE_LORA_SUFFIX = re.compile(r"\.(?P<tag>lora(?:[._](?:A|B|down|up)))(?:\.[^.]+)*\.weight$")
|
||||
_RE_ALPHA_SUFFIX = re.compile(r"\.(?:alpha|lora_alpha)(?:\.[^.]+)*$")
|
||||
|
||||
|
||||
def _rename_layer_underscore_layer_name(old_name: str) -> str:
|
||||
rules = [
|
||||
(r"_(\d+)_attn_to_out_(\d+)", r".\1.attn.to_out.\2"),
|
||||
(r"_(\d+)_img_mlp_net_(\d+)_proj", r".\1.img_mlp.net.\2.proj"),
|
||||
(r"_(\d+)_txt_mlp_net_(\d+)_proj", r".\1.txt_mlp.net.\2.proj"),
|
||||
(r"_(\d+)_img_mlp_net_(\d+)", r".\1.img_mlp.net.\2"),
|
||||
(r"_(\d+)_txt_mlp_net_(\d+)", r".\1.txt_mlp.net.\2"),
|
||||
(r"_(\d+)_img_mod_(\d+)", r".\1.img_mod.\2"),
|
||||
(r"_(\d+)_txt_mod_(\d+)", r".\1.txt_mod.\2"),
|
||||
(r"_(\d+)_attn_", r".\1.attn."),
|
||||
]
|
||||
new_name = old_name
|
||||
for pattern, replacement in rules:
|
||||
new_name = re.sub(pattern, replacement, new_name)
|
||||
return new_name
|
||||
|
||||
|
||||
def _is_indexable_module(module):
|
||||
return isinstance(module, (nn.ModuleList, nn.Sequential, list, tuple))
|
||||
|
||||
|
||||
def _get_module_by_name(model: nn.Module, name: str) -> Optional[nn.Module]:
|
||||
if not name:
|
||||
return model
|
||||
module = model
|
||||
for part in name.split("."):
|
||||
if not part:
|
||||
continue
|
||||
if hasattr(module, part):
|
||||
module = getattr(module, part)
|
||||
elif part.isdigit() and _is_indexable_module(module):
|
||||
try:
|
||||
module = module[int(part)]
|
||||
except (IndexError, TypeError):
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
return module
|
||||
|
||||
|
||||
def _resolve_module_name(model: nn.Module, name: str) -> Tuple[str, Optional[nn.Module]]:
|
||||
module = _get_module_by_name(model, name)
|
||||
if module is not None:
|
||||
return name, module
|
||||
|
||||
replacements = [
|
||||
(".attn.to_out.0", ".attn.to_out"),
|
||||
(".attention.to_qkv", ".attention.qkv"),
|
||||
(".attention.to_out.0", ".attention.out"),
|
||||
(".feed_forward.net.0.proj", ".feed_forward.w13"),
|
||||
(".feed_forward.net.2", ".feed_forward.w2"),
|
||||
(".ff.net.0.proj", ".mlp_fc1"),
|
||||
(".ff.net.2", ".mlp_fc2"),
|
||||
(".ff_context.net.0.proj", ".mlp_context_fc1"),
|
||||
(".ff_context.net.2", ".mlp_context_fc2"),
|
||||
]
|
||||
for src, dst in replacements:
|
||||
if src in name:
|
||||
alt = name.replace(src, dst)
|
||||
module = _get_module_by_name(model, alt)
|
||||
if module is not None:
|
||||
return alt, module
|
||||
return name, None
|
||||
|
||||
|
||||
def _classify_and_map_key(key: str) -> Optional[Tuple[str, str, Optional[str], str]]:
|
||||
normalized = key
|
||||
if normalized.startswith("transformer."):
|
||||
normalized = normalized[len("transformer."):]
|
||||
if normalized.startswith("diffusion_model."):
|
||||
normalized = normalized[len("diffusion_model."):]
|
||||
if normalized.startswith("lora_unet_"):
|
||||
normalized = _rename_layer_underscore_layer_name(normalized[len("lora_unet_"):])
|
||||
|
||||
match = _RE_LORA_SUFFIX.search(normalized)
|
||||
if match:
|
||||
tag = match.group("tag")
|
||||
base = normalized[:match.start()]
|
||||
ab = "A" if ("lora_A" in tag or tag.endswith(".A") or "down" in tag) else "B"
|
||||
else:
|
||||
match = _RE_ALPHA_SUFFIX.search(normalized)
|
||||
if not match:
|
||||
return None
|
||||
base = normalized[:match.start()]
|
||||
ab = "alpha"
|
||||
|
||||
for pattern, template, group, comp_fn in KEY_MAPPING:
|
||||
key_match = pattern.match(base)
|
||||
if key_match:
|
||||
return group, key_match.expand(template), comp_fn(key_match) if comp_fn else None, ab
|
||||
return None
|
||||
|
||||
|
||||
def _detect_lora_format(lora_state_dict: Dict[str, torch.Tensor]) -> bool:
|
||||
standard_patterns = (
|
||||
".lora_up.",
|
||||
".lora_down.",
|
||||
".lora_A.",
|
||||
".lora_B.",
|
||||
".lora.up.",
|
||||
".lora.down.",
|
||||
".lora.A.",
|
||||
".lora.B.",
|
||||
)
|
||||
return any(pattern in key for key in lora_state_dict for pattern in standard_patterns)
|
||||
|
||||
|
||||
def _load_lora_state_dict(path_or_dict: Union[str, Path, Dict[str, torch.Tensor]]) -> Dict[str, torch.Tensor]:
|
||||
if isinstance(path_or_dict, dict):
|
||||
return path_or_dict
|
||||
path = Path(path_or_dict)
|
||||
if path.suffix == ".safetensors":
|
||||
state_dict: Dict[str, torch.Tensor] = {}
|
||||
with safe_open(path, framework="pt", device="cpu") as handle:
|
||||
for key in handle.keys():
|
||||
state_dict[key] = handle.get_tensor(key)
|
||||
return state_dict
|
||||
return comfy.utils.load_torch_file(str(path), safe_load=True)
|
||||
|
||||
|
||||
def _fuse_glu_lora(glu_weights: Dict[str, torch.Tensor]) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor]]:
|
||||
if "w1_A" not in glu_weights or "w3_A" not in glu_weights:
|
||||
return None, None, None
|
||||
a_w1, b_w1 = glu_weights["w1_A"], glu_weights["w1_B"]
|
||||
a_w3, b_w3 = glu_weights["w3_A"], glu_weights["w3_B"]
|
||||
if a_w1.shape[1] != a_w3.shape[1]:
|
||||
return None, None, None
|
||||
a_fused = torch.cat([a_w1, a_w3], dim=0)
|
||||
out1, out3 = b_w1.shape[0], b_w3.shape[0]
|
||||
rank1, rank3 = b_w1.shape[1], b_w3.shape[1]
|
||||
b_fused = torch.zeros(out1 + out3, rank1 + rank3, dtype=b_w1.dtype, device=b_w1.device)
|
||||
b_fused[:out1, :rank1] = b_w1
|
||||
b_fused[out1:, rank1:] = b_w3
|
||||
return a_fused, b_fused, glu_weights.get("w1_alpha")
|
||||
|
||||
|
||||
def _fuse_qkv_lora(qkv_weights: Dict[str, torch.Tensor], model: Optional[nn.Module] = None, base_key: Optional[str] = None) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor]]:
|
||||
required_keys = ["Q_A", "Q_B", "K_A", "K_B", "V_A", "V_B"]
|
||||
if not all(key in qkv_weights for key in required_keys):
|
||||
return None, None, None
|
||||
a_q, a_k, a_v = qkv_weights["Q_A"], qkv_weights["K_A"], qkv_weights["V_A"]
|
||||
b_q, b_k, b_v = qkv_weights["Q_B"], qkv_weights["K_B"], qkv_weights["V_B"]
|
||||
if not (a_q.shape == a_k.shape == a_v.shape):
|
||||
return None, None, None
|
||||
if not (b_q.shape[1] == b_k.shape[1] == b_v.shape[1]):
|
||||
return None, None, None
|
||||
|
||||
out_features = None
|
||||
if model is not None and base_key is not None:
|
||||
_, module = _resolve_module_name(model, base_key)
|
||||
out_features = getattr(module, "out_features", None) if module is not None else None
|
||||
|
||||
alpha_fused = None
|
||||
alpha_q = qkv_weights.get("Q_alpha")
|
||||
alpha_k = qkv_weights.get("K_alpha")
|
||||
alpha_v = qkv_weights.get("V_alpha")
|
||||
if alpha_q is not None and alpha_k is not None and alpha_v is not None and alpha_q.item() == alpha_k.item() == alpha_v.item():
|
||||
alpha_fused = alpha_q
|
||||
|
||||
a_fused = torch.cat([a_q, a_k, a_v], dim=0)
|
||||
rank = b_q.shape[1]
|
||||
out_q, out_k, out_v = b_q.shape[0], b_k.shape[0], b_v.shape[0]
|
||||
total_out = out_features if out_features is not None else out_q + out_k + out_v
|
||||
b_fused = torch.zeros(total_out, 3 * rank, dtype=b_q.dtype, device=b_q.device)
|
||||
b_fused[:out_q, :rank] = b_q
|
||||
b_fused[out_q:out_q + out_k, rank:2 * rank] = b_k
|
||||
b_fused[out_q + out_k:out_q + out_k + out_v, 2 * rank:] = b_v
|
||||
return a_fused, b_fused, alpha_fused
|
||||
|
||||
|
||||
def _handle_proj_out_split(lora_dict: Dict[str, Dict[str, torch.Tensor]], base_key: str, model: nn.Module) -> Tuple[Dict[str, Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]], List[str]]:
|
||||
result: Dict[str, Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]] = {}
|
||||
consumed: List[str] = []
|
||||
match = re.search(r"single_transformer_blocks\.(\d+)", base_key)
|
||||
if not match or base_key not in lora_dict:
|
||||
return result, consumed
|
||||
block_idx = match.group(1)
|
||||
block = _get_module_by_name(model, f"single_transformer_blocks.{block_idx}")
|
||||
if block is None:
|
||||
return result, consumed
|
||||
a_full = lora_dict[base_key].get("A")
|
||||
b_full = lora_dict[base_key].get("B")
|
||||
alpha = lora_dict[base_key].get("alpha")
|
||||
attn_to_out = getattr(getattr(block, "attn", None), "to_out", None)
|
||||
mlp_fc2 = getattr(block, "mlp_fc2", None)
|
||||
if a_full is None or b_full is None or attn_to_out is None or mlp_fc2 is None:
|
||||
return result, consumed
|
||||
attn_in = getattr(attn_to_out, "in_features", None)
|
||||
mlp_in = getattr(mlp_fc2, "in_features", None)
|
||||
if attn_in is None or mlp_in is None or a_full.shape[1] != attn_in + mlp_in:
|
||||
return result, consumed
|
||||
result[f"single_transformer_blocks.{block_idx}.attn.to_out"] = (a_full[:, :attn_in], b_full.clone(), alpha)
|
||||
result[f"single_transformer_blocks.{block_idx}.mlp_fc2"] = (a_full[:, attn_in:], b_full.clone(), alpha)
|
||||
consumed.append(base_key)
|
||||
return result, consumed
|
||||
|
||||
|
||||
def _apply_lora_to_module(module: nn.Module, a_tensor: torch.Tensor, b_tensor: torch.Tensor, module_name: str, model: nn.Module) -> None:
|
||||
if not hasattr(module, "in_features") or not hasattr(module, "out_features"):
|
||||
raise ValueError(f"{module_name}: unsupported module without in/out features")
|
||||
if a_tensor.shape[1] != module.in_features or b_tensor.shape[0] != module.out_features:
|
||||
raise ValueError(f"{module_name}: LoRA shape mismatch")
|
||||
|
||||
if module.__class__.__name__ == "AWQW4A16Linear" and hasattr(module, "qweight"):
|
||||
if not hasattr(module, "_lora_original_forward"):
|
||||
module._lora_original_forward = module.forward
|
||||
if not hasattr(module, "_nunchaku_lora_bundle"):
|
||||
module._nunchaku_lora_bundle = []
|
||||
module._nunchaku_lora_bundle.append((a_tensor, b_tensor))
|
||||
|
||||
def _awq_lora_forward(x, *args, **kwargs):
|
||||
out = module._lora_original_forward(x, *args, **kwargs)
|
||||
x_flat = x.reshape(-1, module.in_features)
|
||||
for local_a, local_b in module._nunchaku_lora_bundle:
|
||||
local_a = local_a.to(device=out.device, dtype=out.dtype)
|
||||
local_b = local_b.to(device=out.device, dtype=out.dtype)
|
||||
lora_term = (x_flat @ local_a.transpose(0, 1)) @ local_b.transpose(0, 1)
|
||||
try:
|
||||
out = out + lora_term.reshape(out.shape)
|
||||
except Exception:
|
||||
pass
|
||||
return out
|
||||
|
||||
module.forward = _awq_lora_forward
|
||||
if not hasattr(model, "_lora_slots"):
|
||||
model._lora_slots = {}
|
||||
model._lora_slots[module_name] = {"type": "awq_w4a16"}
|
||||
return
|
||||
|
||||
if hasattr(module, "proj_down") and hasattr(module, "proj_up"):
|
||||
proj_down = unpack_lowrank_weight(module.proj_down.data, down=True)
|
||||
proj_up = unpack_lowrank_weight(module.proj_up.data, down=False)
|
||||
base_rank = proj_down.shape[0] if proj_down.shape[1] == module.in_features else proj_down.shape[1]
|
||||
if proj_down.shape[1] == module.in_features:
|
||||
updated_down = torch.cat([proj_down, a_tensor], dim=0)
|
||||
axis_down = 0
|
||||
else:
|
||||
updated_down = torch.cat([proj_down, a_tensor.T], dim=1)
|
||||
axis_down = 1
|
||||
updated_up = torch.cat([proj_up, b_tensor], dim=1)
|
||||
module.proj_down.data = pack_lowrank_weight(updated_down, down=True)
|
||||
module.proj_up.data = pack_lowrank_weight(updated_up, down=False)
|
||||
module.rank = base_rank + a_tensor.shape[0]
|
||||
if not hasattr(model, "_lora_slots"):
|
||||
model._lora_slots = {}
|
||||
model._lora_slots[module_name] = {
|
||||
"type": "nunchaku",
|
||||
"base_rank": base_rank,
|
||||
"axis_down": axis_down,
|
||||
}
|
||||
return
|
||||
|
||||
if isinstance(module, nn.Linear):
|
||||
if not hasattr(model, "_lora_slots"):
|
||||
model._lora_slots = {}
|
||||
if module_name not in model._lora_slots:
|
||||
model._lora_slots[module_name] = {
|
||||
"type": "linear",
|
||||
"original_weight": module.weight.detach().cpu().clone(),
|
||||
}
|
||||
module.weight.data.add_((b_tensor @ a_tensor).to(dtype=module.weight.dtype, device=module.weight.device))
|
||||
return
|
||||
|
||||
raise ValueError(f"{module_name}: unsupported module type {type(module)}")
|
||||
|
||||
|
||||
def reset_lora_v2(model: nn.Module) -> None:
|
||||
slots = getattr(model, "_lora_slots", None)
|
||||
if not slots:
|
||||
return
|
||||
for name, info in list(slots.items()):
|
||||
module = _get_module_by_name(model, name)
|
||||
if module is None:
|
||||
continue
|
||||
module_type = info.get("type", "nunchaku")
|
||||
if module_type == "nunchaku":
|
||||
base_rank = info["base_rank"]
|
||||
proj_down = unpack_lowrank_weight(module.proj_down.data, down=True)
|
||||
proj_up = unpack_lowrank_weight(module.proj_up.data, down=False)
|
||||
if info.get("axis_down", 0) == 0:
|
||||
proj_down = proj_down[:base_rank, :].clone()
|
||||
else:
|
||||
proj_down = proj_down[:, :base_rank].clone()
|
||||
proj_up = proj_up[:, :base_rank].clone()
|
||||
module.proj_down.data = pack_lowrank_weight(proj_down, down=True)
|
||||
module.proj_up.data = pack_lowrank_weight(proj_up, down=False)
|
||||
module.rank = base_rank
|
||||
elif module_type == "linear" and "original_weight" in info:
|
||||
module.weight.data.copy_(info["original_weight"].to(device=module.weight.device, dtype=module.weight.dtype))
|
||||
elif module_type == "awq_w4a16":
|
||||
if hasattr(module, "_lora_original_forward"):
|
||||
module.forward = module._lora_original_forward
|
||||
for attr in ("_lora_original_forward", "_nunchaku_lora_bundle"):
|
||||
if hasattr(module, attr):
|
||||
delattr(module, attr)
|
||||
model._lora_slots = {}
|
||||
|
||||
|
||||
def compose_loras_v2(model: nn.Module, lora_configs: List[Tuple[Union[str, Path, Dict[str, torch.Tensor]], float]], apply_awq_mod: bool = True) -> bool:
|
||||
del apply_awq_mod # retained for interface compatibility
|
||||
reset_lora_v2(model)
|
||||
aggregated_weights: Dict[str, List[Dict[str, object]]] = defaultdict(list)
|
||||
saw_supported_format = False
|
||||
unresolved_targets = 0
|
||||
|
||||
for index, (path_or_dict, strength) in enumerate(lora_configs):
|
||||
if abs(strength) < 1e-5:
|
||||
continue
|
||||
lora_name = str(path_or_dict) if not isinstance(path_or_dict, dict) else f"lora_{index}"
|
||||
lora_state_dict = _load_lora_state_dict(path_or_dict)
|
||||
if not lora_state_dict or not _detect_lora_format(lora_state_dict):
|
||||
logger.warning("Skipping unsupported Qwen LoRA: %s", lora_name)
|
||||
continue
|
||||
saw_supported_format = True
|
||||
|
||||
grouped_weights: Dict[str, Dict[str, torch.Tensor]] = defaultdict(dict)
|
||||
for key, value in lora_state_dict.items():
|
||||
parsed = _classify_and_map_key(key)
|
||||
if parsed is None:
|
||||
continue
|
||||
group, base_key, component, ab = parsed
|
||||
if component and ab:
|
||||
grouped_weights[base_key][f"{component}_{ab}"] = value
|
||||
else:
|
||||
grouped_weights[base_key][ab] = value
|
||||
|
||||
processed_groups: Dict[str, Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]] = {}
|
||||
handled: set[str] = set()
|
||||
for base_key, weights in grouped_weights.items():
|
||||
if base_key in handled:
|
||||
continue
|
||||
a_tensor = b_tensor = alpha = None
|
||||
if "qkv" in base_key or "add_qkv_proj" in base_key:
|
||||
a_tensor, b_tensor, alpha = _fuse_qkv_lora(weights, model=model, base_key=base_key)
|
||||
elif "w1_A" in weights or "w3_A" in weights:
|
||||
a_tensor, b_tensor, alpha = _fuse_glu_lora(weights)
|
||||
elif ".proj_out" in base_key and "single_transformer_blocks" in base_key:
|
||||
split_map, consumed = _handle_proj_out_split(grouped_weights, base_key, model)
|
||||
processed_groups.update(split_map)
|
||||
handled.update(consumed)
|
||||
continue
|
||||
else:
|
||||
a_tensor, b_tensor, alpha = weights.get("A"), weights.get("B"), weights.get("alpha")
|
||||
if a_tensor is not None and b_tensor is not None:
|
||||
processed_groups[base_key] = (a_tensor, b_tensor, alpha)
|
||||
|
||||
for module_name, (a_tensor, b_tensor, alpha) in processed_groups.items():
|
||||
aggregated_weights[module_name].append({
|
||||
"A": a_tensor,
|
||||
"B": b_tensor,
|
||||
"alpha": alpha,
|
||||
"strength": strength,
|
||||
})
|
||||
|
||||
for module_name, weight_list in aggregated_weights.items():
|
||||
resolved_name, module = _resolve_module_name(model, module_name)
|
||||
if module is None:
|
||||
logger.warning("Skipping unresolved Qwen LoRA target: %s", module_name)
|
||||
unresolved_targets += 1
|
||||
continue
|
||||
all_a = []
|
||||
all_b_scaled = []
|
||||
for item in weight_list:
|
||||
a_tensor = item["A"]
|
||||
b_tensor = item["B"]
|
||||
alpha = item["alpha"]
|
||||
strength = float(item["strength"])
|
||||
rank = a_tensor.shape[0]
|
||||
scale = strength * ((alpha / rank) if alpha is not None else 1.0)
|
||||
if module.__class__.__name__ == "AWQW4A16Linear" and hasattr(module, "qweight"):
|
||||
target_dtype = torch.float16
|
||||
target_device = module.qweight.device
|
||||
elif hasattr(module, "proj_down"):
|
||||
target_dtype = module.proj_down.dtype
|
||||
target_device = module.proj_down.device
|
||||
elif hasattr(module, "weight"):
|
||||
target_dtype = module.weight.dtype
|
||||
target_device = module.weight.device
|
||||
else:
|
||||
target_dtype = torch.float16
|
||||
target_device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
all_a.append(a_tensor.to(dtype=target_dtype, device=target_device))
|
||||
all_b_scaled.append((b_tensor * scale).to(dtype=target_dtype, device=target_device))
|
||||
if not all_a:
|
||||
continue
|
||||
_apply_lora_to_module(module, torch.cat(all_a, dim=0), torch.cat(all_b_scaled, dim=1), resolved_name, model)
|
||||
|
||||
slot_count = len(getattr(model, "_lora_slots", {}) or {})
|
||||
logger.info(
|
||||
"Qwen LoRA composition finished: requested=%d supported=%s applied_targets=%d unresolved=%d",
|
||||
len(lora_configs),
|
||||
saw_supported_format,
|
||||
slot_count,
|
||||
unresolved_targets,
|
||||
)
|
||||
return saw_supported_format
|
||||
|
||||
|
||||
class ComfyQwenImageWrapperLM(nn.Module):
|
||||
def __init__(self, model: nn.Module, config=None, apply_awq_mod: bool = True):
|
||||
super().__init__()
|
||||
self.model = model
|
||||
self.config = {} if config is None else config
|
||||
self.dtype = next(model.parameters()).dtype
|
||||
self.loras: List[Tuple[Union[str, Path, Dict[str, torch.Tensor]], float]] = []
|
||||
self._applied_loras: Optional[List[Tuple[Union[str, Path, Dict[str, torch.Tensor]], float]]] = None
|
||||
self.apply_awq_mod = apply_awq_mod
|
||||
|
||||
def __getattr__(self, name):
|
||||
try:
|
||||
inner = object.__getattribute__(self, "_modules").get("model")
|
||||
except (AttributeError, KeyError):
|
||||
inner = None
|
||||
if inner is None:
|
||||
raise AttributeError(f"{type(self).__name__!s} has no attribute {name}")
|
||||
if name == "model":
|
||||
return inner
|
||||
return getattr(inner, name)
|
||||
|
||||
def process_img(self, *args, **kwargs):
|
||||
return self.model.process_img(*args, **kwargs)
|
||||
|
||||
def _ensure_composed(self):
|
||||
if self._applied_loras != self.loras or (not self.loras and getattr(self.model, "_lora_slots", None)):
|
||||
is_supported_format = compose_loras_v2(self.model, self.loras, apply_awq_mod=self.apply_awq_mod)
|
||||
self._applied_loras = self.loras.copy()
|
||||
has_slots = bool(getattr(self.model, "_lora_slots", None))
|
||||
if self.loras and is_supported_format and not has_slots:
|
||||
logger.warning("Qwen LoRA compose produced 0 target modules. Resetting and retrying once.")
|
||||
reset_lora_v2(self.model)
|
||||
compose_loras_v2(self.model, self.loras, apply_awq_mod=self.apply_awq_mod)
|
||||
has_slots = bool(getattr(self.model, "_lora_slots", None))
|
||||
logger.info("Qwen LoRA retry result: applied_targets=%d", len(getattr(self.model, "_lora_slots", {}) or {}))
|
||||
|
||||
offload_manager = getattr(self.model, "offload_manager", None)
|
||||
if offload_manager is not None:
|
||||
offload_settings = {
|
||||
"num_blocks_on_gpu": getattr(offload_manager, "num_blocks_on_gpu", 1),
|
||||
"use_pin_memory": getattr(offload_manager, "use_pin_memory", False),
|
||||
}
|
||||
logger.info(
|
||||
"Rebuilding Qwen offload manager after LoRA compose: num_blocks_on_gpu=%s use_pin_memory=%s",
|
||||
offload_settings["num_blocks_on_gpu"],
|
||||
offload_settings["use_pin_memory"],
|
||||
)
|
||||
self.model.set_offload(False)
|
||||
self.model.set_offload(True, **offload_settings)
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
self._ensure_composed()
|
||||
return self.model(*args, **kwargs)
|
||||
|
||||
|
||||
def _get_qwen_wrapper_and_transformer(model):
|
||||
model_wrapper = model.model.diffusion_model
|
||||
if hasattr(model_wrapper, "model") and hasattr(model_wrapper, "loras"):
|
||||
transformer = model_wrapper.model
|
||||
if transformer.__class__.__name__.endswith("NunchakuQwenImageTransformer2DModel"):
|
||||
return model_wrapper, transformer
|
||||
if model_wrapper.__class__.__name__.endswith("NunchakuQwenImageTransformer2DModel"):
|
||||
wrapped_model = ComfyQwenImageWrapperLM(model_wrapper, getattr(model_wrapper, "config", {}))
|
||||
model.model.diffusion_model = wrapped_model
|
||||
return wrapped_model, wrapped_model.model
|
||||
raise TypeError(f"This LoRA loader only works with Nunchaku Qwen Image models, but got {type(model_wrapper).__name__}.")
|
||||
|
||||
|
||||
def nunchaku_load_qwen_loras(model, lora_configs: List[Tuple[str, float]], apply_awq_mod: bool = True):
|
||||
model_wrapper, transformer = _get_qwen_wrapper_and_transformer(model)
|
||||
model_wrapper.apply_awq_mod = apply_awq_mod
|
||||
|
||||
saved_config = None
|
||||
if hasattr(model, "model") and hasattr(model.model, "model_config"):
|
||||
saved_config = model.model.model_config
|
||||
model.model.model_config = None
|
||||
|
||||
model_wrapper.model = None
|
||||
try:
|
||||
ret_model = copy.deepcopy(model)
|
||||
finally:
|
||||
if saved_config is not None:
|
||||
model.model.model_config = saved_config
|
||||
model_wrapper.model = transformer
|
||||
|
||||
ret_model_wrapper = ret_model.model.diffusion_model
|
||||
if saved_config is not None:
|
||||
ret_model.model.model_config = saved_config
|
||||
ret_model_wrapper.model = transformer
|
||||
ret_model_wrapper.apply_awq_mod = apply_awq_mod
|
||||
ret_model_wrapper.loras = list(getattr(model_wrapper, "loras", []))
|
||||
|
||||
for lora_name, lora_strength in lora_configs:
|
||||
lora_path = lora_name if os.path.isfile(lora_name) else folder_paths.get_full_path("loras", lora_name)
|
||||
if not lora_path or not os.path.isfile(lora_path):
|
||||
logger.warning("Skipping Qwen LoRA '%s' because it could not be found", lora_name)
|
||||
continue
|
||||
ret_model_wrapper.loras.append((lora_path, lora_strength))
|
||||
|
||||
return ret_model
|
||||
@@ -1,15 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
import inspect
|
||||
|
||||
from ..services.wildcard_service import (
|
||||
contains_dynamic_syntax,
|
||||
get_wildcard_service,
|
||||
is_trigger_words_input,
|
||||
)
|
||||
|
||||
class _AllContainer:
|
||||
"""Container that accepts any key for dynamic input validation."""
|
||||
|
||||
def __contains__(self, item):
|
||||
return True
|
||||
class _PromptOptionalInputs:
|
||||
"""Lookup that preserves explicit optional inputs and dynamic trigger slots."""
|
||||
|
||||
def __getitem__(self, key):
|
||||
return ("STRING", {"forceInput": True})
|
||||
def __init__(self, explicit_inputs: dict[str, tuple[str, dict[str, Any]]]) -> None:
|
||||
self._explicit_inputs = explicit_inputs
|
||||
|
||||
def __contains__(self, item: object) -> bool:
|
||||
if not isinstance(item, str):
|
||||
return False
|
||||
return item in self._explicit_inputs or is_trigger_words_input(item)
|
||||
|
||||
def __getitem__(self, key: str) -> tuple[str, dict[str, Any]]:
|
||||
if key in self._explicit_inputs:
|
||||
return self._explicit_inputs[key]
|
||||
if is_trigger_words_input(key):
|
||||
return (
|
||||
"STRING",
|
||||
{
|
||||
"forceInput": True,
|
||||
"tooltip": "Trigger words to prepend. Connect to add more inputs.",
|
||||
},
|
||||
)
|
||||
raise KeyError(key)
|
||||
|
||||
|
||||
class PromptLM:
|
||||
@@ -20,12 +43,19 @@ class PromptLM:
|
||||
DESCRIPTION = (
|
||||
"Encodes a text prompt using a CLIP model into an embedding that can be used "
|
||||
"to guide the diffusion model towards generating specific images. "
|
||||
"Supports dynamic trigger words inputs."
|
||||
"Supports dynamic trigger words inputs and runtime wildcard expansion."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
dyn_inputs = {
|
||||
optional_inputs: dict[str, tuple[str, dict[str, Any]]] = {
|
||||
"seed": (
|
||||
"INT",
|
||||
{
|
||||
"forceInput": True,
|
||||
"tooltip": "Optional seed for wildcard generation. Leave unconnected for non-deterministic wildcard expansion.",
|
||||
},
|
||||
),
|
||||
"trigger_words1": (
|
||||
"STRING",
|
||||
{
|
||||
@@ -35,10 +65,9 @@ class PromptLM:
|
||||
),
|
||||
}
|
||||
|
||||
# Bypass validation for dynamic inputs during graph execution
|
||||
stack = inspect.stack()
|
||||
if len(stack) > 2 and stack[2].function == "get_input_info":
|
||||
dyn_inputs = _AllContainer()
|
||||
optional_inputs = _PromptOptionalInputs(optional_inputs) # type: ignore[assignment]
|
||||
|
||||
return {
|
||||
"required": {
|
||||
@@ -46,8 +75,8 @@ class PromptLM:
|
||||
"AUTOCOMPLETE_TEXT_PROMPT,STRING",
|
||||
{
|
||||
"widgetType": "AUTOCOMPLETE_TEXT_PROMPT",
|
||||
"placeholder": "Enter prompt... /char, /artist for quick tag search",
|
||||
"tooltip": "The text to be encoded.",
|
||||
"placeholder": "Enter prompt... /character, /artist, /wildcard for quick search",
|
||||
"tooltip": "The text to be encoded. Wildcard references inserted with /wildcard are expanded at runtime.",
|
||||
},
|
||||
),
|
||||
"clip": (
|
||||
@@ -55,7 +84,7 @@ class PromptLM:
|
||||
{"tooltip": "The CLIP model used for encoding the text."},
|
||||
),
|
||||
},
|
||||
"optional": dyn_inputs,
|
||||
"optional": optional_inputs,
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("CONDITIONING", "STRING")
|
||||
@@ -65,20 +94,39 @@ class PromptLM:
|
||||
)
|
||||
FUNCTION = "encode"
|
||||
|
||||
def encode(self, text: str, clip: Any, **kwargs):
|
||||
# Collect all trigger words from dynamic inputs
|
||||
@classmethod
|
||||
def IS_CHANGED(
|
||||
cls,
|
||||
text: str,
|
||||
clip: Any | None = None,
|
||||
seed: int | None = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
del clip, kwargs
|
||||
if contains_dynamic_syntax(text) and seed is None:
|
||||
return float("NaN")
|
||||
return False
|
||||
|
||||
def encode(
|
||||
self,
|
||||
text: str,
|
||||
clip: Any,
|
||||
seed: int | None = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
expanded_text = get_wildcard_service().expand_text(text, seed=seed)
|
||||
|
||||
trigger_words = []
|
||||
for key, value in kwargs.items():
|
||||
if key.startswith("trigger_words") and value:
|
||||
if is_trigger_words_input(key) and value:
|
||||
trigger_words.append(value)
|
||||
|
||||
# Build final prompt
|
||||
if trigger_words:
|
||||
prompt = ", ".join(trigger_words + [text])
|
||||
prompt = ", ".join(trigger_words + [expanded_text])
|
||||
else:
|
||||
prompt = text
|
||||
prompt = expanded_text
|
||||
|
||||
from nodes import CLIPTextEncode # type: ignore
|
||||
|
||||
conditioning = CLIPTextEncode().encode(clip, prompt)[0]
|
||||
return (conditioning, prompt)
|
||||
return (conditioning, prompt)
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, Optional
|
||||
import numpy as np
|
||||
import folder_paths # type: ignore
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
from ..metadata_collector.metadata_processor import MetadataProcessor
|
||||
from ..metadata_collector import get_metadata
|
||||
from ..utils.constants import CARD_PREVIEW_WIDTH
|
||||
from ..utils.exif_utils import ExifUtils
|
||||
from ..utils.utils import calculate_recipe_fingerprint, sanitize_folder_name
|
||||
from PIL import Image, PngImagePlugin
|
||||
import piexif
|
||||
import logging
|
||||
@@ -72,6 +77,13 @@ class SaveImageLM:
|
||||
"tooltip": "Embeds the complete workflow data into the image metadata. Only works with PNG and WebP formats.",
|
||||
},
|
||||
),
|
||||
"save_with_metadata": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": True,
|
||||
"tooltip": "When enabled, embeds generation parameters into the saved image metadata. Disable to skip writing generation metadata.",
|
||||
},
|
||||
),
|
||||
"add_counter_to_filename": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
@@ -79,6 +91,13 @@ class SaveImageLM:
|
||||
"tooltip": "Adds an incremental counter to filenames to prevent overwriting previous images.",
|
||||
},
|
||||
),
|
||||
"save_as_recipe": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "Also saves each generated image as a LoRA Manager recipe.",
|
||||
},
|
||||
),
|
||||
},
|
||||
"hidden": {
|
||||
"id": "UNIQUE_ID",
|
||||
@@ -279,7 +298,12 @@ class SaveImageLM:
|
||||
key = parts[0]
|
||||
|
||||
if key == "seed" and "seed" in metadata_dict:
|
||||
filename = filename.replace(segment, str(metadata_dict.get("seed", "")))
|
||||
seed_value = metadata_dict.get("seed")
|
||||
if seed_value is not None:
|
||||
filename = filename.replace(segment, str(seed_value))
|
||||
else:
|
||||
# Fallback if seed was not captured by metadata collector
|
||||
filename = filename.replace(segment, "0")
|
||||
elif key == "width" and "size" in metadata_dict:
|
||||
size = metadata_dict.get("size", "x")
|
||||
w = size.split("x")[0] if isinstance(size, str) else size[0]
|
||||
@@ -290,12 +314,14 @@ class SaveImageLM:
|
||||
filename = filename.replace(segment, str(h))
|
||||
elif key == "pprompt" and "prompt" in metadata_dict:
|
||||
prompt = metadata_dict.get("prompt", "").replace("\n", " ")
|
||||
prompt = sanitize_folder_name(prompt)
|
||||
if len(parts) >= 2:
|
||||
length = int(parts[1])
|
||||
prompt = prompt[:length]
|
||||
filename = filename.replace(segment, prompt.strip())
|
||||
elif key == "nprompt" and "negative_prompt" in metadata_dict:
|
||||
prompt = metadata_dict.get("negative_prompt", "").replace("\n", " ")
|
||||
prompt = sanitize_folder_name(prompt)
|
||||
if len(parts) >= 2:
|
||||
length = int(parts[1])
|
||||
prompt = prompt[:length]
|
||||
@@ -309,6 +335,7 @@ class SaveImageLM:
|
||||
model = "model_unavailable"
|
||||
else:
|
||||
model = os.path.splitext(os.path.basename(model_value))[0]
|
||||
model = sanitize_folder_name(model)
|
||||
if len(parts) >= 2:
|
||||
length = int(parts[1])
|
||||
model = model[:length]
|
||||
@@ -339,6 +366,203 @@ class SaveImageLM:
|
||||
|
||||
return filename
|
||||
|
||||
@staticmethod
|
||||
def _get_cached_model_by_name(scanner, name):
|
||||
cache = getattr(scanner, "_cache", None)
|
||||
if cache is None or not name:
|
||||
return None
|
||||
|
||||
candidates = [
|
||||
name,
|
||||
os.path.basename(name),
|
||||
os.path.splitext(os.path.basename(name))[0],
|
||||
]
|
||||
for model in getattr(cache, "raw_data", []):
|
||||
file_name = model.get("file_name")
|
||||
if file_name in candidates:
|
||||
return model
|
||||
return None
|
||||
|
||||
def _build_recipe_loras(self, recipe_scanner, lora_stack):
|
||||
lora_matches = re.findall(r"<lora:([^:]+):([^>]+)>", lora_stack or "")
|
||||
lora_scanner = getattr(recipe_scanner, "_lora_scanner", None)
|
||||
loras_data = []
|
||||
base_model_counts = {}
|
||||
|
||||
for name, strength in lora_matches:
|
||||
lora_info = self._get_cached_model_by_name(lora_scanner, name)
|
||||
civitai = (lora_info or {}).get("civitai") or {}
|
||||
civitai_model = civitai.get("model") or {}
|
||||
try:
|
||||
parsed_strength = float(strength)
|
||||
except (TypeError, ValueError):
|
||||
parsed_strength = 1.0
|
||||
|
||||
loras_data.append(
|
||||
{
|
||||
"file_name": name,
|
||||
"strength": parsed_strength,
|
||||
"hash": ((lora_info or {}).get("sha256") or "").lower(),
|
||||
"modelVersionId": civitai.get("id", 0),
|
||||
"modelName": civitai_model.get("name", name) if lora_info else "",
|
||||
"modelVersionName": civitai.get("name", "") if lora_info else "",
|
||||
"isDeleted": False,
|
||||
"exclude": False,
|
||||
}
|
||||
)
|
||||
|
||||
base_model = (lora_info or {}).get("base_model")
|
||||
if base_model:
|
||||
base_model_counts[base_model] = base_model_counts.get(base_model, 0) + 1
|
||||
|
||||
return lora_matches, loras_data, base_model_counts
|
||||
|
||||
def _build_recipe_checkpoint(self, recipe_scanner, checkpoint_raw):
|
||||
if not isinstance(checkpoint_raw, str) or not checkpoint_raw.strip():
|
||||
return None
|
||||
|
||||
checkpoint_name = checkpoint_raw.strip()
|
||||
file_name = os.path.splitext(os.path.basename(checkpoint_name))[0]
|
||||
checkpoint_scanner = getattr(recipe_scanner, "_checkpoint_scanner", None)
|
||||
checkpoint_info = self._get_cached_model_by_name(
|
||||
checkpoint_scanner, checkpoint_name
|
||||
)
|
||||
|
||||
if not checkpoint_info:
|
||||
return {
|
||||
"type": "checkpoint",
|
||||
"name": checkpoint_name,
|
||||
"file_name": file_name,
|
||||
"hash": self.get_checkpoint_hash(checkpoint_name) or "",
|
||||
}
|
||||
|
||||
civitai = checkpoint_info.get("civitai") or {}
|
||||
civitai_model = civitai.get("model") or {}
|
||||
file_path = checkpoint_info.get("file_path") or checkpoint_info.get("path") or ""
|
||||
cached_file_name = (
|
||||
checkpoint_info.get("file_name")
|
||||
or (os.path.splitext(os.path.basename(file_path))[0] if file_path else "")
|
||||
or file_name
|
||||
)
|
||||
|
||||
return {
|
||||
"type": "checkpoint",
|
||||
"modelId": civitai_model.get("id", 0),
|
||||
"modelVersionId": civitai.get("id", 0),
|
||||
"name": civitai_model.get("name")
|
||||
or checkpoint_info.get("model_name")
|
||||
or checkpoint_name,
|
||||
"version": civitai.get("name", ""),
|
||||
"hash": (
|
||||
checkpoint_info.get("sha256") or checkpoint_info.get("hash") or ""
|
||||
).lower(),
|
||||
"file_name": cached_file_name,
|
||||
"modelName": civitai_model.get("name", ""),
|
||||
"modelVersionName": civitai.get("name", ""),
|
||||
"baseModel": checkpoint_info.get("base_model")
|
||||
or civitai.get("baseModel", ""),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _derive_recipe_name(lora_matches):
|
||||
recipe_name_parts = [
|
||||
f"{name.strip()}-{float(strength):.2f}" for name, strength in lora_matches[:3]
|
||||
]
|
||||
return "_".join(recipe_name_parts) or "recipe"
|
||||
|
||||
@staticmethod
|
||||
def _sync_recipe_cache(recipe_scanner, recipe_data, json_path):
|
||||
cache = getattr(recipe_scanner, "_cache", None)
|
||||
if cache is not None:
|
||||
cache.raw_data.append(recipe_data)
|
||||
cache.sorted_by_name = sorted(
|
||||
cache.raw_data, key=lambda item: item.get("title", "").lower()
|
||||
)
|
||||
cache.sorted_by_date = sorted(
|
||||
cache.raw_data,
|
||||
key=lambda item: (
|
||||
item.get("modified", item.get("created_date", 0)),
|
||||
item.get("file_path", ""),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
recipe_scanner._update_folder_metadata(cache)
|
||||
recipe_scanner._update_fts_index_for_recipe(recipe_data, "add")
|
||||
|
||||
recipe_id = str(recipe_data.get("id", ""))
|
||||
if recipe_id:
|
||||
recipe_scanner._json_path_map[recipe_id] = json_path
|
||||
persistent_cache = getattr(recipe_scanner, "_persistent_cache", None)
|
||||
if persistent_cache:
|
||||
persistent_cache.update_recipe(recipe_data, json_path)
|
||||
|
||||
def _save_image_as_recipe(self, file_path, metadata_dict):
|
||||
if not metadata_dict:
|
||||
raise ValueError("No generation metadata found")
|
||||
|
||||
recipe_scanner = ServiceRegistry.get_service_sync("recipe_scanner")
|
||||
if recipe_scanner is None:
|
||||
raise RuntimeError("Recipe scanner unavailable")
|
||||
|
||||
recipes_dir = recipe_scanner.recipes_dir
|
||||
if not recipes_dir:
|
||||
raise RuntimeError("Recipes directory unavailable")
|
||||
os.makedirs(recipes_dir, exist_ok=True)
|
||||
|
||||
recipe_id = str(uuid.uuid4())
|
||||
optimized_image, extension = ExifUtils.optimize_image(
|
||||
image_data=file_path,
|
||||
target_width=CARD_PREVIEW_WIDTH,
|
||||
format="webp",
|
||||
quality=85,
|
||||
preserve_metadata=True,
|
||||
)
|
||||
image_path = os.path.normpath(os.path.join(recipes_dir, f"{recipe_id}{extension}"))
|
||||
with open(image_path, "wb") as file_obj:
|
||||
file_obj.write(optimized_image)
|
||||
|
||||
lora_stack = metadata_dict.get("loras", "")
|
||||
lora_matches, loras_data, base_model_counts = self._build_recipe_loras(
|
||||
recipe_scanner, lora_stack
|
||||
)
|
||||
checkpoint_entry = self._build_recipe_checkpoint(
|
||||
recipe_scanner, metadata_dict.get("checkpoint")
|
||||
)
|
||||
most_common_base_model = (
|
||||
max(base_model_counts.items(), key=lambda item: item[1])[0]
|
||||
if base_model_counts
|
||||
else ""
|
||||
)
|
||||
current_time = time.time()
|
||||
recipe_data = {
|
||||
"id": recipe_id,
|
||||
"file_path": image_path,
|
||||
"title": self._derive_recipe_name(lora_matches),
|
||||
"modified": current_time,
|
||||
"created_date": current_time,
|
||||
"base_model": most_common_base_model
|
||||
or (checkpoint_entry or {}).get("baseModel", ""),
|
||||
"loras": loras_data,
|
||||
"gen_params": {
|
||||
key: value
|
||||
for key, value in metadata_dict.items()
|
||||
if key not in ["checkpoint", "loras"]
|
||||
},
|
||||
"loras_stack": lora_stack,
|
||||
"fingerprint": calculate_recipe_fingerprint(loras_data),
|
||||
}
|
||||
if checkpoint_entry:
|
||||
recipe_data["checkpoint"] = checkpoint_entry
|
||||
|
||||
json_path = os.path.normpath(
|
||||
os.path.join(recipes_dir, f"{recipe_id}.recipe.json")
|
||||
)
|
||||
with open(json_path, "w", encoding="utf-8") as file_obj:
|
||||
json.dump(recipe_data, file_obj, indent=4, ensure_ascii=False)
|
||||
|
||||
ExifUtils.append_recipe_metadata(image_path, recipe_data)
|
||||
self._sync_recipe_cache(recipe_scanner, recipe_data, json_path)
|
||||
|
||||
def save_images(
|
||||
self,
|
||||
images,
|
||||
@@ -350,7 +574,9 @@ class SaveImageLM:
|
||||
lossless_webp=True,
|
||||
quality=100,
|
||||
embed_workflow=False,
|
||||
save_with_metadata=True,
|
||||
add_counter_to_filename=True,
|
||||
save_as_recipe=False,
|
||||
):
|
||||
"""Save images with metadata"""
|
||||
results = []
|
||||
@@ -382,7 +608,7 @@ class SaveImageLM:
|
||||
img = Image.fromarray(np.clip(img, 0, 255).astype(np.uint8))
|
||||
|
||||
# Generate filename with counter if needed
|
||||
base_filename = filename
|
||||
base_filename = filename.replace("%batch_num%", str(i))
|
||||
if add_counter_to_filename:
|
||||
# Use counter + i to ensure unique filenames for all images in batch
|
||||
current_counter = counter + i
|
||||
@@ -421,7 +647,7 @@ class SaveImageLM:
|
||||
try:
|
||||
if file_format == "png":
|
||||
assert pnginfo is not None
|
||||
if metadata:
|
||||
if save_with_metadata and metadata:
|
||||
pnginfo.add_text("parameters", metadata)
|
||||
if embed_workflow and extra_pnginfo is not None:
|
||||
workflow_json = json.dumps(extra_pnginfo["workflow"])
|
||||
@@ -430,7 +656,7 @@ class SaveImageLM:
|
||||
img.save(file_path, format="PNG", **save_kwargs)
|
||||
elif file_format == "jpeg":
|
||||
# For JPEG, use piexif
|
||||
if metadata:
|
||||
if save_with_metadata and metadata:
|
||||
try:
|
||||
exif_dict = {
|
||||
"Exif": {
|
||||
@@ -448,7 +674,7 @@ class SaveImageLM:
|
||||
# For WebP, use piexif for metadata
|
||||
exif_dict = {}
|
||||
|
||||
if metadata:
|
||||
if save_with_metadata and metadata:
|
||||
exif_dict["Exif"] = {
|
||||
piexif.ExifIFD.UserComment: b"UNICODE\0"
|
||||
+ metadata.encode("utf-16be")
|
||||
@@ -469,6 +695,14 @@ class SaveImageLM:
|
||||
|
||||
img.save(file_path, format="WEBP", **save_kwargs)
|
||||
|
||||
if save_as_recipe:
|
||||
try:
|
||||
self._save_image_as_recipe(file_path, metadata_dict)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to save image as recipe: %s", e, exc_info=True
|
||||
)
|
||||
|
||||
results.append(
|
||||
{"filename": file, "subfolder": subfolder, "type": self.type}
|
||||
)
|
||||
@@ -489,7 +723,9 @@ class SaveImageLM:
|
||||
lossless_webp=True,
|
||||
quality=100,
|
||||
embed_workflow=False,
|
||||
save_with_metadata=True,
|
||||
add_counter_to_filename=True,
|
||||
save_as_recipe=False,
|
||||
):
|
||||
"""Process and save image with metadata"""
|
||||
# Make sure the output directory exists
|
||||
@@ -516,7 +752,12 @@ class SaveImageLM:
|
||||
lossless_webp,
|
||||
quality,
|
||||
embed_workflow,
|
||||
save_with_metadata,
|
||||
add_counter_to_filename,
|
||||
save_as_recipe,
|
||||
)
|
||||
|
||||
return (images,)
|
||||
return {
|
||||
"result": (images,),
|
||||
"ui": {"images": results},
|
||||
}
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from ..services.wildcard_service import contains_dynamic_syntax, get_wildcard_service
|
||||
|
||||
|
||||
class TextLM:
|
||||
"""A simple text node with autocomplete support."""
|
||||
|
||||
NAME = "Text (LoraManager)"
|
||||
CATEGORY = "Lora Manager/utils"
|
||||
DESCRIPTION = (
|
||||
"A simple text input node with autocomplete support for tags and styles."
|
||||
"A simple text input node with autocomplete support for tags, styles, and wildcard expansion."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -15,8 +20,17 @@ class TextLM:
|
||||
"AUTOCOMPLETE_TEXT_PROMPT,STRING",
|
||||
{
|
||||
"widgetType": "AUTOCOMPLETE_TEXT_PROMPT",
|
||||
"placeholder": "Enter text... /char, /artist for quick tag search",
|
||||
"tooltip": "The text output.",
|
||||
"placeholder": "Enter text... /character, /artist, /wildcard for quick search",
|
||||
"tooltip": "The text output. Wildcard references inserted with /wildcard are expanded at runtime.",
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"seed": (
|
||||
"INT",
|
||||
{
|
||||
"forceInput": True,
|
||||
"tooltip": "Optional seed for wildcard generation. Leave unconnected for non-deterministic wildcard expansion.",
|
||||
},
|
||||
),
|
||||
},
|
||||
@@ -24,10 +38,14 @@ class TextLM:
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("STRING",)
|
||||
OUTPUT_TOOLTIPS = (
|
||||
"The text output.",
|
||||
)
|
||||
OUTPUT_TOOLTIPS = ("The text output.",)
|
||||
FUNCTION = "process"
|
||||
|
||||
def process(self, text: str):
|
||||
return (text,)
|
||||
@classmethod
|
||||
def IS_CHANGED(cls, text: str, seed: int | None = None):
|
||||
if contains_dynamic_syntax(text) and seed is None:
|
||||
return float("NaN")
|
||||
return False
|
||||
|
||||
def process(self, text: str, seed: int | None = None):
|
||||
return (get_wildcard_service().expand_text(text, seed=seed),)
|
||||
|
||||
@@ -76,6 +76,9 @@ class TriggerWordToggleLM:
|
||||
# Filter out empty strings and return as set
|
||||
return set(word for word in words if word)
|
||||
|
||||
def _group_has_child_items(self, item):
|
||||
return isinstance(item, dict) and isinstance(item.get("items"), list)
|
||||
|
||||
def process_trigger_words(
|
||||
self,
|
||||
id,
|
||||
@@ -112,7 +115,11 @@ class TriggerWordToggleLM:
|
||||
|
||||
if isinstance(trigger_data, list):
|
||||
if group_mode:
|
||||
if allow_strength_adjustment:
|
||||
if any(self._group_has_child_items(item) for item in trigger_data):
|
||||
filtered_groups = self._process_group_items(
|
||||
trigger_data, allow_strength_adjustment
|
||||
)
|
||||
elif allow_strength_adjustment:
|
||||
parsed_items = [
|
||||
self._parse_trigger_item(
|
||||
item, allow_strength_adjustment
|
||||
@@ -174,6 +181,41 @@ class TriggerWordToggleLM:
|
||||
|
||||
return (filtered_triggers,)
|
||||
|
||||
def _process_group_items(self, trigger_data, allow_strength_adjustment):
|
||||
filtered_groups = []
|
||||
|
||||
for item in trigger_data:
|
||||
group = self._parse_trigger_item(item, allow_strength_adjustment)
|
||||
if not group["text"] or not group["active"]:
|
||||
continue
|
||||
|
||||
raw_items = item.get("items") if isinstance(item, dict) else None
|
||||
if isinstance(raw_items, list):
|
||||
active_items = []
|
||||
for raw_item in raw_items:
|
||||
child = self._parse_trigger_item(
|
||||
raw_item, allow_strength_adjustment=False
|
||||
)
|
||||
if child["text"] and child["active"]:
|
||||
active_items.append(child["text"])
|
||||
|
||||
if not active_items:
|
||||
continue
|
||||
|
||||
group_text = ", ".join(active_items)
|
||||
else:
|
||||
group_text = group["text"]
|
||||
|
||||
filtered_groups.append(
|
||||
self._format_word_output(
|
||||
group_text,
|
||||
group["strength"],
|
||||
allow_strength_adjustment,
|
||||
)
|
||||
)
|
||||
|
||||
return filtered_groups
|
||||
|
||||
def _parse_trigger_item(self, item, allow_strength_adjustment):
|
||||
text = (item.get("text") or "").strip()
|
||||
active = bool(item.get("active", False))
|
||||
|
||||
205
py/nodes/unet_loader.py
Normal file
205
py/nodes/unet_loader.py
Normal file
@@ -0,0 +1,205 @@
|
||||
import logging
|
||||
import os
|
||||
from typing import List, Tuple
|
||||
import comfy.sd # type: ignore
|
||||
from ..utils.utils import get_checkpoint_info_absolute, _format_model_name_for_comfyui
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class UNETLoaderLM:
|
||||
"""UNET Loader with support for extra folder paths
|
||||
|
||||
Loads diffusion models/UNets from both standard ComfyUI folders and LoRA Manager's
|
||||
extra folder paths, providing a unified interface for UNET loading.
|
||||
Supports both regular diffusion models and GGUF format models.
|
||||
"""
|
||||
|
||||
NAME = "Unet Loader (LoraManager)"
|
||||
CATEGORY = "Lora Manager/loaders"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
# Get list of unet names from scanner (includes extra folder paths)
|
||||
unet_names = s._get_unet_names()
|
||||
return {
|
||||
"required": {
|
||||
"unet_name": (
|
||||
unet_names,
|
||||
{"tooltip": "The name of the diffusion model to load."},
|
||||
),
|
||||
"weight_dtype": (
|
||||
["default", "fp8_e4m3fn", "fp8_e4m3fn_fast", "fp8_e5m2"],
|
||||
{"tooltip": "The dtype to use for the model weights."},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("MODEL",)
|
||||
RETURN_NAMES = ("MODEL",)
|
||||
OUTPUT_TOOLTIPS = ("The model used for denoising latents.",)
|
||||
FUNCTION = "load_unet"
|
||||
|
||||
@classmethod
|
||||
def _get_unet_names(cls) -> List[str]:
|
||||
"""Get list of diffusion model names from scanner cache in ComfyUI format (relative path with extension)"""
|
||||
try:
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
import asyncio
|
||||
|
||||
async def _get_names():
|
||||
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||
cache = await scanner.get_cached_data()
|
||||
|
||||
# Get all model roots for calculating relative paths
|
||||
model_roots = scanner.get_model_roots()
|
||||
|
||||
# Filter only diffusion_model type and format names
|
||||
names = []
|
||||
for item in cache.raw_data:
|
||||
if item.get("sub_type") == "diffusion_model":
|
||||
file_path = item.get("file_path", "")
|
||||
if file_path:
|
||||
# Format using relative path with OS-native separator
|
||||
formatted_name = _format_model_name_for_comfyui(
|
||||
file_path, model_roots
|
||||
)
|
||||
if formatted_name:
|
||||
names.append(formatted_name)
|
||||
|
||||
return sorted(names)
|
||||
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
import concurrent.futures
|
||||
|
||||
def run_in_thread():
|
||||
new_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(new_loop)
|
||||
try:
|
||||
return new_loop.run_until_complete(_get_names())
|
||||
finally:
|
||||
new_loop.close()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
future = executor.submit(run_in_thread)
|
||||
return future.result()
|
||||
except RuntimeError:
|
||||
return asyncio.run(_get_names())
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting unet names: {e}")
|
||||
return []
|
||||
|
||||
def load_unet(self, unet_name: str, weight_dtype: str) -> Tuple:
|
||||
"""Load a diffusion model by name, supporting extra folder paths
|
||||
|
||||
Args:
|
||||
unet_name: The name of the diffusion model to load (relative path with extension)
|
||||
weight_dtype: The dtype to use for model weights
|
||||
|
||||
Returns:
|
||||
Tuple of (MODEL,)
|
||||
"""
|
||||
import torch
|
||||
|
||||
# Get absolute path from cache using ComfyUI-style name
|
||||
unet_path, metadata = get_checkpoint_info_absolute(unet_name)
|
||||
|
||||
if metadata is None:
|
||||
raise FileNotFoundError(
|
||||
f"Diffusion model '{unet_name}' not found in LoRA Manager cache. "
|
||||
"Make sure the model is indexed and try again."
|
||||
)
|
||||
|
||||
# Check if it's a GGUF model
|
||||
if unet_path.endswith(".gguf"):
|
||||
return self._load_gguf_unet(unet_path, unet_name, weight_dtype)
|
||||
|
||||
# Load regular diffusion model using ComfyUI's API
|
||||
logger.info(f"Loading diffusion model from: {unet_path}")
|
||||
|
||||
# Build model options based on weight_dtype
|
||||
model_options = {}
|
||||
if weight_dtype == "fp8_e4m3fn":
|
||||
model_options["dtype"] = torch.float8_e4m3fn
|
||||
elif weight_dtype == "fp8_e4m3fn_fast":
|
||||
model_options["dtype"] = torch.float8_e4m3fn
|
||||
model_options["fp8_optimizations"] = True
|
||||
elif weight_dtype == "fp8_e5m2":
|
||||
model_options["dtype"] = torch.float8_e5m2
|
||||
|
||||
model = comfy.sd.load_diffusion_model(unet_path, model_options=model_options)
|
||||
return (model,)
|
||||
|
||||
def _load_gguf_unet(
|
||||
self, unet_path: str, unet_name: str, weight_dtype: str
|
||||
) -> Tuple:
|
||||
"""Load a GGUF format diffusion model
|
||||
|
||||
Args:
|
||||
unet_path: Absolute path to the GGUF file
|
||||
unet_name: Name of the model for error messages
|
||||
weight_dtype: The dtype to use for model weights
|
||||
|
||||
Returns:
|
||||
Tuple of (MODEL,)
|
||||
"""
|
||||
import torch
|
||||
from .gguf_import_helper import get_gguf_modules
|
||||
|
||||
# Get ComfyUI-GGUF modules using helper (handles various import scenarios)
|
||||
try:
|
||||
loader_module, ops_module, nodes_module = get_gguf_modules()
|
||||
gguf_sd_loader = getattr(loader_module, "gguf_sd_loader")
|
||||
GGMLOps = getattr(ops_module, "GGMLOps")
|
||||
GGUFModelPatcher = getattr(nodes_module, "GGUFModelPatcher")
|
||||
except RuntimeError as e:
|
||||
raise RuntimeError(f"Cannot load GGUF model '{unet_name}'. {str(e)}")
|
||||
|
||||
logger.info(f"Loading GGUF diffusion model from: {unet_path}")
|
||||
|
||||
try:
|
||||
# Load GGUF state dict
|
||||
sd, extra = gguf_sd_loader(unet_path)
|
||||
|
||||
# Prepare kwargs for metadata if supported
|
||||
kwargs = {}
|
||||
import inspect
|
||||
|
||||
valid_params = inspect.signature(
|
||||
comfy.sd.load_diffusion_model_state_dict
|
||||
).parameters
|
||||
if "metadata" in valid_params:
|
||||
kwargs["metadata"] = extra.get("metadata", {})
|
||||
|
||||
# Setup custom operations with GGUF support
|
||||
ops = GGMLOps()
|
||||
|
||||
# Handle weight_dtype for GGUF models
|
||||
if weight_dtype in ("default", None):
|
||||
ops.Linear.dequant_dtype = None
|
||||
elif weight_dtype in ["target"]:
|
||||
ops.Linear.dequant_dtype = weight_dtype
|
||||
else:
|
||||
ops.Linear.dequant_dtype = getattr(torch, weight_dtype, None)
|
||||
|
||||
# Load the model
|
||||
model = comfy.sd.load_diffusion_model_state_dict(
|
||||
sd, model_options={"custom_operations": ops}, **kwargs
|
||||
)
|
||||
|
||||
if model is None:
|
||||
raise RuntimeError(
|
||||
f"Could not detect model type for GGUF diffusion model: {unet_path}"
|
||||
)
|
||||
|
||||
# Wrap with GGUFModelPatcher
|
||||
model = GGUFModelPatcher.clone(model)
|
||||
|
||||
return (model,)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error loading GGUF diffusion model '{unet_name}': {e}")
|
||||
raise RuntimeError(
|
||||
f"Failed to load GGUF diffusion model '{unet_name}': {str(e)}"
|
||||
)
|
||||
@@ -44,11 +44,29 @@ import folder_paths # type: ignore
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_lora_syntax_format():
|
||||
try:
|
||||
from ..services.settings_manager import get_settings_manager
|
||||
return get_settings_manager().get("lora_syntax_format", "legacy")
|
||||
except Exception:
|
||||
return "legacy"
|
||||
|
||||
|
||||
def apply_lora_syntax_format(name):
|
||||
fmt = get_lora_syntax_format()
|
||||
if fmt == "legacy":
|
||||
return name.replace("\\", "/").rstrip("/").split("/")[-1]
|
||||
return name
|
||||
|
||||
|
||||
def extract_lora_name(lora_path):
|
||||
"""Extract the lora name from a lora path (e.g., 'IL\\aorunIllstrious.safetensors' -> 'aorunIllstrious')"""
|
||||
# Get the basename without extension
|
||||
basename = os.path.basename(lora_path)
|
||||
return os.path.splitext(basename)[0]
|
||||
normalized = lora_path.replace("\\", "/")
|
||||
basename = os.path.basename(normalized)
|
||||
name_no_ext = os.path.splitext(basename)[0]
|
||||
dirname = os.path.dirname(normalized)
|
||||
if dirname and dirname not in (".", "/") and not normalized.startswith("/"):
|
||||
return apply_lora_syntax_format(f"{dirname}/{name_no_ext}")
|
||||
return apply_lora_syntax_format(name_no_ext)
|
||||
|
||||
|
||||
def get_loras_list(kwargs):
|
||||
@@ -158,3 +176,24 @@ def nunchaku_load_lora(model, lora_name, lora_strength):
|
||||
ret_model.model.model_config.unet_config["in_channels"] = new_in_channels
|
||||
|
||||
return ret_model
|
||||
|
||||
|
||||
def detect_nunchaku_model_kind(model):
|
||||
"""Return the supported Nunchaku model kind for a Comfy model, if any."""
|
||||
try:
|
||||
model_wrapper = model.model.diffusion_model
|
||||
except (AttributeError, TypeError):
|
||||
return None
|
||||
|
||||
wrapper_name = model_wrapper.__class__.__name__
|
||||
if wrapper_name == "ComfyFluxWrapper":
|
||||
return "flux"
|
||||
|
||||
inner_model = getattr(model_wrapper, "model", None)
|
||||
inner_name = inner_model.__class__.__name__ if inner_model is not None else ""
|
||||
if wrapper_name.endswith("NunchakuQwenImageTransformer2DModel"):
|
||||
return "qwen_image"
|
||||
if inner_name.endswith("NunchakuQwenImageTransformer2DModel"):
|
||||
return "qwen_image"
|
||||
|
||||
return None
|
||||
|
||||
@@ -1,10 +1,22 @@
|
||||
import folder_paths # type: ignore
|
||||
from ..utils.utils import get_lora_info
|
||||
import os
|
||||
from ..utils.utils import get_lora_info_absolute
|
||||
from ..config import config
|
||||
from .utils import FlexibleOptionalInputType, any_type, get_loras_list
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _relpath_within_loras(abs_path):
|
||||
"""Return abs_path relative to the first matching lora root, or basename as fallback."""
|
||||
all_roots = list(config.loras_roots or []) + list(config.extra_loras_roots or [])
|
||||
for root in all_roots:
|
||||
try:
|
||||
return os.path.relpath(abs_path, root)
|
||||
except ValueError:
|
||||
continue
|
||||
return os.path.basename(abs_path)
|
||||
|
||||
class WanVideoLoraSelectLM:
|
||||
NAME = "WanVideo Lora Select (LoraManager)"
|
||||
CATEGORY = "Lora Manager/stackers"
|
||||
@@ -56,13 +68,13 @@ class WanVideoLoraSelectLM:
|
||||
clip_strength = float(lora.get('clipStrength', model_strength))
|
||||
|
||||
# Get lora path and trigger words
|
||||
lora_path, trigger_words = get_lora_info(lora_name)
|
||||
lora_path, trigger_words = get_lora_info_absolute(lora_name)
|
||||
|
||||
# Create lora item for WanVideo format
|
||||
lora_item = {
|
||||
"path": folder_paths.get_full_path("loras", lora_path),
|
||||
"path": lora_path,
|
||||
"strength": model_strength,
|
||||
"name": lora_path.split(".")[0],
|
||||
"name": os.path.splitext(_relpath_within_loras(lora_path))[0],
|
||||
"blocks": selected_blocks,
|
||||
"layer_filter": layer_filter,
|
||||
"low_mem_load": low_mem_load,
|
||||
|
||||
@@ -1,11 +1,23 @@
|
||||
import folder_paths # type: ignore
|
||||
from ..utils.utils import get_lora_info
|
||||
import os
|
||||
from ..utils.utils import get_lora_info_absolute
|
||||
from ..config import config
|
||||
from .utils import any_type
|
||||
import logging
|
||||
|
||||
# 初始化日志记录器
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _relpath_within_loras(abs_path):
|
||||
"""Return abs_path relative to the first matching lora root, or basename as fallback."""
|
||||
all_roots = list(config.loras_roots or []) + list(config.extra_loras_roots or [])
|
||||
for root in all_roots:
|
||||
try:
|
||||
return os.path.relpath(abs_path, root)
|
||||
except ValueError:
|
||||
continue
|
||||
return os.path.basename(abs_path)
|
||||
|
||||
# 定义新节点的类
|
||||
class WanVideoLoraTextSelectLM:
|
||||
# 节点在UI中显示的名称
|
||||
@@ -87,12 +99,12 @@ class WanVideoLoraTextSelectLM:
|
||||
else:
|
||||
continue
|
||||
|
||||
lora_path, trigger_words = get_lora_info(lora_name_raw)
|
||||
lora_path, trigger_words = get_lora_info_absolute(lora_name_raw)
|
||||
|
||||
lora_item = {
|
||||
"path": folder_paths.get_full_path("loras", lora_path),
|
||||
"path": lora_path,
|
||||
"strength": model_strength,
|
||||
"name": lora_path.split(".")[0],
|
||||
"name": os.path.splitext(_relpath_within_loras(lora_path))[0],
|
||||
"blocks": selected_blocks,
|
||||
"layer_filter": layer_filter,
|
||||
"low_mem_load": low_mem_load,
|
||||
|
||||
@@ -7,7 +7,7 @@ import re
|
||||
from typing import Dict, List, Any, Optional, Tuple
|
||||
from abc import ABC, abstractmethod
|
||||
from ..config import config
|
||||
from ..utils.constants import VALID_LORA_TYPES
|
||||
from ..utils.constants import VALID_LORA_TYPES, VALID_CHECKPOINT_SUB_TYPES
|
||||
from ..utils.civitai_utils import rewrite_preview_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -58,9 +58,52 @@ class RecipeMetadataParser(ABC):
|
||||
civitai_info, error_msg = civitai_info_tuple if isinstance(civitai_info_tuple, tuple) else (civitai_info_tuple, None)
|
||||
|
||||
if not civitai_info or error_msg == "Model not found":
|
||||
# Model not found or deleted
|
||||
lora_entry['isDeleted'] = True
|
||||
lora_entry['thumbnailUrl'] = '/loras_static/images/no-preview.png'
|
||||
# CivitAI may fail to resolve a hash that is still being
|
||||
# computed (known CivitAI issue). Before marking as deleted,
|
||||
# try to reconcile with a local model that has the same
|
||||
# filename and matching AutoV3 hash.
|
||||
reconciled = False
|
||||
file_name = lora_entry.get("file_name")
|
||||
if file_name and recipe_scanner and hash_value:
|
||||
lora_scanner = getattr(recipe_scanner, "_lora_scanner", None)
|
||||
if lora_scanner:
|
||||
try:
|
||||
# Local import to avoid circular dependency:
|
||||
# base.py → file_utils → settings_manager → ...
|
||||
# → recipe_scanner → enrichment → base.py
|
||||
from ..utils.file_utils import calculate_autov3 # fmt: skip
|
||||
cache = await lora_scanner.get_cached_data()
|
||||
for item in getattr(cache, "raw_data", []):
|
||||
if item.get("file_name") == file_name:
|
||||
local_path = item.get("file_path")
|
||||
if local_path and os.path.exists(local_path):
|
||||
local_autov3 = calculate_autov3(local_path)
|
||||
if local_autov3 and local_autov3 == hash_value:
|
||||
lora_entry["existsLocally"] = True
|
||||
lora_entry["localPath"] = local_path
|
||||
lora_entry["hash"] = item.get("sha256", hash_value)
|
||||
if "preview_url" in item:
|
||||
lora_entry["thumbnailUrl"] = config.get_preview_static_url(item["preview_url"])
|
||||
civ = item.get("civitai") or {}
|
||||
if isinstance(civ, dict):
|
||||
if civ.get("id") is not None:
|
||||
lora_entry["id"] = civ["id"]
|
||||
if civ.get("modelId") is not None:
|
||||
lora_entry["modelId"] = civ["modelId"]
|
||||
if civ.get("name"):
|
||||
lora_entry["version"] = civ["name"]
|
||||
# model_name is the CivitAI model display
|
||||
# name stored directly in the cache column.
|
||||
cached_model_name = item.get("model_name")
|
||||
if cached_model_name:
|
||||
lora_entry["name"] = cached_model_name
|
||||
reconciled = True
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
if not reconciled:
|
||||
lora_entry['isDeleted'] = True
|
||||
lora_entry['thumbnailUrl'] = '/loras_static/images/no-preview.png'
|
||||
return lora_entry
|
||||
|
||||
# Get model type and validate
|
||||
@@ -173,6 +216,20 @@ class RecipeMetadataParser(ABC):
|
||||
checkpoint['isDeleted'] = True
|
||||
return checkpoint
|
||||
|
||||
# Validate that the model type is actually a checkpoint.
|
||||
# Unlike populate_lora_from_civitai which has this check,
|
||||
# this function was missing type validation — allowing LoRA
|
||||
# version data to be saved as the recipe's checkpoint when the
|
||||
# wrong version ID was passed downstream (fixed in v2.7+).
|
||||
model_type = civitai_data.get('model', {}).get('type', '').lower()
|
||||
if model_type not in VALID_CHECKPOINT_SUB_TYPES:
|
||||
logger.warning(
|
||||
f"Cannot populate checkpoint: model version {civitai_data.get('id')} "
|
||||
f"has type '{model_type}', expected one of {VALID_CHECKPOINT_SUB_TYPES}. "
|
||||
f"Skipping checkpoint enrichment."
|
||||
)
|
||||
return checkpoint
|
||||
|
||||
if 'model' in civitai_data and 'name' in civitai_data['model']:
|
||||
checkpoint['name'] = civitai_data['model']['name']
|
||||
|
||||
|
||||
@@ -13,4 +13,5 @@ GEN_PARAM_KEYS = [
|
||||
'seed',
|
||||
'size',
|
||||
'clip_skip',
|
||||
'denoising_strength',
|
||||
]
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import logging
|
||||
import json
|
||||
import re
|
||||
import os
|
||||
from typing import Any, Dict, Optional
|
||||
from .merger import GenParamsMerger
|
||||
from .base import RecipeMetadataParser
|
||||
from ..services.metadata_service import get_default_metadata_provider
|
||||
from ..utils.civitai_utils import extract_civitai_image_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -16,54 +16,65 @@ class RecipeEnricher:
|
||||
async def enrich_recipe(
|
||||
recipe: Dict[str, Any],
|
||||
civitai_client: Any,
|
||||
request_params: Optional[Dict[str, Any]] = None
|
||||
request_params: Optional[Dict[str, Any]] = None,
|
||||
prefetched_civitai_meta_raw: Optional[Dict[str, Any]] = None,
|
||||
prefetched_model_version_id: Optional[int] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
Enrich a recipe dictionary in-place with metadata from Civitai and embedded params.
|
||||
|
||||
|
||||
Args:
|
||||
recipe: The recipe dictionary to enrich. Must have 'gen_params' initialized.
|
||||
civitai_client: Authenticated Civitai client instance.
|
||||
request_params: (Optional) Parameters from a user request (e.g. import).
|
||||
|
||||
prefetched_civitai_meta_raw: (Optional) Pre-fetched raw meta from Civitai
|
||||
get_image_info, avoiding a duplicate API call.
|
||||
prefetched_model_version_id: (Optional) Pre-fetched model version ID.
|
||||
|
||||
Returns:
|
||||
bool: True if the recipe was modified, False otherwise.
|
||||
"""
|
||||
updated = False
|
||||
gen_params = recipe.get("gen_params", {})
|
||||
|
||||
# 1. Fetch Civitai Info if available
|
||||
|
||||
# 1. Obtain Civitai metadata
|
||||
civitai_meta = None
|
||||
model_version_id = None
|
||||
|
||||
source_url = recipe.get("source_url") or recipe.get("source_path", "")
|
||||
|
||||
# Check if it's a Civitai image URL
|
||||
image_id_match = re.search(r'civitai\.com/images/(\d+)', str(source_url))
|
||||
if image_id_match:
|
||||
image_id = image_id_match.group(1)
|
||||
try:
|
||||
image_info = await civitai_client.get_image_info(image_id)
|
||||
if image_info:
|
||||
# Handle nested meta often found in Civitai API responses
|
||||
raw_meta = image_info.get("meta")
|
||||
if isinstance(raw_meta, dict):
|
||||
if "meta" in raw_meta and isinstance(raw_meta["meta"], dict):
|
||||
civitai_meta = raw_meta["meta"]
|
||||
else:
|
||||
civitai_meta = raw_meta
|
||||
|
||||
model_version_id = image_info.get("modelVersionId")
|
||||
|
||||
# If not at top level, check resources in meta
|
||||
if not model_version_id and civitai_meta:
|
||||
resources = civitai_meta.get("civitaiResources", [])
|
||||
for res in resources:
|
||||
if res.get("type") == "checkpoint":
|
||||
model_version_id = res.get("modelVersionId")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to fetch Civitai image info: {e}")
|
||||
model_version_id = prefetched_model_version_id
|
||||
|
||||
source_path = recipe.get("source_path", "")
|
||||
|
||||
if prefetched_civitai_meta_raw is not None:
|
||||
raw_meta = prefetched_civitai_meta_raw
|
||||
if isinstance(raw_meta, dict):
|
||||
if "meta" in raw_meta and isinstance(raw_meta["meta"], dict):
|
||||
civitai_meta = raw_meta["meta"]
|
||||
else:
|
||||
civitai_meta = raw_meta
|
||||
else:
|
||||
image_id = extract_civitai_image_id(str(source_path))
|
||||
if image_id:
|
||||
try:
|
||||
image_info = await civitai_client.get_image_info(
|
||||
image_id, source_url=str(source_path)
|
||||
)
|
||||
if image_info:
|
||||
raw_meta = image_info.get("meta")
|
||||
if isinstance(raw_meta, dict):
|
||||
if "meta" in raw_meta and isinstance(raw_meta["meta"], dict):
|
||||
civitai_meta = raw_meta["meta"]
|
||||
else:
|
||||
civitai_meta = raw_meta
|
||||
|
||||
model_version_id = image_info.get("modelVersionId")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to fetch Civitai image info: {e}")
|
||||
|
||||
if not model_version_id and civitai_meta:
|
||||
resources = civitai_meta.get("civitaiResources", [])
|
||||
for res in resources:
|
||||
if res.get("type") == "checkpoint":
|
||||
model_version_id = res.get("modelVersionId")
|
||||
break
|
||||
|
||||
# 2. Merge Parameters
|
||||
# Priority: request_params > civitai_meta > embedded (existing gen_params)
|
||||
@@ -179,27 +190,42 @@ class RecipeEnricher:
|
||||
existing_cp = recipe.get("checkpoint")
|
||||
if existing_cp is None:
|
||||
existing_cp = {}
|
||||
|
||||
# Extract baseModel from raw civitai_info before populate_checkpoint_from_civitai
|
||||
# (populate may reject non-checkpoint types and lose this data)
|
||||
base_model_from_civitai: str = ""
|
||||
if isinstance(civitai_info, dict):
|
||||
base_model_from_civitai = civitai_info.get("baseModel", "") or ""
|
||||
elif isinstance(civitai_info, tuple) and len(civitai_info) > 0 and isinstance(civitai_info[0], dict):
|
||||
base_model_from_civitai = civitai_info[0].get("baseModel", "") or ""
|
||||
|
||||
checkpoint_data = await RecipeMetadataParser.populate_checkpoint_from_civitai(existing_cp, civitai_info)
|
||||
# 1. First, resolve base_model using full data before we format it away
|
||||
|
||||
# 1. Resolve base_model from checkpoint_data first, then fall back to raw civitai_info
|
||||
current_base_model = recipe.get("base_model")
|
||||
resolved_base_model = checkpoint_data.get("baseModel")
|
||||
resolved_base_model = checkpoint_data.get("baseModel") or base_model_from_civitai
|
||||
if resolved_base_model:
|
||||
# Update if empty OR if it matches our generic prefix but is less specific
|
||||
is_generic = not current_base_model or current_base_model.lower() in ["flux", "sdxl", "sd15"]
|
||||
if is_generic and resolved_base_model != current_base_model:
|
||||
recipe["base_model"] = resolved_base_model
|
||||
|
||||
# 2. Format according to requirements: type, modelId, modelVersionId, modelName, modelVersionName
|
||||
formatted_checkpoint = {
|
||||
"type": "checkpoint",
|
||||
"modelId": checkpoint_data.get("modelId"),
|
||||
"modelVersionId": checkpoint_data.get("id") or checkpoint_data.get("modelVersionId"),
|
||||
"modelName": checkpoint_data.get("name"), # In base.py, 'name' is populated from civitai_data['model']['name']
|
||||
"modelVersionName": checkpoint_data.get("version") # In base.py, 'version' is populated from civitai_data['name']
|
||||
}
|
||||
# Remove None values
|
||||
recipe["checkpoint"] = {k: v for k, v in formatted_checkpoint.items() if v is not None}
|
||||
|
||||
|
||||
# 2. Only format and save checkpoint if it has real data (not just type after type rejection)
|
||||
has_checkpoint_data = any([
|
||||
checkpoint_data.get("modelId"),
|
||||
checkpoint_data.get("id") or checkpoint_data.get("modelVersionId"),
|
||||
checkpoint_data.get("name"),
|
||||
checkpoint_data.get("version"),
|
||||
])
|
||||
if has_checkpoint_data:
|
||||
formatted_checkpoint = {
|
||||
"type": "checkpoint",
|
||||
"modelId": checkpoint_data.get("modelId"),
|
||||
"modelVersionId": checkpoint_data.get("id") or checkpoint_data.get("modelVersionId"),
|
||||
"modelName": checkpoint_data.get("name"),
|
||||
"modelVersionName": checkpoint_data.get("version"),
|
||||
}
|
||||
recipe["checkpoint"] = {k: v for k, v in formatted_checkpoint.items() if v is not None}
|
||||
|
||||
return True
|
||||
else:
|
||||
# Fallback to name extraction if we don't already have one
|
||||
|
||||
@@ -7,6 +7,7 @@ from .parsers import (
|
||||
MetaFormatParser,
|
||||
AutomaticMetadataParser,
|
||||
CivitaiApiMetadataParser,
|
||||
SuiImageParamsParser,
|
||||
)
|
||||
from .base import RecipeMetadataParser
|
||||
|
||||
@@ -55,6 +56,13 @@ class RecipeParserFactory:
|
||||
# If JSON parsing fails, move on to other parsers
|
||||
pass
|
||||
|
||||
# Try SuiImageParamsParser for SuiImage metadata format
|
||||
try:
|
||||
if SuiImageParamsParser().is_metadata_matching(metadata_str):
|
||||
return SuiImageParamsParser()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Check other parsers that expect string input
|
||||
if RecipeFormatParser().is_metadata_matching(metadata_str):
|
||||
return RecipeFormatParser()
|
||||
|
||||
@@ -1,27 +1,33 @@
|
||||
from typing import Any, Dict, Optional
|
||||
import logging
|
||||
|
||||
from .constants import GEN_PARAM_KEYS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GenParamsMerger:
|
||||
"""Utility to merge generation parameters from multiple sources with priority."""
|
||||
|
||||
ALLOWED_KEYS = set(GEN_PARAM_KEYS)
|
||||
|
||||
BLACKLISTED_KEYS = {
|
||||
"id", "url", "userId", "username", "createdAt", "updatedAt", "hash", "meta",
|
||||
"draft", "extra", "width", "height", "process", "quantity", "workflow",
|
||||
"baseModel", "resources", "disablePoi", "aspectRatio", "Created Date",
|
||||
"experimental", "civitaiResources", "civitai_resources", "Civitai resources",
|
||||
"modelVersionId", "modelId", "hashes", "Model", "Model hash", "checkpoint_hash",
|
||||
"checkpoint", "checksum", "model_checksum"
|
||||
"checkpoint", "checksum", "model_checksum", "raw_metadata",
|
||||
}
|
||||
|
||||
|
||||
NORMALIZATION_MAPPING = {
|
||||
# Civitai specific
|
||||
"cfg": "cfg_scale",
|
||||
"cfgScale": "cfg_scale",
|
||||
"clipSkip": "clip_skip",
|
||||
"negativePrompt": "negative_prompt",
|
||||
# Case variations
|
||||
"Sampler": "sampler",
|
||||
"sampler_name": "sampler",
|
||||
"scheduler": "sampler",
|
||||
"Steps": "steps",
|
||||
"Seed": "seed",
|
||||
"Size": "size",
|
||||
@@ -36,63 +42,40 @@ class GenParamsMerger:
|
||||
def merge(
|
||||
request_params: Optional[Dict[str, Any]] = None,
|
||||
civitai_meta: Optional[Dict[str, Any]] = None,
|
||||
embedded_metadata: Optional[Dict[str, Any]] = None
|
||||
embedded_metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Merge generation parameters from three sources.
|
||||
|
||||
Priority: request_params > civitai_meta > embedded_metadata
|
||||
|
||||
Args:
|
||||
request_params: Params provided directly in the import request
|
||||
civitai_meta: Params from Civitai Image API 'meta' field
|
||||
embedded_metadata: Params extracted from image EXIF/embedded metadata
|
||||
|
||||
Returns:
|
||||
Merged parameters dictionary
|
||||
"""
|
||||
result = {}
|
||||
|
||||
# 1. Start with embedded metadata (lowest priority)
|
||||
Priority: request_params > civitai_meta > embedded_metadata
|
||||
"""
|
||||
result: Dict[str, Any] = {}
|
||||
|
||||
if embedded_metadata:
|
||||
# If it's a full recipe metadata, we use its gen_params
|
||||
if "gen_params" in embedded_metadata and isinstance(embedded_metadata["gen_params"], dict):
|
||||
if "gen_params" in embedded_metadata and isinstance(
|
||||
embedded_metadata["gen_params"], dict
|
||||
):
|
||||
GenParamsMerger._update_normalized(result, embedded_metadata["gen_params"])
|
||||
else:
|
||||
# Otherwise assume the dict itself contains gen_params
|
||||
GenParamsMerger._update_normalized(result, embedded_metadata)
|
||||
|
||||
# 2. Layer Civitai meta (medium priority)
|
||||
if civitai_meta:
|
||||
GenParamsMerger._update_normalized(result, civitai_meta)
|
||||
|
||||
# 3. Layer request params (highest priority)
|
||||
if request_params:
|
||||
GenParamsMerger._update_normalized(result, request_params)
|
||||
|
||||
# Filter out blacklisted keys and also the original camelCase keys if they were normalized
|
||||
final_result = {}
|
||||
for k, v in result.items():
|
||||
if k in GenParamsMerger.BLACKLISTED_KEYS:
|
||||
continue
|
||||
if k in GenParamsMerger.NORMALIZATION_MAPPING:
|
||||
continue
|
||||
final_result[k] = v
|
||||
|
||||
return final_result
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _update_normalized(target: Dict[str, Any], source: Dict[str, Any]) -> None:
|
||||
"""Update target dict with normalized keys from source."""
|
||||
for k, v in source.items():
|
||||
normalized_key = GenParamsMerger.NORMALIZATION_MAPPING.get(k, k)
|
||||
target[normalized_key] = v
|
||||
# Also keep the original key for now if it's not the same,
|
||||
# so we can filter at the end or avoid losing it if it wasn't supposed to be renamed?
|
||||
# Actually, if we rename it, we should probably NOT keep both in 'target'
|
||||
# because we want to filter them out at the end anyway.
|
||||
if normalized_key != k:
|
||||
# If we are overwriting an existing snake_case key with a camelCase one's value,
|
||||
# that's fine because of the priority order of calls to _update_normalized.
|
||||
pass
|
||||
target[k] = v
|
||||
"""Update target dict with normalized, persistence-safe keys from source."""
|
||||
for key, value in source.items():
|
||||
if key in GenParamsMerger.BLACKLISTED_KEYS:
|
||||
continue
|
||||
|
||||
normalized_key = GenParamsMerger.NORMALIZATION_MAPPING.get(key, key)
|
||||
if normalized_key not in GenParamsMerger.ALLOWED_KEYS:
|
||||
continue
|
||||
|
||||
target[normalized_key] = value
|
||||
|
||||
@@ -5,6 +5,7 @@ from .comfy import ComfyMetadataParser
|
||||
from .meta_format import MetaFormatParser
|
||||
from .automatic import AutomaticMetadataParser
|
||||
from .civitai_image import CivitaiApiMetadataParser
|
||||
from .sui_image_params import SuiImageParamsParser
|
||||
|
||||
__all__ = [
|
||||
'RecipeFormatParser',
|
||||
@@ -12,4 +13,5 @@ __all__ = [
|
||||
'MetaFormatParser',
|
||||
'AutomaticMetadataParser',
|
||||
'CivitaiApiMetadataParser',
|
||||
'SuiImageParamsParser',
|
||||
]
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Dict, Any, Union
|
||||
from ..base import RecipeMetadataParser
|
||||
from ..constants import GEN_PARAM_KEYS
|
||||
from ...services.metadata_service import get_default_metadata_provider
|
||||
from ...config import config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -42,6 +43,7 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
"height",
|
||||
"Model",
|
||||
"Model hash",
|
||||
"modelVersionIds",
|
||||
)
|
||||
return any(key in payload for key in civitai_image_fields)
|
||||
|
||||
@@ -72,7 +74,8 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
return False
|
||||
|
||||
async def parse_metadata( # type: ignore[override]
|
||||
self, user_comment, recipe_scanner=None, civitai_client=None
|
||||
self, user_comment, recipe_scanner=None, civitai_client=None,
|
||||
local_cache: dict[str, Any] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Parse metadata from Civitai image format
|
||||
|
||||
@@ -80,6 +83,8 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
user_comment: The metadata from the image (dict)
|
||||
recipe_scanner: Optional recipe scanner service
|
||||
civitai_client: Optional Civitai API client (deprecated, use metadata_provider instead)
|
||||
local_cache: Optional dict mapping sha256/autov3 hash → scanner cache item.
|
||||
When provided, matching models skip CivitAI API calls.
|
||||
|
||||
Returns:
|
||||
Dict containing parsed recipe data
|
||||
@@ -184,8 +189,77 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
# Process standard resources array
|
||||
if "resources" in metadata and isinstance(metadata["resources"], list):
|
||||
for resource in metadata["resources"]:
|
||||
resource_type = resource.get("type", "lora")
|
||||
|
||||
# Track resources with type "model" — these are checkpoint models.
|
||||
# The resources array is the most reliable source for checkpoint
|
||||
# identification because it has an explicit type field and hash,
|
||||
# unlike modelVersionIds which is a flat list with no type info.
|
||||
if resource_type == "model":
|
||||
checkpoint_entry = {
|
||||
"id": 0,
|
||||
"modelId": 0,
|
||||
"name": resource.get("name", "Unknown Model"),
|
||||
"version": "",
|
||||
"type": resource.get("type", "model"),
|
||||
"existsLocally": False,
|
||||
"localPath": None,
|
||||
"file_name": resource.get("name", ""),
|
||||
"hash": resource.get("hash", "") or "",
|
||||
"thumbnailUrl": "/loras_static/images/no-preview.png",
|
||||
"baseModel": "",
|
||||
"size": 0,
|
||||
"downloadUrl": "",
|
||||
"isDeleted": False,
|
||||
}
|
||||
|
||||
# Try to look up base model from the checkpoint hash
|
||||
cp_hash = checkpoint_entry.get("hash")
|
||||
if cp_hash and metadata_provider:
|
||||
local_cached = local_cache.get(cp_hash) if local_cache else None
|
||||
if local_cached:
|
||||
self._populate_entry_from_cache(
|
||||
checkpoint_entry, local_cached
|
||||
)
|
||||
bm = checkpoint_entry.get("baseModel", "")
|
||||
if bm and not result["base_model"]:
|
||||
result["base_model"] = bm
|
||||
else:
|
||||
try:
|
||||
civitai_info = (
|
||||
await metadata_provider.get_model_by_hash(
|
||||
cp_hash
|
||||
)
|
||||
)
|
||||
civitai_data, error_msg = (
|
||||
(civitai_info, None)
|
||||
if not isinstance(civitai_info, tuple)
|
||||
else civitai_info
|
||||
)
|
||||
if civitai_data and error_msg != "Model not found":
|
||||
if 'model' in civitai_data and 'name' in civitai_data['model']:
|
||||
checkpoint_entry['name'] = civitai_data['model']['name']
|
||||
checkpoint_entry['id'] = civitai_data.get('id', 0)
|
||||
checkpoint_entry['modelId'] = civitai_data.get('modelId', 0)
|
||||
if 'name' in civitai_data:
|
||||
checkpoint_entry['version'] = civitai_data['name']
|
||||
base_model = civitai_data.get('baseModel', '')
|
||||
if base_model:
|
||||
checkpoint_entry['baseModel'] = base_model
|
||||
if not result['base_model']:
|
||||
result['base_model'] = base_model
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching checkpoint info for hash "
|
||||
f"{cp_hash}: {e}"
|
||||
)
|
||||
|
||||
if result["model"] is None:
|
||||
result["model"] = checkpoint_entry
|
||||
continue
|
||||
|
||||
# Modified to process resources without a type field as potential LoRAs
|
||||
if resource.get("type", "lora") == "lora":
|
||||
if resource_type == "lora":
|
||||
lora_hash = resource.get("hash", "")
|
||||
|
||||
# Try to get hash from the hashes field if not present in resource
|
||||
@@ -219,34 +293,45 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
}
|
||||
|
||||
# Try to get info from Civitai if hash is available
|
||||
if lora_entry["hash"] and metadata_provider:
|
||||
try:
|
||||
civitai_info = (
|
||||
await metadata_provider.get_model_by_hash(lora_hash)
|
||||
if lora_hash and metadata_provider:
|
||||
local_cached = local_cache.get(lora_hash) if local_cache else None
|
||||
if local_cached:
|
||||
self._populate_entry_from_cache(
|
||||
lora_entry, local_cached
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
continue # Skip invalid LoRA types
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
# If we have a version ID from Civitai, track it for deduplication
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
# Track by version ID for deduplication
|
||||
if lora_entry.get("id"):
|
||||
added_loras[str(lora_entry["id"])] = len(
|
||||
result["loras"]
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_entry['hash']}: {e}"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
civitai_info = (
|
||||
await metadata_provider.get_model_by_hash(lora_hash)
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
continue # Skip invalid LoRA types
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
# If we have a version ID from Civitai, track it for deduplication
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
added_loras[str(lora_entry["id"])] = len(
|
||||
result["loras"]
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_entry['hash']}: {e}"
|
||||
)
|
||||
|
||||
# Track by hash if we have it
|
||||
if lora_hash:
|
||||
@@ -429,6 +514,65 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
|
||||
result["loras"].append(lora_entry)
|
||||
|
||||
# Process modelVersionIds from Civitai image API
|
||||
# These are model version IDs returned at root level when meta doesn't contain resources
|
||||
if "modelVersionIds" in metadata and isinstance(
|
||||
metadata["modelVersionIds"], list
|
||||
):
|
||||
for version_id in metadata["modelVersionIds"]:
|
||||
version_id_str = str(version_id)
|
||||
|
||||
# Skip if we've already added this LoRA by version ID
|
||||
if version_id_str in added_loras:
|
||||
continue
|
||||
|
||||
# Initialize lora entry with version ID
|
||||
lora_entry = {
|
||||
"id": version_id,
|
||||
"modelId": 0,
|
||||
"name": "Unknown LoRA",
|
||||
"version": "",
|
||||
"type": "lora",
|
||||
"weight": 1.0,
|
||||
"existsLocally": False,
|
||||
"thumbnailUrl": "/loras_static/images/no-preview.png",
|
||||
"baseModel": "",
|
||||
"size": 0,
|
||||
"downloadUrl": "",
|
||||
"isDeleted": False,
|
||||
}
|
||||
|
||||
# Fetch model info from Civitai
|
||||
if metadata_provider and version_id_str:
|
||||
try:
|
||||
civitai_info = (
|
||||
await metadata_provider.get_model_version_info(
|
||||
version_id_str
|
||||
)
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
continue # Skip invalid LoRA types
|
||||
|
||||
lora_entry = populated_entry
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for model version {version_id}: {e}"
|
||||
)
|
||||
|
||||
# Track this LoRA for deduplication
|
||||
if version_id_str:
|
||||
added_loras[version_id_str] = len(result["loras"])
|
||||
|
||||
result["loras"].append(lora_entry)
|
||||
|
||||
# If we found LoRA hashes in the metadata but haven't already
|
||||
# populated entries for them, fall back to creating LoRAs from
|
||||
# the hashes section. Some Civitai image responses only include
|
||||
@@ -565,3 +709,41 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing Civitai image metadata: {e}", exc_info=True)
|
||||
return {"error": str(e), "loras": []}
|
||||
|
||||
@staticmethod
|
||||
def _populate_entry_from_cache(
|
||||
entry: dict[str, Any],
|
||||
cache_item: dict[str, Any],
|
||||
) -> None:
|
||||
"""Fill a lora/checkpoint entry from a scanner cache item.
|
||||
|
||||
Avoids CivitAI API calls for models that exist locally.
|
||||
Mirrors the population logic in
|
||||
``RecipeMetadataParser.populate_lora_from_civitai()`` but operates
|
||||
entirely on cached data.
|
||||
"""
|
||||
civ = cache_item.get("civitai") or {}
|
||||
if isinstance(civ, dict):
|
||||
if civ.get("id") is not None:
|
||||
entry["id"] = civ["id"]
|
||||
if civ.get("modelId") is not None:
|
||||
entry["modelId"] = civ["modelId"]
|
||||
if civ.get("name"):
|
||||
entry["version"] = civ["name"]
|
||||
cached_name = cache_item.get("model_name")
|
||||
if cached_name:
|
||||
entry["name"] = cached_name
|
||||
entry["existsLocally"] = True
|
||||
local_path = cache_item.get("file_path")
|
||||
if local_path:
|
||||
entry["localPath"] = local_path
|
||||
sha256 = cache_item.get("sha256")
|
||||
if sha256:
|
||||
entry["hash"] = sha256
|
||||
if "preview_url" in cache_item:
|
||||
entry["thumbnailUrl"] = config.get_preview_static_url(
|
||||
cache_item["preview_url"]
|
||||
)
|
||||
base_model = cache_item.get("base_model", "")
|
||||
if base_model:
|
||||
entry["baseModel"] = base_model
|
||||
|
||||
188
py/recipes/parsers/sui_image_params.py
Normal file
188
py/recipes/parsers/sui_image_params.py
Normal file
@@ -0,0 +1,188 @@
|
||||
"""Parser for SuiImage (Stable Diffusion WebUI) metadata format."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Dict, Any, Optional, List
|
||||
from ..base import RecipeMetadataParser
|
||||
from ...services.metadata_service import get_default_metadata_provider
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SuiImageParamsParser(RecipeMetadataParser):
|
||||
"""Parser for SuiImage metadata JSON format.
|
||||
|
||||
This format is used by some Stable Diffusion WebUI variants.
|
||||
Structure:
|
||||
{
|
||||
"sui_image_params": {
|
||||
"prompt": "...",
|
||||
"negativeprompt": "...",
|
||||
"model": "...",
|
||||
"seed": ...,
|
||||
"steps": ...,
|
||||
...
|
||||
},
|
||||
"sui_models": [
|
||||
{"name": "...", "param": "model", "hash": "..."},
|
||||
...
|
||||
],
|
||||
"sui_extra_data": {...}
|
||||
}
|
||||
"""
|
||||
|
||||
def is_metadata_matching(self, user_comment: str) -> bool:
|
||||
"""Check if the user comment matches the SuiImage metadata format"""
|
||||
try:
|
||||
data = json.loads(user_comment)
|
||||
return isinstance(data, dict) and 'sui_image_params' in data
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return False
|
||||
|
||||
async def parse_metadata(self, user_comment: str, recipe_scanner=None, civitai_client=None) -> Dict[str, Any]:
|
||||
"""Parse metadata from SuiImage metadata format"""
|
||||
try:
|
||||
metadata_provider = await get_default_metadata_provider()
|
||||
|
||||
data = json.loads(user_comment)
|
||||
params = data.get('sui_image_params', {})
|
||||
models = data.get('sui_models', [])
|
||||
|
||||
# Extract prompt and negative prompt
|
||||
prompt = params.get('prompt', '')
|
||||
negative_prompt = params.get('negativeprompt', '') or params.get('negative_prompt', '')
|
||||
|
||||
# Extract generation parameters
|
||||
gen_params = {}
|
||||
if prompt:
|
||||
gen_params['prompt'] = prompt
|
||||
if negative_prompt:
|
||||
gen_params['negative_prompt'] = negative_prompt
|
||||
|
||||
# Map standard parameters
|
||||
param_mapping = {
|
||||
'steps': 'steps',
|
||||
'seed': 'seed',
|
||||
'cfgscale': 'cfg_scale',
|
||||
'cfg_scale': 'cfg_scale',
|
||||
'width': 'width',
|
||||
'height': 'height',
|
||||
'sampler': 'sampler',
|
||||
'scheduler': 'scheduler',
|
||||
'model': 'model',
|
||||
'vae': 'vae',
|
||||
}
|
||||
|
||||
for src_key, dest_key in param_mapping.items():
|
||||
if src_key in params and params[src_key] is not None:
|
||||
gen_params[dest_key] = params[src_key]
|
||||
|
||||
# Add size info if available
|
||||
if 'width' in gen_params and 'height' in gen_params:
|
||||
gen_params['size'] = f"{gen_params['width']}x{gen_params['height']}"
|
||||
|
||||
# Process models - extract checkpoint and loras
|
||||
loras: List[Dict[str, Any]] = []
|
||||
checkpoint: Optional[Dict[str, Any]] = None
|
||||
|
||||
for model in models:
|
||||
model_name = model.get('name', '')
|
||||
param_type = model.get('param', '')
|
||||
model_hash = model.get('hash', '')
|
||||
|
||||
# Remove .safetensors extension for cleaner name
|
||||
clean_name = model_name.replace('.safetensors', '') if model_name else ''
|
||||
|
||||
# Check if this is a LoRA by looking at the name or param type
|
||||
is_lora = 'lora' in model_name.lower() or param_type.lower().startswith('lora')
|
||||
|
||||
if is_lora:
|
||||
lora_entry = {
|
||||
'id': 0,
|
||||
'modelId': 0,
|
||||
'name': clean_name,
|
||||
'version': '',
|
||||
'type': 'lora',
|
||||
'weight': 1.0,
|
||||
'existsLocally': False,
|
||||
'localPath': None,
|
||||
'file_name': model_name,
|
||||
'hash': model_hash.replace('0x', '') if model_hash.startswith('0x') else model_hash,
|
||||
'thumbnailUrl': '/loras_static/images/no-preview.png',
|
||||
'baseModel': '',
|
||||
'size': 0,
|
||||
'downloadUrl': '',
|
||||
'isDeleted': False
|
||||
}
|
||||
|
||||
# Try to get additional info from metadata provider
|
||||
if metadata_provider and model_hash:
|
||||
try:
|
||||
civitai_info = await metadata_provider.get_model_by_hash(
|
||||
model_hash.replace('0x', '') if model_hash.startswith('0x') else model_hash
|
||||
)
|
||||
if civitai_info:
|
||||
lora_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry, civitai_info, recipe_scanner
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Error fetching info for LoRA {clean_name}: {e}")
|
||||
|
||||
if lora_entry:
|
||||
loras.append(lora_entry)
|
||||
elif param_type == 'model' or 'lora' not in model_name.lower():
|
||||
# This is likely a checkpoint
|
||||
checkpoint_entry = {
|
||||
'id': 0,
|
||||
'modelId': 0,
|
||||
'name': clean_name,
|
||||
'version': '',
|
||||
'type': 'checkpoint',
|
||||
'hash': model_hash.replace('0x', '') if model_hash.startswith('0x') else model_hash,
|
||||
'existsLocally': False,
|
||||
'localPath': None,
|
||||
'file_name': model_name,
|
||||
'thumbnailUrl': '/loras_static/images/no-preview.png',
|
||||
'baseModel': '',
|
||||
'size': 0,
|
||||
'downloadUrl': '',
|
||||
'isDeleted': False
|
||||
}
|
||||
|
||||
# Try to get additional info from metadata provider
|
||||
if metadata_provider and model_hash:
|
||||
try:
|
||||
civitai_info = await metadata_provider.get_model_by_hash(
|
||||
model_hash.replace('0x', '') if model_hash.startswith('0x') else model_hash
|
||||
)
|
||||
if civitai_info:
|
||||
checkpoint_entry = await self.populate_checkpoint_from_civitai(
|
||||
checkpoint_entry, civitai_info
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Error fetching info for checkpoint {clean_name}: {e}")
|
||||
|
||||
checkpoint = checkpoint_entry
|
||||
|
||||
# Determine base model from loras or checkpoint
|
||||
base_model = None
|
||||
if loras:
|
||||
base_models = [lora.get('baseModel') for lora in loras if lora.get('baseModel')]
|
||||
if base_models:
|
||||
from collections import Counter
|
||||
base_model_counts = Counter(base_models)
|
||||
base_model = base_model_counts.most_common(1)[0][0]
|
||||
elif checkpoint and checkpoint.get('baseModel'):
|
||||
base_model = checkpoint['baseModel']
|
||||
|
||||
return {
|
||||
'base_model': base_model,
|
||||
'loras': loras,
|
||||
'checkpoint': checkpoint,
|
||||
'gen_params': gen_params,
|
||||
'from_sui_image_params': True
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing SuiImage metadata: {e}", exc_info=True)
|
||||
return {"error": str(e), "loras": []}
|
||||
@@ -251,7 +251,7 @@ class BaseModelRoutes(ABC):
|
||||
|
||||
def _find_model_file(self, files):
|
||||
"""Find the appropriate model file from the files list - can be overridden by subclasses."""
|
||||
return next((file for file in files if file.get("type") == "Model" and file.get("primary") is True), None)
|
||||
return next((file for file in files if file.get("type") in ("Model", "Diffusion Model") and file.get("primary") is True), None)
|
||||
|
||||
def get_handler(self, name: str) -> Callable[[web.Request], web.StreamResponse]:
|
||||
"""Expose handlers for subclasses or tests."""
|
||||
|
||||
141
py/routes/handlers/base_model_handlers.py
Normal file
141
py/routes/handlers/base_model_handlers.py
Normal file
@@ -0,0 +1,141 @@
|
||||
"""Handlers for base model related endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Awaitable, Callable, Dict
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from ...services.civitai_base_model_service import get_civitai_base_model_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BaseModelHandlerSet:
|
||||
"""Collection of handlers for base model operations."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_model_service_factory: Callable[[], Any] = get_civitai_base_model_service,
|
||||
) -> None:
|
||||
self._base_model_service_factory = base_model_service_factory
|
||||
|
||||
def to_route_mapping(
|
||||
self,
|
||||
) -> Dict[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||
"""Return mapping of route names to handler methods."""
|
||||
return {
|
||||
"get_base_models": self.get_base_models,
|
||||
"refresh_base_models": self.refresh_base_models,
|
||||
"get_base_model_categories": self.get_base_model_categories,
|
||||
"get_base_model_cache_status": self.get_base_model_cache_status,
|
||||
}
|
||||
|
||||
async def get_base_models(self, request: web.Request) -> web.Response:
|
||||
"""Get merged base models (hardcoded + remote from Civitai).
|
||||
|
||||
Query Parameters:
|
||||
refresh: If 'true', force refresh from API
|
||||
|
||||
Returns:
|
||||
JSON response with:
|
||||
- models: List of base model names
|
||||
- source: 'cache', 'api', or 'fallback'
|
||||
- last_updated: ISO timestamp
|
||||
- counts: hardcoded_count, remote_count, merged_count
|
||||
"""
|
||||
try:
|
||||
service = await self._base_model_service_factory()
|
||||
|
||||
# Check for refresh parameter
|
||||
force_refresh = request.query.get("refresh", "").lower() == "true"
|
||||
|
||||
result = await service.get_base_models(force_refresh=force_refresh)
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"data": result,
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in get_base_models: {e}")
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(e)},
|
||||
status=500,
|
||||
)
|
||||
|
||||
async def refresh_base_models(self, request: web.Request) -> web.Response:
|
||||
"""Force refresh base models from Civitai API.
|
||||
|
||||
Returns:
|
||||
JSON response with refreshed data
|
||||
"""
|
||||
try:
|
||||
service = await self._base_model_service_factory()
|
||||
result = await service.refresh_cache()
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"data": result,
|
||||
"message": "Base models cache refreshed successfully",
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in refresh_base_models: {e}")
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(e)},
|
||||
status=500,
|
||||
)
|
||||
|
||||
async def get_base_model_categories(self, request: web.Request) -> web.Response:
|
||||
"""Get categorized base models.
|
||||
|
||||
Returns:
|
||||
JSON response with categorized models
|
||||
"""
|
||||
try:
|
||||
service = await self._base_model_service_factory()
|
||||
categories = service.get_model_categories()
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"data": categories,
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in get_base_model_categories: {e}")
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(e)},
|
||||
status=500,
|
||||
)
|
||||
|
||||
async def get_base_model_cache_status(self, request: web.Request) -> web.Response:
|
||||
"""Get cache status for base models.
|
||||
|
||||
Returns:
|
||||
JSON response with cache status
|
||||
"""
|
||||
try:
|
||||
service = await self._base_model_service_factory()
|
||||
status = service.get_cache_status()
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"data": status,
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in get_base_model_cache_status: {e}")
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(e)},
|
||||
status=500,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -16,9 +16,14 @@ import jinja2
|
||||
|
||||
from ...config import config
|
||||
from ...services.download_coordinator import DownloadCoordinator
|
||||
from ...services.connectivity_guard import (
|
||||
OFFLINE_FRIENDLY_MESSAGE,
|
||||
is_expected_offline_error,
|
||||
)
|
||||
from ...services.metadata_sync_service import MetadataSyncService
|
||||
from ...services.model_file_service import ModelMoveService
|
||||
from ...services.preview_asset_service import PreviewAssetService
|
||||
from ...services.service_registry import ServiceRegistry
|
||||
from ...services.settings_manager import SettingsManager, get_settings_manager
|
||||
from ...services.tag_update_service import TagUpdateService
|
||||
from ...services.use_cases import (
|
||||
@@ -32,6 +37,7 @@ from ...services.use_cases import (
|
||||
)
|
||||
from ...services.websocket_manager import WebSocketManager
|
||||
from ...services.websocket_progress_callback import WebSocketProgressCallback
|
||||
from ...services.download_queue_service import DownloadQueueService
|
||||
from ...services.errors import RateLimitError, ResourceNotFoundError
|
||||
from ...utils.civitai_utils import resolve_license_payload
|
||||
from ...utils.file_utils import calculate_sha256
|
||||
@@ -64,7 +70,6 @@ class ModelPageView:
|
||||
self._settings = settings_service
|
||||
self._server_i18n = server_i18n
|
||||
self._logger = logger
|
||||
self._app_version = self._get_app_version()
|
||||
|
||||
def _load_supporters(self) -> dict:
|
||||
"""Load supporters data from JSON file."""
|
||||
@@ -155,7 +160,7 @@ class ModelPageView:
|
||||
"request": request,
|
||||
"folders": [],
|
||||
"t": self._server_i18n.get_translation,
|
||||
"version": self._app_version,
|
||||
"version": self._get_app_version(),
|
||||
}
|
||||
|
||||
if not is_initializing:
|
||||
@@ -224,6 +229,44 @@ class ModelListingHandler:
|
||||
)
|
||||
return web.json_response({"error": str(exc)}, status=500)
|
||||
|
||||
async def get_excluded_models(self, request: web.Request) -> web.Response:
|
||||
start_time = time.perf_counter()
|
||||
try:
|
||||
params = self._parse_common_params(request)
|
||||
# group_by_model is meaningless for excluded view; strip it
|
||||
params.pop("group_by_model", None)
|
||||
result = await self._service.get_excluded_paginated_data(**params)
|
||||
|
||||
format_start = time.perf_counter()
|
||||
formatted_result = {
|
||||
"items": [
|
||||
await self._service.format_response(item)
|
||||
for item in result["items"]
|
||||
],
|
||||
"total": result["total"],
|
||||
"page": result["page"],
|
||||
"page_size": result["page_size"],
|
||||
"total_pages": result["total_pages"],
|
||||
}
|
||||
format_duration = time.perf_counter() - format_start
|
||||
|
||||
duration = time.perf_counter() - start_time
|
||||
self._logger.debug(
|
||||
"Request for %s/excluded took %.3fs (formatting: %.3fs)",
|
||||
self._service.model_type,
|
||||
duration,
|
||||
format_duration,
|
||||
)
|
||||
return web.json_response(formatted_result)
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error retrieving excluded %ss: %s",
|
||||
self._service.model_type,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
return web.json_response({"error": str(exc)}, status=500)
|
||||
|
||||
def _parse_common_params(self, request: web.Request) -> Dict:
|
||||
page = int(request.query.get("page", "1"))
|
||||
page_size = min(int(request.query.get("page_size", "20")), 100)
|
||||
@@ -261,6 +304,15 @@ class ModelListingHandler:
|
||||
for tag in exclude_tags:
|
||||
if tag:
|
||||
tag_filters[tag] = "exclude"
|
||||
|
||||
auto_tag_filters: Dict[str, str] = {}
|
||||
for tag in request.query.getall("auto_tag_include", []):
|
||||
if tag:
|
||||
auto_tag_filters[tag] = "include"
|
||||
for tag in request.query.getall("auto_tag_exclude", []):
|
||||
if tag:
|
||||
auto_tag_filters[tag] = "exclude"
|
||||
|
||||
favorites_only = request.query.get("favorites_only", "false").lower() == "true"
|
||||
|
||||
search_options = {
|
||||
@@ -309,6 +361,26 @@ class ModelListingHandler:
|
||||
else:
|
||||
allow_selling_generated_content = None # None means no filter applied
|
||||
|
||||
# Name pattern filters for LoRA Pool
|
||||
name_pattern_include = request.query.getall("name_pattern_include", [])
|
||||
name_pattern_exclude = request.query.getall("name_pattern_exclude", [])
|
||||
name_pattern_use_regex = (
|
||||
request.query.get("name_pattern_use_regex", "false").lower() == "true"
|
||||
)
|
||||
|
||||
# Group-by-model flag: deduplicate versions sharing the same civitai modelId
|
||||
group_by_model = (
|
||||
request.query.get("group_by_model", "false").lower() == "true"
|
||||
)
|
||||
|
||||
# View-local-versions filter: show all local versions of a specific model
|
||||
civitai_model_id = request.query.get("civitai_model_id")
|
||||
if civitai_model_id is not None:
|
||||
try:
|
||||
civitai_model_id = int(civitai_model_id)
|
||||
except (TypeError, ValueError):
|
||||
civitai_model_id = None
|
||||
|
||||
return {
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
@@ -320,6 +392,7 @@ class ModelListingHandler:
|
||||
"fuzzy_search": fuzzy_search,
|
||||
"base_models": base_models,
|
||||
"tags": tag_filters,
|
||||
"auto_tags": auto_tag_filters,
|
||||
"tag_logic": tag_logic,
|
||||
"search_options": search_options,
|
||||
"hash_filters": hash_filters,
|
||||
@@ -328,6 +401,11 @@ class ModelListingHandler:
|
||||
"credit_required": credit_required,
|
||||
"allow_selling_generated_content": allow_selling_generated_content,
|
||||
"model_types": model_types,
|
||||
"name_pattern_include": name_pattern_include,
|
||||
"name_pattern_exclude": name_pattern_exclude,
|
||||
"name_pattern_use_regex": name_pattern_use_regex,
|
||||
"group_by_model": group_by_model,
|
||||
"civitai_model_id": civitai_model_id,
|
||||
**self._parse_specific_params(request),
|
||||
}
|
||||
|
||||
@@ -382,6 +460,21 @@ class ModelManagementHandler:
|
||||
self._logger.error("Error excluding model: %s", exc, exc_info=True)
|
||||
return web.Response(text=str(exc), status=500)
|
||||
|
||||
async def unexclude_model(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
data = await request.json()
|
||||
file_path = data.get("file_path")
|
||||
if not file_path:
|
||||
return web.Response(text="Model path is required", status=400)
|
||||
|
||||
result = await self._lifecycle_service.unexclude_model(file_path)
|
||||
return web.json_response(result)
|
||||
except ValueError as exc:
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=400)
|
||||
except Exception as exc:
|
||||
self._logger.error("Error restoring model: %s", exc, exc_info=True)
|
||||
return web.Response(text=str(exc), status=500)
|
||||
|
||||
async def fetch_civitai(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
data = await request.json()
|
||||
@@ -443,6 +536,11 @@ class ModelManagementHandler:
|
||||
formatted_metadata = await self._service.format_response(model_data)
|
||||
return web.json_response({"success": True, "metadata": formatted_metadata})
|
||||
except Exception as exc:
|
||||
if is_expected_offline_error(str(exc)):
|
||||
return web.json_response(
|
||||
{"success": False, "error": OFFLINE_FRIENDLY_MESSAGE},
|
||||
status=503,
|
||||
)
|
||||
self._logger.error("Error fetching from CivitAI: %s", exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
@@ -489,6 +587,11 @@ class ModelManagementHandler:
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
if is_expected_offline_error(str(exc)):
|
||||
return web.json_response(
|
||||
{"success": False, "error": OFFLINE_FRIENDLY_MESSAGE},
|
||||
status=503,
|
||||
)
|
||||
self._logger.error("Error re-linking to CivitAI: %s", exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
@@ -703,7 +806,7 @@ class ModelManagementHandler:
|
||||
|
||||
metadata_updates = {k: v for k, v in data.items() if k != "file_path"}
|
||||
|
||||
await self._metadata_sync.save_metadata_updates(
|
||||
updated_metadata = await self._metadata_sync.save_metadata_updates(
|
||||
file_path=file_path,
|
||||
updates=metadata_updates,
|
||||
metadata_loader=self._metadata_sync.load_local_metadata,
|
||||
@@ -714,7 +817,12 @@ class ModelManagementHandler:
|
||||
cache = await self._service.scanner.get_cached_data()
|
||||
await cache.resort()
|
||||
|
||||
return web.json_response({"success": True})
|
||||
from ...services.auto_tag_service import extract_auto_tags
|
||||
auto_tags = extract_auto_tags(updated_metadata)
|
||||
|
||||
return web.json_response(
|
||||
{"success": True, "auto_tags": auto_tags}
|
||||
)
|
||||
except Exception as exc:
|
||||
self._logger.error("Error saving metadata: %s", exc, exc_info=True)
|
||||
return web.Response(text=str(exc), status=500)
|
||||
@@ -731,14 +839,16 @@ class ModelManagementHandler:
|
||||
if not isinstance(new_tags, list):
|
||||
return web.Response(text="Tags must be a list", status=400)
|
||||
|
||||
tags = await self._tag_update_service.add_tags(
|
||||
tags, auto_tags = await self._tag_update_service.add_tags(
|
||||
file_path=file_path,
|
||||
new_tags=new_tags,
|
||||
metadata_loader=self._metadata_sync.load_local_metadata,
|
||||
update_cache=self._service.scanner.update_single_model_cache,
|
||||
)
|
||||
|
||||
return web.json_response({"success": True, "tags": tags})
|
||||
return web.json_response(
|
||||
{"success": True, "tags": tags, "auto_tags": auto_tags}
|
||||
)
|
||||
except Exception as exc:
|
||||
self._logger.error("Error adding tags: %s", exc, exc_info=True)
|
||||
return web.Response(text=str(exc), status=500)
|
||||
@@ -849,7 +959,7 @@ class ModelQueryHandler:
|
||||
async def get_base_models(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
limit = int(request.query.get("limit", "20"))
|
||||
if limit < 1 or limit > 100:
|
||||
if limit < 0 or limit > 100:
|
||||
limit = 20
|
||||
base_models = await self._service.get_base_models(limit)
|
||||
return web.json_response({"success": True, "base_models": base_models})
|
||||
@@ -1085,6 +1195,12 @@ class ModelQueryHandler:
|
||||
|
||||
async def find_filename_conflicts(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
settings = get_settings_manager()
|
||||
if settings.get("lora_syntax_format", "legacy") == "full":
|
||||
return web.json_response(
|
||||
{"success": True, "conflicts": [], "count": 0}
|
||||
)
|
||||
|
||||
duplicates = self._service.find_duplicate_filenames()
|
||||
result = []
|
||||
cache = await self._service.scanner.get_cached_data()
|
||||
@@ -1173,6 +1289,14 @@ class ModelQueryHandler:
|
||||
license_flags = (model_data or {}).get("license_flags")
|
||||
if license_flags is not None:
|
||||
response_payload["license_flags"] = int(license_flags)
|
||||
# Include the user's license icon style preference so the
|
||||
# ComfyUI tooltip can pick the right set without a separate
|
||||
# API call.
|
||||
try:
|
||||
settings = get_settings_manager()
|
||||
response_payload["use_new_license_icons"] = settings.get("use_new_license_icons", True)
|
||||
except Exception:
|
||||
pass
|
||||
return web.json_response(response_payload)
|
||||
return web.json_response(
|
||||
{
|
||||
@@ -1374,6 +1498,21 @@ class ModelDownloadHandler:
|
||||
)
|
||||
return web.Response(status=500, text=str(exc))
|
||||
|
||||
async def skip_download_get(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
download_id = request.query.get("download_id")
|
||||
if not download_id:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Download ID is required"}, status=400
|
||||
)
|
||||
result = await self._download_coordinator.skip_download(download_id)
|
||||
return web.json_response(result)
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error skipping download via GET: %s", exc, exc_info=True
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def cancel_download_get(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
download_id = request.query.get("download_id")
|
||||
@@ -1454,6 +1593,291 @@ class ModelDownloadHandler:
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Download queue / history handlers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def get_download_queue(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
service = await DownloadQueueService.get_instance()
|
||||
queue = await service.get_queue()
|
||||
stats = await service.get_stats()
|
||||
return web.json_response({"success": True, "queue": queue, "stats": stats})
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error getting download queue: %s", exc, exc_info=True
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def add_to_download_queue(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
import uuid
|
||||
|
||||
download_id = request.query.get("download_id") or str(uuid.uuid4())
|
||||
model_id_str = request.query.get("model_id")
|
||||
model_version_id_str = request.query.get("model_version_id")
|
||||
model_name = request.query.get("model_name", "")
|
||||
version_name = request.query.get("version_name", "")
|
||||
thumbnail_url = request.query.get("thumbnail_url", "")
|
||||
source = request.query.get("source")
|
||||
file_params_json = request.query.get("file_params")
|
||||
|
||||
model_id = int(model_id_str) if model_id_str else None
|
||||
model_version_id = int(model_version_id_str) if model_version_id_str else None
|
||||
file_params = json.loads(file_params_json) if file_params_json else None
|
||||
|
||||
service = await DownloadQueueService.get_instance()
|
||||
item = await service.add_to_queue(
|
||||
download_id=download_id,
|
||||
model_id=model_id,
|
||||
model_version_id=model_version_id,
|
||||
model_name=model_name,
|
||||
version_name=version_name,
|
||||
thumbnail_url=thumbnail_url,
|
||||
source=source,
|
||||
file_params=file_params,
|
||||
)
|
||||
return web.json_response({"success": True, "item": item})
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error adding to download queue: %s", exc, exc_info=True
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def remove_from_download_queue(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
download_id = request.query.get("download_id")
|
||||
if not download_id:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "download_id is required"}, status=400
|
||||
)
|
||||
|
||||
service = await DownloadQueueService.get_instance()
|
||||
removed = await service.remove_from_queue(download_id)
|
||||
return web.json_response({"success": removed})
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error removing from download queue: %s", exc, exc_info=True
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def move_queue_item_to_top(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
download_id = request.query.get("download_id")
|
||||
if not download_id:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "download_id is required"}, status=400
|
||||
)
|
||||
|
||||
service = await DownloadQueueService.get_instance()
|
||||
moved = await service.move_to_top(download_id)
|
||||
return web.json_response({"success": moved})
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error moving queue item to top: %s", exc, exc_info=True
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def move_queue_item_to_end(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
download_id = request.query.get("download_id")
|
||||
if not download_id:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "download_id is required"}, status=400
|
||||
)
|
||||
|
||||
service = await DownloadQueueService.get_instance()
|
||||
moved = await service.move_to_end(download_id)
|
||||
return web.json_response({"success": moved})
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error moving queue item to end: %s", exc, exc_info=True
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def clear_download_queue(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
status_filter = request.query.get("status") or None
|
||||
service = await DownloadQueueService.get_instance()
|
||||
cleared = await service.clear_queue(status_filter=status_filter)
|
||||
return web.json_response({"success": True, "cleared": cleared})
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error clearing download queue: %s", exc, exc_info=True
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def get_download_history(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
limit = min(int(request.query.get("limit", "50")), 500)
|
||||
offset = int(request.query.get("offset", "0"))
|
||||
status_filter = request.query.get("status") or None
|
||||
service = await DownloadQueueService.get_instance()
|
||||
result = await service.get_history(
|
||||
limit=limit, offset=offset, status_filter=status_filter
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"items": result["items"],
|
||||
"total": result["total"],
|
||||
"limit": result["limit"],
|
||||
"offset": result["offset"],
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error getting download history: %s", exc, exc_info=True
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def clear_download_history(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
status_filter = request.query.get("status") or None
|
||||
service = await DownloadQueueService.get_instance()
|
||||
cleared = await service.clear_history(status_filter=status_filter)
|
||||
return web.json_response({"success": True, "cleared": cleared})
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error clearing download history: %s", exc, exc_info=True
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def delete_download_history_item(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
item_id = int(request.query.get("id", "0"))
|
||||
if not item_id:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "id is required"}, status=400
|
||||
)
|
||||
|
||||
service = await DownloadQueueService.get_instance()
|
||||
deleted = await service.delete_history_item(item_id)
|
||||
return web.json_response({"success": deleted})
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error deleting download history item: %s", exc, exc_info=True
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def retry_download_from_history(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
item_id = int(request.query.get("id", "0"))
|
||||
if not item_id:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "id is required"}, status=400
|
||||
)
|
||||
|
||||
service = await DownloadQueueService.get_instance()
|
||||
item = await service.retry_from_history(item_id)
|
||||
if item is None:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "History item not found or not retryable"},
|
||||
status=404,
|
||||
)
|
||||
return web.json_response({"success": True, "item": item})
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error retrying download from history: %s", exc, exc_info=True
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def retry_all_failed_downloads(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
service = await DownloadQueueService.get_instance()
|
||||
retry_count = await service.retry_all_failed()
|
||||
return web.json_response({"success": True, "retry_count": retry_count})
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error retrying all failed downloads: %s", exc, exc_info=True
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def complete_download_in_queue(self, request: web.Request) -> web.Response:
|
||||
"""Atomically move a download from queue to history with terminal status."""
|
||||
try:
|
||||
download_id = request.query.get("download_id")
|
||||
if not download_id:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "download_id is required"}, status=400
|
||||
)
|
||||
status = request.query.get("status", "completed")
|
||||
error = request.query.get("error")
|
||||
file_path = request.query.get("file_path")
|
||||
try:
|
||||
bytes_downloaded = int(request.query.get("bytes_downloaded", "0"))
|
||||
except (TypeError, ValueError):
|
||||
bytes_downloaded = 0
|
||||
total_bytes_raw = request.query.get("total_bytes")
|
||||
total_bytes = int(total_bytes_raw) if total_bytes_raw else None
|
||||
completed_at_raw = request.query.get("completed_at")
|
||||
completed_at = float(completed_at_raw) if completed_at_raw else None
|
||||
|
||||
service = await DownloadQueueService.get_instance()
|
||||
item = await service.complete_download(
|
||||
download_id=download_id,
|
||||
status=status,
|
||||
error=error,
|
||||
file_path=file_path,
|
||||
bytes_downloaded=bytes_downloaded,
|
||||
total_bytes=total_bytes,
|
||||
completed_at=completed_at,
|
||||
)
|
||||
if item is None:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Download not found in queue"}, status=404
|
||||
)
|
||||
return web.json_response({"success": True, "item": item})
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error completing download: %s", exc, exc_info=True
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def get_download_stats(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
service = await DownloadQueueService.get_instance()
|
||||
stats = await service.get_stats()
|
||||
return web.json_response({"success": True, "stats": stats})
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error getting download stats: %s", exc, exc_info=True
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def update_download_queue_status(self, request: web.Request) -> web.Response:
|
||||
"""Update the status of a queue item (non-terminal transitions).
|
||||
|
||||
Supported transitions include ``queued → downloading``,
|
||||
``downloading → paused``, ``paused → downloading``, etc.
|
||||
Terminal transitions (``completed``, ``failed``, ``canceled``)
|
||||
should use ``complete_download_in_queue`` instead.
|
||||
"""
|
||||
try:
|
||||
download_id = request.query.get("download_id")
|
||||
status = request.query.get("status")
|
||||
if not download_id or not status:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "download_id and status are required",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
service = await DownloadQueueService.get_instance()
|
||||
updated = await service.update_status(download_id, status)
|
||||
if not updated:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Download not found in queue"},
|
||||
status=404,
|
||||
)
|
||||
return web.json_response({"success": True})
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error updating download queue status: %s", exc, exc_info=True
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
|
||||
class ModelCivitaiHandler:
|
||||
"""CivitAI integration endpoints."""
|
||||
@@ -1495,7 +1919,9 @@ class ModelCivitaiHandler:
|
||||
return web.json_response(result)
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error in fetch_all_civitai for %ss: %s", self._service.model_type, exc
|
||||
"Error in fetch_all_civitai for %ss: %s",
|
||||
self._service.model_type, exc,
|
||||
exc_info=True,
|
||||
)
|
||||
return web.Response(text=str(exc), status=500)
|
||||
|
||||
@@ -1522,6 +1948,20 @@ class ModelCivitaiHandler:
|
||||
|
||||
cache = await self._service.scanner.get_cached_data()
|
||||
version_index = cache.version_index
|
||||
downloaded_version_ids: set[int] = set()
|
||||
try:
|
||||
history_service = await ServiceRegistry.get_downloaded_version_history_service()
|
||||
downloaded_version_ids = set(
|
||||
await history_service.get_downloaded_version_ids(
|
||||
self._service.model_type,
|
||||
model_id,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - defensive logging
|
||||
self._logger.debug(
|
||||
"Failed to load download history for CivitAI versions: %s",
|
||||
exc,
|
||||
)
|
||||
|
||||
for version in versions:
|
||||
version_id = None
|
||||
@@ -1538,6 +1978,9 @@ class ModelCivitaiHandler:
|
||||
else None
|
||||
)
|
||||
version["existsLocally"] = cache_entry is not None
|
||||
version["hasBeenDownloaded"] = (
|
||||
version_id in downloaded_version_ids if version_id is not None else False
|
||||
)
|
||||
if cache_entry and isinstance(cache_entry, Mapping):
|
||||
local_path = cache_entry.get("file_path")
|
||||
if local_path:
|
||||
@@ -1780,6 +2223,11 @@ class ModelUpdateHandler:
|
||||
status=429,
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - defensive log
|
||||
if is_expected_offline_error(str(exc)):
|
||||
return web.json_response(
|
||||
{"success": False, "error": OFFLINE_FRIENDLY_MESSAGE},
|
||||
status=503,
|
||||
)
|
||||
self._logger.error("Failed to fetch license info: %s", exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
@@ -1840,6 +2288,10 @@ class ModelUpdateHandler:
|
||||
if target_model_ids:
|
||||
target_model_ids = sorted(set(target_model_ids))
|
||||
|
||||
folder_path: Optional[str] = payload.get("folder_path")
|
||||
if folder_path is not None and not isinstance(folder_path, str):
|
||||
folder_path = None
|
||||
|
||||
provider = await self._get_civitai_provider()
|
||||
if provider is None:
|
||||
return web.json_response(
|
||||
@@ -1854,6 +2306,7 @@ class ModelUpdateHandler:
|
||||
provider,
|
||||
force_refresh=force_refresh,
|
||||
target_model_ids=target_model_ids or None,
|
||||
folder_path=folder_path,
|
||||
)
|
||||
if self._service.scanner.is_cancelled():
|
||||
return web.json_response(
|
||||
@@ -1868,15 +2321,29 @@ class ModelUpdateHandler:
|
||||
{"success": False, "error": str(exc) or "Rate limited"}, status=429
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - defensive logging
|
||||
self._logger.error(
|
||||
"Failed to refresh model updates: %s", exc, exc_info=True
|
||||
)
|
||||
if is_expected_offline_error(str(exc)):
|
||||
return web.json_response(
|
||||
{"success": False, "error": OFFLINE_FRIENDLY_MESSAGE},
|
||||
status=503,
|
||||
)
|
||||
self._logger.error("Failed to refresh model updates: %s", exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
hide_early_access = False
|
||||
if self._settings is not None:
|
||||
try:
|
||||
hide_early_access = bool(
|
||||
self._settings.get("hide_early_access_updates", False)
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
serialized_records = []
|
||||
for record in records.values():
|
||||
has_update_fn = getattr(record, "has_update", None)
|
||||
if callable(has_update_fn) and has_update_fn():
|
||||
if callable(has_update_fn) and has_update_fn(
|
||||
hide_early_access=hide_early_access
|
||||
):
|
||||
serialized_records.append(self._serialize_record(record))
|
||||
|
||||
return web.json_response(
|
||||
@@ -2256,7 +2723,7 @@ class ModelUpdateHandler:
|
||||
self,
|
||||
record,
|
||||
*,
|
||||
version_context: Optional[Dict[int, Dict[str, Optional[str]]]] = None,
|
||||
version_context: Optional[Dict[int, Dict[str, Any]]] = None,
|
||||
) -> Dict:
|
||||
context = version_context or {}
|
||||
# Check user setting for hiding early access versions
|
||||
@@ -2285,7 +2752,7 @@ class ModelUpdateHandler:
|
||||
|
||||
@staticmethod
|
||||
def _serialize_version(
|
||||
version, context: Optional[Dict[str, Optional[str]]]
|
||||
version, context: Optional[Dict[str, Any]]
|
||||
) -> Dict:
|
||||
context = context or {}
|
||||
preview_override = context.get("preview_override")
|
||||
@@ -2319,17 +2786,42 @@ class ModelUpdateHandler:
|
||||
"sizeBytes": version.size_bytes,
|
||||
"previewUrl": preview_url,
|
||||
"isInLibrary": version.is_in_library,
|
||||
"hasBeenDownloaded": bool(context.get("has_been_downloaded", False)),
|
||||
"shouldIgnore": version.should_ignore,
|
||||
"earlyAccessEndsAt": version.early_access_ends_at,
|
||||
"isEarlyAccess": is_early_access,
|
||||
"usageControl": version.usage_control,
|
||||
"filePath": context.get("file_path"),
|
||||
"fileName": context.get("file_name"),
|
||||
}
|
||||
|
||||
async def _build_version_context(
|
||||
self, record
|
||||
) -> Dict[int, Dict[str, Optional[str]]]:
|
||||
context: Dict[int, Dict[str, Optional[str]]] = {}
|
||||
) -> Dict[int, Dict[str, Any]]:
|
||||
context: Dict[int, Dict[str, Any]] = {}
|
||||
downloaded_version_ids: set[int] = set()
|
||||
try:
|
||||
history_service = await ServiceRegistry.get_downloaded_version_history_service()
|
||||
downloaded_version_ids = set(
|
||||
await history_service.get_downloaded_version_ids(
|
||||
record.model_type,
|
||||
record.model_id,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - defensive logging
|
||||
self._logger.debug(
|
||||
"Failed to load download history while building version context: %s",
|
||||
exc,
|
||||
)
|
||||
|
||||
for version in record.versions:
|
||||
context[version.version_id] = {
|
||||
"file_path": None,
|
||||
"file_name": None,
|
||||
"preview_override": None,
|
||||
"has_been_downloaded": version.version_id in downloaded_version_ids,
|
||||
}
|
||||
|
||||
try:
|
||||
cache = await self._service.scanner.get_cached_data()
|
||||
except Exception as exc: # pragma: no cover - defensive logging
|
||||
@@ -2348,16 +2840,21 @@ class ModelUpdateHandler:
|
||||
cache_entry = version_index.get(version.version_id)
|
||||
if isinstance(cache_entry, Mapping):
|
||||
preview = cache_entry.get("preview_url")
|
||||
context_entry: Dict[str, Optional[str]] = {
|
||||
"file_path": cache_entry.get("file_path"),
|
||||
"file_name": cache_entry.get("file_name"),
|
||||
"preview_override": None,
|
||||
}
|
||||
context_entry = context.setdefault(
|
||||
version.version_id,
|
||||
{
|
||||
"file_path": None,
|
||||
"file_name": None,
|
||||
"preview_override": None,
|
||||
"has_been_downloaded": version.version_id in downloaded_version_ids,
|
||||
},
|
||||
)
|
||||
context_entry["file_path"] = cache_entry.get("file_path")
|
||||
context_entry["file_name"] = cache_entry.get("file_name")
|
||||
if isinstance(preview, str) and preview:
|
||||
context_entry["preview_override"] = config.get_preview_static_url(
|
||||
preview
|
||||
)
|
||||
context[version.version_id] = context_entry
|
||||
return context
|
||||
|
||||
|
||||
@@ -2381,8 +2878,10 @@ class ModelHandlerSet:
|
||||
return {
|
||||
"handle_models_page": self.page_view.handle,
|
||||
"get_models": self.listing.get_models,
|
||||
"get_excluded_models": self.listing.get_excluded_models,
|
||||
"delete_model": self.management.delete_model,
|
||||
"exclude_model": self.management.exclude_model,
|
||||
"unexclude_model": self.management.unexclude_model,
|
||||
"fetch_civitai": self.management.fetch_civitai,
|
||||
"fetch_all_civitai": self.civitai.fetch_all_civitai,
|
||||
"relink_civitai": self.management.relink_civitai,
|
||||
@@ -2406,9 +2905,24 @@ class ModelHandlerSet:
|
||||
"download_model": self.download.download_model,
|
||||
"download_model_get": self.download.download_model_get,
|
||||
"cancel_download_get": self.download.cancel_download_get,
|
||||
"skip_download_get": self.download.skip_download_get,
|
||||
"pause_download_get": self.download.pause_download_get,
|
||||
"resume_download_get": self.download.resume_download_get,
|
||||
"get_download_progress": self.download.get_download_progress,
|
||||
"get_download_queue": self.download.get_download_queue,
|
||||
"add_to_download_queue": self.download.add_to_download_queue,
|
||||
"remove_from_download_queue": self.download.remove_from_download_queue,
|
||||
"move_queue_item_to_top": self.download.move_queue_item_to_top,
|
||||
"move_queue_item_to_end": self.download.move_queue_item_to_end,
|
||||
"clear_download_queue": self.download.clear_download_queue,
|
||||
"get_download_history": self.download.get_download_history,
|
||||
"clear_download_history": self.download.clear_download_history,
|
||||
"delete_download_history_item": self.download.delete_download_history_item,
|
||||
"retry_download_from_history": self.download.retry_download_from_history,
|
||||
"retry_all_failed_downloads": self.download.retry_all_failed_downloads,
|
||||
"complete_download_in_queue": self.download.complete_download_in_queue,
|
||||
"get_download_stats": self.download.get_download_stats,
|
||||
"update_download_queue_status": self.download.update_download_queue_status,
|
||||
"get_civitai_versions": self.civitai.get_civitai_versions,
|
||||
"get_civitai_model_by_version": self.civitai.get_civitai_model_by_version,
|
||||
"get_civitai_model_by_hash": self.civitai.get_civitai_model_by_hash,
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import mimetypes
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
@@ -12,6 +13,12 @@ from ...config import config as global_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CHUNK_SIZE = 1024 * 1024 # 1 MB — balance between streaming iteration overhead and per-chunk memory
|
||||
|
||||
# Video file extensions that bypass native sendfile on Windows
|
||||
# to avoid IOCP/ProactorEventLoop crashes during client disconnect.
|
||||
_VIDEO_EXTENSIONS = frozenset({".mp4", ".webm", ".mov", ".avi", ".mkv"})
|
||||
|
||||
|
||||
class PreviewHandler:
|
||||
"""Serve preview assets for the active library at request time."""
|
||||
@@ -48,8 +55,58 @@ class PreviewHandler:
|
||||
logger.debug("Preview file not found at %s", str(resolved))
|
||||
raise web.HTTPNotFound(text="Preview file not found")
|
||||
|
||||
# aiohttp's FileResponse handles range requests and content headers for us.
|
||||
return web.FileResponse(path=resolved, chunk_size=256 * 1024)
|
||||
# aiohttp's FileResponse handles range requests, content headers, and
|
||||
# uses kernel sendfile (zero-copy DMA) on Linux/macOS. On Windows it
|
||||
# uses IOCP-based _sendfile_native which can crash when the client
|
||||
# disconnects mid-transfer during fast scrolling. The _stream_file()
|
||||
# fallback is kept for a future compat toggle.
|
||||
#
|
||||
# Set explicit Cache-Control so the browser can cache video (and image)
|
||||
# previews across VirtualScroller recycling cycles. Without this,
|
||||
# Chrome does not cache 206 Partial Content responses for <video>
|
||||
# elements, causing the same video to be re-downloaded on every scroll.
|
||||
resp = web.FileResponse(path=resolved, chunk_size=_CHUNK_SIZE)
|
||||
resp.headers["Cache-Control"] = "public, max-age=86400"
|
||||
return resp
|
||||
|
||||
async def _stream_file(
|
||||
self, request: web.Request, path: Path
|
||||
) -> web.StreamResponse:
|
||||
"""Stream a file chunk-by-chunk, bypassing native sendfile.
|
||||
|
||||
This avoids the Windows IOCP ``_sendfile_native`` crash that occurs
|
||||
when the client disconnects during a large file transfer.
|
||||
"""
|
||||
content_type, _ = mimetypes.guess_type(str(path))
|
||||
if content_type is None:
|
||||
content_type = "application/octet-stream"
|
||||
|
||||
file_size = path.stat().st_size
|
||||
resp = web.StreamResponse()
|
||||
resp.content_type = content_type
|
||||
resp.content_length = file_size
|
||||
|
||||
# Allow browser caching: video previews rarely change during a session.
|
||||
# The frontend already appends ?t={version} to bust cache on update.
|
||||
resp.headers["Cache-Control"] = "public, max-age=86400"
|
||||
|
||||
await resp.prepare(request)
|
||||
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
while True:
|
||||
chunk = f.read(_CHUNK_SIZE)
|
||||
if not chunk:
|
||||
break
|
||||
await resp.write(chunk)
|
||||
except (ConnectionResetError, ConnectionAbortedError):
|
||||
# Client disconnected during streaming — expected when scrolling
|
||||
# rapidly through a library with animated previews.
|
||||
pass
|
||||
except OSError as exc:
|
||||
logger.debug("I/O error streaming preview %s: %s", path, exc)
|
||||
|
||||
return resp
|
||||
|
||||
|
||||
__all__ = ["PreviewHandler"]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -22,11 +22,17 @@ class RouteDefinition:
|
||||
MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("GET", "/api/lm/settings", "get_settings"),
|
||||
RouteDefinition("POST", "/api/lm/settings", "update_settings"),
|
||||
RouteDefinition("GET", "/api/lm/doctor/diagnostics", "get_doctor_diagnostics"),
|
||||
RouteDefinition("POST", "/api/lm/doctor/repair-cache", "repair_doctor_cache"),
|
||||
RouteDefinition("POST", "/api/lm/doctor/resolve-filename-conflicts", "resolve_doctor_filename_conflicts"),
|
||||
RouteDefinition("POST", "/api/lm/doctor/export-bundle", "export_doctor_bundle"),
|
||||
RouteDefinition("GET", "/api/lm/priority-tags", "get_priority_tags"),
|
||||
RouteDefinition("GET", "/api/lm/settings/libraries", "get_settings_libraries"),
|
||||
RouteDefinition("POST", "/api/lm/settings/libraries/activate", "activate_library"),
|
||||
RouteDefinition("GET", "/api/lm/health-check", "health_check"),
|
||||
RouteDefinition("GET", "/api/lm/supporters", "get_supporters"),
|
||||
RouteDefinition("GET", "/api/lm/wildcards/search", "search_wildcards"),
|
||||
RouteDefinition("POST", "/api/lm/wildcards/open-location", "open_wildcards_location"),
|
||||
RouteDefinition("POST", "/api/lm/open-file-location", "open_file_location"),
|
||||
RouteDefinition("POST", "/api/lm/update-usage-stats", "update_usage_stats"),
|
||||
RouteDefinition("GET", "/api/lm/get-usage-stats", "get_usage_stats"),
|
||||
@@ -37,6 +43,22 @@ MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("POST", "/api/lm/update-node-widget", "update_node_widget"),
|
||||
RouteDefinition("GET", "/api/lm/get-registry", "get_registry"),
|
||||
RouteDefinition("GET", "/api/lm/check-model-exists", "check_model_exists"),
|
||||
RouteDefinition("GET", "/api/lm/check-models-exist", "check_models_exist"),
|
||||
RouteDefinition(
|
||||
"GET",
|
||||
"/api/lm/model-version-download-status",
|
||||
"get_model_version_download_status",
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST",
|
||||
"/api/lm/model-version-download-status",
|
||||
"set_model_version_download_status",
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET",
|
||||
"/api/lm/set-model-version-download-status",
|
||||
"set_model_version_download_status",
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/civitai/user-models", "get_civitai_user_models"),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/download-metadata-archive", "download_metadata_archive"
|
||||
@@ -47,6 +69,10 @@ MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/metadata-archive-status", "get_metadata_archive_status"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/backup/status", "get_backup_status"),
|
||||
RouteDefinition("POST", "/api/lm/backup/export", "export_backup"),
|
||||
RouteDefinition("POST", "/api/lm/backup/import", "import_backup"),
|
||||
RouteDefinition("POST", "/api/lm/backup/open-location", "open_backup_location"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/model-versions-status", "get_model_versions_status"
|
||||
),
|
||||
@@ -56,6 +82,18 @@ MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/example-workflows/{filename}", "get_example_workflow"
|
||||
),
|
||||
# Base model management routes
|
||||
RouteDefinition("GET", "/api/lm/base-models", "get_base_models"),
|
||||
RouteDefinition("POST", "/api/lm/base-models/refresh", "refresh_base_models"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/base-models/categories", "get_base_model_categories"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/base-models/cache-status", "get_base_model_cache_status"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/delete-model-version", "delete_model_version"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -19,10 +19,12 @@ from ..services.downloader import get_downloader
|
||||
from ..utils.usage_stats import UsageStats
|
||||
from .handlers.misc_handlers import (
|
||||
CustomWordsHandler,
|
||||
DoctorHandler,
|
||||
ExampleWorkflowsHandler,
|
||||
FileSystemHandler,
|
||||
HealthCheckHandler,
|
||||
LoraCodeHandler,
|
||||
BackupHandler,
|
||||
MetadataArchiveHandler,
|
||||
MiscHandlerSet,
|
||||
ModelExampleFilesHandler,
|
||||
@@ -33,8 +35,10 @@ from .handlers.misc_handlers import (
|
||||
SupportersHandler,
|
||||
TrainedWordsHandler,
|
||||
UsageStatsHandler,
|
||||
WildcardsHandler,
|
||||
build_service_registry_adapter,
|
||||
)
|
||||
from .handlers.base_model_handlers import BaseModelHandlerSet
|
||||
from .misc_route_registrar import MiscRouteRegistrar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -115,6 +119,7 @@ class MiscRoutes:
|
||||
settings_service=self._settings,
|
||||
metadata_provider_updater=self._metadata_provider_updater,
|
||||
)
|
||||
backup = BackupHandler()
|
||||
filesystem = FileSystemHandler(settings_service=self._settings)
|
||||
node_registry_handler = NodeRegistryHandler(
|
||||
node_registry=self._node_registry,
|
||||
@@ -126,8 +131,11 @@ class MiscRoutes:
|
||||
metadata_provider_factory=self._metadata_provider_factory,
|
||||
)
|
||||
custom_words = CustomWordsHandler()
|
||||
wildcards = WildcardsHandler()
|
||||
supporters = SupportersHandler()
|
||||
doctor = DoctorHandler(settings_service=self._settings)
|
||||
example_workflows = ExampleWorkflowsHandler()
|
||||
base_model = BaseModelHandlerSet()
|
||||
|
||||
return self._handler_set_factory(
|
||||
health=health,
|
||||
@@ -139,10 +147,14 @@ class MiscRoutes:
|
||||
node_registry=node_registry_handler,
|
||||
model_library=model_library,
|
||||
metadata_archive=metadata_archive,
|
||||
backup=backup,
|
||||
filesystem=filesystem,
|
||||
custom_words=custom_words,
|
||||
wildcards=wildcards,
|
||||
supporters=supporters,
|
||||
doctor=doctor,
|
||||
example_workflows=example_workflows,
|
||||
base_model=base_model,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -22,8 +22,10 @@ class RouteDefinition:
|
||||
|
||||
COMMON_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/list", "get_models"),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/excluded", "get_excluded_models"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/delete", "delete_model"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/exclude", "exclude_model"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/unexclude", "unexclude_model"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/fetch-civitai", "fetch_civitai"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/fetch-all-civitai", "fetch_all_civitai"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/relink-civitai", "relink_civitai"),
|
||||
@@ -99,11 +101,46 @@ COMMON_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("POST", "/api/lm/download-model", "download_model"),
|
||||
RouteDefinition("GET", "/api/lm/download-model-get", "download_model_get"),
|
||||
RouteDefinition("GET", "/api/lm/cancel-download-get", "cancel_download_get"),
|
||||
RouteDefinition("GET", "/api/lm/skip-download", "skip_download_get"),
|
||||
RouteDefinition("GET", "/api/lm/pause-download", "pause_download_get"),
|
||||
RouteDefinition("GET", "/api/lm/resume-download", "resume_download_get"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/download-progress/{download_id}", "get_download_progress"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/downloads/queue", "get_download_queue"),
|
||||
RouteDefinition("GET", "/api/lm/downloads/queue/add", "add_to_download_queue"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/queue/remove", "remove_from_download_queue"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/queue/move-to-top", "move_queue_item_to_top"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/queue/move-to-end", "move_queue_item_to_end"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/queue/clear", "clear_download_queue"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/downloads/history", "get_download_history"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/history/clear", "clear_download_history"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/history/delete", "delete_download_history_item"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/history/retry", "retry_download_from_history"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/history/retry-all", "retry_all_failed_downloads"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/downloads/stats", "get_download_stats"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/queue/complete", "complete_download_in_queue"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/queue/status", "update_download_queue_status"
|
||||
),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/cancel-task", "cancel_task"),
|
||||
RouteDefinition("GET", "/{prefix}", "handle_models_page"),
|
||||
)
|
||||
|
||||
@@ -51,10 +51,14 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
"POST", "/api/lm/recipes/save-from-widget", "save_recipe_from_widget"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/recipes/for-lora", "get_recipes_for_lora"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/recipes/for-checkpoint", "get_recipes_for_checkpoint"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/recipes/scan", "scan_recipes"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/repair", "repair_recipes"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/cancel-repair", "cancel_repair"),
|
||||
RouteDefinition("POST", "/api/lm/recipe/{recipe_id}/repair", "repair_recipe"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/repair-bulk", "repair_recipes_bulk"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/repair-progress", "get_repair_progress"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/batch-import/start", "start_batch_import"),
|
||||
RouteDefinition(
|
||||
@@ -67,6 +71,16 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
"POST", "/api/lm/recipes/batch-import/directory", "start_directory_import"
|
||||
),
|
||||
RouteDefinition("POST", "/api/lm/recipes/browse-directory", "browse_directory"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/recipes/check-image-exists", "check_image_exists"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/recipes/import-from-url", "import_from_url"),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/recipes/create-from-example", "create_from_example"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/recipe/{recipe_id}/reimport", "reimport_recipe"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -11,6 +11,8 @@ from ..config import config
|
||||
from ..services.settings_manager import get_settings_manager
|
||||
from ..services.server_i18n import server_i18n
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
from ..services.model_query import normalize_sub_type, resolve_sub_type
|
||||
from ..utils.constants import VALID_LORA_SUB_TYPES, VALID_CHECKPOINT_SUB_TYPES
|
||||
from ..utils.usage_stats import UsageStats
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -140,6 +142,21 @@ class StatsRoutes:
|
||||
# Get usage statistics
|
||||
usage_data = await self.usage_stats.get_stats()
|
||||
|
||||
# CivitAI model type distribution across all model types
|
||||
# Use the same logic as the filter panel: normalize_sub_type(resolve_sub_type(entry))
|
||||
# with sub-type validation per model type
|
||||
model_types_counter: Counter[str] = Counter()
|
||||
for entry in lora_cache.raw_data:
|
||||
ntype = normalize_sub_type(resolve_sub_type(entry))
|
||||
if ntype and ntype in VALID_LORA_SUB_TYPES:
|
||||
model_types_counter[ntype] += 1
|
||||
for entry in checkpoint_cache.raw_data:
|
||||
ntype = normalize_sub_type(resolve_sub_type(entry))
|
||||
if ntype and ntype in VALID_CHECKPOINT_SUB_TYPES:
|
||||
model_types_counter[ntype] += 1
|
||||
# Embeddings: always count as "embedding" regardless of CivitAI sub-type
|
||||
model_types_counter['embedding'] = len(embedding_cache.raw_data)
|
||||
|
||||
return web.json_response({
|
||||
'success': True,
|
||||
'data': {
|
||||
@@ -154,7 +171,8 @@ class StatsRoutes:
|
||||
'total_generations': usage_data.get('total_executions', 0),
|
||||
'unused_loras': self._count_unused_models(lora_cache.raw_data, usage_data.get('loras', {})),
|
||||
'unused_checkpoints': self._count_unused_models(checkpoint_cache.raw_data, usage_data.get('checkpoints', {})),
|
||||
'unused_embeddings': self._count_unused_models(embedding_cache.raw_data, usage_data.get('embeddings', {}))
|
||||
'unused_embeddings': self._count_unused_models(embedding_cache.raw_data, usage_data.get('embeddings', {})),
|
||||
'model_types_distribution': dict(model_types_counter.most_common())
|
||||
}
|
||||
})
|
||||
|
||||
@@ -459,9 +477,12 @@ class StatsRoutes:
|
||||
if unused_lora_percent > 50:
|
||||
insights.append({
|
||||
'type': 'warning',
|
||||
'title': 'High Number of Unused LoRAs',
|
||||
'description': f'{unused_lora_percent:.1f}% of your LoRAs ({unused_loras}/{total_loras}) have never been used.',
|
||||
'suggestion': 'Consider organizing or archiving unused models to free up storage space.'
|
||||
'key': 'insights.unusedLoras.high',
|
||||
'params': {
|
||||
'percent': f'{unused_lora_percent:.1f}',
|
||||
'count': str(unused_loras),
|
||||
'total': str(total_loras)
|
||||
}
|
||||
})
|
||||
|
||||
if total_checkpoints > 0:
|
||||
@@ -469,9 +490,12 @@ class StatsRoutes:
|
||||
if unused_checkpoint_percent > 30:
|
||||
insights.append({
|
||||
'type': 'warning',
|
||||
'title': 'Unused Checkpoints Detected',
|
||||
'description': f'{unused_checkpoint_percent:.1f}% of your checkpoints ({unused_checkpoints}/{total_checkpoints}) have never been used.',
|
||||
'suggestion': 'Review and consider removing checkpoints you no longer need.'
|
||||
'key': 'insights.unusedCheckpoints.detected',
|
||||
'params': {
|
||||
'percent': f'{unused_checkpoint_percent:.1f}',
|
||||
'count': str(unused_checkpoints),
|
||||
'total': str(total_checkpoints)
|
||||
}
|
||||
})
|
||||
|
||||
if total_embeddings > 0:
|
||||
@@ -479,9 +503,12 @@ class StatsRoutes:
|
||||
if unused_embedding_percent > 50:
|
||||
insights.append({
|
||||
'type': 'warning',
|
||||
'title': 'High Number of Unused Embeddings',
|
||||
'description': f'{unused_embedding_percent:.1f}% of your embeddings ({unused_embeddings}/{total_embeddings}) have never been used.',
|
||||
'suggestion': 'Consider organizing or archiving unused embeddings to optimize your collection.'
|
||||
'key': 'insights.unusedEmbeddings.high',
|
||||
'params': {
|
||||
'percent': f'{unused_embedding_percent:.1f}',
|
||||
'count': str(unused_embeddings),
|
||||
'total': str(total_embeddings)
|
||||
}
|
||||
})
|
||||
|
||||
# Storage insights
|
||||
@@ -492,18 +519,20 @@ class StatsRoutes:
|
||||
if total_size > 100 * 1024 * 1024 * 1024: # 100GB
|
||||
insights.append({
|
||||
'type': 'info',
|
||||
'title': 'Large Collection Detected',
|
||||
'description': f'Your model collection is using {self._format_size(total_size)} of storage.',
|
||||
'suggestion': 'Consider using external storage or cloud solutions for better organization.'
|
||||
'key': 'insights.collection.large',
|
||||
'params': {
|
||||
'size': self._format_size(total_size)
|
||||
}
|
||||
})
|
||||
|
||||
# Recent activity insight
|
||||
if usage_data.get('total_executions', 0) > 100:
|
||||
insights.append({
|
||||
'type': 'success',
|
||||
'title': 'Active User',
|
||||
'description': f'You\'ve completed {usage_data["total_executions"]} generations so far!',
|
||||
'suggestion': 'Keep exploring and creating amazing content with your models.'
|
||||
'key': 'insights.activity.active',
|
||||
'params': {
|
||||
'count': str(usage_data['total_executions'])
|
||||
}
|
||||
})
|
||||
|
||||
return web.json_response({
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import os
|
||||
import logging
|
||||
import toml
|
||||
import git
|
||||
import zipfile
|
||||
import shutil
|
||||
import tempfile
|
||||
@@ -11,6 +10,7 @@ from typing import Dict, List
|
||||
|
||||
from ..utils.settings_paths import ensure_settings_file
|
||||
from ..services.downloader import get_downloader
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -212,8 +212,19 @@ class UpdateRoutes:
|
||||
|
||||
zip_path = tmp_zip_path
|
||||
|
||||
# Skip both settings.json, civitai and model cache folder
|
||||
UpdateRoutes._clean_plugin_folder(plugin_root, skip_files=['settings.json', 'civitai', 'model_cache'])
|
||||
# Close the downloaded-versions SQLite connection before cleaning,
|
||||
# so that shutil.rmtree() does not fail on Windows (the process
|
||||
# cannot delete a file with an outstanding open handle).
|
||||
try:
|
||||
history_svc = ServiceRegistry._services.get("downloaded_version_history_service")
|
||||
if history_svc is not None:
|
||||
history_svc.close()
|
||||
logger.info("Closed downloaded-version history database connection")
|
||||
except Exception:
|
||||
logger.debug("Could not close downloaded-version history database", exc_info=True)
|
||||
|
||||
# Skip settings.json, civitai, model cache and runtime cache folders
|
||||
UpdateRoutes._clean_plugin_folder(plugin_root, skip_files=['settings.json', 'civitai', 'model_cache', 'cache', 'wildcards', 'backups', 'stats'])
|
||||
|
||||
# Extract ZIP to temp dir
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
@@ -222,16 +233,17 @@ class UpdateRoutes:
|
||||
# Find extracted folder (GitHub ZIP contains a root folder)
|
||||
extracted_root = next(os.scandir(tmp_dir)).path
|
||||
|
||||
# Copy files, skipping settings.json and civitai folder
|
||||
# Copy files, skipping user data that should be preserved
|
||||
skip_items = {'settings.json', 'civitai', 'wildcards', 'backups', 'stats'}
|
||||
for item in os.listdir(extracted_root):
|
||||
if item == 'settings.json' or item == 'civitai':
|
||||
if item in skip_items:
|
||||
continue
|
||||
src = os.path.join(extracted_root, item)
|
||||
dst = os.path.join(plugin_root, item)
|
||||
if os.path.isdir(src):
|
||||
if os.path.exists(dst):
|
||||
shutil.rmtree(dst)
|
||||
shutil.copytree(src, dst, ignore=shutil.ignore_patterns('settings.json', 'civitai'))
|
||||
shutil.copytree(src, dst, ignore=shutil.ignore_patterns(*skip_items))
|
||||
else:
|
||||
shutil.copy2(src, dst)
|
||||
|
||||
@@ -239,15 +251,17 @@ class UpdateRoutes:
|
||||
# for ComfyUI Manager to work properly
|
||||
tracking_info_file = os.path.join(plugin_root, '.tracking')
|
||||
tracking_files = []
|
||||
skip_tracked = {'civitai', 'wildcards', 'backups', 'stats'}
|
||||
for root, dirs, files in os.walk(extracted_root):
|
||||
# Skip civitai folder and its contents
|
||||
# Skip user data directories and their contents
|
||||
rel_root = os.path.relpath(root, extracted_root)
|
||||
if rel_root == 'civitai' or rel_root.startswith('civitai' + os.sep):
|
||||
top_dir = rel_root.split(os.sep)[0] if rel_root != '.' else ''
|
||||
if top_dir in skip_tracked:
|
||||
continue
|
||||
for file in files:
|
||||
rel_path = os.path.relpath(os.path.join(root, file), extracted_root)
|
||||
# Skip settings.json and any file under civitai
|
||||
if rel_path == 'settings.json' or rel_path.startswith('civitai' + os.sep):
|
||||
# Skip settings.json and any file under user data dirs
|
||||
if rel_path == 'settings.json' or rel_path.split(os.sep)[0] in skip_tracked:
|
||||
continue
|
||||
tracking_files.append(rel_path.replace("\\", "/"))
|
||||
with open(tracking_info_file, "w", encoding='utf-8') as file:
|
||||
@@ -342,6 +356,15 @@ class UpdateRoutes:
|
||||
Returns:
|
||||
tuple: (success, new_version)
|
||||
"""
|
||||
try:
|
||||
import git
|
||||
except ImportError:
|
||||
logger.error(
|
||||
"GitPython is not available: the git executable was not found in PATH. "
|
||||
"Install git or set $GIT_PYTHON_GIT_EXECUTABLE to the git binary path."
|
||||
)
|
||||
return False, ""
|
||||
|
||||
try:
|
||||
# Open the Git repository
|
||||
repo = git.Repo(plugin_root)
|
||||
@@ -438,6 +461,7 @@ class UpdateRoutes:
|
||||
if not os.path.exists(os.path.join(plugin_root, '.git')):
|
||||
return git_info
|
||||
|
||||
import git
|
||||
repo = git.Repo(plugin_root)
|
||||
commit = repo.head.commit
|
||||
git_info['commit_hash'] = commit.hexsha
|
||||
|
||||
602
py/services/aria2_downloader.py
Normal file
602
py/services/aria2_downloader.py
Normal file
@@ -0,0 +1,602 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
import shutil
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import aiohttp
|
||||
|
||||
from .downloader import DownloadProgress, get_downloader, is_ssl_cert_verify_error
|
||||
from .aria2_transfer_state import Aria2TransferStateStore
|
||||
from .settings_manager import get_settings_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def _try_certifi_ca_path() -> str | None:
|
||||
"""Return the certifi CA bundle path if available, else None."""
|
||||
try:
|
||||
import certifi # type: ignore[import-untyped]
|
||||
|
||||
path = certifi.where()
|
||||
if os.path.isfile(path):
|
||||
logger.debug(
|
||||
"aria2 --ca-certificate: using certifi CA bundle at %s", path
|
||||
)
|
||||
return path
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
logger.debug("aria2 --ca-certificate: certifi not available")
|
||||
return None
|
||||
|
||||
|
||||
CIVITAI_DOWNLOAD_URL_PREFIXES = (
|
||||
"https://civitai.com/api/download/",
|
||||
"https://civitai.red/api/download/",
|
||||
)
|
||||
|
||||
|
||||
class Aria2Error(RuntimeError):
|
||||
"""Raised when aria2 integration fails."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Aria2Transfer:
|
||||
"""Track an aria2 download registered by the Python coordinator."""
|
||||
|
||||
gid: str
|
||||
save_path: str
|
||||
|
||||
|
||||
class Aria2Downloader:
|
||||
"""Manage an aria2 RPC daemon for recommended model downloads."""
|
||||
|
||||
_instance = None
|
||||
_lock = asyncio.Lock()
|
||||
|
||||
@classmethod
|
||||
async def get_instance(cls) -> "Aria2Downloader":
|
||||
async with cls._lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = cls()
|
||||
return cls._instance
|
||||
|
||||
def __init__(self) -> None:
|
||||
if hasattr(self, "_initialized"):
|
||||
return
|
||||
|
||||
self._initialized = True
|
||||
self._process: Optional[asyncio.subprocess.Process] = None
|
||||
self._rpc_port: Optional[int] = None
|
||||
self._rpc_secret = ""
|
||||
self._rpc_url = ""
|
||||
self._rpc_session: Optional[aiohttp.ClientSession] = None
|
||||
self._rpc_session_lock = asyncio.Lock()
|
||||
self._process_lock = asyncio.Lock()
|
||||
self._transfers: Dict[str, Aria2Transfer] = {}
|
||||
self._poll_interval = 0.5
|
||||
self._state_store = Aria2TransferStateStore()
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
return self._process is not None and self._process.returncode is None
|
||||
|
||||
async def download_file(
|
||||
self,
|
||||
url: str,
|
||||
save_path: str,
|
||||
*,
|
||||
download_id: str,
|
||||
progress_callback=None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> Tuple[bool, str]:
|
||||
"""Download a file using aria2 RPC and wait for completion."""
|
||||
|
||||
await self._ensure_process()
|
||||
save_path = os.path.abspath(save_path)
|
||||
transfer = self._transfers.get(download_id)
|
||||
if transfer is None or os.path.abspath(transfer.save_path) != save_path:
|
||||
gid = await self._schedule_download(
|
||||
url,
|
||||
save_path,
|
||||
download_id=download_id,
|
||||
headers=headers,
|
||||
)
|
||||
transfer = Aria2Transfer(gid=gid, save_path=save_path)
|
||||
self._transfers[download_id] = transfer
|
||||
|
||||
try:
|
||||
while True:
|
||||
status = await self.get_status(download_id)
|
||||
if status is None:
|
||||
return False, "aria2 download not found"
|
||||
|
||||
snapshot = self._build_progress_snapshot(status)
|
||||
if progress_callback is not None:
|
||||
await self._dispatch_progress(progress_callback, snapshot)
|
||||
|
||||
state = status.get("status", "")
|
||||
if state == "complete":
|
||||
completed_path = self._resolve_completed_path(status, save_path)
|
||||
return True, completed_path
|
||||
if state == "error":
|
||||
return False, status.get("errorMessage") or "aria2 download failed"
|
||||
if state == "removed":
|
||||
return False, "Download was cancelled"
|
||||
|
||||
await asyncio.sleep(self._poll_interval)
|
||||
finally:
|
||||
self._transfers.pop(download_id, None)
|
||||
|
||||
async def _schedule_download(
|
||||
self,
|
||||
url: str,
|
||||
save_path: str,
|
||||
*,
|
||||
download_id: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> str:
|
||||
save_dir = os.path.dirname(save_path)
|
||||
out_name = os.path.basename(save_path)
|
||||
|
||||
Path(save_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
resolved_url = url
|
||||
request_headers = headers
|
||||
if headers and url.startswith(CIVITAI_DOWNLOAD_URL_PREFIXES):
|
||||
resolved_url = await self._resolve_authenticated_redirect_url(url, headers)
|
||||
if resolved_url != url:
|
||||
request_headers = None
|
||||
logger.debug(
|
||||
"Resolved Civitai download %s to signed URL for aria2",
|
||||
download_id,
|
||||
)
|
||||
|
||||
options: Dict[str, str] = {
|
||||
"dir": save_dir,
|
||||
"out": out_name,
|
||||
"continue": "true",
|
||||
"max-connection-per-server": "4",
|
||||
"split": "4",
|
||||
"min-split-size": "1M",
|
||||
"allow-overwrite": "true",
|
||||
"auto-file-renaming": "false",
|
||||
"file-allocation": "none",
|
||||
}
|
||||
if request_headers:
|
||||
options["header"] = [
|
||||
f"{key}: {value}" for key, value in request_headers.items()
|
||||
]
|
||||
|
||||
logger.debug(
|
||||
"Submitting aria2 download %s -> %s (auth=%s, civitai_signed=%s)",
|
||||
download_id,
|
||||
save_path,
|
||||
bool(request_headers),
|
||||
resolved_url != url,
|
||||
)
|
||||
|
||||
try:
|
||||
gid = await self._rpc_call("aria2.addUri", [[resolved_url], options])
|
||||
except Exception as exc:
|
||||
raise Aria2Error(f"Failed to schedule aria2 download: {exc}") from exc
|
||||
|
||||
logger.debug("aria2 accepted download %s with gid %s", download_id, gid)
|
||||
await self._state_store.upsert(
|
||||
download_id,
|
||||
{
|
||||
"gid": gid,
|
||||
"save_path": save_path,
|
||||
"status": "downloading",
|
||||
"url": url,
|
||||
},
|
||||
)
|
||||
return gid
|
||||
|
||||
async def get_status(self, download_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Return the raw aria2 status payload for a known download."""
|
||||
|
||||
transfer = self._transfers.get(download_id)
|
||||
if transfer is None:
|
||||
return None
|
||||
|
||||
keys = [
|
||||
"gid",
|
||||
"status",
|
||||
"totalLength",
|
||||
"completedLength",
|
||||
"downloadSpeed",
|
||||
"errorMessage",
|
||||
"files",
|
||||
]
|
||||
try:
|
||||
status = await self._rpc_call("aria2.tellStatus", [transfer.gid, keys])
|
||||
except Exception as exc:
|
||||
raise Aria2Error(f"Failed to query aria2 download status: {exc}") from exc
|
||||
|
||||
if isinstance(status, dict):
|
||||
return status
|
||||
return None
|
||||
|
||||
async def get_status_by_gid(self, gid: str) -> Optional[Dict[str, Any]]:
|
||||
keys = [
|
||||
"gid",
|
||||
"status",
|
||||
"totalLength",
|
||||
"completedLength",
|
||||
"downloadSpeed",
|
||||
"errorMessage",
|
||||
"files",
|
||||
]
|
||||
try:
|
||||
status = await self._rpc_call("aria2.tellStatus", [gid, keys])
|
||||
except Exception as exc:
|
||||
message = str(exc)
|
||||
if "cannot be found" in message.lower() or "not found" in message.lower():
|
||||
return None
|
||||
raise Aria2Error(f"Failed to query aria2 download status: {exc}") from exc
|
||||
|
||||
if isinstance(status, dict):
|
||||
return status
|
||||
return None
|
||||
|
||||
async def restore_transfer(self, download_id: str, gid: str, save_path: str) -> None:
|
||||
await self._ensure_process()
|
||||
self._transfers[download_id] = Aria2Transfer(
|
||||
gid=gid,
|
||||
save_path=os.path.abspath(save_path),
|
||||
)
|
||||
|
||||
async def reassign_transfer(
|
||||
self, from_download_id: str, to_download_id: str
|
||||
) -> Optional[Aria2Transfer]:
|
||||
transfer = self._transfers.get(from_download_id)
|
||||
if transfer is None:
|
||||
return None
|
||||
|
||||
self._transfers[to_download_id] = transfer
|
||||
if from_download_id != to_download_id:
|
||||
self._transfers.pop(from_download_id, None)
|
||||
return transfer
|
||||
|
||||
async def has_transfer(self, download_id: str) -> bool:
|
||||
return download_id in self._transfers
|
||||
|
||||
async def pause_download(self, download_id: str) -> Dict[str, Any]:
|
||||
transfer = self._transfers.get(download_id)
|
||||
if transfer is None:
|
||||
return {"success": False, "error": "Download task not found"}
|
||||
|
||||
try:
|
||||
await self._rpc_call("aria2.forcePause", [transfer.gid])
|
||||
except Exception as exc:
|
||||
return {"success": False, "error": str(exc)}
|
||||
|
||||
await self._state_store.upsert(download_id, {"status": "paused"})
|
||||
return {"success": True, "message": "Download paused successfully"}
|
||||
|
||||
async def resume_download(self, download_id: str) -> Dict[str, Any]:
|
||||
transfer = self._transfers.get(download_id)
|
||||
if transfer is None:
|
||||
return {"success": False, "error": "Download task not found"}
|
||||
|
||||
try:
|
||||
await self._rpc_call("aria2.unpause", [transfer.gid])
|
||||
except Exception as exc:
|
||||
return {"success": False, "error": str(exc)}
|
||||
|
||||
await self._state_store.upsert(download_id, {"status": "downloading"})
|
||||
return {"success": True, "message": "Download resumed successfully"}
|
||||
|
||||
async def cancel_download(self, download_id: str) -> Dict[str, Any]:
|
||||
transfer = self._transfers.get(download_id)
|
||||
if transfer is None:
|
||||
return {"success": False, "error": "Download task not found"}
|
||||
|
||||
try:
|
||||
await self._rpc_call("aria2.forceRemove", [transfer.gid])
|
||||
except Exception as exc:
|
||||
return {"success": False, "error": str(exc)}
|
||||
|
||||
await self._state_store.remove(download_id)
|
||||
return {"success": True, "message": "Download cancelled successfully"}
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Shut down the RPC process and session."""
|
||||
|
||||
if self._rpc_session is not None:
|
||||
await self._rpc_session.close()
|
||||
self._rpc_session = None
|
||||
|
||||
process = self._process
|
||||
self._process = None
|
||||
self._transfers.clear()
|
||||
|
||||
if process is None:
|
||||
return
|
||||
|
||||
if process.returncode is None:
|
||||
process.terminate()
|
||||
try:
|
||||
await asyncio.wait_for(process.wait(), timeout=5.0)
|
||||
except asyncio.TimeoutError:
|
||||
process.kill()
|
||||
await process.wait()
|
||||
|
||||
async def _dispatch_progress(self, callback, snapshot: DownloadProgress) -> None:
|
||||
try:
|
||||
result = callback(snapshot, snapshot)
|
||||
except TypeError:
|
||||
result = callback(snapshot.percent_complete)
|
||||
|
||||
if asyncio.iscoroutine(result):
|
||||
await result
|
||||
elif hasattr(result, "__await__"):
|
||||
await result
|
||||
|
||||
def _build_progress_snapshot(self, status: Dict[str, Any]) -> DownloadProgress:
|
||||
completed = self._parse_int(status.get("completedLength"))
|
||||
total = self._parse_int(status.get("totalLength"))
|
||||
speed = float(self._parse_int(status.get("downloadSpeed")))
|
||||
percent = 0.0
|
||||
if total > 0:
|
||||
percent = (completed / total) * 100.0
|
||||
|
||||
return DownloadProgress(
|
||||
percent_complete=max(0.0, min(percent, 100.0)),
|
||||
bytes_downloaded=completed,
|
||||
total_bytes=total or None,
|
||||
bytes_per_second=speed,
|
||||
timestamp=datetime.now().timestamp(),
|
||||
)
|
||||
|
||||
def _resolve_completed_path(self, status: Dict[str, Any], default_path: str) -> str:
|
||||
files = status.get("files")
|
||||
if isinstance(files, list) and files:
|
||||
first = files[0]
|
||||
if isinstance(first, dict):
|
||||
candidate = first.get("path")
|
||||
if isinstance(candidate, str) and candidate:
|
||||
return candidate
|
||||
return default_path
|
||||
|
||||
@staticmethod
|
||||
def _parse_int(value: Any) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
async def _resolve_authenticated_redirect_url(
|
||||
self,
|
||||
url: str,
|
||||
headers: Dict[str, str],
|
||||
) -> str:
|
||||
downloader = await get_downloader()
|
||||
session = await downloader.session
|
||||
request_headers = dict(downloader.default_headers)
|
||||
request_headers.update(headers)
|
||||
request_headers["Accept-Encoding"] = "identity"
|
||||
|
||||
try:
|
||||
async with session.get(
|
||||
url,
|
||||
headers=request_headers,
|
||||
allow_redirects=False,
|
||||
proxy=downloader.proxy_url,
|
||||
) as response:
|
||||
if response.status in {301, 302, 303, 307, 308}:
|
||||
location = response.headers.get("Location")
|
||||
if location:
|
||||
return location
|
||||
raise Aria2Error(
|
||||
"Authenticated Civitai redirect did not include a Location header"
|
||||
)
|
||||
|
||||
if response.status == 200:
|
||||
return url
|
||||
|
||||
body = await response.text()
|
||||
raise Aria2Error(
|
||||
f"Failed to resolve authenticated Civitai redirect: status={response.status} body={body[:300]}"
|
||||
)
|
||||
except aiohttp.ClientError as exc:
|
||||
if is_ssl_cert_verify_error(exc):
|
||||
logger.error(
|
||||
"SSL certificate verification failed during Civitai redirect "
|
||||
"resolution for %s. This is usually caused by an outdated CA "
|
||||
"certificate bundle. Recommended fixes:\n"
|
||||
" 1. pip install --upgrade certifi\n"
|
||||
" 2. pip install pip-system-certs",
|
||||
url,
|
||||
)
|
||||
raise Aria2Error(
|
||||
f"Failed to resolve authenticated Civitai redirect: {exc}"
|
||||
) from exc
|
||||
|
||||
async def _ensure_process(self) -> None:
|
||||
async with self._process_lock:
|
||||
if self.is_running and await self._ping():
|
||||
return
|
||||
|
||||
await self.close()
|
||||
|
||||
executable = self._resolve_executable()
|
||||
self._rpc_port = self._find_free_port()
|
||||
self._rpc_secret = secrets.token_hex(16)
|
||||
self._rpc_url = f"http://127.0.0.1:{self._rpc_port}/jsonrpc"
|
||||
|
||||
command = [
|
||||
executable,
|
||||
"--enable-rpc=true",
|
||||
"--rpc-listen-all=false",
|
||||
f"--rpc-listen-port={self._rpc_port}",
|
||||
f"--rpc-secret={self._rpc_secret}",
|
||||
"--check-certificate=true",
|
||||
# Point aria2 at certifi's CA bundle when available so it uses
|
||||
# the same certificate store as Python downloads.
|
||||
*((
|
||||
f"--ca-certificate={ca_cert}",
|
||||
) if (ca_cert := _try_certifi_ca_path()) else ()),
|
||||
"--allow-overwrite=true",
|
||||
"--auto-file-renaming=false",
|
||||
"--file-allocation=none",
|
||||
"--max-concurrent-downloads=5",
|
||||
"--continue=true",
|
||||
"--daemon=false",
|
||||
"--quiet=true",
|
||||
f"--stop-with-process={os.getpid()}",
|
||||
]
|
||||
|
||||
logger.info("Starting aria2 RPC daemon from %s", executable)
|
||||
self._process = await asyncio.create_subprocess_exec(
|
||||
*command,
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
|
||||
await self._wait_until_ready()
|
||||
|
||||
def _resolve_executable(self) -> str:
|
||||
settings = get_settings_manager()
|
||||
configured_path = (settings.get("aria2c_path") or "").strip()
|
||||
candidate = configured_path or "aria2c"
|
||||
|
||||
resolved = shutil.which(candidate)
|
||||
if resolved:
|
||||
return resolved
|
||||
|
||||
if configured_path and os.path.isfile(configured_path) and os.access(
|
||||
configured_path, os.X_OK
|
||||
):
|
||||
return configured_path
|
||||
|
||||
raise Aria2Error(
|
||||
"aria2c executable was not found. Install aria2 or configure aria2c_path."
|
||||
)
|
||||
|
||||
async def _wait_until_ready(self) -> None:
|
||||
assert self._process is not None
|
||||
|
||||
start_time = asyncio.get_running_loop().time()
|
||||
last_error = ""
|
||||
while asyncio.get_running_loop().time() - start_time < 10.0:
|
||||
if self._process.returncode is not None:
|
||||
stderr_output = ""
|
||||
if self._process.stderr is not None:
|
||||
try:
|
||||
stderr_output = (
|
||||
await asyncio.wait_for(self._process.stderr.read(), timeout=0.2)
|
||||
).decode("utf-8", errors="replace")
|
||||
except Exception:
|
||||
stderr_output = ""
|
||||
raise Aria2Error(
|
||||
f"aria2 RPC process exited early with code {self._process.returncode}: {stderr_output.strip()}"
|
||||
)
|
||||
|
||||
try:
|
||||
if await self._ping():
|
||||
return
|
||||
except Exception as exc: # pragma: no cover - startup race
|
||||
last_error = str(exc)
|
||||
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
raise Aria2Error(
|
||||
f"Timed out waiting for aria2 RPC to become ready{': ' + last_error if last_error else ''}"
|
||||
)
|
||||
|
||||
async def _ping(self) -> bool:
|
||||
try:
|
||||
result = await self._rpc_call("aria2.getVersion", [])
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
return isinstance(result, dict)
|
||||
|
||||
async def _rpc_call(self, method: str, params: list[Any]) -> Any:
|
||||
if not self._rpc_url:
|
||||
raise Aria2Error("aria2 RPC endpoint is not initialized")
|
||||
|
||||
session = await self._get_rpc_session()
|
||||
payload = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": secrets.token_hex(8),
|
||||
"method": method,
|
||||
"params": [f"token:{self._rpc_secret}", *params],
|
||||
}
|
||||
|
||||
async with session.post(self._rpc_url, json=payload) as response:
|
||||
text = await response.text()
|
||||
|
||||
try:
|
||||
body = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
body = None
|
||||
|
||||
if body is None:
|
||||
if response.status != 200:
|
||||
raise Aria2Error(
|
||||
f"aria2 RPC returned status {response.status} with non-JSON body: {text}"
|
||||
)
|
||||
raise Aria2Error(f"Invalid aria2 RPC response: {text}")
|
||||
|
||||
if "error" in body:
|
||||
error = body["error"] or {}
|
||||
code = error.get("code") if isinstance(error, dict) else None
|
||||
message = error.get("message") if isinstance(error, dict) else str(error)
|
||||
logger.error(
|
||||
"aria2 RPC %s failed with HTTP %s, code=%s, message=%s",
|
||||
method,
|
||||
response.status,
|
||||
code,
|
||||
message,
|
||||
)
|
||||
status_message = (
|
||||
f"aria2 RPC {method} failed with status {response.status}: {message}"
|
||||
if response.status != 200
|
||||
else message
|
||||
)
|
||||
raise Aria2Error(status_message or "Unknown aria2 RPC error")
|
||||
|
||||
if response.status != 200:
|
||||
logger.error(
|
||||
"aria2 RPC %s returned unexpected HTTP status %s without error payload: %s",
|
||||
method,
|
||||
response.status,
|
||||
body,
|
||||
)
|
||||
raise Aria2Error(
|
||||
f"aria2 RPC {method} returned unexpected status {response.status}"
|
||||
)
|
||||
|
||||
return body.get("result")
|
||||
|
||||
async def _get_rpc_session(self) -> aiohttp.ClientSession:
|
||||
if self._rpc_session is None or self._rpc_session.closed:
|
||||
async with self._rpc_session_lock:
|
||||
if self._rpc_session is None or self._rpc_session.closed:
|
||||
timeout = aiohttp.ClientTimeout(total=30)
|
||||
self._rpc_session = aiohttp.ClientSession(timeout=timeout)
|
||||
return self._rpc_session
|
||||
|
||||
@staticmethod
|
||||
def _find_free_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
sock.listen(1)
|
||||
return int(sock.getsockname()[1])
|
||||
|
||||
|
||||
async def get_aria2_downloader() -> Aria2Downloader:
|
||||
"""Get the singleton aria2 downloader."""
|
||||
|
||||
return await Aria2Downloader.get_instance()
|
||||
108
py/services/aria2_transfer_state.py
Normal file
108
py/services/aria2_transfer_state.py
Normal file
@@ -0,0 +1,108 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from copy import deepcopy
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from ..utils.cache_paths import get_cache_base_dir
|
||||
|
||||
|
||||
def get_aria2_state_path() -> str:
|
||||
base_dir = get_cache_base_dir(create=True)
|
||||
state_dir = os.path.join(base_dir, "aria2")
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
return os.path.join(state_dir, "downloads.json")
|
||||
|
||||
|
||||
class Aria2TransferStateStore:
|
||||
"""Persist aria2 transfer metadata needed for restart recovery."""
|
||||
|
||||
_locks_by_path: Dict[str, asyncio.Lock] = {}
|
||||
|
||||
def __init__(self, state_path: Optional[str] = None) -> None:
|
||||
self._state_path = os.path.abspath(state_path or get_aria2_state_path())
|
||||
self._lock = self._locks_by_path.setdefault(self._state_path, asyncio.Lock())
|
||||
|
||||
def _read_all_unlocked(self) -> Dict[str, Dict[str, Any]]:
|
||||
try:
|
||||
with open(self._state_path, "r", encoding="utf-8") as handle:
|
||||
data = json.load(handle)
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
|
||||
normalized: Dict[str, Dict[str, Any]] = {}
|
||||
for download_id, entry in data.items():
|
||||
if isinstance(download_id, str) and isinstance(entry, dict):
|
||||
normalized[download_id] = entry
|
||||
return normalized
|
||||
|
||||
def _write_all_unlocked(self, data: Dict[str, Dict[str, Any]]) -> None:
|
||||
directory = os.path.dirname(self._state_path)
|
||||
if directory:
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
|
||||
temp_path = f"{self._state_path}.tmp"
|
||||
with open(temp_path, "w", encoding="utf-8") as handle:
|
||||
json.dump(data, handle, ensure_ascii=True, indent=2, sort_keys=True)
|
||||
os.replace(temp_path, self._state_path)
|
||||
|
||||
async def load_all(self) -> Dict[str, Dict[str, Any]]:
|
||||
async with self._lock:
|
||||
return deepcopy(self._read_all_unlocked())
|
||||
|
||||
async def get(self, download_id: str) -> Optional[Dict[str, Any]]:
|
||||
async with self._lock:
|
||||
return deepcopy(self._read_all_unlocked().get(download_id))
|
||||
|
||||
async def upsert(self, download_id: str, payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
async with self._lock:
|
||||
data = self._read_all_unlocked()
|
||||
current = data.get(download_id, {})
|
||||
current.update(payload)
|
||||
data[download_id] = current
|
||||
self._write_all_unlocked(data)
|
||||
return deepcopy(current)
|
||||
|
||||
async def remove(self, download_id: str) -> None:
|
||||
async with self._lock:
|
||||
data = self._read_all_unlocked()
|
||||
if download_id in data:
|
||||
del data[download_id]
|
||||
self._write_all_unlocked(data)
|
||||
|
||||
async def find_by_save_path(
|
||||
self, save_path: str, *, exclude_download_id: Optional[str] = None
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
normalized_target = os.path.abspath(save_path)
|
||||
async with self._lock:
|
||||
data = self._read_all_unlocked()
|
||||
for download_id, entry in data.items():
|
||||
if exclude_download_id and download_id == exclude_download_id:
|
||||
continue
|
||||
candidate = entry.get("save_path")
|
||||
if isinstance(candidate, str) and os.path.abspath(candidate) == normalized_target:
|
||||
result = dict(entry)
|
||||
result["download_id"] = download_id
|
||||
return result
|
||||
return None
|
||||
|
||||
async def reassign(self, from_download_id: str, to_download_id: str) -> Optional[Dict[str, Any]]:
|
||||
async with self._lock:
|
||||
data = self._read_all_unlocked()
|
||||
existing = data.get(from_download_id)
|
||||
if existing is None:
|
||||
return None
|
||||
updated = dict(existing)
|
||||
updated["download_id"] = to_download_id
|
||||
data[to_download_id] = updated
|
||||
if from_download_id != to_download_id:
|
||||
data.pop(from_download_id, None)
|
||||
self._write_all_unlocked(data)
|
||||
return deepcopy(updated)
|
||||
139
py/services/auto_tag_service.py
Normal file
139
py/services/auto_tag_service.py
Normal file
@@ -0,0 +1,139 @@
|
||||
"""
|
||||
Auto-tag extraction service for model cards.
|
||||
|
||||
Extracts implicit model attributes (HIGH/LOW, I2V/T2V/TI2V, Lightning, Turbo)
|
||||
from filename, base_model, and CivitAI version name — no manual tagging required.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Dict, List, Set
|
||||
|
||||
# ── Tag category definitions ──────────────────────────────────────────
|
||||
# Each category maps a display label to a regex pattern.
|
||||
# Patterns are case-insensitive and matched against filename, base_model,
|
||||
# and civitai version name.
|
||||
|
||||
# Use (?<![a-zA-Z0-9]) and (?![a-zA-Z0-9]) instead of \b because
|
||||
# Python's \b treats underscore as a word character, so \bHIGH\b
|
||||
# won't match '_HIGH_' in filenames.
|
||||
_B = r"(?<![a-zA-Z0-9])" # left boundary
|
||||
_E = r"(?![a-zA-Z0-9])" # right boundary
|
||||
|
||||
AUTO_TAG_CATEGORIES: Dict[str, str] = {
|
||||
"HIGH": _B + r"HIGH" + _E,
|
||||
"LOW": _B + r"(?<!F)LOW" + _E,
|
||||
"I2V": _B + r"I2V" + _E,
|
||||
"T2V": _B + r"T2V" + _E,
|
||||
"TI2V": _B + r"TI2V" + _E,
|
||||
"Lightning": _B + r"Lightning" + _E,
|
||||
"Turbo": _B + r"Turbo" + _E,
|
||||
}
|
||||
|
||||
# Tags that belong to the "mode" group (HIGH/LOW)
|
||||
MODE_TAGS = {"HIGH", "LOW"}
|
||||
|
||||
# Tags that belong to the "video mode" group (I2V/T2V/TI2V)
|
||||
VIDEO_MODE_TAGS = {"I2V", "T2V", "TI2V"}
|
||||
|
||||
# Tags that belong to the "speed/optimization" group
|
||||
SPEED_TAGS = {"Lightning", "Turbo"}
|
||||
|
||||
# ── Display category groups (for settings UI) ─────────────────────────
|
||||
|
||||
AUTO_TAG_GROUPS = {
|
||||
"mode": {"HIGH", "LOW"},
|
||||
"video": {"I2V", "T2V", "TI2V"},
|
||||
"speed": {"Lightning", "Turbo"},
|
||||
}
|
||||
|
||||
# Default enabled categories
|
||||
DEFAULT_ENABLED_GROUPS = {"mode", "video"}
|
||||
|
||||
|
||||
def _collect_sources(model_data: Dict) -> List[str]:
|
||||
"""Collect all text sources from model data for tag matching."""
|
||||
sources: List[str] = []
|
||||
|
||||
file_name = model_data.get("file_name", "")
|
||||
if file_name:
|
||||
sources.append(file_name)
|
||||
|
||||
base_model = model_data.get("base_model", "")
|
||||
if base_model:
|
||||
sources.append(base_model)
|
||||
|
||||
civitai = model_data.get("civitai", {})
|
||||
if isinstance(civitai, dict):
|
||||
version_name = civitai.get("name", "")
|
||||
if version_name:
|
||||
sources.append(version_name)
|
||||
|
||||
return sources
|
||||
|
||||
|
||||
def extract_auto_tags(model_data: Dict) -> List[str]:
|
||||
"""Extract auto-detected tags from model metadata.
|
||||
|
||||
Uses a two-layer approach:
|
||||
Layer 1 — Regex-based detection against filename, base_model, and
|
||||
CivitAI version name.
|
||||
Layer 2 — Merge in any user-defined tags that overlap with known
|
||||
auto-tag categories. This provides a manual fallback when
|
||||
auto-detection fails (e.g. "I2V HN" or unlabeled models).
|
||||
|
||||
HIGH/LOW tags are only returned when the base_model indicates a Wan
|
||||
family model — no other model architecture uses this distinction.
|
||||
|
||||
Args:
|
||||
model_data: Model metadata dict with keys:
|
||||
file_name, base_model, civitai (with optional 'name' field),
|
||||
tags (user-defined tag list, used as fallback).
|
||||
|
||||
Returns:
|
||||
Sorted list of unique auto-tag strings (e.g. ["I2V"]).
|
||||
"""
|
||||
sources = _collect_sources(model_data)
|
||||
base_model = model_data.get("base_model", "")
|
||||
is_wan = "wan" in base_model.lower()
|
||||
|
||||
found: Set[str] = set()
|
||||
|
||||
# ── Layer 1: regex-based detection ────────────────────────────
|
||||
if sources:
|
||||
for label, pattern in AUTO_TAG_CATEGORIES.items():
|
||||
# HIGH/LOW are Wan-specific — skip for non-Wan to avoid noise
|
||||
if label in ("HIGH", "LOW"):
|
||||
if not is_wan:
|
||||
continue
|
||||
# Use case-insensitive character class + case-sensitive boundary,
|
||||
# so "HighNoise" (camelCase) matches but "highlight" doesn't.
|
||||
# Boundary: not followed by lowercase letter (= word has ended).
|
||||
ci = "".join(f"[{c.lower()}{c.upper()}]" for c in label)
|
||||
if label == "LOW":
|
||||
regex = re.compile(r"(?<![Ff])" + ci + r"(?![a-z])")
|
||||
else:
|
||||
regex = re.compile(ci + r"(?![a-z])")
|
||||
else:
|
||||
regex = re.compile(pattern, re.IGNORECASE)
|
||||
for source in sources:
|
||||
if regex.search(source):
|
||||
found.add(label)
|
||||
break
|
||||
|
||||
# ── Layer 2: user-defined tags as manual fallback ─────────────
|
||||
# When auto-detection fails (abbreviated names like "Hi"/"Lo",
|
||||
# "I2V HN", or unlabeled models), users can add canonical tags
|
||||
# (HIGH, LOW, I2V, etc.) to the model's regular tags for correct
|
||||
# badge display and filtering. Matching is case-insensitive so
|
||||
# "high"/"High"/"HIGH" all resolve to the canonical label.
|
||||
user_tags = model_data.get("tags")
|
||||
if user_tags:
|
||||
label_map = {label.lower(): label for label in AUTO_TAG_CATEGORIES}
|
||||
for t in user_tags:
|
||||
canonical = label_map.get(t.lower())
|
||||
if canonical:
|
||||
found.add(canonical)
|
||||
|
||||
return sorted(found)
|
||||
423
py/services/backup_service.py
Normal file
423
py/services/backup_service.py
Normal file
@@ -0,0 +1,423 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Optional
|
||||
|
||||
from ..utils.cache_paths import CacheType, get_cache_base_dir, get_cache_file_path
|
||||
from ..utils.settings_paths import get_settings_dir
|
||||
from .settings_manager import get_settings_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
BACKUP_MANIFEST_VERSION = 1
|
||||
DEFAULT_BACKUP_RETENTION_COUNT = 5
|
||||
DEFAULT_BACKUP_INTERVAL_SECONDS = 24 * 60 * 60
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackupEntry:
|
||||
kind: str
|
||||
archive_path: str
|
||||
target_path: str
|
||||
sha256: str
|
||||
size: int
|
||||
mtime: float
|
||||
|
||||
|
||||
class BackupService:
|
||||
"""Create and restore user-state backup archives."""
|
||||
|
||||
_instance: "BackupService | None" = None
|
||||
_instance_lock = asyncio.Lock()
|
||||
|
||||
def __init__(self, *, settings_manager=None, backup_dir: str | None = None) -> None:
|
||||
self._settings = settings_manager or get_settings_manager()
|
||||
self._backup_dir = Path(backup_dir or self._resolve_backup_dir())
|
||||
self._backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._lock = asyncio.Lock()
|
||||
self._auto_task: asyncio.Task[None] | None = None
|
||||
|
||||
@classmethod
|
||||
async def get_instance(cls) -> "BackupService":
|
||||
async with cls._instance_lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = cls()
|
||||
cls._instance._ensure_auto_snapshot_task()
|
||||
return cls._instance
|
||||
|
||||
@staticmethod
|
||||
def _resolve_backup_dir() -> str:
|
||||
return os.path.join(get_settings_dir(create=True), "backups")
|
||||
|
||||
def get_backup_dir(self) -> str:
|
||||
return str(self._backup_dir)
|
||||
|
||||
def _ensure_auto_snapshot_task(self) -> None:
|
||||
if self._auto_task is not None and not self._auto_task.done():
|
||||
return
|
||||
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return
|
||||
|
||||
self._auto_task = loop.create_task(self._auto_backup_loop())
|
||||
|
||||
def _get_setting_bool(self, key: str, default: bool) -> bool:
|
||||
try:
|
||||
return bool(self._settings.get(key, default))
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
def _get_setting_int(self, key: str, default: int) -> int:
|
||||
try:
|
||||
value = self._settings.get(key, default)
|
||||
return max(1, int(value))
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
def _settings_file_path(self) -> str:
|
||||
settings_file = getattr(self._settings, "settings_file", None)
|
||||
if settings_file:
|
||||
return str(settings_file)
|
||||
return os.path.join(get_settings_dir(create=True), "settings.json")
|
||||
|
||||
def _download_history_path(self) -> str:
|
||||
base_dir = get_cache_base_dir(create=True)
|
||||
history_dir = os.path.join(base_dir, "download_history")
|
||||
os.makedirs(history_dir, exist_ok=True)
|
||||
return os.path.join(history_dir, "downloaded_versions.sqlite")
|
||||
|
||||
def _model_update_dir(self) -> str:
|
||||
return str(Path(get_cache_file_path(CacheType.MODEL_UPDATE, create_dir=True)).parent)
|
||||
|
||||
def _model_update_targets(self) -> list[tuple[str, str, str]]:
|
||||
"""Return (kind, archive_path, target_path) tuples for backup."""
|
||||
|
||||
targets: list[tuple[str, str, str]] = []
|
||||
|
||||
settings_path = self._settings_file_path()
|
||||
targets.append(("settings", "settings/settings.json", settings_path))
|
||||
|
||||
history_path = self._download_history_path()
|
||||
targets.append(
|
||||
(
|
||||
"download_history",
|
||||
"cache/download_history/downloaded_versions.sqlite",
|
||||
history_path,
|
||||
)
|
||||
)
|
||||
|
||||
symlink_path = get_cache_file_path(CacheType.SYMLINK, create_dir=True)
|
||||
targets.append(
|
||||
(
|
||||
"symlink_map",
|
||||
"cache/symlink/symlink_map.json",
|
||||
symlink_path,
|
||||
)
|
||||
)
|
||||
|
||||
model_update_dir = Path(self._model_update_dir())
|
||||
if model_update_dir.exists():
|
||||
for sqlite_file in sorted(model_update_dir.glob("*.sqlite")):
|
||||
targets.append(
|
||||
(
|
||||
"model_update",
|
||||
f"cache/model_update/{sqlite_file.name}",
|
||||
str(sqlite_file),
|
||||
)
|
||||
)
|
||||
|
||||
stats_path = os.path.join(get_settings_dir(create=True), "stats", "lora_manager_stats.json")
|
||||
if os.path.exists(stats_path):
|
||||
targets.append(
|
||||
(
|
||||
"usage_stats",
|
||||
"stats/lora_manager_stats.json",
|
||||
stats_path,
|
||||
)
|
||||
)
|
||||
|
||||
return targets
|
||||
|
||||
@staticmethod
|
||||
def _hash_file(path: str) -> tuple[str, int, float]:
|
||||
digest = hashlib.sha256()
|
||||
total = 0
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
total += len(chunk)
|
||||
digest.update(chunk)
|
||||
mtime = os.path.getmtime(path)
|
||||
return digest.hexdigest(), total, mtime
|
||||
|
||||
def _build_manifest(self, entries: Iterable[BackupEntry], *, snapshot_type: str) -> dict[str, Any]:
|
||||
created_at = datetime.now(timezone.utc).isoformat()
|
||||
active_library = None
|
||||
try:
|
||||
active_library = self._settings.get_active_library_name()
|
||||
except Exception:
|
||||
active_library = None
|
||||
|
||||
return {
|
||||
"manifest_version": BACKUP_MANIFEST_VERSION,
|
||||
"created_at": created_at,
|
||||
"snapshot_type": snapshot_type,
|
||||
"active_library": active_library,
|
||||
"files": [
|
||||
{
|
||||
"kind": entry.kind,
|
||||
"archive_path": entry.archive_path,
|
||||
"target_path": entry.target_path,
|
||||
"sha256": entry.sha256,
|
||||
"size": entry.size,
|
||||
"mtime": entry.mtime,
|
||||
}
|
||||
for entry in entries
|
||||
],
|
||||
}
|
||||
|
||||
def _write_archive(self, archive_path: str, entries: list[BackupEntry], manifest: dict[str, Any]) -> None:
|
||||
with zipfile.ZipFile(
|
||||
archive_path,
|
||||
mode="w",
|
||||
compression=zipfile.ZIP_DEFLATED,
|
||||
compresslevel=6,
|
||||
) as zf:
|
||||
zf.writestr(
|
||||
"manifest.json",
|
||||
json.dumps(manifest, indent=2, ensure_ascii=False).encode("utf-8"),
|
||||
)
|
||||
for entry in entries:
|
||||
zf.write(entry.target_path, arcname=entry.archive_path)
|
||||
|
||||
async def create_snapshot(self, *, snapshot_type: str = "manual", persist: bool = False) -> dict[str, Any]:
|
||||
"""Create a backup archive.
|
||||
|
||||
If ``persist`` is true, the archive is stored in the backup directory
|
||||
and retained according to the configured retention policy.
|
||||
"""
|
||||
|
||||
async with self._lock:
|
||||
raw_targets = self._model_update_targets()
|
||||
entries: list[BackupEntry] = []
|
||||
for kind, archive_path, target_path in raw_targets:
|
||||
if not os.path.exists(target_path):
|
||||
continue
|
||||
sha256, size, mtime = self._hash_file(target_path)
|
||||
entries.append(
|
||||
BackupEntry(
|
||||
kind=kind,
|
||||
archive_path=archive_path,
|
||||
target_path=target_path,
|
||||
sha256=sha256,
|
||||
size=size,
|
||||
mtime=mtime,
|
||||
)
|
||||
)
|
||||
|
||||
if not entries:
|
||||
raise FileNotFoundError("No backupable files were found")
|
||||
|
||||
manifest = self._build_manifest(entries, snapshot_type=snapshot_type)
|
||||
archive_name = self._build_archive_name(snapshot_type=snapshot_type)
|
||||
fd, temp_path = tempfile.mkstemp(suffix=".zip", dir=str(self._backup_dir))
|
||||
os.close(fd)
|
||||
|
||||
try:
|
||||
self._write_archive(temp_path, entries, manifest)
|
||||
if persist:
|
||||
final_path = self._backup_dir / archive_name
|
||||
os.replace(temp_path, final_path)
|
||||
self._prune_snapshots()
|
||||
return {
|
||||
"archive_path": str(final_path),
|
||||
"archive_name": final_path.name,
|
||||
"manifest": manifest,
|
||||
}
|
||||
|
||||
with open(temp_path, "rb") as handle:
|
||||
data = handle.read()
|
||||
return {
|
||||
"archive_name": archive_name,
|
||||
"archive_bytes": data,
|
||||
"manifest": manifest,
|
||||
}
|
||||
finally:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.remove(temp_path)
|
||||
|
||||
def _build_archive_name(self, *, snapshot_type: str) -> str:
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
return f"lora-manager-backup-{timestamp}-{snapshot_type}.zip"
|
||||
|
||||
def _prune_snapshots(self) -> None:
|
||||
retention = self._get_setting_int(
|
||||
"backup_retention_count", DEFAULT_BACKUP_RETENTION_COUNT
|
||||
)
|
||||
archives = sorted(
|
||||
self._backup_dir.glob("lora-manager-backup-*-auto.zip"),
|
||||
key=lambda path: path.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
for path in archives[retention:]:
|
||||
with contextlib.suppress(OSError):
|
||||
path.unlink()
|
||||
|
||||
async def restore_snapshot(self, archive_path: str) -> dict[str, Any]:
|
||||
"""Restore backup contents from a ZIP archive."""
|
||||
|
||||
async with self._lock:
|
||||
try:
|
||||
zf = zipfile.ZipFile(archive_path, mode="r")
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise ValueError("Backup archive is not a valid ZIP file") from exc
|
||||
|
||||
with zf:
|
||||
try:
|
||||
manifest = json.loads(zf.read("manifest.json").decode("utf-8"))
|
||||
except KeyError as exc:
|
||||
raise ValueError("Backup archive is missing manifest.json") from exc
|
||||
|
||||
if not isinstance(manifest, dict):
|
||||
raise ValueError("Backup manifest is invalid")
|
||||
if manifest.get("manifest_version") != BACKUP_MANIFEST_VERSION:
|
||||
raise ValueError("Backup manifest version is not supported")
|
||||
|
||||
files = manifest.get("files", [])
|
||||
if not isinstance(files, list):
|
||||
raise ValueError("Backup manifest file list is invalid")
|
||||
|
||||
extracted_paths: list[tuple[str, str]] = []
|
||||
temp_dir = Path(tempfile.mkdtemp(prefix="lora-manager-restore-"))
|
||||
try:
|
||||
for item in files:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
archive_member = item.get("archive_path")
|
||||
if not isinstance(archive_member, str) or not archive_member:
|
||||
continue
|
||||
archive_member_path = Path(archive_member)
|
||||
if archive_member_path.is_absolute() or ".." in archive_member_path.parts:
|
||||
raise ValueError(f"Invalid archive member path: {archive_member}")
|
||||
|
||||
kind = item.get("kind")
|
||||
target_path = self._resolve_restore_target(kind, archive_member)
|
||||
if target_path is None:
|
||||
continue
|
||||
|
||||
extracted_path = temp_dir / archive_member_path
|
||||
extracted_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(archive_member) as source, open(
|
||||
extracted_path, "wb"
|
||||
) as destination:
|
||||
shutil.copyfileobj(source, destination)
|
||||
|
||||
expected_hash = item.get("sha256")
|
||||
if isinstance(expected_hash, str) and expected_hash:
|
||||
actual_hash, _, _ = self._hash_file(str(extracted_path))
|
||||
if actual_hash != expected_hash:
|
||||
raise ValueError(
|
||||
f"Checksum mismatch for {archive_member}"
|
||||
)
|
||||
|
||||
extracted_paths.append((str(extracted_path), target_path))
|
||||
|
||||
for extracted_path, target_path in extracted_paths:
|
||||
os.makedirs(os.path.dirname(target_path), exist_ok=True)
|
||||
os.replace(extracted_path, target_path)
|
||||
finally:
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"restored_files": len(extracted_paths),
|
||||
"snapshot_type": manifest.get("snapshot_type"),
|
||||
}
|
||||
|
||||
def _resolve_restore_target(self, kind: Any, archive_member: str) -> str | None:
|
||||
if kind == "settings":
|
||||
return self._settings_file_path()
|
||||
if kind == "download_history":
|
||||
return self._download_history_path()
|
||||
if kind == "symlink_map":
|
||||
return get_cache_file_path(CacheType.SYMLINK, create_dir=True)
|
||||
if kind == "model_update":
|
||||
filename = os.path.basename(archive_member)
|
||||
return str(Path(get_cache_file_path(CacheType.MODEL_UPDATE, create_dir=True)).parent / filename)
|
||||
if kind == "usage_stats":
|
||||
return os.path.join(get_settings_dir(create=True), "stats", "lora_manager_stats.json")
|
||||
return None
|
||||
|
||||
async def create_auto_snapshot_if_due(self) -> Optional[dict[str, Any]]:
|
||||
if not self._get_setting_bool("backup_auto_enabled", True):
|
||||
return None
|
||||
|
||||
latest = self.get_latest_auto_snapshot()
|
||||
now = time.time()
|
||||
if latest and now - latest["mtime"] < DEFAULT_BACKUP_INTERVAL_SECONDS:
|
||||
return None
|
||||
|
||||
return await self.create_snapshot(snapshot_type="auto", persist=True)
|
||||
|
||||
async def _auto_backup_loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
await self.create_auto_snapshot_if_due()
|
||||
await asyncio.sleep(DEFAULT_BACKUP_INTERVAL_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # pragma: no cover - defensive guard
|
||||
logger.warning("Automatic backup snapshot failed: %s", exc, exc_info=True)
|
||||
await asyncio.sleep(60)
|
||||
|
||||
def get_available_snapshots(self) -> list[dict[str, Any]]:
|
||||
snapshots: list[dict[str, Any]] = []
|
||||
for path in sorted(self._backup_dir.glob("lora-manager-backup-*.zip")):
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError:
|
||||
continue
|
||||
snapshots.append(
|
||||
{
|
||||
"name": path.name,
|
||||
"path": str(path),
|
||||
"size": stat.st_size,
|
||||
"mtime": stat.st_mtime,
|
||||
"is_auto": path.name.endswith("-auto.zip"),
|
||||
}
|
||||
)
|
||||
snapshots.sort(key=lambda item: item["mtime"], reverse=True)
|
||||
return snapshots
|
||||
|
||||
def get_latest_auto_snapshot(self) -> Optional[dict[str, Any]]:
|
||||
autos = [snapshot for snapshot in self.get_available_snapshots() if snapshot["is_auto"]]
|
||||
if not autos:
|
||||
return None
|
||||
return autos[0]
|
||||
|
||||
def get_status(self) -> dict[str, Any]:
|
||||
snapshots = self.get_available_snapshots()
|
||||
return {
|
||||
"backupDir": self.get_backup_dir(),
|
||||
"enabled": self._get_setting_bool("backup_auto_enabled", True),
|
||||
"retentionCount": self._get_setting_int(
|
||||
"backup_retention_count", DEFAULT_BACKUP_RETENTION_COUNT
|
||||
),
|
||||
"snapshotCount": len(snapshots),
|
||||
"latestSnapshot": snapshots[0] if snapshots else None,
|
||||
"latestAutoSnapshot": self.get_latest_auto_snapshot(),
|
||||
}
|
||||
@@ -20,6 +20,7 @@ from .model_query import (
|
||||
resolve_sub_type,
|
||||
)
|
||||
from .settings_manager import get_settings_manager
|
||||
from ..utils.civitai_utils import build_civitai_model_page_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -76,6 +77,7 @@ class BaseModelService(ABC):
|
||||
base_models: list = None,
|
||||
model_types: list = None,
|
||||
tags: Optional[Dict[str, str]] = None,
|
||||
auto_tags: Optional[Dict[str, str]] = None,
|
||||
search_options: dict = None,
|
||||
hash_filters: dict = None,
|
||||
favorites_only: bool = False,
|
||||
@@ -94,9 +96,108 @@ class BaseModelService(ABC):
|
||||
sorted_data = await self._fetch_with_usage_sort(sort_params)
|
||||
else:
|
||||
sorted_data = await self.cache_repository.fetch_sorted(sort_params)
|
||||
# Pre-compute auto_tags for every item — needed for both filtering
|
||||
# and display. Computation is cheap (string regex on 2-3 fields).
|
||||
from .auto_tag_service import extract_auto_tags
|
||||
for item in sorted_data:
|
||||
item["auto_tags"] = extract_auto_tags(item)
|
||||
fetch_duration = time.perf_counter() - t0
|
||||
initial_count = len(sorted_data)
|
||||
|
||||
# Optionally filter by civitai model ID (shows all local versions of a specific model)
|
||||
civitai_model_id = kwargs.get("civitai_model_id")
|
||||
if civitai_model_id is not None:
|
||||
sorted_data = [
|
||||
item for item in sorted_data
|
||||
if self._extract_model_id(item) == civitai_model_id
|
||||
]
|
||||
# VLM mode: always sort by version ID descending (newest version first),
|
||||
# regardless of the current sort_by preference.
|
||||
sorted_data.sort(
|
||||
key=lambda x: self._extract_version_id(x) or 0,
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
# Optionally group by civitai modelId, showing only the latest version per model
|
||||
dedup_lost = 0
|
||||
if kwargs.get("group_by_model") and civitai_model_id is None:
|
||||
# Determine whether to further sub-group by base model
|
||||
# When version_grouping is "same_base", versions with different
|
||||
# base models are effectively different groups — the dedup key
|
||||
# needs to include base_model so the version count and VLM flow
|
||||
# stay consistent (card shows correct count for its base model).
|
||||
ufs = self.settings.get("version_grouping", "same_base")
|
||||
group_by_base = ufs == "same_base"
|
||||
|
||||
dedup_map = {} # (modelId [,base_model]) -> (item, version_id)
|
||||
version_counter = {} # same-key -> count
|
||||
standalone = []
|
||||
for item in sorted_data:
|
||||
mid = self._extract_model_id(item)
|
||||
if mid is None:
|
||||
standalone.append(item)
|
||||
continue
|
||||
key = (mid, item.get("base_model") or "") if group_by_base else mid
|
||||
# Count all versions per key
|
||||
version_counter[key] = version_counter.get(key, 0) + 1
|
||||
vid = self._extract_version_id(item) or 0
|
||||
if key not in dedup_map or vid > dedup_map[key][1]:
|
||||
dedup_map[key] = (item, vid)
|
||||
# Attach version_count to each surviving grouped item (shallow copy
|
||||
# to avoid mutating cached dicts — the cache is shared across requests)
|
||||
for key, (item, vid) in dedup_map.items():
|
||||
item = dict(item)
|
||||
item["version_count"] = version_counter[key]
|
||||
dedup_map[key] = (item, vid)
|
||||
dedup_lost = len(sorted_data) - (len(dedup_map) + len(standalone))
|
||||
sorted_data = [entry[0] for entry in dedup_map.values()] + standalone
|
||||
|
||||
# Re-sort by version_count (grouped: after dedup; non-grouped: group internally, sort, expand)
|
||||
if sort_params.key == "versions_count" and civitai_model_id is None:
|
||||
reverse = sort_params.order == "desc"
|
||||
if kwargs.get("group_by_model"):
|
||||
# Grouped mode: items are already dedup'd with version_count attached
|
||||
sorted_data.sort(
|
||||
key=lambda x: (
|
||||
x.get("version_count", 0),
|
||||
(x.get("model_name") or x.get("file_name") or "").lower(),
|
||||
x.get("file_path", "").lower(),
|
||||
),
|
||||
reverse=reverse,
|
||||
)
|
||||
else:
|
||||
# Non-grouped mode: group internally, sort groups by count, expand
|
||||
# Respect the version_grouping setting (same logic as grouped dedup)
|
||||
ufs = self.settings.get("version_grouping", "same_base")
|
||||
group_by_base = ufs == "same_base"
|
||||
|
||||
model_groups: Dict[Any, List[Dict]] = {}
|
||||
ungrouped_standalone: List[Dict] = []
|
||||
for item in sorted_data:
|
||||
mid = self._extract_model_id(item)
|
||||
if mid is None:
|
||||
ungrouped_standalone.append(item)
|
||||
continue
|
||||
key = (mid, item.get("base_model") or "") if group_by_base else mid
|
||||
model_groups.setdefault(key, []).append(item)
|
||||
# Sort versions within each group by version id descending
|
||||
for items in model_groups.values():
|
||||
items.sort(
|
||||
key=lambda x: self._extract_version_id(x) or 0,
|
||||
reverse=True,
|
||||
)
|
||||
# Sort groups by version count
|
||||
sorted_groups = sorted(
|
||||
model_groups.values(),
|
||||
key=lambda items: len(items),
|
||||
reverse=reverse,
|
||||
)
|
||||
# Flatten: grouped items first, standalone items last
|
||||
sorted_data = []
|
||||
for items in sorted_groups:
|
||||
sorted_data.extend(items)
|
||||
sorted_data.extend(ungrouped_standalone)
|
||||
|
||||
t1 = time.perf_counter()
|
||||
if hash_filters:
|
||||
filtered_data = await self._apply_hash_filters(sorted_data, hash_filters)
|
||||
@@ -109,6 +210,7 @@ class BaseModelService(ABC):
|
||||
base_models=base_models,
|
||||
model_types=model_types,
|
||||
tags=tags,
|
||||
auto_tags=auto_tags,
|
||||
favorites_only=favorites_only,
|
||||
search_options=search_options,
|
||||
tag_logic=tag_logic,
|
||||
@@ -164,7 +266,7 @@ class BaseModelService(ABC):
|
||||
overall_duration = time.perf_counter() - overall_start
|
||||
logger.debug(
|
||||
"%s.get_paginated_data took %.3fs (fetch: %.3fs, filter: %.3fs, update_filter: %.3fs, pagination: %.3fs, annotate: %.3fs). "
|
||||
"Counts: initial=%d, post_filter=%d, final=%d",
|
||||
"Counts: initial=%d, dedup=%d, post_filter=%d, final=%d",
|
||||
self.__class__.__name__,
|
||||
overall_duration,
|
||||
fetch_duration,
|
||||
@@ -173,11 +275,63 @@ class BaseModelService(ABC):
|
||||
pagination_duration,
|
||||
annotate_duration,
|
||||
initial_count,
|
||||
dedup_lost,
|
||||
post_filter_count,
|
||||
final_count,
|
||||
)
|
||||
return paginated
|
||||
|
||||
async def get_excluded_paginated_data(
|
||||
self,
|
||||
page: int,
|
||||
page_size: int,
|
||||
sort_by: str = "name",
|
||||
search: str = None,
|
||||
fuzzy_search: bool = False,
|
||||
search_options: dict = None,
|
||||
**kwargs,
|
||||
) -> Dict:
|
||||
"""Get paginated excluded model data."""
|
||||
excluded_paths = list(self.scanner.get_excluded_models())
|
||||
excluded_entries: List[Dict[str, Any]] = []
|
||||
stale_paths: List[str] = []
|
||||
|
||||
for file_path in excluded_paths:
|
||||
if not file_path or not os.path.exists(file_path):
|
||||
stale_paths.append(file_path)
|
||||
continue
|
||||
|
||||
entry = await self._build_excluded_entry(file_path)
|
||||
if entry:
|
||||
excluded_entries.append(entry)
|
||||
else:
|
||||
stale_paths.append(file_path)
|
||||
|
||||
if stale_paths:
|
||||
current_excluded = getattr(self.scanner, "_excluded_models", None)
|
||||
if isinstance(current_excluded, list):
|
||||
stale_set = set(stale_paths)
|
||||
self.scanner._excluded_models = [
|
||||
path for path in current_excluded if path not in stale_set
|
||||
]
|
||||
persist_current_cache = getattr(self.scanner, "_persist_current_cache", None)
|
||||
if callable(persist_current_cache):
|
||||
await persist_current_cache()
|
||||
|
||||
excluded_entries = self._sort_entries(excluded_entries, sort_by)
|
||||
|
||||
if search:
|
||||
excluded_entries = await self._apply_search_filters(
|
||||
excluded_entries,
|
||||
search,
|
||||
fuzzy_search,
|
||||
search_options,
|
||||
)
|
||||
|
||||
paginated = self._paginate(excluded_entries, page, page_size)
|
||||
paginated["items"] = await self._annotate_update_flags(paginated["items"])
|
||||
return paginated
|
||||
|
||||
async def _fetch_with_usage_sort(self, sort_params):
|
||||
"""Fetch data sorted by usage count (desc/asc)."""
|
||||
cache = await self.cache_repository.get_cache()
|
||||
@@ -208,11 +362,71 @@ class BaseModelService(ABC):
|
||||
|
||||
reverse = sort_params.order == "desc"
|
||||
annotated.sort(
|
||||
key=lambda x: (x.get("usage_count", 0), x.get("model_name", "").lower()),
|
||||
key=lambda x: (
|
||||
x.get("usage_count", 0),
|
||||
x.get("model_name", "").lower(),
|
||||
x.get("file_path", "").lower()
|
||||
),
|
||||
reverse=reverse,
|
||||
)
|
||||
return annotated
|
||||
|
||||
def _sort_entries(self, data: List[Dict[str, Any]], sort_by: str) -> List[Dict[str, Any]]:
|
||||
sort_params = self.cache_repository.parse_sort(sort_by)
|
||||
key_name = sort_params.key
|
||||
|
||||
if key_name == "date":
|
||||
key_fn = lambda item: (
|
||||
float(item.get("modified", 0.0) or 0.0),
|
||||
(item.get("model_name") or item.get("file_name") or "").lower(),
|
||||
item.get("file_path", "").lower(),
|
||||
)
|
||||
elif key_name == "size":
|
||||
key_fn = lambda item: (
|
||||
int(item.get("size", 0) or 0),
|
||||
(item.get("model_name") or item.get("file_name") or "").lower(),
|
||||
item.get("file_path", "").lower(),
|
||||
)
|
||||
elif key_name == "usage":
|
||||
key_fn = lambda item: (
|
||||
int(item.get("usage_count", 0) or 0),
|
||||
(item.get("model_name") or item.get("file_name") or "").lower(),
|
||||
item.get("file_path", "").lower(),
|
||||
)
|
||||
else:
|
||||
key_fn = lambda item: (
|
||||
(item.get("model_name") or item.get("file_name") or "").lower(),
|
||||
item.get("file_path", "").lower(),
|
||||
)
|
||||
|
||||
return sorted(data, key=key_fn, reverse=sort_params.order == "desc")
|
||||
|
||||
async def _build_excluded_entry(self, file_path: str) -> Optional[Dict[str, Any]]:
|
||||
root_path = self.scanner._find_root_for_file(file_path)
|
||||
if not root_path:
|
||||
return None
|
||||
|
||||
metadata, should_skip = await MetadataManager.load_metadata(
|
||||
file_path,
|
||||
self.metadata_class,
|
||||
)
|
||||
if should_skip:
|
||||
return None
|
||||
|
||||
if metadata is None:
|
||||
metadata = await self.scanner._create_default_metadata(file_path)
|
||||
if metadata is None:
|
||||
return None
|
||||
|
||||
metadata = self.scanner.adjust_metadata(metadata, file_path, root_path)
|
||||
folder = os.path.dirname(os.path.relpath(file_path, root_path)).replace(
|
||||
os.path.sep, "/"
|
||||
)
|
||||
entry = self.scanner._build_cache_entry(metadata, folder=folder)
|
||||
entry = self.scanner.adjust_cached_entry(entry)
|
||||
entry["exclude"] = True
|
||||
return entry
|
||||
|
||||
async def _apply_hash_filters(
|
||||
self, data: List[Dict], hash_filters: Dict
|
||||
) -> List[Dict]:
|
||||
@@ -242,6 +456,7 @@ class BaseModelService(ABC):
|
||||
base_models: list = None,
|
||||
model_types: list = None,
|
||||
tags: Optional[Dict[str, str]] = None,
|
||||
auto_tags: Optional[Dict[str, str]] = None,
|
||||
favorites_only: bool = False,
|
||||
search_options: dict = None,
|
||||
tag_logic: str = "any",
|
||||
@@ -255,6 +470,7 @@ class BaseModelService(ABC):
|
||||
base_models=base_models,
|
||||
model_types=model_types,
|
||||
tags=tags,
|
||||
auto_tags=auto_tags,
|
||||
favorites_only=favorites_only,
|
||||
search_options=normalized_options,
|
||||
tag_logic=tag_logic,
|
||||
@@ -374,7 +590,7 @@ class BaseModelService(ABC):
|
||||
if not ordered_ids:
|
||||
return annotated
|
||||
|
||||
strategy_value = self.settings.get("update_flag_strategy")
|
||||
strategy_value = self.settings.get("version_grouping")
|
||||
if isinstance(strategy_value, str) and strategy_value.strip():
|
||||
strategy = strategy_value.strip().lower()
|
||||
else:
|
||||
@@ -749,30 +965,86 @@ class BaseModelService(ABC):
|
||||
"""Get the static preview URL for a model file"""
|
||||
cache = await self.scanner.get_cached_data()
|
||||
|
||||
name_normalized = model_name.replace("\\", "/")
|
||||
name_no_ext = name_normalized
|
||||
for ext in (".safetensors", ".ckpt", ".pt", ".bin"):
|
||||
if name_no_ext.lower().endswith(ext):
|
||||
name_no_ext = name_no_ext[: -len(ext)]
|
||||
break
|
||||
|
||||
has_path = "/" in name_no_ext
|
||||
basename = os.path.basename(name_no_ext) if has_path else name_no_ext
|
||||
best_fallback = None
|
||||
|
||||
for model in cache.raw_data:
|
||||
if model["file_name"] == model_name:
|
||||
file_name = model.get("file_name", "")
|
||||
folder = model.get("folder", "")
|
||||
file_name_no_ext = file_name
|
||||
for ext in (".safetensors", ".ckpt", ".pt", ".bin"):
|
||||
if file_name_no_ext.lower().endswith(ext):
|
||||
file_name_no_ext = file_name_no_ext[: -len(ext)]
|
||||
break
|
||||
path_name = f"{folder}/{file_name_no_ext}".replace("\\", "/") if folder else file_name_no_ext
|
||||
|
||||
if name_no_ext == file_name_no_ext or name_no_ext == path_name:
|
||||
preview_url = model.get("preview_url")
|
||||
if preview_url:
|
||||
from ..config import config
|
||||
|
||||
return config.get_preview_static_url(preview_url)
|
||||
|
||||
if has_path and file_name_no_ext == basename:
|
||||
if folder and name_no_ext.startswith(folder.replace("\\", "/") + "/"):
|
||||
best_fallback = model
|
||||
elif best_fallback is None:
|
||||
best_fallback = model
|
||||
|
||||
if best_fallback:
|
||||
preview_url = best_fallback.get("preview_url")
|
||||
if preview_url:
|
||||
from ..config import config
|
||||
|
||||
return config.get_preview_static_url(preview_url)
|
||||
|
||||
return "/loras_static/images/no-preview.png"
|
||||
|
||||
async def get_model_civitai_url(self, model_name: str) -> Dict[str, Optional[str]]:
|
||||
"""Get the Civitai URL for a model file"""
|
||||
cache = await self.scanner.get_cached_data()
|
||||
|
||||
name_normalized = model_name.replace("\\", "/")
|
||||
name_no_ext = name_normalized
|
||||
for ext in (".safetensors", ".ckpt", ".pt", ".bin"):
|
||||
if name_no_ext.lower().endswith(ext):
|
||||
name_no_ext = name_no_ext[: -len(ext)]
|
||||
break
|
||||
|
||||
has_path = "/" in name_no_ext
|
||||
basename = os.path.basename(name_no_ext) if has_path else name_no_ext
|
||||
best_fallback = None
|
||||
|
||||
for model in cache.raw_data:
|
||||
if model["file_name"] == model_name:
|
||||
file_name = model.get("file_name", "")
|
||||
folder = model.get("folder", "")
|
||||
file_name_no_ext = file_name
|
||||
for ext in (".safetensors", ".ckpt", ".pt", ".bin"):
|
||||
if file_name_no_ext.lower().endswith(ext):
|
||||
file_name_no_ext = file_name_no_ext[: -len(ext)]
|
||||
break
|
||||
path_name = f"{folder}/{file_name_no_ext}".replace("\\", "/") if folder else file_name_no_ext
|
||||
|
||||
if name_no_ext == file_name_no_ext or name_no_ext == path_name:
|
||||
civitai_data = model.get("civitai", {})
|
||||
model_id = civitai_data.get("modelId")
|
||||
version_id = civitai_data.get("id")
|
||||
|
||||
if model_id:
|
||||
civitai_url = f"https://civitai.com/models/{model_id}"
|
||||
if version_id:
|
||||
civitai_url += f"?modelVersionId={version_id}"
|
||||
civitai_host = self.settings.get("civitai_host", "civitai.com")
|
||||
civitai_url = build_civitai_model_page_url(
|
||||
model_id,
|
||||
version_id,
|
||||
host=civitai_host,
|
||||
)
|
||||
|
||||
return {
|
||||
"civitai_url": civitai_url,
|
||||
@@ -780,6 +1052,27 @@ class BaseModelService(ABC):
|
||||
"version_id": str(version_id) if version_id else None,
|
||||
}
|
||||
|
||||
if has_path and file_name_no_ext == basename:
|
||||
if folder and name_no_ext.startswith(folder.replace("\\", "/") + "/"):
|
||||
best_fallback = model
|
||||
elif best_fallback is None:
|
||||
best_fallback = model
|
||||
|
||||
if best_fallback:
|
||||
civitai_data = best_fallback.get("civitai", {})
|
||||
model_id = civitai_data.get("modelId")
|
||||
if model_id:
|
||||
version_id = civitai_data.get("id")
|
||||
civitai_host = self.settings.get("civitai_host", "civitai.com")
|
||||
civitai_url = build_civitai_model_page_url(
|
||||
model_id, version_id, host=civitai_host
|
||||
)
|
||||
return {
|
||||
"civitai_url": civitai_url,
|
||||
"model_id": str(model_id),
|
||||
"version_id": str(version_id) if version_id else None,
|
||||
}
|
||||
|
||||
return {"civitai_url": None, "model_id": None, "version_id": None}
|
||||
|
||||
async def get_model_metadata(self, file_path: str) -> Optional[Dict]:
|
||||
@@ -793,6 +1086,17 @@ class BaseModelService(ABC):
|
||||
)
|
||||
if should_skip or metadata is None:
|
||||
return None
|
||||
|
||||
# Prune stale example-image metadata entries whose files no longer
|
||||
# exist on disk (e.g. a user deleted the files manually).
|
||||
from ..utils.example_images_metadata import MetadataUpdater
|
||||
|
||||
was_modified = await MetadataUpdater.prune_stale_example_images(metadata)
|
||||
if was_modified:
|
||||
asyncio.create_task(
|
||||
MetadataManager.save_metadata(file_path, metadata)
|
||||
)
|
||||
|
||||
return self.filter_civitai_data(metadata.to_dict().get("civitai", {}))
|
||||
|
||||
async def get_model_description(self, file_path: str) -> Optional[str]:
|
||||
|
||||
@@ -224,7 +224,7 @@ class BatchImportService:
|
||||
return False
|
||||
|
||||
for recipe in getattr(cache, "raw_data", []):
|
||||
source_path = recipe.get("source_path") or recipe.get("source_url")
|
||||
source_path = recipe.get("source_path")
|
||||
if source_path and source_path == source:
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -58,6 +58,7 @@ class CacheEntryValidator:
|
||||
'preview_nsfw_level': (0, False),
|
||||
'notes': ('', False),
|
||||
'usage_tips': ('', False),
|
||||
'hash_status': ('completed', False),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
@@ -90,13 +91,31 @@ class CacheEntryValidator:
|
||||
|
||||
errors: List[str] = []
|
||||
repaired = False
|
||||
|
||||
# If auto_repair is on, we work on a copy. If not, we still need a safe way to check fields.
|
||||
working_entry = dict(entry) if auto_repair else entry
|
||||
|
||||
# Determine effective hash_status for validation logic
|
||||
hash_status = entry.get('hash_status')
|
||||
if hash_status is None:
|
||||
if auto_repair:
|
||||
working_entry['hash_status'] = 'completed'
|
||||
repaired = True
|
||||
hash_status = 'completed'
|
||||
|
||||
for field_name, (default_value, is_required) in cls.CORE_FIELDS.items():
|
||||
value = working_entry.get(field_name)
|
||||
# Get current value from the original entry to avoid side effects during validation
|
||||
value = entry.get(field_name)
|
||||
|
||||
# Check if field is missing or None
|
||||
if value is None:
|
||||
# Special case: sha256 can be None/empty if hash_status is pending
|
||||
if field_name == 'sha256' and hash_status == 'pending':
|
||||
if auto_repair:
|
||||
working_entry[field_name] = ''
|
||||
repaired = True
|
||||
continue
|
||||
|
||||
if is_required:
|
||||
errors.append(f"Required field '{field_name}' is missing or None")
|
||||
if auto_repair:
|
||||
@@ -107,6 +126,10 @@ class CacheEntryValidator:
|
||||
# Validate field type and value
|
||||
field_error = cls._validate_field(field_name, value, default_value)
|
||||
if field_error:
|
||||
# Special case: allow empty string for sha256 if pending
|
||||
if field_name == 'sha256' and hash_status == 'pending' and value == '':
|
||||
continue
|
||||
|
||||
errors.append(field_error)
|
||||
if auto_repair:
|
||||
working_entry[field_name] = cls._get_default_copy(default_value)
|
||||
@@ -127,7 +150,7 @@ class CacheEntryValidator:
|
||||
# Special validation: sha256 must not be empty for required field
|
||||
# BUT allow empty sha256 when hash_status is pending (lazy hash calculation)
|
||||
sha256 = working_entry.get('sha256', '')
|
||||
hash_status = working_entry.get('hash_status', 'completed')
|
||||
# Use the effective hash_status we determined earlier
|
||||
if not sha256 or (isinstance(sha256, str) and not sha256.strip()):
|
||||
# Allow empty sha256 for lazy hash calculation (checkpoints)
|
||||
if hash_status != 'pending':
|
||||
@@ -144,8 +167,13 @@ class CacheEntryValidator:
|
||||
if isinstance(sha256, str):
|
||||
normalized_sha = sha256.lower().strip()
|
||||
if normalized_sha != sha256:
|
||||
working_entry['sha256'] = normalized_sha
|
||||
repaired = True
|
||||
if auto_repair:
|
||||
working_entry['sha256'] = normalized_sha
|
||||
repaired = True
|
||||
else:
|
||||
# If not auto-repairing, we don't consider case difference as a "critical error"
|
||||
# that invalidates the entry, but we also don't mark it repaired.
|
||||
pass
|
||||
|
||||
# Determine if entry is valid
|
||||
# Entry is valid if no critical required field errors remain after repair
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -13,22 +14,38 @@ from .model_hash_index import ModelHashIndex
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CheckpointScanner(ModelScanner):
|
||||
"""Service for scanning and managing checkpoint files"""
|
||||
|
||||
|
||||
def __init__(self):
|
||||
# Define supported file extensions
|
||||
file_extensions = {'.ckpt', '.pt', '.pt2', '.bin', '.pth', '.safetensors', '.pkl', '.sft', '.gguf'}
|
||||
file_extensions = {
|
||||
".ckpt",
|
||||
".pt",
|
||||
".pt2",
|
||||
".bin",
|
||||
".pth",
|
||||
".safetensors",
|
||||
".pkl",
|
||||
".sft",
|
||||
".gguf",
|
||||
}
|
||||
super().__init__(
|
||||
model_type="checkpoint",
|
||||
model_class=CheckpointMetadata,
|
||||
file_extensions=file_extensions,
|
||||
hash_index=ModelHashIndex()
|
||||
hash_index=ModelHashIndex(),
|
||||
)
|
||||
if not hasattr(self, "_hash_calculation_lock"):
|
||||
self._hash_calculation_lock = asyncio.Lock()
|
||||
self._hash_calculation_tasks: dict[str, asyncio.Task[Optional[str]]] = {}
|
||||
|
||||
async def _create_default_metadata(self, file_path: str) -> Optional[CheckpointMetadata]:
|
||||
async def _create_default_metadata(
|
||||
self, file_path: str
|
||||
) -> Optional[CheckpointMetadata]:
|
||||
"""Create default metadata for checkpoint without calculating hash (lazy hash).
|
||||
|
||||
|
||||
Checkpoints are typically large (10GB+), so we skip hash calculation during initial
|
||||
scanning to improve startup performance. Hash will be calculated on-demand when
|
||||
fetching metadata from Civitai.
|
||||
@@ -38,13 +55,13 @@ class CheckpointScanner(ModelScanner):
|
||||
if not os.path.exists(real_path):
|
||||
logger.error(f"File not found: {file_path}")
|
||||
return None
|
||||
|
||||
|
||||
base_name = os.path.splitext(os.path.basename(file_path))[0]
|
||||
dir_path = os.path.dirname(file_path)
|
||||
|
||||
|
||||
# Find preview image
|
||||
preview_url = find_preview_file(base_name, dir_path)
|
||||
|
||||
|
||||
# Create metadata WITHOUT calculating hash
|
||||
metadata = CheckpointMetadata(
|
||||
file_name=base_name,
|
||||
@@ -59,70 +76,133 @@ class CheckpointScanner(ModelScanner):
|
||||
modelDescription="",
|
||||
sub_type="checkpoint",
|
||||
from_civitai=False, # Mark as local model since no hash yet
|
||||
hash_status="pending" # Mark hash as pending
|
||||
hash_status="pending", # Mark hash as pending
|
||||
)
|
||||
|
||||
|
||||
# Save the created metadata
|
||||
logger.info(f"Creating checkpoint metadata (hash pending) for {file_path}")
|
||||
await MetadataManager.save_metadata(file_path, metadata)
|
||||
|
||||
|
||||
return metadata
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating default checkpoint metadata for {file_path}: {e}")
|
||||
logger.error(
|
||||
f"Error creating default checkpoint metadata for {file_path}: {e}"
|
||||
)
|
||||
return None
|
||||
|
||||
async def calculate_hash_for_model(self, file_path: str) -> Optional[str]:
|
||||
"""Calculate hash for a checkpoint on-demand.
|
||||
|
||||
"""Calculate hash for a checkpoint on-demand with per-file singleflight.
|
||||
|
||||
Args:
|
||||
file_path: Path to the model file
|
||||
|
||||
|
||||
Returns:
|
||||
SHA256 hash string, or None if calculation failed
|
||||
"""
|
||||
from ..utils.file_utils import calculate_sha256
|
||||
|
||||
try:
|
||||
real_path = os.path.realpath(file_path)
|
||||
if not os.path.exists(real_path):
|
||||
logger.error(f"File not found for hash calculation: {file_path}")
|
||||
return None
|
||||
|
||||
|
||||
metadata, _ = await MetadataManager.load_metadata(
|
||||
file_path, self.model_class
|
||||
)
|
||||
if (
|
||||
metadata is not None
|
||||
and metadata.hash_status == "completed"
|
||||
and metadata.sha256
|
||||
):
|
||||
return metadata.sha256
|
||||
|
||||
async with self._hash_calculation_lock:
|
||||
metadata, _ = await MetadataManager.load_metadata(
|
||||
file_path, self.model_class
|
||||
)
|
||||
if (
|
||||
metadata is not None
|
||||
and metadata.hash_status == "completed"
|
||||
and metadata.sha256
|
||||
):
|
||||
return metadata.sha256
|
||||
|
||||
task = self._hash_calculation_tasks.get(real_path)
|
||||
if task is None:
|
||||
task = asyncio.create_task(
|
||||
self._run_hash_calculation_task(file_path, real_path)
|
||||
)
|
||||
self._hash_calculation_tasks[real_path] = task
|
||||
|
||||
return await asyncio.shield(task)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error calculating hash for {file_path}: {e}")
|
||||
return None
|
||||
|
||||
async def _run_hash_calculation_task(
|
||||
self, file_path: str, real_path: str
|
||||
) -> Optional[str]:
|
||||
"""Run a hash calculation task and remove it from the in-flight map."""
|
||||
try:
|
||||
return await self._calculate_hash_for_model_uncached(file_path, real_path)
|
||||
finally:
|
||||
task = asyncio.current_task()
|
||||
async with self._hash_calculation_lock:
|
||||
if self._hash_calculation_tasks.get(real_path) is task:
|
||||
del self._hash_calculation_tasks[real_path]
|
||||
|
||||
async def _calculate_hash_for_model_uncached(
|
||||
self, file_path: str, real_path: str
|
||||
) -> Optional[str]:
|
||||
"""Calculate hash for a checkpoint without checking in-flight tasks."""
|
||||
from ..utils.file_utils import calculate_sha256
|
||||
|
||||
try:
|
||||
# Load current metadata
|
||||
metadata, _ = await MetadataManager.load_metadata(file_path, self.model_class)
|
||||
metadata, should_skip = await MetadataManager.load_metadata(
|
||||
file_path, self.model_class
|
||||
)
|
||||
if metadata is None:
|
||||
logger.error(f"No metadata found for {file_path}")
|
||||
return None
|
||||
|
||||
if should_skip:
|
||||
logger.error(f"Invalid metadata found for {file_path}")
|
||||
return None
|
||||
created_metadata = await self._create_default_metadata(file_path)
|
||||
if created_metadata is None:
|
||||
logger.error(f"No metadata found for {file_path}")
|
||||
return None
|
||||
metadata = created_metadata
|
||||
|
||||
# Check if hash is already calculated
|
||||
if metadata.hash_status == "completed" and metadata.sha256:
|
||||
return metadata.sha256
|
||||
|
||||
|
||||
# Update status to calculating
|
||||
metadata.hash_status = "calculating"
|
||||
await MetadataManager.save_metadata(file_path, metadata)
|
||||
|
||||
|
||||
# Calculate hash
|
||||
logger.info(f"Calculating hash for checkpoint: {file_path}")
|
||||
sha256 = await calculate_sha256(real_path)
|
||||
|
||||
|
||||
# Update metadata with hash
|
||||
metadata.sha256 = sha256
|
||||
metadata.hash_status = "completed"
|
||||
await MetadataManager.save_metadata(file_path, metadata)
|
||||
|
||||
|
||||
# Update hash index
|
||||
self._hash_index.add_entry(sha256.lower(), file_path)
|
||||
|
||||
|
||||
logger.info(f"Hash calculated for checkpoint: {file_path}")
|
||||
return sha256
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error calculating hash for {file_path}: {e}")
|
||||
# Update status to failed
|
||||
try:
|
||||
metadata, _ = await MetadataManager.load_metadata(file_path, self.model_class)
|
||||
metadata, _ = await MetadataManager.load_metadata(
|
||||
file_path, self.model_class
|
||||
)
|
||||
if metadata:
|
||||
metadata.hash_status = "failed"
|
||||
await MetadataManager.save_metadata(file_path, metadata)
|
||||
@@ -130,43 +210,46 @@ class CheckpointScanner(ModelScanner):
|
||||
pass
|
||||
return None
|
||||
|
||||
async def calculate_all_pending_hashes(self, progress_callback=None) -> Dict[str, int]:
|
||||
async def calculate_all_pending_hashes(
|
||||
self, progress_callback=None
|
||||
) -> Dict[str, int]:
|
||||
"""Calculate hashes for all checkpoints with pending hash status.
|
||||
|
||||
|
||||
If cache is not initialized, scans filesystem directly for metadata files
|
||||
with hash_status != 'completed'.
|
||||
|
||||
|
||||
Args:
|
||||
progress_callback: Optional callback(progress, total, current_file)
|
||||
|
||||
|
||||
Returns:
|
||||
Dict with 'completed', 'failed', 'total' counts
|
||||
"""
|
||||
# Try to get from cache first
|
||||
cache = await self.get_cached_data()
|
||||
|
||||
|
||||
if cache and cache.raw_data:
|
||||
# Use cache if available
|
||||
pending_models = [
|
||||
item for item in cache.raw_data
|
||||
if item.get('hash_status') != 'completed' or not item.get('sha256')
|
||||
item
|
||||
for item in cache.raw_data
|
||||
if item.get("hash_status") != "completed" or not item.get("sha256")
|
||||
]
|
||||
else:
|
||||
# Cache not initialized, scan filesystem directly
|
||||
pending_models = await self._find_pending_models_from_filesystem()
|
||||
|
||||
|
||||
if not pending_models:
|
||||
return {'completed': 0, 'failed': 0, 'total': 0}
|
||||
|
||||
return {"completed": 0, "failed": 0, "total": 0}
|
||||
|
||||
total = len(pending_models)
|
||||
completed = 0
|
||||
failed = 0
|
||||
|
||||
|
||||
for i, model_data in enumerate(pending_models):
|
||||
file_path = model_data.get('file_path')
|
||||
file_path = model_data.get("file_path")
|
||||
if not file_path:
|
||||
continue
|
||||
|
||||
|
||||
try:
|
||||
sha256 = await self.calculate_hash_for_model(file_path)
|
||||
if sha256:
|
||||
@@ -176,77 +259,102 @@ class CheckpointScanner(ModelScanner):
|
||||
except Exception as e:
|
||||
logger.error(f"Error calculating hash for {file_path}: {e}")
|
||||
failed += 1
|
||||
|
||||
|
||||
if progress_callback:
|
||||
try:
|
||||
await progress_callback(i + 1, total, file_path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {
|
||||
'completed': completed,
|
||||
'failed': failed,
|
||||
'total': total
|
||||
}
|
||||
|
||||
|
||||
return {"completed": completed, "failed": failed, "total": total}
|
||||
|
||||
async def _find_pending_models_from_filesystem(self) -> List[Dict[str, Any]]:
|
||||
"""Scan filesystem for checkpoint metadata files with pending hash status."""
|
||||
pending_models = []
|
||||
|
||||
|
||||
for root_path in self.get_model_roots():
|
||||
if not os.path.exists(root_path):
|
||||
continue
|
||||
|
||||
|
||||
for dirpath, _dirnames, filenames in os.walk(root_path):
|
||||
for filename in filenames:
|
||||
if not filename.endswith('.metadata.json'):
|
||||
if not filename.endswith(".metadata.json"):
|
||||
continue
|
||||
|
||||
|
||||
metadata_path = os.path.join(dirpath, filename)
|
||||
try:
|
||||
with open(metadata_path, 'r', encoding='utf-8') as f:
|
||||
with open(metadata_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
|
||||
# Check if hash is pending
|
||||
hash_status = data.get('hash_status', 'completed')
|
||||
sha256 = data.get('sha256', '')
|
||||
|
||||
if hash_status != 'completed' or not sha256:
|
||||
hash_status = data.get("hash_status", "completed")
|
||||
sha256 = data.get("sha256", "")
|
||||
|
||||
if hash_status != "completed" or not sha256:
|
||||
# Find corresponding model file
|
||||
model_name = filename.replace('.metadata.json', '')
|
||||
model_name = filename.replace(".metadata.json", "")
|
||||
model_path = None
|
||||
|
||||
|
||||
# Look for model file with matching name
|
||||
for ext in self.file_extensions:
|
||||
potential_path = os.path.join(dirpath, model_name + ext)
|
||||
if os.path.exists(potential_path):
|
||||
model_path = potential_path
|
||||
break
|
||||
|
||||
|
||||
if model_path:
|
||||
pending_models.append({
|
||||
'file_path': model_path.replace(os.sep, '/'),
|
||||
'hash_status': hash_status,
|
||||
'sha256': sha256,
|
||||
**{k: v for k, v in data.items() if k not in ['file_path', 'hash_status', 'sha256']}
|
||||
})
|
||||
pending_models.append(
|
||||
{
|
||||
"file_path": model_path.replace(os.sep, "/"),
|
||||
"hash_status": hash_status,
|
||||
"sha256": sha256,
|
||||
**{
|
||||
k: v
|
||||
for k, v in data.items()
|
||||
if k
|
||||
not in [
|
||||
"file_path",
|
||||
"hash_status",
|
||||
"sha256",
|
||||
]
|
||||
},
|
||||
}
|
||||
)
|
||||
except (json.JSONDecodeError, Exception) as e:
|
||||
logger.debug(f"Error reading metadata file {metadata_path}: {e}")
|
||||
logger.debug(
|
||||
f"Error reading metadata file {metadata_path}: {e}"
|
||||
)
|
||||
continue
|
||||
|
||||
|
||||
return pending_models
|
||||
|
||||
def _resolve_sub_type(self, root_path: Optional[str]) -> Optional[str]:
|
||||
"""Resolve the sub-type based on the root path."""
|
||||
"""Resolve the sub-type based on the root path.
|
||||
|
||||
Checks both standard ComfyUI paths and LoRA Manager's extra folder paths.
|
||||
"""
|
||||
if not root_path:
|
||||
return None
|
||||
|
||||
# Check standard ComfyUI checkpoint paths
|
||||
if config.checkpoints_roots and root_path in config.checkpoints_roots:
|
||||
return "checkpoint"
|
||||
|
||||
# Check extra checkpoint paths
|
||||
if (
|
||||
config.extra_checkpoints_roots
|
||||
and root_path in config.extra_checkpoints_roots
|
||||
):
|
||||
return "checkpoint"
|
||||
|
||||
# Check standard ComfyUI unet paths
|
||||
if config.unet_roots and root_path in config.unet_roots:
|
||||
return "diffusion_model"
|
||||
|
||||
# Check extra unet paths
|
||||
if config.extra_unet_roots and root_path in config.extra_unet_roots:
|
||||
return "diffusion_model"
|
||||
|
||||
return None
|
||||
|
||||
def adjust_metadata(self, metadata, file_path, root_path):
|
||||
|
||||
@@ -3,6 +3,7 @@ import logging
|
||||
from typing import Dict
|
||||
|
||||
from .base_model_service import BaseModelService
|
||||
from .auto_tag_service import extract_auto_tags
|
||||
from ..utils.models import CheckpointMetadata
|
||||
from ..config import config
|
||||
|
||||
@@ -42,9 +43,12 @@ class CheckpointService(BaseModelService):
|
||||
"notes": checkpoint_data.get("notes", ""),
|
||||
"sub_type": sub_type,
|
||||
"favorite": checkpoint_data.get("favorite", False),
|
||||
"exclude": bool(checkpoint_data.get("exclude", False)),
|
||||
"update_available": bool(checkpoint_data.get("update_available", False)),
|
||||
"skip_metadata_refresh": bool(checkpoint_data.get("skip_metadata_refresh", False)),
|
||||
"civitai": self.filter_civitai_data(checkpoint_data.get("civitai", {}), minimal=True)
|
||||
"civitai": self.filter_civitai_data(checkpoint_data.get("civitai", {}), minimal=True),
|
||||
"auto_tags": checkpoint_data.get("auto_tags") or extract_auto_tags(checkpoint_data),
|
||||
"version_count": checkpoint_data.get("version_count"),
|
||||
}
|
||||
|
||||
def find_duplicate_hashes(self) -> Dict:
|
||||
|
||||
@@ -186,6 +186,22 @@ class CivArchiveClient:
|
||||
if "metadata" in file_data:
|
||||
transformed["metadata"] = file_data["metadata"]
|
||||
|
||||
# Infer metadata.format from filename extension
|
||||
name = transformed.get("name")
|
||||
if name and isinstance(name, str):
|
||||
lower_name = name.lower()
|
||||
if lower_name.endswith(".safetensors"):
|
||||
inferred_format = "SafeTensor"
|
||||
elif lower_name.endswith(".ckpt"):
|
||||
inferred_format = "PickleTensor"
|
||||
else:
|
||||
inferred_format = None
|
||||
if inferred_format:
|
||||
if "metadata" not in transformed:
|
||||
transformed["metadata"] = {}
|
||||
if isinstance(transformed["metadata"], dict):
|
||||
transformed["metadata"].setdefault("format", inferred_format)
|
||||
|
||||
if file_data.get("modelVersionId") is not None:
|
||||
transformed["modelVersionId"] = file_data.get("modelVersionId")
|
||||
elif file_data.get("model_version_id") is not None:
|
||||
@@ -213,6 +229,20 @@ class CivArchiveClient:
|
||||
for file_data in candidates:
|
||||
if isinstance(file_data, dict):
|
||||
transformed_files.append(self._transform_file_entry(file_data))
|
||||
|
||||
# Sort: .safetensors first, .ckpt second, others last
|
||||
# so the backend fallback (no file_params) prefers safetensors
|
||||
def _sort_key(f: Dict) -> int:
|
||||
fname = f.get("name") or ""
|
||||
if isinstance(fname, str):
|
||||
lower = fname.lower()
|
||||
if lower.endswith(".safetensors"):
|
||||
return 0
|
||||
elif lower.endswith(".ckpt"):
|
||||
return 1
|
||||
return 2
|
||||
|
||||
transformed_files.sort(key=_sort_key)
|
||||
return transformed_files
|
||||
|
||||
def _transform_version(
|
||||
|
||||
436
py/services/civitai_base_model_service.py
Normal file
436
py/services/civitai_base_model_service.py
Normal file
@@ -0,0 +1,436 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||
|
||||
from ..utils.constants import SUPPORTED_DOWNLOAD_SKIP_BASE_MODELS
|
||||
from .downloader import get_downloader
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CivitaiBaseModelService:
|
||||
"""Service for fetching and managing Civitai base models.
|
||||
|
||||
This service provides:
|
||||
- Fetching base models from Civitai API
|
||||
- Caching with TTL (7 days default)
|
||||
- Merging hardcoded and remote base models
|
||||
- Generating abbreviations for new/unknown models
|
||||
"""
|
||||
|
||||
_instance: Optional[CivitaiBaseModelService] = None
|
||||
_lock = asyncio.Lock()
|
||||
|
||||
# Default TTL for cache in seconds (7 days)
|
||||
DEFAULT_CACHE_TTL = 7 * 24 * 60 * 60
|
||||
|
||||
# Civitai API endpoint for enums
|
||||
CIVITAI_ENUMS_URL = "https://civitai.red/api/v1/enums"
|
||||
|
||||
@classmethod
|
||||
async def get_instance(cls) -> CivitaiBaseModelService:
|
||||
"""Get singleton instance of the service."""
|
||||
async with cls._lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = cls()
|
||||
return cls._instance
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the service."""
|
||||
if hasattr(self, "_initialized"):
|
||||
return
|
||||
self._initialized = True
|
||||
|
||||
# Cache storage
|
||||
self._cache: Optional[Dict[str, Any]] = None
|
||||
self._cache_timestamp: Optional[datetime] = None
|
||||
self._cache_ttl = self.DEFAULT_CACHE_TTL
|
||||
|
||||
# Hardcoded models for fallback
|
||||
self._hardcoded_models = set(SUPPORTED_DOWNLOAD_SKIP_BASE_MODELS)
|
||||
|
||||
logger.info("CivitaiBaseModelService initialized")
|
||||
|
||||
async def get_base_models(self, force_refresh: bool = False) -> Dict[str, Any]:
|
||||
"""Get merged base models (hardcoded + remote).
|
||||
|
||||
Args:
|
||||
force_refresh: If True, fetch from API regardless of cache state.
|
||||
|
||||
Returns:
|
||||
Dictionary containing:
|
||||
- models: List of merged base model names
|
||||
- source: 'cache', 'api', or 'fallback'
|
||||
- last_updated: ISO timestamp of last successful API fetch
|
||||
- hardcoded_count: Number of hardcoded models
|
||||
- remote_count: Number of remote models
|
||||
- merged_count: Total unique models
|
||||
"""
|
||||
# Check if cache is valid
|
||||
if not force_refresh and self._is_cache_valid():
|
||||
logger.debug("Returning cached base models")
|
||||
return self._build_response("cache")
|
||||
|
||||
# Try to fetch from API
|
||||
try:
|
||||
remote_models = await self._fetch_from_civitai()
|
||||
if remote_models:
|
||||
self._update_cache(remote_models)
|
||||
return self._build_response("api")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch base models from Civitai: {e}")
|
||||
|
||||
# Fallback to hardcoded models
|
||||
return self._build_response("fallback")
|
||||
|
||||
async def refresh_cache(self) -> Dict[str, Any]:
|
||||
"""Force refresh the cache from Civitai API.
|
||||
|
||||
Returns:
|
||||
Response dict same as get_base_models()
|
||||
"""
|
||||
return await self.get_base_models(force_refresh=True)
|
||||
|
||||
def get_cache_status(self) -> Dict[str, Any]:
|
||||
"""Get current cache status.
|
||||
|
||||
Returns:
|
||||
Dictionary containing:
|
||||
- has_cache: Whether cache exists
|
||||
- last_updated: ISO timestamp or None
|
||||
- is_expired: Whether cache is expired
|
||||
- ttl_seconds: TTL in seconds
|
||||
- age_seconds: Age of cache in seconds (if exists)
|
||||
"""
|
||||
if self._cache is None or self._cache_timestamp is None:
|
||||
return {
|
||||
"has_cache": False,
|
||||
"last_updated": None,
|
||||
"is_expired": True,
|
||||
"ttl_seconds": self._cache_ttl,
|
||||
"age_seconds": None,
|
||||
}
|
||||
|
||||
age = (datetime.now(timezone.utc) - self._cache_timestamp).total_seconds()
|
||||
return {
|
||||
"has_cache": True,
|
||||
"last_updated": self._cache_timestamp.isoformat(),
|
||||
"is_expired": age > self._cache_ttl,
|
||||
"ttl_seconds": self._cache_ttl,
|
||||
"age_seconds": int(age),
|
||||
}
|
||||
|
||||
def generate_abbreviation(self, model_name: str) -> str:
|
||||
"""Generate abbreviation for a base model name.
|
||||
|
||||
Algorithm:
|
||||
1. Extract version patterns (e.g., "2.5" from "Wan Video 2.5")
|
||||
2. Extract main acronym (e.g., "SD" from "SD 1.5")
|
||||
3. Handle special cases (Flux, Wan, etc.)
|
||||
4. Fallback to first letters of words (max 4 chars)
|
||||
|
||||
Args:
|
||||
model_name: Full base model name
|
||||
|
||||
Returns:
|
||||
Generated abbreviation (max 4 characters)
|
||||
"""
|
||||
if not model_name or not isinstance(model_name, str):
|
||||
return "OTH"
|
||||
|
||||
name = model_name.strip()
|
||||
if not name:
|
||||
return "OTH"
|
||||
|
||||
# Check if it's already in hardcoded abbreviations
|
||||
# This is a simplified check - in practice you'd have a mapping
|
||||
lower_name = name.lower()
|
||||
|
||||
# Special cases
|
||||
special_cases = {
|
||||
"sd 1.4": "SD1",
|
||||
"sd 1.5": "SD1",
|
||||
"sd 1.5 lcm": "SD1",
|
||||
"sd 1.5 hyper": "SD1",
|
||||
"sd 2.0": "SD2",
|
||||
"sd 2.1": "SD2",
|
||||
"sd 3": "SD3",
|
||||
"sd 3.5": "SD3",
|
||||
"sd 3.5 medium": "SD3",
|
||||
"sd 3.5 large": "SD3",
|
||||
"sd 3.5 large turbo": "SD3",
|
||||
"sdxl 1.0": "XL",
|
||||
"sdxl lightning": "XL",
|
||||
"sdxl hyper": "XL",
|
||||
"flux.1 d": "F1D",
|
||||
"flux.1 s": "F1S",
|
||||
"flux.1 krea": "F1KR",
|
||||
"flux.1 kontext": "F1KX",
|
||||
"flux.2 d": "F2D",
|
||||
"flux.2 klein 9b": "FK9",
|
||||
"flux.2 klein 9b-base": "FK9B",
|
||||
"flux.2 klein 4b": "FK4",
|
||||
"flux.2 klein 4b-base": "FK4B",
|
||||
"auraflow": "AF",
|
||||
"chroma": "CHR",
|
||||
"pixart a": "PXA",
|
||||
"pixart e": "PXE",
|
||||
"hunyuan 1": "HY",
|
||||
"hunyuan video": "HYV",
|
||||
"lumina": "L",
|
||||
"kolors": "KLR",
|
||||
"noobai": "NAI",
|
||||
"illustrious": "IL",
|
||||
"pony": "PONY",
|
||||
"pony v7": "PNY7",
|
||||
"hidream": "HID",
|
||||
"qwen": "QWEN",
|
||||
"zimageturbo": "ZIT",
|
||||
"zimagebase": "ZIB",
|
||||
"anima": "ANI",
|
||||
"ernie": "ERNI",
|
||||
"ernie turbo": "ETRB",
|
||||
"nucleus": "NUCL",
|
||||
"svd": "SVD",
|
||||
"ltxv": "LTXV",
|
||||
"ltxv2": "LTV2",
|
||||
"ltxv 2.3": "LTX",
|
||||
"cogvideox": "CVX",
|
||||
"mochi": "MCHI",
|
||||
"wan video": "WAN",
|
||||
"wan video 1.3b t2v": "WAN",
|
||||
"wan video 14b t2v": "WAN",
|
||||
"wan video 14b i2v 480p": "WAN",
|
||||
"wan video 14b i2v 720p": "WAN",
|
||||
"wan video 2.2 ti2v-5b": "WAN",
|
||||
"wan video 2.2 t2v-a14b": "WAN",
|
||||
"wan video 2.2 i2v-a14b": "WAN",
|
||||
"wan video 2.5 t2v": "WAN",
|
||||
"wan video 2.5 i2v": "WAN",
|
||||
}
|
||||
|
||||
if lower_name in special_cases:
|
||||
return special_cases[lower_name]
|
||||
|
||||
# Try to extract acronym from version pattern
|
||||
# e.g., "Model Name 2.5" -> "MN25"
|
||||
version_match = re.search(r"(\d+(?:\.\d+)?)", name)
|
||||
version = version_match.group(1) if version_match else ""
|
||||
|
||||
# Remove version and common words
|
||||
words = re.sub(r"\d+(?:\.\d+)?", "", name)
|
||||
words = re.sub(
|
||||
r"\b(model|video|diffusion|checkpoint|textualinversion)\b",
|
||||
"",
|
||||
words,
|
||||
flags=re.I,
|
||||
)
|
||||
words = words.strip()
|
||||
|
||||
# Get first letters of remaining words
|
||||
tokens = re.findall(r"[A-Za-z]+", words)
|
||||
if tokens:
|
||||
# Build abbreviation from first letters
|
||||
abbrev = "".join(token[0].upper() for token in tokens)
|
||||
# Add version if present
|
||||
if version:
|
||||
# Clean version (remove dots for abbreviation)
|
||||
version_clean = version.replace(".", "")
|
||||
abbrev = abbrev[: 4 - len(version_clean)] + version_clean
|
||||
return abbrev[:4]
|
||||
|
||||
# Final fallback: just take first 4 alphanumeric chars
|
||||
alphanumeric = re.sub(r"[^A-Za-z0-9]", "", name)
|
||||
if alphanumeric:
|
||||
return alphanumeric[:4].upper()
|
||||
|
||||
return "OTH"
|
||||
|
||||
async def _fetch_from_civitai(self) -> Optional[Set[str]]:
|
||||
"""Fetch base models from Civitai API.
|
||||
|
||||
Returns:
|
||||
Set of base model names, or None if failed
|
||||
"""
|
||||
try:
|
||||
downloader = await get_downloader()
|
||||
success, result = await downloader.make_request(
|
||||
"GET",
|
||||
self.CIVITAI_ENUMS_URL,
|
||||
use_auth=False, # enums endpoint doesn't require auth
|
||||
)
|
||||
|
||||
if not success:
|
||||
logger.warning(f"Failed to fetch enums from Civitai: {result}")
|
||||
return None
|
||||
|
||||
if isinstance(result, str):
|
||||
data = json.loads(result)
|
||||
else:
|
||||
data = result
|
||||
|
||||
# Extract base models from response
|
||||
base_models = set()
|
||||
|
||||
# Use ActiveBaseModel if available (recommended active models)
|
||||
if "ActiveBaseModel" in data:
|
||||
base_models.update(data["ActiveBaseModel"])
|
||||
logger.info(f"Fetched {len(base_models)} models from ActiveBaseModel")
|
||||
# Fallback to full BaseModel list
|
||||
elif "BaseModel" in data:
|
||||
base_models.update(data["BaseModel"])
|
||||
logger.info(f"Fetched {len(base_models)} models from BaseModel")
|
||||
else:
|
||||
logger.warning("No base model data found in Civitai response")
|
||||
return None
|
||||
|
||||
return base_models
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching from Civitai: {e}")
|
||||
return None
|
||||
|
||||
def _update_cache(self, remote_models: Set[str]) -> None:
|
||||
"""Update internal cache with fetched models.
|
||||
|
||||
Args:
|
||||
remote_models: Set of base model names from API
|
||||
"""
|
||||
self._cache = {
|
||||
"remote_models": sorted(remote_models),
|
||||
"hardcoded_models": sorted(self._hardcoded_models),
|
||||
}
|
||||
self._cache_timestamp = datetime.now(timezone.utc)
|
||||
logger.info(f"Cache updated with {len(remote_models)} remote models")
|
||||
|
||||
def _is_cache_valid(self) -> bool:
|
||||
"""Check if current cache is valid (not expired).
|
||||
|
||||
Returns:
|
||||
True if cache exists and is not expired
|
||||
"""
|
||||
if self._cache is None or self._cache_timestamp is None:
|
||||
return False
|
||||
|
||||
age = (datetime.now(timezone.utc) - self._cache_timestamp).total_seconds()
|
||||
return age <= self._cache_ttl
|
||||
|
||||
def _build_response(self, source: str) -> Dict[str, Any]:
|
||||
"""Build response dictionary.
|
||||
|
||||
Args:
|
||||
source: 'cache', 'api', or 'fallback'
|
||||
|
||||
Returns:
|
||||
Response dictionary
|
||||
"""
|
||||
if source == "fallback" or self._cache is None:
|
||||
# Use only hardcoded models
|
||||
merged = sorted(self._hardcoded_models)
|
||||
return {
|
||||
"models": merged,
|
||||
"source": source,
|
||||
"last_updated": None,
|
||||
"hardcoded_count": len(self._hardcoded_models),
|
||||
"remote_count": 0,
|
||||
"merged_count": len(merged),
|
||||
}
|
||||
|
||||
# Merge hardcoded and remote models
|
||||
remote_set = set(self._cache.get("remote_models", []))
|
||||
merged = sorted(self._hardcoded_models | remote_set)
|
||||
|
||||
return {
|
||||
"models": merged,
|
||||
"source": source,
|
||||
"last_updated": self._cache_timestamp.isoformat()
|
||||
if self._cache_timestamp
|
||||
else None,
|
||||
"hardcoded_count": len(self._hardcoded_models),
|
||||
"remote_count": len(remote_set),
|
||||
"merged_count": len(merged),
|
||||
}
|
||||
|
||||
def get_model_categories(self) -> Dict[str, List[str]]:
|
||||
"""Get categorized base models.
|
||||
|
||||
Returns:
|
||||
Dictionary mapping category names to lists of model names
|
||||
"""
|
||||
# Define category patterns
|
||||
categories = {
|
||||
"Stable Diffusion 1.x": ["SD 1.4", "SD 1.5", "SD 1.5 LCM", "SD 1.5 Hyper"],
|
||||
"Stable Diffusion 2.x": ["SD 2.0", "SD 2.1"],
|
||||
"Stable Diffusion 3.x": [
|
||||
"SD 3",
|
||||
"SD 3.5",
|
||||
"SD 3.5 Medium",
|
||||
"SD 3.5 Large",
|
||||
"SD 3.5 Large Turbo",
|
||||
],
|
||||
"SDXL": ["SDXL 1.0", "SDXL Lightning", "SDXL Hyper"],
|
||||
"Flux Models": [
|
||||
"Flux.1 D",
|
||||
"Flux.1 S",
|
||||
"Flux.1 Krea",
|
||||
"Flux.1 Kontext",
|
||||
"Flux.2 D",
|
||||
"Flux.2 Klein 9B",
|
||||
"Flux.2 Klein 9B-base",
|
||||
"Flux.2 Klein 4B",
|
||||
"Flux.2 Klein 4B-base",
|
||||
],
|
||||
"Video Models": [
|
||||
"SVD",
|
||||
"LTXV",
|
||||
"LTXV2",
|
||||
"LTXV 2.3",
|
||||
"CogVideoX",
|
||||
"Mochi",
|
||||
"Hunyuan Video",
|
||||
"Wan Video",
|
||||
"Wan Video 1.3B t2v",
|
||||
"Wan Video 14B t2v",
|
||||
"Wan Video 14B i2v 480p",
|
||||
"Wan Video 14B i2v 720p",
|
||||
"Wan Video 2.2 TI2V-5B",
|
||||
"Wan Video 2.2 T2V-A14B",
|
||||
"Wan Video 2.2 I2V-A14B",
|
||||
"Wan Video 2.5 T2V",
|
||||
"Wan Video 2.5 I2V",
|
||||
],
|
||||
"Other Models": [
|
||||
"Illustrious",
|
||||
"Pony",
|
||||
"Pony V7",
|
||||
"HiDream",
|
||||
"Qwen",
|
||||
"AuraFlow",
|
||||
"Chroma",
|
||||
"ZImageTurbo",
|
||||
"ZImageBase",
|
||||
"PixArt a",
|
||||
"PixArt E",
|
||||
"Hunyuan 1",
|
||||
"Lumina",
|
||||
"Kolors",
|
||||
"NoobAI",
|
||||
"Anima",
|
||||
"Ernie",
|
||||
"Ernie Turbo",
|
||||
"Nucleus",
|
||||
],
|
||||
}
|
||||
|
||||
return categories
|
||||
|
||||
|
||||
# Convenience function for getting the singleton instance
|
||||
async def get_civitai_base_model_service() -> CivitaiBaseModelService:
|
||||
"""Get the singleton instance of CivitaiBaseModelService."""
|
||||
return await CivitaiBaseModelService.get_instance()
|
||||
@@ -2,7 +2,13 @@ import asyncio
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
from collections import OrderedDict
|
||||
from typing import Any, Optional, Dict, Tuple, List, Sequence
|
||||
from .connectivity_guard import (
|
||||
OFFLINE_FRIENDLY_MESSAGE,
|
||||
is_expected_offline_error,
|
||||
is_offline_cooldown_error,
|
||||
)
|
||||
from .model_metadata_provider import (
|
||||
CivitaiModelMetadataProvider,
|
||||
ModelMetadataProviderManager,
|
||||
@@ -39,7 +45,18 @@ class CivitaiClient:
|
||||
return
|
||||
self._initialized = True
|
||||
|
||||
self.base_url = "https://civitai.com/api/v1"
|
||||
self.base_url = "https://civitai.red/api/v1"
|
||||
# In-memory cache to avoid redundant get_model_version_info calls
|
||||
# within the same import/scan flow. Only successful results are cached.
|
||||
# Uses OrderedDict with LRU eviction at MAX_CACHE_ENTRIES to prevent
|
||||
# unbounded growth in long-running server processes.
|
||||
self._version_info_cache: OrderedDict[
|
||||
str, Tuple[Optional[Dict], Optional[str]]
|
||||
] = OrderedDict()
|
||||
self._MAX_CACHE_ENTRIES = 500
|
||||
|
||||
def _build_image_info_url(self, image_id: str) -> str:
|
||||
return f"{self.base_url}/images?imageId={image_id}&nsfw=X"
|
||||
|
||||
async def _make_request(
|
||||
self,
|
||||
@@ -49,20 +66,57 @@ class CivitaiClient:
|
||||
use_auth: bool = False,
|
||||
**kwargs,
|
||||
) -> Tuple[bool, Dict | str]:
|
||||
"""Wrapper around downloader.make_request that surfaces rate limits."""
|
||||
"""Wrapper around downloader.make_request that surfaces rate limits,
|
||||
with retry for transient server errors (5xx, Cloudflare 524, network flakiness)."""
|
||||
|
||||
downloader = await get_downloader()
|
||||
success, result = await downloader.make_request(
|
||||
method,
|
||||
url,
|
||||
use_auth=use_auth,
|
||||
**kwargs,
|
||||
)
|
||||
if not success and isinstance(result, RateLimitError):
|
||||
if result.provider is None:
|
||||
result.provider = "civitai_api"
|
||||
raise result
|
||||
return success, result
|
||||
max_retries = 3
|
||||
for attempt in range(max_retries):
|
||||
downloader = await get_downloader()
|
||||
success, result = await downloader.make_request(
|
||||
method,
|
||||
url,
|
||||
use_auth=use_auth,
|
||||
**kwargs,
|
||||
)
|
||||
if success:
|
||||
return True, result
|
||||
|
||||
if isinstance(result, RateLimitError):
|
||||
if result.provider is None:
|
||||
result.provider = "civitai_api"
|
||||
raise result
|
||||
|
||||
if is_offline_cooldown_error(result):
|
||||
return False, OFFLINE_FRIENDLY_MESSAGE
|
||||
|
||||
# Transient server error — retry with exponential backoff
|
||||
if self._is_transient_server_error(str(result)):
|
||||
if attempt < max_retries - 1:
|
||||
wait = 2**attempt # 1s, 2s, 4s
|
||||
logger.info(
|
||||
"Transient error on %s %s, retrying in %ds "
|
||||
"(attempt %d/%d): %s",
|
||||
method,
|
||||
url,
|
||||
wait,
|
||||
attempt + 1,
|
||||
max_retries,
|
||||
result,
|
||||
)
|
||||
await asyncio.sleep(wait)
|
||||
continue
|
||||
logger.warning(
|
||||
"All %d retries exhausted for %s %s: %s",
|
||||
max_retries,
|
||||
method,
|
||||
url,
|
||||
result,
|
||||
)
|
||||
return False, result
|
||||
|
||||
return False, result
|
||||
|
||||
return False, "Unexpected error in _make_request"
|
||||
|
||||
@staticmethod
|
||||
def _remove_comfy_metadata(model_version: Optional[Dict]) -> None:
|
||||
@@ -121,6 +175,8 @@ class CivitaiClient:
|
||||
)
|
||||
if not success:
|
||||
message = str(version)
|
||||
if is_expected_offline_error(message):
|
||||
return None, OFFLINE_FRIENDLY_MESSAGE
|
||||
if "not found" in message.lower():
|
||||
return None, "Model not found"
|
||||
|
||||
@@ -161,6 +217,9 @@ class CivitaiClient:
|
||||
return True
|
||||
return False
|
||||
except Exception as e:
|
||||
if is_expected_offline_error(str(e)):
|
||||
logger.debug("Preview download skipped due to offline state.")
|
||||
return False
|
||||
logger.error(f"Download Error: {str(e)}")
|
||||
return False
|
||||
|
||||
@@ -186,11 +245,36 @@ class CivitaiClient:
|
||||
|
||||
return _from_value(payload)
|
||||
|
||||
@staticmethod
|
||||
def _is_transient_server_error(message: str) -> bool:
|
||||
"""Return True when the message indicates a transient upstream failure.
|
||||
|
||||
Recognises Cloudflare 524, generic 5xx, and connectivity-level flakiness
|
||||
that should not be treated as a permanent failure.
|
||||
"""
|
||||
normalized = message.lower()
|
||||
if "status 5" in normalized or "status 524" in normalized:
|
||||
return True
|
||||
if any(
|
||||
keyword in normalized
|
||||
for keyword in (
|
||||
"connection refused",
|
||||
"connection reset",
|
||||
"temporary failure",
|
||||
"name resolution",
|
||||
"connection closed",
|
||||
)
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict]:
|
||||
"""Get all versions of a model with local availability info"""
|
||||
try:
|
||||
success, result = await self._make_request(
|
||||
"GET", f"{self.base_url}/models/{model_id}", use_auth=True
|
||||
"GET",
|
||||
f"{self.base_url}/models/{model_id}",
|
||||
use_auth=True,
|
||||
)
|
||||
if success:
|
||||
# Also return model type along with versions
|
||||
@@ -202,7 +286,17 @@ class CivitaiClient:
|
||||
message = self._extract_error_message(result)
|
||||
if message and "not found" in message.lower():
|
||||
raise ResourceNotFoundError(f"Resource not found for model {model_id}")
|
||||
if is_expected_offline_error(message):
|
||||
logger.info("Civitai request skipped: %s", OFFLINE_FRIENDLY_MESSAGE)
|
||||
return None
|
||||
if message:
|
||||
if self._is_transient_server_error(message):
|
||||
logger.info(
|
||||
"Transient server error for model %s: %s",
|
||||
model_id,
|
||||
message,
|
||||
)
|
||||
return None
|
||||
raise RuntimeError(message)
|
||||
return None
|
||||
except RateLimitError:
|
||||
@@ -237,7 +331,7 @@ class CivitaiClient:
|
||||
"GET",
|
||||
f"{self.base_url}/models",
|
||||
use_auth=True,
|
||||
params={"ids": query},
|
||||
params={"ids": query, "nsfw": "true"},
|
||||
)
|
||||
if not success:
|
||||
return None
|
||||
@@ -316,6 +410,25 @@ class CivitaiClient:
|
||||
return None
|
||||
|
||||
target_version = self._select_target_version(model_data, model_id, version_id)
|
||||
|
||||
# If modelVersions is empty (e.g. CivitAI cache lag for newly published
|
||||
# models) but a specific version_id is known, fall back to fetching the
|
||||
# version directly via the individual model-versions endpoint, then
|
||||
# enrich it with the model-level data we already have.
|
||||
if target_version is None and version_id is not None:
|
||||
logger.info(
|
||||
"modelVersions empty for model %s; falling back to direct "
|
||||
"version lookup for %s",
|
||||
model_id,
|
||||
version_id,
|
||||
)
|
||||
version = await self._fetch_version_by_id(version_id)
|
||||
if version:
|
||||
self._enrich_version_with_model_data(version, model_data)
|
||||
self._remove_comfy_metadata(version)
|
||||
return version
|
||||
return None
|
||||
|
||||
if target_version is None:
|
||||
return None
|
||||
|
||||
@@ -346,10 +459,14 @@ class CivitaiClient:
|
||||
|
||||
async def _fetch_model_data(self, model_id: int) -> Optional[Dict]:
|
||||
success, data = await self._make_request(
|
||||
"GET", f"{self.base_url}/models/{model_id}", use_auth=True
|
||||
"GET",
|
||||
f"{self.base_url}/models/{model_id}",
|
||||
use_auth=True,
|
||||
)
|
||||
if success:
|
||||
return data
|
||||
if is_expected_offline_error(data):
|
||||
return None
|
||||
logger.warning(f"Failed to fetch model data for model {model_id}")
|
||||
return None
|
||||
|
||||
@@ -358,10 +475,14 @@ class CivitaiClient:
|
||||
return None
|
||||
|
||||
success, version = await self._make_request(
|
||||
"GET", f"{self.base_url}/model-versions/{version_id}", use_auth=True
|
||||
"GET",
|
||||
f"{self.base_url}/model-versions/{version_id}",
|
||||
use_auth=True,
|
||||
)
|
||||
if success:
|
||||
return version
|
||||
if is_expected_offline_error(version):
|
||||
return None
|
||||
|
||||
logger.warning(f"Failed to fetch version by id {version_id}")
|
||||
return None
|
||||
@@ -371,10 +492,14 @@ class CivitaiClient:
|
||||
return None
|
||||
|
||||
success, version = await self._make_request(
|
||||
"GET", f"{self.base_url}/model-versions/by-hash/{model_hash}", use_auth=True
|
||||
"GET",
|
||||
f"{self.base_url}/model-versions/by-hash/{model_hash}",
|
||||
use_auth=True,
|
||||
)
|
||||
if success:
|
||||
return version
|
||||
if is_expected_offline_error(version):
|
||||
return None
|
||||
|
||||
logger.warning(f"Failed to fetch version by hash {model_hash}")
|
||||
return None
|
||||
@@ -450,20 +575,33 @@ class CivitaiClient:
|
||||
- The model version data or None if not found
|
||||
- An error message if there was an error, or None on success
|
||||
"""
|
||||
# In-memory cache avoids redundant API calls within the same
|
||||
# import/scan flow (e.g. _resolve_base_model_from_checkpoint
|
||||
# followed by _resolve_and_populate_checkpoint with the same id).
|
||||
if version_id in self._version_info_cache:
|
||||
logger.debug("Cache hit for model version info: %s", version_id)
|
||||
self._version_info_cache.move_to_end(version_id) # LRU bump
|
||||
return self._version_info_cache[version_id]
|
||||
|
||||
try:
|
||||
url = f"{self.base_url}/model-versions/{version_id}"
|
||||
|
||||
logger.debug(f"Resolving DNS for model version info: {url}")
|
||||
logger.debug("Resolving Civitai model version info: %s", url)
|
||||
success, result = await self._make_request("GET", url, use_auth=True)
|
||||
|
||||
if success:
|
||||
logger.debug(
|
||||
f"Successfully fetched model version info for: {version_id}"
|
||||
)
|
||||
logger.debug("Successfully fetched model version info for: %s", version_id)
|
||||
self._remove_comfy_metadata(result)
|
||||
self._version_info_cache[version_id] = (result, None)
|
||||
self._version_info_cache.move_to_end(version_id)
|
||||
# Evict oldest entry when over capacity
|
||||
if len(self._version_info_cache) > self._MAX_CACHE_ENTRIES:
|
||||
self._version_info_cache.popitem(last=False)
|
||||
return result, None
|
||||
|
||||
# Handle specific error cases
|
||||
if is_expected_offline_error(result):
|
||||
return None, OFFLINE_FRIENDLY_MESSAGE
|
||||
if "not found" in str(result):
|
||||
error_msg = f"Model not found"
|
||||
logger.warning(f"Model version not found: {version_id} - {error_msg}")
|
||||
@@ -479,47 +617,149 @@ class CivitaiClient:
|
||||
logger.error(error_msg)
|
||||
return None, error_msg
|
||||
|
||||
async def get_image_info(self, image_id: str) -> Optional[Dict]:
|
||||
async def get_image_info(
|
||||
self, image_id: str, source_url: str | None = None
|
||||
) -> Optional[Dict]:
|
||||
"""Fetch image information from Civitai API
|
||||
|
||||
Args:
|
||||
image_id: The Civitai image ID
|
||||
source_url: Original image page URL. Accepted for caller compatibility;
|
||||
API requests always target ``civitai.red``.
|
||||
|
||||
Returns:
|
||||
Optional[Dict]: The image data or None if not found
|
||||
"""
|
||||
try:
|
||||
url = f"{self.base_url}/images?imageId={image_id}&nsfw=X"
|
||||
|
||||
logger.debug(f"Fetching image info for ID: {image_id}")
|
||||
requested_id = int(image_id)
|
||||
url = self._build_image_info_url(image_id)
|
||||
success, result = await self._make_request("GET", url, use_auth=True)
|
||||
|
||||
if success:
|
||||
if result and "items" in result and len(result["items"]) > 0:
|
||||
logger.debug(f"Successfully fetched image info for ID: {image_id}")
|
||||
return result["items"][0]
|
||||
logger.warning(f"No image found with ID: {image_id}")
|
||||
if not success:
|
||||
if is_expected_offline_error(result):
|
||||
return None
|
||||
if self._is_transient_server_error(str(result)):
|
||||
logger.info(
|
||||
"Transient server error fetching image info for ID %s: %s",
|
||||
image_id,
|
||||
result,
|
||||
)
|
||||
return None
|
||||
logger.error(
|
||||
"Failed to fetch image info for ID %s from civitai.red: %s",
|
||||
image_id,
|
||||
result,
|
||||
)
|
||||
return None
|
||||
|
||||
logger.error(f"Failed to fetch image info for ID: {image_id}: {result}")
|
||||
if result and "items" in result and isinstance(result["items"], list):
|
||||
items = result["items"]
|
||||
|
||||
for item in items:
|
||||
if isinstance(item, dict) and item.get("id") == requested_id:
|
||||
logger.debug(
|
||||
"Successfully fetched image info for ID %s from civitai.red",
|
||||
image_id,
|
||||
)
|
||||
return item
|
||||
|
||||
returned_ids = [
|
||||
item.get("id")
|
||||
for item in items
|
||||
if isinstance(item, dict) and "id" in item
|
||||
]
|
||||
|
||||
logger.warning(
|
||||
"CivitAI API returned no matching image for requested ID %s from civitai.red. Returned %d item(s) with IDs: %s. This may indicate the image was deleted, hidden, or there is a database lag.",
|
||||
image_id,
|
||||
len(items),
|
||||
returned_ids,
|
||||
)
|
||||
return None
|
||||
|
||||
logger.warning("No image found with ID: %s", image_id)
|
||||
return None
|
||||
except RateLimitError:
|
||||
raise
|
||||
except ValueError as e:
|
||||
error_msg = f"Invalid image ID format: {image_id}"
|
||||
logger.error(error_msg)
|
||||
return None
|
||||
except Exception as e:
|
||||
error_msg = f"Error fetching image info: {e}"
|
||||
logger.error(error_msg)
|
||||
return None
|
||||
|
||||
async def get_model_versions_by_hashes(
|
||||
self, hashes: List[str]
|
||||
) -> Optional[List[Dict]]:
|
||||
"""Fetch full version details for up to 100 SHA256 hashes via the batch endpoint.
|
||||
|
||||
Uses POST /api/v1/model-versions/by-hash which returns full version
|
||||
details including ``usageControl`` and ``earlyAccessEndsAt`` that are
|
||||
not available from the model-level API.
|
||||
|
||||
Args:
|
||||
hashes: List of SHA256 hashes (max 100 per batch; auto-split).
|
||||
|
||||
Returns:
|
||||
List of version dicts or None on failure.
|
||||
"""
|
||||
if not hashes:
|
||||
return []
|
||||
|
||||
BATCH_SIZE = 100
|
||||
all_versions: List[Dict] = []
|
||||
|
||||
for start in range(0, len(hashes), BATCH_SIZE):
|
||||
batch = hashes[start : start + BATCH_SIZE]
|
||||
try:
|
||||
success, result = await self._make_request(
|
||||
"POST",
|
||||
f"{self.base_url}/model-versions/by-hash",
|
||||
use_auth=True,
|
||||
json=batch,
|
||||
)
|
||||
if not success:
|
||||
logger.warning(
|
||||
"Batch by-hash request failed for %d hashes: %s",
|
||||
len(batch),
|
||||
result,
|
||||
)
|
||||
continue
|
||||
|
||||
if isinstance(result, list):
|
||||
all_versions.extend(result)
|
||||
else:
|
||||
logger.debug(
|
||||
"Unexpected by-hash response type: %s", type(result)
|
||||
)
|
||||
except RateLimitError:
|
||||
raise
|
||||
except Exception as exc: # pragma: no cover - defensive logging
|
||||
logger.error(
|
||||
"Error fetching model versions by hashes: %s", exc
|
||||
)
|
||||
|
||||
return all_versions if all_versions else None
|
||||
|
||||
async def get_user_models(self, username: str) -> Optional[List[Dict]]:
|
||||
"""Fetch all models for a specific Civitai user."""
|
||||
if not username:
|
||||
return None
|
||||
|
||||
try:
|
||||
url = f"{self.base_url}/models?username={username}"
|
||||
success, result = await self._make_request("GET", url, use_auth=True)
|
||||
success, result = await self._make_request(
|
||||
"GET",
|
||||
f"{self.base_url}/models",
|
||||
use_auth=True,
|
||||
params={"username": username, "nsfw": "true"},
|
||||
)
|
||||
|
||||
if not success:
|
||||
if is_expected_offline_error(result):
|
||||
logger.info("User model fetch skipped: %s", OFFLINE_FRIENDLY_MESSAGE)
|
||||
return None
|
||||
logger.error("Failed to fetch models for %s: %s", username, result)
|
||||
return None
|
||||
|
||||
|
||||
204
py/services/connectivity_guard.py
Normal file
204
py/services/connectivity_guard.py
Normal file
@@ -0,0 +1,204 @@
|
||||
"""In-memory connectivity guard to suppress repeated network retries when offline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import errno
|
||||
import logging
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OFFLINE_COOLDOWN_ERROR = "offline_cooldown"
|
||||
OFFLINE_FRIENDLY_MESSAGE = "Network offline, will retry automatically later"
|
||||
|
||||
|
||||
def is_offline_cooldown_error(value: Any) -> bool:
|
||||
"""Return True when a response payload represents guard short-circuit."""
|
||||
return isinstance(value, str) and value == OFFLINE_COOLDOWN_ERROR
|
||||
|
||||
|
||||
def is_expected_offline_error(value: Any) -> bool:
|
||||
"""Return True when payload is an expected offline-related result."""
|
||||
if is_offline_cooldown_error(value):
|
||||
return True
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
normalized = value.lower()
|
||||
return "network offline" in normalized or "offline" in normalized
|
||||
|
||||
|
||||
class ConnectivityGuard:
|
||||
"""Tracks network failures and gates outbound requests during cooldown."""
|
||||
|
||||
_instance: "ConnectivityGuard | None" = None
|
||||
_instance_lock = asyncio.Lock()
|
||||
|
||||
@classmethod
|
||||
async def get_instance(cls) -> "ConnectivityGuard":
|
||||
async with cls._instance_lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = cls()
|
||||
return cls._instance
|
||||
|
||||
def __init__(self) -> None:
|
||||
if hasattr(self, "_initialized"):
|
||||
return
|
||||
self._initialized = True
|
||||
self._default_destination = "__global__"
|
||||
self._destination_states: dict[str, _DestinationState] = {
|
||||
self._default_destination: _DestinationState()
|
||||
}
|
||||
self.base_backoff_seconds = 30
|
||||
self.max_backoff_seconds = 300
|
||||
self.failure_threshold = 3
|
||||
|
||||
@property
|
||||
def online(self) -> bool:
|
||||
return self._state_for_destination(None).online
|
||||
|
||||
@online.setter
|
||||
def online(self, value: bool) -> None:
|
||||
self._state_for_destination(None).online = value
|
||||
|
||||
@property
|
||||
def failure_count(self) -> int:
|
||||
return self._state_for_destination(None).failure_count
|
||||
|
||||
@failure_count.setter
|
||||
def failure_count(self, value: int) -> None:
|
||||
self._state_for_destination(None).failure_count = value
|
||||
|
||||
@property
|
||||
def cooldown_until(self) -> datetime | None:
|
||||
return self._state_for_destination(None).cooldown_until
|
||||
|
||||
@cooldown_until.setter
|
||||
def cooldown_until(self, value: datetime | None) -> None:
|
||||
self._state_for_destination(None).cooldown_until = value
|
||||
|
||||
def _now(self) -> datetime:
|
||||
return datetime.now()
|
||||
|
||||
def _normalize_destination(self, destination: str | None) -> str:
|
||||
if destination is None or not destination.strip():
|
||||
return self._default_destination
|
||||
return destination.lower().strip()
|
||||
|
||||
def _state_for_destination(self, destination: str | None) -> "_DestinationState":
|
||||
destination_key = self._normalize_destination(destination)
|
||||
if destination_key not in self._destination_states:
|
||||
self._destination_states[destination_key] = _DestinationState()
|
||||
return self._destination_states[destination_key]
|
||||
|
||||
def in_cooldown(self, destination: str | None = None) -> bool:
|
||||
state = self._state_for_destination(destination)
|
||||
if state.cooldown_until is None:
|
||||
return False
|
||||
return self._now() < state.cooldown_until
|
||||
|
||||
def cooldown_remaining_seconds(self, destination: str | None = None) -> float:
|
||||
state = self._state_for_destination(destination)
|
||||
if state.cooldown_until is None:
|
||||
return 0.0
|
||||
return max(0.0, (state.cooldown_until - self._now()).total_seconds())
|
||||
|
||||
def should_block_request(self, destination: str | None = None) -> bool:
|
||||
return self.in_cooldown(destination)
|
||||
|
||||
def register_success(self, destination: str | None = None) -> None:
|
||||
destination_key = self._normalize_destination(destination)
|
||||
state = self._state_for_destination(destination_key)
|
||||
was_offline = (not state.online) or state.cooldown_until is not None
|
||||
state.online = True
|
||||
state.failure_count = 0
|
||||
state.cooldown_until = None
|
||||
if was_offline:
|
||||
logger.info(
|
||||
"Connectivity restored for destination '%s'; requests resumed.",
|
||||
destination_key,
|
||||
)
|
||||
|
||||
def register_network_failure(
|
||||
self, exc: Exception, destination: str | None = None
|
||||
) -> None:
|
||||
destination_key = self._normalize_destination(destination)
|
||||
state = self._state_for_destination(destination_key)
|
||||
state.online = False
|
||||
state.failure_count += 1
|
||||
|
||||
if state.failure_count < self.failure_threshold:
|
||||
logger.debug(
|
||||
"Network failure tracked for destination '%s' (%d/%d): %s",
|
||||
destination_key,
|
||||
state.failure_count,
|
||||
self.failure_threshold,
|
||||
exc,
|
||||
)
|
||||
return
|
||||
|
||||
retry_step = state.failure_count - self.failure_threshold
|
||||
backoff = min(
|
||||
self.max_backoff_seconds,
|
||||
self.base_backoff_seconds * (2**retry_step),
|
||||
)
|
||||
should_log_warning = not self.in_cooldown(destination_key)
|
||||
state.cooldown_until = self._now() + timedelta(seconds=backoff)
|
||||
|
||||
if should_log_warning:
|
||||
logger.warning(
|
||||
"Connectivity offline for destination '%s'; enter cooldown for %ss after %d network failures.",
|
||||
destination_key,
|
||||
int(backoff),
|
||||
state.failure_count,
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
"Cooldown still active for destination '%s'; failure_count=%d, backoff=%ss.",
|
||||
destination_key,
|
||||
state.failure_count,
|
||||
int(backoff),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def is_network_unreachable_error(exc: Exception) -> bool:
|
||||
"""Return whether the exception should count as connectivity failure."""
|
||||
if isinstance(exc, asyncio.CancelledError):
|
||||
return False
|
||||
|
||||
if isinstance(
|
||||
exc,
|
||||
(
|
||||
asyncio.TimeoutError,
|
||||
TimeoutError,
|
||||
ConnectionRefusedError,
|
||||
socket.gaierror,
|
||||
aiohttp.ServerTimeoutError,
|
||||
aiohttp.ConnectionTimeoutError,
|
||||
aiohttp.ClientConnectorError,
|
||||
aiohttp.ClientConnectionError,
|
||||
),
|
||||
):
|
||||
return True
|
||||
|
||||
if isinstance(exc, OSError) and exc.errno in {
|
||||
errno.ENETUNREACH,
|
||||
errno.EHOSTUNREACH,
|
||||
errno.ETIMEDOUT,
|
||||
errno.ECONNREFUSED,
|
||||
}:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
@dataclass
|
||||
class _DestinationState:
|
||||
online: bool = True
|
||||
failure_count: int = 0
|
||||
cooldown_until: datetime | None = None
|
||||
@@ -7,11 +7,13 @@ with category filtering and enriched results including post counts.
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_EMBEDDED_COMMAND_PATTERN = re.compile(r"\s/\w")
|
||||
class CustomWordsService:
|
||||
"""Service for autocomplete via TagFTSIndex.
|
||||
|
||||
@@ -77,12 +79,28 @@ class CustomWordsService:
|
||||
Returns:
|
||||
List of dicts with tag_name, category, and post_count.
|
||||
"""
|
||||
normalized_search = search_term.strip()
|
||||
if not normalized_search:
|
||||
return []
|
||||
|
||||
# Prompt widgets should only send the active token, but guard against
|
||||
# accidental full-prompt queries reaching the FTS path.
|
||||
if (
|
||||
"__" in normalized_search
|
||||
or "," in normalized_search
|
||||
or ">" in normalized_search
|
||||
or "\n" in normalized_search
|
||||
or "\r" in normalized_search
|
||||
or _EMBEDDED_COMMAND_PATTERN.search(normalized_search)
|
||||
):
|
||||
logger.debug("Skipping prompt-like custom words query: %s", normalized_search)
|
||||
return []
|
||||
|
||||
tag_index = self._get_tag_index()
|
||||
if tag_index is not None:
|
||||
results = tag_index.search(
|
||||
search_term, categories=categories, limit=limit, offset=offset
|
||||
return tag_index.search(
|
||||
normalized_search, categories=categories, limit=limit, offset=offset
|
||||
)
|
||||
return results
|
||||
|
||||
logger.debug("TagFTSIndex not available, returning empty results")
|
||||
return []
|
||||
|
||||
@@ -110,6 +110,23 @@ class DownloadCoordinator:
|
||||
|
||||
return result
|
||||
|
||||
async def skip_download(self, download_id: str) -> Dict[str, Any]:
|
||||
"""Skip a download while preserving all partial files on disk."""
|
||||
download_manager = await self._download_manager_factory()
|
||||
result = await download_manager.skip_download(download_id)
|
||||
|
||||
await self._ws_manager.broadcast_download_progress(
|
||||
download_id,
|
||||
{
|
||||
"status": "skipped",
|
||||
"progress": 0,
|
||||
"download_id": download_id,
|
||||
"message": "Download skipped by user (partial files preserved)",
|
||||
},
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
async def pause_download(self, download_id: str) -> Dict[str, Any]:
|
||||
"""Pause an active download and notify listeners."""
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
871
py/services/download_queue_service.py
Normal file
871
py/services/download_queue_service.py
Normal file
@@ -0,0 +1,871 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
from ..utils.cache_paths import get_cache_base_dir
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _resolve_database_path() -> str:
|
||||
base_dir = get_cache_base_dir(create=True)
|
||||
history_dir = os.path.join(base_dir, "download_history")
|
||||
os.makedirs(history_dir, exist_ok=True)
|
||||
return os.path.join(history_dir, "download_queue.sqlite")
|
||||
|
||||
|
||||
class DownloadQueueService:
|
||||
"""Persistent download queue and history manager backed by SQLite.
|
||||
|
||||
Provides a singleton interface for managing a download queue and
|
||||
corresponding history table, both stored in a single SQLite database
|
||||
under the cache directory.
|
||||
"""
|
||||
|
||||
_instance: Optional[DownloadQueueService] = None
|
||||
_class_lock: asyncio.Lock = asyncio.Lock()
|
||||
|
||||
_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS download_queue (
|
||||
download_id TEXT PRIMARY KEY,
|
||||
model_id INTEGER,
|
||||
model_version_id INTEGER,
|
||||
model_name TEXT NOT NULL DEFAULT '',
|
||||
version_name TEXT DEFAULT '',
|
||||
thumbnail_url TEXT DEFAULT '',
|
||||
source TEXT,
|
||||
file_params TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'queued',
|
||||
priority INTEGER DEFAULT 0,
|
||||
progress INTEGER DEFAULT 0,
|
||||
bytes_downloaded INTEGER DEFAULT 0,
|
||||
total_bytes INTEGER,
|
||||
bytes_per_second REAL DEFAULT 0.0,
|
||||
error TEXT,
|
||||
file_path TEXT,
|
||||
added_at REAL NOT NULL,
|
||||
started_at REAL,
|
||||
completed_at REAL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_dq_status ON download_queue(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_dq_added ON download_queue(added_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS download_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
download_id TEXT,
|
||||
model_id INTEGER,
|
||||
model_version_id INTEGER,
|
||||
model_name TEXT NOT NULL DEFAULT '',
|
||||
version_name TEXT DEFAULT '',
|
||||
thumbnail_url TEXT DEFAULT '',
|
||||
status TEXT NOT NULL,
|
||||
error TEXT,
|
||||
file_path TEXT,
|
||||
bytes_downloaded INTEGER DEFAULT 0,
|
||||
total_bytes INTEGER,
|
||||
completed_at REAL NOT NULL,
|
||||
is_already_exists INTEGER DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_dh_completed ON download_history(completed_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_dh_status ON download_history(status);
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
async def get_instance(cls) -> DownloadQueueService:
|
||||
"""Return the singleton instance, creating it if necessary."""
|
||||
async with cls._class_lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = cls()
|
||||
await cls._instance.deduplicate()
|
||||
return cls._instance
|
||||
|
||||
def __init__(self, db_path: Optional[str] = None) -> None:
|
||||
self._db_path = db_path or _resolve_database_path()
|
||||
self._lock = asyncio.Lock()
|
||||
self._conn: Optional[sqlite3.Connection] = None
|
||||
self._schema_initialized = False
|
||||
self._ensure_directory()
|
||||
self._initialize_schema()
|
||||
|
||||
def _ensure_directory(self) -> None:
|
||||
directory = os.path.dirname(self._db_path)
|
||||
if directory:
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(self._db_path, check_same_thread=False)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
def _get_conn(self) -> sqlite3.Connection:
|
||||
if self._conn is None:
|
||||
self._conn = sqlite3.connect(self._db_path, check_same_thread=False)
|
||||
self._conn.row_factory = sqlite3.Row
|
||||
return self._conn
|
||||
|
||||
def _initialize_schema(self) -> None:
|
||||
if self._schema_initialized:
|
||||
return
|
||||
with self._connect() as conn:
|
||||
conn.executescript(self._SCHEMA)
|
||||
conn.commit()
|
||||
self._schema_initialized = True
|
||||
|
||||
def get_database_path(self) -> str:
|
||||
"""Return the resolved database file path."""
|
||||
return self._db_path
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the persistent SQLite connection, if open.
|
||||
|
||||
This is called before plugin update operations to release the
|
||||
database file lock on Windows, allowing ``shutil.rmtree()`` to
|
||||
succeed when the cache resides inside the plugin directory.
|
||||
"""
|
||||
if self._conn is not None:
|
||||
try:
|
||||
self._conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
self._conn = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Queue methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def add_to_queue(
|
||||
self,
|
||||
download_id: str,
|
||||
model_id: Optional[int] = None,
|
||||
model_version_id: Optional[int] = None,
|
||||
model_name: str = "",
|
||||
version_name: str = "",
|
||||
thumbnail_url: str = "",
|
||||
source: Optional[str] = None,
|
||||
file_params: Optional[dict[str, Any]] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Insert a new download into the queue.
|
||||
|
||||
Returns the inserted row as a dict (or an empty dict if the
|
||||
download_id already exists).
|
||||
"""
|
||||
now = time.time()
|
||||
file_params_json = json.dumps(file_params) if file_params is not None else None
|
||||
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO download_queue (
|
||||
download_id, model_id, model_version_id, model_name,
|
||||
version_name, thumbnail_url, source, file_params,
|
||||
status, priority, added_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?)
|
||||
""",
|
||||
(
|
||||
download_id,
|
||||
model_id,
|
||||
model_version_id,
|
||||
model_name,
|
||||
version_name,
|
||||
thumbnail_url,
|
||||
source,
|
||||
file_params_json,
|
||||
now,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
row = conn.execute(
|
||||
"SELECT * FROM download_queue WHERE download_id = ?",
|
||||
(download_id,),
|
||||
).fetchone()
|
||||
|
||||
return dict(row) if row else {}
|
||||
|
||||
async def get_queue(self) -> list[dict[str, Any]]:
|
||||
"""Return all items in the queue ordered by priority then added time."""
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM download_queue ORDER BY priority DESC, added_at ASC"
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
async def get_queued_count(self) -> int:
|
||||
"""Return the number of items with status ``'queued'``."""
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) AS cnt FROM download_queue WHERE status = 'queued'"
|
||||
).fetchone()
|
||||
return row["cnt"] if row else 0
|
||||
|
||||
async def update_status(
|
||||
self,
|
||||
download_id: str,
|
||||
status: str,
|
||||
**extra: Any,
|
||||
) -> bool:
|
||||
"""Update the status and/or extra fields of a queue item.
|
||||
|
||||
Accepted extra keyword arguments:
|
||||
``progress``, ``error``, ``file_path``, ``bytes_downloaded``,
|
||||
``total_bytes``, ``bytes_per_second``.
|
||||
|
||||
Returns ``True`` if a row was updated.
|
||||
"""
|
||||
allowed_extra = {
|
||||
"progress",
|
||||
"error",
|
||||
"file_path",
|
||||
"bytes_downloaded",
|
||||
"total_bytes",
|
||||
"bytes_per_second",
|
||||
}
|
||||
|
||||
set_clauses: list[str] = ["status = ?"]
|
||||
params: list[Any] = [status]
|
||||
now = time.time()
|
||||
|
||||
if status in ("downloading",):
|
||||
set_clauses.append("started_at = COALESCE(started_at, ?)")
|
||||
params.append(now)
|
||||
if status in ("completed", "failed", "canceled"):
|
||||
set_clauses.append("completed_at = ?")
|
||||
params.append(now)
|
||||
|
||||
for key, value in extra.items():
|
||||
if key in allowed_extra:
|
||||
set_clauses.append(f"{key} = ?")
|
||||
params.append(value)
|
||||
|
||||
params.append(download_id)
|
||||
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
cursor = conn.execute(
|
||||
f"UPDATE download_queue SET {', '.join(set_clauses)} "
|
||||
"WHERE download_id = ?",
|
||||
params,
|
||||
)
|
||||
conn.commit()
|
||||
return cursor.rowcount > 0
|
||||
|
||||
async def remove_from_queue(self, download_id: str) -> bool:
|
||||
"""Remove a single item from the queue by download_id.
|
||||
|
||||
Returns ``True`` if a row was deleted.
|
||||
"""
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
cursor = conn.execute(
|
||||
"DELETE FROM download_queue WHERE download_id = ?",
|
||||
(download_id,),
|
||||
)
|
||||
conn.commit()
|
||||
return cursor.rowcount > 0
|
||||
|
||||
async def move_to_top(self, download_id: str) -> bool:
|
||||
"""Move an item to the front of the queue (highest priority).
|
||||
|
||||
Returns ``True`` if the item was found and updated.
|
||||
"""
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
row = conn.execute(
|
||||
"SELECT priority FROM download_queue WHERE download_id = ?",
|
||||
(download_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return False
|
||||
|
||||
max_row = conn.execute(
|
||||
"SELECT MAX(priority) AS mx FROM download_queue"
|
||||
).fetchone()
|
||||
max_priority: int = max_row["mx"] if max_row["mx"] is not None else 0
|
||||
|
||||
conn.execute(
|
||||
"UPDATE download_queue SET priority = ? WHERE download_id = ?",
|
||||
(max_priority + 1, download_id),
|
||||
)
|
||||
conn.commit()
|
||||
return True
|
||||
|
||||
async def move_to_end(self, download_id: str) -> bool:
|
||||
"""Move an item to the end of the queue (lowest priority).
|
||||
|
||||
Returns ``True`` if the item was found and updated.
|
||||
"""
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
row = conn.execute(
|
||||
"SELECT priority FROM download_queue WHERE download_id = ?",
|
||||
(download_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return False
|
||||
|
||||
min_row = conn.execute(
|
||||
"SELECT MIN(priority) AS mn FROM download_queue"
|
||||
).fetchone()
|
||||
min_priority: int = min_row["mn"] if min_row["mn"] is not None else 0
|
||||
|
||||
conn.execute(
|
||||
"UPDATE download_queue SET priority = ? WHERE download_id = ?",
|
||||
(min_priority - 1, download_id),
|
||||
)
|
||||
conn.commit()
|
||||
return True
|
||||
|
||||
async def clear_queue(self, status_filter: Optional[str] = None) -> int:
|
||||
"""Remove items from the queue.
|
||||
|
||||
When *status_filter* is provided only items with that status are
|
||||
deleted. Returns the number of deleted rows.
|
||||
"""
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
if status_filter is not None:
|
||||
cursor = conn.execute(
|
||||
"DELETE FROM download_queue WHERE status = ?",
|
||||
(status_filter,),
|
||||
)
|
||||
else:
|
||||
cursor = conn.execute("DELETE FROM download_queue")
|
||||
conn.commit()
|
||||
return cursor.rowcount
|
||||
|
||||
async def complete_download(
|
||||
self,
|
||||
download_id: str,
|
||||
status: str = "completed",
|
||||
error: Optional[str] = None,
|
||||
file_path: Optional[str] = None,
|
||||
bytes_downloaded: int = 0,
|
||||
total_bytes: Optional[int] = None,
|
||||
completed_at: Optional[float] = None,
|
||||
) -> Optional[dict[str, Any]]:
|
||||
"""Atomically move a download from the queue into the history table.
|
||||
|
||||
Looks up the queue record by ``download_id``, deletes it from the
|
||||
queue, and inserts a corresponding history entry with the given
|
||||
terminal status (``completed``, ``failed``, or ``canceled``).
|
||||
|
||||
When *completed_at* is provided it is used as the completion
|
||||
timestamp; otherwise ``time.time()`` is used.
|
||||
|
||||
Returns the original queue record (before deletion) on success,
|
||||
or ``None`` if the download was not found in the queue.
|
||||
"""
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
row = conn.execute(
|
||||
"SELECT * FROM download_queue WHERE download_id = ?",
|
||||
(download_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
now = completed_at if completed_at is not None else time.time()
|
||||
conn.execute(
|
||||
"DELETE FROM download_queue WHERE download_id = ?",
|
||||
(download_id,),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO download_history (
|
||||
download_id, model_id, model_version_id, model_name,
|
||||
version_name, thumbnail_url, status, error, file_path,
|
||||
bytes_downloaded, total_bytes, completed_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
row["download_id"],
|
||||
row["model_id"],
|
||||
row["model_version_id"],
|
||||
row["model_name"],
|
||||
row["version_name"],
|
||||
row["thumbnail_url"],
|
||||
status,
|
||||
error,
|
||||
file_path,
|
||||
bytes_downloaded,
|
||||
total_bytes,
|
||||
now,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
return dict(row)
|
||||
|
||||
async def pop_next_download(self) -> Optional[dict[str, Any]]:
|
||||
"""Atomically fetch and mark the next queued item as ``downloading``.
|
||||
|
||||
The item with the highest priority (and earliest ``added_at``
|
||||
among ties) whose status is ``'queued'`` is selected, set to
|
||||
``'downloading'``, and returned as a dict. Returns ``None`` if
|
||||
the queue is empty.
|
||||
"""
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT * FROM download_queue
|
||||
WHERE status = 'queued'
|
||||
ORDER BY priority DESC, added_at ASC
|
||||
LIMIT 1
|
||||
"""
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
download_id = row["download_id"]
|
||||
now = time.time()
|
||||
conn.execute(
|
||||
"UPDATE download_queue SET status = 'downloading', "
|
||||
"started_at = COALESCE(started_at, ?) "
|
||||
"WHERE download_id = ?",
|
||||
(now, download_id),
|
||||
)
|
||||
conn.commit()
|
||||
updated = conn.execute(
|
||||
"SELECT * FROM download_queue WHERE download_id = ?",
|
||||
(download_id,),
|
||||
).fetchone()
|
||||
|
||||
return dict(updated) if updated else None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# History methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def add_to_history(
|
||||
self,
|
||||
download_id: Optional[str] = None,
|
||||
model_id: Optional[int] = None,
|
||||
model_version_id: Optional[int] = None,
|
||||
model_name: str = "",
|
||||
version_name: str = "",
|
||||
thumbnail_url: str = "",
|
||||
status: str = "completed",
|
||||
error: Optional[str] = None,
|
||||
file_path: Optional[str] = None,
|
||||
bytes_downloaded: int = 0,
|
||||
total_bytes: Optional[int] = None,
|
||||
is_already_exists: int = 0,
|
||||
) -> int:
|
||||
"""Insert a record into the download history.
|
||||
|
||||
Returns the ``id`` (AUTOINCREMENT primary key) of the newly
|
||||
inserted row.
|
||||
"""
|
||||
now = time.time()
|
||||
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
cursor = conn.execute(
|
||||
"""
|
||||
INSERT INTO download_history (
|
||||
download_id, model_id, model_version_id, model_name,
|
||||
version_name, thumbnail_url, status, error, file_path,
|
||||
bytes_downloaded, total_bytes, completed_at, is_already_exists
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
download_id,
|
||||
model_id,
|
||||
model_version_id,
|
||||
model_name,
|
||||
version_name,
|
||||
thumbnail_url,
|
||||
status,
|
||||
error,
|
||||
file_path,
|
||||
bytes_downloaded,
|
||||
total_bytes,
|
||||
now,
|
||||
is_already_exists,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
return cursor.lastrowid or 0
|
||||
|
||||
async def get_history(
|
||||
self,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
status_filter: Optional[str] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Return a page of download history entries.
|
||||
|
||||
Returns a dict with keys ``items``, ``total``, ``limit``, and
|
||||
``offset``.
|
||||
"""
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
|
||||
if status_filter is not None:
|
||||
count_row = conn.execute(
|
||||
"SELECT COUNT(*) AS cnt FROM download_history WHERE status = ?",
|
||||
(status_filter,),
|
||||
).fetchone()
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM download_history WHERE status = ? "
|
||||
"ORDER BY completed_at DESC LIMIT ? OFFSET ?",
|
||||
(status_filter, limit, offset),
|
||||
).fetchall()
|
||||
else:
|
||||
count_row = conn.execute(
|
||||
"SELECT COUNT(*) AS cnt FROM download_history"
|
||||
).fetchone()
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM download_history "
|
||||
"ORDER BY completed_at DESC LIMIT ? OFFSET ?",
|
||||
(limit, offset),
|
||||
).fetchall()
|
||||
|
||||
return {
|
||||
"items": [dict(row) for row in rows],
|
||||
"total": count_row["cnt"] if count_row else 0,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
}
|
||||
|
||||
async def delete_history_item(self, id: int) -> bool:
|
||||
"""Delete a single history entry by its *id*.
|
||||
|
||||
Returns ``True`` if a row was deleted.
|
||||
"""
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
cursor = conn.execute(
|
||||
"DELETE FROM download_history WHERE id = ?",
|
||||
(id,),
|
||||
)
|
||||
conn.commit()
|
||||
return cursor.rowcount > 0
|
||||
|
||||
async def clear_history(
|
||||
self,
|
||||
status_filter: Optional[str] = None,
|
||||
before_timestamp: Optional[float] = None,
|
||||
) -> int:
|
||||
"""Remove history entries matching the optional filters.
|
||||
|
||||
Both ``status_filter`` and ``before_timestamp`` can be combined
|
||||
(AND logic). Returns the number of deleted rows.
|
||||
"""
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
|
||||
clauses: list[str] = []
|
||||
params: list[Any] = []
|
||||
|
||||
if status_filter is not None:
|
||||
clauses.append("status = ?")
|
||||
params.append(status_filter)
|
||||
if before_timestamp is not None:
|
||||
clauses.append("completed_at < ?")
|
||||
params.append(before_timestamp)
|
||||
|
||||
where = ""
|
||||
if clauses:
|
||||
where = " WHERE " + " AND ".join(clauses)
|
||||
|
||||
cursor = conn.execute(
|
||||
f"DELETE FROM download_history{where}",
|
||||
params,
|
||||
)
|
||||
conn.commit()
|
||||
return cursor.rowcount
|
||||
|
||||
async def get_history_count(self, status_filter: Optional[str] = None) -> int:
|
||||
"""Return the number of history entries, optionally filtered by status."""
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
if status_filter is not None:
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) AS cnt FROM download_history WHERE status = ?",
|
||||
(status_filter,),
|
||||
).fetchone()
|
||||
else:
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) AS cnt FROM download_history"
|
||||
).fetchone()
|
||||
return row["cnt"] if row else 0
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Retry
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def retry_from_history(self, item_id: int) -> Optional[dict[str, Any]]:
|
||||
"""Re-queue a failed or canceled download from history.
|
||||
|
||||
Looks up the history record by its primary key. If the status is
|
||||
``failed`` or ``canceled`` a new queue entry is created with the
|
||||
same model metadata and a fresh download id, and the original
|
||||
history entry is **deleted** to prevent exponential growth when
|
||||
the retried item is later canceled or fails again and re-retried.
|
||||
"""
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
row = conn.execute(
|
||||
"SELECT * FROM download_history WHERE id = ?",
|
||||
(item_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
status = str(row["status"])
|
||||
if status not in ("failed", "canceled"):
|
||||
return None
|
||||
|
||||
import uuid
|
||||
|
||||
new_id = str(uuid.uuid4())
|
||||
now = time.time()
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO download_queue (
|
||||
download_id, model_id, model_version_id, model_name,
|
||||
version_name, thumbnail_url, source, file_params,
|
||||
status, priority, added_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, NULL, 'queued', 0, ?)
|
||||
""",
|
||||
(
|
||||
new_id,
|
||||
row["model_id"],
|
||||
row["model_version_id"],
|
||||
row["model_name"],
|
||||
row["version_name"],
|
||||
row["thumbnail_url"],
|
||||
"retry",
|
||||
now,
|
||||
),
|
||||
)
|
||||
conn.execute(
|
||||
"DELETE FROM download_history WHERE id = ?",
|
||||
(item_id,),
|
||||
)
|
||||
conn.commit()
|
||||
queued = conn.execute(
|
||||
"SELECT * FROM download_queue WHERE download_id = ?",
|
||||
(new_id,),
|
||||
).fetchone()
|
||||
|
||||
return dict(queued) if queued else None
|
||||
|
||||
async def retry_all_failed(self) -> int:
|
||||
"""Re-queue all failed and canceled downloads from history.
|
||||
|
||||
Each history entry is **deleted** after being re-queued so that
|
||||
repeated retry-all calls do not cause exponential growth.
|
||||
|
||||
Returns the number of items that were re-queued.
|
||||
"""
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM download_history WHERE status IN ('failed', 'canceled')"
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
import uuid
|
||||
|
||||
now = time.time()
|
||||
count = 0
|
||||
for row in rows:
|
||||
new_id = str(uuid.uuid4())
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO download_queue (
|
||||
download_id, model_id, model_version_id, model_name,
|
||||
version_name, thumbnail_url, source, file_params,
|
||||
status, priority, added_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, NULL, 'queued', 0, ?)
|
||||
""",
|
||||
(
|
||||
new_id,
|
||||
row["model_id"],
|
||||
row["model_version_id"],
|
||||
row["model_name"],
|
||||
row["version_name"],
|
||||
row["thumbnail_url"],
|
||||
"retry",
|
||||
now,
|
||||
),
|
||||
)
|
||||
conn.execute(
|
||||
"DELETE FROM download_history WHERE id = ?",
|
||||
(row["id"],),
|
||||
)
|
||||
count += 1
|
||||
conn.commit()
|
||||
|
||||
return count
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Stats
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def get_stats(self) -> dict[str, int]:
|
||||
"""Return aggregate counts across both tables.
|
||||
|
||||
Returns a dict with keys ``queued``, ``downloading``, ``paused``
|
||||
(all from the queue table) and ``completed``, ``failed``,
|
||||
``canceled`` (all from the history table).
|
||||
"""
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
|
||||
queue_rows = conn.execute(
|
||||
"SELECT status, COUNT(*) AS cnt FROM download_queue GROUP BY status"
|
||||
).fetchall()
|
||||
queue_stats: dict[str, int] = {}
|
||||
for row in queue_rows:
|
||||
queue_stats[str(row["status"])] = row["cnt"]
|
||||
|
||||
history_rows = conn.execute(
|
||||
"SELECT status, COUNT(*) AS cnt FROM download_history GROUP BY status"
|
||||
).fetchall()
|
||||
history_stats: dict[str, int] = {}
|
||||
for row in history_rows:
|
||||
history_stats[str(row["status"])] = row["cnt"]
|
||||
|
||||
return {
|
||||
"queued": queue_stats.get("queued", 0),
|
||||
"downloading": queue_stats.get("downloading", 0),
|
||||
"paused": queue_stats.get("paused", 0),
|
||||
"completed": history_stats.get("completed", 0),
|
||||
"failed": history_stats.get("failed", 0),
|
||||
"canceled": history_stats.get("canceled", 0),
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Deduplication (one-time cleanup for bug #980)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def deduplicate(self) -> dict[str, int]:
|
||||
"""Remove duplicate entries caused by the retry-amplification bug.
|
||||
|
||||
The bug (issue #980) caused the same download to appear N times in
|
||||
both the queue and history tables when ``retry_all_failed`` was
|
||||
called repeatedly without deleting the original history rows.
|
||||
|
||||
This method is called **once** when the singleton is first created.
|
||||
It is idempotent — after the first run there will be no duplicates
|
||||
to remove, so subsequent calls are a no-op.
|
||||
|
||||
Returns a dict with the count of removed rows per table.
|
||||
"""
|
||||
result: dict[str, int] = {
|
||||
"removed_history": 0,
|
||||
"removed_queue": 0,
|
||||
"removed_orphan_queue": 0,
|
||||
}
|
||||
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
|
||||
# 1. History: for each (model_id, model_version_id, status) triplet
|
||||
# keep only the row with the highest id (most recently inserted).
|
||||
conn.execute("""
|
||||
DELETE FROM download_history
|
||||
WHERE id NOT IN (
|
||||
SELECT MAX(id)
|
||||
FROM download_history
|
||||
GROUP BY model_id, model_version_id, status
|
||||
)
|
||||
""")
|
||||
result["removed_history"] = conn.execute(
|
||||
"SELECT changes()"
|
||||
).fetchone()[0]
|
||||
|
||||
# 2. Cross-status dedup: for each (model_id, model_version_id),
|
||||
# keep only the entry with the highest-priority terminal status.
|
||||
# Priority: completed (3) > failed (2) > canceled (1).
|
||||
# This prevents the same model version from having both a
|
||||
# 'failed' and a 'canceled' entry (or a 'completed' alongside
|
||||
# either) after the bug-created duplicates are removed.
|
||||
conn.execute("""
|
||||
DELETE FROM download_history
|
||||
WHERE id NOT IN (
|
||||
SELECT dh.id
|
||||
FROM download_history dh
|
||||
INNER JOIN (
|
||||
SELECT model_id, model_version_id,
|
||||
MAX(CASE status
|
||||
WHEN 'completed' THEN 3
|
||||
WHEN 'failed' THEN 2
|
||||
WHEN 'canceled' THEN 1
|
||||
ELSE 0
|
||||
END) AS best_prio
|
||||
FROM download_history
|
||||
GROUP BY model_id, model_version_id
|
||||
) best
|
||||
ON dh.model_id = best.model_id
|
||||
AND dh.model_version_id = best.model_version_id
|
||||
AND CASE dh.status
|
||||
WHEN 'completed' THEN 3
|
||||
WHEN 'failed' THEN 2
|
||||
WHEN 'canceled' THEN 1
|
||||
ELSE 0
|
||||
END = best.best_prio
|
||||
GROUP BY dh.model_id, dh.model_version_id
|
||||
HAVING dh.id = MAX(dh.id)
|
||||
)
|
||||
""")
|
||||
result["removed_history"] += conn.execute(
|
||||
"SELECT changes()"
|
||||
).fetchone()[0]
|
||||
|
||||
# 3. Queue: for each (model_id, model_version_id) keep only the
|
||||
# row with the latest added_at (most recently enqueued).
|
||||
conn.execute("""
|
||||
DELETE FROM download_queue
|
||||
WHERE rowid NOT IN (
|
||||
SELECT MAX(rowid)
|
||||
FROM download_queue
|
||||
WHERE status IN ('queued', 'downloading', 'paused', 'waiting')
|
||||
GROUP BY model_id, model_version_id
|
||||
)
|
||||
AND status IN ('queued', 'downloading', 'paused', 'waiting')
|
||||
""")
|
||||
result["removed_queue"] = conn.execute(
|
||||
"SELECT changes()"
|
||||
).fetchone()[0]
|
||||
|
||||
# 4. Remove orphaned queue entries — items that were re-queued
|
||||
# (source='retry') but whose model version already has a
|
||||
# terminal history entry. These are artifacts of the buggy
|
||||
# retry cycle that were never cleaned up.
|
||||
conn.execute("""
|
||||
DELETE FROM download_queue
|
||||
WHERE source = 'retry'
|
||||
AND (model_id, model_version_id) IN (
|
||||
SELECT model_id, model_version_id
|
||||
FROM download_history
|
||||
WHERE status IN ('failed', 'canceled')
|
||||
)
|
||||
AND status IN ('queued', 'waiting')
|
||||
""")
|
||||
result["removed_orphan_queue"] = conn.execute(
|
||||
"SELECT changes()"
|
||||
).fetchone()[0]
|
||||
|
||||
conn.commit()
|
||||
|
||||
logger.info(
|
||||
"Deduplicate: removed %s history rows, %s queue rows, "
|
||||
"%s orphaned queue rows",
|
||||
result["removed_history"],
|
||||
result["removed_queue"],
|
||||
result["removed_orphan_queue"],
|
||||
)
|
||||
return result
|
||||
335
py/services/downloaded_version_history_service.py
Normal file
335
py/services/downloaded_version_history_service.py
Normal file
@@ -0,0 +1,335 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import time
|
||||
from typing import Iterable, Mapping, Optional, Sequence
|
||||
|
||||
from ..utils.cache_paths import get_cache_base_dir
|
||||
from .settings_manager import get_settings_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _normalize_model_type(model_type: str | None) -> Optional[str]:
|
||||
if not isinstance(model_type, str):
|
||||
return None
|
||||
normalized = model_type.strip().lower()
|
||||
if normalized in {"lora", "locon", "dora"}:
|
||||
return "lora"
|
||||
if normalized == "checkpoint":
|
||||
return "checkpoint"
|
||||
if normalized in {"embedding", "textualinversion"}:
|
||||
return "embedding"
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_int(value) -> Optional[int]:
|
||||
try:
|
||||
if value is None:
|
||||
return None
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_database_path() -> str:
|
||||
base_dir = get_cache_base_dir(create=True)
|
||||
history_dir = os.path.join(base_dir, "download_history")
|
||||
os.makedirs(history_dir, exist_ok=True)
|
||||
return os.path.join(history_dir, "downloaded_versions.sqlite")
|
||||
|
||||
|
||||
class DownloadedVersionHistoryService:
|
||||
_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS downloaded_model_versions (
|
||||
model_type TEXT NOT NULL,
|
||||
version_id INTEGER NOT NULL,
|
||||
model_id INTEGER,
|
||||
first_seen_at REAL NOT NULL,
|
||||
last_seen_at REAL NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
last_file_path TEXT,
|
||||
last_library_name TEXT,
|
||||
is_deleted_override INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (model_type, version_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_downloaded_model_versions_model
|
||||
ON downloaded_model_versions(model_type, model_id);
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str | None = None, *, settings_manager=None) -> None:
|
||||
self._db_path = db_path or _resolve_database_path()
|
||||
self._settings = settings_manager or get_settings_manager()
|
||||
self._lock = asyncio.Lock()
|
||||
self._conn: sqlite3.Connection | None = None
|
||||
self._schema_initialized = False
|
||||
self._ensure_directory()
|
||||
self._initialize_schema()
|
||||
|
||||
def _ensure_directory(self) -> None:
|
||||
directory = os.path.dirname(self._db_path)
|
||||
if directory:
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(self._db_path, check_same_thread=False)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
def _get_conn(self) -> sqlite3.Connection:
|
||||
if self._conn is None:
|
||||
self._conn = sqlite3.connect(self._db_path, check_same_thread=False)
|
||||
self._conn.row_factory = sqlite3.Row
|
||||
return self._conn
|
||||
|
||||
def _initialize_schema(self) -> None:
|
||||
if self._schema_initialized:
|
||||
return
|
||||
with self._connect() as conn:
|
||||
conn.executescript(self._SCHEMA)
|
||||
conn.commit()
|
||||
self._schema_initialized = True
|
||||
|
||||
def get_database_path(self) -> str:
|
||||
return self._db_path
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the persistent SQLite connection, if open.
|
||||
|
||||
This is called before plugin update operations to release the
|
||||
database file lock on Windows, allowing ``shutil.rmtree()`` to
|
||||
succeed when the cache resides inside the plugin directory.
|
||||
"""
|
||||
if self._conn is not None:
|
||||
try:
|
||||
self._conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
self._conn = None
|
||||
|
||||
def _get_active_library_name(self) -> str | None:
|
||||
try:
|
||||
value = self._settings.get_active_library_name()
|
||||
except Exception:
|
||||
return None
|
||||
return value or None
|
||||
|
||||
async def mark_downloaded(
|
||||
self,
|
||||
model_type: str,
|
||||
version_id: int,
|
||||
*,
|
||||
model_id: int | None = None,
|
||||
source: str = "manual",
|
||||
file_path: str | None = None,
|
||||
library_name: str | None = None,
|
||||
) -> None:
|
||||
normalized_type = _normalize_model_type(model_type)
|
||||
normalized_version_id = _normalize_int(version_id)
|
||||
normalized_model_id = _normalize_int(model_id)
|
||||
if normalized_type is None or normalized_version_id is None:
|
||||
return
|
||||
|
||||
active_library_name = library_name or self._get_active_library_name()
|
||||
timestamp = time.time()
|
||||
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO downloaded_model_versions (
|
||||
model_type, version_id, model_id, first_seen_at, last_seen_at,
|
||||
source, last_file_path, last_library_name, is_deleted_override
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0)
|
||||
ON CONFLICT(model_type, version_id) DO UPDATE SET
|
||||
model_id = COALESCE(excluded.model_id, downloaded_model_versions.model_id),
|
||||
last_seen_at = excluded.last_seen_at,
|
||||
source = excluded.source,
|
||||
last_file_path = COALESCE(excluded.last_file_path, downloaded_model_versions.last_file_path),
|
||||
last_library_name = COALESCE(excluded.last_library_name, downloaded_model_versions.last_library_name),
|
||||
is_deleted_override = 0
|
||||
""",
|
||||
(
|
||||
normalized_type,
|
||||
normalized_version_id,
|
||||
normalized_model_id,
|
||||
timestamp,
|
||||
timestamp,
|
||||
source,
|
||||
file_path,
|
||||
active_library_name,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
async def mark_downloaded_bulk(
|
||||
self,
|
||||
model_type: str,
|
||||
records: Sequence[Mapping[str, object]],
|
||||
*,
|
||||
source: str = "scan",
|
||||
library_name: str | None = None,
|
||||
) -> None:
|
||||
normalized_type = _normalize_model_type(model_type)
|
||||
if normalized_type is None or not records:
|
||||
return
|
||||
|
||||
timestamp = time.time()
|
||||
active_library_name = library_name or self._get_active_library_name()
|
||||
payload: list[tuple[object, ...]] = []
|
||||
for record in records:
|
||||
version_id = _normalize_int(record.get("version_id"))
|
||||
if version_id is None:
|
||||
continue
|
||||
payload.append(
|
||||
(
|
||||
normalized_type,
|
||||
version_id,
|
||||
_normalize_int(record.get("model_id")),
|
||||
timestamp,
|
||||
timestamp,
|
||||
source,
|
||||
record.get("file_path"),
|
||||
active_library_name,
|
||||
)
|
||||
)
|
||||
|
||||
if not payload:
|
||||
return
|
||||
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
conn.executemany(
|
||||
"""
|
||||
INSERT INTO downloaded_model_versions (
|
||||
model_type, version_id, model_id, first_seen_at, last_seen_at,
|
||||
source, last_file_path, last_library_name, is_deleted_override
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0)
|
||||
ON CONFLICT(model_type, version_id) DO UPDATE SET
|
||||
model_id = COALESCE(excluded.model_id, downloaded_model_versions.model_id),
|
||||
last_seen_at = excluded.last_seen_at,
|
||||
source = excluded.source,
|
||||
last_file_path = COALESCE(excluded.last_file_path, downloaded_model_versions.last_file_path),
|
||||
last_library_name = COALESCE(excluded.last_library_name, downloaded_model_versions.last_library_name),
|
||||
is_deleted_override = 0
|
||||
""",
|
||||
payload,
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
async def mark_as_deleted(self, model_type: str, version_id: int) -> None:
|
||||
normalized_type = _normalize_model_type(model_type)
|
||||
normalized_version_id = _normalize_int(version_id)
|
||||
if normalized_type is None or normalized_version_id is None:
|
||||
return
|
||||
|
||||
timestamp = time.time()
|
||||
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO downloaded_model_versions (
|
||||
model_type, version_id, model_id, first_seen_at, last_seen_at,
|
||||
source, last_file_path, last_library_name, is_deleted_override
|
||||
) VALUES (?, ?, NULL, ?, ?, 'manual', NULL, ?, 1)
|
||||
ON CONFLICT(model_type, version_id) DO UPDATE SET
|
||||
last_seen_at = excluded.last_seen_at,
|
||||
source = excluded.source,
|
||||
last_library_name = COALESCE(excluded.last_library_name, downloaded_model_versions.last_library_name),
|
||||
is_deleted_override = 1
|
||||
""",
|
||||
(
|
||||
normalized_type,
|
||||
normalized_version_id,
|
||||
timestamp,
|
||||
timestamp,
|
||||
self._get_active_library_name(),
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
async def has_been_downloaded(self, model_type: str, version_id: int) -> bool:
|
||||
normalized_type = _normalize_model_type(model_type)
|
||||
normalized_version_id = _normalize_int(version_id)
|
||||
if normalized_type is None or normalized_version_id is None:
|
||||
return False
|
||||
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT is_deleted_override
|
||||
FROM downloaded_model_versions
|
||||
WHERE model_type = ? AND version_id = ?
|
||||
""",
|
||||
(normalized_type, normalized_version_id),
|
||||
).fetchone()
|
||||
return bool(row) and not bool(row["is_deleted_override"])
|
||||
|
||||
async def get_downloaded_version_ids(
|
||||
self, model_type: str, model_id: int
|
||||
) -> list[int]:
|
||||
normalized_type = _normalize_model_type(model_type)
|
||||
normalized_model_id = _normalize_int(model_id)
|
||||
if normalized_type is None or normalized_model_id is None:
|
||||
return []
|
||||
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT version_id
|
||||
FROM downloaded_model_versions
|
||||
WHERE model_type = ? AND model_id = ? AND is_deleted_override = 0
|
||||
ORDER BY version_id ASC
|
||||
""",
|
||||
(normalized_type, normalized_model_id),
|
||||
).fetchall()
|
||||
return [int(row["version_id"]) for row in rows]
|
||||
|
||||
async def get_downloaded_version_ids_bulk(
|
||||
self, model_type: str, model_ids: Iterable[int]
|
||||
) -> dict[int, set[int]]:
|
||||
normalized_type = _normalize_model_type(model_type)
|
||||
if normalized_type is None:
|
||||
return {}
|
||||
|
||||
normalized_model_ids = sorted(
|
||||
{
|
||||
value
|
||||
for value in (_normalize_int(model_id) for model_id in model_ids)
|
||||
if value is not None
|
||||
}
|
||||
)
|
||||
if not normalized_model_ids:
|
||||
return {}
|
||||
|
||||
placeholders = ", ".join(["?"] * len(normalized_model_ids))
|
||||
params: list[object] = [normalized_type, *normalized_model_ids]
|
||||
|
||||
async with self._lock:
|
||||
conn = self._get_conn()
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT model_id, version_id
|
||||
FROM downloaded_model_versions
|
||||
WHERE model_type = ?
|
||||
AND model_id IN ({placeholders})
|
||||
AND is_deleted_override = 0
|
||||
""",
|
||||
params,
|
||||
).fetchall()
|
||||
|
||||
result: dict[int, set[int]] = {}
|
||||
for row in rows:
|
||||
model_id = _normalize_int(row["model_id"])
|
||||
version_id = _normalize_int(row["version_id"])
|
||||
if model_id is None or version_id is None:
|
||||
continue
|
||||
result.setdefault(model_id, set()).add(version_id)
|
||||
return result
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,6 +3,7 @@ import logging
|
||||
from typing import Dict
|
||||
|
||||
from .base_model_service import BaseModelService
|
||||
from .auto_tag_service import extract_auto_tags
|
||||
from ..utils.models import EmbeddingMetadata
|
||||
from ..config import config
|
||||
|
||||
@@ -42,9 +43,12 @@ class EmbeddingService(BaseModelService):
|
||||
"notes": embedding_data.get("notes", ""),
|
||||
"sub_type": sub_type,
|
||||
"favorite": embedding_data.get("favorite", False),
|
||||
"exclude": bool(embedding_data.get("exclude", False)),
|
||||
"update_available": bool(embedding_data.get("update_available", False)),
|
||||
"skip_metadata_refresh": bool(embedding_data.get("skip_metadata_refresh", False)),
|
||||
"civitai": self.filter_civitai_data(embedding_data.get("civitai", {}), minimal=True)
|
||||
"civitai": self.filter_civitai_data(embedding_data.get("civitai", {}), minimal=True),
|
||||
"auto_tags": embedding_data.get("auto_tags") or extract_auto_tags(embedding_data),
|
||||
"version_count": embedding_data.get("version_count"),
|
||||
}
|
||||
|
||||
def find_duplicate_hashes(self) -> Dict:
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import os
|
||||
import logging
|
||||
import json
|
||||
import os
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .base_model_service import BaseModelService
|
||||
from .model_query import resolve_sub_type
|
||||
from .auto_tag_service import extract_auto_tags
|
||||
from ..utils.models import LoraMetadata
|
||||
from ..config import config
|
||||
|
||||
@@ -27,7 +29,7 @@ class LoraService(BaseModelService):
|
||||
# Resolve sub_type using priority: sub_type > model_type > civitai.model.type > default
|
||||
# Normalize to lowercase for consistent API responses
|
||||
sub_type = resolve_sub_type(lora_data).lower()
|
||||
|
||||
|
||||
return {
|
||||
"model_name": lora_data["model_name"],
|
||||
"file_name": lora_data["file_name"],
|
||||
@@ -47,12 +49,17 @@ class LoraService(BaseModelService):
|
||||
"usage_tips": lora_data.get("usage_tips", ""),
|
||||
"notes": lora_data.get("notes", ""),
|
||||
"favorite": lora_data.get("favorite", False),
|
||||
"exclude": bool(lora_data.get("exclude", False)),
|
||||
"update_available": bool(lora_data.get("update_available", False)),
|
||||
"skip_metadata_refresh": bool(lora_data.get("skip_metadata_refresh", False)),
|
||||
"skip_metadata_refresh": bool(
|
||||
lora_data.get("skip_metadata_refresh", False)
|
||||
),
|
||||
"sub_type": sub_type,
|
||||
"civitai": self.filter_civitai_data(
|
||||
lora_data.get("civitai", {}), minimal=True
|
||||
),
|
||||
"auto_tags": lora_data.get("auto_tags") or extract_auto_tags(lora_data),
|
||||
"version_count": lora_data.get("version_count"),
|
||||
}
|
||||
|
||||
async def _apply_specific_filters(self, data: List[Dict], **kwargs) -> List[Dict]:
|
||||
@@ -62,6 +69,68 @@ class LoraService(BaseModelService):
|
||||
if first_letter:
|
||||
data = self._filter_by_first_letter(data, first_letter)
|
||||
|
||||
# Handle name pattern filters
|
||||
name_pattern_include = kwargs.get("name_pattern_include", [])
|
||||
name_pattern_exclude = kwargs.get("name_pattern_exclude", [])
|
||||
name_pattern_use_regex = kwargs.get("name_pattern_use_regex", False)
|
||||
|
||||
if name_pattern_include or name_pattern_exclude:
|
||||
import re
|
||||
|
||||
def matches_pattern(name, pattern, use_regex):
|
||||
"""Check if name matches pattern (regex or substring)"""
|
||||
if not name:
|
||||
return False
|
||||
if use_regex:
|
||||
try:
|
||||
return bool(re.search(pattern, name, re.IGNORECASE))
|
||||
except re.error:
|
||||
# Invalid regex, fall back to substring match
|
||||
return pattern.lower() in name.lower()
|
||||
else:
|
||||
return pattern.lower() in name.lower()
|
||||
|
||||
def matches_any_pattern(name, patterns, use_regex):
|
||||
"""Check if name matches any of the patterns"""
|
||||
if not patterns:
|
||||
return True
|
||||
return any(matches_pattern(name, p, use_regex) for p in patterns)
|
||||
|
||||
filtered = []
|
||||
for lora in data:
|
||||
model_name = lora.get("model_name", "")
|
||||
file_name = lora.get("file_name", "")
|
||||
names_to_check = [n for n in [model_name, file_name] if n]
|
||||
|
||||
# Check exclude patterns first
|
||||
excluded = False
|
||||
if name_pattern_exclude:
|
||||
for name in names_to_check:
|
||||
if matches_any_pattern(
|
||||
name, name_pattern_exclude, name_pattern_use_regex
|
||||
):
|
||||
excluded = True
|
||||
break
|
||||
|
||||
if excluded:
|
||||
continue
|
||||
|
||||
# Check include patterns
|
||||
if name_pattern_include:
|
||||
included = False
|
||||
for name in names_to_check:
|
||||
if matches_any_pattern(
|
||||
name, name_pattern_include, name_pattern_use_regex
|
||||
):
|
||||
included = True
|
||||
break
|
||||
if not included:
|
||||
continue
|
||||
|
||||
filtered.append(lora)
|
||||
|
||||
data = filtered
|
||||
|
||||
return data
|
||||
|
||||
def _filter_by_first_letter(self, data: List[Dict], letter: str) -> List[Dict]:
|
||||
@@ -214,6 +283,57 @@ class LoraService(BaseModelService):
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_recommended_strength_from_lora_data(lora_data: Dict) -> Optional[float]:
|
||||
"""Parse usage_tips JSON and extract recommended model strength."""
|
||||
try:
|
||||
usage_tips = lora_data.get("usage_tips", "")
|
||||
if not usage_tips:
|
||||
return None
|
||||
tips_data = json.loads(usage_tips)
|
||||
return tips_data.get("strength")
|
||||
except (json.JSONDecodeError, TypeError, AttributeError):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_recommended_clip_strength_from_lora_data(
|
||||
lora_data: Dict,
|
||||
) -> Optional[float]:
|
||||
"""Parse usage_tips JSON and extract recommended clip strength."""
|
||||
try:
|
||||
usage_tips = lora_data.get("usage_tips", "")
|
||||
if not usage_tips:
|
||||
return None
|
||||
tips_data = json.loads(usage_tips)
|
||||
return tips_data.get("clipStrength")
|
||||
except (json.JSONDecodeError, TypeError, AttributeError):
|
||||
return None
|
||||
|
||||
async def get_lora_metadata_by_filename(self, filename: str) -> Optional[Dict]:
|
||||
"""Return cached raw metadata for a LoRA matching the given filename."""
|
||||
cache = await self.scanner.get_cached_data(force_refresh=False)
|
||||
|
||||
fn_normalized = filename.replace("\\", "/")
|
||||
fn_no_ext = fn_normalized
|
||||
for ext in (".safetensors", ".ckpt", ".pt", ".bin"):
|
||||
if fn_no_ext.lower().endswith(ext):
|
||||
fn_no_ext = fn_no_ext[: -len(ext)]
|
||||
break
|
||||
|
||||
for lora in cache.raw_data if cache else []:
|
||||
file_name = lora.get("file_name", "")
|
||||
folder = lora.get("folder", "")
|
||||
file_name_no_ext = file_name
|
||||
for ext in (".safetensors", ".ckpt", ".pt", ".bin"):
|
||||
if file_name_no_ext.lower().endswith(ext):
|
||||
file_name_no_ext = file_name_no_ext[: -len(ext)]
|
||||
break
|
||||
path_name = f"{folder}/{file_name_no_ext}".replace("\\", "/") if folder else file_name_no_ext
|
||||
if fn_no_ext in (file_name_no_ext, path_name):
|
||||
return lora
|
||||
|
||||
return None
|
||||
|
||||
def find_duplicate_hashes(self) -> Dict:
|
||||
"""Find LoRAs with duplicate SHA256 hashes"""
|
||||
return self.scanner._hash_index.get_duplicate_hashes()
|
||||
@@ -264,34 +384,10 @@ class LoraService(BaseModelService):
|
||||
List of LoRA dicts with randomized strengths
|
||||
"""
|
||||
import random
|
||||
import json
|
||||
|
||||
# Use a local Random instance to avoid affecting global random state
|
||||
# This ensures each execution with a different seed produces different results
|
||||
rng = random.Random(seed)
|
||||
|
||||
def get_recommended_strength(lora_data: Dict) -> Optional[float]:
|
||||
"""Parse usage_tips JSON and extract recommended strength"""
|
||||
try:
|
||||
usage_tips = lora_data.get("usage_tips", "")
|
||||
if not usage_tips:
|
||||
return None
|
||||
tips_data = json.loads(usage_tips)
|
||||
return tips_data.get("strength")
|
||||
except (json.JSONDecodeError, TypeError, AttributeError):
|
||||
return None
|
||||
|
||||
def get_recommended_clip_strength(lora_data: Dict) -> Optional[float]:
|
||||
"""Parse usage_tips JSON and extract recommended clip strength"""
|
||||
try:
|
||||
usage_tips = lora_data.get("usage_tips", "")
|
||||
if not usage_tips:
|
||||
return None
|
||||
tips_data = json.loads(usage_tips)
|
||||
return tips_data.get("clipStrength")
|
||||
except (json.JSONDecodeError, TypeError, AttributeError):
|
||||
return None
|
||||
|
||||
if locked_loras is None:
|
||||
locked_loras = []
|
||||
|
||||
@@ -321,7 +417,10 @@ class LoraService(BaseModelService):
|
||||
locked_loras = locked_loras[:target_count]
|
||||
|
||||
# Filter out locked LoRAs from available pool
|
||||
locked_names = {lora["name"] for lora in locked_loras}
|
||||
locked_names = {
|
||||
os.path.basename(lora["name"]) if "/" in str(lora.get("name", "")) else lora["name"]
|
||||
for lora in locked_loras
|
||||
}
|
||||
available_pool = [
|
||||
l for l in available_loras if l["file_name"] not in locked_names
|
||||
]
|
||||
@@ -339,7 +438,9 @@ class LoraService(BaseModelService):
|
||||
result_loras = []
|
||||
for lora in selected:
|
||||
if use_recommended_strength:
|
||||
recommended_strength = get_recommended_strength(lora)
|
||||
recommended_strength = self.get_recommended_strength_from_lora_data(
|
||||
lora
|
||||
)
|
||||
if recommended_strength is not None:
|
||||
scale = rng.uniform(
|
||||
recommended_strength_scale_min, recommended_strength_scale_max
|
||||
@@ -357,7 +458,9 @@ class LoraService(BaseModelService):
|
||||
if use_same_clip_strength:
|
||||
clip_str = model_str
|
||||
elif use_recommended_strength:
|
||||
recommended_clip_strength = get_recommended_clip_strength(lora)
|
||||
recommended_clip_strength = (
|
||||
self.get_recommended_clip_strength_from_lora_data(lora)
|
||||
)
|
||||
if recommended_clip_strength is not None:
|
||||
scale = rng.uniform(
|
||||
recommended_strength_scale_min, recommended_strength_scale_max
|
||||
@@ -368,13 +471,11 @@ class LoraService(BaseModelService):
|
||||
rng.uniform(clip_strength_min, clip_strength_max), 2
|
||||
)
|
||||
else:
|
||||
clip_str = round(
|
||||
rng.uniform(clip_strength_min, clip_strength_max), 2
|
||||
)
|
||||
clip_str = round(rng.uniform(clip_strength_min, clip_strength_max), 2)
|
||||
|
||||
result_loras.append(
|
||||
{
|
||||
"name": lora["file_name"],
|
||||
"name": f"{lora['folder']}/{lora['file_name']}" if lora.get("folder") else lora["file_name"],
|
||||
"strength": model_str,
|
||||
"clipStrength": clip_str,
|
||||
"active": True,
|
||||
@@ -485,12 +586,69 @@ class LoraService(BaseModelService):
|
||||
if bool(lora.get("license_flags", 127) & (1 << 1))
|
||||
]
|
||||
|
||||
# Apply name pattern filters
|
||||
name_patterns = filter_section.get("namePatterns", {})
|
||||
include_patterns = name_patterns.get("include", [])
|
||||
exclude_patterns = name_patterns.get("exclude", [])
|
||||
use_regex = name_patterns.get("useRegex", False)
|
||||
|
||||
if include_patterns or exclude_patterns:
|
||||
import re
|
||||
|
||||
def matches_pattern(name, pattern, use_regex):
|
||||
"""Check if name matches pattern (regex or substring)"""
|
||||
if not name:
|
||||
return False
|
||||
if use_regex:
|
||||
try:
|
||||
return bool(re.search(pattern, name, re.IGNORECASE))
|
||||
except re.error:
|
||||
# Invalid regex, fall back to substring match
|
||||
return pattern.lower() in name.lower()
|
||||
else:
|
||||
return pattern.lower() in name.lower()
|
||||
|
||||
def matches_any_pattern(name, patterns, use_regex):
|
||||
"""Check if name matches any of the patterns"""
|
||||
if not patterns:
|
||||
return True
|
||||
return any(matches_pattern(name, p, use_regex) for p in patterns)
|
||||
|
||||
filtered = []
|
||||
for lora in available_loras:
|
||||
model_name = lora.get("model_name", "")
|
||||
file_name = lora.get("file_name", "")
|
||||
names_to_check = [n for n in [model_name, file_name] if n]
|
||||
|
||||
# Check exclude patterns first
|
||||
excluded = False
|
||||
if exclude_patterns:
|
||||
for name in names_to_check:
|
||||
if matches_any_pattern(name, exclude_patterns, use_regex):
|
||||
excluded = True
|
||||
break
|
||||
|
||||
if excluded:
|
||||
continue
|
||||
|
||||
# Check include patterns
|
||||
if include_patterns:
|
||||
included = False
|
||||
for name in names_to_check:
|
||||
if matches_any_pattern(name, include_patterns, use_regex):
|
||||
included = True
|
||||
break
|
||||
if not included:
|
||||
continue
|
||||
|
||||
filtered.append(lora)
|
||||
|
||||
available_loras = filtered
|
||||
|
||||
return available_loras
|
||||
|
||||
async def get_cycler_list(
|
||||
self,
|
||||
pool_config: Optional[Dict] = None,
|
||||
sort_by: str = "filename"
|
||||
self, pool_config: Optional[Dict] = None, sort_by: str = "filename"
|
||||
) -> List[Dict]:
|
||||
"""
|
||||
Get filtered and sorted LoRA list for cycling.
|
||||
@@ -516,19 +674,26 @@ class LoraService(BaseModelService):
|
||||
if sort_by == "model_name":
|
||||
available_loras = sorted(
|
||||
available_loras,
|
||||
key=lambda x: (x.get("model_name") or x.get("file_name", "")).lower()
|
||||
key=lambda x: (
|
||||
(x.get("model_name") or x.get("file_name", "")).lower(),
|
||||
x.get("file_path", "").lower(),
|
||||
),
|
||||
)
|
||||
else: # Default to filename
|
||||
available_loras = sorted(
|
||||
available_loras,
|
||||
key=lambda x: x.get("file_name", "").lower()
|
||||
key=lambda x: (
|
||||
x.get("file_name", "").lower(),
|
||||
x.get("file_path", "").lower(),
|
||||
),
|
||||
)
|
||||
|
||||
# Return minimal data needed for cycling
|
||||
return [
|
||||
{
|
||||
"file_name": lora["file_name"],
|
||||
"file_name": f"{lora['folder']}/{lora['file_name']}" if lora.get("folder") else lora["file_name"],
|
||||
"model_name": lora.get("model_name", lora["file_name"]),
|
||||
"folder": lora.get("folder", ""),
|
||||
}
|
||||
for lora in available_loras
|
||||
]
|
||||
|
||||
@@ -122,11 +122,25 @@ async def get_metadata_provider(provider_name: str = None):
|
||||
|
||||
provider_manager = await ModelMetadataProviderManager.get_instance()
|
||||
|
||||
provider = (
|
||||
provider_manager._get_provider(provider_name)
|
||||
if provider_name
|
||||
else provider_manager._get_provider()
|
||||
)
|
||||
try:
|
||||
provider = (
|
||||
provider_manager._get_provider(provider_name)
|
||||
if provider_name
|
||||
else provider_manager._get_provider()
|
||||
)
|
||||
except ValueError as e:
|
||||
# Provider not initialized, attempt to initialize
|
||||
if "No default provider set" in str(e) or "not registered" in str(e):
|
||||
logger.warning(f"Metadata provider not initialized ({e}), initializing now...")
|
||||
await initialize_metadata_providers()
|
||||
provider_manager = await ModelMetadataProviderManager.get_instance()
|
||||
provider = (
|
||||
provider_manager._get_provider(provider_name)
|
||||
if provider_name
|
||||
else provider_manager._get_provider()
|
||||
)
|
||||
else:
|
||||
raise
|
||||
|
||||
return _wrap_provider_with_rate_limit(provider_name, provider)
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ from typing import Any, Awaitable, Callable, Dict, Iterable, Optional
|
||||
from ..services.settings_manager import SettingsManager
|
||||
from ..utils.civitai_utils import resolve_license_payload
|
||||
from ..utils.model_utils import determine_base_model
|
||||
from .connectivity_guard import OFFLINE_FRIENDLY_MESSAGE, is_expected_offline_error
|
||||
from .errors import RateLimitError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -215,13 +216,19 @@ class MetadataSyncService:
|
||||
provider_used: Optional[str] = None
|
||||
last_error: Optional[str] = None
|
||||
civitai_api_not_found = False
|
||||
any_rate_limited = False
|
||||
|
||||
for provider_name, provider in provider_attempts:
|
||||
try:
|
||||
civitai_metadata_candidate, error = await provider.get_model_by_hash(sha256)
|
||||
except RateLimitError as exc:
|
||||
exc.provider = exc.provider or (provider_name or provider.__class__.__name__)
|
||||
raise
|
||||
logger.warning(
|
||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||
provider_name or provider.__class__.__name__,
|
||||
exc.retry_after or 0,
|
||||
)
|
||||
any_rate_limited = True
|
||||
continue
|
||||
except Exception as exc: # pragma: no cover - defensive logging
|
||||
logger.error("Provider %s failed for hash %s: %s", provider_name, sha256, exc)
|
||||
civitai_metadata_candidate, error = None, str(exc)
|
||||
@@ -257,6 +264,14 @@ class MetadataSyncService:
|
||||
model_data["last_checked_at"] = datetime.now().timestamp()
|
||||
needs_save = True
|
||||
|
||||
# When the model was already classified as "not on CivitAI" via
|
||||
# .metadata.json (civitai_deleted=True) but the SQLite cache is
|
||||
# stale (because the pre-fix code never persisted these flags),
|
||||
# ensure the flags are written to the scanner cache + SQLite.
|
||||
if not needs_save and model_data.get("civitai_deleted") is True:
|
||||
model_data["last_checked_at"] = datetime.now().timestamp()
|
||||
needs_save = True
|
||||
|
||||
# Save metadata if any state was updated
|
||||
if needs_save:
|
||||
data_to_save = model_data.copy()
|
||||
@@ -265,6 +280,7 @@ class MetadataSyncService:
|
||||
if "last_checked_at" not in data_to_save:
|
||||
data_to_save["last_checked_at"] = datetime.now().timestamp()
|
||||
await self._metadata_manager.save_metadata(file_path, data_to_save)
|
||||
await update_cache_func(file_path, file_path, data_to_save)
|
||||
|
||||
default_error = (
|
||||
"CivitAI model is deleted and metadata archive DB is not enabled"
|
||||
@@ -274,11 +290,19 @@ class MetadataSyncService:
|
||||
else "No provider returned metadata"
|
||||
)
|
||||
|
||||
resolved_error = last_error or default_error
|
||||
if any_rate_limited and "Rate limited" not in resolved_error:
|
||||
resolved_error = "Rate limited"
|
||||
if is_expected_offline_error(resolved_error):
|
||||
resolved_error = OFFLINE_FRIENDLY_MESSAGE
|
||||
|
||||
error_msg = (
|
||||
f"Error fetching metadata: {last_error or default_error} "
|
||||
f"(model_name={model_data.get('model_name', '')})"
|
||||
f"Error fetching metadata: {resolved_error} "
|
||||
f"(file={os.path.basename(file_path)}, sha256={sha256})"
|
||||
)
|
||||
logger.error(error_msg)
|
||||
# Use case layer (BulkMetadataRefreshUseCase) logs failed models at WARNING level,
|
||||
# so this level is demoted to DEBUG to avoid duplicate user-visible logging.
|
||||
logger.debug(error_msg)
|
||||
return False, error_msg
|
||||
|
||||
model_data["from_civitai"] = True
|
||||
@@ -347,6 +371,9 @@ class MetadataSyncService:
|
||||
return False, error_msg
|
||||
except Exception as exc: # pragma: no cover - error path
|
||||
error_msg = f"Error fetching metadata: {exc}"
|
||||
if is_expected_offline_error(str(exc)):
|
||||
logger.info(OFFLINE_FRIENDLY_MESSAGE)
|
||||
return False, OFFLINE_FRIENDLY_MESSAGE
|
||||
logger.error(error_msg, exc_info=True)
|
||||
return False, error_msg
|
||||
|
||||
@@ -400,7 +427,18 @@ class MetadataSyncService:
|
||||
metadata = await metadata_loader(metadata_path)
|
||||
|
||||
for key, value in updates.items():
|
||||
if isinstance(value, dict) and isinstance(metadata.get(key), dict):
|
||||
if key == "tags" and isinstance(value, list):
|
||||
# Normalize tags: trim, lowercase, deduplicate
|
||||
normalized = []
|
||||
seen = set()
|
||||
for tag in value:
|
||||
if isinstance(tag, str):
|
||||
t = tag.strip().lower()
|
||||
if t and t not in seen:
|
||||
normalized.append(t)
|
||||
seen.add(t)
|
||||
metadata[key] = normalized
|
||||
elif isinstance(value, dict) and isinstance(metadata.get(key), dict):
|
||||
metadata[key].update(value)
|
||||
else:
|
||||
metadata[key] = value
|
||||
|
||||
@@ -18,6 +18,8 @@ SUPPORTED_SORT_MODES = [
|
||||
('size', 'desc'),
|
||||
('usage', 'asc'),
|
||||
('usage', 'desc'),
|
||||
('versions_count', 'asc'),
|
||||
('versions_count', 'desc'),
|
||||
]
|
||||
# Is this in use?
|
||||
|
||||
@@ -221,33 +223,56 @@ class ModelCache:
|
||||
start_time = time.perf_counter()
|
||||
reverse = (order == 'desc')
|
||||
if sort_key == 'name':
|
||||
# Natural sort by configured display name, case-insensitive
|
||||
# Natural sort by configured display name, case-insensitive, with file_path as tie-breaker
|
||||
result = natsorted(
|
||||
data,
|
||||
key=lambda x: self._get_display_name(x).lower(),
|
||||
key=lambda x: (
|
||||
self._get_display_name(x).lower(),
|
||||
x.get('file_path', '').lower()
|
||||
),
|
||||
reverse=reverse
|
||||
)
|
||||
elif sort_key == 'date':
|
||||
# Sort by modified timestamp (use .get() with default to handle missing fields)
|
||||
# Sort by modified timestamp, fallback to name and path for stability
|
||||
result = sorted(
|
||||
data,
|
||||
key=lambda x: x.get('modified', 0.0),
|
||||
key=lambda x: (
|
||||
x.get('modified', 0.0),
|
||||
self._get_display_name(x).lower(),
|
||||
x.get('file_path', '').lower()
|
||||
),
|
||||
reverse=reverse
|
||||
)
|
||||
elif sort_key == 'size':
|
||||
# Sort by file size (use .get() with default to handle missing fields)
|
||||
# Sort by file size, fallback to name and path for stability
|
||||
result = sorted(
|
||||
data,
|
||||
key=lambda x: x.get('size', 0),
|
||||
key=lambda x: (
|
||||
x.get('size', 0),
|
||||
self._get_display_name(x).lower(),
|
||||
x.get('file_path', '').lower()
|
||||
),
|
||||
reverse=reverse
|
||||
)
|
||||
elif sort_key == 'usage':
|
||||
# Sort by usage count, fallback to 0, then name for stability
|
||||
# Sort by usage count, fallback to 0, then name and path for stability
|
||||
return sorted(
|
||||
data,
|
||||
key=lambda x: (
|
||||
x.get('usage_count', 0),
|
||||
self._get_display_name(x).lower()
|
||||
self._get_display_name(x).lower(),
|
||||
x.get('file_path', '').lower()
|
||||
),
|
||||
reverse=reverse
|
||||
)
|
||||
elif sort_key == 'versions_count':
|
||||
# Pre-dedup sort: fall back to name sort.
|
||||
# Actual re-sort by version_count happens in get_paginated_data after dedup.
|
||||
result = natsorted(
|
||||
data,
|
||||
key=lambda x: (
|
||||
self._get_display_name(x).lower(),
|
||||
x.get('file_path', '').lower()
|
||||
),
|
||||
reverse=reverse
|
||||
)
|
||||
|
||||
@@ -7,6 +7,7 @@ class ModelHashIndex:
|
||||
def __init__(self):
|
||||
self._hash_to_path: Dict[str, str] = {}
|
||||
self._filename_to_hash: Dict[str, str] = {}
|
||||
self._autov2_to_path: Dict[str, str] = {}
|
||||
# New data structures for tracking duplicates
|
||||
self._duplicate_hashes: Dict[str, List[str]] = {} # sha256 -> list of paths
|
||||
self._duplicate_filenames: Dict[str, List[str]] = {} # filename -> list of paths
|
||||
@@ -63,6 +64,9 @@ class ModelHashIndex:
|
||||
# Add new mappings
|
||||
self._hash_to_path[sha256] = file_path
|
||||
self._filename_to_hash[filename] = sha256
|
||||
# AutoV2 = first 10 chars of SHA256
|
||||
if len(sha256) >= 10:
|
||||
self._autov2_to_path[sha256[:10]] = file_path
|
||||
|
||||
def _get_filename_from_path(self, file_path: str) -> str:
|
||||
"""Extract filename without extension from path"""
|
||||
@@ -79,6 +83,12 @@ class ModelHashIndex:
|
||||
hash_val = h
|
||||
break
|
||||
|
||||
if hash_val is None:
|
||||
for h, paths in self._duplicate_hashes.items():
|
||||
if file_path in paths:
|
||||
hash_val = h
|
||||
break
|
||||
|
||||
# If we didn't find a hash, nothing to do
|
||||
if not hash_val:
|
||||
return
|
||||
@@ -151,7 +161,12 @@ class ModelHashIndex:
|
||||
del self._duplicate_filenames[filename]
|
||||
if filename in self._filename_to_hash:
|
||||
del self._filename_to_hash[filename]
|
||||
|
||||
|
||||
# Remove from AutoV2 index
|
||||
autov2_keys_to_remove = [k for k, v in self._autov2_to_path.items() if v == file_path]
|
||||
for k in autov2_keys_to_remove:
|
||||
del self._autov2_to_path[k]
|
||||
|
||||
def remove_by_hash(self, sha256: str) -> None:
|
||||
"""Remove entry by hash"""
|
||||
sha256 = sha256.lower()
|
||||
@@ -171,6 +186,10 @@ class ModelHashIndex:
|
||||
# Remove hash-to-path mapping
|
||||
del self._hash_to_path[sha256]
|
||||
|
||||
autov2_key = sha256[:10]
|
||||
if autov2_key in self._autov2_to_path:
|
||||
del self._autov2_to_path[autov2_key]
|
||||
|
||||
# Update filename-to-hash and duplicate filenames for all paths
|
||||
for path_to_remove in paths_to_remove:
|
||||
fname = self._get_filename_from_path(path_to_remove)
|
||||
@@ -189,13 +208,24 @@ class ModelHashIndex:
|
||||
# If only one entry remains, it's no longer a duplicate
|
||||
del self._duplicate_filenames[fname]
|
||||
|
||||
def has_hash(self, sha256: str) -> bool:
|
||||
"""Check if hash exists in index"""
|
||||
return sha256.lower() in self._hash_to_path
|
||||
|
||||
def get_path(self, sha256: str) -> Optional[str]:
|
||||
"""Get file path for a hash"""
|
||||
return self._hash_to_path.get(sha256.lower())
|
||||
def has_hash(self, hash_value: str) -> bool:
|
||||
"""Check if hash exists in index (SHA256 or AutoV2)"""
|
||||
normalized = hash_value.lower()
|
||||
if normalized in self._hash_to_path:
|
||||
return True
|
||||
if len(normalized) == 10:
|
||||
return normalized in self._autov2_to_path
|
||||
return False
|
||||
|
||||
def get_path(self, hash_value: str) -> Optional[str]:
|
||||
"""Get file path for a hash (SHA256 or AutoV2)"""
|
||||
normalized = hash_value.lower()
|
||||
path = self._hash_to_path.get(normalized)
|
||||
if path is not None:
|
||||
return path
|
||||
if len(normalized) == 10:
|
||||
return self._autov2_to_path.get(normalized)
|
||||
return None
|
||||
|
||||
def get_hash(self, file_path: str) -> Optional[str]:
|
||||
"""Get hash for a file path"""
|
||||
@@ -203,13 +233,16 @@ class ModelHashIndex:
|
||||
return self._filename_to_hash.get(filename)
|
||||
|
||||
def get_hash_by_filename(self, filename: str) -> Optional[str]:
|
||||
"""Get hash for a filename without extension"""
|
||||
"""Get hash for a filename (bare basename or path-prefixed name)"""
|
||||
if "/" in filename or "\\" in filename:
|
||||
filename = os.path.splitext(os.path.basename(filename.replace("\\", "/")))[0]
|
||||
return self._filename_to_hash.get(filename)
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear all entries"""
|
||||
self._hash_to_path.clear()
|
||||
self._filename_to_hash.clear()
|
||||
self._autov2_to_path.clear()
|
||||
self._duplicate_hashes.clear()
|
||||
self._duplicate_filenames.clear()
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import Any, Awaitable, Callable, Dict, Iterable, List, Mapping, Opti
|
||||
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
from ..utils.constants import PREVIEW_EXTENSIONS
|
||||
from ..utils.metadata_manager import MetadataManager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -110,6 +111,11 @@ class ModelLifecycleService:
|
||||
self._scanner._hash_index.remove_by_path(file_path)
|
||||
|
||||
await self._sync_update_for_model(model_id)
|
||||
|
||||
persist_current_cache = getattr(self._scanner, "_persist_current_cache", None)
|
||||
if callable(persist_current_cache):
|
||||
await persist_current_cache()
|
||||
|
||||
return {"success": True, "deleted_files": deleted_files}
|
||||
|
||||
@staticmethod
|
||||
@@ -207,11 +213,56 @@ class ModelLifecycleService:
|
||||
|
||||
excluded = getattr(self._scanner, "_excluded_models", None)
|
||||
if isinstance(excluded, list):
|
||||
excluded.append(file_path)
|
||||
if file_path not in excluded:
|
||||
excluded.append(file_path)
|
||||
|
||||
persist_current_cache = getattr(self._scanner, "_persist_current_cache", None)
|
||||
if callable(persist_current_cache):
|
||||
await persist_current_cache()
|
||||
|
||||
message = f"Model {os.path.basename(file_path)} excluded"
|
||||
return {"success": True, "message": message}
|
||||
|
||||
async def unexclude_model(self, file_path: str) -> Dict[str, object]:
|
||||
"""Restore a previously excluded model to the active cache."""
|
||||
|
||||
if not file_path:
|
||||
raise ValueError("Model path is required")
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
raise ValueError("Model file does not exist")
|
||||
|
||||
metadata_path = os.path.splitext(file_path)[0] + ".metadata.json"
|
||||
metadata_payload = await self._metadata_loader(metadata_path)
|
||||
metadata_payload["exclude"] = False
|
||||
|
||||
await self._metadata_manager.save_metadata(file_path, metadata_payload)
|
||||
|
||||
metadata, should_skip = await MetadataManager.load_metadata(
|
||||
file_path,
|
||||
self._scanner.model_class,
|
||||
)
|
||||
if should_skip:
|
||||
metadata = None
|
||||
if metadata is None:
|
||||
metadata = metadata_payload
|
||||
|
||||
excluded = getattr(self._scanner, "_excluded_models", None)
|
||||
if isinstance(excluded, list):
|
||||
self._scanner._excluded_models = [
|
||||
path for path in excluded if path != file_path
|
||||
]
|
||||
|
||||
await self._scanner.update_single_model_cache(
|
||||
file_path,
|
||||
file_path,
|
||||
metadata,
|
||||
recalculate_type=True,
|
||||
)
|
||||
|
||||
message = f"Model {os.path.basename(file_path)} restored"
|
||||
return {"success": True, "message": message}
|
||||
|
||||
async def bulk_delete_models(self, file_paths: Iterable[str]) -> Dict[str, object]:
|
||||
"""Delete a collection of models via the scanner bulk operation."""
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ import logging
|
||||
import random
|
||||
from typing import Optional, Dict, Tuple, Any, List, Sequence
|
||||
from .downloader import get_downloader
|
||||
from .errors import RateLimitError
|
||||
from .errors import RateLimitError, ResourceNotFoundError
|
||||
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
@@ -65,7 +65,14 @@ class _RateLimitRetryHelper:
|
||||
return await func(*args, **kwargs)
|
||||
except RateLimitError as exc:
|
||||
attempt += 1
|
||||
if attempt >= self._retry_limit:
|
||||
|
||||
# Determine effective retry limit based on rate-limit magnitude
|
||||
effective_retry_limit = self._retry_limit # default: 3
|
||||
if exc.retry_after is not None and exc.retry_after >= 120.0:
|
||||
# Long rate-limit window (>=2 min) — retries are futile
|
||||
effective_retry_limit = 1 # total 1 attempt = 0 retries
|
||||
|
||||
if attempt >= effective_retry_limit:
|
||||
exc.provider = exc.provider or label
|
||||
raise
|
||||
|
||||
@@ -81,7 +88,11 @@ class _RateLimitRetryHelper:
|
||||
|
||||
def _calculate_delay(self, retry_after: Optional[float], attempt: int) -> float:
|
||||
if retry_after is not None:
|
||||
return min(self._max_delay, max(0.0, retry_after))
|
||||
# Cap at 1800s (30 min) as a safety ceiling. The old 30s cap was
|
||||
# too low — CivArchive can return retry_after ~1500s, causing all
|
||||
# retries to fail. A generous ceiling protects against pathological
|
||||
# server values while still respecting the server's guidance.
|
||||
return min(1800.0, max(0.0, retry_after))
|
||||
|
||||
base_delay = self._base_delay * (2 ** max(0, attempt - 1))
|
||||
jitter_span = base_delay * self._jitter_ratio
|
||||
@@ -108,6 +119,18 @@ class ModelMetadataProvider(ABC):
|
||||
) -> Optional[Dict[int, Dict]]:
|
||||
"""Fetch model versions for multiple model ids when supported."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_model_versions_by_hashes(
|
||||
self, hashes: List[str]
|
||||
) -> Optional[List[Dict]]:
|
||||
"""Fetch full version details for multiple SHA256 hashes.
|
||||
|
||||
Used specifically to retrieve ``usageControl`` which is only
|
||||
available from the per-version / by-hash API, not from model-level
|
||||
responses. Providers that cannot resolve hashes should let the
|
||||
default ``NotImplementedError`` propagate.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_model_version(self, model_id: int = None, version_id: int = None) -> Optional[Dict]:
|
||||
@@ -140,6 +163,11 @@ class CivitaiModelMetadataProvider(ModelMetadataProvider):
|
||||
self, model_ids: Sequence[int]
|
||||
) -> Optional[Dict[int, Dict]]:
|
||||
return await self.client.get_model_versions_bulk(model_ids)
|
||||
|
||||
async def get_model_versions_by_hashes(
|
||||
self, hashes: List[str]
|
||||
) -> Optional[List[Dict]]:
|
||||
return await self.client.get_model_versions_by_hashes(hashes)
|
||||
|
||||
async def get_model_version(self, model_id: int = None, version_id: int = None) -> Optional[Dict]:
|
||||
return await self.client.get_model_version(model_id, version_id)
|
||||
@@ -457,14 +485,19 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
if result:
|
||||
return result, error
|
||||
except RateLimitError as exc:
|
||||
exc.provider = exc.provider or label
|
||||
raise exc
|
||||
logger.warning(
|
||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||
label,
|
||||
exc.retry_after or 0,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.debug("Provider %s failed for get_model_by_hash: %s", label, e)
|
||||
continue
|
||||
return None, "Model not found"
|
||||
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict]:
|
||||
not_found_confirmed = False
|
||||
for provider, label in self._iter_providers():
|
||||
try:
|
||||
result = await self._call_with_rate_limit(
|
||||
@@ -475,8 +508,20 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
if result:
|
||||
return result
|
||||
except RateLimitError as exc:
|
||||
exc.provider = exc.provider or label
|
||||
raise exc
|
||||
logger.warning(
|
||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||
label,
|
||||
exc.retry_after or 0,
|
||||
)
|
||||
continue
|
||||
except ResourceNotFoundError:
|
||||
not_found_confirmed = True
|
||||
logger.debug(
|
||||
"Provider %s reports model %s as not found",
|
||||
label,
|
||||
model_id,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.debug("Provider %s failed for get_model_versions: %s", label, e)
|
||||
continue
|
||||
@@ -494,8 +539,12 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
if result:
|
||||
return result
|
||||
except RateLimitError as exc:
|
||||
exc.provider = exc.provider or label
|
||||
raise exc
|
||||
logger.warning(
|
||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||
label,
|
||||
exc.retry_after or 0,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.debug("Provider %s failed for get_model_version: %s", label, e)
|
||||
continue
|
||||
@@ -512,13 +561,47 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
if result:
|
||||
return result, error
|
||||
except RateLimitError as exc:
|
||||
exc.provider = exc.provider or label
|
||||
raise exc
|
||||
logger.warning(
|
||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||
label,
|
||||
exc.retry_after or 0,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.debug("Provider %s failed for get_model_version_info: %s", label, e)
|
||||
continue
|
||||
return None, "No provider could retrieve the data"
|
||||
|
||||
async def get_model_versions_by_hashes(
|
||||
self, hashes: List[str]
|
||||
) -> Optional[List[Dict]]:
|
||||
for provider, label in self._iter_providers():
|
||||
try:
|
||||
result = await self._call_with_rate_limit(
|
||||
label,
|
||||
provider.get_model_versions_by_hashes,
|
||||
hashes,
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
except NotImplementedError:
|
||||
continue
|
||||
except RateLimitError as exc:
|
||||
logger.warning(
|
||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||
label,
|
||||
exc.retry_after or 0,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
"Provider %s failed for get_model_versions_by_hashes: %s",
|
||||
label,
|
||||
e,
|
||||
)
|
||||
continue
|
||||
return None
|
||||
|
||||
async def get_user_models(self, username: str) -> Optional[List[Dict]]:
|
||||
for provider, label in self._iter_providers():
|
||||
try:
|
||||
@@ -530,8 +613,12 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
if result is not None:
|
||||
return result
|
||||
except RateLimitError as exc:
|
||||
exc.provider = exc.provider or label
|
||||
raise exc
|
||||
logger.warning(
|
||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
||||
label,
|
||||
exc.retry_after or 0,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.debug("Provider %s failed for get_user_models: %s", label, e)
|
||||
continue
|
||||
@@ -593,6 +680,15 @@ class RateLimitRetryingProvider(ModelMetadataProvider):
|
||||
model_ids,
|
||||
)
|
||||
|
||||
async def get_model_versions_by_hashes(
|
||||
self, hashes: List[str]
|
||||
) -> Optional[List[Dict]]:
|
||||
return await self._rate_limit_helper.run(
|
||||
self._label,
|
||||
self._provider.get_model_versions_by_hashes,
|
||||
hashes,
|
||||
)
|
||||
|
||||
async def get_model_version(self, model_id: int = None, version_id: int = None) -> Optional[Dict]:
|
||||
return await self._rate_limit_helper.run(
|
||||
self._label,
|
||||
@@ -669,6 +765,17 @@ class ModelMetadataProviderManager:
|
||||
provider = self._get_provider(provider_name)
|
||||
return await provider.get_model_version_info(version_id)
|
||||
|
||||
async def get_model_versions_by_hashes(
|
||||
self,
|
||||
hashes: List[str],
|
||||
provider_name: str = None,
|
||||
) -> Optional[List[Dict]]:
|
||||
provider = self._get_provider(provider_name)
|
||||
try:
|
||||
return await provider.get_model_versions_by_hashes(hashes)
|
||||
except NotImplementedError:
|
||||
return None
|
||||
|
||||
async def get_user_models(self, username: str, provider_name: str = None) -> Optional[List[Dict]]:
|
||||
"""Fetch models owned by the specified user"""
|
||||
provider = self._get_provider(provider_name)
|
||||
|
||||
@@ -96,6 +96,7 @@ class FilterCriteria:
|
||||
folder_exclude: Optional[Sequence[str]] = None
|
||||
base_models: Optional[Sequence[str]] = None
|
||||
tags: Optional[Dict[str, str]] = None
|
||||
auto_tags: Optional[Dict[str, str]] = None
|
||||
favorites_only: bool = False
|
||||
search_options: Optional[Dict[str, Any]] = None
|
||||
model_types: Optional[Sequence[str]] = None
|
||||
@@ -293,12 +294,14 @@ class ModelFilterSet:
|
||||
for tag, state in tag_filters.items():
|
||||
if not tag:
|
||||
continue
|
||||
# Normalize to lowercase for case-insensitive matching
|
||||
normalized = tag.strip().lower()
|
||||
if state == "exclude":
|
||||
exclude_tags.add(tag)
|
||||
exclude_tags.add(normalized)
|
||||
else:
|
||||
include_tags.add(tag)
|
||||
include_tags.add(normalized)
|
||||
else:
|
||||
include_tags = {tag for tag in tag_filters if tag}
|
||||
include_tags = {tag.strip().lower() for tag in tag_filters if tag}
|
||||
|
||||
if include_tags:
|
||||
tag_logic = criteria.tag_logic.lower() if criteria.tag_logic else "any"
|
||||
@@ -317,13 +320,17 @@ class ModelFilterSet:
|
||||
return True
|
||||
# Otherwise, check if all non-special tags match
|
||||
if non_special_tags:
|
||||
return all(tag in (item_tags or []) for tag in non_special_tags)
|
||||
# Case-insensitive: normalize item tags too
|
||||
normalized_item_tags = {t.strip().lower() for t in (item_tags or []) if isinstance(t, str)}
|
||||
return all(tag in normalized_item_tags for tag in non_special_tags)
|
||||
return True
|
||||
# Normal case: all tags must match
|
||||
return all(tag in (item_tags or []) for tag in non_special_tags)
|
||||
# Normal case: all tags must match (case-insensitive)
|
||||
normalized_item_tags = {t.strip().lower() for t in (item_tags or []) if isinstance(t, str)}
|
||||
return all(tag in normalized_item_tags for tag in non_special_tags)
|
||||
else:
|
||||
# OR logic (default): item must have ANY include tag
|
||||
return any(tag in include_tags for tag in (item_tags or []))
|
||||
# OR logic (default): item must have ANY include tag (case-insensitive)
|
||||
normalized_item_tags = {t.strip().lower() for t in (item_tags or []) if isinstance(t, str)}
|
||||
return bool(normalized_item_tags & include_tags)
|
||||
|
||||
items = [item for item in items if matches_include(item.get("tags"))]
|
||||
|
||||
@@ -332,7 +339,9 @@ class ModelFilterSet:
|
||||
def matches_exclude(item_tags):
|
||||
if not item_tags and "__no_tags__" in exclude_tags:
|
||||
return True
|
||||
return any(tag in exclude_tags for tag in (item_tags or []))
|
||||
# Case-insensitive: normalize item tags
|
||||
normalized_item_tags = {t.strip().lower() for t in (item_tags or []) if isinstance(t, str)}
|
||||
return bool(normalized_item_tags & exclude_tags)
|
||||
|
||||
items = [
|
||||
item for item in items if not matches_exclude(item.get("tags"))
|
||||
@@ -359,10 +368,37 @@ class ModelFilterSet:
|
||||
]
|
||||
model_types_duration = time.perf_counter() - t0
|
||||
|
||||
auto_tags_duration = 0
|
||||
auto_tag_filters = criteria.auto_tags or {}
|
||||
if auto_tag_filters:
|
||||
t0 = time.perf_counter()
|
||||
include_at = set()
|
||||
exclude_at = set()
|
||||
for tag, state in auto_tag_filters.items():
|
||||
if not tag:
|
||||
continue
|
||||
if state == "exclude":
|
||||
exclude_at.add(tag)
|
||||
else:
|
||||
include_at.add(tag)
|
||||
|
||||
if include_at:
|
||||
items = [
|
||||
item for item in items
|
||||
if any(tag in include_at for tag in (item.get("auto_tags") or []))
|
||||
]
|
||||
|
||||
if exclude_at:
|
||||
items = [
|
||||
item for item in items
|
||||
if not any(tag in exclude_at for tag in (item.get("auto_tags") or []))
|
||||
]
|
||||
auto_tags_duration = time.perf_counter() - t0
|
||||
|
||||
duration = time.perf_counter() - overall_start
|
||||
if duration > 0.1: # Only log if it's potentially slow
|
||||
logger.debug(
|
||||
"ModelFilterSet.apply took %.3fs (sfw: %.3fs, fav: %.3fs, folder: %.3fs, base: %.3fs, tags: %.3fs, types: %.3fs). "
|
||||
"ModelFilterSet.apply took %.3fs (sfw: %.3fs, fav: %.3fs, folder: %.3fs, base: %.3fs, tags: %.3fs, types: %.3fs, auto_tags: %.3fs). "
|
||||
"Count: %d -> %d",
|
||||
duration,
|
||||
sfw_duration,
|
||||
@@ -371,6 +407,7 @@ class ModelFilterSet:
|
||||
base_models_duration,
|
||||
tags_duration,
|
||||
model_types_duration,
|
||||
auto_tags_duration,
|
||||
initial_count,
|
||||
len(items),
|
||||
)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user