mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-09-21 19:21:27 -03:00
Compare commits
571
Commits
v1.0.1
...
27027c4497
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
27027c4497 | ||
|
|
86c85c08ec | ||
|
|
196c8ffc3e | ||
|
|
cfc95ee02a | ||
|
|
479fa36997 | ||
|
|
3e1216e9bc | ||
|
|
007883b7d1 | ||
|
|
dc9200a12c | ||
|
|
d2f955266d | ||
|
|
8e724538bd | ||
|
|
6fcdeb799d | ||
|
|
97b9b1f62b | ||
|
|
4bf9a4b640 | ||
|
|
c5088772e8 | ||
|
|
56acefbd6c | ||
|
|
5ab06c4aae | ||
|
|
c11f4b5c68 | ||
|
|
86376284f4 | ||
|
|
2b8a2fc7d8 | ||
|
|
f26e1b41c8 | ||
|
|
c1671af99f | ||
|
|
ac7707d0f6 | ||
|
|
381cd710a2 | ||
|
|
ad0d18cb79 | ||
|
|
7980ee77d0 | ||
|
|
916b8bb327 | ||
|
|
87e3d4dea9 | ||
|
|
76a913f5e0 | ||
|
|
d8c192e647 | ||
|
|
c453437620 | ||
|
|
720fa6d909 | ||
|
|
b4f71089f4 | ||
|
|
83e6657ead | ||
|
|
7ea6df4111 | ||
|
|
d9ab92602a | ||
|
|
5ffadaed31 | ||
|
|
24f5f7df5d | ||
|
|
daf01fb1d6 | ||
|
|
0f11b6def9 | ||
|
|
7df83f44b8 | ||
|
|
169fa7bed6 | ||
|
|
027b504fe8 | ||
|
|
186ef4da78 | ||
|
|
dc674098e7 | ||
|
|
9087b4b07c | ||
|
|
8e45c22d7a | ||
|
|
191c4e03cd | ||
|
|
ab4154c57d | ||
|
|
28e93d12ff | ||
|
|
75e63c758b | ||
|
|
823f71f269 | ||
|
|
042dd4088d | ||
|
|
eaa791a9eb | ||
|
|
2228627ff4 | ||
|
|
4c647ad9c8 | ||
|
|
8ca3e6c33f | ||
|
|
dd6bdbf297 | ||
|
|
b47dde87e4 | ||
|
|
99e65cccd8 | ||
|
|
3bdacb8f46 | ||
|
|
b4f9c224d3 | ||
|
|
5ec0399c81 | ||
|
|
b464fdc333 | ||
|
|
53825500db | ||
|
|
f2ac790752 | ||
|
|
0d8805cdee | ||
|
|
656e24ac9b | ||
|
|
6718b37403 | ||
|
|
c9e5e784fc | ||
|
|
f92f958682 | ||
|
|
f63fab0676 | ||
|
|
cfc4903c0c | ||
|
|
a527a847fe | ||
|
|
91b0bf8933 | ||
|
|
66d1c96783 | ||
|
|
986128076e | ||
|
|
1de0a53241 | ||
|
|
0ec7eaf606 | ||
|
|
d9fcb0e92b | ||
|
|
f49b4ba4db | ||
|
|
84e708328b | ||
|
|
125bed3f09 | ||
|
|
077e70169d | ||
|
|
e6dc169a05 | ||
|
|
f34c02756d | ||
|
|
1e4c315481 | ||
|
|
a8283a0d00 | ||
|
|
55896669fc | ||
|
|
e341e0b9d2 | ||
|
|
e6538c83bb | ||
|
|
92e1285ea5 | ||
|
|
2aabd1d90e | ||
|
|
7b8b778f83 | ||
|
|
7c8dc57d55 | ||
|
|
fe95fae5f2 | ||
|
|
ce8a95abf7 | ||
|
|
c8e7e543d6 | ||
|
|
a9dbb15ffa | ||
|
|
cf64043f7d | ||
|
|
ccaff92c18 | ||
|
|
585b5c922a | ||
|
|
ea80c2224c | ||
|
|
8b0f56c1a6 | ||
|
|
8022d12f03 | ||
|
|
3939f7f91b | ||
|
|
aebf2e37dd | ||
|
|
f53f859a71 | ||
|
|
d916375abe | ||
|
|
57983df4bd | ||
|
|
c68d7559a0 | ||
|
|
9a8f5bf2d6 | ||
|
|
a8d742b031 | ||
|
|
c27e4d1bfc | ||
|
|
d15a8aa9a2 | ||
|
|
74a7d12ca4 | ||
|
|
2f94a9773e | ||
|
|
37bdfa21ea | ||
|
|
f0bf2728c9 | ||
|
|
dc715aa273 | ||
|
|
7ee2361e87 | ||
|
|
e04c22f83f | ||
|
|
681cc13e90 | ||
|
|
090e0297d4 | ||
|
|
6f71335be4 | ||
|
|
7f51812c1e | ||
|
|
a9dc4d7b9d | ||
|
|
5d50ddb5d4 | ||
|
|
f86198d234 | ||
|
|
ffe65d983c | ||
|
|
b0b5be913c | ||
|
|
01efcbc584 | ||
|
|
02c249917a | ||
|
|
419bbc90b2 | ||
|
|
b0c4510fdb | ||
|
|
bf6a614e0d | ||
|
|
feab01cd9c | ||
|
|
966024e534 | ||
|
|
2018722cc8 | ||
|
|
9d85c2a44a | ||
|
|
03dd047e62 | ||
|
|
86b547c1e0 | ||
|
|
bab9752c8b | ||
|
|
774cc1be86 | ||
|
|
234b73c8a2 | ||
|
|
abd06c48f4 | ||
|
|
6ca411e4e4 | ||
|
|
6470021e77 | ||
|
|
71658ab37b | ||
|
|
4f016a8024 | ||
|
|
f362ed585b | ||
|
|
196172624f | ||
|
|
316702b7ab | ||
|
|
a7625b009f | ||
|
|
5d4a33c90d | ||
|
|
041a6b8525 | ||
|
|
2638109ad6 | ||
|
|
b019326747 | ||
|
|
54b44131b6 | ||
|
|
a1d948025c | ||
|
|
a90b2514ba | ||
|
|
cb4ad27813 | ||
|
|
637831248b | ||
|
|
00228deaaa | ||
|
|
2373edf73c | ||
|
|
e0e1b804a7 | ||
|
|
fecbe8241f | ||
|
|
5983eaa1ce | ||
|
|
07fa454f72 | ||
|
|
4b5aa45379 | ||
|
|
9a0d866be4 | ||
|
|
308d8f71b8 | ||
|
|
d0e8938039 | ||
|
|
13ed898b6b | ||
|
|
e1dfd1c2a6 | ||
|
|
e3e944911b | ||
|
|
51c0135250 | ||
|
|
7b19bbb14e | ||
|
|
5494a70f40 | ||
|
|
26c9ade1c9 | ||
|
|
87db23825f | ||
|
|
8fb00998a7 | ||
|
|
dd3aa97d0a | ||
|
|
8bee8f4069 | ||
|
|
817fe21b3e | ||
|
|
905c37290f | ||
|
|
f7632a47f9 | ||
|
|
646f1ddfb1 | ||
|
|
170c8068c5 | ||
|
|
3494037d20 | ||
|
|
a1fd4e150b | ||
|
|
b22f09bd1d | ||
|
|
4ed9169646 | ||
|
|
f06c60bd47 | ||
|
|
ee8250c26c | ||
|
|
88349bf944 | ||
|
|
a8adcaf023 | ||
|
|
63785f82b5 | ||
|
|
cf898da193 | ||
|
|
3c83e78d9f | ||
|
|
d7291f73c9 | ||
|
|
fe90f7f9b1 | ||
|
|
8b344ea39f | ||
|
|
8348a0cef8 | ||
|
|
7cf785b72f | ||
|
|
e8913f4481 | ||
|
|
f9c3d8dc97 | ||
|
|
09ca91fc0e | ||
|
|
16f5222efd | ||
|
|
28e7c04b37 | ||
|
|
28f99c46d3 | ||
|
|
205194f4e6 | ||
|
|
402d8b07cf | ||
|
|
3e303ab316 | ||
|
|
e9e8c31ad1 | ||
|
|
703a6a4ea0 | ||
|
|
283730cf38 | ||
|
|
20417797e8 | ||
|
|
004c69b9ef | ||
|
|
47fe2d3783 | ||
|
|
36ef840a22 | ||
|
|
09c2445ac9 | ||
|
|
8a6d23f9c7 | ||
|
|
3d207b6744 | ||
|
|
b3edda62ad | ||
|
|
a429e6b1c3 | ||
|
|
c1bf9c6221 | ||
|
|
75fffc1e25 | ||
|
|
f264bab65c | ||
|
|
154fcd803b | ||
|
|
4ef32d3a96 | ||
|
|
d2d109a69c | ||
|
|
3a2941d751 | ||
|
|
0ac10dfd42 | ||
|
|
9c95856b2f | ||
|
|
5ce4667d32 | ||
|
|
be53fda6df | ||
|
|
f48de05102 | ||
|
|
93ad81ed87 | ||
|
|
ea14d211be | ||
|
|
8052cefd46 | ||
|
|
845815b9b7 | ||
|
|
609dc5d783 | ||
|
|
7a71b34b54 | ||
|
|
71a459422f | ||
|
|
cd2628a0ee | ||
|
|
85da7175bc | ||
|
|
d3bf0a164b | ||
|
|
afb6ca1b8d | ||
|
|
94f43426d7 | ||
|
|
2b361f4f5d | ||
|
|
7438072f8c | ||
|
|
26c54fd358 | ||
|
|
7cb6b04c63 | ||
|
|
fc29cde82a | ||
|
|
559ca946dc | ||
|
|
2b8e7c7504 | ||
|
|
6816d75933 | ||
|
|
b58abbad7c | ||
|
|
999814ca87 | ||
|
|
3c2760a803 | ||
|
|
0edbd7bcca | ||
|
|
21e89fa7de | ||
|
|
968d6d1d1f | ||
|
|
cf0fd0e0ad | ||
|
|
16e5dcf7b2 | ||
|
|
ab6bb25d46 | ||
|
|
07f49559be | ||
|
|
b24b1a7e57 | ||
|
|
faf64f8986 | ||
|
|
a617487a43 | ||
|
|
3012a7aef3 | ||
|
|
499e19de34 | ||
|
|
9161762ca9 | ||
|
|
9bbd26efe6 | ||
|
|
258b2622d5 | ||
|
|
80ec9085dd | ||
|
|
c5c7373e10 | ||
|
|
b7721866e5 | ||
|
|
8314b9bedb | ||
|
|
75298a402f | ||
|
|
92b5efd414 | ||
|
|
33ee392b7b | ||
|
|
5237f8b7dc | ||
|
|
5107313fd1 | ||
|
|
95bbc66919 | ||
|
|
e268e59419 | ||
|
|
547e1f9498 | ||
|
|
bf32d8b6fd | ||
|
|
8299881024 | ||
|
|
da02268196 | ||
|
|
8c4b9a1e70 | ||
|
|
0906c484e9 | ||
|
|
4199c30fec | ||
|
|
4a8084cdbc | ||
|
|
6263e6848c | ||
|
|
58c266ad07 | ||
|
|
2939813e1a | ||
|
|
a9e5ee7e79 | ||
|
|
a17b0e9901 | ||
|
|
8f23d966bf | ||
|
|
7a76fc72d0 | ||
|
|
518a4dd5ee | ||
|
|
2b6d4e5d8b | ||
|
|
1f4edbeb9d | ||
|
|
a256558a0e | ||
|
|
818b9113f0 | ||
|
|
6a4fd020dc | ||
|
|
7a23040452 | ||
|
|
138024aefe | ||
|
|
a19ddc14f6 | ||
|
|
7001ced694 | ||
|
|
a5c861646c | ||
|
|
3e0bb73793 | ||
|
|
ac51f6a2f6 | ||
|
|
bef222c77d | ||
|
|
7cd6a53447 | ||
|
|
6850b35770 | ||
|
|
237a015cde | ||
|
|
1ae2778baa | ||
|
|
84fcdb5f20 | ||
|
|
8a0b368b44 | ||
|
|
3990535505 | ||
|
|
3e961a9860 | ||
|
|
d6669f1d04 | ||
|
|
519bafebc8 | ||
|
|
d87863b423 | ||
|
|
84e9fe2dfb | ||
|
|
46cbcf94c8 | ||
|
|
05f3018495 | ||
|
|
f565cc35ca | ||
|
|
dd1cdce16d | ||
|
|
a9e0e7dc8d | ||
|
|
b302d1db7d | ||
|
|
7cbddd9cf7 | ||
|
|
cb8c699224 | ||
|
|
451f74b874 | ||
|
|
a1d248baa6 | ||
|
|
18577fa336 | ||
|
|
5797ce9408 | ||
|
|
826f06255a | ||
|
|
84e16b5c5b | ||
|
|
eb22054580 | ||
|
|
08afb05ece | ||
|
|
f51f125cf1 | ||
|
|
24b2078f21 | ||
|
|
130fb5d2d5 | ||
|
|
23c6863a3a | ||
|
|
c0e2578640 | ||
|
|
e3c812367e | ||
|
|
4d239008a6 | ||
|
|
00177a06d0 | ||
|
|
568daa351e | ||
|
|
5a4664fa12 | ||
|
|
dd5b213adc | ||
|
|
d9ee9b3155 | ||
|
|
01dac57c35 | ||
|
|
7f92d09239 | ||
|
|
62f9e3f44a | ||
|
|
e55895786d | ||
|
|
82b77bf593 | ||
|
|
1beef5dea9 | ||
|
|
c8beaa64e1 | ||
|
|
fb443ed6ae | ||
|
|
151a467598 | ||
|
|
98e1d168b0 | ||
|
|
716f18e0ed | ||
|
|
b060dc99fc | ||
|
|
54bcdfab38 | ||
|
|
2e7532eecc | ||
|
|
7e5e3b1ec7 | ||
|
|
df67bd396a | ||
|
|
dd5d9cfcb2 | ||
|
|
d9fd60bec1 | ||
|
|
b633b22779 | ||
|
|
1ffa543160 | ||
|
|
cdc940586e | ||
|
|
ccf1c6f2ae | ||
|
|
bfe7b5e1c7 | ||
|
|
85c020cd12 | ||
|
|
1b202f8ec7 | ||
|
|
d02a0611d3 | ||
|
|
92166a161a | ||
|
|
b509f27cb7 | ||
|
|
5c2ef48917 | ||
|
|
ad2bd82c67 | ||
|
|
17ba350153 | ||
|
|
60175334b5 | ||
|
|
f65a01df00 | ||
|
|
430e24d70b | ||
|
|
14f0c48fdd | ||
|
|
34791c2ad7 | ||
|
|
3f6824eef6 | ||
|
|
3919dfa3f4 | ||
|
|
7124b5293f | ||
|
|
d2a04f8993 | ||
|
|
7027a7c270 | ||
|
|
0a1d7dfd4c | ||
|
|
3962b1a96d | ||
|
|
8b856276bf | ||
|
|
c97c802956 | ||
|
|
24e2909627 | ||
|
|
b768f1368f | ||
|
|
37ccd29fc0 | ||
|
|
7416080cfb | ||
|
|
26be187d42 | ||
|
|
d7caa1fa47 | ||
|
|
2629fcce23 | ||
|
|
438e7d07b9 | ||
|
|
e9932ea870 | ||
|
|
5dd8b96422 | ||
|
|
5e1cf68bbd | ||
|
|
1044fa3c83 | ||
|
|
397892bb7f | ||
|
|
f105500740 | ||
|
|
806555cf06 | ||
|
|
5cd7204101 | ||
|
|
3b602a3698 | ||
|
|
15dfaed462 | ||
|
|
0e51851025 | ||
|
|
0d0f4defca | ||
|
|
818fa34a48 | ||
|
|
78303b2a5e | ||
|
|
9ce56dd40c | ||
|
|
4e3ede23b7 | ||
|
|
33e5f3d85d | ||
|
|
031d5e4f40 | ||
|
|
4ff5774e34 | ||
|
|
94e1a8ac7b | ||
|
|
cc20d3b992 | ||
|
|
a74cbe7aa2 | ||
|
|
94edfaa190 | ||
|
|
31c54ff068 | ||
|
|
21872a8e9e | ||
|
|
612612f1c7 | ||
|
|
ff240db5b1 | ||
|
|
bcfed4b874 | ||
|
|
1352c6ecbe | ||
|
|
30b01b8a92 | ||
|
|
a105cb322b | ||
|
|
3bf396d003 | ||
|
|
60cfb3b8e0 | ||
|
|
6763abb83c | ||
|
|
5c53968caa | ||
|
|
b4f7dd75af | ||
|
|
86118d0654 | ||
|
|
df1410535e | ||
|
|
75f74d54d8 | ||
|
|
ab6100f596 | ||
|
|
5d3ab3bbf8 | ||
|
|
d9dc0dba8d | ||
|
|
3631c5eb10 | ||
|
|
6d5b4b7312 | ||
|
|
7803bd542d | ||
|
|
f0a86dbbc0 | ||
|
|
682e964f89 | ||
|
|
908464bc0a | ||
|
|
0ffee3a854 | ||
|
|
8aa9739c44 | ||
|
|
50739bbb43 | ||
|
|
e849303763 | ||
|
|
241b2e15d2 | ||
|
|
88da754504 | ||
|
|
b4a706651f | ||
|
|
ff7cc6d9bb | ||
|
|
454210a47c | ||
|
|
2d7c404ebb | ||
|
|
e23d803ecf | ||
|
|
0cc640cfaa | ||
|
|
2ac0eb0f9d | ||
|
|
f028625ce9 | ||
|
|
06acc7f576 | ||
|
|
d324b57274 | ||
|
|
502b7eab31 | ||
|
|
be75ad930e | ||
|
|
763c4f4dad | ||
|
|
d32c492bdb | ||
|
|
5dcfde36ea | ||
|
|
1d035361a4 | ||
|
|
25605c5e78 | ||
|
|
f3268a6179 | ||
|
|
055e94d77b | ||
|
|
47fcd530a0 | ||
|
|
3c32b9e088 | ||
|
|
ffe0670a27 | ||
|
|
cc147a1795 | ||
|
|
e81409bea4 | ||
|
|
b31fae4e51 | ||
|
|
c6e5467907 | ||
|
|
df0e5797d0 | ||
|
|
ebdbb36271 | ||
|
|
2eef629821 | ||
|
|
658a04736d | ||
|
|
ef7f677933 | ||
|
|
63f0942452 | ||
|
|
a1dff6dd47 | ||
|
|
7fa40023b0 | ||
|
|
3c8acdb65e | ||
|
|
1e9a7812d6 | ||
|
|
37f0e8f213 | ||
|
|
ecf7ea21e4 | ||
|
|
79dd9a1b29 | ||
|
|
ef4923fd94 | ||
|
|
1eeba666f5 | ||
|
|
89e26d9292 | ||
|
|
fc19a145ff | ||
|
|
34f03d6495 | ||
|
|
9443175abc | ||
|
|
dc5072628f | ||
|
|
ff4b8ec849 | ||
|
|
7ab271c752 | ||
|
|
5a7f4dc88b | ||
|
|
761108bfd1 | ||
|
|
24dd3a777c | ||
|
|
1c530ea013 | ||
|
|
0ced53c059 | ||
|
|
67ad68a23f | ||
|
|
d9ec9c512e | ||
|
|
0bcd8e09a9 | ||
|
|
fa049a28c8 | ||
|
|
89fd2b43d6 | ||
|
|
c53f44e7ef | ||
|
|
ae7bfdb517 | ||
|
|
68bf8442eb | ||
|
|
605fbf4117 | ||
|
|
406d5fea6a | ||
|
|
af2146f96c | ||
|
|
bdc8dec860 | ||
|
|
c4fa1631ee | ||
|
|
506d763dc2 | ||
|
|
a2cd09b619 | ||
|
|
cdd77029b6 | ||
|
|
439679e15f | ||
|
|
2640258902 | ||
|
|
b910388d54 | ||
|
|
083de395b1 | ||
|
|
4514ca94b7 | ||
|
|
62247bdd87 | ||
|
|
6d0d9600a7 | ||
|
|
70cd3f4e1b | ||
|
|
a95c518b30 | ||
|
|
ba1800095e | ||
|
|
39c083db79 | ||
|
|
55e9e4bb6f | ||
|
|
0253d001e6 | ||
|
|
9998da3241 | ||
|
|
6666a72775 | ||
|
|
5f1bd894b9 | ||
|
|
1817142a7b | ||
|
|
25fa175aa2 | ||
|
|
39643eb2bc | ||
|
|
4ac78f8aa8 | ||
|
|
0bcca0ba68 | ||
|
|
72f8e0d1be | ||
|
|
85b6c91192 | ||
|
|
908016cbd6 | ||
|
|
a5ac9cf81b | ||
|
|
32875042bd | ||
|
|
51fe7aa07e | ||
|
|
db4726a961 | ||
|
|
e13d70248a | ||
|
|
1c4919a3e8 | ||
|
|
18ddadc9ec | ||
|
|
b6dd6938b0 | ||
|
|
b711ac468a | ||
|
|
727d0ef043 | ||
|
|
9344d86332 | ||
|
|
d36b16c213 | ||
|
|
33a7f07558 | ||
|
|
4f599aeced | ||
|
|
30db8c3d1d | ||
|
|
05636712f0 |
@@ -0,0 +1,69 @@
|
||||
---
|
||||
name: lora-manager-runtime-context
|
||||
description: Inspect ComfyUI LoRA Manager runtime configuration and local diagnostic state. Use when debugging LoRA Manager issues that require locating or reading settings.json, active library paths, model metadata JSON sidecars, recipe metadata JSON files, example image folders, SQLite caches, symlink maps, download history, aria2 state, or other cache files under the LoRA Manager user config directory.
|
||||
---
|
||||
|
||||
# LoRA Manager Runtime Context
|
||||
|
||||
## Core Rules
|
||||
|
||||
- Treat runtime state as local user data. Prefer read-only inspection unless the user explicitly asks for mutation.
|
||||
- Never print secret-like settings values. Redact keys containing `key`, `token`, `secret`, `password`, `auth`, or `credential`, including `civitai_api_key`.
|
||||
- Resolve paths from the runtime configuration before guessing. In this environment the settings file is normally `/home/miao/.config/ComfyUI-LoRA-Manager/settings.json`, but portable settings can override this through the repository `settings.json`.
|
||||
- Use the active library when selecting per-library caches and paths. Read `active_library` from settings; fall back to `default` if missing.
|
||||
- Normalize and expand `~` before comparing paths. Symlinks are common in this repo.
|
||||
|
||||
## Quick Start
|
||||
|
||||
Use the bundled helper for a safe first pass:
|
||||
|
||||
```bash
|
||||
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py summary
|
||||
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py caches
|
||||
```
|
||||
|
||||
The script redacts sensitive settings, opens SQLite databases read-only, and reports inaccessible or locked databases as warnings.
|
||||
|
||||
For focused checks:
|
||||
|
||||
```bash
|
||||
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py recipes
|
||||
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py model --path /path/to/model.safetensors
|
||||
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py sqlite --db /path/to/cache.sqlite --limit 3
|
||||
```
|
||||
|
||||
## Runtime Path Rules
|
||||
|
||||
- Settings directory: use `py/utils/settings_paths.py`. Default platform path is `platformdirs.user_config_dir("ComfyUI-LoRA-Manager", appauthor=False)`.
|
||||
- Settings file: `<settings_dir>/settings.json`.
|
||||
- Cache root: `<settings_dir>/cache`.
|
||||
- Canonical cache files:
|
||||
- Model cache: `cache/model/<active_library>.sqlite`.
|
||||
- Recipe cache: `cache/recipe/<active_library>.sqlite`.
|
||||
- Model update cache: `cache/model_update/<active_library>.sqlite`.
|
||||
- Recipe FTS: `cache/fts/recipe_fts.sqlite`.
|
||||
- Tag FTS: `cache/fts/tag_fts.sqlite`.
|
||||
- Symlink map: `cache/symlink/symlink_map.json`.
|
||||
- Download history: `cache/download_history/downloaded_versions.sqlite`.
|
||||
- aria2 state: `cache/aria2/downloads.json`.
|
||||
- Legacy cache locations may exist; prefer canonical paths unless diagnosing migrations.
|
||||
|
||||
## Data Location Rules
|
||||
|
||||
- Model roots come from `settings.folder_paths` and the active library payload under `settings.libraries[active_library]`.
|
||||
- Model metadata JSON sidecars live next to the model file as `<model basename>.metadata.json`.
|
||||
- Recipes root is `settings.recipes_path` when it is a non-empty string. If empty, use the first configured LoRA root plus `/recipes`.
|
||||
- Recipe JSON files are named `*.recipe.json` under the recipes root and may be nested in folders.
|
||||
- Example image root is `settings.example_images_path`.
|
||||
- If multiple libraries are configured, example images are stored under `<example_images_path>/<sanitized_library>/<sha256>/`; otherwise they are under `<example_images_path>/<sha256>/`.
|
||||
|
||||
## Useful Cache Tables
|
||||
|
||||
- Model cache: `models`, `model_tags`, `hash_index`, `excluded_models`.
|
||||
- Recipe cache: `recipes`, `cache_metadata`.
|
||||
- Model update cache: `model_update_status`, `model_update_versions`.
|
||||
- Tag FTS cache: `tags`, `fts_metadata`, plus FTS internal tables.
|
||||
- Recipe FTS cache: `recipe_rowid`, `fts_metadata`, plus FTS internal tables.
|
||||
- Download history: `downloaded_model_versions`.
|
||||
|
||||
Prefer querying only counts, schema, and a few sample rows unless the user asks for full output.
|
||||
@@ -0,0 +1,4 @@
|
||||
interface:
|
||||
display_name: "LoRA Manager Runtime Context"
|
||||
short_description: "Inspect LoRA Manager runtime state"
|
||||
default_prompt: "Use $lora-manager-runtime-context to inspect LoRA Manager settings, metadata paths, and caches for debugging."
|
||||
+381
@@ -0,0 +1,381 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
SECRET_PATTERN = re.compile(r"(key|token|secret|password|auth|credential)", re.IGNORECASE)
|
||||
APP_NAME = "ComfyUI-LoRA-Manager"
|
||||
CACHE_SQLITE = {
|
||||
"model": ("model", "{library}.sqlite"),
|
||||
"recipe": ("recipe", "{library}.sqlite"),
|
||||
"model_update": ("model_update", "{library}.sqlite"),
|
||||
"recipe_fts": ("fts", "recipe_fts.sqlite"),
|
||||
"tag_fts": ("fts", "tag_fts.sqlite"),
|
||||
"download_history": ("download_history", "downloaded_versions.sqlite"),
|
||||
}
|
||||
CACHE_JSON = {
|
||||
"symlink": ("symlink", "symlink_map.json"),
|
||||
"aria2": ("aria2", "downloads.json"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Inspect LoRA Manager runtime state read-only.")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
subparsers.add_parser("summary", help="Print redacted settings and resolved paths.")
|
||||
subparsers.add_parser("caches", help="Print cache paths and SQLite table summaries.")
|
||||
subparsers.add_parser("recipes", help="Print resolved recipes root and recipe JSON count.")
|
||||
|
||||
model_parser = subparsers.add_parser("model", help="Inspect a model metadata sidecar path.")
|
||||
model_parser.add_argument("--path", required=True, help="Path to a model file or metadata JSON file.")
|
||||
|
||||
sqlite_parser = subparsers.add_parser("sqlite", help="Inspect a SQLite database read-only.")
|
||||
sqlite_parser.add_argument("--db", required=True, help="Path to the SQLite database.")
|
||||
sqlite_parser.add_argument("--limit", type=int, default=3, help="Rows to sample from each user table.")
|
||||
|
||||
args = parser.parse_args()
|
||||
context = build_context()
|
||||
|
||||
if args.command == "summary":
|
||||
print_json(summary_payload(context))
|
||||
elif args.command == "caches":
|
||||
print_json(caches_payload(context))
|
||||
elif args.command == "recipes":
|
||||
print_json(recipes_payload(context))
|
||||
elif args.command == "model":
|
||||
print_json(model_payload(args.path))
|
||||
elif args.command == "sqlite":
|
||||
print_json(sqlite_payload(Path(args.db).expanduser(), args.limit))
|
||||
return 0
|
||||
|
||||
|
||||
def build_context() -> dict[str, Any]:
|
||||
settings_path = resolve_settings_path()
|
||||
settings = load_json(settings_path)
|
||||
settings_dir = settings_path.parent
|
||||
active_library = settings.get("active_library") or "default"
|
||||
safe_library = sanitize_library_name(str(active_library))
|
||||
cache_root = settings_dir / "cache"
|
||||
return {
|
||||
"settings_path": str(settings_path),
|
||||
"settings_dir": str(settings_dir),
|
||||
"settings": settings,
|
||||
"active_library": active_library,
|
||||
"safe_library": safe_library,
|
||||
"cache_root": str(cache_root),
|
||||
"cache_paths": resolve_cache_paths(cache_root, safe_library),
|
||||
}
|
||||
|
||||
|
||||
def resolve_settings_path() -> Path:
|
||||
repo_root = find_repo_root()
|
||||
portable = repo_root / "settings.json"
|
||||
if portable.exists():
|
||||
payload = load_json(portable)
|
||||
if isinstance(payload, dict) and payload.get("use_portable_settings") is True:
|
||||
return portable
|
||||
|
||||
config_home = os.environ.get("XDG_CONFIG_HOME")
|
||||
if config_home:
|
||||
return Path(config_home).expanduser() / APP_NAME / "settings.json"
|
||||
return Path.home() / ".config" / APP_NAME / "settings.json"
|
||||
|
||||
|
||||
def find_repo_root() -> Path:
|
||||
current = Path(__file__).resolve()
|
||||
for parent in current.parents:
|
||||
if (parent / "py").is_dir() and (parent / "standalone.py").exists():
|
||||
return parent
|
||||
return Path.cwd()
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
except json.JSONDecodeError as exc:
|
||||
return {"_error": f"invalid JSON: {exc}"}
|
||||
except OSError as exc:
|
||||
return {"_error": f"unreadable: {exc}"}
|
||||
return payload if isinstance(payload, dict) else {"_error": "JSON root is not an object"}
|
||||
|
||||
|
||||
def resolve_cache_paths(cache_root: Path, library: str) -> dict[str, str]:
|
||||
paths: dict[str, str] = {}
|
||||
for name, (subdir, filename) in CACHE_SQLITE.items():
|
||||
paths[name] = str(cache_root / subdir / filename.format(library=library))
|
||||
for name, (subdir, filename) in CACHE_JSON.items():
|
||||
paths[name] = str(cache_root / subdir / filename)
|
||||
return paths
|
||||
|
||||
|
||||
def summary_payload(context: dict[str, Any]) -> dict[str, Any]:
|
||||
settings = context["settings"]
|
||||
return {
|
||||
"settings_path": context["settings_path"],
|
||||
"settings_dir": context["settings_dir"],
|
||||
"active_library": context["active_library"],
|
||||
"settings": redact(settings),
|
||||
"model_roots": model_roots(settings, context["active_library"]),
|
||||
"recipes_root": str(resolve_recipes_root(settings, context["active_library"]) or ""),
|
||||
"example_images": example_images_payload(settings, context["active_library"]),
|
||||
"cache_root": context["cache_root"],
|
||||
"cache_paths": context["cache_paths"],
|
||||
}
|
||||
|
||||
|
||||
def caches_payload(context: dict[str, Any]) -> dict[str, Any]:
|
||||
caches: dict[str, Any] = {}
|
||||
for name, path_string in context["cache_paths"].items():
|
||||
path = Path(path_string)
|
||||
item: dict[str, Any] = {
|
||||
"path": str(path),
|
||||
"exists": path.exists(),
|
||||
"size": path.stat().st_size if path.exists() else None,
|
||||
}
|
||||
if path.suffix == ".sqlite":
|
||||
item["sqlite"] = sqlite_payload(path, limit=0)
|
||||
elif path.suffix == ".json":
|
||||
item["json"] = json_file_summary(path)
|
||||
caches[name] = item
|
||||
return {"active_library": context["active_library"], "caches": caches}
|
||||
|
||||
|
||||
def recipes_payload(context: dict[str, Any]) -> dict[str, Any]:
|
||||
root = resolve_recipes_root(context["settings"], context["active_library"])
|
||||
files: list[str] = []
|
||||
if root and root.exists():
|
||||
files = [str(path) for path in sorted(root.rglob("*.recipe.json"))[:20]]
|
||||
return {
|
||||
"recipes_root": str(root or ""),
|
||||
"exists": bool(root and root.exists()),
|
||||
"recipe_json_count": count_recipe_files(root),
|
||||
"sample_recipe_json": files,
|
||||
"recipe_cache": context["cache_paths"].get("recipe"),
|
||||
}
|
||||
|
||||
|
||||
def model_payload(raw_path: str) -> dict[str, Any]:
|
||||
path = Path(raw_path).expanduser()
|
||||
metadata_path = path if path.name.endswith(".metadata.json") else path.with_suffix(".metadata.json")
|
||||
payload = {
|
||||
"input_path": str(path),
|
||||
"metadata_path": str(metadata_path),
|
||||
"model_exists": path.exists(),
|
||||
"metadata_exists": metadata_path.exists(),
|
||||
}
|
||||
if metadata_path.exists():
|
||||
data = load_json(metadata_path)
|
||||
payload["metadata_summary"] = redact(summarize_value(data))
|
||||
return payload
|
||||
|
||||
|
||||
def sqlite_payload(path: Path, limit: int = 3, allow_copy: bool = True) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {"path": str(path), "exists": path.exists(), "tables": {}}
|
||||
if not path.exists():
|
||||
return result
|
||||
try:
|
||||
conn = connect_sqlite_readonly(path)
|
||||
except sqlite3.Error as exc:
|
||||
result["error"] = str(exc)
|
||||
return result
|
||||
try:
|
||||
table_rows = conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
|
||||
).fetchall()
|
||||
for table_row in table_rows:
|
||||
table = table_row["name"]
|
||||
columns = [
|
||||
row["name"]
|
||||
for row in conn.execute(f"PRAGMA table_info({quote_identifier(table)})").fetchall()
|
||||
]
|
||||
table_info: dict[str, Any] = {"columns": columns}
|
||||
try:
|
||||
table_info["count"] = conn.execute(
|
||||
f"SELECT COUNT(*) FROM {quote_identifier(table)}"
|
||||
).fetchone()[0]
|
||||
except sqlite3.Error as exc:
|
||||
table_info["count_error"] = str(exc)
|
||||
if limit > 0 and columns and not is_internal_sqlite_table(table):
|
||||
try:
|
||||
rows = conn.execute(
|
||||
f"SELECT * FROM {quote_identifier(table)} LIMIT ?", (limit,)
|
||||
).fetchall()
|
||||
table_info["sample"] = [redact(dict(row)) for row in rows]
|
||||
except sqlite3.Error as exc:
|
||||
table_info["sample_error"] = str(exc)
|
||||
result["tables"][table] = table_info
|
||||
except sqlite3.Error as exc:
|
||||
fallback = sqlite_copy_payload(path, limit, str(exc)) if allow_copy else None
|
||||
if fallback is not None:
|
||||
result.update(fallback)
|
||||
else:
|
||||
result["error"] = str(exc)
|
||||
finally:
|
||||
conn.close()
|
||||
return result
|
||||
|
||||
|
||||
def connect_sqlite_readonly(path: Path) -> sqlite3.Connection:
|
||||
errors: list[str] = []
|
||||
for query in ("mode=ro", "mode=ro&immutable=1"):
|
||||
try:
|
||||
conn = sqlite3.connect(f"file:{path}?{query}", uri=True)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
except sqlite3.Error as exc:
|
||||
errors.append(f"{query}: {exc}")
|
||||
raise sqlite3.OperationalError("; ".join(errors))
|
||||
|
||||
|
||||
def sqlite_copy_payload(path: Path, limit: int, original_error: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
with tempfile.TemporaryDirectory(prefix="lm-cache-inspect-") as temp_dir:
|
||||
copy_path = Path(temp_dir) / path.name
|
||||
shutil.copy2(path, copy_path)
|
||||
payload = sqlite_payload(copy_path, limit, allow_copy=False)
|
||||
payload["path"] = str(path)
|
||||
payload["inspected_copy"] = True
|
||||
payload["original_error"] = original_error
|
||||
return payload
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def json_file_summary(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {"exists": False}
|
||||
data = load_json(path)
|
||||
return {"exists": True, "summary": redact(summarize_value(data))}
|
||||
|
||||
|
||||
def model_roots(settings: dict[str, Any], active_library: str) -> dict[str, list[str]]:
|
||||
roots: dict[str, list[str]] = {}
|
||||
sources = [settings]
|
||||
library = settings.get("libraries", {}).get(active_library)
|
||||
if isinstance(library, dict):
|
||||
sources.insert(0, library)
|
||||
for source in sources:
|
||||
folder_paths = source.get("folder_paths")
|
||||
if isinstance(folder_paths, dict):
|
||||
for key, value in folder_paths.items():
|
||||
roots.setdefault(key, []).extend(normalize_path_list(value))
|
||||
for default_key, folder_key in (
|
||||
("default_lora_root", "loras"),
|
||||
("default_checkpoint_root", "checkpoints"),
|
||||
("default_embedding_root", "embeddings"),
|
||||
("default_unet_root", "unet"),
|
||||
):
|
||||
value = settings.get(default_key)
|
||||
if isinstance(value, str) and value:
|
||||
roots.setdefault(folder_key, []).append(expand_path(value))
|
||||
return {key: dedupe(values) for key, values in roots.items()}
|
||||
|
||||
|
||||
def resolve_recipes_root(settings: dict[str, Any], active_library: str) -> Path | None:
|
||||
recipes_path = settings.get("recipes_path")
|
||||
library = settings.get("libraries", {}).get(active_library)
|
||||
if isinstance(library, dict) and isinstance(library.get("recipes_path"), str):
|
||||
recipes_path = library["recipes_path"] or recipes_path
|
||||
if isinstance(recipes_path, str) and recipes_path.strip():
|
||||
return Path(expand_path(recipes_path.strip()))
|
||||
lora_roots = model_roots(settings, active_library).get("loras") or []
|
||||
return Path(lora_roots[0]) / "recipes" if lora_roots else None
|
||||
|
||||
|
||||
def example_images_payload(settings: dict[str, Any], active_library: str) -> dict[str, Any]:
|
||||
root = settings.get("example_images_path") or ""
|
||||
libraries = settings.get("libraries")
|
||||
library_count = len(libraries) if isinstance(libraries, dict) else 0
|
||||
scoped = library_count > 1
|
||||
root_path = Path(expand_path(root)) if isinstance(root, str) and root else None
|
||||
library_root = root_path / sanitize_library_name(active_library) if root_path and scoped else root_path
|
||||
return {
|
||||
"root": str(root_path or ""),
|
||||
"uses_library_scoped_folders": scoped,
|
||||
"library_root": str(library_root or ""),
|
||||
}
|
||||
|
||||
|
||||
def count_recipe_files(root: Path | None) -> int:
|
||||
if not root or not root.exists():
|
||||
return 0
|
||||
return sum(1 for _ in root.rglob("*.recipe.json"))
|
||||
|
||||
|
||||
def normalize_path_list(value: Any) -> list[str]:
|
||||
if isinstance(value, str):
|
||||
return [expand_path(value)] if value else []
|
||||
if isinstance(value, list):
|
||||
return [expand_path(item) for item in value if isinstance(item, str) and item]
|
||||
return []
|
||||
|
||||
|
||||
def expand_path(value: str) -> str:
|
||||
return str(Path(value).expanduser().resolve(strict=False))
|
||||
|
||||
|
||||
def sanitize_library_name(name: str) -> str:
|
||||
safe = re.sub(r"[^A-Za-z0-9_.-]", "_", name or "default")
|
||||
return safe or "default"
|
||||
|
||||
|
||||
def dedupe(values: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
for value in values:
|
||||
if value not in seen:
|
||||
result.append(value)
|
||||
seen.add(value)
|
||||
return result
|
||||
|
||||
|
||||
def redact(value: Any, key: str = "") -> Any:
|
||||
if key and SECRET_PATTERN.search(key):
|
||||
return "<redacted>"
|
||||
if isinstance(value, dict):
|
||||
return {str(k): redact(v, str(k)) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [redact(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def summarize_value(value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
return {key: summarize_value(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return {
|
||||
"type": "array",
|
||||
"length": len(value),
|
||||
"first": summarize_value(value[0]) if value else None,
|
||||
}
|
||||
return value
|
||||
|
||||
|
||||
def quote_identifier(identifier: str) -> str:
|
||||
return '"' + identifier.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def is_internal_sqlite_table(table: str) -> bool:
|
||||
return table.startswith("sqlite_") or table.endswith(("_data", "_idx", "_docsize", "_config", "_content"))
|
||||
|
||||
|
||||
def print_json(payload: Any) -> None:
|
||||
json.dump(payload, sys.stdout, indent=2, ensure_ascii=False)
|
||||
sys.stdout.write("\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,153 +0,0 @@
|
||||
# Recipe Batch Import Feature Design
|
||||
|
||||
## Overview
|
||||
Enable users to import multiple images as recipes in a single operation, rather than processing them individually. This feature addresses the need for efficient bulk recipe creation from existing image collections.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Frontend │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ BatchImportManager.js │
|
||||
│ ├── InputCollector (收集URL列表/目录路径) │
|
||||
│ ├── ConcurrencyController (自适应并发控制) │
|
||||
│ ├── ProgressTracker (进度追踪) │
|
||||
│ └── ResultAggregator (结果汇总) │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ batch_import_modal.html │
|
||||
│ └── 批量导入UI组件 │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ batch_import_progress.css │
|
||||
│ └── 进度显示样式 │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Backend │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ py/routes/handlers/recipe_handlers.py │
|
||||
│ ├── start_batch_import() - 启动批量导入 │
|
||||
│ ├── get_batch_import_progress() - 查询进度 │
|
||||
│ └── cancel_batch_import() - 取消导入 │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ py/services/batch_import_service.py │
|
||||
│ ├── 自适应并发执行 │
|
||||
│ ├── 结果汇总 │
|
||||
│ └── WebSocket进度广播 │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## API Endpoints
|
||||
|
||||
| 端点 | 方法 | 说明 |
|
||||
|------|------|------|
|
||||
| `/api/lm/recipes/batch-import/start` | POST | 启动批量导入,返回 operation_id |
|
||||
| `/api/lm/recipes/batch-import/progress` | GET | 查询进度状态 |
|
||||
| `/api/lm/recipes/batch-import/cancel` | POST | 取消导入 |
|
||||
|
||||
## Backend Implementation Details
|
||||
|
||||
### BatchImportService
|
||||
|
||||
Location: `py/services/batch_import_service.py`
|
||||
|
||||
Key classes:
|
||||
- `BatchImportItem`: Dataclass for individual import item
|
||||
- `BatchImportProgress`: Dataclass for tracking progress
|
||||
- `BatchImportService`: Main service class
|
||||
|
||||
Features:
|
||||
- Adaptive concurrency control (adjusts based on success/failure rate)
|
||||
- WebSocket progress broadcasting
|
||||
- Graceful error handling (individual failures don't stop the batch)
|
||||
- Result aggregation
|
||||
|
||||
### WebSocket Message Format
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "batch_import_progress",
|
||||
"operation_id": "xxx",
|
||||
"total": 50,
|
||||
"completed": 23,
|
||||
"success": 21,
|
||||
"failed": 2,
|
||||
"skipped": 0,
|
||||
"current_item": "image_024.png",
|
||||
"status": "running"
|
||||
}
|
||||
```
|
||||
|
||||
### Input Types
|
||||
|
||||
1. **URL List**: Array of URLs (http/https)
|
||||
2. **Local Paths**: Array of local file paths
|
||||
3. **Directory**: Path to directory with optional recursive flag
|
||||
|
||||
### Error Handling
|
||||
|
||||
- Invalid URLs/paths: Skip and record error
|
||||
- Download failures: Record error, continue
|
||||
- Metadata extraction failures: Mark as "no metadata"
|
||||
- Duplicate detection: Option to skip duplicates
|
||||
|
||||
## Frontend Implementation Details (TODO)
|
||||
|
||||
### UI Components
|
||||
|
||||
1. **BatchImportModal**: Main modal with tabs for URLs/Directory input
|
||||
2. **ProgressDisplay**: Real-time progress bar and status
|
||||
3. **ResultsSummary**: Final results with success/failure breakdown
|
||||
|
||||
### Adaptive Concurrency Controller
|
||||
|
||||
```javascript
|
||||
class AdaptiveConcurrencyController {
|
||||
constructor(options = {}) {
|
||||
this.minConcurrency = options.minConcurrency || 1;
|
||||
this.maxConcurrency = options.maxConcurrency || 5;
|
||||
this.currentConcurrency = options.initialConcurrency || 3;
|
||||
}
|
||||
|
||||
adjustConcurrency(taskDuration, success) {
|
||||
if (success && taskDuration < 1000 && this.currentConcurrency < this.maxConcurrency) {
|
||||
this.currentConcurrency = Math.min(this.currentConcurrency + 1, this.maxConcurrency);
|
||||
}
|
||||
if (!success || taskDuration > 10000) {
|
||||
this.currentConcurrency = Math.max(this.currentConcurrency - 1, this.minConcurrency);
|
||||
}
|
||||
return this.currentConcurrency;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
Backend (implemented):
|
||||
├── py/services/batch_import_service.py # 后端服务
|
||||
├── py/routes/handlers/batch_import_handler.py # API处理器 (added to recipe_handlers.py)
|
||||
├── tests/services/test_batch_import_service.py # 单元测试
|
||||
└── tests/routes/test_batch_import_routes.py # API集成测试
|
||||
|
||||
Frontend (TODO):
|
||||
├── static/js/managers/BatchImportManager.js # 主管理器
|
||||
├── static/js/managers/batch/ # 子模块
|
||||
│ ├── ConcurrencyController.js # 并发控制
|
||||
│ ├── ProgressTracker.js # 进度追踪
|
||||
│ └── ResultAggregator.js # 结果汇总
|
||||
├── static/css/components/batch-import-modal.css # 样式
|
||||
└── templates/components/batch_import_modal.html # Modal模板
|
||||
```
|
||||
|
||||
## Implementation Status
|
||||
|
||||
- [x] Backend BatchImportService
|
||||
- [x] Backend API handlers
|
||||
- [x] WebSocket progress broadcasting
|
||||
- [x] Unit tests
|
||||
- [x] Integration tests
|
||||
- [ ] Frontend BatchImportManager
|
||||
- [ ] Frontend UI components
|
||||
- [ ] E2E tests
|
||||
@@ -13,8 +13,5 @@ A clear and concise description of what the problem is. Ex. I'm always frustrate
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
||||
|
||||
+18
-1
@@ -7,15 +7,25 @@ py/run_test.py
|
||||
.vscode/
|
||||
cache/
|
||||
civitai/
|
||||
stats/
|
||||
wildcards/
|
||||
backups/
|
||||
logs/
|
||||
node_modules/
|
||||
coverage/
|
||||
.coverage
|
||||
model_cache/
|
||||
|
||||
# agent
|
||||
# agent / dev tooling
|
||||
.opencode/
|
||||
.claude/
|
||||
.sisyphus/
|
||||
.codex
|
||||
.omo
|
||||
reasonix.toml
|
||||
.reasonix/
|
||||
.codegraph/
|
||||
.playwright-mcp/
|
||||
|
||||
# Vue widgets development cache (but keep build output)
|
||||
vue-widgets/node_modules/
|
||||
@@ -24,3 +34,10 @@ vue-widgets/dist/
|
||||
|
||||
# Hypothesis test cache
|
||||
.hypothesis/
|
||||
|
||||
# Working/research notes (not committed)
|
||||
.docs/
|
||||
|
||||
# HF enrichment validation baseline snapshots (contain potentially
|
||||
# NSFW README content fetched from community model repos)
|
||||
tests/enrich_hf_validation/baselines/
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
# Embeddings Usage Tracking — Hybrid Approach (Plan C)
|
||||
|
||||
> **Status**: Reference document for future implementation
|
||||
> **Current implementation**: Plan A (prompt text parsing only, see `usage_stats.py:_process_embeddings`)
|
||||
> **Next step**: Add Plan B as a supplement when edge-case coverage is needed
|
||||
|
||||
## Problem
|
||||
|
||||
Embeddings in ComfyUI are not loaded through dedicated ComfyUI nodes like LoRAs or
|
||||
Checkpoints. They are resolved during CLIP tokenization when the prompt text contains
|
||||
`embedding:<name>` syntax (see `comfy/sd1_clip.py:SDTokenizer.tokenize_with_weights`).
|
||||
|
||||
This means the existing metadata_collector hook (which intercepts node execution via
|
||||
`_map_node_over_list`) cannot capture embeddings the same way it captures LoRAs and
|
||||
checkpoints — there is no "EmbeddingLoader" node to intercept.
|
||||
|
||||
## Solution Architecture
|
||||
|
||||
The hybrid approach combines **two complementary mechanisms** to capture embedding
|
||||
usage from all possible paths.
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ Plan A (已实现) │
|
||||
│ │
|
||||
│ MetadataRegistry.prompt_metadata["prompts"] │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ _process_embeddings() │
|
||||
│ │ │
|
||||
│ ├─ Iterate all prompt node texts │
|
||||
│ ├─ regex extract "embedding:<name>" │
|
||||
│ ├─ resolve name → sha256 via EmbeddingScanner │
|
||||
│ └─ UsageStats.stats["embeddings"][sha256]++ │
|
||||
│ │
|
||||
│ Coverage: ~95% — all CLIPTextEncode/Flux/etc nodes │
|
||||
│ │
|
||||
│ Gap: Custom nodes that load embeddings programmatically │
|
||||
│ without putting embedding:name in prompt text │
|
||||
└─────────────────────────────────────────────────────────┘
|
||||
|
||||
+
|
||||
↓ (future: enable Plan B when needed)
|
||||
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ Plan B (未来 — monkey-patch) │
|
||||
│ │
|
||||
│ comfy/sd1_clip.py:load_embed() │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ Monkey-patch intercepts EVERY embedding file load │
|
||||
│ │ │
|
||||
│ ├─ Records embedding_name + success/failure │
|
||||
│ ├─ Associates with current prompt_id (via registry)│
|
||||
│ └─ Feeds into UsageStats same as Plan A │
|
||||
│ │
|
||||
│ Coverage: 100% — catches ALL embedding loads │
|
||||
│ │
|
||||
│ Cost: Requires patching into ComfyUI internals │
|
||||
│ (sd1_clip.py, sdxl_clip.py, some text_encoders) │
|
||||
└─────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Plan B Detail — Monkey-patch `load_embed`
|
||||
|
||||
### Target Function
|
||||
|
||||
**`comfy.sd1_clip.load_embed(embedding_name, embedding_directory, embedding_size, embed_key=None)`**
|
||||
at line 415 of `sd1_clip.py`.
|
||||
|
||||
This is the **single choke point** for all embedding file loads in ComfyUI. Every
|
||||
CLIP variant (SD1, SDXL, SD3, Flux) calls this same function.
|
||||
|
||||
### Implementation Sketch
|
||||
|
||||
```python
|
||||
# In metadata_collector/metadata_hook.py (or a new module)
|
||||
import comfy.sd1_clip as sd1_clip
|
||||
|
||||
_original_load_embed = sd1_clip.load_embed
|
||||
|
||||
def _patched_load_embed(embedding_name, embedding_directory, embedding_size, embed_key=None):
|
||||
result = _original_load_embed(
|
||||
embedding_name, embedding_directory, embedding_size, embed_key
|
||||
)
|
||||
if result is not None:
|
||||
_record_embedding_usage(embedding_name)
|
||||
return result
|
||||
|
||||
sd1_clip.load_embed = _patched_load_embed
|
||||
```
|
||||
|
||||
### Prompt ID Association
|
||||
|
||||
The challenge is associating the `load_embed` call with the current `prompt_id`.
|
||||
Options:
|
||||
|
||||
1. **Thread-local / contextvar**: Store current `prompt_id` in a `contextvars.ContextVar`
|
||||
that the metadata_collector sets at the start of each prompt execution.
|
||||
|
||||
2. **MetadataRegistry singleton**: The MetadataRegistry already has `current_prompt_id`.
|
||||
The patch can read it directly since both run in the same thread.
|
||||
|
||||
3. **Lazy aggregation**: Instead of associating with prompt_id at load time, collect
|
||||
all loaded embedding names in a global set during execution, then flush to
|
||||
UsageStats after the prompt completes.
|
||||
|
||||
### Files to Patch
|
||||
|
||||
| File | Function | Coverage |
|
||||
|------|----------|----------|
|
||||
| `comfy/sd1_clip.py:415` | `load_embed()` | Primary — SD1.x, SDXL, SD3, Flux |
|
||||
| `comfy/sdxl_clip.py` | Not needed (calls `sd1_clip.SDTokenizer`) | — |
|
||||
| `comfy/text_encoders/sd3_clip.py` | Not needed (calls `sd1_clip.SDTokenizer`) | — |
|
||||
| `comfy/text_encoders/flux.py` | Not needed (calls `sd1_clip.SDTokenizer`) | — |
|
||||
|
||||
The SD1 tokenizer is the base class for all CLIP variants' tokenizers, so patching
|
||||
`load_embed` covers them all.
|
||||
|
||||
### Edge Cases
|
||||
|
||||
| Edge Case | Plan A | Plan B |
|
||||
|-----------|--------|--------|
|
||||
| `embedding:name` in CLIPTextEncode | ✅ | ✅ |
|
||||
| `embedding:name` in CLIPTextEncodeFlux | ✅ | ✅ |
|
||||
| `embedding:name` in PromptLM (LoRA Manager) | ✅ | ✅ |
|
||||
| `embedding:name` in WAS_Text_to_Conditioning | ✅ | ✅ |
|
||||
| Custom node that loads embedding programmatically | ❌ | ✅ |
|
||||
| Embedding loaded multiple times in same prompt | ✅ (dedup via set) | ✅ (dedup via set) |
|
||||
| Embedding file not found | N/A | ✅ (can log) |
|
||||
| Embedding dimension mismatch | N/A | ✅ (can log) |
|
||||
| Text encoder with non-standard tokenizer (LLaMA, T5...) | Partial | ✅ (if it calls load_embed) |
|
||||
|
||||
## Migration Path: Standalone → Hybrid
|
||||
|
||||
### Phase 1 — Plan A (当前状态)
|
||||
- Prompt text parsing only
|
||||
- No monkey-patching required
|
||||
- Covers all standard workflows
|
||||
|
||||
### Phase 2 — Enable Plan B (未来工作)
|
||||
1. Add monkey-patch of `load_embed` in `metadata_collector/metadata_hook.py` (alongside
|
||||
the existing `_map_node_over_list` hook)
|
||||
2. Collect loaded embedding names in a `set()` on the registry
|
||||
3. In `UsageStats._process_embeddings()`, merge the Plan A results (from prompt text)
|
||||
with the Plan B results (from the patch)
|
||||
4. Add `prompt_data` field on MetadataRegistry to store loaded embeddings per prompt
|
||||
|
||||
### Deduplication
|
||||
|
||||
```python
|
||||
# Merge Plan A + Plan B results in _process_embeddings
|
||||
plan_a_names = extract_from_prompt_texts(prompts_data)
|
||||
plan_b_names = registry.get_loaded_embeddings(prompt_id)
|
||||
|
||||
all_names = plan_a_names | plan_b_names
|
||||
```
|
||||
|
||||
## Testing the Hybrid
|
||||
|
||||
| Scenario | What to verify |
|
||||
|----------|---------------|
|
||||
| Standard `embedding:name` in prompt | Plan A captures it |
|
||||
| Embedding loaded by custom node script | Plan B captures it |
|
||||
| Both paths fire for same embedding | No double-counting (dedup) |
|
||||
| Embedding name resolves to hash | EmbeddingScanner.get_hash_by_filename works |
|
||||
| No embedding scanner available | Graceful skip, no crash |
|
||||
| Missing embedding file | Plan B logs warning, Plan A skips gracefully |
|
||||
| Empty prompt | No crash, no entries |
|
||||
| Standalone mode | Both plans disabled gracefully |
|
||||
|
||||
## Key Files Reference
|
||||
|
||||
| File | Role |
|
||||
|------|------|
|
||||
| `py/utils/usage_stats.py` | Core — `_process_embeddings()` for Plan A |
|
||||
| `py/metadata_collector/constants.py` | `EMBEDDINGS` category constant |
|
||||
| `py/metadata_collector/metadata_hook.py` | Future — monkey-patch for Plan B |
|
||||
| `py/services/embedding_scanner.py` | Hash resolution service |
|
||||
| `py/routes/stats_routes.py` | Already handles `usage_data.get('embeddings', {})` |
|
||||
| `comfy/sd1_clip.py` (ComfyUI) | `load_embed()` — Plan B target |
|
||||
@@ -31,7 +31,7 @@ COVERAGE_FILE=coverage/backend/.coverage pytest \
|
||||
--cov-report=xml:coverage/backend/coverage.xml
|
||||
```
|
||||
|
||||
### Frontend Development (Standalone Web UI)
|
||||
### Frontend Development (LoRA Manager Web UI)
|
||||
|
||||
```bash
|
||||
npm install
|
||||
@@ -102,6 +102,7 @@ npm run test:coverage # Generate coverage report
|
||||
- ComfyUI: `app.registerExtension()`, `node.addDOMWidget(name, type, element, options)`
|
||||
- Event handlers via `addEventListener` or widget callbacks
|
||||
- Shared utilities: `web/comfyui/utils.js`
|
||||
- Dual-mode rendering patterns (canvas vs Vue): see `docs/comfyui-dual-mode-widgets.md`
|
||||
|
||||
### Vue Composables Pattern
|
||||
|
||||
@@ -136,7 +137,13 @@ npm run test:coverage # Generate coverage report
|
||||
- Dual mode: ComfyUI plugin (folder_paths) vs standalone (settings.json)
|
||||
- Detection: `os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"`
|
||||
- Run `python scripts/sync_translation_keys.py` after adding UI strings to `locales/en.json`
|
||||
- Symlinks require normalized paths
|
||||
- Symlinks require normalized paths.
|
||||
**Business paths vs real paths**: All stored paths and operation routing use the
|
||||
original paths as they appear under configured model roots — symlinks are NOT
|
||||
resolved. `os.path.realpath` is only for scanner dedup and the symlink cache.
|
||||
Any path passed to `os.remove`/`os.rename`/`shutil.move` or validated by a
|
||||
containment check MUST use the business path (i.e. `os.path.abspath`, not
|
||||
`realpath`).
|
||||
|
||||
## Git / Commit Messages
|
||||
|
||||
@@ -147,9 +154,9 @@ npm run test:coverage # Generate coverage report
|
||||
|
||||
## Frontend UI Architecture
|
||||
|
||||
### 1. Standalone Web UI
|
||||
### 1. LoRA Manager Web UI
|
||||
- Location: `./static/` and `./templates/`
|
||||
- Tech: Vanilla JS + CSS, served by standalone server
|
||||
- Tech: Vanilla JS + CSS, served by the hosting server (ComfyUI app in plugin mode, `standalone.py` in standalone mode)
|
||||
- Tests via npm in root directory
|
||||
|
||||
### 2. ComfyUI Custom Node Widgets
|
||||
|
||||
+18
@@ -15,6 +15,10 @@ try: # pragma: no cover - import fallback for pytest collection
|
||||
from .py.nodes.lora_pool import LoraPoolLM
|
||||
from .py.nodes.lora_randomizer import LoraRandomizerLM
|
||||
from .py.nodes.lora_cycler import LoraCyclerLM
|
||||
from .py.nodes.lora_info import LoraInfoLM
|
||||
from .py.nodes.lora_syntax_to_path import LoraSyntaxToPath
|
||||
from .py.nodes.create_hook_lora import CreateHookLoraLM
|
||||
from .py.nodes.metadata_overwrite import MetadataOverwriteLM
|
||||
from .py.metadata_collector import init as init_metadata_collector
|
||||
except (
|
||||
ImportError
|
||||
@@ -56,6 +60,16 @@ except (
|
||||
"py.nodes.lora_randomizer"
|
||||
).LoraRandomizerLM
|
||||
LoraCyclerLM = importlib.import_module("py.nodes.lora_cycler").LoraCyclerLM
|
||||
LoraInfoLM = importlib.import_module("py.nodes.lora_info").LoraInfoLM
|
||||
LoraSyntaxToPath = importlib.import_module(
|
||||
"py.nodes.lora_syntax_to_path"
|
||||
).LoraSyntaxToPath
|
||||
CreateHookLoraLM = importlib.import_module(
|
||||
"py.nodes.create_hook_lora"
|
||||
).CreateHookLoraLM
|
||||
MetadataOverwriteLM = importlib.import_module(
|
||||
"py.nodes.metadata_overwrite"
|
||||
).MetadataOverwriteLM
|
||||
init_metadata_collector = importlib.import_module("py.metadata_collector").init
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
@@ -75,6 +89,10 @@ NODE_CLASS_MAPPINGS = {
|
||||
LoraPoolLM.NAME: LoraPoolLM,
|
||||
LoraRandomizerLM.NAME: LoraRandomizerLM,
|
||||
LoraCyclerLM.NAME: LoraCyclerLM,
|
||||
LoraInfoLM.NAME: LoraInfoLM,
|
||||
LoraSyntaxToPath.NAME: LoraSyntaxToPath,
|
||||
CreateHookLoraLM.NAME: CreateHookLoraLM,
|
||||
MetadataOverwriteLM.NAME: MetadataOverwriteLM,
|
||||
}
|
||||
|
||||
WEB_DIRECTORY = "./web/comfyui"
|
||||
|
||||
+521
-297
File diff suppressed because it is too large
Load Diff
@@ -1,183 +0,0 @@
|
||||
## Overview
|
||||
|
||||
The **LoRA Manager Civitai Extension** is a Browser extension designed to work seamlessly with [LoRA Manager](https://github.com/willmiao/ComfyUI-Lora-Manager) to significantly enhance your browsing experience on [Civitai](https://civitai.com). With this extension, you can:
|
||||
|
||||
✅ Instantly see which models are already present in your local library
|
||||
✅ Download new models with a single click
|
||||
✅ Manage downloads efficiently with queue and parallel download support
|
||||
✅ Keep your downloaded models automatically organized according to your custom settings
|
||||
|
||||

|
||||
|
||||
**Update:** It now also supports browsing on [CivArchive](https://civarchive.com/) (formerly CivitaiArchive).
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
## Why Supporter Access?
|
||||
|
||||
LoRA Manager is built with love for the Stable Diffusion and ComfyUI communities. Your support makes it possible for me to keep improving and maintaining the tool full-time.
|
||||
|
||||
Supporter-exclusive features help ensure the long-term sustainability of LoRA Manager, allowing continuous updates, new features, and better performance for everyone.
|
||||
|
||||
Every contribution directly fuels development and keeps the core LoRA Manager free and open-source. In addition to monthly supporters, one-time donation supporters will also receive a license key, with the duration scaling according to the contribution amount. Thank you for helping keep this project alive and growing. ❤️
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
|
||||
### Supported Browsers & Installation Methods
|
||||
|
||||
| Browser | Installation Method |
|
||||
|--------------------|-------------------------------------------------------------------------------------|
|
||||
| **Google Chrome** | [Chrome Web Store link](https://chromewebstore.google.com/detail/capigligggeijgmocnaflanlbghnamgm?utm_source=item-share-cb) |
|
||||
| **Microsoft Edge** | Install via Chrome Web Store (compatible) |
|
||||
| **Brave Browser** | Install via Chrome Web Store (compatible) |
|
||||
| **Opera** | Install via Chrome Web Store (compatible) |
|
||||
| **Firefox** | <div id="firefox-install" class="install-ok"><a href="https://github.com/willmiao/lm-civitai-extension-firefox/releases/latest/download/extension.xpi">📦 Install Firefox Extension (reviewed and verified by Mozilla)</a></div> |
|
||||
|
||||
For non-Chrome browsers (e.g., Microsoft Edge), you can typically install extensions from the Chrome Web Store by following these steps: open the extension’s Chrome Web Store page, click 'Get extension', then click 'Allow' when prompted to enable installations from other stores, and finally click 'Add extension' to complete the installation.
|
||||
|
||||
---
|
||||
|
||||
## Privacy & Security
|
||||
|
||||
I understand concerns around browser extensions and privacy, and I want to be fully transparent about how the **LM Civitai Extension** works:
|
||||
|
||||
- **Reviewed and Verified**
|
||||
This extension has been **manually reviewed and approved by the Chrome Web Store**. The Firefox version uses the **exact same code** (only the packaging format differs) and has passed **Mozilla’s Add-on review**.
|
||||
|
||||
- **Minimal Network Access**
|
||||
The only external server this extension connects to is:
|
||||
**`https://willmiao.shop`** — used solely for **license validation**.
|
||||
|
||||
It does **not collect, transmit, or store any personal or usage data**.
|
||||
No browsing history, no user IDs, no analytics, no hidden trackers.
|
||||
|
||||
- **Local-Only Model Detection**
|
||||
Model detection and LoRA Manager communication all happen **locally** within your browser, directly interacting with your local LoRA Manager backend.
|
||||
|
||||
I value your trust and are committed to keeping your local setup private and secure. If you have any questions, feel free to reach out!
|
||||
|
||||
---
|
||||
|
||||
## How to Use
|
||||
|
||||
After installing the extension, you'll automatically receive a **7-day trial** to explore all features.
|
||||
|
||||
When the extension is correctly installed and your license is valid:
|
||||
|
||||
- Open **Civitai**, and you'll see visual indicators added by the extension on model cards, showing:
|
||||
- ✅ Models already present in your local library
|
||||
- ⬇️ A download button for models not in your library
|
||||
|
||||
Clicking the download button adds the corresponding model version to the download queue, waiting to be downloaded. You can set up to **5 models to download simultaneously**.
|
||||
|
||||
### Visual Indicators Appear On:
|
||||
|
||||
- **Home Page** — Featured models
|
||||
- **Models Page**
|
||||
- **Creator Profiles** — If the creator has set their models to be visible
|
||||
- **Recommended Resources** — On individual model pages
|
||||
|
||||
### Version Buttons on Model Pages
|
||||
|
||||
On a specific model page, visual indicators also appear on version buttons, showing which versions are already in your local library.
|
||||
|
||||
**Starting from v0.4.8**, model pages use a dedicated download button for better compatibility. When switching to a specific version by clicking a version button:
|
||||
|
||||
- The new **dedicated download button** directly triggers download via **LoRA Manager**
|
||||
- The **original download button** remains unchanged for standard browser downloads
|
||||
|
||||

|
||||
|
||||
### Hide Models Already in Library (Beta)
|
||||
|
||||
**New in v0.4.8**: A new **Hide models already in library (Beta)** option makes it easier to focus on models you haven't added yet. It can be enabled from Settings, or toggled quickly using **Ctrl + Shift + H** (macOS: **Command + Shift + H**).
|
||||
|
||||
### Resources on Image Pages — now shows in-library indicators for image resources plus one-click recipe import
|
||||
|
||||
- **One-Click Import Civitai Image as Recipe** — Import any Civitai image as a recipe with a single click in the Resources Used panel.
|
||||
- **Auto-Queue Missing Assets** — In Settings you can decide if LoRAs or checkpoints referenced by that image should automatically be added to your download queue.
|
||||
- **More Accurate Metadata** — Importing directly from the page is faster than copying inside LM and keeps on-site tags and other metadata perfectly aligned.
|
||||
|
||||

|
||||
|
||||
[](https://github.com/user-attachments/assets/41fd4240-c949-4f83-bde7-8f3124c09494)
|
||||
|
||||
---
|
||||
|
||||
## Model Download Location & LoRA Manager Settings
|
||||
|
||||
To use the **one-click download function**, you must first set:
|
||||
|
||||
- Your **Default LoRAs Root**
|
||||
- Your **Default Checkpoints Root**
|
||||
|
||||
These are set within LoRA Manager's settings.
|
||||
|
||||
When everything is configured, downloaded model files will be placed in:
|
||||
|
||||
`<Default_Models_Root>/<Base_Model_of_the_Model>/<First_Tag_of_the_Model>`
|
||||
|
||||
|
||||
### Update: Default Path Customization (2025-07-21)
|
||||
|
||||
A new setting to customize the default download path has been added in the nightly version. You can now personalize where models are saved when downloading via the LM Civitai Extension.
|
||||
|
||||

|
||||
|
||||
The previous YAML path mapping file will be deprecated—settings will now be unified in settings.json to simplify configuration.
|
||||
|
||||
---
|
||||
|
||||
## Backend Port Configuration
|
||||
|
||||
If your **ComfyUI** or **LoRA Manager** backend is running on a port **other than the default 8188**, you must configure the backend port in the extension's settings.
|
||||
|
||||
After correctly setting and saving the port, you'll see in the extension's header area:
|
||||
- A **Healthy** status with the tooltip: `Connected to LoRA Manager on port xxxx`
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Connecting to a Remote LoRA Manager
|
||||
|
||||
If your LoRA Manager is running on another computer, you can still connect from your browser using port forwarding.
|
||||
|
||||
> **Why can't you set a remote IP directly?**
|
||||
>
|
||||
> For privacy and security, the extension only requests access to `http://127.0.0.1/*`. Supporting remote IPs would require much broader permissions, which may be rejected by browser stores and could raise user concerns.
|
||||
|
||||
**Solution: Port Forwarding with `socat`**
|
||||
|
||||
On your browser computer, run:
|
||||
|
||||
`socat TCP-LISTEN:8188,bind=127.0.0.1,fork TCP:REMOTE.IP.ADDRESS.HERE:8188`
|
||||
|
||||
- Replace `REMOTE.IP.ADDRESS.HERE` with the IP of the machine running LoRA Manager.
|
||||
- Adjust the port if needed.
|
||||
|
||||
This lets the extension connect to `127.0.0.1:8188` as usual, with traffic forwarded to your remote server.
|
||||
|
||||
_Thanks to user **Temikus** for sharing this solution!_
|
||||
|
||||
---
|
||||
|
||||
## Roadmap
|
||||
|
||||
The extension will evolve alongside **LoRA Manager** improvements. Planned features include:
|
||||
|
||||
- [x] Support for **additional model types** (e.g., embeddings)
|
||||
- [x] One-click **Recipe Import**
|
||||
- [x] Display of in-library status for all resources in the **Resources Used** section of the image page
|
||||
- [x] One-click **Auto-organize Models**
|
||||
- [x] **Hide models already in library (Beta)** - Focus on models you haven't added yet
|
||||
|
||||
**Stay tuned — and thank you for your support!**
|
||||
|
||||
---
|
||||
@@ -0,0 +1,208 @@
|
||||
# Agent Skills System
|
||||
|
||||
The LoRA Manager agent skills system enables LLM-powered metadata enrichment and other AI-driven tasks. Users configure their own LLM provider (BYOK), and skills are executed through right-click context menu actions.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ LoRA Manager Backend │
|
||||
│ │
|
||||
│ ┌──────────────┐ ┌────────────────┐ │
|
||||
│ │ LLMService │───▶│ LLM Provider │ │
|
||||
│ │ (BYOK config, │◀───│ (OpenAI/Ollama │ │
|
||||
│ │ API calls) │ │ /custom) │ │
|
||||
│ └───────┬───────┘ └────────────────┘ │
|
||||
│ │ │
|
||||
│ ┌───────▼───────────────────────┐ │
|
||||
│ │ AgentService │ │
|
||||
│ │ (orchestration: validate │ │
|
||||
│ │ → LLM call → post-process │ │
|
||||
│ │ → WebSocket broadcast) │ │
|
||||
│ └───────┬───────────────────────┘ │
|
||||
│ │ │
|
||||
│ ┌───────▼───────────────────────┐ │
|
||||
│ │ SkillRegistry │ │
|
||||
│ │ ┌─────────────────────────┐ │ │
|
||||
│ │ │ enrich_hf_metadata: │ │ │
|
||||
│ │ │ - skill.yaml │ │ │
|
||||
│ │ │ - prompt.md │ │ │
|
||||
│ │ │ - handler.py │ │ │
|
||||
│ │ └─────────────────────────┘ │ │
|
||||
│ └───────────────────────────────┘ │
|
||||
└──────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Key Design Principle
|
||||
|
||||
**Skills define *what* to do (prompt + post-processing). The AgentService handles *how* (LLM calls, validation, progress).**
|
||||
|
||||
Skills never call the LLM directly. This keeps BYOK configuration centralized and provider-agnostic.
|
||||
|
||||
## BYOK Configuration
|
||||
|
||||
Users configure their LLM provider in **Settings → AI Provider**:
|
||||
|
||||
| Setting | Description | Example |
|
||||
|---|---|---|
|
||||
| `llm_provider` | Provider type | `openai`, `ollama`, or `custom` |
|
||||
| `llm_api_key` | API key (not needed for local Ollama) | `sk-...` |
|
||||
| `llm_api_base` | Custom API base URL (empty = provider default) | `https://api.openai.com/v1` |
|
||||
| `llm_model` | Model name | `gpt-4o-mini` |
|
||||
|
||||
Environment variable overrides: `LLM_API_KEY`, `LLM_MODEL`, `LLM_API_BASE`, `LLM_PROVIDER`.
|
||||
|
||||
### Supported Providers
|
||||
|
||||
- **OpenAI**: Uses `https://api.openai.com/v1` by default
|
||||
- **Ollama** (local): Uses `http://localhost:11434/v1`, no API key required
|
||||
- **Custom**: Any OpenAI-compatible endpoint (vLLM, LM Studio, etc.) — set `llm_api_base` explicitly
|
||||
|
||||
## Available Skills
|
||||
|
||||
### enrich_hf_metadata
|
||||
|
||||
Enriches HuggingFace-downloaded models with metadata extracted by an LLM from the HF model card.
|
||||
|
||||
**Entry point**: Right-click context menu → "Enrich Metadata (Agent)"
|
||||
|
||||
**What it does**:
|
||||
1. Reads the model's `.metadata.json` to get the `hf_url`
|
||||
2. Fetches the README.md from the HuggingFace repository
|
||||
3. Sends the README + local metadata to the LLM for structured extraction
|
||||
4. Writes extracted fields to `.metadata.json`:
|
||||
- `base_model` — only if current value is empty
|
||||
- `trainedWords` — trigger words (LoRA only, if none exist)
|
||||
- `modelDescription` — concise summary (if none exists)
|
||||
- `tags` — merged with existing tags, deduplicated
|
||||
- `metadata_source` — audit trail: `agent:enrich_hf_metadata`
|
||||
- `llm_enriched_at` — ISO timestamp
|
||||
5. Downloads and optimizes preview image (if LLM found one in the README)
|
||||
6. Updates the scanner cache
|
||||
7. Broadcasts WebSocket progress events
|
||||
|
||||
**Model types**: LoRA, Checkpoint, Embedding
|
||||
|
||||
## Adding a New Skill
|
||||
|
||||
### 1. Create the skill directory
|
||||
|
||||
```
|
||||
py/services/agent/skills/<skill_name>/
|
||||
├── skill.yaml # Skill metadata and schemas
|
||||
├── prompt.md # LLM prompt template
|
||||
└── handler.py # Pre-processing and post-processing
|
||||
```
|
||||
|
||||
### 2. Write skill.yaml
|
||||
|
||||
```yaml
|
||||
name: my_skill
|
||||
title: "My Skill"
|
||||
description: "What this skill does"
|
||||
llm_required: true
|
||||
model_type_filter: ["lora"] # or null for all types
|
||||
input_schema:
|
||||
type: object
|
||||
properties:
|
||||
model_paths:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
required:
|
||||
- model_paths
|
||||
output_schema:
|
||||
type: object
|
||||
properties:
|
||||
# ... JSON schema for LLM output
|
||||
permissions:
|
||||
write_metadata: true
|
||||
write_previews: false
|
||||
network_domains:
|
||||
- "example.com"
|
||||
```
|
||||
|
||||
### 3. Write prompt.md
|
||||
|
||||
Use `{{variable}}` placeholders that will be replaced with data from the `prepare` function:
|
||||
|
||||
```markdown
|
||||
You are an expert assistant...
|
||||
|
||||
Model URL: {{hf_url}}
|
||||
README content:
|
||||
{{readme_content}}
|
||||
|
||||
Current metadata:
|
||||
{{current_metadata}}
|
||||
```
|
||||
|
||||
### 4. Write handler.py
|
||||
|
||||
```python
|
||||
async def prepare(model_path: str, input_data: dict) -> dict:
|
||||
"""Gather context for the LLM prompt. Returns variables for template rendering."""
|
||||
return {
|
||||
"model_path": model_path,
|
||||
# ... other variables used in prompt.md
|
||||
}
|
||||
|
||||
async def post_process(context) -> dict:
|
||||
"""Apply the LLM-extracted data to the model."""
|
||||
llm_response = context.llm_response
|
||||
# ... write metadata, download previews, update cache
|
||||
return {
|
||||
"success": True,
|
||||
"updated_fields": ["base_model", "tags"],
|
||||
"errors": [],
|
||||
}
|
||||
```
|
||||
|
||||
**Important**: Use absolute imports (`from py.utils.metadata_manager import MetadataManager`) because skills are loaded via `importlib.util.spec_from_file_location`, which doesn't support relative imports.
|
||||
|
||||
### 5. Test
|
||||
|
||||
The skill is automatically discovered by `SkillRegistry` on startup. Test with:
|
||||
|
||||
```python
|
||||
pytest tests/services/test_agent_service.py
|
||||
```
|
||||
|
||||
## API Endpoints
|
||||
|
||||
| Method | Path | Description |
|
||||
|---|---|---|
|
||||
| GET | `/api/lm/agent/skills` | List available skills |
|
||||
| POST | `/api/lm/agent/execute/{skill_name}` | Execute a skill (body: `{"model_paths": [...]}`) |
|
||||
| POST | `/api/lm/agent/cancel` | Cancel running skill (stub) |
|
||||
|
||||
## WebSocket Events
|
||||
|
||||
| Type | When | Key fields |
|
||||
|---|---|---|
|
||||
| `agent_progress` | Skill started/processing | `skill`, `status`, `total`, `processed`, `success`, `current_path` |
|
||||
| `agent_progress` | Skill completed | `skill`, `status`, `updated_models`, `errors`, `summary` |
|
||||
| `agent_progress` | Skill error | `skill`, `status`, `error` |
|
||||
|
||||
## Security Model
|
||||
|
||||
Skills declare permissions in `skill.yaml`:
|
||||
- `write_metadata` — can write `.metadata.json` files
|
||||
- `write_previews` — can download/replace preview images
|
||||
- `network_domains` — allowed domains for HTTP requests
|
||||
|
||||
These are declarative constraints checked by `AgentService`. They are defense-in-depth, not a sandbox — the Python process can technically do anything, but the contract is clear and auditable.
|
||||
|
||||
## File Locations
|
||||
|
||||
| Component | Path |
|
||||
|---|---|
|
||||
| LLMService | `py/services/llm_service.py` |
|
||||
| AgentService | `py/services/agent/agent_service.py` |
|
||||
| SkillRegistry | `py/services/agent/skill_registry.py` |
|
||||
| SkillDefinition | `py/services/agent/skill_definition.py` |
|
||||
| Skills directory | `py/services/agent/skills/` |
|
||||
| Route handlers | `py/routes/handlers/agent_handlers.py` |
|
||||
| Frontend manager | `static/js/managers/AgentManager.js` |
|
||||
| Settings UI | `templates/components/modals/settings_modal.html` |
|
||||
| Context menu | `templates/components/context_menu.html` |
|
||||
@@ -0,0 +1,65 @@
|
||||
# ComfyUI Dual-Mode Widget Rendering
|
||||
|
||||
ComfyUI custom node widgets render in one of two modes. Patterns that work in one often fail silently in the other. Test both.
|
||||
|
||||
## Mode Detection
|
||||
|
||||
```js
|
||||
typeof LiteGraph !== 'undefined' && LiteGraph.vueNodesMode
|
||||
```
|
||||
|
||||
In Vue SFCs, `window.LiteGraph` is unavailable — pass as a prop from `main.ts`.
|
||||
|
||||
## Canvas Mode Layout
|
||||
|
||||
Uses `computeLayoutSize()` + `distributeSpace()` to allocate widget height within the node. Widgets with `computeLayoutSize` participate in space distribution; those with `computeSize` have fixed height.
|
||||
|
||||
- `getMinHeight()` in `addDOMWidget` options → minimum widget height
|
||||
- `widget.computeLayoutSize()` → `{ minHeight, minWidth, maxHeight? }`
|
||||
- Avoid `getMaxHeight()` unless the widget genuinely needs a fixed cap (prevents user resize)
|
||||
|
||||
## Vue Mode Layout
|
||||
|
||||
Uses CSS Grid (`grid-template-rows`) + `ResizeObserver`. The ResizeObserver watches the widget's DOM and feeds back into grid row sizing. This creates a feedback loop: content grows → row resizes → more space for content → content reflows/grows → row resizes again.
|
||||
|
||||
### Height Containment
|
||||
|
||||
The fix: `contain: layout size` on the widget root. This tells the browser the element's intrinsic size is CSS-determined, not driven by descendant content. The ResizeObserver sees a stable size and the loop is broken.
|
||||
|
||||
```css
|
||||
.widget-root.lm-vue-node {
|
||||
height: 100%;
|
||||
min-height: var(--comfy-widget-min-height, 200px);
|
||||
contain: layout size;
|
||||
}
|
||||
```
|
||||
|
||||
Existing examples: `.lm-loras-container.lm-vue-node` and `.comfy-tags-container.lm-vue-node` in `web/comfyui/lm_styles.css`.
|
||||
|
||||
**Do NOT** fix height issues with `maxHeight`, `getMaxHeight()`, or inline `max-height` — these prevent the user from resizing the node.
|
||||
|
||||
## Scroll Wheel Isolation
|
||||
|
||||
Both modes need to distinguish "user wants to scroll widget content" from "user wants to zoom canvas".
|
||||
|
||||
**Canvas mode:** Add `@wheel` on widget root. Check `event.target.closest(selector)` for scrollable sub-areas. If scrollable → `event.stopPropagation()`. Otherwise → `app.canvas.processMouseWheel(event)`.
|
||||
|
||||
**Vue mode:** Add CSS class `lm-wheel-scrollable` to scrollable elements. The global capture-phase hook in `web/comfyui/utils.js` (`enableListWheelScroll`) detects wheel events on marked elements and manually scrolls them via `element.scrollTop`, consuming the event before canvas zoom sees it.
|
||||
|
||||
## DOM Structure
|
||||
|
||||
`main.ts` creates an outer `<div>` container, then `vueApp.mount(container)`. The Vue app renders its own root element inside.
|
||||
|
||||
- `container.id` / `container.style.*` → outer element
|
||||
- Vue scoped `<style>` → `[data-v-hash]` applies only to Vue root
|
||||
|
||||
Classes needed by scoped Vue CSS must go on the Vue root element. Pass data as props and bind with `:class` rather than manipulating the DOM from `main.ts`.
|
||||
|
||||
## Serialization
|
||||
|
||||
For stateful widgets that need workflow persistence:
|
||||
|
||||
- `serialize: true` in `addDOMWidget` options
|
||||
- `serializeValue()` → state snapshot (called on workflow save)
|
||||
- `onSetValue(v)` → restore state (called on workflow load)
|
||||
- Always handle missing keys in restored value for backward compatibility with old workflows
|
||||
@@ -39,6 +39,7 @@ These fields are present in all model metadata files.
|
||||
| `metadata_source` | string\|null | ❌ No | ✅ Yes | Last provider that supplied metadata (see below) |
|
||||
| `last_checked_at` | float | ❌ No (default: `0`) | ✅ Yes | Unix timestamp of last metadata check |
|
||||
| `hash_status` | string | ❌ No (default: `"completed"`) | ✅ Yes | Hash calculation status: `"pending"`, `"calculating"`, `"completed"`, `"failed"` |
|
||||
| `autov3` | string\|null | ❌ No | ✅ Yes | CivitAI AutoV3 hash (first 12 chars, lowercase hex) sourced from the safetensors embedded metadata (`sshs_model_hash` / `modelspec.hash_sha256`). **Absent** = not yet checked (may be backfilled later); **`null`** = checked but unavailable (header has no recognized hash); **12-char hex string** = value |
|
||||
|
||||
---
|
||||
|
||||
@@ -287,6 +288,7 @@ These fields are automatically synchronized with the filesystem:
|
||||
- `preview_url` — Updated if preview file is moved/removed
|
||||
- `sha256` — Updated during hash calculation (when `hash_status="pending"`)
|
||||
- `hash_status` — Updated during hash calculation
|
||||
- `autov3` — Set when metadata is first created (from safetensors header); may be backfilled later for entries where it is absent
|
||||
- `last_checked_at` — Timestamp of scan
|
||||
- `metadata_source` — Set based on metadata provider
|
||||
|
||||
@@ -345,6 +347,7 @@ These fields can be edited by users at any time through the Lora Manager UI or b
|
||||
| `metadata_source` | `null` |
|
||||
| `last_checked_at` | `0` |
|
||||
| `hash_status` | `"completed"` |
|
||||
| `autov3` | absent (not checked) or `null` (checked, no value) |
|
||||
| `usage_tips` | `"{}"` (LoRA only) |
|
||||
| `model_type` | `"checkpoint"` or `"embedding"` (not present in LoRA models) |
|
||||
|
||||
@@ -354,6 +357,7 @@ These fields can be edited by users at any time through the Lora Manager UI or b
|
||||
|
||||
| Version | Date | Changes |
|
||||
|---------|------|---------|
|
||||
| 1.1 | 2026-08 | Added `autov3` field (CivitAI AutoV3 hash with three-state semantics) |
|
||||
| 1.0 | 2026-03 | Initial schema documentation |
|
||||
|
||||
---
|
||||
|
||||
File diff suppressed because one or more lines are too long
+532
-52
File diff suppressed because it is too large
Load Diff
+2243
-1763
File diff suppressed because it is too large
Load Diff
+533
-53
File diff suppressed because it is too large
Load Diff
+533
-53
File diff suppressed because it is too large
Load Diff
+533
-53
File diff suppressed because it is too large
Load Diff
+532
-52
File diff suppressed because it is too large
Load Diff
+532
-52
File diff suppressed because it is too large
Load Diff
+533
-53
File diff suppressed because it is too large
Load Diff
+540
-60
File diff suppressed because it is too large
Load Diff
+532
-52
File diff suppressed because it is too large
Load Diff
+262
-17
@@ -1,12 +1,19 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
import os
|
||||
import platform
|
||||
import posixpath
|
||||
import threading
|
||||
from pathlib import Path
|
||||
import folder_paths # type: ignore
|
||||
import folder_paths # pyright: ignore[reportMissingImports]
|
||||
from typing import Any, Dict, Iterable, List, Mapping, Optional, Set, Tuple
|
||||
import logging
|
||||
import json
|
||||
import urllib.parse
|
||||
import sys as _sys
|
||||
import types as _types
|
||||
import time
|
||||
|
||||
from .utils.cache_paths import CacheType, get_cache_file_path, get_legacy_cache_paths
|
||||
@@ -25,21 +32,57 @@ standalone_mode = (
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _normalize_root_identity(path: str) -> str:
|
||||
"""Normalize a root path for comparisons across slash styles."""
|
||||
|
||||
normalized = posixpath.normpath(path.strip().replace("\\", "/"))
|
||||
if len(normalized) >= 2 and normalized[1] == ":":
|
||||
return normalized.lower()
|
||||
return normalized
|
||||
|
||||
|
||||
def _resolve_valid_default_root(
|
||||
current: str, primary_paths: List[str], name: str
|
||||
current: str, primary_paths: List[str], allowed_paths: List[str], name: str
|
||||
) -> str:
|
||||
"""Return a valid default root from the current primary path set."""
|
||||
"""Return a valid default root from the current primary/extra path set."""
|
||||
|
||||
valid_paths = [path for path in primary_paths if isinstance(path, str) and path.strip()]
|
||||
if not valid_paths:
|
||||
return ""
|
||||
fallback_paths: List[str] = []
|
||||
seen: Set[str] = set()
|
||||
for path in allowed_paths:
|
||||
if not isinstance(path, str):
|
||||
continue
|
||||
stripped = path.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
identity = _normalize_root_identity(stripped)
|
||||
if identity in seen:
|
||||
continue
|
||||
seen.add(identity)
|
||||
fallback_paths.append(stripped)
|
||||
|
||||
if current in valid_paths:
|
||||
allowed = {_normalize_root_identity(path) for path in fallback_paths}
|
||||
|
||||
if current and _normalize_root_identity(current) in allowed:
|
||||
return current
|
||||
|
||||
if not valid_paths:
|
||||
if not fallback_paths:
|
||||
return ""
|
||||
if current:
|
||||
logger.info(
|
||||
"Repaired stale %s from '%s' to '%s' because it is not present in primary or extra roots",
|
||||
name,
|
||||
current,
|
||||
fallback_paths[0],
|
||||
)
|
||||
else:
|
||||
logger.info("Auto-setting %s to '%s'", name, fallback_paths[0])
|
||||
return fallback_paths[0]
|
||||
|
||||
if current:
|
||||
logger.info(
|
||||
"Repaired stale %s from '%s' to '%s'",
|
||||
"Repaired stale %s from '%s' to '%s' because it is not present in primary or extra roots",
|
||||
name,
|
||||
current,
|
||||
valid_paths[0],
|
||||
@@ -51,7 +94,7 @@ def _resolve_valid_default_root(
|
||||
|
||||
|
||||
def _normalize_folder_paths_for_comparison(
|
||||
folder_paths: Mapping[str, Iterable[str]],
|
||||
folder_paths: Mapping[str, Any],
|
||||
) -> Dict[str, Set[str]]:
|
||||
"""Normalize folder paths for comparison across libraries."""
|
||||
|
||||
@@ -134,6 +177,12 @@ class Config:
|
||||
self.extra_checkpoints_roots: List[str] = []
|
||||
self.extra_unet_roots: List[str] = []
|
||||
self.extra_embeddings_roots: List[str] = []
|
||||
self.recipes_path: str = ""
|
||||
|
||||
# Load extra folder paths from active library settings before symlink scan
|
||||
# so both primary and extra paths are discovered in a single pass.
|
||||
self._load_extra_paths_from_settings()
|
||||
|
||||
# Scan symbolic links during initialization
|
||||
self._initialize_symlink_mappings()
|
||||
|
||||
@@ -141,6 +190,98 @@ class Config:
|
||||
# Save the paths to settings.json when running in ComfyUI mode
|
||||
self.save_folder_paths_to_settings()
|
||||
|
||||
def _load_extra_paths_from_settings(self) -> None:
|
||||
"""Read extra folder paths from the active library and apply them.
|
||||
|
||||
Called during ``Config.__init__`` before the symlink scan so both primary and
|
||||
extra paths are discovered in a single pass. Mirrors the extra-path
|
||||
portion of ``_apply_library_paths`` without replacing the primary roots
|
||||
that were already resolved via ``folder_paths.get_folder_paths``.
|
||||
"""
|
||||
try:
|
||||
from .services.settings_manager import get_settings_manager
|
||||
|
||||
settings_manager = get_settings_manager()
|
||||
library_name = settings_manager.get_active_library_name()
|
||||
libraries = settings_manager.get_libraries()
|
||||
|
||||
if not library_name or library_name not in libraries:
|
||||
return
|
||||
|
||||
library_config = libraries[library_name]
|
||||
if not isinstance(library_config, dict):
|
||||
return
|
||||
|
||||
# Always read recipes_path — it is independent of extra folder paths
|
||||
# and must be set before any early returns below.
|
||||
recipes_path = library_config.get("recipes_path", "")
|
||||
if isinstance(recipes_path, str) and recipes_path:
|
||||
self.recipes_path = recipes_path
|
||||
|
||||
extra_folder_paths = library_config.get("extra_folder_paths")
|
||||
if not isinstance(extra_folder_paths, dict):
|
||||
return
|
||||
|
||||
extra_lora = extra_folder_paths.get("loras", []) or []
|
||||
extra_checkpoint = extra_folder_paths.get("checkpoints", []) or []
|
||||
extra_unet = extra_folder_paths.get("unet", []) or []
|
||||
extra_embedding = extra_folder_paths.get("embeddings", []) or []
|
||||
|
||||
if not any([extra_lora, extra_checkpoint, extra_unet, extra_embedding]):
|
||||
return
|
||||
|
||||
filtered_extra_lora = self._filter_overlapping_extra_lora_paths(
|
||||
self.loras_roots, extra_lora
|
||||
)
|
||||
self.extra_loras_roots = self._prepare_lora_paths(filtered_extra_lora)
|
||||
(
|
||||
_,
|
||||
self.extra_checkpoints_roots,
|
||||
self.extra_unet_roots,
|
||||
) = self._prepare_checkpoint_paths(extra_checkpoint, extra_unet)
|
||||
self.extra_embeddings_roots = self._prepare_embedding_paths(
|
||||
extra_embedding
|
||||
)
|
||||
|
||||
if self.extra_loras_roots:
|
||||
logger.info(
|
||||
"Found extra LoRA roots:"
|
||||
+ "\n - "
|
||||
+ "\n - ".join(self.extra_loras_roots)
|
||||
)
|
||||
if self.extra_checkpoints_roots:
|
||||
logger.info(
|
||||
"Found extra checkpoint roots:"
|
||||
+ "\n - "
|
||||
+ "\n - ".join(self.extra_checkpoints_roots)
|
||||
)
|
||||
if self.extra_unet_roots:
|
||||
logger.info(
|
||||
"Found extra diffusion model roots:"
|
||||
+ "\n - "
|
||||
+ "\n - ".join(self.extra_unet_roots)
|
||||
)
|
||||
if self.extra_embeddings_roots:
|
||||
logger.info(
|
||||
"Found extra embedding roots:"
|
||||
+ "\n - "
|
||||
+ "\n - ".join(self.extra_embeddings_roots)
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Applied library settings for '%s' with extra paths: loras=%s, "
|
||||
"checkpoints=%s, embeddings=%s",
|
||||
library_name,
|
||||
extra_lora,
|
||||
extra_checkpoint,
|
||||
extra_embedding,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Could not load extra paths from library settings: %s", exc
|
||||
)
|
||||
|
||||
def save_folder_paths_to_settings(self):
|
||||
"""Persist ComfyUI-derived folder paths to the multi-library settings."""
|
||||
try:
|
||||
@@ -222,42 +363,120 @@ class Config:
|
||||
"Failed to rename legacy 'default' library: %s", rename_error
|
||||
)
|
||||
|
||||
# Clean up a stale "default" library entry that has no meaningful
|
||||
# paths configured (e.g. leftover bootstrap artifact). This only
|
||||
# fires when "comfyui" already exists so we never delete the last
|
||||
# remaining library.
|
||||
if (
|
||||
"default" in libraries
|
||||
and "comfyui" in libraries
|
||||
and isinstance(default_library, Mapping)
|
||||
):
|
||||
default_folder_paths = _normalize_library_folder_paths(
|
||||
default_library
|
||||
)
|
||||
default_extra_paths = default_library.get("extra_folder_paths", {})
|
||||
has_meaningful_paths = bool(default_folder_paths) or bool(
|
||||
default_extra_paths
|
||||
) or any(
|
||||
default_library.get(key)
|
||||
for key in (
|
||||
"default_lora_root",
|
||||
"default_checkpoint_root",
|
||||
"default_unet_root",
|
||||
"default_embedding_root",
|
||||
"recipes_path",
|
||||
)
|
||||
)
|
||||
if not has_meaningful_paths:
|
||||
try:
|
||||
settings_service.delete_library("default")
|
||||
libraries_changed = True
|
||||
logger.info(
|
||||
"Removed stale 'default' library entry "
|
||||
"with no meaningful paths configured"
|
||||
)
|
||||
libraries = settings_service.get_libraries()
|
||||
comfy_library = libraries.get("comfyui", {})
|
||||
except Exception as delete_error:
|
||||
logger.debug(
|
||||
"Failed to remove stale 'default' library: %s",
|
||||
delete_error,
|
||||
)
|
||||
|
||||
default_lora_root = _resolve_valid_default_root(
|
||||
comfy_library.get("default_lora_root", ""),
|
||||
list(self.loras_roots or []),
|
||||
list(self.loras_roots or [])
|
||||
+ list(comfy_library.get("extra_folder_paths", {}).get("loras", []) or []),
|
||||
"default_lora_root",
|
||||
)
|
||||
|
||||
default_checkpoint_root = _resolve_valid_default_root(
|
||||
comfy_library.get("default_checkpoint_root", ""),
|
||||
list(self.checkpoints_roots or []),
|
||||
list(self.checkpoints_roots or [])
|
||||
+ list(comfy_library.get("extra_folder_paths", {}).get("checkpoints", []) or []),
|
||||
"default_checkpoint_root",
|
||||
)
|
||||
|
||||
default_embedding_root = _resolve_valid_default_root(
|
||||
comfy_library.get("default_embedding_root", ""),
|
||||
list(self.embeddings_roots or []),
|
||||
list(self.embeddings_roots or [])
|
||||
+ list(comfy_library.get("extra_folder_paths", {}).get("embeddings", []) or []),
|
||||
"default_embedding_root",
|
||||
)
|
||||
|
||||
metadata = dict(comfy_library.get("metadata", {}))
|
||||
metadata.setdefault("display_name", "ComfyUI")
|
||||
metadata["source"] = "comfyui"
|
||||
extra_folder_paths = {}
|
||||
if isinstance(comfy_library, Mapping):
|
||||
existing_extra_paths = comfy_library.get("extra_folder_paths", {})
|
||||
if isinstance(existing_extra_paths, Mapping):
|
||||
extra_folder_paths = {
|
||||
key: list(value) if isinstance(value, list) else []
|
||||
for key, value in existing_extra_paths.items()
|
||||
}
|
||||
|
||||
active_library_name = settings_service.get_active_library_name()
|
||||
should_activate = (
|
||||
active_library_name == "comfyui"
|
||||
or self._should_activate_comfy_library(libraries, libraries_changed)
|
||||
)
|
||||
|
||||
settings_service.upsert_library(
|
||||
"comfyui",
|
||||
folder_paths=target_folder_paths,
|
||||
extra_folder_paths=extra_folder_paths,
|
||||
default_lora_root=default_lora_root,
|
||||
default_checkpoint_root=default_checkpoint_root,
|
||||
default_embedding_root=default_embedding_root,
|
||||
metadata=metadata,
|
||||
activate=True,
|
||||
activate=should_activate,
|
||||
)
|
||||
|
||||
logger.info("Updated 'comfyui' library with current folder paths")
|
||||
if should_activate:
|
||||
logger.info("Updated 'comfyui' library with current folder paths")
|
||||
else:
|
||||
logger.info(
|
||||
"Updated 'comfyui' library with current folder paths without activating it"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save folder paths: {e}")
|
||||
|
||||
def _should_activate_comfy_library(
|
||||
self, libraries: Mapping[str, Any], libraries_changed: bool
|
||||
) -> bool:
|
||||
"""Return whether startup sync should make the ComfyUI library active."""
|
||||
|
||||
if libraries_changed:
|
||||
return True
|
||||
if not libraries:
|
||||
return True
|
||||
return "comfyui" in libraries and len(libraries) == 1
|
||||
|
||||
def _is_link(self, path: str) -> bool:
|
||||
try:
|
||||
if os.path.islink(path):
|
||||
@@ -267,7 +486,7 @@ class Config:
|
||||
import ctypes
|
||||
|
||||
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
|
||||
attrs = ctypes.windll.kernel32.GetFileAttributesW(str(path)) # type: ignore[attr-defined]
|
||||
attrs = ctypes.windll.kernel32.GetFileAttributesW(str(path)) # pyright: ignore[reportAttributeAccessIssue]
|
||||
return attrs != -1 and (attrs & FILE_ATTRIBUTE_REPARSE_POINT)
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking Windows reparse point: {e}")
|
||||
@@ -276,7 +495,7 @@ class Config:
|
||||
logger.error(f"Error checking link status for {path}: {e}")
|
||||
return False
|
||||
|
||||
def _entry_is_symlink(self, entry: os.DirEntry) -> bool:
|
||||
def _entry_is_symlink(self, entry: os.DirEntry[str]) -> bool:
|
||||
"""Check if a directory entry is a symlink, including Windows junctions."""
|
||||
if entry.is_symlink():
|
||||
return True
|
||||
@@ -285,7 +504,7 @@ class Config:
|
||||
import ctypes
|
||||
|
||||
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
|
||||
attrs = ctypes.windll.kernel32.GetFileAttributesW(entry.path) # type: ignore[attr-defined]
|
||||
attrs = ctypes.windll.kernel32.GetFileAttributesW(entry.path) # pyright: ignore[reportAttributeAccessIssue]
|
||||
return attrs != -1 and (attrs & FILE_ATTRIBUTE_REPARSE_POINT)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -652,6 +871,8 @@ class Config:
|
||||
preview_roots.update(self._expand_preview_root(root))
|
||||
for root in self.extra_embeddings_roots or []:
|
||||
preview_roots.update(self._expand_preview_root(root))
|
||||
if self.recipes_path:
|
||||
preview_roots.update(self._expand_preview_root(self.recipes_path))
|
||||
|
||||
for target, link in self._path_mappings.items():
|
||||
preview_roots.update(self._expand_preview_root(target))
|
||||
@@ -909,11 +1130,13 @@ class Config:
|
||||
|
||||
def _apply_library_paths(
|
||||
self,
|
||||
folder_paths: Mapping[str, Iterable[str]],
|
||||
extra_folder_paths: Optional[Mapping[str, Iterable[str]]] = None,
|
||||
folder_paths: Mapping[str, Any],
|
||||
extra_folder_paths: Optional[Mapping[str, Any]] = None,
|
||||
recipes_path: str = "",
|
||||
) -> None:
|
||||
self._path_mappings.clear()
|
||||
self._preview_root_paths = set()
|
||||
self.recipes_path = recipes_path if isinstance(recipes_path, str) else ""
|
||||
|
||||
lora_paths = folder_paths.get("loras", []) or []
|
||||
checkpoint_paths = folder_paths.get("checkpoints", []) or []
|
||||
@@ -1169,7 +1392,12 @@ class Config:
|
||||
if not isinstance(extra_folder_paths, Mapping):
|
||||
extra_folder_paths = None
|
||||
|
||||
self._apply_library_paths(folder_paths, extra_folder_paths)
|
||||
recipes_path = (
|
||||
str(library_config.get("recipes_path", ""))
|
||||
if isinstance(library_config, Mapping)
|
||||
else ""
|
||||
)
|
||||
self._apply_library_paths(folder_paths, extra_folder_paths, recipes_path)
|
||||
|
||||
logger.info(
|
||||
"Applied library settings with %d lora roots (%d extra), %d checkpoint roots (%d extra), and %d embedding roots (%d extra)",
|
||||
@@ -1200,4 +1428,21 @@ class Config:
|
||||
|
||||
|
||||
# Global config instance
|
||||
config = Config()
|
||||
# NOTE: Guard against re-import. When ServiceRegistry.get_lora_scanner() triggers
|
||||
# a fresh import of lora_scanner → config, we must NOT re-execute Config.__init__()
|
||||
# (which re-scans all roots, re-registers libraries, etc.).
|
||||
#
|
||||
# Strategy: store the config instance in a dedicated sentinel module
|
||||
# ('_lm_config_cache') that is NEVER removed from sys.modules (its key does
|
||||
# NOT start with 'py.'), so it survives re-imports of py.* modules.
|
||||
_CONFIG_SENTINEL = "_lm_config_cache"
|
||||
config: Config
|
||||
if _CONFIG_SENTINEL in _sys.modules:
|
||||
# Re-import: reuse the existing singleton from the sentinel.
|
||||
config = _sys.modules[_CONFIG_SENTINEL].config
|
||||
else:
|
||||
config = Config()
|
||||
# Register the sentinel so re-imports of py.config find us.
|
||||
_sentinel_mod = _types.ModuleType(_CONFIG_SENTINEL)
|
||||
setattr(_sentinel_mod, "config", config)
|
||||
_sys.modules[_CONFIG_SENTINEL] = _sentinel_mod
|
||||
|
||||
+32
-34
@@ -14,7 +14,7 @@ standalone_mode = (
|
||||
if not standalone_mode:
|
||||
setup_logging()
|
||||
|
||||
from server import PromptServer # type: ignore
|
||||
from server import PromptServer # pyright: ignore[reportMissingImports]
|
||||
|
||||
from .config import config
|
||||
from .services.model_service_factory import (
|
||||
@@ -33,6 +33,7 @@ from .utils.example_images_migration import ExampleImagesMigration
|
||||
from .services.websocket_manager import ws_manager
|
||||
from .services.example_images_cleanup_service import ExampleImagesCleanupService
|
||||
from .middleware.csp_middleware import relax_csp_for_remote_media
|
||||
from .middleware.error_middleware import api_json_error
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -76,6 +77,11 @@ class LoraManager:
|
||||
"""Initialize and register all routes using the new refactored architecture"""
|
||||
app = PromptServer.instance.app
|
||||
|
||||
# Register JSON error middleware for /api/* routes as the outermost
|
||||
# middleware so it catches errors from all other middlewares.
|
||||
if api_json_error not in app.middlewares:
|
||||
app.middlewares.insert(0, api_json_error)
|
||||
|
||||
if relax_csp_for_remote_media not in app.middlewares:
|
||||
# Ensure CSP relaxer executes after ComfyUI's block_external_middleware so it can
|
||||
# see and extend the restrictive header instead of being overwritten by it.
|
||||
@@ -184,45 +190,17 @@ class LoraManager:
|
||||
async def _initialize_services(cls):
|
||||
"""Initialize all services using the ServiceRegistry"""
|
||||
try:
|
||||
# Apply library settings to load extra folder paths before scanning
|
||||
# Only apply if extra paths haven't been loaded yet (preserves test mocks)
|
||||
try:
|
||||
from .services.settings_manager import get_settings_manager
|
||||
|
||||
settings_manager = get_settings_manager()
|
||||
library_name = settings_manager.get_active_library_name()
|
||||
libraries = settings_manager.get_libraries()
|
||||
if library_name and library_name in libraries:
|
||||
library_config = libraries[library_name]
|
||||
# Only apply settings if extra paths are not already configured
|
||||
# This preserves values set by tests via monkeypatch
|
||||
extra_paths = library_config.get("extra_folder_paths", {})
|
||||
has_extra_paths = (
|
||||
config.extra_loras_roots
|
||||
or config.extra_checkpoints_roots
|
||||
or config.extra_unet_roots
|
||||
or config.extra_embeddings_roots
|
||||
)
|
||||
if not has_extra_paths and any(extra_paths.values()):
|
||||
config.apply_library_settings(library_config)
|
||||
logger.info(
|
||||
"Applied library settings for '%s' with extra paths: loras=%s, checkpoints=%s, embeddings=%s",
|
||||
library_name,
|
||||
extra_paths.get("loras", []),
|
||||
extra_paths.get("checkpoints", []),
|
||||
extra_paths.get("embeddings", []),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to apply library settings during initialization: %s", exc
|
||||
)
|
||||
|
||||
# Initialize CivitaiClient first to ensure it's ready for other services
|
||||
await ServiceRegistry.get_civitai_client()
|
||||
|
||||
# Register DownloadManager with ServiceRegistry
|
||||
await ServiceRegistry.get_download_manager()
|
||||
|
||||
# Initialize DownloadQueueService for persistent queue/history
|
||||
await ServiceRegistry.get_download_queue_service()
|
||||
|
||||
await ServiceRegistry.get_backup_service()
|
||||
|
||||
from .services.metadata_service import initialize_metadata_providers
|
||||
|
||||
await initialize_metadata_providers()
|
||||
@@ -230,6 +208,10 @@ class LoraManager:
|
||||
# Initialize WebSocket manager
|
||||
await ServiceRegistry.get_websocket_manager()
|
||||
|
||||
# Preload LLM model catalog (background task, non-blocking)
|
||||
from .services.llm_service import LLMService
|
||||
await LLMService.get_instance()
|
||||
|
||||
# Initialize scanners in background
|
||||
lora_scanner = await ServiceRegistry.get_lora_scanner()
|
||||
checkpoint_scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||
@@ -458,5 +440,21 @@ class LoraManager:
|
||||
try:
|
||||
logger.info("LoRA Manager: Cleaning up services")
|
||||
|
||||
# Cancel any in-flight scanner initialization tasks so thread-pool
|
||||
# workers (e.g. _initialize_cache_sync) can break out of their loops
|
||||
# when the server shuts down (e.g. Ctrl+C on WSL).
|
||||
for name in ("lora_scanner", "checkpoint_scanner", "embedding_scanner"):
|
||||
scanner = ServiceRegistry.get_service_sync(name)
|
||||
if scanner is not None and hasattr(scanner, "cancel_task"):
|
||||
scanner.cancel_task()
|
||||
logger.debug("LoRA Manager: Cancelled %s", name)
|
||||
|
||||
# Close shared aiohttp sessions to avoid "Unclosed client session" warnings
|
||||
try:
|
||||
from py.routes.handlers.hf_handlers import close_hf_api_session
|
||||
await close_hf_api_session()
|
||||
except Exception as exc:
|
||||
logger.debug("Error closing HF API session: %s", exc)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during cleanup: {e}", exc_info=True)
|
||||
|
||||
@@ -22,7 +22,7 @@ if not standalone_mode:
|
||||
|
||||
logger.info("ComfyUI Metadata Collector initialized")
|
||||
|
||||
def get_metadata(prompt_id=None): # type: ignore[no-redef]
|
||||
def get_metadata(prompt_id=None): # pyright: ignore[reportRedeclaration]
|
||||
"""Helper function to get metadata from the registry"""
|
||||
registry = MetadataRegistry()
|
||||
return registry.get_metadata(prompt_id)
|
||||
@@ -31,6 +31,6 @@ else:
|
||||
def init():
|
||||
logger.info("ComfyUI Metadata Collector disabled in standalone mode")
|
||||
|
||||
def get_metadata(prompt_id=None): # type: ignore[no-redef]
|
||||
def get_metadata(prompt_id=None): # pyright: ignore[reportRedeclaration]
|
||||
"""Dummy implementation for standalone mode"""
|
||||
return {}
|
||||
|
||||
@@ -1,13 +1,28 @@
|
||||
"""Constants used by the metadata collector"""
|
||||
|
||||
# Sentinel value for clip_skip to distinguish "unconnected / widget default"
|
||||
# from "user wired value 0". Both ComfyUI CLIPSetLastLayer (-24..-1) and
|
||||
# A1111 conventions treat 0 as meaningless for clip skipping, but users may
|
||||
# explicitly wire 0 to the overwrite node to express "no clip skip / default".
|
||||
CLIP_SKIP_SENTINEL = -25
|
||||
|
||||
# Metadata categories
|
||||
MODELS = "models"
|
||||
PROMPTS = "prompts"
|
||||
SAMPLING = "sampling"
|
||||
LORAS = "loras"
|
||||
EMBEDDINGS = "embeddings"
|
||||
SIZE = "size"
|
||||
IMAGES = "images"
|
||||
IS_SAMPLER = "is_sampler" # New constant to mark sampler nodes
|
||||
OVERWRITE = "overwrite" # Manual metadata overwrite from MetadataOverwriteLM node
|
||||
|
||||
# Field names that the MetadataOverwriteLM node and its extractor share
|
||||
METADATA_OVERWRITE_FIELDS = (
|
||||
"prompt", "negative_prompt", "seed", "steps", "cfg_scale",
|
||||
"sampler", "scheduler", "model", "loras", "size",
|
||||
"clip_skip", "additional_data",
|
||||
)
|
||||
|
||||
# Complete list of categories to track
|
||||
METADATA_CATEGORIES = [MODELS, PROMPTS, SAMPLING, LORAS, SIZE, IMAGES]
|
||||
METADATA_CATEGORIES = [MODELS, PROMPTS, SAMPLING, LORAS, EMBEDDINGS, SIZE, IMAGES, OVERWRITE]
|
||||
|
||||
@@ -16,7 +16,7 @@ class MetadataHook:
|
||||
execution = None
|
||||
try:
|
||||
# Try direct import first
|
||||
import execution # type: ignore
|
||||
import execution # pyright: ignore[reportMissingImports]
|
||||
except ImportError:
|
||||
# Try to locate from system modules
|
||||
for module_name in sys.modules:
|
||||
@@ -83,7 +83,8 @@ class MetadataHook:
|
||||
|
||||
# Record inputs before execution
|
||||
if node_id is not None:
|
||||
registry.record_node_execution(node_id, class_type, input_data_all, None)
|
||||
return_types = getattr(obj, 'RETURN_TYPES', None)
|
||||
registry.record_node_execution(node_id, class_type, input_data_all, None, return_types=return_types)
|
||||
except Exception as e:
|
||||
logger.error(f"Error collecting metadata (pre-execution): {str(e)}")
|
||||
|
||||
@@ -114,7 +115,8 @@ class MetadataHook:
|
||||
|
||||
# Record outputs after execution
|
||||
if node_id is not None:
|
||||
registry.update_node_execution(node_id, class_type, results)
|
||||
return_types = getattr(obj, 'RETURN_TYPES', None)
|
||||
registry.update_node_execution(node_id, class_type, results, return_types=return_types)
|
||||
except Exception as e:
|
||||
logger.error(f"Error collecting metadata (post-execution): {str(e)}")
|
||||
|
||||
@@ -135,10 +137,13 @@ class MetadataHook:
|
||||
# Store the dynprompt reference for node lookups
|
||||
if hasattr(prompt, 'original_prompt'):
|
||||
registry.set_current_prompt(prompt)
|
||||
|
||||
|
||||
# Store extra_data for accessing full workflow node properties
|
||||
registry.set_extra_data(extra_data)
|
||||
|
||||
# Execute the original function
|
||||
return original_execute(*args, **kwargs)
|
||||
|
||||
|
||||
# Replace the functions
|
||||
execution._map_node_over_list = map_node_over_list_with_metadata
|
||||
execution.execute = execute_with_prompt_tracking
|
||||
@@ -163,7 +168,8 @@ class MetadataHook:
|
||||
class_type = obj.__class__.__name__
|
||||
node_id = unique_id
|
||||
if node_id is not None:
|
||||
registry.record_node_execution(node_id, class_type, input_data_all, None)
|
||||
return_types = getattr(obj, 'RETURN_TYPES', None)
|
||||
registry.record_node_execution(node_id, class_type, input_data_all, None, return_types=return_types)
|
||||
except Exception as e:
|
||||
logger.error(f"Error collecting metadata (pre-execution): {str(e)}")
|
||||
|
||||
@@ -180,7 +186,8 @@ class MetadataHook:
|
||||
class_type = obj.__class__.__name__
|
||||
node_id = unique_id
|
||||
if node_id is not None:
|
||||
registry.update_node_execution(node_id, class_type, results)
|
||||
return_types = getattr(obj, 'RETURN_TYPES', None)
|
||||
registry.update_node_execution(node_id, class_type, results, return_types=return_types)
|
||||
except Exception as e:
|
||||
logger.error(f"Error collecting metadata (post-execution): {str(e)}")
|
||||
|
||||
@@ -202,6 +209,9 @@ class MetadataHook:
|
||||
if hasattr(prompt, 'original_prompt'):
|
||||
registry.set_current_prompt(prompt)
|
||||
|
||||
# Store extra_data for accessing full workflow node properties
|
||||
registry.set_extra_data(extra_data)
|
||||
|
||||
# Execute the original function
|
||||
return await original_execute(*args, **kwargs)
|
||||
|
||||
|
||||
@@ -1,15 +1,68 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from .constants import IMAGES
|
||||
|
||||
# Check if running in standalone mode
|
||||
standalone_mode = os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1" or os.environ.get("HF_HUB_DISABLE_TELEMETRY", "0") == "0"
|
||||
|
||||
from .constants import MODELS, PROMPTS, SAMPLING, LORAS, SIZE, IS_SAMPLER
|
||||
from .constants import MODELS, PROMPTS, SAMPLING, LORAS, SIZE, IS_SAMPLER, OVERWRITE
|
||||
from .node_extractors import NODE_EXTRACTORS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Keys that identify metadata hint marks stored in node.properties.lm_marker_role
|
||||
_META_MARK_PREFIX = "meta_"
|
||||
_MARK_PRIMARY_MODEL = "primary_model"
|
||||
_MARK_PRIMARY_SAMPLER = "primary_sampler"
|
||||
_MARK_POSITIVE_PROMPT = "positive_prompt"
|
||||
_MARK_NEGATIVE_PROMPT = "negative_prompt"
|
||||
|
||||
class MetadataProcessor:
|
||||
"""Process and format collected metadata"""
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _get_user_marks(metadata):
|
||||
"""Scan workflow nodes (from extra_data.extra_pnginfo.workflow) for user-assigned
|
||||
metadata hint marks stored in node.properties.lm_marker_role.
|
||||
|
||||
Returns a dict mapping mark type keys to node IDs.
|
||||
Example: {'primary_model': '42', 'primary_sampler': '17'}
|
||||
"""
|
||||
marks: dict[str, str] = {}
|
||||
|
||||
# Primary source: extra_data.extra_pnginfo.workflow.nodes (has full properties)
|
||||
extra_data = metadata.get("extra_data")
|
||||
if extra_data and isinstance(extra_data, dict):
|
||||
extra_pnginfo = extra_data.get("extra_pnginfo", {})
|
||||
if isinstance(extra_pnginfo, dict):
|
||||
workflow = extra_pnginfo.get("workflow", {})
|
||||
nodes = workflow.get("nodes", [])
|
||||
for node in nodes:
|
||||
node_id = str(node.get("id", ""))
|
||||
role = node.get("properties", {}).get("lm_marker_role", "")
|
||||
if role.startswith(_META_MARK_PREFIX):
|
||||
mark_type = role[len(_META_MARK_PREFIX):]
|
||||
if mark_type in marks:
|
||||
logger.warning(
|
||||
"Duplicate meta hint '%s': node %s (previous: %s), "
|
||||
"last match wins",
|
||||
mark_type, node_id, marks[mark_type],
|
||||
)
|
||||
marks[mark_type] = node_id
|
||||
|
||||
# Fallback: try prompt.original_prompt (API-only submissions may not have workflow)
|
||||
if not marks:
|
||||
prompt = metadata.get("current_prompt")
|
||||
if prompt and getattr(prompt, "original_prompt", None):
|
||||
for node_id, node_data in prompt.original_prompt.items():
|
||||
role = node_data.get("properties", {}).get("lm_marker_role", "")
|
||||
if role.startswith(_META_MARK_PREFIX):
|
||||
mark_type = role[len(_META_MARK_PREFIX):]
|
||||
marks[mark_type] = node_id
|
||||
|
||||
return marks
|
||||
|
||||
@staticmethod
|
||||
def find_primary_sampler(metadata, downstream_id=None):
|
||||
"""
|
||||
@@ -352,50 +405,101 @@ class MetadataProcessor:
|
||||
|
||||
# Check if we have stored conditioning objects for this sampler
|
||||
if sampler_id in metadata.get(PROMPTS, {}) and (
|
||||
"pos_conditioning" in metadata[PROMPTS][sampler_id] or
|
||||
"neg_conditioning" in metadata[PROMPTS][sampler_id]):
|
||||
|
||||
"pos_conditioning" in metadata[PROMPTS][sampler_id] or
|
||||
"neg_conditioning" in metadata[PROMPTS][sampler_id]
|
||||
):
|
||||
pos_conditioning = metadata[PROMPTS][sampler_id].get("pos_conditioning")
|
||||
neg_conditioning = metadata[PROMPTS][sampler_id].get("neg_conditioning")
|
||||
|
||||
# Helper function to recursively find prompt text for a conditioning object
|
||||
def find_prompt_text_for_conditioning(conditioning_obj, is_positive=True):
|
||||
|
||||
def extend_unique(target, values):
|
||||
for value in values:
|
||||
if value and value not in target:
|
||||
target.append(value)
|
||||
|
||||
# Helper function to recursively find prompt texts for a conditioning object.
|
||||
# Transform nodes can map one output conditioning to multiple source conditionings.
|
||||
def find_prompt_texts_for_conditioning(
|
||||
conditioning_obj, is_positive=True, visited=None
|
||||
):
|
||||
if conditioning_obj is None:
|
||||
return ""
|
||||
|
||||
return []
|
||||
|
||||
if visited is None:
|
||||
visited = set()
|
||||
|
||||
conditioning_id = id(conditioning_obj)
|
||||
if conditioning_id in visited:
|
||||
return []
|
||||
visited.add(conditioning_id)
|
||||
|
||||
prompt_texts = []
|
||||
|
||||
# Try to match conditioning objects with those stored by extractors
|
||||
for prompt_node_id, prompt_data in metadata[PROMPTS].items():
|
||||
# For nodes with single conditioning output
|
||||
if "conditioning" in prompt_data:
|
||||
if id(prompt_data["conditioning"]) == id(conditioning_obj):
|
||||
return prompt_data.get("text", "")
|
||||
|
||||
# For nodes with separate pos_conditioning and neg_conditioning outputs (like TSC_EfficientLoader)
|
||||
if is_positive and "positive_encoded" in prompt_data:
|
||||
if id(prompt_data["positive_encoded"]) == id(conditioning_obj):
|
||||
if "positive_text" in prompt_data:
|
||||
return prompt_data["positive_text"]
|
||||
else:
|
||||
orig_conditioning = prompt_data.get("orig_pos_cond", None)
|
||||
if orig_conditioning is not None:
|
||||
# Recursively find the prompt text for the original conditioning
|
||||
return find_prompt_text_for_conditioning(orig_conditioning, is_positive=True)
|
||||
|
||||
if not is_positive and "negative_encoded" in prompt_data:
|
||||
if id(prompt_data["negative_encoded"]) == id(conditioning_obj):
|
||||
if "negative_text" in prompt_data:
|
||||
return prompt_data["negative_text"]
|
||||
else:
|
||||
orig_conditioning = prompt_data.get("orig_neg_cond", None)
|
||||
if orig_conditioning is not None:
|
||||
# Recursively find the prompt text for the original conditioning
|
||||
return find_prompt_text_for_conditioning(orig_conditioning, is_positive=False)
|
||||
|
||||
return ""
|
||||
|
||||
if not isinstance(prompt_data, dict):
|
||||
continue
|
||||
|
||||
# For CLIP text nodes with a single conditioning output.
|
||||
if id(prompt_data.get("conditioning")) == conditioning_id:
|
||||
text = prompt_data.get("text", "")
|
||||
if text:
|
||||
extend_unique(prompt_texts, [text])
|
||||
|
||||
# Generic provenance for passthrough/transform/combine nodes.
|
||||
for source in prompt_data.get("conditioning_sources", []):
|
||||
if id(source.get("output")) != conditioning_id:
|
||||
continue
|
||||
for input_conditioning in source.get("inputs", []):
|
||||
extend_unique(
|
||||
prompt_texts,
|
||||
find_prompt_texts_for_conditioning(
|
||||
input_conditioning, is_positive, visited
|
||||
),
|
||||
)
|
||||
|
||||
# For nodes with separate pos_conditioning and neg_conditioning outputs
|
||||
# like TSC_EfficientLoader and existing ControlNet-style metadata.
|
||||
if (
|
||||
is_positive
|
||||
and id(prompt_data.get("positive_encoded")) == conditioning_id
|
||||
):
|
||||
if prompt_data.get("positive_text"):
|
||||
extend_unique(prompt_texts, [prompt_data["positive_text"]])
|
||||
else:
|
||||
extend_unique(
|
||||
prompt_texts,
|
||||
find_prompt_texts_for_conditioning(
|
||||
prompt_data.get("orig_pos_cond"),
|
||||
is_positive=True,
|
||||
visited=visited,
|
||||
),
|
||||
)
|
||||
|
||||
if (
|
||||
not is_positive
|
||||
and id(prompt_data.get("negative_encoded")) == conditioning_id
|
||||
):
|
||||
if prompt_data.get("negative_text"):
|
||||
extend_unique(prompt_texts, [prompt_data["negative_text"]])
|
||||
else:
|
||||
extend_unique(
|
||||
prompt_texts,
|
||||
find_prompt_texts_for_conditioning(
|
||||
prompt_data.get("orig_neg_cond"),
|
||||
is_positive=False,
|
||||
visited=visited,
|
||||
),
|
||||
)
|
||||
|
||||
return prompt_texts
|
||||
|
||||
# Find prompt texts using the helper function
|
||||
result["prompt"] = find_prompt_text_for_conditioning(pos_conditioning, is_positive=True)
|
||||
result["negative_prompt"] = find_prompt_text_for_conditioning(neg_conditioning, is_positive=False)
|
||||
result["prompt"] = ", ".join(
|
||||
find_prompt_texts_for_conditioning(pos_conditioning, is_positive=True)
|
||||
)
|
||||
result["negative_prompt"] = ", ".join(
|
||||
find_prompt_texts_for_conditioning(neg_conditioning, is_positive=False)
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
@@ -420,20 +524,57 @@ class MetadataProcessor:
|
||||
"checkpoint": None,
|
||||
"loras": "",
|
||||
"size": None,
|
||||
"clip_skip": None
|
||||
"clip_skip": None,
|
||||
"additional_data": "",
|
||||
}
|
||||
|
||||
# Get the prompt object for node relationship tracing
|
||||
prompt = metadata.get("current_prompt")
|
||||
|
||||
# Find the primary KSampler node
|
||||
primary_sampler_id, primary_sampler = MetadataProcessor.find_primary_sampler(metadata, id)
|
||||
|
||||
# Directly get checkpoint from metadata instead of tracing
|
||||
# Pass primary_sampler_id to avoid redundant calculation
|
||||
checkpoint = MetadataProcessor.find_primary_checkpoint(metadata, id, primary_sampler_id)
|
||||
if checkpoint:
|
||||
params["checkpoint"] = checkpoint
|
||||
|
||||
# ---- User marks: override heuristic inference with user-assigned hints ----
|
||||
user_marks = MetadataProcessor._get_user_marks(metadata)
|
||||
|
||||
# Find the primary KSampler node (user mark takes priority)
|
||||
primary_sampler_id = None
|
||||
primary_sampler = None
|
||||
if _MARK_PRIMARY_SAMPLER in user_marks:
|
||||
marked_id = user_marks[_MARK_PRIMARY_SAMPLER]
|
||||
sampler_data = metadata.get(SAMPLING, {}).get(marked_id)
|
||||
if sampler_data and sampler_data.get(IS_SAMPLER):
|
||||
primary_sampler_id = marked_id
|
||||
primary_sampler = sampler_data
|
||||
else:
|
||||
logger.warning(
|
||||
"User-marked primary sampler %s has no runtime metadata, "
|
||||
"falling back to heuristic",
|
||||
marked_id,
|
||||
)
|
||||
if primary_sampler is None:
|
||||
primary_sampler_id, primary_sampler = MetadataProcessor.find_primary_sampler(metadata, id)
|
||||
|
||||
# Resolve checkpoint / model (user mark takes priority)
|
||||
if _MARK_PRIMARY_MODEL in user_marks:
|
||||
marked_id = user_marks[_MARK_PRIMARY_MODEL]
|
||||
if marked_id in metadata.get(MODELS, {}):
|
||||
params["checkpoint"] = metadata[MODELS][marked_id].get("name")
|
||||
else:
|
||||
extra_data = metadata.get("extra_data")
|
||||
extra_pnginfo = extra_data.get("extra_pnginfo", {}) if extra_data and isinstance(extra_data, dict) else {}
|
||||
workflow = extra_pnginfo.get("workflow", {}) if isinstance(extra_pnginfo, dict) else {}
|
||||
node_type = "unknown"
|
||||
for n in workflow.get("nodes", []):
|
||||
if str(n.get("id", "")) == marked_id:
|
||||
node_type = n.get("type", "unknown")
|
||||
break
|
||||
logger.warning(
|
||||
"User-marked primary model %s (type=%s, registered=%s) has no runtime metadata, "
|
||||
"falling back to heuristic",
|
||||
marked_id, node_type, node_type in NODE_EXTRACTORS,
|
||||
)
|
||||
if params["checkpoint"] is None:
|
||||
checkpoint = MetadataProcessor.find_primary_checkpoint(metadata, id, primary_sampler_id)
|
||||
if checkpoint:
|
||||
params["checkpoint"] = checkpoint
|
||||
|
||||
# Check if guidance parameter exists in any sampling node
|
||||
for node_id, sampler_info in metadata.get(SAMPLING, {}).items():
|
||||
@@ -488,7 +629,22 @@ class MetadataProcessor:
|
||||
|
||||
# For SamplerCustom, handle any additional parameters
|
||||
MetadataProcessor.handle_custom_advanced_sampler(metadata, prompt, primary_sampler_id, params)
|
||||
|
||||
|
||||
# ---- User marks: override prompts with explicitly tagged nodes ----
|
||||
prompts_data = metadata.get(PROMPTS, {})
|
||||
if _MARK_POSITIVE_PROMPT in user_marks:
|
||||
pos_id = user_marks[_MARK_POSITIVE_PROMPT]
|
||||
if pos_id in prompts_data:
|
||||
prompt_text = prompts_data[pos_id].get("text") or prompts_data[pos_id].get("positive_text")
|
||||
if prompt_text:
|
||||
params["prompt"] = prompt_text
|
||||
if _MARK_NEGATIVE_PROMPT in user_marks:
|
||||
neg_id = user_marks[_MARK_NEGATIVE_PROMPT]
|
||||
if neg_id in prompts_data:
|
||||
prompt_text = prompts_data[neg_id].get("text") or prompts_data[neg_id].get("negative_text")
|
||||
if prompt_text:
|
||||
params["negative_prompt"] = prompt_text
|
||||
|
||||
# Size extraction is same for all sampler types
|
||||
# Check if the sampler itself has size information (from latent_image)
|
||||
if primary_sampler_id in metadata.get(SIZE, {}):
|
||||
@@ -509,9 +665,34 @@ class MetadataProcessor:
|
||||
|
||||
params["loras"] = " ".join(lora_parts)
|
||||
|
||||
# Set default clip_skip value
|
||||
params["clip_skip"] = "1" # Common default
|
||||
|
||||
# Extract clip_skip from any SAMPLING node that provides it
|
||||
for sampler_info in metadata.get(SAMPLING, {}).values():
|
||||
clip_skip = sampler_info.get("parameters", {}).get("clip_skip")
|
||||
if clip_skip is not None:
|
||||
params["clip_skip"] = clip_skip
|
||||
break
|
||||
if params["clip_skip"] is None:
|
||||
params["clip_skip"] = "1"
|
||||
|
||||
# ---- Apply manual metadata overwrites ----
|
||||
for overwrite_info in metadata.get(OVERWRITE, {}).values():
|
||||
overwrite_params = overwrite_info.get("parameters", {})
|
||||
for key, value in overwrite_params.items():
|
||||
if key == "clip_skip":
|
||||
# Accept any value from overwrite node (sentinel -25 already
|
||||
# filtered upstream). Needed because falsy check treats 0
|
||||
# as "not set" even though 0 is a valid wired input here.
|
||||
params[key] = value
|
||||
elif value: # truthy check — only overwrite when user provided a real value
|
||||
params[key] = value
|
||||
|
||||
# Bridge: the overwrite node exposes the field as "model" (more accurate),
|
||||
# but the internal pipeline key remains "checkpoint" for backward compatibility
|
||||
# with A1111 metadata format and downstream consumers.
|
||||
if params.get("model"):
|
||||
params["checkpoint"] = params["model"]
|
||||
del params["model"]
|
||||
|
||||
return params
|
||||
|
||||
@staticmethod
|
||||
@@ -595,6 +776,15 @@ class MetadataProcessor:
|
||||
if negative_node_id and negative_node_id in metadata.get(PROMPTS, {}):
|
||||
params["negative_prompt"] = metadata[PROMPTS][negative_node_id].get("text", "")
|
||||
else:
|
||||
positive_node_id = MetadataProcessor.trace_node_input(prompt, guider_node_id, "conditioning", max_depth=10)
|
||||
# Generic guider nodes often expose separate positive/negative inputs.
|
||||
positive_node_id = MetadataProcessor.trace_node_input(prompt, guider_node_id, "positive", max_depth=10)
|
||||
if not positive_node_id:
|
||||
positive_node_id = MetadataProcessor.trace_node_input(prompt, guider_node_id, "conditioning", max_depth=10)
|
||||
if positive_node_id and positive_node_id in metadata.get(PROMPTS, {}):
|
||||
params["prompt"] = metadata[PROMPTS][positive_node_id].get("text", "")
|
||||
|
||||
negative_node_id = MetadataProcessor.trace_node_input(prompt, guider_node_id, "negative", max_depth=10)
|
||||
if not negative_node_id:
|
||||
negative_node_id = MetadataProcessor.trace_node_input(prompt, guider_node_id, "conditioning", max_depth=10)
|
||||
if negative_node_id and negative_node_id in metadata.get(PROMPTS, {}):
|
||||
params["negative_prompt"] = metadata[PROMPTS][negative_node_id].get("text", "")
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import time
|
||||
from nodes import NODE_CLASS_MAPPINGS # type: ignore
|
||||
from typing import Any
|
||||
from nodes import NODE_CLASS_MAPPINGS # pyright: ignore[reportMissingImports, reportAttributeAccessIssue]
|
||||
from .node_extractors import NODE_EXTRACTORS, GenericNodeExtractor
|
||||
from .constants import METADATA_CATEGORIES, IMAGES
|
||||
from .constants import METADATA_CATEGORIES, IMAGES, OVERWRITE
|
||||
|
||||
|
||||
class MetadataRegistry:
|
||||
@@ -9,6 +10,15 @@ class MetadataRegistry:
|
||||
|
||||
_instance = None
|
||||
|
||||
current_prompt_id: Any = None
|
||||
current_prompt: Any = None
|
||||
metadata: dict[str, Any] = {}
|
||||
prompt_metadata: dict[str, Any] = {}
|
||||
executed_nodes: set[str] = set()
|
||||
node_cache: dict[str, Any] = {}
|
||||
max_prompt_history: int = 3
|
||||
metadata_categories: list[str] = METADATA_CATEGORIES
|
||||
|
||||
def __new__(cls):
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
@@ -61,6 +71,7 @@ class MetadataRegistry:
|
||||
{
|
||||
"execution_order": [],
|
||||
"current_prompt": None, # Will store the prompt object
|
||||
"extra_data": None, # Will store the API extra_data for workflow metadata
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
)
|
||||
@@ -75,6 +86,11 @@ class MetadataRegistry:
|
||||
# Store the prompt in the metadata for later relationship tracing
|
||||
self.prompt_metadata[self.current_prompt_id]["current_prompt"] = prompt
|
||||
|
||||
def set_extra_data(self, extra_data):
|
||||
"""Store the API extra_data (contains extra_pnginfo.workflow with node properties)"""
|
||||
if self.current_prompt_id and self.current_prompt_id in self.prompt_metadata:
|
||||
self.prompt_metadata[self.current_prompt_id]["extra_data"] = extra_data
|
||||
|
||||
def get_metadata(self, prompt_id=None):
|
||||
"""Get collected metadata for a prompt"""
|
||||
key = prompt_id if prompt_id is not None else self.current_prompt_id
|
||||
@@ -122,20 +138,28 @@ class MetadataRegistry:
|
||||
cache_key = f"{node_id}:{class_type}"
|
||||
|
||||
# Check if this node type is relevant for metadata collection
|
||||
if class_type in NODE_EXTRACTORS:
|
||||
if class_type in NODE_EXTRACTORS or cache_key in self.node_cache:
|
||||
# Check if we have cached metadata for this node
|
||||
if cache_key in self.node_cache:
|
||||
cached_data = self.node_cache[cache_key]
|
||||
|
||||
# Detect bypass (mode=4) / mute (mode=2) — these nodes
|
||||
# were intentionally disabled and should not contribute
|
||||
# overwrite values from a previous execution's cache.
|
||||
node_mode = node_data.get("mode", 0)
|
||||
node_is_disabled = node_mode in (2, 4)
|
||||
|
||||
# Apply cached metadata to the current metadata
|
||||
for category in self.metadata_categories:
|
||||
if category == OVERWRITE and node_is_disabled:
|
||||
continue
|
||||
if category in cached_data and node_id in cached_data[category]:
|
||||
if node_id not in metadata[category]:
|
||||
metadata[category][node_id] = cached_data[category][
|
||||
node_id
|
||||
]
|
||||
|
||||
def record_node_execution(self, node_id, class_type, inputs, outputs):
|
||||
def record_node_execution(self, node_id, class_type, inputs, outputs, return_types=None):
|
||||
"""Record information about a node's execution"""
|
||||
if not self.current_prompt_id:
|
||||
return
|
||||
@@ -158,17 +182,18 @@ class MetadataRegistry:
|
||||
|
||||
# Extract node-specific metadata
|
||||
extractor = NODE_EXTRACTORS.get(class_type, GenericNodeExtractor)
|
||||
extractor.extract(
|
||||
node_id,
|
||||
processed_inputs,
|
||||
outputs,
|
||||
self.prompt_metadata[self.current_prompt_id],
|
||||
)
|
||||
if extractor is GenericNodeExtractor:
|
||||
extractor.extract(node_id, processed_inputs, outputs,
|
||||
self.prompt_metadata[self.current_prompt_id],
|
||||
return_types=return_types)
|
||||
else:
|
||||
extractor.extract(node_id, processed_inputs, outputs,
|
||||
self.prompt_metadata[self.current_prompt_id])
|
||||
|
||||
# Cache this node's metadata
|
||||
self._cache_node_metadata(node_id, class_type)
|
||||
|
||||
def update_node_execution(self, node_id, class_type, outputs):
|
||||
def update_node_execution(self, node_id, class_type, outputs, return_types=None):
|
||||
"""Update node metadata with output information"""
|
||||
if not self.current_prompt_id:
|
||||
return
|
||||
@@ -179,9 +204,17 @@ class MetadataRegistry:
|
||||
# Use the same extractor to update with outputs
|
||||
extractor = NODE_EXTRACTORS.get(class_type, GenericNodeExtractor)
|
||||
if hasattr(extractor, "update"):
|
||||
extractor.update(
|
||||
node_id, processed_outputs, self.prompt_metadata[self.current_prompt_id]
|
||||
)
|
||||
if extractor is GenericNodeExtractor:
|
||||
extractor.update(
|
||||
node_id, processed_outputs,
|
||||
self.prompt_metadata[self.current_prompt_id],
|
||||
return_types=return_types,
|
||||
)
|
||||
else:
|
||||
extractor.update(
|
||||
node_id, processed_outputs,
|
||||
self.prompt_metadata[self.current_prompt_id],
|
||||
)
|
||||
|
||||
# Update the cached metadata for this node
|
||||
self._cache_node_metadata(node_id, class_type)
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
from .constants import MODELS, PROMPTS, SAMPLING, LORAS, SIZE, IMAGES, IS_SAMPLER
|
||||
from .constants import MODELS, PROMPTS, SAMPLING, LORAS, SIZE, IMAGES, IS_SAMPLER, OVERWRITE
|
||||
from .overwrite_utils import collect_overwrite_params
|
||||
|
||||
|
||||
def _store_checkpoint_metadata(metadata, node_id, model_name):
|
||||
@@ -29,11 +32,78 @@ class NodeMetadataExtractor:
|
||||
pass
|
||||
|
||||
class GenericNodeExtractor(NodeMetadataExtractor):
|
||||
"""Default extractor for nodes without specific handling"""
|
||||
"""Fallback extractor with type-signature-based detection.
|
||||
|
||||
When a node is not in the NODE_EXTRACTORS registry, the hook layer
|
||||
passes ``return_types`` from ``obj.RETURN_TYPES``:
|
||||
|
||||
* ``MODEL`` output: common input fields (ckpt_name, unet_name, etc.)
|
||||
are checked for a model file name and stored as checkpoint metadata.
|
||||
* ``CONDITIONING`` output: common text input fields are checked for
|
||||
prompt text and stored as prompt metadata.
|
||||
"""
|
||||
|
||||
# Input field names that carry a model path in loader-style nodes.
|
||||
_MODEL_NAME_FIELDS = (
|
||||
"ckpt_name", "unet_name", "model_path", "model_name", "gguf_name",
|
||||
)
|
||||
|
||||
# Extensions used by checkpoint_scanner.py — only record values that look
|
||||
# like real model filenames to avoid capturing unrelated string fields.
|
||||
_MODEL_EXTENSIONS = {
|
||||
".ckpt", ".pt", ".pt2", ".bin", ".pth", ".safetensors", ".pkl", ".sft", ".gguf",
|
||||
}
|
||||
|
||||
# Input field names that may carry prompt text in encoder-style nodes.
|
||||
_TEXT_FIELDS = ("text", "clip_l", "t5xxl", "prompt", "positive", "negative")
|
||||
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
pass
|
||||
|
||||
def extract(node_id, inputs, outputs, metadata, return_types=None):
|
||||
if return_types is None:
|
||||
return
|
||||
|
||||
# — MODEL loader detection (checkpoint / UNET / GGUF) —
|
||||
if "MODEL" in return_types or any("MODEL" in str(t) for t in return_types):
|
||||
for field in GenericNodeExtractor._MODEL_NAME_FIELDS:
|
||||
val = inputs.get(field)
|
||||
if val and isinstance(val, str) and val.strip():
|
||||
name = val.strip()
|
||||
if not any(name.lower().endswith(ext) for ext in GenericNodeExtractor._MODEL_EXTENSIONS):
|
||||
continue
|
||||
_store_checkpoint_metadata(metadata, node_id, name)
|
||||
return
|
||||
|
||||
# — CONDITIONING encoder detection (CLIPTextEncode, Flux, custom) —
|
||||
if "CONDITIONING" in return_types or any("CONDITIONING" in str(t) for t in return_types):
|
||||
text = None
|
||||
for field in GenericNodeExtractor._TEXT_FIELDS:
|
||||
val = inputs.get(field)
|
||||
if val and isinstance(val, str) and val.strip():
|
||||
text = val.strip()
|
||||
break
|
||||
if text:
|
||||
prompt_data = metadata.setdefault(PROMPTS, {})
|
||||
prompt_data[node_id] = {
|
||||
"text": text,
|
||||
"node_id": node_id,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata, return_types=None):
|
||||
if return_types is None:
|
||||
return
|
||||
if "CONDITIONING" not in return_types and not any(
|
||||
"CONDITIONING" in str(t) for t in return_types
|
||||
):
|
||||
return
|
||||
if node_id not in metadata.get(PROMPTS, {}):
|
||||
return
|
||||
if outputs and isinstance(outputs, list) and len(outputs) > 0:
|
||||
if isinstance(outputs[0], tuple) and len(outputs[0]) > 0:
|
||||
cond = outputs[0][0]
|
||||
if cond is not None:
|
||||
metadata[PROMPTS][node_id]["conditioning"] = cond
|
||||
|
||||
class CheckpointLoaderExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
@@ -142,6 +212,118 @@ class TSCCheckpointLoaderExtractor(NodeMetadataExtractor):
|
||||
metadata[PROMPTS][node_id]["positive_encoded"] = positive_conditioning
|
||||
metadata[PROMPTS][node_id]["negative_encoded"] = negative_conditioning
|
||||
|
||||
|
||||
class EasyComfyLoaderExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
if "ckpt_name" in inputs:
|
||||
_store_checkpoint_metadata(metadata, node_id, inputs["ckpt_name"])
|
||||
|
||||
# Only extract from optional_lora_stack — skip the single lora_name to
|
||||
# avoid double-counting LoRAs that come through the LORA_STACK path.
|
||||
active_loras = []
|
||||
optional_lora_stack = inputs.get("optional_lora_stack")
|
||||
if optional_lora_stack is not None and isinstance(optional_lora_stack, (list, tuple)):
|
||||
for item in optional_lora_stack:
|
||||
if isinstance(item, (list, tuple)) and len(item) >= 2:
|
||||
lora_path = item[0]
|
||||
model_strength = item[1]
|
||||
lora_name = os.path.splitext(os.path.basename(lora_path))[0]
|
||||
active_loras.append({
|
||||
"name": lora_name,
|
||||
"strength": model_strength
|
||||
})
|
||||
|
||||
if active_loras:
|
||||
metadata[LORAS][node_id] = {
|
||||
"lora_list": active_loras,
|
||||
"node_id": node_id
|
||||
}
|
||||
|
||||
positive_text = inputs.get("positive", "")
|
||||
negative_text = inputs.get("negative", "")
|
||||
|
||||
if positive_text or negative_text:
|
||||
if node_id not in metadata[PROMPTS]:
|
||||
metadata[PROMPTS][node_id] = {"node_id": node_id}
|
||||
metadata[PROMPTS][node_id]["positive_text"] = positive_text
|
||||
metadata[PROMPTS][node_id]["negative_text"] = negative_text
|
||||
|
||||
if "clip_skip" in inputs:
|
||||
clip_skip = inputs["clip_skip"]
|
||||
if node_id not in metadata[SAMPLING]:
|
||||
metadata[SAMPLING][node_id] = {"parameters": {}, "node_id": node_id}
|
||||
metadata[SAMPLING][node_id]["parameters"]["clip_skip"] = clip_skip
|
||||
|
||||
width = inputs.get("empty_latent_width")
|
||||
height = inputs.get("empty_latent_height")
|
||||
if width is not None and height is not None:
|
||||
if SIZE not in metadata:
|
||||
metadata[SIZE] = {}
|
||||
metadata[SIZE][node_id] = {
|
||||
"width": int(width),
|
||||
"height": int(height),
|
||||
"node_id": node_id
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata):
|
||||
# outputs: [(pipe_dict, model, vae), ...]
|
||||
if not outputs or not isinstance(outputs, list) or len(outputs) == 0:
|
||||
return
|
||||
first_output = outputs[0]
|
||||
if not isinstance(first_output, tuple) or len(first_output) < 1:
|
||||
return
|
||||
pipe = first_output[0]
|
||||
if not isinstance(pipe, dict):
|
||||
return
|
||||
|
||||
positive_conditioning = pipe.get("positive")
|
||||
negative_conditioning = pipe.get("negative")
|
||||
|
||||
if positive_conditioning is not None or negative_conditioning is not None:
|
||||
if node_id not in metadata[PROMPTS]:
|
||||
metadata[PROMPTS][node_id] = {"node_id": node_id}
|
||||
if positive_conditioning is not None:
|
||||
metadata[PROMPTS][node_id]["positive_encoded"] = positive_conditioning
|
||||
if negative_conditioning is not None:
|
||||
metadata[PROMPTS][node_id]["negative_encoded"] = negative_conditioning
|
||||
|
||||
|
||||
class EasyPreSamplingExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
sampling_params = {}
|
||||
for key in ("steps", "cfg", "sampler_name", "scheduler", "denoise", "seed"):
|
||||
if key in inputs:
|
||||
sampling_params[key] = inputs[key]
|
||||
|
||||
metadata[SAMPLING][node_id] = {
|
||||
"parameters": sampling_params,
|
||||
"node_id": node_id,
|
||||
IS_SAMPLER: True
|
||||
}
|
||||
|
||||
|
||||
class EasySeedExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs or "seed" not in inputs:
|
||||
return
|
||||
|
||||
metadata[SAMPLING][node_id] = {
|
||||
"parameters": {"seed": inputs["seed"]},
|
||||
"node_id": node_id,
|
||||
IS_SAMPLER: False
|
||||
}
|
||||
|
||||
|
||||
class CLIPTextEncodeExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
@@ -161,6 +343,251 @@ class CLIPTextEncodeExtractor(NodeMetadataExtractor):
|
||||
conditioning = outputs[0][0]
|
||||
metadata[PROMPTS][node_id]["conditioning"] = conditioning
|
||||
|
||||
|
||||
class MyOriginalWaifuTextExtractor(NodeMetadataExtractor):
|
||||
"""Extractor for ComfyUI-MyOriginalWaifu TextProvider nodes."""
|
||||
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
positive_text = inputs.get("positive", "")
|
||||
negative_text = inputs.get("negative", "")
|
||||
|
||||
if positive_text or negative_text:
|
||||
metadata[PROMPTS][node_id] = {
|
||||
"positive_text": positive_text,
|
||||
"negative_text": negative_text,
|
||||
"node_id": node_id,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata):
|
||||
output_tuple = _first_output_tuple(outputs)
|
||||
if not output_tuple or len(output_tuple) < 2:
|
||||
return
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
prompt_metadata["positive_text"] = output_tuple[0]
|
||||
prompt_metadata["negative_text"] = output_tuple[1]
|
||||
|
||||
|
||||
class MyOriginalWaifuClipExtractor(NodeMetadataExtractor):
|
||||
"""Extractor for ComfyUI-MyOriginalWaifu ClipProvider nodes."""
|
||||
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
positive_text = inputs.get("positive", "")
|
||||
negative_text = inputs.get("negative", "")
|
||||
|
||||
if positive_text or negative_text:
|
||||
metadata[PROMPTS][node_id] = {
|
||||
"positive_text": positive_text,
|
||||
"negative_text": negative_text,
|
||||
"node_id": node_id,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata):
|
||||
output_tuple = _first_output_tuple(outputs)
|
||||
if not output_tuple or len(output_tuple) < 2:
|
||||
return
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
prompt_metadata["positive_encoded"] = output_tuple[0]
|
||||
prompt_metadata["negative_encoded"] = output_tuple[1]
|
||||
|
||||
|
||||
def _ensure_prompt_metadata(metadata, node_id):
|
||||
if node_id not in metadata[PROMPTS]:
|
||||
metadata[PROMPTS][node_id] = {"node_id": node_id}
|
||||
return metadata[PROMPTS][node_id]
|
||||
|
||||
|
||||
def _first_output_tuple(outputs):
|
||||
if not outputs or not isinstance(outputs, list) or len(outputs) == 0:
|
||||
return None
|
||||
first_output = outputs[0]
|
||||
if isinstance(first_output, tuple):
|
||||
return first_output
|
||||
return None
|
||||
|
||||
|
||||
def _record_conditioning_source(
|
||||
metadata, node_id, output_conditioning, input_conditionings
|
||||
):
|
||||
if output_conditioning is None:
|
||||
return
|
||||
|
||||
sources = [
|
||||
conditioning for conditioning in input_conditionings if conditioning is not None
|
||||
]
|
||||
if not sources:
|
||||
return
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
prompt_metadata.setdefault("conditioning_sources", []).append(
|
||||
{
|
||||
"output": output_conditioning,
|
||||
"inputs": sources,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _get_variable_name(inputs):
|
||||
for key in ("key", "name", "variable_name", "tag", "text"):
|
||||
value = inputs.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _get_node_variable_name(metadata, node_id, inputs):
|
||||
variable_name = _get_variable_name(inputs)
|
||||
if variable_name:
|
||||
return variable_name
|
||||
|
||||
prompt = metadata.get("current_prompt")
|
||||
original_prompt = getattr(prompt, "original_prompt", None)
|
||||
if not original_prompt or node_id not in original_prompt:
|
||||
return None
|
||||
|
||||
node_data = original_prompt[node_id]
|
||||
variable_name = _get_variable_name(node_data.get("inputs", {}))
|
||||
if variable_name:
|
||||
return variable_name
|
||||
|
||||
widgets_values = node_data.get("widgets_values", [])
|
||||
if widgets_values and isinstance(widgets_values[0], str):
|
||||
return widgets_values[0]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class ControlNetApplyAdvancedExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
if inputs.get("positive") is not None:
|
||||
prompt_metadata["orig_pos_cond"] = inputs["positive"]
|
||||
if inputs.get("negative") is not None:
|
||||
prompt_metadata["orig_neg_cond"] = inputs["negative"]
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata):
|
||||
output_tuple = _first_output_tuple(outputs)
|
||||
if not output_tuple:
|
||||
return
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
positive_input = prompt_metadata.get("orig_pos_cond")
|
||||
negative_input = prompt_metadata.get("orig_neg_cond")
|
||||
|
||||
if len(output_tuple) >= 1:
|
||||
prompt_metadata["positive_encoded"] = output_tuple[0]
|
||||
_record_conditioning_source(
|
||||
metadata, node_id, output_tuple[0], [positive_input]
|
||||
)
|
||||
if len(output_tuple) >= 2:
|
||||
prompt_metadata["negative_encoded"] = output_tuple[1]
|
||||
_record_conditioning_source(
|
||||
metadata, node_id, output_tuple[1], [negative_input]
|
||||
)
|
||||
|
||||
|
||||
class ConditioningCombineExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
input_conditionings = []
|
||||
for input_name in inputs:
|
||||
if (
|
||||
input_name.startswith("conditioning")
|
||||
and inputs[input_name] is not None
|
||||
):
|
||||
input_conditionings.append(inputs[input_name])
|
||||
|
||||
if input_conditionings:
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
prompt_metadata["orig_conditionings"] = input_conditionings
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata):
|
||||
output_tuple = _first_output_tuple(outputs)
|
||||
if not output_tuple or len(output_tuple) < 1:
|
||||
return
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
output_conditioning = output_tuple[0]
|
||||
prompt_metadata["conditioning"] = output_conditioning
|
||||
_record_conditioning_source(
|
||||
metadata,
|
||||
node_id,
|
||||
output_conditioning,
|
||||
prompt_metadata.get("orig_conditionings", []),
|
||||
)
|
||||
|
||||
|
||||
class SetNodeExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
variable_name = _get_node_variable_name(metadata, node_id, inputs)
|
||||
conditioning = inputs.get("CONDITIONING")
|
||||
if conditioning is None:
|
||||
conditioning = inputs.get("conditioning")
|
||||
if conditioning is None:
|
||||
return
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
prompt_metadata["conditioning"] = conditioning
|
||||
if variable_name:
|
||||
prompt_metadata["variable_name"] = variable_name
|
||||
metadata[PROMPTS].setdefault("__conditioning_variables__", {})[
|
||||
variable_name
|
||||
] = conditioning
|
||||
|
||||
|
||||
class GetNodeExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
variable_name = _get_node_variable_name(metadata, node_id, inputs or {})
|
||||
if variable_name:
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
prompt_metadata["variable_name"] = variable_name
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata):
|
||||
output_tuple = _first_output_tuple(outputs)
|
||||
if not output_tuple or len(output_tuple) < 1:
|
||||
return
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
output_conditioning = output_tuple[0]
|
||||
prompt_metadata["conditioning"] = output_conditioning
|
||||
|
||||
variable_name = prompt_metadata.get("variable_name")
|
||||
if not variable_name:
|
||||
return
|
||||
|
||||
input_conditioning = metadata[PROMPTS].get("__conditioning_variables__", {}).get(
|
||||
variable_name
|
||||
)
|
||||
_record_conditioning_source(
|
||||
metadata, node_id, output_conditioning, [input_conditioning]
|
||||
)
|
||||
|
||||
# Base Sampler Extractor to reduce code redundancy
|
||||
class BaseSamplerExtractor(NodeMetadataExtractor):
|
||||
"""Base extractor for sampler nodes with common functionality"""
|
||||
@@ -427,6 +854,75 @@ class ImageSizeExtractor(NodeMetadataExtractor):
|
||||
"node_id": node_id
|
||||
}
|
||||
|
||||
class RgthreePowerLoraLoaderExtractor(NodeMetadataExtractor):
|
||||
"""Extract LoRA metadata from rgthree Power Lora Loader.
|
||||
|
||||
The node passes LoRAs as dynamic kwargs: LORA_1, LORA_2, ... each containing
|
||||
{'on': bool, 'lora': filename, 'strength': float, 'strengthTwo': float}.
|
||||
"""
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
active_loras = []
|
||||
for key, value in inputs.items():
|
||||
if not key.upper().startswith('LORA_'):
|
||||
continue
|
||||
if not isinstance(value, dict):
|
||||
continue
|
||||
if not value.get('on') or not value.get('lora'):
|
||||
continue
|
||||
lora_name = os.path.splitext(os.path.basename(value['lora']))[0]
|
||||
active_loras.append({
|
||||
"name": lora_name,
|
||||
"strength": round(float(value.get('strength', 1.0)), 2)
|
||||
})
|
||||
|
||||
if active_loras:
|
||||
metadata[LORAS][node_id] = {
|
||||
"lora_list": active_loras,
|
||||
"node_id": node_id
|
||||
}
|
||||
|
||||
|
||||
class TensorRTLoaderExtractor(NodeMetadataExtractor):
|
||||
"""Extract checkpoint metadata from TensorRT Loader.
|
||||
|
||||
extract() parses the engine filename from 'unet_name' as a best-effort
|
||||
fallback (strips profile suffix after '_$' and counter suffix).
|
||||
|
||||
update() checks if the output MODEL has attachments["source_model"]
|
||||
set by the node (NubeBuster fork) and overrides with the real name.
|
||||
Vanilla TRT doesn't set this — the filename parse stands.
|
||||
"""
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs or "unet_name" not in inputs:
|
||||
return
|
||||
unet_name = inputs.get("unet_name")
|
||||
# Strip path and extension, then drop the $_profile suffix
|
||||
model_name = os.path.splitext(os.path.basename(unet_name))[0]
|
||||
if "_$" in model_name:
|
||||
model_name = model_name[:model_name.index("_$")]
|
||||
# Strip counter suffix (e.g. _00001_) left by ComfyUI's save path
|
||||
model_name = re.sub(r'_\d+_?$', '', model_name)
|
||||
_store_checkpoint_metadata(metadata, node_id, model_name)
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata):
|
||||
if not outputs or not isinstance(outputs, list) or len(outputs) == 0:
|
||||
return
|
||||
first_output = outputs[0]
|
||||
if not isinstance(first_output, tuple) or len(first_output) < 1:
|
||||
return
|
||||
model = first_output[0]
|
||||
# NubeBuster fork sets attachments["source_model"] on the ModelPatcher
|
||||
source_model = getattr(model, 'attachments', {}).get("source_model")
|
||||
if source_model:
|
||||
_store_checkpoint_metadata(metadata, node_id, source_model)
|
||||
|
||||
|
||||
class LoraLoaderManagerExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
@@ -473,6 +969,55 @@ class LoraLoaderManagerExtractor(NodeMetadataExtractor):
|
||||
"node_id": node_id
|
||||
}
|
||||
|
||||
class LoraTextLoaderManagerExtractor(NodeMetadataExtractor):
|
||||
"""Extract LoRA metadata from LoraTextLoaderLM (LoRA Text Loader).
|
||||
|
||||
The node accepts a `lora_syntax` STRING containing <lora:name:strength> tags
|
||||
(same format as the ComfyUI prompt), plus an optional `lora_stack`.
|
||||
This extractor parses the syntax string using the same regex as the node.
|
||||
"""
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
active_loras = []
|
||||
|
||||
# Process lora_stack if available (optional input)
|
||||
if "lora_stack" in inputs:
|
||||
lora_stack = inputs.get("lora_stack", [])
|
||||
for item in lora_stack:
|
||||
# lora_stack entries are (path, model_strength, clip_strength) tuples
|
||||
if isinstance(item, (list, tuple)) and len(item) >= 2:
|
||||
lora_path = item[0]
|
||||
model_strength = item[1]
|
||||
lora_name = os.path.splitext(os.path.basename(lora_path))[0]
|
||||
active_loras.append({
|
||||
"name": lora_name,
|
||||
"strength": round(float(model_strength), 2)
|
||||
})
|
||||
|
||||
# Process lora_syntax string input
|
||||
if "lora_syntax" in inputs:
|
||||
lora_syntax = inputs.get("lora_syntax", "")
|
||||
if lora_syntax and isinstance(lora_syntax, str):
|
||||
pattern = r"<lora:([^:>]+):([^:>]+)(?::([^:>]+))?>"
|
||||
matches = re.findall(pattern, lora_syntax, re.IGNORECASE)
|
||||
for match in matches:
|
||||
lora_name = match[0]
|
||||
model_strength = float(match[1])
|
||||
active_loras.append({
|
||||
"name": lora_name,
|
||||
"strength": round(model_strength, 2)
|
||||
})
|
||||
|
||||
if active_loras:
|
||||
metadata[LORAS][node_id] = {
|
||||
"lora_list": active_loras,
|
||||
"node_id": node_id
|
||||
}
|
||||
|
||||
|
||||
class FluxGuidanceExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
@@ -577,8 +1122,6 @@ class SamplerCustomAdvancedExtractor(BaseSamplerExtractor):
|
||||
# Extract latent dimensions
|
||||
BaseSamplerExtractor.extract_latent_dimensions(node_id, inputs, metadata)
|
||||
|
||||
import json
|
||||
|
||||
class CLIPTextEncodeFluxExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
@@ -679,6 +1222,28 @@ class CR_ApplyControlNetStackExtractor(NodeMetadataExtractor):
|
||||
metadata[PROMPTS][node_id]["positive_encoded"] = transformed_positive
|
||||
metadata[PROMPTS][node_id]["negative_encoded"] = transformed_negative
|
||||
|
||||
class MetadataOverwriteExtractor(NodeMetadataExtractor):
|
||||
"""Extract manually specified metadata from MetadataOverwriteLM node.
|
||||
|
||||
Stores truthy input values under the OVERWRITE category so that
|
||||
extract_generation_params can merge them over the inferred params.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
overwrite_params = collect_overwrite_params(inputs)
|
||||
|
||||
if overwrite_params:
|
||||
metadata.setdefault(OVERWRITE, {})
|
||||
metadata[OVERWRITE][node_id] = {
|
||||
"parameters": overwrite_params,
|
||||
"node_id": node_id,
|
||||
}
|
||||
|
||||
|
||||
# Registry of node-specific extractors
|
||||
# Keys are node class names
|
||||
NODE_EXTRACTORS = {
|
||||
@@ -699,9 +1264,12 @@ NODE_EXTRACTORS = {
|
||||
"KSamplerSelect": KSamplerSelectExtractor, # Add KSamplerSelect
|
||||
"BasicScheduler": BasicSchedulerExtractor, # Add BasicScheduler
|
||||
"AlignYourStepsScheduler": BasicSchedulerExtractor, # Add AlignYourStepsScheduler
|
||||
# ComfyUI-Easy-Use pre-sampling / seed
|
||||
"samplerSettings": EasyPreSamplingExtractor, # easy preSampling
|
||||
"easySeed": EasySeedExtractor, # easy seed
|
||||
# Loaders
|
||||
"CheckpointLoaderSimple": CheckpointLoaderExtractor,
|
||||
"comfyLoader": CheckpointLoaderExtractor, # easy comfyLoader
|
||||
"comfyLoader": EasyComfyLoaderExtractor, # ComfyUI-Easy-Use easy comfyLoader
|
||||
"CheckpointLoaderSimpleWithImages": CheckpointLoaderExtractor, # CheckpointLoader|pysssss
|
||||
"TSC_EfficientLoader": TSCCheckpointLoaderExtractor, # Efficient Nodes
|
||||
"NunchakuFluxDiTLoader": NunchakuFluxDiTLoaderExtractor, # ComfyUI-Nunchaku
|
||||
@@ -711,12 +1279,18 @@ NODE_EXTRACTORS = {
|
||||
"GGUFLoaderKJ": KJNodesModelLoaderExtractor, # KJNodes
|
||||
"DiffusionModelLoaderKJ": KJNodesModelLoaderExtractor, # KJNodes
|
||||
"CheckpointLoaderKJ": CheckpointLoaderExtractor, # KJNodes
|
||||
"CheckpointLoaderLM": CheckpointLoaderExtractor, # LoRA Manager
|
||||
"UNETLoader": UNETLoaderExtractor, # Updated to use dedicated extractor
|
||||
"UnetLoaderGGUF": UNETLoaderExtractor, # Updated to use dedicated extractor
|
||||
"UNETLoaderLM": UNETLoaderExtractor, # LoRA Manager
|
||||
"LoraLoader": LoraLoaderExtractor,
|
||||
"LoraLoaderLM": LoraLoaderManagerExtractor,
|
||||
"LoraTextLoaderLM": LoraTextLoaderManagerExtractor,
|
||||
"RgthreePowerLoraLoader": RgthreePowerLoraLoaderExtractor,
|
||||
"TensorRTLoader": TensorRTLoaderExtractor,
|
||||
# Conditioning
|
||||
"CLIPTextEncode": CLIPTextEncodeExtractor,
|
||||
"CLIPTextEncodeAttentionBias": CLIPTextEncodeExtractor, # From https://github.com/silveroxides/ComfyUI_PromptAttention
|
||||
"PromptLM": CLIPTextEncodeExtractor,
|
||||
"CLIPTextEncodeFlux": CLIPTextEncodeFluxExtractor, # Add CLIPTextEncodeFlux
|
||||
"WAS_Text_to_Conditioning": CLIPTextEncodeExtractor,
|
||||
@@ -724,6 +1298,12 @@ NODE_EXTRACTORS = {
|
||||
"smZ_CLIPTextEncode": CLIPTextEncodeExtractor, # From https://github.com/shiimizu/ComfyUI_smZNodes
|
||||
"CR_ApplyControlNetStack": CR_ApplyControlNetStackExtractor, # Add CR_ApplyControlNetStack
|
||||
"PCTextEncode": CLIPTextEncodeExtractor, # From https://github.com/asagi4/comfyui-prompt-control
|
||||
"TextProvider": MyOriginalWaifuTextExtractor, # ComfyUI-MyOriginalWaifu
|
||||
"ClipProvider": MyOriginalWaifuClipExtractor, # ComfyUI-MyOriginalWaifu
|
||||
"ControlNetApplyAdvanced": ControlNetApplyAdvancedExtractor,
|
||||
"ConditioningCombine": ConditioningCombineExtractor,
|
||||
"SetNode": SetNodeExtractor,
|
||||
"GetNode": GetNodeExtractor,
|
||||
# Latent
|
||||
"EmptyLatentImage": ImageSizeExtractor,
|
||||
# Flux
|
||||
@@ -731,5 +1311,7 @@ NODE_EXTRACTORS = {
|
||||
"CFGGuider": CFGGuiderExtractor, # Add CFGGuider
|
||||
# Image
|
||||
"VAEDecode": VAEDecodeExtractor, # Added VAEDecode extractor
|
||||
# Metadata overwrite
|
||||
"MetadataOverwriteLM": MetadataOverwriteExtractor,
|
||||
# Add other nodes as needed
|
||||
}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Shared helpers for Metadata Overwrite node metadata collection.
|
||||
|
||||
Used by both the MetadataOverwriteLM node (execution time) and the
|
||||
MetadataOverwriteExtractor (hook time) so the conversion/filtering logic
|
||||
cannot drift between the two paths.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
from ..utils.utils import model_patcher_to_name
|
||||
from .constants import CLIP_SKIP_SENTINEL, METADATA_OVERWRITE_FIELDS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def collect_overwrite_params(values: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Convert node input values into non-default overwrite parameters.
|
||||
|
||||
For most fields, a falsy value (empty string, 0) means "not set" and is
|
||||
skipped. clip_skip uses a dedicated sentinel (-25) so that a wired value
|
||||
of 0 is preserved. The ``model`` field accepts either a manual string or
|
||||
a wired MODEL (ModelPatcher) connection; in the latter case the source
|
||||
model name is extracted from the patcher's ``cached_patcher_init`` and
|
||||
stored as a ComfyUI-style relative path.
|
||||
"""
|
||||
result: Dict[str, Any] = {}
|
||||
for key in METADATA_OVERWRITE_FIELDS:
|
||||
value = values.get(key)
|
||||
if key == "model" and not isinstance(value, str):
|
||||
value = model_patcher_to_name(value)
|
||||
if value is None:
|
||||
logger.warning(
|
||||
"Could not extract model name from wired MODEL input "
|
||||
"(no cached_patcher_init); model metadata overwrite skipped"
|
||||
)
|
||||
if key == "clip_skip":
|
||||
if value != CLIP_SKIP_SENTINEL:
|
||||
result[key] = value
|
||||
elif value:
|
||||
result[key] = value
|
||||
return result
|
||||
@@ -0,0 +1,233 @@
|
||||
"""Metadata operations — thin in-process wrappers around LoRA Manager internal services.
|
||||
|
||||
All functions are simple Python async functions that delegate to the
|
||||
appropriate internal service. They use **relative imports** within the
|
||||
``py`` package, so ``sys.modules`` caching works normally and there is no
|
||||
risk of double import or circular dependencies.
|
||||
|
||||
Usage (in-process, primary)::
|
||||
|
||||
from py.metadata_ops import list_base_models, read_metadata
|
||||
|
||||
models = await list_base_models()
|
||||
meta = await read_metadata("/path/to/model.safetensors")
|
||||
|
||||
Usage (subprocess, debugging / external)::
|
||||
|
||||
python -m py.metadata_ops base-models list
|
||||
python -m py.metadata_ops metadata read /path/to/model.safetensors
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SCANNER_TYPE_MAP: dict[str, str] = {
|
||||
"get_lora_scanner": "lora",
|
||||
"get_checkpoint_scanner": "checkpoint",
|
||||
"get_embedding_scanner": "embedding",
|
||||
}
|
||||
|
||||
SCANNER_GETTER_NAMES = tuple(SCANNER_TYPE_MAP.keys())
|
||||
|
||||
|
||||
async def _find_model_entry(
|
||||
model_path: str,
|
||||
) -> tuple[Any, object, str | None] | tuple[None, None, None]:
|
||||
"""Iterate all scanners and return the first (scanner, entry, getter_name)
|
||||
that owns *model_path*. Returns ``(None, None, None)`` when no scanner
|
||||
claims it.
|
||||
"""
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
|
||||
normalized = os.path.normpath(model_path)
|
||||
for getter_name in SCANNER_GETTER_NAMES:
|
||||
getter = getattr(ServiceRegistry, getter_name, None)
|
||||
if getter is None:
|
||||
continue
|
||||
try:
|
||||
scanner = await getter()
|
||||
if scanner is None:
|
||||
continue
|
||||
cache = await scanner.get_cached_data()
|
||||
for entry in cache.raw_data:
|
||||
if os.path.normpath(entry.get("file_path", "")) == normalized:
|
||||
return scanner, entry, getter_name
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Scanner %s check failed for %s: %s",
|
||||
getter_name, model_path, exc,
|
||||
)
|
||||
return None, None, None
|
||||
|
||||
|
||||
async def _find_scanner_for_model(
|
||||
model_path: str,
|
||||
) -> tuple[Any, object] | tuple[None, None]:
|
||||
"""Find the (scanner, cache_entry) responsible for *model_path*."""
|
||||
scanner, entry, _ = await _find_model_entry(model_path)
|
||||
return scanner, entry
|
||||
|
||||
|
||||
async def identify_model_type(model_path: str) -> str:
|
||||
"""Determine the model type (``\"lora\"``, ``\"checkpoint\"``, or
|
||||
``\"embedding\"``) for *model_path*.
|
||||
|
||||
Falls back to ``\"lora\"`` when unknown.
|
||||
"""
|
||||
_, _, getter_name = await _find_model_entry(model_path)
|
||||
return SCANNER_TYPE_MAP[getter_name] if getter_name else "lora"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def list_base_models(limit: int = 0) -> List[str]:
|
||||
"""Return all valid CivitAI base model names.
|
||||
|
||||
Uses ``CivitaiBaseModelService.get_base_models()`` which merges a
|
||||
hardcoded list (``SUPPORTED_DOWNLOAD_SKIP_BASE_MODELS``) with remote
|
||||
models fetched from the CivitAI API. Never empty — the hardcoded
|
||||
fallback always provides a complete set.
|
||||
|
||||
The result is sorted alphabetically. Pass *limit* = 0 for all models.
|
||||
"""
|
||||
from ..services.civitai_base_model_service import (
|
||||
CivitaiBaseModelService,
|
||||
)
|
||||
|
||||
try:
|
||||
service = await CivitaiBaseModelService.get_instance()
|
||||
response = await service.get_base_models()
|
||||
names: List[str] = response.get("models", [])
|
||||
except Exception as exc:
|
||||
logger.warning("list_base_models failed: %s", exc)
|
||||
names = []
|
||||
if limit > 0:
|
||||
return names[:limit]
|
||||
return names
|
||||
|
||||
|
||||
async def read_metadata(model_path: str) -> Dict[str, Any]:
|
||||
"""Load the full metadata payload for *model_path* from disk.
|
||||
|
||||
Returns an empty dict when the metadata file does not exist or cannot
|
||||
be parsed — never raises.
|
||||
"""
|
||||
from ..utils.metadata_manager import MetadataManager
|
||||
|
||||
try:
|
||||
return await MetadataManager.load_metadata_payload(model_path) or {}
|
||||
except Exception as exc:
|
||||
logger.warning("read_metadata failed for %s: %s", model_path, exc)
|
||||
return {}
|
||||
|
||||
|
||||
async def apply_metadata_updates(
|
||||
model_path: str,
|
||||
updates: Dict[str, Any],
|
||||
) -> List[str]:
|
||||
"""Merge *updates* into the model's on-disk metadata and persist.
|
||||
|
||||
Returns the list of field names that actually changed.
|
||||
"""
|
||||
from ..utils.metadata_manager import MetadataManager
|
||||
|
||||
metadata = await read_metadata(model_path)
|
||||
updated_fields: List[str] = []
|
||||
for key, value in updates.items():
|
||||
old = metadata.get(key)
|
||||
if old != value:
|
||||
metadata[key] = value
|
||||
updated_fields.append(key)
|
||||
if updated_fields:
|
||||
await MetadataManager.save_metadata(model_path, metadata)
|
||||
return updated_fields
|
||||
|
||||
|
||||
async def download_preview(
|
||||
model_path: str,
|
||||
url: str,
|
||||
*,
|
||||
target_width: int = 480,
|
||||
quality: int = 85,
|
||||
) -> str | None:
|
||||
"""Download a preview image from *url*, optimise to .webp, and save it.
|
||||
|
||||
The output file is placed alongside the model file with a ``.webp``
|
||||
extension. Returns the local file path on success, ``None`` on failure.
|
||||
"""
|
||||
from ..services.downloader import get_downloader
|
||||
from ..utils.exif_utils import ExifUtils
|
||||
|
||||
if not url or not url.strip():
|
||||
return None
|
||||
|
||||
base_name = os.path.splitext(os.path.basename(model_path))[0]
|
||||
preview_dir = os.path.dirname(model_path)
|
||||
output_path = os.path.join(preview_dir, base_name + ".webp")
|
||||
|
||||
downloader = await get_downloader()
|
||||
|
||||
# Try in-memory download + optimise first
|
||||
success, content, _headers = await downloader.download_to_memory(
|
||||
url, use_auth=False,
|
||||
)
|
||||
if success and content:
|
||||
try:
|
||||
optimized_data, _ = ExifUtils.optimize_image(
|
||||
image_data=content,
|
||||
target_width=target_width,
|
||||
format="webp",
|
||||
quality=quality,
|
||||
preserve_metadata=False,
|
||||
)
|
||||
with open(output_path, "wb") as f:
|
||||
f.write(optimized_data)
|
||||
return output_path
|
||||
except Exception as exc:
|
||||
logger.warning("Preview optimisation failed, saving raw: %s", exc)
|
||||
# Fall through to raw save
|
||||
|
||||
# Fallback: download directly to file
|
||||
try:
|
||||
ok, _ = await downloader.download_file(url, output_path, use_auth=False)
|
||||
if ok:
|
||||
return output_path
|
||||
except Exception as exc:
|
||||
logger.warning("Preview fallback download failed for %s: %s", model_path, exc)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def refresh_cache(model_path: str) -> bool:
|
||||
"""Invalidate and reload the scanner cache entry for *model_path*.
|
||||
|
||||
Returns ``True`` when the model was found and the cache was refreshed.
|
||||
"""
|
||||
scanner, entry = await _find_scanner_for_model(model_path)
|
||||
if scanner is None:
|
||||
logger.warning("refresh_cache: no scanner found for %s", model_path)
|
||||
return False
|
||||
try:
|
||||
metadata = await read_metadata(model_path)
|
||||
if not metadata:
|
||||
logger.warning("refresh_cache: no metadata for %s", model_path)
|
||||
return False
|
||||
await scanner.update_single_model_cache(model_path, model_path, metadata)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning("refresh_cache failed for %s: %s", model_path, exc)
|
||||
return False
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Subprocess entry point for ``metadata_ops`` (debugging / external use).
|
||||
|
||||
Usage::
|
||||
|
||||
python -m py.metadata_ops base-models list [--limit N]
|
||||
python -m py.metadata_ops metadata read <path>
|
||||
python -m py.metadata_ops metadata update <path> --json '{...}'
|
||||
python -m py.metadata_ops preview download <path> --url <url>
|
||||
python -m py.metadata_ops cache refresh <path>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from typing import Any, Dict, List
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(prog="lmcli", description="LoRA Manager Agent CLI")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
# base-models list
|
||||
base_models = sub.add_parser("base-models", aliases=["bm"])
|
||||
base_models_cmds = base_models.add_subparsers(dest="subcommand", required=True)
|
||||
base_models_list = base_models_cmds.add_parser("list")
|
||||
base_models_list.add_argument(
|
||||
"--limit", type=int, default=0, help="Max number of models (0 = all)"
|
||||
)
|
||||
|
||||
# metadata read
|
||||
meta = sub.add_parser("metadata", aliases=["md"])
|
||||
meta_cmds = meta.add_subparsers(dest="subcommand", required=True)
|
||||
meta_read = meta_cmds.add_parser("read")
|
||||
meta_read.add_argument("path", type=str, help="Model file path")
|
||||
|
||||
# metadata update
|
||||
meta_update = meta_cmds.add_parser("update")
|
||||
meta_update.add_argument("path", type=str, help="Model file path")
|
||||
meta_update.add_argument(
|
||||
"--json",
|
||||
type=str,
|
||||
required=True,
|
||||
help='JSON object of fields to update, e.g. \'{"base_model": "SDXL 1.0"}\'',
|
||||
)
|
||||
|
||||
# preview download
|
||||
prev = sub.add_parser("preview", aliases=["pv"])
|
||||
prev_cmds = prev.add_subparsers(dest="subcommand", required=True)
|
||||
prev_dl = prev_cmds.add_parser("download")
|
||||
prev_dl.add_argument("path", type=str, help="Model file path")
|
||||
prev_dl.add_argument("--url", type=str, required=True, help="Preview image URL")
|
||||
|
||||
# cache refresh
|
||||
cache = sub.add_parser("cache")
|
||||
cache_cmds = cache.add_subparsers(dest="subcommand", required=True)
|
||||
cache_refresh = cache_cmds.add_parser("refresh")
|
||||
cache_refresh.add_argument("path", type=str, help="Model file path")
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
async def _run(args: argparse.Namespace) -> Any:
|
||||
from . import ( # lazy import so startup is fast
|
||||
list_base_models,
|
||||
read_metadata,
|
||||
apply_metadata_updates,
|
||||
download_preview,
|
||||
refresh_cache,
|
||||
)
|
||||
|
||||
cmd = args.command
|
||||
sub = args.subcommand
|
||||
|
||||
if cmd in ("base-models", "bm") and sub == "list":
|
||||
return await list_base_models(limit=args.limit)
|
||||
|
||||
if cmd in ("metadata", "md") and sub == "read":
|
||||
return await read_metadata(args.path)
|
||||
|
||||
if cmd in ("metadata", "md") and sub == "update":
|
||||
updates: Dict[str, Any] = json.loads(args.json)
|
||||
return await apply_metadata_updates(args.path, updates)
|
||||
|
||||
if cmd in ("preview", "pv") and sub == "download":
|
||||
return await download_preview(args.path, args.url)
|
||||
|
||||
if cmd == "cache" and sub == "refresh":
|
||||
return await refresh_cache(args.path)
|
||||
|
||||
raise ValueError(f"Unknown command: {cmd} {sub}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = _build_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
result = asyncio.run(_run(args))
|
||||
# Always print as JSON so callers can parse reliably
|
||||
if isinstance(result, list):
|
||||
for item in result:
|
||||
print(item)
|
||||
elif isinstance(result, dict):
|
||||
json.dump(result, sys.stdout, ensure_ascii=False, indent=2)
|
||||
print()
|
||||
else:
|
||||
print(json.dumps(result))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -16,6 +16,8 @@ IMG_EXTENSIONS = (
|
||||
".tif",
|
||||
".tiff",
|
||||
".webp",
|
||||
".avif",
|
||||
".jxl",
|
||||
".mp4"
|
||||
)
|
||||
|
||||
|
||||
@@ -4,15 +4,21 @@ from typing import Awaitable, Callable, Dict, List
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
# Use wildcard for CivitAI to support their CDN subdomains (e.g., image-b2.civitai.com)
|
||||
# Security note: This is acceptable because:
|
||||
# 1. CSP img-src only controls image/video loading, not script execution
|
||||
# 2. All *.civitai.com subdomains are controlled by Civitai
|
||||
# 3. Explicit domain list would require constant updates as Civitai adds CDN nodes
|
||||
REMOTE_MEDIA_SOURCES = (
|
||||
"https://image.civitai.com",
|
||||
"https://*.civitai.com",
|
||||
"https://img.genur.art",
|
||||
)
|
||||
|
||||
|
||||
@web.middleware
|
||||
async def relax_csp_for_remote_media(
|
||||
request: web.Request, handler: Callable[[web.Request], Awaitable[web.StreamResponse]]
|
||||
request: web.Request,
|
||||
handler: Callable[[web.Request], Awaitable[web.StreamResponse]],
|
||||
) -> web.StreamResponse:
|
||||
"""Allow LoRA Manager media previews to load from trusted remote domains.
|
||||
|
||||
@@ -43,7 +49,9 @@ async def relax_csp_for_remote_media(
|
||||
directive_order.append(name)
|
||||
directives[name] = values
|
||||
|
||||
def merge_sources(name: str, sources: List[str], defaults: List[str] | None = None) -> None:
|
||||
def merge_sources(
|
||||
name: str, sources: List[str], defaults: List[str] | None = None
|
||||
) -> None:
|
||||
existing = directives.get(name, list(defaults or []))
|
||||
|
||||
for source in sources:
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""JSON error middleware for API routes.
|
||||
|
||||
Ensures all responses to /api/* requests return valid JSON that the
|
||||
browser-extension frontend can JSON.parse() without crashing, even when
|
||||
the route does not exist (404) or the handler raises an exception (500).
|
||||
|
||||
Extension consumers call response.json() unconditionally — an HTML error
|
||||
page causes ``SyntaxError: unexpected end of data`` that leaks into the
|
||||
popup UI as a toast notification.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Awaitable, Callable
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@web.middleware
|
||||
async def api_json_error(
|
||||
request: web.Request,
|
||||
handler: Callable[[web.Request], Awaitable[web.Response]],
|
||||
) -> web.Response:
|
||||
"""Return JSON ``{"success": false, "error": "..."}`` for API errors.
|
||||
|
||||
Only intercepts paths starting with ``/api/`` — all other routes
|
||||
(frontend pages, static files, WebSocket upgrades) pass through
|
||||
unchanged.
|
||||
"""
|
||||
if not request.path.startswith("/api/"):
|
||||
return await handler(request)
|
||||
|
||||
try:
|
||||
response = await handler(request)
|
||||
return response
|
||||
except web.HTTPException as exc:
|
||||
# Let redirects (301, 302, 307, 308) propagate — they are not errors.
|
||||
if exc.status < 400:
|
||||
raise
|
||||
|
||||
# Preview 404 is routine (file deleted from disk) — not worth a warning.
|
||||
logger_method = logger.warning
|
||||
if request.path.startswith("/api/lm/previews") and exc.status == 404:
|
||||
logger_method = logger.debug
|
||||
|
||||
logger_method(
|
||||
"API %s %s returned HTTP %d: %s",
|
||||
request.method,
|
||||
request.path,
|
||||
exc.status,
|
||||
exc.reason,
|
||||
)
|
||||
|
||||
return web.json_response(
|
||||
{"success": False, "error": f"{exc.status}: {exc.reason}"},
|
||||
status=exc.status,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"API %s %s raised unhandled exception: %s",
|
||||
request.method,
|
||||
request.path,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": f"500: Internal Server Error ({type(exc).__name__})",
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
@@ -1,7 +1,7 @@
|
||||
import logging
|
||||
from typing import List, Tuple
|
||||
import comfy.sd # type: ignore
|
||||
import folder_paths # type: ignore
|
||||
from typing import Any, List, Tuple
|
||||
import comfy.sd # pyright: ignore[reportMissingImports]
|
||||
import folder_paths # pyright: ignore[reportMissingImports]
|
||||
from ..utils.utils import get_checkpoint_info_absolute, _format_model_name_for_comfyui
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -18,9 +18,9 @@ class CheckpointLoaderLM:
|
||||
CATEGORY = "Lora Manager/loaders"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
def INPUT_TYPES(cls):
|
||||
# Get list of checkpoint names from scanner (includes extra folder paths)
|
||||
checkpoint_names = s._get_checkpoint_names()
|
||||
checkpoint_names = cls._get_checkpoint_names()
|
||||
return {
|
||||
"required": {
|
||||
"ckpt_name": (
|
||||
@@ -89,7 +89,7 @@ class CheckpointLoaderLM:
|
||||
logger.error(f"Error getting checkpoint names: {e}")
|
||||
return []
|
||||
|
||||
def load_checkpoint(self, ckpt_name: str) -> Tuple:
|
||||
def load_checkpoint(self, ckpt_name: str) -> Tuple[Any, Any, Any]:
|
||||
"""Load a checkpoint by name, supporting extra folder paths
|
||||
|
||||
Args:
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Create Hook LoRA (LoraManager) — multi-LoRA hook node compatible with ComfyUI's built-in hook pipeline.
|
||||
|
||||
Produces ``("HOOKS",)`` output that chains seamlessly with downstream hook consumers
|
||||
(ConditioningSetProperties, SetHookKeyframes, CombineHooks, SetClipHooks, etc.).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from ..utils.utils import get_lora_info_absolute
|
||||
from .utils import (
|
||||
FlexibleOptionalInputType,
|
||||
any_type,
|
||||
apply_lora_syntax_format,
|
||||
get_loras_list,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CreateHookLoraLM:
|
||||
NAME = "Create Hook LoRA (LoraManager)"
|
||||
CATEGORY = "Lora Manager/hooks"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"text": (
|
||||
"AUTOCOMPLETE_TEXT_LORAS",
|
||||
{
|
||||
"placeholder": "Search LoRAs to add...",
|
||||
"tooltip": (
|
||||
"Search and select LoRAs. Each LoRA gets its own "
|
||||
"model/clip strength. Hooks chain with prev_hooks."
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": FlexibleOptionalInputType(any_type),
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("HOOKS", "STRING", "STRING")
|
||||
RETURN_NAMES = ("HOOKS", "trigger_words", "active_loras")
|
||||
FUNCTION = "create_hook"
|
||||
|
||||
def create_hook(self, text: str, **kwargs):
|
||||
"""Create a HookGroup from the selected LoRAs, chained with prev_hooks.
|
||||
|
||||
Each active LoRA from the widget is loaded and wrapped in a WeightHook
|
||||
via :func:`comfy.hooks.create_hook_lora`. All hooks are combined into a
|
||||
single group and returned alongside trigger words and a human-readable
|
||||
summary of the active LoRAs.
|
||||
"""
|
||||
del text # used by the frontend widget only
|
||||
|
||||
# Lazy imports: comfy is not available in CI/test environment at module level
|
||||
import comfy.hooks # pyright: ignore[reportMissingImports] # noqa: C0415
|
||||
import comfy.utils # pyright: ignore[reportMissingImports] # noqa: C0415
|
||||
|
||||
prev_hooks: comfy.hooks.HookGroup | None = kwargs.get("prev_hooks")
|
||||
|
||||
hook_group = prev_hooks.clone() if prev_hooks is not None else comfy.hooks.HookGroup()
|
||||
|
||||
all_trigger_words: list[str] = []
|
||||
active_loras: list[tuple[str, float, float]] = []
|
||||
|
||||
for lora in get_loras_list(kwargs):
|
||||
if not lora.get("active", False):
|
||||
continue
|
||||
|
||||
lora_name = apply_lora_syntax_format(lora["name"])
|
||||
model_strength = float(lora["strength"])
|
||||
clip_strength = float(lora.get("clipStrength", model_strength))
|
||||
|
||||
# Skip useless no-op entries (both strengths are zero)
|
||||
if model_strength == 0.0 and clip_strength == 0.0:
|
||||
continue
|
||||
|
||||
lora_path, trigger_words = get_lora_info_absolute(lora_name)
|
||||
if not lora_path or not os.path.isfile(lora_path):
|
||||
logger.warning("LoRA '%s' not found — skipping", lora_name)
|
||||
continue
|
||||
|
||||
try:
|
||||
lora_weights = comfy.utils.load_torch_file(lora_path, safe_load=True)
|
||||
|
||||
lora_hooks = comfy.hooks.create_hook_lora(
|
||||
lora=lora_weights,
|
||||
strength_model=model_strength,
|
||||
strength_clip=clip_strength,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to load LoRA '%s' — skipping", lora_name)
|
||||
continue
|
||||
hook_group = hook_group.clone_and_combine(lora_hooks)
|
||||
|
||||
active_loras.append((lora_name, model_strength, clip_strength))
|
||||
all_trigger_words.extend(trigger_words)
|
||||
|
||||
# Format trigger words (group mode separator)
|
||||
trigger_words_text = ",, ".join(all_trigger_words) if all_trigger_words else ""
|
||||
|
||||
# Format active LoRAs summary
|
||||
formatted_loras = []
|
||||
for name, model_s, clip_s in active_loras:
|
||||
if abs(model_s - clip_s) > 0.001:
|
||||
formatted_loras.append(
|
||||
f"<lora:{name}:{model_s}:{clip_s}>"
|
||||
)
|
||||
else:
|
||||
formatted_loras.append(f"<lora:{name}:{model_s}>")
|
||||
active_loras_text = " ".join(formatted_loras)
|
||||
|
||||
return (hook_group, trigger_words_text, active_loras_text)
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Lora Info display node — pure frontend node for showing selected LoRA info.
|
||||
|
||||
This node does NOT participate in workflow execution. Its single optional
|
||||
"lora_source" input exists solely as a wire-connection anchor so that the
|
||||
frontend can traverse the graph and push selection data to connected info nodes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class LoraInfoLM:
|
||||
"""Display node that shows filename and notes for the selected LoRA."""
|
||||
|
||||
NAME = "Lora Info (LoraManager)"
|
||||
CATEGORY = "Lora Manager/utils"
|
||||
DESCRIPTION = (
|
||||
"Displays information (filename, notes) about the currently selected "
|
||||
"LoRA. Connect any output from a LoRA Loader or Stacker to the "
|
||||
"lora_source input, then select a LoRA in the source widget — the "
|
||||
"info updates automatically. Does not affect workflow execution."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ()
|
||||
RETURN_NAMES = ()
|
||||
OUTPUT_NODE = False
|
||||
FUNCTION = "noop"
|
||||
|
||||
def noop(self, **kwargs):
|
||||
# This node is display-only — no workflow execution needed.
|
||||
return ()
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
LoraInfoLM.NAME: LoraInfoLM,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
LoraInfoLM.NAME: "Lora Info (LoraManager)",
|
||||
}
|
||||
+6
-20
@@ -1,18 +1,19 @@
|
||||
import importlib
|
||||
import logging
|
||||
import re
|
||||
|
||||
import comfy.sd # type: ignore
|
||||
import comfy.utils # type: ignore
|
||||
import comfy.sd # pyright: ignore[reportMissingImports]
|
||||
import comfy.utils # pyright: ignore[reportMissingImports]
|
||||
|
||||
from ..utils.utils import get_lora_info_absolute
|
||||
from .utils import (
|
||||
FlexibleOptionalInputType,
|
||||
any_type,
|
||||
apply_lora_syntax_format,
|
||||
detect_nunchaku_model_kind,
|
||||
extract_lora_name,
|
||||
get_loras_list,
|
||||
nunchaku_load_lora,
|
||||
parse_lora_syntax,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -52,7 +53,7 @@ def _collect_widget_entries(kwargs):
|
||||
for lora in get_loras_list(kwargs):
|
||||
if not lora.get("active", False):
|
||||
continue
|
||||
lora_name = lora["name"]
|
||||
lora_name = apply_lora_syntax_format(lora["name"])
|
||||
model_strength = float(lora["strength"])
|
||||
clip_strength = float(lora.get("clipStrength", model_strength))
|
||||
lora_path, trigger_words = get_lora_info_absolute(lora_name)
|
||||
@@ -188,25 +189,10 @@ class LoraTextLoaderLM:
|
||||
RETURN_NAMES = ("MODEL", "CLIP", "trigger_words", "loaded_loras")
|
||||
FUNCTION = "load_loras_from_text"
|
||||
|
||||
def parse_lora_syntax(self, text):
|
||||
"""Parse LoRA syntax from text input."""
|
||||
pattern = r"<lora:([^:>]+):([^:>]+)(?::([^:>]+))?>"
|
||||
matches = re.findall(pattern, text, re.IGNORECASE)
|
||||
|
||||
loras = []
|
||||
for match in matches:
|
||||
model_strength = float(match[1])
|
||||
loras.append({
|
||||
"name": match[0],
|
||||
"model_strength": model_strength,
|
||||
"clip_strength": float(match[2]) if match[2] else model_strength,
|
||||
})
|
||||
return loras
|
||||
|
||||
def load_loras_from_text(self, model, lora_syntax, clip=None, lora_stack=None):
|
||||
"""Load LoRAs based on text syntax input."""
|
||||
lora_entries = _collect_stack_entries(lora_stack)
|
||||
for lora in self.parse_lora_syntax(lora_syntax):
|
||||
for lora in parse_lora_syntax(lora_syntax):
|
||||
lora_path, trigger_words = get_lora_info_absolute(lora["name"])
|
||||
lora_entries.append({
|
||||
"name": lora["name"],
|
||||
|
||||
@@ -1,26 +1,102 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
_STACK_INPUT_PATTERN = re.compile(r"^lora_stack(?:_([ab])|(\d+))$")
|
||||
|
||||
|
||||
def _is_stack_input(name: str) -> bool:
|
||||
return bool(_STACK_INPUT_PATTERN.match(name))
|
||||
|
||||
|
||||
def _stack_slot_number(name: str) -> int:
|
||||
"""Numeric slot used to order stack inputs; legacy a/b map to 1/2."""
|
||||
match = _STACK_INPUT_PATTERN.match(name)
|
||||
if not match:
|
||||
return -1
|
||||
letter, digits = match.group(1), match.group(2)
|
||||
if digits is not None:
|
||||
return int(digits)
|
||||
return 1 if letter == "a" else 2
|
||||
|
||||
|
||||
class _LoraStackOptionalInputs:
|
||||
"""Lookup that preserves explicit optional inputs and dynamic lora_stack slots."""
|
||||
|
||||
def __init__(self, explicit_inputs: dict[str, tuple[str, dict[str, Any]]]) -> None:
|
||||
self._explicit_inputs = explicit_inputs
|
||||
|
||||
def __contains__(self, item: object) -> bool:
|
||||
if not isinstance(item, str):
|
||||
return False
|
||||
return item in self._explicit_inputs or _is_stack_input(item)
|
||||
|
||||
def __getitem__(self, key: str) -> tuple[str, dict[str, Any]]:
|
||||
if key in self._explicit_inputs:
|
||||
return self._explicit_inputs[key]
|
||||
if _is_stack_input(key):
|
||||
return (
|
||||
"LORA_STACK",
|
||||
{
|
||||
"tooltip": "A LoRA stack to combine. Connect to add more inputs.",
|
||||
},
|
||||
)
|
||||
raise KeyError(key)
|
||||
|
||||
|
||||
class LoraStackCombinerLM:
|
||||
NAME = "Lora Stack Combiner (LoraManager)"
|
||||
CATEGORY = "Lora Manager/stackers"
|
||||
DESCRIPTION = (
|
||||
"Combines multiple LoRA stacks into a single stack. "
|
||||
"Supports dynamic inputs: connect a stack to add more inputs."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
optional_inputs: dict[str, tuple[str, dict[str, Any]]] = {
|
||||
"lora_stack1": (
|
||||
"LORA_STACK",
|
||||
{
|
||||
"tooltip": "A LoRA stack to combine. Connect to add more inputs.",
|
||||
},
|
||||
),
|
||||
"lora_stack2": (
|
||||
"LORA_STACK",
|
||||
{
|
||||
"tooltip": "A LoRA stack to combine. Connect to add more inputs.",
|
||||
},
|
||||
),
|
||||
}
|
||||
|
||||
stack = inspect.stack()
|
||||
if len(stack) > 2 and stack[2].function == "get_input_info":
|
||||
optional_inputs = _LoraStackOptionalInputs(optional_inputs) # pyright: ignore[reportAssignmentType]
|
||||
|
||||
return {
|
||||
"required": {
|
||||
"lora_stack_a": ("LORA_STACK",),
|
||||
"lora_stack_b": ("LORA_STACK",),
|
||||
},
|
||||
"required": {},
|
||||
"optional": optional_inputs,
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("LORA_STACK",)
|
||||
RETURN_NAMES = ("LORA_STACK",)
|
||||
FUNCTION = "combine_stacks"
|
||||
|
||||
def combine_stacks(self, lora_stack_a, lora_stack_b):
|
||||
combined_stack = []
|
||||
def combine_stacks(self, lora_stack1=None, lora_stack2=None, **kwargs):
|
||||
stacks = {
|
||||
"lora_stack1": lora_stack1,
|
||||
"lora_stack2": lora_stack2,
|
||||
}
|
||||
for key, value in kwargs.items():
|
||||
if _is_stack_input(key) and value is not None:
|
||||
stacks[key] = value
|
||||
|
||||
if lora_stack_a:
|
||||
combined_stack.extend(lora_stack_a)
|
||||
if lora_stack_b:
|
||||
combined_stack.extend(lora_stack_b)
|
||||
combined_stack = []
|
||||
for key in sorted(stacks, key=_stack_slot_number):
|
||||
stack = stacks[key]
|
||||
if stack:
|
||||
combined_stack.extend(stack)
|
||||
|
||||
return (combined_stack,)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
from ..utils.utils import get_lora_info
|
||||
from .utils import FlexibleOptionalInputType, any_type, extract_lora_name, get_loras_list
|
||||
from .utils import FlexibleOptionalInputType, any_type, apply_lora_syntax_format, extract_lora_name, get_loras_list
|
||||
|
||||
import logging
|
||||
|
||||
@@ -48,7 +48,7 @@ class LoraStackerLM:
|
||||
if not lora.get('active', False):
|
||||
continue
|
||||
|
||||
lora_name = lora['name']
|
||||
lora_name = apply_lora_syntax_format(lora['name'])
|
||||
model_strength = float(lora['strength'])
|
||||
# Get clip strength - use model strength as default if not specified
|
||||
clip_strength = float(lora.get('clipStrength', model_strength))
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Node to resolve `<lora:name:strength>` syntax to absolute file system paths.
|
||||
|
||||
Takes the loaded_loras / active_loras STRING output from LoraLoaderLM or
|
||||
LoraStackerLM and resolves each lora name to its absolute path on disk via
|
||||
the scanner cache. Unknown names are returned as-is.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from ..utils.utils import get_lora_info_absolute
|
||||
from .utils import parse_lora_syntax
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LoraSyntaxToPath:
|
||||
NAME = "LoRA Syntax → Path (LoraManager)"
|
||||
CATEGORY = "Lora Manager/utils"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"lora_syntax": (
|
||||
"STRING",
|
||||
{
|
||||
"forceInput": True,
|
||||
"multiline": True,
|
||||
"tooltip": (
|
||||
"<lora:name:strength> formatted text from "
|
||||
"loaded_loras / active_loras output"
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("paths",)
|
||||
FUNCTION = "resolve"
|
||||
|
||||
def resolve(self, lora_syntax: str) -> tuple[str]:
|
||||
"""Parse <lora:...> syntax and resolve each name to its absolute path."""
|
||||
if not lora_syntax or not lora_syntax.strip():
|
||||
logger.info("Received empty lora_syntax input")
|
||||
return ("",)
|
||||
|
||||
parsed = parse_lora_syntax(lora_syntax)
|
||||
if not parsed:
|
||||
logger.info("No valid <lora:...> entries found in input")
|
||||
return ("",)
|
||||
|
||||
paths: list[str] = []
|
||||
for entry in parsed:
|
||||
try:
|
||||
absolute_path, _ = get_lora_info_absolute(entry["name"])
|
||||
paths.append(absolute_path)
|
||||
except Exception:
|
||||
logger.warning("Failed to resolve lora '%s', skipping", entry["name"])
|
||||
continue
|
||||
|
||||
return ("\n".join(paths),)
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Metadata Overwrite node — allows users to manually specify generation parameters
|
||||
that override the automatically collected/inferred metadata.
|
||||
|
||||
Most inputs have falsy defaults (empty string / 0) which are skipped.
|
||||
clip_skip uses a sentinel default (-25) so that a wired value of 0 is
|
||||
preserved — both ComfyUI and A1111 conventions have no meaningful 0 value,
|
||||
but users may wire 0 to express "no clip skip / default".
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from ..metadata_collector.constants import CLIP_SKIP_SENTINEL as _CLIP_SKIP_SENTINEL
|
||||
from ..metadata_collector.overwrite_utils import collect_overwrite_params
|
||||
|
||||
|
||||
class MetadataOverwriteLM:
|
||||
NAME = "Metadata Overwrite (LoraManager)"
|
||||
CATEGORY = "Lora Manager/utils"
|
||||
DESCRIPTION = (
|
||||
"Manually specify generation parameters to override automatically collected "
|
||||
"metadata. Only filled/connected inputs will take effect — empty defaults "
|
||||
"are ignored."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls) -> dict[str, Any]:
|
||||
return {
|
||||
"optional": {
|
||||
"prompt": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"multiline": True,
|
||||
"tooltip": "Positive prompt. Only overwrites when non-empty.",
|
||||
},
|
||||
),
|
||||
"negative_prompt": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"multiline": True,
|
||||
"tooltip": "Negative prompt. Only overwrites when non-empty.",
|
||||
},
|
||||
),
|
||||
"seed": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"min": 0,
|
||||
"max": 0xFFFFFFFFFFFFFFFF,
|
||||
"control_after_generate": False,
|
||||
"tooltip": "Seed value. Only overwrites when > 0.",
|
||||
},
|
||||
),
|
||||
"steps": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"min": 0,
|
||||
"max": 10000,
|
||||
"tooltip": "Number of steps. Only overwrites when > 0.",
|
||||
},
|
||||
),
|
||||
"cfg_scale": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.0,
|
||||
"min": 0.0,
|
||||
"max": 100.0,
|
||||
"tooltip": "CFG scale. Only overwrites when > 0.",
|
||||
},
|
||||
),
|
||||
"sampler": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"tooltip": "Sampler name. Only overwrites when non-empty.",
|
||||
},
|
||||
),
|
||||
"scheduler": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"tooltip": "Scheduler name. Only overwrites when non-empty.",
|
||||
},
|
||||
),
|
||||
"model": (
|
||||
"STRING,MODEL",
|
||||
{
|
||||
"default": "",
|
||||
"widgetType": "STRING",
|
||||
"tooltip": (
|
||||
"The checkpoint or diffusion model (UNet) used "
|
||||
"for generation. Fill in the name manually or "
|
||||
"connect a MODEL output — the model name is then "
|
||||
"extracted automatically. Only overwrites when "
|
||||
"non-empty."
|
||||
),
|
||||
},
|
||||
),
|
||||
"loras": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"multiline": True,
|
||||
"tooltip": (
|
||||
"LoRA syntax, e.g. <lora:name:strength> "
|
||||
"or <lora:name:model_strength:clip_strength>, "
|
||||
"separated by spaces. Only overwrites when non-empty."
|
||||
),
|
||||
},
|
||||
),
|
||||
"size": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"tooltip": (
|
||||
"Image size in WIDTHxHEIGHT format (e.g. 512x768). "
|
||||
"Only overwrites when non-empty."
|
||||
),
|
||||
},
|
||||
),
|
||||
"clip_skip": (
|
||||
"INT",
|
||||
{
|
||||
"default": _CLIP_SKIP_SENTINEL,
|
||||
"min": -25,
|
||||
"max": 24,
|
||||
"tooltip": (
|
||||
"Clip skip (ComfyUI: -24..-1, A1111: 1+). "
|
||||
"Default -25 means not set — any other value "
|
||||
"overwrites."
|
||||
),
|
||||
},
|
||||
),
|
||||
"additional_data": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"multiline": True,
|
||||
"tooltip": (
|
||||
"Additional data to embed in the image metadata. "
|
||||
"Inserted between Clip skip and Model hash in the "
|
||||
"A1111-compatible parameters string. "
|
||||
'Example: "Copyright": "Some license info"'
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("METADATA",)
|
||||
RETURN_NAMES = ("metadata",)
|
||||
FUNCTION = "collect_metadata"
|
||||
OUTPUT_NODE = True
|
||||
|
||||
def collect_metadata(self, **kwargs: Any) -> tuple[dict[str, Any]]:
|
||||
"""Collect non-default input values into a metadata dict.
|
||||
|
||||
For most fields, a falsy value (empty string, 0) means "not set"
|
||||
and is skipped. clip_skip uses a dedicated sentinel (-25) so that
|
||||
a wired value of 0 is preserved and reaches the metadata pipeline.
|
||||
|
||||
The ``model`` field accepts either a manual string or a wired MODEL
|
||||
(ModelPatcher) connection; in the latter case the underlying model
|
||||
name is extracted from the patcher's ``cached_patcher_init`` and
|
||||
stored as a ComfyUI-style relative path.
|
||||
"""
|
||||
return (collect_overwrite_params(kwargs),)
|
||||
+12
-13
@@ -15,15 +15,15 @@ import os
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple, Union
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union, cast
|
||||
|
||||
import comfy.utils # type: ignore
|
||||
import folder_paths # type: ignore
|
||||
import comfy.utils # pyright: ignore[reportMissingImports]
|
||||
import folder_paths # pyright: ignore[reportMissingImports]
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from safetensors import safe_open
|
||||
|
||||
from nunchaku.lora.flux.nunchaku_converter import (
|
||||
from nunchaku.lora.flux.nunchaku_converter import ( # pyright: ignore[reportMissingTypeStubs]
|
||||
pack_lowrank_weight,
|
||||
unpack_lowrank_weight,
|
||||
)
|
||||
@@ -87,10 +87,6 @@ def _rename_layer_underscore_layer_name(old_name: str) -> str:
|
||||
return new_name
|
||||
|
||||
|
||||
def _is_indexable_module(module):
|
||||
return isinstance(module, (nn.ModuleList, nn.Sequential, list, tuple))
|
||||
|
||||
|
||||
def _get_module_by_name(model: nn.Module, name: str) -> Optional[nn.Module]:
|
||||
if not name:
|
||||
return model
|
||||
@@ -100,7 +96,7 @@ def _get_module_by_name(model: nn.Module, name: str) -> Optional[nn.Module]:
|
||||
continue
|
||||
if hasattr(module, part):
|
||||
module = getattr(module, part)
|
||||
elif part.isdigit() and _is_indexable_module(module):
|
||||
elif part.isdigit() and isinstance(module, (nn.ModuleList, nn.Sequential, list, tuple)):
|
||||
try:
|
||||
module = module[int(part)]
|
||||
except (IndexError, TypeError):
|
||||
@@ -267,7 +263,9 @@ def _handle_proj_out_split(lora_dict: Dict[str, Dict[str, torch.Tensor]], base_k
|
||||
return result, consumed
|
||||
|
||||
|
||||
def _apply_lora_to_module(module: nn.Module, a_tensor: torch.Tensor, b_tensor: torch.Tensor, module_name: str, model: nn.Module) -> None:
|
||||
def _apply_lora_to_module(module: Any, a_tensor: torch.Tensor, b_tensor: torch.Tensor, module_name: str, model: Any) -> None:
|
||||
# These modules are dynamic torch containers; monkey-patched attributes
|
||||
# below are set at runtime, so the module/model types are deliberately Any.
|
||||
if not hasattr(module, "in_features") or not hasattr(module, "out_features"):
|
||||
raise ValueError(f"{module_name}: unsupported module without in/out features")
|
||||
if a_tensor.shape[1] != module.in_features or b_tensor.shape[0] != module.out_features:
|
||||
@@ -336,7 +334,7 @@ def _apply_lora_to_module(module: nn.Module, a_tensor: torch.Tensor, b_tensor: t
|
||||
raise ValueError(f"{module_name}: unsupported module type {type(module)}")
|
||||
|
||||
|
||||
def reset_lora_v2(model: nn.Module) -> None:
|
||||
def reset_lora_v2(model: Any) -> None:
|
||||
slots = getattr(model, "_lora_slots", None)
|
||||
if not slots:
|
||||
return
|
||||
@@ -344,6 +342,7 @@ def reset_lora_v2(model: nn.Module) -> None:
|
||||
module = _get_module_by_name(model, name)
|
||||
if module is None:
|
||||
continue
|
||||
module = cast(Any, module)
|
||||
module_type = info.get("type", "nunchaku")
|
||||
if module_type == "nunchaku":
|
||||
base_rank = info["base_rank"]
|
||||
@@ -371,7 +370,7 @@ def reset_lora_v2(model: nn.Module) -> None:
|
||||
def compose_loras_v2(model: nn.Module, lora_configs: List[Tuple[Union[str, Path, Dict[str, torch.Tensor]], float]], apply_awq_mod: bool = True) -> bool:
|
||||
del apply_awq_mod # retained for interface compatibility
|
||||
reset_lora_v2(model)
|
||||
aggregated_weights: Dict[str, List[Dict[str, object]]] = defaultdict(list)
|
||||
aggregated_weights: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
|
||||
saw_supported_format = False
|
||||
unresolved_targets = 0
|
||||
|
||||
@@ -471,7 +470,7 @@ def compose_loras_v2(model: nn.Module, lora_configs: List[Tuple[Union[str, Path,
|
||||
class ComfyQwenImageWrapperLM(nn.Module):
|
||||
def __init__(self, model: nn.Module, config=None, apply_awq_mod: bool = True):
|
||||
super().__init__()
|
||||
self.model = model
|
||||
self.model: Any = model
|
||||
self.config = {} if config is None else config
|
||||
self.dtype = next(model.parameters()).dtype
|
||||
self.loras: List[Tuple[Union[str, Path, Dict[str, torch.Tensor]], float]] = []
|
||||
|
||||
+69
-21
@@ -1,15 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
import inspect
|
||||
|
||||
from ..services.wildcard_service import (
|
||||
contains_dynamic_syntax,
|
||||
get_wildcard_service,
|
||||
is_trigger_words_input,
|
||||
)
|
||||
|
||||
class _AllContainer:
|
||||
"""Container that accepts any key for dynamic input validation."""
|
||||
|
||||
def __contains__(self, item):
|
||||
return True
|
||||
class _PromptOptionalInputs:
|
||||
"""Lookup that preserves explicit optional inputs and dynamic trigger slots."""
|
||||
|
||||
def __getitem__(self, key):
|
||||
return ("STRING", {"forceInput": True})
|
||||
def __init__(self, explicit_inputs: dict[str, tuple[str, dict[str, Any]]]) -> None:
|
||||
self._explicit_inputs = explicit_inputs
|
||||
|
||||
def __contains__(self, item: object) -> bool:
|
||||
if not isinstance(item, str):
|
||||
return False
|
||||
return item in self._explicit_inputs or is_trigger_words_input(item)
|
||||
|
||||
def __getitem__(self, key: str) -> tuple[str, dict[str, Any]]:
|
||||
if key in self._explicit_inputs:
|
||||
return self._explicit_inputs[key]
|
||||
if is_trigger_words_input(key):
|
||||
return (
|
||||
"STRING",
|
||||
{
|
||||
"forceInput": True,
|
||||
"tooltip": "Trigger words to prepend. Connect to add more inputs.",
|
||||
},
|
||||
)
|
||||
raise KeyError(key)
|
||||
|
||||
|
||||
class PromptLM:
|
||||
@@ -20,12 +43,19 @@ class PromptLM:
|
||||
DESCRIPTION = (
|
||||
"Encodes a text prompt using a CLIP model into an embedding that can be used "
|
||||
"to guide the diffusion model towards generating specific images. "
|
||||
"Supports dynamic trigger words inputs."
|
||||
"Supports dynamic trigger words inputs and runtime wildcard expansion."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
dyn_inputs = {
|
||||
optional_inputs: dict[str, tuple[str, dict[str, Any]]] = {
|
||||
"seed": (
|
||||
"INT",
|
||||
{
|
||||
"forceInput": True,
|
||||
"tooltip": "Optional seed for wildcard generation. Leave unconnected for non-deterministic wildcard expansion.",
|
||||
},
|
||||
),
|
||||
"trigger_words1": (
|
||||
"STRING",
|
||||
{
|
||||
@@ -35,10 +65,9 @@ class PromptLM:
|
||||
),
|
||||
}
|
||||
|
||||
# Bypass validation for dynamic inputs during graph execution
|
||||
stack = inspect.stack()
|
||||
if len(stack) > 2 and stack[2].function == "get_input_info":
|
||||
dyn_inputs = _AllContainer()
|
||||
optional_inputs = _PromptOptionalInputs(optional_inputs) # pyright: ignore[reportAssignmentType]
|
||||
|
||||
return {
|
||||
"required": {
|
||||
@@ -46,8 +75,8 @@ class PromptLM:
|
||||
"AUTOCOMPLETE_TEXT_PROMPT,STRING",
|
||||
{
|
||||
"widgetType": "AUTOCOMPLETE_TEXT_PROMPT",
|
||||
"placeholder": "Enter prompt... /char, /artist for quick tag search",
|
||||
"tooltip": "The text to be encoded.",
|
||||
"placeholder": "Enter prompt... /character, /artist, /wildcard for quick search",
|
||||
"tooltip": "The text to be encoded. Wildcard references inserted with /wildcard are expanded at runtime.",
|
||||
},
|
||||
),
|
||||
"clip": (
|
||||
@@ -55,7 +84,7 @@ class PromptLM:
|
||||
{"tooltip": "The CLIP model used for encoding the text."},
|
||||
),
|
||||
},
|
||||
"optional": dyn_inputs,
|
||||
"optional": optional_inputs,
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("CONDITIONING", "STRING")
|
||||
@@ -65,20 +94,39 @@ class PromptLM:
|
||||
)
|
||||
FUNCTION = "encode"
|
||||
|
||||
def encode(self, text: str, clip: Any, **kwargs):
|
||||
# Collect all trigger words from dynamic inputs
|
||||
@classmethod
|
||||
def IS_CHANGED(
|
||||
cls,
|
||||
text: str,
|
||||
clip: Any | None = None,
|
||||
seed: int | None = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
del clip, kwargs
|
||||
if contains_dynamic_syntax(text) and seed is None:
|
||||
return float("NaN")
|
||||
return False
|
||||
|
||||
def encode(
|
||||
self,
|
||||
text: str,
|
||||
clip: Any,
|
||||
seed: int | None = None,
|
||||
**kwargs: Any,
|
||||
):
|
||||
expanded_text = get_wildcard_service().expand_text(text, seed=seed)
|
||||
|
||||
trigger_words = []
|
||||
for key, value in kwargs.items():
|
||||
if key.startswith("trigger_words") and value:
|
||||
if is_trigger_words_input(key) and value:
|
||||
trigger_words.append(value)
|
||||
|
||||
# Build final prompt
|
||||
if trigger_words:
|
||||
prompt = ", ".join(trigger_words + [text])
|
||||
prompt = ", ".join(trigger_words + [expanded_text])
|
||||
else:
|
||||
prompt = text
|
||||
prompt = expanded_text
|
||||
|
||||
from nodes import CLIPTextEncode # type: ignore
|
||||
from nodes import CLIPTextEncode # pyright: ignore[reportMissingImports, reportAttributeAccessIssue]
|
||||
|
||||
conditioning = CLIPTextEncode().encode(clip, prompt)[0]
|
||||
return (conditioning, prompt)
|
||||
return (conditioning, prompt)
|
||||
|
||||
+592
-132
@@ -1,16 +1,171 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, Optional
|
||||
import numpy as np
|
||||
import folder_paths # type: ignore
|
||||
import folder_paths # pyright: ignore[reportMissingImports]
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
from ..metadata_collector.metadata_processor import MetadataProcessor
|
||||
from ..metadata_collector import get_metadata
|
||||
from ..utils.constants import CARD_PREVIEW_WIDTH
|
||||
from ..utils.exif_utils import ExifUtils
|
||||
from ..utils.utils import calculate_recipe_fingerprint, sanitize_folder_name
|
||||
from PIL import Image, PngImagePlugin
|
||||
import piexif
|
||||
import piexif # pyright: ignore[reportMissingTypeStubs]
|
||||
import logging
|
||||
|
||||
# Civitai-compatible sampler name mapping: ComfyUI internal → A1111 display name
|
||||
CIVITAI_SAMPLER_MAP = {
|
||||
"euler": "Euler",
|
||||
"euler_ancestral": "Euler a",
|
||||
"lms": "LMS",
|
||||
"heun": "Heun",
|
||||
"dpm_2": "DPM2",
|
||||
"dpm_2_ancestral": "DPM2 a",
|
||||
"dpmpp_2s_ancestral": "DPM++ 2S a",
|
||||
"dpmpp_2m": "DPM++ 2M",
|
||||
"dpmpp_sde": "DPM++ SDE",
|
||||
"dpmpp_sde_gpu": "DPM++ SDE",
|
||||
"dpmpp_2m_sde": "DPM++ 2M SDE",
|
||||
"dpmpp_2m_sde_gpu": "DPM++ 2M SDE",
|
||||
"dpmpp_3m_sde": "DPM++ 3M SDE",
|
||||
"dpm_fast": "DPM fast",
|
||||
"dpm_adaptive": "DPM adaptive",
|
||||
"ddim": "DDIM",
|
||||
"plms": "PLMS",
|
||||
"uni_pc_bh2": "UniPC",
|
||||
"uni_pc": "UniPC",
|
||||
"lcm": "LCM",
|
||||
}
|
||||
|
||||
# Base model display name → AIR URN slug
|
||||
# Sourced from civitai source: src/shared/constants/basemodel.constants.ts
|
||||
BASE_MODEL_AIR_SLUG = {
|
||||
# Stable Diffusion family
|
||||
"SD 1.4": "sd1",
|
||||
"SD 1.5": "sd1",
|
||||
"SD 1.5 LCM": "sd1",
|
||||
"SD 1.5 Hyper": "sd1",
|
||||
"SD 2.0": "sd2",
|
||||
"SD 2.0 768": "sd2",
|
||||
"SD 2.1": "sd2",
|
||||
"SD 2.1 768": "sd2",
|
||||
"SD 2.1 Unclip": "sd2",
|
||||
"SD 3.0": "sd3",
|
||||
"SD 3.5": "sd35",
|
||||
"SD 3.5 Large": "sd35",
|
||||
"SD 3.5 Large Turbo": "sd35",
|
||||
"SD 3.5 Medium": "sd35",
|
||||
"SDXL 0.9": "sdxl",
|
||||
"SDXL 1.0": "sdxl",
|
||||
"SDXL 1.0 LCM": "sdxl",
|
||||
"SDXL Lightning": "sdxl",
|
||||
"SDXL Hyper": "sdxl",
|
||||
"SDXL Turbo": "sdxl",
|
||||
"SDXL Distilled": "sdxldistilled",
|
||||
"Stable Cascade": "scascade",
|
||||
"Stable Video Diffusion": "svd",
|
||||
"SVD": "svd",
|
||||
"SVD XT": "svdxt",
|
||||
|
||||
# SDXL community fine-tunes
|
||||
"Pony": "pony",
|
||||
"Pony Diffusion": "pony",
|
||||
"Illustrious": "illustrious",
|
||||
"NoobAI": "noobai",
|
||||
"Animagine": "illustrious",
|
||||
|
||||
# Flux family
|
||||
"Flux.1": "flux1",
|
||||
"Flux.1 D": "flux1",
|
||||
"Flux.1 S": "flux1",
|
||||
"Flux.1 Krea": "fluxkrea",
|
||||
"Flux.1 Kontext": "flux1kontext",
|
||||
"Flux.2": "flux2",
|
||||
"Flux.2 D": "flux2",
|
||||
"Flux.2 Klein 9B": "flux2klein_9b",
|
||||
"Flux.2 Klein 9B Base": "flux2klein_9b_base",
|
||||
"Flux.2 Klein 4B": "flux2klein_4b",
|
||||
"Flux.2 Klein 4B Base": "flux2klein_4b_base",
|
||||
|
||||
# Other image models (sorted alphabetically)
|
||||
"AuraFlow": "auraflow",
|
||||
"Chroma": "chroma",
|
||||
"HiDream": "hidream",
|
||||
"HiDream-O1": "hidream-o1",
|
||||
"Hunyuan DiT": "hydit1",
|
||||
"Hunyuan Video": "hyv1",
|
||||
"Kolors": "kolors",
|
||||
"Lumina": "lumina",
|
||||
"Mochi": "mochi",
|
||||
"ODOR": "odor",
|
||||
"PixArt Alpha": "pixarta",
|
||||
"PixArt Sigma": "pixarte",
|
||||
"Playground v2": "playgroundv2",
|
||||
"Playground v2.5": "playgroundv2",
|
||||
"Pony Diffusion V7": "ponyv7",
|
||||
|
||||
# Video models
|
||||
"CogVideoX": "cogvideox",
|
||||
"LTX Video": "ltxv",
|
||||
"LTX Video 2": "ltxv2",
|
||||
"LTX Video 2.3": "ltxv23",
|
||||
"Wan Video": "wanvideo",
|
||||
"Wan Video 1.3B T2V": "wanvideo_13b_t2v",
|
||||
"Wan Video 14B T2V": "wanvideo_14b_t2v",
|
||||
"Wan Video 14B I2V 480p": "wanvideo_14b_i2v_480p",
|
||||
"Wan Video 14B I2V 720p": "wanvideo_14b_i2v_720p",
|
||||
|
||||
# Third-party / proprietary image models
|
||||
"Boogu": "boogu",
|
||||
"Ernie": "ernie",
|
||||
"Grok": "grok",
|
||||
"HappyHorse": "happyhorse",
|
||||
"Ideogram": "ideogram",
|
||||
"Ideogram 4.0": "ideogram",
|
||||
"Imagen": "imagen4",
|
||||
"Imagen 4": "imagen4",
|
||||
"Krea": "krea2",
|
||||
"Krea 2": "krea2",
|
||||
"Lens": "lens",
|
||||
"MAI": "mai",
|
||||
"Nano Banana": "nanobanana",
|
||||
"OpenAI": "openai",
|
||||
"Reve": "reve",
|
||||
"Reve 2": "reve",
|
||||
"Reve 2.1": "reve",
|
||||
"Seedream": "seedream",
|
||||
"Sora": "sora2",
|
||||
"Sora 2": "sora2",
|
||||
"Veo": "veo3",
|
||||
"Veo 2": "veo3",
|
||||
"Veo 3": "veo3",
|
||||
"ZImageTurbo": "zimageturbo",
|
||||
"ZImageBase": "zimagebase",
|
||||
"ZImage": "zimagebase",
|
||||
|
||||
# Third-party video models
|
||||
"Hailuo by MiniMax": "minimax",
|
||||
"Haiper": "haiper",
|
||||
"Kling": "kling",
|
||||
"Lightricks": "lightricks",
|
||||
"Seedance": "seedance",
|
||||
"Vidu": "vidu",
|
||||
|
||||
# Qwen family
|
||||
"Qwen": "qwen",
|
||||
"Qwen 2": "qwen2",
|
||||
|
||||
# Anima
|
||||
"Anima": "anima",
|
||||
|
||||
# Special
|
||||
"Upscaler": "upscaler",
|
||||
"Other": "other",
|
||||
}
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -65,11 +220,29 @@ class SaveImageLM:
|
||||
"tooltip": "Compression quality for JPEG and lossy WebP formats (1-100). Higher values mean better quality but larger files.",
|
||||
},
|
||||
),
|
||||
"webp_method": (
|
||||
"INT",
|
||||
{
|
||||
"default": 6,
|
||||
"min": 0,
|
||||
"max": 6,
|
||||
"tooltip": "WebP compression method (0-6). 0=fastest/largest, 6=slowest/smallest. Only applies when file_format is 'webp'.",
|
||||
},
|
||||
),
|
||||
"jpeg_subsampling": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"min": 0,
|
||||
"max": 2,
|
||||
"tooltip": "JPEG chroma subsampling level. 0=4:4:4 (best quality), 1=4:2:2, 2=4:2:0 (smallest files). Only applies when file_format is 'jpeg'.",
|
||||
},
|
||||
),
|
||||
"embed_workflow": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "Embeds the complete workflow data into the image metadata. Only works with PNG and WebP formats.",
|
||||
"tooltip": "When enabled, saved images store the complete workflow. Drag the image back into ComfyUI to restore the original node graph. PNG and WebP only.",
|
||||
},
|
||||
),
|
||||
"save_with_metadata": (
|
||||
@@ -79,6 +252,13 @@ class SaveImageLM:
|
||||
"tooltip": "When enabled, embeds generation parameters into the saved image metadata. Disable to skip writing generation metadata.",
|
||||
},
|
||||
),
|
||||
"add_loras_to_prompt": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "When enabled, appends the LoRA syntax line (e.g. <lora:name:strength>) after the positive prompt in the saved metadata.",
|
||||
},
|
||||
),
|
||||
"add_counter_to_filename": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
@@ -86,6 +266,13 @@ class SaveImageLM:
|
||||
"tooltip": "Adds an incremental counter to filenames to prevent overwriting previous images.",
|
||||
},
|
||||
),
|
||||
"save_as_recipe": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "Also saves each generated image as a LoRA Manager recipe.",
|
||||
},
|
||||
),
|
||||
},
|
||||
"hidden": {
|
||||
"id": "UNIQUE_ID",
|
||||
@@ -130,148 +317,197 @@ class SaveImageLM:
|
||||
|
||||
return None
|
||||
|
||||
def format_metadata(self, metadata_dict):
|
||||
"""Format metadata in the requested format similar to userComment example"""
|
||||
if not metadata_dict:
|
||||
return ""
|
||||
def _resolve_model_cache_entry(self, scanner_type: str, name: str):
|
||||
"""Resolve model hash, civitai metadata, and base_model from scanner cache.
|
||||
Returns (hash_str, civitai_dict, base_model_str). All values are empty defaults when not found."""
|
||||
scanner = ServiceRegistry.get_service_sync(scanner_type)
|
||||
if scanner is None or not name:
|
||||
return "", {}, ""
|
||||
|
||||
# Helper function to only add parameter if value is not None
|
||||
def add_param_if_not_none(param_list, label, value):
|
||||
if value is not None:
|
||||
param_list.append(f"{label}: {value}")
|
||||
entry = self._get_cached_model_by_name(scanner, name)
|
||||
if entry is None:
|
||||
basename = os.path.splitext(os.path.basename(name))[0]
|
||||
hash_val = scanner.get_hash_by_filename(basename)
|
||||
return (hash_val or "").lower(), {}, ""
|
||||
|
||||
hash_val = (entry.get("sha256") or "").lower()
|
||||
civitai = entry.get("civitai") or {}
|
||||
base_model = entry.get("base_model") or ""
|
||||
return hash_val, civitai, base_model
|
||||
|
||||
@staticmethod
|
||||
def _get_civitai_sampler_name(sampler_name: str, scheduler: str) -> str:
|
||||
if sampler_name in CIVITAI_SAMPLER_MAP:
|
||||
civitai_name = CIVITAI_SAMPLER_MAP[sampler_name]
|
||||
if scheduler == "karras":
|
||||
civitai_name += " Karras"
|
||||
elif scheduler == "exponential":
|
||||
civitai_name += " Exponential"
|
||||
return civitai_name
|
||||
else:
|
||||
if scheduler and scheduler != "normal":
|
||||
return f"{sampler_name}_{scheduler}"
|
||||
return sampler_name
|
||||
|
||||
@staticmethod
|
||||
def _build_air_string(base_model: str, model_type: str, model_id: int, version_id: int) -> str:
|
||||
slug = BASE_MODEL_AIR_SLUG.get(base_model, "other")
|
||||
type_lower = model_type.lower() if model_type else "other"
|
||||
return f"urn:air:{slug}:{type_lower}:civitai:{model_id}@{version_id}"
|
||||
|
||||
def format_metadata(self, metadata_dict: dict[str, Any], add_loras_to_prompt: bool = False) -> str:
|
||||
"""Format metadata as A1111-compatible parameters string with Hashes JSON and Civitai resources."""
|
||||
if not metadata_dict: return ""
|
||||
|
||||
# Extract the prompt and negative prompt
|
||||
prompt = metadata_dict.get("prompt", "")
|
||||
negative_prompt = metadata_dict.get("negative_prompt", "")
|
||||
|
||||
# Extract loras from the prompt if present
|
||||
steps = metadata_dict.get("steps")
|
||||
cfg = metadata_dict.get("guidance")
|
||||
if cfg is None:
|
||||
cfg = metadata_dict.get("cfg_scale")
|
||||
if cfg is None:
|
||||
cfg = metadata_dict.get("cfg")
|
||||
seed = metadata_dict.get("seed")
|
||||
size = metadata_dict.get("size")
|
||||
sampler = metadata_dict.get("sampler") or ""
|
||||
scheduler = metadata_dict.get("scheduler") or "normal"
|
||||
checkpoint = metadata_dict.get("checkpoint") or ""
|
||||
loras_text = metadata_dict.get("loras", "")
|
||||
lora_hashes = {}
|
||||
clip_skip = metadata_dict.get("clip_skip")
|
||||
|
||||
# If loras are found, add them on a new line after the prompt
|
||||
# Parse LoRA entries from <lora:name:strength> format
|
||||
lora_entries: list[tuple[str, float]] = []
|
||||
if loras_text:
|
||||
prompt_with_loras = f"{prompt}\n{loras_text}"
|
||||
for match in re.findall(r"<lora:([^:]+):([^>]+)>", loras_text):
|
||||
lora_name, strength_str = match
|
||||
try:
|
||||
strength = float(strength_str)
|
||||
except (ValueError, TypeError):
|
||||
strength = 1.0
|
||||
lora_entries.append((lora_name, strength))
|
||||
|
||||
# Extract lora names from the format <lora:name:strength>
|
||||
lora_matches = re.findall(r"<lora:([^:]+):([^>]+)>", loras_text)
|
||||
# Resolve checkpoint hash and Civitai data from local cache
|
||||
ckpt_hash, ckpt_civitai, ckpt_base_model = "", {}, ""
|
||||
ckpt_display_name = ""
|
||||
if checkpoint:
|
||||
ckpt_hash, ckpt_civitai, ckpt_base_model = self._resolve_model_cache_entry(
|
||||
"checkpoint_scanner", checkpoint
|
||||
)
|
||||
ckpt_display_name = os.path.splitext(os.path.basename(checkpoint))[0]
|
||||
|
||||
# Get hash for each lora
|
||||
for lora_name, strength in lora_matches:
|
||||
hash_value = self.get_lora_hash(lora_name)
|
||||
if hash_value:
|
||||
lora_hashes[lora_name] = hash_value
|
||||
else:
|
||||
prompt_with_loras = prompt
|
||||
# Resolve LoRA hash and Civitai data from local cache
|
||||
loras_data: list[dict[str, Any]] = []
|
||||
for lora_name, strength in lora_entries:
|
||||
lora_hash, lora_civitai, lora_base_model = self._resolve_model_cache_entry(
|
||||
"lora_scanner", lora_name
|
||||
)
|
||||
loras_data.append({
|
||||
"name": lora_name,
|
||||
"strength": strength,
|
||||
"hash": lora_hash,
|
||||
"civitai": lora_civitai,
|
||||
"base_model": lora_base_model,
|
||||
})
|
||||
|
||||
# Format the first part (prompt and loras)
|
||||
metadata_parts = [prompt_with_loras]
|
||||
# Build Hashes JSON (A1111 / Civitai standard format)
|
||||
hashes: dict[str, str] = {}
|
||||
if ckpt_hash:
|
||||
hashes["model"] = ckpt_hash[:10].upper()
|
||||
for lora in loras_data:
|
||||
if lora["hash"]:
|
||||
hashes[f"LORA:{lora['name']}"] = lora["hash"][:10].upper()
|
||||
|
||||
# Add negative prompt
|
||||
# Build Civitai resources JSON array
|
||||
civitai_resources: list[dict[str, Any]] = []
|
||||
if ckpt_civitai.get("id", 0) > 0:
|
||||
ckpt_resource: dict[str, Any] = {}
|
||||
ckpt_type = (ckpt_civitai.get("model") or {}).get("type", "Checkpoint")
|
||||
model_id = ckpt_civitai.get("modelId", 0)
|
||||
version_id = ckpt_civitai.get("id", 0)
|
||||
if model_id and version_id:
|
||||
ckpt_resource["air"] = self._build_air_string(
|
||||
ckpt_base_model, ckpt_type, int(model_id), int(version_id)
|
||||
)
|
||||
elif version_id:
|
||||
ckpt_resource["modelVersionId"] = int(version_id)
|
||||
if ckpt_civitai.get("name"):
|
||||
ckpt_resource["versionName"] = ckpt_civitai["name"]
|
||||
if ckpt_resource:
|
||||
civitai_resources.append(ckpt_resource)
|
||||
|
||||
for lora in loras_data:
|
||||
lora_civitai = lora["civitai"]
|
||||
if not lora_civitai or lora_civitai.get("id", 0) <= 0:
|
||||
continue
|
||||
lora_resource: dict[str, Any] = {"weight": lora["strength"]}
|
||||
lora_type = (lora_civitai.get("model") or {}).get("type", "LORA")
|
||||
model_id = lora_civitai.get("modelId", 0)
|
||||
version_id = lora_civitai.get("id", 0)
|
||||
if model_id and version_id:
|
||||
lora_resource["air"] = self._build_air_string(
|
||||
lora["base_model"], lora_type, int(model_id), int(version_id)
|
||||
)
|
||||
elif version_id:
|
||||
lora_resource["modelVersionId"] = int(version_id)
|
||||
if lora_civitai.get("name"):
|
||||
lora_resource["versionName"] = lora_civitai["name"]
|
||||
civitai_resources.append(lora_resource)
|
||||
|
||||
sampler_name = CIVITAI_SAMPLER_MAP.get(sampler, sampler) if sampler else None
|
||||
|
||||
scheduler_mapping = {
|
||||
"normal": "Normal",
|
||||
"karras": "Karras",
|
||||
"exponential": "Exponential",
|
||||
"sgm_uniform": "SGM Uniform",
|
||||
"sgm_quadratic": "SGM Quadratic",
|
||||
}
|
||||
scheduler_name = scheduler_mapping.get(scheduler, scheduler) if scheduler else None
|
||||
|
||||
# Build output lines
|
||||
prompt_line = prompt if prompt else ""
|
||||
if add_loras_to_prompt and loras_text:
|
||||
prompt_line = f"{prompt_line}\n{loras_text}" if prompt_line else loras_text
|
||||
lines = [prompt_line] if prompt_line else [""]
|
||||
if negative_prompt:
|
||||
metadata_parts.append(f"Negative prompt: {negative_prompt}")
|
||||
lines.append(f"Negative prompt: {negative_prompt}")
|
||||
|
||||
# Format the second part (generation parameters)
|
||||
params = []
|
||||
|
||||
# Add standard parameters in the correct order
|
||||
if "steps" in metadata_dict:
|
||||
add_param_if_not_none(params, "Steps", metadata_dict.get("steps"))
|
||||
|
||||
# Combine sampler and scheduler information
|
||||
sampler_name = None
|
||||
scheduler_name = None
|
||||
|
||||
if "sampler" in metadata_dict:
|
||||
sampler = metadata_dict.get("sampler")
|
||||
# Convert ComfyUI sampler names to user-friendly names
|
||||
sampler_mapping = {
|
||||
"euler": "Euler",
|
||||
"euler_ancestral": "Euler a",
|
||||
"dpm_2": "DPM2",
|
||||
"dpm_2_ancestral": "DPM2 a",
|
||||
"heun": "Heun",
|
||||
"dpm_fast": "DPM fast",
|
||||
"dpm_adaptive": "DPM adaptive",
|
||||
"lms": "LMS",
|
||||
"dpmpp_2s_ancestral": "DPM++ 2S a",
|
||||
"dpmpp_sde": "DPM++ SDE",
|
||||
"dpmpp_sde_gpu": "DPM++ SDE",
|
||||
"dpmpp_2m": "DPM++ 2M",
|
||||
"dpmpp_2m_sde": "DPM++ 2M SDE",
|
||||
"dpmpp_2m_sde_gpu": "DPM++ 2M SDE",
|
||||
"ddim": "DDIM",
|
||||
}
|
||||
sampler_name = sampler_mapping.get(sampler, sampler)
|
||||
|
||||
if "scheduler" in metadata_dict:
|
||||
scheduler = metadata_dict.get("scheduler")
|
||||
scheduler_mapping = {
|
||||
"normal": "Simple",
|
||||
"karras": "Karras",
|
||||
"exponential": "Exponential",
|
||||
"sgm_uniform": "SGM Uniform",
|
||||
"sgm_quadratic": "SGM Quadratic",
|
||||
}
|
||||
scheduler_name = scheduler_mapping.get(scheduler, scheduler)
|
||||
|
||||
# Add combined sampler and scheduler information
|
||||
params: list[str] = []
|
||||
if steps is not None:
|
||||
params.append(f"Steps: {steps}")
|
||||
if sampler_name:
|
||||
if scheduler_name:
|
||||
params.append(f"Sampler: {sampler_name} {scheduler_name}")
|
||||
else:
|
||||
params.append(f"Sampler: {sampler_name}")
|
||||
if cfg is not None:
|
||||
params.append(f"CFG scale: {cfg}")
|
||||
if seed is not None:
|
||||
params.append(f"Seed: {seed}")
|
||||
if size:
|
||||
params.append(f"Size: {size}")
|
||||
if clip_skip is not None:
|
||||
try:
|
||||
params.append(f"Clip skip: {abs(int(clip_skip))}")
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
additional_data = metadata_dict.get("additional_data", "")
|
||||
if additional_data:
|
||||
params.append(additional_data)
|
||||
if ckpt_hash:
|
||||
params.append(f"Model hash: {ckpt_hash[:10].upper()}")
|
||||
if ckpt_display_name:
|
||||
params.append(f"Model: {ckpt_display_name}")
|
||||
if hashes:
|
||||
params.append(f"Hashes: {json.dumps(hashes, separators=(',', ':'))}")
|
||||
params.append("Version: ComfyUI")
|
||||
if civitai_resources:
|
||||
params.append(
|
||||
f"Civitai resources: {json.dumps(civitai_resources, separators=(',', ':'))}"
|
||||
)
|
||||
|
||||
# CFG scale (Use guidance if available, otherwise fall back to cfg_scale or cfg)
|
||||
if "guidance" in metadata_dict:
|
||||
add_param_if_not_none(params, "CFG scale", metadata_dict.get("guidance"))
|
||||
elif "cfg_scale" in metadata_dict:
|
||||
add_param_if_not_none(params, "CFG scale", metadata_dict.get("cfg_scale"))
|
||||
elif "cfg" in metadata_dict:
|
||||
add_param_if_not_none(params, "CFG scale", metadata_dict.get("cfg"))
|
||||
|
||||
# Seed
|
||||
if "seed" in metadata_dict:
|
||||
add_param_if_not_none(params, "Seed", metadata_dict.get("seed"))
|
||||
|
||||
# Size
|
||||
if "size" in metadata_dict:
|
||||
add_param_if_not_none(params, "Size", metadata_dict.get("size"))
|
||||
|
||||
# Model info
|
||||
if "checkpoint" in metadata_dict:
|
||||
# Ensure checkpoint is a string before processing
|
||||
checkpoint = metadata_dict.get("checkpoint")
|
||||
if checkpoint is not None:
|
||||
# Get model hash
|
||||
model_hash = self.get_checkpoint_hash(checkpoint)
|
||||
|
||||
# Extract basename without path
|
||||
checkpoint_name = os.path.basename(checkpoint)
|
||||
# Remove extension if present
|
||||
checkpoint_name = os.path.splitext(checkpoint_name)[0]
|
||||
|
||||
# Add model hash if available
|
||||
if model_hash:
|
||||
params.append(
|
||||
f"Model hash: {model_hash[:10]}, Model: {checkpoint_name}"
|
||||
)
|
||||
else:
|
||||
params.append(f"Model: {checkpoint_name}")
|
||||
|
||||
# Add LoRA hashes if available
|
||||
if lora_hashes:
|
||||
lora_hash_parts = []
|
||||
for lora_name, hash_value in lora_hashes.items():
|
||||
lora_hash_parts.append(f"{lora_name}: {hash_value[:10]}")
|
||||
|
||||
if lora_hash_parts:
|
||||
params.append(f'Lora hashes: "{", ".join(lora_hash_parts)}"')
|
||||
|
||||
# Combine all parameters with commas
|
||||
metadata_parts.append(", ".join(params))
|
||||
|
||||
# Join all parts with a new line
|
||||
return "\n".join(metadata_parts)
|
||||
lines.append(", ".join(params))
|
||||
return "\n".join(lines)
|
||||
|
||||
# credit to nkchocoai
|
||||
# Add format_filename method to handle pattern substitution
|
||||
@@ -286,7 +522,12 @@ class SaveImageLM:
|
||||
key = parts[0]
|
||||
|
||||
if key == "seed" and "seed" in metadata_dict:
|
||||
filename = filename.replace(segment, str(metadata_dict.get("seed", "")))
|
||||
seed_value = metadata_dict.get("seed")
|
||||
if seed_value is not None:
|
||||
filename = filename.replace(segment, str(seed_value))
|
||||
else:
|
||||
# Fallback if seed was not captured by metadata collector
|
||||
filename = filename.replace(segment, "0")
|
||||
elif key == "width" and "size" in metadata_dict:
|
||||
size = metadata_dict.get("size", "x")
|
||||
w = size.split("x")[0] if isinstance(size, str) else size[0]
|
||||
@@ -297,12 +538,14 @@ class SaveImageLM:
|
||||
filename = filename.replace(segment, str(h))
|
||||
elif key == "pprompt" and "prompt" in metadata_dict:
|
||||
prompt = metadata_dict.get("prompt", "").replace("\n", " ")
|
||||
prompt = sanitize_folder_name(prompt)
|
||||
if len(parts) >= 2:
|
||||
length = int(parts[1])
|
||||
prompt = prompt[:length]
|
||||
filename = filename.replace(segment, prompt.strip())
|
||||
elif key == "nprompt" and "negative_prompt" in metadata_dict:
|
||||
prompt = metadata_dict.get("negative_prompt", "").replace("\n", " ")
|
||||
prompt = sanitize_folder_name(prompt)
|
||||
if len(parts) >= 2:
|
||||
length = int(parts[1])
|
||||
prompt = prompt[:length]
|
||||
@@ -316,6 +559,7 @@ class SaveImageLM:
|
||||
model = "model_unavailable"
|
||||
else:
|
||||
model = os.path.splitext(os.path.basename(model_value))[0]
|
||||
model = sanitize_folder_name(model)
|
||||
if len(parts) >= 2:
|
||||
length = int(parts[1])
|
||||
model = model[:length]
|
||||
@@ -346,6 +590,203 @@ class SaveImageLM:
|
||||
|
||||
return filename
|
||||
|
||||
@staticmethod
|
||||
def _get_cached_model_by_name(scanner, name):
|
||||
cache = getattr(scanner, "_cache", None)
|
||||
if cache is None or not name:
|
||||
return None
|
||||
|
||||
candidates = [
|
||||
name,
|
||||
os.path.basename(name),
|
||||
os.path.splitext(os.path.basename(name))[0],
|
||||
]
|
||||
for model in getattr(cache, "raw_data", []):
|
||||
file_name = model.get("file_name")
|
||||
if file_name in candidates:
|
||||
return model
|
||||
return None
|
||||
|
||||
def _build_recipe_loras(self, recipe_scanner, lora_stack):
|
||||
lora_matches = re.findall(r"<lora:([^:]+):([^>]+)>", lora_stack or "")
|
||||
lora_scanner = getattr(recipe_scanner, "_lora_scanner", None)
|
||||
loras_data = []
|
||||
base_model_counts = {}
|
||||
|
||||
for name, strength in lora_matches:
|
||||
lora_info = self._get_cached_model_by_name(lora_scanner, name)
|
||||
civitai = (lora_info or {}).get("civitai") or {}
|
||||
civitai_model = civitai.get("model") or {}
|
||||
try:
|
||||
parsed_strength = float(strength)
|
||||
except (TypeError, ValueError):
|
||||
parsed_strength = 1.0
|
||||
|
||||
loras_data.append(
|
||||
{
|
||||
"file_name": name,
|
||||
"strength": parsed_strength,
|
||||
"hash": ((lora_info or {}).get("sha256") or "").lower(),
|
||||
"modelVersionId": civitai.get("id", 0),
|
||||
"modelName": civitai_model.get("name", name) if lora_info else "",
|
||||
"modelVersionName": civitai.get("name", "") if lora_info else "",
|
||||
"isDeleted": False,
|
||||
"exclude": False,
|
||||
}
|
||||
)
|
||||
|
||||
base_model = (lora_info or {}).get("base_model")
|
||||
if base_model:
|
||||
base_model_counts[base_model] = base_model_counts.get(base_model, 0) + 1
|
||||
|
||||
return lora_matches, loras_data, base_model_counts
|
||||
|
||||
def _build_recipe_checkpoint(self, recipe_scanner, checkpoint_raw):
|
||||
if not isinstance(checkpoint_raw, str) or not checkpoint_raw.strip():
|
||||
return None
|
||||
|
||||
checkpoint_name = checkpoint_raw.strip()
|
||||
file_name = os.path.splitext(os.path.basename(checkpoint_name))[0]
|
||||
checkpoint_scanner = getattr(recipe_scanner, "_checkpoint_scanner", None)
|
||||
checkpoint_info = self._get_cached_model_by_name(
|
||||
checkpoint_scanner, checkpoint_name
|
||||
)
|
||||
|
||||
if not checkpoint_info:
|
||||
return {
|
||||
"type": "checkpoint",
|
||||
"name": checkpoint_name,
|
||||
"file_name": file_name,
|
||||
"hash": self.get_checkpoint_hash(checkpoint_name) or "",
|
||||
}
|
||||
|
||||
civitai = checkpoint_info.get("civitai") or {}
|
||||
civitai_model = civitai.get("model") or {}
|
||||
file_path = checkpoint_info.get("file_path") or checkpoint_info.get("path") or ""
|
||||
cached_file_name = (
|
||||
checkpoint_info.get("file_name")
|
||||
or (os.path.splitext(os.path.basename(file_path))[0] if file_path else "")
|
||||
or file_name
|
||||
)
|
||||
|
||||
return {
|
||||
"type": "checkpoint",
|
||||
"modelId": civitai_model.get("id", 0),
|
||||
"modelVersionId": civitai.get("id", 0),
|
||||
"name": civitai_model.get("name")
|
||||
or checkpoint_info.get("model_name")
|
||||
or checkpoint_name,
|
||||
"version": civitai.get("name", ""),
|
||||
"hash": (
|
||||
checkpoint_info.get("sha256") or checkpoint_info.get("hash") or ""
|
||||
).lower(),
|
||||
"file_name": cached_file_name,
|
||||
"modelName": civitai_model.get("name", ""),
|
||||
"modelVersionName": civitai.get("name", ""),
|
||||
"baseModel": checkpoint_info.get("base_model")
|
||||
or civitai.get("baseModel", ""),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _derive_recipe_name(lora_matches):
|
||||
recipe_name_parts = [
|
||||
f"{name.strip()}-{float(strength):.2f}" for name, strength in lora_matches[:3]
|
||||
]
|
||||
return "_".join(recipe_name_parts) or "recipe"
|
||||
|
||||
@staticmethod
|
||||
def _sync_recipe_cache(recipe_scanner, recipe_data, json_path):
|
||||
cache = getattr(recipe_scanner, "_cache", None)
|
||||
if cache is not None:
|
||||
cache.raw_data.append(recipe_data)
|
||||
cache.sorted_by_name = sorted(
|
||||
cache.raw_data, key=lambda item: item.get("title", "").lower()
|
||||
)
|
||||
cache.sorted_by_date = sorted(
|
||||
cache.raw_data,
|
||||
key=lambda item: (
|
||||
item.get("modified", item.get("created_date", 0)),
|
||||
item.get("file_path", ""),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
recipe_scanner._update_folder_metadata(cache)
|
||||
recipe_scanner._update_fts_index_for_recipe(recipe_data, "add")
|
||||
|
||||
recipe_id = str(recipe_data.get("id", ""))
|
||||
if recipe_id:
|
||||
recipe_scanner._json_path_map[recipe_id] = json_path
|
||||
persistent_cache = getattr(recipe_scanner, "_persistent_cache", None)
|
||||
if persistent_cache:
|
||||
persistent_cache.update_recipe(recipe_data, json_path)
|
||||
|
||||
def _save_image_as_recipe(self, file_path, metadata_dict):
|
||||
if not metadata_dict:
|
||||
raise ValueError("No generation metadata found")
|
||||
|
||||
recipe_scanner = ServiceRegistry.get_service_sync("recipe_scanner")
|
||||
if recipe_scanner is None:
|
||||
raise RuntimeError("Recipe scanner unavailable")
|
||||
|
||||
recipes_dir = recipe_scanner.recipes_dir
|
||||
if not recipes_dir:
|
||||
raise RuntimeError("Recipes directory unavailable")
|
||||
os.makedirs(recipes_dir, exist_ok=True)
|
||||
|
||||
recipe_id = str(uuid.uuid4())
|
||||
optimized_image, extension = ExifUtils.optimize_image(
|
||||
image_data=file_path,
|
||||
target_width=CARD_PREVIEW_WIDTH,
|
||||
format="webp",
|
||||
quality=85,
|
||||
preserve_metadata=True,
|
||||
)
|
||||
image_path = os.path.normpath(os.path.join(recipes_dir, f"{recipe_id}{extension}"))
|
||||
with open(image_path, "wb") as file_obj:
|
||||
file_obj.write(optimized_image)
|
||||
|
||||
lora_stack = metadata_dict.get("loras", "")
|
||||
lora_matches, loras_data, base_model_counts = self._build_recipe_loras(
|
||||
recipe_scanner, lora_stack
|
||||
)
|
||||
checkpoint_entry = self._build_recipe_checkpoint(
|
||||
recipe_scanner, metadata_dict.get("checkpoint")
|
||||
)
|
||||
most_common_base_model = (
|
||||
max(base_model_counts.items(), key=lambda item: item[1])[0]
|
||||
if base_model_counts
|
||||
else ""
|
||||
)
|
||||
current_time = time.time()
|
||||
recipe_data = {
|
||||
"id": recipe_id,
|
||||
"file_path": image_path,
|
||||
"title": self._derive_recipe_name(lora_matches),
|
||||
"modified": current_time,
|
||||
"created_date": current_time,
|
||||
"base_model": most_common_base_model
|
||||
or (checkpoint_entry or {}).get("baseModel", ""),
|
||||
"loras": loras_data,
|
||||
"gen_params": {
|
||||
key: value
|
||||
for key, value in metadata_dict.items()
|
||||
if key not in ["checkpoint", "loras"]
|
||||
},
|
||||
"loras_stack": lora_stack,
|
||||
"fingerprint": calculate_recipe_fingerprint(loras_data),
|
||||
}
|
||||
if checkpoint_entry:
|
||||
recipe_data["checkpoint"] = checkpoint_entry
|
||||
|
||||
json_path = os.path.normpath(
|
||||
os.path.join(recipes_dir, f"{recipe_id}.recipe.json")
|
||||
)
|
||||
with open(json_path, "w", encoding="utf-8") as file_obj:
|
||||
json.dump(recipe_data, file_obj, indent=4, ensure_ascii=False)
|
||||
|
||||
ExifUtils.append_recipe_metadata(image_path, recipe_data)
|
||||
self._sync_recipe_cache(recipe_scanner, recipe_data, json_path)
|
||||
|
||||
def save_images(
|
||||
self,
|
||||
images,
|
||||
@@ -356,9 +797,13 @@ class SaveImageLM:
|
||||
extra_pnginfo=None,
|
||||
lossless_webp=True,
|
||||
quality=100,
|
||||
webp_method=6,
|
||||
jpeg_subsampling=0,
|
||||
embed_workflow=False,
|
||||
save_with_metadata=True,
|
||||
add_counter_to_filename=True,
|
||||
save_as_recipe=False,
|
||||
add_loras_to_prompt=False,
|
||||
):
|
||||
"""Save images with metadata"""
|
||||
results = []
|
||||
@@ -367,7 +812,7 @@ class SaveImageLM:
|
||||
raw_metadata = get_metadata()
|
||||
metadata_dict = MetadataProcessor.to_dict(raw_metadata, id)
|
||||
|
||||
metadata = self.format_metadata(metadata_dict)
|
||||
metadata = self.format_metadata(metadata_dict, add_loras_to_prompt)
|
||||
|
||||
# Process filename_prefix with pattern substitution
|
||||
filename_prefix = self.format_filename(filename_prefix, metadata_dict)
|
||||
@@ -390,7 +835,7 @@ class SaveImageLM:
|
||||
img = Image.fromarray(np.clip(img, 0, 255).astype(np.uint8))
|
||||
|
||||
# Generate filename with counter if needed
|
||||
base_filename = filename
|
||||
base_filename = filename.replace("%batch_num%", str(i))
|
||||
if add_counter_to_filename:
|
||||
# Use counter + i to ensure unique filenames for all images in batch
|
||||
current_counter = counter + i
|
||||
@@ -409,15 +854,14 @@ class SaveImageLM:
|
||||
elif file_format == "jpeg":
|
||||
file = base_filename + ".jpg"
|
||||
file_extension = ".jpg"
|
||||
save_kwargs = {"quality": quality, "optimize": True}
|
||||
save_kwargs = {"quality": quality, "optimize": True, "subsampling": jpeg_subsampling}
|
||||
elif file_format == "webp":
|
||||
file = base_filename + ".webp"
|
||||
file_extension = ".webp"
|
||||
# Add optimization param to control performance
|
||||
save_kwargs = {
|
||||
"quality": quality,
|
||||
"lossless": lossless_webp,
|
||||
"method": 0,
|
||||
"method": webp_method,
|
||||
}
|
||||
else:
|
||||
raise ValueError(f"Unsupported file format: {file_format}")
|
||||
@@ -477,6 +921,14 @@ class SaveImageLM:
|
||||
|
||||
img.save(file_path, format="WEBP", **save_kwargs)
|
||||
|
||||
if save_as_recipe:
|
||||
try:
|
||||
self._save_image_as_recipe(file_path, metadata_dict)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to save image as recipe: %s", e, exc_info=True
|
||||
)
|
||||
|
||||
results.append(
|
||||
{"filename": file, "subfolder": subfolder, "type": self.type}
|
||||
)
|
||||
@@ -496,9 +948,13 @@ class SaveImageLM:
|
||||
extra_pnginfo=None,
|
||||
lossless_webp=True,
|
||||
quality=100,
|
||||
webp_method=6,
|
||||
jpeg_subsampling=0,
|
||||
embed_workflow=False,
|
||||
save_with_metadata=True,
|
||||
add_counter_to_filename=True,
|
||||
save_as_recipe=False,
|
||||
add_loras_to_prompt=False,
|
||||
):
|
||||
"""Process and save image with metadata"""
|
||||
# Make sure the output directory exists
|
||||
@@ -524,9 +980,13 @@ class SaveImageLM:
|
||||
extra_pnginfo,
|
||||
lossless_webp,
|
||||
quality,
|
||||
webp_method,
|
||||
jpeg_subsampling,
|
||||
embed_workflow,
|
||||
save_with_metadata,
|
||||
add_counter_to_filename,
|
||||
save_as_recipe,
|
||||
add_loras_to_prompt,
|
||||
)
|
||||
|
||||
return {
|
||||
|
||||
+26
-8
@@ -1,10 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from ..services.wildcard_service import contains_dynamic_syntax, get_wildcard_service
|
||||
|
||||
|
||||
class TextLM:
|
||||
"""A simple text node with autocomplete support."""
|
||||
|
||||
NAME = "Text (LoraManager)"
|
||||
CATEGORY = "Lora Manager/utils"
|
||||
DESCRIPTION = (
|
||||
"A simple text input node with autocomplete support for tags and styles."
|
||||
"A simple text input node with autocomplete support for tags, styles, and wildcard expansion."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -15,8 +20,17 @@ class TextLM:
|
||||
"AUTOCOMPLETE_TEXT_PROMPT,STRING",
|
||||
{
|
||||
"widgetType": "AUTOCOMPLETE_TEXT_PROMPT",
|
||||
"placeholder": "Enter text... /char, /artist for quick tag search",
|
||||
"tooltip": "The text output.",
|
||||
"placeholder": "Enter text... /character, /artist, /wildcard for quick search",
|
||||
"tooltip": "The text output. Wildcard references inserted with /wildcard are expanded at runtime.",
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": {
|
||||
"seed": (
|
||||
"INT",
|
||||
{
|
||||
"forceInput": True,
|
||||
"tooltip": "Optional seed for wildcard generation. Leave unconnected for non-deterministic wildcard expansion.",
|
||||
},
|
||||
),
|
||||
},
|
||||
@@ -24,10 +38,14 @@ class TextLM:
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("STRING",)
|
||||
OUTPUT_TOOLTIPS = (
|
||||
"The text output.",
|
||||
)
|
||||
OUTPUT_TOOLTIPS = ("The text output.",)
|
||||
FUNCTION = "process"
|
||||
|
||||
def process(self, text: str):
|
||||
return (text,)
|
||||
@classmethod
|
||||
def IS_CHANGED(cls, text: str, seed: int | None = None):
|
||||
if contains_dynamic_syntax(text) and seed is None:
|
||||
return float("NaN")
|
||||
return False
|
||||
|
||||
def process(self, text: str, seed: int | None = None):
|
||||
return (get_wildcard_service().expand_text(text, seed=seed),)
|
||||
|
||||
@@ -76,6 +76,9 @@ class TriggerWordToggleLM:
|
||||
# Filter out empty strings and return as set
|
||||
return set(word for word in words if word)
|
||||
|
||||
def _group_has_child_items(self, item):
|
||||
return isinstance(item, dict) and isinstance(item.get("items"), list)
|
||||
|
||||
def process_trigger_words(
|
||||
self,
|
||||
id,
|
||||
@@ -112,7 +115,11 @@ class TriggerWordToggleLM:
|
||||
|
||||
if isinstance(trigger_data, list):
|
||||
if group_mode:
|
||||
if allow_strength_adjustment:
|
||||
if any(self._group_has_child_items(item) for item in trigger_data):
|
||||
filtered_groups = self._process_group_items(
|
||||
trigger_data, allow_strength_adjustment
|
||||
)
|
||||
elif allow_strength_adjustment:
|
||||
parsed_items = [
|
||||
self._parse_trigger_item(
|
||||
item, allow_strength_adjustment
|
||||
@@ -174,6 +181,41 @@ class TriggerWordToggleLM:
|
||||
|
||||
return (filtered_triggers,)
|
||||
|
||||
def _process_group_items(self, trigger_data, allow_strength_adjustment):
|
||||
filtered_groups = []
|
||||
|
||||
for item in trigger_data:
|
||||
group = self._parse_trigger_item(item, allow_strength_adjustment)
|
||||
if not group["text"] or not group["active"]:
|
||||
continue
|
||||
|
||||
raw_items = item.get("items") if isinstance(item, dict) else None
|
||||
if isinstance(raw_items, list):
|
||||
active_items = []
|
||||
for raw_item in raw_items:
|
||||
child = self._parse_trigger_item(
|
||||
raw_item, allow_strength_adjustment=False
|
||||
)
|
||||
if child["text"] and child["active"]:
|
||||
active_items.append(child["text"])
|
||||
|
||||
if not active_items:
|
||||
continue
|
||||
|
||||
group_text = ", ".join(active_items)
|
||||
else:
|
||||
group_text = group["text"]
|
||||
|
||||
filtered_groups.append(
|
||||
self._format_word_output(
|
||||
group_text,
|
||||
group["strength"],
|
||||
allow_strength_adjustment,
|
||||
)
|
||||
)
|
||||
|
||||
return filtered_groups
|
||||
|
||||
def _parse_trigger_item(self, item, allow_strength_adjustment):
|
||||
text = (item.get("text") or "").strip()
|
||||
active = bool(item.get("active", False))
|
||||
|
||||
+27
-6
@@ -1,12 +1,27 @@
|
||||
import logging
|
||||
import os
|
||||
from typing import List, Tuple
|
||||
import comfy.sd # type: ignore
|
||||
from typing import Any, List, Tuple
|
||||
import comfy.sd # pyright: ignore[reportMissingImports]
|
||||
from ..utils.utils import get_checkpoint_info_absolute, _format_model_name_for_comfyui
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _reload_gguf_unet(
|
||||
unet_path: str, weight_dtype: str, disable_dynamic: bool = False
|
||||
) -> object:
|
||||
"""Reload a GGUF diffusion model from disk (cached_patcher_init factory).
|
||||
|
||||
Mirrors the GGUF branch of UNETLoaderLM.load_unet so ModelPatcher
|
||||
deepclone/dynamic machinery can rebuild GGUF models with the correct
|
||||
GGMLOps. ``disable_dynamic`` is accepted for signature compatibility
|
||||
with core ComfyUI loaders.
|
||||
"""
|
||||
loader = UNETLoaderLM()
|
||||
model, = loader._load_gguf_unet(unet_path, unet_path, weight_dtype)
|
||||
return model
|
||||
|
||||
|
||||
class UNETLoaderLM:
|
||||
"""UNET Loader with support for extra folder paths
|
||||
|
||||
@@ -19,9 +34,9 @@ class UNETLoaderLM:
|
||||
CATEGORY = "Lora Manager/loaders"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
def INPUT_TYPES(cls):
|
||||
# Get list of unet names from scanner (includes extra folder paths)
|
||||
unet_names = s._get_unet_names()
|
||||
unet_names = cls._get_unet_names()
|
||||
return {
|
||||
"required": {
|
||||
"unet_name": (
|
||||
@@ -90,7 +105,7 @@ class UNETLoaderLM:
|
||||
logger.error(f"Error getting unet names: {e}")
|
||||
return []
|
||||
|
||||
def load_unet(self, unet_name: str, weight_dtype: str) -> Tuple:
|
||||
def load_unet(self, unet_name: str, weight_dtype: str) -> Tuple[Any, ...]:
|
||||
"""Load a diffusion model by name, supporting extra folder paths
|
||||
|
||||
Args:
|
||||
@@ -133,7 +148,7 @@ class UNETLoaderLM:
|
||||
|
||||
def _load_gguf_unet(
|
||||
self, unet_path: str, unet_name: str, weight_dtype: str
|
||||
) -> Tuple:
|
||||
) -> Tuple[Any, ...]:
|
||||
"""Load a GGUF format diffusion model
|
||||
|
||||
Args:
|
||||
@@ -196,6 +211,12 @@ class UNETLoaderLM:
|
||||
# Wrap with GGUFModelPatcher
|
||||
model = GGUFModelPatcher.clone(model)
|
||||
|
||||
# Register a reload factory so the MODEL carries its source path
|
||||
# (cached_patcher_init) like core ComfyUI loaders do — required
|
||||
# for model-name extraction downstream and for ModelPatcher
|
||||
# deepclone/dynamic machinery.
|
||||
model.cached_patcher_init = (_reload_gguf_unet, (unet_path, weight_dtype))
|
||||
|
||||
return (model,)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
+48
-6
@@ -1,3 +1,6 @@
|
||||
from typing import Any
|
||||
|
||||
|
||||
class AnyType(str):
|
||||
"""A special class that is always equal in not equal comparisons. Credit to pythongosssss"""
|
||||
|
||||
@@ -6,7 +9,7 @@ class AnyType(str):
|
||||
|
||||
|
||||
# Credit to Regis Gaughan, III (rgthree)
|
||||
class FlexibleOptionalInputType(dict):
|
||||
class FlexibleOptionalInputType(dict[str, Any]):
|
||||
"""A special class to make flexible nodes that pass data to our python handlers.
|
||||
|
||||
Enables both flexible/dynamic input types (like for Any Switch) or a dynamic number of inputs
|
||||
@@ -23,6 +26,7 @@ class FlexibleOptionalInputType(dict):
|
||||
"""
|
||||
|
||||
def __init__(self, type):
|
||||
super().__init__()
|
||||
self.type = type
|
||||
|
||||
def __getitem__(self, key):
|
||||
@@ -36,19 +40,57 @@ any_type = AnyType("*")
|
||||
|
||||
# Common methods extracted from lora_loader.py and lora_stacker.py
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
import copy
|
||||
import sys
|
||||
import folder_paths # type: ignore
|
||||
import folder_paths # pyright: ignore[reportMissingImports]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_lora_syntax_format():
|
||||
try:
|
||||
from ..services.settings_manager import get_settings_manager
|
||||
return get_settings_manager().get("lora_syntax_format", "legacy")
|
||||
except Exception:
|
||||
return "legacy"
|
||||
|
||||
|
||||
def apply_lora_syntax_format(name):
|
||||
fmt = get_lora_syntax_format()
|
||||
if fmt == "legacy":
|
||||
return name.replace("\\", "/").rstrip("/").split("/")[-1]
|
||||
return name
|
||||
|
||||
|
||||
def extract_lora_name(lora_path):
|
||||
"""Extract the lora name from a lora path (e.g., 'IL\\aorunIllstrious.safetensors' -> 'aorunIllstrious')"""
|
||||
# Get the basename without extension
|
||||
basename = os.path.basename(lora_path)
|
||||
return os.path.splitext(basename)[0]
|
||||
normalized = lora_path.replace("\\", "/")
|
||||
basename = os.path.basename(normalized)
|
||||
name_no_ext = os.path.splitext(basename)[0]
|
||||
dirname = os.path.dirname(normalized)
|
||||
if dirname and dirname not in (".", "/") and not normalized.startswith("/"):
|
||||
return apply_lora_syntax_format(f"{dirname}/{name_no_ext}")
|
||||
return apply_lora_syntax_format(name_no_ext)
|
||||
|
||||
|
||||
def parse_lora_syntax(text: str) -> list[dict[str, Any]]:
|
||||
"""Parse <lora:name:strength> syntax from text input into a list of dicts.
|
||||
|
||||
Each entry contains: name, model_strength, clip_strength.
|
||||
Supports both ``<lora:name:strength>`` and ``<lora:name:model_strength:clip_strength>``.
|
||||
"""
|
||||
pattern = r"<lora:([^:>]+):([^:>]+)(?::([^:>]+))?>"
|
||||
matches = re.findall(pattern, text, re.IGNORECASE)
|
||||
loras = []
|
||||
for match in matches:
|
||||
model_strength = float(match[1])
|
||||
loras.append({
|
||||
"name": match[0],
|
||||
"model_strength": model_strength,
|
||||
"clip_strength": float(match[2]) if match[2] else model_strength,
|
||||
})
|
||||
return loras
|
||||
|
||||
|
||||
def get_loras_list(kwargs):
|
||||
|
||||
@@ -1,10 +1,22 @@
|
||||
import folder_paths # type: ignore
|
||||
from ..utils.utils import get_lora_info
|
||||
import os
|
||||
from ..utils.utils import get_lora_info_absolute
|
||||
from ..config import config
|
||||
from .utils import FlexibleOptionalInputType, any_type, get_loras_list
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _relpath_within_loras(abs_path):
|
||||
"""Return abs_path relative to the first matching lora root, or basename as fallback."""
|
||||
all_roots = list(config.loras_roots or []) + list(config.extra_loras_roots or [])
|
||||
for root in all_roots:
|
||||
try:
|
||||
return os.path.relpath(abs_path, root)
|
||||
except ValueError:
|
||||
continue
|
||||
return os.path.basename(abs_path)
|
||||
|
||||
class WanVideoLoraSelectLM:
|
||||
NAME = "WanVideo Lora Select (LoraManager)"
|
||||
CATEGORY = "Lora Manager/stackers"
|
||||
@@ -56,13 +68,13 @@ class WanVideoLoraSelectLM:
|
||||
clip_strength = float(lora.get('clipStrength', model_strength))
|
||||
|
||||
# Get lora path and trigger words
|
||||
lora_path, trigger_words = get_lora_info(lora_name)
|
||||
lora_path, trigger_words = get_lora_info_absolute(lora_name)
|
||||
|
||||
# Create lora item for WanVideo format
|
||||
lora_item = {
|
||||
"path": folder_paths.get_full_path("loras", lora_path),
|
||||
"path": lora_path,
|
||||
"strength": model_strength,
|
||||
"name": lora_path.split(".")[0],
|
||||
"name": os.path.splitext(_relpath_within_loras(lora_path))[0],
|
||||
"blocks": selected_blocks,
|
||||
"layer_filter": layer_filter,
|
||||
"low_mem_load": low_mem_load,
|
||||
|
||||
@@ -1,11 +1,23 @@
|
||||
import folder_paths # type: ignore
|
||||
from ..utils.utils import get_lora_info
|
||||
import os
|
||||
from ..utils.utils import get_lora_info_absolute
|
||||
from ..config import config
|
||||
from .utils import any_type
|
||||
import logging
|
||||
|
||||
# 初始化日志记录器
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _relpath_within_loras(abs_path):
|
||||
"""Return abs_path relative to the first matching lora root, or basename as fallback."""
|
||||
all_roots = list(config.loras_roots or []) + list(config.extra_loras_roots or [])
|
||||
for root in all_roots:
|
||||
try:
|
||||
return os.path.relpath(abs_path, root)
|
||||
except ValueError:
|
||||
continue
|
||||
return os.path.basename(abs_path)
|
||||
|
||||
# 定义新节点的类
|
||||
class WanVideoLoraTextSelectLM:
|
||||
# 节点在UI中显示的名称
|
||||
@@ -87,12 +99,12 @@ class WanVideoLoraTextSelectLM:
|
||||
else:
|
||||
continue
|
||||
|
||||
lora_path, trigger_words = get_lora_info(lora_name_raw)
|
||||
lora_path, trigger_words = get_lora_info_absolute(lora_name_raw)
|
||||
|
||||
lora_item = {
|
||||
"path": folder_paths.get_full_path("loras", lora_path),
|
||||
"path": lora_path,
|
||||
"strength": model_strength,
|
||||
"name": lora_path.split(".")[0],
|
||||
"name": os.path.splitext(_relpath_within_loras(lora_path))[0],
|
||||
"blocks": selected_blocks,
|
||||
"layer_filter": layer_filter,
|
||||
"low_mem_load": low_mem_load,
|
||||
|
||||
+78
-9
@@ -1,3 +1,7 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
"""Base classes for recipe parsers."""
|
||||
|
||||
import json
|
||||
@@ -7,7 +11,7 @@ import re
|
||||
from typing import Dict, List, Any, Optional, Tuple
|
||||
from abc import ABC, abstractmethod
|
||||
from ..config import config
|
||||
from ..utils.constants import VALID_LORA_TYPES
|
||||
from ..utils.constants import VALID_LORA_TYPES, VALID_CHECKPOINT_SUB_TYPES
|
||||
from ..utils.civitai_utils import rewrite_preview_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -38,7 +42,7 @@ class RecipeMetadataParser(ABC):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
async def populate_lora_from_civitai(lora_entry: Dict[str, Any], civitai_info_tuple: Tuple[Dict[str, Any], Optional[str]],
|
||||
async def populate_lora_from_civitai(lora_entry: Dict[str, Any], civitai_info_tuple: Tuple[Dict[str, Any] | None, str | None] | Dict[str, Any],
|
||||
recipe_scanner=None, base_model_counts=None, hash_value=None) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Populate a lora entry with information from Civitai API response
|
||||
@@ -58,9 +62,52 @@ class RecipeMetadataParser(ABC):
|
||||
civitai_info, error_msg = civitai_info_tuple if isinstance(civitai_info_tuple, tuple) else (civitai_info_tuple, None)
|
||||
|
||||
if not civitai_info or error_msg == "Model not found":
|
||||
# Model not found or deleted
|
||||
lora_entry['isDeleted'] = True
|
||||
lora_entry['thumbnailUrl'] = '/loras_static/images/no-preview.png'
|
||||
# CivitAI may fail to resolve a hash that is still being
|
||||
# computed (known CivitAI issue). Before marking as deleted,
|
||||
# try to reconcile with a local model that has the same
|
||||
# filename and matching AutoV3 hash.
|
||||
reconciled = False
|
||||
file_name = lora_entry.get("file_name")
|
||||
if file_name and recipe_scanner and hash_value:
|
||||
lora_scanner = getattr(recipe_scanner, "_lora_scanner", None)
|
||||
if lora_scanner:
|
||||
try:
|
||||
# Local import to avoid circular dependency:
|
||||
# base.py → file_utils → settings_manager → ...
|
||||
# → recipe_scanner → enrichment → base.py
|
||||
from ..utils.file_utils import calculate_autov3 # fmt: skip
|
||||
cache = await lora_scanner.get_cached_data()
|
||||
for item in getattr(cache, "raw_data", []):
|
||||
if item.get("file_name") == file_name:
|
||||
local_path = item.get("file_path")
|
||||
if local_path and os.path.exists(local_path):
|
||||
local_autov3 = calculate_autov3(local_path)
|
||||
if local_autov3 and local_autov3 == hash_value:
|
||||
lora_entry["existsLocally"] = True
|
||||
lora_entry["localPath"] = local_path
|
||||
lora_entry["hash"] = item.get("sha256", hash_value)
|
||||
if "preview_url" in item:
|
||||
lora_entry["thumbnailUrl"] = config.get_preview_static_url(item["preview_url"])
|
||||
civ = item.get("civitai") or {}
|
||||
if isinstance(civ, dict):
|
||||
if civ.get("id") is not None:
|
||||
lora_entry["id"] = civ["id"]
|
||||
if civ.get("modelId") is not None:
|
||||
lora_entry["modelId"] = civ["modelId"]
|
||||
if civ.get("name"):
|
||||
lora_entry["version"] = civ["name"]
|
||||
# model_name is the CivitAI model display
|
||||
# name stored directly in the cache column.
|
||||
cached_model_name = item.get("model_name")
|
||||
if cached_model_name:
|
||||
lora_entry["name"] = cached_model_name
|
||||
reconciled = True
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
if not reconciled:
|
||||
lora_entry['isDeleted'] = True
|
||||
lora_entry['thumbnailUrl'] = '/loras_static/images/no-preview.png'
|
||||
return lora_entry
|
||||
|
||||
# Get model type and validate
|
||||
@@ -132,10 +179,18 @@ class RecipeMetadataParser(ABC):
|
||||
lora_entry['localPath'] = local_path
|
||||
lora_entry['file_name'] = os.path.splitext(os.path.basename(local_path))[0]
|
||||
|
||||
# Get thumbnail from local preview if available
|
||||
# Get thumbnail from local preview if available.
|
||||
# Match the cache item by local path first (get_path_by_hash
|
||||
# cascade: 10-char autov2 / 12-char autov3), then by hash.
|
||||
lora_cache = await lora_scanner.get_cached_data()
|
||||
lora_item = next((item for item in lora_cache.raw_data
|
||||
if item['sha256'].lower() == lora_entry['hash'].lower()), None)
|
||||
h = (lora_entry.get("hash") or "").lower()
|
||||
lora_item = next((item for item in lora_cache.raw_data
|
||||
if (item.get("file_path") or "") == local_path), None)
|
||||
if lora_item is None:
|
||||
lora_item = next((item for item in lora_cache.raw_data
|
||||
if (item.get("sha256") or "").lower() == h
|
||||
or (item.get("autov3") or "").lower() == h
|
||||
or (item.get("sha256") or "")[:10].lower() == h), None)
|
||||
if lora_item and 'preview_url' in lora_item:
|
||||
lora_entry['thumbnailUrl'] = config.get_preview_static_url(lora_item['preview_url'])
|
||||
except Exception as e:
|
||||
@@ -151,7 +206,7 @@ class RecipeMetadataParser(ABC):
|
||||
return lora_entry
|
||||
|
||||
@staticmethod
|
||||
async def populate_checkpoint_from_civitai(checkpoint: Dict[str, Any], civitai_info: Dict[str, Any]) -> Dict[str, Any]:
|
||||
async def populate_checkpoint_from_civitai(checkpoint: Dict[str, Any], civitai_info: Dict[str, Any] | Tuple[Dict[str, Any] | None, str | None] | None) -> Dict[str, Any]:
|
||||
"""
|
||||
Populate checkpoint information from Civitai API response
|
||||
|
||||
@@ -173,6 +228,20 @@ class RecipeMetadataParser(ABC):
|
||||
checkpoint['isDeleted'] = True
|
||||
return checkpoint
|
||||
|
||||
# Validate that the model type is actually a checkpoint.
|
||||
# Unlike populate_lora_from_civitai which has this check,
|
||||
# this function was missing type validation — allowing LoRA
|
||||
# version data to be saved as the recipe's checkpoint when the
|
||||
# wrong version ID was passed downstream (fixed in v2.7+).
|
||||
model_type = civitai_data.get('model', {}).get('type', '').lower()
|
||||
if model_type not in VALID_CHECKPOINT_SUB_TYPES:
|
||||
logger.warning(
|
||||
f"Cannot populate checkpoint: model version {civitai_data.get('id')} "
|
||||
f"has type '{model_type}', expected one of {VALID_CHECKPOINT_SUB_TYPES}. "
|
||||
f"Skipping checkpoint enrichment."
|
||||
)
|
||||
return checkpoint
|
||||
|
||||
if 'model' in civitai_data and 'name' in civitai_data['model']:
|
||||
checkpoint['name'] = civitai_data['model']['name']
|
||||
|
||||
|
||||
@@ -13,4 +13,5 @@ GEN_PARAM_KEYS = [
|
||||
'seed',
|
||||
'size',
|
||||
'clip_skip',
|
||||
'denoising_strength',
|
||||
]
|
||||
|
||||
+81
-51
@@ -1,11 +1,15 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
import logging
|
||||
import json
|
||||
import re
|
||||
import os
|
||||
from typing import Any, Dict, Optional
|
||||
from .merger import GenParamsMerger
|
||||
from .base import RecipeMetadataParser
|
||||
from ..services.metadata_service import get_default_metadata_provider
|
||||
from ..utils.civitai_utils import extract_civitai_image_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -16,54 +20,65 @@ class RecipeEnricher:
|
||||
async def enrich_recipe(
|
||||
recipe: Dict[str, Any],
|
||||
civitai_client: Any,
|
||||
request_params: Optional[Dict[str, Any]] = None
|
||||
request_params: Optional[Dict[str, Any]] = None,
|
||||
prefetched_civitai_meta_raw: Optional[Dict[str, Any]] = None,
|
||||
prefetched_model_version_id: Optional[int] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
Enrich a recipe dictionary in-place with metadata from Civitai and embedded params.
|
||||
|
||||
|
||||
Args:
|
||||
recipe: The recipe dictionary to enrich. Must have 'gen_params' initialized.
|
||||
civitai_client: Authenticated Civitai client instance.
|
||||
request_params: (Optional) Parameters from a user request (e.g. import).
|
||||
|
||||
prefetched_civitai_meta_raw: (Optional) Pre-fetched raw meta from Civitai
|
||||
get_image_info, avoiding a duplicate API call.
|
||||
prefetched_model_version_id: (Optional) Pre-fetched model version ID.
|
||||
|
||||
Returns:
|
||||
bool: True if the recipe was modified, False otherwise.
|
||||
"""
|
||||
updated = False
|
||||
gen_params = recipe.get("gen_params", {})
|
||||
|
||||
# 1. Fetch Civitai Info if available
|
||||
|
||||
# 1. Obtain Civitai metadata
|
||||
civitai_meta = None
|
||||
model_version_id = None
|
||||
|
||||
source_url = recipe.get("source_url") or recipe.get("source_path", "")
|
||||
|
||||
# Check if it's a Civitai image URL
|
||||
image_id_match = re.search(r'civitai\.com/images/(\d+)', str(source_url))
|
||||
if image_id_match:
|
||||
image_id = image_id_match.group(1)
|
||||
try:
|
||||
image_info = await civitai_client.get_image_info(image_id)
|
||||
if image_info:
|
||||
# Handle nested meta often found in Civitai API responses
|
||||
raw_meta = image_info.get("meta")
|
||||
if isinstance(raw_meta, dict):
|
||||
if "meta" in raw_meta and isinstance(raw_meta["meta"], dict):
|
||||
civitai_meta = raw_meta["meta"]
|
||||
else:
|
||||
civitai_meta = raw_meta
|
||||
|
||||
model_version_id = image_info.get("modelVersionId")
|
||||
|
||||
# If not at top level, check resources in meta
|
||||
if not model_version_id and civitai_meta:
|
||||
resources = civitai_meta.get("civitaiResources", [])
|
||||
for res in resources:
|
||||
if res.get("type") == "checkpoint":
|
||||
model_version_id = res.get("modelVersionId")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to fetch Civitai image info: {e}")
|
||||
model_version_id = prefetched_model_version_id
|
||||
|
||||
source_path = recipe.get("source_path", "")
|
||||
|
||||
if prefetched_civitai_meta_raw is not None:
|
||||
raw_meta = prefetched_civitai_meta_raw
|
||||
if isinstance(raw_meta, dict):
|
||||
if "meta" in raw_meta and isinstance(raw_meta["meta"], dict):
|
||||
civitai_meta = raw_meta["meta"]
|
||||
else:
|
||||
civitai_meta = raw_meta
|
||||
else:
|
||||
image_id = extract_civitai_image_id(str(source_path))
|
||||
if image_id:
|
||||
try:
|
||||
image_info = await civitai_client.get_image_info(
|
||||
image_id, source_url=str(source_path)
|
||||
)
|
||||
if image_info:
|
||||
raw_meta = image_info.get("meta")
|
||||
if isinstance(raw_meta, dict):
|
||||
if "meta" in raw_meta and isinstance(raw_meta["meta"], dict):
|
||||
civitai_meta = raw_meta["meta"]
|
||||
else:
|
||||
civitai_meta = raw_meta
|
||||
|
||||
model_version_id = image_info.get("modelVersionId")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to fetch Civitai image info: {e}")
|
||||
|
||||
if not model_version_id and civitai_meta:
|
||||
resources = civitai_meta.get("civitaiResources", [])
|
||||
for res in resources:
|
||||
if res.get("type") == "checkpoint":
|
||||
model_version_id = res.get("modelVersionId")
|
||||
break
|
||||
|
||||
# 2. Merge Parameters
|
||||
# Priority: request_params > civitai_meta > embedded (existing gen_params)
|
||||
@@ -179,27 +194,42 @@ class RecipeEnricher:
|
||||
existing_cp = recipe.get("checkpoint")
|
||||
if existing_cp is None:
|
||||
existing_cp = {}
|
||||
|
||||
# Extract baseModel from raw civitai_info before populate_checkpoint_from_civitai
|
||||
# (populate may reject non-checkpoint types and lose this data)
|
||||
base_model_from_civitai: str = ""
|
||||
if isinstance(civitai_info, dict):
|
||||
base_model_from_civitai = civitai_info.get("baseModel", "") or ""
|
||||
elif isinstance(civitai_info, tuple) and len(civitai_info) > 0 and isinstance(civitai_info[0], dict):
|
||||
base_model_from_civitai = civitai_info[0].get("baseModel", "") or ""
|
||||
|
||||
checkpoint_data = await RecipeMetadataParser.populate_checkpoint_from_civitai(existing_cp, civitai_info)
|
||||
# 1. First, resolve base_model using full data before we format it away
|
||||
|
||||
# 1. Resolve base_model from checkpoint_data first, then fall back to raw civitai_info
|
||||
current_base_model = recipe.get("base_model")
|
||||
resolved_base_model = checkpoint_data.get("baseModel")
|
||||
resolved_base_model = checkpoint_data.get("baseModel") or base_model_from_civitai
|
||||
if resolved_base_model:
|
||||
# Update if empty OR if it matches our generic prefix but is less specific
|
||||
is_generic = not current_base_model or current_base_model.lower() in ["flux", "sdxl", "sd15"]
|
||||
if is_generic and resolved_base_model != current_base_model:
|
||||
recipe["base_model"] = resolved_base_model
|
||||
|
||||
# 2. Format according to requirements: type, modelId, modelVersionId, modelName, modelVersionName
|
||||
formatted_checkpoint = {
|
||||
"type": "checkpoint",
|
||||
"modelId": checkpoint_data.get("modelId"),
|
||||
"modelVersionId": checkpoint_data.get("id") or checkpoint_data.get("modelVersionId"),
|
||||
"modelName": checkpoint_data.get("name"), # In base.py, 'name' is populated from civitai_data['model']['name']
|
||||
"modelVersionName": checkpoint_data.get("version") # In base.py, 'version' is populated from civitai_data['name']
|
||||
}
|
||||
# Remove None values
|
||||
recipe["checkpoint"] = {k: v for k, v in formatted_checkpoint.items() if v is not None}
|
||||
|
||||
|
||||
# 2. Only format and save checkpoint if it has real data (not just type after type rejection)
|
||||
has_checkpoint_data = any([
|
||||
checkpoint_data.get("modelId"),
|
||||
checkpoint_data.get("id") or checkpoint_data.get("modelVersionId"),
|
||||
checkpoint_data.get("name"),
|
||||
checkpoint_data.get("version"),
|
||||
])
|
||||
if has_checkpoint_data:
|
||||
formatted_checkpoint = {
|
||||
"type": "checkpoint",
|
||||
"modelId": checkpoint_data.get("modelId"),
|
||||
"modelVersionId": checkpoint_data.get("id") or checkpoint_data.get("modelVersionId"),
|
||||
"modelName": checkpoint_data.get("name"),
|
||||
"modelVersionName": checkpoint_data.get("version"),
|
||||
}
|
||||
recipe["checkpoint"] = {k: v for k, v in formatted_checkpoint.items() if v is not None}
|
||||
|
||||
return True
|
||||
else:
|
||||
# Fallback to name extraction if we don't already have one
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Factory for creating recipe metadata parsers."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
from .parsers import (
|
||||
RecipeFormatParser,
|
||||
ComfyMetadataParser,
|
||||
@@ -31,7 +32,8 @@ class RecipeParserFactory:
|
||||
# First, try CivitaiApiMetadataParser for dict input
|
||||
if isinstance(metadata, dict):
|
||||
try:
|
||||
if CivitaiApiMetadataParser().is_metadata_matching(metadata):
|
||||
user_comment: Any = metadata
|
||||
if CivitaiApiMetadataParser().is_metadata_matching(user_comment):
|
||||
return CivitaiApiMetadataParser()
|
||||
except Exception as e:
|
||||
logger.debug(f"CivitaiApiMetadataParser check failed: {e}")
|
||||
|
||||
+29
-46
@@ -1,27 +1,33 @@
|
||||
from typing import Any, Dict, Optional
|
||||
import logging
|
||||
|
||||
from .constants import GEN_PARAM_KEYS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GenParamsMerger:
|
||||
"""Utility to merge generation parameters from multiple sources with priority."""
|
||||
|
||||
ALLOWED_KEYS = set(GEN_PARAM_KEYS)
|
||||
|
||||
BLACKLISTED_KEYS = {
|
||||
"id", "url", "userId", "username", "createdAt", "updatedAt", "hash", "meta",
|
||||
"draft", "extra", "width", "height", "process", "quantity", "workflow",
|
||||
"baseModel", "resources", "disablePoi", "aspectRatio", "Created Date",
|
||||
"experimental", "civitaiResources", "civitai_resources", "Civitai resources",
|
||||
"modelVersionId", "modelId", "hashes", "Model", "Model hash", "checkpoint_hash",
|
||||
"checkpoint", "checksum", "model_checksum"
|
||||
"checkpoint", "checksum", "model_checksum", "raw_metadata",
|
||||
}
|
||||
|
||||
|
||||
NORMALIZATION_MAPPING = {
|
||||
# Civitai specific
|
||||
"cfg": "cfg_scale",
|
||||
"cfgScale": "cfg_scale",
|
||||
"clipSkip": "clip_skip",
|
||||
"negativePrompt": "negative_prompt",
|
||||
# Case variations
|
||||
"Sampler": "sampler",
|
||||
"sampler_name": "sampler",
|
||||
"scheduler": "sampler",
|
||||
"Steps": "steps",
|
||||
"Seed": "seed",
|
||||
"Size": "size",
|
||||
@@ -36,63 +42,40 @@ class GenParamsMerger:
|
||||
def merge(
|
||||
request_params: Optional[Dict[str, Any]] = None,
|
||||
civitai_meta: Optional[Dict[str, Any]] = None,
|
||||
embedded_metadata: Optional[Dict[str, Any]] = None
|
||||
embedded_metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Merge generation parameters from three sources.
|
||||
|
||||
Priority: request_params > civitai_meta > embedded_metadata
|
||||
|
||||
Args:
|
||||
request_params: Params provided directly in the import request
|
||||
civitai_meta: Params from Civitai Image API 'meta' field
|
||||
embedded_metadata: Params extracted from image EXIF/embedded metadata
|
||||
|
||||
Returns:
|
||||
Merged parameters dictionary
|
||||
"""
|
||||
result = {}
|
||||
|
||||
# 1. Start with embedded metadata (lowest priority)
|
||||
Priority: request_params > civitai_meta > embedded_metadata
|
||||
"""
|
||||
result: Dict[str, Any] = {}
|
||||
|
||||
if embedded_metadata:
|
||||
# If it's a full recipe metadata, we use its gen_params
|
||||
if "gen_params" in embedded_metadata and isinstance(embedded_metadata["gen_params"], dict):
|
||||
if "gen_params" in embedded_metadata and isinstance(
|
||||
embedded_metadata["gen_params"], dict
|
||||
):
|
||||
GenParamsMerger._update_normalized(result, embedded_metadata["gen_params"])
|
||||
else:
|
||||
# Otherwise assume the dict itself contains gen_params
|
||||
GenParamsMerger._update_normalized(result, embedded_metadata)
|
||||
|
||||
# 2. Layer Civitai meta (medium priority)
|
||||
if civitai_meta:
|
||||
GenParamsMerger._update_normalized(result, civitai_meta)
|
||||
|
||||
# 3. Layer request params (highest priority)
|
||||
if request_params:
|
||||
GenParamsMerger._update_normalized(result, request_params)
|
||||
|
||||
# Filter out blacklisted keys and also the original camelCase keys if they were normalized
|
||||
final_result = {}
|
||||
for k, v in result.items():
|
||||
if k in GenParamsMerger.BLACKLISTED_KEYS:
|
||||
continue
|
||||
if k in GenParamsMerger.NORMALIZATION_MAPPING:
|
||||
continue
|
||||
final_result[k] = v
|
||||
|
||||
return final_result
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _update_normalized(target: Dict[str, Any], source: Dict[str, Any]) -> None:
|
||||
"""Update target dict with normalized keys from source."""
|
||||
for k, v in source.items():
|
||||
normalized_key = GenParamsMerger.NORMALIZATION_MAPPING.get(k, k)
|
||||
target[normalized_key] = v
|
||||
# Also keep the original key for now if it's not the same,
|
||||
# so we can filter at the end or avoid losing it if it wasn't supposed to be renamed?
|
||||
# Actually, if we rename it, we should probably NOT keep both in 'target'
|
||||
# because we want to filter them out at the end anyway.
|
||||
if normalized_key != k:
|
||||
# If we are overwriting an existing snake_case key with a camelCase one's value,
|
||||
# that's fine because of the priority order of calls to _update_normalized.
|
||||
pass
|
||||
target[k] = v
|
||||
"""Update target dict with normalized, persistence-safe keys from source."""
|
||||
for key, value in source.items():
|
||||
if key in GenParamsMerger.BLACKLISTED_KEYS:
|
||||
continue
|
||||
|
||||
normalized_key = GenParamsMerger.NORMALIZATION_MAPPING.get(key, key)
|
||||
if normalized_key not in GenParamsMerger.ALLOWED_KEYS:
|
||||
continue
|
||||
|
||||
target[normalized_key] = value
|
||||
|
||||
@@ -52,7 +52,7 @@ class AutomaticMetadataParser(RecipeMetadataParser):
|
||||
negative_and_params = ""
|
||||
|
||||
# Initialize metadata
|
||||
metadata = {
|
||||
metadata: Dict[str, Any] = {
|
||||
"prompt": prompt,
|
||||
"loras": []
|
||||
}
|
||||
@@ -123,24 +123,39 @@ class AutomaticMetadataParser(RecipeMetadataParser):
|
||||
if model_hash_from_hashes:
|
||||
metadata["model_hash"] = model_hash_from_hashes
|
||||
|
||||
# Extract Lora hashes in alternative format
|
||||
# Extract Lora hashes in alternative format.
|
||||
# Run unconditionally (not just as fallback) so that
|
||||
# non-empty hashes from Lora hashes fill in the gaps left
|
||||
# by empty values in the Hashes JSON dict. Some WebUI
|
||||
# builds write real hash values only to Lora hashes and
|
||||
# leave the Hashes JSON values empty.
|
||||
lora_hashes_match = re.search(self.LORA_HASHES_REGEX, params_section)
|
||||
if not hashes_match and lora_hashes_match:
|
||||
if lora_hashes_match:
|
||||
try:
|
||||
lora_hashes_str = lora_hashes_match.group(1)
|
||||
lora_hash_entries = lora_hashes_str.split(', ')
|
||||
|
||||
# Initialize hashes dict if it doesn't exist
|
||||
if "hashes" not in metadata:
|
||||
metadata["hashes"] = {}
|
||||
|
||||
|
||||
# Parse each lora hash entry (format: "name: hash")
|
||||
for entry in lora_hash_entries:
|
||||
if ': ' in entry:
|
||||
lora_name, lora_hash = entry.split(': ', 1)
|
||||
# Add as lora type in the same format as regular hashes
|
||||
metadata["hashes"][f"lora:{lora_name}"] = lora_hash.strip()
|
||||
|
||||
lora_hash = lora_hash.strip()
|
||||
if not lora_hash:
|
||||
# Skip entries without a hash value
|
||||
continue
|
||||
# Initialize hashes dict if it doesn't exist
|
||||
if "hashes" not in metadata:
|
||||
metadata["hashes"] = {}
|
||||
# Add as lora type in the same format as
|
||||
# regular hashes. Only override an
|
||||
# existing entry if its value is empty
|
||||
# (Lora hashes is the more reliable
|
||||
# source when Hashes JSON has blanks).
|
||||
key = f"lora:{lora_name}"
|
||||
existing = metadata["hashes"].get(key, "")
|
||||
if not existing:
|
||||
metadata["hashes"][key] = lora_hash
|
||||
|
||||
# Remove lora hashes from params section
|
||||
params_section = params_section.replace(lora_hashes_match.group(0), '')
|
||||
except Exception as e:
|
||||
@@ -362,6 +377,12 @@ class AutomaticMetadataParser(RecipeMetadataParser):
|
||||
# Only process lora or hypernet types
|
||||
if not hash_key.startswith(("lora:", "hypernet:")):
|
||||
continue
|
||||
|
||||
# Skip entries without a hash value — they can't be
|
||||
# resolved via CivitAI and would only produce a
|
||||
# useless "Deleted" entry in the recipe.
|
||||
if not lora_hash:
|
||||
continue
|
||||
|
||||
lora_type, lora_name = hash_key.split(':', 1)
|
||||
|
||||
@@ -387,11 +408,7 @@ class AutomaticMetadataParser(RecipeMetadataParser):
|
||||
# Try to get info from Civitai
|
||||
if metadata_provider:
|
||||
try:
|
||||
if lora_hash:
|
||||
# If we have hash, use it for lookup
|
||||
civitai_info = await metadata_provider.get_model_by_hash(lora_hash)
|
||||
else:
|
||||
civitai_info = None
|
||||
civitai_info = await metadata_provider.get_model_by_hash(lora_hash)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
|
||||
@@ -4,8 +4,9 @@ import json
|
||||
import logging
|
||||
from typing import Dict, Any, Union
|
||||
from ..base import RecipeMetadataParser
|
||||
from ..constants import GEN_PARAM_KEYS
|
||||
from ..constants import GEN_PARAM_KEYS, VALID_LORA_TYPES
|
||||
from ...services.metadata_service import get_default_metadata_provider
|
||||
from ...config import config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -13,15 +14,16 @@ logger = logging.getLogger(__name__)
|
||||
class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
"""Parser for Civitai image metadata format"""
|
||||
|
||||
def is_metadata_matching(self, metadata) -> bool:
|
||||
def is_metadata_matching(self, user_comment) -> bool:
|
||||
"""Check if the metadata matches the Civitai image metadata format
|
||||
|
||||
Args:
|
||||
metadata: The metadata from the image (dict)
|
||||
user_comment: The metadata from the image (dict)
|
||||
|
||||
Returns:
|
||||
bool: True if this parser can handle the metadata
|
||||
"""
|
||||
metadata = user_comment
|
||||
if not metadata or not isinstance(metadata, dict):
|
||||
return False
|
||||
|
||||
@@ -72,8 +74,9 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
|
||||
return False
|
||||
|
||||
async def parse_metadata( # type: ignore[override]
|
||||
self, user_comment, recipe_scanner=None, civitai_client=None
|
||||
async def parse_metadata( # pyright: ignore[reportIncompatibleMethodOverride]
|
||||
self, user_comment, recipe_scanner=None, civitai_client=None,
|
||||
local_cache: dict[str, Any] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Parse metadata from Civitai image format
|
||||
|
||||
@@ -81,12 +84,13 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
user_comment: The metadata from the image (dict)
|
||||
recipe_scanner: Optional recipe scanner service
|
||||
civitai_client: Optional Civitai API client (deprecated, use metadata_provider instead)
|
||||
local_cache: Optional dict mapping sha256/autov3 hash → scanner cache item.
|
||||
When provided, matching models skip CivitAI API calls.
|
||||
|
||||
Returns:
|
||||
Dict containing parsed recipe data
|
||||
"""
|
||||
metadata: Dict[str, Any] = user_comment # type: ignore[assignment]
|
||||
metadata = user_comment
|
||||
metadata: Dict[str, Any] = user_comment
|
||||
try:
|
||||
# Get metadata provider instead of using civitai_client directly
|
||||
metadata_provider = await get_default_metadata_provider()
|
||||
@@ -112,7 +116,7 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
metadata = inner_meta
|
||||
|
||||
# Initialize result structure
|
||||
result = {
|
||||
result: Dict[str, Any] = {
|
||||
"base_model": None,
|
||||
"loras": [],
|
||||
"model": None,
|
||||
@@ -121,10 +125,10 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
}
|
||||
|
||||
# Track already added LoRAs to prevent duplicates
|
||||
added_loras = {} # key: model_version_id or hash, value: index in result["loras"]
|
||||
added_loras: Dict[str, Any] = {} # key: model_version_id or hash, value: index in result["loras"]
|
||||
|
||||
# Extract hash information from hashes field for LoRA matching
|
||||
lora_hashes = {}
|
||||
lora_hashes: Dict[str, Any] = {}
|
||||
if "hashes" in metadata and isinstance(metadata["hashes"], dict):
|
||||
for key, hash_value in metadata["hashes"].items():
|
||||
key_str = str(key)
|
||||
@@ -180,13 +184,83 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
if model_info:
|
||||
result["base_model"] = model_info.get("baseModel", "")
|
||||
|
||||
base_model_counts = {}
|
||||
base_model_counts: Dict[str, int] = {}
|
||||
|
||||
# Process standard resources array
|
||||
if "resources" in metadata and isinstance(metadata["resources"], list):
|
||||
for resource in metadata["resources"]:
|
||||
resource_type = resource.get("type", "lora")
|
||||
|
||||
# Track resources with type "model" — these are checkpoint models.
|
||||
# The resources array is the most reliable source for checkpoint
|
||||
# identification because it has an explicit type field and hash,
|
||||
# unlike modelVersionIds which is a flat list with no type info.
|
||||
if resource_type == "model":
|
||||
checkpoint_entry: Dict[str, Any] = {
|
||||
"id": 0,
|
||||
"modelId": 0,
|
||||
"name": resource.get("name", "Unknown Model"),
|
||||
"version": "",
|
||||
"type": resource.get("type", "model"),
|
||||
"existsLocally": False,
|
||||
"localPath": None,
|
||||
"file_name": resource.get("name", ""),
|
||||
"hash": resource.get("hash", "") or "",
|
||||
"thumbnailUrl": "/loras_static/images/no-preview.png",
|
||||
"baseModel": "",
|
||||
"size": 0,
|
||||
"downloadUrl": "",
|
||||
"isDeleted": False,
|
||||
}
|
||||
|
||||
# Try to look up base model from the checkpoint hash
|
||||
cp_hash = checkpoint_entry.get("hash")
|
||||
if cp_hash and metadata_provider:
|
||||
# local_cache keys are stored lowercase
|
||||
local_cached = local_cache.get(cp_hash.lower()) if local_cache else None
|
||||
if local_cached:
|
||||
self._populate_entry_from_cache(
|
||||
checkpoint_entry, local_cached
|
||||
)
|
||||
bm = checkpoint_entry.get("baseModel", "")
|
||||
if bm and not result["base_model"]:
|
||||
result["base_model"] = bm
|
||||
else:
|
||||
try:
|
||||
civitai_info = (
|
||||
await metadata_provider.get_model_by_hash(
|
||||
cp_hash
|
||||
)
|
||||
)
|
||||
civitai_data, error_msg = (
|
||||
(civitai_info, None)
|
||||
if not isinstance(civitai_info, tuple)
|
||||
else civitai_info
|
||||
)
|
||||
if civitai_data and error_msg != "Model not found":
|
||||
if 'model' in civitai_data and 'name' in civitai_data['model']:
|
||||
checkpoint_entry['name'] = civitai_data['model']['name']
|
||||
checkpoint_entry['id'] = civitai_data.get('id', 0)
|
||||
checkpoint_entry['modelId'] = civitai_data.get('modelId', 0)
|
||||
if 'name' in civitai_data:
|
||||
checkpoint_entry['version'] = civitai_data['name']
|
||||
base_model = civitai_data.get('baseModel', '')
|
||||
if base_model:
|
||||
checkpoint_entry['baseModel'] = base_model
|
||||
if not result['base_model']:
|
||||
result['base_model'] = base_model
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching checkpoint info for hash "
|
||||
f"{cp_hash}: {e}"
|
||||
)
|
||||
|
||||
if result["model"] is None:
|
||||
result["model"] = checkpoint_entry
|
||||
continue
|
||||
|
||||
# Modified to process resources without a type field as potential LoRAs
|
||||
if resource.get("type", "lora") == "lora":
|
||||
if resource_type == "lora":
|
||||
lora_hash = resource.get("hash", "")
|
||||
|
||||
# Try to get hash from the hashes field if not present in resource
|
||||
@@ -220,34 +294,58 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
}
|
||||
|
||||
# Try to get info from Civitai if hash is available
|
||||
if lora_entry["hash"] and metadata_provider:
|
||||
try:
|
||||
civitai_info = (
|
||||
await metadata_provider.get_model_by_hash(lora_hash)
|
||||
if lora_hash and metadata_provider:
|
||||
# local_cache keys are stored lowercase
|
||||
local_cached = local_cache.get(lora_hash.lower()) if local_cache else None
|
||||
if local_cached:
|
||||
cached_type = self._cache_item_model_type(local_cached)
|
||||
if cached_type and cached_type not in VALID_LORA_TYPES:
|
||||
logger.debug(
|
||||
f"Skipping non-LoRA cache item for hash {lora_hash}"
|
||||
)
|
||||
continue
|
||||
self._populate_entry_from_cache(
|
||||
lora_entry, local_cached
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
continue # Skip invalid LoRA types
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
# If we have a version ID from Civitai, track it for deduplication
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
# Track by version ID for deduplication
|
||||
if lora_entry.get("id"):
|
||||
added_loras[str(lora_entry["id"])] = len(
|
||||
result["loras"]
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_entry['hash']}: {e}"
|
||||
)
|
||||
# Mirror base.py:150-151 counts for API-path loras
|
||||
bm = local_cached.get("base_model") or ""
|
||||
if bm:
|
||||
base_model_counts[bm] = base_model_counts.get(
|
||||
bm, 0
|
||||
) + 1
|
||||
else:
|
||||
try:
|
||||
civitai_info = (
|
||||
await metadata_provider.get_model_by_hash(lora_hash)
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
continue # Skip invalid LoRA types
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
# If we have a version ID from Civitai, track it for deduplication
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
added_loras[str(lora_entry["id"])] = len(
|
||||
result["loras"]
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_entry['hash']}: {e}"
|
||||
)
|
||||
|
||||
# Track by hash if we have it
|
||||
if lora_hash:
|
||||
@@ -430,11 +528,21 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
|
||||
result["loras"].append(lora_entry)
|
||||
|
||||
# Process modelVersionIds from Civitai image API
|
||||
# These are model version IDs returned at root level when meta doesn't contain resources
|
||||
if "modelVersionIds" in metadata and isinstance(
|
||||
metadata["modelVersionIds"], list
|
||||
# Process modelVersionIds from Civitai image API.
|
||||
# These are version IDs returned at root level of the API response.
|
||||
# When resources or civitaiResources are already present in metadata
|
||||
# (which they are when ?withMeta=true is passed), those sections have
|
||||
# complete hash/type information — modelVersionIds is a fallback for
|
||||
# when meta is null and only the flat ID list is available. Skipping
|
||||
# it here avoids duplicates: the same file hash often resolves to
|
||||
# different version IDs via hash lookup (resources) vs the original
|
||||
# version ID in modelVersionIds, and both paths would create entries.
|
||||
if (
|
||||
"modelVersionIds" in metadata
|
||||
and isinstance(metadata["modelVersionIds"], list)
|
||||
and not result.get("loras")
|
||||
):
|
||||
|
||||
for version_id in metadata["modelVersionIds"]:
|
||||
version_id_str = str(version_id)
|
||||
|
||||
@@ -442,6 +550,13 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
if version_id_str in added_loras:
|
||||
continue
|
||||
|
||||
# Skip if this version ID is already the recipe's checkpoint
|
||||
# (resolved earlier from embedded resources/Model hash,
|
||||
# avoiding a duplicate CivitAI API call).
|
||||
existing_model = result.get("model")
|
||||
if existing_model and str(existing_model.get("id")) == version_id_str:
|
||||
continue
|
||||
|
||||
# Initialize lora entry with version ID
|
||||
lora_entry = {
|
||||
"id": version_id,
|
||||
@@ -475,9 +590,40 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
continue # Skip invalid LoRA types
|
||||
# Not a LoRA — try as checkpoint (only if we
|
||||
# don't already have one). Reuses the same
|
||||
# civitai_info from the API call above so no
|
||||
# extra query is made.
|
||||
if result["model"] is None:
|
||||
checkpoint_entry = {
|
||||
"id": version_id,
|
||||
"modelId": 0,
|
||||
"name": "Unknown Model",
|
||||
"version": "",
|
||||
"type": "checkpoint",
|
||||
"existsLocally": False,
|
||||
"localPath": None,
|
||||
"file_name": "",
|
||||
"hash": "",
|
||||
"thumbnailUrl": (
|
||||
"/loras_static/images/no-preview.png"
|
||||
),
|
||||
"baseModel": "",
|
||||
"size": 0,
|
||||
"downloadUrl": "",
|
||||
"isDeleted": False,
|
||||
}
|
||||
cp_populated = await (
|
||||
self.populate_checkpoint_from_civitai(
|
||||
checkpoint_entry, civitai_info
|
||||
)
|
||||
)
|
||||
if cp_populated.get("modelId"):
|
||||
result["model"] = cp_populated
|
||||
continue # Not a LoRA, don't add to loras
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for model version {version_id}: {e}"
|
||||
@@ -517,30 +663,47 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
}
|
||||
|
||||
if metadata_provider:
|
||||
try:
|
||||
civitai_info = await metadata_provider.get_model_by_hash(
|
||||
lora_hash
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
# local_cache keys are stored lowercase
|
||||
local_cached = local_cache.get(lora_hash.lower()) if local_cache else None
|
||||
if local_cached:
|
||||
cached_type = self._cache_item_model_type(local_cached)
|
||||
if cached_type and cached_type not in VALID_LORA_TYPES:
|
||||
logger.debug(
|
||||
f"Skipping non-LoRA cache item for hash {lora_hash}"
|
||||
)
|
||||
continue
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
self._populate_entry_from_cache(lora_entry, local_cached)
|
||||
# Mirror base.py:150-151 counts for API-path loras
|
||||
bm = local_cached.get("base_model") or ""
|
||||
if bm:
|
||||
base_model_counts[bm] = base_model_counts.get(bm, 0) + 1
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_hash}: {e}"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
civitai_info = await metadata_provider.get_model_by_hash(
|
||||
lora_hash
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
continue
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_hash}: {e}"
|
||||
)
|
||||
|
||||
added_loras[lora_hash] = len(result["loras"])
|
||||
result["loras"].append(lora_entry)
|
||||
@@ -579,32 +742,51 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
|
||||
# Try to get info from Civitai if hash is available
|
||||
if lora_entry["hash"] and metadata_provider:
|
||||
try:
|
||||
civitai_info = await metadata_provider.get_model_by_hash(
|
||||
lora_hash
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
# local_cache keys are stored lowercase
|
||||
local_cached = local_cache.get(lora_hash.lower()) if local_cache else None
|
||||
if local_cached:
|
||||
cached_type = self._cache_item_model_type(local_cached)
|
||||
if cached_type and cached_type not in VALID_LORA_TYPES:
|
||||
logger.debug(
|
||||
f"Skipping non-LoRA cache item for hash {lora_hash}"
|
||||
)
|
||||
lora_index += 1
|
||||
continue # Skip invalid LoRA types
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
continue # Skip non-LoRA cache items
|
||||
self._populate_entry_from_cache(lora_entry, local_cached)
|
||||
# Mirror base.py:150-151 counts for API-path loras
|
||||
bm = local_cached.get("base_model") or ""
|
||||
if bm:
|
||||
base_model_counts[bm] = base_model_counts.get(bm, 0) + 1
|
||||
# If we have a version ID from Civitai, track it for deduplication
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_entry['hash']}: {e}"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
civitai_info = await metadata_provider.get_model_by_hash(
|
||||
lora_hash
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
lora_index += 1
|
||||
continue # Skip invalid LoRA types
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
# If we have a version ID from Civitai, track it for deduplication
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_entry['hash']}: {e}"
|
||||
)
|
||||
|
||||
# Track by hash if we have it
|
||||
if lora_hash:
|
||||
@@ -625,3 +807,52 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing Civitai image metadata: {e}", exc_info=True)
|
||||
return {"error": str(e), "loras": []}
|
||||
|
||||
@staticmethod
|
||||
def _populate_entry_from_cache(
|
||||
entry: dict[str, Any],
|
||||
cache_item: dict[str, Any],
|
||||
) -> None:
|
||||
"""Fill a lora/checkpoint entry from a scanner cache item.
|
||||
|
||||
Avoids CivitAI API calls for models that exist locally.
|
||||
Mirrors the population logic in
|
||||
``RecipeMetadataParser.populate_lora_from_civitai()`` but operates
|
||||
entirely on cached data.
|
||||
"""
|
||||
civ = cache_item.get("civitai") or {}
|
||||
if isinstance(civ, dict):
|
||||
if civ.get("id") is not None:
|
||||
entry["id"] = civ["id"]
|
||||
if civ.get("modelId") is not None:
|
||||
entry["modelId"] = civ["modelId"]
|
||||
if civ.get("name"):
|
||||
entry["version"] = civ["name"]
|
||||
cached_name = cache_item.get("model_name")
|
||||
if cached_name:
|
||||
entry["name"] = cached_name
|
||||
entry["existsLocally"] = True
|
||||
local_path = cache_item.get("file_path")
|
||||
if local_path:
|
||||
entry["localPath"] = local_path
|
||||
sha256 = cache_item.get("sha256")
|
||||
if sha256:
|
||||
entry["hash"] = sha256
|
||||
if "preview_url" in cache_item:
|
||||
entry["thumbnailUrl"] = config.get_preview_static_url(
|
||||
cache_item["preview_url"]
|
||||
)
|
||||
base_model = cache_item.get("base_model", "")
|
||||
if base_model:
|
||||
entry["baseModel"] = base_model
|
||||
|
||||
@staticmethod
|
||||
def _cache_item_model_type(cache_item: dict[str, Any]) -> str:
|
||||
"""Lowercased civitai.model.type of a cache item, or '' when unknown."""
|
||||
civ = cache_item.get("civitai")
|
||||
if not isinstance(civ, dict):
|
||||
return ""
|
||||
model_info = civ.get("model")
|
||||
if not isinstance(model_info, dict):
|
||||
return ""
|
||||
return (model_info.get("type") or "").lower()
|
||||
|
||||
@@ -30,7 +30,7 @@ class MetaFormatParser(RecipeMetadataParser):
|
||||
prompt = parts[0].strip()
|
||||
|
||||
# Initialize metadata
|
||||
metadata = {"prompt": prompt, "loras": []}
|
||||
metadata: Dict[str, Any] = {"prompt": prompt, "loras": []}
|
||||
|
||||
# Extract negative prompt and parameters if available
|
||||
if len(parts) > 1:
|
||||
|
||||
@@ -91,7 +91,15 @@ class RecipeFormatParser(RecipeMetadataParser):
|
||||
exists_locally = lora_scanner.has_hash(lora['hash'])
|
||||
if exists_locally:
|
||||
lora_cache = await lora_scanner.get_cached_data()
|
||||
lora_item = next((item for item in lora_cache.raw_data if item['sha256'].lower() == lora['hash'].lower()), None)
|
||||
# Cascade match: full sha256, stored autov3, or autov2 (sha256[:10]).
|
||||
h = (lora.get('hash') or '').lower()
|
||||
lora_item = next(
|
||||
(item for item in lora_cache.raw_data
|
||||
if (item.get("sha256") or "").lower() == h
|
||||
or (item.get("autov3") or "").lower() == h
|
||||
or (item.get("sha256") or "")[:10].lower() == h),
|
||||
None
|
||||
)
|
||||
if lora_item:
|
||||
lora_entry['existsLocally'] = True
|
||||
lora_entry['inLibrary'] = True
|
||||
@@ -148,7 +156,7 @@ class RecipeFormatParser(RecipeMetadataParser):
|
||||
checkpoint_data = recipe_metadata.get('checkpoint') or {}
|
||||
if isinstance(checkpoint_data, dict) and checkpoint_data:
|
||||
version_id = checkpoint_data.get('modelVersionId') or checkpoint_data.get('id')
|
||||
checkpoint_entry = {
|
||||
checkpoint_entry: Dict[str, Any] = {
|
||||
'id': version_id or 0,
|
||||
'modelId': checkpoint_data.get('modelId', 0),
|
||||
'name': checkpoint_data.get('name', 'Unknown Checkpoint'),
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, Callable, Dict, Mapping
|
||||
from typing import TYPE_CHECKING, Awaitable, Callable, Dict, Mapping
|
||||
|
||||
import jinja2
|
||||
from aiohttp import web
|
||||
@@ -84,7 +84,7 @@ class BaseModelRoutes(ABC):
|
||||
self.metadata_progress_callback = WebSocketBroadcastCallback()
|
||||
|
||||
self._handler_set: ModelHandlerSet | None = None
|
||||
self._handler_mapping: Dict[str, Callable[[web.Request], web.StreamResponse]] | None = None
|
||||
self._handler_mapping: Dict[str, Callable[[web.Request], Awaitable[web.Response]]] | None = None
|
||||
|
||||
self._preview_service = PreviewAssetService(
|
||||
metadata_manager=MetadataManager,
|
||||
@@ -131,7 +131,7 @@ class BaseModelRoutes(ABC):
|
||||
self._handler_set = None
|
||||
self._handler_mapping = None
|
||||
|
||||
def _ensure_handler_mapping(self) -> Mapping[str, Callable[[web.Request], web.StreamResponse]]:
|
||||
def _ensure_handler_mapping(self) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||
if self._handler_mapping is None:
|
||||
handler_set = self._create_handler_set()
|
||||
self._handler_set = handler_set
|
||||
@@ -220,7 +220,7 @@ class BaseModelRoutes(ABC):
|
||||
)
|
||||
|
||||
@property
|
||||
def route_handlers(self) -> Mapping[str, Callable[[web.Request], web.StreamResponse]]:
|
||||
def route_handlers(self) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||
return self._ensure_handler_mapping()
|
||||
|
||||
def setup_routes(self, app: web.Application, prefix: str) -> None:
|
||||
@@ -237,7 +237,7 @@ class BaseModelRoutes(ABC):
|
||||
"""Setup model-specific routes."""
|
||||
raise NotImplementedError
|
||||
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict:
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict[str, Any]:
|
||||
"""Parse model-specific parameters - to be overridden by subclasses."""
|
||||
return {}
|
||||
|
||||
@@ -251,9 +251,9 @@ class BaseModelRoutes(ABC):
|
||||
|
||||
def _find_model_file(self, files):
|
||||
"""Find the appropriate model file from the files list - can be overridden by subclasses."""
|
||||
return next((file for file in files if file.get("type") == "Model" and file.get("primary") is True), None)
|
||||
return next((file for file in files if file.get("type") in ("Model", "Diffusion Model") and file.get("primary") is True), None)
|
||||
|
||||
def get_handler(self, name: str) -> Callable[[web.Request], web.StreamResponse]:
|
||||
def get_handler(self, name: str) -> Callable[[web.Request], Awaitable[web.StreamResponse]]:
|
||||
"""Expose handlers for subclasses or tests."""
|
||||
return self._ensure_handler_mapping()[name]
|
||||
|
||||
@@ -285,7 +285,7 @@ class BaseModelRoutes(ABC):
|
||||
)
|
||||
return self.model_lifecycle_service
|
||||
|
||||
def _make_handler_proxy(self, name: str) -> Callable[[web.Request], web.StreamResponse]:
|
||||
def _make_handler_proxy(self, name: str) -> Callable[[web.Request], Awaitable[web.StreamResponse]]:
|
||||
async def proxy(request: web.Request) -> web.StreamResponse:
|
||||
try:
|
||||
handler = self.get_handler(name)
|
||||
|
||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Callable, Mapping
|
||||
from typing import Awaitable, Callable, Mapping
|
||||
|
||||
import jinja2
|
||||
from aiohttp import web
|
||||
@@ -61,7 +61,9 @@ class BaseRecipeRoutes:
|
||||
self._i18n_registered = False
|
||||
self._startup_hooks_registered = False
|
||||
self._handler_set: RecipeHandlerSet | None = None
|
||||
self._handler_mapping: dict[str, Callable] | None = None
|
||||
self._handler_mapping: Mapping[
|
||||
str, Callable[[web.Request], Awaitable[web.StreamResponse]]
|
||||
] | None = None
|
||||
|
||||
async def attach_dependencies(self, app: web.Application | None = None) -> None:
|
||||
"""Resolve shared services from the registry."""
|
||||
@@ -84,7 +86,9 @@ class BaseRecipeRoutes:
|
||||
app.on_startup.append(self.attach_dependencies)
|
||||
self._startup_hooks_registered = True
|
||||
|
||||
def to_route_mapping(self) -> Mapping[str, Callable]:
|
||||
def to_route_mapping(
|
||||
self,
|
||||
) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||
"""Return a mapping of handler name to coroutine for registrar binding."""
|
||||
|
||||
if self._handler_mapping is None:
|
||||
@@ -124,17 +128,17 @@ class BaseRecipeRoutes:
|
||||
or os.environ.get("HF_HUB_DISABLE_TELEMETRY", "0") == "0"
|
||||
)
|
||||
if not standalone_mode:
|
||||
from ..metadata_collector import get_metadata # type: ignore[import-not-found]
|
||||
from ..metadata_collector.metadata_processor import ( # type: ignore[import-not-found]
|
||||
from ..metadata_collector import get_metadata # pyright: ignore[reportMissingImports]
|
||||
from ..metadata_collector.metadata_processor import ( # pyright: ignore[reportMissingImports]
|
||||
MetadataProcessor,
|
||||
)
|
||||
from ..metadata_collector.metadata_registry import ( # type: ignore[import-not-found]
|
||||
from ..metadata_collector.metadata_registry import ( # pyright: ignore[reportMissingImports]
|
||||
MetadataRegistry,
|
||||
)
|
||||
else: # pragma: no cover - optional dependency path
|
||||
get_metadata = None # type: ignore[assignment]
|
||||
MetadataProcessor = None # type: ignore[assignment]
|
||||
MetadataRegistry = None # type: ignore[assignment]
|
||||
get_metadata = None # pyright: ignore[reportAssignmentType]
|
||||
MetadataProcessor = None # pyright: ignore[reportAssignmentType]
|
||||
MetadataRegistry = None # pyright: ignore[reportAssignmentType]
|
||||
|
||||
analysis_service = RecipeAnalysisService(
|
||||
exif_utils=ExifUtils,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import logging
|
||||
from typing import Dict, List, Set
|
||||
from typing import Any, Dict, List, Set
|
||||
from aiohttp import web
|
||||
|
||||
from .base_model_routes import BaseModelRoutes
|
||||
@@ -28,13 +28,13 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
# Attach service dependencies
|
||||
self.attach_service(self.service)
|
||||
|
||||
def setup_routes(self, app: web.Application):
|
||||
def setup_routes(self, app: web.Application, prefix: str = "checkpoints"):
|
||||
"""Setup Checkpoint routes"""
|
||||
# Schedule service initialization on app startup
|
||||
app.on_startup.append(lambda _: self.initialize_services())
|
||||
|
||||
|
||||
# Setup common routes with 'checkpoints' prefix (includes page route)
|
||||
super().setup_routes(app, 'checkpoints')
|
||||
super().setup_routes(app, prefix)
|
||||
|
||||
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
||||
"""Setup Checkpoint-specific routes"""
|
||||
@@ -53,9 +53,9 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
"""Get expected model types string for error messages"""
|
||||
return "Checkpoint"
|
||||
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict:
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict[str, Any]:
|
||||
"""Parse Checkpoint-specific parameters"""
|
||||
params: Dict = {}
|
||||
params: Dict[str, Any] = {}
|
||||
|
||||
if 'checkpoint_hash' in request.query:
|
||||
params['hash_filters'] = {'single_hash': request.query['checkpoint_hash'].lower()}
|
||||
@@ -70,7 +70,7 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
"""Get detailed information for a specific checkpoint by name"""
|
||||
try:
|
||||
name = request.match_info.get('name', '')
|
||||
checkpoint_info = await self.service.get_model_info_by_name(name)
|
||||
checkpoint_info = await self.service.get_model_info_by_name(name) # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
if checkpoint_info:
|
||||
return web.json_response(checkpoint_info)
|
||||
@@ -89,7 +89,7 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
roots.extend(config.checkpoints_roots or [])
|
||||
roots.extend(config.extra_checkpoints_roots or [])
|
||||
# Remove duplicates while preserving order
|
||||
seen: set = set()
|
||||
seen: set[str] = set()
|
||||
unique_roots: List[str] = []
|
||||
for root in roots:
|
||||
if root and root not in seen:
|
||||
@@ -114,7 +114,7 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
roots.extend(config.unet_roots or [])
|
||||
roots.extend(config.extra_unet_roots or [])
|
||||
# Remove duplicates while preserving order
|
||||
seen: set = set()
|
||||
seen: set[str] = set()
|
||||
unique_roots: List[str] = []
|
||||
for root in roots:
|
||||
if root and root not in seen:
|
||||
|
||||
@@ -26,13 +26,13 @@ class EmbeddingRoutes(BaseModelRoutes):
|
||||
# Attach service dependencies
|
||||
self.attach_service(self.service)
|
||||
|
||||
def setup_routes(self, app: web.Application):
|
||||
def setup_routes(self, app: web.Application, prefix: str = "embeddings"):
|
||||
"""Setup Embedding routes"""
|
||||
# Schedule service initialization on app startup
|
||||
app.on_startup.append(lambda _: self.initialize_services())
|
||||
|
||||
|
||||
# Setup common routes with 'embeddings' prefix (includes page route)
|
||||
super().setup_routes(app, 'embeddings')
|
||||
super().setup_routes(app, prefix)
|
||||
|
||||
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
||||
"""Setup Embedding-specific routes"""
|
||||
@@ -51,7 +51,7 @@ class EmbeddingRoutes(BaseModelRoutes):
|
||||
"""Get detailed information for a specific embedding by name"""
|
||||
try:
|
||||
name = request.match_info.get('name', '')
|
||||
embedding_info = await self.service.get_model_info_by_name(name)
|
||||
embedding_info = await self.service.get_model_info_by_name(name) # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
if embedding_info:
|
||||
return web.json_response(embedding_info)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Callable, Mapping
|
||||
from typing import Any, Awaitable, Callable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -35,7 +35,7 @@ class ExampleImagesRoutes:
|
||||
*,
|
||||
ws_manager,
|
||||
download_manager: DownloadManager | None = None,
|
||||
processor=ExampleImagesProcessor,
|
||||
processor: Any = ExampleImagesProcessor,
|
||||
file_manager=ExampleImagesFileManager,
|
||||
cleanup_service: ExampleImagesCleanupService | None = None,
|
||||
) -> None:
|
||||
@@ -46,7 +46,9 @@ class ExampleImagesRoutes:
|
||||
self._file_manager = file_manager
|
||||
self._cleanup_service = cleanup_service or ExampleImagesCleanupService()
|
||||
self._handler_set: ExampleImagesHandlerSet | None = None
|
||||
self._handler_mapping: Mapping[str, Callable[[web.Request], web.StreamResponse]] | None = None
|
||||
self._handler_mapping: Mapping[
|
||||
str, Callable[[web.Request], Awaitable[web.StreamResponse]]
|
||||
] | None = None
|
||||
|
||||
@classmethod
|
||||
def setup_routes(cls, app: web.Application, *, ws_manager) -> None:
|
||||
@@ -61,7 +63,9 @@ class ExampleImagesRoutes:
|
||||
registrar = ExampleImagesRouteRegistrar(app)
|
||||
registrar.register_routes(self.to_route_mapping())
|
||||
|
||||
def to_route_mapping(self) -> Mapping[str, Callable[[web.Request], web.StreamResponse]]:
|
||||
def to_route_mapping(
|
||||
self,
|
||||
) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||
"""Return the registrar-compatible mapping of handler names to callables."""
|
||||
|
||||
if self._handler_mapping is None:
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
"""HTTP route handlers for agent skill endpoints.
|
||||
|
||||
These handlers expose the :class:`AgentService` via HTTP, allowing the
|
||||
frontend to list available skills and execute them on selected models.
|
||||
Progress is reported via WebSocket broadcast.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from ...services.agent import AgentService, AgentProgressReporter
|
||||
from ...services.llm_service import LLMNotConfiguredError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AgentHandler:
|
||||
"""HTTP handler for agent skill operations."""
|
||||
|
||||
def __init__(self, agent_service: AgentService | None = None) -> None:
|
||||
self._agent_service = agent_service
|
||||
|
||||
async def _ensure_service(self) -> AgentService:
|
||||
if self._agent_service is None:
|
||||
self._agent_service = await AgentService.get_instance()
|
||||
return self._agent_service
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# GET /api/lm/agent/skills
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def get_agent_skills(self, request: web.Request) -> web.Response:
|
||||
"""Return a list of available agent skills."""
|
||||
|
||||
service = await self._ensure_service()
|
||||
skills = await service.list_skills()
|
||||
return web.json_response({"skills": skills})
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# POST /api/lm/agent/execute/{skill_name}
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def execute_agent_skill(self, request: web.Request) -> web.Response:
|
||||
"""Execute an agent skill on the provided model paths.
|
||||
|
||||
Request body::
|
||||
|
||||
{"model_paths": ["/path/to/model1.safetensors", ...], "options": {}}
|
||||
|
||||
Returns immediately with a task ID. Execution runs in the
|
||||
background; progress and completion are pushed via WebSocket
|
||||
events of type ``agent_progress``.
|
||||
"""
|
||||
|
||||
skill_name = request.match_info.get("skill_name", "")
|
||||
if not skill_name:
|
||||
return web.json_response(
|
||||
{"error": "Skill name is required"}, status=400
|
||||
)
|
||||
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception:
|
||||
return web.json_response(
|
||||
{"error": "Invalid JSON body"}, status=400
|
||||
)
|
||||
|
||||
model_paths = body.get("model_paths", [])
|
||||
if not model_paths or not isinstance(model_paths, list):
|
||||
return web.json_response(
|
||||
{"error": "model_paths must be a non-empty array"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
service = await self._ensure_service()
|
||||
|
||||
# Validate LLM configuration early for skills that need it
|
||||
# (fail fast rather than after starting background work)
|
||||
try:
|
||||
from ...services.llm_service import LLMService
|
||||
|
||||
llm = await LLMService.get_instance()
|
||||
if not llm.is_configured():
|
||||
return web.json_response(
|
||||
{
|
||||
"error": "LLM provider is not configured. "
|
||||
"Enable it in Settings → AI Provider.",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to check LLM configuration: %s", exc)
|
||||
|
||||
# Launch execution in the background
|
||||
progress_reporter = AgentProgressReporter()
|
||||
logger.info(
|
||||
"LLM enrichment '%s' starting for %d model(s)",
|
||||
skill_name, len(model_paths),
|
||||
)
|
||||
|
||||
async def _run() -> None:
|
||||
try:
|
||||
result = await service.execute_skill(
|
||||
skill_name=skill_name,
|
||||
input_data={"model_paths": model_paths},
|
||||
progress_callback=progress_reporter,
|
||||
)
|
||||
logger.info(
|
||||
"LLM enrichment '%s' finished: success=%s, summary='%s', errors=%s",
|
||||
skill_name, result.success, result.summary, result.errors,
|
||||
)
|
||||
except LLMNotConfiguredError as exc:
|
||||
logger.warning("LLM enrichment '%s' not configured: %s", skill_name, exc)
|
||||
await progress_reporter.on_progress(
|
||||
{
|
||||
"type": "agent_progress",
|
||||
"skill": skill_name,
|
||||
"status": "error",
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("LLM enrichment '%s' failed: %s", skill_name, exc, exc_info=True)
|
||||
await progress_reporter.on_progress(
|
||||
{
|
||||
"type": "agent_progress",
|
||||
"skill": skill_name,
|
||||
"status": "error",
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
|
||||
# Fire and forget — progress comes via WebSocket
|
||||
asyncio.create_task(_run())
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
"status": "started",
|
||||
"skill": skill_name,
|
||||
"model_count": len(model_paths),
|
||||
}
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# POST /api/lm/agent/cancel
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def cancel_agent_skill(self, request: web.Request) -> web.Response:
|
||||
"""Cancel a running agent skill.
|
||||
|
||||
NOTE: Cancellation is a stub for now — the AgentService processes
|
||||
models sequentially and does not yet support mid-execution
|
||||
cancellation. This endpoint exists for API completeness.
|
||||
"""
|
||||
|
||||
# TODO: implement cooperative cancellation in AgentService
|
||||
return web.json_response(
|
||||
{"status": "acknowledged", "note": "Cancellation not yet implemented"},
|
||||
status=200,
|
||||
)
|
||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Mapping
|
||||
from typing import Awaitable, Callable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -170,7 +170,7 @@ class ExampleImagesHandlerSet:
|
||||
management: ExampleImagesManagementHandler
|
||||
files: ExampleImagesFileHandler
|
||||
|
||||
def to_route_mapping(self) -> Mapping[str, Callable[[web.Request], web.StreamResponse]]:
|
||||
def to_route_mapping(self) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||
"""Flatten handler methods into the registrar mapping."""
|
||||
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,508 @@
|
||||
"""Handlers for Hugging Face model listing and download.
|
||||
|
||||
Minimal MVP implementation — uses direct HTTP to the HF API for file
|
||||
listing and the project's existing aiohttp-based Downloader for
|
||||
downloading. No huggingface_hub dependency required.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
from aiohttp import web
|
||||
|
||||
from ...config import config
|
||||
from ...services.downloader import (
|
||||
DownloadProgress,
|
||||
get_downloader,
|
||||
)
|
||||
from ...services.aria2_downloader import Aria2Downloader
|
||||
from ...services.settings_manager import get_settings_manager
|
||||
from ...services.service_registry import ServiceRegistry
|
||||
from ...services.websocket_manager import ws_manager
|
||||
from ...utils.constants import MODEL_FILE_EXTENSIONS
|
||||
from ...utils.metadata_manager import MetadataManager
|
||||
from ...utils.models import LoraMetadata, CheckpointMetadata, EmbeddingMetadata
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_MODEL_CLASS = LoraMetadata
|
||||
_DEFAULT_SCANNER_GETTER = "get_lora_scanner"
|
||||
|
||||
# Shared aiohttp session for HF API calls (created on first use)
|
||||
_hf_api_session: aiohttp.ClientSession | None = None
|
||||
|
||||
|
||||
async def _get_hf_api_session() -> aiohttp.ClientSession:
|
||||
"""Get or create the shared aiohttp session for HF API calls."""
|
||||
global _hf_api_session # needed because we reassign the module-level name
|
||||
if _hf_api_session is None or _hf_api_session.closed:
|
||||
_hf_api_session = aiohttp.ClientSession(
|
||||
headers={"User-Agent": "ComfyUI-LoRA-Manager/1.0"},
|
||||
timeout=aiohttp.ClientTimeout(total=30),
|
||||
)
|
||||
return _hf_api_session
|
||||
|
||||
|
||||
async def close_hf_api_session() -> None:
|
||||
"""Close the shared HF API session, if it was ever created."""
|
||||
global _hf_api_session
|
||||
if _hf_api_session is not None and not _hf_api_session.closed:
|
||||
await _hf_api_session.close()
|
||||
_hf_api_session = None
|
||||
|
||||
|
||||
def _infer_model_type(model_root: str) -> tuple[Any, str]:
|
||||
"""Determine model class and scanner by matching ``model_root`` against the
|
||||
configured root paths for each model type (from ``Config``).
|
||||
|
||||
The ``model_root`` value comes from the frontend's model-root dropdown,
|
||||
which is populated from the current page's scanner roots. By checking
|
||||
which scanner's root list it belongs to, we avoid fragile heuristics
|
||||
like substring-matching path names.
|
||||
"""
|
||||
norm = os.path.normpath(model_root).replace(os.sep, "/")
|
||||
|
||||
# LoRA roots
|
||||
for p in (config.loras_roots or []) + (config.extra_loras_roots or []):
|
||||
if os.path.normpath(p).replace(os.sep, "/") == norm:
|
||||
return LoraMetadata, "get_lora_scanner"
|
||||
|
||||
# Checkpoint / UNet roots
|
||||
for p in (
|
||||
(config.checkpoints_roots or [])
|
||||
+ (config.extra_checkpoints_roots or [])
|
||||
+ (config.unet_roots or [])
|
||||
+ (config.extra_unet_roots or [])
|
||||
):
|
||||
if os.path.normpath(p).replace(os.sep, "/") == norm:
|
||||
return CheckpointMetadata, "get_checkpoint_scanner"
|
||||
|
||||
# Embedding roots
|
||||
for p in (config.embeddings_roots or []) + (config.extra_embeddings_roots or []):
|
||||
if os.path.normpath(p).replace(os.sep, "/") == norm:
|
||||
return EmbeddingMetadata, "get_embedding_scanner"
|
||||
|
||||
# Fallback — should not happen in normal use
|
||||
logger.warning(
|
||||
"Could not determine model type for root '%s'; defaulting to LoRA",
|
||||
model_root,
|
||||
)
|
||||
return _DEFAULT_MODEL_CLASS, _DEFAULT_SCANNER_GETTER
|
||||
|
||||
|
||||
async def _save_hf_metadata(dest_path: str, repo: str, model_root: str) -> None:
|
||||
"""Create a proper .metadata.json and add the model to the scanner cache.
|
||||
|
||||
Uses ``MetadataManager.create_default_metadata()`` which computes the
|
||||
SHA256 hash, extracts safetensors header metadata (base_model), and
|
||||
produces a fully-populated ``LoraMetadata`` (or ``CheckpointMetadata`` /
|
||||
``EmbeddingMetadata``) object. We then overlay HF-specific fields and
|
||||
register the model in the in-memory scanner cache so it appears
|
||||
immediately without a full filesystem walk.
|
||||
"""
|
||||
try:
|
||||
hf_url = f"https://huggingface.co/{repo}"
|
||||
model_class, scanner_getter_name = _infer_model_type(model_root)
|
||||
|
||||
# 1. Create proper metadata (computes SHA256, reads safetensors headers)
|
||||
metadata = await MetadataManager.create_default_metadata(
|
||||
dest_path, model_class=model_class
|
||||
)
|
||||
if metadata is None:
|
||||
logger.warning("create_default_metadata returned None for %s", dest_path)
|
||||
return
|
||||
|
||||
# 2. Overlay HF-specific fields
|
||||
metadata._unknown_fields["hf_url"] = hf_url
|
||||
metadata.from_civitai = False # HF models are not from CivitAI
|
||||
|
||||
metadata_dict = metadata.to_dict()
|
||||
if "trainedWords" in metadata_dict and not metadata_dict["trainedWords"]:
|
||||
del metadata_dict["trainedWords"]
|
||||
|
||||
# 3. Save metadata atomically
|
||||
await MetadataManager.save_metadata(dest_path, metadata_dict)
|
||||
logger.info("Saved HF metadata (with hf_url) for %s", dest_path)
|
||||
|
||||
# 4. Determine relative folder path for cache
|
||||
# model_root is an absolute path; dest_path is under it
|
||||
folder = ""
|
||||
if os.path.isabs(model_root) and dest_path.startswith(model_root):
|
||||
rel = os.path.relpath(os.path.dirname(dest_path), model_root)
|
||||
folder = rel.replace(os.sep, "/") if rel != "." else ""
|
||||
|
||||
# 5. Add to scanner cache (same as CivitAI's _execute_download does)
|
||||
scanner_getter = getattr(ServiceRegistry, scanner_getter_name, None)
|
||||
if scanner_getter is not None:
|
||||
scanner = await scanner_getter()
|
||||
if scanner is not None:
|
||||
metadata_dict = metadata.to_dict()
|
||||
metadata_dict["hf_url"] = hf_url
|
||||
await scanner.add_model_to_cache(metadata_dict, folder)
|
||||
logger.info("Added %s to scanner cache (folder=%s)", dest_path, folder)
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to save HF metadata for %s: %s", dest_path, exc)
|
||||
|
||||
|
||||
def _find_matching_root(dest_dir: str) -> str | None:
|
||||
"""Walk up *dest_dir* to find which configured scanner root it belongs to."""
|
||||
norm = os.path.normpath(dest_dir).replace(os.sep, "/")
|
||||
all_roots = []
|
||||
for root_list in (
|
||||
config.loras_roots or [],
|
||||
config.extra_loras_roots or [],
|
||||
config.checkpoints_roots or [],
|
||||
config.extra_checkpoints_roots or [],
|
||||
config.unet_roots or [],
|
||||
config.extra_unet_roots or [],
|
||||
config.embeddings_roots or [],
|
||||
config.extra_embeddings_roots or [],
|
||||
):
|
||||
all_roots.extend([os.path.normpath(p).replace(os.sep, "/") for p in root_list])
|
||||
# Find the longest matching prefix
|
||||
match: str | None = None
|
||||
for root in all_roots:
|
||||
if norm.startswith(root):
|
||||
if match is None or len(root) > len(match):
|
||||
match = root
|
||||
return match
|
||||
|
||||
|
||||
async def _add_to_scanner_cache(dest_path: str, metadata: dict[str, Any]) -> None:
|
||||
model_dir = os.path.dirname(dest_path)
|
||||
model_root = _find_matching_root(model_dir)
|
||||
if not model_root:
|
||||
raise ValueError(f"File path {dest_path} is not within any configured scanner root")
|
||||
scanner_getter_name = _infer_model_type(model_root)[1]
|
||||
scanner_getter = getattr(ServiceRegistry, scanner_getter_name, None)
|
||||
if scanner_getter is None:
|
||||
raise RuntimeError(f"Scanner getter '{scanner_getter_name}' not found in ServiceRegistry")
|
||||
scanner = await scanner_getter()
|
||||
if scanner is None:
|
||||
raise RuntimeError(f"Scanner '{scanner_getter_name}' returned None")
|
||||
await scanner.update_single_model_cache(dest_path, dest_path, metadata)
|
||||
|
||||
|
||||
class HfHandler:
|
||||
"""Handle Hugging Face model browsing and download."""
|
||||
|
||||
async def set_hf_url(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
payload: dict[str, Any] = await request.json()
|
||||
except json.JSONDecodeError:
|
||||
return web.json_response({"success": False, "error": "Invalid JSON"}, status=400)
|
||||
|
||||
file_path = (payload.get("file_path") or "").strip()
|
||||
hf_url = (payload.get("hf_url") or "").strip()
|
||||
|
||||
if not file_path or not hf_url:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Missing required fields: 'file_path' and 'hf_url'"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
m = re.match(r"^https?://huggingface\.co/([^/]+/[^/]+)/?$", hf_url)
|
||||
if not m:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "Invalid HuggingFace URL. Expected format: https://huggingface.co/user/repo",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
|
||||
if not os.path.isfile(file_path):
|
||||
return web.json_response(
|
||||
{"success": False, "error": f"File not found: {file_path}"},
|
||||
status=404,
|
||||
)
|
||||
|
||||
model_root = _find_matching_root(os.path.dirname(file_path))
|
||||
if not model_root:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "File is not within any configured model directory. Cannot link to HuggingFace.",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
|
||||
try:
|
||||
existing = await MetadataManager.load_metadata_payload(file_path)
|
||||
if existing.get("hf_url") == hf_url:
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": "hf_url already set",
|
||||
"hf_url": hf_url,
|
||||
})
|
||||
|
||||
existing["hf_url"] = hf_url
|
||||
existing["from_civitai"] = False
|
||||
await MetadataManager.save_metadata(file_path, existing)
|
||||
|
||||
await _add_to_scanner_cache(file_path, existing)
|
||||
|
||||
logger.info("Set hf_url=%s for %s", hf_url, file_path)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"hf_url set to {hf_url}",
|
||||
"hf_url": hf_url,
|
||||
})
|
||||
except Exception as exc:
|
||||
logger.error("Failed to set hf_url for %s: %s", file_path, exc)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(exc)},
|
||||
status=500,
|
||||
)
|
||||
|
||||
async def get_hf_repo_files(self, request: web.Request) -> web.Response:
|
||||
"""List model-weight files from a HF repo with real file sizes.
|
||||
|
||||
Uses the HF tree API endpoint which returns accurate file sizes
|
||||
(including LFS-tracked files), unlike the model info endpoint.
|
||||
"""
|
||||
repo = request.query.get("repo", "").strip()
|
||||
if not repo or "/" not in repo:
|
||||
return web.json_response(
|
||||
{"error": "Missing or invalid 'repo' parameter (expected user/repo)"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
url = f"https://huggingface.co/api/models/{repo}/tree/main"
|
||||
|
||||
try:
|
||||
session = await _get_hf_api_session()
|
||||
async with session.get(url) as resp:
|
||||
if resp.status == 404:
|
||||
return web.json_response(
|
||||
{"error": f"Repo '{repo}' not found"}, status=404
|
||||
)
|
||||
if resp.status != 200:
|
||||
text = await resp.text()
|
||||
return web.json_response(
|
||||
{"error": f"HF API error {resp.status}: {text[:200]}"},
|
||||
status=resp.status,
|
||||
)
|
||||
tree: list[dict[str, Any]] = await resp.json()
|
||||
except Exception as exc:
|
||||
logger.error("Failed to fetch HF repo files: %s", exc)
|
||||
return web.json_response({"error": str(exc)}, status=502)
|
||||
|
||||
files: list[dict[str, Any]] = []
|
||||
for entry in tree:
|
||||
path: str = entry.get("path", "")
|
||||
ext = os.path.splitext(path)[1].lower()
|
||||
if ext not in MODEL_FILE_EXTENSIONS:
|
||||
continue
|
||||
size = entry.get("size", 0) or 0
|
||||
if size == 0 and "lfs" in entry:
|
||||
size = entry["lfs"].get("size", 0) or 0
|
||||
files.append({
|
||||
"filename": path,
|
||||
"size": size,
|
||||
})
|
||||
|
||||
files.sort(key=lambda f: f["size"], reverse=True)
|
||||
return web.json_response(files)
|
||||
|
||||
async def download_hf_model(self, request: web.Request) -> web.Response:
|
||||
"""Download a single file from Hugging Face into the model directory.
|
||||
|
||||
POST JSON body::
|
||||
|
||||
{
|
||||
"repo": "dx8152/Flux2-Klein-9B-Consistency",
|
||||
"filename": "Flux2-Klein-9B-consistency-V2.safetensors",
|
||||
"revision": "main",
|
||||
"model_root": "loras",
|
||||
"relative_path": "",
|
||||
"use_default_paths": false,
|
||||
"download_id": "optional-batch-id"
|
||||
}
|
||||
|
||||
If ``download_id`` is provided, real-time progress (bytes, speed,
|
||||
percentage) is broadcast via the WebSocket progress system, matching
|
||||
the CivitAI download experience.
|
||||
|
||||
Respects the ``download_backend`` setting (``aria2`` or ``default``).
|
||||
"""
|
||||
try:
|
||||
payload: dict[str, Any] = await request.json()
|
||||
except json.JSONDecodeError:
|
||||
return web.json_response({"error": "Invalid JSON"}, status=400)
|
||||
|
||||
repo = (payload.get("repo") or "").strip()
|
||||
filename = (payload.get("filename") or "").strip()
|
||||
revision = (payload.get("revision") or "main").strip()
|
||||
model_root = (payload.get("model_root") or "").strip()
|
||||
relative_path = (payload.get("relative_path") or "").strip()
|
||||
use_default_paths = bool(payload.get("use_default_paths", False))
|
||||
download_id: str | None = payload.get("download_id")
|
||||
|
||||
logger.info(
|
||||
"download_hf_model: repo=%s file=%s root=%s download_id=%s",
|
||||
repo, filename, model_root, download_id,
|
||||
)
|
||||
|
||||
if not repo or not filename:
|
||||
return web.json_response(
|
||||
{"error": "Missing required fields: 'repo' and 'filename'"}, status=400
|
||||
)
|
||||
|
||||
# Validate repo format — must be user/repo_name
|
||||
if repo.count("/") != 1 or not re.match(r"^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$", repo):
|
||||
return web.json_response({"error": f"Invalid repo format: {repo}"}, status=400)
|
||||
author, repo_name = repo.split("/", 1)
|
||||
if ".." in (author, repo_name) or "." in (author, repo_name):
|
||||
return web.json_response({"error": f"Invalid repo format: {repo}"}, status=400)
|
||||
|
||||
# Validate filename — must not contain path traversal
|
||||
if ".." in filename:
|
||||
return web.json_response({"error": "Invalid filename"}, status=400)
|
||||
|
||||
# Validate relative_path — must not be absolute or escape base directory
|
||||
if relative_path:
|
||||
if os.path.isabs(relative_path):
|
||||
return web.json_response({"error": "relative_path must not be absolute"}, status=400)
|
||||
if ".." in relative_path.split("/") or "\\" in relative_path:
|
||||
return web.json_response({"error": "Invalid relative_path"}, status=400)
|
||||
|
||||
# Use model_root directly as the base directory — same approach as
|
||||
# CivitAI's download path (download_manager.py). No realpath, no
|
||||
# allowed-roots validation, no path-traversal check; those are
|
||||
# unnecessary when the frontend sends the path from its own dropdown
|
||||
# (populated from scanner roots). Using the "business path" directly
|
||||
# keeps dest_path consistent with scanner roots so that later folder
|
||||
# derivation (in _save_hf_metadata) works correctly.
|
||||
if os.path.isabs(model_root):
|
||||
base_dir = os.path.normpath(model_root)
|
||||
else:
|
||||
base_dir = os.path.normpath(os.path.join(os.getcwd(), "models", model_root))
|
||||
|
||||
if use_default_paths:
|
||||
target_dir = os.path.join(base_dir, "huggingface", author, repo_name)
|
||||
elif relative_path:
|
||||
target_dir = os.path.join(base_dir, relative_path)
|
||||
else:
|
||||
target_dir = base_dir
|
||||
|
||||
# Strip HF repo subdirectory — "diffusion_models/xxx.safetensors"
|
||||
# is an HF repo convention, not meaningful for local storage.
|
||||
file_base = os.path.basename(filename)
|
||||
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
dest_path = os.path.join(target_dir, file_base)
|
||||
|
||||
# Check if already exists (simple skip)
|
||||
if os.path.exists(dest_path) and os.path.getsize(dest_path) > 0:
|
||||
logger.info("download_hf_model: file already exists, skipping — %s", dest_path)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"File already exists: {dest_path}",
|
||||
"path": dest_path,
|
||||
})
|
||||
|
||||
# Build HF resolve URL
|
||||
resolve_url = (
|
||||
f"https://huggingface.co/{repo}/resolve/{revision}/{filename}"
|
||||
)
|
||||
|
||||
# Set up progress callback if download_id is provided
|
||||
progress_callback = None
|
||||
if download_id:
|
||||
|
||||
async def _progress_callback(
|
||||
progress: float | DownloadProgress,
|
||||
snapshot: DownloadProgress | None = None,
|
||||
) -> None:
|
||||
percent = 0.0
|
||||
metrics = snapshot if isinstance(snapshot, DownloadProgress) else None
|
||||
|
||||
if isinstance(progress, DownloadProgress):
|
||||
percent = progress.percent_complete
|
||||
metrics = progress
|
||||
elif isinstance(snapshot, DownloadProgress):
|
||||
percent = snapshot.percent_complete
|
||||
else:
|
||||
percent = float(progress)
|
||||
|
||||
broadcast: dict[str, Any] = {
|
||||
"status": "progress",
|
||||
"progress": round(percent),
|
||||
}
|
||||
if metrics:
|
||||
broadcast["bytes_downloaded"] = metrics.bytes_downloaded
|
||||
broadcast["total_bytes"] = metrics.total_bytes
|
||||
broadcast["bytes_per_second"] = metrics.bytes_per_second
|
||||
|
||||
await ws_manager.broadcast_download_progress(download_id, broadcast)
|
||||
|
||||
progress_callback = _progress_callback
|
||||
|
||||
# Respect download backend setting (aria2 vs default)
|
||||
download_backend = (
|
||||
get_settings_manager().get("download_backend", "default")
|
||||
)
|
||||
|
||||
if download_backend == "aria2":
|
||||
aria2 = await Aria2Downloader.get_instance()
|
||||
aid = download_id or f"hf_{repo}_{filename}"
|
||||
try:
|
||||
hf_success, hf_result = await aria2.download_file(
|
||||
url=resolve_url,
|
||||
save_path=dest_path,
|
||||
download_id=aid,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
if hf_success:
|
||||
await _save_hf_metadata(dest_path, repo, model_root)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"Downloaded to {dest_path}",
|
||||
"path": dest_path,
|
||||
})
|
||||
else:
|
||||
return web.json_response(
|
||||
{"success": False, "error": hf_result or "aria2 download failed"},
|
||||
status=500,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("HF download (aria2) failed: %s", exc)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(exc)}, status=500
|
||||
)
|
||||
|
||||
# Default: use built-in aiohttp Downloader
|
||||
downloader = await get_downloader()
|
||||
try:
|
||||
success, result = await downloader.download_file(
|
||||
url=resolve_url,
|
||||
save_path=dest_path,
|
||||
use_auth=False,
|
||||
allow_resume=True,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
if success:
|
||||
await _save_hf_metadata(dest_path, repo, model_root)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"Downloaded to {result}",
|
||||
"path": result,
|
||||
})
|
||||
else:
|
||||
return web.json_response(
|
||||
{"success": False, "error": result or "Download failed"},
|
||||
status=500,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("HF download failed: %s", exc)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(exc)}, status=500
|
||||
)
|
||||
+2300
-137
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -2,7 +2,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import mimetypes
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
@@ -12,6 +14,12 @@ from ...config import config as global_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CHUNK_SIZE = 1024 * 1024 # 1 MB — balance between streaming iteration overhead and per-chunk memory
|
||||
|
||||
# Video file extensions that bypass native sendfile on Windows
|
||||
# to avoid IOCP/ProactorEventLoop crashes during client disconnect.
|
||||
_VIDEO_EXTENSIONS = frozenset({".mp4", ".webm", ".mov", ".avi", ".mkv"})
|
||||
|
||||
|
||||
class PreviewHandler:
|
||||
"""Serve preview assets for the active library at request time."""
|
||||
@@ -46,10 +54,90 @@ class PreviewHandler:
|
||||
|
||||
if not resolved.is_file():
|
||||
logger.debug("Preview file not found at %s", str(resolved))
|
||||
asyncio.create_task(self._cleanup_stale_preview_url(normalized))
|
||||
raise web.HTTPNotFound(text="Preview file not found")
|
||||
|
||||
# aiohttp's FileResponse handles range requests and content headers for us.
|
||||
return web.FileResponse(path=resolved, chunk_size=256 * 1024)
|
||||
# aiohttp's FileResponse handles range requests, content headers, and
|
||||
# uses kernel sendfile (zero-copy DMA) on Linux/macOS. On Windows it
|
||||
# uses IOCP-based _sendfile_native which can crash when the client
|
||||
# disconnects mid-transfer during fast scrolling. The _stream_file()
|
||||
# fallback is kept for a future compat toggle.
|
||||
#
|
||||
# Set explicit Cache-Control so the browser can cache video (and image)
|
||||
# previews across VirtualScroller recycling cycles. Without this,
|
||||
# Chrome does not cache 206 Partial Content responses for <video>
|
||||
# elements, causing the same video to be re-downloaded on every scroll.
|
||||
resp = web.FileResponse(path=resolved, chunk_size=_CHUNK_SIZE)
|
||||
resp.headers["Cache-Control"] = "public, max-age=86400"
|
||||
return resp
|
||||
|
||||
async def _cleanup_stale_preview_url(self, normalized_preview_path: str) -> None:
|
||||
"""Fire-and-forget: clear stale preview_url from all model caches.
|
||||
|
||||
When a preview file is no longer on disk, remove its reference from
|
||||
every cached entry so subsequent list API responses return an empty
|
||||
``preview_url``, letting the frontend show the no-preview placeholder.
|
||||
"""
|
||||
try:
|
||||
from ...services.service_registry import ServiceRegistry
|
||||
|
||||
for service_name in ("lora_scanner", "checkpoint_scanner", "embedding_scanner"):
|
||||
scanner = ServiceRegistry.get_service_sync(service_name)
|
||||
if scanner is None or not hasattr(scanner, "_cache"):
|
||||
continue
|
||||
cache = getattr(scanner, "_cache", None)
|
||||
if cache is None or not hasattr(cache, "clear_preview_by_path"):
|
||||
continue
|
||||
cleared = await cache.clear_preview_by_path(normalized_preview_path)
|
||||
if cleared and hasattr(scanner, "_persist_current_cache"):
|
||||
await scanner._persist_current_cache()
|
||||
logger.info(
|
||||
"Cleared stale preview_url for %d %s entries (%s)",
|
||||
cleared,
|
||||
service_name,
|
||||
normalized_preview_path,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to clean up stale preview_url: %s", exc)
|
||||
|
||||
async def _stream_file(
|
||||
self, request: web.Request, path: Path
|
||||
) -> web.StreamResponse:
|
||||
"""Stream a file chunk-by-chunk, bypassing native sendfile.
|
||||
|
||||
This avoids the Windows IOCP ``_sendfile_native`` crash that occurs
|
||||
when the client disconnects during a large file transfer.
|
||||
"""
|
||||
content_type, _ = mimetypes.guess_type(str(path))
|
||||
if content_type is None:
|
||||
content_type = "application/octet-stream"
|
||||
|
||||
file_size = path.stat().st_size
|
||||
resp = web.StreamResponse()
|
||||
resp.content_type = content_type
|
||||
resp.content_length = file_size
|
||||
|
||||
# Allow browser caching: video previews rarely change during a session.
|
||||
# The frontend already appends ?t={version} to bust cache on update.
|
||||
resp.headers["Cache-Control"] = "public, max-age=86400"
|
||||
|
||||
await resp.prepare(request)
|
||||
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
while True:
|
||||
chunk = f.read(_CHUNK_SIZE)
|
||||
if not chunk:
|
||||
break
|
||||
await resp.write(chunk)
|
||||
except (ConnectionResetError, ConnectionAbortedError):
|
||||
# Client disconnected during streaming — expected when scrolling
|
||||
# rapidly through a library with animated previews.
|
||||
pass
|
||||
except OSError as exc:
|
||||
logger.debug("I/O error streaming preview %s: %s", path, exc)
|
||||
|
||||
return resp
|
||||
|
||||
|
||||
__all__ = ["PreviewHandler"]
|
||||
|
||||
+1267
-129
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,8 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from aiohttp import web
|
||||
from typing import Dict
|
||||
from server import PromptServer # type: ignore
|
||||
from typing import Any, Dict
|
||||
from server import PromptServer # pyright: ignore[reportMissingImports]
|
||||
|
||||
from .base_model_routes import BaseModelRoutes
|
||||
from .model_route_registrar import ModelRouteRegistrar
|
||||
@@ -31,13 +31,13 @@ class LoraRoutes(BaseModelRoutes):
|
||||
# Attach service dependencies
|
||||
self.attach_service(self.service)
|
||||
|
||||
def setup_routes(self, app: web.Application):
|
||||
def setup_routes(self, app: web.Application, prefix: str = "loras"):
|
||||
"""Setup LoRA routes"""
|
||||
# Schedule service initialization on app startup
|
||||
app.on_startup.append(lambda _: self.initialize_services())
|
||||
|
||||
# Setup common routes with 'loras' prefix (includes page route)
|
||||
super().setup_routes(app, "loras")
|
||||
super().setup_routes(app, prefix)
|
||||
|
||||
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
||||
"""Setup LoRA-specific routes"""
|
||||
@@ -73,7 +73,7 @@ class LoraRoutes(BaseModelRoutes):
|
||||
"POST", "/api/lm/{prefix}/get_trigger_words", prefix, self.get_trigger_words
|
||||
)
|
||||
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict:
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict[str, Any]:
|
||||
"""Parse LoRA-specific parameters"""
|
||||
params = {}
|
||||
|
||||
@@ -119,25 +119,6 @@ class LoraRoutes(BaseModelRoutes):
|
||||
logger.error(f"Error getting letter counts: {e}")
|
||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||
|
||||
async def get_lora_notes(self, request: web.Request) -> web.Response:
|
||||
"""Get notes for a specific LoRA file"""
|
||||
try:
|
||||
lora_name = request.query.get("name")
|
||||
if not lora_name:
|
||||
return web.Response(text="Lora file name is required", status=400)
|
||||
|
||||
notes = await self.service.get_lora_notes(lora_name)
|
||||
if notes is not None:
|
||||
return web.json_response({"success": True, "notes": notes})
|
||||
else:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "LoRA not found in cache"}, status=404
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting lora notes: {e}", exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||
|
||||
async def get_lora_trigger_words(self, request: web.Request) -> web.Response:
|
||||
"""Get trigger words for a specific LoRA file"""
|
||||
try:
|
||||
@@ -168,52 +149,6 @@ class LoraRoutes(BaseModelRoutes):
|
||||
logger.error(f"Error getting lora usage tips by path: {e}", exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||
|
||||
async def get_lora_preview_url(self, request: web.Request) -> web.Response:
|
||||
"""Get the static preview URL for a LoRA file"""
|
||||
try:
|
||||
lora_name = request.query.get("name")
|
||||
if not lora_name:
|
||||
return web.Response(text="Lora file name is required", status=400)
|
||||
|
||||
preview_url = await self.service.get_lora_preview_url(lora_name)
|
||||
if preview_url:
|
||||
return web.json_response({"success": True, "preview_url": preview_url})
|
||||
else:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "No preview URL found for the specified lora",
|
||||
},
|
||||
status=404,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting lora preview URL: {e}", exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||
|
||||
async def get_lora_civitai_url(self, request: web.Request) -> web.Response:
|
||||
"""Get the Civitai URL for a LoRA file"""
|
||||
try:
|
||||
lora_name = request.query.get("name")
|
||||
if not lora_name:
|
||||
return web.Response(text="Lora file name is required", status=400)
|
||||
|
||||
result = await self.service.get_lora_civitai_url(lora_name)
|
||||
if result["civitai_url"]:
|
||||
return web.json_response({"success": True, **result})
|
||||
else:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "No Civitai data found for the specified lora",
|
||||
},
|
||||
status=404,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting lora Civitai URL: {e}", exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||
|
||||
async def get_random_loras(self, request: web.Request) -> web.Response:
|
||||
"""Get random LoRAs based on filters and strength ranges"""
|
||||
try:
|
||||
@@ -337,7 +272,7 @@ class LoraRoutes(BaseModelRoutes):
|
||||
graph_identifier = entry.get("graph_id")
|
||||
|
||||
try:
|
||||
parsed_node_id = int(node_identifier)
|
||||
parsed_node_id = int(node_identifier) # pyright: ignore[reportArgumentType]
|
||||
except (TypeError, ValueError):
|
||||
parsed_node_id = node_identifier
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ miscellaneous endpoints share a consistent registration flow.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Iterable, Mapping
|
||||
from typing import Any, Callable, Iterable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -22,21 +22,47 @@ class RouteDefinition:
|
||||
MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("GET", "/api/lm/settings", "get_settings"),
|
||||
RouteDefinition("POST", "/api/lm/settings", "update_settings"),
|
||||
RouteDefinition("GET", "/api/lm/llm/models", "get_llm_models"),
|
||||
RouteDefinition("GET", "/api/lm/llm/provider-models", "get_provider_models"),
|
||||
RouteDefinition("GET", "/api/lm/doctor/diagnostics", "get_doctor_diagnostics"),
|
||||
RouteDefinition("POST", "/api/lm/doctor/repair-cache", "repair_doctor_cache"),
|
||||
RouteDefinition("POST", "/api/lm/doctor/resolve-filename-conflicts", "resolve_doctor_filename_conflicts"),
|
||||
RouteDefinition("POST", "/api/lm/doctor/export-bundle", "export_doctor_bundle"),
|
||||
RouteDefinition("GET", "/api/lm/priority-tags", "get_priority_tags"),
|
||||
RouteDefinition("GET", "/api/lm/settings/libraries", "get_settings_libraries"),
|
||||
RouteDefinition("POST", "/api/lm/settings/libraries/activate", "activate_library"),
|
||||
RouteDefinition("GET", "/api/lm/health-check", "health_check"),
|
||||
RouteDefinition("GET", "/api/lm/supporters", "get_supporters"),
|
||||
RouteDefinition("GET", "/api/lm/wildcards/search", "search_wildcards"),
|
||||
RouteDefinition("POST", "/api/lm/wildcards/open-location", "open_wildcards_location"),
|
||||
RouteDefinition("POST", "/api/lm/open-file-location", "open_file_location"),
|
||||
RouteDefinition("POST", "/api/lm/update-usage-stats", "update_usage_stats"),
|
||||
RouteDefinition("GET", "/api/lm/get-usage-stats", "get_usage_stats"),
|
||||
RouteDefinition("POST", "/api/lm/update-lora-code", "update_lora_code"),
|
||||
RouteDefinition("GET", "/api/lm/update-lora-code", "get_update_lora_code"),
|
||||
RouteDefinition("GET", "/api/lm/trained-words", "get_trained_words"),
|
||||
RouteDefinition("GET", "/api/lm/model-example-files", "get_model_example_files"),
|
||||
RouteDefinition("POST", "/api/lm/register-nodes", "register_nodes"),
|
||||
RouteDefinition("POST", "/api/lm/update-node-widget", "update_node_widget"),
|
||||
RouteDefinition("GET", "/api/lm/update-node-widget", "get_update_node_widget"),
|
||||
RouteDefinition("GET", "/api/lm/get-registry", "get_registry"),
|
||||
RouteDefinition("GET", "/api/lm/check-model-exists", "check_model_exists"),
|
||||
RouteDefinition("GET", "/api/lm/check-models-exist", "check_models_exist"),
|
||||
RouteDefinition(
|
||||
"GET",
|
||||
"/api/lm/model-version-download-status",
|
||||
"get_model_version_download_status",
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST",
|
||||
"/api/lm/model-version-download-status",
|
||||
"set_model_version_download_status",
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET",
|
||||
"/api/lm/set-model-version-download-status",
|
||||
"set_model_version_download_status",
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/civitai/user-models", "get_civitai_user_models"),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/download-metadata-archive", "download_metadata_archive"
|
||||
@@ -47,6 +73,10 @@ MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/metadata-archive-status", "get_metadata_archive_status"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/backup/status", "get_backup_status"),
|
||||
RouteDefinition("POST", "/api/lm/backup/export", "export_backup"),
|
||||
RouteDefinition("POST", "/api/lm/backup/import", "import_backup"),
|
||||
RouteDefinition("POST", "/api/lm/backup/open-location", "open_backup_location"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/model-versions-status", "get_model_versions_status"
|
||||
),
|
||||
@@ -65,6 +95,29 @@ MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/base-models/cache-status", "get_base_model_cache_status"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/delete-model-version", "delete_model_version"
|
||||
),
|
||||
# Hugging Face model endpoints
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/hf-repo-files", "get_hf_repo_files"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/download-hf-model", "download_hf_model"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/set-hf-url", "set_hf_url"
|
||||
),
|
||||
# Agent skill endpoints
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/agent/skills", "get_agent_skills"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/agent/execute/{skill_name}", "execute_agent_skill"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/agent/cancel", "cancel_agent_skill"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -94,7 +147,7 @@ class MiscRouteRegistrar:
|
||||
handler_lookup[definition.handler_name],
|
||||
)
|
||||
|
||||
def _bind(self, method: str, path: str, handler: Callable) -> None:
|
||||
def _bind(self, method: str, path: str, handler: Callable[..., Any]) -> None:
|
||||
add_method_name = self._METHOD_MAP[method.upper()]
|
||||
add_method = getattr(self._app.router, add_method_name)
|
||||
add_method(path, handler)
|
||||
|
||||
@@ -7,7 +7,7 @@ import os
|
||||
from typing import Awaitable, Callable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
from server import PromptServer # type: ignore
|
||||
from server import PromptServer # pyright: ignore[reportMissingImports]
|
||||
|
||||
from ..services.metadata_service import (
|
||||
get_metadata_archive_manager,
|
||||
@@ -19,10 +19,12 @@ from ..services.downloader import get_downloader
|
||||
from ..utils.usage_stats import UsageStats
|
||||
from .handlers.misc_handlers import (
|
||||
CustomWordsHandler,
|
||||
DoctorHandler,
|
||||
ExampleWorkflowsHandler,
|
||||
FileSystemHandler,
|
||||
HealthCheckHandler,
|
||||
LoraCodeHandler,
|
||||
BackupHandler,
|
||||
MetadataArchiveHandler,
|
||||
MiscHandlerSet,
|
||||
ModelExampleFilesHandler,
|
||||
@@ -33,9 +35,12 @@ from .handlers.misc_handlers import (
|
||||
SupportersHandler,
|
||||
TrainedWordsHandler,
|
||||
UsageStatsHandler,
|
||||
WildcardsHandler,
|
||||
build_service_registry_adapter,
|
||||
)
|
||||
from .handlers.base_model_handlers import BaseModelHandlerSet
|
||||
from .handlers.hf_handlers import HfHandler
|
||||
from .handlers.agent_handlers import AgentHandler
|
||||
from .misc_route_registrar import MiscRouteRegistrar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -116,6 +121,7 @@ class MiscRoutes:
|
||||
settings_service=self._settings,
|
||||
metadata_provider_updater=self._metadata_provider_updater,
|
||||
)
|
||||
backup = BackupHandler()
|
||||
filesystem = FileSystemHandler(settings_service=self._settings)
|
||||
node_registry_handler = NodeRegistryHandler(
|
||||
node_registry=self._node_registry,
|
||||
@@ -127,9 +133,13 @@ class MiscRoutes:
|
||||
metadata_provider_factory=self._metadata_provider_factory,
|
||||
)
|
||||
custom_words = CustomWordsHandler()
|
||||
wildcards = WildcardsHandler()
|
||||
supporters = SupportersHandler()
|
||||
doctor = DoctorHandler(settings_service=self._settings)
|
||||
example_workflows = ExampleWorkflowsHandler()
|
||||
base_model = BaseModelHandlerSet()
|
||||
hf_handler = HfHandler()
|
||||
agent_handler = AgentHandler()
|
||||
|
||||
return self._handler_set_factory(
|
||||
health=health,
|
||||
@@ -141,11 +151,16 @@ class MiscRoutes:
|
||||
node_registry=node_registry_handler,
|
||||
model_library=model_library,
|
||||
metadata_archive=metadata_archive,
|
||||
backup=backup,
|
||||
filesystem=filesystem,
|
||||
custom_words=custom_words,
|
||||
wildcards=wildcards,
|
||||
supporters=supporters,
|
||||
doctor=doctor,
|
||||
example_workflows=example_workflows,
|
||||
base_model=base_model,
|
||||
hf_handler=hf_handler,
|
||||
agent_handler=agent_handler,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Iterable, Mapping
|
||||
from typing import Any, Callable, Iterable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -22,8 +22,10 @@ class RouteDefinition:
|
||||
|
||||
COMMON_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/list", "get_models"),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/excluded", "get_excluded_models"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/delete", "delete_model"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/exclude", "exclude_model"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/unexclude", "unexclude_model"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/fetch-civitai", "fetch_civitai"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/fetch-all-civitai", "fetch_all_civitai"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/relink-civitai", "relink_civitai"),
|
||||
@@ -44,6 +46,7 @@ COMMON_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
"GET", "/api/lm/{prefix}/auto-organize-progress", "get_auto_organize_progress"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/top-tags", "get_top_tags"),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/search-tags", "search_tags"),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/base-models", "get_base_models"),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/model-types", "get_model_types"),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/scan", "scan_models"),
|
||||
@@ -99,11 +102,46 @@ COMMON_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("POST", "/api/lm/download-model", "download_model"),
|
||||
RouteDefinition("GET", "/api/lm/download-model-get", "download_model_get"),
|
||||
RouteDefinition("GET", "/api/lm/cancel-download-get", "cancel_download_get"),
|
||||
RouteDefinition("GET", "/api/lm/skip-download", "skip_download_get"),
|
||||
RouteDefinition("GET", "/api/lm/pause-download", "pause_download_get"),
|
||||
RouteDefinition("GET", "/api/lm/resume-download", "resume_download_get"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/download-progress/{download_id}", "get_download_progress"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/downloads/queue", "get_download_queue"),
|
||||
RouteDefinition("GET", "/api/lm/downloads/queue/add", "add_to_download_queue"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/queue/remove", "remove_from_download_queue"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/queue/move-to-top", "move_queue_item_to_top"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/queue/move-to-end", "move_queue_item_to_end"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/queue/clear", "clear_download_queue"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/downloads/history", "get_download_history"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/history/clear", "clear_download_history"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/history/delete", "delete_download_history_item"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/history/retry", "retry_download_from_history"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/history/retry-all", "retry_all_failed_downloads"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/downloads/stats", "get_download_stats"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/queue/complete", "complete_download_in_queue"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/queue/status", "update_download_queue_status"
|
||||
),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/cancel-task", "cancel_task"),
|
||||
RouteDefinition("GET", "/{prefix}", "handle_models_page"),
|
||||
)
|
||||
@@ -136,15 +174,15 @@ class ModelRouteRegistrar:
|
||||
handler_lookup[definition.handler_name],
|
||||
)
|
||||
|
||||
def add_route(self, method: str, path: str, handler: Callable) -> None:
|
||||
def add_route(self, method: str, path: str, handler: Callable[..., Any]) -> None:
|
||||
self._bind_route(method, path, handler)
|
||||
|
||||
def add_prefixed_route(
|
||||
self, method: str, path_template: str, prefix: str, handler: Callable
|
||||
self, method: str, path_template: str, prefix: str, handler: Callable[..., Any]
|
||||
) -> None:
|
||||
self._bind_route(method, path_template.replace("{prefix}", prefix), handler)
|
||||
|
||||
def _bind_route(self, method: str, path: str, handler: Callable) -> None:
|
||||
def _bind_route(self, method: str, path: str, handler: Callable[..., Any]) -> None:
|
||||
add_method_name = self._METHOD_MAP[method.upper()]
|
||||
add_method = getattr(self._app.router, add_method_name)
|
||||
add_method(path, handler)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Mapping
|
||||
from typing import Any, Callable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -29,6 +29,7 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("POST", "/api/lm/recipes/save", "save_recipe"),
|
||||
RouteDefinition("DELETE", "/api/lm/recipe/{recipe_id}", "delete_recipe"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/top-tags", "get_top_tags"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/search-tags", "search_tags"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/base-models", "get_base_models"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/roots", "get_roots"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/folders", "get_folders"),
|
||||
@@ -58,6 +59,7 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("POST", "/api/lm/recipes/repair", "repair_recipes"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/cancel-repair", "cancel_repair"),
|
||||
RouteDefinition("POST", "/api/lm/recipe/{recipe_id}/repair", "repair_recipe"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/repair-bulk", "repair_recipes_bulk"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/repair-progress", "get_repair_progress"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/batch-import/start", "start_batch_import"),
|
||||
RouteDefinition(
|
||||
@@ -70,6 +72,16 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
"POST", "/api/lm/recipes/batch-import/directory", "start_directory_import"
|
||||
),
|
||||
RouteDefinition("POST", "/api/lm/recipes/browse-directory", "browse_directory"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/recipes/check-image-exists", "check_image_exists"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/recipes/import-from-url", "import_from_url"),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/recipes/create-from-example", "create_from_example"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/recipe/{recipe_id}/reimport", "reimport_recipe"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -93,7 +105,7 @@ class RecipeRouteRegistrar:
|
||||
handler = handler_lookup[definition.handler_name]
|
||||
self._bind_route(definition.method, definition.path, handler)
|
||||
|
||||
def _bind_route(self, method: str, path: str, handler: Callable) -> None:
|
||||
def _bind_route(self, method: str, path: str, handler: Callable[..., Any]) -> None:
|
||||
add_method_name = self._METHOD_MAP[method.upper()]
|
||||
add_method = getattr(self._app.router, add_method_name)
|
||||
add_method(path, handler)
|
||||
|
||||
+56
-26
@@ -11,6 +11,8 @@ from ..config import config
|
||||
from ..services.settings_manager import get_settings_manager
|
||||
from ..services.server_i18n import server_i18n
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
from ..services.model_query import normalize_sub_type, resolve_sub_type
|
||||
from ..utils.constants import VALID_LORA_SUB_TYPES, VALID_CHECKPOINT_SUB_TYPES
|
||||
from ..utils.usage_stats import UsageStats
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -38,10 +40,11 @@ class StatsRoutes:
|
||||
"""Route handlers for Statistics page and API endpoints"""
|
||||
|
||||
def __init__(self):
|
||||
self.lora_scanner = None
|
||||
self.checkpoint_scanner = None
|
||||
self.embedding_scanner = None
|
||||
self.usage_stats = None
|
||||
self.lora_scanner: Any = None
|
||||
self.checkpoint_scanner: Any = None
|
||||
self.embedding_scanner: Any = None
|
||||
self.usage_stats: Any = None
|
||||
self._i18n_filter_added = False
|
||||
self.template_env = jinja2.Environment(
|
||||
loader=jinja2.FileSystemLoader(config.templates_path),
|
||||
autoescape=True
|
||||
@@ -93,9 +96,9 @@ class StatsRoutes:
|
||||
server_i18n.set_locale(user_language)
|
||||
|
||||
# 为模板环境添加i18n过滤器
|
||||
if not hasattr(self.template_env, '_i18n_filter_added'):
|
||||
if not self._i18n_filter_added:
|
||||
self.template_env.filters['t'] = server_i18n.create_template_filter()
|
||||
self.template_env._i18n_filter_added = True
|
||||
self._i18n_filter_added = True
|
||||
|
||||
template = self.template_env.get_template('statistics.html')
|
||||
rendered = template.render(
|
||||
@@ -140,6 +143,21 @@ class StatsRoutes:
|
||||
# Get usage statistics
|
||||
usage_data = await self.usage_stats.get_stats()
|
||||
|
||||
# CivitAI model type distribution across all model types
|
||||
# Use the same logic as the filter panel: normalize_sub_type(resolve_sub_type(entry))
|
||||
# with sub-type validation per model type
|
||||
model_types_counter: Counter[str] = Counter()
|
||||
for entry in lora_cache.raw_data:
|
||||
ntype = normalize_sub_type(resolve_sub_type(entry))
|
||||
if ntype and ntype in VALID_LORA_SUB_TYPES:
|
||||
model_types_counter[ntype] += 1
|
||||
for entry in checkpoint_cache.raw_data:
|
||||
ntype = normalize_sub_type(resolve_sub_type(entry))
|
||||
if ntype and ntype in VALID_CHECKPOINT_SUB_TYPES:
|
||||
model_types_counter[ntype] += 1
|
||||
# Embeddings: always count as "embedding" regardless of CivitAI sub-type
|
||||
model_types_counter['embedding'] = len(embedding_cache.raw_data)
|
||||
|
||||
return web.json_response({
|
||||
'success': True,
|
||||
'data': {
|
||||
@@ -154,7 +172,8 @@ class StatsRoutes:
|
||||
'total_generations': usage_data.get('total_executions', 0),
|
||||
'unused_loras': self._count_unused_models(lora_cache.raw_data, usage_data.get('loras', {})),
|
||||
'unused_checkpoints': self._count_unused_models(checkpoint_cache.raw_data, usage_data.get('checkpoints', {})),
|
||||
'unused_embeddings': self._count_unused_models(embedding_cache.raw_data, usage_data.get('embeddings', {}))
|
||||
'unused_embeddings': self._count_unused_models(embedding_cache.raw_data, usage_data.get('embeddings', {})),
|
||||
'model_types_distribution': dict(model_types_counter.most_common())
|
||||
}
|
||||
})
|
||||
|
||||
@@ -459,9 +478,12 @@ class StatsRoutes:
|
||||
if unused_lora_percent > 50:
|
||||
insights.append({
|
||||
'type': 'warning',
|
||||
'title': 'High Number of Unused LoRAs',
|
||||
'description': f'{unused_lora_percent:.1f}% of your LoRAs ({unused_loras}/{total_loras}) have never been used.',
|
||||
'suggestion': 'Consider organizing or archiving unused models to free up storage space.'
|
||||
'key': 'insights.unusedLoras.high',
|
||||
'params': {
|
||||
'percent': f'{unused_lora_percent:.1f}',
|
||||
'count': str(unused_loras),
|
||||
'total': str(total_loras)
|
||||
}
|
||||
})
|
||||
|
||||
if total_checkpoints > 0:
|
||||
@@ -469,9 +491,12 @@ class StatsRoutes:
|
||||
if unused_checkpoint_percent > 30:
|
||||
insights.append({
|
||||
'type': 'warning',
|
||||
'title': 'Unused Checkpoints Detected',
|
||||
'description': f'{unused_checkpoint_percent:.1f}% of your checkpoints ({unused_checkpoints}/{total_checkpoints}) have never been used.',
|
||||
'suggestion': 'Review and consider removing checkpoints you no longer need.'
|
||||
'key': 'insights.unusedCheckpoints.detected',
|
||||
'params': {
|
||||
'percent': f'{unused_checkpoint_percent:.1f}',
|
||||
'count': str(unused_checkpoints),
|
||||
'total': str(total_checkpoints)
|
||||
}
|
||||
})
|
||||
|
||||
if total_embeddings > 0:
|
||||
@@ -479,9 +504,12 @@ class StatsRoutes:
|
||||
if unused_embedding_percent > 50:
|
||||
insights.append({
|
||||
'type': 'warning',
|
||||
'title': 'High Number of Unused Embeddings',
|
||||
'description': f'{unused_embedding_percent:.1f}% of your embeddings ({unused_embeddings}/{total_embeddings}) have never been used.',
|
||||
'suggestion': 'Consider organizing or archiving unused embeddings to optimize your collection.'
|
||||
'key': 'insights.unusedEmbeddings.high',
|
||||
'params': {
|
||||
'percent': f'{unused_embedding_percent:.1f}',
|
||||
'count': str(unused_embeddings),
|
||||
'total': str(total_embeddings)
|
||||
}
|
||||
})
|
||||
|
||||
# Storage insights
|
||||
@@ -492,18 +520,20 @@ class StatsRoutes:
|
||||
if total_size > 100 * 1024 * 1024 * 1024: # 100GB
|
||||
insights.append({
|
||||
'type': 'info',
|
||||
'title': 'Large Collection Detected',
|
||||
'description': f'Your model collection is using {self._format_size(total_size)} of storage.',
|
||||
'suggestion': 'Consider using external storage or cloud solutions for better organization.'
|
||||
'key': 'insights.collection.large',
|
||||
'params': {
|
||||
'size': self._format_size(total_size)
|
||||
}
|
||||
})
|
||||
|
||||
# Recent activity insight
|
||||
if usage_data.get('total_executions', 0) > 100:
|
||||
insights.append({
|
||||
'type': 'success',
|
||||
'title': 'Active User',
|
||||
'description': f'You\'ve completed {usage_data["total_executions"]} generations so far!',
|
||||
'suggestion': 'Keep exploring and creating amazing content with your models.'
|
||||
'key': 'insights.activity.active',
|
||||
'params': {
|
||||
'count': str(usage_data['total_executions'])
|
||||
}
|
||||
})
|
||||
|
||||
return web.json_response({
|
||||
@@ -520,7 +550,7 @@ class StatsRoutes:
|
||||
'error': str(e)
|
||||
}, status=500)
|
||||
|
||||
def _count_unused_models(self, models: List[Dict], usage_data: Dict) -> int:
|
||||
def _count_unused_models(self, models: List[Dict[str, Any]], usage_data: Dict[str, Any]) -> int:
|
||||
"""Count models that have never been used"""
|
||||
used_hashes = set(usage_data.keys())
|
||||
unused_count = 0
|
||||
@@ -531,7 +561,7 @@ class StatsRoutes:
|
||||
|
||||
return unused_count
|
||||
|
||||
def _get_top_used_models(self, usage_data: Dict, model_map: Dict, limit: int) -> List[Dict]:
|
||||
def _get_top_used_models(self, usage_data: Dict[str, Any], model_map: Dict[str, Any], limit: int) -> List[Dict[str, Any]]:
|
||||
"""Get top used models with their metadata"""
|
||||
sorted_usage = sorted(usage_data.items(), key=lambda x: x[1].get('total', 0), reverse=True)
|
||||
|
||||
@@ -549,7 +579,7 @@ class StatsRoutes:
|
||||
|
||||
return top_models
|
||||
|
||||
def _get_usage_timeline(self, usage_data: Dict, days: int) -> List[Dict]:
|
||||
def _get_usage_timeline(self, usage_data: Dict[str, Any], days: int) -> List[Dict[str, Any]]:
|
||||
"""Get usage timeline for the past N days"""
|
||||
timeline = []
|
||||
today = datetime.now()
|
||||
@@ -585,7 +615,7 @@ class StatsRoutes:
|
||||
|
||||
return list(reversed(timeline)) # Oldest to newest
|
||||
|
||||
def _format_size(self, size_bytes: int) -> str:
|
||||
def _format_size(self, size_bytes: float) -> str:
|
||||
"""Format file size in human readable format"""
|
||||
for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
|
||||
if size_bytes < 1024.0:
|
||||
|
||||
+383
-63
@@ -1,21 +1,120 @@
|
||||
import os
|
||||
import logging
|
||||
import toml
|
||||
import git
|
||||
import zipfile
|
||||
import shutil
|
||||
import tempfile
|
||||
import asyncio
|
||||
from aiohttp import web, ClientError
|
||||
from typing import Dict, List
|
||||
from typing import Any, Dict, List, cast
|
||||
|
||||
from ..utils.settings_paths import ensure_settings_file
|
||||
from ..services.downloader import get_downloader
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
NETWORK_EXCEPTIONS = (ClientError, OSError, asyncio.TimeoutError)
|
||||
|
||||
# User-managed directories that live inside the plugin folder (portable
|
||||
# mode) and must survive a Git-based update. ``git clean -fd`` would
|
||||
# otherwise delete them because they are untracked and, in released tags,
|
||||
# not listed in ``.gitignore``. ``-e`` excludes a path from cleaning
|
||||
# regardless of whether it is ignored.
|
||||
_PRESERVE_DIRS = ('settings.json', 'civitai', 'wildcards', 'backups', 'stats', 'logs', 'cache', 'model_cache')
|
||||
|
||||
|
||||
def _clean_excludes() -> List[str]:
|
||||
"""Build the ``-e`` arguments for ``git clean`` from :data:`_PRESERVE_DIRS`."""
|
||||
excludes: List[str] = []
|
||||
for name in _PRESERVE_DIRS:
|
||||
excludes.append('-e')
|
||||
excludes.append(name)
|
||||
# For directories, also exclude nested matches explicitly
|
||||
# (``-e dir`` alone matches the dir entry; ``-e dir/**`` guards
|
||||
# contents under all git versions as defense-in-depth).
|
||||
excludes.append('-e')
|
||||
excludes.append(f'{name}/**')
|
||||
return excludes
|
||||
|
||||
|
||||
def _stage_preserved_items(plugin_root: str) -> tuple[str, list[str]]:
|
||||
"""Move preserved user-data items to a temp directory outside *plugin_root*.
|
||||
|
||||
This ensures that ``git reset --hard``, ``git clean -fd``, and ZIP-based
|
||||
replacement cannot touch these files even when ``-e`` exclusion patterns
|
||||
are mishandled (e.g. on Windows where forward-slash patterns may not
|
||||
match backslash-prefixed paths in some Git builds, or where file locks
|
||||
prevent deletion/recreation).
|
||||
|
||||
Returns:
|
||||
``(backup_root, staged_names)``: the temp directory path and the
|
||||
list of item names that were successfully moved.
|
||||
"""
|
||||
backup_root = tempfile.mkdtemp(prefix='lora_manager_update_')
|
||||
staged: list[str] = []
|
||||
for name in _PRESERVE_DIRS:
|
||||
src = os.path.join(plugin_root, name)
|
||||
if not os.path.lexists(src):
|
||||
continue
|
||||
dst = os.path.join(backup_root, name)
|
||||
try:
|
||||
shutil.move(src, dst)
|
||||
staged.append(name)
|
||||
logger.debug("Staged '%s' for update safety", name)
|
||||
except OSError:
|
||||
# ``shutil.move`` may fail on Windows if a file handle inside
|
||||
# the directory is still open (e.g. a SQLite WAL file). Fall
|
||||
# back to copy-then-remove.
|
||||
logger.debug("Move failed for '%s', falling back to copy", name)
|
||||
try:
|
||||
if os.path.isdir(src) and not os.path.islink(src):
|
||||
shutil.copytree(src, dst, symlinks=True)
|
||||
shutil.rmtree(src, ignore_errors=True)
|
||||
else:
|
||||
shutil.copy2(src, dst)
|
||||
os.remove(src)
|
||||
staged.append(name)
|
||||
logger.info("Copied (then removed) '%s' for update safety", name)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Could not stage '%s': %s (will rely on git -e / skip lists)", name, exc
|
||||
)
|
||||
return backup_root, staged
|
||||
|
||||
|
||||
def _restore_preserved_items(plugin_root: str, backup_root: str, staged: list[str]) -> None:
|
||||
"""Move staged items back from *backup_root* into *plugin_root*.
|
||||
|
||||
Any leftover placeholder at the destination (created by git checkout or
|
||||
ZIP extraction) is removed before the move.
|
||||
"""
|
||||
for name in staged:
|
||||
src = os.path.join(backup_root, name)
|
||||
dst = os.path.join(plugin_root, name)
|
||||
try:
|
||||
if os.path.lexists(dst):
|
||||
if os.path.isdir(dst) and not os.path.islink(dst):
|
||||
shutil.rmtree(dst, ignore_errors=True)
|
||||
else:
|
||||
os.remove(dst)
|
||||
shutil.move(src, dst)
|
||||
logger.debug("Restored '%s' after update", name)
|
||||
except OSError:
|
||||
logger.debug("Move failed restoring '%s', falling back to copy", name)
|
||||
try:
|
||||
if os.path.isdir(src) and not os.path.islink(src):
|
||||
shutil.copytree(src, dst, symlinks=True, dirs_exist_ok=True)
|
||||
shutil.rmtree(src, ignore_errors=True)
|
||||
else:
|
||||
shutil.copy2(src, dst)
|
||||
os.remove(src)
|
||||
logger.info("Copied '%s' back after update", name)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to restore '%s': %s", name, exc)
|
||||
shutil.rmtree(backup_root, ignore_errors=True)
|
||||
|
||||
|
||||
|
||||
class UpdateRoutes:
|
||||
"""Routes for handling plugin update checks"""
|
||||
@@ -26,6 +125,7 @@ class UpdateRoutes:
|
||||
app.router.add_get('/api/lm/check-updates', UpdateRoutes.check_updates)
|
||||
app.router.add_get('/api/lm/version-info', UpdateRoutes.get_version_info)
|
||||
app.router.add_post('/api/lm/perform-update', UpdateRoutes.perform_update)
|
||||
app.router.add_post('/api/lm/switch-channel', UpdateRoutes.switch_channel)
|
||||
|
||||
@staticmethod
|
||||
async def check_updates(request):
|
||||
@@ -44,10 +144,17 @@ class UpdateRoutes:
|
||||
|
||||
# Fetch remote version from GitHub
|
||||
if nightly:
|
||||
remote_version, changelog = await UpdateRoutes._get_nightly_version()
|
||||
releases = None
|
||||
local_hash = git_info.get('short_hash', '')
|
||||
nightly_version, releases_result = await asyncio.gather(
|
||||
UpdateRoutes._get_nightly_version(local_hash),
|
||||
UpdateRoutes._get_remote_version()
|
||||
)
|
||||
remote_version, _, behind_by, commit_date = nightly_version
|
||||
_, changelog, releases = releases_result
|
||||
else:
|
||||
remote_version, changelog, releases = await UpdateRoutes._get_remote_version()
|
||||
behind_by = 0
|
||||
commit_date = ''
|
||||
|
||||
# Compare versions
|
||||
if nightly:
|
||||
@@ -60,6 +167,10 @@ class UpdateRoutes:
|
||||
remote_version.replace('v', '')
|
||||
)
|
||||
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
plugin_root = os.path.dirname(os.path.dirname(current_dir))
|
||||
has_git = os.path.exists(os.path.join(plugin_root, '.git'))
|
||||
|
||||
response_data = {
|
||||
'success': True,
|
||||
'current_version': local_version,
|
||||
@@ -67,13 +178,13 @@ class UpdateRoutes:
|
||||
'update_available': update_available,
|
||||
'changelog': changelog,
|
||||
'git_info': git_info,
|
||||
'nightly': nightly
|
||||
'nightly': nightly,
|
||||
'has_git': has_git,
|
||||
'releases': releases,
|
||||
'behind_by': behind_by,
|
||||
'commit_date': commit_date
|
||||
}
|
||||
|
||||
# Include releases list for stable mode
|
||||
if releases is not None:
|
||||
response_data['releases'] = releases
|
||||
|
||||
return web.json_response(response_data)
|
||||
|
||||
except NETWORK_EXCEPTIONS as e:
|
||||
@@ -105,9 +216,14 @@ class UpdateRoutes:
|
||||
# Format: version-short_hash
|
||||
version_string = f"{local_version}-{short_hash}"
|
||||
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
plugin_root = os.path.dirname(os.path.dirname(current_dir))
|
||||
has_git = os.path.exists(os.path.join(plugin_root, '.git'))
|
||||
|
||||
return web.json_response({
|
||||
'success': True,
|
||||
'version': version_string
|
||||
'version': version_string,
|
||||
'has_git': has_git
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
@@ -135,20 +251,22 @@ class UpdateRoutes:
|
||||
if os.path.exists(settings_path):
|
||||
with open(settings_path, 'r', encoding='utf-8') as f:
|
||||
settings_backup = f.read()
|
||||
logger.info("Backed up settings.json")
|
||||
logger.debug("Backed up settings.json (%d bytes)", len(settings_backup))
|
||||
|
||||
git_folder = os.path.join(plugin_root, '.git')
|
||||
if os.path.exists(git_folder):
|
||||
# Git update
|
||||
success, new_version = await UpdateRoutes._perform_git_update(plugin_root, nightly)
|
||||
else:
|
||||
# Fallback: Download ZIP and replace files
|
||||
success, new_version = await UpdateRoutes._download_and_replace_zip(plugin_root)
|
||||
staged_backup_dir, staged_items = _stage_preserved_items(plugin_root)
|
||||
try:
|
||||
git_folder = os.path.join(plugin_root, '.git')
|
||||
if os.path.exists(git_folder):
|
||||
success, new_version = await UpdateRoutes._perform_git_update(plugin_root, nightly)
|
||||
else:
|
||||
success, new_version = await UpdateRoutes._download_and_replace_zip(plugin_root)
|
||||
finally:
|
||||
_restore_preserved_items(plugin_root, staged_backup_dir, staged_items)
|
||||
|
||||
if settings_backup and success:
|
||||
with open(settings_path, 'w', encoding='utf-8') as f:
|
||||
f.write(settings_backup)
|
||||
logger.info("Restored settings.json")
|
||||
logger.debug("Restored settings.json content (%d bytes)", len(settings_backup))
|
||||
|
||||
if success:
|
||||
return web.json_response({
|
||||
@@ -169,6 +287,164 @@ class UpdateRoutes:
|
||||
'error': str(e)
|
||||
})
|
||||
|
||||
@staticmethod
|
||||
async def switch_channel(request):
|
||||
"""
|
||||
Switch between release and nightly update channels.
|
||||
|
||||
ZIP/CNR install → Nightly: git init + checkout main (one-way upgrade)
|
||||
Git install → Release: git checkout latest tag (.git preserved)
|
||||
ZIP/CNR install → Release: ZIP download (no .git, stays in ZIP mode)
|
||||
Git install → Nightly: git checkout main + pull
|
||||
"""
|
||||
try:
|
||||
body = await request.json() if request.has_body else {}
|
||||
channel = body.get('channel', '')
|
||||
|
||||
if channel not in ('release', 'nightly'):
|
||||
return web.json_response({
|
||||
'success': False,
|
||||
'error': f'Invalid channel: {channel}. Must be "release" or "nightly".'
|
||||
})
|
||||
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
plugin_root = os.path.dirname(os.path.dirname(current_dir))
|
||||
|
||||
settings_path = ensure_settings_file(logger)
|
||||
settings_backup = None
|
||||
if os.path.exists(settings_path):
|
||||
with open(settings_path, 'r', encoding='utf-8') as f:
|
||||
settings_backup = f.read()
|
||||
logger.debug("Backed up settings.json before channel switch (%d bytes)", len(settings_backup))
|
||||
|
||||
staged_backup_dir, staged_items = _stage_preserved_items(plugin_root)
|
||||
try:
|
||||
git_folder = os.path.join(plugin_root, '.git')
|
||||
|
||||
if channel == 'nightly':
|
||||
git_backup = None
|
||||
if os.path.exists(git_folder):
|
||||
git_backup = UpdateRoutes._backup_git(git_folder, 'nightly')
|
||||
|
||||
success = False
|
||||
new_version = ''
|
||||
try:
|
||||
if os.path.exists(git_folder):
|
||||
success, new_version = await UpdateRoutes._perform_git_update(
|
||||
plugin_root, nightly=True
|
||||
)
|
||||
else:
|
||||
success, new_version = UpdateRoutes._init_git_repo(plugin_root)
|
||||
finally:
|
||||
UpdateRoutes._restore_git(git_backup, git_folder, success, 'nightly')
|
||||
else:
|
||||
success = False
|
||||
new_version = ''
|
||||
if os.path.exists(git_folder):
|
||||
success, new_version = await UpdateRoutes._perform_git_update(
|
||||
plugin_root, nightly=False
|
||||
)
|
||||
else:
|
||||
tracking_file = os.path.join(plugin_root, '.tracking')
|
||||
if os.path.exists(tracking_file):
|
||||
os.remove(tracking_file)
|
||||
success, new_version = await UpdateRoutes._download_and_replace_zip(plugin_root)
|
||||
finally:
|
||||
_restore_preserved_items(plugin_root, staged_backup_dir, staged_items)
|
||||
|
||||
if settings_backup and success:
|
||||
with open(settings_path, 'w', encoding='utf-8') as f:
|
||||
f.write(settings_backup)
|
||||
logger.debug("Restored settings.json content after channel switch (%d bytes)", len(settings_backup))
|
||||
|
||||
if success:
|
||||
return web.json_response({
|
||||
'success': True,
|
||||
'channel': channel,
|
||||
'new_version': new_version,
|
||||
'message': f'Switched to {channel} channel'
|
||||
})
|
||||
else:
|
||||
return web.json_response({
|
||||
'success': False,
|
||||
'error': f'Failed to switch to {channel} channel'
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to switch channel: %s", e, exc_info=True)
|
||||
return web.json_response({
|
||||
'success': False,
|
||||
'error': str(e)
|
||||
})
|
||||
|
||||
@staticmethod
|
||||
def _init_git_repo(plugin_root: str) -> tuple[bool, str]:
|
||||
"""
|
||||
Initialize a Git repository in a ZIP-installed plugin folder.
|
||||
Clones the remote history and checks out main branch.
|
||||
"""
|
||||
try:
|
||||
import git
|
||||
except ImportError:
|
||||
logger.error(
|
||||
"GitPython is not available: cannot initialize git repo. "
|
||||
"Install git or set $GIT_PYTHON_GIT_EXECUTABLE to the git binary path."
|
||||
)
|
||||
return False, ""
|
||||
|
||||
clean_excludes = _clean_excludes()
|
||||
|
||||
try:
|
||||
repo = git.Repo.init(plugin_root)
|
||||
origin = repo.create_remote(
|
||||
'origin',
|
||||
'https://github.com/willmiao/ComfyUI-Lora-Manager.git'
|
||||
)
|
||||
origin.fetch()
|
||||
|
||||
repo.create_head('main', origin.refs.main)
|
||||
repo.git.checkout('main', '--force')
|
||||
repo.git.reset('--hard')
|
||||
repo.git.clean('-fd', *clean_excludes)
|
||||
|
||||
tracking_file = os.path.join(plugin_root, '.tracking')
|
||||
if os.path.exists(tracking_file):
|
||||
os.remove(tracking_file)
|
||||
logger.info("Removed .tracking file (now in git mode)")
|
||||
|
||||
new_version = f"main-{repo.head.commit.hexsha[:7]}"
|
||||
logger.info("Initialized git repo on main branch: %s", new_version)
|
||||
return True, new_version
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to initialize git repo: %s", e, exc_info=True)
|
||||
return False, ""
|
||||
|
||||
@staticmethod
|
||||
def _backup_git(git_folder, label):
|
||||
try:
|
||||
backup_dir = tempfile.mkdtemp()
|
||||
backup = os.path.join(backup_dir, '.git')
|
||||
shutil.copytree(git_folder, backup)
|
||||
logger.info("Backed up .git before switching to %s", label)
|
||||
return backup
|
||||
except Exception as e:
|
||||
logger.error("Failed to backup .git before %s switch: %s", label, e)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _restore_git(git_backup, git_folder, success, label):
|
||||
if git_backup and not success:
|
||||
try:
|
||||
if os.path.exists(git_folder):
|
||||
shutil.rmtree(git_folder)
|
||||
shutil.copytree(git_backup, git_folder)
|
||||
logger.info("Restored .git after failed %s switch", label)
|
||||
except Exception as e:
|
||||
logger.error("Failed to restore .git after %s switch: %s", label, e)
|
||||
if git_backup:
|
||||
shutil.rmtree(os.path.dirname(git_backup), ignore_errors=True)
|
||||
|
||||
@staticmethod
|
||||
async def _download_and_replace_zip(plugin_root: str) -> tuple[bool, str]:
|
||||
"""
|
||||
@@ -191,9 +467,10 @@ class UpdateRoutes:
|
||||
if not success:
|
||||
logger.error(f"Failed to fetch release info: {data}")
|
||||
return False, ""
|
||||
|
||||
zip_url = data.get("zipball_url")
|
||||
version = data.get("tag_name", "unknown")
|
||||
|
||||
release_payload = cast(dict[str, Any], data)
|
||||
zip_url = release_payload.get("zipball_url", "")
|
||||
version = release_payload.get("tag_name", "unknown")
|
||||
|
||||
# Download ZIP to temporary file
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".zip") as tmp_zip:
|
||||
@@ -212,8 +489,18 @@ class UpdateRoutes:
|
||||
|
||||
zip_path = tmp_zip_path
|
||||
|
||||
# Skip both settings.json, civitai and model cache folder
|
||||
UpdateRoutes._clean_plugin_folder(plugin_root, skip_files=['settings.json', 'civitai', 'model_cache'])
|
||||
# Close the downloaded-versions SQLite connection before cleaning,
|
||||
# so that shutil.rmtree() does not fail on Windows (the process
|
||||
# cannot delete a file with an outstanding open handle).
|
||||
try:
|
||||
history_svc = ServiceRegistry._services.get("downloaded_version_history_service")
|
||||
if history_svc is not None:
|
||||
history_svc.close()
|
||||
logger.info("Closed downloaded-version history database connection")
|
||||
except Exception:
|
||||
logger.debug("Could not close downloaded-version history database", exc_info=True)
|
||||
|
||||
UpdateRoutes._clean_plugin_folder(plugin_root, skip_files=list(_PRESERVE_DIRS))
|
||||
|
||||
# Extract ZIP to temp dir
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
@@ -222,16 +509,17 @@ class UpdateRoutes:
|
||||
# Find extracted folder (GitHub ZIP contains a root folder)
|
||||
extracted_root = next(os.scandir(tmp_dir)).path
|
||||
|
||||
# Copy files, skipping settings.json and civitai folder
|
||||
# Copy files, skipping user data that should be preserved
|
||||
skip_items = set(_PRESERVE_DIRS)
|
||||
for item in os.listdir(extracted_root):
|
||||
if item == 'settings.json' or item == 'civitai':
|
||||
if item in skip_items:
|
||||
continue
|
||||
src = os.path.join(extracted_root, item)
|
||||
dst = os.path.join(plugin_root, item)
|
||||
if os.path.isdir(src):
|
||||
if os.path.exists(dst):
|
||||
shutil.rmtree(dst)
|
||||
shutil.copytree(src, dst, ignore=shutil.ignore_patterns('settings.json', 'civitai'))
|
||||
shutil.copytree(src, dst, ignore=shutil.ignore_patterns(*skip_items))
|
||||
else:
|
||||
shutil.copy2(src, dst)
|
||||
|
||||
@@ -239,15 +527,17 @@ class UpdateRoutes:
|
||||
# for ComfyUI Manager to work properly
|
||||
tracking_info_file = os.path.join(plugin_root, '.tracking')
|
||||
tracking_files = []
|
||||
skip_tracked = set(_PRESERVE_DIRS) - {'settings.json'}
|
||||
for root, dirs, files in os.walk(extracted_root):
|
||||
# Skip civitai folder and its contents
|
||||
# Skip user data directories and their contents
|
||||
rel_root = os.path.relpath(root, extracted_root)
|
||||
if rel_root == 'civitai' or rel_root.startswith('civitai' + os.sep):
|
||||
top_dir = rel_root.split(os.sep)[0] if rel_root != '.' else ''
|
||||
if top_dir in skip_tracked:
|
||||
continue
|
||||
for file in files:
|
||||
rel_path = os.path.relpath(os.path.join(root, file), extracted_root)
|
||||
# Skip settings.json and any file under civitai
|
||||
if rel_path == 'settings.json' or rel_path.startswith('civitai' + os.sep):
|
||||
# Skip settings.json and any file under user data dirs
|
||||
if rel_path == 'settings.json' or rel_path.split(os.sep)[0] in skip_tracked:
|
||||
continue
|
||||
tracking_files.append(rel_path.replace("\\", "/"))
|
||||
with open(tracking_info_file, "w", encoding='utf-8') as file:
|
||||
@@ -260,7 +550,8 @@ class UpdateRoutes:
|
||||
except Exception as e:
|
||||
logger.error(f"ZIP update failed: {e}", exc_info=True)
|
||||
return False, ""
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _clean_plugin_folder(plugin_root, skip_files=None):
|
||||
skip_files = skip_files or []
|
||||
for item in os.listdir(plugin_root):
|
||||
@@ -273,41 +564,56 @@ class UpdateRoutes:
|
||||
os.remove(path)
|
||||
|
||||
@staticmethod
|
||||
async def _get_nightly_version() -> tuple[str, List[str]]:
|
||||
"""
|
||||
Fetch latest commit from main branch
|
||||
"""
|
||||
async def _get_nightly_version(local_hash: str = "") -> tuple[str, List[str], int, str]:
|
||||
repo_owner = "willmiao"
|
||||
repo_name = "ComfyUI-Lora-Manager"
|
||||
|
||||
# Use GitHub API to fetch the latest commit from main branch
|
||||
|
||||
github_url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/commits/main"
|
||||
|
||||
|
||||
try:
|
||||
downloader = await get_downloader()
|
||||
success, data = await downloader.make_request('GET', github_url, custom_headers={'Accept': 'application/vnd.github+json'})
|
||||
|
||||
success, data = await downloader.make_request(
|
||||
'GET', github_url,
|
||||
custom_headers={'Accept': 'application/vnd.github+json'}
|
||||
)
|
||||
|
||||
if not success:
|
||||
logger.warning(f"Failed to fetch GitHub commit: {data}")
|
||||
return "main", []
|
||||
|
||||
commit_sha = data.get('sha', '')[:7] # Short hash
|
||||
commit_message = data.get('commit', {}).get('message', '')
|
||||
|
||||
# Format as "main-{short_hash}"
|
||||
logger.warning("Failed to fetch GitHub commit: %s", data)
|
||||
return "main", [], 0, ""
|
||||
|
||||
commit_payload = cast(dict[str, Any], data)
|
||||
commit_sha = commit_payload.get('sha', '')[:7]
|
||||
commit_message = commit_payload.get('commit', {}).get('message', '')
|
||||
commit_date = commit_payload.get('commit', {}).get('committer', {}).get('date', '')[:10]
|
||||
|
||||
version = f"main-{commit_sha}"
|
||||
|
||||
# Use commit message as changelog
|
||||
changelog = [commit_message] if commit_message else []
|
||||
|
||||
return version, changelog
|
||||
|
||||
|
||||
behind_by = 0
|
||||
if local_hash and local_hash not in ('unknown', 'stable'):
|
||||
compare_url = (
|
||||
f"https://api.github.com/repos/{repo_owner}/{repo_name}"
|
||||
f"/compare/{local_hash}...main"
|
||||
)
|
||||
c_ok, c_data = await downloader.make_request(
|
||||
'GET', compare_url,
|
||||
custom_headers={'Accept': 'application/vnd.github+json'}
|
||||
)
|
||||
if c_ok:
|
||||
compare_payload = cast(dict[str, Any], c_data)
|
||||
if compare_payload.get('status') in ('ahead', 'diverged'):
|
||||
behind_by = compare_payload.get('ahead_by', 0)
|
||||
else:
|
||||
behind_by = compare_payload.get('behind_by', 0)
|
||||
|
||||
return version, changelog, behind_by, commit_date
|
||||
|
||||
except NETWORK_EXCEPTIONS as e:
|
||||
logger.warning("Unable to reach GitHub for nightly version: %s", e)
|
||||
return "main", []
|
||||
return "main", [], 0, ""
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching nightly version: {e}", exc_info=True)
|
||||
return "main", []
|
||||
logger.error("Error fetching nightly version: %s", e, exc_info=True)
|
||||
return "main", [], 0, ""
|
||||
|
||||
@staticmethod
|
||||
def _compare_nightly_versions(local_git_info: Dict[str, str], remote_version: str) -> bool:
|
||||
@@ -342,6 +648,17 @@ class UpdateRoutes:
|
||||
Returns:
|
||||
tuple: (success, new_version)
|
||||
"""
|
||||
try:
|
||||
import git
|
||||
except ImportError:
|
||||
logger.error(
|
||||
"GitPython is not available: the git executable was not found in PATH. "
|
||||
"Install git or set $GIT_PYTHON_GIT_EXECUTABLE to the git binary path."
|
||||
)
|
||||
return False, ""
|
||||
|
||||
clean_excludes = _clean_excludes()
|
||||
|
||||
try:
|
||||
# Open the Git repository
|
||||
repo = git.Repo(plugin_root)
|
||||
@@ -353,8 +670,9 @@ class UpdateRoutes:
|
||||
if nightly:
|
||||
# Reset to discard any local changes
|
||||
repo.git.reset('--hard')
|
||||
# Clean untracked files
|
||||
repo.git.clean('-fd')
|
||||
# Clean untracked files, but preserve user-managed directories
|
||||
# (wildcards, backups, stats, civitai, caches, settings.json).
|
||||
repo.git.clean('-fd', *clean_excludes)
|
||||
|
||||
# Switch to main branch and pull latest
|
||||
main_branch = 'main'
|
||||
@@ -371,8 +689,9 @@ class UpdateRoutes:
|
||||
else:
|
||||
# Reset to discard any local changes
|
||||
repo.git.reset('--hard')
|
||||
# Clean untracked files
|
||||
repo.git.clean('-fd')
|
||||
# Clean untracked files, but preserve user-managed directories
|
||||
# (wildcards, backups, stats, civitai, caches, settings.json).
|
||||
repo.git.clean('-fd', *clean_excludes)
|
||||
|
||||
# Get latest release tag
|
||||
tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime, reverse=True)
|
||||
@@ -390,7 +709,7 @@ class UpdateRoutes:
|
||||
logger.info(f"Successfully updated to {new_version}")
|
||||
return True, new_version
|
||||
|
||||
except git.exc.GitError as e:
|
||||
except git.exc.GitError as e: # pyright: ignore[reportAttributeAccessIssue]
|
||||
logger.error(f"Git error during update: {e}")
|
||||
return False, ""
|
||||
except Exception as e:
|
||||
@@ -438,6 +757,7 @@ class UpdateRoutes:
|
||||
if not os.path.exists(os.path.join(plugin_root, '.git')):
|
||||
return git_info
|
||||
|
||||
import git
|
||||
repo = git.Repo(plugin_root)
|
||||
commit = repo.head.commit
|
||||
git_info['commit_hash'] = commit.hexsha
|
||||
@@ -450,7 +770,7 @@ class UpdateRoutes:
|
||||
return git_info
|
||||
|
||||
@staticmethod
|
||||
async def _get_remote_version() -> tuple[str, List[str], List[Dict]]:
|
||||
async def _get_remote_version() -> tuple[str, List[str], List[Dict[str, Any]]]:
|
||||
"""
|
||||
Fetch remote version from GitHub
|
||||
Returns:
|
||||
@@ -472,7 +792,7 @@ class UpdateRoutes:
|
||||
|
||||
# Parse releases
|
||||
releases = []
|
||||
for i, release in enumerate(data):
|
||||
for i, release in enumerate(cast(list[dict[str, Any]], data)):
|
||||
version = release.get('tag_name', '')
|
||||
if not version.startswith('v'):
|
||||
version = f"v{version}"
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
"""LLM-powered metadata enrichment pipeline infrastructure.
|
||||
|
||||
This package provides the orchestration layer for LLM-powered features.
|
||||
Skills define *what* to do (prompt template). The :class:`AgentService`
|
||||
handles *how* (LLM calls, context gathering, validation, progress).
|
||||
|
||||
NOTE: The current implementation is a code-driven pipeline, not a true
|
||||
agent loop. Future agent orchestration (LLM-driven tool selection) will
|
||||
live alongside this package with its own namespace.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .skill_definition import SkillDefinition, SkillPermissions
|
||||
from .skill_registry import SkillRegistry
|
||||
from .agent_service import AgentService, AgentProgressReporter, SkillResult
|
||||
from .post_processor import PostProcessor
|
||||
|
||||
__all__ = [
|
||||
"AgentProgressReporter",
|
||||
"AgentService",
|
||||
"PostProcessor",
|
||||
"SkillDefinition",
|
||||
"SkillPermissions",
|
||||
"SkillRegistry",
|
||||
"SkillResult",
|
||||
]
|
||||
@@ -0,0 +1,489 @@
|
||||
"""Pipeline orchestration service.
|
||||
|
||||
The :class:`AgentService` coordinates LLM-powered pipeline execution:
|
||||
|
||||
1. Look up the pipeline definition in :class:`SkillRegistry`
|
||||
2. Validate input against its ``input_schema``
|
||||
3. Prepare context via :mod:`~py.metadata_ops` (read metadata, list base models, fetch HF README)
|
||||
4. If ``llm_required``: call :class:`LLMService` with the rendered prompt
|
||||
5. Post-process via :class:`PostProcessor` (delegates I/O to :mod:`~py.metadata_ops`)
|
||||
6. Broadcast progress and completion via :class:`WebSocketManager`
|
||||
|
||||
Pipeline definitions (*skills*) describe *what* to do (prompt template).
|
||||
The AgentService handles *how* (LLM calls, context gathering, validation,
|
||||
progress).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import aiohttp
|
||||
|
||||
import os
|
||||
|
||||
from ...config import config
|
||||
from ..llm_service import LLMService
|
||||
from ..websocket_manager import ws_manager
|
||||
from .post_processor import PostProcessor
|
||||
from .skill_registry import SkillRegistry
|
||||
from .skills.enrich_hf_metadata.readme_processor import (
|
||||
clean_readme_for_llm,
|
||||
extract_relevant_section,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AgentProgressReporter:
|
||||
"""Protocol-compatible progress reporter backed by WebSocket broadcast."""
|
||||
|
||||
async def on_progress(self, payload: Dict[str, Any]) -> None:
|
||||
await ws_manager.broadcast(payload)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SkillResult:
|
||||
"""Outcome of a skill execution."""
|
||||
|
||||
success: bool
|
||||
updated_models: List[Dict[str, Any]] = field(default_factory=list)
|
||||
errors: List[str] = field(default_factory=list)
|
||||
summary: str = ""
|
||||
|
||||
|
||||
def _validate_schema(data: Any, schema: Dict[str, Any], path: str = "") -> List[str]:
|
||||
"""Minimal JSON schema validator.
|
||||
|
||||
Supports a subset of JSON Schema: ``type``, ``properties``, ``required``,
|
||||
``items``, ``enum``. Returns a list of error messages (empty = valid).
|
||||
"""
|
||||
|
||||
errors: List[str] = []
|
||||
if not schema:
|
||||
return errors
|
||||
|
||||
expected_type = schema.get("type")
|
||||
if expected_type:
|
||||
type_map = {
|
||||
"string": str,
|
||||
"number": (int, float),
|
||||
"integer": int,
|
||||
"boolean": bool,
|
||||
"array": list,
|
||||
"object": dict,
|
||||
"null": type(None),
|
||||
}
|
||||
expected_py = type_map.get(expected_type)
|
||||
if expected_py is not None and not isinstance(data, expected_py):
|
||||
errors.append(f"{path or 'root'}: expected {expected_type}, got {type(data).__name__}")
|
||||
return errors
|
||||
|
||||
if expected_type == "object" and isinstance(data, dict):
|
||||
properties = schema.get("properties", {})
|
||||
required = schema.get("required", [])
|
||||
for req_key in required:
|
||||
if req_key not in data:
|
||||
errors.append(f"{path or 'root'}: missing required property '{req_key}'")
|
||||
for key, value in data.items():
|
||||
if key in properties:
|
||||
errors.extend(_validate_schema(value, properties[key], f"{path}.{key}"))
|
||||
|
||||
if expected_type == "array" and isinstance(data, list):
|
||||
items_schema = schema.get("items")
|
||||
if items_schema:
|
||||
for i, item in enumerate(data):
|
||||
errors.extend(_validate_schema(item, items_schema, f"{path}[{i}]"))
|
||||
|
||||
if "enum" in schema and data not in schema["enum"]:
|
||||
errors.append(f"{path or 'root'}: value '{data}' not in enum {schema['enum']}")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Prompt template rendering
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
def _render_prompt(template: str, variables: Dict[str, Any]) -> str:
|
||||
"""Render a prompt template with ``{{variable}}`` placeholders.
|
||||
|
||||
Uses simple regex substitution — no Jinja2 dependency needed.
|
||||
"""
|
||||
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
key = match.group(1).strip()
|
||||
value = variables.get(key, "")
|
||||
if isinstance(value, (dict, list)):
|
||||
return json.dumps(value, ensure_ascii=False, indent=2)
|
||||
return str(value)
|
||||
|
||||
return re.sub(r"\{\{(\w+)\}\}", replace, template)
|
||||
|
||||
|
||||
class AgentService:
|
||||
"""Orchestrate agent skill execution.
|
||||
|
||||
Usage::
|
||||
|
||||
service = await AgentService.get_instance()
|
||||
result = await service.execute_skill(
|
||||
skill_name="enrich_hf_metadata",
|
||||
input_data={"model_paths": ["/path/to/model.safetensors"]},
|
||||
progress_callback=AgentProgressReporter(),
|
||||
)
|
||||
"""
|
||||
|
||||
_instance: Optional["AgentService"] = None
|
||||
_lock: asyncio.Lock = asyncio.Lock()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
skill_registry: Optional[SkillRegistry] = None,
|
||||
llm_service: Optional[LLMService] = None,
|
||||
) -> None:
|
||||
self._registry = skill_registry
|
||||
self._llm_service = llm_service
|
||||
|
||||
@classmethod
|
||||
async def get_instance(cls) -> "AgentService":
|
||||
"""Return the lazily-initialised global ``AgentService``."""
|
||||
|
||||
if cls._instance is None:
|
||||
async with cls._lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = cls(
|
||||
skill_registry=await SkillRegistry.get_instance(),
|
||||
llm_service=await LLMService.get_instance(),
|
||||
)
|
||||
return cls._instance
|
||||
|
||||
@classmethod
|
||||
def reset_instance(cls) -> None:
|
||||
"""Reset the cached singleton — primarily for tests."""
|
||||
|
||||
cls._instance = None
|
||||
|
||||
async def _ensure_registry(self) -> SkillRegistry:
|
||||
if self._registry is None:
|
||||
self._registry = await SkillRegistry.get_instance()
|
||||
return self._registry
|
||||
|
||||
async def _ensure_llm(self) -> LLMService:
|
||||
if self._llm_service is None:
|
||||
self._llm_service = await LLMService.get_instance()
|
||||
return self._llm_service
|
||||
|
||||
async def list_skills(self) -> List[Dict[str, Any]]:
|
||||
"""Return a JSON-serialisable list of available skills."""
|
||||
|
||||
registry = await self._ensure_registry()
|
||||
return [
|
||||
{
|
||||
"name": s.name,
|
||||
"title": s.title,
|
||||
"description": s.description,
|
||||
"llm_required": s.llm_required,
|
||||
"model_type_filter": s.model_type_filter,
|
||||
}
|
||||
for s in registry.list_skills()
|
||||
]
|
||||
|
||||
async def execute_skill(
|
||||
self,
|
||||
*,
|
||||
skill_name: str,
|
||||
input_data: Dict[str, Any],
|
||||
progress_callback: Optional[AgentProgressReporter] = None,
|
||||
) -> SkillResult:
|
||||
"""Execute a pipeline (skill) on the given models.
|
||||
|
||||
Args:
|
||||
skill_name: Name of the pipeline to execute
|
||||
input_data: Input validated against the pipeline's ``input_schema``
|
||||
progress_callback: Optional WebSocket progress reporter
|
||||
|
||||
Returns:
|
||||
:class:`SkillResult` with success status and updated model info
|
||||
"""
|
||||
|
||||
registry = await self._ensure_registry()
|
||||
skill = registry.get_skill(skill_name)
|
||||
if skill is None:
|
||||
return SkillResult(
|
||||
success=False,
|
||||
errors=[f"Skill not found: {skill_name}"],
|
||||
summary=f"Skill '{skill_name}' does not exist",
|
||||
)
|
||||
|
||||
input_errors = _validate_schema(input_data, skill.input_schema)
|
||||
if input_errors:
|
||||
return SkillResult(
|
||||
success=False,
|
||||
errors=input_errors,
|
||||
summary=f"Invalid input: {'; '.join(input_errors)}",
|
||||
)
|
||||
|
||||
model_paths = input_data.get("model_paths", [])
|
||||
if not model_paths:
|
||||
return SkillResult(
|
||||
success=False,
|
||||
errors=["No model_paths provided"],
|
||||
summary="No models to process",
|
||||
)
|
||||
|
||||
total = len(model_paths)
|
||||
processed = 0
|
||||
success_count = 0
|
||||
skipped_count = 0
|
||||
updated_models: List[Dict[str, Any]] = []
|
||||
errors: List[str] = []
|
||||
post_processor = PostProcessor()
|
||||
|
||||
await self._emit_progress(
|
||||
progress_callback, skill_name, status="started",
|
||||
total=total, processed=0, success=0,
|
||||
)
|
||||
|
||||
llm = await self._ensure_llm()
|
||||
llm_configured = llm.is_configured() if skill.llm_required else True
|
||||
|
||||
for model_path in model_paths:
|
||||
model_filename = os.path.basename(model_path)
|
||||
logger.info(
|
||||
"[%s] [%d/%d] %s",
|
||||
skill_name, processed + 1, total, model_filename,
|
||||
)
|
||||
updated_data: Dict[str, Any] = {}
|
||||
skip_model = False
|
||||
try:
|
||||
from ...metadata_ops import read_metadata
|
||||
metadata = await read_metadata(model_path)
|
||||
|
||||
# Fast-fail: enrich_hf_metadata requires hf_url to have HF README context
|
||||
if skill_name == "enrich_hf_metadata" and not metadata.get("hf_url", ""):
|
||||
logger.info(
|
||||
"[%s] SKIP %s — no hf_url in metadata",
|
||||
skill_name, model_filename,
|
||||
)
|
||||
skipped_count += 1
|
||||
skip_model = True
|
||||
|
||||
if not skip_model:
|
||||
prompt_vars: Dict[str, Any] = {"model_path": model_path}
|
||||
if skill.llm_required and llm_configured:
|
||||
prompt_vars = await self._build_prompt_context(
|
||||
skill_name, model_path, metadata, registry, llm,
|
||||
)
|
||||
|
||||
llm_response: Optional[Dict[str, Any]] = None
|
||||
if skill.llm_required and llm_configured:
|
||||
prompt_template = registry.load_prompt(skill_name)
|
||||
rendered = _render_prompt(prompt_template, prompt_vars)
|
||||
llm_response = await llm.chat_completion_json(
|
||||
system_prompt=prompt_vars.get(
|
||||
"system_prompt",
|
||||
"You are a helpful assistant that extracts structured metadata.",
|
||||
),
|
||||
user_prompt=rendered,
|
||||
)
|
||||
if llm_response:
|
||||
logger.info(
|
||||
"[%s] [%d/%d] %s → base_model=%s confidence=%s",
|
||||
skill_name, processed + 1, total, model_filename,
|
||||
(llm_response.get("base_model") or "?")[:50],
|
||||
llm_response.get("confidence", "?"),
|
||||
)
|
||||
|
||||
model_result = await post_processor.process(
|
||||
skill_name=skill_name,
|
||||
model_path=model_path,
|
||||
llm_output=llm_response or {},
|
||||
metadata=metadata,
|
||||
readme_content=prompt_vars.get("readme_content_full", ""),
|
||||
)
|
||||
|
||||
if model_result.get("success", True):
|
||||
success_count += 1
|
||||
uf = model_result.get("updated_fields", [])
|
||||
if uf:
|
||||
updated_models.append({"path": model_path, "updated_fields": uf})
|
||||
updated_data = model_result.get("updates", {})
|
||||
if "preview_url" in updated_data and updated_data["preview_url"]:
|
||||
updated_data["preview_url"] = config.get_preview_static_url(
|
||||
updated_data["preview_url"]
|
||||
)
|
||||
else:
|
||||
errors.extend(
|
||||
model_result.get("errors", [model_result.get("error", "Unknown error")])
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Skill %s failed for %s: %s", skill_name, model_path, exc)
|
||||
errors.append(f"{model_path}: {exc}")
|
||||
|
||||
processed += 1
|
||||
await self._emit_progress(
|
||||
progress_callback, skill_name, status="processing",
|
||||
total=total, processed=processed, success=success_count,
|
||||
skipped=skipped_count,
|
||||
current_path=model_path,
|
||||
updated_data=updated_data,
|
||||
)
|
||||
|
||||
result = SkillResult(
|
||||
success=success_count > 0,
|
||||
updated_models=updated_models,
|
||||
errors=errors,
|
||||
summary=f"Processed {processed}/{total} models, {success_count} succeeded, {skipped_count} skipped",
|
||||
)
|
||||
|
||||
await self._emit_progress(
|
||||
progress_callback, skill_name, status="completed",
|
||||
total=total, processed=processed, success=success_count,
|
||||
skipped=skipped_count,
|
||||
updated_models=updated_models, errors=errors, summary=result.summary,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Base model grouping (keeps the prompt compact)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _format_base_models(models: List[str]) -> str:
|
||||
"""Format the base model list as a flat, one-per-line list.
|
||||
|
||||
Attempts to group by family consistently degraded LLM extraction
|
||||
accuracy — the LLM finds individual model names harder to spot
|
||||
in comma-separated groups than in a simple ``- Name`` list.
|
||||
"""
|
||||
return "\n".join(f"- {m}" for m in models)
|
||||
|
||||
async def _build_prompt_context(
|
||||
self,
|
||||
skill_name: str,
|
||||
model_path: str,
|
||||
metadata: Dict[str, Any],
|
||||
registry: SkillRegistry,
|
||||
llm: Any,
|
||||
) -> Dict[str, Any]:
|
||||
"""Gather variables for the skill's prompt template.
|
||||
|
||||
Reads metadata, fetches the HF README (if applicable), lists available
|
||||
base models, loads user priority tags, and returns a dict that maps to
|
||||
``{{variable}}`` placeholders in ``prompt.md``.
|
||||
"""
|
||||
from ...metadata_ops import identify_model_type, list_base_models
|
||||
from ..settings_manager import SettingsManager
|
||||
|
||||
context: Dict[str, Any] = {
|
||||
"model_path": model_path,
|
||||
"model_basename": "",
|
||||
"hf_url": "",
|
||||
"repo": "",
|
||||
"readme_content": "",
|
||||
"readme_content_full": "",
|
||||
"current_metadata": {},
|
||||
"base_models": [],
|
||||
"priority_tags": "",
|
||||
}
|
||||
|
||||
# Extract model basename (filename without extension) for the LLM
|
||||
# to use when locating the matching section in collection repos.
|
||||
raw_basename = os.path.splitext(os.path.basename(model_path))[0]
|
||||
context["model_basename"] = raw_basename or ""
|
||||
|
||||
context["current_metadata"] = {
|
||||
"file_name": metadata.get("file_name", ""),
|
||||
"base_model": metadata.get("base_model", ""),
|
||||
"tags": metadata.get("tags", []),
|
||||
"modelDescription": metadata.get("modelDescription", ""),
|
||||
"trainedWords": metadata.get("trainedWords", []),
|
||||
"sha256": (metadata.get("sha256") or "")[:16] + "..." if metadata.get("sha256") else "",
|
||||
"size": metadata.get("size", 0),
|
||||
}
|
||||
|
||||
hf_url = metadata.get("hf_url", "")
|
||||
context["hf_url"] = hf_url
|
||||
repo = self._extract_repo_from_url(hf_url) if hf_url else ""
|
||||
context["repo"] = repo or ""
|
||||
if repo:
|
||||
readme = await self._fetch_readme(repo)
|
||||
# Trim README to the section relevant to this model file
|
||||
# (collection repos often have multiple models in one README).
|
||||
if readme and raw_basename:
|
||||
trimmed = extract_relevant_section(readme, raw_basename)
|
||||
cleaned = clean_readme_for_llm(trimmed) if trimmed else ""
|
||||
else:
|
||||
cleaned = clean_readme_for_llm(readme) if readme else ""
|
||||
context["readme_content"] = cleaned if cleaned else "(README not available)"
|
||||
context["readme_content_full"] = readme or ""
|
||||
|
||||
try:
|
||||
raw_models = await list_base_models()
|
||||
context["base_models"] = self._format_base_models(raw_models)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to list base models: %s", exc)
|
||||
context["base_models"] = "</not available>"
|
||||
|
||||
# Determine model type and load the corresponding priority_tags
|
||||
try:
|
||||
model_type = await identify_model_type(model_path)
|
||||
context["model_type"] = model_type
|
||||
settings = SettingsManager()
|
||||
priority_config = settings.get_priority_tag_config()
|
||||
context["priority_tags"] = priority_config.get(model_type, "")
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to load priority tags: %s", exc)
|
||||
context["model_type"] = "lora"
|
||||
context["priority_tags"] = ""
|
||||
|
||||
return context
|
||||
|
||||
@staticmethod
|
||||
def _extract_repo_from_url(hf_url: str) -> Optional[str]:
|
||||
"""Extract ``user/repo`` from a HuggingFace URL."""
|
||||
if not hf_url:
|
||||
return None
|
||||
m = re.match(r"https?://huggingface\.co/([^/]+/[^/]+)", hf_url)
|
||||
return m.group(1) if m else None
|
||||
|
||||
@staticmethod
|
||||
async def _fetch_readme(repo: str) -> str:
|
||||
"""Fetch README.md from HuggingFace (tries ``main``, then ``master``)."""
|
||||
async with aiohttp.ClientSession(
|
||||
headers={"User-Agent": "ComfyUI-LoRA-Manager/1.0"},
|
||||
timeout=aiohttp.ClientTimeout(total=30),
|
||||
) as session:
|
||||
for branch in ("main", "master"):
|
||||
url = f"https://huggingface.co/{repo}/raw/{branch}/README.md"
|
||||
try:
|
||||
async with session.get(url) as resp:
|
||||
if resp.status == 200:
|
||||
return await resp.text()
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to fetch README from %s: %s", url, exc)
|
||||
return ""
|
||||
|
||||
async def _emit_progress(
|
||||
self,
|
||||
callback: Optional[AgentProgressReporter],
|
||||
skill_name: str,
|
||||
*,
|
||||
status: str,
|
||||
**extra: Any,
|
||||
) -> None:
|
||||
"""Send a progress update via WebSocket (if callback is set)."""
|
||||
payload: Dict[str, Any] = {"type": "agent_progress", "skill": skill_name, "status": status}
|
||||
payload.update(extra)
|
||||
if callback is not None:
|
||||
await callback.on_progress(payload)
|
||||
@@ -0,0 +1,336 @@
|
||||
"""Post-processing engine for skill pipeline outputs.
|
||||
|
||||
The :class:`PostProcessor` takes the LLM's structured JSON output and applies
|
||||
it to a model's on-disk metadata via the :mod:`~py.metadata_ops` functions.
|
||||
|
||||
It handles all the skill-specific business logic — conditions, transformations,
|
||||
and orchestration of multiple side-effects (write metadata, download preview,
|
||||
refresh cache). All actual I/O is delegated to :mod:`~py.metadata_ops`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PostProcessor:
|
||||
"""Deterministic post-processor for skill pipeline outputs.
|
||||
|
||||
Usage (called by :class:`~py.services.agent.agent_service.AgentService`)::
|
||||
|
||||
processor = PostProcessor()
|
||||
result = await processor.process(
|
||||
skill_name="enrich_hf_metadata",
|
||||
model_path="/path/to/model.safetensors",
|
||||
llm_output={...},
|
||||
metadata={...}, # from metadata_ops.read_metadata()
|
||||
)
|
||||
"""
|
||||
|
||||
async def process(
|
||||
self,
|
||||
*,
|
||||
skill_name: str,
|
||||
model_path: str,
|
||||
llm_output: Dict[str, Any],
|
||||
metadata: Dict[str, Any],
|
||||
readme_content: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
"""Route *llm_output* to the correct skill post-processor.
|
||||
|
||||
*readme_content* is optional raw markdown content (e.g. HF README)
|
||||
that is converted to HTML and stored as ``modelDescription`` for
|
||||
the description tab.
|
||||
|
||||
Returns a dict with keys ``success`` (bool), ``updated_fields`` (list),
|
||||
``preview_downloaded`` (bool), and ``errors`` (list).
|
||||
"""
|
||||
if skill_name == "enrich_hf_metadata":
|
||||
return await self._process_enrich_hf_metadata(
|
||||
model_path, llm_output, metadata, readme_content,
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
"updated_fields": [],
|
||||
"errors": [f"No post-processor registered for skill: {skill_name}"],
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# enrich_hf_metadata
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _process_enrich_hf_metadata(
|
||||
self,
|
||||
model_path: str,
|
||||
llm_output: Dict[str, Any],
|
||||
metadata: Dict[str, Any],
|
||||
readme_content: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
from ...metadata_ops import (
|
||||
apply_metadata_updates,
|
||||
download_preview,
|
||||
refresh_cache,
|
||||
)
|
||||
from .skills.enrich_hf_metadata.readme_processor import (
|
||||
convert_readme_to_html,
|
||||
extract_gallery_images,
|
||||
extract_gallery_table_images,
|
||||
extract_relevant_section,
|
||||
extract_simple_markdown_images,
|
||||
extract_html_img_tags,
|
||||
extract_repo_from_hf_url,
|
||||
)
|
||||
|
||||
updated_fields: List[str] = []
|
||||
preview_downloaded = False
|
||||
|
||||
# -- Determine whether this is an HF-sourced model -----------------
|
||||
is_hf_model = not metadata.get("from_civitai", True)
|
||||
|
||||
# -- Collect updates -----------------------------------------------
|
||||
updates: Dict[str, Any] = {}
|
||||
|
||||
# base_model
|
||||
new_base = (llm_output.get("base_model") or "").strip()
|
||||
current_base = metadata.get("base_model", "") or ""
|
||||
if new_base and self._should_overwrite(current_base, is_hf_model):
|
||||
updates["base_model"] = new_base
|
||||
|
||||
# trigger words → civitai.trainedWords
|
||||
new_triggers = llm_output.get("trigger_words", [])
|
||||
trigger_words_empty = True
|
||||
if isinstance(new_triggers, list):
|
||||
cleaned = [t.strip() for t in new_triggers if t.strip()]
|
||||
cleaned = [t for t in cleaned if t.lower() not in ("none", "null", "n/a")]
|
||||
trigger_words_empty = not cleaned
|
||||
current_civitai = metadata.get("civitai") or {}
|
||||
current_triggers = current_civitai.get("trainedWords") or []
|
||||
if self._should_overwrite_list(current_triggers, is_hf_model):
|
||||
trig_civitai = dict(current_civitai)
|
||||
if "civitai" in updates and isinstance(updates["civitai"], dict):
|
||||
trig_civitai.update(updates["civitai"])
|
||||
trig_civitai["trainedWords"] = cleaned
|
||||
updates["civitai"] = trig_civitai
|
||||
|
||||
# modelDescription — from raw README content (converted to HTML)
|
||||
if readme_content and is_hf_model:
|
||||
converted = convert_readme_to_html(readme_content)
|
||||
if converted:
|
||||
updates["modelDescription"] = converted
|
||||
|
||||
# short_description → civitai.description (for "About this version")
|
||||
short_desc = (llm_output.get("short_description") or "").strip()
|
||||
if short_desc and is_hf_model:
|
||||
current_civitai = metadata.get("civitai") or {}
|
||||
desc_civitai = dict(current_civitai)
|
||||
if "civitai" in updates and isinstance(updates["civitai"], dict):
|
||||
desc_civitai.update(updates["civitai"])
|
||||
desc_civitai["description"] = short_desc
|
||||
updates["civitai"] = desc_civitai
|
||||
|
||||
# gallery images → civitai.images (from YAML frontmatter widget entries
|
||||
# and Sample Gallery markdown tables in the README body)
|
||||
gallery_images: List[Dict[str, Any]] = []
|
||||
if readme_content and is_hf_model:
|
||||
hf_url = metadata.get("hf_url", "") or ""
|
||||
repo = extract_repo_from_hf_url(hf_url)
|
||||
if repo:
|
||||
rec_w = llm_output.get("recommended_width") or 0
|
||||
rec_h = llm_output.get("recommended_height") or 0
|
||||
|
||||
# 1. Widget images (YAML frontmatter)
|
||||
gallery = extract_gallery_images(
|
||||
readme_content, repo,
|
||||
default_width=rec_w, default_height=rec_h,
|
||||
)
|
||||
|
||||
# 2. Sample Gallery table images (markdown body), deduplicated
|
||||
existing_urls = {img["url"] for img in gallery if img.get("url")}
|
||||
table_images = extract_gallery_table_images(
|
||||
readme_content, repo,
|
||||
existing_urls=existing_urls,
|
||||
default_width=rec_w, default_height=rec_h,
|
||||
)
|
||||
existing_urls.update(img["url"] for img in table_images if img.get("url"))
|
||||
|
||||
# 3. Simple markdown images `` in the body
|
||||
simple_images = extract_simple_markdown_images(
|
||||
readme_content, repo,
|
||||
existing_urls=existing_urls,
|
||||
default_width=rec_w, default_height=rec_h,
|
||||
)
|
||||
existing_urls.update(img["url"] for img in simple_images if img.get("url"))
|
||||
|
||||
# 4. HTML `<img>` tags (used by many collection repos)
|
||||
html_images = extract_html_img_tags(
|
||||
readme_content, repo,
|
||||
existing_urls=existing_urls,
|
||||
default_width=rec_w, default_height=rec_h,
|
||||
)
|
||||
|
||||
all_images = gallery + table_images + simple_images + html_images
|
||||
if all_images:
|
||||
gallery_images = all_images
|
||||
current_civitai = metadata.get("civitai") or {}
|
||||
gallery_civitai = dict(current_civitai)
|
||||
if "civitai" in updates and isinstance(updates["civitai"], dict):
|
||||
gallery_civitai.update(updates["civitai"])
|
||||
gallery_civitai["images"] = all_images
|
||||
updates["civitai"] = gallery_civitai
|
||||
|
||||
# tags
|
||||
new_tags = llm_output.get("tags", [])
|
||||
if isinstance(new_tags, list) and new_tags:
|
||||
existing_tags = metadata.get("tags") or []
|
||||
merged = self._merge_tags(existing_tags, new_tags)
|
||||
if len(merged) > len(existing_tags) or is_hf_model:
|
||||
updates["tags"] = merged
|
||||
|
||||
# metadata_source & llm_enriched_at (always set)
|
||||
updates["metadata_source"] = "agent:enrich_hf_metadata"
|
||||
updates["llm_enriched_at"] = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
# Store LLM confidence in metadata so it's accessible for evaluation
|
||||
raw_confidence = (llm_output.get("confidence") or "").strip()
|
||||
if raw_confidence:
|
||||
updates["_llm_confidence"] = raw_confidence
|
||||
|
||||
# Fallback: extract instance_prompt from YAML frontmatter when the LLM
|
||||
# returned empty trigger words but the README has instance_prompt.
|
||||
if trigger_words_empty:
|
||||
instance_prompt = _extract_yaml_instance_prompt(readme_content)
|
||||
if instance_prompt:
|
||||
current_civitai = metadata.get("civitai") or {}
|
||||
trig_civitai = dict(current_civitai)
|
||||
if "civitai" in updates and isinstance(updates["civitai"], dict):
|
||||
trig_civitai.update(updates["civitai"])
|
||||
trig_civitai["trainedWords"] = [instance_prompt]
|
||||
updates["civitai"] = trig_civitai
|
||||
|
||||
preview_remote_url = (llm_output.get("preview_url") or "").strip()
|
||||
# Fallback: if the LLM couldn't find a preview image in the cleaned
|
||||
# README, find the first gallery image from the *model-specific
|
||||
# section* of the README (not the repo-wide first image, which
|
||||
# belongs to a different model in collection repos).
|
||||
if not preview_remote_url and readme_content and is_hf_model:
|
||||
model_basename = os.path.splitext(os.path.basename(model_path))[0]
|
||||
relevant_section = extract_relevant_section(
|
||||
readme_content, model_basename,
|
||||
)
|
||||
if relevant_section and relevant_section != readme_content:
|
||||
for img in gallery_images:
|
||||
img_url = img.get("url", "")
|
||||
if img_url and img_url in relevant_section:
|
||||
preview_remote_url = img_url
|
||||
break
|
||||
# Last resort: use the first gallery image from the full README.
|
||||
if not preview_remote_url and gallery_images:
|
||||
preview_remote_url = gallery_images[0].get("url", "")
|
||||
current_preview = metadata.get("preview_url") or ""
|
||||
if preview_remote_url and not (current_preview and os.path.exists(current_preview)):
|
||||
local_path = await download_preview(model_path, preview_remote_url)
|
||||
if local_path:
|
||||
preview_downloaded = True
|
||||
updates["preview_url"] = local_path
|
||||
|
||||
# notes — plain-text summary of usage info from the LLM
|
||||
new_notes = (llm_output.get("notes") or "").strip()
|
||||
if new_notes:
|
||||
updates["notes"] = new_notes
|
||||
|
||||
# usage_tips — JSON string (e.g. {"strength_min":0.85,"strength_max":1.4})
|
||||
raw_tips = (llm_output.get("usage_tips") or "").strip()
|
||||
if raw_tips and raw_tips != "{}":
|
||||
try:
|
||||
json.loads(raw_tips)
|
||||
updates["usage_tips"] = raw_tips
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
logger.warning(
|
||||
"LLM returned invalid usage_tips JSON: %s", raw_tips[:200]
|
||||
)
|
||||
|
||||
if updates:
|
||||
updated_fields = await apply_metadata_updates(model_path, updates)
|
||||
|
||||
# -- Refresh scanner cache ------------------------------------------
|
||||
if updated_fields or preview_downloaded:
|
||||
await refresh_cache(model_path)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"updated_fields": updated_fields,
|
||||
"preview_downloaded": preview_downloaded,
|
||||
"updates": updates,
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _should_overwrite(current_value: str, is_hf_model: bool) -> bool:
|
||||
"""Return ``True`` when a scalar field should be overwritten."""
|
||||
return is_hf_model or not current_value or current_value.lower() in (
|
||||
"", "unknown",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _should_overwrite_list(current_list: List[str], is_hf_model: bool) -> bool:
|
||||
"""Return ``True`` when a list field should be overwritten."""
|
||||
return is_hf_model or not current_list
|
||||
|
||||
@staticmethod
|
||||
def _merge_tags(existing: List[str], new: List[str]) -> List[str]:
|
||||
"""Merge *new* tags into *existing*, all lowercased.
|
||||
|
||||
This matches the behaviour of :class:`TagUpdateService` which
|
||||
normalises every tag to lowercase for case-insensitive dedup.
|
||||
"""
|
||||
merged: List[str] = []
|
||||
seen: set[str] = set()
|
||||
for tag in list(existing) + list(new):
|
||||
t = tag.strip().lower()
|
||||
if t and t not in seen:
|
||||
merged.append(t)
|
||||
seen.add(t)
|
||||
return merged
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Module-level helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
def _extract_yaml_instance_prompt(readme_content: str) -> str:
|
||||
"""Extract ``instance_prompt`` from the YAML frontmatter of a HF README.
|
||||
|
||||
Returns the prompt text, or empty string if not found. Handles
|
||||
``null`` / ``~`` YAML null values by returning empty string.
|
||||
"""
|
||||
if not readme_content or not readme_content.startswith("---"):
|
||||
return ""
|
||||
|
||||
# Find end of frontmatter
|
||||
end = readme_content.find("---", 3)
|
||||
if end == -1:
|
||||
return ""
|
||||
frontmatter = readme_content[3:end]
|
||||
|
||||
for line in frontmatter.split("\n"):
|
||||
line = line.strip()
|
||||
m = re.match(r"^instance_prompt:\s*(.*)", line)
|
||||
if m:
|
||||
val = m.group(1).strip().strip('"').strip("'")
|
||||
if val.lower() in ("null", "~", "none", ""):
|
||||
return ""
|
||||
return val
|
||||
|
||||
return ""
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Skill definition data structures.
|
||||
|
||||
Each skill is described by a :class:`SkillDefinition` that declares its
|
||||
input/output schemas, whether it needs an LLM call, and what permissions
|
||||
its post-processor has.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SkillPermissions:
|
||||
"""Declarative permission scope for a skill's post-processor.
|
||||
|
||||
These are auditable constraints — the :class:`AgentService` checks them
|
||||
before invoking the handler. They are defense-in-depth, not a sandbox.
|
||||
"""
|
||||
|
||||
write_metadata: bool = True
|
||||
write_previews: bool = True
|
||||
network_domains: Tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SkillDefinition:
|
||||
"""Immutable description of an agent skill."""
|
||||
|
||||
name: str
|
||||
title: str
|
||||
description: str
|
||||
llm_required: bool
|
||||
input_schema: Dict[str, Any] = field(default_factory=dict)
|
||||
output_schema: Dict[str, Any] = field(default_factory=dict)
|
||||
model_type_filter: Optional[List[str]] = None
|
||||
permissions: SkillPermissions = field(default_factory=SkillPermissions)
|
||||
|
||||
def applies_to_model_type(self, model_type: str) -> bool:
|
||||
"""Return ``True`` if this skill can run on the given model type."""
|
||||
|
||||
if self.model_type_filter is None:
|
||||
return True
|
||||
return model_type in self.model_type_filter
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Discovery and loading of prompt-based skills.
|
||||
|
||||
Skills live in ``py/services/agent/skills/<name>/`` directories. Each
|
||||
directory must contain a ``prompt.md`` file with YAML frontmatter::
|
||||
|
||||
---
|
||||
name: my_skill
|
||||
title: "My Skill"
|
||||
description: "What this skill does"
|
||||
llm_required: true
|
||||
---
|
||||
|
||||
Prompt template with ``{{variable}}`` placeholders.
|
||||
|
||||
Legacy ``SKILL.md`` files are also supported for backward compatibility.
|
||||
|
||||
The registry scans the skills directory on first access and caches results.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import yaml
|
||||
|
||||
from .skill_definition import SkillDefinition, SkillPermissions
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Directory where built-in skills are stored
|
||||
_SKILLS_DIR = Path(__file__).parent / "skills"
|
||||
|
||||
#: Preferred file names for prompt definition files (tried in order).
|
||||
#: ``prompt.md`` is the current convention; ``SKILL.md`` is the legacy name
|
||||
#: kept for backward compatibility.
|
||||
_PROMPT_FILE_NAMES: tuple[str, ...] = ("prompt.md", "SKILL.md")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Frontmatter parser
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_FRONTMATTER_RE = re.compile(
|
||||
r"^---\s*\n(.*?\n)---\s*\n?(.*)", re.DOTALL
|
||||
)
|
||||
|
||||
|
||||
def _parse_skill_file(path: Path) -> tuple[dict[str, Any], str]:
|
||||
"""Read a prompt definition file (``prompt.md`` or legacy ``SKILL.md``) and
|
||||
return (frontmatter_dict, body_text).
|
||||
|
||||
Raises ``ValueError`` if the file lacks valid YAML frontmatter.
|
||||
"""
|
||||
text = path.read_text(encoding="utf-8")
|
||||
m = _FRONTMATTER_RE.match(text)
|
||||
if not m:
|
||||
raise ValueError(f"Missing or invalid YAML frontmatter in {path}")
|
||||
frontmatter = yaml.safe_load(m.group(1))
|
||||
if not isinstance(frontmatter, dict):
|
||||
raise ValueError(f"Frontmatter in {path} is not a mapping")
|
||||
body = m.group(2).strip()
|
||||
return frontmatter, body
|
||||
|
||||
|
||||
class SkillRegistry:
|
||||
"""Discover and load agent skills from the filesystem."""
|
||||
|
||||
_instance: Optional["SkillRegistry"] = None
|
||||
_lock: asyncio.Lock = asyncio.Lock()
|
||||
|
||||
def __init__(self, skills_dir: Path = _SKILLS_DIR) -> None:
|
||||
self._skills_dir = skills_dir
|
||||
self._skills: Dict[str, SkillDefinition] = {}
|
||||
self._loaded: bool = False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Singleton access
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@classmethod
|
||||
async def get_instance(cls) -> "SkillRegistry":
|
||||
"""Return the lazily-initialised global ``SkillRegistry``."""
|
||||
|
||||
if cls._instance is None:
|
||||
async with cls._lock:
|
||||
if cls._instance is None:
|
||||
registry = cls()
|
||||
registry._discover()
|
||||
cls._instance = registry
|
||||
return cls._instance
|
||||
|
||||
@classmethod
|
||||
def reset_instance(cls) -> None:
|
||||
"""Reset the cached singleton — primarily for tests."""
|
||||
|
||||
cls._instance = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Discovery
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _find_prompt_file(skill_dir: Path) -> Path | None:
|
||||
"""Return the first prompt definition file that exists in *skill_dir*.
|
||||
|
||||
Tries ``_PROMPT_FILE_NAMES`` in order so that new conventions
|
||||
(``prompt.md``) take precedence while legacy ``SKILL.md`` files
|
||||
still load without changes.
|
||||
"""
|
||||
for name in _PROMPT_FILE_NAMES:
|
||||
candidate = skill_dir / name
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
return None
|
||||
|
||||
def _discover(self) -> None:
|
||||
"""Scan the skills directory and load all valid skill definitions."""
|
||||
|
||||
self._skills.clear()
|
||||
if not self._skills_dir.is_dir():
|
||||
logger.warning("Skills directory does not exist: %s", self._skills_dir)
|
||||
self._loaded = True
|
||||
return
|
||||
|
||||
for entry in sorted(self._skills_dir.iterdir()):
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
prompt_file = self._find_prompt_file(entry)
|
||||
if prompt_file is None:
|
||||
continue
|
||||
try:
|
||||
definition = self._load_skill_definition(prompt_file)
|
||||
if definition is not None:
|
||||
self._skills[definition.name] = definition
|
||||
logger.debug("Loaded skill: %s", definition.name)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to load skill from %s: %s", prompt_file, exc)
|
||||
|
||||
self._loaded = True
|
||||
logger.info("Discovered %d prompt-based skills", len(self._skills))
|
||||
|
||||
def _load_skill_definition(self, path: Path) -> Optional[SkillDefinition]:
|
||||
"""Parse a prompt definition file's frontmatter into a
|
||||
:class:`SkillDefinition`."""
|
||||
|
||||
try:
|
||||
data, _body = _parse_skill_file(path)
|
||||
except (ValueError, yaml.YAMLError) as exc:
|
||||
logger.warning("Failed to parse prompt file %s: %s", path, exc)
|
||||
return None
|
||||
|
||||
if "name" not in data:
|
||||
logger.warning("Prompt file %s missing required 'name' field", path)
|
||||
return None
|
||||
|
||||
perm_data = data.get("permissions", {})
|
||||
permissions = SkillPermissions(
|
||||
write_metadata=perm_data.get("write_metadata", True),
|
||||
write_previews=perm_data.get("write_previews", True),
|
||||
network_domains=tuple(perm_data.get("network_domains", [])),
|
||||
)
|
||||
|
||||
return SkillDefinition(
|
||||
name=data["name"],
|
||||
title=data.get("title", data["name"]),
|
||||
description=data.get("description", ""),
|
||||
llm_required=data.get("llm_required", False),
|
||||
input_schema=data.get("input_schema", {}),
|
||||
output_schema=data.get("output_schema", {}),
|
||||
model_type_filter=data.get("model_type_filter"),
|
||||
permissions=permissions,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def list_skills(self) -> List[SkillDefinition]:
|
||||
"""Return all discovered skill definitions."""
|
||||
|
||||
if not self._loaded:
|
||||
self._discover()
|
||||
return list(self._skills.values())
|
||||
|
||||
def get_skill(self, name: str) -> Optional[SkillDefinition]:
|
||||
"""Return the skill definition for ``name``, or ``None`` if not found."""
|
||||
|
||||
if not self._loaded:
|
||||
self._discover()
|
||||
return self._skills.get(name)
|
||||
|
||||
def load_prompt(self, name: str) -> str:
|
||||
"""Load and return the prompt template body for the named skill."""
|
||||
|
||||
skill_dir = self._skills_dir / name
|
||||
skill_path = self._find_prompt_file(skill_dir)
|
||||
if skill_path is None:
|
||||
raise FileNotFoundError(
|
||||
f"Prompt file not found for skill '{name}' in {skill_dir} "
|
||||
f"(tried {list(_PROMPT_FILE_NAMES)})"
|
||||
)
|
||||
try:
|
||||
_frontmatter, body = _parse_skill_file(skill_path)
|
||||
return body
|
||||
except (ValueError, yaml.YAMLError) as exc:
|
||||
raise ValueError(f"Failed to parse prompt from {skill_path}: {exc}") from exc
|
||||
@@ -0,0 +1,165 @@
|
||||
---
|
||||
name: enrich_hf_metadata
|
||||
title: "Enrich Metadata from HuggingFace"
|
||||
description: >
|
||||
Parse the HuggingFace model card via LLM to extract description, trigger
|
||||
words, base model, tags, and preview image URL.
|
||||
llm_required: true
|
||||
---
|
||||
|
||||
You are an expert assistant for AI image generation models. Your task is to extract structured metadata from a HuggingFace model card (README.md).
|
||||
|
||||
## Model Information
|
||||
|
||||
- **Repository**: {{hf_url}}
|
||||
- **Model file path**: {{model_path}}
|
||||
- **Model filename**: {{model_basename}}
|
||||
- **Repository ID**: {{repo}}
|
||||
|
||||
## Current Metadata (may be incomplete)
|
||||
|
||||
```json
|
||||
{{current_metadata}}
|
||||
```
|
||||
|
||||
## User Priority Tags Reference
|
||||
|
||||
The user has configured the following list of **meaningful tag categories** for this model type (`{{model_type}}`):
|
||||
|
||||
```
|
||||
{{priority_tags}}
|
||||
```
|
||||
|
||||
These are the subjects, styles, and concepts the user considers useful for categorization. Use this list as a **reference** when evaluating tags (see the **tags** section below).
|
||||
|
||||
## Available Base Models
|
||||
|
||||
The following base models are currently valid in this system. Use the EXACT
|
||||
name listed — do not invent aliases or modify variant suffixes.
|
||||
|
||||
{{base_models}}
|
||||
|
||||
## HuggingFace README Content
|
||||
|
||||
```
|
||||
{{readme_content}}
|
||||
```
|
||||
|
||||
## Extraction Instructions
|
||||
|
||||
Extract the following information from the README content above:
|
||||
|
||||
### base_model
|
||||
The base model this model was trained on. Use EXACTLY one of the names from the **Available Base Models** list above. Do not invent new names or use aliases.
|
||||
|
||||
Check the YAML frontmatter for ``base_model:`` first. If the frontmatter has no ``base_model:``, look at the **model filename** (``{{model_basename}}``), YAML ``tags:``, README title and first paragraph for clues — the base model family is often embedded in the name
|
||||
|
||||
### trigger_words
|
||||
The trigger words or activation prompts needed to use this LoRA. Look for:
|
||||
- `instance_prompt:` in the YAML frontmatter
|
||||
- Phrases like "trigger word:", "trigger:", "use this prompt:", "activation prompt:"
|
||||
- In collection repos: the trigger section **specific to this model file** (look near matching download links or anchor IDs)
|
||||
- Example prompts at the start (usually the first word or phrase before any description)
|
||||
Return as an array of strings. If none found, return an empty array `[]`. **Never** return `["None"]` or any placeholder value — a truly empty list means no trigger words exist.
|
||||
|
||||
### short_description
|
||||
A concise 1-2 sentence summary of what this model does. Extract from the "Model description" section or the first paragraph. For collection repos, focus on the **specific model version** matching `{{model_basename}}`, not the repo as a whole. Return empty string if the README is too minimal.
|
||||
|
||||
### tags
|
||||
3-8 relevant tags for categorizing this model. **Quality over quantity.**
|
||||
|
||||
Sources to consider:
|
||||
- The YAML frontmatter `tags:` list (filter out technical ones — see below)
|
||||
- The subject, style, character, or concept the model represents
|
||||
- The model filename itself may give clues (e.g. "pokemon", "anime", "pixelart")
|
||||
|
||||
**Critical filtering rules — apply them strictly:**
|
||||
|
||||
1. **Exclude technical/generic tags.** Reject any tag that describes the model's **training methodology, framework, architecture, or modality** rather than its content. Examples to exclude: `text-to-image`, `diffusers`, `lora`, `dreambooth`, `diffusers-training`, `flux`, `sdxl`, `checkpoint`, `pytorch`, `safetensors`, `fine-tuning`, `stable-diffusion`, and any variant of these.
|
||||
|
||||
2. **Cross-reference against the priority_tags reference.** Only include a tag if it meaningfully describes what the model actually creates (subject, style, character type) and is semantically close to one of the priority_tags. If none of the README's tags match meaningful categories, prefer returning a smaller set or an empty array over including low-value tags.
|
||||
|
||||
3. **All lowercase, no spaces, no hyphens** (use single words like `"photorealistic"`, `"anime"`, `"character"`).
|
||||
|
||||
Return empty array if no meaningful content tags remain after filtering.
|
||||
|
||||
### recommended_width, recommended_height
|
||||
The recommended image generation resolution for this model, in pixels. Look for sections like "Best Dimensions", "Recommended size", "Suggested resolution", or similar phrasing in the README. Prefer the explicitly marked "Best" or default resolution. If the table/list has multiple entries (e.g. "768 x 1024 (Best)" and "1024 x 1024 (Default)"), use the one marked "Best". Return integers. If no resolution can be determined, return 0 for both.
|
||||
|
||||
### preview_url
|
||||
The URL of the most suitable preview image from the README. Look for:
|
||||
- Image tags near the section matching the model filename (`{{model_basename}}`)
|
||||
- The YAML frontmatter `widget:` section (which often has `output.url` fields)
|
||||
- In collection repos: the sample images listed **under the section** for this specific model version
|
||||
- Generic `` in the body
|
||||
Choose the first image that appears to be a generation example (not a logo or diagram). Construct the absolute URL as `https://huggingface.co/{{repo}}/resolve/main/{filename}`. If no suitable image is found, return an empty string.
|
||||
|
||||
### notes
|
||||
A plain-text summary of the model card's key practical usage information. Combine trigger words, style modifiers, recommended parameters (steps, CFG, resolution, sampler), and any setup tips into a readable paragraph. For collection repos, focus on the **specific model version** matching `{{model_basename}}`. Return empty string if the README has no useful usage info.
|
||||
|
||||
### usage_tips
|
||||
A JSON string with structured usage recommendations. Extract from the README any explicit ranges or recommended values (e.g. "Set LoRA strength: **0.85 - 1.4**", "CLIP strength: 0.5"). Possible fields (include only those you can determine):
|
||||
|
||||
```json
|
||||
{
|
||||
"strength_min": 0.85,
|
||||
"strength_max": 1.4,
|
||||
"strength_range": "0.85-1.4",
|
||||
"strength": 0.6,
|
||||
"clip_strength": 0.5,
|
||||
"clip_skip": 2
|
||||
}
|
||||
```
|
||||
|
||||
Return the JSON string (e.g. `'{"strength_min":0.85,"strength_max":1.4}'`). Return `"{}"` if nothing useful is found.
|
||||
|
||||
### confidence
|
||||
Your confidence level in the extracted data:
|
||||
- "high" — most fields were explicitly stated in the README
|
||||
- "medium" — some fields were inferred from context
|
||||
- "low" — most fields are guesses based on limited information
|
||||
|
||||
## Important: Handling Collection Repos (multiple model files)
|
||||
|
||||
Many HuggingFace repos contain **multiple model files** in a single repository
|
||||
(e.g. a "LoRA collection" with different styles/characters in separate files).
|
||||
|
||||
The model file currently being enriched is: **`{{model_basename}}`**
|
||||
|
||||
To find the correct section in the README:
|
||||
|
||||
1. **Search for download links** containing the filename — the surrounding paragraph is your section.
|
||||
2. **Search for anchor IDs** (`<a id="...">`) or section headings whose text matches words from the filename.
|
||||
3. **Search for HTML headings** (`<h1>`, `<h2>`, `<span>`) containing parts of the filename.
|
||||
4. If no match is found, use the full README as usual — the model may be the only one in the repo.
|
||||
|
||||
When a matching section IS found, prefer metadata from that section.
|
||||
When no section matches (e.g. single-model repos or repos without per-file sections),
|
||||
extract metadata from the full README normally. Do not return empty data just
|
||||
because the filename doesn't appear in the README.
|
||||
|
||||
## Output Format
|
||||
|
||||
Return ONLY a JSON object with exactly these fields (no markdown fences, no extra text):
|
||||
|
||||
```json
|
||||
{
|
||||
"model_path": "{{model_path}}",
|
||||
"base_model": "<canonical name or empty string>",
|
||||
"trigger_words": ["<word1>", "<word2>"],
|
||||
"short_description": "<1-2 sentence summary>",
|
||||
"tags": ["<tag1>", "<tag2>"],
|
||||
"recommended_width": 768,
|
||||
"recommended_height": 1024,
|
||||
"preview_url": "<image URL or empty string>",
|
||||
"notes": "<plain-text usage summary or empty string>",
|
||||
"usage_tips": "<JSON string like '{\"strength_min\":0.85,\"strength_max\":1.4}' or '{}'>",
|
||||
"confidence": "<high|medium|low>"
|
||||
}
|
||||
```
|
||||
|
||||
Important:
|
||||
- Only include the JSON object, no other text
|
||||
- If a field cannot be determined, use an empty string or empty array
|
||||
- Do not fabricate information not supported by the README
|
||||
- Never use placeholder values like `"None"` or `"unknown"` for missing data — use empty string or empty array
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,683 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
import shutil
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import aiohttp
|
||||
|
||||
from .downloader import DownloadProgress, get_downloader, is_ssl_cert_verify_error
|
||||
from .aria2_transfer_state import Aria2TransferStateStore
|
||||
from .settings_manager import get_settings_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def _try_certifi_ca_path() -> str | None:
|
||||
"""Return the certifi CA bundle path if available, else None."""
|
||||
try:
|
||||
import certifi # pyright: ignore[reportMissingTypeStubs]
|
||||
|
||||
path = certifi.where()
|
||||
if os.path.isfile(path):
|
||||
logger.debug(
|
||||
"aria2 --ca-certificate: using certifi CA bundle at %s", path
|
||||
)
|
||||
return path
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
logger.debug("aria2 --ca-certificate: certifi not available")
|
||||
return None
|
||||
|
||||
|
||||
CIVITAI_DOWNLOAD_URL_PREFIXES = (
|
||||
"https://civitai.com/api/download/",
|
||||
"https://civitai.red/api/download/",
|
||||
)
|
||||
|
||||
|
||||
class Aria2Error(RuntimeError):
|
||||
"""Raised when aria2 integration fails."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Aria2Transfer:
|
||||
"""Track an aria2 download registered by the Python coordinator."""
|
||||
|
||||
gid: str
|
||||
save_path: str
|
||||
|
||||
|
||||
class Aria2Downloader:
|
||||
"""Manage an aria2 RPC daemon for recommended model downloads."""
|
||||
|
||||
_instance = None
|
||||
_lock = asyncio.Lock()
|
||||
|
||||
@classmethod
|
||||
async def get_instance(cls) -> "Aria2Downloader":
|
||||
async with cls._lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = cls()
|
||||
return cls._instance
|
||||
|
||||
def __init__(self) -> None:
|
||||
if hasattr(self, "_initialized"):
|
||||
return
|
||||
|
||||
self._initialized = True
|
||||
self._process: Optional[asyncio.subprocess.Process] = None
|
||||
self._rpc_port: Optional[int] = None
|
||||
self._rpc_secret = ""
|
||||
self._rpc_url = ""
|
||||
self._rpc_session: Optional[aiohttp.ClientSession] = None
|
||||
self._rpc_session_lock = asyncio.Lock()
|
||||
self._process_lock = asyncio.Lock()
|
||||
self._transfers: Dict[str, Aria2Transfer] = {}
|
||||
self._poll_interval = 0.5
|
||||
self._state_store = Aria2TransferStateStore()
|
||||
self._stderr_reader_task: Optional[asyncio.Task[Any]] = None
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
return self._process is not None and self._process.returncode is None
|
||||
|
||||
async def download_file(
|
||||
self,
|
||||
url: str,
|
||||
save_path: str,
|
||||
*,
|
||||
download_id: str,
|
||||
progress_callback=None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> Tuple[bool, str]:
|
||||
"""Download a file using aria2 RPC and wait for completion."""
|
||||
|
||||
await self._ensure_process()
|
||||
save_path = os.path.abspath(save_path)
|
||||
transfer = self._transfers.get(download_id)
|
||||
if transfer is None or os.path.abspath(transfer.save_path) != save_path:
|
||||
gid = await self._schedule_download(
|
||||
url,
|
||||
save_path,
|
||||
download_id=download_id,
|
||||
headers=headers,
|
||||
)
|
||||
transfer = Aria2Transfer(gid=gid, save_path=save_path)
|
||||
self._transfers[download_id] = transfer
|
||||
|
||||
try:
|
||||
while True:
|
||||
status = await self._get_status_with_retry(download_id)
|
||||
if status is None:
|
||||
return False, "aria2 download not found"
|
||||
|
||||
snapshot = self._build_progress_snapshot(status)
|
||||
if progress_callback is not None:
|
||||
await self._dispatch_progress(progress_callback, snapshot)
|
||||
|
||||
state = status.get("status", "")
|
||||
if state == "complete":
|
||||
completed_path = self._resolve_completed_path(status, save_path)
|
||||
return True, completed_path
|
||||
if state == "error":
|
||||
return False, status.get("errorMessage") or "aria2 download failed"
|
||||
if state == "removed":
|
||||
return False, "Download was cancelled"
|
||||
|
||||
await asyncio.sleep(self._poll_interval)
|
||||
finally:
|
||||
self._transfers.pop(download_id, None)
|
||||
|
||||
async def _get_status_with_retry(
|
||||
self, download_id: str, *, max_retries: int = 4, retry_delay: float = 3.0
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Call get_status with retry for transient RPC failures.
|
||||
|
||||
Only retries on :exc:`Aria2Error` (RPC-level failure). Returns
|
||||
``None`` immediately when the download_id is not tracked (a missing
|
||||
transfer is not a transient condition, so retrying is pointless).
|
||||
|
||||
A single failed RPC call should not immediately fail the download,
|
||||
because aria2 may be temporarily busy (e.g. finalizing multiple
|
||||
concurrent downloads) and a retry will often succeed.
|
||||
"""
|
||||
last_exc: Optional[Exception] = None
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return await self.get_status(download_id)
|
||||
except Aria2Error as exc:
|
||||
last_exc = exc
|
||||
if attempt < max_retries - 1:
|
||||
logger.warning(
|
||||
"aria2 get_status transient failure (attempt %d/%d) for %s: %s",
|
||||
attempt + 1, max_retries, download_id, exc,
|
||||
)
|
||||
await asyncio.sleep(retry_delay)
|
||||
raise Aria2Error(
|
||||
f"Failed to query aria2 download status after {max_retries} attempts: {last_exc}"
|
||||
) from last_exc
|
||||
|
||||
async def _schedule_download(
|
||||
self,
|
||||
url: str,
|
||||
save_path: str,
|
||||
*,
|
||||
download_id: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> str:
|
||||
save_dir = os.path.dirname(save_path)
|
||||
out_name = os.path.basename(save_path)
|
||||
|
||||
Path(save_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
resolved_url = url
|
||||
request_headers = headers
|
||||
if headers and url.startswith(CIVITAI_DOWNLOAD_URL_PREFIXES):
|
||||
resolved_url = await self._resolve_authenticated_redirect_url(url, headers)
|
||||
if resolved_url != url:
|
||||
request_headers = None
|
||||
logger.debug(
|
||||
"Resolved Civitai download %s to signed URL for aria2",
|
||||
download_id,
|
||||
)
|
||||
|
||||
options: Dict[str, Any] = {
|
||||
"dir": save_dir,
|
||||
"out": out_name,
|
||||
"continue": "true",
|
||||
"max-connection-per-server": "4",
|
||||
"split": "4",
|
||||
"min-split-size": "1M",
|
||||
"allow-overwrite": "true",
|
||||
"auto-file-renaming": "false",
|
||||
"file-allocation": "none",
|
||||
}
|
||||
|
||||
# Pass proxy to aria2 so the actual file transfer goes through the
|
||||
# same proxy used by the aiohttp-based URL resolution step above.
|
||||
downloader = await get_downloader()
|
||||
if downloader.proxy_url:
|
||||
options["all-proxy"] = downloader.proxy_url
|
||||
|
||||
if request_headers:
|
||||
options["header"] = [
|
||||
f"{key}: {value}" for key, value in request_headers.items()
|
||||
]
|
||||
|
||||
logger.debug(
|
||||
"Submitting aria2 download %s -> %s (auth=%s, civitai_signed=%s)",
|
||||
download_id,
|
||||
save_path,
|
||||
bool(request_headers),
|
||||
resolved_url != url,
|
||||
)
|
||||
|
||||
try:
|
||||
gid = await self._rpc_call("aria2.addUri", [[resolved_url], options])
|
||||
except Exception as exc:
|
||||
raise Aria2Error(f"Failed to schedule aria2 download: {exc}") from exc
|
||||
|
||||
logger.debug("aria2 accepted download %s with gid %s", download_id, gid)
|
||||
await self._state_store.upsert(
|
||||
download_id,
|
||||
{
|
||||
"gid": gid,
|
||||
"save_path": save_path,
|
||||
"status": "downloading",
|
||||
"url": url,
|
||||
},
|
||||
)
|
||||
return gid
|
||||
|
||||
async def get_status(self, download_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Return the raw aria2 status payload for a known download."""
|
||||
|
||||
transfer = self._transfers.get(download_id)
|
||||
if transfer is None:
|
||||
return None
|
||||
|
||||
keys = [
|
||||
"gid",
|
||||
"status",
|
||||
"totalLength",
|
||||
"completedLength",
|
||||
"downloadSpeed",
|
||||
"errorMessage",
|
||||
"files",
|
||||
]
|
||||
try:
|
||||
status = await self._rpc_call("aria2.tellStatus", [transfer.gid, keys])
|
||||
except Exception as exc:
|
||||
raise Aria2Error(f"Failed to query aria2 download status: {exc}") from exc
|
||||
|
||||
if isinstance(status, dict):
|
||||
return status
|
||||
return None
|
||||
|
||||
async def get_status_by_gid(self, gid: str) -> Optional[Dict[str, Any]]:
|
||||
keys = [
|
||||
"gid",
|
||||
"status",
|
||||
"totalLength",
|
||||
"completedLength",
|
||||
"downloadSpeed",
|
||||
"errorMessage",
|
||||
"files",
|
||||
]
|
||||
try:
|
||||
status = await self._rpc_call("aria2.tellStatus", [gid, keys])
|
||||
except Exception as exc:
|
||||
message = str(exc)
|
||||
if "cannot be found" in message.lower() or "not found" in message.lower():
|
||||
return None
|
||||
raise Aria2Error(f"Failed to query aria2 download status: {exc}") from exc
|
||||
|
||||
if isinstance(status, dict):
|
||||
return status
|
||||
return None
|
||||
|
||||
async def restore_transfer(self, download_id: str, gid: str, save_path: str) -> None:
|
||||
await self._ensure_process()
|
||||
self._transfers[download_id] = Aria2Transfer(
|
||||
gid=gid,
|
||||
save_path=os.path.abspath(save_path),
|
||||
)
|
||||
|
||||
async def reassign_transfer(
|
||||
self, from_download_id: str, to_download_id: str
|
||||
) -> Optional[Aria2Transfer]:
|
||||
transfer = self._transfers.get(from_download_id)
|
||||
if transfer is None:
|
||||
return None
|
||||
|
||||
self._transfers[to_download_id] = transfer
|
||||
if from_download_id != to_download_id:
|
||||
self._transfers.pop(from_download_id, None)
|
||||
return transfer
|
||||
|
||||
async def has_transfer(self, download_id: str) -> bool:
|
||||
return download_id in self._transfers
|
||||
|
||||
async def pause_download(self, download_id: str) -> Dict[str, Any]:
|
||||
transfer = self._transfers.get(download_id)
|
||||
if transfer is None:
|
||||
return {"success": False, "error": "Download task not found"}
|
||||
|
||||
try:
|
||||
await self._rpc_call("aria2.forcePause", [transfer.gid])
|
||||
except Exception as exc:
|
||||
return {"success": False, "error": str(exc)}
|
||||
|
||||
await self._state_store.upsert(download_id, {"status": "paused"})
|
||||
return {"success": True, "message": "Download paused successfully"}
|
||||
|
||||
async def resume_download(self, download_id: str) -> Dict[str, Any]:
|
||||
transfer = self._transfers.get(download_id)
|
||||
if transfer is None:
|
||||
return {"success": False, "error": "Download task not found"}
|
||||
|
||||
try:
|
||||
await self._rpc_call("aria2.unpause", [transfer.gid])
|
||||
except Exception as exc:
|
||||
return {"success": False, "error": str(exc)}
|
||||
|
||||
await self._state_store.upsert(download_id, {"status": "downloading"})
|
||||
return {"success": True, "message": "Download resumed successfully"}
|
||||
|
||||
async def cancel_download(self, download_id: str) -> Dict[str, Any]:
|
||||
transfer = self._transfers.get(download_id)
|
||||
if transfer is None:
|
||||
return {"success": False, "error": "Download task not found"}
|
||||
|
||||
try:
|
||||
await self._rpc_call("aria2.forceRemove", [transfer.gid])
|
||||
except Exception as exc:
|
||||
return {"success": False, "error": str(exc)}
|
||||
|
||||
await self._state_store.remove(download_id)
|
||||
return {"success": True, "message": "Download cancelled successfully"}
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Shut down the RPC process and session."""
|
||||
|
||||
# Cancel the background stderr reader first so it stops reading
|
||||
# from the pipe before the subprocess is terminated.
|
||||
if self._stderr_reader_task is not None:
|
||||
self._stderr_reader_task.cancel()
|
||||
try:
|
||||
await asyncio.wait_for(self._stderr_reader_task, timeout=2.0)
|
||||
except (asyncio.CancelledError, asyncio.TimeoutError):
|
||||
pass
|
||||
self._stderr_reader_task = None
|
||||
|
||||
if self._rpc_session is not None:
|
||||
await self._rpc_session.close()
|
||||
self._rpc_session = None
|
||||
|
||||
process = self._process
|
||||
self._process = None
|
||||
self._transfers.clear()
|
||||
|
||||
if process is None:
|
||||
return
|
||||
|
||||
if process.returncode is None:
|
||||
process.terminate()
|
||||
try:
|
||||
await asyncio.wait_for(process.wait(), timeout=5.0)
|
||||
except asyncio.TimeoutError:
|
||||
process.kill()
|
||||
await process.wait()
|
||||
|
||||
async def _drain_stderr(self) -> None:
|
||||
"""Continuously drain aria2's stderr pipe so it never blocks.
|
||||
|
||||
When the 64 KB pipe buffer fills up, aria2's ``write()`` to stderr
|
||||
blocks, which freezes the entire ``aria2c`` process — including its
|
||||
RPC handler. This background task reads lines from stderr as they
|
||||
arrive and forwards them to Python's logger.
|
||||
"""
|
||||
try:
|
||||
assert self._process is not None and self._process.stderr is not None
|
||||
async for line in self._process.stderr:
|
||||
text = line.decode("utf-8", errors="replace").rstrip()
|
||||
if text:
|
||||
logger.debug("aria2 stderr: %s", text)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def _dispatch_progress(self, callback, snapshot: DownloadProgress) -> None:
|
||||
try:
|
||||
result = callback(snapshot, snapshot)
|
||||
except TypeError:
|
||||
result = callback(snapshot.percent_complete)
|
||||
|
||||
if asyncio.iscoroutine(result):
|
||||
await result
|
||||
elif hasattr(result, "__await__"):
|
||||
await result
|
||||
|
||||
def _build_progress_snapshot(self, status: Dict[str, Any]) -> DownloadProgress:
|
||||
completed = self._parse_int(status.get("completedLength"))
|
||||
total = self._parse_int(status.get("totalLength"))
|
||||
speed = float(self._parse_int(status.get("downloadSpeed")))
|
||||
percent = 0.0
|
||||
if total > 0:
|
||||
percent = (completed / total) * 100.0
|
||||
|
||||
return DownloadProgress(
|
||||
percent_complete=max(0.0, min(percent, 100.0)),
|
||||
bytes_downloaded=completed,
|
||||
total_bytes=total or None,
|
||||
bytes_per_second=speed,
|
||||
timestamp=datetime.now().timestamp(),
|
||||
)
|
||||
|
||||
def _resolve_completed_path(self, status: Dict[str, Any], default_path: str) -> str:
|
||||
files = status.get("files")
|
||||
if isinstance(files, list) and files:
|
||||
first = files[0]
|
||||
if isinstance(first, dict):
|
||||
candidate = first.get("path")
|
||||
if isinstance(candidate, str) and candidate:
|
||||
return candidate
|
||||
return default_path
|
||||
|
||||
@staticmethod
|
||||
def _parse_int(value: Any) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
async def _resolve_authenticated_redirect_url(
|
||||
self,
|
||||
url: str,
|
||||
headers: Dict[str, str],
|
||||
) -> str:
|
||||
downloader = await get_downloader()
|
||||
session = await downloader.session
|
||||
request_headers = dict(downloader.default_headers)
|
||||
request_headers.update(headers)
|
||||
request_headers["Accept-Encoding"] = "identity"
|
||||
|
||||
try:
|
||||
async with session.get(
|
||||
url,
|
||||
headers=request_headers,
|
||||
allow_redirects=False,
|
||||
proxy=downloader.proxy_url,
|
||||
) as response:
|
||||
if response.status in {301, 302, 303, 307, 308}:
|
||||
location = response.headers.get("Location")
|
||||
if location:
|
||||
return location
|
||||
raise Aria2Error(
|
||||
"Authenticated Civitai redirect did not include a Location header"
|
||||
)
|
||||
|
||||
if response.status == 200:
|
||||
return url
|
||||
|
||||
body = await response.text()
|
||||
raise Aria2Error(
|
||||
f"Failed to resolve authenticated Civitai redirect: status={response.status} body={body[:300]}"
|
||||
)
|
||||
except aiohttp.ClientError as exc:
|
||||
if is_ssl_cert_verify_error(exc):
|
||||
logger.error(
|
||||
"SSL certificate verification failed during Civitai redirect "
|
||||
"resolution for %s. This is usually caused by an outdated CA "
|
||||
"certificate bundle. Recommended fixes:\n"
|
||||
" 1. pip install --upgrade certifi\n"
|
||||
" 2. pip install pip-system-certs",
|
||||
url,
|
||||
)
|
||||
raise Aria2Error(
|
||||
f"Failed to resolve authenticated Civitai redirect: {exc}"
|
||||
) from exc
|
||||
|
||||
async def _ensure_process(self) -> None:
|
||||
async with self._process_lock:
|
||||
if self.is_running and await self._ping():
|
||||
return
|
||||
|
||||
await self.close()
|
||||
|
||||
executable = self._resolve_executable()
|
||||
self._rpc_port = self._find_free_port()
|
||||
self._rpc_secret = secrets.token_hex(16)
|
||||
self._rpc_url = f"http://127.0.0.1:{self._rpc_port}/jsonrpc"
|
||||
|
||||
command = [
|
||||
executable,
|
||||
"--enable-rpc=true",
|
||||
"--rpc-listen-all=false",
|
||||
f"--rpc-listen-port={self._rpc_port}",
|
||||
f"--rpc-secret={self._rpc_secret}",
|
||||
"--check-certificate=true",
|
||||
# Point aria2 at certifi's CA bundle when available so it uses
|
||||
# the same certificate store as Python downloads.
|
||||
*((
|
||||
f"--ca-certificate={ca_cert}",
|
||||
) if (ca_cert := _try_certifi_ca_path()) else ()),
|
||||
"--allow-overwrite=true",
|
||||
"--auto-file-renaming=false",
|
||||
"--file-allocation=none",
|
||||
"--max-concurrent-downloads=5",
|
||||
"--continue=true",
|
||||
"--daemon=false",
|
||||
"--quiet=true",
|
||||
f"--stop-with-process={os.getpid()}",
|
||||
]
|
||||
|
||||
logger.info("Starting aria2 RPC daemon from %s", executable)
|
||||
self._process = await asyncio.create_subprocess_exec(
|
||||
*command,
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
|
||||
await self._wait_until_ready()
|
||||
|
||||
# Drain aria2's stderr in a background task so the pipe buffer
|
||||
# never fills up. If the pipe blocks, aria2 itself freezes and
|
||||
# cannot respond to RPC — this was the root cause of the
|
||||
# "Failed to query aria2 download status" timeout bug.
|
||||
# Must start AFTER _wait_until_ready to avoid a race where the
|
||||
# drain task consumes aria2's early-exit error message before
|
||||
# _wait_until_ready can read it.
|
||||
self._stderr_reader_task = asyncio.create_task(
|
||||
self._drain_stderr()
|
||||
)
|
||||
|
||||
def _resolve_executable(self) -> str:
|
||||
settings = get_settings_manager()
|
||||
configured_path = (settings.get("aria2c_path") or "").strip()
|
||||
candidate = configured_path or "aria2c"
|
||||
|
||||
resolved = shutil.which(candidate)
|
||||
if resolved:
|
||||
return resolved
|
||||
|
||||
if configured_path and os.path.isfile(configured_path) and os.access(
|
||||
configured_path, os.X_OK
|
||||
):
|
||||
return configured_path
|
||||
|
||||
raise Aria2Error(
|
||||
"aria2c executable was not found. Install aria2 or configure aria2c_path."
|
||||
)
|
||||
|
||||
async def _wait_until_ready(self) -> None:
|
||||
assert self._process is not None
|
||||
|
||||
start_time = asyncio.get_running_loop().time()
|
||||
last_error = ""
|
||||
while asyncio.get_running_loop().time() - start_time < 10.0:
|
||||
if self._process.returncode is not None:
|
||||
stderr_output = ""
|
||||
if self._process.stderr is not None:
|
||||
try:
|
||||
stderr_output = (
|
||||
await asyncio.wait_for(self._process.stderr.read(), timeout=0.2)
|
||||
).decode("utf-8", errors="replace")
|
||||
except Exception:
|
||||
stderr_output = ""
|
||||
raise Aria2Error(
|
||||
f"aria2 RPC process exited early with code {self._process.returncode}: {stderr_output.strip()}"
|
||||
)
|
||||
|
||||
try:
|
||||
if await self._ping():
|
||||
return
|
||||
except Exception as exc: # pragma: no cover - startup race
|
||||
last_error = str(exc)
|
||||
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
raise Aria2Error(
|
||||
f"Timed out waiting for aria2 RPC to become ready{': ' + last_error if last_error else ''}"
|
||||
)
|
||||
|
||||
async def _ping(self) -> bool:
|
||||
try:
|
||||
result = await self._rpc_call("aria2.getVersion", [])
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
return isinstance(result, dict)
|
||||
|
||||
async def _rpc_call(self, method: str, params: list[Any]) -> Any:
|
||||
if not self._rpc_url:
|
||||
raise Aria2Error("aria2 RPC endpoint is not initialized")
|
||||
|
||||
session = await self._get_rpc_session()
|
||||
payload = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": secrets.token_hex(8),
|
||||
"method": method,
|
||||
"params": [f"token:{self._rpc_secret}", *params],
|
||||
}
|
||||
|
||||
async with session.post(self._rpc_url, json=payload) as response:
|
||||
text = await response.text()
|
||||
|
||||
try:
|
||||
body = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
body = None
|
||||
|
||||
if body is None:
|
||||
if response.status != 200:
|
||||
raise Aria2Error(
|
||||
f"aria2 RPC returned status {response.status} with non-JSON body: {text}"
|
||||
)
|
||||
raise Aria2Error(f"Invalid aria2 RPC response: {text}")
|
||||
|
||||
if "error" in body:
|
||||
error = body["error"] or {}
|
||||
code = error.get("code") if isinstance(error, dict) else None
|
||||
message = error.get("message") if isinstance(error, dict) else str(error)
|
||||
logger.error(
|
||||
"aria2 RPC %s failed with HTTP %s, code=%s, message=%s",
|
||||
method,
|
||||
response.status,
|
||||
code,
|
||||
message,
|
||||
)
|
||||
status_message = (
|
||||
f"aria2 RPC {method} failed with status {response.status}: {message}"
|
||||
if response.status != 200
|
||||
else message
|
||||
)
|
||||
raise Aria2Error(status_message or "Unknown aria2 RPC error")
|
||||
|
||||
if response.status != 200:
|
||||
logger.error(
|
||||
"aria2 RPC %s returned unexpected HTTP status %s without error payload: %s",
|
||||
method,
|
||||
response.status,
|
||||
body,
|
||||
)
|
||||
raise Aria2Error(
|
||||
f"aria2 RPC {method} returned unexpected status {response.status}"
|
||||
)
|
||||
|
||||
return body.get("result")
|
||||
|
||||
async def _get_rpc_session(self) -> aiohttp.ClientSession:
|
||||
if self._rpc_session is None or self._rpc_session.closed:
|
||||
async with self._rpc_session_lock:
|
||||
if self._rpc_session is None or self._rpc_session.closed:
|
||||
timeout = aiohttp.ClientTimeout(
|
||||
total=None, sock_connect=10, sock_read=60
|
||||
)
|
||||
self._rpc_session = aiohttp.ClientSession(timeout=timeout)
|
||||
return self._rpc_session
|
||||
|
||||
@staticmethod
|
||||
def _find_free_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
sock.listen(1)
|
||||
return int(sock.getsockname()[1])
|
||||
|
||||
|
||||
async def get_aria2_downloader() -> Aria2Downloader:
|
||||
"""Get the singleton aria2 downloader."""
|
||||
|
||||
return await Aria2Downloader.get_instance()
|
||||
@@ -0,0 +1,108 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from copy import deepcopy
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from ..utils.cache_paths import get_cache_base_dir
|
||||
|
||||
|
||||
def get_aria2_state_path() -> str:
|
||||
base_dir = get_cache_base_dir(create=True)
|
||||
state_dir = os.path.join(base_dir, "aria2")
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
return os.path.join(state_dir, "downloads.json")
|
||||
|
||||
|
||||
class Aria2TransferStateStore:
|
||||
"""Persist aria2 transfer metadata needed for restart recovery."""
|
||||
|
||||
_locks_by_path: Dict[str, asyncio.Lock] = {}
|
||||
|
||||
def __init__(self, state_path: Optional[str] = None) -> None:
|
||||
self._state_path = os.path.abspath(state_path or get_aria2_state_path())
|
||||
self._lock = self._locks_by_path.setdefault(self._state_path, asyncio.Lock())
|
||||
|
||||
def _read_all_unlocked(self) -> Dict[str, Dict[str, Any]]:
|
||||
try:
|
||||
with open(self._state_path, "r", encoding="utf-8") as handle:
|
||||
data = json.load(handle)
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
|
||||
normalized: Dict[str, Dict[str, Any]] = {}
|
||||
for download_id, entry in data.items():
|
||||
if isinstance(download_id, str) and isinstance(entry, dict):
|
||||
normalized[download_id] = entry
|
||||
return normalized
|
||||
|
||||
def _write_all_unlocked(self, data: Dict[str, Dict[str, Any]]) -> None:
|
||||
directory = os.path.dirname(self._state_path)
|
||||
if directory:
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
|
||||
temp_path = f"{self._state_path}.tmp"
|
||||
with open(temp_path, "w", encoding="utf-8") as handle:
|
||||
json.dump(data, handle, ensure_ascii=True, indent=2, sort_keys=True)
|
||||
os.replace(temp_path, self._state_path)
|
||||
|
||||
async def load_all(self) -> Dict[str, Dict[str, Any]]:
|
||||
async with self._lock:
|
||||
return deepcopy(self._read_all_unlocked())
|
||||
|
||||
async def get(self, download_id: str) -> Optional[Dict[str, Any]]:
|
||||
async with self._lock:
|
||||
return deepcopy(self._read_all_unlocked().get(download_id))
|
||||
|
||||
async def upsert(self, download_id: str, payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
async with self._lock:
|
||||
data = self._read_all_unlocked()
|
||||
current = data.get(download_id, {})
|
||||
current.update(payload)
|
||||
data[download_id] = current
|
||||
self._write_all_unlocked(data)
|
||||
return deepcopy(current)
|
||||
|
||||
async def remove(self, download_id: str) -> None:
|
||||
async with self._lock:
|
||||
data = self._read_all_unlocked()
|
||||
if download_id in data:
|
||||
del data[download_id]
|
||||
self._write_all_unlocked(data)
|
||||
|
||||
async def find_by_save_path(
|
||||
self, save_path: str, *, exclude_download_id: Optional[str] = None
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
normalized_target = os.path.abspath(save_path)
|
||||
async with self._lock:
|
||||
data = self._read_all_unlocked()
|
||||
for download_id, entry in data.items():
|
||||
if exclude_download_id and download_id == exclude_download_id:
|
||||
continue
|
||||
candidate = entry.get("save_path")
|
||||
if isinstance(candidate, str) and os.path.abspath(candidate) == normalized_target:
|
||||
result = dict(entry)
|
||||
result["download_id"] = download_id
|
||||
return result
|
||||
return None
|
||||
|
||||
async def reassign(self, from_download_id: str, to_download_id: str) -> Optional[Dict[str, Any]]:
|
||||
async with self._lock:
|
||||
data = self._read_all_unlocked()
|
||||
existing = data.get(from_download_id)
|
||||
if existing is None:
|
||||
return None
|
||||
updated = dict(existing)
|
||||
updated["download_id"] = to_download_id
|
||||
data[to_download_id] = updated
|
||||
if from_download_id != to_download_id:
|
||||
data.pop(from_download_id, None)
|
||||
self._write_all_unlocked(data)
|
||||
return deepcopy(updated)
|
||||
@@ -0,0 +1,139 @@
|
||||
"""
|
||||
Auto-tag extraction service for model cards.
|
||||
|
||||
Extracts implicit model attributes (HIGH/LOW, I2V/T2V/TI2V, Lightning, Turbo)
|
||||
from filename, base_model, and CivitAI version name — no manual tagging required.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Dict, List, Set
|
||||
|
||||
# ── Tag category definitions ──────────────────────────────────────────
|
||||
# Each category maps a display label to a regex pattern.
|
||||
# Patterns are case-insensitive and matched against filename, base_model,
|
||||
# and civitai version name.
|
||||
|
||||
# Use (?<![a-zA-Z0-9]) and (?![a-zA-Z0-9]) instead of \b because
|
||||
# Python's \b treats underscore as a word character, so \bHIGH\b
|
||||
# won't match '_HIGH_' in filenames.
|
||||
_B = r"(?<![a-zA-Z0-9])" # left boundary
|
||||
_E = r"(?![a-zA-Z0-9])" # right boundary
|
||||
|
||||
AUTO_TAG_CATEGORIES: Dict[str, str] = {
|
||||
"HIGH": _B + r"HIGH" + _E,
|
||||
"LOW": _B + r"(?<!F)LOW" + _E,
|
||||
"I2V": _B + r"I2V" + _E,
|
||||
"T2V": _B + r"T2V" + _E,
|
||||
"TI2V": _B + r"TI2V" + _E,
|
||||
"Lightning": _B + r"Lightning" + _E,
|
||||
"Turbo": _B + r"Turbo" + _E,
|
||||
}
|
||||
|
||||
# Tags that belong to the "mode" group (HIGH/LOW)
|
||||
MODE_TAGS = {"HIGH", "LOW"}
|
||||
|
||||
# Tags that belong to the "video mode" group (I2V/T2V/TI2V)
|
||||
VIDEO_MODE_TAGS = {"I2V", "T2V", "TI2V"}
|
||||
|
||||
# Tags that belong to the "speed/optimization" group
|
||||
SPEED_TAGS = {"Lightning", "Turbo"}
|
||||
|
||||
# ── Display category groups (for settings UI) ─────────────────────────
|
||||
|
||||
AUTO_TAG_GROUPS = {
|
||||
"mode": {"HIGH", "LOW"},
|
||||
"video": {"I2V", "T2V", "TI2V"},
|
||||
"speed": {"Lightning", "Turbo"},
|
||||
}
|
||||
|
||||
# Default enabled categories
|
||||
DEFAULT_ENABLED_GROUPS = {"mode", "video"}
|
||||
|
||||
|
||||
def _collect_sources(model_data: Dict[str, Any]) -> List[str]:
|
||||
"""Collect all text sources from model data for tag matching."""
|
||||
sources: List[str] = []
|
||||
|
||||
file_name = model_data.get("file_name", "")
|
||||
if file_name:
|
||||
sources.append(file_name)
|
||||
|
||||
base_model = model_data.get("base_model", "")
|
||||
if base_model:
|
||||
sources.append(base_model)
|
||||
|
||||
civitai = model_data.get("civitai", {})
|
||||
if isinstance(civitai, dict):
|
||||
version_name = civitai.get("name", "")
|
||||
if version_name:
|
||||
sources.append(version_name)
|
||||
|
||||
return sources
|
||||
|
||||
|
||||
def extract_auto_tags(model_data: Dict[str, Any]) -> List[str]:
|
||||
"""Extract auto-detected tags from model metadata.
|
||||
|
||||
Uses a two-layer approach:
|
||||
Layer 1 — Regex-based detection against filename, base_model, and
|
||||
CivitAI version name.
|
||||
Layer 2 — Merge in any user-defined tags that overlap with known
|
||||
auto-tag categories. This provides a manual fallback when
|
||||
auto-detection fails (e.g. "I2V HN" or unlabeled models).
|
||||
|
||||
HIGH/LOW tags are only returned when the base_model indicates a Wan
|
||||
family model — no other model architecture uses this distinction.
|
||||
|
||||
Args:
|
||||
model_data: Model metadata dict with keys:
|
||||
file_name, base_model, civitai (with optional 'name' field),
|
||||
tags (user-defined tag list, used as fallback).
|
||||
|
||||
Returns:
|
||||
Sorted list of unique auto-tag strings (e.g. ["I2V"]).
|
||||
"""
|
||||
sources = _collect_sources(model_data)
|
||||
base_model = model_data.get("base_model", "")
|
||||
is_wan = "wan" in base_model.lower()
|
||||
|
||||
found: Set[str] = set()
|
||||
|
||||
# ── Layer 1: regex-based detection ────────────────────────────
|
||||
if sources:
|
||||
for label, pattern in AUTO_TAG_CATEGORIES.items():
|
||||
# HIGH/LOW are Wan-specific — skip for non-Wan to avoid noise
|
||||
if label in ("HIGH", "LOW"):
|
||||
if not is_wan:
|
||||
continue
|
||||
# Use case-insensitive character class + case-sensitive boundary,
|
||||
# so "HighNoise" (camelCase) matches but "highlight" doesn't.
|
||||
# Boundary: not followed by lowercase letter (= word has ended).
|
||||
ci = "".join(f"[{c.lower()}{c.upper()}]" for c in label)
|
||||
if label == "LOW":
|
||||
regex = re.compile(r"(?<![Ff])" + ci + r"(?![a-z])")
|
||||
else:
|
||||
regex = re.compile(ci + r"(?![a-z])")
|
||||
else:
|
||||
regex = re.compile(pattern, re.IGNORECASE)
|
||||
for source in sources:
|
||||
if regex.search(source):
|
||||
found.add(label)
|
||||
break
|
||||
|
||||
# ── Layer 2: user-defined tags as manual fallback ─────────────
|
||||
# When auto-detection fails (abbreviated names like "Hi"/"Lo",
|
||||
# "I2V HN", or unlabeled models), users can add canonical tags
|
||||
# (HIGH, LOW, I2V, etc.) to the model's regular tags for correct
|
||||
# badge display and filtering. Matching is case-insensitive so
|
||||
# "high"/"High"/"HIGH" all resolve to the canonical label.
|
||||
user_tags = model_data.get("tags")
|
||||
if user_tags:
|
||||
label_map = {label.lower(): label for label in AUTO_TAG_CATEGORIES}
|
||||
for t in user_tags:
|
||||
canonical = label_map.get(t.lower())
|
||||
if canonical:
|
||||
found.add(canonical)
|
||||
|
||||
return sorted(found)
|
||||
@@ -0,0 +1,144 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
"""Backfill the AutoV3 checked state for models loaded from a persisted snapshot.
|
||||
|
||||
The SQLite persistent cache predates the AutoV3 feature, so entries hydrated
|
||||
from it have a NULL ``autov3`` column (the "not checked yet" state). This
|
||||
service computes the embedded AutoV3 hash for each such model — once per
|
||||
process — and persists it through the scanner's single write path
|
||||
(:meth:`ModelScanner.update_autov3_for_model`), marking every visited row so a
|
||||
subsequent run finds nothing left to do.
|
||||
|
||||
Three-state contract honored here:
|
||||
|
||||
- ``NULL`` (sqlite) / absent (dict) = not checked yet → backfill computes it
|
||||
- ``''`` (sqlite/dict) / JSON null = checked, no value available → never recompute
|
||||
- 12-char lowercase hex = value → never recompute
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover - type-check only; runtime imports are local
|
||||
from .model_scanner import ModelScanner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _resolve_autov3(file_path: str) -> str:
|
||||
"""Resolve the AutoV3 hash for a model file.
|
||||
|
||||
Prefers the Civitai AutoV3 reported for the file whose SHA256 matches
|
||||
(the authoritative value for recipe matching); falls back to the embedded
|
||||
safetensors header hash. Returns ``''`` when neither is available.
|
||||
"""
|
||||
try:
|
||||
metadata_path = f"{os.path.splitext(file_path)[0]}.metadata.json"
|
||||
if os.path.exists(metadata_path):
|
||||
with open(metadata_path, "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
if isinstance(payload, dict):
|
||||
from ..utils.models import autov3_from_civitai_files # local import avoids cycles
|
||||
|
||||
sha256 = (payload.get("sha256") or "").lower()
|
||||
civitai_autov3 = autov3_from_civitai_files(payload.get("civitai"), sha256)
|
||||
if civitai_autov3:
|
||||
return civitai_autov3
|
||||
except Exception:
|
||||
pass
|
||||
from ..utils.file_utils import calculate_autov3 # local import avoids cycles
|
||||
|
||||
return calculate_autov3(file_path) or ""
|
||||
|
||||
|
||||
class Autov3BackfillService:
|
||||
"""Compute and persist AutoV3 hashes for models missing a checked state."""
|
||||
|
||||
_instance: Optional["Autov3BackfillService"] = None
|
||||
_instance_lock = threading.Lock()
|
||||
|
||||
def __init__(self) -> None:
|
||||
# Re-entrancy guard per model type: scanners for different model types
|
||||
# initialize concurrently (lora_manager.py), so a global guard would
|
||||
# silently skip every type but the first to start. Each model type
|
||||
# runs its own backfill; a duplicate trigger for the same type no-ops.
|
||||
self._running_types: set[str] = set()
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls) -> "Autov3BackfillService":
|
||||
"""Return the process-wide singleton instance."""
|
||||
if cls._instance is None:
|
||||
with cls._instance_lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = cls()
|
||||
return cls._instance
|
||||
|
||||
async def backfill(self, scanner: "ModelScanner") -> int:
|
||||
"""Compute AutoV3 for every un-checked model of ``scanner.model_type``.
|
||||
|
||||
Each candidate file is read once via :func:`~py.utils.file_utils.calculate_autov3`
|
||||
(cheap: safetensors header only) and the result is persisted through
|
||||
``scanner.update_autov3_for_model``. Files that no longer exist on
|
||||
disk are skipped — they are intentionally NOT marked, because scanner
|
||||
cleanup removes the stale row later.
|
||||
|
||||
Returns:
|
||||
The number of models successfully updated. Never raises; on any
|
||||
failure a warning is logged and ``0`` is returned. A duplicate
|
||||
trigger for a model type that is already being backfilled returns
|
||||
``0`` immediately; different model types run concurrently.
|
||||
"""
|
||||
model_type = scanner.model_type
|
||||
if model_type in self._running_types:
|
||||
return 0
|
||||
self._running_types.add(model_type)
|
||||
try:
|
||||
# Local imports avoid import cycles at module load time.
|
||||
from .persistent_model_cache import get_persistent_cache
|
||||
from ..utils.file_utils import calculate_autov3
|
||||
|
||||
persistent = getattr(scanner, "_persistent_cache", None) or get_persistent_cache()
|
||||
paths = persistent.get_models_missing_autov3(model_type)
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
count = 0
|
||||
for path in paths:
|
||||
# A file that no longer exists must not be marked; scanner
|
||||
# cleanup removes the stale row later. The existence check and
|
||||
# hash resolution run in the executor so the loop stays
|
||||
# responsive to API requests while the backfill iterates a
|
||||
# large library.
|
||||
if not await loop.run_in_executor(None, os.path.exists, path):
|
||||
continue
|
||||
autov3 = await loop.run_in_executor(None, _resolve_autov3, path)
|
||||
if await scanner.update_autov3_for_model(model_type, path, autov3):
|
||||
count += 1
|
||||
|
||||
if paths:
|
||||
logger.info(
|
||||
"AutoV3 backfill: updated %d/%d models for %s",
|
||||
count,
|
||||
len(paths),
|
||||
model_type,
|
||||
)
|
||||
else:
|
||||
# Steady state after the first run: nothing left to backfill.
|
||||
logger.debug("AutoV3 backfill: nothing to process for %s", model_type)
|
||||
return count
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"AutoV3 backfill failed for %s: %s",
|
||||
getattr(scanner, "model_type", "?"),
|
||||
exc,
|
||||
)
|
||||
return 0
|
||||
finally:
|
||||
self._running_types.discard(model_type)
|
||||
@@ -0,0 +1,427 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Optional
|
||||
|
||||
from ..utils.cache_paths import CacheType, get_cache_base_dir, get_cache_file_path
|
||||
from ..utils.settings_paths import get_settings_dir
|
||||
from .settings_manager import get_settings_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
BACKUP_MANIFEST_VERSION = 1
|
||||
DEFAULT_BACKUP_RETENTION_COUNT = 5
|
||||
DEFAULT_BACKUP_INTERVAL_SECONDS = 24 * 60 * 60
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackupEntry:
|
||||
kind: str
|
||||
archive_path: str
|
||||
target_path: str
|
||||
sha256: str
|
||||
size: int
|
||||
mtime: float
|
||||
|
||||
|
||||
class BackupService:
|
||||
"""Create and restore user-state backup archives."""
|
||||
|
||||
_instance: "BackupService | None" = None
|
||||
_instance_lock = asyncio.Lock()
|
||||
|
||||
def __init__(self, *, settings_manager=None, backup_dir: str | None = None) -> None:
|
||||
self._settings = settings_manager or get_settings_manager()
|
||||
self._backup_dir = Path(backup_dir or self._resolve_backup_dir())
|
||||
self._backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._lock = asyncio.Lock()
|
||||
self._auto_task: asyncio.Task[None] | None = None
|
||||
|
||||
@classmethod
|
||||
async def get_instance(cls) -> "BackupService":
|
||||
async with cls._instance_lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = cls()
|
||||
cls._instance._ensure_auto_snapshot_task()
|
||||
return cls._instance
|
||||
|
||||
@staticmethod
|
||||
def _resolve_backup_dir() -> str:
|
||||
return os.path.join(get_settings_dir(create=True), "backups")
|
||||
|
||||
def get_backup_dir(self) -> str:
|
||||
return str(self._backup_dir)
|
||||
|
||||
def _ensure_auto_snapshot_task(self) -> None:
|
||||
if self._auto_task is not None and not self._auto_task.done():
|
||||
return
|
||||
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return
|
||||
|
||||
self._auto_task = loop.create_task(self._auto_backup_loop())
|
||||
|
||||
def _get_setting_bool(self, key: str, default: bool) -> bool:
|
||||
try:
|
||||
return bool(self._settings.get(key, default))
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
def _get_setting_int(self, key: str, default: int) -> int:
|
||||
try:
|
||||
value = self._settings.get(key, default)
|
||||
return max(1, int(value))
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
def _settings_file_path(self) -> str:
|
||||
settings_file = getattr(self._settings, "settings_file", None)
|
||||
if settings_file:
|
||||
return str(settings_file)
|
||||
return os.path.join(get_settings_dir(create=True), "settings.json")
|
||||
|
||||
def _download_history_path(self) -> str:
|
||||
base_dir = get_cache_base_dir(create=True)
|
||||
history_dir = os.path.join(base_dir, "download_history")
|
||||
os.makedirs(history_dir, exist_ok=True)
|
||||
return os.path.join(history_dir, "downloaded_versions.sqlite")
|
||||
|
||||
def _model_update_dir(self) -> str:
|
||||
return str(Path(get_cache_file_path(CacheType.MODEL_UPDATE, create_dir=True)).parent)
|
||||
|
||||
def _model_update_targets(self) -> list[tuple[str, str, str]]:
|
||||
"""Return (kind, archive_path, target_path) tuples for backup."""
|
||||
|
||||
targets: list[tuple[str, str, str]] = []
|
||||
|
||||
settings_path = self._settings_file_path()
|
||||
targets.append(("settings", "settings/settings.json", settings_path))
|
||||
|
||||
history_path = self._download_history_path()
|
||||
targets.append(
|
||||
(
|
||||
"download_history",
|
||||
"cache/download_history/downloaded_versions.sqlite",
|
||||
history_path,
|
||||
)
|
||||
)
|
||||
|
||||
symlink_path = get_cache_file_path(CacheType.SYMLINK, create_dir=True)
|
||||
targets.append(
|
||||
(
|
||||
"symlink_map",
|
||||
"cache/symlink/symlink_map.json",
|
||||
symlink_path,
|
||||
)
|
||||
)
|
||||
|
||||
model_update_dir = Path(self._model_update_dir())
|
||||
if model_update_dir.exists():
|
||||
for sqlite_file in sorted(model_update_dir.glob("*.sqlite")):
|
||||
targets.append(
|
||||
(
|
||||
"model_update",
|
||||
f"cache/model_update/{sqlite_file.name}",
|
||||
str(sqlite_file),
|
||||
)
|
||||
)
|
||||
|
||||
stats_path = os.path.join(get_settings_dir(create=True), "stats", "lora_manager_stats.json")
|
||||
if os.path.exists(stats_path):
|
||||
targets.append(
|
||||
(
|
||||
"usage_stats",
|
||||
"stats/lora_manager_stats.json",
|
||||
stats_path,
|
||||
)
|
||||
)
|
||||
|
||||
return targets
|
||||
|
||||
@staticmethod
|
||||
def _hash_file(path: str) -> tuple[str, int, float]:
|
||||
digest = hashlib.sha256()
|
||||
total = 0
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
total += len(chunk)
|
||||
digest.update(chunk)
|
||||
mtime = os.path.getmtime(path)
|
||||
return digest.hexdigest(), total, mtime
|
||||
|
||||
def _build_manifest(self, entries: Iterable[BackupEntry], *, snapshot_type: str) -> dict[str, Any]:
|
||||
created_at = datetime.now(timezone.utc).isoformat()
|
||||
active_library = None
|
||||
try:
|
||||
active_library = self._settings.get_active_library_name()
|
||||
except Exception:
|
||||
active_library = None
|
||||
|
||||
return {
|
||||
"manifest_version": BACKUP_MANIFEST_VERSION,
|
||||
"created_at": created_at,
|
||||
"snapshot_type": snapshot_type,
|
||||
"active_library": active_library,
|
||||
"files": [
|
||||
{
|
||||
"kind": entry.kind,
|
||||
"archive_path": entry.archive_path,
|
||||
"target_path": entry.target_path,
|
||||
"sha256": entry.sha256,
|
||||
"size": entry.size,
|
||||
"mtime": entry.mtime,
|
||||
}
|
||||
for entry in entries
|
||||
],
|
||||
}
|
||||
|
||||
def _write_archive(self, archive_path: str, entries: list[BackupEntry], manifest: dict[str, Any]) -> None:
|
||||
with zipfile.ZipFile(
|
||||
archive_path,
|
||||
mode="w",
|
||||
compression=zipfile.ZIP_DEFLATED,
|
||||
compresslevel=6,
|
||||
) as zf:
|
||||
zf.writestr(
|
||||
"manifest.json",
|
||||
json.dumps(manifest, indent=2, ensure_ascii=False).encode("utf-8"),
|
||||
)
|
||||
for entry in entries:
|
||||
zf.write(entry.target_path, arcname=entry.archive_path)
|
||||
|
||||
async def create_snapshot(self, *, snapshot_type: str = "manual", persist: bool = False) -> dict[str, Any]:
|
||||
"""Create a backup archive.
|
||||
|
||||
If ``persist`` is true, the archive is stored in the backup directory
|
||||
and retained according to the configured retention policy.
|
||||
"""
|
||||
|
||||
async with self._lock:
|
||||
raw_targets = self._model_update_targets()
|
||||
entries: list[BackupEntry] = []
|
||||
for kind, archive_path, target_path in raw_targets:
|
||||
if not os.path.exists(target_path):
|
||||
continue
|
||||
sha256, size, mtime = self._hash_file(target_path)
|
||||
entries.append(
|
||||
BackupEntry(
|
||||
kind=kind,
|
||||
archive_path=archive_path,
|
||||
target_path=target_path,
|
||||
sha256=sha256,
|
||||
size=size,
|
||||
mtime=mtime,
|
||||
)
|
||||
)
|
||||
|
||||
if not entries:
|
||||
raise FileNotFoundError("No backupable files were found")
|
||||
|
||||
manifest = self._build_manifest(entries, snapshot_type=snapshot_type)
|
||||
archive_name = self._build_archive_name(snapshot_type=snapshot_type)
|
||||
fd, temp_path = tempfile.mkstemp(suffix=".zip", dir=str(self._backup_dir))
|
||||
os.close(fd)
|
||||
|
||||
try:
|
||||
self._write_archive(temp_path, entries, manifest)
|
||||
if persist:
|
||||
final_path = self._backup_dir / archive_name
|
||||
os.replace(temp_path, final_path)
|
||||
self._prune_snapshots()
|
||||
return {
|
||||
"archive_path": str(final_path),
|
||||
"archive_name": final_path.name,
|
||||
"manifest": manifest,
|
||||
}
|
||||
|
||||
with open(temp_path, "rb") as handle:
|
||||
data = handle.read()
|
||||
return {
|
||||
"archive_name": archive_name,
|
||||
"archive_bytes": data,
|
||||
"manifest": manifest,
|
||||
}
|
||||
finally:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.remove(temp_path)
|
||||
|
||||
def _build_archive_name(self, *, snapshot_type: str) -> str:
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
return f"lora-manager-backup-{timestamp}-{snapshot_type}.zip"
|
||||
|
||||
def _prune_snapshots(self) -> None:
|
||||
retention = self._get_setting_int(
|
||||
"backup_retention_count", DEFAULT_BACKUP_RETENTION_COUNT
|
||||
)
|
||||
archives = sorted(
|
||||
self._backup_dir.glob("lora-manager-backup-*-auto.zip"),
|
||||
key=lambda path: path.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
for path in archives[retention:]:
|
||||
with contextlib.suppress(OSError):
|
||||
path.unlink()
|
||||
|
||||
async def restore_snapshot(self, archive_path: str) -> dict[str, Any]:
|
||||
"""Restore backup contents from a ZIP archive."""
|
||||
|
||||
async with self._lock:
|
||||
try:
|
||||
zf = zipfile.ZipFile(archive_path, mode="r")
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise ValueError("Backup archive is not a valid ZIP file") from exc
|
||||
|
||||
with zf:
|
||||
try:
|
||||
manifest = json.loads(zf.read("manifest.json").decode("utf-8"))
|
||||
except KeyError as exc:
|
||||
raise ValueError("Backup archive is missing manifest.json") from exc
|
||||
|
||||
if not isinstance(manifest, dict):
|
||||
raise ValueError("Backup manifest is invalid")
|
||||
if manifest.get("manifest_version") != BACKUP_MANIFEST_VERSION:
|
||||
raise ValueError("Backup manifest version is not supported")
|
||||
|
||||
files = manifest.get("files", [])
|
||||
if not isinstance(files, list):
|
||||
raise ValueError("Backup manifest file list is invalid")
|
||||
|
||||
extracted_paths: list[tuple[str, str]] = []
|
||||
temp_dir = Path(tempfile.mkdtemp(prefix="lora-manager-restore-"))
|
||||
try:
|
||||
for item in files:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
archive_member = item.get("archive_path")
|
||||
if not isinstance(archive_member, str) or not archive_member:
|
||||
continue
|
||||
archive_member_path = Path(archive_member)
|
||||
if archive_member_path.is_absolute() or ".." in archive_member_path.parts:
|
||||
raise ValueError(f"Invalid archive member path: {archive_member}")
|
||||
|
||||
kind = item.get("kind")
|
||||
target_path = self._resolve_restore_target(kind, archive_member)
|
||||
if target_path is None:
|
||||
continue
|
||||
|
||||
extracted_path = temp_dir / archive_member_path
|
||||
extracted_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(archive_member) as source, open(
|
||||
extracted_path, "wb"
|
||||
) as destination:
|
||||
shutil.copyfileobj(source, destination)
|
||||
|
||||
expected_hash = item.get("sha256")
|
||||
if isinstance(expected_hash, str) and expected_hash:
|
||||
actual_hash, _, _ = self._hash_file(str(extracted_path))
|
||||
if actual_hash != expected_hash:
|
||||
raise ValueError(
|
||||
f"Checksum mismatch for {archive_member}"
|
||||
)
|
||||
|
||||
extracted_paths.append((str(extracted_path), target_path))
|
||||
|
||||
for extracted_path, target_path in extracted_paths:
|
||||
os.makedirs(os.path.dirname(target_path), exist_ok=True)
|
||||
os.replace(extracted_path, target_path)
|
||||
finally:
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"restored_files": len(extracted_paths),
|
||||
"snapshot_type": manifest.get("snapshot_type"),
|
||||
}
|
||||
|
||||
def _resolve_restore_target(self, kind: Any, archive_member: str) -> str | None:
|
||||
if kind == "settings":
|
||||
return self._settings_file_path()
|
||||
if kind == "download_history":
|
||||
return self._download_history_path()
|
||||
if kind == "symlink_map":
|
||||
return get_cache_file_path(CacheType.SYMLINK, create_dir=True)
|
||||
if kind == "model_update":
|
||||
filename = os.path.basename(archive_member)
|
||||
return str(Path(get_cache_file_path(CacheType.MODEL_UPDATE, create_dir=True)).parent / filename)
|
||||
if kind == "usage_stats":
|
||||
return os.path.join(get_settings_dir(create=True), "stats", "lora_manager_stats.json")
|
||||
return None
|
||||
|
||||
async def create_auto_snapshot_if_due(self) -> Optional[dict[str, Any]]:
|
||||
if not self._get_setting_bool("backup_auto_enabled", True):
|
||||
return None
|
||||
|
||||
latest = self.get_latest_auto_snapshot()
|
||||
now = time.time()
|
||||
if latest and now - latest["mtime"] < DEFAULT_BACKUP_INTERVAL_SECONDS:
|
||||
return None
|
||||
|
||||
return await self.create_snapshot(snapshot_type="auto", persist=True)
|
||||
|
||||
async def _auto_backup_loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
await self.create_auto_snapshot_if_due()
|
||||
await asyncio.sleep(DEFAULT_BACKUP_INTERVAL_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # pragma: no cover - defensive guard
|
||||
logger.warning("Automatic backup snapshot failed: %s", exc, exc_info=True)
|
||||
await asyncio.sleep(60)
|
||||
|
||||
def get_available_snapshots(self) -> list[dict[str, Any]]:
|
||||
snapshots: list[dict[str, Any]] = []
|
||||
for path in sorted(self._backup_dir.glob("lora-manager-backup-*.zip")):
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError:
|
||||
continue
|
||||
snapshots.append(
|
||||
{
|
||||
"name": path.name,
|
||||
"path": str(path),
|
||||
"size": stat.st_size,
|
||||
"mtime": stat.st_mtime,
|
||||
"is_auto": path.name.endswith("-auto.zip"),
|
||||
}
|
||||
)
|
||||
snapshots.sort(key=lambda item: item["mtime"], reverse=True)
|
||||
return snapshots
|
||||
|
||||
def get_latest_auto_snapshot(self) -> Optional[dict[str, Any]]:
|
||||
autos = [snapshot for snapshot in self.get_available_snapshots() if snapshot["is_auto"]]
|
||||
if not autos:
|
||||
return None
|
||||
return autos[0]
|
||||
|
||||
def get_status(self) -> dict[str, Any]:
|
||||
snapshots = self.get_available_snapshots()
|
||||
return {
|
||||
"backupDir": self.get_backup_dir(),
|
||||
"enabled": self._get_setting_bool("backup_auto_enabled", True),
|
||||
"retentionCount": self._get_setting_int(
|
||||
"backup_retention_count", DEFAULT_BACKUP_RETENTION_COUNT
|
||||
),
|
||||
"snapshotCount": len(snapshots),
|
||||
"latestSnapshot": snapshots[0] if snapshots else None,
|
||||
"latestAutoSnapshot": self.get_latest_auto_snapshot(),
|
||||
}
|
||||
@@ -1,7 +1,8 @@
|
||||
from abc import ABC, abstractmethod
|
||||
import asyncio
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional, Type, TYPE_CHECKING
|
||||
import random
|
||||
from typing import Any, Awaitable, Dict, List, Optional, Type, Union, TYPE_CHECKING, cast
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
@@ -20,6 +21,7 @@ from .model_query import (
|
||||
resolve_sub_type,
|
||||
)
|
||||
from .settings_manager import get_settings_manager
|
||||
from ..utils.civitai_utils import build_civitai_model_page_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -68,23 +70,24 @@ class BaseModelService(ABC):
|
||||
page: int,
|
||||
page_size: int,
|
||||
sort_by: str = "name",
|
||||
folder: str = None,
|
||||
folder_include: list = None,
|
||||
folder_exclude: list = None,
|
||||
search: str = None,
|
||||
folder: str | None = None,
|
||||
folder_include: list[str] | None = None,
|
||||
folder_exclude: list[str] | None = None,
|
||||
search: str | None = None,
|
||||
fuzzy_search: bool = False,
|
||||
base_models: list = None,
|
||||
model_types: list = None,
|
||||
base_models: list[str] | None = None,
|
||||
model_types: list[str] | None = None,
|
||||
tags: Optional[Dict[str, str]] = None,
|
||||
search_options: dict = None,
|
||||
hash_filters: dict = None,
|
||||
auto_tags: Optional[Dict[str, str]] = None,
|
||||
search_options: dict[str, Any] | None = None,
|
||||
hash_filters: dict[str, Any] | None = None,
|
||||
favorites_only: bool = False,
|
||||
update_available_only: bool = False,
|
||||
credit_required: Optional[bool] = None,
|
||||
allow_selling_generated_content: Optional[bool] = None,
|
||||
tag_logic: str = "any",
|
||||
**kwargs,
|
||||
) -> Dict:
|
||||
) -> Dict[str, Any]:
|
||||
"""Get paginated and filtered model data"""
|
||||
overall_start = time.perf_counter()
|
||||
|
||||
@@ -94,9 +97,117 @@ class BaseModelService(ABC):
|
||||
sorted_data = await self._fetch_with_usage_sort(sort_params)
|
||||
else:
|
||||
sorted_data = await self.cache_repository.fetch_sorted(sort_params)
|
||||
# Pre-compute auto_tags for every item — needed for both filtering
|
||||
# and display. Computation is cheap (string regex on 2-3 fields).
|
||||
from .auto_tag_service import extract_auto_tags
|
||||
for item in sorted_data:
|
||||
item["auto_tags"] = extract_auto_tags(item)
|
||||
fetch_duration = time.perf_counter() - t0
|
||||
initial_count = len(sorted_data)
|
||||
|
||||
# Optionally filter by civitai model ID (shows all local versions of a specific model)
|
||||
civitai_model_id = kwargs.get("civitai_model_id")
|
||||
if civitai_model_id is not None:
|
||||
sorted_data = [
|
||||
item for item in sorted_data
|
||||
if self._extract_group_key(item) == civitai_model_id
|
||||
]
|
||||
# VLM mode: always sort by version ID descending (newest version first),
|
||||
# regardless of the current sort_by preference.
|
||||
# Fall back to modified timestamp for non-CivitAI sources.
|
||||
sorted_data.sort(
|
||||
key=lambda x: self._extract_version_id(x)
|
||||
or x.get("modified", 0)
|
||||
or 0,
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
# Optionally group by civitai modelId, showing only the latest version per model
|
||||
dedup_lost = 0
|
||||
if kwargs.get("group_by_model") and civitai_model_id is None:
|
||||
# Determine whether to further sub-group by base model
|
||||
# When version_grouping is "same_base", versions with different
|
||||
# base models are effectively different groups — the dedup key
|
||||
# needs to include base_model so the version count and VLM flow
|
||||
# stay consistent (card shows correct count for its base model).
|
||||
ufs = self.settings.get("version_grouping", "same_base")
|
||||
group_by_base = ufs == "same_base"
|
||||
|
||||
dedup_map = {} # (modelId [,base_model]) -> (item, version_or_modified)
|
||||
version_counter = {} # same-key -> count
|
||||
standalone = []
|
||||
for item in sorted_data:
|
||||
mid = self._extract_group_key(item)
|
||||
if mid is None:
|
||||
standalone.append(item)
|
||||
continue
|
||||
key = (mid, item.get("base_model") or "") if group_by_base else mid
|
||||
# Count all versions per key
|
||||
version_counter[key] = version_counter.get(key, 0) + 1
|
||||
# Prefer CivitAI version_id; fall back to modified timestamp
|
||||
vid = self._extract_version_id(item)
|
||||
if vid is None:
|
||||
vid = item.get("modified", 0) or 0
|
||||
if key not in dedup_map or vid > dedup_map[key][1]:
|
||||
dedup_map[key] = (item, vid)
|
||||
# Attach version_count to each surviving grouped item (shallow copy
|
||||
# to avoid mutating cached dicts — the cache is shared across requests)
|
||||
for key, (item, vid) in dedup_map.items():
|
||||
item = dict(item)
|
||||
item["version_count"] = version_counter[key]
|
||||
dedup_map[key] = (item, vid)
|
||||
dedup_lost = len(sorted_data) - (len(dedup_map) + len(standalone))
|
||||
sorted_data = [entry[0] for entry in dedup_map.values()] + standalone
|
||||
|
||||
# Re-sort by version_count (grouped: after dedup; non-grouped: group internally, sort, expand)
|
||||
if sort_params.key == "versions_count" and civitai_model_id is None:
|
||||
reverse = sort_params.order == "desc"
|
||||
if kwargs.get("group_by_model"):
|
||||
# Grouped mode: items are already dedup'd with version_count attached
|
||||
sorted_data.sort(
|
||||
key=lambda x: (
|
||||
x.get("version_count", 0),
|
||||
(x.get("model_name") or x.get("file_name") or "").lower(),
|
||||
x.get("file_path", "").lower(),
|
||||
),
|
||||
reverse=reverse,
|
||||
)
|
||||
else:
|
||||
# Non-grouped mode: group internally, sort groups by count, expand
|
||||
# Respect the version_grouping setting (same logic as grouped dedup)
|
||||
ufs = self.settings.get("version_grouping", "same_base")
|
||||
group_by_base = ufs == "same_base"
|
||||
|
||||
model_groups: Dict[Any, List[Dict[str, Any]]] = {}
|
||||
ungrouped_standalone: List[Dict[str, Any]] = []
|
||||
for item in sorted_data:
|
||||
mid = self._extract_group_key(item)
|
||||
if mid is None:
|
||||
ungrouped_standalone.append(item)
|
||||
continue
|
||||
key = (mid, item.get("base_model") or "") if group_by_base else mid
|
||||
model_groups.setdefault(key, []).append(item)
|
||||
# Sort versions within each group by version id (descending);
|
||||
# fall back to modified timestamp for non-CivitAI sources.
|
||||
for items in model_groups.values():
|
||||
items.sort(
|
||||
key=lambda x: self._extract_version_id(x)
|
||||
or x.get("modified", 0)
|
||||
or 0,
|
||||
reverse=True,
|
||||
)
|
||||
# Sort groups by version count
|
||||
sorted_groups = sorted(
|
||||
model_groups.values(),
|
||||
key=lambda items: len(items),
|
||||
reverse=reverse,
|
||||
)
|
||||
# Flatten: grouped items first, standalone items last
|
||||
sorted_data = []
|
||||
for items in sorted_groups:
|
||||
sorted_data.extend(items)
|
||||
sorted_data.extend(ungrouped_standalone)
|
||||
|
||||
t1 = time.perf_counter()
|
||||
if hash_filters:
|
||||
filtered_data = await self._apply_hash_filters(sorted_data, hash_filters)
|
||||
@@ -109,6 +220,7 @@ class BaseModelService(ABC):
|
||||
base_models=base_models,
|
||||
model_types=model_types,
|
||||
tags=tags,
|
||||
auto_tags=auto_tags,
|
||||
favorites_only=favorites_only,
|
||||
search_options=search_options,
|
||||
tag_logic=tag_logic,
|
||||
@@ -137,7 +249,7 @@ class BaseModelService(ABC):
|
||||
filter_duration = time.perf_counter() - t1
|
||||
post_filter_count = len(filtered_data)
|
||||
|
||||
annotated_for_filter: Optional[List[Dict]] = None
|
||||
annotated_for_filter: Optional[List[Dict[str, Any]]] = None
|
||||
t2 = time.perf_counter()
|
||||
if update_available_only:
|
||||
annotated_for_filter = await self._annotate_update_flags(filtered_data)
|
||||
@@ -164,7 +276,7 @@ class BaseModelService(ABC):
|
||||
overall_duration = time.perf_counter() - overall_start
|
||||
logger.debug(
|
||||
"%s.get_paginated_data took %.3fs (fetch: %.3fs, filter: %.3fs, update_filter: %.3fs, pagination: %.3fs, annotate: %.3fs). "
|
||||
"Counts: initial=%d, post_filter=%d, final=%d",
|
||||
"Counts: initial=%d, dedup=%d, post_filter=%d, final=%d",
|
||||
self.__class__.__name__,
|
||||
overall_duration,
|
||||
fetch_duration,
|
||||
@@ -173,11 +285,63 @@ class BaseModelService(ABC):
|
||||
pagination_duration,
|
||||
annotate_duration,
|
||||
initial_count,
|
||||
dedup_lost,
|
||||
post_filter_count,
|
||||
final_count,
|
||||
)
|
||||
return paginated
|
||||
|
||||
async def get_excluded_paginated_data(
|
||||
self,
|
||||
page: int,
|
||||
page_size: int,
|
||||
sort_by: str = "name",
|
||||
search: str | None = None,
|
||||
fuzzy_search: bool = False,
|
||||
search_options: dict[str, Any] | None = None,
|
||||
**kwargs,
|
||||
) -> Dict[str, Any]:
|
||||
"""Get paginated excluded model data."""
|
||||
excluded_paths = list(self.scanner.get_excluded_models())
|
||||
excluded_entries: List[Dict[str, Any]] = []
|
||||
stale_paths: List[str] = []
|
||||
|
||||
for file_path in excluded_paths:
|
||||
if not file_path or not os.path.exists(file_path):
|
||||
stale_paths.append(file_path)
|
||||
continue
|
||||
|
||||
entry = await self._build_excluded_entry(file_path)
|
||||
if entry:
|
||||
excluded_entries.append(entry)
|
||||
else:
|
||||
stale_paths.append(file_path)
|
||||
|
||||
if stale_paths:
|
||||
current_excluded = getattr(self.scanner, "_excluded_models", None)
|
||||
if isinstance(current_excluded, list):
|
||||
stale_set = set(stale_paths)
|
||||
self.scanner._excluded_models = [
|
||||
path for path in current_excluded if path not in stale_set
|
||||
]
|
||||
persist_current_cache = getattr(self.scanner, "_persist_current_cache", None)
|
||||
if callable(persist_current_cache):
|
||||
await cast(Awaitable[Any], persist_current_cache())
|
||||
|
||||
excluded_entries = self._sort_entries(excluded_entries, sort_by)
|
||||
|
||||
if search:
|
||||
excluded_entries = await self._apply_search_filters(
|
||||
excluded_entries,
|
||||
search,
|
||||
fuzzy_search,
|
||||
search_options,
|
||||
)
|
||||
|
||||
paginated = self._paginate(excluded_entries, page, page_size)
|
||||
paginated["items"] = await self._annotate_update_flags(paginated["items"])
|
||||
return paginated
|
||||
|
||||
async def _fetch_with_usage_sort(self, sort_params):
|
||||
"""Fetch data sorted by usage count (desc/asc)."""
|
||||
cache = await self.cache_repository.get_cache()
|
||||
@@ -217,39 +381,113 @@ class BaseModelService(ABC):
|
||||
)
|
||||
return annotated
|
||||
|
||||
def _sort_entries(self, data: List[Dict[str, Any]], sort_by: str) -> List[Dict[str, Any]]:
|
||||
sort_params = self.cache_repository.parse_sort(sort_by)
|
||||
key_name = sort_params.key
|
||||
|
||||
if key_name == "date":
|
||||
key_fn = lambda item: (
|
||||
float(item.get("modified", 0.0) or 0.0),
|
||||
(item.get("model_name") or item.get("file_name") or "").lower(),
|
||||
item.get("file_path", "").lower(),
|
||||
)
|
||||
elif key_name == "random":
|
||||
# Seeded random shuffle: same seed -> same order (stable pagination)
|
||||
rng = random.Random(sort_params.seed or "random")
|
||||
result = list(data)
|
||||
rng.shuffle(result)
|
||||
return result
|
||||
elif key_name == "size":
|
||||
key_fn = lambda item: (
|
||||
int(item.get("size", 0) or 0),
|
||||
(item.get("model_name") or item.get("file_name") or "").lower(),
|
||||
item.get("file_path", "").lower(),
|
||||
)
|
||||
elif key_name == "usage":
|
||||
key_fn = lambda item: (
|
||||
int(item.get("usage_count", 0) or 0),
|
||||
(item.get("model_name") or item.get("file_name") or "").lower(),
|
||||
item.get("file_path", "").lower(),
|
||||
)
|
||||
else:
|
||||
key_fn = lambda item: (
|
||||
(item.get("model_name") or item.get("file_name") or "").lower(),
|
||||
item.get("file_path", "").lower(),
|
||||
)
|
||||
|
||||
return sorted(data, key=key_fn, reverse=sort_params.order == "desc")
|
||||
|
||||
async def _build_excluded_entry(self, file_path: str) -> Optional[Dict[str, Any]]:
|
||||
root_path = self.scanner._find_root_for_file(file_path)
|
||||
if not root_path:
|
||||
return None
|
||||
|
||||
metadata, should_skip = await MetadataManager.load_metadata(
|
||||
file_path,
|
||||
self.metadata_class,
|
||||
)
|
||||
if should_skip:
|
||||
return None
|
||||
|
||||
if metadata is None:
|
||||
metadata = await self.scanner._create_default_metadata(file_path)
|
||||
if metadata is None:
|
||||
return None
|
||||
|
||||
metadata = self.scanner.adjust_metadata(metadata, file_path, root_path)
|
||||
folder = os.path.dirname(os.path.relpath(file_path, root_path)).replace(
|
||||
os.path.sep, "/"
|
||||
)
|
||||
entry = self.scanner._build_cache_entry(metadata, folder=folder)
|
||||
entry = self.scanner.adjust_cached_entry(entry)
|
||||
entry["exclude"] = True
|
||||
return entry
|
||||
|
||||
async def _apply_hash_filters(
|
||||
self, data: List[Dict], hash_filters: Dict
|
||||
) -> List[Dict]:
|
||||
"""Apply hash-based filtering"""
|
||||
self, data: List[Dict[str, Any]], hash_filters: Dict[str, Any]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Apply hash-based filtering (SHA256 and AutoV3)."""
|
||||
|
||||
def matches_hash_set(item: Dict[str, Any], hash_set: set[str]) -> bool:
|
||||
"""Check whether an item matches any hash in the set.
|
||||
|
||||
Compares the item's ``sha256`` field and its non-empty ``autov3``
|
||||
field, both case-insensitively.
|
||||
"""
|
||||
if item.get("sha256", "").lower() in hash_set:
|
||||
return True
|
||||
autov3 = item.get("autov3", "")
|
||||
return bool(autov3) and autov3.lower() in hash_set
|
||||
|
||||
single_hash = hash_filters.get("single_hash")
|
||||
multiple_hashes = hash_filters.get("multiple_hashes")
|
||||
|
||||
if single_hash:
|
||||
# Filter by single hash
|
||||
single_hash = single_hash.lower()
|
||||
# Filter by single hash (SHA256 or AutoV3)
|
||||
return [
|
||||
item for item in data if item.get("sha256", "").lower() == single_hash
|
||||
item for item in data if matches_hash_set(item, {single_hash.lower()})
|
||||
]
|
||||
elif multiple_hashes:
|
||||
# Filter by multiple hashes
|
||||
hash_set = set(hash.lower() for hash in multiple_hashes)
|
||||
return [item for item in data if item.get("sha256", "").lower() in hash_set]
|
||||
# Filter by multiple hashes (SHA256 or AutoV3)
|
||||
hash_set = {hash.lower() for hash in multiple_hashes}
|
||||
return [item for item in data if matches_hash_set(item, hash_set)]
|
||||
|
||||
return data
|
||||
|
||||
async def _apply_common_filters(
|
||||
self,
|
||||
data: List[Dict],
|
||||
folder: str = None,
|
||||
folder_include: list = None,
|
||||
folder_exclude: list = None,
|
||||
base_models: list = None,
|
||||
model_types: list = None,
|
||||
data: List[Dict[str, Any]],
|
||||
folder: str | None = None,
|
||||
folder_include: list[str] | None = None,
|
||||
folder_exclude: list[str] | None = None,
|
||||
base_models: list[str] | None = None,
|
||||
model_types: list[str] | None = None,
|
||||
tags: Optional[Dict[str, str]] = None,
|
||||
auto_tags: Optional[Dict[str, str]] = None,
|
||||
favorites_only: bool = False,
|
||||
search_options: dict = None,
|
||||
search_options: dict[str, Any] | None = None,
|
||||
tag_logic: str = "any",
|
||||
) -> List[Dict]:
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Apply common filters that work across all model types"""
|
||||
normalized_options = self.search_strategy.normalize_options(search_options)
|
||||
criteria = FilterCriteria(
|
||||
@@ -259,6 +497,7 @@ class BaseModelService(ABC):
|
||||
base_models=base_models,
|
||||
model_types=model_types,
|
||||
tags=tags,
|
||||
auto_tags=auto_tags,
|
||||
favorites_only=favorites_only,
|
||||
search_options=normalized_options,
|
||||
tag_logic=tag_logic,
|
||||
@@ -267,24 +506,24 @@ class BaseModelService(ABC):
|
||||
|
||||
async def _apply_search_filters(
|
||||
self,
|
||||
data: List[Dict],
|
||||
data: List[Dict[str, Any]],
|
||||
search: str,
|
||||
fuzzy_search: bool,
|
||||
search_options: dict,
|
||||
) -> List[Dict]:
|
||||
search_options: dict[str, Any] | None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Apply search filtering"""
|
||||
normalized_options = self.search_strategy.normalize_options(search_options)
|
||||
return self.search_strategy.apply(
|
||||
data, search, normalized_options, fuzzy_search
|
||||
)
|
||||
|
||||
async def _apply_specific_filters(self, data: List[Dict], **kwargs) -> List[Dict]:
|
||||
async def _apply_specific_filters(self, data: List[Dict[str, Any]], **kwargs) -> List[Dict[str, Any]]:
|
||||
"""Apply model-specific filters - to be overridden by subclasses if needed"""
|
||||
return data
|
||||
|
||||
async def _apply_credit_required_filter(
|
||||
self, data: List[Dict], credit_required: bool
|
||||
) -> List[Dict]:
|
||||
self, data: List[Dict[str, Any]], credit_required: bool
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Apply credit required filtering based on license_flags.
|
||||
|
||||
Args:
|
||||
@@ -314,8 +553,8 @@ class BaseModelService(ABC):
|
||||
return filtered_data
|
||||
|
||||
async def _apply_allow_selling_filter(
|
||||
self, data: List[Dict], allow_selling: bool
|
||||
) -> List[Dict]:
|
||||
self, data: List[Dict[str, Any]], allow_selling: bool
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Apply allow selling generated content filtering based on license_flags.
|
||||
|
||||
Args:
|
||||
@@ -347,8 +586,8 @@ class BaseModelService(ABC):
|
||||
|
||||
async def _annotate_update_flags(
|
||||
self,
|
||||
items: List[Dict],
|
||||
) -> List[Dict]:
|
||||
items: List[Dict[str, Any]],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Attach an update_available flag to each response item.
|
||||
|
||||
Items without a civitai model id default to False.
|
||||
@@ -363,7 +602,7 @@ class BaseModelService(ABC):
|
||||
item["update_available"] = False
|
||||
return annotated
|
||||
|
||||
id_to_items: Dict[int, List[Dict]] = {}
|
||||
id_to_items: Dict[int, List[Dict[str, Any]]] = {}
|
||||
ordered_ids: List[int] = []
|
||||
for item in annotated:
|
||||
model_id = self._extract_model_id(item)
|
||||
@@ -378,7 +617,7 @@ class BaseModelService(ABC):
|
||||
if not ordered_ids:
|
||||
return annotated
|
||||
|
||||
strategy_value = self.settings.get("update_flag_strategy")
|
||||
strategy_value = self.settings.get("version_grouping")
|
||||
if isinstance(strategy_value, str) and strategy_value.strip():
|
||||
strategy = strategy_value.strip().lower()
|
||||
else:
|
||||
@@ -400,7 +639,7 @@ class BaseModelService(ABC):
|
||||
record_method = getattr(self.update_service, "get_records_bulk", None)
|
||||
if callable(record_method):
|
||||
try:
|
||||
records = await record_method(self.model_type, ordered_ids)
|
||||
records = await cast(Awaitable[Any], record_method(self.model_type, ordered_ids))
|
||||
resolved = {
|
||||
model_id: record.has_update(hide_early_access=hide_early_access)
|
||||
for model_id, record in records.items()
|
||||
@@ -420,11 +659,11 @@ class BaseModelService(ABC):
|
||||
bulk_method = getattr(self.update_service, "has_updates_bulk", None)
|
||||
if callable(bulk_method):
|
||||
try:
|
||||
resolved = await bulk_method(
|
||||
resolved = await cast(Awaitable[Any], bulk_method(
|
||||
self.model_type,
|
||||
ordered_ids,
|
||||
hide_early_access=hide_early_access,
|
||||
)
|
||||
))
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"Failed to resolve update status in bulk for %s models (%s): %s",
|
||||
@@ -486,7 +725,34 @@ class BaseModelService(ABC):
|
||||
return annotated
|
||||
|
||||
@staticmethod
|
||||
def _extract_model_id(item: Dict) -> Optional[int]:
|
||||
def _extract_hf_group_key(item: Dict[str, Any]) -> Optional[str]:
|
||||
"""Extract `hf:{owner}/{repo}` from item's ``hf_url``, or None."""
|
||||
hf_url = item.get("hf_url") if isinstance(item, dict) else None
|
||||
if not hf_url or not isinstance(hf_url, str):
|
||||
return None
|
||||
m = re.match(
|
||||
r"https?://huggingface\.co/([^/]+/[^/]+)", hf_url.strip()
|
||||
)
|
||||
if not m:
|
||||
return None
|
||||
return f"hf:{m.group(1)}"
|
||||
|
||||
@staticmethod
|
||||
def _extract_group_key(item: Dict[str, Any]) -> Union[int, str, None]:
|
||||
"""Return the group identity key: CivitAI modelId (int) or HF repo (str).
|
||||
|
||||
Preference order:
|
||||
1. CivitAI ``modelId`` (int)
|
||||
2. HF repo identity ``hf:{owner}/{repo}`` (str)
|
||||
3. ``None`` (no known grouping source)
|
||||
"""
|
||||
mid = BaseModelService._extract_model_id(item)
|
||||
if mid is not None:
|
||||
return mid
|
||||
return BaseModelService._extract_hf_group_key(item)
|
||||
|
||||
@staticmethod
|
||||
def _extract_model_id(item: Dict[str, Any]) -> Optional[int]:
|
||||
civitai = item.get("civitai") if isinstance(item, dict) else None
|
||||
if not isinstance(civitai, dict):
|
||||
return None
|
||||
@@ -499,7 +765,7 @@ class BaseModelService(ABC):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_version_id(item: Dict) -> Optional[int]:
|
||||
def _extract_version_id(item: Dict[str, Any]) -> Optional[int]:
|
||||
civitai = item.get("civitai") if isinstance(item, dict) else None
|
||||
if not isinstance(civitai, dict):
|
||||
return None
|
||||
@@ -512,7 +778,7 @@ class BaseModelService(ABC):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_base_model(item: Dict) -> Optional[str]:
|
||||
def _extract_base_model(item: Dict[str, Any]) -> Optional[str]:
|
||||
value = item.get("base_model")
|
||||
if value is None:
|
||||
return None
|
||||
@@ -564,7 +830,7 @@ class BaseModelService(ABC):
|
||||
|
||||
return highest_by_base
|
||||
|
||||
def _paginate(self, data: List[Dict], page: int, page_size: int) -> Dict:
|
||||
def _paginate(self, data: List[Dict[str, Any]], page: int, page_size: int) -> Dict[str, Any]:
|
||||
"""Apply pagination to filtered data"""
|
||||
total_items = len(data)
|
||||
start_idx = (page - 1) * page_size
|
||||
@@ -579,16 +845,26 @@ class BaseModelService(ABC):
|
||||
}
|
||||
|
||||
@abstractmethod
|
||||
async def format_response(self, model_data: Dict) -> Dict:
|
||||
"""Format model data for API response - must be implemented by subclasses"""
|
||||
async def format_response(self, model_data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""Format model data for API response - must be implemented by subclasses.
|
||||
|
||||
Subclasses should return None for corrupted entries so the handler
|
||||
layer can filter them out. See issue #730.
|
||||
"""
|
||||
pass
|
||||
|
||||
# Common service methods that delegate to scanner
|
||||
async def get_top_tags(self, limit: int = 20) -> List[Dict]:
|
||||
async def get_top_tags(self, limit: int = 20) -> List[Dict[str, Any]]:
|
||||
"""Get top tags sorted by frequency"""
|
||||
return await self.scanner.get_top_tags(limit)
|
||||
|
||||
async def get_base_models(self, limit: int = 20) -> List[Dict]:
|
||||
async def search_tags(
|
||||
self, query: str, limit: int = 50
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search tags by substring, sorted by frequency"""
|
||||
return await self.scanner.search_tags(query, limit)
|
||||
|
||||
async def get_base_models(self, limit: int = 20) -> List[Dict[str, Any]]:
|
||||
"""Get base models sorted by frequency"""
|
||||
return await self.scanner.get_base_models(limit)
|
||||
|
||||
@@ -655,7 +931,7 @@ class BaseModelService(ABC):
|
||||
"""Get model root directories"""
|
||||
return self.scanner.get_model_roots()
|
||||
|
||||
def filter_civitai_data(self, data: Dict, minimal: bool = False) -> Dict:
|
||||
def filter_civitai_data(self, data: Dict[str, Any], minimal: bool = False) -> Dict[str, Any]:
|
||||
"""Filter relevant fields from CivitAI data"""
|
||||
if not data:
|
||||
return {}
|
||||
@@ -681,7 +957,7 @@ class BaseModelService(ABC):
|
||||
)
|
||||
return {k: data[k] for k in fields if k in data}
|
||||
|
||||
async def get_folder_tree(self, model_root: str) -> Dict:
|
||||
async def get_folder_tree(self, model_root: str) -> Dict[str, Any]:
|
||||
"""Get hierarchical folder tree for a specific model root"""
|
||||
cache = await self.scanner.get_cached_data()
|
||||
|
||||
@@ -710,7 +986,7 @@ class BaseModelService(ABC):
|
||||
|
||||
return tree
|
||||
|
||||
async def get_unified_folder_tree(self) -> Dict:
|
||||
async def get_unified_folder_tree(self) -> Dict[str, Any]:
|
||||
"""Get unified folder tree across all model roots"""
|
||||
cache = await self.scanner.get_cached_data()
|
||||
|
||||
@@ -739,13 +1015,21 @@ class BaseModelService(ABC):
|
||||
|
||||
return unified_tree
|
||||
|
||||
async def get_model_notes(self, model_name: str) -> Optional[str]:
|
||||
"""Get notes for a specific model file"""
|
||||
async def get_model_notes(self, model_name: str) -> Optional[dict[str, Any]]:
|
||||
"""Get notes and file_path for a specific model file.
|
||||
|
||||
Supports both simple names (``OWSMianne_ANIMA_V1``) and full-path
|
||||
syntax (``Anima/character/OWSMianne_ANIMA_V1``).
|
||||
"""
|
||||
cache = await self.scanner.get_cached_data()
|
||||
|
||||
for model in cache.raw_data:
|
||||
if model["file_name"] == model_name:
|
||||
return model.get("notes", "")
|
||||
file_name = model.get("file_name", "")
|
||||
if file_name == model_name or model_name.endswith("/" + file_name) or model_name.endswith("\\" + file_name):
|
||||
return {
|
||||
"notes": model.get("notes", ""),
|
||||
"file_path": model.get("file_path", ""),
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
@@ -753,30 +1037,86 @@ class BaseModelService(ABC):
|
||||
"""Get the static preview URL for a model file"""
|
||||
cache = await self.scanner.get_cached_data()
|
||||
|
||||
name_normalized = model_name.replace("\\", "/")
|
||||
name_no_ext = name_normalized
|
||||
for ext in (".safetensors", ".ckpt", ".pt", ".bin"):
|
||||
if name_no_ext.lower().endswith(ext):
|
||||
name_no_ext = name_no_ext[: -len(ext)]
|
||||
break
|
||||
|
||||
has_path = "/" in name_no_ext
|
||||
basename = os.path.basename(name_no_ext) if has_path else name_no_ext
|
||||
best_fallback = None
|
||||
|
||||
for model in cache.raw_data:
|
||||
if model["file_name"] == model_name:
|
||||
file_name = model.get("file_name", "")
|
||||
folder = model.get("folder", "")
|
||||
file_name_no_ext = file_name
|
||||
for ext in (".safetensors", ".ckpt", ".pt", ".bin"):
|
||||
if file_name_no_ext.lower().endswith(ext):
|
||||
file_name_no_ext = file_name_no_ext[: -len(ext)]
|
||||
break
|
||||
path_name = f"{folder}/{file_name_no_ext}".replace("\\", "/") if folder else file_name_no_ext
|
||||
|
||||
if name_no_ext == file_name_no_ext or name_no_ext == path_name:
|
||||
preview_url = model.get("preview_url")
|
||||
if preview_url:
|
||||
from ..config import config
|
||||
|
||||
return config.get_preview_static_url(preview_url)
|
||||
|
||||
if has_path and file_name_no_ext == basename:
|
||||
if folder and name_no_ext.startswith(folder.replace("\\", "/") + "/"):
|
||||
best_fallback = model
|
||||
elif best_fallback is None:
|
||||
best_fallback = model
|
||||
|
||||
if best_fallback:
|
||||
preview_url = best_fallback.get("preview_url")
|
||||
if preview_url:
|
||||
from ..config import config
|
||||
|
||||
return config.get_preview_static_url(preview_url)
|
||||
|
||||
return "/loras_static/images/no-preview.png"
|
||||
|
||||
async def get_model_civitai_url(self, model_name: str) -> Dict[str, Optional[str]]:
|
||||
"""Get the Civitai URL for a model file"""
|
||||
cache = await self.scanner.get_cached_data()
|
||||
|
||||
name_normalized = model_name.replace("\\", "/")
|
||||
name_no_ext = name_normalized
|
||||
for ext in (".safetensors", ".ckpt", ".pt", ".bin"):
|
||||
if name_no_ext.lower().endswith(ext):
|
||||
name_no_ext = name_no_ext[: -len(ext)]
|
||||
break
|
||||
|
||||
has_path = "/" in name_no_ext
|
||||
basename = os.path.basename(name_no_ext) if has_path else name_no_ext
|
||||
best_fallback = None
|
||||
|
||||
for model in cache.raw_data:
|
||||
if model["file_name"] == model_name:
|
||||
file_name = model.get("file_name", "")
|
||||
folder = model.get("folder", "")
|
||||
file_name_no_ext = file_name
|
||||
for ext in (".safetensors", ".ckpt", ".pt", ".bin"):
|
||||
if file_name_no_ext.lower().endswith(ext):
|
||||
file_name_no_ext = file_name_no_ext[: -len(ext)]
|
||||
break
|
||||
path_name = f"{folder}/{file_name_no_ext}".replace("\\", "/") if folder else file_name_no_ext
|
||||
|
||||
if name_no_ext == file_name_no_ext or name_no_ext == path_name:
|
||||
civitai_data = model.get("civitai", {})
|
||||
model_id = civitai_data.get("modelId")
|
||||
version_id = civitai_data.get("id")
|
||||
|
||||
if model_id:
|
||||
civitai_url = f"https://civitai.com/models/{model_id}"
|
||||
if version_id:
|
||||
civitai_url += f"?modelVersionId={version_id}"
|
||||
civitai_host = self.settings.get("civitai_host", "civitai.com")
|
||||
civitai_url = build_civitai_model_page_url(
|
||||
model_id,
|
||||
version_id,
|
||||
host=civitai_host,
|
||||
)
|
||||
|
||||
return {
|
||||
"civitai_url": civitai_url,
|
||||
@@ -784,19 +1124,69 @@ class BaseModelService(ABC):
|
||||
"version_id": str(version_id) if version_id else None,
|
||||
}
|
||||
|
||||
if has_path and file_name_no_ext == basename:
|
||||
if folder and name_no_ext.startswith(folder.replace("\\", "/") + "/"):
|
||||
best_fallback = model
|
||||
elif best_fallback is None:
|
||||
best_fallback = model
|
||||
|
||||
if best_fallback:
|
||||
civitai_data = best_fallback.get("civitai", {})
|
||||
model_id = civitai_data.get("modelId")
|
||||
if model_id:
|
||||
version_id = civitai_data.get("id")
|
||||
civitai_host = self.settings.get("civitai_host", "civitai.com")
|
||||
civitai_url = build_civitai_model_page_url(
|
||||
model_id, version_id, host=civitai_host
|
||||
)
|
||||
return {
|
||||
"civitai_url": civitai_url,
|
||||
"model_id": str(model_id),
|
||||
"version_id": str(version_id) if version_id else None,
|
||||
}
|
||||
|
||||
return {"civitai_url": None, "model_id": None, "version_id": None}
|
||||
|
||||
async def get_model_metadata(self, file_path: str) -> Optional[Dict]:
|
||||
async def get_model_metadata(self, file_path: str) -> Optional[Dict[str, Any]]:
|
||||
"""Load full metadata for a single model.
|
||||
|
||||
Listing/search endpoints return lightweight cache entries; this method performs
|
||||
a lazy read of the on-disk metadata snapshot when callers need full detail.
|
||||
|
||||
As a beneficial side effect, the in-memory and persistent caches are
|
||||
opportunistically synchronised with the on-disk metadata — this keeps the
|
||||
caches fresh even when a ``.metadata.json`` file was edited outside of the
|
||||
normal save path (e.g. manually or by an external script).
|
||||
"""
|
||||
metadata, should_skip = await MetadataManager.load_metadata(
|
||||
file_path, self.metadata_class
|
||||
)
|
||||
if should_skip or metadata is None:
|
||||
return None
|
||||
|
||||
# Prune stale example-image metadata entries whose files no longer
|
||||
# exist on disk (e.g. a user deleted the files manually).
|
||||
from ..utils.example_images_metadata import MetadataUpdater
|
||||
|
||||
was_modified = await MetadataUpdater.prune_stale_example_images(metadata)
|
||||
if was_modified:
|
||||
asyncio.create_task(
|
||||
MetadataManager.save_metadata(file_path, metadata)
|
||||
)
|
||||
|
||||
# Opportunistically sync the in-memory + persistent caches.
|
||||
# The .metadata.json disk read is already paid for; the sync only
|
||||
# performs work when the cache is actually stale, and uses targeted,
|
||||
# in-place operations to minimise overhead even with large model sets.
|
||||
#
|
||||
# Fire-and-forget by design: the task is intentionally untracked.
|
||||
# sync_cache_from_metadata handles its own errors internally.
|
||||
asyncio.create_task(
|
||||
self.scanner.sync_cache_from_metadata(
|
||||
file_path, metadata.to_dict()
|
||||
)
|
||||
)
|
||||
|
||||
return self.filter_civitai_data(metadata.to_dict().get("civitai", {}))
|
||||
|
||||
async def get_model_description(self, file_path: str) -> Optional[str]:
|
||||
@@ -853,7 +1243,7 @@ class BaseModelService(ABC):
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _relative_path_sort_key(relative_path: str, include_terms: List[str]) -> tuple:
|
||||
def _relative_path_sort_key(relative_path: str, include_terms: List[str]) -> tuple[int, int, int, str]:
|
||||
"""Sort paths by how well they satisfy the include tokens.
|
||||
|
||||
Sorts based on path without extension for consistent ordering.
|
||||
@@ -880,19 +1270,87 @@ class BaseModelService(ABC):
|
||||
)
|
||||
|
||||
async def search_relative_paths(
|
||||
self, search_term: str, limit: int = 15, offset: int = 0
|
||||
self,
|
||||
search_term: str,
|
||||
limit: int = 15,
|
||||
offset: int = 0,
|
||||
*,
|
||||
folder: Optional[str] = None,
|
||||
folder_include: Optional[list[str]] = None,
|
||||
folder_exclude: Optional[list[str]] = None,
|
||||
base_models: Optional[list[str]] = None,
|
||||
model_types: Optional[list[str]] = None,
|
||||
tags: Optional[dict[str, str]] = None,
|
||||
auto_tags: Optional[dict[str, str]] = None,
|
||||
tag_logic: str = "any",
|
||||
credit_required: Optional[bool] = None,
|
||||
allow_selling_generated_content: Optional[bool] = None,
|
||||
recursive: bool = True,
|
||||
apply_filters: bool = False,
|
||||
) -> List[str]:
|
||||
"""Search model relative file paths for autocomplete functionality"""
|
||||
"""Search model relative file paths for autocomplete functionality.
|
||||
|
||||
Optional filter kwargs mirror the filters used by the list endpoint
|
||||
(/api/lm/{prefix}/list). When no filter kwargs are provided the
|
||||
behavior is identical to plain token-based path matching.
|
||||
"""
|
||||
cache = await self.scanner.get_cached_data()
|
||||
include_terms, exclude_terms = self._parse_search_tokens(search_term)
|
||||
|
||||
data = cache.raw_data
|
||||
has_filters = any(
|
||||
[
|
||||
apply_filters,
|
||||
folder is not None,
|
||||
folder_include,
|
||||
folder_exclude,
|
||||
base_models,
|
||||
model_types,
|
||||
tags,
|
||||
auto_tags,
|
||||
credit_required is not None,
|
||||
allow_selling_generated_content is not None,
|
||||
]
|
||||
)
|
||||
if has_filters:
|
||||
# Auto-tags are not stored in the scanner cache — they are computed
|
||||
# on the fly. Pre-compute them only when an auto-tag filter is
|
||||
# active to avoid mutating cache entries unnecessarily.
|
||||
if auto_tags:
|
||||
from .auto_tag_service import extract_auto_tags
|
||||
|
||||
for item in data:
|
||||
if not item.get("auto_tags"):
|
||||
item["auto_tags"] = extract_auto_tags(item)
|
||||
|
||||
criteria = FilterCriteria(
|
||||
folder=folder,
|
||||
folder_include=folder_include,
|
||||
folder_exclude=folder_exclude,
|
||||
base_models=base_models,
|
||||
model_types=model_types,
|
||||
tags=tags,
|
||||
auto_tags=auto_tags,
|
||||
search_options={"recursive": recursive},
|
||||
tag_logic=tag_logic,
|
||||
)
|
||||
data = self.filter_set.apply(data, criteria)
|
||||
if credit_required is not None:
|
||||
data = await self._apply_credit_required_filter(
|
||||
data, credit_required
|
||||
)
|
||||
if allow_selling_generated_content is not None:
|
||||
data = await self._apply_allow_selling_filter(
|
||||
data, allow_selling_generated_content
|
||||
)
|
||||
|
||||
matching_paths = []
|
||||
|
||||
# Get model roots for path calculation
|
||||
model_roots = self.scanner.get_model_roots()
|
||||
|
||||
# Collect all matching paths first (needed for proper sorting and offset)
|
||||
for model in cache.raw_data:
|
||||
for model in data:
|
||||
file_path = model.get("file_path", "")
|
||||
if not file_path:
|
||||
continue
|
||||
|
||||
@@ -224,7 +224,7 @@ class BatchImportService:
|
||||
return False
|
||||
|
||||
for recipe in getattr(cache, "raw_data", []):
|
||||
source_path = recipe.get("source_path") or recipe.get("source_url")
|
||||
source_path = recipe.get("source_path")
|
||||
if source_path and source_path == source:
|
||||
return True
|
||||
return False
|
||||
@@ -523,6 +523,10 @@ class BatchImportService:
|
||||
if payload.get("checkpoint"):
|
||||
metadata["checkpoint"] = payload["checkpoint"]
|
||||
|
||||
nsfw = payload.get("preview_nsfw_level")
|
||||
if isinstance(nsfw, int) and nsfw > 0:
|
||||
metadata["preview_nsfw_level"] = nsfw
|
||||
|
||||
image_bytes = None
|
||||
image_base64 = payload.get("image_base64")
|
||||
|
||||
|
||||
@@ -59,6 +59,7 @@ class CacheEntryValidator:
|
||||
'notes': ('', False),
|
||||
'usage_tips': ('', False),
|
||||
'hash_status': ('completed', False),
|
||||
'autov3': (None, False),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
@@ -119,8 +120,13 @@ class CacheEntryValidator:
|
||||
if is_required:
|
||||
errors.append(f"Required field '{field_name}' is missing or None")
|
||||
if auto_repair:
|
||||
working_entry[field_name] = cls._get_default_copy(default_value)
|
||||
repaired = True
|
||||
# A missing optional field whose default is None is already
|
||||
# semantically equal to its default (e.g. autov3: absent
|
||||
# means "not checked") — writing None back is a no-op, not
|
||||
# a repair.
|
||||
if default_value is not None:
|
||||
working_entry[field_name] = cls._get_default_copy(default_value)
|
||||
repaired = True
|
||||
continue
|
||||
|
||||
# Validate field type and value
|
||||
@@ -175,6 +181,15 @@ class CacheEntryValidator:
|
||||
# that invalidates the entry, but we also don't mark it repaired.
|
||||
pass
|
||||
|
||||
# Normalize autov3 to lowercase if needed (optional field, never stripped).
|
||||
autov3 = working_entry.get('autov3')
|
||||
if isinstance(autov3, str) and autov3:
|
||||
normalized_autov3 = autov3.lower()
|
||||
if normalized_autov3 != autov3:
|
||||
if auto_repair:
|
||||
working_entry['autov3'] = normalized_autov3
|
||||
repaired = True
|
||||
|
||||
# Determine if entry is valid
|
||||
# Entry is valid if no critical required field errors remain after repair
|
||||
# Critical fields are file_path and sha256
|
||||
@@ -242,6 +257,19 @@ class CacheEntryValidator:
|
||||
"""
|
||||
expected_type = type(default_value)
|
||||
|
||||
# Special case: autov3 is optional with a three-state contract.
|
||||
# None = not checked, "" = checked but unavailable, otherwise a
|
||||
# 12-character hex string (case-insensitive here; normalized to
|
||||
# lowercase separately).
|
||||
if field_name == 'autov3':
|
||||
if value is None or value == "":
|
||||
return None
|
||||
if not isinstance(value, str):
|
||||
return f"Field 'autov3' should be string or None, got {type(value).__name__}"
|
||||
if len(value) != 12 or any(c not in '0123456789abcdefABCDEF' for c in value):
|
||||
return "Field 'autov3' should be a 12-character hex string"
|
||||
return None
|
||||
|
||||
# Special handling for numeric types
|
||||
if expected_type == int:
|
||||
if not isinstance(value, (int, float)):
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user