mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-09-21 11:11:26 -03:00
Compare commits
568 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 16b0bdf70a | |||
| e09fe5888b | |||
| f67689b0f9 | |||
| 1d6da1787a | |||
| d572292142 | |||
| 1b1a8d63db | |||
| a0a5b13ab0 | |||
| 779bd18e75 | |||
| 01137eed88 | |||
| c6c44b741a | |||
| 5095b23eb2 | |||
| cc25bb3dc2 | |||
| 3b54a13cae | |||
| 9bbe57ee85 | |||
| 4938faa049 | |||
| cc8eedcff7 | |||
| 9734df15b4 | |||
| 2ceb1e2850 | |||
| 942717f0b6 | |||
| 0f160e157f | |||
| e9e9ee20c6 | |||
| f0ee30fc68 | |||
| 51de85a6ca | |||
| 4064ea7d3a | |||
| 35b291ab19 | |||
| e711e643f1 | |||
| db38ad80e6 | |||
| 326df32933 | |||
| 31ef9ffa06 | |||
| 38d4c59b4c | |||
| b9bf006998 | |||
| 5ab0e88abc | |||
| 84146b62fd | |||
| adeb40bfff | |||
| 8a21837ca2 | |||
| 3302147a43 | |||
| 4d87ae7637 | |||
| b1a653f18f | |||
| 6fe0543d2e | |||
| 931dfbe1d3 | |||
| b5c1331911 | |||
| 37f2cba72d | |||
| 0e789cb38c | |||
| f3b3393a16 | |||
| 480a3f4ea5 | |||
| 69a62d739c | |||
| 28fbb86dce | |||
| f88fe2665c | |||
| 3592eab48c | |||
| 1dbdf5b00c | |||
| fc3b2d7c13 | |||
| f2a7297cb9 | |||
| 57729375b6 | |||
| fa7ce725c1 | |||
| 27da7b3ca3 | |||
| 3070838a42 | |||
| fe160134d0 | |||
| 6d3f82976f | |||
| 91b2735dad | |||
| 3112869a21 | |||
| aa630bf85b | |||
| e0052cd237 | |||
| 3cdc5ba7a2 | |||
| 04485e384f | |||
| a03dc4002f | |||
| cc9d3bff42 | |||
| 2672b3331b | |||
| 4963bf2b2e | |||
| 51cad6f852 | |||
| 1b5cbbbaa0 | |||
| e747946f7a | |||
| 53fa22f39c | |||
| 82b34097fb | |||
| a7995db009 | |||
| 5ae4aef30e | |||
| 08023f0cd9 | |||
| 6e2185c182 | |||
| 41302e75ba | |||
| a17399d667 | |||
| e2d85a0a21 | |||
| 303833bbae | |||
| f86b7b55d6 | |||
| 782bb53784 | |||
| 139231e225 | |||
| 121d8d5cea | |||
| ec147bd677 | |||
| 93fc28b499 | |||
| 7afed1a14b | |||
| e6f5142e48 | |||
| 87f05fb66c | |||
| cf64e5baa8 | |||
| 634ea7f299 | |||
| 6ba64ebb3c | |||
| 03569c62df | |||
| a61840b366 | |||
| 726fc178f1 | |||
| 8260bd022d | |||
| b309becdf9 | |||
| 1e375bb8d9 | |||
| 14da8a6f17 | |||
| da71985c3e | |||
| 7c4c8b8f30 | |||
| 77109b3cf8 | |||
| 00095a5398 | |||
| 6b41c3bbb4 | |||
| b37238d790 | |||
| bc33e32c6f | |||
| 49704d801c | |||
| 34ca14d7fc | |||
| f7b247f9e8 | |||
| 3005d2877e | |||
| ed2a17970f | |||
| 9584fa85c9 | |||
| 1fd7cc0123 | |||
| 39e7c1376c | |||
| 2a3c632dc5 | |||
| 8d46d26abe | |||
| d761ac77f7 | |||
| c8b9db5bf4 | |||
| bce7d1d30c | |||
| bccd494a56 | |||
| 3fd29f6943 | |||
| 838a374a56 | |||
| 6e31da7a70 | |||
| fc9088bfd6 | |||
| 675421ea84 | |||
| 2ff98ae089 | |||
| c972c755fc | |||
| ebe3df7d22 | |||
| be44a75b74 | |||
| fd1227d3b8 | |||
| 3a9e02137d | |||
| d8a2be8edc | |||
| 1c46b2e8c3 | |||
| 3c3ac49f2f | |||
| 1a1be95a64 | |||
| 7a36659a20 | |||
| cb18281b14 | |||
| 856c9a87ac | |||
| a7d65fe84a | |||
| 15bf079af2 | |||
| 65ba750634 | |||
| 17dcbd3d4f | |||
| e914a0e19d | |||
| 2ba04bb1bd | |||
| 1b7314591a | |||
| 2bfb987312 | |||
| df34efafbc | |||
| c2a2048c8b | |||
| 1e1921cabb | |||
| ee233548e5 | |||
| 574dfbbe55 | |||
| 1d3bcdfe47 | |||
| 74369940bf | |||
| d188cec306 | |||
| 641a61f804 | |||
| 3025c64fea | |||
| c52cfc7e7a | |||
| 4ed9f775f6 | |||
| 0b08ad283a | |||
| 08895f77ff | |||
| 74f889f160 | |||
| c51090ab16 | |||
| cdb044cb45 | |||
| c83b26b556 | |||
| a202c666bc | |||
| e05046af10 | |||
| 41ed03e5c6 | |||
| da071e8452 | |||
| a0bb6df2b8 | |||
| 6f5c444ec5 | |||
| 20f66a4fe1 | |||
| 879745da53 | |||
| 3afec0a0be | |||
| 06c270a6e1 | |||
| 87e93636dc | |||
| 074d1f2e51 | |||
| 40f922b0e8 | |||
| a7214b6cff | |||
| 8ca66e72eb | |||
| 90be5799e4 | |||
| 1a93b0eca2 | |||
| c2360a35ad | |||
| 030a32f8fa | |||
| 25e72b43ce | |||
| 41e9883daa | |||
| ae461ebc81 | |||
| 3ebf256c5d | |||
| 0905e2be6e | |||
| bd380bc1a1 | |||
| cb4fd3a0e6 | |||
| bbe0acac5c | |||
| 45e7c25308 | |||
| 86aa1d8059 | |||
| 74254756ef | |||
| 259e08e47c | |||
| 6647c45731 | |||
| b614a5c447 | |||
| b80830913c | |||
| e57e11897e | |||
| 8a16034135 | |||
| 7fc3b7e5be | |||
| b0c7a1baae | |||
| 6411d83d46 | |||
| 74a063b0e5 | |||
| 96376e5cce | |||
| e7c26bf722 | |||
| cef4129fc9 | |||
| 0a28500848 | |||
| fc3f3f3bdb | |||
| fa58297973 | |||
| 5d1a22fb8f | |||
| d2f50f26f1 | |||
| 4a6042d0b4 | |||
| 846206d958 | |||
| 0daf4924f0 | |||
| d38a3d091d | |||
| 94dd08646d | |||
| 658f88ca48 | |||
| f53352efb2 | |||
| 38809a9d1b | |||
| 395682509c | |||
| ef3e7d7bf4 | |||
| c85b6b64a1 | |||
| 34c87d4934 | |||
| 93472e5d67 | |||
| ae185ee714 | |||
| 795036275a | |||
| d43ab6e32f | |||
| 280181f92e | |||
| f8d98934ad | |||
| 303cca0d85 | |||
| c2f16784b3 | |||
| 5bc6d8286c | |||
| 3f8381ffee | |||
| 1ca99294c9 | |||
| 680f0a57f5 | |||
| 94e3f54571 | |||
| 5c2b2aedcc | |||
| ebc31fb963 | |||
| 9659df6ad9 | |||
| 04d131e9dc | |||
| 78fe6282c7 | |||
| 0c00ee22fc | |||
| 5fd4946b1f | |||
| f1d3ac0cdc | |||
| e2c45905f0 | |||
| b2c68e6a65 | |||
| eb0f6dd3b6 | |||
| 0bf87f9092 | |||
| 1da2433bb2 | |||
| 2d6cf545b9 | |||
| 6a259a14fa | |||
| 41e1fd1e1f | |||
| 95fb3c7fc9 | |||
| 8237e5f9ea | |||
| aa75986178 | |||
| b887922055 | |||
| 68fa0f29c7 | |||
| d9d362c9c9 | |||
| d0bc4be0dc | |||
| 420530f532 | |||
| 3001f0f0ef | |||
| b2a1307d23 | |||
| 64da845a58 | |||
| 27027c4497 | |||
| 86c85c08ec | |||
| 196c8ffc3e | |||
| cfc95ee02a | |||
| 479fa36997 | |||
| 3e1216e9bc | |||
| 007883b7d1 | |||
| dc9200a12c | |||
| d2f955266d | |||
| 8e724538bd | |||
| 6fcdeb799d | |||
| 97b9b1f62b | |||
| 4bf9a4b640 | |||
| c5088772e8 | |||
| 56acefbd6c | |||
| 5ab06c4aae | |||
| c11f4b5c68 | |||
| 86376284f4 | |||
| 2b8a2fc7d8 | |||
| f26e1b41c8 | |||
| c1671af99f | |||
| ac7707d0f6 | |||
| 381cd710a2 | |||
| ad0d18cb79 | |||
| 7980ee77d0 | |||
| 916b8bb327 | |||
| 87e3d4dea9 | |||
| 76a913f5e0 | |||
| d8c192e647 | |||
| c453437620 | |||
| 720fa6d909 | |||
| b4f71089f4 | |||
| 83e6657ead | |||
| 7ea6df4111 | |||
| d9ab92602a | |||
| 5ffadaed31 | |||
| 24f5f7df5d | |||
| daf01fb1d6 | |||
| 0f11b6def9 | |||
| 7df83f44b8 | |||
| 169fa7bed6 | |||
| 027b504fe8 | |||
| 186ef4da78 | |||
| dc674098e7 | |||
| 9087b4b07c | |||
| 8e45c22d7a | |||
| 191c4e03cd | |||
| ab4154c57d | |||
| 28e93d12ff | |||
| 75e63c758b | |||
| 823f71f269 | |||
| 042dd4088d | |||
| eaa791a9eb | |||
| 2228627ff4 | |||
| 4c647ad9c8 | |||
| 8ca3e6c33f | |||
| dd6bdbf297 | |||
| b47dde87e4 | |||
| 99e65cccd8 | |||
| 3bdacb8f46 | |||
| b4f9c224d3 | |||
| 5ec0399c81 | |||
| b464fdc333 | |||
| 53825500db | |||
| f2ac790752 | |||
| 0d8805cdee | |||
| 656e24ac9b | |||
| 6718b37403 | |||
| c9e5e784fc | |||
| f92f958682 | |||
| f63fab0676 | |||
| cfc4903c0c | |||
| a527a847fe | |||
| 91b0bf8933 | |||
| 66d1c96783 | |||
| 986128076e | |||
| 1de0a53241 | |||
| 0ec7eaf606 | |||
| d9fcb0e92b | |||
| f49b4ba4db | |||
| 84e708328b | |||
| 125bed3f09 | |||
| 077e70169d | |||
| e6dc169a05 | |||
| f34c02756d | |||
| 1e4c315481 | |||
| a8283a0d00 | |||
| 55896669fc | |||
| e341e0b9d2 | |||
| e6538c83bb | |||
| 92e1285ea5 | |||
| 2aabd1d90e | |||
| 7b8b778f83 | |||
| 7c8dc57d55 | |||
| fe95fae5f2 | |||
| ce8a95abf7 | |||
| c8e7e543d6 | |||
| a9dbb15ffa | |||
| cf64043f7d | |||
| ccaff92c18 | |||
| 585b5c922a | |||
| ea80c2224c | |||
| 8b0f56c1a6 | |||
| 8022d12f03 | |||
| 3939f7f91b | |||
| aebf2e37dd | |||
| f53f859a71 | |||
| d916375abe | |||
| 57983df4bd | |||
| c68d7559a0 | |||
| 9a8f5bf2d6 | |||
| a8d742b031 | |||
| c27e4d1bfc | |||
| d15a8aa9a2 | |||
| 74a7d12ca4 | |||
| 2f94a9773e | |||
| 37bdfa21ea | |||
| f0bf2728c9 | |||
| dc715aa273 | |||
| 7ee2361e87 | |||
| e04c22f83f | |||
| 681cc13e90 | |||
| 090e0297d4 | |||
| 6f71335be4 | |||
| 7f51812c1e | |||
| a9dc4d7b9d | |||
| 5d50ddb5d4 | |||
| f86198d234 | |||
| ffe65d983c | |||
| b0b5be913c | |||
| 01efcbc584 | |||
| 02c249917a | |||
| 419bbc90b2 | |||
| b0c4510fdb | |||
| bf6a614e0d | |||
| feab01cd9c | |||
| 966024e534 | |||
| 2018722cc8 | |||
| 9d85c2a44a | |||
| 03dd047e62 | |||
| 86b547c1e0 | |||
| bab9752c8b | |||
| 774cc1be86 | |||
| 234b73c8a2 | |||
| abd06c48f4 | |||
| 6ca411e4e4 | |||
| 6470021e77 | |||
| 71658ab37b | |||
| 4f016a8024 | |||
| f362ed585b | |||
| 196172624f | |||
| 316702b7ab | |||
| a7625b009f | |||
| 5d4a33c90d | |||
| 041a6b8525 | |||
| 2638109ad6 | |||
| b019326747 | |||
| 54b44131b6 | |||
| a1d948025c | |||
| a90b2514ba | |||
| cb4ad27813 | |||
| 637831248b | |||
| 00228deaaa | |||
| 2373edf73c | |||
| e0e1b804a7 | |||
| fecbe8241f | |||
| 5983eaa1ce | |||
| 07fa454f72 | |||
| 4b5aa45379 | |||
| 9a0d866be4 | |||
| 308d8f71b8 | |||
| d0e8938039 | |||
| 13ed898b6b | |||
| e1dfd1c2a6 | |||
| e3e944911b | |||
| 51c0135250 | |||
| 7b19bbb14e | |||
| 5494a70f40 | |||
| 26c9ade1c9 | |||
| 87db23825f | |||
| 8fb00998a7 | |||
| dd3aa97d0a | |||
| 8bee8f4069 | |||
| 817fe21b3e | |||
| 905c37290f | |||
| f7632a47f9 | |||
| 646f1ddfb1 | |||
| 170c8068c5 | |||
| 3494037d20 | |||
| a1fd4e150b | |||
| b22f09bd1d | |||
| 4ed9169646 | |||
| f06c60bd47 | |||
| ee8250c26c | |||
| 88349bf944 | |||
| a8adcaf023 | |||
| 63785f82b5 | |||
| cf898da193 | |||
| 3c83e78d9f | |||
| d7291f73c9 | |||
| fe90f7f9b1 | |||
| 8b344ea39f | |||
| 8348a0cef8 | |||
| 7cf785b72f | |||
| e8913f4481 | |||
| f9c3d8dc97 | |||
| 09ca91fc0e | |||
| 16f5222efd | |||
| 28e7c04b37 | |||
| 28f99c46d3 | |||
| 205194f4e6 | |||
| 402d8b07cf | |||
| 3e303ab316 | |||
| e9e8c31ad1 | |||
| 703a6a4ea0 | |||
| 283730cf38 | |||
| 20417797e8 | |||
| 004c69b9ef | |||
| 47fe2d3783 | |||
| 36ef840a22 | |||
| 09c2445ac9 | |||
| 8a6d23f9c7 | |||
| 3d207b6744 | |||
| b3edda62ad | |||
| a429e6b1c3 | |||
| c1bf9c6221 | |||
| 75fffc1e25 | |||
| f264bab65c | |||
| 154fcd803b | |||
| 4ef32d3a96 | |||
| d2d109a69c | |||
| 3a2941d751 | |||
| 0ac10dfd42 | |||
| 9c95856b2f | |||
| 5ce4667d32 | |||
| be53fda6df | |||
| f48de05102 | |||
| 93ad81ed87 | |||
| ea14d211be | |||
| 8052cefd46 | |||
| 845815b9b7 | |||
| 609dc5d783 | |||
| 7a71b34b54 | |||
| 71a459422f | |||
| cd2628a0ee | |||
| 85da7175bc | |||
| d3bf0a164b | |||
| afb6ca1b8d | |||
| 94f43426d7 | |||
| 2b361f4f5d | |||
| 7438072f8c | |||
| 26c54fd358 | |||
| 7cb6b04c63 | |||
| fc29cde82a | |||
| 559ca946dc | |||
| 2b8e7c7504 | |||
| 6816d75933 | |||
| b58abbad7c | |||
| 999814ca87 | |||
| 3c2760a803 | |||
| 0edbd7bcca | |||
| 21e89fa7de | |||
| 968d6d1d1f | |||
| cf0fd0e0ad | |||
| 16e5dcf7b2 | |||
| ab6bb25d46 | |||
| 07f49559be | |||
| b24b1a7e57 | |||
| faf64f8986 | |||
| a617487a43 | |||
| 3012a7aef3 | |||
| 499e19de34 | |||
| 9161762ca9 | |||
| 9bbd26efe6 | |||
| 258b2622d5 | |||
| 80ec9085dd | |||
| c5c7373e10 | |||
| b7721866e5 | |||
| 8314b9bedb | |||
| 75298a402f | |||
| 92b5efd414 | |||
| 33ee392b7b | |||
| 5237f8b7dc | |||
| 5107313fd1 | |||
| 95bbc66919 | |||
| e268e59419 | |||
| 547e1f9498 | |||
| bf32d8b6fd | |||
| 8299881024 | |||
| da02268196 | |||
| 8c4b9a1e70 | |||
| 0906c484e9 | |||
| 4199c30fec | |||
| 4a8084cdbc | |||
| 6263e6848c | |||
| 58c266ad07 | |||
| 2939813e1a | |||
| a9e5ee7e79 | |||
| a17b0e9901 | |||
| 8f23d966bf | |||
| 7a76fc72d0 | |||
| 518a4dd5ee | |||
| 2b6d4e5d8b |
@@ -1,201 +0,0 @@
|
||||
---
|
||||
name: lora-manager-e2e
|
||||
description: End-to-end testing and validation for LoRa Manager features. Use when performing automated E2E validation of LoRa Manager standalone mode, including starting/restarting the server, using Chrome DevTools MCP to interact with the web UI at http://127.0.0.1:8188/loras, and verifying frontend-to-backend functionality. Covers workflow validation, UI interaction testing, and integration testing between the standalone Python backend and the browser frontend.
|
||||
---
|
||||
|
||||
# LoRa Manager E2E Testing
|
||||
|
||||
This skill provides workflows and utilities for end-to-end testing of LoRa Manager using Chrome DevTools MCP.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- LoRa Manager project cloned and dependencies installed (`pip install -r requirements.txt`)
|
||||
- Chrome browser available for debugging
|
||||
- Chrome DevTools MCP connected
|
||||
|
||||
## Quick Start Workflow
|
||||
|
||||
### 1. Start LoRa Manager Standalone
|
||||
|
||||
```python
|
||||
# Use the provided script to start the server
|
||||
python .agents/skills/lora-manager-e2e/scripts/start_server.py --port 8188
|
||||
```
|
||||
|
||||
Or manually:
|
||||
```bash
|
||||
cd /home/miao/workspace/ComfyUI/custom_nodes/ComfyUI-Lora-Manager
|
||||
python standalone.py --port 8188
|
||||
```
|
||||
|
||||
Wait for server ready message before proceeding.
|
||||
|
||||
### 2. Open Chrome Debug Mode
|
||||
|
||||
```bash
|
||||
# Chrome with remote debugging on port 9222
|
||||
google-chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:8188/loras
|
||||
```
|
||||
|
||||
### 3. Connect Chrome DevTools MCP
|
||||
|
||||
Ensure the MCP server is connected to Chrome at `http://localhost:9222`.
|
||||
|
||||
### 4. Navigate and Interact
|
||||
|
||||
Use Chrome DevTools MCP tools to:
|
||||
- Take snapshots: `take_snapshot`
|
||||
- Click elements: `click`
|
||||
- Fill forms: `fill` or `fill_form`
|
||||
- Evaluate scripts: `evaluate_script`
|
||||
- Wait for elements: `wait_for`
|
||||
|
||||
## Common E2E Test Patterns
|
||||
|
||||
### Pattern: Full Page Load Verification
|
||||
|
||||
```python
|
||||
# Navigate to LoRA list page
|
||||
navigate_page(type="url", url="http://127.0.0.1:8188/loras")
|
||||
|
||||
# Wait for page to load
|
||||
wait_for(text="LoRAs", timeout=10000)
|
||||
|
||||
# Take snapshot to verify UI state
|
||||
snapshot = take_snapshot()
|
||||
```
|
||||
|
||||
### Pattern: Restart Server for Configuration Changes
|
||||
|
||||
```python
|
||||
# Stop current server (if running)
|
||||
# Start with new configuration
|
||||
python .agents/skills/lora-manager-e2e/scripts/start_server.py --port 8188 --restart
|
||||
|
||||
# Wait and refresh browser
|
||||
navigate_page(type="reload", ignoreCache=True)
|
||||
wait_for(text="LoRAs", timeout=15000)
|
||||
```
|
||||
|
||||
### Pattern: Verify Backend API via Frontend
|
||||
|
||||
```python
|
||||
# Execute script in browser to call backend API
|
||||
result = evaluate_script(function="""
|
||||
async () => {
|
||||
const response = await fetch('/loras/api/list');
|
||||
const data = await response.json();
|
||||
return { count: data.length, firstItem: data[0]?.name };
|
||||
}
|
||||
""")
|
||||
```
|
||||
|
||||
### Pattern: Form Submission Flow
|
||||
|
||||
```python
|
||||
# Fill a form (e.g., search or filter)
|
||||
fill_form(elements=[
|
||||
{"uid": "search-input", "value": "character"},
|
||||
])
|
||||
|
||||
# Click submit button
|
||||
click(uid="search-button")
|
||||
|
||||
# Wait for results
|
||||
wait_for(text="Results", timeout=5000)
|
||||
|
||||
# Verify results via snapshot
|
||||
snapshot = take_snapshot()
|
||||
```
|
||||
|
||||
### Pattern: Modal Dialog Interaction
|
||||
|
||||
```python
|
||||
# Open modal (e.g., add LoRA)
|
||||
click(uid="add-lora-button")
|
||||
|
||||
# Wait for modal to appear
|
||||
wait_for(text="Add LoRA", timeout=3000)
|
||||
|
||||
# Fill modal form
|
||||
fill_form(elements=[
|
||||
{"uid": "lora-name", "value": "Test LoRA"},
|
||||
{"uid": "lora-path", "value": "/path/to/lora.safetensors"},
|
||||
])
|
||||
|
||||
# Submit
|
||||
click(uid="modal-submit-button")
|
||||
|
||||
# Wait for success message or close
|
||||
wait_for(text="Success", timeout=5000)
|
||||
```
|
||||
|
||||
## Available Scripts
|
||||
|
||||
### scripts/start_server.py
|
||||
|
||||
Starts or restarts the LoRa Manager standalone server.
|
||||
|
||||
```bash
|
||||
python scripts/start_server.py [--port PORT] [--restart] [--wait]
|
||||
```
|
||||
|
||||
Options:
|
||||
- `--port`: Server port (default: 8188)
|
||||
- `--restart`: Kill existing server before starting
|
||||
- `--wait`: Wait for server to be ready before exiting
|
||||
|
||||
### scripts/wait_for_server.py
|
||||
|
||||
Polls server until ready or timeout.
|
||||
|
||||
```bash
|
||||
python scripts/wait_for_server.py [--port PORT] [--timeout SECONDS]
|
||||
```
|
||||
|
||||
## Test Scenarios Reference
|
||||
|
||||
See [references/test-scenarios.md](references/test-scenarios.md) for detailed test scenarios including:
|
||||
- LoRA list display and filtering
|
||||
- Model metadata editing
|
||||
- Recipe creation and management
|
||||
- Settings configuration
|
||||
- Import/export functionality
|
||||
|
||||
## Network Request Verification
|
||||
|
||||
Use `list_network_requests` and `get_network_request` to verify API calls:
|
||||
|
||||
```python
|
||||
# List recent XHR/fetch requests
|
||||
requests = list_network_requests(resourceTypes=["xhr", "fetch"])
|
||||
|
||||
# Get details of specific request
|
||||
details = get_network_request(reqid=123)
|
||||
```
|
||||
|
||||
## Console Message Monitoring
|
||||
|
||||
```python
|
||||
# Check for errors or warnings
|
||||
messages = list_console_messages(types=["error", "warn"])
|
||||
```
|
||||
|
||||
## Performance Testing
|
||||
|
||||
```python
|
||||
# Start performance trace
|
||||
performance_start_trace(reload=True, autoStop=False)
|
||||
|
||||
# Perform actions...
|
||||
|
||||
# Stop and analyze
|
||||
results = performance_stop_trace()
|
||||
```
|
||||
|
||||
## Cleanup
|
||||
|
||||
Always ensure proper cleanup after tests:
|
||||
1. Stop the standalone server
|
||||
2. Close browser pages (keep at least one open)
|
||||
3. Clear temporary data if needed
|
||||
@@ -1,324 +0,0 @@
|
||||
# Chrome DevTools MCP Cheatsheet for LoRa Manager
|
||||
|
||||
Quick reference for common MCP commands used in LoRa Manager E2E testing.
|
||||
|
||||
## Navigation
|
||||
|
||||
```python
|
||||
# Navigate to LoRA list page
|
||||
navigate_page(type="url", url="http://127.0.0.1:8188/loras")
|
||||
|
||||
# Reload page with cache clear
|
||||
navigate_page(type="reload", ignoreCache=True)
|
||||
|
||||
# Go back/forward
|
||||
navigate_page(type="back")
|
||||
navigate_page(type="forward")
|
||||
```
|
||||
|
||||
## Waiting
|
||||
|
||||
```python
|
||||
# Wait for text to appear
|
||||
wait_for(text="LoRAs", timeout=10000)
|
||||
|
||||
# Wait for specific element (via evaluate_script)
|
||||
evaluate_script(function="""
|
||||
() => {
|
||||
return new Promise((resolve) => {
|
||||
const check = () => {
|
||||
if (document.querySelector('.lora-card')) {
|
||||
resolve(true);
|
||||
} else {
|
||||
setTimeout(check, 100);
|
||||
}
|
||||
};
|
||||
check();
|
||||
});
|
||||
}
|
||||
""")
|
||||
```
|
||||
|
||||
## Taking Snapshots
|
||||
|
||||
```python
|
||||
# Full page snapshot
|
||||
snapshot = take_snapshot()
|
||||
|
||||
# Verbose snapshot (more details)
|
||||
snapshot = take_snapshot(verbose=True)
|
||||
|
||||
# Save to file
|
||||
take_snapshot(filePath="test-snapshots/page-load.json")
|
||||
```
|
||||
|
||||
## Element Interaction
|
||||
|
||||
```python
|
||||
# Click element
|
||||
click(uid="element-uid-from-snapshot")
|
||||
|
||||
# Double click
|
||||
click(uid="element-uid", dblClick=True)
|
||||
|
||||
# Fill input
|
||||
fill(uid="search-input", value="test query")
|
||||
|
||||
# Fill multiple inputs
|
||||
fill_form(elements=[
|
||||
{"uid": "input-1", "value": "value 1"},
|
||||
{"uid": "input-2", "value": "value 2"},
|
||||
])
|
||||
|
||||
# Hover
|
||||
hover(uid="lora-card-1")
|
||||
|
||||
# Upload file
|
||||
upload_file(uid="file-input", filePath="/path/to/file.safetensors")
|
||||
```
|
||||
|
||||
## Keyboard Input
|
||||
|
||||
```python
|
||||
# Press key
|
||||
press_key(key="Enter")
|
||||
press_key(key="Escape")
|
||||
press_key(key="Tab")
|
||||
|
||||
# Keyboard shortcuts
|
||||
press_key(key="Control+A") # Select all
|
||||
press_key(key="Control+F") # Find
|
||||
```
|
||||
|
||||
## JavaScript Evaluation
|
||||
|
||||
```python
|
||||
# Simple evaluation
|
||||
result = evaluate_script(function="() => document.title")
|
||||
|
||||
# Async evaluation
|
||||
result = evaluate_script(function="""
|
||||
async () => {
|
||||
const response = await fetch('/loras/api/list');
|
||||
return await response.json();
|
||||
}
|
||||
""")
|
||||
|
||||
# Check element existence
|
||||
exists = evaluate_script(function="""
|
||||
() => document.querySelector('.lora-card') !== null
|
||||
""")
|
||||
|
||||
# Get element count
|
||||
count = evaluate_script(function="""
|
||||
() => document.querySelectorAll('.lora-card').length
|
||||
""")
|
||||
```
|
||||
|
||||
## Network Monitoring
|
||||
|
||||
```python
|
||||
# List all network requests
|
||||
requests = list_network_requests()
|
||||
|
||||
# Filter by resource type
|
||||
xhr_requests = list_network_requests(resourceTypes=["xhr", "fetch"])
|
||||
|
||||
# Get specific request details
|
||||
details = get_network_request(reqid=123)
|
||||
|
||||
# Include preserved requests from previous navigations
|
||||
all_requests = list_network_requests(includePreservedRequests=True)
|
||||
```
|
||||
|
||||
## Console Monitoring
|
||||
|
||||
```python
|
||||
# List all console messages
|
||||
messages = list_console_messages()
|
||||
|
||||
# Filter by type
|
||||
errors = list_console_messages(types=["error", "warn"])
|
||||
|
||||
# Include preserved messages
|
||||
all_messages = list_console_messages(includePreservedMessages=True)
|
||||
|
||||
# Get specific message
|
||||
details = get_console_message(msgid=1)
|
||||
```
|
||||
|
||||
## Performance Testing
|
||||
|
||||
```python
|
||||
# Start trace with page reload
|
||||
performance_start_trace(reload=True, autoStop=False)
|
||||
|
||||
# Start trace without reload
|
||||
performance_start_trace(reload=False, autoStop=True, filePath="trace.json.gz")
|
||||
|
||||
# Stop trace
|
||||
results = performance_stop_trace()
|
||||
|
||||
# Stop and save
|
||||
performance_stop_trace(filePath="trace-results.json.gz")
|
||||
|
||||
# Analyze specific insight
|
||||
insight = performance_analyze_insight(
|
||||
insightSetId="results.insightSets[0].id",
|
||||
insightName="LCPBreakdown"
|
||||
)
|
||||
```
|
||||
|
||||
## Page Management
|
||||
|
||||
```python
|
||||
# List open pages
|
||||
pages = list_pages()
|
||||
|
||||
# Select a page
|
||||
select_page(pageId=0, bringToFront=True)
|
||||
|
||||
# Create new page
|
||||
new_page(url="http://127.0.0.1:8188/loras")
|
||||
|
||||
# Close page (keep at least one open!)
|
||||
close_page(pageId=1)
|
||||
|
||||
# Resize page
|
||||
resize_page(width=1920, height=1080)
|
||||
```
|
||||
|
||||
## Screenshots
|
||||
|
||||
```python
|
||||
# Full page screenshot
|
||||
take_screenshot(fullPage=True)
|
||||
|
||||
# Viewport screenshot
|
||||
take_screenshot()
|
||||
|
||||
# Element screenshot
|
||||
take_screenshot(uid="lora-card-1")
|
||||
|
||||
# Save to file
|
||||
take_screenshot(filePath="screenshots/page.png", format="png")
|
||||
|
||||
# JPEG with quality
|
||||
take_screenshot(filePath="screenshots/page.jpg", format="jpeg", quality=90)
|
||||
```
|
||||
|
||||
## Dialog Handling
|
||||
|
||||
```python
|
||||
# Accept dialog
|
||||
handle_dialog(action="accept")
|
||||
|
||||
# Accept with text input
|
||||
handle_dialog(action="accept", promptText="user input")
|
||||
|
||||
# Dismiss dialog
|
||||
handle_dialog(action="dismiss")
|
||||
```
|
||||
|
||||
## Device Emulation
|
||||
|
||||
```python
|
||||
# Mobile viewport
|
||||
emulate(viewport={"width": 375, "height": 667, "isMobile": True, "hasTouch": True})
|
||||
|
||||
# Tablet viewport
|
||||
emulate(viewport={"width": 768, "height": 1024, "isMobile": True, "hasTouch": True})
|
||||
|
||||
# Desktop viewport
|
||||
emulate(viewport={"width": 1920, "height": 1080})
|
||||
|
||||
# Network throttling
|
||||
emulate(networkConditions="Slow 3G")
|
||||
emulate(networkConditions="Fast 4G")
|
||||
|
||||
# CPU throttling
|
||||
emulate(cpuThrottlingRate=4) # 4x slowdown
|
||||
|
||||
# Geolocation
|
||||
emulate(geolocation={"latitude": 37.7749, "longitude": -122.4194})
|
||||
|
||||
# User agent
|
||||
emulate(userAgent="Mozilla/5.0 (Custom)")
|
||||
|
||||
# Reset emulation
|
||||
emulate(viewport=None, networkConditions="No emulation", userAgent=None)
|
||||
```
|
||||
|
||||
## Drag and Drop
|
||||
|
||||
```python
|
||||
# Drag element to another
|
||||
drag(from_uid="draggable-item", to_uid="drop-zone")
|
||||
```
|
||||
|
||||
## Common LoRa Manager Test Patterns
|
||||
|
||||
### Verify LoRA Cards Loaded
|
||||
|
||||
```python
|
||||
navigate_page(type="url", url="http://127.0.0.1:8188/loras")
|
||||
wait_for(text="LoRAs", timeout=10000)
|
||||
|
||||
# Check if cards loaded
|
||||
result = evaluate_script(function="""
|
||||
() => {
|
||||
const cards = document.querySelectorAll('.lora-card');
|
||||
return {
|
||||
count: cards.length,
|
||||
hasData: cards.length > 0
|
||||
};
|
||||
}
|
||||
""")
|
||||
```
|
||||
|
||||
### Search and Verify Results
|
||||
|
||||
```python
|
||||
fill(uid="search-input", value="character")
|
||||
press_key(key="Enter")
|
||||
wait_for(timeout=2000) # Wait for debounce
|
||||
|
||||
# Check results
|
||||
result = evaluate_script(function="""
|
||||
() => {
|
||||
const cards = document.querySelectorAll('.lora-card');
|
||||
const names = Array.from(cards).map(c => c.dataset.name || c.textContent);
|
||||
return { count: cards.length, names };
|
||||
}
|
||||
""")
|
||||
```
|
||||
|
||||
### Check API Response
|
||||
|
||||
```python
|
||||
# Trigger API call
|
||||
evaluate_script(function="""
|
||||
() => window.loraApiCallPromise = fetch('/loras/api/list').then(r => r.json())
|
||||
""")
|
||||
|
||||
# Wait and get result
|
||||
import time
|
||||
time.sleep(1)
|
||||
|
||||
result = evaluate_script(function="""
|
||||
async () => await window.loraApiCallPromise
|
||||
""")
|
||||
```
|
||||
|
||||
### Monitor Console for Errors
|
||||
|
||||
```python
|
||||
# Before test: clear console (navigate reloads)
|
||||
navigate_page(type="reload")
|
||||
|
||||
# ... perform actions ...
|
||||
|
||||
# Check for errors
|
||||
errors = list_console_messages(types=["error"])
|
||||
assert len(errors) == 0, f"Console errors: {errors}"
|
||||
```
|
||||
@@ -1,272 +0,0 @@
|
||||
# LoRa Manager E2E Test Scenarios
|
||||
|
||||
This document provides detailed test scenarios for end-to-end validation of LoRa Manager features.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [LoRA List Page](#lora-list-page)
|
||||
2. [Model Details](#model-details)
|
||||
3. [Recipes](#recipes)
|
||||
4. [Settings](#settings)
|
||||
5. [Import/Export](#importexport)
|
||||
|
||||
---
|
||||
|
||||
## LoRA List Page
|
||||
|
||||
### Scenario: Page Load and Display
|
||||
|
||||
**Objective**: Verify the LoRA list page loads correctly and displays models.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to `http://127.0.0.1:8188/loras`
|
||||
2. Wait for page title "LoRAs" to appear
|
||||
3. Take snapshot to verify:
|
||||
- Header with "LoRAs" title is visible
|
||||
- Search/filter controls are present
|
||||
- Grid/list view toggle exists
|
||||
- LoRA cards are displayed (if models exist)
|
||||
- Pagination controls (if applicable)
|
||||
|
||||
**Expected Result**: Page loads without errors, UI elements are present.
|
||||
|
||||
### Scenario: Search Functionality
|
||||
|
||||
**Objective**: Verify search filters LoRA models correctly.
|
||||
|
||||
**Steps**:
|
||||
1. Ensure at least one LoRA exists with known name (e.g., "test-character")
|
||||
2. Navigate to LoRA list page
|
||||
3. Enter search term in search box: "test"
|
||||
4. Press Enter or click search button
|
||||
5. Wait for results to update
|
||||
|
||||
**Expected Result**: Only LoRAs matching search term are displayed.
|
||||
|
||||
**Verification Script**:
|
||||
```python
|
||||
# After search, verify filtered results
|
||||
evaluate_script(function="""
|
||||
() => {
|
||||
const cards = document.querySelectorAll('.lora-card');
|
||||
const names = Array.from(cards).map(c => c.dataset.name);
|
||||
return { count: cards.length, names };
|
||||
}
|
||||
""")
|
||||
```
|
||||
|
||||
### Scenario: Filter by Tags
|
||||
|
||||
**Objective**: Verify tag filtering works correctly.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to LoRA list page
|
||||
2. Click on a tag (e.g., "character", "style")
|
||||
3. Wait for filtered results
|
||||
|
||||
**Expected Result**: Only LoRAs with selected tag are displayed.
|
||||
|
||||
### Scenario: View Mode Toggle
|
||||
|
||||
**Objective**: Verify grid/list view toggle works.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to LoRA list page
|
||||
2. Click list view button
|
||||
3. Verify list layout
|
||||
4. Click grid view button
|
||||
5. Verify grid layout
|
||||
|
||||
**Expected Result**: View mode changes correctly, layout updates.
|
||||
|
||||
---
|
||||
|
||||
## Model Details
|
||||
|
||||
### Scenario: Open Model Details
|
||||
|
||||
**Objective**: Verify clicking a LoRA opens its details.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to LoRA list page
|
||||
2. Click on a LoRA card
|
||||
3. Wait for details panel/modal to open
|
||||
|
||||
**Expected Result**: Details panel shows:
|
||||
- Model name
|
||||
- Preview image
|
||||
- Metadata (trigger words, tags, etc.)
|
||||
- Action buttons (edit, delete, etc.)
|
||||
|
||||
### Scenario: Edit Model Metadata
|
||||
|
||||
**Objective**: Verify metadata editing works end-to-end.
|
||||
|
||||
**Steps**:
|
||||
1. Open a LoRA's details
|
||||
2. Click "Edit" button
|
||||
3. Modify trigger words field
|
||||
4. Add/remove tags
|
||||
5. Save changes
|
||||
6. Refresh page
|
||||
7. Reopen the same LoRA
|
||||
|
||||
**Expected Result**: Changes persist after refresh.
|
||||
|
||||
### Scenario: Delete Model
|
||||
|
||||
**Objective**: Verify model deletion works.
|
||||
|
||||
**Steps**:
|
||||
1. Open a LoRA's details
|
||||
2. Click "Delete" button
|
||||
3. Confirm deletion in dialog
|
||||
4. Wait for removal
|
||||
|
||||
**Expected Result**: Model removed from list, success message shown.
|
||||
|
||||
---
|
||||
|
||||
## Recipes
|
||||
|
||||
### Scenario: Recipe List Display
|
||||
|
||||
**Objective**: Verify recipes page loads and displays recipes.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to `http://127.0.0.1:8188/recipes`
|
||||
2. Wait for "Recipes" title
|
||||
3. Take snapshot
|
||||
|
||||
**Expected Result**: Recipe list displayed with cards/items.
|
||||
|
||||
### Scenario: Create New Recipe
|
||||
|
||||
**Objective**: Verify recipe creation workflow.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to recipes page
|
||||
2. Click "New Recipe" button
|
||||
3. Fill recipe form:
|
||||
- Name: "Test Recipe"
|
||||
- Description: "E2E test recipe"
|
||||
- Add LoRA models
|
||||
4. Save recipe
|
||||
5. Verify recipe appears in list
|
||||
|
||||
**Expected Result**: New recipe created and displayed.
|
||||
|
||||
### Scenario: Apply Recipe
|
||||
|
||||
**Objective**: Verify applying a recipe to ComfyUI.
|
||||
|
||||
**Steps**:
|
||||
1. Open a recipe
|
||||
2. Click "Apply" or "Load in ComfyUI"
|
||||
3. Verify action completes
|
||||
|
||||
**Expected Result**: Recipe applied successfully.
|
||||
|
||||
---
|
||||
|
||||
## Settings
|
||||
|
||||
### Scenario: Settings Page Load
|
||||
|
||||
**Objective**: Verify settings page displays correctly.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to `http://127.0.0.1:8188/settings`
|
||||
2. Wait for "Settings" title
|
||||
3. Take snapshot
|
||||
|
||||
**Expected Result**: Settings form with various options displayed.
|
||||
|
||||
### Scenario: Change Setting and Restart
|
||||
|
||||
**Objective**: Verify settings persist after restart.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to settings page
|
||||
2. Change a setting (e.g., default view mode)
|
||||
3. Save settings
|
||||
4. Restart server: `python scripts/start_server.py --restart --wait`
|
||||
5. Refresh browser page
|
||||
6. Navigate to settings
|
||||
|
||||
**Expected Result**: Changed setting value persists.
|
||||
|
||||
---
|
||||
|
||||
## Import/Export
|
||||
|
||||
### Scenario: Export Models List
|
||||
|
||||
**Objective**: Verify export functionality.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to LoRA list
|
||||
2. Click "Export" button
|
||||
3. Select format (JSON/CSV)
|
||||
4. Download file
|
||||
|
||||
**Expected Result**: File downloaded with correct data.
|
||||
|
||||
### Scenario: Import Models
|
||||
|
||||
**Objective**: Verify import functionality.
|
||||
|
||||
**Steps**:
|
||||
1. Prepare import file
|
||||
2. Navigate to import page
|
||||
3. Upload file
|
||||
4. Verify import results
|
||||
|
||||
**Expected Result**: Models imported successfully, confirmation shown.
|
||||
|
||||
---
|
||||
|
||||
## API Integration Tests
|
||||
|
||||
### Scenario: Verify API Endpoints
|
||||
|
||||
**Objective**: Verify backend API responds correctly.
|
||||
|
||||
**Test via browser console**:
|
||||
```javascript
|
||||
// List LoRAs
|
||||
fetch('/loras/api/list').then(r => r.json()).then(console.log)
|
||||
|
||||
// Get LoRA details
|
||||
fetch('/loras/api/detail/<id>').then(r => r.json()).then(console.log)
|
||||
|
||||
// Search LoRAs
|
||||
fetch('/loras/api/search?q=test').then(r => r.json()).then(console.log)
|
||||
```
|
||||
|
||||
**Expected Result**: APIs return valid JSON with expected structure.
|
||||
|
||||
---
|
||||
|
||||
## Console Error Monitoring
|
||||
|
||||
During all tests, monitor browser console for errors:
|
||||
|
||||
```python
|
||||
# Check for JavaScript errors
|
||||
messages = list_console_messages(types=["error"])
|
||||
assert len(messages) == 0, f"Console errors found: {messages}"
|
||||
```
|
||||
|
||||
## Network Request Verification
|
||||
|
||||
Verify key API calls are made:
|
||||
|
||||
```python
|
||||
# List XHR requests
|
||||
requests = list_network_requests(resourceTypes=["xhr", "fetch"])
|
||||
|
||||
# Look for specific endpoints
|
||||
lora_list_requests = [r for r in requests if "/api/list" in r.get("url", "")]
|
||||
assert len(lora_list_requests) > 0, "LoRA list API not called"
|
||||
```
|
||||
@@ -1,193 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Example E2E test demonstrating LoRa Manager testing workflow.
|
||||
|
||||
This script shows how to:
|
||||
1. Start the standalone server
|
||||
2. Use Chrome DevTools MCP to interact with the UI
|
||||
3. Verify functionality end-to-end
|
||||
|
||||
Note: This is a template. Actual execution requires Chrome DevTools MCP.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def run_test():
|
||||
"""Run example E2E test flow."""
|
||||
|
||||
print("=" * 60)
|
||||
print("LoRa Manager E2E Test Example")
|
||||
print("=" * 60)
|
||||
|
||||
# Step 1: Start server
|
||||
print("\n[1/5] Starting LoRa Manager standalone server...")
|
||||
result = subprocess.run(
|
||||
[sys.executable, "start_server.py", "--port", "8188", "--wait", "--timeout", "30"],
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
if result.returncode != 0:
|
||||
print(f"Failed to start server: {result.stderr}")
|
||||
return 1
|
||||
print("Server ready!")
|
||||
|
||||
# Step 2: Open Chrome (manual step - show command)
|
||||
print("\n[2/5] Open Chrome with debug mode:")
|
||||
print("google-chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:8188/loras")
|
||||
print("(In actual test, this would be automated via MCP)")
|
||||
|
||||
# Step 3: Navigate and verify page load
|
||||
print("\n[3/5] Page Load Verification:")
|
||||
print("""
|
||||
MCP Commands to execute:
|
||||
1. navigate_page(type="url", url="http://127.0.0.1:8188/loras")
|
||||
2. wait_for(text="LoRAs", timeout=10000)
|
||||
3. snapshot = take_snapshot()
|
||||
""")
|
||||
|
||||
# Step 4: Test search functionality
|
||||
print("\n[4/5] Search Functionality Test:")
|
||||
print("""
|
||||
MCP Commands to execute:
|
||||
1. fill(uid="search-input", value="test")
|
||||
2. press_key(key="Enter")
|
||||
3. wait_for(text="Results", timeout=5000)
|
||||
4. result = evaluate_script(function="""
|
||||
() => {
|
||||
const cards = document.querySelectorAll('.lora-card');
|
||||
return { count: cards.length };
|
||||
}
|
||||
""")
|
||||
""")
|
||||
|
||||
# Step 5: Verify API
|
||||
print("\n[5/5] API Verification:")
|
||||
print("""
|
||||
MCP Commands to execute:
|
||||
1. api_result = evaluate_script(function="""
|
||||
async () => {
|
||||
const response = await fetch('/loras/api/list');
|
||||
const data = await response.json();
|
||||
return { count: data.length, status: response.status };
|
||||
}
|
||||
""")
|
||||
2. Verify api_result['status'] == 200
|
||||
""")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Test flow completed!")
|
||||
print("=" * 60)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def example_restart_flow():
|
||||
"""Example: Testing configuration change that requires restart."""
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Example: Server Restart Flow")
|
||||
print("=" * 60)
|
||||
|
||||
print("""
|
||||
Scenario: Change setting and verify after restart
|
||||
|
||||
Steps:
|
||||
1. Navigate to settings page
|
||||
- navigate_page(type="url", url="http://127.0.0.1:8188/settings")
|
||||
|
||||
2. Change a setting (e.g., theme)
|
||||
- fill(uid="theme-select", value="dark")
|
||||
- click(uid="save-settings-button")
|
||||
|
||||
3. Restart server
|
||||
- subprocess.run([python, "start_server.py", "--restart", "--wait"])
|
||||
|
||||
4. Refresh browser
|
||||
- navigate_page(type="reload", ignoreCache=True)
|
||||
- wait_for(text="LoRAs", timeout=15000)
|
||||
|
||||
5. Verify setting persisted
|
||||
- navigate_page(type="url", url="http://127.0.0.1:8188/settings")
|
||||
- theme = evaluate_script(function="() => document.querySelector('#theme-select').value")
|
||||
- assert theme == "dark"
|
||||
""")
|
||||
|
||||
|
||||
def example_modal_interaction():
|
||||
"""Example: Testing modal dialog interaction."""
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Example: Modal Dialog Interaction")
|
||||
print("=" * 60)
|
||||
|
||||
print("""
|
||||
Scenario: Add new LoRA via modal
|
||||
|
||||
Steps:
|
||||
1. Open modal
|
||||
- click(uid="add-lora-button")
|
||||
- wait_for(text="Add LoRA", timeout=3000)
|
||||
|
||||
2. Fill form
|
||||
- fill_form(elements=[
|
||||
{"uid": "lora-name", "value": "Test Character"},
|
||||
{"uid": "lora-path", "value": "/models/test.safetensors"},
|
||||
])
|
||||
|
||||
3. Submit
|
||||
- click(uid="modal-submit-button")
|
||||
|
||||
4. Verify success
|
||||
- wait_for(text="Successfully added", timeout=5000)
|
||||
- snapshot = take_snapshot()
|
||||
""")
|
||||
|
||||
|
||||
def example_network_monitoring():
|
||||
"""Example: Network request monitoring."""
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Example: Network Request Monitoring")
|
||||
print("=" * 60)
|
||||
|
||||
print("""
|
||||
Scenario: Verify API calls during user interaction
|
||||
|
||||
Steps:
|
||||
1. Clear network log (implicit on navigation)
|
||||
- navigate_page(type="url", url="http://127.0.0.1:8188/loras")
|
||||
|
||||
2. Perform action that triggers API call
|
||||
- fill(uid="search-input", value="character")
|
||||
- press_key(key="Enter")
|
||||
|
||||
3. List network requests
|
||||
- requests = list_network_requests(resourceTypes=["xhr", "fetch"])
|
||||
|
||||
4. Find search API call
|
||||
- search_requests = [r for r in requests if "/api/search" in r.get("url", "")]
|
||||
- assert len(search_requests) > 0, "Search API was not called"
|
||||
|
||||
5. Get request details
|
||||
- if search_requests:
|
||||
details = get_network_request(reqid=search_requests[0]["reqid"])
|
||||
- Verify request method, response status, etc.
|
||||
""")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("LoRa Manager E2E Test Examples\n")
|
||||
print("This script demonstrates E2E testing patterns.\n")
|
||||
print("Note: Actual execution requires Chrome DevTools MCP connection.\n")
|
||||
|
||||
run_test()
|
||||
example_restart_flow()
|
||||
example_modal_interaction()
|
||||
example_network_monitoring()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("All examples shown!")
|
||||
print("=" * 60)
|
||||
@@ -1,169 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Start or restart LoRa Manager standalone server for E2E testing.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import socket
|
||||
import signal
|
||||
import os
|
||||
|
||||
|
||||
def find_server_process(port: int) -> list[int]:
|
||||
"""Find PIDs of processes listening on the given port."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["lsof", "-ti", f":{port}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
return [int(pid) for pid in result.stdout.strip().split("\n") if pid]
|
||||
except FileNotFoundError:
|
||||
# lsof not available, try netstat
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["netstat", "-tlnp"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False
|
||||
)
|
||||
pids = []
|
||||
for line in result.stdout.split("\n"):
|
||||
if f":{port}" in line:
|
||||
parts = line.split()
|
||||
for part in parts:
|
||||
if "/" in part:
|
||||
try:
|
||||
pid = int(part.split("/")[0])
|
||||
pids.append(pid)
|
||||
except ValueError:
|
||||
pass
|
||||
return pids
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def kill_server(port: int) -> None:
|
||||
"""Kill processes using the specified port."""
|
||||
pids = find_server_process(port)
|
||||
for pid in pids:
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
print(f"Sent SIGTERM to process {pid}")
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
# Wait for processes to terminate
|
||||
time.sleep(1)
|
||||
|
||||
# Force kill if still running
|
||||
pids = find_server_process(port)
|
||||
for pid in pids:
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
print(f"Sent SIGKILL to process {pid}")
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
def is_server_ready(port: int, timeout: float = 0.5) -> bool:
|
||||
"""Check if server is accepting connections."""
|
||||
try:
|
||||
with socket.create_connection(("127.0.0.1", port), timeout=timeout):
|
||||
return True
|
||||
except (socket.timeout, ConnectionRefusedError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
def wait_for_server(port: int, timeout: int = 30) -> bool:
|
||||
"""Wait for server to become ready."""
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
if is_server_ready(port):
|
||||
return True
|
||||
time.sleep(0.5)
|
||||
return False
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Start LoRa Manager standalone server for E2E testing"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port",
|
||||
type=int,
|
||||
default=8188,
|
||||
help="Server port (default: 8188)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--restart",
|
||||
action="store_true",
|
||||
help="Kill existing server before starting"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--wait",
|
||||
action="store_true",
|
||||
help="Wait for server to be ready before exiting"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=int,
|
||||
default=30,
|
||||
help="Timeout for waiting (default: 30)"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Get project root (parent of .agents directory)
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
skill_dir = os.path.dirname(script_dir)
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.dirname(skill_dir)))
|
||||
|
||||
# Restart if requested
|
||||
if args.restart:
|
||||
print(f"Killing existing server on port {args.port}...")
|
||||
kill_server(args.port)
|
||||
time.sleep(1)
|
||||
|
||||
# Check if already running
|
||||
if is_server_ready(args.port):
|
||||
print(f"Server already running on port {args.port}")
|
||||
return 0
|
||||
|
||||
# Start server
|
||||
print(f"Starting LoRa Manager standalone server on port {args.port}...")
|
||||
cmd = [sys.executable, "standalone.py", "--port", str(args.port)]
|
||||
|
||||
# Start in background
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
cwd=project_root,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
start_new_session=True
|
||||
)
|
||||
|
||||
print(f"Server process started with PID {process.pid}")
|
||||
|
||||
# Wait for ready if requested
|
||||
if args.wait:
|
||||
print(f"Waiting for server to be ready (timeout: {args.timeout}s)...")
|
||||
if wait_for_server(args.port, args.timeout):
|
||||
print(f"Server ready at http://127.0.0.1:{args.port}/loras")
|
||||
return 0
|
||||
else:
|
||||
print(f"Timeout waiting for server")
|
||||
return 1
|
||||
|
||||
print(f"Server starting at http://127.0.0.1:{args.port}/loras")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -9,7 +9,10 @@ description: Inspect ComfyUI LoRA Manager runtime configuration and local diagno
|
||||
|
||||
- Treat runtime state as local user data. Prefer read-only inspection unless the user explicitly asks for mutation.
|
||||
- Never print secret-like settings values. Redact keys containing `key`, `token`, `secret`, `password`, `auth`, or `credential`, including `civitai_api_key`.
|
||||
- Resolve paths from the runtime configuration before guessing. In this environment the settings file is normally `/home/miao/.config/ComfyUI-LoRA-Manager/settings.json`, but portable settings can override this through the repository `settings.json`.
|
||||
- Resolve paths from the runtime configuration before guessing. Settings-directory precedence (highest first):
|
||||
1. **Explicit override** — env `LORA_MANAGER_SETTINGS_DIR` or standalone `--settings-path` (also accepted by the inspect script as `--settings-path DIR`). Pins EVERYTHING (`settings.json`, `cache/`, `wildcards/`, `backups/`, `logs/`, `stats/`) under the given directory; bypasses portable mode and the user config dir. Common when inspecting a sandboxed/E2E instance.
|
||||
2. **Portable** — repository `<repo-root>/settings.json` with `"use_portable_settings": true` (or `LORA_MANAGER_PORTABLE=1`): settings dir = `<repo-root>`.
|
||||
3. **Default** — `~/.config/ComfyUI-LoRA-Manager` on this machine (`platformdirs.user_config_dir("ComfyUI-LoRA-Manager", appauthor=False)`).
|
||||
- Use the active library when selecting per-library caches and paths. Read `active_library` from settings; fall back to `default` if missing.
|
||||
- Normalize and expand `~` before comparing paths. Symlinks are common in this repo.
|
||||
|
||||
@@ -32,9 +35,17 @@ python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_conte
|
||||
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py sqlite --db /path/to/cache.sqlite --limit 3
|
||||
```
|
||||
|
||||
To inspect a sandboxed/E2E instance that pins its settings directory:
|
||||
|
||||
```bash
|
||||
# --settings-path DIR (or LORA_MANAGER_SETTINGS_DIR) works with every subcommand:
|
||||
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py \
|
||||
--settings-path /tmp/opencode/<plan>-e2e/settings summary
|
||||
```
|
||||
|
||||
## Runtime Path Rules
|
||||
|
||||
- Settings directory: use `py/utils/settings_paths.py`. Default platform path is `platformdirs.user_config_dir("ComfyUI-LoRA-Manager", appauthor=False)`.
|
||||
- Settings directory: resolve via `py/utils/settings_paths.py` — `get_settings_dir()` honors the `LORA_MANAGER_SETTINGS_DIR` / programmatic override first, then portable mode, then `platformdirs.user_config_dir("ComfyUI-LoRA-Manager", appauthor=False)`. The inspect script mirrors this precedence in `resolve_settings_path()`.
|
||||
- Settings file: `<settings_dir>/settings.json`.
|
||||
- Cache root: `<settings_dir>/cache`.
|
||||
- Canonical cache files:
|
||||
|
||||
@@ -14,6 +14,7 @@ from typing import Any
|
||||
|
||||
SECRET_PATTERN = re.compile(r"(key|token|secret|password|auth|credential)", re.IGNORECASE)
|
||||
APP_NAME = "ComfyUI-LoRA-Manager"
|
||||
SETTINGS_DIR_ENV = "LORA_MANAGER_SETTINGS_DIR"
|
||||
CACHE_SQLITE = {
|
||||
"model": ("model", "{library}.sqlite"),
|
||||
"recipe": ("recipe", "{library}.sqlite"),
|
||||
@@ -30,6 +31,15 @@ CACHE_JSON = {
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Inspect LoRA Manager runtime state read-only.")
|
||||
parser.add_argument(
|
||||
"--settings-path",
|
||||
type=str,
|
||||
default=None,
|
||||
metavar="DIR",
|
||||
help="Explicit settings directory (same as LORA_MANAGER_SETTINGS_DIR / "
|
||||
"standalone --settings-path). Overrides portable mode and the default "
|
||||
"user config dir.",
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
subparsers.add_parser("summary", help="Print redacted settings and resolved paths.")
|
||||
@@ -44,6 +54,8 @@ def main() -> int:
|
||||
sqlite_parser.add_argument("--limit", type=int, default=3, help="Rows to sample from each user table.")
|
||||
|
||||
args = parser.parse_args()
|
||||
if args.settings_path:
|
||||
os.environ[SETTINGS_DIR_ENV] = args.settings_path
|
||||
context = build_context()
|
||||
|
||||
if args.command == "summary":
|
||||
@@ -78,6 +90,11 @@ def build_context() -> dict[str, Any]:
|
||||
|
||||
|
||||
def resolve_settings_path() -> Path:
|
||||
# Explicit override: LORA_MANAGER_SETTINGS_DIR env or --settings-path.
|
||||
explicit = os.environ.get(SETTINGS_DIR_ENV)
|
||||
if explicit:
|
||||
return Path(explicit).expanduser() / "settings.json"
|
||||
|
||||
repo_root = find_repo_root()
|
||||
portable = repo_root / "settings.json"
|
||||
if portable.exists():
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
# Recipe Batch Import Feature Design
|
||||
|
||||
## Overview
|
||||
Enable users to import multiple images as recipes in a single operation, rather than processing them individually. This feature addresses the need for efficient bulk recipe creation from existing image collections.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Frontend │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ BatchImportManager.js │
|
||||
│ ├── InputCollector (收集URL列表/目录路径) │
|
||||
│ ├── ConcurrencyController (自适应并发控制) │
|
||||
│ ├── ProgressTracker (进度追踪) │
|
||||
│ └── ResultAggregator (结果汇总) │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ batch_import_modal.html │
|
||||
│ └── 批量导入UI组件 │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ batch_import_progress.css │
|
||||
│ └── 进度显示样式 │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Backend │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ py/routes/handlers/recipe_handlers.py │
|
||||
│ ├── start_batch_import() - 启动批量导入 │
|
||||
│ ├── get_batch_import_progress() - 查询进度 │
|
||||
│ └── cancel_batch_import() - 取消导入 │
|
||||
├─────────────────────────────────────────────────────────────────┤
|
||||
│ py/services/batch_import_service.py │
|
||||
│ ├── 自适应并发执行 │
|
||||
│ ├── 结果汇总 │
|
||||
│ └── WebSocket进度广播 │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## API Endpoints
|
||||
|
||||
| 端点 | 方法 | 说明 |
|
||||
|------|------|------|
|
||||
| `/api/lm/recipes/batch-import/start` | POST | 启动批量导入,返回 operation_id |
|
||||
| `/api/lm/recipes/batch-import/progress` | GET | 查询进度状态 |
|
||||
| `/api/lm/recipes/batch-import/cancel` | POST | 取消导入 |
|
||||
|
||||
## Backend Implementation Details
|
||||
|
||||
### BatchImportService
|
||||
|
||||
Location: `py/services/batch_import_service.py`
|
||||
|
||||
Key classes:
|
||||
- `BatchImportItem`: Dataclass for individual import item
|
||||
- `BatchImportProgress`: Dataclass for tracking progress
|
||||
- `BatchImportService`: Main service class
|
||||
|
||||
Features:
|
||||
- Adaptive concurrency control (adjusts based on success/failure rate)
|
||||
- WebSocket progress broadcasting
|
||||
- Graceful error handling (individual failures don't stop the batch)
|
||||
- Result aggregation
|
||||
|
||||
### WebSocket Message Format
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "batch_import_progress",
|
||||
"operation_id": "xxx",
|
||||
"total": 50,
|
||||
"completed": 23,
|
||||
"success": 21,
|
||||
"failed": 2,
|
||||
"skipped": 0,
|
||||
"current_item": "image_024.png",
|
||||
"status": "running"
|
||||
}
|
||||
```
|
||||
|
||||
### Input Types
|
||||
|
||||
1. **URL List**: Array of URLs (http/https)
|
||||
2. **Local Paths**: Array of local file paths
|
||||
3. **Directory**: Path to directory with optional recursive flag
|
||||
|
||||
### Error Handling
|
||||
|
||||
- Invalid URLs/paths: Skip and record error
|
||||
- Download failures: Record error, continue
|
||||
- Metadata extraction failures: Mark as "no metadata"
|
||||
- Duplicate detection: Option to skip duplicates
|
||||
|
||||
## Frontend Implementation Details (TODO)
|
||||
|
||||
### UI Components
|
||||
|
||||
1. **BatchImportModal**: Main modal with tabs for URLs/Directory input
|
||||
2. **ProgressDisplay**: Real-time progress bar and status
|
||||
3. **ResultsSummary**: Final results with success/failure breakdown
|
||||
|
||||
### Adaptive Concurrency Controller
|
||||
|
||||
```javascript
|
||||
class AdaptiveConcurrencyController {
|
||||
constructor(options = {}) {
|
||||
this.minConcurrency = options.minConcurrency || 1;
|
||||
this.maxConcurrency = options.maxConcurrency || 5;
|
||||
this.currentConcurrency = options.initialConcurrency || 3;
|
||||
}
|
||||
|
||||
adjustConcurrency(taskDuration, success) {
|
||||
if (success && taskDuration < 1000 && this.currentConcurrency < this.maxConcurrency) {
|
||||
this.currentConcurrency = Math.min(this.currentConcurrency + 1, this.maxConcurrency);
|
||||
}
|
||||
if (!success || taskDuration > 10000) {
|
||||
this.currentConcurrency = Math.max(this.currentConcurrency - 1, this.minConcurrency);
|
||||
}
|
||||
return this.currentConcurrency;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
Backend (implemented):
|
||||
├── py/services/batch_import_service.py # 后端服务
|
||||
├── py/routes/handlers/batch_import_handler.py # API处理器 (added to recipe_handlers.py)
|
||||
├── tests/services/test_batch_import_service.py # 单元测试
|
||||
└── tests/routes/test_batch_import_routes.py # API集成测试
|
||||
|
||||
Frontend (TODO):
|
||||
├── static/js/managers/BatchImportManager.js # 主管理器
|
||||
├── static/js/managers/batch/ # 子模块
|
||||
│ ├── ConcurrencyController.js # 并发控制
|
||||
│ ├── ProgressTracker.js # 进度追踪
|
||||
│ └── ResultAggregator.js # 结果汇总
|
||||
├── static/css/components/batch-import-modal.css # 样式
|
||||
└── templates/components/batch_import_modal.html # Modal模板
|
||||
```
|
||||
|
||||
## Implementation Status
|
||||
|
||||
- [x] Backend BatchImportService
|
||||
- [x] Backend API handlers
|
||||
- [x] WebSocket progress broadcasting
|
||||
- [x] Unit tests
|
||||
- [x] Integration tests
|
||||
- [ ] Frontend BatchImportManager
|
||||
- [ ] Frontend UI components
|
||||
- [ ] E2E tests
|
||||
+16
-1
@@ -7,17 +7,25 @@ py/run_test.py
|
||||
.vscode/
|
||||
cache/
|
||||
civitai/
|
||||
stats/
|
||||
wildcards/
|
||||
backups/
|
||||
logs/
|
||||
node_modules/
|
||||
coverage/
|
||||
.coverage
|
||||
model_cache/
|
||||
|
||||
# agent
|
||||
# agent / dev tooling
|
||||
.opencode/
|
||||
.claude/
|
||||
.sisyphus/
|
||||
.codex
|
||||
.omo
|
||||
reasonix.toml
|
||||
.reasonix/
|
||||
.codegraph/
|
||||
.playwright-mcp/
|
||||
|
||||
# Vue widgets development cache (but keep build output)
|
||||
vue-widgets/node_modules/
|
||||
@@ -26,3 +34,10 @@ vue-widgets/dist/
|
||||
|
||||
# Hypothesis test cache
|
||||
.hypothesis/
|
||||
|
||||
# Working/research notes (not committed)
|
||||
.docs/
|
||||
|
||||
# HF enrichment validation baseline snapshots (contain potentially
|
||||
# NSFW README content fetched from community model repos)
|
||||
tests/enrich_hf_validation/baselines/
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
---
|
||||
slug: undo-delete-staging
|
||||
status: drafting
|
||||
intent: clear
|
||||
review_required: false
|
||||
pending-action: write .omo/plans/undo-delete-staging.md
|
||||
approach: "Option B: delayed physical deletion with Undo. Backend: same-volume rename to per-root staging dir (.lm-pending-delete/) [updated 2026-08: model staging moved to a SIBLING dir inside each deleted model's own folder — see 'Symlink fix (2026-08)' under Decisions] + manifest JSON (batch_id, expires_at, staged->original map) + purge (30s TTL timer + startup sweep + opportunistic) + undo-delete endpoint + settings toggle 'skip undo'. Small files (recipes: JSON+preview) copy to global staging under settings dir instead of rename. Frontend: extend toast system with action button + 30s countdown; delete flows (single model / recipe / bulk / duplicates) consume batch_id from delete response and show Undo toast; expired undo -> 'undo expired' toast. Plus confirm-modal friction (C-friction, NO type-to-confirm): delete button delay-activation 1.5s + modal shows file size 'will free X GB' + Cancel gets initial focus. i18n keys + sync_translation_keys.py."
|
||||
---
|
||||
|
||||
# Draft: undo-delete-staging
|
||||
|
||||
## Components (topology ledger)
|
||||
<!-- Lock the SHAPE before depth. One row per top-level component that can succeed or fail independently. -->
|
||||
<!-- id | outcome (one line) | status: active|deferred | evidence path -->
|
||||
- backend staging module (stage/purge/undo + manifest + per-volume dir resolution) | new module, active | pending exploration: model_lifecycle_service.py delete_model / delete_model_artifacts
|
||||
- delete endpoints return batch_id (model/recipe/bulk/duplicates) | active | pending exploration: handlers + response shapes
|
||||
- undo-delete HTTP endpoint + route registration | active | pending exploration: route registrar pattern
|
||||
- purge scheduling (30s timer + startup sweep + opportunistic) | active | pending exploration: app on_startup hooks
|
||||
- settings toggle "skip undo window" | active | pending exploration: settings service read pattern
|
||||
- frontend toast extension (action button + countdown) | active | pending exploration: showToast impl
|
||||
- frontend delete flows consume batch_id + Undo toast | active | pending exploration: call sites
|
||||
- confirm-modal friction (delay-activate + size display + cancel focus) | active | pending exploration: modal focus behavior
|
||||
- i18n keys + sync_translation_keys.py | active | known
|
||||
|
||||
## Open assumptions (announced defaults)
|
||||
<!-- Record any default you adopt instead of asking, so the user can veto it at the gate. -->
|
||||
<!-- assumption | adopted default | rationale | reversible? -->
|
||||
- Undo window TTL = 30s | 30s balances space-freeing intent vs accident recovery | yes (constant)
|
||||
- Staging dir name: `.lm-pending-delete/` under each model root; recipes: `{settings_dir}/.lm-pending-delete/` | hidden, same-volume [updated 2026-08: same-volume is now guaranteed by sibling staging inside the model's own folder, not by the root location], consistent | yes
|
||||
- Staging failure falls back to existing hard delete | user intent is delete; staging is best-effort; hard delete likely fails identically under same conditions | yes
|
||||
- Purge on startup uses expires_at (not purge-all) so a <30s restart with live tab can still undo | robust, matches client-side timer | yes
|
||||
- Settings toggle label: "Delete permanently immediately (skip undo window)" | power users freeing space | yes
|
||||
- C-friction: delete button enabled after 1.5s + modal shows freed size; NO type-to-confirm (user vetoed) | user explicitly rejected type-to-confirm | n/a
|
||||
- Bulk/duplicates delete: one batch id for whole action, one undo restores all | simplest consistent semantics | yes
|
||||
|
||||
## Findings (cited - path:lines)
|
||||
|
||||
### Backend
|
||||
- `delete_model_artifacts` (py/services/model_lifecycle_service.py:19-48) = physical delete via os.remove; patterns: main file + `{name}.metadata.json` + PREVIEW_EXTENSIONS (py/utils/constants.py:22-37). ALSO called by ModelScanner.bulk_delete_models (py/services/model_scanner.py:2221) - single swap point covers bulk models.
|
||||
- `ModelLifecycleService.delete_model` (model_lifecycle_service.py:101-154): fetches `cached_entry` (111-116) - SNAPSHOT available for cache restore; after delete: cache.raw_data removal + resort + bump_cache_version (136-143), `_hash_index.remove_by_path` (145-146), `_sync_update_for_model` (148; update-service only, no recipe JSON rewrites - recipe refs are hash-based, re-resolve on restore), `_persist_current_cache` (150-152), returns `{"success": True, "deleted_files": [...]}` (154).
|
||||
- Handler `delete_model` (py/routes/handlers/model_handlers.py:478-492): POST /api/lm/{prefix}/delete; response passthrough; `_broadcast_models_changed()` (57-74) after success; 400 `{"success":false,"error"}`; 500 plain text.
|
||||
- Recipe delete: handler (recipe_handlers.py:1422-1438) DELETE /api/lm/recipe/{recipe_id} -> persistence_service.delete_recipe (py/services/recipes/persistence_service.py:193-209): os.remove(recipe_json_path) + os.remove(image_path) (204-206), recipe_scanner.remove_recipe (208), returns `{"success": true, "message": ...}`. PersistenceResult dataclass (20-25).
|
||||
- Bulk models: POST /api/lm/{prefix}/bulk-delete (model_route_registrar.py:39) -> handler (model_handlers.py:974-994) -> lifecycle_service.bulk_delete_models (model_lifecycle_service.py:308-318) -> scanner.bulk_delete_models (model_scanner.py:2181-2269) which calls delete_model_artifacts per file (2221) + `_batch_update_cache_for_deleted_models` (2271-2335); response `{"success","status","total_deleted","total_attempted","cache_updated","results"}` (2254-2269).
|
||||
- Bulk recipes: POST /api/lm/recipes/bulk-delete (recipe_route_registrar.py:50) -> handler (recipe_handlers.py:1554-1573) -> persistence_service.bulk_delete (persistence_service.py:439-482): per-id os.remove x2 (464-466), recipe_scanner.bulk_remove (472); response `{"success","deleted","failed","total_deleted","total_failed"}` (474-482).
|
||||
- Duplicates: NO dedicated delete endpoints (find-only: GET /api/lm/{prefix}/find-duplicates model_route_registrar.py:59, GET /api/lm/recipes/find-duplicates recipe_route_registrar.py:49). Duplicate deletion reuses bulk-delete endpoints.
|
||||
- Startup hooks: lora_manager.py:183-187 `app.on_startup.append(lambda app: cls._initialize_services())` (ComfyUI mode, app = PromptServer.instance.app at :78); standalone.py:370-374 same (StandaloneLoraManager.add_routes). Background tasks: `asyncio.create_task(name=...)` (lora_manager.py:224-239; recipe_handlers.py:793). Singleton+asyncio.Lock pattern: model_scanner.py:40-63.
|
||||
- Settings: DEFAULT_SETTINGS (py/services/settings_manager.py:57-119), `get(key, default)` (1390-1392), get_settings_manager() (2215-2228), reset_settings_manager() (2231). Typed-bool getter example: get_skip_previously_downloaded_model_versions (1253-1262). Handlers: base_model_routes.py:70, base_recipe_routes.py:54.
|
||||
- Model roots: ModelScanner.get_model_roots base NotImplementedError (model_scanner.py:1073-1075); impls lora_scanner.py:31-45, checkpoint_scanner.py:428-441, embedding_scanner.py:24-36. `_find_root_for_file(file_path)` (model_scanner.py:1108-1124) returns containing root - for per-root staging dir computation [updated 2026-08: staging no longer uses the containing root; batches are siblings inside the model's own folder]. Business-path rule (AGENTS.md): use os.path.abspath, never realpath, for staging/undo routing.
|
||||
- Cache restore methods: ModelCache has raw_data + resort (conftest mocks: tests/conftest.py:144-154); ModelHashIndex.add_entry(sha256, file_path, autov3) (py/services/model_hash_index.py:16); RecipeScanner.add_recipe(recipe_data) (recipe_scanner.py:2136) -> recipe_cache.add_recipe (recipe_cache.py:64). No single-file incremental model rescan - use snapshot restore instead of rescan.
|
||||
- Route registrar: model_route_registrar.py:177 add_route(method, path, handler), :180 add_prefixed_route - undo endpoint can be a non-prefixed route via add_route.
|
||||
- Tests: tests/services/test_model_lifecycle_service.py (inline tmp_path files, per-test stub scanners ScannerForDelete/VersionAwareScanner etc); conftest MockScanner/MockCache/MockHashIndex (tests/conftest.py:134-212); integration fixtures tests/integration/conftest.py; lifecycle hook tests tests/routes/test_lora_manager_lifecycle.py:177-178, tests/standalone/test_standalone_server.py:83-84.
|
||||
|
||||
### Frontend
|
||||
- 5 delete call sites:
|
||||
a) Single model: static/js/utils/modalUtils.js confirmDelete (27-42) -> getModelApiClient().deleteModel(path); ignores return.
|
||||
b) Recipe single: static/js/components/RecipeCard.js confirmDeleteRecipe (405-449) - RAW fetch DELETE /api/lm/recipe/{id}, checks only response.ok, showToast toast.recipes.deletedSuccessfully, state.virtualScroller.removeItemByFilePath.
|
||||
c) Bulk: static/js/managers/BulkManager.js confirmBulkDelete (633-672) -> getActiveApiClient() (134-142) -> bulkDeleteModels(filePaths); reads result.cancelled/success/deleted_count/error.
|
||||
d) Recipe duplicates: static/js/components/DuplicatesManager.js confirmDeleteDuplicates (457-494) - RAW fetch POST /api/lm/recipes/bulk-delete, reads data.success/data.total_deleted, exitDuplicateMode().
|
||||
e) Model duplicates: static/js/components/ModelDuplicatesManager.js confirmDeleteDuplicates (710-776) - RAW fetch POST /api/lm/{type}/bulk-delete, reads data.total_deleted, then resetAndReload(true) + find-duplicates re-check.
|
||||
Bonus: static/js/components/shared/ModelVersionsTab.js:1136-1144 client.deleteModel (ignores return).
|
||||
- API clients: BaseModelApiClient.deleteModel (static/js/api/baseModelApi.js:184-216) returns true/false, shows its own toasts, does removeItemByFilePath inside; bulkDeleteModels (1591-1642) returns {success, deleted_count, failed_count, errors} or {success:false, cancelled:true}; RecipeSidebarApiClient.bulkDeleteModels (recipeApi.js:623-664) returns {success, deleted_count: total_deleted, ...}. Endpoint map apiConfig.js:56,64.
|
||||
- Toast: showToast(key, params={}, type='info', fallback=null) (static/js/utils/uiHelpers.js:136-193) - textContent only, NO action/button support; durations 2000/5000ms; CSS static/css/components/toast.css (.toast flex gap:12px - button can be added). Closest action pattern: bannerService.registerBanner actions array + onRegister (static/js/managers/BannerService.js; used uiHelpers.js:18-57).
|
||||
- i18n: locales/en.json delete keys (1303-1314 bulkDelete, 1945-1948 recipes, 1987-1991 models, 2124-2130 duplicates, 2166-2170 toast.api); t()/interpolate (static/js/i18n/index.js:193-248); translate wrapper (utils/i18nHelpers.js:13-23); sync script scripts/sync_translation_keys.py (en reference, [TODO: Translate] placeholders).
|
||||
- Refresh after undo: recipes -> window.recipeManager.loadRecipes(true) (recipes.js:359; used by FilterManager.js:752 etc) or refreshRecipes (recipeApi.js:308); models -> resetAndReload(true) from modelApiFactory (used by ModelDuplicatesManager.js:740).
|
||||
- Size for modal: card.dataset.file_size (ModelCard.js:467), formatFileSize (ModelModal.js:615).
|
||||
- Tests: tests/frontend/utils/uiHelpers.dom.test.js (toast), api/recipeApi.bulk.test.js, components/duplicatesManager.test.js, components/modelDuplicatesManager.test.js, pages/*Page.test.js, i18n tests tests/i18n/test_i18n.py.
|
||||
|
||||
## Decisions (with rationale)
|
||||
|
||||
1. Same-volume rename staging for model files (atomic, no copy cost for multi-GB files); cross-volume rename forbidden. [CORRECTED 2026-08: "same-volume because under the containing root" was only true for plain directories — nested symlinked subdirs could cross volumes. Superseded by sibling staging: `.lm-pending-delete/<batch_id>/` inside the deleted model's own folder makes stage/undo same-device by construction; see "Symlink fix (2026-08)" below.]
|
||||
2. Copy-to-global-staging for recipes (small files; avoids recipe JSON vs preview image cross-volume problem).
|
||||
3. Manifest JSON files are the only state - no DB changes. Manifest includes model cached_entry snapshot for exact cache restore (no rescan needed).
|
||||
4. Undo endpoint returns restored paths; expired batch -> 404-style error -> frontend 'undo expired' toast.
|
||||
5. Skip-undo setting honored server-side (no batch_id in response -> no undo toast client-side).
|
||||
6. Staging failure falls back to existing hard delete (best-effort undo, never blocks delete).
|
||||
7. Undo window TTL = 30s constant (PENDING_DELETE_TTL_SECONDS); startup sweep uses expires_at (survives restart; browser-tab timer survives).
|
||||
8. Purge triple-trigger: per-batch asyncio timer task + on_startup sweep + opportunistic purge at each stage/undo.
|
||||
9. Frontend: new showActionToast (keep showToast signature untouched; extract shared createToastElement/appendToast internals); undo click -> shared handleUndoDelete(batchId, refreshFn); full list refresh after undo (recipes: window.recipeManager.loadRecipes(true); models: resetAndReload(true)).
|
||||
10. C-friction wave (NO type-to-confirm - user vetoed): delete buttons delay-activate 1.5s after modal open, initial focus on Cancel, model delete modal gains "permanently deleted from disk" warning + file size display (card.dataset.file_size + formatFileSize).
|
||||
11. Model cache restore on undo: append snapshot to cache.raw_data (dedupe by file_path) + resort + bump_cache_version + _persist_current_cache + _hash_index.add_entry + _broadcast_models_changed. Recipe restore: copy back files + recipe_scanner.add_recipe(recipe_data loaded from restored JSON).
|
||||
|
||||
### Symlink fix (2026-08)
|
||||
|
||||
Post-execution addendum (plan `.omo/plans/undo-delete-symlink-fix.md`, commits 5fd4946b / 0c00ee22):
|
||||
|
||||
12. Model staging moved from `<model_root>/.lm-pending-delete/<batch_id>/` to `<model_dir>/.lm-pending-delete/<batch_id>/` (sibling of the model artifacts, inside the deleted model's own folder). Stage/undo renames are same-device BY CONSTRUCTION — EXDEV is impossible even when the business path traverses nested symlinks to other volumes (the decision-1 "containing root" guarantee covered only plain directories). EXDEV remains possible only for cross-volume merges, which keep the batch_ids-array fallback. Accepted edge: deleting the model's whole FOLDER during the 30s window destroys that batch (undo returns 404). Batch discovery uses an in-memory registry (`_known_batch_dirs`) with a startup reconciliation scan (`purge_expired(scan_roots=True)`) covering restarts and crash leftovers. Recipe batches unchanged (copy-based settings-dir staging with the `_restore_file` EXDEV fallback).
|
||||
|
||||
## Scope IN
|
||||
|
||||
- Model single delete (model_handlers delete_model / model_lifecycle_service)
|
||||
- Recipe delete (recipe_handlers delete_recipe / persistence_service)
|
||||
- Bulk delete (models scanner + recipes persistence) + duplicates (reuse bulk endpoints)
|
||||
- Undo endpoint POST /api/lm/undo-delete (models + recipes, one batch space)
|
||||
- Purge: timer + startup sweep + opportunistic
|
||||
- Settings toggle delete_undo_enabled + settings page checkbox
|
||||
- Frontend: showActionToast + all 5 delete flows + shared undo handler
|
||||
- C-friction modal changes (delay-activate + cancel focus + warning copy + size display)
|
||||
- i18n keys + sync_translation_keys.py
|
||||
- Backend + frontend tests
|
||||
|
||||
## Scope OUT (Must NOT have)
|
||||
|
||||
- NO type-to-confirm / hold-to-confirm friction (user vetoed)
|
||||
- NO OS trash integration (send2trash) in this iteration
|
||||
- NO persistent recycle-bin UI (no trash browsing page)
|
||||
- NO changes to exclude/unexclude flow
|
||||
- NO DB migrations
|
||||
- NO new dependencies (no send2trash)
|
||||
- NO changes to download flows
|
||||
- NO recipe-JSON rewriting on model undo (hash-based refs re-resolve themselves)
|
||||
|
||||
## Open questions
|
||||
|
||||
None - all implementation details resolved by exploration. Design decisions settled in conversation (B+C, no type-to-confirm).
|
||||
|
||||
## Approval gate
|
||||
status: approved
|
||||
<!-- Approach approved -> rerun scaffold without --draft-only, run Metis gap analysis, APPEND todo batches, fill TL;DR last, run structural self-check, then Phase 4 handoff. -->
|
||||
|
||||
## Review round state (ulw-plan-review-round-state-contract)
|
||||
```json
|
||||
{
|
||||
"transition": "replace",
|
||||
"phase": "review_round_initialized",
|
||||
"applies_when": ["retry_after_plan_change"],
|
||||
"atomic": true,
|
||||
"review_required": true,
|
||||
"plan_path": ".omo/plans/undo-delete-staging.md",
|
||||
"plan_sha256": "8cf7c9be38a76d8ef1fb832aba043d6d7e82b60465b1bf28c6eafa7045117adc",
|
||||
"review_round_id": "rr-undo-del-20260811-006",
|
||||
"round_status": "active",
|
||||
"pending-action": "review .omo/plans/undo-delete-staging.md",
|
||||
"review": {
|
||||
"momus": { "status": "pending", "workspace_root": "/mnt/data/reinstall-backup-2026-04-12/data/workspace/ComfyUI/custom_nodes/ComfyUI-Lora-Manager", "runtime_home": null, "target": ".omo/plans/undo-delete-staging.md", "round_id": "rr-undo-del-20260811-006", "plan_sha256": "8cf7c9be38a76d8ef1fb832aba043d6d7e82b60465b1bf28c6eafa7045117adc", "launch_id": null, "session": null, "result": null },
|
||||
"independent": { "status": "pending", "workspace_root": "/mnt/data/reinstall-backup-2026-04-12/data/workspace/ComfyUI/custom_nodes/ComfyUI-Lora-Manager", "runtime_home": null, "target": ".omo/plans/undo-delete-staging.md", "round_id": "rr-undo-del-20260811-006", "plan_sha256": "8cf7c9be38a76d8ef1fb832aba043d6d7e82b60465b1bf28c6eafa7045117adc", "launch_id": null, "session": null, "result": null }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Review results + fix/retry ledger
|
||||
|
||||
### Round 1 (rr-undo-del-20260811-001, plan sha256 6c52bf99...)
|
||||
- momus: APPROVE (non-blocking notes: todo1+7 duplicate DEFAULT_SETTINGS key -> fixed todo 7 to verify-only; "batch_ids" plural in todos 8/9 acceptance -> fixed; purge OSError note -> folded into todo 1 purge semantics)
|
||||
- independent (oracle): CHANGES_REQUESTED
|
||||
- BLOCKING S1: scanner walks would index .lm-pending-delete staged files as ghost entries -> fixed: todo 1 now mandates scanner walk exclusion at model_scanner.py:706/:867/:1404/_process_model_file + acceptance (o) scanner-visibility test
|
||||
- BLOCKING S2: manifest lacks model_type, undo could restore into wrong cache/hash index -> fixed: manifest now carries model_type + todo 5 resolves per-type scanner via registrar pattern + acceptance (b) checkpoint-batch test
|
||||
- S3 merged-batch expires_at re-anchor -> fixed: merge_batches re-anchors now+TTL in todo 1 + todo 3/4 assertions
|
||||
- S4 manifest-less dir policy -> fixed: quarantine to <batch_id>.orphaned, never delete (todo 1 + acceptance g)
|
||||
- S5 partial-undo retry semantics -> fixed: per-entry restored flag write-through + retry test (acceptance e)
|
||||
- S6 purge locked-file failure semantics -> fixed: skip file, keep batch, never rmtree past errors (todo 1 + acceptance i)
|
||||
- T8 undo-after-restart test -> fixed: todo 5 acceptance (f)
|
||||
- T9 recipe undo -> re-delete test -> fixed: todo 5 acceptance (h)
|
||||
- T7 rescan-stale-entry test -> fixed: todo 5 acceptance (g)
|
||||
- Route registration pinned to shared routes class per mode (NOT per-model-type registrar which registers 3x) -> fixed: todo 5 now creates py/routes/pending_delete_routes.py registered once in lora_manager.py:170-172 + standalone.py:356-358 + duplicate-route test (e)
|
||||
- Version-index staleness on single-delete undo -> fixed: todo 5 follows bulk cache-update pattern incl. rebuild_version_index (model_scanner.py:2324)
|
||||
- Cancelled-bulk batch_id frontend handling -> fixed: todo 9 shows action toast on cancelled+staged-subset
|
||||
- Single-instance assumption -> added to Scope OUT
|
||||
- Occupied-refusal loss UX -> accepted-intent documented in success criteria + modal copy
|
||||
|
||||
### Round 2 (rr-undo-del-20260811-002, plan sha256 f3d52235...)
|
||||
- momus: APPROVE (all 12 round-1 fixes verified present; zero dead references; non-blocking nits only)
|
||||
- independent (oracle): CHANGES_REQUESTED
|
||||
- BLOCK-1: merge_batches file-movement semantics unspecified (silent data-loss vector) -> fixed: todo 1 now specifies move-into-winner-dir + entry re-point + loser-dirs-removed-only-when-empty + abort-on-move-failure (all batches intact) + merge inside service lock + acceptance (k) file-survival assertions + acceptance (l) merge-failure abort test
|
||||
- BLOCK-2: same-file parallel edits within waves (todo 5 vs 6 on lora_manager.py; todo 8 vs 9 on baseModelApi.js) -> fixed: waves/matrix now serialize 5->6 and 8->9 with explicit reasons; matrix updated
|
||||
- Recommended: checkpoint_scanner.py:331 exclusion -> fixed (todo 1 + acceptance p); S5 pre-check skips restored:true entries -> fixed (todo 1); _tags_count restore on undo -> fixed (todo 5 + acceptance j); undo-blind flows documented (ModelVersionsTab + misc_handlers:2456) -> fixed (todo 8 note + Scope OUT); merge-failure no-merge fallback contract (batch_ids array) -> fixed (todos 3/4/9)
|
||||
|
||||
### Round 3 (rr-undo-del-20260811-003, plan sha256 8f2dfd46...)
|
||||
- momus: APPROVE (all round-2 fixes verified present + spot-checked refs; no new contradictions)
|
||||
- independent (oracle): CHANGES_REQUESTED
|
||||
- BLOCKING A: merged batches never timer-purged after re-anchor (winner's original timer no-ops at old expiry; no fresh timer for re-anchored expiry; idle server -> merged batch lingers, violating "30s purge" success criterion; affects EVERY bulk delete) -> fixed: todo 1 merge_batches now ARMS A FRESH PURGE TIMER for the winner with re-anchored expiry + acceptance (q) fresh-timer test + purge_expired must enumerate ALL scanner types' roots (explicit in todo 1)
|
||||
- BLOCKING B: dependency matrix contradicted same-file policy for todos 8/9<->11 (5 shared files) and 12<->11 -> fixed: todo 11 now "Blocked by: 8, 9 (same files...)"; todo 12 blocked by 11 (sync after 11); wave text updated (11, then 12 AFTER 11); "Can parallelize with" columns corrected
|
||||
- BLOCKING C: frontend batch_ids sequential-undo fallback has NO test + merge->undo loser-restore + merge->purge assertions missing -> fixed: todo 9 acceptance now tests the batch_ids fallback path; todo 1 acceptance now has (k2)/(k3)
|
||||
- Notes folded: sub-second toast-tail expiry race accepted; EXDEV fallback = NORMAL path for cross-volume bulks [annotated 2026-08: after the sibling-staging fix, EXDEV can only arise during cross-volume MERGES, never during single stage/undo renames]
|
||||
|
||||
### Round 4 (rr-undo-del-20260811-004, plan sha256 179e7ff7...)
|
||||
- momus: APPROVE (round-3 fixes verified; one non-blocking nit: todo 11 inline "Blocked by: —" stale -> fixed to "8, 9")
|
||||
- independent (oracle): CHANGES_REQUESTED
|
||||
- BLOCKING GAP-1 (NEW, introduced by round-3 fix): todo 8 handleUndoDelete always-refresh/always-toast contract contradicted todo 9's sequential loop "exactly ONE final refresh" -> fixed: handleUndoDelete(batchId, refreshFn, {showToast, refresh}) suppression options; todo 9 loop uses suppressed calls + one final refresh/toast; acceptance extended (loop failure mid-way -> stop + error toast + no final refresh; 404 body discrimination expired vs occupied)
|
||||
- BLOCKING GAP-2: no cross-type purge enumeration test -> fixed: todo 1 acceptance (r) purges expired batches across lora root + checkpoint root + recipe staging dir in one call
|
||||
- Non-blocking folded: GAP-3 404-copy discrimination -> fixed in todo 8 (d); GAP-4 merge partial-failure rollback direction (move back + restore manifests, extended (l) asserts sequential constituent undo still restores everything) -> fixed in todo 1; GAP-5 post-restart timer-loss residual gap documented -> fixed in todo 6; GAP-6 usage_stats.py:424 walk added to exclusion mandate + todo 5 acceptance (k) embeddings undo test
|
||||
|
||||
### Round 5 (rr-undo-del-20260811-005, plan sha256 dfaa39ea...)
|
||||
- momus: APPROVE (all round-4 fixes verified; no new contradictions)
|
||||
- independent (oracle): CHANGES_REQUESTED
|
||||
- BLOCK-1: lock-ordering deadlock ambiguity (asyncio.Lock not re-entrant: opportunistic purge_expired called while stage/undo hold the lock would deadlock on first use) -> fixed: todo 1 now has explicit LOCK HIERARCHY (lock acquired ONLY by stage/merge/undo/purge_batch; purge_expired is lock-free and must be called BEFORE lock acquisition); todo 6 (c) updated with the same rule + acceptance (u) lock-no-deadlock test
|
||||
- BLOCK-2: purge edge semantics unspecified -> fixed: purge_batch treats missing staged files (partially-restored batches) as already-purged (FileNotFoundError silent no-op); sweep skips `.orphaned`-suffixed dirs (quarantine is terminal); acceptance (s) partially-restored purge + (t) quarantine-terminal tests
|
||||
- Non-blocking folded: todo 2/3 test-file collision -> todo 3's bulk tests moved to tests/services/test_model_scanner.py; todo 9 (d) DuplicatesManager refreshFn stated explicitly (recipes loadRecipes / models resetAndReload); modal-copy + bulk-count trade-offs acknowledged in success criteria; acceptance (r) extended with embeddings root
|
||||
|
||||
### Round 6 (rr-undo-del-20260811-006, plan sha256 8cf7c9be...)
|
||||
- momus: APPROVE (all round-5 fixes verified; no new contradictions; references verified)
|
||||
- independent (oracle): APPROVE — no blocking issues; all round-5 items fixed with working, tested solutions; no new race/data-loss/consistency defects
|
||||
- Deferred optional improvements (non-blocking, recorded for executor awareness; plan file left untouched to preserve the approved digest):
|
||||
1. Tag-count asymmetry: single delete_model never decrements _tags_count (lifecycle 101-154), bulk does (scanner 2297-2303); undo re-increment is exact for bulk, over-counts for single until rescan (cosmetic, self-healing). Optional fix riding in todo 2: decrement tags in the single-delete path to mirror bulk.
|
||||
2. Todo 5 factual nit: ModelCache.resort() already rebuilds the version index — explicit rebuild in undo is belt-and-braces, no action needed.
|
||||
3. Todo 8 premise nit: ModelVersionsTab call ignores deleteModel's return entirely — nothing breaks, no adaptation needed.
|
||||
4. Todo 3's pytest command includes test_model_lifecycle_service.py which todo 2 edits in the same wave — run that file's tests after todo 2 lands.
|
||||
5. merge_batches with a missing/quarantined constituent id: any sane fallback (abort -> batch_ids, or skip missing) acceptable — files stay staged either way.
|
||||
|
||||
## Review lifecycle
|
||||
- rounds: 6 (rr-undo-del-20260811-001..006); final round both lanes APPROVE
|
||||
- final live-plan validation: sha256 = 8cf7c9be38a76d8ef1fb832aba043d6d7e82b60465b1bf28c6eafa7045117adc — MATCHES approved round-6 digest
|
||||
- status: APPROVED — ready for execution handoff ($start-work undo-delete-staging)
|
||||
File diff suppressed because one or more lines are too long
@@ -2,6 +2,10 @@
|
||||
|
||||
This file provides guidance for agentic coding assistants working in this repository.
|
||||
|
||||
## Overview
|
||||
|
||||
ComfyUI LoRA Manager is a comprehensive LoRA management system for ComfyUI that combines a Python backend with browser-based widgets. It provides model organization, downloading from CivitAI/CivArchive, recipe management, and one-click workflow integration.
|
||||
|
||||
## Development Commands
|
||||
|
||||
### Backend Development
|
||||
@@ -28,16 +32,21 @@ COVERAGE_FILE=coverage/backend/.coverage pytest \
|
||||
--cov=py --cov=standalone \
|
||||
--cov-report=term-missing \
|
||||
--cov-report=html:coverage/backend/html \
|
||||
--cov-report=xml:coverage/backend/coverage.xml
|
||||
--cov-report=xml:coverage/backend/coverage.xml \
|
||||
--cov-report=json:coverage/backend/coverage.json
|
||||
```
|
||||
|
||||
### Frontend Development (Standalone Web UI)
|
||||
### Frontend Development (LoRA Manager Web UI)
|
||||
|
||||
```bash
|
||||
# Install dependencies (root and Vue widgets)
|
||||
npm install
|
||||
cd vue-widgets && npm install && cd ..
|
||||
|
||||
npm test # Run all tests (JS + Vue)
|
||||
npm run test:js # Run JS tests only
|
||||
npm run test:watch # Watch mode
|
||||
npm run test:vue # Run Vue widget tests only
|
||||
npm run test:watch # Watch mode (JS tests only)
|
||||
npm run test:coverage # Generate coverage report
|
||||
```
|
||||
|
||||
@@ -54,105 +63,206 @@ npm run test:watch # Watch mode
|
||||
npm run test:coverage # Generate coverage report
|
||||
```
|
||||
|
||||
## Python Code Style
|
||||
### Localization
|
||||
|
||||
### Imports & Formatting
|
||||
```bash
|
||||
# Sync translation keys after UI string updates
|
||||
python scripts/sync_translation_keys.py
|
||||
```
|
||||
|
||||
Locale files are in `locales/` (en, zh-CN, zh-TW, ja, ko, fr, de, es, ru, he).
|
||||
|
||||
After adding keys to `en.json` and syncing, **stop**: the `[TODO: Translate]` placeholders in
|
||||
the other locales are the expected end state during feature development. Do NOT translate
|
||||
proactively — translate only when the feature owner explicitly asks (see
|
||||
`docs/i18n-translation-guidelines.md` §7).
|
||||
|
||||
**Before translating anything, read `docs/i18n-translation-guidelines.md`** — it defines the
|
||||
term conventions (e.g. "Recipe" stays untranslated in French, 配方 in Chinese; model-type and
|
||||
brand names are never translated), per-locale preferred renderings, placeholder rules, and
|
||||
the known confusion hot-spots.
|
||||
|
||||
## Code Style
|
||||
|
||||
### Python
|
||||
|
||||
#### Imports & Formatting
|
||||
|
||||
- Use `from __future__ import annotations` for forward references
|
||||
- Group imports: standard library, third-party, local (blank line separated)
|
||||
- Use `TYPE_CHECKING` guard for type-checking-only imports
|
||||
- Absolute imports within `py/`: `from ..services import X`
|
||||
- PEP 8 with 4-space indentation, type hints required
|
||||
|
||||
### Naming Conventions
|
||||
#### Naming Conventions
|
||||
|
||||
- Files: `snake_case.py`, Classes: `PascalCase`, Functions/vars: `snake_case`
|
||||
- Constants: `UPPER_SNAKE_CASE`, Private: `_protected`, `__mangled`
|
||||
|
||||
### Error Handling & Async
|
||||
#### Error Handling & Async
|
||||
|
||||
- Use `logging.getLogger(__name__)`, define custom exceptions in `py/services/errors.py`
|
||||
- `async def` for I/O, `@pytest.mark.asyncio` for async tests
|
||||
- Singleton with `asyncio.Lock`: see `ModelScanner.get_instance()`
|
||||
- Return `aiohttp.web.json_response` or `web.Response`
|
||||
|
||||
### Testing
|
||||
### JavaScript/TypeScript
|
||||
|
||||
- `pytest` with `--import-mode=importlib`
|
||||
- Fixtures in `tests/conftest.py`, use `tmp_path_factory` for isolation
|
||||
- Mark tests needing real paths: `@pytest.mark.no_settings_dir_isolation`
|
||||
- Mock ComfyUI dependencies via conftest patterns
|
||||
|
||||
## JavaScript/TypeScript Code Style
|
||||
|
||||
### Imports & Modules
|
||||
#### Imports & Modules
|
||||
|
||||
- ES modules: `import { app } from "../../scripts/app.js"` for ComfyUI
|
||||
- Vue: `import { ref, computed } from 'vue'`, type imports: `import type { Foo }`
|
||||
- Export named functions: `export function foo() {}`
|
||||
|
||||
### Naming & Formatting
|
||||
#### Naming & Formatting
|
||||
|
||||
- camelCase for functions/vars/props, PascalCase for classes
|
||||
- Constants: `UPPER_SNAKE_CASE`, Files: `snake_case.js` or `kebab-case.js`
|
||||
- 2-space indentation preferred (follow existing file conventions)
|
||||
- Vue Single File Components: `<script setup lang="ts">` preferred
|
||||
|
||||
### Widget Development
|
||||
#### Widget Development
|
||||
|
||||
- Prefer vanilla JS for `web/comfyui/` widgets; avoid framework dependencies (except the Vue widgets in `vue-widgets/`)
|
||||
- ComfyUI: `app.registerExtension()`, `node.addDOMWidget(name, type, element, options)`
|
||||
- Event handlers via `addEventListener` or widget callbacks
|
||||
- Shared utilities: `web/comfyui/utils.js`
|
||||
- Dual-mode rendering patterns (canvas vs Vue): see `docs/comfyui-dual-mode-widgets.md`
|
||||
|
||||
### Vue Composables Pattern
|
||||
#### Vue Composables Pattern
|
||||
|
||||
- Use composition API: `useXxxState(widget)`, return reactive refs and methods
|
||||
- Guard restoration loops with flag: `let isRestoring = false`
|
||||
- Build config from state: `const buildConfig = (): Config => { ... }`
|
||||
|
||||
## Architecture Patterns
|
||||
## Architecture
|
||||
|
||||
### Dual Mode Operation
|
||||
|
||||
The system runs in two modes:
|
||||
- **ComfyUI plugin mode**: Integrates with ComfyUI's PromptServer, uses `folder_paths` for model discovery
|
||||
- **Standalone mode**: `standalone.py` mocks ComfyUI dependencies, reads paths from `settings.json`
|
||||
- Detection: `os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"`
|
||||
|
||||
### Backend Entry Points
|
||||
|
||||
- `__init__.py` — ComfyUI plugin entry: registers nodes via `NODE_CLASS_MAPPINGS`, sets `WEB_DIRECTORY`, calls `LoraManager.add_routes()`
|
||||
- `standalone.py` — Standalone server: mocks `folder_paths` and node modules, starts aiohttp server
|
||||
- `py/lora_manager.py` — Main `LoraManager` class that registers all HTTP routes
|
||||
|
||||
### Service Layer
|
||||
|
||||
- `ServiceRegistry` singleton for DI, services use `get_instance()` classmethod
|
||||
- `BaseModelService` abstract base → `LoraService`, `CheckpointService`, `EmbeddingService`
|
||||
- `ModelScanner` base → `LoraScanner`, `CheckpointScanner`, `EmbeddingScanner` for file discovery with hash-based deduplication
|
||||
- `PersistentModelCache` (SQLite) for metadata persistence
|
||||
- `MetadataSyncService` — background sync from CivitAI/CivArchive APIs
|
||||
- `SettingsManager` — settings with schema migration support
|
||||
- `WebSocketManager` — real-time progress broadcasting
|
||||
- `ModelServiceFactory` — creates the right service for each model type
|
||||
- Use cases in `py/services/use_cases/` orchestrate complex business logic (auto-organize, bulk refresh, downloads)
|
||||
- Separate scanners (discovery) from services (business logic)
|
||||
- Handlers in `py/routes/handlers/` are pure functions with deps as params
|
||||
|
||||
### Model Types & Routes
|
||||
|
||||
- `BaseModelService` base for LoRA, Checkpoint, Embedding
|
||||
- `ModelScanner` for file discovery, hash deduplication
|
||||
- `PersistentModelCache` (SQLite) for persistence
|
||||
- Route registrars: `ModelRouteRegistrar`, endpoints: `/loras/*`, `/checkpoints/*`, `/embeddings/*`
|
||||
- WebSocket via `WebSocketManager` for real-time updates
|
||||
- API endpoints follow `/loras/*`, `/checkpoints/*`, `/embeddings/*`, `/other/*` patterns
|
||||
- Route registrars organize endpoints by domain: `ModelRouteRegistrar`, `RecipeRouteRegistrar`, etc.
|
||||
- Request handlers in `py/routes/handlers/` implement route logic
|
||||
- All routes use aiohttp, return `web.json_response` or `web.Response`
|
||||
- Endpoints consumed by the companion browser extension (lm-civitai-extension)
|
||||
MUST also accept `GET` with query-string params: the extension is GET-only by
|
||||
convention (see its AGENTS.md), even for state-changing operations such as
|
||||
`GET /api/lm/recipe/{recipe_id}/reimport`
|
||||
|
||||
### Recipe System
|
||||
|
||||
- Base: `py/recipes/base.py`, Enrichment: `RecipeEnrichmentService`
|
||||
- Parsers: `py/recipes/parsers/`
|
||||
- Base: `py/recipes/base.py`, Enrichment: `RecipeEnrichmentService` in `py/recipes/enrichment.py`
|
||||
- Parsers: `py/recipes/parsers/` for PNG metadata, JSON, and workflow formats
|
||||
|
||||
### Custom Nodes
|
||||
|
||||
- Location: `py/nodes/`, all nodes registered in `__init__.py`
|
||||
- Each node class has a `NAME` class attribute used as key in `NODE_CLASS_MAPPINGS`
|
||||
- Standard ComfyUI node pattern: `INPUT_TYPES()` classmethod, `RETURN_TYPES`, `FUNCTION`
|
||||
|
||||
### Configuration
|
||||
|
||||
- `py/config.py` manages folder paths for models and handles symlink mappings
|
||||
- Auto-saves paths to `settings.json` in ComfyUI mode
|
||||
- `settings.json.example` is intentionally minimal (see Important Notes); all
|
||||
other defaults live in `DEFAULT_SETTINGS` (`py/services/settings_manager.py`)
|
||||
|
||||
### Frontend UI Architecture
|
||||
|
||||
#### 1. LoRA Manager Web UI
|
||||
- Location: `./static/` (JS/CSS) and `./templates/` (HTML)
|
||||
- Tech: Vanilla JS + CSS, served by the hosting server (ComfyUI app in plugin mode, `standalone.py` in standalone mode)
|
||||
- Tests: `tests/frontend/**/*.test.js` (vitest + jsdom)
|
||||
|
||||
#### 2. ComfyUI Custom Node Widgets
|
||||
- Location: `./web/comfyui/` (Vanilla JS) + `./vue-widgets/` (Vue)
|
||||
- Primary styles: `./web/comfyui/lm_styles.css` (NOT `./static/css/`)
|
||||
- Vue widgets: Vue 3 + TypeScript + PrimeVue + vue-i18n, e.g. `LoraPoolWidget`, `LoraRandomizerWidget`, `LoraCyclerWidget`, `AutocompleteTextWidget`
|
||||
- Vue builds to `./web/comfyui/vue-widgets/`; auto-built on ComfyUI startup via `py/vue_widget_builder.py`, typecheck via `vue-tsc`
|
||||
- Widget registration: `app.registerExtension()` and `getCustomWidgets` hooks; `node.addDOMWidget(...)` embeds HTML in LiteGraph nodes
|
||||
- See `docs/dom_widget_dev_guide.md` for the DOMWidget development guide
|
||||
|
||||
## Testing
|
||||
|
||||
### Backend (pytest)
|
||||
|
||||
- Config in `pytest.ini`: `--import-mode=importlib`, testpaths=`tests`
|
||||
- Fixtures in `tests/conftest.py` mock ComfyUI dependencies; use `tmp_path_factory` for isolation
|
||||
- Markers: `@pytest.mark.asyncio`, `@pytest.mark.no_settings_dir_isolation` (tests needing real settings paths)
|
||||
|
||||
### Frontend (vitest)
|
||||
|
||||
- Vanilla JS tests: `tests/frontend/**/*.test.js` with jsdom; setup in `tests/frontend/setup.js`
|
||||
- Vue widget tests: `vue-widgets/tests/**/*.test.ts` with jsdom + `@vue/test-utils`
|
||||
|
||||
### UI Verification (manual default)
|
||||
|
||||
UI/layout changes are verified by the user by eye — do NOT spin up a sandbox,
|
||||
standalone server, or browser automation to "prove" a visual fix. Ask the user to
|
||||
look instead. The full browser E2E ceremony (server + Chrome DevTools MCP +
|
||||
screenshots) is slow, token-heavy, and fragile; reserve it for genuine
|
||||
server+browser integration bugs, and only when the user explicitly agrees.
|
||||
|
||||
If a cross-layer issue ever needs a live server, the sandboxed helpers live in
|
||||
`scripts/e2e/` (`start_server.py`, `wait_for_server.py`). Non-negotiable rules:
|
||||
|
||||
- Always launch with `--settings-path <sandbox>/settings` and sandboxed
|
||||
`folder_paths` under `/tmp` — the repo folder is the real plugin folder and a
|
||||
`settings.json` there is read by the live instance. Never touch real config or
|
||||
real model libraries.
|
||||
- Never kill a process you did not start; `start_server.py` tracks its own PIDs
|
||||
via pidfile and refuses to touch unrelated processes on the port.
|
||||
- Abort after ~30 minutes or 3 consecutive tool failures; report `BLOCKED` with
|
||||
observed state instead of retrying blindly. Clean up sandbox and server after.
|
||||
|
||||
## Key Integration Points
|
||||
|
||||
- **Settings:** Stored in the user config directory (via `platformdirs`) or portable mode (`"use_portable_settings": true`)
|
||||
- **CivitAI/CivArchive:** API clients for metadata sync and model downloads; CivitAI API key stored in settings
|
||||
- **Symlinks:** Config scans symlinks to map virtual→physical paths; fingerprinting prevents redundant rescans
|
||||
- **WebSocket:** Broadcasts real-time progress for downloads, scans, and metadata sync
|
||||
- **Model scanning flow:** Walk folders → compute hashes → deduplicate → extract safetensors metadata → cache in SQLite → background CivitAI sync → WebSocket broadcast
|
||||
|
||||
## Important Notes
|
||||
|
||||
- ALWAYS use English for comments (per copilot-instructions.md)
|
||||
- Dual mode: ComfyUI plugin (folder_paths) vs standalone (settings.json)
|
||||
- Detection: `os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"`
|
||||
- **`settings.json.example` must stay minimal**: only `use_portable_settings`,
|
||||
`civitai_api_key`, and the four core `folder_paths` keys (`loras`,
|
||||
`checkpoints`, `unet`, `embeddings`). Do NOT add optional/default keys
|
||||
(model-category folders, `default_*_root`, `auto_organize_exclusions`, etc.)
|
||||
to this file unless the user explicitly asks for it. Defaults belong in
|
||||
`DEFAULT_SETTINGS` in `py/services/settings_manager.py`.
|
||||
- Run `python scripts/sync_translation_keys.py` after adding UI strings to `locales/en.json`
|
||||
- Symlinks require normalized paths
|
||||
|
||||
## Git / Commit Messages
|
||||
|
||||
- Follow the style of recent repository commits when writing commit messages
|
||||
- Prefer the repo's existing `feat(...)`, `fix(...)`, `chore:` style where applicable
|
||||
- If the user has provided a GitHub issue link or issue ID for the task, mention that issue in the commit message, for example `(#871)`
|
||||
- When unrelated local changes exist, stage and commit only the files relevant to the requested task
|
||||
|
||||
## Frontend UI Architecture
|
||||
|
||||
### 1. Standalone Web UI
|
||||
- Location: `./static/` and `./templates/`
|
||||
- Tech: Vanilla JS + CSS, served by standalone server
|
||||
- Tests via npm in root directory
|
||||
|
||||
### 2. ComfyUI Custom Node Widgets
|
||||
- Location: `./web/comfyui/` (Vanilla JS) + `./vue-widgets/` (Vue)
|
||||
- Primary styles: `./web/comfyui/lm_styles.css` (NOT `./static/css/`)
|
||||
- Vue builds to `./web/comfyui/vue-widgets/`, typecheck via `vue-tsc`
|
||||
- Symlinks require normalized paths.
|
||||
**Business paths vs real paths**: All stored paths and operation routing use the
|
||||
original paths as they appear under configured model roots — symlinks are NOT
|
||||
resolved. `os.path.realpath` is only for scanner dedup and the symlink cache.
|
||||
Any path passed to `os.remove`/`os.rename`/`shutil.move` or validated by a
|
||||
containment check MUST use the business path (i.e. `os.path.abspath`, not
|
||||
`realpath`).
|
||||
@@ -1,189 +0,0 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Overview
|
||||
|
||||
ComfyUI LoRA Manager is a comprehensive LoRA management system for ComfyUI that combines a Python backend with browser-based widgets. It provides model organization, downloading from CivitAI/CivArchive, recipe management, and one-click workflow integration.
|
||||
|
||||
## Development Commands
|
||||
|
||||
### Backend
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
pip install -r requirements-dev.txt
|
||||
|
||||
# Run standalone server (port 8188 by default)
|
||||
python standalone.py --port 8188
|
||||
|
||||
# Run all backend tests
|
||||
pytest
|
||||
|
||||
# Run specific test file or function
|
||||
pytest tests/test_recipes.py
|
||||
pytest tests/test_recipes.py::test_function_name
|
||||
|
||||
# Run backend tests with coverage
|
||||
COVERAGE_FILE=coverage/backend/.coverage pytest \
|
||||
--cov=py \
|
||||
--cov=standalone \
|
||||
--cov-report=term-missing \
|
||||
--cov-report=html:coverage/backend/html \
|
||||
--cov-report=xml:coverage/backend/coverage.xml \
|
||||
--cov-report=json:coverage/backend/coverage.json
|
||||
```
|
||||
|
||||
### Frontend
|
||||
|
||||
There are three test suites run by `npm test`: vanilla JS tests (vitest at root) and Vue widget tests (`vue-widgets/` vitest).
|
||||
|
||||
```bash
|
||||
npm install
|
||||
cd vue-widgets && npm install && cd ..
|
||||
|
||||
# Run all frontend tests (JS + Vue)
|
||||
npm test
|
||||
|
||||
# Run only vanilla JS tests
|
||||
npm run test:js
|
||||
|
||||
# Run only Vue widget tests
|
||||
npm run test:vue
|
||||
|
||||
# Watch mode (JS tests only)
|
||||
npm run test:watch
|
||||
|
||||
# Frontend coverage
|
||||
npm run test:coverage
|
||||
|
||||
# Build Vue widgets (output to web/comfyui/vue-widgets/)
|
||||
cd vue-widgets && npm run build
|
||||
|
||||
# Vue widget dev mode (watch + rebuild)
|
||||
cd vue-widgets && npm run dev
|
||||
|
||||
# Typecheck Vue widgets
|
||||
cd vue-widgets && npm run typecheck
|
||||
```
|
||||
|
||||
### Localization
|
||||
|
||||
```bash
|
||||
# Sync translation keys after UI string updates
|
||||
python scripts/sync_translation_keys.py
|
||||
```
|
||||
|
||||
Locale files are in `locales/` (en, zh-CN, zh-TW, ja, ko, fr, de, es, ru, he).
|
||||
|
||||
## Architecture
|
||||
|
||||
### Dual Mode Operation
|
||||
|
||||
The system runs in two modes:
|
||||
- **ComfyUI plugin mode**: Integrates with ComfyUI's PromptServer, uses `folder_paths` for model discovery
|
||||
- **Standalone mode**: `standalone.py` mocks ComfyUI dependencies, reads paths from `settings.json`
|
||||
- Detection: `os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"`
|
||||
|
||||
### Backend (Python)
|
||||
|
||||
**Entry points:**
|
||||
- `__init__.py` — ComfyUI plugin entry: registers nodes via `NODE_CLASS_MAPPINGS`, sets `WEB_DIRECTORY`, calls `LoraManager.add_routes()`
|
||||
- `standalone.py` — Standalone server: mocks `folder_paths` and node modules, starts aiohttp server
|
||||
- `py/lora_manager.py` — Main `LoraManager` class that registers all HTTP routes
|
||||
|
||||
**Service layer** (`py/services/`):
|
||||
- `ServiceRegistry` singleton for dependency injection; services follow `get_instance()` singleton pattern
|
||||
- `BaseModelService` abstract base → `LoraService`, `CheckpointService`, `EmbeddingService`
|
||||
- `ModelScanner` base → `LoraScanner`, `CheckpointScanner`, `EmbeddingScanner` for file discovery with hash-based deduplication
|
||||
- `PersistentModelCache` — SQLite-based metadata cache
|
||||
- `MetadataSyncService` — Background sync from CivitAI/CivArchive APIs
|
||||
- `SettingsManager` — Settings with schema migration support
|
||||
- `WebSocketManager` — Real-time progress broadcasting
|
||||
- `ModelServiceFactory` — Creates the right service for each model type
|
||||
- Use cases in `py/services/use_cases/` orchestrate complex business logic (auto-organize, bulk refresh, downloads)
|
||||
|
||||
**Routes** (`py/routes/`):
|
||||
- Route registrars organize endpoints by domain: `ModelRouteRegistrar`, `RecipeRouteRegistrar`, etc.
|
||||
- Request handlers in `py/routes/handlers/` implement route logic
|
||||
- API endpoints follow `/loras/*`, `/checkpoints/*`, `/embeddings/*` patterns
|
||||
- All routes use aiohttp, return `web.json_response` or `web.Response`
|
||||
|
||||
**Recipe system** (`py/recipes/`):
|
||||
- `base.py` — Recipe metadata structure
|
||||
- `enrichment.py` — Enriches recipes with model metadata
|
||||
- `parsers/` — Parsers for PNG metadata, JSON, and workflow formats
|
||||
|
||||
**Custom nodes** (`py/nodes/`):
|
||||
- Each node class has a `NAME` class attribute used as key in `NODE_CLASS_MAPPINGS`
|
||||
- Standard ComfyUI node pattern: `INPUT_TYPES()` classmethod, `RETURN_TYPES`, `FUNCTION`
|
||||
- All nodes registered in `__init__.py`
|
||||
|
||||
**Configuration** (`py/config.py`):
|
||||
- Manages folder paths for models, handles symlink mappings
|
||||
- Auto-saves paths to settings.json in ComfyUI mode
|
||||
|
||||
### Frontend — Two Distinct UI Systems
|
||||
|
||||
#### 1. Standalone Manager Web UI
|
||||
- **Location:** `static/` (JS/CSS) and `templates/` (HTML)
|
||||
- **Tech:** Vanilla JS + CSS, served by standalone server
|
||||
- **Structure:** `static/js/core.js` (shared), `loras.js`, `checkpoints.js`, `embeddings.js`, `recipes.js`, `statistics.js`
|
||||
- **Tests:** `tests/frontend/**/*.test.js` (vitest + jsdom)
|
||||
|
||||
#### 2. ComfyUI Custom Node Widgets
|
||||
- **Vanilla JS widgets:** `web/comfyui/*.js` — ES modules extending ComfyUI's LiteGraph UI
|
||||
- `loras_widget.js` / `loras_widget_events.js` — Main LoRA selection widget
|
||||
- `autocomplete.js` — Trigger word and embedding autocomplete
|
||||
- `preview_tooltip.js` — Model card preview tooltips
|
||||
- `top_menu_extension.js` — "Launch LoRA Manager" menu item
|
||||
- `utils.js` — Shared utilities and API helpers
|
||||
- Widget styling in `web/comfyui/lm_styles.css` (NOT `static/css/`)
|
||||
- **Vue widgets:** `vue-widgets/src/` → built to `web/comfyui/vue-widgets/`
|
||||
- Vue 3 + TypeScript + PrimeVue + vue-i18n
|
||||
- Vite build with CSS-injected-by-JS plugin
|
||||
- Components: `LoraPoolWidget`, `LoraRandomizerWidget`, `LoraCyclerWidget`, `AutocompleteTextWidget`
|
||||
- Auto-built on ComfyUI startup via `py/vue_widget_builder.py`
|
||||
- Tests: `vue-widgets/tests/**/*.test.ts` (vitest)
|
||||
|
||||
**Widget registration pattern:**
|
||||
- Widgets use `app.registerExtension()` and `getCustomWidgets` hooks
|
||||
- `node.addDOMWidget(name, type, element, options)` embeds HTML in LiteGraph nodes
|
||||
- See `docs/dom_widget_dev_guide.md` for DOMWidget development guide
|
||||
|
||||
## Code Style
|
||||
|
||||
**Python:**
|
||||
- PEP 8, 4-space indentation, English comments only
|
||||
- Use `from __future__ import annotations` for forward references
|
||||
- Use `TYPE_CHECKING` guard for type-checking-only imports
|
||||
- Loggers via `logging.getLogger(__name__)`
|
||||
- Custom exceptions in `py/services/errors.py`
|
||||
- Async patterns: `async def` for I/O, `@pytest.mark.asyncio` for async tests
|
||||
- Singleton pattern with class-level `asyncio.Lock` (see `ModelScanner.get_instance()`)
|
||||
|
||||
**JavaScript:**
|
||||
- ES modules, camelCase functions/variables, PascalCase classes
|
||||
- Widget files use `*_widget.js` suffix
|
||||
- Prefer vanilla JS for `web/comfyui/` widgets, avoid framework dependencies (except Vue widgets)
|
||||
|
||||
## Testing
|
||||
|
||||
**Backend (pytest):**
|
||||
- Config in `pytest.ini`: `--import-mode=importlib`, testpaths=`tests`
|
||||
- Fixtures in `tests/conftest.py` handle ComfyUI dependency mocking
|
||||
- Markers: `@pytest.mark.asyncio`, `@pytest.mark.no_settings_dir_isolation`
|
||||
- Uses `tmp_path_factory` for directory isolation
|
||||
|
||||
**Frontend (vitest):**
|
||||
- Vanilla JS tests: `tests/frontend/**/*.test.js` with jsdom
|
||||
- Vue widget tests: `vue-widgets/tests/**/*.test.ts` with jsdom + @vue/test-utils
|
||||
- Setup in `tests/frontend/setup.js`
|
||||
|
||||
## Key Integration Points
|
||||
|
||||
- **Settings:** Stored in user directory (via `platformdirs`) or portable mode (`"use_portable_settings": true`)
|
||||
- **CivitAI/CivArchive:** API clients for metadata sync and model downloads; CivitAI API key in settings
|
||||
- **Symlink handling:** Config scans symlinks to map virtual→physical paths; fingerprinting prevents redundant rescans
|
||||
- **WebSocket:** Broadcasts real-time progress for downloads, scans, and metadata sync
|
||||
- **Model scanning flow:** Walk folders → compute hashes → deduplicate → extract safetensors metadata → cache in SQLite → background CivitAI sync → WebSocket broadcast
|
||||
+18
@@ -15,6 +15,10 @@ try: # pragma: no cover - import fallback for pytest collection
|
||||
from .py.nodes.lora_pool import LoraPoolLM
|
||||
from .py.nodes.lora_randomizer import LoraRandomizerLM
|
||||
from .py.nodes.lora_cycler import LoraCyclerLM
|
||||
from .py.nodes.lora_info import LoraInfoLM
|
||||
from .py.nodes.lora_syntax_to_path import LoraSyntaxToPath
|
||||
from .py.nodes.create_hook_lora import CreateHookLoraLM
|
||||
from .py.nodes.metadata_overwrite import MetadataOverwriteLM
|
||||
from .py.metadata_collector import init as init_metadata_collector
|
||||
except (
|
||||
ImportError
|
||||
@@ -56,6 +60,16 @@ except (
|
||||
"py.nodes.lora_randomizer"
|
||||
).LoraRandomizerLM
|
||||
LoraCyclerLM = importlib.import_module("py.nodes.lora_cycler").LoraCyclerLM
|
||||
LoraInfoLM = importlib.import_module("py.nodes.lora_info").LoraInfoLM
|
||||
LoraSyntaxToPath = importlib.import_module(
|
||||
"py.nodes.lora_syntax_to_path"
|
||||
).LoraSyntaxToPath
|
||||
CreateHookLoraLM = importlib.import_module(
|
||||
"py.nodes.create_hook_lora"
|
||||
).CreateHookLoraLM
|
||||
MetadataOverwriteLM = importlib.import_module(
|
||||
"py.nodes.metadata_overwrite"
|
||||
).MetadataOverwriteLM
|
||||
init_metadata_collector = importlib.import_module("py.metadata_collector").init
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
@@ -75,6 +89,10 @@ NODE_CLASS_MAPPINGS = {
|
||||
LoraPoolLM.NAME: LoraPoolLM,
|
||||
LoraRandomizerLM.NAME: LoraRandomizerLM,
|
||||
LoraCyclerLM.NAME: LoraCyclerLM,
|
||||
LoraInfoLM.NAME: LoraInfoLM,
|
||||
LoraSyntaxToPath.NAME: LoraSyntaxToPath,
|
||||
CreateHookLoraLM.NAME: CreateHookLoraLM,
|
||||
MetadataOverwriteLM.NAME: MetadataOverwriteLM,
|
||||
}
|
||||
|
||||
WEB_DIRECTORY = "./web/comfyui"
|
||||
|
||||
+511
-355
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,324 @@
|
||||
# Agent Skills System
|
||||
|
||||
The LoRA Manager agent skills system enables LLM-powered metadata enrichment and other AI-driven tasks. Users configure their own LLM provider (BYOK), and skills are executed through right-click context menu actions.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ LoRA Manager Backend │
|
||||
│ │
|
||||
│ ┌──────────────┐ ┌────────────────┐ │
|
||||
│ │ LLMService │───▶│ LLM Provider │ │
|
||||
│ │ (BYOK config, │◀───│ (OpenAI/Ollama │ │
|
||||
│ │ API calls) │ │ /custom) │ │
|
||||
│ └───────┬───────┘ └────────────────┘ │
|
||||
│ │ │
|
||||
│ ┌───────▼───────────────────────┐ │
|
||||
│ │ AgentService │ │
|
||||
│ │ (orchestration: validate │ │
|
||||
│ │ → LLM call → post-process │ │
|
||||
│ │ → WebSocket broadcast) │ │
|
||||
│ └───────┬───────────────────────┘ │
|
||||
│ │ │
|
||||
│ ┌───────▼───────────────────────┐ │
|
||||
│ │ SkillRegistry │ │
|
||||
│ │ ┌─────────────────────────┐ │ │
|
||||
│ │ │ enrich_hf_metadata: │ │ │
|
||||
│ │ │ - skill.yaml │ │ │
|
||||
│ │ │ - prompt.md │ │ │
|
||||
│ │ │ - handler.py │ │ │
|
||||
│ │ └─────────────────────────┘ │ │
|
||||
│ └───────────────────────────────┘ │
|
||||
└──────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Key Design Principle
|
||||
|
||||
**Skills define *what* to do (prompt + post-processing). The AgentService handles *how* (LLM calls, validation, progress).**
|
||||
|
||||
Skills never call the LLM directly. This keeps BYOK configuration centralized and provider-agnostic.
|
||||
|
||||
## BYOK Configuration
|
||||
|
||||
Users configure their LLM provider in **Settings → AI Provider**:
|
||||
|
||||
| Setting | Description | Example |
|
||||
|---|---|---|
|
||||
| `llm_provider` | Provider type | `openai`, `ollama`, or `custom` |
|
||||
| `llm_api_key` | API key (not needed for local Ollama) | `sk-...` |
|
||||
| `llm_api_base` | Custom API base URL (empty = provider default) | `https://api.openai.com/v1` |
|
||||
| `llm_model` | Model name | `gpt-4o-mini` |
|
||||
|
||||
Environment variable overrides: `LLM_API_KEY`, `LLM_MODEL`, `LLM_API_BASE`, `LLM_PROVIDER`.
|
||||
|
||||
### Supported Providers
|
||||
|
||||
- **OpenAI**: Uses `https://api.openai.com/v1` by default
|
||||
- **Ollama** (local): Uses `http://localhost:11434/v1`, no API key required
|
||||
- **Custom**: Any OpenAI-compatible endpoint (vLLM, LM Studio, etc.) — set `llm_api_base` explicitly
|
||||
|
||||
## Available Skills
|
||||
|
||||
### enrich_hf_metadata
|
||||
|
||||
Enriches models linked to an external model site with metadata extracted by an LLM from the site's model card (README).
|
||||
|
||||
**Entry point**: Right-click context menu → "Enrich Metadata with AI"
|
||||
|
||||
**Supported model sources**:
|
||||
|
||||
| Platform | Link | AI enrichment | Direct download |
|
||||
| --- | --- | --- | --- |
|
||||
| Hugging Face | yes | yes | yes |
|
||||
| ModelScope (`modelscope.cn`) | yes | yes | yes |
|
||||
| ModelScope International (`modelscope.ai`) | yes | yes | yes |
|
||||
| TensorArt | yes | no (see below) | no |
|
||||
|
||||
`modelscope.cn` and `modelscope.ai` are **separate catalogues, not mirrors** — a
|
||||
repository published on one is routinely absent from the other — so each is
|
||||
registered as its own source (`ModelScopeSource` / `ModelScopeIntlSource` in
|
||||
`py/services/model_sources/modelscope.py`). The host therefore decides which
|
||||
API and CDN a model resolves against, and the two deployments get separate
|
||||
version groups (`ms:` / `msai:`) and default download directories. Keep the two
|
||||
tables in `modelSourceHelpers.js` and `registry.py` in step when adding a site.
|
||||
|
||||
TensorArt is link-only: `tensor.art` sits behind a Cloudflare managed challenge and its internal API requires session authorization, so the backend cannot read its model pages. Linking still stores the canonical page URL and the "View on TensorArt" link works.
|
||||
|
||||
**What it does**:
|
||||
1. Reads the model's `.metadata.json` to get the source (`source_platform` + `source_url`, or the legacy `hf_url`)
|
||||
2. Fetches the model card through the provider in `py/services/model_sources/` — the README via `fetch_model_card()`, plus any extras the site keeps outside it via `fetch_model_card_context()`
|
||||
3. Sends the README + site-provided extras + local metadata to the LLM for structured extraction
|
||||
4. Writes extracted fields to `.metadata.json`:
|
||||
- `base_model` — only if current value is empty
|
||||
- `trainedWords` — trigger words (LoRA only, if none exist)
|
||||
- `modelDescription` — the site's author description (if any) followed by the README rendered as HTML
|
||||
- `tags` — merged with existing tags, deduplicated
|
||||
- `civitai.images` — example images
|
||||
- `metadata_source` — audit trail: `agent:enrich_hf_metadata`
|
||||
- `llm_enriched_at` — ISO timestamp
|
||||
5. Downloads and optimizes a preview image, using the per-file example image the
|
||||
site publishes when the README has none
|
||||
6. Updates the scanner cache
|
||||
7. Broadcasts WebSocket progress events
|
||||
|
||||
#### Site-provided card extras (`fetch_model_card_context`)
|
||||
|
||||
A model card is not always just `README.md`. ModelScope keeps the author's
|
||||
summary (`Description`), the site-curated tags (`OfficialTags`), and — per
|
||||
published version — the model filenames together with that file's example
|
||||
images (`MuseInfo.versions[].coverImages`) and trigger words in its
|
||||
model-detail API. AIGC repositories there often ship an auto-generated
|
||||
boilerplate README and put everything useful in `Description`, so reading only
|
||||
the README yields almost nothing.
|
||||
|
||||
Providers opt in by overriding `ModelSource.fetch_model_card_context()`, which
|
||||
returns a `ModelCardContext`. The wanted file is identified by its sha256 when
|
||||
the caller knows it (the scanner already records one) and by **basename**
|
||||
otherwise, so each checkpoint in a collection repo gets its own images — and
|
||||
keeps getting them after the user renames the weights, which is the only
|
||||
identifier a rename cannot invalidate. Sites with no such extras inherit an
|
||||
empty context, and the pipeline behaves exactly as before.
|
||||
|
||||
The README and the repository metadata describe the whole repository, not one
|
||||
file, so `execute_skill()` creates a `ModelSourceCache` for the duration of a
|
||||
run and passes it down. Enriching the eight checkpoints of one ModelScope
|
||||
repository costs two HTTP requests instead of sixteen; only the per-file
|
||||
selection is redone for each file. Nothing is cached across runs, and download
|
||||
URLs never go through it.
|
||||
|
||||
#### Deterministic data is applied whether or not an LLM is configured
|
||||
|
||||
`AgentService._load_source_card()` runs for every source-backed enrichment, and
|
||||
the post-processor applies what it returns before the LLM output is merged. A
|
||||
user with **no** provider configured therefore still gets the author summary,
|
||||
the example images, the preview, the site-curated tags, the trigger words and
|
||||
the README rendered as the model description.
|
||||
|
||||
The LLM is always consulted when one is configured — invoking **Enrich Metadata
|
||||
with AI** must call the provider every time, and the site data is never treated
|
||||
as a reason to skip it. The deterministic values act as fallbacks that fill
|
||||
gaps the LLM leaves behind:
|
||||
|
||||
| Field | Deterministic source | LLM role |
|
||||
| --- | --- | --- |
|
||||
| `model_name` | site display name (`Name`), written only while the value is still the file stem | — |
|
||||
| `modelDescription` | author summary + README as HTML | — |
|
||||
| `civitai.name` | the matched version's label (`modelVersion.showName`) | — |
|
||||
| `civitai.images` | site example images, then README images | — |
|
||||
| `preview_url` | first available example image | may propose one from the README |
|
||||
| `tags` | site-curated tags, always merged in | proposes additional content tags |
|
||||
| `civitai.description` | author summary | richer 1-2 sentence summary wins |
|
||||
| `base_model` | site hints resolved against the canonical vocabulary (`py/services/agent/base_model_resolver.py`) | mapping it is the LLM's job; the resolver only fills in when the LLM returns nothing |
|
||||
| `trainedWords` | per-file site trigger words, then YAML `instance_prompt` | primary extraction |
|
||||
| `usage_tips` | regex over an explicitly stated strength range | primary extraction |
|
||||
| `notes` | — | LLM-only |
|
||||
|
||||
Models with no source, an unknown source, or a source without model-card access (TensorArt) are skipped with an explicit reason and counted in the run summary.
|
||||
|
||||
**Model types**: LoRA, Checkpoint, Embedding
|
||||
|
||||
### Download-time hydration
|
||||
|
||||
The same deterministic mapping runs automatically when a model is downloaded
|
||||
from a model source, so a ModelScope or Hugging Face download lands with the
|
||||
populated card a CivitAI download produces instead of a bare filename and
|
||||
hash. Nothing needs to be triggered by hand and no provider is called.
|
||||
|
||||
`py/services/model_sources/hydration.py` owns this path:
|
||||
|
||||
* `_save_source_metadata()` in `py/routes/handlers/model_source_handlers.py`
|
||||
creates the sidecar (hash, source link, scanner-cache entry) and then calls
|
||||
`hydrate_from_source()`. It also runs for a file that was already on disk, so
|
||||
models downloaded before this existed get topped up on the next attempt.
|
||||
* Metadata is created through the **owning scanner**
|
||||
(`scanner._create_default_metadata()`) rather than
|
||||
`MetadataManager.create_default_metadata()`, so the per-type lazy-hash rule
|
||||
applies: `CheckpointScanner` and `OtherScanner` store
|
||||
`hash_status="pending"` with an empty `sha256` for their multi-GB files, and
|
||||
the generic helper would read a 10 GB checkpoint end to end inside the
|
||||
download request. Hydration copes with the empty hash — `_matching_versions()`
|
||||
falls back to the repository basename, which the download just wrote.
|
||||
* Hydration reuses `PostProcessor` with an empty `llm_output`, so the two paths
|
||||
cannot drift apart. It reports `metadata_source = "source:<platform>"` rather
|
||||
than the skill's `agent:enrich_hf_metadata`, and — because no provider ran —
|
||||
it does not stamp `llm_enriched_at`.
|
||||
* `model_name` is only written while it still equals the file stem: once a user
|
||||
renames a model, that choice is kept.
|
||||
* Only a model whose stored `source_platform`/`source_url` match the repository
|
||||
being downloaded is updated; a local file that merely shares a name must not
|
||||
receive another model's card.
|
||||
* The README and repository payload describe the *repository*, so a short-lived
|
||||
process-wide `ModelSourceCache` (`shared_source_cache`, 300 s, 32 entries)
|
||||
keeps a batch over one repository to two HTTP requests.
|
||||
* Every failure — unreachable site, changed payload shape, broken post-processor
|
||||
— is logged and swallowed. Metadata hydration can never fail a download.
|
||||
* Neither stage advances the byte counter, so both are announced to the
|
||||
progress UI (`_report_phase()` → `{"status": "metadata", "stage": ...}`) as
|
||||
they start. Without that the bar sits at 100% reporting `0 B/s` for several
|
||||
seconds and the download looks stuck. `stage` and `platform` are
|
||||
machine-readable; the wording is localised in `LoadingManager`.
|
||||
|
||||
## Adding a New Skill
|
||||
|
||||
### 1. Create the skill directory
|
||||
|
||||
```
|
||||
py/services/agent/skills/<skill_name>/
|
||||
├── skill.yaml # Skill metadata and schemas
|
||||
├── prompt.md # LLM prompt template
|
||||
└── handler.py # Pre-processing and post-processing
|
||||
```
|
||||
|
||||
### 2. Write skill.yaml
|
||||
|
||||
```yaml
|
||||
name: my_skill
|
||||
title: "My Skill"
|
||||
description: "What this skill does"
|
||||
llm_required: true
|
||||
model_type_filter: ["lora"] # or null for all types
|
||||
input_schema:
|
||||
type: object
|
||||
properties:
|
||||
model_paths:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
required:
|
||||
- model_paths
|
||||
output_schema:
|
||||
type: object
|
||||
properties:
|
||||
# ... JSON schema for LLM output
|
||||
permissions:
|
||||
write_metadata: true
|
||||
write_previews: false
|
||||
network_domains:
|
||||
- "example.com"
|
||||
```
|
||||
|
||||
### 3. Write prompt.md
|
||||
|
||||
Use `{{variable}}` placeholders that will be replaced with data from the `prepare` function:
|
||||
|
||||
```markdown
|
||||
You are an expert assistant...
|
||||
|
||||
Model URL: {{source_url}}
|
||||
README content:
|
||||
{{readme_content}}
|
||||
|
||||
Current metadata:
|
||||
{{current_metadata}}
|
||||
```
|
||||
|
||||
### 4. Write handler.py
|
||||
|
||||
```python
|
||||
async def prepare(model_path: str, input_data: dict) -> dict:
|
||||
"""Gather context for the LLM prompt. Returns variables for template rendering."""
|
||||
return {
|
||||
"model_path": model_path,
|
||||
# ... other variables used in prompt.md
|
||||
}
|
||||
|
||||
async def post_process(context) -> dict:
|
||||
"""Apply the LLM-extracted data to the model."""
|
||||
llm_response = context.llm_response
|
||||
# ... write metadata, download previews, update cache
|
||||
return {
|
||||
"success": True,
|
||||
"updated_fields": ["base_model", "tags"],
|
||||
"errors": [],
|
||||
}
|
||||
```
|
||||
|
||||
**Important**: Use absolute imports (`from py.utils.metadata_manager import MetadataManager`) because skills are loaded via `importlib.util.spec_from_file_location`, which doesn't support relative imports.
|
||||
|
||||
### 5. Test
|
||||
|
||||
The skill is automatically discovered by `SkillRegistry` on startup. Test with:
|
||||
|
||||
```python
|
||||
pytest tests/services/test_agent_service.py
|
||||
```
|
||||
|
||||
## API Endpoints
|
||||
|
||||
| Method | Path | Description |
|
||||
|---|---|---|
|
||||
| GET | `/api/lm/agent/skills` | List available skills |
|
||||
| POST | `/api/lm/agent/execute/{skill_name}` | Execute a skill (body: `{"model_paths": [...]}`) |
|
||||
| POST | `/api/lm/agent/cancel` | Cancel running skill (stub) |
|
||||
|
||||
## WebSocket Events
|
||||
|
||||
| Type | When | Key fields |
|
||||
|---|---|---|
|
||||
| `agent_progress` | Skill started/processing | `skill`, `status`, `total`, `processed`, `success`, `current_path` |
|
||||
| `agent_progress` | Skill completed | `skill`, `status`, `updated_models`, `errors`, `summary` |
|
||||
| `agent_progress` | Skill error | `skill`, `status`, `error` |
|
||||
|
||||
## Security Model
|
||||
|
||||
Skills declare permissions in `skill.yaml`:
|
||||
- `write_metadata` — can write `.metadata.json` files
|
||||
- `write_previews` — can download/replace preview images
|
||||
- `network_domains` — allowed domains for HTTP requests
|
||||
|
||||
These are declarative constraints checked by `AgentService`. They are defense-in-depth, not a sandbox — the Python process can technically do anything, but the contract is clear and auditable.
|
||||
|
||||
## File Locations
|
||||
|
||||
| Component | Path |
|
||||
|---|---|
|
||||
| LLMService | `py/services/llm_service.py` |
|
||||
| AgentService | `py/services/agent/agent_service.py` |
|
||||
| SkillRegistry | `py/services/agent/skill_registry.py` |
|
||||
| SkillDefinition | `py/services/agent/skill_definition.py` |
|
||||
| Skills directory | `py/services/agent/skills/` |
|
||||
| Route handlers | `py/routes/handlers/agent_handlers.py` |
|
||||
| Frontend manager | `static/js/managers/AgentManager.js` |
|
||||
| Settings UI | `templates/components/modals/settings_modal.html` |
|
||||
| Context menu | `templates/components/context_menu.html` |
|
||||
@@ -54,7 +54,7 @@ The dedicated services encapsulate long-running work so handlers stay thin.
|
||||
| Use case | Entry point | Dependencies | Guarantees |
|
||||
| --- | --- | --- | --- |
|
||||
| `RecipeAnalysisService` | `analyze_uploaded_image`, `analyze_remote_image`, `analyze_local_image`, `analyze_widget_metadata` | `ExifUtils`, `RecipeParserFactory`, downloader factory, optional metadata collector/processor | Normalises missing/invalid payloads into `RecipeValidationError`; generates consistent fingerprint data to keep duplicate detection stable; temporary files are cleaned up after every analysis path. |
|
||||
| `RecipePersistenceService` | `save_recipe`, `delete_recipe`, `update_recipe`, `reconnect_lora`, `bulk_delete`, `save_recipe_from_widget` | `ExifUtils`, recipe scanner, card preview sizing constants | Writes images/JSON metadata atomically; updates scanner caches and hash indices before returning; recalculates fingerprints whenever LoRA assignments change. |
|
||||
| `RecipePersistenceService` | `save_recipe`, `delete_recipe`, `update_recipe`, `reconnect_lora`, `get_reconnect_suggestions`, `bulk_delete`, `save_recipe_from_widget` | `ExifUtils`, recipe scanner, card preview sizing constants | Writes images/JSON metadata atomically; updates scanner caches and hash indices before returning; recalculates fingerprints whenever LoRA assignments change. |
|
||||
| `RecipeSharingService` | `share_recipe`, `prepare_download` | `tempfile`, recipe scanner | Copies originals to TTL-managed temp files; metadata lookups re-use the scanner; expired shares trigger cleanup and `RecipeNotFoundError`. |
|
||||
|
||||
## Maintaining critical invariants
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
# ComfyUI Dual-Mode Widget Rendering
|
||||
|
||||
ComfyUI custom node widgets render in one of two modes. Patterns that work in one often fail silently in the other. Test both.
|
||||
|
||||
## Mode Detection
|
||||
|
||||
```js
|
||||
typeof LiteGraph !== 'undefined' && LiteGraph.vueNodesMode
|
||||
```
|
||||
|
||||
In Vue SFCs, `window.LiteGraph` is unavailable — pass as a prop from `main.ts`.
|
||||
|
||||
## Canvas Mode Layout
|
||||
|
||||
Uses `computeLayoutSize()` + `distributeSpace()` to allocate widget height within the node. Widgets with `computeLayoutSize` participate in space distribution; those with `computeSize` have fixed height.
|
||||
|
||||
- `getMinHeight()` in `addDOMWidget` options → minimum widget height
|
||||
- `widget.computeLayoutSize()` → `{ minHeight, minWidth, maxHeight? }`
|
||||
- Avoid `getMaxHeight()` unless the widget genuinely needs a fixed cap (prevents user resize)
|
||||
|
||||
## Vue Mode Layout
|
||||
|
||||
Uses CSS Grid (`grid-template-rows`) + `ResizeObserver`. The ResizeObserver watches the widget's DOM and feeds back into grid row sizing. This creates a feedback loop: content grows → row resizes → more space for content → content reflows/grows → row resizes again.
|
||||
|
||||
### Height Containment
|
||||
|
||||
The fix: `contain: layout size` on the widget root. This tells the browser the element's intrinsic size is CSS-determined, not driven by descendant content. The ResizeObserver sees a stable size and the loop is broken.
|
||||
|
||||
```css
|
||||
.widget-root.lm-vue-node {
|
||||
height: 100%;
|
||||
min-height: var(--comfy-widget-min-height, 200px);
|
||||
contain: layout size;
|
||||
}
|
||||
```
|
||||
|
||||
Existing examples: `.lm-loras-container.lm-vue-node` and `.comfy-tags-container.lm-vue-node` in `web/comfyui/lm_styles.css`.
|
||||
|
||||
**Do NOT** fix height issues with `maxHeight`, `getMaxHeight()`, or inline `max-height` — these prevent the user from resizing the node.
|
||||
|
||||
## Scroll Wheel Isolation
|
||||
|
||||
Both modes need to distinguish "user wants to scroll widget content" from "user wants to zoom canvas".
|
||||
|
||||
**Canvas mode:** Add `@wheel` on widget root. Check `event.target.closest(selector)` for scrollable sub-areas. If scrollable → `event.stopPropagation()`. Otherwise → `app.canvas.processMouseWheel(event)`.
|
||||
|
||||
**Vue mode:** Add CSS class `lm-wheel-scrollable` to scrollable elements. The global capture-phase hook in `web/comfyui/utils.js` (`enableListWheelScroll`) detects wheel events on marked elements and manually scrolls them via `element.scrollTop`, consuming the event before canvas zoom sees it.
|
||||
|
||||
## DOM Structure
|
||||
|
||||
`main.ts` creates an outer `<div>` container, then `vueApp.mount(container)`. The Vue app renders its own root element inside.
|
||||
|
||||
- `container.id` / `container.style.*` → outer element
|
||||
- Vue scoped `<style>` → `[data-v-hash]` applies only to Vue root
|
||||
|
||||
Classes needed by scoped Vue CSS must go on the Vue root element. Pass data as props and bind with `:class` rather than manipulating the DOM from `main.ts`.
|
||||
|
||||
## Serialization
|
||||
|
||||
For stateful widgets that need workflow persistence:
|
||||
|
||||
- `serialize: true` in `addDOMWidget` options
|
||||
- `serializeValue()` → state snapshot (called on workflow save)
|
||||
- `onSetValue(v)` → restore state (called on workflow load)
|
||||
- Always handle missing keys in restored value for backward compatibility with old workflows
|
||||
@@ -0,0 +1,531 @@
|
||||
# i18n Translation Guidelines
|
||||
|
||||
This document is the canonical set of conventions for translating LoRA Manager UI strings.
|
||||
It applies to **human translators and AI agents** alike. Read it before editing anything in
|
||||
`locales/`.
|
||||
|
||||
Source of truth: `locales/en.json` (10 locales, 2025 leaf keys; all locales share the exact
|
||||
same key structure).
|
||||
|
||||
Locales: `en`, `zh-CN`, `zh-TW`, `ja`, `ko`, `fr`, `de`, `es`, `ru`, `he` (RTL).
|
||||
|
||||
> **Status (2026-08 sweep):** a full audit was executed and the terminology, placeholder,
|
||||
> stale-text, and untranslated-block fixes described in §2–§6 were applied across all locales
|
||||
> (commits `3c3ac49f` … `fd1227d3`). The tables below are now the **normative target state**,
|
||||
> not a to-do list — future edits should preserve these renderings and only add what is new.
|
||||
>
|
||||
> **Status (2026-09, Other Models):** the `other` model type (VAE / Upscaler / Text Encoder /
|
||||
> CLIP Vision / ControlNet) and the Other Models opt-in toggles added 36 new keys; all of them
|
||||
> are now translated in all 9 locales (terminology in §2 "Other Models feature"). There are no
|
||||
> remaining `[TODO: Translate]` placeholders in any locale.
|
||||
>
|
||||
> **Status (2026-09, revision):** `other.disabled.description`, `banners.otherModels.content` and
|
||||
> `settings.folderSettings.enableOtherModelsHelp` were refreshed in `en.json` to name all five
|
||||
> sub_types (they had listed four, which read as "these are what enabling manages") and
|
||||
> re-translated in all 9 locales in the same pass. `clip_vision` and `controlnet` are now both
|
||||
> opt-in, so the first two describe **capability** and the third the **master switch**, not the
|
||||
> default set — keep all three enumerating the full five (`VAE / upscaler / text encoder /
|
||||
> CLIP vision / ControlNet` in `en`; locale slash-list casing follows each file's existing
|
||||
> `VAE / Upscaler / Text Encoder / …` style, de compounds as `CLIP-Vision- und ControlNet-Ordner`).
|
||||
>
|
||||
> **Status (2026-09, "no folders found" state):** the Other Models page gained an *enabled but
|
||||
> nothing to scan* empty state with 6 new keys (`other.noPaths.*`); translated in all 9 locales
|
||||
> in the same pass. The `folder_paths` JSON snippet shown in that state lives in
|
||||
> `templates/other.html`, **not** in the locale files, so it is never translated — only the
|
||||
> surrounding prose is. Terminology added in §2.
|
||||
>
|
||||
> **Status (2026-09, model sources):** models can now be linked to ModelScope and TensorArt
|
||||
> alongside Hugging Face, which added 15 keys (`modelCard.actions.viewOnSource`,
|
||||
> `loras.contextMenu.linkModelSource`, `modals.linkModelSource.*`,
|
||||
> `modals.model.versions.sourceGroupInfo`, `toast.contextMenu.enrichNeedsSource`,
|
||||
> `toast.contextMenu.enrichUnsupportedSource`) and refreshed the two `enrichHfAgent` labels,
|
||||
> which had hardcoded "HF" for a button that now also enriches ModelScope models. The
|
||||
> `modals.linkModelSource.urlPlaceholder` value stays byte-identical to `en.json` (it is a URL,
|
||||
> the §6 exception). Terminology in §2, "Model source feature".
|
||||
>
|
||||
> **Status (2026-09, folder sidebar):** the model-root sidebar gained on-disk folder management
|
||||
> (create / rename / delete folders, show empty folders, tree vs list view) plus its `...`
|
||||
> view-options menu, adding 35 `sidebar.*` keys. Those were the only `[TODO: Translate]`
|
||||
> placeholders left behind by the feature series, and all 35 are now translated in all 9
|
||||
> locales, so the "no remaining placeholders" claim above holds again. Terminology in §2,
|
||||
> "Folder sidebar feature".
|
||||
>
|
||||
> **Status (2026-09, chip reordering):** model tags and trigger words now share one drag/`⠿`
|
||||
> grip reorder affordance, which added the single `common.reorder.dragHandle` key (it lives
|
||||
> under `common` because both editors render it). All 9 locales are translated (renderings in
|
||||
> §2, "Chip reordering"). Reordering is pointer-only by design: an `Alt + Arrow` shortcut was
|
||||
> prototyped and removed because it collided with the browser's Alt + Arrow handling and the
|
||||
> modal's arrow-key navigation.
|
||||
|
||||
---
|
||||
|
||||
## 1. Hard rules (do not violate)
|
||||
|
||||
### R1 — Key structure is sacred
|
||||
- Only `locales/en.json` may add/remove/rename keys. All other locales must keep the exact
|
||||
same nested key set. `tests/i18n/test_i18n.py` enforces this.
|
||||
- When a new UI string is added to `en.json`, run
|
||||
`python scripts/sync_translation_keys.py` (adds the missing keys to all locales with
|
||||
`[TODO: Translate]` placeholder copies) — **then stop**. Do NOT translate proactively:
|
||||
placeholders are the expected end state during feature development, and translations are
|
||||
filled in only when the feature owner explicitly asks (workflow details in §7).
|
||||
- Never reorder, re-indent, or reformat a locale file "for tidiness". The sync script
|
||||
preserves formatting; manual reformatting creates noisy diffs.
|
||||
|
||||
### R2 — Placeholders and HTML must be preserved verbatim
|
||||
- `{name}`-style placeholders must appear in the translation exactly as in `en.json`.
|
||||
Do not invent placeholders the source string does not have — the caller may not pass them
|
||||
(example bug: `zh-CN recipes.controls.import.downloadLocationPreview` added `{path}`; the
|
||||
template renders this key with no parameters, so the literal text `{path}` shows in the UI).
|
||||
- `{{...}}` in a locale value is an escaped literal brace — keep it identical.
|
||||
- Keep embedded HTML tags (e.g. `<strong>...</strong>`, `<code>...</code>`) intact.
|
||||
You may move the tag around the sentence if the target language needs different word order.
|
||||
|
||||
### R3 — Never translate or transliterate these
|
||||
- Model types: **LoRA, Checkpoint, Embedding, Diffusion Model**
|
||||
- Products/brands: **LoRA Manager, ComfyUI, CivitAI, CivArchive, HuggingFace, Ko-fi**
|
||||
- Ecosystem names: **LyCORIS, DoRA**, trigger-adjacent jargon **Prompt, Workflow**
|
||||
(these are used as-is in the target-language SD community; see §2 per-language policy)
|
||||
- Theme names: **Nord, Midnight, Monokai, Dracula, Solarized**
|
||||
|
||||
### R4 — The "Recipe" convention (the most important domain term)
|
||||
Product intent: a *Recipe* records a **LoRA combination + generation parameters**
|
||||
(prompt, seed, sampler, …) that reproduces an image style. The metaphor is a **cooking
|
||||
recipe** — "follow it and you get a similar dish". It is **not** a menu, not a dish list,
|
||||
not a prescription.
|
||||
|
||||
Decision per language — translate only into a word whose everyday primary meaning is a
|
||||
cooking recipe; where that word would mislead users, **keep the English "Recipe(s)"**:
|
||||
|
||||
| Locale | Use | Never use |
|
||||
|---|---|---|
|
||||
| fr | **Recipe / Recipes** (keep English) | recette(s) — cooking reading is secondary and it was explicitly judged misleading |
|
||||
| zh-CN / zh-TW | 配方 | 食谱 (reads as "food cookbook") |
|
||||
| ja | レシピ | — (leftover English "Recipe" in `initialization.recipes.title` / `toast.recipes.recipeSaved` → translate) |
|
||||
| ko | 레시피 | — |
|
||||
| de | Rezept / Rezepte | — (cooking meaning dominant; prescription reading acceptable) |
|
||||
| es | receta / recetas | — (cooking meaning dominant) |
|
||||
| ru | рецепт / рецепты | — (leftover English "Recipe" in `initialization.recipes.title` / `toast.recipes.recipeSaved` → translate) |
|
||||
| he | מתכון / מתכונים | — (cooking meaning dominant) |
|
||||
|
||||
Whatever the choice, **one concept = one noun within a locale**. Currently violated in:
|
||||
- `fr` — "Recipe" (~97 keys, incl. nav) mixed with "recette" (~58 keys)
|
||||
- `zh-CN` / `zh-TW` — 配方 (126/122 keys) mixed with 食谱 / 食譜 (14/17 keys, all in the
|
||||
*rematch* flow: `globalContextMenu.rematchRecipes.*`, `toast.recipes.rematch*`)
|
||||
- `de` — "Rezept" (136 keys) mixed with leftover English "Recipe" (5 keys)
|
||||
- `ja` / `ru` — leftover English "Recipe" in `initialization.recipes.title` ("Recipe Manager
|
||||
zu initialisieren" / «Инициализация Recipe Manager») and `toast.recipes.recipeSaved`
|
||||
|
||||
### R5 — One term, one rendering (within each locale)
|
||||
Same source word must not be translated several ways in one file. Known offender areas
|
||||
(see §5 for the full fix list): recipe, Checkpoint, Embedding, prompt, base model, preset,
|
||||
workflow, hash, metadata, tags, bulk. Every locale currently mixes variants of at least one
|
||||
of these — pick the preferred form in the §2 tables and normalize.
|
||||
|
||||
### R6 — Register consistency
|
||||
- `zh-CN` / `zh-TW`: pick 你 or 您 once. Do not mix (zh-CN has 44×你 + 5×您; zh-TW has
|
||||
27×您 + 18×你).
|
||||
- `de`: pick "du" or "Sie" once (currently 143×Sie + ~7×du).
|
||||
- `es`: pick "tú" or "usted" once.
|
||||
|
||||
### R7 — Punctuation per script
|
||||
- Full-width punctuation `:()` is correct **only in CJK locales** (zh-CN, zh-TW, ja, ko).
|
||||
- Latin/Cyrillic/Hebrew locales must use ASCII `: ()` — full-width colons leaked in there
|
||||
are machine-translation artifacts. Known: `fr toast.recipes.createError/createFailed`,
|
||||
`es toast.recipes.createError/createFailed` (e.g. "…de la receta:" should be "…de la receta:").
|
||||
- `fr` apostrophes must be U+2019 `'` / ASCII `'`, never a straight double quote:
|
||||
`fr header.filter.allowSellingGeneratedContentTooltip` currently reads
|
||||
`vendre d"images` → fix to `d'images`. Do not mix `'` and `'` in one file (fr has 299 vs 15).
|
||||
- Ellipsis: use ASCII `...` (project style). Don't introduce `…`.
|
||||
- Keep the sentence-ending period/omission consistent with the source string where the
|
||||
language allows it.
|
||||
- `he` is RTL: mix of Hebrew and Latin scripts is normal; keep Latin term ordering natural.
|
||||
|
||||
### R8 — No untranslated English leftovers
|
||||
Full sentences left byte-identical to `en.json` are bugs (brand names and URL placeholders
|
||||
are the exception). Every locale has them; see §6 for the per-locale checklist.
|
||||
`[TODO: Translate]` placeholders are the sanctioned intermediate state during feature
|
||||
development (see §7) — do not "fix" them unless the feature owner asked for translations.
|
||||
|
||||
### R9 — Mirror the source even when the source is wrong
|
||||
If `en.json` itself contains an inconsistency (e.g. the `Civitai` vs `CivitAI` casing split,
|
||||
or the `CivitArchive` typo in `modals.relinkCivitai.helpText.format4`), translate/transcribe
|
||||
it as-is in your locale and instead **fix the source** in `en.json` (then propagate by
|
||||
re-syncing and re-translating affected keys). Do not silently diverge in one locale only.
|
||||
|
||||
---
|
||||
|
||||
## 2. Per-language term maps
|
||||
|
||||
Preferred rendering per term. "Fix" means the locale currently contains the wrong variant
|
||||
and must be normalized. `en` = keep the English word as-is.
|
||||
|
||||
### fr
|
||||
|
||||
| Term | Use | Fix |
|
||||
|---|---|---|
|
||||
| recipe | Recipe(s) | Replace all "recette(s)" (58 keys, e.g. `recipes.actions.deleteRecipeWithShortcut`, `toast.recipes.rematchComplete`) with "Recipe(s)" |
|
||||
| Checkpoint | Checkpoint | `statistics.modelTypes.checkpoint` = "Point de contrôle" → "Checkpoint" |
|
||||
| trigger words | mot(s)-clé(s) | unify: `modals.model.triggerWords.editWord` uses "mot déclencheur" — pick one |
|
||||
| prompt / negative prompt | Prompt / prompt négatif | — |
|
||||
| base model | modèle(s) de base | — |
|
||||
| preset | préréglage | unify: `modals.model.usageTips.addPresetParameter` "prédéfini", `toast.presets.restored` "par défaut" |
|
||||
| hash | hash | `conflictConfirm.message` "hachage" → "hash" |
|
||||
| tags | tags | `settings.sections.priorityTags` "Étiquettes" → "Tags" |
|
||||
| metadata | métadonnées | `loras.controls.refresh.fullTooltip` keeps English "metadata" |
|
||||
| duplicates | doublon(s) | unify with "dupliqué(e)s" |
|
||||
| bulk | groupé(e) | unify with "par lot / mode lot" variants |
|
||||
|
||||
### de
|
||||
|
||||
| Term | Use | Fix |
|
||||
|---|---|---|
|
||||
| recipe | Rezept/Rezepte | leftover English "Recipe" keys → Rezept (e.g. `toast.recipes.recipeSaved`) |
|
||||
| base model | pick Basis-Modell or Basismodell | currently 27× hyphenated vs 15× closed |
|
||||
| metadata | Metadaten | 4 keys use "Modelldaten" (`onboarding.steps.fetch.title/content`) → Metadaten |
|
||||
| bulk | pick Massen- or Sammelmodus | `loras.controls.bulk.action` = "Massen" reads as "crowds" — use "Massenbearbeitung"/"Mehrfachauswahl" |
|
||||
| register | Sie (formal) | 7 keys use "du/dein" (`settings.backup.managementHelp`, `modals.checkUpdates.message/tip`, `doctor.footer`, …) |
|
||||
|
||||
### es
|
||||
|
||||
| Term | Use | Fix |
|
||||
|---|---|---|
|
||||
| recipe | receta(s) | — |
|
||||
| Checkpoint | Checkpoint | 5 statistics keys "Punto(s) de control" → "Checkpoints" (`statistics.metrics.checkpoints`, `statistics.insights.unusedCheckpoints.*`, `statistics.modelTypes.checkpoint`) |
|
||||
| trigger words | palabra(s) de activación | 2 keys already use it; ~15 keys "palabra(s) clave" (reads as search keyword) → unify |
|
||||
| base model | modelo base | — |
|
||||
| preset | preajuste | 3 keys keep English "preset", 1 "preestablecido" → preajuste |
|
||||
| workflow | pick flujo de trabajo or workflow | currently 21× "flujo de trabajo" vs 10× "workflow" |
|
||||
| bulk | masivo / por lotes | unify; "Batch Import" → traducción |
|
||||
| tags | etiquetas | — |
|
||||
|
||||
### ru
|
||||
|
||||
| Term | Use | Fix |
|
||||
|---|---|---|
|
||||
| recipe | рецепт(ы) | English leftovers: `initialization.recipes.title`, `recipes.batchImport.*`, `toast.recipes.recipeSaved` → translate |
|
||||
| Checkpoint | Checkpoint (recommended) | 3 variants today: "Checkpoint" (17 keys), «Чекпойнт», «Контрольная точка» (statistics, 6 keys) — statistics MUST drop «Контрольная точка» |
|
||||
| Embedding | Embedding | «Эмбеддинг» variant exists in `settings.priorityTags.modelTypes.embedding` — unify |
|
||||
| prompt | промпт | 8 keys use «запрос» (reads as "database/HTTP request") → «промпт» |
|
||||
| base model | базовая модель | — |
|
||||
| preset | пресет | `header.theme.presets` "Предустановки" → пресеты |
|
||||
| workflow | Workflow (recommended) | «рабочий процесс» used in 4 keys — unify |
|
||||
| hash | pick хеш or хэш | both spellings co-occur |
|
||||
| tag(s) | тег(и) | — |
|
||||
| typos | — | `settings.misc.loraSyntaxFormatHelp`: «безпотерьного» → «беспотерьного» |
|
||||
|
||||
### he
|
||||
|
||||
| Term | Use | Fix |
|
||||
|---|---|---|
|
||||
| recipe | מתכון / מתכונים | — |
|
||||
| Checkpoint | Checkpoint | 5 statistics keys «נקודת/נקודות ביקורת» (road/security checkpoint) → "Checkpoint(s)" (`statistics.metrics.checkpoints`, `statistics.modelTypes.checkpoint`, `statistics.insights.unusedCheckpoints.*`) |
|
||||
| Embedding | Embedding | `statistics` keys use הטמעות → Embedding |
|
||||
| prompt | pick הנחיה or פרומפט | 9 keys הנחיה vs 3 פרומפט — unify (recommend פרומפט, SD-community loanword) |
|
||||
| preset | קביעה מראש | `header.filter.presetOverwriteConfirm` uses פריסט → unify |
|
||||
| hash | pick one of האש / גיבוב / hash | 3 variants co-occur — unify (recommend hash or גיבוב) |
|
||||
| metadata | pick מטא-דאטה or מטא-נתונים | 38 vs 17 keys — unify |
|
||||
| model | מודל | 13 keys use דגם/דגמים — unify |
|
||||
| bulk | pick one of 5 variants | 5 different renderings ("כמות גדולה", "המוני", "קבוצתי", "אצווה", …) — unify; `loras.controls.bulk.action` "כמות גדולה" reads as "large quantity" |
|
||||
|
||||
### ja
|
||||
|
||||
| Term | Use | Fix |
|
||||
|---|---|---|
|
||||
| recipe | レシピ | `initialization.recipes.title` keeps English "Recipe Manager" — translate to レシピマネージャー |
|
||||
| Checkpoint | Checkpoint or チェックポイント (pick one) | 3 variants: Checkpoint (~14), checkpoint lowercase (4), チェックポイント (4, e.g. `settings.priorityTags.modelTypes.checkpoint`) |
|
||||
| Embedding | Embedding | 4 keys lowercase "embedding" mid-sentence |
|
||||
| bulk | 一括 | `modals.checkUpdates.tip` "バルクモード" → 一括モード |
|
||||
| recipe counter | 件 or 個 | `globalContextMenu.rematchRecipes.success` uses 件, `.cancelled` uses 個 — unify |
|
||||
|
||||
### ko
|
||||
|
||||
| Term | Use | Fix |
|
||||
|---|---|---|
|
||||
| recipe | 레시피 | — |
|
||||
| Checkpoint | Checkpoint (recommended) | 4 keys transliterate 체크포인트 (`settings.priorityTags.modelTypes.checkpoint`, `toast.recipes.missingCheckpointPath/missingCheckpointInfo/downloadCheckpointFailed`) |
|
||||
| Embedding | Embedding | 3 keys 임베딩 (`settings.priorityTags.modelTypes.embedding`, `uiHelpers.nodeSelector.embedding`) |
|
||||
| base model | 베이스 모델 | 6 keys «기본 모델» read as "default model" → 베이스 모델 (`settings.downloadSkipBaseModels.*`, `toast.loras.downloadSkippedByBaseModel`) |
|
||||
| workflow | pick 워크플로 or 워크플로우 | 26 vs 6 keys — unify |
|
||||
| bulk | 일괄 | `modals.checkUpdates.tip` "벌크 모드" → 일괄 모드 |
|
||||
| tag logic | — | `header.filter.tagLogicAny` = "모든 태그 일치 (OR)" is **inverted** (should be "하나 이상의 태그 일치") and identical to `tagLogicAll` |
|
||||
| particle | — | `modelCard.sendToWorkflow.checkpointNotImplemented`: "Checkpoint을" → "Checkpoint를" |
|
||||
|
||||
### zh-CN / zh-TW
|
||||
|
||||
| Term | zh-CN | zh-TW |
|
||||
|---|---|---|
|
||||
| recipe | 配方 (fix 食谱 → 配方, 14 keys in rematch flow) | 配方 (fix 食譜 → 配方, 17 keys in rematch flow) |
|
||||
| Checkpoint | Checkpoint (fix 检查点 → Checkpoint, 5 keys: `toast.recipes.missingCheckpointPath/missingCheckpointInfo/downloadCheckpointFailed`, `modelCard.actions.checkpointNameCopied`, `modelCard.sendToWorkflow.checkpointNotImplemented`) | Checkpoint (fix 檢查點 → Checkpoint, 4 keys: `modelCard.actions.copyCheckpointName`, `toast.recipes.missing*`×2, `toast.recipes.downloadCheckpointFailed`) |
|
||||
| base model | 基础模型 (fix 基模型 → 基础模型, 3 keys in `modals.model.versions.filters.*`) | 基礎模型 ✓ consistent |
|
||||
| prompt | 提示词 ✓ | 提示詞 ✓ |
|
||||
| preset | 预设 ✓ | 預設 ✓ |
|
||||
| workflow | 工作流 ✓ | 工作流 ✓ |
|
||||
| trigger words | 触发词 ✓ | 觸發詞 ✓ |
|
||||
| hash | 哈希 (哈希值 variant OK) | 雜湊 ✓ |
|
||||
| register | 你 (fix 5×您 → 你) | 您 (fix 18×你 → 您) |
|
||||
|
||||
### Other Models feature (VAE / Upscaler / Text Encoder / CLIP Vision / ControlNet)
|
||||
|
||||
The `other` model type exposes five sub_types. They are **model-type names**, so they follow
|
||||
R3 and stay in Latin in every locale. The `settings.folderSettings.subType*` values are
|
||||
therefore **intentionally byte-identical to `en.json`** (same precedent as
|
||||
`settings.priorityTags.modelTypes` / `checkpoints.modelTypes.checkpoint`) — a §6 sweep must
|
||||
not "fix" them.
|
||||
|
||||
| Term | Rendering | Note |
|
||||
|---|---|---|
|
||||
| VAE | `VAE` everywhere | acronym, always upper-case |
|
||||
| Upscaler | `Upscaler` everywhere | CivitAI `ModelType` name |
|
||||
| Text Encoder | `Text Encoder` everywhere | de compounds as `Text-Encoder-Stammordner` |
|
||||
| CLIP Vision | `CLIP Vision` everywhere | de compounds as `CLIP-Vision-Stammordner` |
|
||||
| ControlNet | `ControlNet` everywhere | brand casing, capital N |
|
||||
|
||||
In prose these names sit next to localized nouns the same way `Diffusion Model` does
|
||||
(zh `VAE 根目录`, ja `VAEルート`, ko `VAE 루트`, ru `Корневая папка VAE`).
|
||||
|
||||
**"Other Models" is the page/feature name, not a model type — translate it:**
|
||||
|
||||
| Locale | `other.title` | `header.navigation.other` |
|
||||
|---|---|---|
|
||||
| fr | Autres modèles | Autres |
|
||||
| zh-CN | 其他模型 | 其他 |
|
||||
| zh-TW | 其他模型 | 其他 |
|
||||
| ja | その他のモデル | その他 |
|
||||
| ko | 기타 모델 | 기타 |
|
||||
| de | Weitere Modelle | Andere |
|
||||
| es | Otros modelos | Otros |
|
||||
| ru | Другие модели | Другое |
|
||||
| he | מודלים אחרים | אחרים |
|
||||
|
||||
`settings.folderSettings.otherSubTypes` ("Managed Types") must name **model** types, matching
|
||||
each locale's `header.filter.modelTypes` rendering (zh `管理的模型类型`, ja `管理するモデルタイプ`,
|
||||
de `Verwaltete Modelltypen`, …).
|
||||
|
||||
The "no folders found" empty state (`other.noPaths.*`) uses two phrases that must stay
|
||||
consistent whenever that copy is edited. `folder key` means the `folder_paths` key name
|
||||
(`vae`, `upscale_models`, … — Latin per the table above); `on disk` means the folder must
|
||||
physically exist:
|
||||
|
||||
| Phrase | Rendering |
|
||||
|---|---|
|
||||
| folder key | zh-CN 文件夹键 · zh-TW 資料夾鍵 · ja フォルダーキー · ko 폴더 키 · fr clé de dossier · de Ordnerschlüssel · es clave de carpeta · ru ключ папки · he מפתח תיקייה |
|
||||
| on disk | zh-CN 在磁盘上 · zh-TW 在磁碟上 · ja ディスク上 · ko 디스크에 · fr sur le disque · de auf dem Datenträger · es en el disco · ru на диске · he בדיסק |
|
||||
|
||||
`settings.json` and `ComfyUI` stay verbatim in every locale; "reload this page" / "restart
|
||||
LoRA Manager" reuse each locale's existing restart wording (`settings.extraFolderPaths.*`).
|
||||
|
||||
### Model source feature (Hugging Face / ModelScope / TensorArt)
|
||||
|
||||
A model file can be linked to the page of an external model site. **Hugging Face**,
|
||||
**ModelScope** and **TensorArt** are brand names and stay Latin in every locale (R3); the
|
||||
generic nouns around them are translated:
|
||||
|
||||
| Term | Rendering |
|
||||
|---|---|
|
||||
| model source | zh-CN 模型来源 · zh-TW 模型來源 · ja モデルソース · ko 모델 소스 · fr source de modèle · de Modellquelle · es fuente de modelo · ru источник модели · he מקור מודל |
|
||||
| model page | zh-CN 模型页面 · zh-TW 模型頁面 · ja モデルページ · ko 모델 페이지 · fr page du modèle · de Modellseite · es página del modelo · ru страница модели · he עמוד המודל |
|
||||
| model card | zh-CN 模型卡 · zh-TW 模型卡 · ja モデルカード · ko 모델 카드 · fr fiche de modèle · de Modellkarte · es ficha de modelo · ru карточка модели · he כרטיס מודל |
|
||||
| AI enrichment (noun) | reuse the existing pair per locale: zh-CN 增强 · zh-TW 增強 · ja 補完 · ko 보강 · fr enrichissement (par IA) · de Anreicherung (KI-) · es enriquecimiento (con IA) · ru обогащение (с помощью ИИ) · he העשרה (AI) |
|
||||
|
||||
`modelCard.actions.viewOnSource` ("View on {source}") follows each locale's existing
|
||||
`viewOnHuggingFace` pattern — de `Auf … ansehen`, ru `Открыть …`, he `צפייה ב-…`,
|
||||
ja `… で見る`, ko `…에서 보기`, zh `在 … 查看`, fr `Voir sur …`, es `Ver en …`. `{source}` is
|
||||
replaced at runtime with the untranslated platform name, so the brand never appears inside the
|
||||
translated text.
|
||||
|
||||
`modals.linkModelSource.enrichNote` states the rule that only sites exposing a readable model
|
||||
card can be enriched and names TensorArt as the current exception. Keep the parenthetical
|
||||
exception in sync if another link-only source is ever added — the sentence is deliberately
|
||||
phrased as a rule, not as an apology for one site.
|
||||
|
||||
The context-menu and bulk-operation enrichment entry points read **"Enrich Metadata with AI"**
|
||||
in `en`, not "Enrich HF Metadata": they cover ModelScope as well, so no locale may reintroduce
|
||||
an `HF` qualifier in `loras.contextMenu.enrichHfAgent` / `loras.bulkOperations.enrichHfAgent`
|
||||
(the key names keep the historical `Hf`; only the values changed).
|
||||
|
||||
### Folder sidebar feature (create / rename / delete folders, empty folders, view options)
|
||||
|
||||
The model-root sidebar manages on-disk folders. "Folder" reuses the noun already fixed in §2
|
||||
(the `folder key` row); the rest is new surface:
|
||||
|
||||
| Term | Rendering |
|
||||
|---|---|
|
||||
| folder | zh-CN 文件夹 · zh-TW 資料夾 · ja フォルダ · ko 폴더 · fr dossier · de Ordner · es carpeta · ru папка · he תיקייה |
|
||||
| model root (as in "no model root is configured") | zh-CN 模型根目录 · zh-TW 模型根目錄 · ja モデルルート · ko 모델 루트 · fr racine de modèle · de Modell-Stammverzeichnis · es raíz de modelo · ru корневая папка моделей · he שורש מודלים — note `sidebar.modelRoot` alone is the shorter 根目录 / 根目錄 / ルート / 루트 / Racine / Stammverzeichnis / Raíz / Корень / שורש |
|
||||
| tree view / list view | zh-CN 树形视图 / 列表视图 · zh-TW 樹狀檢視 / 清單檢視 · ja ツリー表示 / リスト表示 · ko 트리 보기 / 목록 보기 · fr Vue arborescente / Vue liste · de Baumansicht / Listenansicht · es Vista de árbol / Vista de lista · ru Дерево / Список · he תצוגת עץ / תצוגת רשימה |
|
||||
| sidebar | reuse each locale's `sidebar.hideOnThisPage` noun: zh-CN 侧边栏 · zh-TW 側邊欄 · ja サイドバー · ko 사이드바 · fr barre latérale · de Seitenleiste · es barra lateral · ru боковая панель · he סרגל צד |
|
||||
|
||||
Deleting a folder **never cascades over model files** — the backend refuses it and
|
||||
`sidebar.deleteFolderModal.notEmptyMessage` states the rule in every locale, so keep that
|
||||
clause (and its `—`) when the copy is edited. The `{name}` / `{count}` / `{message}` tokens in
|
||||
`sidebar.createFolderResult.*`, `sidebar.deleteFolderResult.*` and `sidebar.renameFolderResult.*`
|
||||
are verbatim §1-R2 placeholders; `successWithFiles` is the only key carrying `{count}`.
|
||||
|
||||
### Chip reordering (model tags / trigger words)
|
||||
|
||||
Model tags and trigger-word chips share a single reorder affordance (drag the chip, or its
|
||||
`⠿` grip where the chip body is click-to-edit), so the copy sits in `common.reorder.dragHandle`
|
||||
instead of a feature namespace. It is used twice per editor: as the grip tooltip and as the
|
||||
hint shown in the edit controls row. There is deliberately **no keyboard shortcut** — an
|
||||
`Alt + Arrow` binding fought the browser's own Alt + Arrow handling and the modal's arrow-key
|
||||
navigation, so reordering is pointer-only and the grip is a decorative, non-focusable
|
||||
affordance. Do not reintroduce a shortcut or a "position X of Y" screen-reader string without
|
||||
re-adding the corresponding keys.
|
||||
|
||||
`dragHandle` is a fragment, not a sentence: it labels both the grip and the hint, so keep it
|
||||
short and imperative and do not append a keyboard hint in any locale.
|
||||
|
||||
| Term | Rendering |
|
||||
|---|---|
|
||||
| drag to reorder | zh-CN 拖拽以调整顺序 · zh-TW 拖曳以調整順序 · ja ドラッグして並べ替え · ko 드래그하여 순서 변경 · fr Glisser pour réordonner · de Zum Neuordnen ziehen · es Arrastra para reordenar · ru Перетащите, чтобы изменить порядок · he גרור כדי לשנות סדר |
|
||||
|
||||
The grip itself is an icon and is never translated.
|
||||
|
||||
---
|
||||
|
||||
## 3. Cross-cutting confusion hot-spots (must-fix list)
|
||||
|
||||
All items below were **resolved** in the 2026-08 sweep — treat them as a regression
|
||||
watch-list: do not reintroduce these renderings.
|
||||
|
||||
1. **Checkpoint rendered as a literal security/road checkpoint** — fr, es, ru, he, zh-CN,
|
||||
zh-TW all had 4–6 keys in the `statistics.*` domain reading as "control point"; reverted
|
||||
to "Checkpoint".
|
||||
2. **"recipe" variants that break the one-noun rule** — fr "recette" → "Recipe", zh
|
||||
食谱/食譜 → 配方, de/ja/ru leftover English "Recipe" translated.
|
||||
3. **ko `header.filter.tagLogicAny`** — was inverted ("모든 태그 일치 (OR)") and identical
|
||||
to `tagLogicAll`; now "어느 하나의 태그와 일치 (OR)".
|
||||
4. **ja `modals.model.versions.actions.viewLocalTooltip`** — was the stale "近日対応予定"
|
||||
("coming soon"); all 9 locales now describe the actual action.
|
||||
5. **Stale help texts** — `settings.downloadSkipBaseModels.help`,
|
||||
`settings.aiProvider.apiBaseHelp`, `settings.hideEarlyAccessUpdates.help` retranslated
|
||||
in all locales to the current `en.json` wording.
|
||||
6. **en.json source bugs** (fixed in source, then mirrored):
|
||||
- "Civitai" → "CivitAI" brand casing (values only; key names `relinkCivitai` etc. keep
|
||||
their lowercase form and must not be renamed)
|
||||
- `modals.relinkCivitai.helpText.format4` "CivitArchive" typo → "CivArchive"
|
||||
- `zh-CN recipes.controls.import.downloadLocationPreview` invented `{path}` removed
|
||||
|
||||
---
|
||||
|
||||
## 4. Placeholder contract deviations (current)
|
||||
|
||||
`{...}` token sets must match `en.json` per key. All deviations found in the 2026-08 sweep
|
||||
were fixed, with one *intentional* exception:
|
||||
|
||||
**`toast.settings.mappingsUpdated`** — the caller passes a hardcoded English inflection
|
||||
(`plural: count !== 1 ? 's' : ''`). Languages that cannot build a plural by appending that
|
||||
`s` (zh-CN/zh-TW, ja, ko, de, ru, he) **drop `{plural}`** and render a count-friendly form
|
||||
(`({count})` or a measure word); fr and es keep it (`mappage{plural}`, `mapeo{plural}`).
|
||||
|
||||
```python
|
||||
# keep a copy of this rule next to the key if it ever moves:
|
||||
# fr/es: "... ({count} mappage{plural})"
|
||||
# de/ru/he: "... ({count})"
|
||||
# zh-CN: "({count} 条映射)" / zh-TW: "({count} 個對應)" / ja: "({count} マッピング)"
|
||||
```
|
||||
|
||||
Do NOT add `{...}` tokens the source lacks (the caller will not supply them, and the literal
|
||||
text renders in the UI), and do NOT rename source tokens (`{typePlural}` stays `{typePlural}`).
|
||||
|
||||
---
|
||||
|
||||
## 5. One term, one rendering — offender matrix
|
||||
|
||||
Cross-locale summary of §2 inconsistencies. "✓" = already consistent. All ✗ cells were
|
||||
resolved in the 2026-08 sweep; the row shows the single rendering now in force per locale.
|
||||
|
||||
| Term | fr | de | es | ru | he | ja | ko | zh-CN | zh-TW |
|
||||
|---|---|---|---|---|---|---|---|---|---|
|
||||
| recipe | Recipe | Rezept | receta | рецепт | מתכון | レシピ | 레시피 | 配方 | 配方 |
|
||||
| Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint |
|
||||
| Embedding | Embedding | Embedding | Embedding | Embedding | Embedding | Embedding | Embedding | Embedding | Embedding |
|
||||
| prompt | Prompt | Prompt | prompt | промпт | פרומפט | プロンプト | 프롬프트 | 提示词 | 提示詞 |
|
||||
| base model | modèle de base | Basismodell | modelo base | базовая модель | מודל בסיס | ベースモデル | 베이스 모델 | 基础模型 | 基礎模型 |
|
||||
| preset | préréglage | Voreinstellung | preajuste | пресет | קביעה מראש | プリセット | 프리셋 | 预设 | 預設 |
|
||||
| workflow | Workflow | Workflow | workflow | Workflow | workflow | ワークフロー | 워크플로 | 工作流 | 工作流 |
|
||||
| hash | hash | Hash | hash | хеш | hash | ハッシュ | 해시 | 哈希 | 雜湊 |
|
||||
| metadata | métadonnées | Metadaten | metadatos | метаданные | מטא-נתונים | メタデータ | 메타데이터 | 元数据 | 中繼資料 |
|
||||
| tags | Tags | Tags | etiquetas | теги | תגיות | タグ | 태그 | 标签 | 標籤 |
|
||||
| duplicates | en double | Duplikate | duplicados | дубликаты | כפילויות | 重複 | 중복 | 重复项 | 重複項 |
|
||||
| bulk | groupé | Massen- | por lotes | пакетный | בכמות גדולה | 一括 | 일괄 | 批量 | 批量 |
|
||||
|
||||
Watch: ja/ko keep the model-type names **Checkpoint/Embedding** and `Diffusion Model` in
|
||||
Latin (consistent with their model-type sections) — do not transliterate them as
|
||||
チェックポイント/체크포인트.
|
||||
|
||||
---
|
||||
|
||||
## 6. Untranslated English leftovers (status)
|
||||
|
||||
Values byte-identical to `en.json` that are actual UI sentences are bugs (brand names and
|
||||
URL placeholders are the exception). As of the 2026-08 sweep, **all previously untranslated
|
||||
blocks are translated** in every locale: `recipes.batchImport.*` + `toast.recipes.batchImport*`
|
||||
(fr/de/es/ru/he/ja/ko), `banners.communitySupport.*`, `modals.model.license.*`,
|
||||
`globalContextMenu.fetchMissingLicenses.*`, the `doctor.*` issue/action/label subset,
|
||||
`toast.settings.libraryLoadFailed` / `libraryActivateFailed`, `toast.api.moveFailed`,
|
||||
`settings.extraFolderPaths.restartRequired`, `toast.recipes.recipeSaved`,
|
||||
`sidebar.dragDrop.moveUnsupported`, `checkpoints.modelTypes.diffusion_model`
|
||||
(ja/ko keep the English loanword), `initialization.recipes.title`.
|
||||
|
||||
The only values that remain intentionally identical to `en.json` are non-translatable:
|
||||
URL/path placeholders (`https://…`, `C:/…`), numeric presets (`5 (1080p), 6 (2K), 8 (4K)`),
|
||||
example token lists (`character, concept, style(toon|toon_style)`), service/provider names
|
||||
(`CivitAI → CivArchive → Archive DB`), model-type names (`settings.priorityTags.modelTypes.*`,
|
||||
`settings.folderSettings.subTypeVae` … `subTypeControlnet` — see §2), and the external playlist
|
||||
title (`help.updateVlogs.playlistTitle`, de: translated to "LoRA Manager-Update-Playlist").
|
||||
|
||||
Rule for `uiHelpers.workflow.noPromptTargets`: the second line (`Mark as → Send Prompt
|
||||
Target`) quotes literal ComfyUI context-menu items — keep those menu labels in English in
|
||||
every locale because that is what the user actually sees in ComfyUI.
|
||||
|
||||
License labels (`modals.model.license.*`): the restriction labels are now translated in all
|
||||
locales (the sibling `creditRequired` has always been translated).
|
||||
|
||||
---
|
||||
|
||||
## 7. Workflow for agents and translators
|
||||
|
||||
### Adding a new UI string
|
||||
1. Add the key to `locales/en.json` only.
|
||||
2. Run `python scripts/sync_translation_keys.py` — it inserts the key into the other 9
|
||||
locales (as a `[TODO: Translate]` placeholder) preserving formatting.
|
||||
3. **During feature development, stop here.** While the UI copy is still in flux, leave the
|
||||
`[TODO: Translate]` placeholders as-is — translating churning strings into 9 locales is
|
||||
wasted work. Placeholders are a normal intermediate state, not a bug.
|
||||
4. Once the wording is final and the feature owner explicitly asks for translations,
|
||||
translate **all** pending `[TODO: Translate]` keys in every locale (not just the latest
|
||||
feature's), applying §1–§3 (placeholders verbatim, Recipe rule, term maps, register).
|
||||
Find pending keys with: `grep -c "TODO: Translate" locales/*.json`
|
||||
5. If the new string contains new terminology, extend §2 tables.
|
||||
|
||||
### Fixing a translation bug
|
||||
1. Locate the key (dotted path) in the relevant locale file.
|
||||
2. Check the corresponding `en.json` value and the actual caller (grep `static/js` or
|
||||
`web/comfyui` for the key) to learn which placeholders are passed.
|
||||
3. Fix trivially; for normalization sweeps (e.g. "recette" → "Recipe"), do it file-wide for
|
||||
the offending keys only — do not touch unrelated lines.
|
||||
4. If the bug is in `en.json` itself (R9), fix the source first, then re-sync and update all
|
||||
locales.
|
||||
|
||||
### Verification
|
||||
```bash
|
||||
pytest tests/i18n/test_i18n.py # key parity + JSON validity + JS key references
|
||||
python scripts/sync_translation_keys.py --dry-run # shows which keys would change; add --verbose for per-key detail
|
||||
npm test # frontend tests incl. i18n helpers
|
||||
```
|
||||
|
||||
`pytest tests/i18n` only checks structure. Quality conventions in this document are not
|
||||
machine-enforced — a human/agent review pass is required.
|
||||
|
||||
### Anti-patterns checklist
|
||||
- [ ] Placeholders `{x}` / `{{x}}` differ from `en.json`
|
||||
- [ ] Same source term translated 2+ ways in the same file (see §5)
|
||||
- [ ] "Checkpoint" became a literal checkpoint; "recipe" became menu/prescription/food-cookbook
|
||||
- [ ] Brand names translated or transliterated (LoRA, CivitAI, ComfyUI, …)
|
||||
- [ ] Latin locale using full-width `:()`; fr using `"` as apostrophe
|
||||
- [ ] Mixed 你/您, du/Sie, tú/usted
|
||||
- [ ] Full English sentences left behind (see §6)
|
||||
- [ ] Register/typos/mojibake; source string is stale vs `en.json` (compare semantics, not
|
||||
just words)
|
||||
@@ -39,6 +39,7 @@ These fields are present in all model metadata files.
|
||||
| `metadata_source` | string\|null | ❌ No | ✅ Yes | Last provider that supplied metadata (see below) |
|
||||
| `last_checked_at` | float | ❌ No (default: `0`) | ✅ Yes | Unix timestamp of last metadata check |
|
||||
| `hash_status` | string | ❌ No (default: `"completed"`) | ✅ Yes | Hash calculation status: `"pending"`, `"calculating"`, `"completed"`, `"failed"` |
|
||||
| `autov3` | string\|null | ❌ No | ✅ Yes | CivitAI AutoV3 hash (first 12 chars, lowercase hex) sourced from the safetensors embedded metadata (`sshs_model_hash` / `modelspec.hash_sha256`). **Absent** = not yet checked (may be backfilled later); **`null`** = checked but unavailable (header has no recognized hash); **12-char hex string** = value |
|
||||
|
||||
---
|
||||
|
||||
@@ -287,6 +288,7 @@ These fields are automatically synchronized with the filesystem:
|
||||
- `preview_url` — Updated if preview file is moved/removed
|
||||
- `sha256` — Updated during hash calculation (when `hash_status="pending"`)
|
||||
- `hash_status` — Updated during hash calculation
|
||||
- `autov3` — Set when metadata is first created (from safetensors header); may be backfilled later for entries where it is absent
|
||||
- `last_checked_at` — Timestamp of scan
|
||||
- `metadata_source` — Set based on metadata provider
|
||||
|
||||
@@ -345,6 +347,7 @@ These fields can be edited by users at any time through the Lora Manager UI or b
|
||||
| `metadata_source` | `null` |
|
||||
| `last_checked_at` | `0` |
|
||||
| `hash_status` | `"completed"` |
|
||||
| `autov3` | absent (not checked) or `null` (checked, no value) |
|
||||
| `usage_tips` | `"{}"` (LoRA only) |
|
||||
| `model_type` | `"checkpoint"` or `"embedding"` (not present in LoRA models) |
|
||||
|
||||
@@ -354,6 +357,7 @@ These fields can be edited by users at any time through the Lora Manager UI or b
|
||||
|
||||
| Version | Date | Changes |
|
||||
|---------|------|---------|
|
||||
| 1.1 | 2026-08 | Added `autov3` field (CivitAI AutoV3 hash with three-state semantics) |
|
||||
| 1.0 | 2026-03 | Initial schema documentation |
|
||||
|
||||
---
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
# Plan: Multi-File Downloads Within a Single CivitAI Model Version
|
||||
|
||||
**Issue:** [#1058 — Cannot download multiple file variants from the same model version](https://github.com/willmiao/ComfyUI-Lora-Manager/issues/1058)
|
||||
**Status:** v2 — revised after adversarial review (backend correctness + frontend/tests)
|
||||
**Scope:** CivitAI/CivArchive downloads of `lora`, `checkpoint`, `embedding` model types. HuggingFace downloads are out of scope (already per-file).
|
||||
|
||||
> v2 changelog: incorporated 18 review findings. Key changes vs v1:
|
||||
> shared file resolver + `resolved_version_id` for the gate (R1); `file_params` normalization at API boundary (R2); D2 hash-matching rule fixed for empty-hash cases (R6/R7); D3 extended to re-point `version_index` on removal (R4); D4 replaced with a child table (R3); `delete_model_version` interaction documented (R5); `ModelVersionsTab` surface added to phase 2 (F6); phase-2 multi-file loop requires a reload-deferred download variant (F7); queue-retry `file_params=NULL` known issue recorded (R9); test-fixture gaps and revised estimates (F10).
|
||||
|
||||
---
|
||||
|
||||
## 1. Problem Statement
|
||||
|
||||
A CivitAI model version can contain multiple downloadable weight files (e.g. fp16/fp32, safetensors/ckpt, different sizes). LoRA Manager already has a working file-selection pipeline (frontend file dialog → `fileParams` → backend file matching), but downloaded state is tracked at the **model-version** level. After any single file of a version is downloaded:
|
||||
|
||||
1. The version is marked **In Library** and the file-selection entry point disappears.
|
||||
2. The backend rejects further download attempts for that version.
|
||||
|
||||
There is no way to download the remaining files of the same version through LoRA Manager.
|
||||
|
||||
## 2. Current State (verified against code; all references confirmed by review)
|
||||
|
||||
### 2.1 Download gating — backend (`py/services/download_manager.py`)
|
||||
|
||||
`_execute_original_download` enforces two version-level gates:
|
||||
|
||||
- **Library gate, early** (lines 1157–1184, before metadata fetch, fires when `model_version_id` given) and **late** (lines 1350–1376, fires only when `model_version_id is None`): `scanner.check_model_version_exists(version_id)` across lora/checkpoint/embedding scanners → hard error `"Model version already exists in ... library"`.
|
||||
- **History gate** (lines 1238–1279): when `skip_previously_downloaded_model_versions` setting is on, `_has_been_downloaded(model_type, version_id)` → silent skip. History DB primary key is `(model_type, version_id)` (`py/services/downloaded_version_history_service.py:61`).
|
||||
|
||||
File selection works: `file_params {id, type, format, size, fp}` is matched against `version_info.files` (lines 1498–1569), **but only under `if file_params and model_version_id:` (line 1499)** — with `model_id`-only requests the selection silently falls back to the primary file (1571–1619). `file_params` currently carries no file `name` or hash.
|
||||
|
||||
### 2.2 Downloaded-state surfacing — backend (`py/routes/handlers/model_handlers.py`)
|
||||
|
||||
`get_civitai_versions` (lines 2148–2188) sets per-version `existsLocally` via `cache.version_index.get(version_id)` (plus a single `localPath` from that entry) and `hasBeenDownloaded` via the history service. No per-file granularity.
|
||||
|
||||
### 2.3 Frontend blockers (`static/js/managers/DownloadManager.js`)
|
||||
|
||||
Three independent gates prevent re-entering the file dialog:
|
||||
|
||||
1. **Line 598:** file-select badge rendered only when `modelFiles.length > 1 && !existsLocally`.
|
||||
2. **Lines 666–681 (`updateNextButtonState`):** Next button disabled with "Already in Library" when `currentVersion.existsLocally`.
|
||||
3. **Lines 784–787 (`proceedToLocation`):** toast + abort when `currentVersion.existsLocally`.
|
||||
|
||||
The badge path (`confirmFileSelection` lines 737–759 → `proceedToLocationContent` → `startDownload` single mode → `executeDownloadWithProgress` → POST `file_params`, `static/js/api/baseModelApi.js:1236–1250`) has **zero** `existsLocally` guards (all 12 occurrences enumerated; none on this path; `import/DownloadManager.js` has none either). The `.exists-locally` CSS class is purely visual (`download-modal.css:496–499`). **Making the badge visible again is sufficient to unlock the flow** for phase 1.
|
||||
|
||||
Post-download refresh is clean: the modal closes and `resetAndReload(true)` performs a full library refetch (`DownloadManager.js:1063`); dialog reopen resets state and refetches versions with no client-side cache. No same-session staleness.
|
||||
|
||||
### 2.4 Local identity of the downloaded file
|
||||
|
||||
`LoraMetadata/CheckpointMetadata/EmbeddingMetadata.from_civitai_info(version_info, file_info, ...)` (`py/utils/models.py:245–369`) persists:
|
||||
|
||||
- `sha256` = `file_info.hashes.SHA256` (lowercased, defaults to `""`) — a stable per-file identity;
|
||||
- `civitai` = the full `version_info` payload (including the `files` list).
|
||||
|
||||
Metadata refresh (`metadata_sync_service.py:104–105`) replaces the `civitai` blob wholesale but never overwrites top-level `sha256`; `verify_duplicate_hashes` (481–526) corrects it to the on-disk hash. Top-level-sha256 matching is refresh-robust.
|
||||
|
||||
**Caveats (review R6/R7):**
|
||||
- SHA256 is not guaranteed: CivArchive's transform only sets `hashes` when source data carries it (`civarchive_client.py:185–189`); `from_civitai_info` defaults to `""`.
|
||||
- Name fallback is unreliable exactly when it matters: local `file_name` is extension-less (`models.py:264`) and `generate_unique_filename` rewrites it with a hash suffix on conflict (`download_manager.py:1125–1136`); checkpoints with `hash_status='pending'` keep empty sha256 until on-demand hashing (`model_scanner.py:1232–1240`).
|
||||
|
||||
### 2.5 Version index collision (pre-existing hazard)
|
||||
|
||||
`ModelCache.version_index` is single-valued (`model_cache.py:133`: `version_index[version_id] = item`). Two files of the same version in the library → second entry overwrites the first; `remove_from_version_index` (lines 151–181) drops the whole version key when the indexed entry is removed, even if a sibling file remains. ~10 read sites depend on this index (48 grep touch points total; readers include `recipe_scanner.py:2682–2726`, `recipe_format.py:37–40`, `misc_handlers.py:2440–2444`, `model_handlers.py`, `model_scanner.check_model_version_exists:2444`).
|
||||
|
||||
Review correction (F3): bulk paths `remove_models` (`model_scanner.py:2376`) and `update_single_model_cache` (`:1689`) call `rebuild_version_index()` right after, so a sibling re-enters the index in those flows — the hazard is narrower than v1 stated, but direct `remove_from_version_index` callers (e.g. `model_scanner.py:1018`) still drop the key, and the user-visible artifact in phase 1 is real: `localPath` in the dialog flips to whichever file was indexed last.
|
||||
|
||||
### 2.6 Entry points that send / don't send `file_params` (fully enumerated by review)
|
||||
|
||||
**Send `file_params` (user-initiated dialog flows only):** `DownloadManager.js:1611–1639` (single mode). API surface accepting arbitrary JSON `file_params`: GET `/api/lm/download-model-get` (`model_handlers.py:1634–1686`), POST `/api/lm/downloads/queue/add` (`model_handlers.py:1799–1832`).
|
||||
|
||||
**Never send `file_params` (keep version-level semantics):** batch download (`DownloadManager.js:1756–1766`; batch also filters out in-library versions at `:1648`), `downloadVersionWithDefaults` (`:1810–1830`), recipe import (`import/DownloadManager.js:269–276`), bulk missing-LoRA (`BulkMissingLoraDownloadManager.js:292–299`), `RecipeModal.js:1728–1736`, `ModelVersionsTab.js:1427`. `web/comfyui/` and `vue-widgets/src` contain **no** download triggers at all (grep-verified). `py/services/use_cases/` has only `download_model_use_case.py` (pass-through).
|
||||
|
||||
### 2.7 Paths that do NOT need changes (verified)
|
||||
|
||||
- **aria2 pause/resume** (`_resume_restored_aria2_download`, line 754+): resumes from persisted `resume_context`; never re-runs existence gates.
|
||||
- **`download_coordinator.py:90`**: pure pass-through of `file_params`.
|
||||
- **Update checker / plugin self-update** (`update_routes.py:496–501`): only closes the history DB handle.
|
||||
- **History delete semantics**: `mark_as_deleted` sets `is_deleted_override=1` and `has_been_downloaded` then returns False (`downloaded_version_history_service.py:276`) — LM-initiated deletes already reset the history skip.
|
||||
|
||||
### 2.8 Related pre-existing issues (record, not necessarily fix)
|
||||
|
||||
- **Queue retry drops file selection** (R9): `download_queue_service.retry_from_history` / `retry_all_failed` re-queue with `file_params=NULL` (`download_queue_service.py:705, 758`) although the queue table has a `file_params` column (`:43`) — a retried non-primary download silently reverts to the primary file. Fix alongside phase 1 (small: persist and reuse the column).
|
||||
- **`delete_model_version`** (`misc_handlers.py:2410–2487`): resolves the file via the single-valued `version_index` (2440–2444), deletes only that one file, and `mark_as_deleted` flags the **entire version** as deleted in history (2479) even when a sibling file remains in the library. See phase 2 item 6.1.5.
|
||||
|
||||
## 3. Goals / Non-Goals
|
||||
|
||||
**Goals**
|
||||
|
||||
- G1: A user can download any not-yet-downloaded file of a version already partially in the library (issue repro steps 6–8).
|
||||
- G2: True duplicates stay blocked: downloading the *same* file of the same version twice is rejected.
|
||||
- G3: Per-file downloaded state visible in the file dialog; multiple files selectable and downloadable in one pass.
|
||||
- G4: No regression for version-level semantics relied on by batch download, recipe missing-LoRA detection, and `skip_previously_downloaded_model_versions`.
|
||||
|
||||
**Non-Goals**
|
||||
|
||||
- No change to recipe `inLibrary` semantics ("any file of the version present" remains sufficient).
|
||||
- No change to the update-checker (version-level comparison).
|
||||
- No primary-key rebuild of the history database.
|
||||
- HuggingFace download flow untouched.
|
||||
|
||||
## 4. Design Decisions
|
||||
|
||||
- **D1 — Explicit file selection bypasses the history gate, version-level gates stay for everyone else.** The history skip exists to dedupe automated flows. A user explicitly picking a file is unambiguous intent; the file-level library gate (G2) still prevents real duplicates. **Guard conditions use normalized truthiness** (see D1a). All confirmed `file_params` senders are user-initiated dialog flows (2.6), and LM-initiated deletes already reset history (2.7), so the bypass only affects "downloaded but not LM-deleted" versions with the setting on — intended.
|
||||
- **D1a — `file_params` normalization at the boundary (R2).** `download-model-get` and `downloads/queue/add` accept arbitrary JSON; `{}` is `not None` but falsy and would bypass gates while downloading the primary file. Normalize `file_params = file_params or None` in the coordinator/handlers, and treat the bypass as active only when a target file id is resolvable.
|
||||
- **D2 — File identity matching rule (R6/R7):** hash-compare **only when both sides are non-empty** (lowercase SHA256 equality); name-compare when either side is empty. Never let `"" == ""` match. Name fallback caveats from 2.4 apply (renamed files, pending checkpoint hashes) — acceptable residual risk, worst case is a blocked re-download the user can retry after hashing completes.
|
||||
- **D3 — Cache indexes: additive multi-index + removal re-pointing (R4).** Add `version_files_index: Dict[int, List[dict]]` maintained alongside `version_index` by the same add/remove/rebuild methods; existing readers of `version_index` untouched. Additionally fix `remove_from_version_index`: when the popped entry has a surviving sibling (per the multi-index), re-point `version_index[version_id]` to the sibling instead of dropping the key; same for the `model_id_index` descriptor. This closes the 2.5 hazard for existing readers (`check_model_version_exists`, `existsLocally`, recipe matching) without restructuring anything.
|
||||
- **D4 — Per-file history via a child table (R3).** v1's additive-column approach is structurally impossible on a `(model_type, version_id)` PK (`ON CONFLICT DO UPDATE` would keep only the last file). Instead add `downloaded_version_files(model_type, version_id, file_id, file_name, downloaded_at, PRIMARY KEY(model_type, version_id, file_id))` — additive, no PK rebuild, honors the Non-Goal. Existing version-level table and queries unchanged. New per-file queries are opt-in. `_initialize_schema` uses `CREATE TABLE IF NOT EXISTS`, so the new table is created for existing DBs without any ALTER.
|
||||
- **D5 — UI flow reuse, with an extracted inner download function for multi-file (F7).** Phase 1 unlocks the existing badge → file dialog → location → download pipeline. Phase 2 upgrades the dialog to multi-select; iterating `executeDownloadWithProgress` as-is would produce N full library reloads, N toasts, and competing failure-summary modals — so phase 2 extracts a reload-deferred, failure-aggregating inner variant and runs one reload + one summary at the end.
|
||||
|
||||
## 5. Implementation — Phase 1 (fix the issue; independently shippable)
|
||||
|
||||
### 5.1 Backend — `py/services/download_manager.py`
|
||||
|
||||
1. **Normalize `file_params`** at the boundary (D1a): `download_coordinator.schedule_download` and the two API handlers (`model_handlers.py:1649–1666`, `1810–1832`) apply `file_params = file_params or None`.
|
||||
2. **Extract a shared file resolver** (R1): pull the matching logic at 1498–1569 into `_resolve_target_file(version_info, file_params) -> Optional[dict]`, used by **both** the new gate and the download-selection path. The selection path's condition (line 1499) switches from `model_version_id` to `resolved_version_id` (already computed at 1230–1236 from `version_info.id`), so gate and download always agree on the target file — including the `model_id`-only case.
|
||||
3. **New helper** `_find_local_file_entry(version_id, target_file) -> Optional[dict]`: iterate the three scanners' cached `raw_data` (NOT `version_index` — single-valued); candidates = entries whose `civitai.id` normalizes to `version_id`; match per D2.
|
||||
4. **Gate restructure in `_execute_original_download`**:
|
||||
- Early scanner gate (1157–1184): add `file_params is None` guard; with normalized `file_params`, defer (file identity not resolvable before metadata fetch).
|
||||
- After `version_info` fetch + `resolved_version_id` (~1229): when `file_params` present, resolve target file via the shared resolver; unresolvable → hard error "No matching file" (fail closed, prevents empty-dict bypass). Resolvable → `_find_local_file_entry`; hit → same hard error shape as today with the file name in the message.
|
||||
- History gate (1238–1279): add `file_params is None` (D1). Base-model skip (1281–1324) unchanged — still applies.
|
||||
- Late gate (1350–1376): add `file_params is None` guard (F2) — the post-fetch file-level check above already covers this case.
|
||||
- Nothing between the early gate and the post-fetch point assumes the version is absent (review task 6: only provider selection + metadata fetch; no DB writes; `_persist_aria2_state` runs only when actually downloading at 1659).
|
||||
5. **Queue retry fix** (2.8, small): persist `file_params` into the queue table on enqueue and reuse it in `retry_from_history` / `retry_all_failed`.
|
||||
6. Logging: `[download]` lines for file-level allow/block, consistent with existing style.
|
||||
|
||||
**Estimated:** ~150–220 LOC + resolver extraction.
|
||||
|
||||
### 5.2 Frontend — `static/js/managers/DownloadManager.js`
|
||||
|
||||
1. Line 598: drop `&& !existsLocally` from the badge condition (badge shows whenever `modelFiles.length > 1`).
|
||||
2. `fileParams` construction (1611–1616): add `name: this.selectedFile.name`.
|
||||
3. Surface the backend "file already in library" hard error as a toast instead of only the batch-summary modal (R10/F12 nit; reuse existing error message field).
|
||||
4. No changes to `updateNextButtonState` / `proceedToLocation` in phase 1; no template or CSS changes.
|
||||
|
||||
**Known phase-1 UX limitations (acknowledged, fixed in phase 2):** with all files downloaded the badge still renders and re-picking a downloaded file fails late (backend error after the location step); `localPath` may point at a sibling file; batch-preview "In Library" badge stays version-level and gives no hint of remaining files.
|
||||
|
||||
**Estimated:** ~10–30 LOC (confirmed realistic by review).
|
||||
|
||||
### 5.3 Phase 1 tests
|
||||
|
||||
Backend — extend `tests/services/test_download_manager_basic.py` (1694 lines; all fixture patterns exist):
|
||||
|
||||
- **Fixture gaps to add (F10):** `DummyScanner.get_cached_data()`/`raw_data` stub (~10 lines); `hashes.SHA256` in the metadata-provider payload's `files`.
|
||||
- Cases: same version + different SHA256 in library + `file_params` → proceeds; same SHA256 → hard error; `file_params=None` + version in library → hard error (unchanged); history-skip on + `file_params` → not skipped; without → skipped (unchanged); empty-dict `file_params` normalized → version-level behavior; `model_id`-only + `file_params` → gate and selection resolve the same file; legacy metadata (empty local sha256) matched by name; target file with empty SHA256 → name fallback, no `""==""` false positive.
|
||||
- Queue retry: `file_params` survives retry.
|
||||
- Assert proceed/abort via the existing `_execute_download` mock pattern.
|
||||
|
||||
Frontend (`tests/frontend/`): badge renders for multi-file version with `existsLocally=true` (pattern from `downloadManager.history.test.js`).
|
||||
|
||||
**Estimated:** ~150–250 LOC (confirmed realistic).
|
||||
|
||||
## 6. Implementation — Phase 2 (per-file status + multi-select + index hardening)
|
||||
|
||||
### 6.1 Backend
|
||||
|
||||
1. **`py/services/model_cache.py`** (D3): add `version_files_index`; maintain in `add_to_version_index` / `remove_from_version_index` / `rebuild_version_index`; removal re-points `version_index[version_id]` (and the `model_id_index` descriptor) to a surviving sibling instead of dropping the key.
|
||||
2. **`py/services/model_scanner.py`**: expose `get_files_for_version(version_id) -> List[dict]`.
|
||||
3. **`py/routes/handlers/model_handlers.py` `get_civitai_versions`**: annotate each version with `downloadedFiles: [{fileId, fileName, filePath}]` via `version_files_index` + D2 matching against `version.files`.
|
||||
4. **`py/services/downloaded_version_history_service.py`** (D4): new child table `downloaded_version_files`; `mark_downloaded` also upserts the child row when `file_id` known; `mark_as_deleted` clears the version's child rows only when no sibling remains in the library; new `get_downloaded_file_ids(model_type, version_id) -> set[int]`. `_record_downloaded_version_history` passes `file_info` through.
|
||||
5. **`delete_model_version`** (`misc_handlers.py:2410–2487`, R5): resolve **all** local files of the version via `version_files_index`; delete all (current endpoint semantics are version-level) or — if kept per-file — only `mark_as_deleted` when no sibling remains. Decide at implementation time; minimum is documenting current behavior.
|
||||
6. **`ModelVersionsTab` backend support**: none needed beyond item 3 (`downloadedFiles`); the tab consumes the same versions payload.
|
||||
|
||||
### 6.2 Frontend
|
||||
|
||||
1. **File dialog multi-select** — change surface (F8): option markup (`DownloadManager.js:712–724`), the single-select click handler (`727–734`), the `input[type="radio"]:checked` selector in `confirmFileSelection` (`738`); template `templates/components/modals/download_modal.html:48–60` (confirm-button label only); CSS `download-modal.css` — checkbox variant of `.file-option-radio input` (595–604) and a **new** `.file-option.disabled` style (does not exist). Files whose id ∈ `downloadedFiles` render disabled with an "In Library" tag.
|
||||
2. **Mixed-type guard (F8):** multi-select is restricted to files sharing the same routing target (`_isDiffusionModel` is computed once from a single `selectedFile` at 798–803; e.g. "Model" + "UNet" files route to different roots). Disallow mixed-type multi-select (simplest, predictable); single-file selection unchanged.
|
||||
3. **Multi-file download loop (D5/F7):** extract from `executeDownloadWithProgress` a reload-deferred, no-toast inner function; iterate per selected file with per-file progress; one `resetAndReload(true)` + one aggregated success/failure summary at the end (reuse `showDownloadBatchSummary`).
|
||||
4. **`updateNextButtonState` / `proceedToLocation`:** for multi-file versions, Next routes into the file dialog; hard block only when *every* weight file is downloaded.
|
||||
5. **`ModelVersionsTab.js` (F6):** the Download action (`:576` hidden when `isInLibrary`) — for multi-file versions with remaining files, show it and route into the download modal's file dialog; keep hidden when all files present.
|
||||
6. **Batch preview (F5):** `batch-preview-local-badge` (`:1320`) gains a "partially downloaded" hint for multi-file versions with remaining files.
|
||||
7. New i18n keys (`modals.download.fileSelection.inLibrary`, `downloadSelected`, partial-download tooltip, etc.) → run `python scripts/sync_translation_keys.py`.
|
||||
|
||||
### 6.3 Phase 2 tests
|
||||
|
||||
- `model_cache` (`tests/services/test_model_cache.py` already covers add/remove at 44–55): multi-valued index; sibling re-point on removal; rebuild.
|
||||
- `get_civitai_versions`: `downloadedFiles` correctness (hash match, name fallback, no match, CivArchive no-hash payload).
|
||||
- History service (`tests/services/test_downloaded_version_history_service.py` uses real SQLite on tmp_path): child-table creation on a legacy DB; per-file record/query; `mark_as_deleted` sibling semantics.
|
||||
- Frontend: dialog checkbox rendering/disabled state and multi-file confirm — **greenfield behavior coverage** (F10: no existing test exercises `showFileSelectionStep`/`confirmFileSelection`; infra exists, patterns must be built).
|
||||
|
||||
## 7. Risks and Mitigations
|
||||
|
||||
| Risk | Impact | Mitigation |
|
||||
|---|---|---|
|
||||
| History-gate bypass (D1) causes unwanted re-downloads in automated flows | Large checkpoint files re-downloaded | Bypass only with normalized, resolvable `file_params` (D1a); all such senders are user-initiated dialog flows (2.6, verified); tests pin batch/recipe/bulk behavior. |
|
||||
| Empty-hash matching edge cases (R6) | Duplicate download of the same file, or false block | D2 rule: hash only when both non-empty; name otherwise; never `""==""`. Residual risk documented (2.4). |
|
||||
| Phase-1 late-failure UX (F12) | User picks a downloaded file, fails only after location step | Toast surfacing (5.2.3); phase 2 disables downloaded files up front. |
|
||||
| Phase-2 index change corrupts existing behavior | Recipe matching, delete flows | Additive index + re-point only; `version_index` read semantics unchanged; `remove_models`/`update_single_model_cache` already rebuild (F3); tests. |
|
||||
| `delete_model_version` marks whole version deleted while sibling remains (R5) | History wrongly suppresses re-download of the surviving sibling's version | Phase 2 item 6.1.5; documented until then. |
|
||||
| History child-table migration failure on user installs | Service init crash | `CREATE TABLE IF NOT EXISTS` in `_initialize_schema`; failure degrades to version-level behavior (per-file queries return empty). |
|
||||
| Batch-preview badge misleading for partial versions (F5) | Minor UX confusion | Acknowledged in phase 1; fixed in phase 2 item 6.2.6. |
|
||||
| UI confusion: version shows "In Library" while files remain downloadable | Support burden | Phase 2: per-file disabled state + partial-download tooltip. |
|
||||
| Hash-identical sibling files (repacked content) | Second file blocked | Acceptable: scanner hash dedup already collapses them. |
|
||||
|
||||
## 8. Rollout
|
||||
|
||||
1. **Commit 1** — `fix(download): allow downloading additional files of an in-library model version (#1058)` → Phase 1 (5.1–5.3).
|
||||
2. **Commit 2** — `feat(download): per-file download status and multi-file selection (#1058)` → Phase 2 (6.1–6.3).
|
||||
|
||||
Phase 1 alone resolves the issue as reported; phase 2 can ship in a later release if review prefers smaller increments.
|
||||
|
||||
## 9. Effort Estimate (revised after review)
|
||||
|
||||
| Phase | Backend | Frontend | Tests | Risk |
|
||||
|---|---|---|---|---|
|
||||
| 1 | ~150–220 LOC (+ queue-retry fix ~30) | ~10–30 LOC | ~150–250 LOC | Low |
|
||||
| 2 | ~250–350 LOC | ~250–350 LOC (multi-file loop refactor + ModelVersionsTab + batch badge) | ~250–350 LOC (dialog tests greenfield) | Medium |
|
||||
@@ -0,0 +1,337 @@
|
||||
# Plan: Global Rate-Limit Abidance for Recipe Ingest & Metadata Fetching
|
||||
|
||||
**Issue:** [#1085 — Large Recipe Ingest Appears to not abide by vendor rate limits, possibly a few other errors?](https://github.com/willmiao/ComfyUI-Lora-Manager/issues/1085)
|
||||
**Status:** v2 — reviewed; decisions recorded in §10. **Phase 1 implemented**
|
||||
(2026-08-27, commit `c2a2048c`): coordinator + downloader gate + Fix C
|
||||
failover semantics + helper double-wait fix + settings. **Phase 2
|
||||
implemented** (2026-08-27): batch-import rate-limit failures map to
|
||||
`SKIPPED` + `rate_limited` WebSocket flag + UI slowdown hint (toast + status
|
||||
text, i18n keys synced); `download_to_memory` / `get_response_headers` /
|
||||
`download_file` register 429 cooldowns. Changes vs v1: Fix C moved to
|
||||
Phase 1, helper double-wait resolved in Phase 1, gate/guard ordering
|
||||
specified.
|
||||
**Scope:** HTTP API traffic to CivitAI (`civitai.red`) and CivArchive (`civarchive.com`) from metadata fetching (bulk refresh, metadata sync, recipe analysis/enrichment, usage-control lookups). Large binary downloads (model files / preview images via `download_file`) are out of scope for *pacing* (they are already single-connection transfers) but their 429 responses should still be *registered*.
|
||||
|
||||
> Context: a first batch of fixes for this issue was already committed as
|
||||
> `ee233548` ("fix(recipes): enforce batch-import concurrency bound and harden
|
||||
> ingest errors (#1085)"): the batch-import concurrency controller now shares a
|
||||
> real semaphore (bounds 1–5 actually apply), the Comfy parser tolerates
|
||||
> list/`None` `ckpt_name`, CivArchive treats empty error payloads as failures,
|
||||
> and offline-cooldown short-circuits log at DEBUG. This plan covers the two
|
||||
> remaining orchestration-level fixes:
|
||||
> **Fix 2** — slow down globally when a vendor rate limit is hit (respect
|
||||
> `Retry-After`, queue instead of hammering); **Fix 3** — stop immediately
|
||||
> failing over to CivArchive when CivitAI is rate-limited.
|
||||
|
||||
---
|
||||
|
||||
## 1. Problem Statement
|
||||
|
||||
During a large recipe ingest (e.g. importing the example-images directory,
|
||||
which can be thousands of images), the manager fires one metadata request per
|
||||
checkpoint + per LoRA per image through the fallback provider chain
|
||||
(`civitai_api → civarchive_api → sqlite`). Consequences observed in #1085:
|
||||
|
||||
1. **CivitAI gets hammered** → 429s. The consumer then *immediately* tries
|
||||
CivArchive for the same lookup, so **CivArchive gets hammered too** before
|
||||
it was ever naturally needed (its only real job is recovering metadata for
|
||||
models deleted from CivitAI).
|
||||
2. Requests are retried per-call after `Retry-After`, but **each concurrent
|
||||
call sleeps independently** → thundering herd: thousands of coroutines wake
|
||||
at the same moment and re-flood the vendor.
|
||||
3. While CivArchive is in the `ConnectivityGuard` cooldown, every batch item
|
||||
short-circuits and is marked `FAILED` — the batch import's success/failure
|
||||
accounting is polluted by a transient vendor state (log spam was fixed in
|
||||
`ee233548`; the item-failure accounting is not).
|
||||
4. `ConnectivityGuard` (`py/services/connectivity_guard.py`) only treats
|
||||
transport-level unreachability as offline; **HTTP 429 is invisible to it**,
|
||||
so nothing ever intentionally paces request rate.
|
||||
|
||||
User expectation from the issue: *"once a vendor rate limit time out is hit,
|
||||
you should trigger a slow down with intentional reduction in request rate"*.
|
||||
|
||||
## 2. Current State (verified against code)
|
||||
|
||||
### 2.1 Where 429s are surfaced
|
||||
|
||||
- `Downloader.make_request` (`py/services/downloader.py:1120-1132`): HTTP 429 →
|
||||
returns `RateLimitError(message, retry_after=…)` parsed from `Retry-After`
|
||||
(missing header defaults to `None`).
|
||||
- `CivitaiClient._make_request` (`py/services/civitai_client.py:97-100`):
|
||||
converts `RateLimitError` to a raise immediately; no waiting. Transient
|
||||
5xx/connection errors are retried 3× with 1s/2s/4s backoff.
|
||||
- `CivArchiveClient._make_request` (`py/services/civarchive_client.py`):
|
||||
raises `RateLimitError` with `provider="civarchive_api"` when not set.
|
||||
- `_RateLimitRetryHelper` (`py/services/model_metadata_provider.py:45-102`):
|
||||
per-call retry loop — sleeps `retry_after` (capped at 1800 s; `≥120 s` ⇒ no
|
||||
retry), then re-raises. Because every concurrent call runs its own helper,
|
||||
they sleep in parallel and re-fire in parallel.
|
||||
- `FallbackMetadataProvider` (`py/services/model_metadata_provider.py:488-508,
|
||||
564-584` etc.): on a final `RateLimitError` from one provider it logs
|
||||
"skipping to next provider" and **continues to the next network provider** —
|
||||
this is the direct cause of the CivArchive flood.
|
||||
- `MetadataSyncService.fetch_and_update_model`
|
||||
(`py/services/metadata_sync_service.py:248-333`): manually iterates
|
||||
`provider_attempts`; on `RateLimitError` it `continue`s to the next provider
|
||||
(same failover problem), then reports `"Rate limited"` when nothing
|
||||
succeeded.
|
||||
- `Downloader.make_request` has a per-destination scope already available:
|
||||
`_guard_destination(url)` returns the hostname (`downloader.py:1194-1199`),
|
||||
used by `ConnectivityGuard`.
|
||||
|
||||
### 2.2 What pacing exists today
|
||||
|
||||
- `ConnectivityGuard`: per-destination cooldown (30 s base, ×2 per extra
|
||||
failure batch, 300 s cap) triggered only by transport errors
|
||||
(`connectivity_guard.py:168-197`).
|
||||
- `AdaptiveConcurrencyController` (batch import, fixed in `ee233548`): shared
|
||||
semaphore enforces 1–5 concurrent items; *duration*-based adjustment only —
|
||||
it never sees HTTP statuses, so it cannot distinguish "slow because rate
|
||||
limited" from "slow because big image".
|
||||
- No token bucket, no minimum inter-request interval, no shared
|
||||
`Retry-After` gate anywhere (`grep` for throttle/token-bucket/rate-limiter:
|
||||
0 hits).
|
||||
|
||||
## 3. Requirements & Constraints
|
||||
|
||||
R1. **Respect `Retry-After`.** After a 429, no further request to that
|
||||
destination may be sent before the vendor's retry window elapses.
|
||||
R2. **No thundering herd.** Concurrent waiters must share one wake-up (gate),
|
||||
not sleep independently.
|
||||
R3. **No double load.** A CivitAI 429 must not trigger a CivArchive request
|
||||
for the same lookup. CivArchive should only be consulted when CivitAI
|
||||
legitimately has no answer (404 / "not found"), or when CivitAI is
|
||||
unreachable long-term.
|
||||
R4. **No spurious item failures.** A rate-limited request must not turn a
|
||||
batch-import item into `FAILED`; it should wait (bounded) and retry, or at
|
||||
worst be `SKIPPED` with a clear "rate limited" reason (re-runnable import).
|
||||
R5. **Never hang forever.** All waiting is bounded by a configurable cap; on
|
||||
expiry the caller receives the `RateLimitError` and can decide.
|
||||
R6. **Keep legitimate failover.** Deleted-model recovery via CivArchive/sqlite
|
||||
must keep working (404 paths unchanged).
|
||||
R7. **Single choke point.** The pacing gate should live where every API call
|
||||
passes (the `Downloader`), so bulk refresh, metadata sync, recipe
|
||||
analysis, and usage-control lookups all benefit without per-feature work.
|
||||
|
||||
## 4. Approach Comparison
|
||||
|
||||
### A. Reactive gate — shared `Retry-After` deadman clock (recommended core)
|
||||
|
||||
A process-wide, per-destination coordinator records the *next-allowed-send*
|
||||
timestamp from each 429 (`now + max(retry_after, backoff)`). Every request
|
||||
through `Downloader.make_request` consults the gate *before sending* and *when
|
||||
a 429 arrives*; waiters block on a shared `asyncio.Event` that fires when the
|
||||
cooldown expires.
|
||||
|
||||
- Pros: single choke point (R7); herd-free (R2); honors server guidance (R1);
|
||||
no guessing at vendor limits; covers all providers automatically; reuses
|
||||
existing per-destination scoping.
|
||||
- Cons: still experiences 429s before slowing down (reactive); long
|
||||
`Retry-After` windows (CivArchive has been observed at ~1500 s) need a sane
|
||||
wait cap + skip/retry UX.
|
||||
|
||||
### B. Preemptive pacing — minimum inter-request interval (recommended companion)
|
||||
|
||||
Per-destination token bucket (simplest form: capacity 1 — at least `N` seconds
|
||||
between consecutive API requests; `N` configurable, default ~0.75 s ≈ 80
|
||||
r/min ceiling).
|
||||
|
||||
- Pros: prevents most 429s before they happen — exactly the "intentional
|
||||
reduction in request rate" the issue asks for; trivial to implement on top
|
||||
of A's coordinator.
|
||||
- Cons: adds latency to bulk operations (thousands of models × `N`); the *exact*
|
||||
vendor limits are unknown (CivitAI anonymous vs keyed vs `civitai.red`
|
||||
mirror differ), so the default must be conservative-but-not-crippling and
|
||||
settings-tunable.
|
||||
|
||||
### C. Fallback semantics change — stop network→network failover on 429 (must-do, low risk)
|
||||
|
||||
`FallbackMetadataProvider` (and `MetadataSyncService.fetch_and_update_model`'s
|
||||
manual loop) must treat a final `RateLimitError` as a **terminal, non-failover
|
||||
result** for network providers. Local-only providers (sqlite archive DB) may
|
||||
stay as a last resort (no vendor cost).
|
||||
|
||||
- Pros: directly removes the CivArchive flood; small, surgical change.
|
||||
- Cons: none significant; requires care to keep 404-failover intact (R6).
|
||||
|
||||
### Rejected / deferred
|
||||
|
||||
- **Per-feature retry queues** (batch import pauses & resumes whole batches):
|
||||
richer UX but much larger change (batch state machine, WebSocket states);
|
||||
unnecessary once A+B make requests wait at the choke point. Defer unless
|
||||
review finds the bounded-wait UX insufficient.
|
||||
- **Full token bucket with burst credit**: overkill; capacity-1 interval is
|
||||
enough given the shared semaphore already caps concurrency at 5.
|
||||
- **Retrying in `connectivity_guard`**: wrong layer — the guard is about
|
||||
transport reachability, not vendor quota.
|
||||
|
||||
## 5. Recommended Architecture
|
||||
|
||||
New singleton **`RateLimitCoordinator`** (`py/services/rate_limit_coordinator.py`,
|
||||
mirroring `ConnectivityGuard`'s singleton + per-destination patterns):
|
||||
|
||||
```
|
||||
state per destination (hostname):
|
||||
next_allowed_send: float (monotonic) # from 429 Retry-After + backoff
|
||||
consecutive_429: int # for backoff growth
|
||||
last_send_at: float # for min-interval pacing
|
||||
waiters: list[Future] | asyncio.Event # shared wake-up per cooldown cycle
|
||||
```
|
||||
|
||||
API:
|
||||
|
||||
- `async wait_for_slot(destination, request_started_within_window: bool)`
|
||||
— called by `Downloader.make_request` *before* sending (blocks until
|
||||
`min(now >= next_allowed_send)` and inter-request interval elapses) and
|
||||
re-armable after a 429.
|
||||
- `register_rate_limit(destination, retry_after: float | None)`
|
||||
— called on 429: `next_allowed_send = max(now + retry_after_or_backoff, current)`;
|
||||
`consecutive_429 += 1`; backoff = `retry_after` honored, else exponential
|
||||
`30 · 2^(n-1)` capped at 1800 s; creates/re-arms the shared wake-up event.
|
||||
- `register_success(destination)` — resets `consecutive_429` (called from the
|
||||
existing 200 path in `make_request`).
|
||||
- `remaining_seconds(destination)`, `in_cooldown(destination)` — for tests and
|
||||
diagnostics.
|
||||
|
||||
Enforcement points:
|
||||
|
||||
1. **`Downloader.make_request`** (`downloader.py:1102-1132`): ordering inside
|
||||
the method is **connectivity-guard fail-fast first** (offline short-circuit
|
||||
costs nothing to check), **then** `await coordinator.wait_for_slot(destination)`
|
||||
before `session.request`. On 429: `coordinator.register_rate_limit(...)`,
|
||||
then *wait for the gate and re-send* (loop, bounded by
|
||||
`rate_limit_max_wait_seconds`, default 300; `retry_after ≥ cap` ⇒ fail
|
||||
immediately). After the loop, return the `RateLimitError` to the caller
|
||||
(unchanged contract) **with `exc.gate_handled = True` set** so downstream
|
||||
retry helpers know the wait already happened. 200 path calls
|
||||
`register_success`.
|
||||
2. **`Downloader.download_to_memory` / `get_response_headers`** (phase 2):
|
||||
register 429s (so API calls queue); waiting only in `make_request`
|
||||
initially.
|
||||
3. **`FallbackMetadataProvider`** (`model_metadata_provider.py`): remove
|
||||
network→network failover on `RateLimitError` — re-raise; only sqlite stays
|
||||
as a local last resort (implementation: per-method `except RateLimitError`
|
||||
handler that marks the chain rate-limited and stops iterating).
|
||||
4. **`MetadataSyncService.fetch_and_update_model`**
|
||||
(`metadata_sync_service.py:248-333`): on `RateLimitError` from the default
|
||||
provider, stop appending further network providers (sqlite may remain);
|
||||
the existing `any_rate_limited` merge already produces `"Rate limited"`.
|
||||
5. **Batch import** (`batch_import_service.py`): no structural change needed —
|
||||
items now wait inside `make_request`; optionally (phase 2) map residual
|
||||
rate-limit failures (after the wait cap) to `SKIPPED` with
|
||||
`"rate limited (retry_after=…s); re-run the import later"` instead of
|
||||
`FAILED`, and surface a `rate_limited` flag in the WebSocket progress
|
||||
broadcast.
|
||||
6. **`_RateLimitRetryHelper` retries** (`model_metadata_provider.py`):
|
||||
**Phase 1** — when the raised `RateLimitError` carries `gate_handled = True`
|
||||
(set by the downloader after honoring the gate), the helper skips its own
|
||||
`retry_after` sleep and re-raises immediately, eliminating the double wait.
|
||||
The wiring stays so a `RateLimitError` still propagates cleanly; full
|
||||
demotion/removal can follow once the gate proves out.
|
||||
|
||||
Settings (`settings.json`, schema extension in `SettingsManager`):
|
||||
|
||||
| key | default | meaning |
|
||||
|---|---|---|
|
||||
| `rate_limit_gate_enabled` | `true` | master switch for the coordinator |
|
||||
| `rate_limit_max_wait_seconds` | `300` | how long `make_request` waits on a 429 gate before returning the error |
|
||||
| `rate_limit_min_interval_seconds` | `0.75` | minimum seconds between API requests per destination (pacing, R6-friendly conservative default) |
|
||||
|
||||
## 6. Changes by File
|
||||
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| `py/services/rate_limit_coordinator.py` (new) | coordinator singleton + per-destination state + tests seam |
|
||||
| `py/services/downloader.py` | gate pre-check + 429 register/wait/retry loop + `register_success`; log the 429 notice at INFO once per cooldown, then DEBUG |
|
||||
| `py/services/model_metadata_provider.py` | `FallbackMetadataProvider`: stop network failover on `RateLimitError`; helper skips its sleep when the error is marked `gate_handled` |
|
||||
| `py/services/metadata_sync_service.py` | `fetch_and_update_model`: same failover semantics; keep sqlite last resort |
|
||||
| `py/services/batch_import_service.py` | (phase 2) rate-limit failures → `SKIPPED` + `rate_limited` progress flag |
|
||||
| `py/services/settings_manager.py` | new settings keys + defaults |
|
||||
| `tests/services/test_rate_limit_coordinator.py` (new) | gate unit tests |
|
||||
| `tests/services/test_civitai_client.py` / `test_civarchive_client.py` | provider-level 429 behavior |
|
||||
| `tests/services/test_metadata_service.py` | failover-chain tests |
|
||||
| `tests/services/test_batch_import_service.py` | SKIPPED-on-rate-limit |
|
||||
|
||||
## 7. Impact, Risks, Open Questions
|
||||
|
||||
- **Behavior change**: with the gate in `make_request`, any request can block
|
||||
up to the wait cap — UI actions that call the API (e.g. a model-details
|
||||
fetch) may take longer during cooldowns. Mitigation: bounded cap + INFO log
|
||||
+ the existing async request handling already tolerates slow responses.
|
||||
**Decided (§10): interactive requests take the same bounded wait** — one
|
||||
behavior, no call-source plumbing; cooldowns are usually short.
|
||||
- **Gate waits occupy batch slots**: with the 1–5 batch semaphore, all slots
|
||||
can park on a gate simultaneously, freezing visible progress for up to one
|
||||
wait cap per wave. Bounded and acceptable; the phase-2 `SKIPPED` mapping +
|
||||
WebSocket `rate_limited` flag (both confirmed in scope, §10) make the stall
|
||||
visible and recoverable.
|
||||
- **Rate limit reality check**: CivitAI anonymous vs keyed limits, and whether
|
||||
`civitai.red` differs, is unverified. Default pacing `0.75 s/req` is a
|
||||
conservative guess (R6). Open question for maintainer: preferred default
|
||||
and whether an API-keyed ceiling should be higher.
|
||||
- **Long CivArchive windows**: `Retry-After ~1500 s` observed in code
|
||||
comments. **Decided (§10): keep the 300 s default cap** — such lookups
|
||||
fail/skip rather than park a request path for 25 minutes; batch import maps
|
||||
them to `SKIPPED` (phase 2) so the user can re-run later.
|
||||
- **Double waiting**: `_RateLimitRetryHelper` + gate could stack waits.
|
||||
**Resolved in Phase 1**: the downloader marks gate-honored errors with
|
||||
`gate_handled = True` and the helper skips its own sleep for those.
|
||||
- **Downloads**: `download_file` 429s return an error to download managers
|
||||
unchanged (already handled); only *registration* is proposed, so future
|
||||
API calls queue behind a large `Retry-After` from a download burst.
|
||||
|
||||
## 8. Test Plan
|
||||
|
||||
1. **Coordinator unit tests** (new file):
|
||||
- 429 with `retry_after` → `wait_for_slot` blocks ~that long, then passes.
|
||||
- N concurrent waiters all wake together (herd test, wall-clock ≈ one
|
||||
window, not N windows).
|
||||
- Consecutive 429s grow backoff; `register_success` resets.
|
||||
- Missing `Retry-After` → default backoff path.
|
||||
- Wait cap: request fails after `rate_limit_max_wait_seconds` with
|
||||
`RateLimitError`.
|
||||
2. **Downloader tests** (mock aiohttp session): 429 then 200 → `make_request`
|
||||
returns success after gate delay; two back-to-back calls to the same
|
||||
destination are spaced ≥ `min_interval`; different destinations are not
|
||||
spaced.
|
||||
3. **Provider tests**: `FallbackMetadataProvider.get_model_version_info` —
|
||||
Civitai raises `RateLimitError` → CivArchive mock **not called**; 404 still
|
||||
falls through to CivArchive; sqlite still tried after network 429.
|
||||
4. **Sync-service test**: `fetch_and_update_model` with a rate-limited default
|
||||
provider → result error contains `"Rate limited"` and sqlite attempt state
|
||||
unchanged.
|
||||
5. **Batch-import test**: analysis provider 429s first, then succeeds →
|
||||
item ends `SUCCESS` (wait path), and post-cap 429 → `SKIPPED` with
|
||||
rate-limit reason (phase 2).
|
||||
6. Full regression: `pytest tests/services tests/routes tests/standalone`
|
||||
(currently 1582 passing).
|
||||
|
||||
## 9. Implementation Phases
|
||||
|
||||
- **Phase 1 (this plan, reviewed):** `RateLimitCoordinator` +
|
||||
`Downloader.make_request` integration (guard fail-fast → gate pre-check
|
||||
pacing → 429 register/wait/retry loop with cap → `gate_handled` marking) +
|
||||
settings + **Fix C failover semantics** (`FallbackMetadataProvider`,
|
||||
`fetch_and_update_model` — moved up from phase 2: smallest diff, kills the
|
||||
CivArchive flood immediately, independent of coordinator correctness) +
|
||||
`_RateLimitRetryHelper` double-wait fix + coordinator/downloader/provider/
|
||||
sync tests.
|
||||
- **Phase 2:** batch-import `SKIPPED`-on-rate-limit + `rate_limited` WebSocket
|
||||
progress flag + slowdown hint (confirmed, §10),
|
||||
`download_to_memory`/HEAD 429 registration, batch tests.
|
||||
- **Phase 3:** full regression + docs + commit referencing `(#1085)`.
|
||||
|
||||
## 10. Review Checklist — Decisions (2026-08-27)
|
||||
|
||||
- [x] Default pacing interval `0.75 s` — **accepted** as conservative default;
|
||||
tunable via `rate_limit_min_interval_seconds`. Revisit if CivitAI
|
||||
publishes keyed/anonymous ceilings.
|
||||
- [x] Wait cap `300 s` — **accepted**; long-window CivArchive lookups fail →
|
||||
batch import marks them `SKIPPED` with a rate-limit reason (phase 2).
|
||||
- [x] Interactive API calls also wait (bounded) — **yes**, same behavior for
|
||||
all callers.
|
||||
- [x] Keep sqlite as last resort behind a network rate limit — **yes**
|
||||
(local-only, no vendor cost).
|
||||
- [x] UI hint — **yes**: WebSocket `rate_limited` flag + "rate limited —
|
||||
slowing down" hint in batch-import progress (phase 2); INFO logging
|
||||
regardless.
|
||||
@@ -0,0 +1,363 @@
|
||||
# Plan: "Other Models" Page — Unified Management for VAE / Upscaler / Text Encoder / etc.
|
||||
|
||||
**Status:** v2 — **Phase 1 implemented** (2026-09-12, commits `27da7b3c` backend + `fa7ce725` frontend; verified live against a running ComfyUI instance: scan/hash/sub_type-derivation/fetch/previews all green). **Phase 2 implemented** (2026-09-12, per §9 design; full pytest + vitest green). **Phase 3 implemented** (§11: opt-in management toggles; default off). **i18n done** (2026-09-13): all 36 new keys translated in the 9 non-English locales — the `[TODO: Translate]` placeholders left by the sync script during development are gone (see `docs/i18n-translation-guidelines.md` §2, "Other Models feature"). **Default set revised (pre-release):** only `vae` / `upscaler` / `text_encoder` are managed by default — `clip_vision` and `controlnet` are both opt-in (§2, §11.1.1).
|
||||
**Scope (Phase 1):** scan + manage (list, search, filter, tags, folders, preview, rename, move, delete/exclude, CivitAI metadata fetch) for a new model type `other`, exposed as a new web page. **Phase 2 (§9):** one-click download from CivitAI for these types.
|
||||
|
||||
## 1. Goal
|
||||
|
||||
Today the manager supports three model types:
|
||||
|
||||
| page | model_type | sub_types |
|
||||
|---|---|---|
|
||||
| `/loras` | `lora` | `lora`, `locon`, `dora` |
|
||||
| `/checkpoints` | `checkpoint` | `checkpoint`, `diffusion_model` |
|
||||
| `/embeddings` | `embedding` | `embedding` |
|
||||
|
||||
Add a fourth page that manages "everything else" — VAE, upscalers, text encoders / CLIP, CLIP vision, optionally ControlNet — with a folder→sub_type mapping table so new ComfyUI folder categories can be added later by configuration, not code.
|
||||
|
||||
## 2. Locked Decisions
|
||||
|
||||
1. **Architecture: one scanner + one service + one page, sub_type derived by location.**
|
||||
Replicates the checkpoint pattern (`CheckpointScanner` aggregates `checkpoints` + `unet` roots and derives `checkpoint` vs `diffusion_model` from the root containing the file, `py/services/checkpoint_scanner.py:384-415`). One `OtherScanner` aggregates all enabled folder roots; `resolve_sub_type_for_path()` maps each root to a sub_type. No per-category scanners.
|
||||
|
||||
2. **Naming: internal `model_type = "other"`, route prefix `/other`, page id `other`.**
|
||||
- `misc` is rejected: `py/routes/misc_routes.py` already owns that name for system/settings routes (`/api/lm/settings`, `/api/lm/doctor/*`).
|
||||
- `components` is rejected: `templates/components/` and `static/js/components/` directories would make `components.html` / `components.js` confusing neighbors.
|
||||
- `other` matches CivitAI's `Other` fallback type semantics. The **display name** is an i18n string (`other.title`, e.g. "Other Models") and can be renamed later without touching code.
|
||||
|
||||
3. **sub_type values:** snake_case, aligned with CivitAI `ModelType` semantics:
|
||||
|
||||
| sub_type | ComfyUI `folder_paths` key(s) | CivitAI ModelType | enabled by default |
|
||||
|---|---|---|---|
|
||||
| `vae` | `vae` | `VAE` | yes |
|
||||
| `upscaler` | `upscale_models` | `Upscaler` | yes |
|
||||
| `text_encoder` | `text_encoders`, `clip` (legacy) | `TextEncoder` (CLIP is retired upstream) | yes |
|
||||
| `clip_vision` | `clip_vision` | `CLIPVision` | no (mapping present, opt-in) |
|
||||
| `controlnet` | `controlnet` | `Controlnet` | no (mapping present, opt-in) |
|
||||
|
||||
New folder categories = one line in the mapping table (see §4.1).
|
||||
|
||||
**Why only three are on by default** (revised in Phase 3, before release):
|
||||
VAE, upscalers and text encoders are dependency-style assets every pipeline
|
||||
needs, and "which one am I actually using" is the recurring problem they
|
||||
solve. `clip_vision` and `controlnet` are workflow-driven instead
|
||||
(IPAdapter/SVD image conditioning; per-workflow ControlNet variants), and
|
||||
ControlNet libraries routinely run to dozens of files, so both are treated
|
||||
symmetrically as opt-in. Enumerating all five as "the default set" was not
|
||||
defensible on demand breadth alone.
|
||||
|
||||
4. **Phase 1 = scan/manage only.** Downloads from CivitAI (`download_manager.py` type mapping, default-root settings keys, download routing) are Phase 2 (§9). CivitAI **metadata fetch** for existing files IS in Phase 1 (hash-based lookup is type-agnostic; only the type-validation hook needs new values).
|
||||
|
||||
5. **Out of scope (default off, revisit later):** usage statistics buckets, recipe matching (`recipe_scanner.py` only merges lora+checkpoint scanners), statistics page, embeddings re-classification (stays its own page — merging would be a breaking change).
|
||||
|
||||
## 3. Why This Works With Minimal Churn
|
||||
|
||||
- `ModelScanner` (`py/services/model_scanner.py:93`) is specialized entirely via constructor params (`model_type`, `model_class`, `file_extensions`) + optional hooks (`adjust_metadata`, `adjust_cached_entry`, `resolve_sub_type_for_path`, `model_scanner.py:1429-1443`).
|
||||
- `BaseModelService` subclasses can be one method (`EmbeddingService` implements only `format_response`, `py/services/embedding_service.py:12`).
|
||||
- Routes: `ModelServiceFactory.register_model_type()` (`py/services/model_service_factory.py:120-136`) + `COMMON_ROUTE_DEFINITIONS` (`py/routes/model_route_registrar.py:23-149`) generate the full `/api/lm/{prefix}/*` surface (~50 endpoints) plus the `GET /{prefix}` page route.
|
||||
- `PersistentModelCache` (`py/services/persistent_model_cache.py:526-606`) is a single `models` table keyed `(model_type, file_path)` with `model_type` as free text — **zero schema change**.
|
||||
- Frontend `apiConfig.js` (`static/js/api/apiConfig.js:51`) generates all endpoints from the model-type string; `ModelCard.js:670-675` renders the sub_type badge from data; the checkpoints page already demonstrates the "one page, multiple sub_types" filter (`header.html:298`).
|
||||
|
||||
## 4. Backend Changes
|
||||
|
||||
### 4.1 New constants — `py/utils/constants.py`
|
||||
|
||||
```python
|
||||
# folder_paths key -> sub_type; single source of truth for extensibility
|
||||
OTHER_MODEL_FOLDER_SUBTYPES = {
|
||||
"vae": "vae",
|
||||
"upscale_models": "upscaler",
|
||||
"text_encoders": "text_encoder",
|
||||
"clip": "text_encoder", # legacy ComfyUI key
|
||||
"clip_vision": "clip_vision",
|
||||
"controlnet": "controlnet",
|
||||
}
|
||||
DEFAULT_OTHER_MODEL_FOLDERS = ("vae", "upscale_models", "text_encoders", "clip", "clip_vision")
|
||||
VALID_OTHER_SUB_TYPES = ["vae", "upscaler", "text_encoder", "clip_vision", "controlnet"]
|
||||
# CivitAI model.type values accepted for this page (fetch-metadata validation)
|
||||
VALID_OTHER_CIVITAI_TYPES = {"vae", "upscaler", "textencoder", "clipvision", "controlnet", "other"}
|
||||
```
|
||||
|
||||
Also extend `CIVITAI_USER_MODEL_TYPES` (`constants.py:90`) if user-model queries should include these types.
|
||||
|
||||
### 4.2 New files (mirror the embedding/checkpoint implementations)
|
||||
|
||||
1. **`py/utils/models.py`** — add `OtherModelMetadata(BaseModelMetadata)`: default `sub_type="vae"` placeholder overridden by scanner hook; `from_civitai_info` mapping CivitAI types → our sub_types (`TextEncoder`→`text_encoder`, `CLIPVision`→`clip_vision`, `Upscaler`→`upscaler`, `VAE`→`vae`, `Controlnet`→`controlnet`, else `other`-ish fallback to folder-derived sub_type).
|
||||
2. **`py/services/other_scanner.py`** — `OtherScanner(ModelScanner)`:
|
||||
- `model_type="other"`, extensions: reuse the checkpoint set (`safetensors/pt/pt2/bin/pth/pkl/sft/gguf`).
|
||||
- `get_model_roots()`: iterate `OTHER_MODEL_FOLDER_SUBTYPES` ∩ enabled keys, pull each from `config` (§4.3); dedupe; build `root → sub_type` map (normalized abspaths; multiple keys may share a sub_type).
|
||||
- Implement all three hooks like `CheckpointScanner` (`checkpoint_scanner.py:384-415`): `resolve_sub_type_for_path` by longest-prefix root match, `adjust_metadata`, `adjust_cached_entry` (sub_type is re-derived on cache load, never persisted).
|
||||
- **Lazy hashing, checkpoint-style**: text encoders (T5-XXL ≈ 10 GB) make eager sha256 painful. Copy the `hash_status="pending"` + singleflight `calculate_hash_for_model` pattern from `CheckpointScanner`.
|
||||
3. **`py/services/other_model_service.py`** — `OtherModelService(BaseModelService)`, `format_response` only (no usage_count, like `EmbeddingService`).
|
||||
4. **`py/routes/other_routes.py`** — `OtherRoutes(BaseModelRoutes)`, `template_name="other.html"`, hooks:
|
||||
- `_validate_civitai_model_type` → `VALID_OTHER_CIVITAI_TYPES`
|
||||
- `_get_expected_model_types`, `_parse_specific_params` (no type-specific download params in Phase 1)
|
||||
- `initialize_services()` on `app.on_startup` pulling `ServiceRegistry.get_other_scanner()`.
|
||||
|
||||
### 4.3 `py/config.py`
|
||||
|
||||
- New `other_roots` property: for each enabled key in `OTHER_MODEL_FOLDER_SUBTYPES`, `folder_paths.get_folder_paths(key)` (plugin mode) — standalone mode needs nothing new: `MockFolderPaths` (`standalone.py:66-105`) already serves arbitrary keys from `settings.json.folder_paths`.
|
||||
- Follow the existing per-type recipe: an `_prepare_other_paths()` (dedupe + symlink registration; also **cross-scanner overlap detection** — warn if an `other` root is already covered by checkpoints/unet/embedding roots, mirroring the checkpoint/unet overlap check).
|
||||
- Wire into: `_apply_library_paths`, `_symlink_roots()`, `_rebuild_preview_roots()` (hard requirement — preview images are served per registered root), `save_folder_paths_to_settings()`.
|
||||
|
||||
### 4.4 Existing-file edits (the "type string scatter" — each is a small branch/entry)
|
||||
|
||||
| file | change |
|
||||
|---|---|
|
||||
| `py/services/model_service_factory.py:120` | register `("other", OtherModelService, OtherRoutes)` in `register_default_model_types()` |
|
||||
| `py/services/service_registry.py` | add `get_other_scanner()` (mirror `:297` `get_embedding_scanner`) |
|
||||
| `py/services/model_scanner.py:67` | `PAGE_TYPE_MAP['other'] = 'other'` (WebSocket progress) |
|
||||
| `py/services/base_model_service.py:896-906` | `get_model_types()` branch → `VALID_OTHER_SUB_TYPES` |
|
||||
| `py/lora_manager.py` | `_initialize_services` scanner task list (`:219-242`), `_cleanup` cancel list (`:463`), `_cleanup_backup_files` roots (`:327-330`) |
|
||||
| `py/routes/handlers/misc_handlers.py` | `scanner_getters` (`:657-661`) + `scanner_factories` (`:757-759`) so Doctor / init-status / refresh-all see the new scanner |
|
||||
| `py/services/pending_delete_service.py` | `_PAGE_TYPE` map (`:57-61`) + scanner getter list (`:983-985`) |
|
||||
| `py/metadata_ops/__init__.py:36-38` | `SCANNER_TYPE_MAP['other']` |
|
||||
| `settings.json.example` | document optional `folder_paths` keys: `vae`, `upscale_models`, `text_encoders`, `clip_vision` |
|
||||
|
||||
**Explicitly NOT touched in Phase 1:** `py/services/download_manager.py`, `py/services/download_routing.py`, `py/services/settings_manager.py` default-root keys, `py/routes/stats_routes.py`, `py/utils/usage_stats.py`, `py/services/recipe_scanner.py`, `py/metadata_collector/`, `py/nodes/`.
|
||||
|
||||
**Zero-change confirmations (verified):** `PersistentModelCache`, `ModelUpdateService`, `DownloadedVersionHistoryService`, `MetadataSyncService` + provider chain (type-agnostic hash lookups), `ModelFileService` / `ModelMoveService` / `ModelLifecycleService` (scanner + model_type injected), `ModelCache` / `ModelHashIndex`, `AutoV3BackfillService`.
|
||||
|
||||
## 5. Frontend Changes
|
||||
|
||||
1. **`static/js/api/apiConfig.js`** — `MODEL_TYPES.OTHER = 'other'`; `MODEL_CONFIG.other` entry (displayName, singularName, `supportsMove`, `supportsBulkOperations`; no letter filter); endpoints come free from `getApiEndpoints()` (`:51`).
|
||||
2. **`static/js/api/otherApi.js`** — thin `OtherApiClient extends BaseModelApiClient` (mirror `embeddingApi.js`); register in `modelApiFactory.js`.
|
||||
3. **`static/js/other.js`** — page entry (mirror `embeddings.js`): `appCore.initialize()` + `createPageControls('other')` + `initializePageFeatures()` + `ModelDuplicatesManager` + `initActiveFiltersSync('other')`.
|
||||
4. **Controls & context menu** — `OtherControls extends PageControls` and `OtherContextMenu` (start from the embedding variants — the smallest); add branches in the two factories (`components/controls/index.js:15`, `components/ContextMenu/index.js:15`). Context-menu template block lives in `templates/other.html` (`{% block additional_components %}`, the checkpoints/embeddings pattern — do NOT touch the shared `context_menu.html`).
|
||||
5. **`templates/other.html`** — copy `embeddings.html`: same content blocks (controls + breadcrumb + duplicates banner + folder sidebar + `#modelGrid`), `data-page="other"`, main script `/loras_static/js/other.js`.
|
||||
6. **`templates/components/header.html`** — nav entry (`:23-43`, active when `request.path.startswith('/other')`); enable the `modelTypes` sub_type filter panel for `other` (`:298-305` pattern from checkpoints); check search-options panel conditions (`:199-224`).
|
||||
7. **`static/js/utils/constants.js`** — `MODEL_SUBTYPE_ABBREVIATIONS` (`:115`): `vae→VAE`, `upscaler→UPS`, `text_encoder→TE`, `clip_vision→CV`, `controlnet→CN`; matching `MODEL_SUBTYPE_DISPLAY_NAMES` (`:99`). (Unknown fallback already uppercases 4 chars, but explicit mappings read better.)
|
||||
8. **`static/js/core.js:110` `getPageType()`** — verify `data-page="other"` flows through `state.pages` generically; add only if the page list is enumerated anywhere.
|
||||
9. No change to `web/comfyui/top_menu_extension.js` (it opens `/loras`; page-to-page nav is the header bar).
|
||||
|
||||
## 6. i18n
|
||||
|
||||
- `locales/en.json`: add `other.title` (e.g. "Other Models") + minimal `other.contextMenu.*` / `other.modelTypes.*` keys; reuse `modelCard.*`, `loras.contextMenu.*`, `common.*` wherever possible (the established pattern — checkpoints/embeddings already reuse lora keys).
|
||||
- Run `python scripts/sync_translation_keys.py`; leave `[TODO: Translate]` placeholders in other locales (per `docs/i18n-translation-guidelines.md` §7 — do not translate proactively).
|
||||
|
||||
## 7. Testing
|
||||
|
||||
Follow existing conventions (`pytest.ini`, `tests/frontend/` vitest):
|
||||
|
||||
1. **Backend (pytest, async where needed):**
|
||||
- `OtherScanner` root aggregation + `resolve_sub_type_for_path` (file under `vae/` root → `vae`; `text_encoders` and legacy `clip` both → `text_encoder`; disabled `controlnet` root not scanned).
|
||||
- Cache round-trip: sub_type re-derived via `adjust_cached_entry` (not persisted).
|
||||
- Lazy hash: `hash_status="pending"` default; `calculate_hash_for_model` singleflight.
|
||||
- `OtherRoutes` registration smoke test: `/api/lm/other/...` endpoints exist; `_validate_civitai_model_type` accepts `vae`/`upscaler`/`textencoder`, rejects `lora`.
|
||||
- Config: `other_roots` in both modes (mock `folder_paths`, and standalone `settings.json.folder_paths`).
|
||||
2. **Frontend (vitest + jsdom, `tests/frontend/`):**
|
||||
- `apiConfig`: `getApiEndpoints('other')` URL shapes; `modelApiFactory` returns the Other client.
|
||||
- `ModelCard` badge rendering for new sub_types.
|
||||
- `createPageControls('other')` / `createPageContextMenu('other')` factories.
|
||||
3. **Manual UI verification by the user** (per AGENTS.md — no sandbox/browser automation): page loads, scans a real library, sub_type filter + badges, context menu actions.
|
||||
|
||||
## 8. Execution Order
|
||||
|
||||
1. `constants.py` + `OtherModelMetadata` + `config.py` roots
|
||||
2. `OtherScanner` (+ registry, factory, `PAGE_TYPE_MAP`) → scanner unit tests green
|
||||
3. `OtherModelService` + `OtherRoutes` + handler/registrar wiring + `lora_manager.py` lifecycle → route tests green
|
||||
4. Doctor/pending-delete/metadata-ops scatter entries
|
||||
5. Template + header nav + frontend API/controls/context-menu/card badges → vitest green
|
||||
6. i18n keys + sync script
|
||||
7. `pytest` + `npm test` full runs; hand to user for manual UI check
|
||||
|
||||
## 9. Phase 2 Detailed Design — CivitAI Downloads for `other`
|
||||
|
||||
Designed 2026-09-12 against the Phase-1 code on this branch; decisions marked **[locked]** follow the same recommendations the feature owner approved for Phase 1.
|
||||
|
||||
### 9.1 Download pipeline touch points
|
||||
|
||||
Flow: `POST /api/lm/download-model` (`py/routes/model_route_registrar.py:104`; GET variant `:105` for the browser extension) → `ModelDownloadHandler.download_model` (`model_handlers.py:1740`) → `DownloadModelUseCase.execute` → `DownloadCoordinator.schedule_download` → `DownloadManager.download_from_civitai` (`download_manager.py:386`) → `_execute_original_download` (`:1415`). Inside, seven scatter points need an `other` branch:
|
||||
|
||||
1. **Type map** (`:1496-1507`): accept `model.type.lower() in VALID_OTHER_CIVITAI_TYPES` → `model_type = "other"` (reuses the Phase-1 set, incl. `"other"` itself).
|
||||
2. **Early version-exists gate** (`:1436-1463`): add `other_scanner.check_model_version_exists`.
|
||||
3. **File-level exists gate** (`:1640-1655` → `_find_local_file_entry` `:320-346` → `_get_scanner_for_model_type` `:230-236`): add explicit `other` branch. **Trap**: the function currently falls through to the lora scanner for unknown types — `"other"` would silently dedupe against loras. Also narrow the fall-through to `"lora"` only / raise on unknown.
|
||||
4. **Version-level fallback gate** (`:1656-1688`): add `elif model_type == "other"`.
|
||||
5. **Default-root selection** (`:1690-1727`): for `other`, first resolve sub_type (§9.2), then read `default_other_roots[sub_type]` (§9.3); if sub_type is undecidable or no default root configured → error guiding the user to pick a folder explicitly.
|
||||
6. **Metadata class selection** (`:1909-1928`) + `_build_metadata_for_resume` (`:969-981`): add `OtherModelMetadata.from_civitai_info` branches.
|
||||
7. **Post-download cache write** (`_execute_download_pipeline` `:2622-2679`): add `other` scanner branch; `adjust_metadata` re-derives sub_type from the on-disk root automatically. `_get_supported_extensions_for_type` (`:2720-2744`): `other` reuses the checkpoint extension set.
|
||||
|
||||
Hooks: `_record_downloaded_version_history` (model_type is free text — zero change); `_sync_downloaded_version` (`:1984` → scanner dispatch `:2130-2135`) add `other`; `py/utils/example_images_download_manager.py` scanner dispatch at `:411-421`, `:591-601`, `:1089+` — add `other` at all three (silent no-scanner otherwise).
|
||||
|
||||
Path templates: `get_download_path_template("other")` is unset, so `other` resolves to a **flat** layout (empty template) — downloads land directly under the resolved sub_type root. This is deliberate: other-model roots are already split per sub_type (`default_other_roots`), and `priority_tags` has no `other` entry, so `{first_tag}` would fall back to an arbitrary CivitAI tag and scatter files into unstable folders. Users who want nesting can still set `download_path_templates["other"]` in `settings.json`. See `DEFAULT_DOWNLOAD_PATH_TEMPLATES` (`py/utils/constants.py`) and `DEFAULT_PATH_TEMPLATES` (`static/js/utils/constants.js`).
|
||||
|
||||
### 9.2 File-level routing (model.type / file.type → sub_type) **[locked]**
|
||||
|
||||
Table-driven, mirroring Phase 1. New in `py/utils/constants.py`:
|
||||
|
||||
```python
|
||||
CIVITAI_FILE_TYPE_TO_OTHER_SUB_TYPE = {
|
||||
"VAE": "vae", "Upscaler": "upscaler", "Text Encoder": "text_encoder",
|
||||
"Vision Encoder": "clip_vision", "CLIPVision": "clip_vision",
|
||||
"ControlNet": "controlnet",
|
||||
}
|
||||
```
|
||||
|
||||
`download_routing.py` gains `resolve_other_download_sub_type(civitai_model_type, file_types, selected_file_type=None)` with fixed priority:
|
||||
|
||||
1. **Explicit user file pick** (`file_params` from #1058's `_resolve_target_file`) — if the picked file's type maps, it wins even when model.type is `Checkpoint`.
|
||||
2. **model.type** via the existing `CIVITAI_TYPE_TO_OTHER_SUB_TYPE` (`constants.py:120-127`).
|
||||
3. **file.type fallback** — only when model.type maps to nothing (e.g. model.type `Other` or retired `CLIP`). MUST NOT override a mapped model.type: checkpoint models routinely bundle VAE/Text Encoder component files, and unconditional file-type routing would misroute them.
|
||||
4. Still undecidable → `None`; `use_default_paths` errors and the UI offers all other roots for manual selection.
|
||||
|
||||
HTTP: extend `DownloadRoutingHandler.get_download_routing` (`download_routing_handlers.py:23`) with an `other` branch returning `{root_kind: "other", sub_type: ...}`; add `GET /api/lm/other/roots_by_subtype` in `OtherRoutes.setup_specific_routes` (data from `config._prepare_other_paths`'s per-key roots, aggregating `text_encoders` + legacy `clip` under `text_encoder`).
|
||||
|
||||
### 9.3 Settings: single dict key `default_other_roots` **[locked]**
|
||||
|
||||
Rejected: four flat keys (`default_vae_root`…) — each flat key costs ~13 touch points in `settings_manager.py` (defaults `:82-85`, `_check_and_auto_set` `:890-895`, `set()` `:1621-1628`, `_update_active_library_entry` `:738-805`, upsert/create signatures `:1953-2132`, `_build_library_payload` `:552-612`, `_sync_active_library_to_root` `:519-547`, three library constructors, frontend `DEFAULT_SETTINGS_BASE`), repeated per future sub_type.
|
||||
|
||||
Chosen: one mapping key `default_other_roots: {sub_type: path}`, copying the `extra_folder_paths` precedent (generic Mapping handling at `:533-535`, `:573-578`, `:763-767`). `_check_and_auto_set` generalizes to per-sub_type candidates (union over that sub_type's folder keys — `text_encoder` → `text_encoders` + `clip`). `set()` validates keys against `VALID_OTHER_SUB_TYPES`.
|
||||
|
||||
Also fix the Phase-1 omission: add `"other_scanner"` to `_notify_library_change` (`:2150-2156`) and `_notify_model_name_display_change` (`:1795-1800`) — otherwise switching libraries leaves the other page stale.
|
||||
|
||||
### 9.4 Settings UI
|
||||
|
||||
- `templates/components/modals/settings/library.html:34-40`: sub_type selectors after the existing four `setting_select`s (Jinja loop; controlnet selector only when `enabled_other_folders` includes it). Dict-subkey save helper `saveOtherRootSetting(subType, value)` alongside the flat `saveSelectSetting`.
|
||||
- `static/js/managers/SettingsManager.js:1547-1697`: `loadOtherRoots()` mirroring `loadUnetRoots()`, fed by `/api/lm/other/roots_by_subtype`; current values from `state.global.settings.default_other_roots`. `state/index.js:24` `DEFAULT_SETTINGS_BASE` += `default_other_roots: {}`.
|
||||
- Optional: one `other` row in the download-path-template block (`library.html:153-211`).
|
||||
- i18n: `settings.folderSettings.*` keys into `locales/en.json` + sync script; other locales keep `[TODO: Translate]`.
|
||||
- Settings GET (`misc_handlers.py:1528-1536`) already returns all non-sensitive keys — new key reaches the frontend for free.
|
||||
|
||||
### 9.5 Frontend download entry
|
||||
|
||||
- `templates/components/controls.html:83`: drop the `page_id != 'other'` exclusion on the download button (keyboard shortcut D self-enables via `PageControls.js:196-198`).
|
||||
- `OtherControls.js:22-55`: add `showDownloadModal: () => downloadManager.showDownloadModal()` (mirror `EmbeddingsControls.js:43-45`).
|
||||
- `DownloadManager.js` `proceedToLocationContent` (`:955-1017`): add `_resolveOtherSubType()` (mirror `_resolveIsDiffusionModel` `:1026`): selected file type → `/api/lm/download/routing` → `otherApiClient.fetchModelRoots(subType)` (new); default-root preselect reads `default_other_roots[subType]` instead of `` `default_${singularType}_root` `` (`:974`). Undecidable → list all other roots (`/api/lm/other/roots`) for manual pick; an explicit save_dir skips backend default-root logic, so the two paths cannot disagree.
|
||||
- `ModelVersionsTab` download buttons are modelType-generic and already work via `getModelApiClient('other')`; context menu has no CivitAI download entry — no change.
|
||||
- Version-list type validation (`get_civitai_versions` → `_validate_civitai_model_type`) already accepts `VALID_OTHER_CIVITAI_TYPES` from Phase 1.
|
||||
|
||||
### 9.6 CivitAI type mapping decisions **[locked]**
|
||||
|
||||
- Download accepts exactly `VALID_OTHER_CIVITAI_TYPES` (`VAE, Upscaler, TextEncoder, CLIP, CLIPVision, Controlnet, Other`) — reuse the Phase-1 tables; do NOT create new ones.
|
||||
- Extend `CIVITAI_USER_MODEL_TYPES` (`constants.py:133-137`) with the 7 aliases, and point them at the other scanner / `"other"` history bucket in `misc_handlers.py` (`type_scanner_map` `:2793-2797`, `downloaded_version_map` `:2821-2827`) — otherwise creator pages silently filter these models while downloads claim support.
|
||||
- Fix (small Phase-1 bug): `OtherModelMetadata.from_civitai_info` (`py/utils/models.py:343`) reads `version_info.get("type")`, but the type lives at `version["model"]["type"]` — the mapping never fires and always degrades to the placeholder. Read `version_info.get("model", {}).get("type")` instead. (`CheckpointMetadata:290` has the same shape; leave it alone here.)
|
||||
|
||||
### 9.7 Tests
|
||||
|
||||
Existing base: `tests/services/test_download_manager_basic.py` (incl. `test_download_rejects_unsupported_model_type` `:1336`), `test_download_manager_error.py`, `test_download_manager_concurrent.py`, `tests/integration/test_download_flow.py`, `tests/services/test_settings_manager.py`; frontend `tests/frontend/managers/downloadManager.routing.test.js`, `settingsManager.library.test.js`.
|
||||
|
||||
Add: (1) `resolve_other_download_sub_type` unit tests — every priority tier, bundled-component anti-misrouting, undecidable → None, civarchive-shaped payload; (2) download_manager — six model.types accepted → other scanner (mock), unknown still rejected, no lora-scanner fall-through, per-sub_type default roots + unconfigured error, resume metadata, extension set; (3) settings_manager — `default_other_roots` defaults/auto-set (incl. text_encoder dual-key union)/library sync/upsert passthrough/illegal sub_type rejection; (4) routes — `/api/lm/download/routing` other branch, `roots_by_subtype` shape; (5) example-images dispatch accepts `other` (3 sites); (6) vitest — `_resolveOtherSubType` + root select + default preselect, `loadOtherRoots`; (7) user-models existsLocally for VAE.
|
||||
|
||||
### 9.8 Phase 2 file list
|
||||
|
||||
Backend: `py/utils/constants.py`, `py/services/download_routing.py`, `py/routes/handlers/download_routing_handlers.py`, `py/services/download_manager.py`, `py/utils/example_images_download_manager.py`, `py/services/settings_manager.py`, `py/utils/models.py`, `py/routes/other_routes.py`, `py/routes/handlers/misc_handlers.py`, `settings.json.example`.
|
||||
Frontend/templates: `templates/components/controls.html`, `static/js/components/controls/OtherControls.js`, `static/js/managers/DownloadManager.js`, `static/js/api/otherApi.js`, `templates/components/modals/settings/library.html`, `static/js/managers/SettingsManager.js`, `static/js/state/index.js`, `locales/en.json` + sync.
|
||||
|
||||
## 10. Risks / Open Questions
|
||||
|
||||
- **Root overlap**: a user may point `text_encoders` at a directory already scanned as checkpoints/unet. Realpath dedup inside one scanner won't catch cross-scanner overlap → the `_prepare_other_paths` overlap warning (§4.3) is the mitigation; duplicate cards across pages are cosmetic, not corrupting (cache keyed by `(model_type, file_path)`).
|
||||
- **Huge text encoders + lazy hash**: CivitAI fetch for a pending-hash model must trigger on-demand hash like checkpoints do — verify that flow (`calculate_hash_for_model`) is reachable from the `other` routes' fetch-metadata handler.
|
||||
- **Retired CivitAI types**: `CLIP`/`CLIPVision` are retired upstream (grandfathered for existing models); metadata fetch must tolerate both retired and current types — `VALID_OTHER_CIVITAI_TYPES` includes them deliberately.
|
||||
- **Standalone users** must add the new `folder_paths` keys to `settings.json` themselves; document in `settings.json.example` and the feature doc.
|
||||
- **Page display name** is i18n-only; if "Other Models" tests poorly, rename `other.title` without code changes.
|
||||
|
||||
### Phase 2 risks
|
||||
|
||||
- **Bundled component files**: checkpoint models routinely ship VAE/Text Encoder component files — file.type routing must stay a fallback (or explicit user pick), never an override (§9.2 priority is load-bearing; test it).
|
||||
- **`_get_scanner_for_model_type` lora fall-through** (`download_manager.py:236`): without an explicit `other` branch, dedupe checks run against the lora scanner — the most insidious trap in Phase 2.
|
||||
- **text_encoder dual folder keys** (`text_encoders` + legacy `clip`): default-root candidates, `roots_by_subtype`, and auto-set must all merge both keys; miss one and the default-root dropdown comes up empty.
|
||||
- **Undecidable sub_type** (model.type `Other` + unknown file types): must error and ask, never silently default to the vae folder.
|
||||
- **Lazy hash after download**: downloads carry CivitAI SHA256 (no recompute needed) — ensure the post-download cache write doesn't leave `hash_status="pending"`, or the next metadata fetch re-hashes a 10 GB file.
|
||||
- **CivArchive source**: same `_execute_original_download` path, same payload shape — cover it once in tests.
|
||||
|
||||
## 11. Phase 3 — Opt-in Management Toggles (implemented)
|
||||
|
||||
Designed 2026-09-13 against the Phase-1/2 code. Other Models is **opt-in**: after
|
||||
Phase 3 the feature ships disabled, so no other-model folder is scanned and the
|
||||
page shows an "enable" empty state until the user turns it on.
|
||||
|
||||
### 11.1 Settings (global, not per-library)
|
||||
|
||||
| key | type | default | meaning |
|
||||
|---|---|---|---|
|
||||
| `enable_other_models` | bool | `false` | master switch |
|
||||
| `enabled_other_sub_types` | list[str] | `["vae","upscaler","text_encoder"]` | allow-list; `clip_vision` and `controlnet` are opt-in (see §2) |
|
||||
|
||||
`enabled_other_folders` (the unreleased, additive, no-UI backend key) was removed
|
||||
and replaced by the sub_type-level allow-list; there is no migration because the
|
||||
feature never shipped. `text_encoder` expands to `text_encoders` + legacy `clip`
|
||||
via `OTHER_SUB_TYPE_FOLDER_KEYS`.
|
||||
|
||||
The default allow-list lives on five surfaces that must stay in sync:
|
||||
`DEFAULT_ENABLED_OTHER_SUB_TYPES` (`py/utils/constants.py`), `DEFAULT_SETTINGS`
|
||||
(`py/services/settings_manager.py`), the two `DEFAULT_SETTINGS_BASE` /
|
||||
`createDefaultSettings` lists (`static/js/state/index.js`), the
|
||||
`updateOtherModelsControls()` fallback (`static/js/managers/SettingsManager.js`)
|
||||
and the server-rendered Jinja fallback
|
||||
(`templates/components/modals/settings/library.html`).
|
||||
|
||||
### 11.1.1 Legacy key handling in `Config._init_other_paths`
|
||||
|
||||
ComfyUI's `folder_paths` rewrites legacy names before every access (`clip` →
|
||||
`text_encoders`, `unet` → `diffusion_models`) and registers both legacy
|
||||
directories under the canonical key, so `get_folder_paths("clip")` returns
|
||||
exactly the same list as `get_folder_paths("text_encoders")`. Querying both keys
|
||||
made the overlap guard fire twice with `please fix your path configuration` for a
|
||||
configuration the user cannot fix. `Config._collapse_legacy_folder_keys()` now
|
||||
drops a key when the host exposes `map_legacy` and resolves it to another queried
|
||||
key, and `_prepare_other_paths()` downgrades a same-`sub_type` duplicate to
|
||||
`debug` (a cross-`sub_type` collision still warns). In standalone mode
|
||||
`MockFolderPaths` has no `map_legacy` and its keys are independent
|
||||
`settings.json` entries, so every key is still queried there.
|
||||
|
||||
`settings.json.example` intentionally stays minimal (only `use_portable_settings`,
|
||||
`civitai_api_key`, and the four core `folder_paths` keys: `loras`, `checkpoints`,
|
||||
`unet`, `embeddings`). Optional keys — including the other-model folder paths and
|
||||
`enable_other_models` — are NOT documented there; they live in `DEFAULT_SETTINGS`
|
||||
and reach the user's `settings.json` on demand. This supersedes the Phase-1/Phase-2
|
||||
notes that proposed adding the other-model folder keys to the example.
|
||||
|
||||
### 11.2 Behaviour matrix
|
||||
|
||||
| state | scan | nav / `/other` | other downloads | `default_other_roots` | Doctor / refresh-all |
|
||||
|---|---|---|---|---|---|
|
||||
| master off | nothing (`other_roots == []`) | nav entry hidden (`nav-item--hidden`); `/other` still renders the disabled empty state + Enable button; one-time dismissible announcement banner on first visit | rejected | preserved, never auto-set | scanner skipped |
|
||||
| sub_type off | that sub_type's folder keys excluded | page keeps working, type disappears from data | auto-routing refused (manual folder still allowed) | preserved, not preselected | normal |
|
||||
| all on (after enabling) | Phase-1/2 behaviour | normal | normal | normal | normal |
|
||||
|
||||
### 11.3 Backend touch points
|
||||
|
||||
- `py/utils/constants.py` — `DEFAULT_ENABLED_OTHER_SUB_TYPES`, `OTHER_SUB_TYPE_FOLDER_KEYS`, `normalize_other_sub_types`.
|
||||
- `py/config.py` — `_get_enabled_other_folder_keys()` is the single scan gate (master switch + allow-list); new `refresh_other_roots()` rebuilds roots + preview roots on toggle.
|
||||
- `py/services/settings_manager.py` — new defaults, `set()` normalization, `is_other_models_enabled()` / `get_enabled_other_sub_types()` / `is_other_sub_type_enabled()`, and `_apply_other_model_settings_change()` which reapplies config and calls `other_scanner.on_library_changed(reconcile=True)`.
|
||||
- `py/services/model_scanner.py` — `_should_keep_cached_entry()` hydration hook (default keep) plus `on_library_changed(reconcile=...)` / `initialize_in_background(reconcile=...)`; the hook filters `raw_data` and the hash/autov3 index rows.
|
||||
- `py/services/other_scanner.py` — drops persisted entries whose folder is no longer a managed root (sub_type is location-derived, so config is the source of truth).
|
||||
- `py/routes/other_routes.py` — `_validate_civitai_model_type` rejects everything while off / mapped-but-disabled sub_types; `_get_page_context_provider()` injects `other_disabled` into the template.
|
||||
- `py/routes/handlers/model_handlers.py` + `base_model_routes.py` — optional `page_context_provider` hook on `ModelPageView`.
|
||||
- `py/routes/handlers/download_routing_handlers.py` — returns `{sub_type: None, disabled: true, reason}` instead of guessing.
|
||||
- `py/services/download_manager.py` — rejects other-type downloads while off; disabled sub_type refuses default-path routing with a "pick a folder" error.
|
||||
- `py/routes/handlers/misc_handlers.py` — Doctor / init-status / refresh-all skip the other scanner while off (`_active_scanner_factories` / `_active_scanner_getters`).
|
||||
- `py/services/pending_delete_service.py` — deliberately untouched: the scanner stays registered so staged deletes still merge.
|
||||
|
||||
### 11.4 Frontend
|
||||
|
||||
Discoverability: the nav entry is hidden while the feature is off, and three
|
||||
lightweight surfaces replace it — a one-time announcement banner, the download
|
||||
toast, and the settings toggle itself.
|
||||
|
||||
- `templates/components/header.html` + `static/css/components/header.css` — `nav-item--hidden` class (server-rendered when off, client-toggled after enabling) and the `fa-shapes` icon.
|
||||
- `templates/other.html` — `other_disabled` branch in `content` + `main_script`; page-scoped CSS for the empty state.
|
||||
- `static/js/other_disabled.js` — boots `appCore` (shared header) and delegates to the shared enable helper.
|
||||
- `static/js/utils/otherModels.js` — shared `enableOtherModels()` (POST settings + reload) and `openOtherModelsSettings()` (settings modal on the Library section); used by the disabled page, the banner and the download modal.
|
||||
- `static/js/managers/BannerService.js` — `other-models-announcement` banner (only when off and not dismissed; `priority: 0`, dismissal persisted via `dismissed_banners`) with Enable / Open Settings actions; `removeOtherModelsAnnouncement()` drops it without persisting a dismissal.
|
||||
- `templates/components/modals/settings/library.html` + `SettingsManager.updateOtherModelsControls()` / `saveEnabledOtherSubTypes()` / `updateOtherModelsNavVisibility()` — master toggle + five sub_type checkboxes; unchecked/disabled sub_types have their default-root select disabled.
|
||||
- `static/js/managers/DownloadManager.js` — a disabled routing answer surfaces a `showActionToast` with an "Enable Other Models" action (opening settings) and falls back to manual selection.
|
||||
- i18n: `settings.folderSettings.*`, `other.disabled.*` and `banners.otherModels.*` keys in `locales/en.json` + `scripts/sync_translation_keys.py` (other locales keep `[TODO: Translate]`).
|
||||
|
||||
### 11.5 Cache consistency
|
||||
|
||||
- Disabling purges rows from the in-memory view at hydration time (the
|
||||
`_should_keep_cached_entry` hook) and from SQLite on the reconcile triggered by
|
||||
the toggle; the `.metadata.json` sidecars survive, so re-enabling rescans
|
||||
without recomputing hashes (critical for multi-GB text encoders).
|
||||
- Enabling triggers a reconcile so newly managed roots are scanned immediately.
|
||||
- Editing `settings.json` while the server is stopped is still covered by the
|
||||
hydration hook, so disabled types never appear after a restart.
|
||||
|
||||
### 11.6 Tests
|
||||
|
||||
Backend: opt-in fixtures added to the other-related suites; new coverage for
|
||||
"default off scans nothing", per-sub_type gating, routing/download rejection,
|
||||
`_should_keep_cached_entry`, settings normalization and `other_disabled` page
|
||||
context. Frontend: `updateOtherModelsControls` / `saveEnabledOtherSubTypes` and
|
||||
the disabled-page enable flow.
|
||||
@@ -0,0 +1,58 @@
|
||||
# CivitAI image imports can end up with 0 LoRAs
|
||||
|
||||
## Symptom
|
||||
|
||||
Importing a CivitAI image URL can produce a recipe with **zero LoRA
|
||||
entries**, even though the image page lists LoRAs in its resource panel.
|
||||
|
||||
Reported example: `https://civitai.red/images/140818889` was imported as a
|
||||
local recipe with 0 LoRAs, while the page shows 3 LoRAs. Some images (e.g.
|
||||
NSFW / higher browsing level) additionally require a login to view, so their
|
||||
data is not publicly reachable at all.
|
||||
|
||||
## Root cause
|
||||
|
||||
URL imports use only two data sources:
|
||||
|
||||
1. **CivitAI REST image API** — `GET /api/v1/images?imageId=<id>&nsfw=X&withMeta=true` → `meta`
|
||||
2. **Embedded image metadata** — EXIF/XMP read from the downloaded bytes
|
||||
|
||||
For the same image both sources can be empty, and the one source that does
|
||||
contain the data is never queried. Verified for image 140818889:
|
||||
|
||||
| Source | What it returned |
|
||||
|---|---|
|
||||
| REST image API | `meta` holds only a prompt; `modelVersionIds: []`; no `resources`/`hashes`; `baseModel: null` |
|
||||
| Downloaded image | PNG with **no EXIF/XMP** (the CDN URL ends in `.jpeg`, the body is PNG) |
|
||||
| Image page HTML | `__NEXT_DATA__` embeds the trpc `image.getGenerationData` result → full `resources` list: 3 LoRAs, each with `modelId`, `modelVersionId`, `modelName`, `modelType`, `versionName`, `baseModel` |
|
||||
|
||||
Key points:
|
||||
|
||||
- The page's resource panel is fed by an **internal, non-public trpc
|
||||
endpoint**, not by the public REST image API.
|
||||
- That internal endpoint is **login-gated** for some content — the
|
||||
"requires login" symptom.
|
||||
- Even with the version IDs in hand, `/model-versions/{id}` for these
|
||||
(Krea) versions returns **no `sha256`**, so an exact local-file hash match
|
||||
is impossible; only model/version identity is recoverable.
|
||||
|
||||
## Conclusion / status
|
||||
|
||||
0-LoRA imports are a data-source gap: public REST meta and image EXIF are
|
||||
both empty, while the only complete source (page generation data) is
|
||||
internal, sometimes login-gated, and not used by the importer.
|
||||
|
||||
Such imports **cannot be reliably auto-repaired/completed** by the backend
|
||||
alone. The old "Repair Metadata" feature only re-fetched the same incomplete
|
||||
REST meta and could not fix them; it was deprecated and has been removed.
|
||||
|
||||
**Fixed via the companion browser extension.** When the extension is
|
||||
installed with a valid license, it scrapes the image page's internal trpc
|
||||
generation data with the user's session and calls the payload-capable
|
||||
re-import endpoint (`POST /api/lm/recipe/{recipe_id}/reimport` with
|
||||
`image_url`/`name`/`resources`/`gen_params`/`base_model`/`tags` query
|
||||
params), which rebuilds the recipe from the caller-supplied metadata. The
|
||||
web UI delegates re-import of CivitAI-image-sourced recipes to the extension
|
||||
automatically (probe + `lm:reimport*` DOM events); without the extension,
|
||||
re-import silently falls back to the native path, which remains limited by
|
||||
the data-source gap documented above.
|
||||
@@ -0,0 +1,92 @@
|
||||
# Reconcile 的 Windows 大小写回退分支 - 待验证清单
|
||||
|
||||
> **状态**: 待 Windows 环境验证 | **创建日期**: 2026-09-11
|
||||
> **相关文件**: `py/services/model_scanner.py` (`ModelScanner._reconcile_cache`)
|
||||
> **相关历史**: #871 (`76ee59cd`, 路径重叠去重)、#1108 (按文件夹扫描的需求)
|
||||
|
||||
---
|
||||
|
||||
## 背景
|
||||
|
||||
Refresh 按钮走的是 `_reconcile_cache()`(快速增量对账)。2026-09-11 做了一轮性能优化,把两处"预防性"的
|
||||
realpath 全量遍历改成按需触发(详见下方"已完成")。优化后,一次零变更 Refresh 在 5 万文件库上从
|
||||
~1400 ms 降到 ~120 ms。
|
||||
|
||||
清理过程中发现**唯一一处遗留的可疑点**:Windows 专属的大小写不敏感回退分支。它无法在 Linux 上验证,
|
||||
因此单独记录,留待 Windows 机器上确认。
|
||||
|
||||
---
|
||||
|
||||
## 待验证分支(现状)
|
||||
|
||||
`py/services/model_scanner.py` 中 `_reconcile_cache()` 的 walk 循环内:
|
||||
|
||||
```python
|
||||
# Try case-insensitive match on Windows
|
||||
if os.name == 'nt':
|
||||
lower_path = file_path.lower()
|
||||
matched = False
|
||||
for cached_path in cached_paths: # 每个未命中文件都全量扫一遍缓存
|
||||
if cached_path.lower() == lower_path:
|
||||
found_paths.add(cached_path)
|
||||
matched = True
|
||||
break
|
||||
if matched:
|
||||
continue
|
||||
```
|
||||
|
||||
它排在精确匹配(`file_path in cached_paths`)和 realpath 别名匹配之后,只有**未命中**的文件才会走到。
|
||||
|
||||
### 为什么可疑
|
||||
|
||||
1. **可能不可达**:Windows 上 `os.path.realpath()` 会返回磁盘上的真实大小写,因此"缓存路径大小写与磁盘
|
||||
不一致"的情形,理论上已经被上一步的 realpath 别名匹配覆盖。若如此,这段就是纯冗余代码。
|
||||
2. **一旦可达就是 O(N×M)**:每个未命中文件都要遍历全部 `cached_paths` 做小写比较。若某种路径写法让
|
||||
整个库都变成"未命中"(例如缓存里的盘符/大小写形式与 walk 结果系统性不一致),一次 Refresh 会退化
|
||||
成 文件数 × 缓存条目数 次字符串比较,比真实 IO 还贵。
|
||||
3. **没有测试覆盖**:`tests/services/test_model_scanner.py` 没有任何针对该分支的用例(它在 Linux 上
|
||||
被 `os.name == 'nt'` 短路,无法覆盖)。
|
||||
|
||||
---
|
||||
|
||||
## 待办
|
||||
|
||||
- [ ] **验证可达性**:在 Windows 上构造"缓存路径与磁盘真实大小写不一致"的场景,确认 realpath 别名匹配
|
||||
是否已经命中,即上面的 `if os.name == 'nt'` 分支是否还有进入的必要。
|
||||
- [ ] **若不可达 / 冗余**:删除该分支,并在删除处留注释说明 realpath 已覆盖大小写归一(附验证记录)。
|
||||
- [ ] **若可达**:保留语义但改成 O(1)——预先构建一次 `lower_path -> cached_path` 映射(与
|
||||
`cached_real_paths` 同样按需、懒构建),把内层全量扫描换成一次字典查询。
|
||||
- [ ] **补一个 Windows-only 的回归测试**(`pytest.mark.skipif(os.name != "nt", ...)`),锁定最终结论。
|
||||
- [ ] 把验证结论回填到本文件,并同步更新状态行。
|
||||
|
||||
---
|
||||
|
||||
## 验证方法(Windows)
|
||||
|
||||
1. **构造不一致的大小写**:让缓存里的 `file_path` 与磁盘实际路径大小写不同(例如改过盘符/目录大小写,
|
||||
或从另一台机器迁移了 `settings.json` 与持久化缓存),然后在 UI 点 Refresh。
|
||||
2. **看后端日志判据**:
|
||||
- 若 realpath 已覆盖 → 日志应显示 `Cache reconciliation completed in X seconds. Added 0, removed 0 models.`,
|
||||
且**没有** `Found N new files to process` / `Processing <path>`。
|
||||
- 若回退分支在起作用 → 同样应该是 `Added 0, removed 0`(因为 `found_paths` 被补上),这是"分支可达"
|
||||
的证据;反之若出现大量 `Processing ...` 并重新 hash,说明连回退分支也没命中,问题更严重
|
||||
(缓存路径被当成了新文件 + 旧条目被删)。
|
||||
3. **跑测试**:`python -m pytest tests/services/test_model_scanner.py -k reconcile`(该文件在 Windows 上会
|
||||
真实执行 `os.name == 'nt'` 分支)。
|
||||
4. **量化**:如果需要,可在 `_reconcile_cache` 里临时插桩统计该分支的进入次数与内层迭代次数,确认是否为 0。
|
||||
|
||||
---
|
||||
|
||||
## 已完成(本轮优化,供对照)
|
||||
|
||||
同一次清理里已经落地并验证的部分(Linux,5 万文件库):
|
||||
|
||||
- `cached_real_paths` 别名映射改为**首次未命中时**懒构建(原来每次 Refresh 都对全部缓存条目算一次 realpath)。
|
||||
- 每个文件的 `realpath` 移到精确命中检查**之后**(原来对每个文件都算,命中即丢弃)。
|
||||
- `get_model_roots()` 在新增文件处理阶段只快照一次(原来每个新文件重读一次)。
|
||||
- 全量去重 pass 加了 O(1) 前置判断(`cached_size_before != len(cached_paths) or total_added > 0`),
|
||||
零变更且缓存干净时跳过;快照本身含重复路径时仍会自愈。
|
||||
|
||||
结果:零变更 Refresh 5 万文件 **~1400 ms → ~120 ms**;根目录顺序/符号链接别名翻转场景仍是
|
||||
`re-processed=0`(不重新读 metadata、不重新 hash)。测试:`tests/services/test_model_scanner.py`
|
||||
47 项、全量后端 2567 项全部通过。
|
||||
File diff suppressed because one or more lines are too long
+913
-265
File diff suppressed because it is too large
Load Diff
+2668
-2020
File diff suppressed because it is too large
Load Diff
+916
-268
File diff suppressed because it is too large
Load Diff
+913
-265
File diff suppressed because it is too large
Load Diff
+915
-267
File diff suppressed because it is too large
Load Diff
+877
-229
File diff suppressed because it is too large
Load Diff
+883
-235
File diff suppressed because it is too large
Load Diff
+885
-237
File diff suppressed because it is too large
Load Diff
+812
-164
File diff suppressed because it is too large
Load Diff
+815
-167
File diff suppressed because it is too large
Load Diff
+356
-16
@@ -1,16 +1,25 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
import os
|
||||
import platform
|
||||
import posixpath
|
||||
import threading
|
||||
from pathlib import Path
|
||||
import folder_paths # type: ignore
|
||||
import folder_paths # pyright: ignore[reportMissingImports]
|
||||
from typing import Any, Dict, Iterable, List, Mapping, Optional, Set, Tuple
|
||||
import logging
|
||||
import json
|
||||
import urllib.parse
|
||||
import sys as _sys
|
||||
import types as _types
|
||||
import time
|
||||
|
||||
from .utils.cache_paths import CacheType, get_cache_file_path, get_legacy_cache_paths
|
||||
from .utils.constants import (
|
||||
OTHER_MODEL_FOLDER_SUBTYPES,
|
||||
)
|
||||
from .utils.settings_paths import (
|
||||
ensure_settings_file,
|
||||
get_settings_dir,
|
||||
@@ -88,7 +97,7 @@ def _resolve_valid_default_root(
|
||||
|
||||
|
||||
def _normalize_folder_paths_for_comparison(
|
||||
folder_paths: Mapping[str, Iterable[str]],
|
||||
folder_paths: Mapping[str, Any],
|
||||
) -> Dict[str, Set[str]]:
|
||||
"""Normalize folder paths for comparison across libraries."""
|
||||
|
||||
@@ -166,6 +175,13 @@ class Config:
|
||||
self.embeddings_roots = None
|
||||
self.base_models_roots = self._init_checkpoint_paths()
|
||||
self.embeddings_roots = self._init_embedding_paths()
|
||||
# Other-model roots (VAE, upscalers, text encoders, ...): flat deduped
|
||||
# list plus a normalized root -> sub_type map and per-folder_paths-key
|
||||
# roots for settings persistence.
|
||||
self.other_roots: Optional[List[str]] = None
|
||||
self.other_root_subtypes: Dict[str, str] = {}
|
||||
self.other_folder_roots: Dict[str, List[str]] = {}
|
||||
self.other_roots = self._init_other_paths()
|
||||
# Extra paths (only for LoRA Manager, not shared with ComfyUI)
|
||||
self.extra_loras_roots: List[str] = []
|
||||
self.extra_checkpoints_roots: List[str] = []
|
||||
@@ -175,8 +191,7 @@ class Config:
|
||||
|
||||
# Load extra folder paths from active library settings before symlink scan
|
||||
# so both primary and extra paths are discovered in a single pass.
|
||||
if not standalone_mode:
|
||||
self._load_extra_paths_from_settings()
|
||||
self._load_extra_paths_from_settings()
|
||||
|
||||
# Scan symbolic links during initialization
|
||||
self._initialize_symlink_mappings()
|
||||
@@ -191,7 +206,7 @@ class Config:
|
||||
Called during ``Config.__init__`` before the symlink scan so both primary and
|
||||
extra paths are discovered in a single pass. Mirrors the extra-path
|
||||
portion of ``_apply_library_paths`` without replacing the primary roots
|
||||
that were already resolved from ComfyUI's ``folder_paths``.
|
||||
that were already resolved via ``folder_paths.get_folder_paths``.
|
||||
"""
|
||||
try:
|
||||
from .services.settings_manager import get_settings_manager
|
||||
@@ -207,6 +222,12 @@ class Config:
|
||||
if not isinstance(library_config, dict):
|
||||
return
|
||||
|
||||
# Always read recipes_path — it is independent of extra folder paths
|
||||
# and must be set before any early returns below.
|
||||
recipes_path = library_config.get("recipes_path", "")
|
||||
if isinstance(recipes_path, str) and recipes_path:
|
||||
self.recipes_path = recipes_path
|
||||
|
||||
extra_folder_paths = library_config.get("extra_folder_paths")
|
||||
if not isinstance(extra_folder_paths, dict):
|
||||
return
|
||||
@@ -232,10 +253,6 @@ class Config:
|
||||
extra_embedding
|
||||
)
|
||||
|
||||
recipes_path = library_config.get("recipes_path", "")
|
||||
if isinstance(recipes_path, str) and recipes_path:
|
||||
self.recipes_path = recipes_path
|
||||
|
||||
if self.extra_loras_roots:
|
||||
logger.info(
|
||||
"Found extra LoRA roots:"
|
||||
@@ -329,6 +346,10 @@ class Config:
|
||||
"unet": list(self.unet_roots or []),
|
||||
"embeddings": list(self.embeddings_roots or []),
|
||||
}
|
||||
# Persist the other-model roots under their original folder_paths
|
||||
# keys so library switching round-trips them.
|
||||
for key, roots in (self.other_folder_roots or {}).items():
|
||||
target_folder_paths[key] = list(roots)
|
||||
|
||||
normalized_target_paths = _normalize_folder_paths_for_comparison(
|
||||
target_folder_paths
|
||||
@@ -356,6 +377,47 @@ class Config:
|
||||
"Failed to rename legacy 'default' library: %s", rename_error
|
||||
)
|
||||
|
||||
# Clean up a stale "default" library entry that has no meaningful
|
||||
# paths configured (e.g. leftover bootstrap artifact). This only
|
||||
# fires when "comfyui" already exists so we never delete the last
|
||||
# remaining library.
|
||||
if (
|
||||
"default" in libraries
|
||||
and "comfyui" in libraries
|
||||
and isinstance(default_library, Mapping)
|
||||
):
|
||||
default_folder_paths = _normalize_library_folder_paths(
|
||||
default_library
|
||||
)
|
||||
default_extra_paths = default_library.get("extra_folder_paths", {})
|
||||
has_meaningful_paths = bool(default_folder_paths) or bool(
|
||||
default_extra_paths
|
||||
) or any(
|
||||
default_library.get(key)
|
||||
for key in (
|
||||
"default_lora_root",
|
||||
"default_checkpoint_root",
|
||||
"default_unet_root",
|
||||
"default_embedding_root",
|
||||
"recipes_path",
|
||||
)
|
||||
)
|
||||
if not has_meaningful_paths:
|
||||
try:
|
||||
settings_service.delete_library("default")
|
||||
libraries_changed = True
|
||||
logger.info(
|
||||
"Removed stale 'default' library entry "
|
||||
"with no meaningful paths configured"
|
||||
)
|
||||
libraries = settings_service.get_libraries()
|
||||
comfy_library = libraries.get("comfyui", {})
|
||||
except Exception as delete_error:
|
||||
logger.debug(
|
||||
"Failed to remove stale 'default' library: %s",
|
||||
delete_error,
|
||||
)
|
||||
|
||||
default_lora_root = _resolve_valid_default_root(
|
||||
comfy_library.get("default_lora_root", ""),
|
||||
list(self.loras_roots or []),
|
||||
@@ -438,7 +500,7 @@ class Config:
|
||||
import ctypes
|
||||
|
||||
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
|
||||
attrs = ctypes.windll.kernel32.GetFileAttributesW(str(path)) # type: ignore[attr-defined]
|
||||
attrs = ctypes.windll.kernel32.GetFileAttributesW(str(path)) # pyright: ignore[reportAttributeAccessIssue]
|
||||
return attrs != -1 and (attrs & FILE_ATTRIBUTE_REPARSE_POINT)
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking Windows reparse point: {e}")
|
||||
@@ -447,7 +509,7 @@ class Config:
|
||||
logger.error(f"Error checking link status for {path}: {e}")
|
||||
return False
|
||||
|
||||
def _entry_is_symlink(self, entry: os.DirEntry) -> bool:
|
||||
def _entry_is_symlink(self, entry: os.DirEntry[str]) -> bool:
|
||||
"""Check if a directory entry is a symlink, including Windows junctions."""
|
||||
if entry.is_symlink():
|
||||
return True
|
||||
@@ -456,7 +518,7 @@ class Config:
|
||||
import ctypes
|
||||
|
||||
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
|
||||
attrs = ctypes.windll.kernel32.GetFileAttributesW(entry.path) # type: ignore[attr-defined]
|
||||
attrs = ctypes.windll.kernel32.GetFileAttributesW(entry.path) # pyright: ignore[reportAttributeAccessIssue]
|
||||
return attrs != -1 and (attrs & FILE_ATTRIBUTE_REPARSE_POINT)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -474,6 +536,7 @@ class Config:
|
||||
roots.extend(self.loras_roots or [])
|
||||
roots.extend(self.base_models_roots or [])
|
||||
roots.extend(self.embeddings_roots or [])
|
||||
roots.extend(self.other_roots or [])
|
||||
# Include extra paths for scanning symlinks
|
||||
roots.extend(self.extra_loras_roots or [])
|
||||
roots.extend(self.extra_checkpoints_roots or [])
|
||||
@@ -814,6 +877,8 @@ class Config:
|
||||
preview_roots.update(self._expand_preview_root(root))
|
||||
for root in self.embeddings_roots or []:
|
||||
preview_roots.update(self._expand_preview_root(root))
|
||||
for root in self.other_roots or []:
|
||||
preview_roots.update(self._expand_preview_root(root))
|
||||
# Include extra paths for preview access
|
||||
for root in self.extra_loras_roots or []:
|
||||
preview_roots.update(self._expand_preview_root(root))
|
||||
@@ -834,7 +899,7 @@ class Config:
|
||||
path for path in preview_roots if path.is_absolute()
|
||||
}
|
||||
logger.debug(
|
||||
"Preview roots rebuilt: %d paths from %d lora roots (%d extra), %d checkpoint roots (%d extra), %d embedding roots (%d extra), %d symlink mappings",
|
||||
"Preview roots rebuilt: %d paths from %d lora roots (%d extra), %d checkpoint roots (%d extra), %d embedding roots (%d extra), %d other roots, %d symlink mappings",
|
||||
len(self._preview_root_paths),
|
||||
len(self.loras_roots or []),
|
||||
len(self.extra_loras_roots or []),
|
||||
@@ -842,6 +907,7 @@ class Config:
|
||||
len(self.extra_checkpoints_roots or []),
|
||||
len(self.embeddings_roots or []),
|
||||
len(self.extra_embeddings_roots or []),
|
||||
len(self.other_roots or []),
|
||||
len(self._path_mappings),
|
||||
)
|
||||
|
||||
@@ -1080,10 +1146,159 @@ class Config:
|
||||
|
||||
return unique_paths
|
||||
|
||||
def _get_enabled_other_folder_keys(self) -> List[str]:
|
||||
"""Return the OTHER_MODEL_FOLDER_SUBTYPES keys that are enabled.
|
||||
|
||||
Other Models management is opt-in: while ``enable_other_models`` is
|
||||
off (the default) no other-model folder is scanned at all. When it is
|
||||
on, only the folder keys of the enabled sub_types are scanned
|
||||
(text_encoder merges ``text_encoders`` with the legacy ``clip`` key).
|
||||
"""
|
||||
try:
|
||||
from .services.settings_manager import get_settings_manager
|
||||
|
||||
enabled_sub_types = get_settings_manager().get_enabled_other_sub_types()
|
||||
except Exception:
|
||||
enabled_sub_types = []
|
||||
if not enabled_sub_types:
|
||||
return []
|
||||
allowed = set(enabled_sub_types)
|
||||
return [
|
||||
key
|
||||
for key, sub_type in OTHER_MODEL_FOLDER_SUBTYPES.items()
|
||||
if sub_type in allowed
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _collapse_legacy_folder_keys(keys: List[str]) -> List[str]:
|
||||
"""Drop folder keys the host already normalizes onto another queried key.
|
||||
|
||||
ComfyUI's ``folder_paths`` rewrites legacy names before every access
|
||||
(``clip`` -> ``text_encoders``, ``unet`` -> ``diffusion_models``), and
|
||||
registers both legacy directories under the canonical key, so
|
||||
``get_folder_paths("clip")`` returns exactly the same list as
|
||||
``get_folder_paths("text_encoders")``. Querying both therefore reports
|
||||
every text-encoder folder twice and trips the overlap guard with a
|
||||
conflict the user cannot fix.
|
||||
|
||||
When the host exposes ``map_legacy`` the alias is provably redundant and
|
||||
is skipped (an empty canonical list implies an empty alias list).
|
||||
Without it - the standalone mock, whose keys are independent
|
||||
``settings.json`` entries - every key is kept, because a ``clip``-only
|
||||
configuration is then genuinely distinct.
|
||||
"""
|
||||
map_legacy = getattr(folder_paths, "map_legacy", None)
|
||||
if not callable(map_legacy):
|
||||
return list(keys)
|
||||
|
||||
queried = set(keys)
|
||||
collapsed: List[str] = []
|
||||
for key in keys:
|
||||
try:
|
||||
canonical = map_legacy(key)
|
||||
except Exception:
|
||||
canonical = key
|
||||
if canonical != key and canonical in queried:
|
||||
logger.debug(
|
||||
"Skipping legacy folder key '%s'; the host resolves it to "
|
||||
"'%s', which is queried as well.",
|
||||
key,
|
||||
canonical,
|
||||
)
|
||||
continue
|
||||
collapsed.append(key)
|
||||
return collapsed
|
||||
|
||||
def _prepare_other_paths(
|
||||
self, folder_path_map: Mapping[str, Iterable[str]]
|
||||
) -> Tuple[List[str], Dict[str, str], Dict[str, List[str]]]:
|
||||
"""Prepare other-model paths from a folder_paths-key -> raw paths map.
|
||||
|
||||
Returns:
|
||||
Tuple of (all_unique_roots, business_root -> sub_type map,
|
||||
folder_paths key -> business roots). This method does NOT modify
|
||||
instance variables - callers must set them.
|
||||
"""
|
||||
unique_paths: List[str] = []
|
||||
sub_type_map: Dict[str, str] = {}
|
||||
per_key_roots: Dict[str, List[str]] = {}
|
||||
# real path -> (business path, sub_type) of the category that claimed it
|
||||
seen_real_paths: Dict[str, Tuple[str, str]] = {}
|
||||
|
||||
# Cross-scanner overlap detection: warn when an "other" root is
|
||||
# already covered by the checkpoints/unet or embeddings scanners.
|
||||
# Kept (not dropped) on purpose - duplicate cards across pages are
|
||||
# cosmetic, while dropping would silently unmanage the files.
|
||||
covered_real_paths = {
|
||||
os.path.normpath(os.path.realpath(path)).replace(os.sep, "/"): path
|
||||
for path in [
|
||||
*(self.base_models_roots or []),
|
||||
*(self.embeddings_roots or []),
|
||||
]
|
||||
if isinstance(path, str) and path.strip() and os.path.exists(path)
|
||||
}
|
||||
|
||||
for key, sub_type in OTHER_MODEL_FOLDER_SUBTYPES.items():
|
||||
raw_paths = folder_path_map.get(key)
|
||||
if not raw_paths:
|
||||
continue
|
||||
path_map = self._dedupe_existing_paths(raw_paths)
|
||||
key_roots: List[str] = []
|
||||
for real_path, business_path in sorted(
|
||||
path_map.items(), key=lambda item: item[1].lower()
|
||||
):
|
||||
seen = seen_real_paths.get(real_path)
|
||||
if seen is not None:
|
||||
seen_business_path, seen_sub_type = seen
|
||||
if seen_sub_type == sub_type:
|
||||
# Same category reached through a second folder_paths
|
||||
# key (legacy alias, or a sub_type spanning two keys).
|
||||
# Expected, so never a "fix your configuration" warning.
|
||||
logger.debug(
|
||||
"Ignoring duplicate folder '%s' for category '%s' "
|
||||
"(already covered by '%s').",
|
||||
business_path,
|
||||
sub_type,
|
||||
seen_business_path,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Detected the same folder '%s' under multiple other-model "
|
||||
"categories ('%s' is already mapped as '%s'). Keeping the "
|
||||
"first category; please fix your path configuration.",
|
||||
business_path,
|
||||
seen_business_path,
|
||||
seen_sub_type,
|
||||
)
|
||||
continue
|
||||
seen_real_paths[real_path] = (business_path, sub_type)
|
||||
unique_paths.append(business_path)
|
||||
key_roots.append(business_path)
|
||||
sub_type_map[business_path] = sub_type
|
||||
|
||||
if real_path != business_path:
|
||||
self.add_path_mapping(business_path, real_path)
|
||||
|
||||
covered_by = covered_real_paths.get(real_path)
|
||||
if covered_by:
|
||||
logger.warning(
|
||||
"Detected an other-model root ('%s', category '%s') that "
|
||||
"overlaps an existing checkpoints/embeddings root ('%s'). "
|
||||
"The same files will appear on both pages; please review "
|
||||
"your path configuration.",
|
||||
business_path,
|
||||
key,
|
||||
covered_by,
|
||||
)
|
||||
if key_roots:
|
||||
per_key_roots[key] = key_roots
|
||||
|
||||
return unique_paths, sub_type_map, per_key_roots
|
||||
|
||||
def _apply_library_paths(
|
||||
self,
|
||||
folder_paths: Mapping[str, Iterable[str]],
|
||||
extra_folder_paths: Optional[Mapping[str, Iterable[str]]] = None,
|
||||
folder_paths: Mapping[str, Any],
|
||||
extra_folder_paths: Optional[Mapping[str, Any]] = None,
|
||||
recipes_path: str = "",
|
||||
) -> None:
|
||||
self._path_mappings.clear()
|
||||
@@ -1103,6 +1318,16 @@ class Config:
|
||||
) = self._prepare_checkpoint_paths(checkpoint_paths, unet_paths)
|
||||
self.embeddings_roots = self._prepare_embedding_paths(embedding_paths)
|
||||
|
||||
other_path_map = {
|
||||
key: folder_paths.get(key, []) or []
|
||||
for key in self._get_enabled_other_folder_keys()
|
||||
}
|
||||
(
|
||||
self.other_roots,
|
||||
self.other_root_subtypes,
|
||||
self.other_folder_roots,
|
||||
) = self._prepare_other_paths(other_path_map)
|
||||
|
||||
# Process extra paths (only for LoRA Manager, not shared with ComfyUI)
|
||||
extra_paths = extra_folder_paths or {}
|
||||
extra_lora_paths = extra_paths.get("loras", []) or []
|
||||
@@ -1219,6 +1444,104 @@ class Config:
|
||||
logger.warning(f"Error initializing embedding paths: {e}")
|
||||
return []
|
||||
|
||||
def _init_other_paths(self) -> List[str]:
|
||||
"""Initialize and validate other-model paths from ComfyUI settings.
|
||||
|
||||
Iterates the enabled OTHER_MODEL_FOLDER_SUBTYPES keys and pulls each
|
||||
from ``folder_paths.get_folder_paths(key)`` (in standalone mode the
|
||||
mock serves arbitrary keys from ``settings.json.folder_paths``).
|
||||
Legacy aliases the host normalizes onto a canonical key (``clip`` ->
|
||||
``text_encoders``) are collapsed first so the same folders are not
|
||||
reported twice.
|
||||
"""
|
||||
try:
|
||||
folder_path_map: Dict[str, List[str]] = {}
|
||||
for key in self._collapse_legacy_folder_keys(
|
||||
self._get_enabled_other_folder_keys()
|
||||
):
|
||||
try:
|
||||
folder_path_map[key] = folder_paths.get_folder_paths(key)
|
||||
except Exception as exc:
|
||||
logger.debug("Error reading folder paths for '%s': %s", key, exc)
|
||||
|
||||
(
|
||||
unique_paths,
|
||||
self.other_root_subtypes,
|
||||
self.other_folder_roots,
|
||||
) = self._prepare_other_paths(folder_path_map)
|
||||
|
||||
logger.info(
|
||||
"Found other model roots:"
|
||||
+ ("\n - " + "\n - ".join(unique_paths) if unique_paths else "[]")
|
||||
)
|
||||
|
||||
if not unique_paths:
|
||||
logger.info("No valid other-model folders found in configuration")
|
||||
return []
|
||||
|
||||
return unique_paths
|
||||
except Exception as e:
|
||||
logger.warning(f"Error initializing other model paths: {e}")
|
||||
return []
|
||||
|
||||
def refresh_other_roots(self) -> None:
|
||||
"""Rebuild other-model roots after the management toggles changed.
|
||||
|
||||
Called when ``enable_other_models`` / ``enabled_other_sub_types`` are
|
||||
updated so the scanner immediately reflects the new folder set without
|
||||
a full application restart.
|
||||
"""
|
||||
self.other_roots = self._init_other_paths()
|
||||
self._rebuild_preview_roots()
|
||||
|
||||
def get_other_models_availability(self) -> Dict[str, Any]:
|
||||
"""Report the other-model folders the host can actually expose.
|
||||
|
||||
Independent of the opt-in ``enable_other_models`` toggle: this answers
|
||||
"could Other Models management work here at all?". ComfyUI mode almost
|
||||
always has these folder keys registered, while standalone mode only
|
||||
knows the keys present in ``settings.json.folder_paths`` - so the UI
|
||||
uses this to decide whether announcing the feature would be actionable.
|
||||
|
||||
Returns:
|
||||
``{"available": bool, "sub_types": {sub_type: [existing roots]}}``.
|
||||
A folder only counts when it exists on disk; an empty folder still
|
||||
counts because CivitAI downloads can target it.
|
||||
"""
|
||||
sub_types: Dict[str, List[str]] = {}
|
||||
try:
|
||||
keys = self._collapse_legacy_folder_keys(
|
||||
list(OTHER_MODEL_FOLDER_SUBTYPES.keys())
|
||||
)
|
||||
except Exception: # pragma: no cover - defensive
|
||||
keys = list(OTHER_MODEL_FOLDER_SUBTYPES.keys())
|
||||
|
||||
for key in keys:
|
||||
sub_type = OTHER_MODEL_FOLDER_SUBTYPES.get(key)
|
||||
if not sub_type:
|
||||
continue
|
||||
try:
|
||||
raw_paths = folder_paths.get_folder_paths(key)
|
||||
except Exception as exc:
|
||||
logger.debug("Error probing folder paths for '%s': %s", key, exc)
|
||||
continue
|
||||
|
||||
bucket = sub_types.setdefault(sub_type, [])
|
||||
for root in sorted(
|
||||
self._dedupe_existing_paths(raw_paths or []).values(),
|
||||
key=lambda path: path.lower(),
|
||||
):
|
||||
if root not in bucket:
|
||||
bucket.append(root)
|
||||
|
||||
available_sub_types = {
|
||||
sub_type: roots for sub_type, roots in sub_types.items() if roots
|
||||
}
|
||||
return {
|
||||
"available": bool(available_sub_types),
|
||||
"sub_types": available_sub_types,
|
||||
}
|
||||
|
||||
def get_preview_static_url(self, preview_path: str) -> str:
|
||||
if not preview_path:
|
||||
return ""
|
||||
@@ -1380,4 +1703,21 @@ class Config:
|
||||
|
||||
|
||||
# Global config instance
|
||||
config = Config()
|
||||
# NOTE: Guard against re-import. When ServiceRegistry.get_lora_scanner() triggers
|
||||
# a fresh import of lora_scanner → config, we must NOT re-execute Config.__init__()
|
||||
# (which re-scans all roots, re-registers libraries, etc.).
|
||||
#
|
||||
# Strategy: store the config instance in a dedicated sentinel module
|
||||
# ('_lm_config_cache') that is NEVER removed from sys.modules (its key does
|
||||
# NOT start with 'py.'), so it survives re-imports of py.* modules.
|
||||
_CONFIG_SENTINEL = "_lm_config_cache"
|
||||
config: Config
|
||||
if _CONFIG_SENTINEL in _sys.modules:
|
||||
# Re-import: reuse the existing singleton from the sentinel.
|
||||
config = _sys.modules[_CONFIG_SENTINEL].config
|
||||
else:
|
||||
config = Config()
|
||||
# Register the sentinel so re-imports of py.config find us.
|
||||
_sentinel_mod = _types.ModuleType(_CONFIG_SENTINEL)
|
||||
setattr(_sentinel_mod, "config", config)
|
||||
_sys.modules[_CONFIG_SENTINEL] = _sentinel_mod
|
||||
|
||||
+37
-1
@@ -14,7 +14,7 @@ standalone_mode = (
|
||||
if not standalone_mode:
|
||||
setup_logging()
|
||||
|
||||
from server import PromptServer # type: ignore
|
||||
from server import PromptServer # pyright: ignore[reportMissingImports]
|
||||
|
||||
from .config import config
|
||||
from .services.model_service_factory import (
|
||||
@@ -25,10 +25,12 @@ from .routes.recipe_routes import RecipeRoutes
|
||||
from .routes.stats_routes import StatsRoutes
|
||||
from .routes.update_routes import UpdateRoutes
|
||||
from .routes.misc_routes import MiscRoutes
|
||||
from .routes.pending_delete_routes import PendingDeleteRoutes
|
||||
from .routes.preview_routes import PreviewRoutes
|
||||
from .routes.example_images_routes import ExampleImagesRoutes
|
||||
from .services.service_registry import ServiceRegistry
|
||||
from .services.settings_manager import get_settings_manager
|
||||
from .services.pending_delete_service import get_pending_delete_service
|
||||
from .utils.example_images_migration import ExampleImagesMigration
|
||||
from .services.websocket_manager import ws_manager
|
||||
from .services.example_images_cleanup_service import ExampleImagesCleanupService
|
||||
@@ -170,6 +172,7 @@ class LoraManager:
|
||||
RecipeRoutes.setup_routes(app)
|
||||
UpdateRoutes.setup_routes(app)
|
||||
MiscRoutes.setup_routes(app)
|
||||
PendingDeleteRoutes.setup_routes(app)
|
||||
ExampleImagesRoutes.setup_routes(app, ws_manager=ws_manager)
|
||||
PreviewRoutes.setup_routes(app)
|
||||
|
||||
@@ -208,10 +211,15 @@ class LoraManager:
|
||||
# Initialize WebSocket manager
|
||||
await ServiceRegistry.get_websocket_manager()
|
||||
|
||||
# Preload LLM model catalog (background task, non-blocking)
|
||||
from .services.llm_service import LLMService
|
||||
await LLMService.get_instance()
|
||||
|
||||
# Initialize scanners in background
|
||||
lora_scanner = await ServiceRegistry.get_lora_scanner()
|
||||
checkpoint_scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||
embedding_scanner = await ServiceRegistry.get_embedding_scanner()
|
||||
other_scanner = await ServiceRegistry.get_other_scanner()
|
||||
|
||||
# Initialize recipe scanner if needed
|
||||
recipe_scanner = await ServiceRegistry.get_recipe_scanner()
|
||||
@@ -229,6 +237,10 @@ class LoraManager:
|
||||
embedding_scanner.initialize_in_background(),
|
||||
name="embedding_cache_init",
|
||||
),
|
||||
asyncio.create_task(
|
||||
other_scanner.initialize_in_background(),
|
||||
name="other_cache_init",
|
||||
),
|
||||
asyncio.create_task(
|
||||
recipe_scanner.initialize_in_background(), name="recipe_cache_init"
|
||||
),
|
||||
@@ -241,6 +253,20 @@ class LoraManager:
|
||||
cls._run_post_initialization_tasks(init_tasks), name="post_init_tasks"
|
||||
)
|
||||
|
||||
# Startup sweep: purge pending-delete batches that expired during a
|
||||
# previous run. Non-blocking (fire-and-forget); purge_expired only
|
||||
# removes already-expired batches, so a staged undo that survived a
|
||||
# restart stays restorable. scan_roots=True runs the reconciliation
|
||||
# pass first so leftover batches (the in-process registry is empty
|
||||
# after a restart) are re-discovered on disk. Covers both plugin
|
||||
# and standalone modes (StandaloneLoraManager reuses this
|
||||
# classmethod).
|
||||
pending_delete_service = await get_pending_delete_service()
|
||||
asyncio.create_task(
|
||||
pending_delete_service.purge_expired(scan_roots=True),
|
||||
name="pending_delete_startup_sweep",
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"LoRA Manager: All services initialized and background tasks scheduled"
|
||||
)
|
||||
@@ -307,6 +333,7 @@ class LoraManager:
|
||||
all_roots.update(config.loras_roots)
|
||||
all_roots.update(config.base_models_roots or [])
|
||||
all_roots.update(config.embeddings_roots or [])
|
||||
all_roots.update(config.other_roots or [])
|
||||
|
||||
total_deleted = 0
|
||||
total_size_freed = 0
|
||||
@@ -436,5 +463,14 @@ class LoraManager:
|
||||
try:
|
||||
logger.info("LoRA Manager: Cleaning up services")
|
||||
|
||||
# Cancel any in-flight scanner initialization tasks so thread-pool
|
||||
# workers (e.g. _initialize_cache_sync) can break out of their loops
|
||||
# when the server shuts down (e.g. Ctrl+C on WSL).
|
||||
for name in ("lora_scanner", "checkpoint_scanner", "embedding_scanner", "other_scanner"):
|
||||
scanner = ServiceRegistry.get_service_sync(name)
|
||||
if scanner is not None and hasattr(scanner, "cancel_task"):
|
||||
scanner.cancel_task()
|
||||
logger.debug("LoRA Manager: Cancelled %s", name)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during cleanup: {e}", exc_info=True)
|
||||
|
||||
@@ -22,7 +22,7 @@ if not standalone_mode:
|
||||
|
||||
logger.info("ComfyUI Metadata Collector initialized")
|
||||
|
||||
def get_metadata(prompt_id=None): # type: ignore[no-redef]
|
||||
def get_metadata(prompt_id=None): # pyright: ignore[reportRedeclaration]
|
||||
"""Helper function to get metadata from the registry"""
|
||||
registry = MetadataRegistry()
|
||||
return registry.get_metadata(prompt_id)
|
||||
@@ -31,6 +31,6 @@ else:
|
||||
def init():
|
||||
logger.info("ComfyUI Metadata Collector disabled in standalone mode")
|
||||
|
||||
def get_metadata(prompt_id=None): # type: ignore[no-redef]
|
||||
def get_metadata(prompt_id=None): # pyright: ignore[reportRedeclaration]
|
||||
"""Dummy implementation for standalone mode"""
|
||||
return {}
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
"""Constants used by the metadata collector"""
|
||||
|
||||
# Sentinel value for clip_skip to distinguish "unconnected / widget default"
|
||||
# from "user wired value 0". Both ComfyUI CLIPSetLastLayer (-24..-1) and
|
||||
# A1111 conventions treat 0 as meaningless for clip skipping, but users may
|
||||
# explicitly wire 0 to the overwrite node to express "no clip skip / default".
|
||||
CLIP_SKIP_SENTINEL = -25
|
||||
|
||||
# Metadata categories
|
||||
MODELS = "models"
|
||||
PROMPTS = "prompts"
|
||||
@@ -9,6 +15,14 @@ EMBEDDINGS = "embeddings"
|
||||
SIZE = "size"
|
||||
IMAGES = "images"
|
||||
IS_SAMPLER = "is_sampler" # New constant to mark sampler nodes
|
||||
OVERWRITE = "overwrite" # Manual metadata overwrite from MetadataOverwriteLM node
|
||||
|
||||
# Field names that the MetadataOverwriteLM node and its extractor share
|
||||
METADATA_OVERWRITE_FIELDS = (
|
||||
"prompt", "negative_prompt", "seed", "steps", "cfg_scale",
|
||||
"sampler", "scheduler", "model", "loras", "size",
|
||||
"clip_skip", "additional_data",
|
||||
)
|
||||
|
||||
# Complete list of categories to track
|
||||
METADATA_CATEGORIES = [MODELS, PROMPTS, SAMPLING, LORAS, EMBEDDINGS, SIZE, IMAGES]
|
||||
METADATA_CATEGORIES = [MODELS, PROMPTS, SAMPLING, LORAS, EMBEDDINGS, SIZE, IMAGES, OVERWRITE]
|
||||
|
||||
@@ -16,7 +16,7 @@ class MetadataHook:
|
||||
execution = None
|
||||
try:
|
||||
# Try direct import first
|
||||
import execution # type: ignore
|
||||
import execution # pyright: ignore[reportMissingImports]
|
||||
except ImportError:
|
||||
# Try to locate from system modules
|
||||
for module_name in sys.modules:
|
||||
@@ -83,7 +83,8 @@ class MetadataHook:
|
||||
|
||||
# Record inputs before execution
|
||||
if node_id is not None:
|
||||
registry.record_node_execution(node_id, class_type, input_data_all, None)
|
||||
return_types = getattr(obj, 'RETURN_TYPES', None)
|
||||
registry.record_node_execution(node_id, class_type, input_data_all, None, return_types=return_types)
|
||||
except Exception as e:
|
||||
logger.error(f"Error collecting metadata (pre-execution): {str(e)}")
|
||||
|
||||
@@ -114,7 +115,8 @@ class MetadataHook:
|
||||
|
||||
# Record outputs after execution
|
||||
if node_id is not None:
|
||||
registry.update_node_execution(node_id, class_type, results)
|
||||
return_types = getattr(obj, 'RETURN_TYPES', None)
|
||||
registry.update_node_execution(node_id, class_type, results, return_types=return_types)
|
||||
except Exception as e:
|
||||
logger.error(f"Error collecting metadata (post-execution): {str(e)}")
|
||||
|
||||
@@ -135,10 +137,13 @@ class MetadataHook:
|
||||
# Store the dynprompt reference for node lookups
|
||||
if hasattr(prompt, 'original_prompt'):
|
||||
registry.set_current_prompt(prompt)
|
||||
|
||||
|
||||
# Store extra_data for accessing full workflow node properties
|
||||
registry.set_extra_data(extra_data)
|
||||
|
||||
# Execute the original function
|
||||
return original_execute(*args, **kwargs)
|
||||
|
||||
|
||||
# Replace the functions
|
||||
execution._map_node_over_list = map_node_over_list_with_metadata
|
||||
execution.execute = execute_with_prompt_tracking
|
||||
@@ -163,7 +168,8 @@ class MetadataHook:
|
||||
class_type = obj.__class__.__name__
|
||||
node_id = unique_id
|
||||
if node_id is not None:
|
||||
registry.record_node_execution(node_id, class_type, input_data_all, None)
|
||||
return_types = getattr(obj, 'RETURN_TYPES', None)
|
||||
registry.record_node_execution(node_id, class_type, input_data_all, None, return_types=return_types)
|
||||
except Exception as e:
|
||||
logger.error(f"Error collecting metadata (pre-execution): {str(e)}")
|
||||
|
||||
@@ -180,7 +186,8 @@ class MetadataHook:
|
||||
class_type = obj.__class__.__name__
|
||||
node_id = unique_id
|
||||
if node_id is not None:
|
||||
registry.update_node_execution(node_id, class_type, results)
|
||||
return_types = getattr(obj, 'RETURN_TYPES', None)
|
||||
registry.update_node_execution(node_id, class_type, results, return_types=return_types)
|
||||
except Exception as e:
|
||||
logger.error(f"Error collecting metadata (post-execution): {str(e)}")
|
||||
|
||||
@@ -202,6 +209,9 @@ class MetadataHook:
|
||||
if hasattr(prompt, 'original_prompt'):
|
||||
registry.set_current_prompt(prompt)
|
||||
|
||||
# Store extra_data for accessing full workflow node properties
|
||||
registry.set_extra_data(extra_data)
|
||||
|
||||
# Execute the original function
|
||||
return await original_execute(*args, **kwargs)
|
||||
|
||||
|
||||
@@ -1,15 +1,68 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from .constants import IMAGES
|
||||
|
||||
# Check if running in standalone mode
|
||||
standalone_mode = os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1" or os.environ.get("HF_HUB_DISABLE_TELEMETRY", "0") == "0"
|
||||
|
||||
from .constants import MODELS, PROMPTS, SAMPLING, LORAS, SIZE, IS_SAMPLER
|
||||
from .constants import MODELS, PROMPTS, SAMPLING, LORAS, SIZE, IS_SAMPLER, OVERWRITE
|
||||
from .node_extractors import NODE_EXTRACTORS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Keys that identify metadata hint marks stored in node.properties.lm_marker_role
|
||||
_META_MARK_PREFIX = "meta_"
|
||||
_MARK_PRIMARY_MODEL = "primary_model"
|
||||
_MARK_PRIMARY_SAMPLER = "primary_sampler"
|
||||
_MARK_POSITIVE_PROMPT = "positive_prompt"
|
||||
_MARK_NEGATIVE_PROMPT = "negative_prompt"
|
||||
|
||||
class MetadataProcessor:
|
||||
"""Process and format collected metadata"""
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _get_user_marks(metadata):
|
||||
"""Scan workflow nodes (from extra_data.extra_pnginfo.workflow) for user-assigned
|
||||
metadata hint marks stored in node.properties.lm_marker_role.
|
||||
|
||||
Returns a dict mapping mark type keys to node IDs.
|
||||
Example: {'primary_model': '42', 'primary_sampler': '17'}
|
||||
"""
|
||||
marks: dict[str, str] = {}
|
||||
|
||||
# Primary source: extra_data.extra_pnginfo.workflow.nodes (has full properties)
|
||||
extra_data = metadata.get("extra_data")
|
||||
if extra_data and isinstance(extra_data, dict):
|
||||
extra_pnginfo = extra_data.get("extra_pnginfo", {})
|
||||
if isinstance(extra_pnginfo, dict):
|
||||
workflow = extra_pnginfo.get("workflow", {})
|
||||
nodes = workflow.get("nodes", [])
|
||||
for node in nodes:
|
||||
node_id = str(node.get("id", ""))
|
||||
role = node.get("properties", {}).get("lm_marker_role", "")
|
||||
if role.startswith(_META_MARK_PREFIX):
|
||||
mark_type = role[len(_META_MARK_PREFIX):]
|
||||
if mark_type in marks:
|
||||
logger.warning(
|
||||
"Duplicate meta hint '%s': node %s (previous: %s), "
|
||||
"last match wins",
|
||||
mark_type, node_id, marks[mark_type],
|
||||
)
|
||||
marks[mark_type] = node_id
|
||||
|
||||
# Fallback: try prompt.original_prompt (API-only submissions may not have workflow)
|
||||
if not marks:
|
||||
prompt = metadata.get("current_prompt")
|
||||
if prompt and getattr(prompt, "original_prompt", None):
|
||||
for node_id, node_data in prompt.original_prompt.items():
|
||||
role = node_data.get("properties", {}).get("lm_marker_role", "")
|
||||
if role.startswith(_META_MARK_PREFIX):
|
||||
mark_type = role[len(_META_MARK_PREFIX):]
|
||||
marks[mark_type] = node_id
|
||||
|
||||
return marks
|
||||
|
||||
@staticmethod
|
||||
def find_primary_sampler(metadata, downstream_id=None):
|
||||
"""
|
||||
@@ -161,6 +214,24 @@ class MetadataProcessor:
|
||||
max_denoise = denoise
|
||||
primary_sampler = sampler_info
|
||||
primary_sampler_id = node_id
|
||||
|
||||
# Last resort: any registered sampler. Samplers without a denoise or
|
||||
# add_noise parameter (e.g. multi-stage samplers like KreaTwoStageSampler)
|
||||
# are not caught by the criteria above. Prefer execution order so the
|
||||
# first executed sampler wins, matching the downstream_id branch.
|
||||
if primary_sampler is None:
|
||||
sampler_ids = [
|
||||
node_id
|
||||
for node_id, sampler_info in metadata.get(SAMPLING, {}).items()
|
||||
if sampler_info.get(IS_SAMPLER, False)
|
||||
]
|
||||
if sampler_ids:
|
||||
if downstream_id and "execution_order" in metadata:
|
||||
for node_id in metadata["execution_order"]:
|
||||
if node_id in sampler_ids:
|
||||
return node_id, metadata[SAMPLING][node_id]
|
||||
primary_sampler_id = sampler_ids[0]
|
||||
primary_sampler = metadata[SAMPLING][sampler_ids[0]]
|
||||
|
||||
return primary_sampler_id, primary_sampler
|
||||
|
||||
@@ -471,20 +542,57 @@ class MetadataProcessor:
|
||||
"checkpoint": None,
|
||||
"loras": "",
|
||||
"size": None,
|
||||
"clip_skip": None
|
||||
"clip_skip": None,
|
||||
"additional_data": "",
|
||||
}
|
||||
|
||||
# Get the prompt object for node relationship tracing
|
||||
prompt = metadata.get("current_prompt")
|
||||
|
||||
# Find the primary KSampler node
|
||||
primary_sampler_id, primary_sampler = MetadataProcessor.find_primary_sampler(metadata, id)
|
||||
|
||||
# Directly get checkpoint from metadata instead of tracing
|
||||
# Pass primary_sampler_id to avoid redundant calculation
|
||||
checkpoint = MetadataProcessor.find_primary_checkpoint(metadata, id, primary_sampler_id)
|
||||
if checkpoint:
|
||||
params["checkpoint"] = checkpoint
|
||||
|
||||
# ---- User marks: override heuristic inference with user-assigned hints ----
|
||||
user_marks = MetadataProcessor._get_user_marks(metadata)
|
||||
|
||||
# Find the primary KSampler node (user mark takes priority)
|
||||
primary_sampler_id = None
|
||||
primary_sampler = None
|
||||
if _MARK_PRIMARY_SAMPLER in user_marks:
|
||||
marked_id = user_marks[_MARK_PRIMARY_SAMPLER]
|
||||
sampler_data = metadata.get(SAMPLING, {}).get(marked_id)
|
||||
if sampler_data and sampler_data.get(IS_SAMPLER):
|
||||
primary_sampler_id = marked_id
|
||||
primary_sampler = sampler_data
|
||||
else:
|
||||
logger.warning(
|
||||
"User-marked primary sampler %s has no runtime metadata, "
|
||||
"falling back to heuristic",
|
||||
marked_id,
|
||||
)
|
||||
if primary_sampler is None:
|
||||
primary_sampler_id, primary_sampler = MetadataProcessor.find_primary_sampler(metadata, id)
|
||||
|
||||
# Resolve checkpoint / model (user mark takes priority)
|
||||
if _MARK_PRIMARY_MODEL in user_marks:
|
||||
marked_id = user_marks[_MARK_PRIMARY_MODEL]
|
||||
if marked_id in metadata.get(MODELS, {}):
|
||||
params["checkpoint"] = metadata[MODELS][marked_id].get("name")
|
||||
else:
|
||||
extra_data = metadata.get("extra_data")
|
||||
extra_pnginfo = extra_data.get("extra_pnginfo", {}) if extra_data and isinstance(extra_data, dict) else {}
|
||||
workflow = extra_pnginfo.get("workflow", {}) if isinstance(extra_pnginfo, dict) else {}
|
||||
node_type = "unknown"
|
||||
for n in workflow.get("nodes", []):
|
||||
if str(n.get("id", "")) == marked_id:
|
||||
node_type = n.get("type", "unknown")
|
||||
break
|
||||
logger.warning(
|
||||
"User-marked primary model %s (type=%s, registered=%s) has no runtime metadata, "
|
||||
"falling back to heuristic",
|
||||
marked_id, node_type, node_type in NODE_EXTRACTORS,
|
||||
)
|
||||
if params["checkpoint"] is None:
|
||||
checkpoint = MetadataProcessor.find_primary_checkpoint(metadata, id, primary_sampler_id)
|
||||
if checkpoint:
|
||||
params["checkpoint"] = checkpoint
|
||||
|
||||
# Check if guidance parameter exists in any sampling node
|
||||
for node_id, sampler_info in metadata.get(SAMPLING, {}).items():
|
||||
@@ -539,7 +647,22 @@ class MetadataProcessor:
|
||||
|
||||
# For SamplerCustom, handle any additional parameters
|
||||
MetadataProcessor.handle_custom_advanced_sampler(metadata, prompt, primary_sampler_id, params)
|
||||
|
||||
|
||||
# ---- User marks: override prompts with explicitly tagged nodes ----
|
||||
prompts_data = metadata.get(PROMPTS, {})
|
||||
if _MARK_POSITIVE_PROMPT in user_marks:
|
||||
pos_id = user_marks[_MARK_POSITIVE_PROMPT]
|
||||
if pos_id in prompts_data:
|
||||
prompt_text = prompts_data[pos_id].get("text") or prompts_data[pos_id].get("positive_text")
|
||||
if prompt_text:
|
||||
params["prompt"] = prompt_text
|
||||
if _MARK_NEGATIVE_PROMPT in user_marks:
|
||||
neg_id = user_marks[_MARK_NEGATIVE_PROMPT]
|
||||
if neg_id in prompts_data:
|
||||
prompt_text = prompts_data[neg_id].get("text") or prompts_data[neg_id].get("negative_text")
|
||||
if prompt_text:
|
||||
params["negative_prompt"] = prompt_text
|
||||
|
||||
# Size extraction is same for all sampler types
|
||||
# Check if the sampler itself has size information (from latent_image)
|
||||
if primary_sampler_id in metadata.get(SIZE, {}):
|
||||
@@ -568,7 +691,26 @@ class MetadataProcessor:
|
||||
break
|
||||
if params["clip_skip"] is None:
|
||||
params["clip_skip"] = "1"
|
||||
|
||||
|
||||
# ---- Apply manual metadata overwrites ----
|
||||
for overwrite_info in metadata.get(OVERWRITE, {}).values():
|
||||
overwrite_params = overwrite_info.get("parameters", {})
|
||||
for key, value in overwrite_params.items():
|
||||
if key == "clip_skip":
|
||||
# Accept any value from overwrite node (sentinel -25 already
|
||||
# filtered upstream). Needed because falsy check treats 0
|
||||
# as "not set" even though 0 is a valid wired input here.
|
||||
params[key] = value
|
||||
elif value: # truthy check — only overwrite when user provided a real value
|
||||
params[key] = value
|
||||
|
||||
# Bridge: the overwrite node exposes the field as "model" (more accurate),
|
||||
# but the internal pipeline key remains "checkpoint" for backward compatibility
|
||||
# with A1111 metadata format and downstream consumers.
|
||||
if params.get("model"):
|
||||
params["checkpoint"] = params["model"]
|
||||
del params["model"]
|
||||
|
||||
return params
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import time
|
||||
from nodes import NODE_CLASS_MAPPINGS # type: ignore
|
||||
from typing import Any
|
||||
from nodes import NODE_CLASS_MAPPINGS # pyright: ignore[reportMissingImports, reportAttributeAccessIssue]
|
||||
from .node_extractors import NODE_EXTRACTORS, GenericNodeExtractor
|
||||
from .constants import METADATA_CATEGORIES, IMAGES
|
||||
from .constants import METADATA_CATEGORIES, IMAGES, OVERWRITE
|
||||
|
||||
|
||||
class MetadataRegistry:
|
||||
@@ -9,6 +10,15 @@ class MetadataRegistry:
|
||||
|
||||
_instance = None
|
||||
|
||||
current_prompt_id: Any = None
|
||||
current_prompt: Any = None
|
||||
metadata: dict[str, Any] = {}
|
||||
prompt_metadata: dict[str, Any] = {}
|
||||
executed_nodes: set[str] = set()
|
||||
node_cache: dict[str, Any] = {}
|
||||
max_prompt_history: int = 3
|
||||
metadata_categories: list[str] = METADATA_CATEGORIES
|
||||
|
||||
def __new__(cls):
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
@@ -61,6 +71,7 @@ class MetadataRegistry:
|
||||
{
|
||||
"execution_order": [],
|
||||
"current_prompt": None, # Will store the prompt object
|
||||
"extra_data": None, # Will store the API extra_data for workflow metadata
|
||||
"timestamp": time.time(),
|
||||
}
|
||||
)
|
||||
@@ -75,6 +86,11 @@ class MetadataRegistry:
|
||||
# Store the prompt in the metadata for later relationship tracing
|
||||
self.prompt_metadata[self.current_prompt_id]["current_prompt"] = prompt
|
||||
|
||||
def set_extra_data(self, extra_data):
|
||||
"""Store the API extra_data (contains extra_pnginfo.workflow with node properties)"""
|
||||
if self.current_prompt_id and self.current_prompt_id in self.prompt_metadata:
|
||||
self.prompt_metadata[self.current_prompt_id]["extra_data"] = extra_data
|
||||
|
||||
def get_metadata(self, prompt_id=None):
|
||||
"""Get collected metadata for a prompt"""
|
||||
key = prompt_id if prompt_id is not None else self.current_prompt_id
|
||||
@@ -122,20 +138,28 @@ class MetadataRegistry:
|
||||
cache_key = f"{node_id}:{class_type}"
|
||||
|
||||
# Check if this node type is relevant for metadata collection
|
||||
if class_type in NODE_EXTRACTORS:
|
||||
if class_type in NODE_EXTRACTORS or cache_key in self.node_cache:
|
||||
# Check if we have cached metadata for this node
|
||||
if cache_key in self.node_cache:
|
||||
cached_data = self.node_cache[cache_key]
|
||||
|
||||
# Detect bypass (mode=4) / mute (mode=2) — these nodes
|
||||
# were intentionally disabled and should not contribute
|
||||
# overwrite values from a previous execution's cache.
|
||||
node_mode = node_data.get("mode", 0)
|
||||
node_is_disabled = node_mode in (2, 4)
|
||||
|
||||
# Apply cached metadata to the current metadata
|
||||
for category in self.metadata_categories:
|
||||
if category == OVERWRITE and node_is_disabled:
|
||||
continue
|
||||
if category in cached_data and node_id in cached_data[category]:
|
||||
if node_id not in metadata[category]:
|
||||
metadata[category][node_id] = cached_data[category][
|
||||
node_id
|
||||
]
|
||||
|
||||
def record_node_execution(self, node_id, class_type, inputs, outputs):
|
||||
def record_node_execution(self, node_id, class_type, inputs, outputs, return_types=None):
|
||||
"""Record information about a node's execution"""
|
||||
if not self.current_prompt_id:
|
||||
return
|
||||
@@ -158,17 +182,18 @@ class MetadataRegistry:
|
||||
|
||||
# Extract node-specific metadata
|
||||
extractor = NODE_EXTRACTORS.get(class_type, GenericNodeExtractor)
|
||||
extractor.extract(
|
||||
node_id,
|
||||
processed_inputs,
|
||||
outputs,
|
||||
self.prompt_metadata[self.current_prompt_id],
|
||||
)
|
||||
if extractor is GenericNodeExtractor:
|
||||
extractor.extract(node_id, processed_inputs, outputs,
|
||||
self.prompt_metadata[self.current_prompt_id],
|
||||
return_types=return_types)
|
||||
else:
|
||||
extractor.extract(node_id, processed_inputs, outputs,
|
||||
self.prompt_metadata[self.current_prompt_id])
|
||||
|
||||
# Cache this node's metadata
|
||||
self._cache_node_metadata(node_id, class_type)
|
||||
|
||||
def update_node_execution(self, node_id, class_type, outputs):
|
||||
def update_node_execution(self, node_id, class_type, outputs, return_types=None):
|
||||
"""Update node metadata with output information"""
|
||||
if not self.current_prompt_id:
|
||||
return
|
||||
@@ -179,9 +204,17 @@ class MetadataRegistry:
|
||||
# Use the same extractor to update with outputs
|
||||
extractor = NODE_EXTRACTORS.get(class_type, GenericNodeExtractor)
|
||||
if hasattr(extractor, "update"):
|
||||
extractor.update(
|
||||
node_id, processed_outputs, self.prompt_metadata[self.current_prompt_id]
|
||||
)
|
||||
if extractor is GenericNodeExtractor:
|
||||
extractor.update(
|
||||
node_id, processed_outputs,
|
||||
self.prompt_metadata[self.current_prompt_id],
|
||||
return_types=return_types,
|
||||
)
|
||||
else:
|
||||
extractor.update(
|
||||
node_id, processed_outputs,
|
||||
self.prompt_metadata[self.current_prompt_id],
|
||||
)
|
||||
|
||||
# Update the cached metadata for this node
|
||||
self._cache_node_metadata(node_id, class_type)
|
||||
|
||||
@@ -2,7 +2,8 @@ import json
|
||||
import os
|
||||
import re
|
||||
|
||||
from .constants import MODELS, PROMPTS, SAMPLING, LORAS, SIZE, IMAGES, IS_SAMPLER
|
||||
from .constants import MODELS, PROMPTS, SAMPLING, LORAS, SIZE, IMAGES, IS_SAMPLER, OVERWRITE
|
||||
from .overwrite_utils import collect_overwrite_params
|
||||
|
||||
|
||||
def _store_checkpoint_metadata(metadata, node_id, model_name):
|
||||
@@ -31,11 +32,95 @@ class NodeMetadataExtractor:
|
||||
pass
|
||||
|
||||
class GenericNodeExtractor(NodeMetadataExtractor):
|
||||
"""Default extractor for nodes without specific handling"""
|
||||
"""Fallback extractor with type-signature-based detection.
|
||||
|
||||
When a node is not in the NODE_EXTRACTORS registry, the hook layer
|
||||
passes ``return_types`` from ``obj.RETURN_TYPES``:
|
||||
|
||||
* ``MODEL`` output: common input fields (ckpt_name, unet_name, etc.)
|
||||
are checked for a model file name and stored as checkpoint metadata.
|
||||
* ``CONDITIONING`` output: common text input fields are checked for
|
||||
prompt text, and conditioning inputs are tracked through transforms.
|
||||
"""
|
||||
|
||||
# Input field names that carry a model path in loader-style nodes.
|
||||
_MODEL_NAME_FIELDS = (
|
||||
"ckpt_name", "unet_name", "model_path", "model_name", "gguf_name",
|
||||
)
|
||||
|
||||
# Extensions used by checkpoint_scanner.py — only record values that look
|
||||
# like real model filenames to avoid capturing unrelated string fields.
|
||||
_MODEL_EXTENSIONS = {
|
||||
".ckpt", ".pt", ".pt2", ".bin", ".pth", ".safetensors", ".pkl", ".sft", ".gguf",
|
||||
}
|
||||
|
||||
# Input field names that may carry prompt text in encoder-style nodes.
|
||||
_TEXT_FIELDS = ("text", "clip_l", "t5xxl", "prompt", "positive", "negative")
|
||||
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
pass
|
||||
|
||||
def extract(node_id, inputs, outputs, metadata, return_types=None):
|
||||
if return_types is None:
|
||||
return
|
||||
|
||||
# — MODEL loader detection (checkpoint / UNET / GGUF) —
|
||||
if "MODEL" in return_types or any("MODEL" in str(t) for t in return_types):
|
||||
for field in GenericNodeExtractor._MODEL_NAME_FIELDS:
|
||||
val = inputs.get(field)
|
||||
if val and isinstance(val, str) and val.strip():
|
||||
name = val.strip()
|
||||
if not any(name.lower().endswith(ext) for ext in GenericNodeExtractor._MODEL_EXTENSIONS):
|
||||
continue
|
||||
_store_checkpoint_metadata(metadata, node_id, name)
|
||||
return
|
||||
|
||||
# — CONDITIONING encoder / transform detection —
|
||||
if "CONDITIONING" in return_types or any("CONDITIONING" in str(t) for t in return_types):
|
||||
text = None
|
||||
for field in GenericNodeExtractor._TEXT_FIELDS:
|
||||
val = inputs.get(field)
|
||||
if val and isinstance(val, str) and val.strip():
|
||||
text = val.strip()
|
||||
break
|
||||
|
||||
input_conditionings = _collect_conditioning_inputs(inputs)
|
||||
if text or input_conditionings:
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
if text:
|
||||
prompt_metadata["text"] = text
|
||||
if input_conditionings:
|
||||
prompt_metadata["orig_conditionings"] = input_conditionings
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata, return_types=None):
|
||||
if return_types is None:
|
||||
return
|
||||
if "CONDITIONING" not in return_types and not any(
|
||||
"CONDITIONING" in str(t) for t in return_types
|
||||
):
|
||||
return
|
||||
if node_id not in metadata.get(PROMPTS, {}):
|
||||
return
|
||||
output_tuple = _first_output_tuple(outputs)
|
||||
if not output_tuple or len(output_tuple) < 1:
|
||||
return
|
||||
|
||||
conditioning_index = _first_conditioning_index(return_types)
|
||||
if conditioning_index is None or len(output_tuple) <= conditioning_index:
|
||||
return
|
||||
|
||||
output_conditioning = output_tuple[conditioning_index]
|
||||
if output_conditioning is None:
|
||||
return
|
||||
|
||||
prompt_metadata = metadata[PROMPTS][node_id]
|
||||
prompt_metadata["conditioning"] = output_conditioning
|
||||
_record_conditioning_source(
|
||||
metadata,
|
||||
node_id,
|
||||
output_conditioning,
|
||||
prompt_metadata.get("orig_conditionings", []),
|
||||
)
|
||||
|
||||
class CheckpointLoaderExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
@@ -349,6 +434,34 @@ def _first_output_tuple(outputs):
|
||||
return None
|
||||
|
||||
|
||||
def _first_conditioning_index(return_types):
|
||||
"""Return the index of the first CONDITIONING output slot, or None."""
|
||||
if not return_types:
|
||||
return None
|
||||
for index, return_type in enumerate(return_types):
|
||||
if "CONDITIONING" in str(return_type):
|
||||
return index
|
||||
return None
|
||||
|
||||
|
||||
def _collect_conditioning_inputs(inputs):
|
||||
"""Collect conditioning object inputs (``conditioning*`` keys).
|
||||
|
||||
Primitive values (None, str, int, float, bool) are excluded so scalar
|
||||
fields like ``conditioning_strength`` are not mistaken for conditioning
|
||||
objects during provenance tracking.
|
||||
"""
|
||||
if not inputs:
|
||||
return []
|
||||
return [
|
||||
value
|
||||
for input_name, value in inputs.items()
|
||||
if input_name.startswith("conditioning")
|
||||
and value is not None
|
||||
and not isinstance(value, (str, int, float, bool))
|
||||
]
|
||||
|
||||
|
||||
def _record_conditioning_source(
|
||||
metadata, node_id, output_conditioning, input_conditionings
|
||||
):
|
||||
@@ -361,6 +474,14 @@ def _record_conditioning_source(
|
||||
if not sources:
|
||||
return
|
||||
|
||||
# Identity-preserving selectors return one of their inputs unchanged:
|
||||
# only that input contributed to the output, so record it alone instead
|
||||
# of treating every input as a combination source.
|
||||
for conditioning in sources:
|
||||
if id(conditioning) == id(output_conditioning):
|
||||
sources = [conditioning]
|
||||
break
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
prompt_metadata.setdefault("conditioning_sources", []).append(
|
||||
{
|
||||
@@ -440,13 +561,7 @@ class ConditioningCombineExtractor(NodeMetadataExtractor):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
input_conditionings = []
|
||||
for input_name in inputs:
|
||||
if (
|
||||
input_name.startswith("conditioning")
|
||||
and inputs[input_name] is not None
|
||||
):
|
||||
input_conditionings.append(inputs[input_name])
|
||||
input_conditionings = _collect_conditioning_inputs(inputs)
|
||||
|
||||
if input_conditionings:
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
@@ -746,6 +861,65 @@ class TSCKSamplerAdvancedExtractor(KSamplerAdvancedExtractor, TSCSamplerBaseExtr
|
||||
|
||||
# Update method is inherited from TSCSamplerBaseExtractor
|
||||
|
||||
class KreaTwoStageSamplerExtractor(BaseSamplerExtractor):
|
||||
"""Extractor for Krea Two/Three Stage Samplers (Auryg/Krea-2-Two-Stage-Sampler).
|
||||
|
||||
The node samples in two (or three) stages with per-stage settings
|
||||
(stage1_steps/stage2_steps, stage1_cfg/stage2_cfg, ...). The canonical
|
||||
metadata fields consumed by ``extract_generation_params`` (steps, cfg,
|
||||
sampler_name, scheduler) are derived from the base stage (stage 1; the
|
||||
three-stage variant reuses stage 1 settings for stage 3), while the full
|
||||
per-stage breakdown is preserved in the raw parameters.
|
||||
"""
|
||||
|
||||
# All per-stage parameter keys present on both node variants.
|
||||
_STAGE_PARAM_KEYS = (
|
||||
"stage1_steps", "stage1_cfg", "stage1_sampler_name", "stage1_scheduler",
|
||||
"stage2_steps", "stage2_cfg", "stage2_sampler_name", "stage2_scheduler",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
BaseSamplerExtractor.extract_sampling_params(
|
||||
node_id,
|
||||
inputs,
|
||||
metadata,
|
||||
("seed", "handoff_percent", "stage3_handoff_percent")
|
||||
+ KreaTwoStageSamplerExtractor._STAGE_PARAM_KEYS,
|
||||
)
|
||||
|
||||
# Derive the canonical fields expected by extract_generation_params.
|
||||
sampling_params = metadata[SAMPLING][node_id]["parameters"]
|
||||
if "stage1_steps" in sampling_params or "stage2_steps" in sampling_params:
|
||||
sampling_params["steps"] = (
|
||||
(sampling_params.get("stage1_steps") or 0)
|
||||
+ (sampling_params.get("stage2_steps") or 0)
|
||||
)
|
||||
if "stage1_cfg" in sampling_params:
|
||||
sampling_params["cfg"] = sampling_params["stage1_cfg"]
|
||||
if "stage1_sampler_name" in sampling_params:
|
||||
sampling_params["sampler_name"] = sampling_params["stage1_sampler_name"]
|
||||
if "stage1_scheduler" in sampling_params:
|
||||
sampling_params["scheduler"] = sampling_params["stage1_scheduler"]
|
||||
|
||||
BaseSamplerExtractor.extract_conditioning(node_id, inputs, metadata)
|
||||
|
||||
# Prefer the final generation resolution; latent dims are the fallback.
|
||||
BaseSamplerExtractor.extract_latent_dimensions(node_id, inputs, metadata)
|
||||
final_width = inputs.get("final_width")
|
||||
final_height = inputs.get("final_height")
|
||||
if final_width and final_height:
|
||||
if SIZE not in metadata:
|
||||
metadata[SIZE] = {}
|
||||
metadata[SIZE][node_id] = {
|
||||
"width": final_width,
|
||||
"height": final_height,
|
||||
"node_id": node_id,
|
||||
}
|
||||
|
||||
class LoraLoaderExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
@@ -786,6 +960,37 @@ class ImageSizeExtractor(NodeMetadataExtractor):
|
||||
"node_id": node_id
|
||||
}
|
||||
|
||||
class KreaDualResolutionSelectorExtractor(NodeMetadataExtractor):
|
||||
"""Extract base resolution from Krea Dual Resolution Selector outputs
|
||||
(Auryg/Krea-2-Two-Stage-Sampler).
|
||||
|
||||
The node computes base/final dimensions at runtime from aspect ratio and
|
||||
megapixel settings, so the values are only available in the update phase
|
||||
(outputs: base_width, base_height, final_width, final_height, seed).
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
# Dimensions are computed at runtime; nothing to do here.
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata):
|
||||
output_tuple = _first_output_tuple(outputs)
|
||||
if not output_tuple or len(output_tuple) < 2:
|
||||
return
|
||||
width, height = output_tuple[0], output_tuple[1]
|
||||
if not isinstance(width, int) or not isinstance(height, int):
|
||||
return
|
||||
|
||||
if SIZE not in metadata:
|
||||
metadata[SIZE] = {}
|
||||
metadata[SIZE][node_id] = {
|
||||
"width": width,
|
||||
"height": height,
|
||||
"node_id": node_id,
|
||||
}
|
||||
|
||||
class RgthreePowerLoraLoaderExtractor(NodeMetadataExtractor):
|
||||
"""Extract LoRA metadata from rgthree Power Lora Loader.
|
||||
|
||||
@@ -901,6 +1106,55 @@ class LoraLoaderManagerExtractor(NodeMetadataExtractor):
|
||||
"node_id": node_id
|
||||
}
|
||||
|
||||
class LoraTextLoaderManagerExtractor(NodeMetadataExtractor):
|
||||
"""Extract LoRA metadata from LoraTextLoaderLM (LoRA Text Loader).
|
||||
|
||||
The node accepts a `lora_syntax` STRING containing <lora:name:strength> tags
|
||||
(same format as the ComfyUI prompt), plus an optional `lora_stack`.
|
||||
This extractor parses the syntax string using the same regex as the node.
|
||||
"""
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
active_loras = []
|
||||
|
||||
# Process lora_stack if available (optional input)
|
||||
if "lora_stack" in inputs:
|
||||
lora_stack = inputs.get("lora_stack", [])
|
||||
for item in lora_stack:
|
||||
# lora_stack entries are (path, model_strength, clip_strength) tuples
|
||||
if isinstance(item, (list, tuple)) and len(item) >= 2:
|
||||
lora_path = item[0]
|
||||
model_strength = item[1]
|
||||
lora_name = os.path.splitext(os.path.basename(lora_path))[0]
|
||||
active_loras.append({
|
||||
"name": lora_name,
|
||||
"strength": round(float(model_strength), 2)
|
||||
})
|
||||
|
||||
# Process lora_syntax string input
|
||||
if "lora_syntax" in inputs:
|
||||
lora_syntax = inputs.get("lora_syntax", "")
|
||||
if lora_syntax and isinstance(lora_syntax, str):
|
||||
pattern = r"<lora:([^:>]+):([^:>]+)(?::([^:>]+))?>"
|
||||
matches = re.findall(pattern, lora_syntax, re.IGNORECASE)
|
||||
for match in matches:
|
||||
lora_name = match[0]
|
||||
model_strength = float(match[1])
|
||||
active_loras.append({
|
||||
"name": lora_name,
|
||||
"strength": round(model_strength, 2)
|
||||
})
|
||||
|
||||
if active_loras:
|
||||
metadata[LORAS][node_id] = {
|
||||
"lora_list": active_loras,
|
||||
"node_id": node_id
|
||||
}
|
||||
|
||||
|
||||
class FluxGuidanceExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
@@ -1105,6 +1359,28 @@ class CR_ApplyControlNetStackExtractor(NodeMetadataExtractor):
|
||||
metadata[PROMPTS][node_id]["positive_encoded"] = transformed_positive
|
||||
metadata[PROMPTS][node_id]["negative_encoded"] = transformed_negative
|
||||
|
||||
class MetadataOverwriteExtractor(NodeMetadataExtractor):
|
||||
"""Extract manually specified metadata from MetadataOverwriteLM node.
|
||||
|
||||
Stores truthy input values under the OVERWRITE category so that
|
||||
extract_generation_params can merge them over the inferred params.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def extract(node_id, inputs, outputs, metadata):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
overwrite_params = collect_overwrite_params(inputs)
|
||||
|
||||
if overwrite_params:
|
||||
metadata.setdefault(OVERWRITE, {})
|
||||
metadata[OVERWRITE][node_id] = {
|
||||
"parameters": overwrite_params,
|
||||
"node_id": node_id,
|
||||
}
|
||||
|
||||
|
||||
# Registry of node-specific extractors
|
||||
# Keys are node class names
|
||||
NODE_EXTRACTORS = {
|
||||
@@ -1116,6 +1392,8 @@ NODE_EXTRACTORS = {
|
||||
"ClownsharKSampler_Beta": SamplerExtractor,
|
||||
"TSC_KSampler": TSCKSamplerExtractor, # Efficient Nodes
|
||||
"TSC_KSamplerAdvanced": TSCKSamplerAdvancedExtractor, # Efficient Nodes
|
||||
"KreaTwoStageSampler": KreaTwoStageSamplerExtractor, # Auryg/Krea-2-Two-Stage-Sampler
|
||||
"KreaThreeStageSampler": KreaTwoStageSamplerExtractor, # Auryg/Krea-2-Two-Stage-Sampler
|
||||
"KSamplerBasicPipe": KSamplerBasicPipeExtractor, # comfyui-impact-pack
|
||||
"KSamplerAdvancedBasicPipe": KSamplerAdvancedBasicPipeExtractor, # comfyui-impact-pack
|
||||
"KSampler_inspire_pipe": KSamplerBasicPipeExtractor, # comfyui-inspire-pack
|
||||
@@ -1146,6 +1424,7 @@ NODE_EXTRACTORS = {
|
||||
"UNETLoaderLM": UNETLoaderExtractor, # LoRA Manager
|
||||
"LoraLoader": LoraLoaderExtractor,
|
||||
"LoraLoaderLM": LoraLoaderManagerExtractor,
|
||||
"LoraTextLoaderLM": LoraTextLoaderManagerExtractor,
|
||||
"RgthreePowerLoraLoader": RgthreePowerLoraLoaderExtractor,
|
||||
"TensorRTLoader": TensorRTLoaderExtractor,
|
||||
# Conditioning
|
||||
@@ -1166,10 +1445,13 @@ NODE_EXTRACTORS = {
|
||||
"GetNode": GetNodeExtractor,
|
||||
# Latent
|
||||
"EmptyLatentImage": ImageSizeExtractor,
|
||||
"KreaDualResolutionSelector": KreaDualResolutionSelectorExtractor, # Auryg/Krea-2-Two-Stage-Sampler
|
||||
# Flux
|
||||
"FluxGuidance": FluxGuidanceExtractor, # Add FluxGuidance
|
||||
"CFGGuider": CFGGuiderExtractor, # Add CFGGuider
|
||||
# Image
|
||||
"VAEDecode": VAEDecodeExtractor, # Added VAEDecode extractor
|
||||
# Metadata overwrite
|
||||
"MetadataOverwriteLM": MetadataOverwriteExtractor,
|
||||
# Add other nodes as needed
|
||||
}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Shared helpers for Metadata Overwrite node metadata collection.
|
||||
|
||||
Used by both the MetadataOverwriteLM node (execution time) and the
|
||||
MetadataOverwriteExtractor (hook time) so the conversion/filtering logic
|
||||
cannot drift between the two paths.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
from ..utils.utils import model_patcher_to_name, sampler_object_to_name
|
||||
from .constants import CLIP_SKIP_SENTINEL, METADATA_OVERWRITE_FIELDS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def collect_overwrite_params(values: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Convert node input values into non-default overwrite parameters.
|
||||
|
||||
For most fields, a falsy value (empty string, 0) means "not set" and is
|
||||
skipped. clip_skip uses a dedicated sentinel (-25) so that a wired value
|
||||
of 0 is preserved. The ``model`` field accepts either a manual string or
|
||||
a wired MODEL (ModelPatcher) connection; in the latter case the source
|
||||
model name is extracted from the patcher's ``cached_patcher_init`` and
|
||||
stored as a ComfyUI-style relative path. The ``sampler`` field likewise
|
||||
accepts a manual string or a wired SAMPLER (KSAMPLER) connection, from
|
||||
which the sampler name is extracted via the sampler function's name.
|
||||
"""
|
||||
result: Dict[str, Any] = {}
|
||||
for key in METADATA_OVERWRITE_FIELDS:
|
||||
value = values.get(key)
|
||||
if key == "model" and not isinstance(value, str):
|
||||
value = model_patcher_to_name(value)
|
||||
if value is None:
|
||||
logger.warning(
|
||||
"Could not extract model name from wired MODEL input "
|
||||
"(no cached_patcher_init); model metadata overwrite skipped"
|
||||
)
|
||||
elif key == "sampler" and not isinstance(value, str):
|
||||
value = sampler_object_to_name(value)
|
||||
if value is None:
|
||||
logger.warning(
|
||||
"Could not extract sampler name from wired SAMPLER input "
|
||||
"(unrecognized sampler function); sampler metadata overwrite skipped"
|
||||
)
|
||||
if key == "clip_skip":
|
||||
if value != CLIP_SKIP_SENTINEL:
|
||||
result[key] = value
|
||||
elif value:
|
||||
result[key] = value
|
||||
return result
|
||||
@@ -0,0 +1,234 @@
|
||||
"""Metadata operations — thin in-process wrappers around LoRA Manager internal services.
|
||||
|
||||
All functions are simple Python async functions that delegate to the
|
||||
appropriate internal service. They use **relative imports** within the
|
||||
``py`` package, so ``sys.modules`` caching works normally and there is no
|
||||
risk of double import or circular dependencies.
|
||||
|
||||
Usage (in-process, primary)::
|
||||
|
||||
from py.metadata_ops import list_base_models, read_metadata
|
||||
|
||||
models = await list_base_models()
|
||||
meta = await read_metadata("/path/to/model.safetensors")
|
||||
|
||||
Usage (subprocess, debugging / external)::
|
||||
|
||||
python -m py.metadata_ops base-models list
|
||||
python -m py.metadata_ops metadata read /path/to/model.safetensors
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SCANNER_TYPE_MAP: dict[str, str] = {
|
||||
"get_lora_scanner": "lora",
|
||||
"get_checkpoint_scanner": "checkpoint",
|
||||
"get_embedding_scanner": "embedding",
|
||||
"get_other_scanner": "other",
|
||||
}
|
||||
|
||||
SCANNER_GETTER_NAMES = tuple(SCANNER_TYPE_MAP.keys())
|
||||
|
||||
|
||||
async def _find_model_entry(
|
||||
model_path: str,
|
||||
) -> tuple[Any, object, str | None] | tuple[None, None, None]:
|
||||
"""Iterate all scanners and return the first (scanner, entry, getter_name)
|
||||
that owns *model_path*. Returns ``(None, None, None)`` when no scanner
|
||||
claims it.
|
||||
"""
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
|
||||
normalized = os.path.normpath(model_path)
|
||||
for getter_name in SCANNER_GETTER_NAMES:
|
||||
getter = getattr(ServiceRegistry, getter_name, None)
|
||||
if getter is None:
|
||||
continue
|
||||
try:
|
||||
scanner = await getter()
|
||||
if scanner is None:
|
||||
continue
|
||||
cache = await scanner.get_cached_data()
|
||||
for entry in cache.raw_data:
|
||||
if os.path.normpath(entry.get("file_path", "")) == normalized:
|
||||
return scanner, entry, getter_name
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Scanner %s check failed for %s: %s",
|
||||
getter_name, model_path, exc,
|
||||
)
|
||||
return None, None, None
|
||||
|
||||
|
||||
async def _find_scanner_for_model(
|
||||
model_path: str,
|
||||
) -> tuple[Any, object] | tuple[None, None]:
|
||||
"""Find the (scanner, cache_entry) responsible for *model_path*."""
|
||||
scanner, entry, _ = await _find_model_entry(model_path)
|
||||
return scanner, entry
|
||||
|
||||
|
||||
async def identify_model_type(model_path: str) -> str:
|
||||
"""Determine the model type (``\"lora\"``, ``\"checkpoint\"``,
|
||||
``\"embedding\"``, or ``\"other\"``) for *model_path*.
|
||||
|
||||
Falls back to ``\"lora\"`` when unknown.
|
||||
"""
|
||||
_, _, getter_name = await _find_model_entry(model_path)
|
||||
return SCANNER_TYPE_MAP[getter_name] if getter_name else "lora"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def list_base_models(limit: int = 0) -> List[str]:
|
||||
"""Return all valid CivitAI base model names.
|
||||
|
||||
Uses ``CivitaiBaseModelService.get_base_models()`` which merges a
|
||||
hardcoded list (``SUPPORTED_DOWNLOAD_SKIP_BASE_MODELS``) with remote
|
||||
models fetched from the CivitAI API. Never empty — the hardcoded
|
||||
fallback always provides a complete set.
|
||||
|
||||
The result is sorted alphabetically. Pass *limit* = 0 for all models.
|
||||
"""
|
||||
from ..services.civitai_base_model_service import (
|
||||
CivitaiBaseModelService,
|
||||
)
|
||||
|
||||
try:
|
||||
service = await CivitaiBaseModelService.get_instance()
|
||||
response = await service.get_base_models()
|
||||
names: List[str] = response.get("models", [])
|
||||
except Exception as exc:
|
||||
logger.warning("list_base_models failed: %s", exc)
|
||||
names = []
|
||||
if limit > 0:
|
||||
return names[:limit]
|
||||
return names
|
||||
|
||||
|
||||
async def read_metadata(model_path: str) -> Dict[str, Any]:
|
||||
"""Load the full metadata payload for *model_path* from disk.
|
||||
|
||||
Returns an empty dict when the metadata file does not exist or cannot
|
||||
be parsed — never raises.
|
||||
"""
|
||||
from ..utils.metadata_manager import MetadataManager
|
||||
|
||||
try:
|
||||
return await MetadataManager.load_metadata_payload(model_path) or {}
|
||||
except Exception as exc:
|
||||
logger.warning("read_metadata failed for %s: %s", model_path, exc)
|
||||
return {}
|
||||
|
||||
|
||||
async def apply_metadata_updates(
|
||||
model_path: str,
|
||||
updates: Dict[str, Any],
|
||||
) -> List[str]:
|
||||
"""Merge *updates* into the model's on-disk metadata and persist.
|
||||
|
||||
Returns the list of field names that actually changed.
|
||||
"""
|
||||
from ..utils.metadata_manager import MetadataManager
|
||||
|
||||
metadata = await read_metadata(model_path)
|
||||
updated_fields: List[str] = []
|
||||
for key, value in updates.items():
|
||||
old = metadata.get(key)
|
||||
if old != value:
|
||||
metadata[key] = value
|
||||
updated_fields.append(key)
|
||||
if updated_fields:
|
||||
await MetadataManager.save_metadata(model_path, metadata)
|
||||
return updated_fields
|
||||
|
||||
|
||||
async def download_preview(
|
||||
model_path: str,
|
||||
url: str,
|
||||
*,
|
||||
target_width: int = 480,
|
||||
quality: int = 85,
|
||||
) -> str | None:
|
||||
"""Download a preview image from *url*, optimise to .webp, and save it.
|
||||
|
||||
The output file is placed alongside the model file with a ``.webp``
|
||||
extension. Returns the local file path on success, ``None`` on failure.
|
||||
"""
|
||||
from ..services.downloader import get_downloader
|
||||
from ..utils.exif_utils import ExifUtils
|
||||
|
||||
if not url or not url.strip():
|
||||
return None
|
||||
|
||||
base_name = os.path.splitext(os.path.basename(model_path))[0]
|
||||
preview_dir = os.path.dirname(model_path)
|
||||
output_path = os.path.join(preview_dir, base_name + ".webp")
|
||||
|
||||
downloader = await get_downloader()
|
||||
|
||||
# Try in-memory download + optimise first
|
||||
success, content, _headers = await downloader.download_to_memory(
|
||||
url, use_auth=False,
|
||||
)
|
||||
if success and content:
|
||||
try:
|
||||
optimized_data, _ = ExifUtils.optimize_image(
|
||||
image_data=content,
|
||||
target_width=target_width,
|
||||
format="webp",
|
||||
quality=quality,
|
||||
preserve_metadata=False,
|
||||
)
|
||||
with open(output_path, "wb") as f:
|
||||
f.write(optimized_data)
|
||||
return output_path
|
||||
except Exception as exc:
|
||||
logger.warning("Preview optimisation failed, saving raw: %s", exc)
|
||||
# Fall through to raw save
|
||||
|
||||
# Fallback: download directly to file
|
||||
try:
|
||||
ok, _ = await downloader.download_file(url, output_path, use_auth=False)
|
||||
if ok:
|
||||
return output_path
|
||||
except Exception as exc:
|
||||
logger.warning("Preview fallback download failed for %s: %s", model_path, exc)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def refresh_cache(model_path: str) -> bool:
|
||||
"""Invalidate and reload the scanner cache entry for *model_path*.
|
||||
|
||||
Returns ``True`` when the model was found and the cache was refreshed.
|
||||
"""
|
||||
scanner, entry = await _find_scanner_for_model(model_path)
|
||||
if scanner is None:
|
||||
logger.warning("refresh_cache: no scanner found for %s", model_path)
|
||||
return False
|
||||
try:
|
||||
metadata = await read_metadata(model_path)
|
||||
if not metadata:
|
||||
logger.warning("refresh_cache: no metadata for %s", model_path)
|
||||
return False
|
||||
await scanner.update_single_model_cache(model_path, model_path, metadata)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning("refresh_cache failed for %s: %s", model_path, exc)
|
||||
return False
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Subprocess entry point for ``metadata_ops`` (debugging / external use).
|
||||
|
||||
Usage::
|
||||
|
||||
python -m py.metadata_ops base-models list [--limit N]
|
||||
python -m py.metadata_ops metadata read <path>
|
||||
python -m py.metadata_ops metadata update <path> --json '{...}'
|
||||
python -m py.metadata_ops preview download <path> --url <url>
|
||||
python -m py.metadata_ops cache refresh <path>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from typing import Any, Dict, List
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(prog="lmcli", description="LoRA Manager Agent CLI")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
# base-models list
|
||||
base_models = sub.add_parser("base-models", aliases=["bm"])
|
||||
base_models_cmds = base_models.add_subparsers(dest="subcommand", required=True)
|
||||
base_models_list = base_models_cmds.add_parser("list")
|
||||
base_models_list.add_argument(
|
||||
"--limit", type=int, default=0, help="Max number of models (0 = all)"
|
||||
)
|
||||
|
||||
# metadata read
|
||||
meta = sub.add_parser("metadata", aliases=["md"])
|
||||
meta_cmds = meta.add_subparsers(dest="subcommand", required=True)
|
||||
meta_read = meta_cmds.add_parser("read")
|
||||
meta_read.add_argument("path", type=str, help="Model file path")
|
||||
|
||||
# metadata update
|
||||
meta_update = meta_cmds.add_parser("update")
|
||||
meta_update.add_argument("path", type=str, help="Model file path")
|
||||
meta_update.add_argument(
|
||||
"--json",
|
||||
type=str,
|
||||
required=True,
|
||||
help='JSON object of fields to update, e.g. \'{"base_model": "SDXL 1.0"}\'',
|
||||
)
|
||||
|
||||
# preview download
|
||||
prev = sub.add_parser("preview", aliases=["pv"])
|
||||
prev_cmds = prev.add_subparsers(dest="subcommand", required=True)
|
||||
prev_dl = prev_cmds.add_parser("download")
|
||||
prev_dl.add_argument("path", type=str, help="Model file path")
|
||||
prev_dl.add_argument("--url", type=str, required=True, help="Preview image URL")
|
||||
|
||||
# cache refresh
|
||||
cache = sub.add_parser("cache")
|
||||
cache_cmds = cache.add_subparsers(dest="subcommand", required=True)
|
||||
cache_refresh = cache_cmds.add_parser("refresh")
|
||||
cache_refresh.add_argument("path", type=str, help="Model file path")
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
async def _run(args: argparse.Namespace) -> Any:
|
||||
from . import ( # lazy import so startup is fast
|
||||
list_base_models,
|
||||
read_metadata,
|
||||
apply_metadata_updates,
|
||||
download_preview,
|
||||
refresh_cache,
|
||||
)
|
||||
|
||||
cmd = args.command
|
||||
sub = args.subcommand
|
||||
|
||||
if cmd in ("base-models", "bm") and sub == "list":
|
||||
return await list_base_models(limit=args.limit)
|
||||
|
||||
if cmd in ("metadata", "md") and sub == "read":
|
||||
return await read_metadata(args.path)
|
||||
|
||||
if cmd in ("metadata", "md") and sub == "update":
|
||||
updates: Dict[str, Any] = json.loads(args.json)
|
||||
return await apply_metadata_updates(args.path, updates)
|
||||
|
||||
if cmd in ("preview", "pv") and sub == "download":
|
||||
return await download_preview(args.path, args.url)
|
||||
|
||||
if cmd == "cache" and sub == "refresh":
|
||||
return await refresh_cache(args.path)
|
||||
|
||||
raise ValueError(f"Unknown command: {cmd} {sub}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = _build_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
result = asyncio.run(_run(args))
|
||||
# Always print as JSON so callers can parse reliably
|
||||
if isinstance(result, list):
|
||||
for item in result:
|
||||
print(item)
|
||||
elif isinstance(result, dict):
|
||||
json.dump(result, sys.stdout, ensure_ascii=False, indent=2)
|
||||
print()
|
||||
else:
|
||||
print(json.dumps(result))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -16,6 +16,8 @@ IMG_EXTENSIONS = (
|
||||
".tif",
|
||||
".tiff",
|
||||
".webp",
|
||||
".avif",
|
||||
".jxl",
|
||||
".mp4"
|
||||
)
|
||||
|
||||
|
||||
@@ -41,7 +41,22 @@ async def api_json_error(
|
||||
if exc.status < 400:
|
||||
raise
|
||||
|
||||
logger.warning(
|
||||
# Preview 404 is routine (file deleted from disk) — not worth a warning.
|
||||
logger_method = logger.warning
|
||||
if request.path.startswith("/api/lm/previews") and exc.status == 404:
|
||||
logger_method = logger.debug
|
||||
|
||||
# Download-progress 404 is routine too: in-memory tracking is removed
|
||||
# once a download finishes/fails, so the extension's final polls 404.
|
||||
# The extension relies on the 404 status itself (failure detection),
|
||||
# so only the log level is lowered.
|
||||
if (
|
||||
request.path.startswith("/api/lm/download-progress/")
|
||||
and exc.status == 404
|
||||
):
|
||||
logger_method = logger.debug
|
||||
|
||||
logger_method(
|
||||
"API %s %s returned HTTP %d: %s",
|
||||
request.method,
|
||||
request.path,
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import logging
|
||||
from typing import List, Tuple
|
||||
import comfy.sd # type: ignore
|
||||
import folder_paths # type: ignore
|
||||
import os
|
||||
from typing import Any, List, Tuple
|
||||
import comfy.sd # pyright: ignore[reportMissingImports]
|
||||
import folder_paths # pyright: ignore[reportMissingImports]
|
||||
from ..utils.utils import get_checkpoint_info_absolute, _format_model_name_for_comfyui
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -12,20 +13,42 @@ class CheckpointLoaderLM:
|
||||
|
||||
Loads checkpoints from both standard ComfyUI folders and LoRA Manager's
|
||||
extra folder paths, providing a unified interface for checkpoint loading.
|
||||
The ckpt_name combo supports ComfyUI's control_after_generate, letting
|
||||
users pick a random checkpoint on every run; the base_model input narrows
|
||||
the random pool through a front-end extension that filters the combo
|
||||
options.
|
||||
"""
|
||||
|
||||
NAME = "Checkpoint Loader (LoraManager)"
|
||||
CATEGORY = "Lora Manager/loaders"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
def INPUT_TYPES(cls):
|
||||
# Get list of checkpoint names from scanner (includes extra folder paths)
|
||||
checkpoint_names = s._get_checkpoint_names()
|
||||
checkpoint_names = cls._get_checkpoint_names()
|
||||
base_models = cls._get_available_base_models()
|
||||
return {
|
||||
"required": {
|
||||
"ckpt_name": (
|
||||
checkpoint_names,
|
||||
{"tooltip": "The name of the checkpoint (model) to load."},
|
||||
{
|
||||
"tooltip": (
|
||||
"The name of the checkpoint (model) to load. Use "
|
||||
"control_after_generate to pick a random model on "
|
||||
"every run."
|
||||
),
|
||||
"control_after_generate": "fixed",
|
||||
},
|
||||
),
|
||||
"base_model": (
|
||||
base_models,
|
||||
{
|
||||
"default": "Any",
|
||||
"tooltip": (
|
||||
"Restrict the random selection pool to this base "
|
||||
"model. 'Any' uses the full pool."
|
||||
),
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -55,10 +78,13 @@ class CheckpointLoaderLM:
|
||||
|
||||
# Filter only checkpoint type (not diffusion_model) and format names
|
||||
names = []
|
||||
for item in cache.raw_data:
|
||||
for item in list(cache.raw_data):
|
||||
if item.get("sub_type") == "checkpoint":
|
||||
file_path = item.get("file_path", "")
|
||||
if file_path:
|
||||
# Only offer models that still exist on disk so ComfyUI
|
||||
# flags missing checkpoints at queue time via
|
||||
# "value not in list" (the scanner cache can be stale).
|
||||
if file_path and os.path.exists(file_path):
|
||||
# Format using relative path with OS-native separator
|
||||
formatted_name = _format_model_name_for_comfyui(
|
||||
file_path, model_roots
|
||||
@@ -89,15 +115,68 @@ class CheckpointLoaderLM:
|
||||
logger.error(f"Error getting checkpoint names: {e}")
|
||||
return []
|
||||
|
||||
def load_checkpoint(self, ckpt_name: str) -> Tuple:
|
||||
@classmethod
|
||||
def _get_available_base_models(cls) -> List[str]:
|
||||
"""Get distinct base_model values present among indexed checkpoints, for the random-selection filter."""
|
||||
try:
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
|
||||
async def _get_base_models():
|
||||
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||
cache = await scanner.get_cached_data()
|
||||
|
||||
base_models = set()
|
||||
for item in list(cache.raw_data):
|
||||
if item.get("sub_type") != "checkpoint":
|
||||
continue
|
||||
base_model = item.get("base_model")
|
||||
file_path = item.get("file_path", "")
|
||||
if base_model and file_path and os.path.exists(file_path):
|
||||
base_models.add(base_model)
|
||||
|
||||
return sorted(base_models)
|
||||
|
||||
return ["Any"] + cls._run_async(_get_base_models)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting available base models: {e}")
|
||||
return ["Any"]
|
||||
|
||||
@staticmethod
|
||||
def _run_async(coro_fn):
|
||||
"""Run an async fetcher, handling the case where an event loop is already running."""
|
||||
import asyncio
|
||||
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
import concurrent.futures
|
||||
|
||||
def run_in_thread():
|
||||
new_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(new_loop)
|
||||
try:
|
||||
return new_loop.run_until_complete(coro_fn())
|
||||
finally:
|
||||
new_loop.close()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
future = executor.submit(run_in_thread)
|
||||
return future.result()
|
||||
except RuntimeError:
|
||||
return asyncio.run(coro_fn())
|
||||
|
||||
def load_checkpoint(
|
||||
self, ckpt_name: str, base_model: str = "Any"
|
||||
) -> Tuple[Any, Any, Any]:
|
||||
"""Load a checkpoint by name, supporting extra folder paths
|
||||
|
||||
Args:
|
||||
ckpt_name: The name of the checkpoint to load (relative path with extension)
|
||||
base_model: Only used by the front-end to filter the random pool
|
||||
|
||||
Returns:
|
||||
Tuple of (MODEL, CLIP, VAE)
|
||||
"""
|
||||
del base_model
|
||||
# Get absolute path from cache using ComfyUI-style name
|
||||
ckpt_path, metadata = get_checkpoint_info_absolute(ckpt_name)
|
||||
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Create Hook LoRA (LoraManager) — multi-LoRA hook node compatible with ComfyUI's built-in hook pipeline.
|
||||
|
||||
Produces ``("HOOKS",)`` output that chains seamlessly with downstream hook consumers
|
||||
(ConditioningSetProperties, SetHookKeyframes, CombineHooks, SetClipHooks, etc.).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from ..utils.utils import get_lora_info_absolute
|
||||
from .utils import (
|
||||
FlexibleOptionalInputType,
|
||||
any_type,
|
||||
apply_lora_syntax_format,
|
||||
get_loras_list,
|
||||
validate_lora_entries,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CreateHookLoraLM:
|
||||
NAME = "Create Hook LoRA (LoraManager)"
|
||||
CATEGORY = "Lora Manager/hooks"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"text": (
|
||||
"AUTOCOMPLETE_TEXT_LORAS",
|
||||
{
|
||||
"placeholder": "Search LoRAs to add...",
|
||||
"tooltip": (
|
||||
"Search and select LoRAs. Each LoRA gets its own "
|
||||
"model/clip strength. Hooks chain with prev_hooks."
|
||||
),
|
||||
},
|
||||
),
|
||||
"loras": ("LORAS", {}),
|
||||
},
|
||||
"optional": FlexibleOptionalInputType(any_type),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, loras=None):
|
||||
"""Queue-time validation: reject missing local LoRAs before execution."""
|
||||
return validate_lora_entries({"loras": loras}) or True
|
||||
|
||||
RETURN_TYPES = ("HOOKS", "STRING", "STRING")
|
||||
RETURN_NAMES = ("HOOKS", "trigger_words", "active_loras")
|
||||
FUNCTION = "create_hook"
|
||||
|
||||
def create_hook(self, text: str, loras, **kwargs):
|
||||
"""Create a HookGroup from the selected LoRAs, chained with prev_hooks.
|
||||
|
||||
Each active LoRA from the widget is loaded and wrapped in a WeightHook
|
||||
via :func:`comfy.hooks.create_hook_lora`. All hooks are combined into a
|
||||
single group and returned alongside trigger words and a human-readable
|
||||
summary of the active LoRAs.
|
||||
"""
|
||||
del text # used by the frontend widget only
|
||||
|
||||
# Lazy imports: comfy is not available in CI/test environment at module level
|
||||
import comfy.hooks # pyright: ignore[reportMissingImports] # noqa: C0415
|
||||
import comfy.utils # pyright: ignore[reportMissingImports] # noqa: C0415
|
||||
|
||||
prev_hooks: comfy.hooks.HookGroup | None = kwargs.get("prev_hooks")
|
||||
|
||||
hook_group = prev_hooks.clone() if prev_hooks is not None else comfy.hooks.HookGroup()
|
||||
|
||||
all_trigger_words: list[str] = []
|
||||
active_loras: list[tuple[str, float, float]] = []
|
||||
|
||||
for lora in get_loras_list({"loras": loras}):
|
||||
if not lora.get("active", False):
|
||||
continue
|
||||
|
||||
lora_name = apply_lora_syntax_format(lora["name"])
|
||||
model_strength = float(lora["strength"])
|
||||
clip_strength = float(lora.get("clipStrength", model_strength))
|
||||
|
||||
# Skip useless no-op entries (both strengths are zero)
|
||||
if model_strength == 0.0 and clip_strength == 0.0:
|
||||
continue
|
||||
|
||||
lora_path, trigger_words = get_lora_info_absolute(lora_name)
|
||||
if not lora_path or not os.path.isfile(lora_path):
|
||||
logger.warning("LoRA '%s' not found — skipping", lora_name)
|
||||
continue
|
||||
|
||||
try:
|
||||
lora_weights = comfy.utils.load_torch_file(lora_path, safe_load=True)
|
||||
|
||||
lora_hooks = comfy.hooks.create_hook_lora(
|
||||
lora=lora_weights,
|
||||
strength_model=model_strength,
|
||||
strength_clip=clip_strength,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to load LoRA '%s' — skipping", lora_name)
|
||||
continue
|
||||
hook_group = hook_group.clone_and_combine(lora_hooks)
|
||||
|
||||
active_loras.append((lora_name, model_strength, clip_strength))
|
||||
all_trigger_words.extend(trigger_words)
|
||||
|
||||
# Format trigger words (group mode separator)
|
||||
trigger_words_text = ",, ".join(all_trigger_words) if all_trigger_words else ""
|
||||
|
||||
# Format active LoRAs summary
|
||||
formatted_loras = []
|
||||
for name, model_s, clip_s in active_loras:
|
||||
if abs(model_s - clip_s) > 0.001:
|
||||
formatted_loras.append(
|
||||
f"<lora:{name}:{model_s}:{clip_s}>"
|
||||
)
|
||||
else:
|
||||
formatted_loras.append(f"<lora:{name}:{model_s}>")
|
||||
active_loras_text = " ".join(formatted_loras)
|
||||
|
||||
return (hook_group, trigger_words_text, active_loras_text)
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Lora Info display node — pure frontend node for showing selected LoRA info.
|
||||
|
||||
This node does NOT participate in workflow execution. Its single optional
|
||||
"lora_source" input exists solely as a wire-connection anchor so that the
|
||||
frontend can traverse the graph and push selection data to connected info nodes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class LoraInfoLM:
|
||||
"""Display node that shows filename and notes for the selected LoRA."""
|
||||
|
||||
NAME = "Lora Info (LoraManager)"
|
||||
CATEGORY = "Lora Manager/utils"
|
||||
DESCRIPTION = (
|
||||
"Displays information (filename, notes) about the currently selected "
|
||||
"LoRA. Connect any output from a LoRA Loader or Stacker to the "
|
||||
"lora_source input, then select a LoRA in the source widget — the "
|
||||
"info updates automatically. Does not affect workflow execution."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ()
|
||||
RETURN_NAMES = ()
|
||||
OUTPUT_NODE = False
|
||||
FUNCTION = "noop"
|
||||
|
||||
def noop(self, **kwargs):
|
||||
# This node is display-only — no workflow execution needed.
|
||||
return ()
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
LoraInfoLM.NAME: LoraInfoLM,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
LoraInfoLM.NAME: "Lora Info (LoraManager)",
|
||||
}
|
||||
+16
-24
@@ -1,9 +1,8 @@
|
||||
import importlib
|
||||
import logging
|
||||
import re
|
||||
|
||||
import comfy.sd # type: ignore
|
||||
import comfy.utils # type: ignore
|
||||
import comfy.sd # pyright: ignore[reportMissingImports]
|
||||
import comfy.utils # pyright: ignore[reportMissingImports]
|
||||
|
||||
from ..utils.utils import get_lora_info_absolute
|
||||
from .utils import (
|
||||
@@ -14,6 +13,8 @@ from .utils import (
|
||||
extract_lora_name,
|
||||
get_loras_list,
|
||||
nunchaku_load_lora,
|
||||
parse_lora_syntax,
|
||||
validate_lora_entries,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -48,9 +49,9 @@ def _collect_stack_entries(lora_stack):
|
||||
return entries
|
||||
|
||||
|
||||
def _collect_widget_entries(kwargs):
|
||||
def _collect_widget_entries(loras):
|
||||
entries = []
|
||||
for lora in get_loras_list(kwargs):
|
||||
for lora in get_loras_list({"loras": loras}):
|
||||
if not lora.get("active", False):
|
||||
continue
|
||||
lora_name = apply_lora_syntax_format(lora["name"])
|
||||
@@ -138,20 +139,26 @@ class LoraLoaderLM:
|
||||
"placeholder": "Search LoRAs to add...",
|
||||
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
||||
}),
|
||||
"loras": ("LORAS", {}),
|
||||
},
|
||||
"optional": FlexibleOptionalInputType(any_type),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, loras=None):
|
||||
"""Queue-time validation: reject missing local LoRAs before execution."""
|
||||
return validate_lora_entries({"loras": loras}) or True
|
||||
|
||||
RETURN_TYPES = ("MODEL", "CLIP", "STRING", "STRING")
|
||||
RETURN_NAMES = ("MODEL", "CLIP", "trigger_words", "loaded_loras")
|
||||
FUNCTION = "load_loras"
|
||||
|
||||
def load_loras(self, model, text, **kwargs):
|
||||
"""Loads multiple LoRAs based on the kwargs input and lora_stack."""
|
||||
def load_loras(self, model, text, loras, **kwargs):
|
||||
"""Loads multiple LoRAs based on the widget input and lora_stack."""
|
||||
del text
|
||||
clip = kwargs.get("clip", None)
|
||||
lora_entries = _collect_stack_entries(kwargs.get("lora_stack", None))
|
||||
lora_entries.extend(_collect_widget_entries(kwargs))
|
||||
lora_entries.extend(_collect_widget_entries(loras))
|
||||
|
||||
nunchaku_model_kind = detect_nunchaku_model_kind(model)
|
||||
if nunchaku_model_kind == "flux":
|
||||
@@ -189,25 +196,10 @@ class LoraTextLoaderLM:
|
||||
RETURN_NAMES = ("MODEL", "CLIP", "trigger_words", "loaded_loras")
|
||||
FUNCTION = "load_loras_from_text"
|
||||
|
||||
def parse_lora_syntax(self, text):
|
||||
"""Parse LoRA syntax from text input."""
|
||||
pattern = r"<lora:([^:>]+):([^:>]+)(?::([^:>]+))?>"
|
||||
matches = re.findall(pattern, text, re.IGNORECASE)
|
||||
|
||||
loras = []
|
||||
for match in matches:
|
||||
model_strength = float(match[1])
|
||||
loras.append({
|
||||
"name": match[0],
|
||||
"model_strength": model_strength,
|
||||
"clip_strength": float(match[2]) if match[2] else model_strength,
|
||||
})
|
||||
return loras
|
||||
|
||||
def load_loras_from_text(self, model, lora_syntax, clip=None, lora_stack=None):
|
||||
"""Load LoRAs based on text syntax input."""
|
||||
lora_entries = _collect_stack_entries(lora_stack)
|
||||
for lora in self.parse_lora_syntax(lora_syntax):
|
||||
for lora in parse_lora_syntax(lora_syntax):
|
||||
lora_path, trigger_words = get_lora_info_absolute(lora["name"])
|
||||
lora_entries.append({
|
||||
"name": lora["name"],
|
||||
|
||||
@@ -9,6 +9,7 @@ and tracks the last used combination for reuse.
|
||||
import logging
|
||||
import os
|
||||
from ..utils.utils import get_lora_info
|
||||
from .utils import validate_lora_entries
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -31,6 +32,11 @@ class LoraRandomizerLM:
|
||||
},
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, loras=None):
|
||||
"""Queue-time validation: reject missing local LoRAs before execution."""
|
||||
return validate_lora_entries({"loras": loras}) or True
|
||||
|
||||
RETURN_TYPES = ("LORA_STACK",)
|
||||
RETURN_NAMES = ("LORA_STACK",)
|
||||
|
||||
|
||||
@@ -1,26 +1,102 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
_STACK_INPUT_PATTERN = re.compile(r"^lora_stack(?:_([ab])|(\d+))$")
|
||||
|
||||
|
||||
def _is_stack_input(name: str) -> bool:
|
||||
return bool(_STACK_INPUT_PATTERN.match(name))
|
||||
|
||||
|
||||
def _stack_slot_number(name: str) -> int:
|
||||
"""Numeric slot used to order stack inputs; legacy a/b map to 1/2."""
|
||||
match = _STACK_INPUT_PATTERN.match(name)
|
||||
if not match:
|
||||
return -1
|
||||
letter, digits = match.group(1), match.group(2)
|
||||
if digits is not None:
|
||||
return int(digits)
|
||||
return 1 if letter == "a" else 2
|
||||
|
||||
|
||||
class _LoraStackOptionalInputs:
|
||||
"""Lookup that preserves explicit optional inputs and dynamic lora_stack slots."""
|
||||
|
||||
def __init__(self, explicit_inputs: dict[str, tuple[str, dict[str, Any]]]) -> None:
|
||||
self._explicit_inputs = explicit_inputs
|
||||
|
||||
def __contains__(self, item: object) -> bool:
|
||||
if not isinstance(item, str):
|
||||
return False
|
||||
return item in self._explicit_inputs or _is_stack_input(item)
|
||||
|
||||
def __getitem__(self, key: str) -> tuple[str, dict[str, Any]]:
|
||||
if key in self._explicit_inputs:
|
||||
return self._explicit_inputs[key]
|
||||
if _is_stack_input(key):
|
||||
return (
|
||||
"LORA_STACK",
|
||||
{
|
||||
"tooltip": "A LoRA stack to combine. Connect to add more inputs.",
|
||||
},
|
||||
)
|
||||
raise KeyError(key)
|
||||
|
||||
|
||||
class LoraStackCombinerLM:
|
||||
NAME = "Lora Stack Combiner (LoraManager)"
|
||||
CATEGORY = "Lora Manager/stackers"
|
||||
DESCRIPTION = (
|
||||
"Combines multiple LoRA stacks into a single stack. "
|
||||
"Supports dynamic inputs: connect a stack to add more inputs."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
optional_inputs: dict[str, tuple[str, dict[str, Any]]] = {
|
||||
"lora_stack1": (
|
||||
"LORA_STACK",
|
||||
{
|
||||
"tooltip": "A LoRA stack to combine. Connect to add more inputs.",
|
||||
},
|
||||
),
|
||||
"lora_stack2": (
|
||||
"LORA_STACK",
|
||||
{
|
||||
"tooltip": "A LoRA stack to combine. Connect to add more inputs.",
|
||||
},
|
||||
),
|
||||
}
|
||||
|
||||
stack = inspect.stack()
|
||||
if len(stack) > 2 and stack[2].function == "get_input_info":
|
||||
optional_inputs = _LoraStackOptionalInputs(optional_inputs) # pyright: ignore[reportAssignmentType]
|
||||
|
||||
return {
|
||||
"required": {
|
||||
"lora_stack_a": ("LORA_STACK",),
|
||||
"lora_stack_b": ("LORA_STACK",),
|
||||
},
|
||||
"required": {},
|
||||
"optional": optional_inputs,
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("LORA_STACK",)
|
||||
RETURN_NAMES = ("LORA_STACK",)
|
||||
FUNCTION = "combine_stacks"
|
||||
|
||||
def combine_stacks(self, lora_stack_a, lora_stack_b):
|
||||
combined_stack = []
|
||||
def combine_stacks(self, lora_stack1=None, lora_stack2=None, **kwargs):
|
||||
stacks = {
|
||||
"lora_stack1": lora_stack1,
|
||||
"lora_stack2": lora_stack2,
|
||||
}
|
||||
for key, value in kwargs.items():
|
||||
if _is_stack_input(key) and value is not None:
|
||||
stacks[key] = value
|
||||
|
||||
if lora_stack_a:
|
||||
combined_stack.extend(lora_stack_a)
|
||||
if lora_stack_b:
|
||||
combined_stack.extend(lora_stack_b)
|
||||
combined_stack = []
|
||||
for key in sorted(stacks, key=_stack_slot_number):
|
||||
stack = stacks[key]
|
||||
if stack:
|
||||
combined_stack.extend(stack)
|
||||
|
||||
return (combined_stack,)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
from ..utils.utils import get_lora_info
|
||||
from .utils import FlexibleOptionalInputType, any_type, apply_lora_syntax_format, extract_lora_name, get_loras_list
|
||||
from .utils import FlexibleOptionalInputType, any_type, apply_lora_syntax_format, extract_lora_name, get_loras_list, validate_lora_entries
|
||||
|
||||
import logging
|
||||
|
||||
@@ -18,16 +18,22 @@ class LoraStackerLM:
|
||||
"placeholder": "Search LoRAs to add...",
|
||||
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
||||
}),
|
||||
"loras": ("LORAS", {}),
|
||||
},
|
||||
"optional": FlexibleOptionalInputType(any_type),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, loras=None):
|
||||
"""Queue-time validation: reject missing local LoRAs before execution."""
|
||||
return validate_lora_entries({"loras": loras}) or True
|
||||
|
||||
RETURN_TYPES = ("LORA_STACK", "STRING", "STRING")
|
||||
RETURN_NAMES = ("LORA_STACK", "trigger_words", "active_loras")
|
||||
FUNCTION = "stack_loras"
|
||||
|
||||
def stack_loras(self, text, **kwargs):
|
||||
"""Stacks multiple LoRAs based on the kwargs input without loading them."""
|
||||
def stack_loras(self, text, loras, **kwargs):
|
||||
"""Stacks multiple LoRAs based on the widget input without loading them."""
|
||||
stack = []
|
||||
active_loras = []
|
||||
all_trigger_words = []
|
||||
@@ -42,8 +48,8 @@ class LoraStackerLM:
|
||||
_, trigger_words = get_lora_info(lora_name)
|
||||
all_trigger_words.extend(trigger_words)
|
||||
|
||||
# Process loras from kwargs with support for both old and new formats
|
||||
loras_list = get_loras_list(kwargs)
|
||||
# Process loras from the widget with support for both old and new formats
|
||||
loras_list = get_loras_list({"loras": loras})
|
||||
for lora in loras_list:
|
||||
if not lora.get('active', False):
|
||||
continue
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Node to resolve `<lora:name:strength>` syntax to absolute file system paths.
|
||||
|
||||
Takes the loaded_loras / active_loras STRING output from LoraLoaderLM or
|
||||
LoraStackerLM and resolves each lora name to its absolute path on disk via
|
||||
the scanner cache. Unknown names are returned as-is.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from ..utils.utils import get_lora_info_absolute
|
||||
from .utils import parse_lora_syntax
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LoraSyntaxToPath:
|
||||
NAME = "LoRA Syntax → Path (LoraManager)"
|
||||
CATEGORY = "Lora Manager/utils"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"lora_syntax": (
|
||||
"STRING",
|
||||
{
|
||||
"forceInput": True,
|
||||
"multiline": True,
|
||||
"tooltip": (
|
||||
"<lora:name:strength> formatted text from "
|
||||
"loaded_loras / active_loras output"
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("paths",)
|
||||
FUNCTION = "resolve"
|
||||
|
||||
def resolve(self, lora_syntax: str) -> tuple[str]:
|
||||
"""Parse <lora:...> syntax and resolve each name to its absolute path."""
|
||||
if not lora_syntax or not lora_syntax.strip():
|
||||
logger.info("Received empty lora_syntax input")
|
||||
return ("",)
|
||||
|
||||
parsed = parse_lora_syntax(lora_syntax)
|
||||
if not parsed:
|
||||
logger.info("No valid <lora:...> entries found in input")
|
||||
return ("",)
|
||||
|
||||
paths: list[str] = []
|
||||
for entry in parsed:
|
||||
try:
|
||||
absolute_path, _ = get_lora_info_absolute(entry["name"])
|
||||
paths.append(absolute_path)
|
||||
except Exception:
|
||||
logger.warning("Failed to resolve lora '%s', skipping", entry["name"])
|
||||
continue
|
||||
|
||||
return ("\n".join(paths),)
|
||||
@@ -0,0 +1,179 @@
|
||||
"""Metadata Overwrite node — allows users to manually specify generation parameters
|
||||
that override the automatically collected/inferred metadata.
|
||||
|
||||
Most inputs have falsy defaults (empty string / 0) which are skipped.
|
||||
clip_skip uses a sentinel default (-25) so that a wired value of 0 is
|
||||
preserved — both ComfyUI and A1111 conventions have no meaningful 0 value,
|
||||
but users may wire 0 to express "no clip skip / default".
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from ..metadata_collector.constants import CLIP_SKIP_SENTINEL as _CLIP_SKIP_SENTINEL
|
||||
from ..metadata_collector.overwrite_utils import collect_overwrite_params
|
||||
|
||||
|
||||
class MetadataOverwriteLM:
|
||||
NAME = "Metadata Overwrite (LoraManager)"
|
||||
CATEGORY = "Lora Manager/utils"
|
||||
DESCRIPTION = (
|
||||
"Manually specify generation parameters to override automatically collected "
|
||||
"metadata. Only filled/connected inputs will take effect — empty defaults "
|
||||
"are ignored."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls) -> dict[str, Any]:
|
||||
return {
|
||||
"optional": {
|
||||
"prompt": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"multiline": True,
|
||||
"tooltip": "Positive prompt. Only overwrites when non-empty.",
|
||||
},
|
||||
),
|
||||
"negative_prompt": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"multiline": True,
|
||||
"tooltip": "Negative prompt. Only overwrites when non-empty.",
|
||||
},
|
||||
),
|
||||
"seed": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"min": 0,
|
||||
"max": 0xFFFFFFFFFFFFFFFF,
|
||||
"control_after_generate": False,
|
||||
"tooltip": "Seed value. Only overwrites when > 0.",
|
||||
},
|
||||
),
|
||||
"steps": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"min": 0,
|
||||
"max": 10000,
|
||||
"tooltip": "Number of steps. Only overwrites when > 0.",
|
||||
},
|
||||
),
|
||||
"cfg_scale": (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.0,
|
||||
"min": 0.0,
|
||||
"max": 100.0,
|
||||
"tooltip": "CFG scale. Only overwrites when > 0.",
|
||||
},
|
||||
),
|
||||
"sampler": (
|
||||
"STRING,SAMPLER",
|
||||
{
|
||||
"default": "",
|
||||
"widgetType": "STRING",
|
||||
"tooltip": (
|
||||
"Sampler name. Fill in the name manually or "
|
||||
"connect a SAMPLER output (e.g. KSamplerSelect) "
|
||||
"— the sampler name is then extracted "
|
||||
"automatically. Note: ddim is recorded as "
|
||||
"euler (ComfyUI internal representation). "
|
||||
"Only overwrites when non-empty."
|
||||
),
|
||||
},
|
||||
),
|
||||
"scheduler": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"tooltip": "Scheduler name. Only overwrites when non-empty.",
|
||||
},
|
||||
),
|
||||
"model": (
|
||||
"STRING,MODEL",
|
||||
{
|
||||
"default": "",
|
||||
"widgetType": "STRING",
|
||||
"tooltip": (
|
||||
"The checkpoint or diffusion model (UNet) used "
|
||||
"for generation. Fill in the name manually or "
|
||||
"connect a MODEL output — the model name is then "
|
||||
"extracted automatically. Only overwrites when "
|
||||
"non-empty."
|
||||
),
|
||||
},
|
||||
),
|
||||
"loras": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"multiline": True,
|
||||
"tooltip": (
|
||||
"LoRA syntax, e.g. <lora:name:strength> "
|
||||
"or <lora:name:model_strength:clip_strength>, "
|
||||
"separated by spaces. Only overwrites when non-empty."
|
||||
),
|
||||
},
|
||||
),
|
||||
"size": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"tooltip": (
|
||||
"Image size in WIDTHxHEIGHT format (e.g. 512x768). "
|
||||
"Only overwrites when non-empty."
|
||||
),
|
||||
},
|
||||
),
|
||||
"clip_skip": (
|
||||
"INT",
|
||||
{
|
||||
"default": _CLIP_SKIP_SENTINEL,
|
||||
"min": -25,
|
||||
"max": 24,
|
||||
"tooltip": (
|
||||
"Clip skip (ComfyUI: -24..-1, A1111: 1+). "
|
||||
"Default -25 means not set — any other value "
|
||||
"overwrites."
|
||||
),
|
||||
},
|
||||
),
|
||||
"additional_data": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"multiline": True,
|
||||
"tooltip": (
|
||||
"Additional data to embed in the image metadata. "
|
||||
"Inserted between Clip skip and Model hash in the "
|
||||
"A1111-compatible parameters string. "
|
||||
'Example: "Copyright": "Some license info"'
|
||||
),
|
||||
},
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("METADATA",)
|
||||
RETURN_NAMES = ("metadata",)
|
||||
FUNCTION = "collect_metadata"
|
||||
OUTPUT_NODE = True
|
||||
|
||||
def collect_metadata(self, **kwargs: Any) -> tuple[dict[str, Any]]:
|
||||
"""Collect non-default input values into a metadata dict.
|
||||
|
||||
For most fields, a falsy value (empty string, 0) means "not set"
|
||||
and is skipped. clip_skip uses a dedicated sentinel (-25) so that
|
||||
a wired value of 0 is preserved and reaches the metadata pipeline.
|
||||
|
||||
The ``model`` field accepts either a manual string or a wired MODEL
|
||||
(ModelPatcher) connection; in the latter case the underlying model
|
||||
name is extracted from the patcher's ``cached_patcher_init`` and
|
||||
stored as a ComfyUI-style relative path. The ``sampler`` field
|
||||
likewise accepts a manual string or a wired SAMPLER (KSAMPLER)
|
||||
connection, from which the sampler name is extracted automatically.
|
||||
"""
|
||||
return (collect_overwrite_params(kwargs),)
|
||||
+12
-13
@@ -15,15 +15,15 @@ import os
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple, Union
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union, cast
|
||||
|
||||
import comfy.utils # type: ignore
|
||||
import folder_paths # type: ignore
|
||||
import comfy.utils # pyright: ignore[reportMissingImports]
|
||||
import folder_paths # pyright: ignore[reportMissingImports]
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from safetensors import safe_open
|
||||
|
||||
from nunchaku.lora.flux.nunchaku_converter import (
|
||||
from nunchaku.lora.flux.nunchaku_converter import ( # pyright: ignore[reportMissingTypeStubs]
|
||||
pack_lowrank_weight,
|
||||
unpack_lowrank_weight,
|
||||
)
|
||||
@@ -87,10 +87,6 @@ def _rename_layer_underscore_layer_name(old_name: str) -> str:
|
||||
return new_name
|
||||
|
||||
|
||||
def _is_indexable_module(module):
|
||||
return isinstance(module, (nn.ModuleList, nn.Sequential, list, tuple))
|
||||
|
||||
|
||||
def _get_module_by_name(model: nn.Module, name: str) -> Optional[nn.Module]:
|
||||
if not name:
|
||||
return model
|
||||
@@ -100,7 +96,7 @@ def _get_module_by_name(model: nn.Module, name: str) -> Optional[nn.Module]:
|
||||
continue
|
||||
if hasattr(module, part):
|
||||
module = getattr(module, part)
|
||||
elif part.isdigit() and _is_indexable_module(module):
|
||||
elif part.isdigit() and isinstance(module, (nn.ModuleList, nn.Sequential, list, tuple)):
|
||||
try:
|
||||
module = module[int(part)]
|
||||
except (IndexError, TypeError):
|
||||
@@ -267,7 +263,9 @@ def _handle_proj_out_split(lora_dict: Dict[str, Dict[str, torch.Tensor]], base_k
|
||||
return result, consumed
|
||||
|
||||
|
||||
def _apply_lora_to_module(module: nn.Module, a_tensor: torch.Tensor, b_tensor: torch.Tensor, module_name: str, model: nn.Module) -> None:
|
||||
def _apply_lora_to_module(module: Any, a_tensor: torch.Tensor, b_tensor: torch.Tensor, module_name: str, model: Any) -> None:
|
||||
# These modules are dynamic torch containers; monkey-patched attributes
|
||||
# below are set at runtime, so the module/model types are deliberately Any.
|
||||
if not hasattr(module, "in_features") or not hasattr(module, "out_features"):
|
||||
raise ValueError(f"{module_name}: unsupported module without in/out features")
|
||||
if a_tensor.shape[1] != module.in_features or b_tensor.shape[0] != module.out_features:
|
||||
@@ -336,7 +334,7 @@ def _apply_lora_to_module(module: nn.Module, a_tensor: torch.Tensor, b_tensor: t
|
||||
raise ValueError(f"{module_name}: unsupported module type {type(module)}")
|
||||
|
||||
|
||||
def reset_lora_v2(model: nn.Module) -> None:
|
||||
def reset_lora_v2(model: Any) -> None:
|
||||
slots = getattr(model, "_lora_slots", None)
|
||||
if not slots:
|
||||
return
|
||||
@@ -344,6 +342,7 @@ def reset_lora_v2(model: nn.Module) -> None:
|
||||
module = _get_module_by_name(model, name)
|
||||
if module is None:
|
||||
continue
|
||||
module = cast(Any, module)
|
||||
module_type = info.get("type", "nunchaku")
|
||||
if module_type == "nunchaku":
|
||||
base_rank = info["base_rank"]
|
||||
@@ -371,7 +370,7 @@ def reset_lora_v2(model: nn.Module) -> None:
|
||||
def compose_loras_v2(model: nn.Module, lora_configs: List[Tuple[Union[str, Path, Dict[str, torch.Tensor]], float]], apply_awq_mod: bool = True) -> bool:
|
||||
del apply_awq_mod # retained for interface compatibility
|
||||
reset_lora_v2(model)
|
||||
aggregated_weights: Dict[str, List[Dict[str, object]]] = defaultdict(list)
|
||||
aggregated_weights: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
|
||||
saw_supported_format = False
|
||||
unresolved_targets = 0
|
||||
|
||||
@@ -471,7 +470,7 @@ def compose_loras_v2(model: nn.Module, lora_configs: List[Tuple[Union[str, Path,
|
||||
class ComfyQwenImageWrapperLM(nn.Module):
|
||||
def __init__(self, model: nn.Module, config=None, apply_awq_mod: bool = True):
|
||||
super().__init__()
|
||||
self.model = model
|
||||
self.model: Any = model
|
||||
self.config = {} if config is None else config
|
||||
self.dtype = next(model.parameters()).dtype
|
||||
self.loras: List[Tuple[Union[str, Path, Dict[str, torch.Tensor]], float]] = []
|
||||
|
||||
+2
-2
@@ -67,7 +67,7 @@ class PromptLM:
|
||||
|
||||
stack = inspect.stack()
|
||||
if len(stack) > 2 and stack[2].function == "get_input_info":
|
||||
optional_inputs = _PromptOptionalInputs(optional_inputs) # type: ignore[assignment]
|
||||
optional_inputs = _PromptOptionalInputs(optional_inputs) # pyright: ignore[reportAssignmentType]
|
||||
|
||||
return {
|
||||
"required": {
|
||||
@@ -126,7 +126,7 @@ class PromptLM:
|
||||
else:
|
||||
prompt = expanded_text
|
||||
|
||||
from nodes import CLIPTextEncode # type: ignore
|
||||
from nodes import CLIPTextEncode # pyright: ignore[reportMissingImports, reportAttributeAccessIssue]
|
||||
|
||||
conditioning = CLIPTextEncode().encode(clip, prompt)[0]
|
||||
return (conditioning, prompt)
|
||||
|
||||
+378
-133
@@ -5,7 +5,7 @@ import time
|
||||
import uuid
|
||||
from typing import Any, Dict, Optional
|
||||
import numpy as np
|
||||
import folder_paths # type: ignore
|
||||
import folder_paths # pyright: ignore[reportMissingImports]
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
from ..metadata_collector.metadata_processor import MetadataProcessor
|
||||
from ..metadata_collector import get_metadata
|
||||
@@ -13,9 +13,159 @@ from ..utils.constants import CARD_PREVIEW_WIDTH
|
||||
from ..utils.exif_utils import ExifUtils
|
||||
from ..utils.utils import calculate_recipe_fingerprint, sanitize_folder_name
|
||||
from PIL import Image, PngImagePlugin
|
||||
import piexif
|
||||
import piexif # pyright: ignore[reportMissingTypeStubs]
|
||||
import logging
|
||||
|
||||
# Civitai-compatible sampler name mapping: ComfyUI internal → A1111 display name
|
||||
CIVITAI_SAMPLER_MAP = {
|
||||
"euler": "Euler",
|
||||
"euler_ancestral": "Euler a",
|
||||
"lms": "LMS",
|
||||
"heun": "Heun",
|
||||
"dpm_2": "DPM2",
|
||||
"dpm_2_ancestral": "DPM2 a",
|
||||
"dpmpp_2s_ancestral": "DPM++ 2S a",
|
||||
"dpmpp_2m": "DPM++ 2M",
|
||||
"dpmpp_sde": "DPM++ SDE",
|
||||
"dpmpp_sde_gpu": "DPM++ SDE",
|
||||
"dpmpp_2m_sde": "DPM++ 2M SDE",
|
||||
"dpmpp_2m_sde_gpu": "DPM++ 2M SDE",
|
||||
"dpmpp_3m_sde": "DPM++ 3M SDE",
|
||||
"dpm_fast": "DPM fast",
|
||||
"dpm_adaptive": "DPM adaptive",
|
||||
"ddim": "DDIM",
|
||||
"plms": "PLMS",
|
||||
"uni_pc_bh2": "UniPC",
|
||||
"uni_pc": "UniPC",
|
||||
"lcm": "LCM",
|
||||
}
|
||||
|
||||
# Base model display name → AIR URN slug
|
||||
# Sourced from civitai source: src/shared/constants/basemodel.constants.ts
|
||||
BASE_MODEL_AIR_SLUG = {
|
||||
# Stable Diffusion family
|
||||
"SD 1.4": "sd1",
|
||||
"SD 1.5": "sd1",
|
||||
"SD 1.5 LCM": "sd1",
|
||||
"SD 1.5 Hyper": "sd1",
|
||||
"SD 2.0": "sd2",
|
||||
"SD 2.0 768": "sd2",
|
||||
"SD 2.1": "sd2",
|
||||
"SD 2.1 768": "sd2",
|
||||
"SD 2.1 Unclip": "sd2",
|
||||
"SD 3.0": "sd3",
|
||||
"SD 3.5": "sd35",
|
||||
"SD 3.5 Large": "sd35",
|
||||
"SD 3.5 Large Turbo": "sd35",
|
||||
"SD 3.5 Medium": "sd35",
|
||||
"SDXL 0.9": "sdxl",
|
||||
"SDXL 1.0": "sdxl",
|
||||
"SDXL 1.0 LCM": "sdxl",
|
||||
"SDXL Lightning": "sdxl",
|
||||
"SDXL Hyper": "sdxl",
|
||||
"SDXL Turbo": "sdxl",
|
||||
"SDXL Distilled": "sdxldistilled",
|
||||
"Stable Cascade": "scascade",
|
||||
"Stable Video Diffusion": "svd",
|
||||
"SVD": "svd",
|
||||
"SVD XT": "svdxt",
|
||||
|
||||
# SDXL community fine-tunes
|
||||
"Pony": "pony",
|
||||
"Pony Diffusion": "pony",
|
||||
"Illustrious": "illustrious",
|
||||
"NoobAI": "noobai",
|
||||
"Animagine": "illustrious",
|
||||
|
||||
# Flux family
|
||||
"Flux.1": "flux1",
|
||||
"Flux.1 D": "flux1",
|
||||
"Flux.1 S": "flux1",
|
||||
"Flux.1 Krea": "fluxkrea",
|
||||
"Flux.1 Kontext": "flux1kontext",
|
||||
"Flux.2": "flux2",
|
||||
"Flux.2 D": "flux2",
|
||||
"Flux.2 Klein 9B": "flux2klein_9b",
|
||||
"Flux.2 Klein 9B Base": "flux2klein_9b_base",
|
||||
"Flux.2 Klein 4B": "flux2klein_4b",
|
||||
"Flux.2 Klein 4B Base": "flux2klein_4b_base",
|
||||
|
||||
# Other image models (sorted alphabetically)
|
||||
"AuraFlow": "auraflow",
|
||||
"Chroma": "chroma",
|
||||
"HiDream": "hidream",
|
||||
"HiDream-O1": "hidream-o1",
|
||||
"Hunyuan DiT": "hydit1",
|
||||
"Hunyuan Video": "hyv1",
|
||||
"Kolors": "kolors",
|
||||
"Lumina": "lumina",
|
||||
"Mochi": "mochi",
|
||||
"ODOR": "odor",
|
||||
"PixArt Alpha": "pixarta",
|
||||
"PixArt Sigma": "pixarte",
|
||||
"Playground v2": "playgroundv2",
|
||||
"Playground v2.5": "playgroundv2",
|
||||
"Pony Diffusion V7": "ponyv7",
|
||||
|
||||
# Video models
|
||||
"CogVideoX": "cogvideox",
|
||||
"LTX Video": "ltxv",
|
||||
"LTX Video 2": "ltxv2",
|
||||
"LTX Video 2.3": "ltxv23",
|
||||
"Wan Video": "wanvideo",
|
||||
"Wan Video 1.3B T2V": "wanvideo_13b_t2v",
|
||||
"Wan Video 14B T2V": "wanvideo_14b_t2v",
|
||||
"Wan Video 14B I2V 480p": "wanvideo_14b_i2v_480p",
|
||||
"Wan Video 14B I2V 720p": "wanvideo_14b_i2v_720p",
|
||||
|
||||
# Third-party / proprietary image models
|
||||
"Boogu": "boogu",
|
||||
"Ernie": "ernie",
|
||||
"Grok": "grok",
|
||||
"HappyHorse": "happyhorse",
|
||||
"Ideogram": "ideogram",
|
||||
"Ideogram 4.0": "ideogram",
|
||||
"Imagen": "imagen4",
|
||||
"Imagen 4": "imagen4",
|
||||
"Krea": "krea2",
|
||||
"Krea 2": "krea2",
|
||||
"Lens": "lens",
|
||||
"MAI": "mai",
|
||||
"Nano Banana": "nanobanana",
|
||||
"OpenAI": "openai",
|
||||
"Reve": "reve",
|
||||
"Reve 2": "reve",
|
||||
"Reve 2.1": "reve",
|
||||
"Seedream": "seedream",
|
||||
"Sora": "sora2",
|
||||
"Sora 2": "sora2",
|
||||
"Veo": "veo3",
|
||||
"Veo 2": "veo3",
|
||||
"Veo 3": "veo3",
|
||||
"ZImageTurbo": "zimageturbo",
|
||||
"ZImageBase": "zimagebase",
|
||||
"ZImage": "zimagebase",
|
||||
|
||||
# Third-party video models
|
||||
"Hailuo by MiniMax": "minimax",
|
||||
"Haiper": "haiper",
|
||||
"Kling": "kling",
|
||||
"Lightricks": "lightricks",
|
||||
"Seedance": "seedance",
|
||||
"Vidu": "vidu",
|
||||
|
||||
# Qwen family
|
||||
"Qwen": "qwen",
|
||||
"Qwen 2": "qwen2",
|
||||
|
||||
# Anima
|
||||
"Anima": "anima",
|
||||
|
||||
# Special
|
||||
"Upscaler": "upscaler",
|
||||
"Other": "other",
|
||||
}
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -70,11 +220,29 @@ class SaveImageLM:
|
||||
"tooltip": "Compression quality for JPEG and lossy WebP formats (1-100). Higher values mean better quality but larger files.",
|
||||
},
|
||||
),
|
||||
"webp_method": (
|
||||
"INT",
|
||||
{
|
||||
"default": 6,
|
||||
"min": 0,
|
||||
"max": 6,
|
||||
"tooltip": "WebP compression method (0-6). 0=fastest/largest, 6=slowest/smallest. Only applies when file_format is 'webp'.",
|
||||
},
|
||||
),
|
||||
"jpeg_subsampling": (
|
||||
"INT",
|
||||
{
|
||||
"default": 0,
|
||||
"min": 0,
|
||||
"max": 2,
|
||||
"tooltip": "JPEG chroma subsampling level. 0=4:4:4 (best quality), 1=4:2:2, 2=4:2:0 (smallest files). Only applies when file_format is 'jpeg'.",
|
||||
},
|
||||
),
|
||||
"embed_workflow": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "Embeds the complete workflow data into the image metadata. Only works with PNG and WebP formats.",
|
||||
"tooltip": "When enabled, saved images store the complete workflow. Drag the image back into ComfyUI to restore the original node graph. PNG and WebP only.",
|
||||
},
|
||||
),
|
||||
"save_with_metadata": (
|
||||
@@ -84,6 +252,13 @@ class SaveImageLM:
|
||||
"tooltip": "When enabled, embeds generation parameters into the saved image metadata. Disable to skip writing generation metadata.",
|
||||
},
|
||||
),
|
||||
"add_loras_to_prompt": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "When enabled, appends the LoRA syntax line (e.g. <lora:name:strength>) after the positive prompt in the saved metadata.",
|
||||
},
|
||||
),
|
||||
"add_counter_to_filename": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
@@ -142,148 +317,197 @@ class SaveImageLM:
|
||||
|
||||
return None
|
||||
|
||||
def format_metadata(self, metadata_dict):
|
||||
"""Format metadata in the requested format similar to userComment example"""
|
||||
if not metadata_dict:
|
||||
return ""
|
||||
def _resolve_model_cache_entry(self, scanner_type: str, name: str):
|
||||
"""Resolve model hash, civitai metadata, and base_model from scanner cache.
|
||||
Returns (hash_str, civitai_dict, base_model_str). All values are empty defaults when not found."""
|
||||
scanner = ServiceRegistry.get_service_sync(scanner_type)
|
||||
if scanner is None or not name:
|
||||
return "", {}, ""
|
||||
|
||||
# Helper function to only add parameter if value is not None
|
||||
def add_param_if_not_none(param_list, label, value):
|
||||
if value is not None:
|
||||
param_list.append(f"{label}: {value}")
|
||||
entry = self._get_cached_model_by_name(scanner, name)
|
||||
if entry is None:
|
||||
basename = os.path.splitext(os.path.basename(name))[0]
|
||||
hash_val = scanner.get_hash_by_filename(basename)
|
||||
return (hash_val or "").lower(), {}, ""
|
||||
|
||||
hash_val = (entry.get("sha256") or "").lower()
|
||||
civitai = entry.get("civitai") or {}
|
||||
base_model = entry.get("base_model") or ""
|
||||
return hash_val, civitai, base_model
|
||||
|
||||
@staticmethod
|
||||
def _get_civitai_sampler_name(sampler_name: str, scheduler: str) -> str:
|
||||
if sampler_name in CIVITAI_SAMPLER_MAP:
|
||||
civitai_name = CIVITAI_SAMPLER_MAP[sampler_name]
|
||||
if scheduler == "karras":
|
||||
civitai_name += " Karras"
|
||||
elif scheduler == "exponential":
|
||||
civitai_name += " Exponential"
|
||||
return civitai_name
|
||||
else:
|
||||
if scheduler and scheduler != "normal":
|
||||
return f"{sampler_name}_{scheduler}"
|
||||
return sampler_name
|
||||
|
||||
@staticmethod
|
||||
def _build_air_string(base_model: str, model_type: str, model_id: int, version_id: int) -> str:
|
||||
slug = BASE_MODEL_AIR_SLUG.get(base_model, "other")
|
||||
type_lower = model_type.lower() if model_type else "other"
|
||||
return f"urn:air:{slug}:{type_lower}:civitai:{model_id}@{version_id}"
|
||||
|
||||
def format_metadata(self, metadata_dict: dict[str, Any], add_loras_to_prompt: bool = False) -> str:
|
||||
"""Format metadata as A1111-compatible parameters string with Hashes JSON and Civitai resources."""
|
||||
if not metadata_dict: return ""
|
||||
|
||||
# Extract the prompt and negative prompt
|
||||
prompt = metadata_dict.get("prompt", "")
|
||||
negative_prompt = metadata_dict.get("negative_prompt", "")
|
||||
|
||||
# Extract loras from the prompt if present
|
||||
steps = metadata_dict.get("steps")
|
||||
cfg = metadata_dict.get("guidance")
|
||||
if cfg is None:
|
||||
cfg = metadata_dict.get("cfg_scale")
|
||||
if cfg is None:
|
||||
cfg = metadata_dict.get("cfg")
|
||||
seed = metadata_dict.get("seed")
|
||||
size = metadata_dict.get("size")
|
||||
sampler = metadata_dict.get("sampler") or ""
|
||||
scheduler = metadata_dict.get("scheduler") or "normal"
|
||||
checkpoint = metadata_dict.get("checkpoint") or ""
|
||||
loras_text = metadata_dict.get("loras", "")
|
||||
lora_hashes = {}
|
||||
clip_skip = metadata_dict.get("clip_skip")
|
||||
|
||||
# If loras are found, add them on a new line after the prompt
|
||||
# Parse LoRA entries from <lora:name:strength> format
|
||||
lora_entries: list[tuple[str, float]] = []
|
||||
if loras_text:
|
||||
prompt_with_loras = f"{prompt}\n{loras_text}"
|
||||
for match in re.findall(r"<lora:([^:]+):([^>]+)>", loras_text):
|
||||
lora_name, strength_str = match
|
||||
try:
|
||||
strength = float(strength_str)
|
||||
except (ValueError, TypeError):
|
||||
strength = 1.0
|
||||
lora_entries.append((lora_name, strength))
|
||||
|
||||
# Extract lora names from the format <lora:name:strength>
|
||||
lora_matches = re.findall(r"<lora:([^:]+):([^>]+)>", loras_text)
|
||||
# Resolve checkpoint hash and Civitai data from local cache
|
||||
ckpt_hash, ckpt_civitai, ckpt_base_model = "", {}, ""
|
||||
ckpt_display_name = ""
|
||||
if checkpoint:
|
||||
ckpt_hash, ckpt_civitai, ckpt_base_model = self._resolve_model_cache_entry(
|
||||
"checkpoint_scanner", checkpoint
|
||||
)
|
||||
ckpt_display_name = os.path.splitext(os.path.basename(checkpoint))[0]
|
||||
|
||||
# Get hash for each lora
|
||||
for lora_name, strength in lora_matches:
|
||||
hash_value = self.get_lora_hash(lora_name)
|
||||
if hash_value:
|
||||
lora_hashes[lora_name] = hash_value
|
||||
else:
|
||||
prompt_with_loras = prompt
|
||||
# Resolve LoRA hash and Civitai data from local cache
|
||||
loras_data: list[dict[str, Any]] = []
|
||||
for lora_name, strength in lora_entries:
|
||||
lora_hash, lora_civitai, lora_base_model = self._resolve_model_cache_entry(
|
||||
"lora_scanner", lora_name
|
||||
)
|
||||
loras_data.append({
|
||||
"name": lora_name,
|
||||
"strength": strength,
|
||||
"hash": lora_hash,
|
||||
"civitai": lora_civitai,
|
||||
"base_model": lora_base_model,
|
||||
})
|
||||
|
||||
# Format the first part (prompt and loras)
|
||||
metadata_parts = [prompt_with_loras]
|
||||
# Build Hashes JSON (A1111 / Civitai standard format)
|
||||
hashes: dict[str, str] = {}
|
||||
if ckpt_hash:
|
||||
hashes["model"] = ckpt_hash[:10].upper()
|
||||
for lora in loras_data:
|
||||
if lora["hash"]:
|
||||
hashes[f"LORA:{lora['name']}"] = lora["hash"][:10].upper()
|
||||
|
||||
# Add negative prompt
|
||||
# Build Civitai resources JSON array
|
||||
civitai_resources: list[dict[str, Any]] = []
|
||||
if ckpt_civitai.get("id", 0) > 0:
|
||||
ckpt_resource: dict[str, Any] = {}
|
||||
ckpt_type = (ckpt_civitai.get("model") or {}).get("type", "Checkpoint")
|
||||
model_id = ckpt_civitai.get("modelId", 0)
|
||||
version_id = ckpt_civitai.get("id", 0)
|
||||
if model_id and version_id:
|
||||
ckpt_resource["air"] = self._build_air_string(
|
||||
ckpt_base_model, ckpt_type, int(model_id), int(version_id)
|
||||
)
|
||||
elif version_id:
|
||||
ckpt_resource["modelVersionId"] = int(version_id)
|
||||
if ckpt_civitai.get("name"):
|
||||
ckpt_resource["versionName"] = ckpt_civitai["name"]
|
||||
if ckpt_resource:
|
||||
civitai_resources.append(ckpt_resource)
|
||||
|
||||
for lora in loras_data:
|
||||
lora_civitai = lora["civitai"]
|
||||
if not lora_civitai or lora_civitai.get("id", 0) <= 0:
|
||||
continue
|
||||
lora_resource: dict[str, Any] = {"weight": lora["strength"]}
|
||||
lora_type = (lora_civitai.get("model") or {}).get("type", "LORA")
|
||||
model_id = lora_civitai.get("modelId", 0)
|
||||
version_id = lora_civitai.get("id", 0)
|
||||
if model_id and version_id:
|
||||
lora_resource["air"] = self._build_air_string(
|
||||
lora["base_model"], lora_type, int(model_id), int(version_id)
|
||||
)
|
||||
elif version_id:
|
||||
lora_resource["modelVersionId"] = int(version_id)
|
||||
if lora_civitai.get("name"):
|
||||
lora_resource["versionName"] = lora_civitai["name"]
|
||||
civitai_resources.append(lora_resource)
|
||||
|
||||
sampler_name = CIVITAI_SAMPLER_MAP.get(sampler, sampler) if sampler else None
|
||||
|
||||
scheduler_mapping = {
|
||||
"normal": "Normal",
|
||||
"karras": "Karras",
|
||||
"exponential": "Exponential",
|
||||
"sgm_uniform": "SGM Uniform",
|
||||
"sgm_quadratic": "SGM Quadratic",
|
||||
}
|
||||
scheduler_name = scheduler_mapping.get(scheduler, scheduler) if scheduler else None
|
||||
|
||||
# Build output lines
|
||||
prompt_line = prompt if prompt else ""
|
||||
if add_loras_to_prompt and loras_text:
|
||||
prompt_line = f"{prompt_line}\n{loras_text}" if prompt_line else loras_text
|
||||
lines = [prompt_line] if prompt_line else [""]
|
||||
if negative_prompt:
|
||||
metadata_parts.append(f"Negative prompt: {negative_prompt}")
|
||||
lines.append(f"Negative prompt: {negative_prompt}")
|
||||
|
||||
# Format the second part (generation parameters)
|
||||
params = []
|
||||
|
||||
# Add standard parameters in the correct order
|
||||
if "steps" in metadata_dict:
|
||||
add_param_if_not_none(params, "Steps", metadata_dict.get("steps"))
|
||||
|
||||
# Combine sampler and scheduler information
|
||||
sampler_name = None
|
||||
scheduler_name = None
|
||||
|
||||
if "sampler" in metadata_dict:
|
||||
sampler = metadata_dict.get("sampler")
|
||||
# Convert ComfyUI sampler names to user-friendly names
|
||||
sampler_mapping = {
|
||||
"euler": "Euler",
|
||||
"euler_ancestral": "Euler a",
|
||||
"dpm_2": "DPM2",
|
||||
"dpm_2_ancestral": "DPM2 a",
|
||||
"heun": "Heun",
|
||||
"dpm_fast": "DPM fast",
|
||||
"dpm_adaptive": "DPM adaptive",
|
||||
"lms": "LMS",
|
||||
"dpmpp_2s_ancestral": "DPM++ 2S a",
|
||||
"dpmpp_sde": "DPM++ SDE",
|
||||
"dpmpp_sde_gpu": "DPM++ SDE",
|
||||
"dpmpp_2m": "DPM++ 2M",
|
||||
"dpmpp_2m_sde": "DPM++ 2M SDE",
|
||||
"dpmpp_2m_sde_gpu": "DPM++ 2M SDE",
|
||||
"ddim": "DDIM",
|
||||
}
|
||||
sampler_name = sampler_mapping.get(sampler, sampler)
|
||||
|
||||
if "scheduler" in metadata_dict:
|
||||
scheduler = metadata_dict.get("scheduler")
|
||||
scheduler_mapping = {
|
||||
"normal": "Simple",
|
||||
"karras": "Karras",
|
||||
"exponential": "Exponential",
|
||||
"sgm_uniform": "SGM Uniform",
|
||||
"sgm_quadratic": "SGM Quadratic",
|
||||
}
|
||||
scheduler_name = scheduler_mapping.get(scheduler, scheduler)
|
||||
|
||||
# Add combined sampler and scheduler information
|
||||
params: list[str] = []
|
||||
if steps is not None:
|
||||
params.append(f"Steps: {steps}")
|
||||
if sampler_name:
|
||||
if scheduler_name:
|
||||
params.append(f"Sampler: {sampler_name} {scheduler_name}")
|
||||
else:
|
||||
params.append(f"Sampler: {sampler_name}")
|
||||
if cfg is not None:
|
||||
params.append(f"CFG scale: {cfg}")
|
||||
if seed is not None:
|
||||
params.append(f"Seed: {seed}")
|
||||
if size:
|
||||
params.append(f"Size: {size}")
|
||||
if clip_skip is not None:
|
||||
try:
|
||||
params.append(f"Clip skip: {abs(int(clip_skip))}")
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
additional_data = metadata_dict.get("additional_data", "")
|
||||
if additional_data:
|
||||
params.append(additional_data)
|
||||
if ckpt_hash:
|
||||
params.append(f"Model hash: {ckpt_hash[:10].upper()}")
|
||||
if ckpt_display_name:
|
||||
params.append(f"Model: {ckpt_display_name}")
|
||||
if hashes:
|
||||
params.append(f"Hashes: {json.dumps(hashes, separators=(',', ':'))}")
|
||||
params.append("Version: ComfyUI")
|
||||
if civitai_resources:
|
||||
params.append(
|
||||
f"Civitai resources: {json.dumps(civitai_resources, separators=(',', ':'))}"
|
||||
)
|
||||
|
||||
# CFG scale (Use guidance if available, otherwise fall back to cfg_scale or cfg)
|
||||
if "guidance" in metadata_dict:
|
||||
add_param_if_not_none(params, "CFG scale", metadata_dict.get("guidance"))
|
||||
elif "cfg_scale" in metadata_dict:
|
||||
add_param_if_not_none(params, "CFG scale", metadata_dict.get("cfg_scale"))
|
||||
elif "cfg" in metadata_dict:
|
||||
add_param_if_not_none(params, "CFG scale", metadata_dict.get("cfg"))
|
||||
|
||||
# Seed
|
||||
if "seed" in metadata_dict:
|
||||
add_param_if_not_none(params, "Seed", metadata_dict.get("seed"))
|
||||
|
||||
# Size
|
||||
if "size" in metadata_dict:
|
||||
add_param_if_not_none(params, "Size", metadata_dict.get("size"))
|
||||
|
||||
# Model info
|
||||
if "checkpoint" in metadata_dict:
|
||||
# Ensure checkpoint is a string before processing
|
||||
checkpoint = metadata_dict.get("checkpoint")
|
||||
if checkpoint is not None:
|
||||
# Get model hash
|
||||
model_hash = self.get_checkpoint_hash(checkpoint)
|
||||
|
||||
# Extract basename without path
|
||||
checkpoint_name = os.path.basename(checkpoint)
|
||||
# Remove extension if present
|
||||
checkpoint_name = os.path.splitext(checkpoint_name)[0]
|
||||
|
||||
# Add model hash if available
|
||||
if model_hash:
|
||||
params.append(
|
||||
f"Model hash: {model_hash[:10]}, Model: {checkpoint_name}"
|
||||
)
|
||||
else:
|
||||
params.append(f"Model: {checkpoint_name}")
|
||||
|
||||
# Add LoRA hashes if available
|
||||
if lora_hashes:
|
||||
lora_hash_parts = []
|
||||
for lora_name, hash_value in lora_hashes.items():
|
||||
lora_hash_parts.append(f"{lora_name}: {hash_value[:10]}")
|
||||
|
||||
if lora_hash_parts:
|
||||
params.append(f'Lora hashes: "{", ".join(lora_hash_parts)}"')
|
||||
|
||||
# Combine all parameters with commas
|
||||
metadata_parts.append(", ".join(params))
|
||||
|
||||
# Join all parts with a new line
|
||||
return "\n".join(metadata_parts)
|
||||
lines.append(", ".join(params))
|
||||
return "\n".join(lines)
|
||||
|
||||
# credit to nkchocoai
|
||||
# Add format_filename method to handle pattern substitution
|
||||
@@ -298,7 +522,12 @@ class SaveImageLM:
|
||||
key = parts[0]
|
||||
|
||||
if key == "seed" and "seed" in metadata_dict:
|
||||
filename = filename.replace(segment, str(metadata_dict.get("seed", "")))
|
||||
seed_value = metadata_dict.get("seed")
|
||||
if seed_value is not None:
|
||||
filename = filename.replace(segment, str(seed_value))
|
||||
else:
|
||||
# Fallback if seed was not captured by metadata collector
|
||||
filename = filename.replace(segment, "0")
|
||||
elif key == "width" and "size" in metadata_dict:
|
||||
size = metadata_dict.get("size", "x")
|
||||
w = size.split("x")[0] if isinstance(size, str) else size[0]
|
||||
@@ -372,7 +601,7 @@ class SaveImageLM:
|
||||
os.path.basename(name),
|
||||
os.path.splitext(os.path.basename(name))[0],
|
||||
]
|
||||
for model in getattr(cache, "raw_data", []):
|
||||
for model in list(getattr(cache, "raw_data", [])):
|
||||
file_name = model.get("file_name")
|
||||
if file_name in candidates:
|
||||
return model
|
||||
@@ -549,6 +778,14 @@ class SaveImageLM:
|
||||
if checkpoint_entry:
|
||||
recipe_data["checkpoint"] = checkpoint_entry
|
||||
|
||||
# The recipe image is the WebP produced above from the output file;
|
||||
# reuse the same metadata extraction to record workflow presence.
|
||||
try:
|
||||
metadata = ExifUtils._load_structured_metadata(image_path)
|
||||
recipe_data["has_workflow"] = bool(metadata.get("workflow"))
|
||||
except Exception:
|
||||
recipe_data["has_workflow"] = False
|
||||
|
||||
json_path = os.path.normpath(
|
||||
os.path.join(recipes_dir, f"{recipe_id}.recipe.json")
|
||||
)
|
||||
@@ -568,10 +805,13 @@ class SaveImageLM:
|
||||
extra_pnginfo=None,
|
||||
lossless_webp=True,
|
||||
quality=100,
|
||||
webp_method=6,
|
||||
jpeg_subsampling=0,
|
||||
embed_workflow=False,
|
||||
save_with_metadata=True,
|
||||
add_counter_to_filename=True,
|
||||
save_as_recipe=False,
|
||||
add_loras_to_prompt=False,
|
||||
):
|
||||
"""Save images with metadata"""
|
||||
results = []
|
||||
@@ -580,7 +820,7 @@ class SaveImageLM:
|
||||
raw_metadata = get_metadata()
|
||||
metadata_dict = MetadataProcessor.to_dict(raw_metadata, id)
|
||||
|
||||
metadata = self.format_metadata(metadata_dict)
|
||||
metadata = self.format_metadata(metadata_dict, add_loras_to_prompt)
|
||||
|
||||
# Process filename_prefix with pattern substitution
|
||||
filename_prefix = self.format_filename(filename_prefix, metadata_dict)
|
||||
@@ -603,7 +843,7 @@ class SaveImageLM:
|
||||
img = Image.fromarray(np.clip(img, 0, 255).astype(np.uint8))
|
||||
|
||||
# Generate filename with counter if needed
|
||||
base_filename = filename
|
||||
base_filename = filename.replace("%batch_num%", str(i))
|
||||
if add_counter_to_filename:
|
||||
# Use counter + i to ensure unique filenames for all images in batch
|
||||
current_counter = counter + i
|
||||
@@ -622,15 +862,14 @@ class SaveImageLM:
|
||||
elif file_format == "jpeg":
|
||||
file = base_filename + ".jpg"
|
||||
file_extension = ".jpg"
|
||||
save_kwargs = {"quality": quality, "optimize": True}
|
||||
save_kwargs = {"quality": quality, "optimize": True, "subsampling": jpeg_subsampling}
|
||||
elif file_format == "webp":
|
||||
file = base_filename + ".webp"
|
||||
file_extension = ".webp"
|
||||
# Add optimization param to control performance
|
||||
save_kwargs = {
|
||||
"quality": quality,
|
||||
"lossless": lossless_webp,
|
||||
"method": 0,
|
||||
"method": webp_method,
|
||||
}
|
||||
else:
|
||||
raise ValueError(f"Unsupported file format: {file_format}")
|
||||
@@ -717,10 +956,13 @@ class SaveImageLM:
|
||||
extra_pnginfo=None,
|
||||
lossless_webp=True,
|
||||
quality=100,
|
||||
webp_method=6,
|
||||
jpeg_subsampling=0,
|
||||
embed_workflow=False,
|
||||
save_with_metadata=True,
|
||||
add_counter_to_filename=True,
|
||||
save_as_recipe=False,
|
||||
add_loras_to_prompt=False,
|
||||
):
|
||||
"""Process and save image with metadata"""
|
||||
# Make sure the output directory exists
|
||||
@@ -746,10 +988,13 @@ class SaveImageLM:
|
||||
extra_pnginfo,
|
||||
lossless_webp,
|
||||
quality,
|
||||
webp_method,
|
||||
jpeg_subsampling,
|
||||
embed_workflow,
|
||||
save_with_metadata,
|
||||
add_counter_to_filename,
|
||||
save_as_recipe,
|
||||
add_loras_to_prompt,
|
||||
)
|
||||
|
||||
return {
|
||||
|
||||
+108
-9
@@ -1,37 +1,74 @@
|
||||
import logging
|
||||
import os
|
||||
from typing import List, Tuple
|
||||
import comfy.sd # type: ignore
|
||||
from typing import Any, List, Tuple
|
||||
import comfy.sd # pyright: ignore[reportMissingImports]
|
||||
from ..utils.utils import get_checkpoint_info_absolute, _format_model_name_for_comfyui
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _reload_gguf_unet(
|
||||
unet_path: str, weight_dtype: str, disable_dynamic: bool = False
|
||||
) -> object:
|
||||
"""Reload a GGUF diffusion model from disk (cached_patcher_init factory).
|
||||
|
||||
Mirrors the GGUF branch of UNETLoaderLM.load_unet so ModelPatcher
|
||||
deepclone/dynamic machinery can rebuild GGUF models with the correct
|
||||
GGMLOps. ``disable_dynamic`` is accepted for signature compatibility
|
||||
with core ComfyUI loaders.
|
||||
"""
|
||||
loader = UNETLoaderLM()
|
||||
model, = loader._load_gguf_unet(unet_path, unet_path, weight_dtype)
|
||||
return model
|
||||
|
||||
|
||||
class UNETLoaderLM:
|
||||
"""UNET Loader with support for extra folder paths
|
||||
|
||||
Loads diffusion models/UNets from both standard ComfyUI folders and LoRA Manager's
|
||||
extra folder paths, providing a unified interface for UNET loading.
|
||||
Supports both regular diffusion models and GGUF format models.
|
||||
The unet_name combo supports ComfyUI's control_after_generate, letting
|
||||
users pick a random diffusion model on every run; the base_model input
|
||||
narrows the random pool through a front-end extension that filters the
|
||||
combo options.
|
||||
"""
|
||||
|
||||
NAME = "Unet Loader (LoraManager)"
|
||||
CATEGORY = "Lora Manager/loaders"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
def INPUT_TYPES(cls):
|
||||
# Get list of unet names from scanner (includes extra folder paths)
|
||||
unet_names = s._get_unet_names()
|
||||
unet_names = cls._get_unet_names()
|
||||
base_models = cls._get_available_base_models()
|
||||
return {
|
||||
"required": {
|
||||
"unet_name": (
|
||||
unet_names,
|
||||
{"tooltip": "The name of the diffusion model to load."},
|
||||
{
|
||||
"tooltip": (
|
||||
"The name of the diffusion model to load. Use "
|
||||
"control_after_generate to pick a random model on "
|
||||
"every run."
|
||||
),
|
||||
"control_after_generate": "fixed",
|
||||
},
|
||||
),
|
||||
"weight_dtype": (
|
||||
["default", "fp8_e4m3fn", "fp8_e4m3fn_fast", "fp8_e5m2"],
|
||||
{"tooltip": "The dtype to use for the model weights."},
|
||||
),
|
||||
"base_model": (
|
||||
base_models,
|
||||
{
|
||||
"default": "Any",
|
||||
"tooltip": (
|
||||
"Restrict the random selection pool to this base "
|
||||
"model. 'Any' uses the full pool."
|
||||
),
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,10 +93,13 @@ class UNETLoaderLM:
|
||||
|
||||
# Filter only diffusion_model type and format names
|
||||
names = []
|
||||
for item in cache.raw_data:
|
||||
for item in list(cache.raw_data):
|
||||
if item.get("sub_type") == "diffusion_model":
|
||||
file_path = item.get("file_path", "")
|
||||
if file_path:
|
||||
# Only offer models that still exist on disk so ComfyUI
|
||||
# flags missing diffusion models at queue time via
|
||||
# "value not in list" (the scanner cache can be stale).
|
||||
if file_path and os.path.exists(file_path):
|
||||
# Format using relative path with OS-native separator
|
||||
formatted_name = _format_model_name_for_comfyui(
|
||||
file_path, model_roots
|
||||
@@ -90,16 +130,69 @@ class UNETLoaderLM:
|
||||
logger.error(f"Error getting unet names: {e}")
|
||||
return []
|
||||
|
||||
def load_unet(self, unet_name: str, weight_dtype: str) -> Tuple:
|
||||
@classmethod
|
||||
def _get_available_base_models(cls) -> List[str]:
|
||||
"""Get distinct base_model values present among indexed diffusion models, for the random-selection filter."""
|
||||
try:
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
|
||||
async def _get_base_models():
|
||||
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||
cache = await scanner.get_cached_data()
|
||||
|
||||
base_models = set()
|
||||
for item in list(cache.raw_data):
|
||||
if item.get("sub_type") != "diffusion_model":
|
||||
continue
|
||||
base_model = item.get("base_model")
|
||||
file_path = item.get("file_path", "")
|
||||
if base_model and file_path and os.path.exists(file_path):
|
||||
base_models.add(base_model)
|
||||
|
||||
return sorted(base_models)
|
||||
|
||||
return ["Any"] + cls._run_async(_get_base_models)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting available base models: {e}")
|
||||
return ["Any"]
|
||||
|
||||
@staticmethod
|
||||
def _run_async(coro_fn):
|
||||
"""Run an async fetcher, handling the case where an event loop is already running."""
|
||||
import asyncio
|
||||
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
import concurrent.futures
|
||||
|
||||
def run_in_thread():
|
||||
new_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(new_loop)
|
||||
try:
|
||||
return new_loop.run_until_complete(coro_fn())
|
||||
finally:
|
||||
new_loop.close()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
future = executor.submit(run_in_thread)
|
||||
return future.result()
|
||||
except RuntimeError:
|
||||
return asyncio.run(coro_fn())
|
||||
|
||||
def load_unet(
|
||||
self, unet_name: str, weight_dtype: str, base_model: str = "Any"
|
||||
) -> Tuple[Any, ...]:
|
||||
"""Load a diffusion model by name, supporting extra folder paths
|
||||
|
||||
Args:
|
||||
unet_name: The name of the diffusion model to load (relative path with extension)
|
||||
weight_dtype: The dtype to use for model weights
|
||||
base_model: Only used by the front-end to filter the random pool
|
||||
|
||||
Returns:
|
||||
Tuple of (MODEL,)
|
||||
"""
|
||||
del base_model
|
||||
import torch
|
||||
|
||||
# Get absolute path from cache using ComfyUI-style name
|
||||
@@ -133,7 +226,7 @@ class UNETLoaderLM:
|
||||
|
||||
def _load_gguf_unet(
|
||||
self, unet_path: str, unet_name: str, weight_dtype: str
|
||||
) -> Tuple:
|
||||
) -> Tuple[Any, ...]:
|
||||
"""Load a GGUF format diffusion model
|
||||
|
||||
Args:
|
||||
@@ -196,6 +289,12 @@ class UNETLoaderLM:
|
||||
# Wrap with GGUFModelPatcher
|
||||
model = GGUFModelPatcher.clone(model)
|
||||
|
||||
# Register a reload factory so the MODEL carries its source path
|
||||
# (cached_patcher_init) like core ComfyUI loaders do — required
|
||||
# for model-name extraction downstream and for ModelPatcher
|
||||
# deepclone/dynamic machinery.
|
||||
model.cached_patcher_init = (_reload_gguf_unet, (unet_path, weight_dtype))
|
||||
|
||||
return (model,)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
+178
-2
@@ -1,3 +1,6 @@
|
||||
from typing import Any
|
||||
|
||||
|
||||
class AnyType(str):
|
||||
"""A special class that is always equal in not equal comparisons. Credit to pythongosssss"""
|
||||
|
||||
@@ -6,7 +9,7 @@ class AnyType(str):
|
||||
|
||||
|
||||
# Credit to Regis Gaughan, III (rgthree)
|
||||
class FlexibleOptionalInputType(dict):
|
||||
class FlexibleOptionalInputType(dict[str, Any]):
|
||||
"""A special class to make flexible nodes that pass data to our python handlers.
|
||||
|
||||
Enables both flexible/dynamic input types (like for Any Switch) or a dynamic number of inputs
|
||||
@@ -23,6 +26,7 @@ class FlexibleOptionalInputType(dict):
|
||||
"""
|
||||
|
||||
def __init__(self, type):
|
||||
super().__init__()
|
||||
self.type = type
|
||||
|
||||
def __getitem__(self, key):
|
||||
@@ -36,10 +40,12 @@ any_type = AnyType("*")
|
||||
|
||||
# Common methods extracted from lora_loader.py and lora_stacker.py
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
import copy
|
||||
import sys
|
||||
import folder_paths # type: ignore
|
||||
import asyncio
|
||||
import folder_paths # pyright: ignore[reportMissingImports]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -69,6 +75,25 @@ def extract_lora_name(lora_path):
|
||||
return apply_lora_syntax_format(name_no_ext)
|
||||
|
||||
|
||||
def parse_lora_syntax(text: str) -> list[dict[str, Any]]:
|
||||
"""Parse <lora:name:strength> syntax from text input into a list of dicts.
|
||||
|
||||
Each entry contains: name, model_strength, clip_strength.
|
||||
Supports both ``<lora:name:strength>`` and ``<lora:name:model_strength:clip_strength>``.
|
||||
"""
|
||||
pattern = r"<lora:([^:>]+):([^:>]+)(?::([^:>]+))?>"
|
||||
matches = re.findall(pattern, text, re.IGNORECASE)
|
||||
loras = []
|
||||
for match in matches:
|
||||
model_strength = float(match[1])
|
||||
loras.append({
|
||||
"name": match[0],
|
||||
"model_strength": model_strength,
|
||||
"clip_strength": float(match[2]) if match[2] else model_strength,
|
||||
})
|
||||
return loras
|
||||
|
||||
|
||||
def get_loras_list(kwargs):
|
||||
"""Helper to extract loras list from either old or new kwargs format"""
|
||||
if "loras" not in kwargs:
|
||||
@@ -87,6 +112,157 @@ def get_loras_list(kwargs):
|
||||
return []
|
||||
|
||||
|
||||
_LORA_EXTENSIONS = (".safetensors", ".ckpt", ".pt", ".bin")
|
||||
|
||||
|
||||
def _strip_lora_extension(name: str) -> str:
|
||||
"""Strip a known LoRA model extension from a name (case-insensitive)."""
|
||||
lowered = name.lower()
|
||||
for ext in _LORA_EXTENSIONS:
|
||||
if lowered.endswith(ext):
|
||||
return name[: -len(ext)]
|
||||
return name
|
||||
|
||||
|
||||
def _find_missing_loras(names: list[str]) -> list[str]:
|
||||
"""Return the names that cannot be resolved to an existing local LoRA file.
|
||||
|
||||
Mirrors the matching semantics of ``get_lora_info_absolute``
|
||||
(py/utils/utils.py): after stripping the extension, a name matches a cached
|
||||
LoRA when it equals the cached file name or the ``folder/file`` path. As a
|
||||
fallback, a name containing a folder that only matches by basename resolves
|
||||
to the first basename match (same behavior as the runtime resolver). Raw
|
||||
absolute paths that exist on disk are always considered available.
|
||||
|
||||
The scanner cache is fetched once for all names; the cache may be stale, so
|
||||
resolved paths are additionally verified with ``os.path.isfile``.
|
||||
"""
|
||||
if not names:
|
||||
return []
|
||||
|
||||
async def _check() -> list[str]:
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
|
||||
scanner = await ServiceRegistry.get_lora_scanner()
|
||||
# The scanner cache may not be hydrated yet (startup, library path
|
||||
# change). An empty cache is not authoritative — treat it as "cannot
|
||||
# verify" and skip validation instead of flagging every active LoRA
|
||||
# as missing.
|
||||
if getattr(scanner, "_cache", None) is None or getattr(
|
||||
scanner, "_is_initializing", False
|
||||
):
|
||||
return []
|
||||
cache = await scanner.get_cached_data()
|
||||
|
||||
lookup = {}
|
||||
basename_candidates = {}
|
||||
for item in list(cache.raw_data):
|
||||
file_path = item.get("file_path")
|
||||
if not file_path:
|
||||
continue
|
||||
file_name = item.get("file_name", "")
|
||||
folder = item.get("folder", "")
|
||||
file_name_no_ext = _strip_lora_extension(file_name)
|
||||
path_name_no_ext = (
|
||||
f"{folder}/{file_name_no_ext}".replace("\\", "/")
|
||||
if folder
|
||||
else file_name_no_ext
|
||||
)
|
||||
lookup.setdefault(file_name_no_ext, file_path)
|
||||
lookup.setdefault(path_name_no_ext, file_path)
|
||||
basename_candidates.setdefault(file_name_no_ext, []).append(
|
||||
(folder, file_path)
|
||||
)
|
||||
|
||||
missing = []
|
||||
for name in names:
|
||||
if not name:
|
||||
continue
|
||||
normalized = name.replace("\\", "/")
|
||||
# Raw absolute paths (outside the library) are usable as-is.
|
||||
if os.path.isfile(normalized):
|
||||
continue
|
||||
no_ext = _strip_lora_extension(normalized)
|
||||
file_path = lookup.get(no_ext)
|
||||
if file_path is None and "/" in no_ext:
|
||||
# A name with a folder that matches only by basename resolves
|
||||
# at runtime like get_lora_info_absolute's fallback does:
|
||||
# prefer a candidate whose folder prefixes the name, else the
|
||||
# first basename match.
|
||||
folder, basename = no_ext.rsplit("/", 1)
|
||||
candidates = basename_candidates.get(basename, [])
|
||||
file_path = next(
|
||||
(
|
||||
fp
|
||||
for fld, fp in candidates
|
||||
if fld and no_ext.startswith(fld + "/")
|
||||
),
|
||||
None,
|
||||
)
|
||||
if file_path is None and candidates:
|
||||
file_path = candidates[0][1]
|
||||
if file_path is None or not os.path.isfile(file_path):
|
||||
missing.append(name)
|
||||
return missing
|
||||
|
||||
try:
|
||||
# Check if we're already in an event loop
|
||||
loop = asyncio.get_running_loop()
|
||||
# If we're in a running loop, run the async check in a separate thread
|
||||
import concurrent.futures
|
||||
|
||||
def run_in_thread():
|
||||
new_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(new_loop)
|
||||
try:
|
||||
return new_loop.run_until_complete(_check())
|
||||
finally:
|
||||
new_loop.close()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
future = executor.submit(run_in_thread)
|
||||
return future.result()
|
||||
except RuntimeError:
|
||||
# No event loop is running, we can use asyncio.run()
|
||||
return asyncio.run(_check())
|
||||
|
||||
|
||||
def validate_lora_entries(kwargs):
|
||||
"""Validate active LoRA widget entries against the local library.
|
||||
|
||||
Used by node ``VALIDATE_INPUTS`` implementations so ComfyUI rejects the
|
||||
prompt at queue time (``custom_validation_failed``) when an active entry
|
||||
references a LoRA that is not available locally — mirroring how built-in
|
||||
loader nodes flag missing models before execution starts.
|
||||
|
||||
Returns:
|
||||
None when every active entry resolves to an existing local file,
|
||||
otherwise a descriptive error string listing the missing LoRAs.
|
||||
Verification failures (e.g. scanner not ready) are treated as valid
|
||||
so queueing is never blocked by validation machinery itself.
|
||||
"""
|
||||
# Missing/empty loras input is always valid; skip get_loras_list so it
|
||||
# does not log a warning for the None case on every queue.
|
||||
if not kwargs.get("loras"):
|
||||
return None
|
||||
loras = get_loras_list(kwargs)
|
||||
active_names = []
|
||||
for lora in loras:
|
||||
if not isinstance(lora, dict):
|
||||
continue
|
||||
if not lora.get("active", False):
|
||||
continue
|
||||
active_names.append(apply_lora_syntax_format(str(lora.get("name") or "")))
|
||||
try:
|
||||
missing = _find_missing_loras(active_names)
|
||||
except Exception:
|
||||
logger.exception("Failed to validate LoRA entries against the local library")
|
||||
return None
|
||||
if not missing:
|
||||
return None
|
||||
return "Missing LoRA(s) in local library: " + ", ".join(missing)
|
||||
|
||||
|
||||
def load_state_dict_in_safetensors(path, device="cpu", filter_prefix=""):
|
||||
"""Simplified version of load_state_dict_in_safetensors that just loads from a local path"""
|
||||
import safetensors.torch
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import os
|
||||
from ..utils.utils import get_lora_info_absolute
|
||||
from ..config import config
|
||||
from .utils import FlexibleOptionalInputType, any_type, get_loras_list
|
||||
from .utils import FlexibleOptionalInputType, any_type, get_loras_list, validate_lora_entries
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -31,15 +31,21 @@ class WanVideoLoraSelectLM:
|
||||
"placeholder": "Search LoRAs to add...",
|
||||
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
||||
}),
|
||||
"loras": ("LORAS", {}),
|
||||
},
|
||||
"optional": FlexibleOptionalInputType(any_type),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, loras=None):
|
||||
"""Queue-time validation: reject missing local LoRAs before execution."""
|
||||
return validate_lora_entries({"loras": loras}) or True
|
||||
|
||||
RETURN_TYPES = ("WANVIDLORA", "STRING", "STRING")
|
||||
RETURN_NAMES = ("lora", "trigger_words", "active_loras")
|
||||
FUNCTION = "process_loras"
|
||||
|
||||
def process_loras(self, text, low_mem_load=False, merge_loras=True, **kwargs):
|
||||
def process_loras(self, text, loras, low_mem_load=False, merge_loras=True, **kwargs):
|
||||
loras_list = []
|
||||
all_trigger_words = []
|
||||
active_loras = []
|
||||
@@ -57,8 +63,8 @@ class WanVideoLoraSelectLM:
|
||||
selected_blocks = blocks.get("selected_blocks", {})
|
||||
layer_filter = blocks.get("layer_filter", "")
|
||||
|
||||
# Process loras from kwargs with support for both old and new formats
|
||||
loras_from_widget = get_loras_list(kwargs)
|
||||
# Process loras from the widget with support for both old and new formats
|
||||
loras_from_widget = get_loras_list({"loras": loras})
|
||||
for lora in loras_from_widget:
|
||||
if not lora.get('active', False):
|
||||
continue
|
||||
|
||||
+65
-9
@@ -1,3 +1,7 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
"""Base classes for recipe parsers."""
|
||||
|
||||
import json
|
||||
@@ -7,7 +11,7 @@ import re
|
||||
from typing import Dict, List, Any, Optional, Tuple
|
||||
from abc import ABC, abstractmethod
|
||||
from ..config import config
|
||||
from ..utils.constants import VALID_LORA_TYPES, VALID_CHECKPOINT_SUB_TYPES
|
||||
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES, VALID_LORA_TYPES, VALID_CHECKPOINT_SUB_TYPES
|
||||
from ..utils.civitai_utils import rewrite_preview_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -38,7 +42,41 @@ class RecipeMetadataParser(ABC):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
async def populate_lora_from_civitai(lora_entry: Dict[str, Any], civitai_info_tuple: Tuple[Dict[str, Any], Optional[str]],
|
||||
def populate_lora_from_local(lora_entry: Dict[str, Any], local_lora: Dict[str, Any], base_model_counts=None) -> Dict[str, Any]:
|
||||
"""Populate a recipe LoRA entry from the local scanner cache."""
|
||||
local_path = local_lora.get('file_path') or ''
|
||||
file_name = local_lora.get('file_name') or os.path.splitext(os.path.basename(local_path))[0]
|
||||
base_model = local_lora.get('base_model') or ''
|
||||
|
||||
lora_entry['name'] = local_lora.get('model_name') or file_name or lora_entry.get('name', '')
|
||||
lora_entry['file_name'] = file_name
|
||||
lora_entry['hash'] = (local_lora.get('sha256') or lora_entry.get('hash') or '').lower()
|
||||
lora_entry['localPath'] = local_path or None
|
||||
lora_entry['size'] = local_lora.get('size', 0) or 0
|
||||
lora_entry['baseModel'] = base_model
|
||||
lora_entry['existsLocally'] = True
|
||||
lora_entry['isDeleted'] = False
|
||||
|
||||
preview_url = local_lora.get('preview_url')
|
||||
if preview_url:
|
||||
lora_entry['thumbnailUrl'] = config.get_preview_static_url(preview_url)
|
||||
|
||||
civitai_info = local_lora.get('civitai') or {}
|
||||
if isinstance(civitai_info, dict):
|
||||
if civitai_info.get('id') is not None:
|
||||
lora_entry['id'] = civitai_info['id']
|
||||
if civitai_info.get('modelId') is not None:
|
||||
lora_entry['modelId'] = civitai_info['modelId']
|
||||
if civitai_info.get('name'):
|
||||
lora_entry['version'] = civitai_info['name']
|
||||
|
||||
if base_model_counts is not None and base_model:
|
||||
base_model_counts[base_model] = base_model_counts.get(base_model, 0) + 1
|
||||
|
||||
return lora_entry
|
||||
|
||||
@staticmethod
|
||||
async def populate_lora_from_civitai(lora_entry: Dict[str, Any], civitai_info_tuple: Tuple[Dict[str, Any] | None, str | None] | Dict[str, Any],
|
||||
recipe_scanner=None, base_model_counts=None, hash_value=None) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Populate a lora entry with information from Civitai API response
|
||||
@@ -151,9 +189,9 @@ class RecipeMetadataParser(ABC):
|
||||
|
||||
# Process file information if available
|
||||
if 'files' in civitai_info:
|
||||
# Find the primary model file (type="Model" and primary=true) in the files list
|
||||
# Find the primary model file (weights-type and primary=true) in the files list
|
||||
model_file = next((file for file in civitai_info.get('files', [])
|
||||
if file.get('type') == 'Model' and file.get('primary') == True), None)
|
||||
if file.get('type') in MODEL_WEIGHT_FILE_TYPES and file.get('primary') == True), None)
|
||||
|
||||
if model_file:
|
||||
# Get size
|
||||
@@ -175,10 +213,18 @@ class RecipeMetadataParser(ABC):
|
||||
lora_entry['localPath'] = local_path
|
||||
lora_entry['file_name'] = os.path.splitext(os.path.basename(local_path))[0]
|
||||
|
||||
# Get thumbnail from local preview if available
|
||||
# Get thumbnail from local preview if available.
|
||||
# Match the cache item by local path first (get_path_by_hash
|
||||
# cascade: 10-char autov2 / 12-char autov3), then by hash.
|
||||
lora_cache = await lora_scanner.get_cached_data()
|
||||
lora_item = next((item for item in lora_cache.raw_data
|
||||
if item['sha256'].lower() == lora_entry['hash'].lower()), None)
|
||||
h = (lora_entry.get("hash") or "").lower()
|
||||
lora_item = next((item for item in lora_cache.raw_data
|
||||
if (item.get("file_path") or "") == local_path), None)
|
||||
if lora_item is None:
|
||||
lora_item = next((item for item in lora_cache.raw_data
|
||||
if (item.get("sha256") or "").lower() == h
|
||||
or (item.get("autov3") or "").lower() == h
|
||||
or (item.get("sha256") or "")[:10].lower() == h), None)
|
||||
if lora_item and 'preview_url' in lora_item:
|
||||
lora_entry['thumbnailUrl'] = config.get_preview_static_url(lora_item['preview_url'])
|
||||
except Exception as e:
|
||||
@@ -194,7 +240,7 @@ class RecipeMetadataParser(ABC):
|
||||
return lora_entry
|
||||
|
||||
@staticmethod
|
||||
async def populate_checkpoint_from_civitai(checkpoint: Dict[str, Any], civitai_info: Dict[str, Any]) -> Dict[str, Any]:
|
||||
async def populate_checkpoint_from_civitai(checkpoint: Dict[str, Any], civitai_info: Dict[str, Any] | Tuple[Dict[str, Any] | None, str | None] | None) -> Dict[str, Any]:
|
||||
"""
|
||||
Populate checkpoint information from Civitai API response
|
||||
|
||||
@@ -249,11 +295,21 @@ class RecipeMetadataParser(ABC):
|
||||
checkpoint['id'] = civitai_data.get('id', 0)
|
||||
|
||||
if 'files' in civitai_data:
|
||||
# Prefer the file CivitAI marked primary; fall back to any
|
||||
# weights-type file (providers without primary flags).
|
||||
model_file = next(
|
||||
(
|
||||
file
|
||||
for file in civitai_data.get('files', [])
|
||||
if file.get('type') == 'Model'
|
||||
if file.get('type') in MODEL_WEIGHT_FILE_TYPES
|
||||
and file.get('primary') is True
|
||||
),
|
||||
None,
|
||||
) or next(
|
||||
(
|
||||
file
|
||||
for file in civitai_data.get('files', [])
|
||||
if file.get('type') in MODEL_WEIGHT_FILE_TYPES
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
import logging
|
||||
import json
|
||||
import os
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Factory for creating recipe metadata parsers."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
from .parsers import (
|
||||
RecipeFormatParser,
|
||||
ComfyMetadataParser,
|
||||
@@ -31,7 +32,8 @@ class RecipeParserFactory:
|
||||
# First, try CivitaiApiMetadataParser for dict input
|
||||
if isinstance(metadata, dict):
|
||||
try:
|
||||
if CivitaiApiMetadataParser().is_metadata_matching(metadata):
|
||||
user_comment: Any = metadata
|
||||
if CivitaiApiMetadataParser().is_metadata_matching(user_comment):
|
||||
return CivitaiApiMetadataParser()
|
||||
except Exception as e:
|
||||
logger.debug(f"CivitaiApiMetadataParser check failed: {e}")
|
||||
|
||||
+245
-69
@@ -8,6 +8,7 @@ from typing import Dict, Any
|
||||
from ..base import RecipeMetadataParser
|
||||
from ..constants import GEN_PARAM_KEYS
|
||||
from ...services.metadata_service import get_default_metadata_provider
|
||||
from ...utils.constants import is_empty_placeholder_hash
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -52,7 +53,7 @@ class AutomaticMetadataParser(RecipeMetadataParser):
|
||||
negative_and_params = ""
|
||||
|
||||
# Initialize metadata
|
||||
metadata = {
|
||||
metadata: Dict[str, Any] = {
|
||||
"prompt": prompt,
|
||||
"loras": []
|
||||
}
|
||||
@@ -123,24 +124,37 @@ class AutomaticMetadataParser(RecipeMetadataParser):
|
||||
if model_hash_from_hashes:
|
||||
metadata["model_hash"] = model_hash_from_hashes
|
||||
|
||||
# Extract Lora hashes in alternative format
|
||||
# Extract Lora hashes in alternative format.
|
||||
# Run unconditionally (not just as fallback) so that
|
||||
# non-empty hashes from Lora hashes fill in the gaps left
|
||||
# by empty values in the Hashes JSON dict. Some WebUI
|
||||
# builds write real hash values only to Lora hashes and
|
||||
# leave the Hashes JSON values empty.
|
||||
lora_hashes_match = re.search(self.LORA_HASHES_REGEX, params_section)
|
||||
if not hashes_match and lora_hashes_match:
|
||||
if lora_hashes_match:
|
||||
try:
|
||||
lora_hashes_str = lora_hashes_match.group(1)
|
||||
lora_hash_entries = lora_hashes_str.split(', ')
|
||||
|
||||
# Initialize hashes dict if it doesn't exist
|
||||
if "hashes" not in metadata:
|
||||
metadata["hashes"] = {}
|
||||
|
||||
|
||||
# Parse each lora hash entry (format: "name: hash")
|
||||
for entry in lora_hash_entries:
|
||||
if ': ' in entry:
|
||||
lora_name, lora_hash = entry.split(': ', 1)
|
||||
# Add as lora type in the same format as regular hashes
|
||||
metadata["hashes"][f"lora:{lora_name}"] = lora_hash.strip()
|
||||
|
||||
lora_hash = lora_hash.strip()
|
||||
if not lora_hash:
|
||||
# Skip entries without a hash value
|
||||
continue
|
||||
# Initialize hashes dict if it doesn't exist
|
||||
if "hashes" not in metadata:
|
||||
metadata["hashes"] = {}
|
||||
# Lora hashes carries the 12-char AutoV3
|
||||
# hash (resolvable on CivitAI and the local
|
||||
# autov3 index); the Hashes JSON value is
|
||||
# only the 10-char AutoV2 prefix, so on
|
||||
# conflict the Lora hashes value wins.
|
||||
key = f"lora:{lora_name}"
|
||||
metadata["hashes"][key] = lora_hash
|
||||
|
||||
# Remove lora hashes from params section
|
||||
params_section = params_section.replace(lora_hashes_match.group(0), '')
|
||||
except Exception as e:
|
||||
@@ -347,66 +361,228 @@ class AutomaticMetadataParser(RecipeMetadataParser):
|
||||
|
||||
checkpoint = checkpoint_entry
|
||||
|
||||
# If no LoRAs from Civitai resources or to supplement, extract from metadata["hashes"]
|
||||
if not loras or len(loras) == 0:
|
||||
# Extract lora weights from extranet tags in prompt (for later use)
|
||||
lora_weights = {}
|
||||
lora_matches = re.findall(self.EXTRANETS_REGEX, prompt)
|
||||
for lora_type, lora_name, lora_weight in lora_matches:
|
||||
key = f"{lora_type}:{lora_name}"
|
||||
lora_weights[key] = round(float(lora_weight), 2)
|
||||
|
||||
# Use hashes from metadata as the primary source
|
||||
if metadata.get("hashes"):
|
||||
for hash_key, lora_hash in metadata.get("hashes", {}).items():
|
||||
# Only process lora or hypernet types
|
||||
if not hash_key.startswith(("lora:", "hypernet:")):
|
||||
def normalize_lora_name(name, basename=False):
|
||||
normalized = str(name or '').replace('\\', '/')
|
||||
if normalized.casefold().endswith('.safetensors'):
|
||||
normalized = normalized[:-12]
|
||||
if basename:
|
||||
normalized = normalized.rsplit('/', 1)[-1]
|
||||
return normalized.casefold()
|
||||
|
||||
def get_version_id(lora):
|
||||
version_id = lora.get('id')
|
||||
if version_id in (None, '', 0, '0'):
|
||||
version_id = lora.get('modelVersionId')
|
||||
if version_id in (None, '', 0, '0'):
|
||||
return None
|
||||
return str(version_id)
|
||||
|
||||
prompt_loras = {}
|
||||
for match in re.findall(self.EXTRANETS_REGEX, prompt):
|
||||
lora_type, lora_name, _ = match
|
||||
prompt_loras[(lora_type, normalize_lora_name(lora_name))] = match
|
||||
|
||||
prompt_by_basename = {}
|
||||
for lora_type, lora_name, lora_weight in prompt_loras.values():
|
||||
key = (lora_type, normalize_lora_name(lora_name, True))
|
||||
prompt_by_basename.setdefault(key, []).append((lora_name, round(float(lora_weight), 2)))
|
||||
|
||||
hash_basenames = {
|
||||
(hash_key.split(':', 1)[0], normalize_lora_name(hash_key.split(':', 1)[1], True))
|
||||
for hash_key, hash_value in metadata.get("hashes", {}).items()
|
||||
if hash_value and hash_key.startswith(("lora:", "hypernet:"))
|
||||
}
|
||||
recipe_base_model = checkpoint.get("baseModel") if checkpoint else None
|
||||
if not recipe_base_model and len(base_model_counts) == 1:
|
||||
recipe_base_model = next(iter(base_model_counts))
|
||||
|
||||
resource_lora_count = len(loras)
|
||||
|
||||
def make_lora_entry(lora_type, lora_name, weight, lora_hash=''):
|
||||
return {
|
||||
'name': lora_name,
|
||||
'type': lora_type,
|
||||
'weight': weight,
|
||||
'hash': lora_hash,
|
||||
'existsLocally': False,
|
||||
'localPath': None,
|
||||
'file_name': lora_name,
|
||||
'thumbnailUrl': '/loras_static/images/no-preview.png',
|
||||
'baseModel': '',
|
||||
'size': 0,
|
||||
'downloadUrl': '',
|
||||
'isDeleted': False
|
||||
}
|
||||
|
||||
def merge_or_append_civitai(civitai_entry, preserve_existing_weight=False):
|
||||
civitai_id = get_version_id(civitai_entry)
|
||||
civitai_hash = (civitai_entry.get('hash') or '').lower()
|
||||
for index, existing in enumerate(loras):
|
||||
existing_id = get_version_id(existing)
|
||||
existing_hash = (existing.get('hash') or '').lower()
|
||||
if not (
|
||||
(civitai_id and existing_id == civitai_id)
|
||||
or (civitai_hash and existing_hash == civitai_hash)
|
||||
):
|
||||
continue
|
||||
|
||||
if preserve_existing_weight:
|
||||
civitai_entry['weight'] = existing.get('weight', civitai_entry['weight'])
|
||||
existing_base = existing.get('baseModel')
|
||||
if not civitai_entry.get('baseModel'):
|
||||
civitai_entry['baseModel'] = existing_base or ''
|
||||
elif existing_base:
|
||||
remaining = base_model_counts.get(existing_base, 0) - 1
|
||||
if remaining > 0:
|
||||
base_model_counts[existing_base] = remaining
|
||||
else:
|
||||
base_model_counts.pop(existing_base, None)
|
||||
loras[index] = civitai_entry
|
||||
return
|
||||
loras.append(civitai_entry)
|
||||
|
||||
def merge_or_append_local(local_entry):
|
||||
local_id = get_version_id(local_entry)
|
||||
local_hash = (local_entry.get('hash') or '').lower()
|
||||
for existing in loras:
|
||||
existing_id = get_version_id(existing)
|
||||
existing_hash = (existing.get('hash') or '').lower()
|
||||
if not (
|
||||
(local_id and existing_id == local_id)
|
||||
or (local_hash and existing_hash == local_hash)
|
||||
):
|
||||
continue
|
||||
|
||||
existing['weight'] = local_entry['weight']
|
||||
existing['hash'] = local_entry['hash']
|
||||
existing['file_name'] = local_entry['file_name']
|
||||
existing['existsLocally'] = True
|
||||
existing['localPath'] = local_entry['localPath']
|
||||
existing['size'] = local_entry['size']
|
||||
existing['isDeleted'] = False
|
||||
if not existing.get('modelId') and local_entry.get('modelId'):
|
||||
existing['modelId'] = local_entry['modelId']
|
||||
if not existing.get('baseModel') and local_entry.get('baseModel'):
|
||||
existing['baseModel'] = local_entry['baseModel']
|
||||
base_model_counts[local_entry['baseModel']] = base_model_counts.get(local_entry['baseModel'], 0) + 1
|
||||
thumbnail_url = local_entry.get('thumbnailUrl')
|
||||
if thumbnail_url and not thumbnail_url.endswith('/images/no-preview.png'):
|
||||
existing['thumbnailUrl'] = thumbnail_url
|
||||
return
|
||||
|
||||
if local_entry.get('baseModel'):
|
||||
base_model = local_entry['baseModel']
|
||||
base_model_counts[base_model] = base_model_counts.get(base_model, 0) + 1
|
||||
loras.append(local_entry)
|
||||
|
||||
resolved_prompt_basenames = set()
|
||||
queried_local_basenames = set()
|
||||
for lora_type, lora_name, lora_weight in prompt_loras.values():
|
||||
weight = round(float(lora_weight), 2)
|
||||
basename_key = (lora_type, normalize_lora_name(lora_name, True))
|
||||
matching_resources = [
|
||||
lora
|
||||
for lora in loras[:resource_lora_count]
|
||||
if lora.get('file_name')
|
||||
and normalize_lora_name(lora['file_name'], True) == basename_key[1]
|
||||
and (
|
||||
(lora_type == 'hypernet' and str(lora.get('type', '')).casefold() in ('hypernet', 'hypernetwork'))
|
||||
or (lora_type == 'lora' and str(lora.get('type', '')).casefold() not in ('hypernet', 'hypernetwork'))
|
||||
)
|
||||
]
|
||||
if len(prompt_by_basename[basename_key]) == 1 and len(matching_resources) == 1:
|
||||
matching_resources[0]['weight'] = weight
|
||||
if basename_key not in hash_basenames:
|
||||
resolved_prompt_basenames.add(basename_key)
|
||||
continue
|
||||
|
||||
if basename_key in hash_basenames:
|
||||
continue
|
||||
|
||||
if not recipe_scanner or lora_type != 'lora':
|
||||
continue
|
||||
queried_local_basenames.add(basename_key)
|
||||
local_lora = await recipe_scanner.get_local_lora(lora_name, recipe_base_model)
|
||||
if not local_lora:
|
||||
continue
|
||||
|
||||
local_entry = self.populate_lora_from_local(
|
||||
make_lora_entry(lora_type, lora_name, weight),
|
||||
local_lora,
|
||||
)
|
||||
merge_or_append_local(local_entry)
|
||||
resolved_prompt_basenames.add(basename_key)
|
||||
|
||||
for hash_key, lora_hash in metadata.get("hashes", {}).items():
|
||||
if not hash_key.startswith(("lora:", "hypernet:")):
|
||||
continue
|
||||
lora_type, lora_name = hash_key.split(':', 1)
|
||||
basename_key = (lora_type, normalize_lora_name(lora_name, True))
|
||||
if basename_key in resolved_prompt_basenames:
|
||||
continue
|
||||
|
||||
prompt_entries = prompt_by_basename.get(basename_key, [])
|
||||
weight = prompt_entries[0][1] if len(prompt_entries) == 1 else 1.0
|
||||
lora_entry = make_lora_entry(lora_type, lora_name, weight, lora_hash)
|
||||
|
||||
if is_empty_placeholder_hash(lora_hash):
|
||||
# The empty-hash placeholder (SHA256 of an empty byte
|
||||
# string) is not a real hash: never look it up in the
|
||||
# local hash index or on CivitAI. Match by filename;
|
||||
# otherwise keep the item as unresolved (no hash, flagged
|
||||
# hashInvalid so the UI shows the unresolvable-hash state
|
||||
# and offers reconnect instead of download) rather than
|
||||
# dropping it.
|
||||
if recipe_scanner and lora_type == 'lora' and basename_key not in queried_local_basenames:
|
||||
local_lora = await recipe_scanner.get_local_lora(lora_name, recipe_base_model)
|
||||
if local_lora:
|
||||
local_entry = self.populate_lora_from_local(lora_entry, local_lora)
|
||||
merge_or_append_local(local_entry)
|
||||
continue
|
||||
|
||||
lora_type, lora_name = hash_key.split(':', 1)
|
||||
|
||||
# Get weight from extranet tags if available, else default to 1.0
|
||||
weight = lora_weights.get(hash_key, 1.0)
|
||||
|
||||
# Initialize lora entry
|
||||
lora_entry = {
|
||||
'name': lora_name,
|
||||
'type': lora_type, # 'lora' or 'hypernet'
|
||||
'weight': weight,
|
||||
'hash': lora_hash,
|
||||
'existsLocally': False,
|
||||
'localPath': None,
|
||||
'file_name': lora_name,
|
||||
'thumbnailUrl': '/loras_static/images/no-preview.png',
|
||||
'baseModel': '',
|
||||
'size': 0,
|
||||
'downloadUrl': '',
|
||||
'isDeleted': False
|
||||
}
|
||||
|
||||
# Try to get info from Civitai
|
||||
if metadata_provider:
|
||||
try:
|
||||
if lora_hash:
|
||||
# If we have hash, use it for lookup
|
||||
civitai_info = await metadata_provider.get_model_by_hash(lora_hash)
|
||||
else:
|
||||
civitai_info = None
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash
|
||||
)
|
||||
if populated_entry is None:
|
||||
continue # Skip invalid LoRA types
|
||||
lora_entry = populated_entry
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching Civitai info for LoRA {lora_name}: {e}")
|
||||
|
||||
lora_entry['hash'] = ''
|
||||
lora_entry['hashInvalid'] = True
|
||||
if not resource_lora_count:
|
||||
loras.append(lora_entry)
|
||||
continue
|
||||
|
||||
if lora_hash and recipe_scanner and lora_type == 'lora':
|
||||
local_lora = await recipe_scanner.get_local_lora_by_hash(lora_hash)
|
||||
if local_lora:
|
||||
local_entry = self.populate_lora_from_local(lora_entry, local_lora)
|
||||
merge_or_append_local(local_entry)
|
||||
continue
|
||||
|
||||
hash_resolved = False
|
||||
if lora_hash and metadata_provider:
|
||||
try:
|
||||
civitai_info = await metadata_provider.get_model_by_hash(lora_hash)
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
if populated_entry is None:
|
||||
continue
|
||||
lora_entry = populated_entry
|
||||
hash_resolved = not lora_entry.get('isDeleted')
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching Civitai info for LoRA {lora_name}: {e}")
|
||||
|
||||
if hash_resolved:
|
||||
merge_or_append_civitai(lora_entry, preserve_existing_weight=not prompt_entries)
|
||||
continue
|
||||
|
||||
if recipe_scanner and lora_type == 'lora' and basename_key not in queried_local_basenames:
|
||||
local_lora = await recipe_scanner.get_local_lora(lora_name, recipe_base_model)
|
||||
if local_lora:
|
||||
local_entry = self.populate_lora_from_local(lora_entry, local_lora)
|
||||
merge_or_append_local(local_entry)
|
||||
continue
|
||||
|
||||
if lora_hash and not resource_lora_count:
|
||||
loras.append(lora_entry)
|
||||
|
||||
# Try to get base model from resources or make educated guess
|
||||
base_model = None
|
||||
|
||||
@@ -4,7 +4,7 @@ import json
|
||||
import logging
|
||||
from typing import Dict, Any, Union
|
||||
from ..base import RecipeMetadataParser
|
||||
from ..constants import GEN_PARAM_KEYS
|
||||
from ..constants import GEN_PARAM_KEYS, VALID_LORA_TYPES
|
||||
from ...services.metadata_service import get_default_metadata_provider
|
||||
from ...config import config
|
||||
|
||||
@@ -14,15 +14,16 @@ logger = logging.getLogger(__name__)
|
||||
class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
"""Parser for Civitai image metadata format"""
|
||||
|
||||
def is_metadata_matching(self, metadata) -> bool:
|
||||
def is_metadata_matching(self, user_comment) -> bool:
|
||||
"""Check if the metadata matches the Civitai image metadata format
|
||||
|
||||
Args:
|
||||
metadata: The metadata from the image (dict)
|
||||
user_comment: The metadata from the image (dict)
|
||||
|
||||
Returns:
|
||||
bool: True if this parser can handle the metadata
|
||||
"""
|
||||
metadata = user_comment
|
||||
if not metadata or not isinstance(metadata, dict):
|
||||
return False
|
||||
|
||||
@@ -73,7 +74,7 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
|
||||
return False
|
||||
|
||||
async def parse_metadata( # type: ignore[override]
|
||||
async def parse_metadata( # pyright: ignore[reportIncompatibleMethodOverride]
|
||||
self, user_comment, recipe_scanner=None, civitai_client=None,
|
||||
local_cache: dict[str, Any] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
@@ -89,8 +90,7 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
Returns:
|
||||
Dict containing parsed recipe data
|
||||
"""
|
||||
metadata: Dict[str, Any] = user_comment # type: ignore[assignment]
|
||||
metadata = user_comment
|
||||
metadata: Dict[str, Any] = user_comment
|
||||
try:
|
||||
# Get metadata provider instead of using civitai_client directly
|
||||
metadata_provider = await get_default_metadata_provider()
|
||||
@@ -115,8 +115,29 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
):
|
||||
metadata = inner_meta
|
||||
|
||||
# Civitai's image API meta parser mangles the A1111 "Lora hashes"
|
||||
# text field into a quote-wrapped dict entry:
|
||||
# '"Daphne Blake Cosplay_v1": "e67ebd5e315f"'
|
||||
# The 12-char AutoV3 it carries is more reliable than the stale
|
||||
# 10-char AutoV2 value in the "hashes" dict, so recover it and
|
||||
# let it override the conflicting entry.
|
||||
if isinstance(metadata, dict):
|
||||
for key, hash_value in list(metadata.items()):
|
||||
if (
|
||||
isinstance(key, str)
|
||||
and key.startswith('"')
|
||||
and isinstance(hash_value, str)
|
||||
and hash_value.endswith('"')
|
||||
):
|
||||
clean_name = key.strip('"').strip()
|
||||
clean_hash = hash_value.strip('"').strip()
|
||||
if clean_name and clean_hash:
|
||||
hashes_dict = metadata.get("hashes")
|
||||
if isinstance(hashes_dict, dict):
|
||||
hashes_dict[f"lora:{clean_name}"] = clean_hash
|
||||
|
||||
# Initialize result structure
|
||||
result = {
|
||||
result: Dict[str, Any] = {
|
||||
"base_model": None,
|
||||
"loras": [],
|
||||
"model": None,
|
||||
@@ -125,10 +146,10 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
}
|
||||
|
||||
# Track already added LoRAs to prevent duplicates
|
||||
added_loras = {} # key: model_version_id or hash, value: index in result["loras"]
|
||||
added_loras: Dict[str, Any] = {} # key: model_version_id or hash, value: index in result["loras"]
|
||||
|
||||
# Extract hash information from hashes field for LoRA matching
|
||||
lora_hashes = {}
|
||||
lora_hashes: Dict[str, Any] = {}
|
||||
if "hashes" in metadata and isinstance(metadata["hashes"], dict):
|
||||
for key, hash_value in metadata["hashes"].items():
|
||||
key_str = str(key)
|
||||
@@ -184,7 +205,7 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
if model_info:
|
||||
result["base_model"] = model_info.get("baseModel", "")
|
||||
|
||||
base_model_counts = {}
|
||||
base_model_counts: Dict[str, int] = {}
|
||||
|
||||
# Process standard resources array
|
||||
if "resources" in metadata and isinstance(metadata["resources"], list):
|
||||
@@ -196,7 +217,7 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
# identification because it has an explicit type field and hash,
|
||||
# unlike modelVersionIds which is a flat list with no type info.
|
||||
if resource_type == "model":
|
||||
checkpoint_entry = {
|
||||
checkpoint_entry: Dict[str, Any] = {
|
||||
"id": 0,
|
||||
"modelId": 0,
|
||||
"name": resource.get("name", "Unknown Model"),
|
||||
@@ -216,7 +237,8 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
# Try to look up base model from the checkpoint hash
|
||||
cp_hash = checkpoint_entry.get("hash")
|
||||
if cp_hash and metadata_provider:
|
||||
local_cached = local_cache.get(cp_hash) if local_cache else None
|
||||
# local_cache keys are stored lowercase
|
||||
local_cached = local_cache.get(cp_hash.lower()) if local_cache else None
|
||||
if local_cached:
|
||||
self._populate_entry_from_cache(
|
||||
checkpoint_entry, local_cached
|
||||
@@ -294,8 +316,15 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
|
||||
# Try to get info from Civitai if hash is available
|
||||
if lora_hash and metadata_provider:
|
||||
local_cached = local_cache.get(lora_hash) if local_cache else None
|
||||
# local_cache keys are stored lowercase
|
||||
local_cached = local_cache.get(lora_hash.lower()) if local_cache else None
|
||||
if local_cached:
|
||||
cached_type = self._cache_item_model_type(local_cached)
|
||||
if cached_type and cached_type not in VALID_LORA_TYPES:
|
||||
logger.debug(
|
||||
f"Skipping non-LoRA cache item for hash {lora_hash}"
|
||||
)
|
||||
continue
|
||||
self._populate_entry_from_cache(
|
||||
lora_entry, local_cached
|
||||
)
|
||||
@@ -304,6 +333,12 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
added_loras[str(lora_entry["id"])] = len(
|
||||
result["loras"]
|
||||
)
|
||||
# Mirror base.py:150-151 counts for API-path loras
|
||||
bm = local_cached.get("base_model") or ""
|
||||
if bm:
|
||||
base_model_counts[bm] = base_model_counts.get(
|
||||
bm, 0
|
||||
) + 1
|
||||
else:
|
||||
try:
|
||||
civitai_info = (
|
||||
@@ -514,11 +549,21 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
|
||||
result["loras"].append(lora_entry)
|
||||
|
||||
# Process modelVersionIds from Civitai image API
|
||||
# These are model version IDs returned at root level when meta doesn't contain resources
|
||||
if "modelVersionIds" in metadata and isinstance(
|
||||
metadata["modelVersionIds"], list
|
||||
# Process modelVersionIds from Civitai image API.
|
||||
# These are version IDs returned at root level of the API response.
|
||||
# When resources or civitaiResources are already present in metadata
|
||||
# (which they are when ?withMeta=true is passed), those sections have
|
||||
# complete hash/type information — modelVersionIds is a fallback for
|
||||
# when meta is null and only the flat ID list is available. Skipping
|
||||
# it here avoids duplicates: the same file hash often resolves to
|
||||
# different version IDs via hash lookup (resources) vs the original
|
||||
# version ID in modelVersionIds, and both paths would create entries.
|
||||
if (
|
||||
"modelVersionIds" in metadata
|
||||
and isinstance(metadata["modelVersionIds"], list)
|
||||
and not result.get("loras")
|
||||
):
|
||||
|
||||
for version_id in metadata["modelVersionIds"]:
|
||||
version_id_str = str(version_id)
|
||||
|
||||
@@ -526,6 +571,13 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
if version_id_str in added_loras:
|
||||
continue
|
||||
|
||||
# Skip if this version ID is already the recipe's checkpoint
|
||||
# (resolved earlier from embedded resources/Model hash,
|
||||
# avoiding a duplicate CivitAI API call).
|
||||
existing_model = result.get("model")
|
||||
if existing_model and str(existing_model.get("id")) == version_id_str:
|
||||
continue
|
||||
|
||||
# Initialize lora entry with version ID
|
||||
lora_entry = {
|
||||
"id": version_id,
|
||||
@@ -559,9 +611,40 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
continue # Skip invalid LoRA types
|
||||
# Not a LoRA — try as checkpoint (only if we
|
||||
# don't already have one). Reuses the same
|
||||
# civitai_info from the API call above so no
|
||||
# extra query is made.
|
||||
if result["model"] is None:
|
||||
checkpoint_entry = {
|
||||
"id": version_id,
|
||||
"modelId": 0,
|
||||
"name": "Unknown Model",
|
||||
"version": "",
|
||||
"type": "checkpoint",
|
||||
"existsLocally": False,
|
||||
"localPath": None,
|
||||
"file_name": "",
|
||||
"hash": "",
|
||||
"thumbnailUrl": (
|
||||
"/loras_static/images/no-preview.png"
|
||||
),
|
||||
"baseModel": "",
|
||||
"size": 0,
|
||||
"downloadUrl": "",
|
||||
"isDeleted": False,
|
||||
}
|
||||
cp_populated = await (
|
||||
self.populate_checkpoint_from_civitai(
|
||||
checkpoint_entry, civitai_info
|
||||
)
|
||||
)
|
||||
if cp_populated.get("modelId"):
|
||||
result["model"] = cp_populated
|
||||
continue # Not a LoRA, don't add to loras
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for model version {version_id}: {e}"
|
||||
@@ -601,30 +684,47 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
}
|
||||
|
||||
if metadata_provider:
|
||||
try:
|
||||
civitai_info = await metadata_provider.get_model_by_hash(
|
||||
lora_hash
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
# local_cache keys are stored lowercase
|
||||
local_cached = local_cache.get(lora_hash.lower()) if local_cache else None
|
||||
if local_cached:
|
||||
cached_type = self._cache_item_model_type(local_cached)
|
||||
if cached_type and cached_type not in VALID_LORA_TYPES:
|
||||
logger.debug(
|
||||
f"Skipping non-LoRA cache item for hash {lora_hash}"
|
||||
)
|
||||
continue
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
self._populate_entry_from_cache(lora_entry, local_cached)
|
||||
# Mirror base.py:150-151 counts for API-path loras
|
||||
bm = local_cached.get("base_model") or ""
|
||||
if bm:
|
||||
base_model_counts[bm] = base_model_counts.get(bm, 0) + 1
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_hash}: {e}"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
civitai_info = await metadata_provider.get_model_by_hash(
|
||||
lora_hash
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
continue
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_hash}: {e}"
|
||||
)
|
||||
|
||||
added_loras[lora_hash] = len(result["loras"])
|
||||
result["loras"].append(lora_entry)
|
||||
@@ -663,32 +763,51 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
|
||||
# Try to get info from Civitai if hash is available
|
||||
if lora_entry["hash"] and metadata_provider:
|
||||
try:
|
||||
civitai_info = await metadata_provider.get_model_by_hash(
|
||||
lora_hash
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
# local_cache keys are stored lowercase
|
||||
local_cached = local_cache.get(lora_hash.lower()) if local_cache else None
|
||||
if local_cached:
|
||||
cached_type = self._cache_item_model_type(local_cached)
|
||||
if cached_type and cached_type not in VALID_LORA_TYPES:
|
||||
logger.debug(
|
||||
f"Skipping non-LoRA cache item for hash {lora_hash}"
|
||||
)
|
||||
lora_index += 1
|
||||
continue # Skip invalid LoRA types
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
continue # Skip non-LoRA cache items
|
||||
self._populate_entry_from_cache(lora_entry, local_cached)
|
||||
# Mirror base.py:150-151 counts for API-path loras
|
||||
bm = local_cached.get("base_model") or ""
|
||||
if bm:
|
||||
base_model_counts[bm] = base_model_counts.get(bm, 0) + 1
|
||||
# If we have a version ID from Civitai, track it for deduplication
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_entry['hash']}: {e}"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
civitai_info = await metadata_provider.get_model_by_hash(
|
||||
lora_hash
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
lora_index += 1
|
||||
continue # Skip invalid LoRA types
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
# If we have a version ID from Civitai, track it for deduplication
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_entry['hash']}: {e}"
|
||||
)
|
||||
|
||||
# Track by hash if we have it
|
||||
if lora_hash:
|
||||
@@ -747,3 +866,14 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
base_model = cache_item.get("base_model", "")
|
||||
if base_model:
|
||||
entry["baseModel"] = base_model
|
||||
|
||||
@staticmethod
|
||||
def _cache_item_model_type(cache_item: dict[str, Any]) -> str:
|
||||
"""Lowercased civitai.model.type of a cache item, or '' when unknown."""
|
||||
civ = cache_item.get("civitai")
|
||||
if not isinstance(civ, dict):
|
||||
return ""
|
||||
model_info = civ.get("model")
|
||||
if not isinstance(model_info, dict):
|
||||
return ""
|
||||
return (model_info.get("type") or "").lower()
|
||||
|
||||
+112
-75
@@ -31,41 +31,106 @@ class ComfyMetadataParser(RecipeMetadataParser):
|
||||
metadata_provider = await get_default_metadata_provider()
|
||||
|
||||
data = json.loads(user_comment)
|
||||
|
||||
checkpoint_nodes = {k: v for k, v in data.items() if isinstance(v, dict) and v.get('class_type') == 'CheckpointLoaderSimple'}
|
||||
checkpoint = None
|
||||
checkpoint_id = None
|
||||
checkpoint_version_id = None
|
||||
if checkpoint_nodes:
|
||||
checkpoint_node = next(iter(checkpoint_nodes.values()))
|
||||
if 'inputs' in checkpoint_node and 'ckpt_name' in checkpoint_node['inputs']:
|
||||
checkpoint_name = checkpoint_node['inputs']['ckpt_name']
|
||||
# Some ComfyUI workflows serialize ckpt_name as a
|
||||
# single-element list (e.g. ["model.safetensors"]) or leave
|
||||
# the value unset (None). Neither is a string, so skip the
|
||||
# CivitAI-URN lookup instead of crashing re.search with a
|
||||
# TypeError that fails the whole image import.
|
||||
if isinstance(checkpoint_name, list):
|
||||
checkpoint_name = (
|
||||
checkpoint_name[0] if checkpoint_name else None
|
||||
)
|
||||
if isinstance(checkpoint_name, str):
|
||||
checkpoint_match = re.search(r'civitai:(\d+)@(\d+)', checkpoint_name)
|
||||
if checkpoint_match:
|
||||
checkpoint_id = checkpoint_match.group(1)
|
||||
checkpoint_version_id = checkpoint_match.group(2)
|
||||
checkpoint = {
|
||||
'id': checkpoint_version_id,
|
||||
'modelId': checkpoint_id,
|
||||
'name': f"Checkpoint {checkpoint_id}",
|
||||
'version': '',
|
||||
'type': 'checkpoint'
|
||||
}
|
||||
if metadata_provider:
|
||||
try:
|
||||
civitai_info_tuple = await metadata_provider.get_model_version_info(checkpoint_version_id)
|
||||
civitai_info, _ = civitai_info_tuple if isinstance(civitai_info_tuple, tuple) else (civitai_info_tuple, None)
|
||||
checkpoint = await self.populate_checkpoint_from_civitai(checkpoint, civitai_info)
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching Civitai info for checkpoint: {e}")
|
||||
|
||||
recipe_base_model = checkpoint.get('baseModel') if checkpoint else None
|
||||
loras = []
|
||||
|
||||
# Find all LoraLoader nodes
|
||||
lora_nodes = {k: v for k, v in data.items() if isinstance(v, dict) and v.get('class_type') == 'LoraLoader'}
|
||||
|
||||
# Process each LoraLoader node
|
||||
for node_id, node in lora_nodes.items():
|
||||
if 'inputs' not in node or 'lora_name' not in node['inputs']:
|
||||
lora_candidates = []
|
||||
for node in data.values():
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
|
||||
lora_name = node['inputs'].get('lora_name', '')
|
||||
|
||||
# Parse the URN to extract model ID and version ID
|
||||
# Format: "urn:air:sdxl:lora:civitai:1107767@1253442"
|
||||
|
||||
inputs = node.get('inputs')
|
||||
if not isinstance(inputs, dict):
|
||||
continue
|
||||
|
||||
if node.get('class_type') == 'LoraLoader':
|
||||
lora_name = inputs.get('lora_name', '')
|
||||
if isinstance(lora_name, str) and lora_name:
|
||||
lora_candidates.append((lora_name, inputs.get('strength_model', 1.0)))
|
||||
continue
|
||||
|
||||
if node.get('class_type') != 'LoraLoaderLM':
|
||||
continue
|
||||
|
||||
loras_data = inputs.get('loras', [])
|
||||
if isinstance(loras_data, dict):
|
||||
loras_data = loras_data.get('__value__', [])
|
||||
if isinstance(loras_data, list) and len(loras_data) == 1 and isinstance(loras_data[0], list):
|
||||
loras_data = loras_data[0]
|
||||
if not isinstance(loras_data, list):
|
||||
continue
|
||||
|
||||
for lora in loras_data:
|
||||
if not isinstance(lora, dict) or not lora.get('active', False) or lora.get('_isDummy', False):
|
||||
continue
|
||||
lora_name = lora.get('name', '')
|
||||
if isinstance(lora_name, str) and lora_name:
|
||||
lora_candidates.append((lora_name, lora.get('strength', 1.0)))
|
||||
|
||||
for lora_name, weight in lora_candidates:
|
||||
if isinstance(weight, str):
|
||||
try:
|
||||
weight = float(weight)
|
||||
except ValueError:
|
||||
weight = 1.0
|
||||
lora_id_match = re.search(r'civitai:(\d+)@(\d+)', lora_name)
|
||||
if not lora_id_match:
|
||||
continue
|
||||
|
||||
model_id = lora_id_match.group(1)
|
||||
model_version_id = lora_id_match.group(2)
|
||||
|
||||
# Get strength from node inputs
|
||||
weight = node['inputs'].get('strength_model', 1.0)
|
||||
|
||||
# Initialize lora entry with default values
|
||||
if lora_id_match:
|
||||
model_id = lora_id_match.group(1)
|
||||
model_version_id = lora_id_match.group(2)
|
||||
entry_name = f"Lora {model_id}"
|
||||
else:
|
||||
model_id = 0
|
||||
model_version_id = 0
|
||||
entry_name = re.split(r'[\\/]', lora_name)[-1]
|
||||
entry_name = re.sub(r'\.[^.]+$', '', entry_name)
|
||||
|
||||
lora_entry = {
|
||||
'id': model_version_id,
|
||||
'modelId': model_id,
|
||||
'name': f"Lora {model_id}", # Default name
|
||||
'name': entry_name,
|
||||
'version': '',
|
||||
'type': 'lora',
|
||||
'weight': weight,
|
||||
'existsLocally': False,
|
||||
'localPath': None,
|
||||
'file_name': '',
|
||||
'file_name': entry_name,
|
||||
'hash': '',
|
||||
'thumbnailUrl': '/loras_static/images/no-preview.png',
|
||||
'baseModel': '',
|
||||
@@ -73,59 +138,31 @@ class ComfyMetadataParser(RecipeMetadataParser):
|
||||
'downloadUrl': '',
|
||||
'isDeleted': False
|
||||
}
|
||||
|
||||
# Get additional info from Civitai if metadata provider is available
|
||||
if metadata_provider:
|
||||
try:
|
||||
civitai_info_tuple = await metadata_provider.get_model_version_info(model_version_id)
|
||||
# Populate lora entry with Civitai info
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info_tuple,
|
||||
recipe_scanner
|
||||
)
|
||||
if populated_entry is None:
|
||||
continue # Skip invalid LoRA types
|
||||
lora_entry = populated_entry
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching Civitai info for LoRA: {e}")
|
||||
|
||||
|
||||
if lora_id_match:
|
||||
if metadata_provider:
|
||||
try:
|
||||
civitai_info_tuple = await metadata_provider.get_model_version_info(model_version_id)
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info_tuple,
|
||||
recipe_scanner
|
||||
)
|
||||
if populated_entry is None:
|
||||
continue
|
||||
lora_entry = populated_entry
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching Civitai info for LoRA: {e}")
|
||||
else:
|
||||
if not recipe_scanner:
|
||||
continue
|
||||
local_lora = await recipe_scanner.get_local_lora(lora_name, recipe_base_model)
|
||||
if not local_lora:
|
||||
continue
|
||||
lora_entry = self.populate_lora_from_local(lora_entry, local_lora)
|
||||
|
||||
loras.append(lora_entry)
|
||||
|
||||
# Find checkpoint info
|
||||
checkpoint_nodes = {k: v for k, v in data.items() if isinstance(v, dict) and v.get('class_type') == 'CheckpointLoaderSimple'}
|
||||
checkpoint = None
|
||||
checkpoint_id = None
|
||||
checkpoint_version_id = None
|
||||
|
||||
if checkpoint_nodes:
|
||||
# Get the first checkpoint node
|
||||
checkpoint_node = next(iter(checkpoint_nodes.values()))
|
||||
if 'inputs' in checkpoint_node and 'ckpt_name' in checkpoint_node['inputs']:
|
||||
checkpoint_name = checkpoint_node['inputs']['ckpt_name']
|
||||
# Parse checkpoint URN
|
||||
checkpoint_match = re.search(r'civitai:(\d+)@(\d+)', checkpoint_name)
|
||||
if checkpoint_match:
|
||||
checkpoint_id = checkpoint_match.group(1)
|
||||
checkpoint_version_id = checkpoint_match.group(2)
|
||||
checkpoint = {
|
||||
'id': checkpoint_version_id,
|
||||
'modelId': checkpoint_id,
|
||||
'name': f"Checkpoint {checkpoint_id}",
|
||||
'version': '',
|
||||
'type': 'checkpoint'
|
||||
}
|
||||
|
||||
# Get additional checkpoint info from Civitai
|
||||
if metadata_provider:
|
||||
try:
|
||||
civitai_info_tuple = await metadata_provider.get_model_version_info(checkpoint_version_id)
|
||||
civitai_info, _ = civitai_info_tuple if isinstance(civitai_info_tuple, tuple) else (civitai_info_tuple, None)
|
||||
# Populate checkpoint with Civitai info
|
||||
checkpoint = await self.populate_checkpoint_from_civitai(checkpoint, civitai_info)
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching Civitai info for checkpoint: {e}")
|
||||
|
||||
# Extract generation parameters
|
||||
gen_params = {}
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ class MetaFormatParser(RecipeMetadataParser):
|
||||
prompt = parts[0].strip()
|
||||
|
||||
# Initialize metadata
|
||||
metadata = {"prompt": prompt, "loras": []}
|
||||
metadata: Dict[str, Any] = {"prompt": prompt, "loras": []}
|
||||
|
||||
# Extract negative prompt and parameters if available
|
||||
if len(parts) > 1:
|
||||
|
||||
@@ -91,7 +91,15 @@ class RecipeFormatParser(RecipeMetadataParser):
|
||||
exists_locally = lora_scanner.has_hash(lora['hash'])
|
||||
if exists_locally:
|
||||
lora_cache = await lora_scanner.get_cached_data()
|
||||
lora_item = next((item for item in lora_cache.raw_data if item['sha256'].lower() == lora['hash'].lower()), None)
|
||||
# Cascade match: full sha256, stored autov3, or autov2 (sha256[:10]).
|
||||
h = (lora.get('hash') or '').lower()
|
||||
lora_item = next(
|
||||
(item for item in lora_cache.raw_data
|
||||
if (item.get("sha256") or "").lower() == h
|
||||
or (item.get("autov3") or "").lower() == h
|
||||
or (item.get("sha256") or "")[:10].lower() == h),
|
||||
None
|
||||
)
|
||||
if lora_item:
|
||||
lora_entry['existsLocally'] = True
|
||||
lora_entry['inLibrary'] = True
|
||||
@@ -148,7 +156,7 @@ class RecipeFormatParser(RecipeMetadataParser):
|
||||
checkpoint_data = recipe_metadata.get('checkpoint') or {}
|
||||
if isinstance(checkpoint_data, dict) and checkpoint_data:
|
||||
version_id = checkpoint_data.get('modelVersionId') or checkpoint_data.get('id')
|
||||
checkpoint_entry = {
|
||||
checkpoint_entry: Dict[str, Any] = {
|
||||
'id': version_id or 0,
|
||||
'modelId': checkpoint_data.get('modelId', 0),
|
||||
'name': checkpoint_data.get('name', 'Unknown Checkpoint'),
|
||||
@@ -188,7 +196,7 @@ class RecipeFormatParser(RecipeMetadataParser):
|
||||
filtered_gen_params[key] = value
|
||||
|
||||
return {
|
||||
'base_model': checkpoint['baseModel'] if checkpoint and checkpoint.get('baseModel') else recipe_metadata.get('base_model', ''),
|
||||
'base_model': checkpoint['baseModel'] if checkpoint and checkpoint.get('baseModel') else (recipe_metadata.get('base_model') or None),
|
||||
'loras': loras,
|
||||
'gen_params': filtered_gen_params,
|
||||
'tags': recipe_metadata.get('tags', []),
|
||||
@@ -200,3 +208,24 @@ class RecipeFormatParser(RecipeMetadataParser):
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing recipe format metadata: {e}", exc_info=True)
|
||||
return {"error": str(e), "loras": []}
|
||||
|
||||
|
||||
def strip_recipe_metadata(metadata_text: str) -> str:
|
||||
"""Strip the ``Recipe metadata: {...}`` block appended by LoRA Manager.
|
||||
|
||||
The saved recipe image carries the original generation metadata followed
|
||||
by an appended recipe JSON block (see ``ExifUtils.append_recipe_metadata``).
|
||||
Re-import wants to re-parse the original embedded metadata, so this returns
|
||||
only the text before the appended marker. The input is returned unchanged
|
||||
when no marker is present.
|
||||
"""
|
||||
if not metadata_text:
|
||||
return metadata_text
|
||||
match = re.search(
|
||||
RecipeFormatParser.METADATA_MARKER,
|
||||
metadata_text,
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
if not match:
|
||||
return metadata_text
|
||||
return metadata_text[: match.start()].strip()
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, Callable, Dict, Mapping
|
||||
from typing import TYPE_CHECKING, Awaitable, Callable, Dict, Mapping
|
||||
|
||||
import jinja2
|
||||
from aiohttp import web
|
||||
@@ -30,6 +30,7 @@ from ..services.websocket_progress_callback import (
|
||||
WebSocketProgressCallback,
|
||||
)
|
||||
from ..utils.exif_utils import ExifUtils
|
||||
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES
|
||||
from ..utils.metadata_manager import MetadataManager
|
||||
from .model_route_registrar import COMMON_ROUTE_DEFINITIONS, ModelRouteRegistrar
|
||||
from .handlers.model_handlers import (
|
||||
@@ -84,7 +85,7 @@ class BaseModelRoutes(ABC):
|
||||
self.metadata_progress_callback = WebSocketBroadcastCallback()
|
||||
|
||||
self._handler_set: ModelHandlerSet | None = None
|
||||
self._handler_mapping: Dict[str, Callable[[web.Request], web.StreamResponse]] | None = None
|
||||
self._handler_mapping: Dict[str, Callable[[web.Request], Awaitable[web.Response]]] | None = None
|
||||
|
||||
self._preview_service = PreviewAssetService(
|
||||
metadata_manager=MetadataManager,
|
||||
@@ -131,7 +132,7 @@ class BaseModelRoutes(ABC):
|
||||
self._handler_set = None
|
||||
self._handler_mapping = None
|
||||
|
||||
def _ensure_handler_mapping(self) -> Mapping[str, Callable[[web.Request], web.StreamResponse]]:
|
||||
def _ensure_handler_mapping(self) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||
if self._handler_mapping is None:
|
||||
handler_set = self._create_handler_set()
|
||||
self._handler_set = handler_set
|
||||
@@ -148,6 +149,7 @@ class BaseModelRoutes(ABC):
|
||||
settings_service=self._settings,
|
||||
server_i18n=self._server_i18n,
|
||||
logger=logger,
|
||||
page_context_provider=self._get_page_context_provider(),
|
||||
)
|
||||
listing = ModelListingHandler(
|
||||
service=service,
|
||||
@@ -220,7 +222,7 @@ class BaseModelRoutes(ABC):
|
||||
)
|
||||
|
||||
@property
|
||||
def route_handlers(self) -> Mapping[str, Callable[[web.Request], web.StreamResponse]]:
|
||||
def route_handlers(self) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||
return self._ensure_handler_mapping()
|
||||
|
||||
def setup_routes(self, app: web.Application, prefix: str) -> None:
|
||||
@@ -237,7 +239,7 @@ class BaseModelRoutes(ABC):
|
||||
"""Setup model-specific routes."""
|
||||
raise NotImplementedError
|
||||
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict:
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict[str, Any]:
|
||||
"""Parse model-specific parameters - to be overridden by subclasses."""
|
||||
return {}
|
||||
|
||||
@@ -249,11 +251,15 @@ class BaseModelRoutes(ABC):
|
||||
"""Get expected model types string for error messages - to be overridden by subclasses."""
|
||||
return "any model type"
|
||||
|
||||
def _get_page_context_provider(self):
|
||||
"""Optional hook returning extra template context for the page view."""
|
||||
return None
|
||||
|
||||
def _find_model_file(self, files):
|
||||
"""Find the appropriate model file from the files list - can be overridden by subclasses."""
|
||||
return next((file for file in files if file.get("type") in ("Model", "Diffusion Model") and file.get("primary") is True), None)
|
||||
return next((file for file in files if file.get("type") in MODEL_WEIGHT_FILE_TYPES and file.get("primary") is True), None)
|
||||
|
||||
def get_handler(self, name: str) -> Callable[[web.Request], web.StreamResponse]:
|
||||
def get_handler(self, name: str) -> Callable[[web.Request], Awaitable[web.StreamResponse]]:
|
||||
"""Expose handlers for subclasses or tests."""
|
||||
return self._ensure_handler_mapping()[name]
|
||||
|
||||
@@ -285,7 +291,7 @@ class BaseModelRoutes(ABC):
|
||||
)
|
||||
return self.model_lifecycle_service
|
||||
|
||||
def _make_handler_proxy(self, name: str) -> Callable[[web.Request], web.StreamResponse]:
|
||||
def _make_handler_proxy(self, name: str) -> Callable[[web.Request], Awaitable[web.StreamResponse]]:
|
||||
async def proxy(request: web.Request) -> web.StreamResponse:
|
||||
try:
|
||||
handler = self.get_handler(name)
|
||||
|
||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Callable, Mapping
|
||||
from typing import Awaitable, Callable, Mapping
|
||||
|
||||
import jinja2
|
||||
from aiohttp import web
|
||||
@@ -32,6 +32,7 @@ from .handlers.recipe_handlers import (
|
||||
RecipePageView,
|
||||
RecipeQueryHandler,
|
||||
RecipeSharingHandler,
|
||||
RecipeWorkflowHandler,
|
||||
)
|
||||
from .recipe_route_registrar import ROUTE_DEFINITIONS
|
||||
|
||||
@@ -61,7 +62,9 @@ class BaseRecipeRoutes:
|
||||
self._i18n_registered = False
|
||||
self._startup_hooks_registered = False
|
||||
self._handler_set: RecipeHandlerSet | None = None
|
||||
self._handler_mapping: dict[str, Callable] | None = None
|
||||
self._handler_mapping: Mapping[
|
||||
str, Callable[[web.Request], Awaitable[web.StreamResponse]]
|
||||
] | None = None
|
||||
|
||||
async def attach_dependencies(self, app: web.Application | None = None) -> None:
|
||||
"""Resolve shared services from the registry."""
|
||||
@@ -84,7 +87,9 @@ class BaseRecipeRoutes:
|
||||
app.on_startup.append(self.attach_dependencies)
|
||||
self._startup_hooks_registered = True
|
||||
|
||||
def to_route_mapping(self) -> Mapping[str, Callable]:
|
||||
def to_route_mapping(
|
||||
self,
|
||||
) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||
"""Return a mapping of handler name to coroutine for registrar binding."""
|
||||
|
||||
if self._handler_mapping is None:
|
||||
@@ -124,17 +129,17 @@ class BaseRecipeRoutes:
|
||||
or os.environ.get("HF_HUB_DISABLE_TELEMETRY", "0") == "0"
|
||||
)
|
||||
if not standalone_mode:
|
||||
from ..metadata_collector import get_metadata # type: ignore[import-not-found]
|
||||
from ..metadata_collector.metadata_processor import ( # type: ignore[import-not-found]
|
||||
from ..metadata_collector import get_metadata # pyright: ignore[reportMissingImports]
|
||||
from ..metadata_collector.metadata_processor import ( # pyright: ignore[reportMissingImports]
|
||||
MetadataProcessor,
|
||||
)
|
||||
from ..metadata_collector.metadata_registry import ( # type: ignore[import-not-found]
|
||||
from ..metadata_collector.metadata_registry import ( # pyright: ignore[reportMissingImports]
|
||||
MetadataRegistry,
|
||||
)
|
||||
else: # pragma: no cover - optional dependency path
|
||||
get_metadata = None # type: ignore[assignment]
|
||||
MetadataProcessor = None # type: ignore[assignment]
|
||||
MetadataRegistry = None # type: ignore[assignment]
|
||||
get_metadata = None # pyright: ignore[reportAssignmentType]
|
||||
MetadataProcessor = None # pyright: ignore[reportAssignmentType]
|
||||
MetadataRegistry = None # pyright: ignore[reportAssignmentType]
|
||||
|
||||
analysis_service = RecipeAnalysisService(
|
||||
exif_utils=ExifUtils,
|
||||
@@ -196,6 +201,18 @@ class BaseRecipeRoutes:
|
||||
sharing_service=sharing_service,
|
||||
)
|
||||
|
||||
# Lazy import: standalone mode replaces the ``server`` module with a
|
||||
# mock, so resolve PromptServer at handler-set build time instead of
|
||||
# module import time. The handler's standalone check guards UX.
|
||||
from server import PromptServer # pyright: ignore[reportMissingImports]
|
||||
|
||||
workflow = RecipeWorkflowHandler(
|
||||
ensure_dependencies_ready=self.ensure_dependencies_ready,
|
||||
recipe_scanner_getter=recipe_scanner_getter,
|
||||
prompt_server=PromptServer,
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
from ..services.websocket_manager import ws_manager
|
||||
|
||||
batch_import_service = BatchImportService(
|
||||
@@ -220,4 +237,5 @@ class BaseRecipeRoutes:
|
||||
analysis=analysis,
|
||||
sharing=sharing,
|
||||
batch_import=batch_import,
|
||||
workflow=workflow,
|
||||
)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
from typing import Dict, List, Set
|
||||
import os
|
||||
from typing import Any, Dict, List, Set
|
||||
from aiohttp import web
|
||||
|
||||
from .base_model_routes import BaseModelRoutes
|
||||
@@ -7,6 +8,7 @@ from .model_route_registrar import ModelRouteRegistrar
|
||||
from ..services.checkpoint_service import CheckpointService
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
from ..config import config
|
||||
from ..utils.utils import _format_model_name_for_comfyui
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -28,13 +30,13 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
# Attach service dependencies
|
||||
self.attach_service(self.service)
|
||||
|
||||
def setup_routes(self, app: web.Application):
|
||||
def setup_routes(self, app: web.Application, prefix: str = "checkpoints"):
|
||||
"""Setup Checkpoint routes"""
|
||||
# Schedule service initialization on app startup
|
||||
app.on_startup.append(lambda _: self.initialize_services())
|
||||
|
||||
|
||||
# Setup common routes with 'checkpoints' prefix (includes page route)
|
||||
super().setup_routes(app, 'checkpoints')
|
||||
super().setup_routes(app, prefix)
|
||||
|
||||
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
||||
"""Setup Checkpoint-specific routes"""
|
||||
@@ -44,7 +46,46 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
# Checkpoint roots and Unet roots
|
||||
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/checkpoints_roots', prefix, self.get_checkpoints_roots)
|
||||
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/unet_roots', prefix, self.get_unet_roots)
|
||||
|
||||
# Name/base_model pool for the Checkpoint/Unet Loader nodes' base_model filtering
|
||||
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/loader-pool', prefix, self.get_loader_pool)
|
||||
|
||||
async def get_loader_pool(self, request: web.Request) -> web.Response:
|
||||
"""Return ComfyUI-formatted model names with their base_model.
|
||||
|
||||
Backing data for the Checkpoint/Unet Loader nodes'
|
||||
control_after_generate feature: the front-end filters the
|
||||
ckpt_name/unet_name combo options by base_model using this pool, so
|
||||
randomize mode picks within the narrowed set.
|
||||
"""
|
||||
try:
|
||||
sub_type = request.query.get("sub_type", "checkpoint")
|
||||
if sub_type not in ("checkpoint", "diffusion_model"):
|
||||
return web.json_response({"error": "invalid sub_type"}, status=400)
|
||||
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||
cache = await scanner.get_cached_data()
|
||||
model_roots = scanner.get_model_roots()
|
||||
items: List[Dict[str, str]] = []
|
||||
for item in cache.raw_data:
|
||||
if item.get("sub_type") != sub_type:
|
||||
continue
|
||||
file_path = item.get("file_path", "")
|
||||
if not file_path or not os.path.exists(file_path):
|
||||
continue
|
||||
formatted_name = _format_model_name_for_comfyui(file_path, model_roots)
|
||||
if formatted_name:
|
||||
items.append(
|
||||
{
|
||||
"name": formatted_name,
|
||||
"base_model": item.get("base_model", "") or "",
|
||||
}
|
||||
)
|
||||
items.sort(key=lambda x: x["name"])
|
||||
return web.json_response({"items": items})
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting loader pool: {e}", exc_info=True)
|
||||
return web.json_response({"error": str(e)}, status=500)
|
||||
|
||||
def _validate_civitai_model_type(self, model_type: str) -> bool:
|
||||
"""Validate CivitAI model type for Checkpoint"""
|
||||
return model_type.lower() == 'checkpoint'
|
||||
@@ -53,9 +94,9 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
"""Get expected model types string for error messages"""
|
||||
return "Checkpoint"
|
||||
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict:
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict[str, Any]:
|
||||
"""Parse Checkpoint-specific parameters"""
|
||||
params: Dict = {}
|
||||
params: Dict[str, Any] = {}
|
||||
|
||||
if 'checkpoint_hash' in request.query:
|
||||
params['hash_filters'] = {'single_hash': request.query['checkpoint_hash'].lower()}
|
||||
@@ -70,7 +111,7 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
"""Get detailed information for a specific checkpoint by name"""
|
||||
try:
|
||||
name = request.match_info.get('name', '')
|
||||
checkpoint_info = await self.service.get_model_info_by_name(name)
|
||||
checkpoint_info = await self.service.get_model_info_by_name(name) # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
if checkpoint_info:
|
||||
return web.json_response(checkpoint_info)
|
||||
@@ -89,7 +130,7 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
roots.extend(config.checkpoints_roots or [])
|
||||
roots.extend(config.extra_checkpoints_roots or [])
|
||||
# Remove duplicates while preserving order
|
||||
seen: set = set()
|
||||
seen: set[str] = set()
|
||||
unique_roots: List[str] = []
|
||||
for root in roots:
|
||||
if root and root not in seen:
|
||||
@@ -114,7 +155,7 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
roots.extend(config.unet_roots or [])
|
||||
roots.extend(config.extra_unet_roots or [])
|
||||
# Remove duplicates while preserving order
|
||||
seen: set = set()
|
||||
seen: set[str] = set()
|
||||
unique_roots: List[str] = []
|
||||
for root in roots:
|
||||
if root and root not in seen:
|
||||
|
||||
@@ -26,13 +26,13 @@ class EmbeddingRoutes(BaseModelRoutes):
|
||||
# Attach service dependencies
|
||||
self.attach_service(self.service)
|
||||
|
||||
def setup_routes(self, app: web.Application):
|
||||
def setup_routes(self, app: web.Application, prefix: str = "embeddings"):
|
||||
"""Setup Embedding routes"""
|
||||
# Schedule service initialization on app startup
|
||||
app.on_startup.append(lambda _: self.initialize_services())
|
||||
|
||||
|
||||
# Setup common routes with 'embeddings' prefix (includes page route)
|
||||
super().setup_routes(app, 'embeddings')
|
||||
super().setup_routes(app, prefix)
|
||||
|
||||
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
||||
"""Setup Embedding-specific routes"""
|
||||
@@ -51,7 +51,7 @@ class EmbeddingRoutes(BaseModelRoutes):
|
||||
"""Get detailed information for a specific embedding by name"""
|
||||
try:
|
||||
name = request.match_info.get('name', '')
|
||||
embedding_info = await self.service.get_model_info_by_name(name)
|
||||
embedding_info = await self.service.get_model_info_by_name(name) # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
if embedding_info:
|
||||
return web.json_response(embedding_info)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Callable, Mapping
|
||||
from typing import Any, Awaitable, Callable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -35,7 +35,7 @@ class ExampleImagesRoutes:
|
||||
*,
|
||||
ws_manager,
|
||||
download_manager: DownloadManager | None = None,
|
||||
processor=ExampleImagesProcessor,
|
||||
processor: Any = ExampleImagesProcessor,
|
||||
file_manager=ExampleImagesFileManager,
|
||||
cleanup_service: ExampleImagesCleanupService | None = None,
|
||||
) -> None:
|
||||
@@ -46,7 +46,9 @@ class ExampleImagesRoutes:
|
||||
self._file_manager = file_manager
|
||||
self._cleanup_service = cleanup_service or ExampleImagesCleanupService()
|
||||
self._handler_set: ExampleImagesHandlerSet | None = None
|
||||
self._handler_mapping: Mapping[str, Callable[[web.Request], web.StreamResponse]] | None = None
|
||||
self._handler_mapping: Mapping[
|
||||
str, Callable[[web.Request], Awaitable[web.StreamResponse]]
|
||||
] | None = None
|
||||
|
||||
@classmethod
|
||||
def setup_routes(cls, app: web.Application, *, ws_manager) -> None:
|
||||
@@ -61,7 +63,9 @@ class ExampleImagesRoutes:
|
||||
registrar = ExampleImagesRouteRegistrar(app)
|
||||
registrar.register_routes(self.to_route_mapping())
|
||||
|
||||
def to_route_mapping(self) -> Mapping[str, Callable[[web.Request], web.StreamResponse]]:
|
||||
def to_route_mapping(
|
||||
self,
|
||||
) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||
"""Return the registrar-compatible mapping of handler names to callables."""
|
||||
|
||||
if self._handler_mapping is None:
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
"""HTTP route handlers for agent skill endpoints.
|
||||
|
||||
These handlers expose the :class:`AgentService` via HTTP, allowing the
|
||||
frontend to list available skills and execute them on selected models.
|
||||
Progress is reported via WebSocket broadcast.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from ...services.agent import AgentService, AgentProgressReporter
|
||||
from ...services.llm_service import LLMNotConfiguredError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AgentHandler:
|
||||
"""HTTP handler for agent skill operations."""
|
||||
|
||||
def __init__(self, agent_service: AgentService | None = None) -> None:
|
||||
self._agent_service = agent_service
|
||||
|
||||
async def _ensure_service(self) -> AgentService:
|
||||
if self._agent_service is None:
|
||||
self._agent_service = await AgentService.get_instance()
|
||||
return self._agent_service
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# GET /api/lm/agent/skills
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def get_agent_skills(self, request: web.Request) -> web.Response:
|
||||
"""Return a list of available agent skills."""
|
||||
|
||||
service = await self._ensure_service()
|
||||
skills = await service.list_skills()
|
||||
return web.json_response({"skills": skills})
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# POST /api/lm/agent/execute/{skill_name}
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def execute_agent_skill(self, request: web.Request) -> web.Response:
|
||||
"""Execute an agent skill on the provided model paths.
|
||||
|
||||
Request body::
|
||||
|
||||
{"model_paths": ["/path/to/model1.safetensors", ...], "options": {}}
|
||||
|
||||
Returns immediately with a task ID. Execution runs in the
|
||||
background; progress and completion are pushed via WebSocket
|
||||
events of type ``agent_progress``.
|
||||
"""
|
||||
|
||||
skill_name = request.match_info.get("skill_name", "")
|
||||
if not skill_name:
|
||||
return web.json_response(
|
||||
{"error": "Skill name is required"}, status=400
|
||||
)
|
||||
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception:
|
||||
return web.json_response(
|
||||
{"error": "Invalid JSON body"}, status=400
|
||||
)
|
||||
|
||||
model_paths = body.get("model_paths", [])
|
||||
if not model_paths or not isinstance(model_paths, list):
|
||||
return web.json_response(
|
||||
{"error": "model_paths must be a non-empty array"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
service = await self._ensure_service()
|
||||
|
||||
# Validate LLM configuration early for skills that need it
|
||||
# (fail fast rather than after starting background work)
|
||||
try:
|
||||
from ...services.llm_service import LLMService
|
||||
|
||||
llm = await LLMService.get_instance()
|
||||
if not llm.is_configured():
|
||||
return web.json_response(
|
||||
{
|
||||
"error": "LLM provider is not configured. "
|
||||
"Enable it in Settings → AI Provider.",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to check LLM configuration: %s", exc)
|
||||
|
||||
# Launch execution in the background
|
||||
progress_reporter = AgentProgressReporter()
|
||||
logger.info(
|
||||
"LLM enrichment '%s' starting for %d model(s)",
|
||||
skill_name, len(model_paths),
|
||||
)
|
||||
|
||||
async def _run() -> None:
|
||||
try:
|
||||
result = await service.execute_skill(
|
||||
skill_name=skill_name,
|
||||
input_data={"model_paths": model_paths},
|
||||
progress_callback=progress_reporter,
|
||||
)
|
||||
logger.info(
|
||||
"LLM enrichment '%s' finished: success=%s, summary='%s', errors=%s",
|
||||
skill_name, result.success, result.summary, result.errors,
|
||||
)
|
||||
except LLMNotConfiguredError as exc:
|
||||
logger.warning("LLM enrichment '%s' not configured: %s", skill_name, exc)
|
||||
await progress_reporter.on_progress(
|
||||
{
|
||||
"type": "agent_progress",
|
||||
"skill": skill_name,
|
||||
"status": "error",
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("LLM enrichment '%s' failed: %s", skill_name, exc, exc_info=True)
|
||||
await progress_reporter.on_progress(
|
||||
{
|
||||
"type": "agent_progress",
|
||||
"skill": skill_name,
|
||||
"status": "error",
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
|
||||
# Fire and forget — progress comes via WebSocket
|
||||
asyncio.create_task(_run())
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
"status": "started",
|
||||
"skill": skill_name,
|
||||
"model_count": len(model_paths),
|
||||
}
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# POST /api/lm/agent/cancel
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def cancel_agent_skill(self, request: web.Request) -> web.Response:
|
||||
"""Cancel a running agent skill.
|
||||
|
||||
NOTE: Cancellation is a stub for now — the AgentService processes
|
||||
models sequentially and does not yet support mid-execution
|
||||
cancellation. This endpoint exists for API completeness.
|
||||
"""
|
||||
|
||||
# TODO: implement cooperative cancellation in AgentService
|
||||
return web.json_response(
|
||||
{"status": "acknowledged", "note": "Cancellation not yet implemented"},
|
||||
status=200,
|
||||
)
|
||||
@@ -0,0 +1,110 @@
|
||||
"""HTTP handler for download target routing decisions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from ...services.download_routing import (
|
||||
is_diffusion_model_download,
|
||||
resolve_other_download_sub_type,
|
||||
)
|
||||
from ...utils.constants import VALID_OTHER_CIVITAI_TYPES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DownloadRoutingHandler:
|
||||
"""Expose the download-time checkpoint/diffusion-model routing decision.
|
||||
|
||||
The web UI calls this when the user reaches the download location step
|
||||
so the root dropdown offers the same root set (checkpoint vs unet) that
|
||||
the download manager would pick for ``use_default_paths``.
|
||||
"""
|
||||
|
||||
async def get_download_routing(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
payload = await request.json()
|
||||
except json.JSONDecodeError:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Invalid JSON payload"}, status=400
|
||||
)
|
||||
|
||||
model_type = payload.get("model_type", "")
|
||||
base_model = payload.get("base_model") or ""
|
||||
file_types = payload.get("file_types") or []
|
||||
selected_file_type = payload.get("selected_file_type")
|
||||
|
||||
if not isinstance(model_type, str) or not model_type:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "model_type is required"}, status=400
|
||||
)
|
||||
if not isinstance(base_model, str) or not isinstance(file_types, list):
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "base_model must be a string and file_types a list",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
if selected_file_type is not None and not isinstance(selected_file_type, str):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "selected_file_type must be a string"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
if model_type.lower() in VALID_OTHER_CIVITAI_TYPES:
|
||||
from ...services.settings_manager import get_settings_manager
|
||||
|
||||
settings = get_settings_manager()
|
||||
if not settings.is_other_models_enabled():
|
||||
# Opt-in feature is off: never auto-route, the UI falls back to
|
||||
# manual folder selection and the download manager rejects it.
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"root_kind": "other",
|
||||
"sub_type": None,
|
||||
"disabled": True,
|
||||
"reason": "other_models_disabled",
|
||||
}
|
||||
)
|
||||
|
||||
sub_type = resolve_other_download_sub_type(
|
||||
model_type,
|
||||
file_types=(str(t) for t in file_types),
|
||||
selected_file_type=selected_file_type,
|
||||
)
|
||||
if sub_type and not settings.is_other_sub_type_enabled(sub_type):
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"root_kind": "other",
|
||||
"sub_type": None,
|
||||
"disabled": True,
|
||||
"reason": "other_sub_type_disabled",
|
||||
"requested_sub_type": sub_type,
|
||||
}
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"root_kind": "other",
|
||||
"sub_type": sub_type,
|
||||
}
|
||||
)
|
||||
|
||||
is_diffusion = is_diffusion_model_download(
|
||||
model_type,
|
||||
file_types=(str(t) for t in file_types),
|
||||
base_model=base_model,
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"is_diffusion_model": is_diffusion,
|
||||
"root_kind": "unet" if is_diffusion else model_type,
|
||||
}
|
||||
)
|
||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Mapping
|
||||
from typing import Awaitable, Callable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -170,7 +170,7 @@ class ExampleImagesHandlerSet:
|
||||
management: ExampleImagesManagementHandler
|
||||
files: ExampleImagesFileHandler
|
||||
|
||||
def to_route_mapping(self) -> Mapping[str, Callable[[web.Request], web.StreamResponse]]:
|
||||
def to_route_mapping(self) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||
"""Flatten handler methods into the registrar mapping."""
|
||||
|
||||
return {
|
||||
|
||||
+980
-147
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,639 @@
|
||||
"""Handlers for external model sources: linking, file listing and downloads.
|
||||
|
||||
Covers every site registered in :mod:`py.services.model_sources`. The module
|
||||
was Hugging Face only (``hf_handlers.py`` / ``HfHandler``) until ModelScope
|
||||
downloads were added; the per-site differences now live in the providers, so
|
||||
this file has no platform branches beyond the capability lookups.
|
||||
|
||||
The historical route paths (``/api/lm/set-hf-url``, ``/api/lm/hf-repo-files``,
|
||||
``/api/lm/download-hf-model``) are still registered as aliases of the generic
|
||||
handlers, so existing callers keep working.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from ...config import config
|
||||
from ...services.downloader import (
|
||||
DownloadProgress,
|
||||
get_downloader,
|
||||
)
|
||||
from ...services.aria2_downloader import Aria2Downloader
|
||||
from ...services.model_sources import (
|
||||
ModelSourceError,
|
||||
SourceRef,
|
||||
detect_source,
|
||||
get_download_source,
|
||||
hydrate_from_source,
|
||||
is_valid_source_id,
|
||||
list_sources,
|
||||
normalize_metadata_source,
|
||||
)
|
||||
from ...services.settings_manager import get_settings_manager
|
||||
from ...services.service_registry import ServiceRegistry
|
||||
from ...services.websocket_manager import ws_manager
|
||||
from ...utils.metadata_manager import MetadataManager
|
||||
from ...utils.models import LoraMetadata, CheckpointMetadata, EmbeddingMetadata
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_MODEL_CLASS = LoraMetadata
|
||||
_DEFAULT_SCANNER_GETTER = "get_lora_scanner"
|
||||
|
||||
|
||||
def _infer_model_type(model_root: str) -> tuple[Any, str]:
|
||||
"""Determine model class and scanner by matching ``model_root`` against the
|
||||
configured root paths for each model type (from ``Config``).
|
||||
|
||||
The ``model_root`` value comes from the frontend's model-root dropdown,
|
||||
which is populated from the current page's scanner roots. By checking
|
||||
which scanner's root list it belongs to, we avoid fragile heuristics
|
||||
like substring-matching path names.
|
||||
"""
|
||||
norm = os.path.normpath(model_root).replace(os.sep, "/")
|
||||
|
||||
# LoRA roots
|
||||
for p in (config.loras_roots or []) + (config.extra_loras_roots or []):
|
||||
if os.path.normpath(p).replace(os.sep, "/") == norm:
|
||||
return LoraMetadata, "get_lora_scanner"
|
||||
|
||||
# Checkpoint / UNet roots
|
||||
for p in (
|
||||
(config.checkpoints_roots or [])
|
||||
+ (config.extra_checkpoints_roots or [])
|
||||
+ (config.unet_roots or [])
|
||||
+ (config.extra_unet_roots or [])
|
||||
):
|
||||
if os.path.normpath(p).replace(os.sep, "/") == norm:
|
||||
return CheckpointMetadata, "get_checkpoint_scanner"
|
||||
|
||||
# Embedding roots
|
||||
for p in (config.embeddings_roots or []) + (config.extra_embeddings_roots or []):
|
||||
if os.path.normpath(p).replace(os.sep, "/") == norm:
|
||||
return EmbeddingMetadata, "get_embedding_scanner"
|
||||
|
||||
# Fallback — should not happen in normal use
|
||||
logger.warning(
|
||||
"Could not determine model type for root '%s'; defaulting to LoRA",
|
||||
model_root,
|
||||
)
|
||||
return _DEFAULT_MODEL_CLASS, _DEFAULT_SCANNER_GETTER
|
||||
|
||||
|
||||
async def _report_phase(
|
||||
download_id: str | None, stage: str, platform: str = ""
|
||||
) -> None:
|
||||
"""Tell the progress UI which post-transfer stage is running.
|
||||
|
||||
A download's byte counter stops the moment the last byte lands, but the
|
||||
backend still has to index the file and read the model site's API. Without
|
||||
this the bar sits at 100% reporting "0 B/s" and the download looks stuck for
|
||||
several seconds. *stage* is machine-readable — the UI localises it — and
|
||||
*platform* lets it name the site the metadata comes from.
|
||||
"""
|
||||
|
||||
if not download_id:
|
||||
return
|
||||
try:
|
||||
await ws_manager.broadcast_download_progress(
|
||||
download_id,
|
||||
{
|
||||
"status": "metadata",
|
||||
"stage": stage,
|
||||
"platform": platform,
|
||||
"progress": 100,
|
||||
},
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - progress must never be fatal
|
||||
logger.debug("Failed to report the '%s' phase: %s", stage, exc)
|
||||
|
||||
|
||||
async def _save_source_metadata(
|
||||
dest_path: str, ref: SourceRef, model_root: str, *, download_id: str | None = None
|
||||
) -> None:
|
||||
"""Create a proper .metadata.json and add the model to the scanner cache.
|
||||
|
||||
The metadata is created through the owning scanner rather than
|
||||
``MetadataManager.create_default_metadata()``, because that is the only
|
||||
factory that knows when hashing must be deferred: ``CheckpointScanner`` and
|
||||
``OtherScanner`` deliberately record ``hash_status="pending"`` with an empty
|
||||
``sha256`` for their multi-GB files, and the generic helper would read a
|
||||
10 GB checkpoint end to end *inside the download request*. Scanners for the
|
||||
small types delegate straight back to it, so nothing changes for them.
|
||||
|
||||
The external-source fields are then overlaid and the model is registered in
|
||||
the in-memory scanner cache so it appears immediately without a full
|
||||
filesystem walk.
|
||||
|
||||
Finally the site's own published metadata is applied (see
|
||||
:func:`~py.services.model_sources.hydration.hydrate_from_source`), so a
|
||||
ModelScope or Hugging Face download lands with the same populated model
|
||||
card a CivitAI download produces instead of a bare filename and hash.
|
||||
|
||||
Both post-transfer stages are reported through *download_id* when the UI is
|
||||
watching one, because neither advances the byte counter.
|
||||
"""
|
||||
try:
|
||||
model_class, scanner_getter_name = _infer_model_type(model_root)
|
||||
|
||||
scanner = None
|
||||
scanner_getter = getattr(ServiceRegistry, scanner_getter_name, None)
|
||||
if scanner_getter is not None:
|
||||
scanner = await scanner_getter()
|
||||
|
||||
# 1. Create proper metadata (reads safetensors headers; hashes only for
|
||||
# the model types whose scanner does not defer it)
|
||||
await _report_phase(download_id, "indexing", ref.platform)
|
||||
create_metadata = getattr(scanner, "_create_default_metadata", None)
|
||||
if create_metadata is not None:
|
||||
metadata = await create_metadata(dest_path)
|
||||
else:
|
||||
metadata = await MetadataManager.create_default_metadata(
|
||||
dest_path, model_class=model_class
|
||||
)
|
||||
if metadata is None:
|
||||
logger.warning("create_default_metadata returned None for %s", dest_path)
|
||||
return
|
||||
|
||||
# 2. Overlay the external-source fields (`hf_url` is written by
|
||||
# normalisation for Hugging Face only)
|
||||
fields = metadata._unknown_fields
|
||||
fields["source_url"] = ref.url
|
||||
fields["source_platform"] = ref.platform
|
||||
if ref.platform == "huggingface":
|
||||
fields["hf_url"] = ref.url
|
||||
metadata.from_civitai = False # externally-sourced models are not from CivitAI
|
||||
|
||||
# 3. Save metadata atomically
|
||||
await MetadataManager.save_metadata(dest_path, metadata)
|
||||
logger.info(
|
||||
"Saved %s metadata (source=%s, hash_status=%s) for %s",
|
||||
ref.platform, ref.url, getattr(metadata, "hash_status", "?"), dest_path,
|
||||
)
|
||||
|
||||
# 4. Determine relative folder path for cache
|
||||
# model_root is an absolute path; dest_path is under it
|
||||
folder = ""
|
||||
if os.path.isabs(model_root) and dest_path.startswith(model_root):
|
||||
rel = os.path.relpath(os.path.dirname(dest_path), model_root)
|
||||
folder = rel.replace(os.sep, "/") if rel != "." else ""
|
||||
|
||||
# 5. Add to scanner cache (same as CivitAI's _execute_download does)
|
||||
if scanner is not None:
|
||||
metadata_dict = normalize_metadata_source(metadata.to_dict())
|
||||
await scanner.add_model_to_cache(metadata_dict, folder)
|
||||
logger.info("Added %s to scanner cache (folder=%s)", dest_path, folder)
|
||||
|
||||
# 6. Top up from the site's public API. Runs last so the scanner-cache
|
||||
# refresh it performs lands on the entry created above. It never
|
||||
# raises and never fails the download.
|
||||
await _report_phase(download_id, "source", ref.platform)
|
||||
await hydrate_from_source(dest_path, ref=ref)
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to save source metadata for %s: %s", dest_path, exc)
|
||||
|
||||
|
||||
def _find_matching_root(dest_dir: str) -> str | None:
|
||||
"""Walk up *dest_dir* to find which configured scanner root it belongs to."""
|
||||
norm = os.path.normpath(dest_dir).replace(os.sep, "/")
|
||||
all_roots = []
|
||||
for root_list in (
|
||||
config.loras_roots or [],
|
||||
config.extra_loras_roots or [],
|
||||
config.checkpoints_roots or [],
|
||||
config.extra_checkpoints_roots or [],
|
||||
config.unet_roots or [],
|
||||
config.extra_unet_roots or [],
|
||||
config.embeddings_roots or [],
|
||||
config.extra_embeddings_roots or [],
|
||||
):
|
||||
all_roots.extend([os.path.normpath(p).replace(os.sep, "/") for p in root_list])
|
||||
# Find the longest matching prefix
|
||||
match: str | None = None
|
||||
for root in all_roots:
|
||||
if norm.startswith(root):
|
||||
if match is None or len(root) > len(match):
|
||||
match = root
|
||||
return match
|
||||
|
||||
|
||||
async def _add_to_scanner_cache(dest_path: str, metadata: dict[str, Any]) -> None:
|
||||
model_dir = os.path.dirname(dest_path)
|
||||
model_root = _find_matching_root(model_dir)
|
||||
if not model_root:
|
||||
raise ValueError(f"File path {dest_path} is not within any configured scanner root")
|
||||
scanner_getter_name = _infer_model_type(model_root)[1]
|
||||
scanner_getter = getattr(ServiceRegistry, scanner_getter_name, None)
|
||||
if scanner_getter is None:
|
||||
raise RuntimeError(f"Scanner getter '{scanner_getter_name}' not found in ServiceRegistry")
|
||||
scanner = await scanner_getter()
|
||||
if scanner is None:
|
||||
raise RuntimeError(f"Scanner '{scanner_getter_name}' returned None")
|
||||
await scanner.update_single_model_cache(dest_path, dest_path, metadata)
|
||||
|
||||
|
||||
def _unsupported_platform_error(platform: str) -> web.Response:
|
||||
supported = ", ".join(source.label for source in list_sources() if source.supports_download)
|
||||
return web.json_response(
|
||||
{"error": f"'{platform}' does not support downloads. Supported: {supported}"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
|
||||
class ModelSourceHandler:
|
||||
"""Handle external model browsing, linking and downloads."""
|
||||
|
||||
async def get_model_sources(self, request: web.Request) -> web.Response:
|
||||
"""List the external model sites the UI can link a model to.
|
||||
|
||||
Used by the "Link Model" dialog to validate URLs client-side, to
|
||||
explain which sites support AI metadata enrichment, and to pick the
|
||||
right download endpoint/revision.
|
||||
"""
|
||||
|
||||
return web.json_response([
|
||||
{
|
||||
"platform": source.platform,
|
||||
"label": source.label,
|
||||
"supports_enrichment": source.supports_enrichment,
|
||||
"supports_download": source.supports_download,
|
||||
"default_revision": source.default_revision,
|
||||
"example_url": source.canonical_url(
|
||||
"user/repo" if source.platform != "tensorart" else "827823520299086029"
|
||||
),
|
||||
}
|
||||
for source in list_sources()
|
||||
])
|
||||
|
||||
async def set_hf_url(self, request: web.Request) -> web.Response:
|
||||
"""Link a model file to its page on an external model site.
|
||||
|
||||
Accepts ``source_url`` (preferred) or the legacy ``hf_url`` / ``url``
|
||||
payload key. Every registered site is recognised and the platform is
|
||||
stored alongside the canonical URL. TensorArt models can be linked and
|
||||
browsed, but not AI-enriched.
|
||||
|
||||
The route path keeps its historical ``set-hf-url`` name.
|
||||
"""
|
||||
|
||||
try:
|
||||
payload: dict[str, Any] = await request.json()
|
||||
except json.JSONDecodeError:
|
||||
return web.json_response({"success": False, "error": "Invalid JSON"}, status=400)
|
||||
|
||||
file_path = (payload.get("file_path") or "").strip()
|
||||
raw_url = (
|
||||
payload.get("source_url")
|
||||
or payload.get("hf_url")
|
||||
or payload.get("url")
|
||||
or ""
|
||||
)
|
||||
source_url = raw_url.strip() if isinstance(raw_url, str) else ""
|
||||
|
||||
if not file_path or not source_url:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "Missing required fields: 'file_path' and 'source_url'",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
|
||||
ref = detect_source(source_url, strict=True)
|
||||
if ref is None:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": (
|
||||
"Unsupported model URL. Supported formats: "
|
||||
+ ", ".join(
|
||||
f"{s.label} ({s.canonical_url('user/repo')})"
|
||||
if s.platform != "tensorart"
|
||||
else f"{s.label} (https://tensor.art/models/<id>)"
|
||||
for s in list_sources()
|
||||
)
|
||||
),
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
|
||||
if not os.path.isfile(file_path):
|
||||
return web.json_response(
|
||||
{"success": False, "error": f"File not found: {file_path}"},
|
||||
status=404,
|
||||
)
|
||||
|
||||
model_root = _find_matching_root(os.path.dirname(file_path))
|
||||
if not model_root:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "File is not within any configured model directory. Cannot link to a model source.",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
|
||||
try:
|
||||
existing = await MetadataManager.load_metadata_payload(file_path)
|
||||
|
||||
already_linked = (
|
||||
(existing.get("source_url") or "").strip() == ref.url
|
||||
and (existing.get("source_platform") or "").strip().lower()
|
||||
== ref.platform
|
||||
) or (
|
||||
not existing.get("source_url")
|
||||
and ref.platform == "huggingface"
|
||||
and (existing.get("hf_url") or "").strip() == ref.url
|
||||
)
|
||||
if already_linked:
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": "source_url already set",
|
||||
"source_url": ref.url,
|
||||
"source_platform": ref.platform,
|
||||
"hf_url": ref.url if ref.platform == "huggingface" else "",
|
||||
})
|
||||
|
||||
existing["source_url"] = ref.url
|
||||
existing["source_platform"] = ref.platform
|
||||
if ref.platform == "huggingface":
|
||||
existing["hf_url"] = ref.url
|
||||
else:
|
||||
existing.pop("hf_url", None)
|
||||
normalize_metadata_source(existing)
|
||||
|
||||
# NOTE: deliberately do NOT touch `from_civitai` here. It records
|
||||
# where the metadata came from, and the UI must show the CivitAI
|
||||
# link whenever CivitAI data is present — linking an external
|
||||
# source must not hide it (#1094). Source provenance is tracked
|
||||
# via `source_platform` / `source_url`.
|
||||
await MetadataManager.save_metadata(file_path, existing)
|
||||
|
||||
await _add_to_scanner_cache(file_path, existing)
|
||||
|
||||
logger.info(
|
||||
"Linked %s to %s source (%s)", file_path, ref.platform, ref.url
|
||||
)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"Linked to {ref.url}",
|
||||
"source_url": ref.url,
|
||||
"source_platform": ref.platform,
|
||||
"hf_url": existing.get("hf_url", ""),
|
||||
})
|
||||
except Exception as exc:
|
||||
logger.error("Failed to link %s to a model source: %s", file_path, exc)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(exc)},
|
||||
status=500,
|
||||
)
|
||||
|
||||
async def list_model_source_files(self, request: web.Request) -> web.Response:
|
||||
"""List the downloadable weight files of an external repository.
|
||||
|
||||
Query params: ``platform``, ``repo`` (``owner/name``), ``revision``
|
||||
(optional; each site has its own default branch).
|
||||
|
||||
Returns a JSON array of ``{"filename", "size"}``, largest first —
|
||||
the same shape the Hugging Face endpoint has always returned.
|
||||
"""
|
||||
|
||||
platform = (request.query.get("platform") or "").strip()
|
||||
repo = (request.query.get("repo") or "").strip()
|
||||
revision = (request.query.get("revision") or "").strip()
|
||||
|
||||
source = get_download_source(platform)
|
||||
if source is None:
|
||||
return _unsupported_platform_error(platform)
|
||||
if not is_valid_source_id(repo):
|
||||
return web.json_response(
|
||||
{"error": "Missing or invalid 'repo' parameter (expected owner/name)"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
try:
|
||||
files = await source.list_files(repo, revision)
|
||||
except ModelSourceError as exc:
|
||||
return web.json_response({"error": str(exc)}, status=exc.status)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to list %s files in %s: %s", platform, repo, exc)
|
||||
return web.json_response({"error": str(exc)}, status=502)
|
||||
|
||||
return web.json_response(files)
|
||||
|
||||
async def download_model_source(self, request: web.Request) -> web.Response:
|
||||
"""Download a single file from an external repository.
|
||||
|
||||
POST JSON body::
|
||||
|
||||
{
|
||||
"platform": "modelscope",
|
||||
"repo": "owner/name",
|
||||
"filename": "subdir/model.safetensors",
|
||||
"revision": "master",
|
||||
"model_root": "loras",
|
||||
"relative_path": "",
|
||||
"use_default_paths": false,
|
||||
"download_id": "optional-batch-id"
|
||||
}
|
||||
|
||||
``platform`` defaults to ``huggingface`` when omitted, which keeps the
|
||||
legacy ``/api/lm/download-hf-model`` payload working unchanged.
|
||||
|
||||
If ``download_id`` is provided, real-time progress (bytes, speed,
|
||||
percentage) is broadcast via the WebSocket progress system.
|
||||
|
||||
Respects the ``download_backend`` setting (``aria2`` or ``default``).
|
||||
"""
|
||||
try:
|
||||
payload: dict[str, Any] = await request.json()
|
||||
except json.JSONDecodeError:
|
||||
return web.json_response({"error": "Invalid JSON"}, status=400)
|
||||
|
||||
platform = (payload.get("platform") or "huggingface").strip()
|
||||
repo = (payload.get("repo") or "").strip()
|
||||
filename = (payload.get("filename") or "").strip()
|
||||
revision = (payload.get("revision") or "").strip()
|
||||
model_root = (payload.get("model_root") or "").strip()
|
||||
relative_path = (payload.get("relative_path") or "").strip()
|
||||
use_default_paths = bool(payload.get("use_default_paths", False))
|
||||
download_id: str | None = payload.get("download_id")
|
||||
|
||||
logger.info(
|
||||
"download_model_source: platform=%s repo=%s file=%s root=%s download_id=%s",
|
||||
platform, repo, filename, model_root, download_id,
|
||||
)
|
||||
|
||||
source = get_download_source(platform)
|
||||
if source is None:
|
||||
return _unsupported_platform_error(platform)
|
||||
|
||||
if not repo or not filename:
|
||||
return web.json_response(
|
||||
{"error": "Missing required fields: 'repo' and 'filename'"}, status=400
|
||||
)
|
||||
|
||||
# `owner/name` only; the components become path segments below.
|
||||
if not is_valid_source_id(repo):
|
||||
return web.json_response({"error": f"Invalid repo format: {repo}"}, status=400)
|
||||
owner, repo_name = repo.split("/", 1)
|
||||
|
||||
# Validate filename — must not contain path traversal
|
||||
if ".." in filename:
|
||||
return web.json_response({"error": "Invalid filename"}, status=400)
|
||||
|
||||
# Validate relative_path — must not be absolute or escape base directory
|
||||
if relative_path:
|
||||
if os.path.isabs(relative_path):
|
||||
return web.json_response({"error": "relative_path must not be absolute"}, status=400)
|
||||
if ".." in relative_path.split("/") or "\\" in relative_path:
|
||||
return web.json_response({"error": "Invalid relative_path"}, status=400)
|
||||
|
||||
# Use model_root directly as the base directory — same approach as
|
||||
# CivitAI's download path (download_manager.py). No realpath, no
|
||||
# allowed-roots validation, no path-traversal check; those are
|
||||
# unnecessary when the frontend sends the path from its own dropdown
|
||||
# (populated from scanner roots). Using the "business path" directly
|
||||
# keeps dest_path consistent with scanner roots so that later folder
|
||||
# derivation (in _save_source_metadata) works correctly.
|
||||
if os.path.isabs(model_root):
|
||||
base_dir = os.path.normpath(model_root)
|
||||
else:
|
||||
base_dir = os.path.normpath(os.path.join(os.getcwd(), "models", model_root))
|
||||
|
||||
if use_default_paths:
|
||||
target_dir = os.path.join(base_dir, source.default_subdir, owner, repo_name)
|
||||
elif relative_path:
|
||||
target_dir = os.path.join(base_dir, relative_path)
|
||||
else:
|
||||
target_dir = base_dir
|
||||
|
||||
# Strip the repository sub-directory — "diffusion_models/xxx.safetensors"
|
||||
# is a repository convention, not meaningful for local storage.
|
||||
file_base = os.path.basename(filename)
|
||||
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
dest_path = os.path.join(target_dir, file_base)
|
||||
|
||||
# Built per request: sites that redirect to a CDN hand out a
|
||||
# time-limited token in the redirect, so the URL must never be cached.
|
||||
resolve_url = source.file_download_url(repo, filename, revision)
|
||||
ref = SourceRef(
|
||||
platform=source.platform, source_id=repo, url=source.canonical_url(repo)
|
||||
)
|
||||
|
||||
# Check if already exists (simple skip)
|
||||
if os.path.exists(dest_path) and os.path.getsize(dest_path) > 0:
|
||||
logger.info("download_model_source: file already exists, skipping — %s", dest_path)
|
||||
# The sidecar may predate the source metadata being fetched, or may
|
||||
# have been deleted, so top it up instead of skipping past it.
|
||||
# Hydration no-ops when there is no sidecar to update.
|
||||
await _report_phase(download_id, "source", source.platform)
|
||||
await hydrate_from_source(dest_path, ref=ref)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"File already exists: {dest_path}",
|
||||
"path": dest_path,
|
||||
})
|
||||
|
||||
# Set up progress callback if download_id is provided
|
||||
progress_callback = None
|
||||
if download_id:
|
||||
|
||||
async def _progress_callback(
|
||||
progress: float | DownloadProgress,
|
||||
snapshot: DownloadProgress | None = None,
|
||||
) -> None:
|
||||
percent = 0.0
|
||||
metrics = snapshot if isinstance(snapshot, DownloadProgress) else None
|
||||
|
||||
if isinstance(progress, DownloadProgress):
|
||||
percent = progress.percent_complete
|
||||
metrics = progress
|
||||
elif isinstance(snapshot, DownloadProgress):
|
||||
percent = snapshot.percent_complete
|
||||
else:
|
||||
percent = float(progress)
|
||||
|
||||
broadcast: dict[str, Any] = {
|
||||
"status": "progress",
|
||||
"progress": round(percent),
|
||||
}
|
||||
if metrics:
|
||||
broadcast["bytes_downloaded"] = metrics.bytes_downloaded
|
||||
broadcast["total_bytes"] = metrics.total_bytes
|
||||
broadcast["bytes_per_second"] = metrics.bytes_per_second
|
||||
|
||||
await ws_manager.broadcast_download_progress(download_id, broadcast)
|
||||
|
||||
progress_callback = _progress_callback
|
||||
|
||||
# Respect download backend setting (aria2 vs default)
|
||||
download_backend = (
|
||||
get_settings_manager().get("download_backend", "default")
|
||||
)
|
||||
|
||||
if download_backend == "aria2":
|
||||
aria2 = await Aria2Downloader.get_instance()
|
||||
aid = download_id or f"{source.platform}_{repo}_{filename}"
|
||||
try:
|
||||
ok, result = await aria2.download_file(
|
||||
url=resolve_url,
|
||||
save_path=dest_path,
|
||||
download_id=aid,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
if ok:
|
||||
await _save_source_metadata(
|
||||
dest_path, ref, model_root, download_id=download_id
|
||||
)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"Downloaded to {dest_path}",
|
||||
"path": dest_path,
|
||||
})
|
||||
return web.json_response(
|
||||
{"success": False, "error": result or "aria2 download failed"},
|
||||
status=500,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("%s download (aria2) failed: %s", platform, exc)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(exc)}, status=500
|
||||
)
|
||||
|
||||
# Default: use built-in aiohttp Downloader
|
||||
downloader = await get_downloader()
|
||||
try:
|
||||
success, result = await downloader.download_file(
|
||||
url=resolve_url,
|
||||
save_path=dest_path,
|
||||
use_auth=False,
|
||||
allow_resume=True,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
if success:
|
||||
await _save_source_metadata(
|
||||
dest_path, ref, model_root, download_id=download_id
|
||||
)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"Downloaded to {result}",
|
||||
"path": result,
|
||||
})
|
||||
return web.json_response(
|
||||
{"success": False, "error": result or "Download failed"},
|
||||
status=500,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("%s download failed: %s", platform, exc)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(exc)}, status=500
|
||||
)
|
||||
@@ -0,0 +1,324 @@
|
||||
"""Handler for the pending-delete undo endpoint.
|
||||
|
||||
Restores a staged delete batch (models or recipes) via
|
||||
``PendingDeleteService.undo`` and then repairs the affected library caches:
|
||||
the model cache entry is restored from the manifest's ``model_snapshot``
|
||||
(including the version index and hash index), tag counts are re-incremented,
|
||||
and the recipe cache is re-populated via ``RecipeScanner.add_recipe``.
|
||||
|
||||
The per-type scanner is resolved from the manifest's ``model_type`` page value
|
||||
through the SAME ServiceRegistry getters the model route registrars use
|
||||
(lora/checkpoint/embedding) - never a hardcoded lora scanner.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Any, Awaitable, Callable, Dict, List, Optional, Set, cast
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from ...services.pending_delete_service import get_pending_delete_service
|
||||
from .model_handlers import _broadcast_models_changed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Manifest ``model_type`` page values -> ServiceRegistry scanner getter names.
|
||||
# The model route registrars resolve per-type scanners via these getters
|
||||
# (lora_routes / checkpoint_routes / embedding_routes); undo must do the same
|
||||
# so the CORRECT cache is restored for the deleted model's type.
|
||||
_MODEL_TYPE_GETTER_NAMES: Dict[str, str] = {
|
||||
"loras": "get_lora_scanner",
|
||||
"checkpoints": "get_checkpoint_scanner",
|
||||
"embeddings": "get_embedding_scanner",
|
||||
"other": "get_other_scanner",
|
||||
}
|
||||
|
||||
# Staged batch ids are ``uuid.uuid4().hex`` (32 lowercase hex chars). The id is
|
||||
# joined into filesystem paths by ``_find_batch_dir``, so reject anything that
|
||||
# does not match this exact shape (blocks path-traversal via batch_id).
|
||||
_BATCH_ID_RE = re.compile(r"^[0-9a-f]{32}$")
|
||||
|
||||
|
||||
class PendingDeleteHandler:
|
||||
"""Handle undo requests for staged model/recipe deletions."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
service_factory: Callable[[], Awaitable[Any]] = get_pending_delete_service,
|
||||
scanner_getter: Optional[Callable[[str], Awaitable[Any]]] = None,
|
||||
recipe_scanner_getter: Optional[Callable[[], Awaitable[Any]]] = None,
|
||||
) -> None:
|
||||
self._service_factory: Callable[[], Awaitable[Any]] = service_factory
|
||||
self._scanner_getter: Callable[[str], Awaitable[Any]] = (
|
||||
scanner_getter or self._resolve_scanner
|
||||
)
|
||||
self._recipe_scanner_getter: Callable[[], Awaitable[Any]] = (
|
||||
recipe_scanner_getter or self._resolve_recipe_scanner
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _resolve_scanner(model_type: str) -> Any:
|
||||
"""Resolve the per-type scanner for a manifest ``model_type``.
|
||||
|
||||
The getter is looked up on the ServiceRegistry module namespace at call
|
||||
time so tests (and the registry stubs) can patch it.
|
||||
"""
|
||||
from ...services import service_registry
|
||||
|
||||
getter_name = _MODEL_TYPE_GETTER_NAMES.get(model_type)
|
||||
if getter_name is None:
|
||||
raise ValueError(f"Unknown model type: {model_type}")
|
||||
getter = getattr(service_registry.ServiceRegistry, getter_name, None)
|
||||
if not callable(getter):
|
||||
raise ValueError(f"No scanner getter for model type: {model_type}")
|
||||
scanner = await cast(Callable[[], Awaitable[Any]], getter)()
|
||||
if scanner is None:
|
||||
raise ValueError(f"No scanner registered for model type: {model_type}")
|
||||
return scanner
|
||||
|
||||
@staticmethod
|
||||
async def _resolve_recipe_scanner() -> Any:
|
||||
"""Resolve the recipe scanner via the ServiceRegistry module namespace."""
|
||||
from ...services import service_registry
|
||||
|
||||
getter = getattr(service_registry.ServiceRegistry, "get_recipe_scanner", None)
|
||||
if not callable(getter):
|
||||
raise ValueError("Recipe scanner getter unavailable")
|
||||
scanner = await cast(Callable[[], Awaitable[Any]], getter)()
|
||||
if scanner is None:
|
||||
raise ValueError("No recipe scanner registered")
|
||||
return scanner
|
||||
|
||||
async def undo_delete(self, request: web.Request) -> web.Response:
|
||||
"""Restore a staged batch and its library cache entry.
|
||||
|
||||
Body: ``{"batch_id": str}``. On success returns
|
||||
``{"success": True, "restored": [<original paths>], "kind": kind}``.
|
||||
Expired/unknown batches and occupied target paths -> 404.
|
||||
"""
|
||||
try:
|
||||
data = await request.json()
|
||||
except Exception:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Invalid JSON body"}, status=400
|
||||
)
|
||||
if not isinstance(data, dict):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Invalid JSON body"}, status=400
|
||||
)
|
||||
batch_id = data.get("batch_id")
|
||||
if not batch_id or not isinstance(batch_id, str):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "batch_id is required"}, status=400
|
||||
)
|
||||
if not _BATCH_ID_RE.fullmatch(batch_id):
|
||||
# batch_id is joined into a path by _find_batch_dir - restrict to
|
||||
# the exact staged-id shape so traversal payloads get 400.
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Invalid batch_id"}, status=400
|
||||
)
|
||||
|
||||
service = await self._service_factory()
|
||||
try:
|
||||
# Read the manifest BEFORE undo: undo() removes the batch dir.
|
||||
manifest = await self._read_staged_manifest(service, batch_id)
|
||||
result = await service.undo(batch_id)
|
||||
except ValueError as exc:
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=404)
|
||||
except Exception as exc:
|
||||
logger.error("Unexpected error undoing batch %s: %s", batch_id, exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
kind = result.get("kind")
|
||||
try:
|
||||
if kind == "model":
|
||||
if manifest is not None:
|
||||
await self._restore_model_cache(manifest)
|
||||
else:
|
||||
# undo() raises when the manifest is missing, so this only
|
||||
# happens defensively - files are restored regardless.
|
||||
logger.warning(
|
||||
"Manifest missing after undo of %s; skipping cache restore",
|
||||
batch_id,
|
||||
)
|
||||
_broadcast_models_changed()
|
||||
elif kind == "recipe":
|
||||
# Recipe undo is client-refresh only: re-add to the scanner
|
||||
# cache, no models_changed broadcast.
|
||||
if manifest is not None:
|
||||
await self._restore_recipe_cache(result, manifest)
|
||||
else:
|
||||
logger.warning(
|
||||
"Manifest missing after undo of %s; skipping cache restore",
|
||||
batch_id,
|
||||
)
|
||||
except Exception as exc:
|
||||
# Files are already restored; only the cache restoration failed.
|
||||
logger.error(
|
||||
"Cache restoration failed after undo of %s: %s",
|
||||
batch_id,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"restored": result.get("restored", []),
|
||||
"kind": kind,
|
||||
}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _read_staged_manifest(
|
||||
service: Any, batch_id: str
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Locate and read the batch manifest while it still exists on disk."""
|
||||
batch_dir = await service._find_batch_dir(batch_id)
|
||||
if not batch_dir:
|
||||
return None
|
||||
manifest_path = os.path.join(batch_dir, "manifest.json")
|
||||
try:
|
||||
with open(manifest_path, "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
logger.debug("Failed to read manifest for batch %s: %s", batch_id, exc)
|
||||
return None
|
||||
return payload if isinstance(payload, dict) else None
|
||||
|
||||
async def _restore_model_cache(self, manifest: Dict[str, Any]) -> None:
|
||||
"""Re-add every deleted model's cache entry from the manifest.
|
||||
|
||||
Each main-file entry carries the deleted model's ``snapshot`` (added at
|
||||
stage time), so a merged bulk manifest holds ALL snapshots - undo must
|
||||
restore every one, not just the top-level winner's. Old-format
|
||||
manifests without entry snapshots fall back to the top-level
|
||||
``model_snapshot`` (backward compat / single-delete path).
|
||||
"""
|
||||
model_type = manifest.get("model_type")
|
||||
if not model_type or not isinstance(model_type, str):
|
||||
raise ValueError(f"Manifest carries no model_type: {manifest.get('batch_id')}")
|
||||
scanner = await self._scanner_getter(model_type)
|
||||
|
||||
# Collect one snapshot per distinct file_path from the entry snapshots.
|
||||
snapshots: List[Dict[str, Any]] = []
|
||||
seen: Set[str] = set()
|
||||
for entry in manifest.get("entries") or []:
|
||||
snapshot = entry.get("snapshot")
|
||||
if not isinstance(snapshot, dict):
|
||||
continue
|
||||
file_path = snapshot.get("file_path")
|
||||
if not file_path or not isinstance(file_path, str):
|
||||
continue
|
||||
if file_path in seen:
|
||||
continue
|
||||
seen.add(file_path)
|
||||
snapshots.append(snapshot)
|
||||
|
||||
if not snapshots:
|
||||
# Backward compat: pre-F3 manifests carry only the top-level
|
||||
# model_snapshot (single-delete path, unchanged behavior).
|
||||
top = manifest.get("model_snapshot")
|
||||
if isinstance(top, dict) and top.get("file_path"):
|
||||
snapshots = [top]
|
||||
else:
|
||||
logger.warning(
|
||||
"Manifest %s has no restorable model snapshot; skipping cache restore",
|
||||
manifest.get("batch_id"),
|
||||
)
|
||||
return
|
||||
|
||||
cache = await scanner.get_cached_data()
|
||||
if cache is None:
|
||||
logger.warning(
|
||||
"Scanner cache unavailable for %s; skipping cache restore", model_type
|
||||
)
|
||||
return
|
||||
|
||||
for snapshot in snapshots:
|
||||
file_path = str(snapshot["file_path"])
|
||||
# A rescan between delete and undo may have re-added a stale entry
|
||||
# for this path - drop it so exactly one (the snapshot) remains.
|
||||
cache.raw_data = [
|
||||
item for item in cache.raw_data if item.get("file_path") != file_path
|
||||
]
|
||||
|
||||
# Restore tag counts (mirror of the bulk-delete decrement in
|
||||
# _batch_update_cache_for_deleted_models: undo re-increments).
|
||||
tags = snapshot.get("tags")
|
||||
if isinstance(tags, list):
|
||||
for tag in tags:
|
||||
if not isinstance(tag, str) or not tag:
|
||||
continue
|
||||
scanner._tags_count[tag] = scanner._tags_count.get(tag, 0) + 1
|
||||
|
||||
cache.raw_data.append(dict(snapshot))
|
||||
|
||||
# Re-register the path in the hash index (add_entry guards a
|
||||
# missing sha256 internally; still guard defensively here).
|
||||
sha256 = snapshot.get("sha256") or ""
|
||||
autov3 = snapshot.get("autov3")
|
||||
hash_index = getattr(scanner, "_hash_index", None)
|
||||
if hash_index is not None and sha256 and file_path:
|
||||
hash_index.add_entry(sha256, file_path, autov3)
|
||||
|
||||
# Follow the bulk-delete cache-update pattern ONCE after all entries,
|
||||
# including the explicit version-index rebuild so the version index
|
||||
# does not go stale.
|
||||
cache.rebuild_version_index()
|
||||
await cache.resort()
|
||||
|
||||
scanner.bump_cache_version()
|
||||
|
||||
persist = getattr(scanner, "_persist_current_cache", None)
|
||||
if callable(persist):
|
||||
result = persist()
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
|
||||
async def _restore_recipe_cache(
|
||||
self, result: Dict[str, Any], manifest: Dict[str, Any]
|
||||
) -> None:
|
||||
"""Re-add a restored recipe via ``RecipeScanner.add_recipe``.
|
||||
|
||||
The recipe JSON embeds the full recipe_data (incl. id/file_path);
|
||||
``add_recipe`` only READS the ``_json_path_map`` so the forced frontend
|
||||
refresh self-heals any transient path-map gap.
|
||||
"""
|
||||
restored = result.get("restored") or []
|
||||
json_path = next(
|
||||
(p for p in restored if isinstance(p, str) and p.endswith(".json")),
|
||||
None,
|
||||
)
|
||||
if not json_path or not os.path.exists(json_path):
|
||||
# Defensive fallback to the manifest's recipe_snapshot file_path.
|
||||
snapshot = manifest.get("recipe_snapshot") or {}
|
||||
fallback = snapshot.get("file_path")
|
||||
if fallback and os.path.exists(fallback):
|
||||
json_path = fallback
|
||||
else:
|
||||
logger.warning(
|
||||
"Restored recipe JSON not found in %s; skipping cache restore",
|
||||
restored,
|
||||
)
|
||||
return
|
||||
try:
|
||||
with open(json_path, "r", encoding="utf-8") as handle:
|
||||
recipe_data = json.load(handle)
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
logger.warning("Failed to load restored recipe JSON %s: %s", json_path, exc)
|
||||
return
|
||||
if not isinstance(recipe_data, dict):
|
||||
return
|
||||
recipe_scanner = await self._recipe_scanner_getter()
|
||||
await recipe_scanner.add_recipe(recipe_data)
|
||||
|
||||
|
||||
__all__ = ["PendingDeleteHandler"]
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import mimetypes
|
||||
import urllib.parse
|
||||
@@ -53,6 +54,7 @@ class PreviewHandler:
|
||||
|
||||
if not resolved.is_file():
|
||||
logger.debug("Preview file not found at %s", str(resolved))
|
||||
asyncio.create_task(self._cleanup_stale_preview_url(normalized))
|
||||
raise web.HTTPNotFound(text="Preview file not found")
|
||||
|
||||
# aiohttp's FileResponse handles range requests, content headers, and
|
||||
@@ -69,6 +71,35 @@ class PreviewHandler:
|
||||
resp.headers["Cache-Control"] = "public, max-age=86400"
|
||||
return resp
|
||||
|
||||
async def _cleanup_stale_preview_url(self, normalized_preview_path: str) -> None:
|
||||
"""Fire-and-forget: clear stale preview_url from all model caches.
|
||||
|
||||
When a preview file is no longer on disk, remove its reference from
|
||||
every cached entry so subsequent list API responses return an empty
|
||||
``preview_url``, letting the frontend show the no-preview placeholder.
|
||||
"""
|
||||
try:
|
||||
from ...services.service_registry import ServiceRegistry
|
||||
|
||||
for service_name in ("lora_scanner", "checkpoint_scanner", "embedding_scanner"):
|
||||
scanner = ServiceRegistry.get_service_sync(service_name)
|
||||
if scanner is None or not hasattr(scanner, "_cache"):
|
||||
continue
|
||||
cache = getattr(scanner, "_cache", None)
|
||||
if cache is None or not hasattr(cache, "clear_preview_by_path"):
|
||||
continue
|
||||
cleared = await cache.clear_preview_by_path(normalized_preview_path)
|
||||
if cleared and hasattr(scanner, "_persist_current_cache"):
|
||||
await scanner._persist_current_cache()
|
||||
logger.info(
|
||||
"Cleared stale preview_url for %d %s entries (%s)",
|
||||
cleared,
|
||||
service_name,
|
||||
normalized_preview_path,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to clean up stale preview_url: %s", exc)
|
||||
|
||||
async def _stream_file(
|
||||
self, request: web.Request, path: Path
|
||||
) -> web.StreamResponse:
|
||||
|
||||
+1095
-290
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,8 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from aiohttp import web
|
||||
from typing import Dict
|
||||
from server import PromptServer # type: ignore
|
||||
from typing import Any, Dict
|
||||
from server import PromptServer # pyright: ignore[reportMissingImports]
|
||||
|
||||
from .base_model_routes import BaseModelRoutes
|
||||
from .model_route_registrar import ModelRouteRegistrar
|
||||
@@ -31,13 +31,13 @@ class LoraRoutes(BaseModelRoutes):
|
||||
# Attach service dependencies
|
||||
self.attach_service(self.service)
|
||||
|
||||
def setup_routes(self, app: web.Application):
|
||||
def setup_routes(self, app: web.Application, prefix: str = "loras"):
|
||||
"""Setup LoRA routes"""
|
||||
# Schedule service initialization on app startup
|
||||
app.on_startup.append(lambda _: self.initialize_services())
|
||||
|
||||
# Setup common routes with 'loras' prefix (includes page route)
|
||||
super().setup_routes(app, "loras")
|
||||
super().setup_routes(app, prefix)
|
||||
|
||||
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
||||
"""Setup LoRA-specific routes"""
|
||||
@@ -73,7 +73,7 @@ class LoraRoutes(BaseModelRoutes):
|
||||
"POST", "/api/lm/{prefix}/get_trigger_words", prefix, self.get_trigger_words
|
||||
)
|
||||
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict:
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict[str, Any]:
|
||||
"""Parse LoRA-specific parameters"""
|
||||
params = {}
|
||||
|
||||
@@ -119,25 +119,6 @@ class LoraRoutes(BaseModelRoutes):
|
||||
logger.error(f"Error getting letter counts: {e}")
|
||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||
|
||||
async def get_lora_notes(self, request: web.Request) -> web.Response:
|
||||
"""Get notes for a specific LoRA file"""
|
||||
try:
|
||||
lora_name = request.query.get("name")
|
||||
if not lora_name:
|
||||
return web.Response(text="Lora file name is required", status=400)
|
||||
|
||||
notes = await self.service.get_lora_notes(lora_name)
|
||||
if notes is not None:
|
||||
return web.json_response({"success": True, "notes": notes})
|
||||
else:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "LoRA not found in cache"}, status=404
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting lora notes: {e}", exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||
|
||||
async def get_lora_trigger_words(self, request: web.Request) -> web.Response:
|
||||
"""Get trigger words for a specific LoRA file"""
|
||||
try:
|
||||
@@ -168,52 +149,6 @@ class LoraRoutes(BaseModelRoutes):
|
||||
logger.error(f"Error getting lora usage tips by path: {e}", exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||
|
||||
async def get_lora_preview_url(self, request: web.Request) -> web.Response:
|
||||
"""Get the static preview URL for a LoRA file"""
|
||||
try:
|
||||
lora_name = request.query.get("name")
|
||||
if not lora_name:
|
||||
return web.Response(text="Lora file name is required", status=400)
|
||||
|
||||
preview_url = await self.service.get_lora_preview_url(lora_name)
|
||||
if preview_url:
|
||||
return web.json_response({"success": True, "preview_url": preview_url})
|
||||
else:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "No preview URL found for the specified lora",
|
||||
},
|
||||
status=404,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting lora preview URL: {e}", exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||
|
||||
async def get_lora_civitai_url(self, request: web.Request) -> web.Response:
|
||||
"""Get the Civitai URL for a LoRA file"""
|
||||
try:
|
||||
lora_name = request.query.get("name")
|
||||
if not lora_name:
|
||||
return web.Response(text="Lora file name is required", status=400)
|
||||
|
||||
result = await self.service.get_lora_civitai_url(lora_name)
|
||||
if result["civitai_url"]:
|
||||
return web.json_response({"success": True, **result})
|
||||
else:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "No Civitai data found for the specified lora",
|
||||
},
|
||||
status=404,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting lora Civitai URL: {e}", exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||
|
||||
async def get_random_loras(self, request: web.Request) -> web.Response:
|
||||
"""Get random LoRAs based on filters and strength ranges"""
|
||||
try:
|
||||
@@ -337,7 +272,7 @@ class LoraRoutes(BaseModelRoutes):
|
||||
graph_identifier = entry.get("graph_id")
|
||||
|
||||
try:
|
||||
parsed_node_id = int(node_identifier)
|
||||
parsed_node_id = int(node_identifier) # pyright: ignore[reportArgumentType]
|
||||
except (TypeError, ValueError):
|
||||
parsed_node_id = node_identifier
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ miscellaneous endpoints share a consistent registration flow.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Iterable, Mapping
|
||||
from typing import Any, Callable, Iterable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -22,6 +22,8 @@ class RouteDefinition:
|
||||
MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("GET", "/api/lm/settings", "get_settings"),
|
||||
RouteDefinition("POST", "/api/lm/settings", "update_settings"),
|
||||
RouteDefinition("GET", "/api/lm/llm/models", "get_llm_models"),
|
||||
RouteDefinition("GET", "/api/lm/llm/provider-models", "get_provider_models"),
|
||||
RouteDefinition("GET", "/api/lm/doctor/diagnostics", "get_doctor_diagnostics"),
|
||||
RouteDefinition("POST", "/api/lm/doctor/repair-cache", "repair_doctor_cache"),
|
||||
RouteDefinition("POST", "/api/lm/doctor/resolve-filename-conflicts", "resolve_doctor_filename_conflicts"),
|
||||
@@ -30,6 +32,7 @@ MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("GET", "/api/lm/settings/libraries", "get_settings_libraries"),
|
||||
RouteDefinition("POST", "/api/lm/settings/libraries/activate", "activate_library"),
|
||||
RouteDefinition("GET", "/api/lm/health-check", "health_check"),
|
||||
RouteDefinition("GET", "/api/lm/init-status", "get_init_status"),
|
||||
RouteDefinition("GET", "/api/lm/supporters", "get_supporters"),
|
||||
RouteDefinition("GET", "/api/lm/wildcards/search", "search_wildcards"),
|
||||
RouteDefinition("POST", "/api/lm/wildcards/open-location", "open_wildcards_location"),
|
||||
@@ -37,10 +40,12 @@ MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("POST", "/api/lm/update-usage-stats", "update_usage_stats"),
|
||||
RouteDefinition("GET", "/api/lm/get-usage-stats", "get_usage_stats"),
|
||||
RouteDefinition("POST", "/api/lm/update-lora-code", "update_lora_code"),
|
||||
RouteDefinition("GET", "/api/lm/update-lora-code", "get_update_lora_code"),
|
||||
RouteDefinition("GET", "/api/lm/trained-words", "get_trained_words"),
|
||||
RouteDefinition("GET", "/api/lm/model-example-files", "get_model_example_files"),
|
||||
RouteDefinition("POST", "/api/lm/register-nodes", "register_nodes"),
|
||||
RouteDefinition("POST", "/api/lm/update-node-widget", "update_node_widget"),
|
||||
RouteDefinition("GET", "/api/lm/update-node-widget", "get_update_node_widget"),
|
||||
RouteDefinition("GET", "/api/lm/get-registry", "get_registry"),
|
||||
RouteDefinition("GET", "/api/lm/check-model-exists", "check_model_exists"),
|
||||
RouteDefinition("GET", "/api/lm/check-models-exist", "check_models_exist"),
|
||||
@@ -94,6 +99,41 @@ MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/delete-model-version", "delete_model_version"
|
||||
),
|
||||
# External model source endpoints (Hugging Face / ModelScope).
|
||||
# The hf-* paths are the historical names, kept as aliases.
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/model-source-files", "list_model_source_files"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/hf-repo-files", "get_hf_repo_files"
|
||||
),
|
||||
# Download target routing decision (checkpoint vs diffusion model roots)
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/download/routing", "get_download_routing"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/download-model-source", "download_model_source"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/download-hf-model", "download_hf_model"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/set-hf-url", "set_hf_url"
|
||||
),
|
||||
# Supported external model sites (Hugging Face / ModelScope / TensorArt)
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/model-sources", "get_model_sources"
|
||||
),
|
||||
# Agent skill endpoints
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/agent/skills", "get_agent_skills"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/agent/execute/{skill_name}", "execute_agent_skill"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/agent/cancel", "cancel_agent_skill"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -123,7 +163,7 @@ class MiscRouteRegistrar:
|
||||
handler_lookup[definition.handler_name],
|
||||
)
|
||||
|
||||
def _bind(self, method: str, path: str, handler: Callable) -> None:
|
||||
def _bind(self, method: str, path: str, handler: Callable[..., Any]) -> None:
|
||||
add_method_name = self._METHOD_MAP[method.upper()]
|
||||
add_method = getattr(self._app.router, add_method_name)
|
||||
add_method(path, handler)
|
||||
|
||||
@@ -7,7 +7,7 @@ import os
|
||||
from typing import Awaitable, Callable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
from server import PromptServer # type: ignore
|
||||
from server import PromptServer # pyright: ignore[reportMissingImports]
|
||||
|
||||
from ..services.metadata_service import (
|
||||
get_metadata_archive_manager,
|
||||
@@ -39,6 +39,9 @@ from .handlers.misc_handlers import (
|
||||
build_service_registry_adapter,
|
||||
)
|
||||
from .handlers.base_model_handlers import BaseModelHandlerSet
|
||||
from .handlers.model_source_handlers import ModelSourceHandler
|
||||
from .handlers.agent_handlers import AgentHandler
|
||||
from .handlers.download_routing_handlers import DownloadRoutingHandler
|
||||
from .misc_route_registrar import MiscRouteRegistrar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -136,6 +139,9 @@ class MiscRoutes:
|
||||
doctor = DoctorHandler(settings_service=self._settings)
|
||||
example_workflows = ExampleWorkflowsHandler()
|
||||
base_model = BaseModelHandlerSet()
|
||||
model_source_handler = ModelSourceHandler()
|
||||
agent_handler = AgentHandler()
|
||||
download_routing = DownloadRoutingHandler()
|
||||
|
||||
return self._handler_set_factory(
|
||||
health=health,
|
||||
@@ -155,6 +161,9 @@ class MiscRoutes:
|
||||
doctor=doctor,
|
||||
example_workflows=example_workflows,
|
||||
base_model=base_model,
|
||||
model_source_handler=model_source_handler,
|
||||
agent_handler=agent_handler,
|
||||
download_routing=download_routing,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Iterable, Mapping
|
||||
from typing import Any, Callable, Iterable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -40,12 +40,16 @@ COMMON_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/verify-duplicates", "verify_duplicates"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/move_model", "move_model"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/move_models_bulk", "move_models_bulk"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/create-folder", "create_folder"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/delete-folder", "delete_folder"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/rename-folder", "rename_folder"),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/auto-organize", "auto_organize_models"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/auto-organize", "auto_organize_models"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/{prefix}/auto-organize-progress", "get_auto_organize_progress"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/top-tags", "get_top_tags"),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/search-tags", "search_tags"),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/base-models", "get_base_models"),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/model-types", "get_model_types"),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/scan", "scan_models"),
|
||||
@@ -67,6 +71,8 @@ COMMON_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
"GET", "/api/lm/{prefix}/model-description", "get_model_description"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/relative-paths", "get_relative_paths"),
|
||||
RouteDefinition("PUT", "/api/lm/{prefix}/active-filters", "update_active_filters"),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/active-filters", "get_active_filters"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/{prefix}/civitai/versions/{model_id}", "get_civitai_versions"
|
||||
),
|
||||
@@ -138,6 +144,9 @@ COMMON_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/queue/complete", "complete_download_in_queue"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/downloads/queue/status", "update_download_queue_status"
|
||||
),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/cancel-task", "cancel_task"),
|
||||
RouteDefinition("GET", "/{prefix}", "handle_models_page"),
|
||||
)
|
||||
@@ -170,15 +179,15 @@ class ModelRouteRegistrar:
|
||||
handler_lookup[definition.handler_name],
|
||||
)
|
||||
|
||||
def add_route(self, method: str, path: str, handler: Callable) -> None:
|
||||
def add_route(self, method: str, path: str, handler: Callable[..., Any]) -> None:
|
||||
self._bind_route(method, path, handler)
|
||||
|
||||
def add_prefixed_route(
|
||||
self, method: str, path_template: str, prefix: str, handler: Callable
|
||||
self, method: str, path_template: str, prefix: str, handler: Callable[..., Any]
|
||||
) -> None:
|
||||
self._bind_route(method, path_template.replace("{prefix}", prefix), handler)
|
||||
|
||||
def _bind_route(self, method: str, path: str, handler: Callable) -> None:
|
||||
def _bind_route(self, method: str, path: str, handler: Callable[..., Any]) -> None:
|
||||
add_method_name = self._METHOD_MAP[method.upper()]
|
||||
add_method = getattr(self._app.router, add_method_name)
|
||||
add_method(path, handler)
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List
|
||||
from aiohttp import web
|
||||
|
||||
from .base_model_routes import BaseModelRoutes
|
||||
from .model_route_registrar import ModelRouteRegistrar
|
||||
from ..config import config
|
||||
from ..services.other_model_service import OtherModelService
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
from ..utils.constants import (
|
||||
CIVITAI_TYPE_TO_OTHER_SUB_TYPE,
|
||||
OTHER_MODEL_FOLDER_SUBTYPES,
|
||||
VALID_OTHER_CIVITAI_TYPES,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OtherRoutes(BaseModelRoutes):
|
||||
"""Other-model-specific route controller (VAE, upscaler, text encoder, ...)"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Other-model routes with OtherModel service"""
|
||||
super().__init__()
|
||||
self.template_name = "other.html"
|
||||
|
||||
async def initialize_services(self):
|
||||
"""Initialize services from ServiceRegistry"""
|
||||
other_scanner = await ServiceRegistry.get_other_scanner()
|
||||
update_service = await ServiceRegistry.get_model_update_service()
|
||||
self.service = OtherModelService(other_scanner, update_service=update_service)
|
||||
self.set_model_update_service(update_service)
|
||||
|
||||
# Attach service dependencies
|
||||
self.attach_service(self.service)
|
||||
|
||||
def setup_routes(self, app: web.Application, prefix: str = "other"):
|
||||
"""Setup Other-model routes"""
|
||||
# Schedule service initialization on app startup
|
||||
app.on_startup.append(lambda _: self.initialize_services())
|
||||
|
||||
# Setup common routes with 'other' prefix (includes page route)
|
||||
super().setup_routes(app, prefix)
|
||||
|
||||
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
||||
"""Setup Other-model-specific routes"""
|
||||
# Other-model info by name
|
||||
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/info/{name}', prefix, self.get_other_model_info)
|
||||
# Other-model roots grouped by sub_type (text_encoders + legacy clip
|
||||
# are aggregated under text_encoder)
|
||||
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/roots_by_subtype', prefix, self.get_roots_by_subtype)
|
||||
|
||||
def _validate_civitai_model_type(self, model_type: str) -> bool:
|
||||
"""Validate CivitAI model type for other models.
|
||||
|
||||
Accepts retired CivitAI types (CLIP, CLIPVision) as well — grandfathered
|
||||
models on CivitAI still carry them. Types whose sub_type is currently
|
||||
disabled (or every type while the opt-in feature is off) are rejected.
|
||||
"""
|
||||
normalized = (model_type or "").strip().lower()
|
||||
if normalized not in VALID_OTHER_CIVITAI_TYPES:
|
||||
return False
|
||||
if not self._settings.is_other_models_enabled():
|
||||
return False
|
||||
|
||||
sub_type = CIVITAI_TYPE_TO_OTHER_SUB_TYPE.get(normalized)
|
||||
if sub_type is None:
|
||||
# CivitAI "Other" has no sub_type of its own; it is only usable
|
||||
# while at least one sub_type is enabled.
|
||||
return bool(self._settings.get_enabled_other_sub_types())
|
||||
return self._settings.is_other_sub_type_enabled(sub_type)
|
||||
|
||||
def _get_page_context_provider(self):
|
||||
"""Expose the opt-in feature state to the Other Models page template."""
|
||||
return self._page_context_for_other
|
||||
|
||||
def _page_context_for_other(self, request: web.Request) -> Dict[str, Any]:
|
||||
if not self._settings.is_other_models_enabled():
|
||||
return {"other_disabled": True, "other_no_paths": False}
|
||||
|
||||
# Enabled but nothing to scan: folder paths for the managed sub_types
|
||||
# resolved to no existing folder. Render an actionable empty state
|
||||
# instead of an apparently broken empty grid.
|
||||
standalone_mode = os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"
|
||||
return {
|
||||
"other_disabled": False,
|
||||
"other_no_paths": not bool(config.other_roots),
|
||||
"standalone_mode": standalone_mode,
|
||||
}
|
||||
|
||||
def _get_expected_model_types(self) -> str:
|
||||
"""Get expected model types string for error messages"""
|
||||
return "VAE, Upscaler, TextEncoder, CLIPVision, Controlnet, or Other"
|
||||
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict[str, Any]:
|
||||
"""Parse other-model-specific parameters (none in Phase 1)."""
|
||||
return {}
|
||||
|
||||
async def get_roots_by_subtype(self, request: web.Request) -> web.Response:
|
||||
"""Return other-model roots grouped by sub_type.
|
||||
|
||||
Aggregates the per-folder_paths-key roots from config
|
||||
(``text_encoders`` and the legacy ``clip`` key both land under
|
||||
``text_encoder``).
|
||||
"""
|
||||
try:
|
||||
roots_by_subtype: Dict[str, List[str]] = {}
|
||||
for key, roots in (config.other_folder_roots or {}).items():
|
||||
sub_type = OTHER_MODEL_FOLDER_SUBTYPES.get(key)
|
||||
if not sub_type:
|
||||
continue
|
||||
bucket = roots_by_subtype.setdefault(sub_type, [])
|
||||
for root in roots:
|
||||
if root and root not in bucket:
|
||||
bucket.append(root)
|
||||
return web.json_response(
|
||||
{"success": True, "roots_by_subtype": roots_by_subtype}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting other roots by sub_type: {e}", exc_info=True)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(e)}, status=500
|
||||
)
|
||||
|
||||
async def get_other_model_info(self, request: web.Request) -> web.Response:
|
||||
"""Get detailed information for a specific other model by name"""
|
||||
try:
|
||||
name = request.match_info.get('name', '')
|
||||
model_info = await self.service.get_model_info_by_name(name) # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
if model_info:
|
||||
return web.json_response(model_info)
|
||||
else:
|
||||
return web.json_response({"error": "Model not found"}, status=404)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in get_other_model_info: {e}", exc_info=True)
|
||||
return web.json_response({"error": str(e)}, status=500)
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Route controller for the pending-delete undo endpoint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .handlers.pending_delete_handler import PendingDeleteHandler
|
||||
|
||||
|
||||
class PendingDeleteRoutes:
|
||||
"""Shared route controller mirroring MiscRoutes/UpdateRoutes.
|
||||
|
||||
Registered ONCE per mode (py/lora_manager.py, standalone.py); NEVER through
|
||||
the per-model-type ModelRouteRegistrar, which is instantiated per model
|
||||
type and would register this non-prefixed route three times.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def setup_routes(app: web.Application) -> None:
|
||||
"""Register the shared undo-delete endpoint."""
|
||||
handler = PendingDeleteHandler()
|
||||
_ = app.router.add_post("/api/lm/undo-delete", handler.undo_delete)
|
||||
|
||||
|
||||
__all__ = ["PendingDeleteRoutes"]
|
||||
@@ -3,7 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Mapping
|
||||
from typing import Any, Callable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -29,6 +29,7 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("POST", "/api/lm/recipes/save", "save_recipe"),
|
||||
RouteDefinition("DELETE", "/api/lm/recipe/{recipe_id}", "delete_recipe"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/top-tags", "get_top_tags"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/search-tags", "search_tags"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/base-models", "get_base_models"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/roots", "get_roots"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/folders", "get_folders"),
|
||||
@@ -42,9 +43,37 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/recipe/{recipe_id}/syntax", "get_recipe_syntax"),
|
||||
RouteDefinition("PUT", "/api/lm/recipe/{recipe_id}/update", "update_recipe"),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/recipe/{recipe_id}/opened", "record_recipe_open"
|
||||
),
|
||||
RouteDefinition("POST", "/api/lm/recipe/move", "move_recipe"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/move-bulk", "move_recipes_bulk"),
|
||||
RouteDefinition("POST", "/api/lm/recipe/lora/reconnect", "reconnect_lora"),
|
||||
RouteDefinition("POST", "/api/lm/recipe/lora/restore", "restore_lora"),
|
||||
RouteDefinition(
|
||||
"GET",
|
||||
"/api/lm/recipe/{recipe_id}/lora/{lora_index}/reconnect-suggestions",
|
||||
"get_reconnect_suggestions",
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/recipe/lora/mark-hash-invalid", "mark_lora_hash_invalid"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/recipe/checkpoint/reconnect", "reconnect_checkpoint"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/recipe/checkpoint/restore", "restore_checkpoint"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET",
|
||||
"/api/lm/recipe/{recipe_id}/checkpoint/reconnect-suggestions",
|
||||
"get_checkpoint_reconnect_suggestions",
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST",
|
||||
"/api/lm/recipe/checkpoint/mark-hash-invalid",
|
||||
"mark_checkpoint_hash_invalid",
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/recipes/find-duplicates", "find_duplicates"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/bulk-delete", "bulk_delete"),
|
||||
RouteDefinition(
|
||||
@@ -55,11 +84,11 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
"GET", "/api/lm/recipes/for-checkpoint", "get_recipes_for_checkpoint"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/recipes/scan", "scan_recipes"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/repair", "repair_recipes"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/cancel-repair", "cancel_repair"),
|
||||
RouteDefinition("POST", "/api/lm/recipe/{recipe_id}/repair", "repair_recipe"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/repair-bulk", "repair_recipes_bulk"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/repair-progress", "get_repair_progress"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/rematch", "rematch_recipes"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/rematch-bulk", "rematch_recipes_bulk"),
|
||||
RouteDefinition("POST", "/api/lm/recipe/{recipe_id}/rematch", "rematch_recipe"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/cancel-rematch", "cancel_rematch"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/rematch-progress", "get_rematch_progress"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/batch-import/start", "start_batch_import"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/recipes/batch-import/progress", "get_batch_import_progress"
|
||||
@@ -81,6 +110,14 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/recipe/{recipe_id}/reimport", "reimport_recipe"
|
||||
),
|
||||
# The companion browser extension only ever issues GET requests, so the
|
||||
# payload-based re-import variant must also be reachable via GET.
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/recipe/{recipe_id}/reimport", "reimport_recipe"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/recipe/{recipe_id}/send-workflow", "send_recipe_workflow"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -104,7 +141,7 @@ class RecipeRouteRegistrar:
|
||||
handler = handler_lookup[definition.handler_name]
|
||||
self._bind_route(definition.method, definition.path, handler)
|
||||
|
||||
def _bind_route(self, method: str, path: str, handler: Callable) -> None:
|
||||
def _bind_route(self, method: str, path: str, handler: Callable[..., Any]) -> None:
|
||||
add_method_name = self._METHOD_MAP[method.upper()]
|
||||
add_method = getattr(self._app.router, add_method_name)
|
||||
add_method(path, handler)
|
||||
|
||||
+56
-26
@@ -11,6 +11,8 @@ from ..config import config
|
||||
from ..services.settings_manager import get_settings_manager
|
||||
from ..services.server_i18n import server_i18n
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
from ..services.model_query import normalize_sub_type, resolve_sub_type
|
||||
from ..utils.constants import VALID_LORA_SUB_TYPES, VALID_CHECKPOINT_SUB_TYPES
|
||||
from ..utils.usage_stats import UsageStats
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -38,10 +40,11 @@ class StatsRoutes:
|
||||
"""Route handlers for Statistics page and API endpoints"""
|
||||
|
||||
def __init__(self):
|
||||
self.lora_scanner = None
|
||||
self.checkpoint_scanner = None
|
||||
self.embedding_scanner = None
|
||||
self.usage_stats = None
|
||||
self.lora_scanner: Any = None
|
||||
self.checkpoint_scanner: Any = None
|
||||
self.embedding_scanner: Any = None
|
||||
self.usage_stats: Any = None
|
||||
self._i18n_filter_added = False
|
||||
self.template_env = jinja2.Environment(
|
||||
loader=jinja2.FileSystemLoader(config.templates_path),
|
||||
autoescape=True
|
||||
@@ -93,9 +96,9 @@ class StatsRoutes:
|
||||
server_i18n.set_locale(user_language)
|
||||
|
||||
# 为模板环境添加i18n过滤器
|
||||
if not hasattr(self.template_env, '_i18n_filter_added'):
|
||||
if not self._i18n_filter_added:
|
||||
self.template_env.filters['t'] = server_i18n.create_template_filter()
|
||||
self.template_env._i18n_filter_added = True
|
||||
self._i18n_filter_added = True
|
||||
|
||||
template = self.template_env.get_template('statistics.html')
|
||||
rendered = template.render(
|
||||
@@ -140,6 +143,21 @@ class StatsRoutes:
|
||||
# Get usage statistics
|
||||
usage_data = await self.usage_stats.get_stats()
|
||||
|
||||
# CivitAI model type distribution across all model types
|
||||
# Use the same logic as the filter panel: normalize_sub_type(resolve_sub_type(entry))
|
||||
# with sub-type validation per model type
|
||||
model_types_counter: Counter[str] = Counter()
|
||||
for entry in lora_cache.raw_data:
|
||||
ntype = normalize_sub_type(resolve_sub_type(entry))
|
||||
if ntype and ntype in VALID_LORA_SUB_TYPES:
|
||||
model_types_counter[ntype] += 1
|
||||
for entry in checkpoint_cache.raw_data:
|
||||
ntype = normalize_sub_type(resolve_sub_type(entry))
|
||||
if ntype and ntype in VALID_CHECKPOINT_SUB_TYPES:
|
||||
model_types_counter[ntype] += 1
|
||||
# Embeddings: always count as "embedding" regardless of CivitAI sub-type
|
||||
model_types_counter['embedding'] = len(embedding_cache.raw_data)
|
||||
|
||||
return web.json_response({
|
||||
'success': True,
|
||||
'data': {
|
||||
@@ -154,7 +172,8 @@ class StatsRoutes:
|
||||
'total_generations': usage_data.get('total_executions', 0),
|
||||
'unused_loras': self._count_unused_models(lora_cache.raw_data, usage_data.get('loras', {})),
|
||||
'unused_checkpoints': self._count_unused_models(checkpoint_cache.raw_data, usage_data.get('checkpoints', {})),
|
||||
'unused_embeddings': self._count_unused_models(embedding_cache.raw_data, usage_data.get('embeddings', {}))
|
||||
'unused_embeddings': self._count_unused_models(embedding_cache.raw_data, usage_data.get('embeddings', {})),
|
||||
'model_types_distribution': dict(model_types_counter.most_common())
|
||||
}
|
||||
})
|
||||
|
||||
@@ -459,9 +478,12 @@ class StatsRoutes:
|
||||
if unused_lora_percent > 50:
|
||||
insights.append({
|
||||
'type': 'warning',
|
||||
'title': 'High Number of Unused LoRAs',
|
||||
'description': f'{unused_lora_percent:.1f}% of your LoRAs ({unused_loras}/{total_loras}) have never been used.',
|
||||
'suggestion': 'Consider organizing or archiving unused models to free up storage space.'
|
||||
'key': 'insights.unusedLoras.high',
|
||||
'params': {
|
||||
'percent': f'{unused_lora_percent:.1f}',
|
||||
'count': str(unused_loras),
|
||||
'total': str(total_loras)
|
||||
}
|
||||
})
|
||||
|
||||
if total_checkpoints > 0:
|
||||
@@ -469,9 +491,12 @@ class StatsRoutes:
|
||||
if unused_checkpoint_percent > 30:
|
||||
insights.append({
|
||||
'type': 'warning',
|
||||
'title': 'Unused Checkpoints Detected',
|
||||
'description': f'{unused_checkpoint_percent:.1f}% of your checkpoints ({unused_checkpoints}/{total_checkpoints}) have never been used.',
|
||||
'suggestion': 'Review and consider removing checkpoints you no longer need.'
|
||||
'key': 'insights.unusedCheckpoints.detected',
|
||||
'params': {
|
||||
'percent': f'{unused_checkpoint_percent:.1f}',
|
||||
'count': str(unused_checkpoints),
|
||||
'total': str(total_checkpoints)
|
||||
}
|
||||
})
|
||||
|
||||
if total_embeddings > 0:
|
||||
@@ -479,9 +504,12 @@ class StatsRoutes:
|
||||
if unused_embedding_percent > 50:
|
||||
insights.append({
|
||||
'type': 'warning',
|
||||
'title': 'High Number of Unused Embeddings',
|
||||
'description': f'{unused_embedding_percent:.1f}% of your embeddings ({unused_embeddings}/{total_embeddings}) have never been used.',
|
||||
'suggestion': 'Consider organizing or archiving unused embeddings to optimize your collection.'
|
||||
'key': 'insights.unusedEmbeddings.high',
|
||||
'params': {
|
||||
'percent': f'{unused_embedding_percent:.1f}',
|
||||
'count': str(unused_embeddings),
|
||||
'total': str(total_embeddings)
|
||||
}
|
||||
})
|
||||
|
||||
# Storage insights
|
||||
@@ -492,18 +520,20 @@ class StatsRoutes:
|
||||
if total_size > 100 * 1024 * 1024 * 1024: # 100GB
|
||||
insights.append({
|
||||
'type': 'info',
|
||||
'title': 'Large Collection Detected',
|
||||
'description': f'Your model collection is using {self._format_size(total_size)} of storage.',
|
||||
'suggestion': 'Consider using external storage or cloud solutions for better organization.'
|
||||
'key': 'insights.collection.large',
|
||||
'params': {
|
||||
'size': self._format_size(total_size)
|
||||
}
|
||||
})
|
||||
|
||||
# Recent activity insight
|
||||
if usage_data.get('total_executions', 0) > 100:
|
||||
insights.append({
|
||||
'type': 'success',
|
||||
'title': 'Active User',
|
||||
'description': f'You\'ve completed {usage_data["total_executions"]} generations so far!',
|
||||
'suggestion': 'Keep exploring and creating amazing content with your models.'
|
||||
'key': 'insights.activity.active',
|
||||
'params': {
|
||||
'count': str(usage_data['total_executions'])
|
||||
}
|
||||
})
|
||||
|
||||
return web.json_response({
|
||||
@@ -520,7 +550,7 @@ class StatsRoutes:
|
||||
'error': str(e)
|
||||
}, status=500)
|
||||
|
||||
def _count_unused_models(self, models: List[Dict], usage_data: Dict) -> int:
|
||||
def _count_unused_models(self, models: List[Dict[str, Any]], usage_data: Dict[str, Any]) -> int:
|
||||
"""Count models that have never been used"""
|
||||
used_hashes = set(usage_data.keys())
|
||||
unused_count = 0
|
||||
@@ -531,7 +561,7 @@ class StatsRoutes:
|
||||
|
||||
return unused_count
|
||||
|
||||
def _get_top_used_models(self, usage_data: Dict, model_map: Dict, limit: int) -> List[Dict]:
|
||||
def _get_top_used_models(self, usage_data: Dict[str, Any], model_map: Dict[str, Any], limit: int) -> List[Dict[str, Any]]:
|
||||
"""Get top used models with their metadata"""
|
||||
sorted_usage = sorted(usage_data.items(), key=lambda x: x[1].get('total', 0), reverse=True)
|
||||
|
||||
@@ -549,7 +579,7 @@ class StatsRoutes:
|
||||
|
||||
return top_models
|
||||
|
||||
def _get_usage_timeline(self, usage_data: Dict, days: int) -> List[Dict]:
|
||||
def _get_usage_timeline(self, usage_data: Dict[str, Any], days: int) -> List[Dict[str, Any]]:
|
||||
"""Get usage timeline for the past N days"""
|
||||
timeline = []
|
||||
today = datetime.now()
|
||||
@@ -585,7 +615,7 @@ class StatsRoutes:
|
||||
|
||||
return list(reversed(timeline)) # Oldest to newest
|
||||
|
||||
def _format_size(self, size_bytes: int) -> str:
|
||||
def _format_size(self, size_bytes: float) -> str:
|
||||
"""Format file size in human readable format"""
|
||||
for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
|
||||
if size_bytes < 1024.0:
|
||||
|
||||
+353
-57
@@ -6,7 +6,7 @@ import shutil
|
||||
import tempfile
|
||||
import asyncio
|
||||
from aiohttp import web, ClientError
|
||||
from typing import Dict, List
|
||||
from typing import Any, Dict, List, cast
|
||||
|
||||
from ..utils.settings_paths import ensure_settings_file
|
||||
from ..services.downloader import get_downloader
|
||||
@@ -16,6 +16,105 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
NETWORK_EXCEPTIONS = (ClientError, OSError, asyncio.TimeoutError)
|
||||
|
||||
# User-managed directories that live inside the plugin folder (portable
|
||||
# mode) and must survive a Git-based update. ``git clean -fd`` would
|
||||
# otherwise delete them because they are untracked and, in released tags,
|
||||
# not listed in ``.gitignore``. ``-e`` excludes a path from cleaning
|
||||
# regardless of whether it is ignored.
|
||||
_PRESERVE_DIRS = ('settings.json', 'civitai', 'wildcards', 'backups', 'stats', 'logs', 'cache', 'model_cache')
|
||||
|
||||
|
||||
def _clean_excludes() -> List[str]:
|
||||
"""Build the ``-e`` arguments for ``git clean`` from :data:`_PRESERVE_DIRS`."""
|
||||
excludes: List[str] = []
|
||||
for name in _PRESERVE_DIRS:
|
||||
excludes.append('-e')
|
||||
excludes.append(name)
|
||||
# For directories, also exclude nested matches explicitly
|
||||
# (``-e dir`` alone matches the dir entry; ``-e dir/**`` guards
|
||||
# contents under all git versions as defense-in-depth).
|
||||
excludes.append('-e')
|
||||
excludes.append(f'{name}/**')
|
||||
return excludes
|
||||
|
||||
|
||||
def _stage_preserved_items(plugin_root: str) -> tuple[str, list[str]]:
|
||||
"""Move preserved user-data items to a temp directory outside *plugin_root*.
|
||||
|
||||
This ensures that ``git reset --hard``, ``git clean -fd``, and ZIP-based
|
||||
replacement cannot touch these files even when ``-e`` exclusion patterns
|
||||
are mishandled (e.g. on Windows where forward-slash patterns may not
|
||||
match backslash-prefixed paths in some Git builds, or where file locks
|
||||
prevent deletion/recreation).
|
||||
|
||||
Returns:
|
||||
``(backup_root, staged_names)``: the temp directory path and the
|
||||
list of item names that were successfully moved.
|
||||
"""
|
||||
backup_root = tempfile.mkdtemp(prefix='lora_manager_update_')
|
||||
staged: list[str] = []
|
||||
for name in _PRESERVE_DIRS:
|
||||
src = os.path.join(plugin_root, name)
|
||||
if not os.path.lexists(src):
|
||||
continue
|
||||
dst = os.path.join(backup_root, name)
|
||||
try:
|
||||
shutil.move(src, dst)
|
||||
staged.append(name)
|
||||
logger.debug("Staged '%s' for update safety", name)
|
||||
except OSError:
|
||||
# ``shutil.move`` may fail on Windows if a file handle inside
|
||||
# the directory is still open (e.g. a SQLite WAL file). Fall
|
||||
# back to copy-then-remove.
|
||||
logger.debug("Move failed for '%s', falling back to copy", name)
|
||||
try:
|
||||
if os.path.isdir(src) and not os.path.islink(src):
|
||||
shutil.copytree(src, dst, symlinks=True)
|
||||
shutil.rmtree(src, ignore_errors=True)
|
||||
else:
|
||||
shutil.copy2(src, dst)
|
||||
os.remove(src)
|
||||
staged.append(name)
|
||||
logger.info("Copied (then removed) '%s' for update safety", name)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Could not stage '%s': %s (will rely on git -e / skip lists)", name, exc
|
||||
)
|
||||
return backup_root, staged
|
||||
|
||||
|
||||
def _restore_preserved_items(plugin_root: str, backup_root: str, staged: list[str]) -> None:
|
||||
"""Move staged items back from *backup_root* into *plugin_root*.
|
||||
|
||||
Any leftover placeholder at the destination (created by git checkout or
|
||||
ZIP extraction) is removed before the move.
|
||||
"""
|
||||
for name in staged:
|
||||
src = os.path.join(backup_root, name)
|
||||
dst = os.path.join(plugin_root, name)
|
||||
try:
|
||||
if os.path.lexists(dst):
|
||||
if os.path.isdir(dst) and not os.path.islink(dst):
|
||||
shutil.rmtree(dst, ignore_errors=True)
|
||||
else:
|
||||
os.remove(dst)
|
||||
shutil.move(src, dst)
|
||||
logger.debug("Restored '%s' after update", name)
|
||||
except OSError:
|
||||
logger.debug("Move failed restoring '%s', falling back to copy", name)
|
||||
try:
|
||||
if os.path.isdir(src) and not os.path.islink(src):
|
||||
shutil.copytree(src, dst, symlinks=True, dirs_exist_ok=True)
|
||||
shutil.rmtree(src, ignore_errors=True)
|
||||
else:
|
||||
shutil.copy2(src, dst)
|
||||
os.remove(src)
|
||||
logger.info("Copied '%s' back after update", name)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to restore '%s': %s", name, exc)
|
||||
shutil.rmtree(backup_root, ignore_errors=True)
|
||||
|
||||
|
||||
|
||||
class UpdateRoutes:
|
||||
"""Routes for handling plugin update checks"""
|
||||
@@ -26,6 +125,7 @@ class UpdateRoutes:
|
||||
app.router.add_get('/api/lm/check-updates', UpdateRoutes.check_updates)
|
||||
app.router.add_get('/api/lm/version-info', UpdateRoutes.get_version_info)
|
||||
app.router.add_post('/api/lm/perform-update', UpdateRoutes.perform_update)
|
||||
app.router.add_post('/api/lm/switch-channel', UpdateRoutes.switch_channel)
|
||||
|
||||
@staticmethod
|
||||
async def check_updates(request):
|
||||
@@ -44,10 +144,17 @@ class UpdateRoutes:
|
||||
|
||||
# Fetch remote version from GitHub
|
||||
if nightly:
|
||||
remote_version, changelog = await UpdateRoutes._get_nightly_version()
|
||||
releases = None
|
||||
local_hash = git_info.get('short_hash', '')
|
||||
nightly_version, releases_result = await asyncio.gather(
|
||||
UpdateRoutes._get_nightly_version(local_hash),
|
||||
UpdateRoutes._get_remote_version()
|
||||
)
|
||||
remote_version, _, behind_by, commit_date = nightly_version
|
||||
_, changelog, releases = releases_result
|
||||
else:
|
||||
remote_version, changelog, releases = await UpdateRoutes._get_remote_version()
|
||||
behind_by = 0
|
||||
commit_date = ''
|
||||
|
||||
# Compare versions
|
||||
if nightly:
|
||||
@@ -60,6 +167,10 @@ class UpdateRoutes:
|
||||
remote_version.replace('v', '')
|
||||
)
|
||||
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
plugin_root = os.path.dirname(os.path.dirname(current_dir))
|
||||
has_git = os.path.exists(os.path.join(plugin_root, '.git'))
|
||||
|
||||
response_data = {
|
||||
'success': True,
|
||||
'current_version': local_version,
|
||||
@@ -67,13 +178,13 @@ class UpdateRoutes:
|
||||
'update_available': update_available,
|
||||
'changelog': changelog,
|
||||
'git_info': git_info,
|
||||
'nightly': nightly
|
||||
'nightly': nightly,
|
||||
'has_git': has_git,
|
||||
'releases': releases,
|
||||
'behind_by': behind_by,
|
||||
'commit_date': commit_date
|
||||
}
|
||||
|
||||
# Include releases list for stable mode
|
||||
if releases is not None:
|
||||
response_data['releases'] = releases
|
||||
|
||||
return web.json_response(response_data)
|
||||
|
||||
except NETWORK_EXCEPTIONS as e:
|
||||
@@ -105,9 +216,14 @@ class UpdateRoutes:
|
||||
# Format: version-short_hash
|
||||
version_string = f"{local_version}-{short_hash}"
|
||||
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
plugin_root = os.path.dirname(os.path.dirname(current_dir))
|
||||
has_git = os.path.exists(os.path.join(plugin_root, '.git'))
|
||||
|
||||
return web.json_response({
|
||||
'success': True,
|
||||
'version': version_string
|
||||
'version': version_string,
|
||||
'has_git': has_git
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
@@ -135,20 +251,22 @@ class UpdateRoutes:
|
||||
if os.path.exists(settings_path):
|
||||
with open(settings_path, 'r', encoding='utf-8') as f:
|
||||
settings_backup = f.read()
|
||||
logger.info("Backed up settings.json")
|
||||
logger.debug("Backed up settings.json (%d bytes)", len(settings_backup))
|
||||
|
||||
git_folder = os.path.join(plugin_root, '.git')
|
||||
if os.path.exists(git_folder):
|
||||
# Git update
|
||||
success, new_version = await UpdateRoutes._perform_git_update(plugin_root, nightly)
|
||||
else:
|
||||
# Fallback: Download ZIP and replace files
|
||||
success, new_version = await UpdateRoutes._download_and_replace_zip(plugin_root)
|
||||
staged_backup_dir, staged_items = _stage_preserved_items(plugin_root)
|
||||
try:
|
||||
git_folder = os.path.join(plugin_root, '.git')
|
||||
if os.path.exists(git_folder):
|
||||
success, new_version = await UpdateRoutes._perform_git_update(plugin_root, nightly)
|
||||
else:
|
||||
success, new_version = await UpdateRoutes._download_and_replace_zip(plugin_root)
|
||||
finally:
|
||||
_restore_preserved_items(plugin_root, staged_backup_dir, staged_items)
|
||||
|
||||
if settings_backup and success:
|
||||
with open(settings_path, 'w', encoding='utf-8') as f:
|
||||
f.write(settings_backup)
|
||||
logger.info("Restored settings.json")
|
||||
logger.debug("Restored settings.json content (%d bytes)", len(settings_backup))
|
||||
|
||||
if success:
|
||||
return web.json_response({
|
||||
@@ -169,6 +287,164 @@ class UpdateRoutes:
|
||||
'error': str(e)
|
||||
})
|
||||
|
||||
@staticmethod
|
||||
async def switch_channel(request):
|
||||
"""
|
||||
Switch between release and nightly update channels.
|
||||
|
||||
ZIP/CNR install → Nightly: git init + checkout main (one-way upgrade)
|
||||
Git install → Release: git checkout latest tag (.git preserved)
|
||||
ZIP/CNR install → Release: ZIP download (no .git, stays in ZIP mode)
|
||||
Git install → Nightly: git checkout main + pull
|
||||
"""
|
||||
try:
|
||||
body = await request.json() if request.has_body else {}
|
||||
channel = body.get('channel', '')
|
||||
|
||||
if channel not in ('release', 'nightly'):
|
||||
return web.json_response({
|
||||
'success': False,
|
||||
'error': f'Invalid channel: {channel}. Must be "release" or "nightly".'
|
||||
})
|
||||
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
plugin_root = os.path.dirname(os.path.dirname(current_dir))
|
||||
|
||||
settings_path = ensure_settings_file(logger)
|
||||
settings_backup = None
|
||||
if os.path.exists(settings_path):
|
||||
with open(settings_path, 'r', encoding='utf-8') as f:
|
||||
settings_backup = f.read()
|
||||
logger.debug("Backed up settings.json before channel switch (%d bytes)", len(settings_backup))
|
||||
|
||||
staged_backup_dir, staged_items = _stage_preserved_items(plugin_root)
|
||||
try:
|
||||
git_folder = os.path.join(plugin_root, '.git')
|
||||
|
||||
if channel == 'nightly':
|
||||
git_backup = None
|
||||
if os.path.exists(git_folder):
|
||||
git_backup = UpdateRoutes._backup_git(git_folder, 'nightly')
|
||||
|
||||
success = False
|
||||
new_version = ''
|
||||
try:
|
||||
if os.path.exists(git_folder):
|
||||
success, new_version = await UpdateRoutes._perform_git_update(
|
||||
plugin_root, nightly=True
|
||||
)
|
||||
else:
|
||||
success, new_version = UpdateRoutes._init_git_repo(plugin_root)
|
||||
finally:
|
||||
UpdateRoutes._restore_git(git_backup, git_folder, success, 'nightly')
|
||||
else:
|
||||
success = False
|
||||
new_version = ''
|
||||
if os.path.exists(git_folder):
|
||||
success, new_version = await UpdateRoutes._perform_git_update(
|
||||
plugin_root, nightly=False
|
||||
)
|
||||
else:
|
||||
tracking_file = os.path.join(plugin_root, '.tracking')
|
||||
if os.path.exists(tracking_file):
|
||||
os.remove(tracking_file)
|
||||
success, new_version = await UpdateRoutes._download_and_replace_zip(plugin_root)
|
||||
finally:
|
||||
_restore_preserved_items(plugin_root, staged_backup_dir, staged_items)
|
||||
|
||||
if settings_backup and success:
|
||||
with open(settings_path, 'w', encoding='utf-8') as f:
|
||||
f.write(settings_backup)
|
||||
logger.debug("Restored settings.json content after channel switch (%d bytes)", len(settings_backup))
|
||||
|
||||
if success:
|
||||
return web.json_response({
|
||||
'success': True,
|
||||
'channel': channel,
|
||||
'new_version': new_version,
|
||||
'message': f'Switched to {channel} channel'
|
||||
})
|
||||
else:
|
||||
return web.json_response({
|
||||
'success': False,
|
||||
'error': f'Failed to switch to {channel} channel'
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to switch channel: %s", e, exc_info=True)
|
||||
return web.json_response({
|
||||
'success': False,
|
||||
'error': str(e)
|
||||
})
|
||||
|
||||
@staticmethod
|
||||
def _init_git_repo(plugin_root: str) -> tuple[bool, str]:
|
||||
"""
|
||||
Initialize a Git repository in a ZIP-installed plugin folder.
|
||||
Clones the remote history and checks out main branch.
|
||||
"""
|
||||
try:
|
||||
import git
|
||||
except ImportError:
|
||||
logger.error(
|
||||
"GitPython is not available: cannot initialize git repo. "
|
||||
"Install git or set $GIT_PYTHON_GIT_EXECUTABLE to the git binary path."
|
||||
)
|
||||
return False, ""
|
||||
|
||||
clean_excludes = _clean_excludes()
|
||||
|
||||
try:
|
||||
repo = git.Repo.init(plugin_root)
|
||||
origin = repo.create_remote(
|
||||
'origin',
|
||||
'https://github.com/willmiao/ComfyUI-Lora-Manager.git'
|
||||
)
|
||||
origin.fetch()
|
||||
|
||||
repo.create_head('main', origin.refs.main)
|
||||
repo.git.checkout('main', '--force')
|
||||
repo.git.reset('--hard')
|
||||
repo.git.clean('-fd', *clean_excludes)
|
||||
|
||||
tracking_file = os.path.join(plugin_root, '.tracking')
|
||||
if os.path.exists(tracking_file):
|
||||
os.remove(tracking_file)
|
||||
logger.info("Removed .tracking file (now in git mode)")
|
||||
|
||||
new_version = f"main-{repo.head.commit.hexsha[:7]}"
|
||||
logger.info("Initialized git repo on main branch: %s", new_version)
|
||||
return True, new_version
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Failed to initialize git repo: %s", e, exc_info=True)
|
||||
return False, ""
|
||||
|
||||
@staticmethod
|
||||
def _backup_git(git_folder, label):
|
||||
try:
|
||||
backup_dir = tempfile.mkdtemp()
|
||||
backup = os.path.join(backup_dir, '.git')
|
||||
shutil.copytree(git_folder, backup)
|
||||
logger.info("Backed up .git before switching to %s", label)
|
||||
return backup
|
||||
except Exception as e:
|
||||
logger.error("Failed to backup .git before %s switch: %s", label, e)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _restore_git(git_backup, git_folder, success, label):
|
||||
if git_backup and not success:
|
||||
try:
|
||||
if os.path.exists(git_folder):
|
||||
shutil.rmtree(git_folder)
|
||||
shutil.copytree(git_backup, git_folder)
|
||||
logger.info("Restored .git after failed %s switch", label)
|
||||
except Exception as e:
|
||||
logger.error("Failed to restore .git after %s switch: %s", label, e)
|
||||
if git_backup:
|
||||
shutil.rmtree(os.path.dirname(git_backup), ignore_errors=True)
|
||||
|
||||
@staticmethod
|
||||
async def _download_and_replace_zip(plugin_root: str) -> tuple[bool, str]:
|
||||
"""
|
||||
@@ -191,9 +467,10 @@ class UpdateRoutes:
|
||||
if not success:
|
||||
logger.error(f"Failed to fetch release info: {data}")
|
||||
return False, ""
|
||||
|
||||
zip_url = data.get("zipball_url")
|
||||
version = data.get("tag_name", "unknown")
|
||||
|
||||
release_payload = cast(dict[str, Any], data)
|
||||
zip_url = release_payload.get("zipball_url", "")
|
||||
version = release_payload.get("tag_name", "unknown")
|
||||
|
||||
# Download ZIP to temporary file
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".zip") as tmp_zip:
|
||||
@@ -223,8 +500,7 @@ class UpdateRoutes:
|
||||
except Exception:
|
||||
logger.debug("Could not close downloaded-version history database", exc_info=True)
|
||||
|
||||
# Skip settings.json, civitai, model cache and runtime cache folders
|
||||
UpdateRoutes._clean_plugin_folder(plugin_root, skip_files=['settings.json', 'civitai', 'model_cache', 'cache', 'wildcards', 'backups', 'stats'])
|
||||
UpdateRoutes._clean_plugin_folder(plugin_root, skip_files=list(_PRESERVE_DIRS))
|
||||
|
||||
# Extract ZIP to temp dir
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
@@ -234,7 +510,7 @@ class UpdateRoutes:
|
||||
extracted_root = next(os.scandir(tmp_dir)).path
|
||||
|
||||
# Copy files, skipping user data that should be preserved
|
||||
skip_items = {'settings.json', 'civitai', 'wildcards', 'backups', 'stats'}
|
||||
skip_items = set(_PRESERVE_DIRS)
|
||||
for item in os.listdir(extracted_root):
|
||||
if item in skip_items:
|
||||
continue
|
||||
@@ -251,7 +527,7 @@ class UpdateRoutes:
|
||||
# for ComfyUI Manager to work properly
|
||||
tracking_info_file = os.path.join(plugin_root, '.tracking')
|
||||
tracking_files = []
|
||||
skip_tracked = {'civitai', 'wildcards', 'backups', 'stats'}
|
||||
skip_tracked = set(_PRESERVE_DIRS) - {'settings.json'}
|
||||
for root, dirs, files in os.walk(extracted_root):
|
||||
# Skip user data directories and their contents
|
||||
rel_root = os.path.relpath(root, extracted_root)
|
||||
@@ -274,7 +550,8 @@ class UpdateRoutes:
|
||||
except Exception as e:
|
||||
logger.error(f"ZIP update failed: {e}", exc_info=True)
|
||||
return False, ""
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _clean_plugin_folder(plugin_root, skip_files=None):
|
||||
skip_files = skip_files or []
|
||||
for item in os.listdir(plugin_root):
|
||||
@@ -287,41 +564,56 @@ class UpdateRoutes:
|
||||
os.remove(path)
|
||||
|
||||
@staticmethod
|
||||
async def _get_nightly_version() -> tuple[str, List[str]]:
|
||||
"""
|
||||
Fetch latest commit from main branch
|
||||
"""
|
||||
async def _get_nightly_version(local_hash: str = "") -> tuple[str, List[str], int, str]:
|
||||
repo_owner = "willmiao"
|
||||
repo_name = "ComfyUI-Lora-Manager"
|
||||
|
||||
# Use GitHub API to fetch the latest commit from main branch
|
||||
|
||||
github_url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/commits/main"
|
||||
|
||||
|
||||
try:
|
||||
downloader = await get_downloader()
|
||||
success, data = await downloader.make_request('GET', github_url, custom_headers={'Accept': 'application/vnd.github+json'})
|
||||
|
||||
success, data = await downloader.make_request(
|
||||
'GET', github_url,
|
||||
custom_headers={'Accept': 'application/vnd.github+json'}
|
||||
)
|
||||
|
||||
if not success:
|
||||
logger.warning(f"Failed to fetch GitHub commit: {data}")
|
||||
return "main", []
|
||||
|
||||
commit_sha = data.get('sha', '')[:7] # Short hash
|
||||
commit_message = data.get('commit', {}).get('message', '')
|
||||
|
||||
# Format as "main-{short_hash}"
|
||||
logger.warning("Failed to fetch GitHub commit: %s", data)
|
||||
return "main", [], 0, ""
|
||||
|
||||
commit_payload = cast(dict[str, Any], data)
|
||||
commit_sha = commit_payload.get('sha', '')[:7]
|
||||
commit_message = commit_payload.get('commit', {}).get('message', '')
|
||||
commit_date = commit_payload.get('commit', {}).get('committer', {}).get('date', '')[:10]
|
||||
|
||||
version = f"main-{commit_sha}"
|
||||
|
||||
# Use commit message as changelog
|
||||
changelog = [commit_message] if commit_message else []
|
||||
|
||||
return version, changelog
|
||||
|
||||
|
||||
behind_by = 0
|
||||
if local_hash and local_hash not in ('unknown', 'stable'):
|
||||
compare_url = (
|
||||
f"https://api.github.com/repos/{repo_owner}/{repo_name}"
|
||||
f"/compare/{local_hash}...main"
|
||||
)
|
||||
c_ok, c_data = await downloader.make_request(
|
||||
'GET', compare_url,
|
||||
custom_headers={'Accept': 'application/vnd.github+json'}
|
||||
)
|
||||
if c_ok:
|
||||
compare_payload = cast(dict[str, Any], c_data)
|
||||
if compare_payload.get('status') in ('ahead', 'diverged'):
|
||||
behind_by = compare_payload.get('ahead_by', 0)
|
||||
else:
|
||||
behind_by = compare_payload.get('behind_by', 0)
|
||||
|
||||
return version, changelog, behind_by, commit_date
|
||||
|
||||
except NETWORK_EXCEPTIONS as e:
|
||||
logger.warning("Unable to reach GitHub for nightly version: %s", e)
|
||||
return "main", []
|
||||
return "main", [], 0, ""
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching nightly version: {e}", exc_info=True)
|
||||
return "main", []
|
||||
logger.error("Error fetching nightly version: %s", e, exc_info=True)
|
||||
return "main", [], 0, ""
|
||||
|
||||
@staticmethod
|
||||
def _compare_nightly_versions(local_git_info: Dict[str, str], remote_version: str) -> bool:
|
||||
@@ -365,6 +657,8 @@ class UpdateRoutes:
|
||||
)
|
||||
return False, ""
|
||||
|
||||
clean_excludes = _clean_excludes()
|
||||
|
||||
try:
|
||||
# Open the Git repository
|
||||
repo = git.Repo(plugin_root)
|
||||
@@ -376,8 +670,9 @@ class UpdateRoutes:
|
||||
if nightly:
|
||||
# Reset to discard any local changes
|
||||
repo.git.reset('--hard')
|
||||
# Clean untracked files
|
||||
repo.git.clean('-fd')
|
||||
# Clean untracked files, but preserve user-managed directories
|
||||
# (wildcards, backups, stats, civitai, caches, settings.json).
|
||||
repo.git.clean('-fd', *clean_excludes)
|
||||
|
||||
# Switch to main branch and pull latest
|
||||
main_branch = 'main'
|
||||
@@ -394,8 +689,9 @@ class UpdateRoutes:
|
||||
else:
|
||||
# Reset to discard any local changes
|
||||
repo.git.reset('--hard')
|
||||
# Clean untracked files
|
||||
repo.git.clean('-fd')
|
||||
# Clean untracked files, but preserve user-managed directories
|
||||
# (wildcards, backups, stats, civitai, caches, settings.json).
|
||||
repo.git.clean('-fd', *clean_excludes)
|
||||
|
||||
# Get latest release tag
|
||||
tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime, reverse=True)
|
||||
@@ -413,7 +709,7 @@ class UpdateRoutes:
|
||||
logger.info(f"Successfully updated to {new_version}")
|
||||
return True, new_version
|
||||
|
||||
except git.exc.GitError as e:
|
||||
except git.exc.GitError as e: # pyright: ignore[reportAttributeAccessIssue]
|
||||
logger.error(f"Git error during update: {e}")
|
||||
return False, ""
|
||||
except Exception as e:
|
||||
@@ -474,7 +770,7 @@ class UpdateRoutes:
|
||||
return git_info
|
||||
|
||||
@staticmethod
|
||||
async def _get_remote_version() -> tuple[str, List[str], List[Dict]]:
|
||||
async def _get_remote_version() -> tuple[str, List[str], List[Dict[str, Any]]]:
|
||||
"""
|
||||
Fetch remote version from GitHub
|
||||
Returns:
|
||||
@@ -496,7 +792,7 @@ class UpdateRoutes:
|
||||
|
||||
# Parse releases
|
||||
releases = []
|
||||
for i, release in enumerate(data):
|
||||
for i, release in enumerate(cast(list[dict[str, Any]], data)):
|
||||
version = release.get('tag_name', '')
|
||||
if not version.startswith('v'):
|
||||
version = f"v{version}"
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
"""In-memory store for the LoRA Manager page's active filters.
|
||||
|
||||
The manager page keeps its filter state in localStorage for its own
|
||||
restoration, but the ComfyUI node autocomplete runs in a potentially
|
||||
different browser/origin (or Electron shell) where that storage is not
|
||||
shared. This store mirrors the active filters server-side so the
|
||||
``/api/lm/{prefix}/relative-paths`` endpoint can inject them into
|
||||
autocomplete searches regardless of which client set them.
|
||||
|
||||
State is process-local and intentionally not persisted; the manager page
|
||||
re-pushes its restored state on load.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Keys copied from the manager page's persisted filter snapshot.
|
||||
_FILTER_KEYS = (
|
||||
"baseModel",
|
||||
"tags",
|
||||
"autoTags",
|
||||
"modelTypes",
|
||||
"tagLogic",
|
||||
"license",
|
||||
)
|
||||
|
||||
|
||||
class ActiveFiltersStore:
|
||||
"""Process-local store of active filters, keyed by model type."""
|
||||
|
||||
_instance: Optional["ActiveFiltersStore"] = None
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._filters: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls) -> "ActiveFiltersStore":
|
||||
if cls._instance is None:
|
||||
cls._instance = cls()
|
||||
return cls._instance
|
||||
|
||||
@classmethod
|
||||
def reset_instance(cls) -> None:
|
||||
"""Drop the singleton (test isolation)."""
|
||||
cls._instance = None
|
||||
|
||||
def set_filters(self, model_type: str, payload: Dict[str, Any]) -> None:
|
||||
"""Replace the stored active filters for a model type.
|
||||
|
||||
Only recognized keys are kept; everything else is discarded.
|
||||
"""
|
||||
filters = payload.get("filters")
|
||||
sanitized: Dict[str, Any] = {
|
||||
"activeFolder": payload.get("activeFolder"),
|
||||
"recursiveSearch": bool(payload.get("recursiveSearch", True)),
|
||||
"filters": (
|
||||
{key: filters[key] for key in _FILTER_KEYS if key in filters}
|
||||
if isinstance(filters, dict)
|
||||
else None
|
||||
),
|
||||
}
|
||||
self._filters[model_type] = sanitized
|
||||
|
||||
def get_filters(self, model_type: str) -> Optional[Dict[str, Any]]:
|
||||
"""Return the stored payload for a model type, or None if unset."""
|
||||
return self._filters.get(model_type)
|
||||
|
||||
def clear(self, model_type: str) -> None:
|
||||
self._filters.pop(model_type, None)
|
||||
|
||||
|
||||
def active_filters_to_query_kwargs(payload: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""Map a stored active-filters payload to ``search_relative_paths`` kwargs.
|
||||
|
||||
Mirrors the query-param mapping that the ComfyUI autocomplete used to
|
||||
build client-side from localStorage (web/comfyui/autocomplete.js).
|
||||
"""
|
||||
kwargs: Dict[str, Any] = {}
|
||||
if not payload:
|
||||
return kwargs
|
||||
|
||||
active_folder = payload.get("activeFolder")
|
||||
recursive = payload.get("recursiveSearch", True)
|
||||
|
||||
if active_folder and active_folder != "null":
|
||||
kwargs["folder"] = active_folder
|
||||
elif not recursive:
|
||||
# Root folder with recursion disabled mirrors the page list,
|
||||
# which matches only root-level files via folder=''.
|
||||
kwargs["folder"] = ""
|
||||
|
||||
filters = payload.get("filters")
|
||||
if isinstance(filters, dict):
|
||||
base_models = filters.get("baseModel")
|
||||
if isinstance(base_models, list):
|
||||
kwargs["base_models"] = [m for m in base_models if m]
|
||||
|
||||
for source_key, target_key in (("tags", "tags"), ("autoTags", "auto_tags")):
|
||||
states = filters.get(source_key)
|
||||
if isinstance(states, dict):
|
||||
mapped = {
|
||||
tag: state
|
||||
for tag, state in states.items()
|
||||
if state in ("include", "exclude")
|
||||
}
|
||||
if mapped:
|
||||
kwargs[target_key] = mapped
|
||||
|
||||
model_types = filters.get("modelTypes")
|
||||
if isinstance(model_types, list):
|
||||
kwargs["model_types"] = [t for t in model_types if t]
|
||||
|
||||
tag_logic = filters.get("tagLogic")
|
||||
if tag_logic:
|
||||
kwargs["tag_logic"] = tag_logic
|
||||
|
||||
license_filter = filters.get("license")
|
||||
if isinstance(license_filter, dict):
|
||||
no_credit = license_filter.get("noCredit")
|
||||
if no_credit == "include":
|
||||
kwargs["credit_required"] = False
|
||||
elif no_credit == "exclude":
|
||||
kwargs["credit_required"] = True
|
||||
allow_selling = license_filter.get("allowSelling")
|
||||
if allow_selling == "include":
|
||||
kwargs["allow_selling_generated_content"] = True
|
||||
elif allow_selling == "exclude":
|
||||
kwargs["allow_selling_generated_content"] = False
|
||||
|
||||
kwargs["recursive"] = recursive
|
||||
return kwargs
|
||||
@@ -0,0 +1,27 @@
|
||||
"""LLM-powered metadata enrichment pipeline infrastructure.
|
||||
|
||||
This package provides the orchestration layer for LLM-powered features.
|
||||
Skills define *what* to do (prompt template). The :class:`AgentService`
|
||||
handles *how* (LLM calls, context gathering, validation, progress).
|
||||
|
||||
NOTE: The current implementation is a code-driven pipeline, not a true
|
||||
agent loop. Future agent orchestration (LLM-driven tool selection) will
|
||||
live alongside this package with its own namespace.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .skill_definition import SkillDefinition, SkillPermissions
|
||||
from .skill_registry import SkillRegistry
|
||||
from .agent_service import AgentService, AgentProgressReporter, SkillResult
|
||||
from .post_processor import PostProcessor
|
||||
|
||||
__all__ = [
|
||||
"AgentProgressReporter",
|
||||
"AgentService",
|
||||
"PostProcessor",
|
||||
"SkillDefinition",
|
||||
"SkillPermissions",
|
||||
"SkillRegistry",
|
||||
"SkillResult",
|
||||
]
|
||||
@@ -0,0 +1,652 @@
|
||||
"""Pipeline orchestration service.
|
||||
|
||||
The :class:`AgentService` coordinates LLM-powered pipeline execution:
|
||||
|
||||
1. Look up the pipeline definition in :class:`SkillRegistry`
|
||||
2. Validate input against its ``input_schema``
|
||||
3. Prepare context via :mod:`~py.metadata_ops` (read metadata, list base models, fetch HF README)
|
||||
4. If ``llm_required``: call :class:`LLMService` with the rendered prompt
|
||||
5. Post-process via :class:`PostProcessor` (delegates I/O to :mod:`~py.metadata_ops`)
|
||||
6. Broadcast progress and completion via :class:`WebSocketManager`
|
||||
|
||||
Pipeline definitions (*skills*) describe *what* to do (prompt template).
|
||||
The AgentService handles *how* (LLM calls, context gathering, validation,
|
||||
progress).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ...config import config
|
||||
from ..llm_service import LLMService
|
||||
from ..model_sources import (
|
||||
ModelCardContext,
|
||||
ModelSourceCache,
|
||||
get_source,
|
||||
resolve_source_ref,
|
||||
source_label,
|
||||
)
|
||||
from ..model_sources.hydration import load_model_card, resolve_site_base_model
|
||||
from ..websocket_manager import ws_manager
|
||||
from .post_processor import PostProcessor
|
||||
from .skill_registry import SkillRegistry
|
||||
from .skills.enrich_hf_metadata.readme_processor import (
|
||||
clean_readme_for_llm,
|
||||
extract_relevant_section,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AgentProgressReporter:
|
||||
"""Protocol-compatible progress reporter backed by WebSocket broadcast."""
|
||||
|
||||
async def on_progress(self, payload: Dict[str, Any]) -> None:
|
||||
await ws_manager.broadcast(payload)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SkillResult:
|
||||
"""Outcome of a skill execution."""
|
||||
|
||||
success: bool
|
||||
updated_models: List[Dict[str, Any]] = field(default_factory=list)
|
||||
errors: List[str] = field(default_factory=list)
|
||||
summary: str = ""
|
||||
|
||||
|
||||
def _validate_schema(data: Any, schema: Dict[str, Any], path: str = "") -> List[str]:
|
||||
"""Minimal JSON schema validator.
|
||||
|
||||
Supports a subset of JSON Schema: ``type``, ``properties``, ``required``,
|
||||
``items``, ``enum``. Returns a list of error messages (empty = valid).
|
||||
"""
|
||||
|
||||
errors: List[str] = []
|
||||
if not schema:
|
||||
return errors
|
||||
|
||||
expected_type = schema.get("type")
|
||||
if expected_type:
|
||||
type_map = {
|
||||
"string": str,
|
||||
"number": (int, float),
|
||||
"integer": int,
|
||||
"boolean": bool,
|
||||
"array": list,
|
||||
"object": dict,
|
||||
"null": type(None),
|
||||
}
|
||||
expected_py = type_map.get(expected_type)
|
||||
if expected_py is not None and not isinstance(data, expected_py):
|
||||
errors.append(f"{path or 'root'}: expected {expected_type}, got {type(data).__name__}")
|
||||
return errors
|
||||
|
||||
if expected_type == "object" and isinstance(data, dict):
|
||||
properties = schema.get("properties", {})
|
||||
required = schema.get("required", [])
|
||||
for req_key in required:
|
||||
if req_key not in data:
|
||||
errors.append(f"{path or 'root'}: missing required property '{req_key}'")
|
||||
for key, value in data.items():
|
||||
if key in properties:
|
||||
errors.extend(_validate_schema(value, properties[key], f"{path}.{key}"))
|
||||
|
||||
if expected_type == "array" and isinstance(data, list):
|
||||
items_schema = schema.get("items")
|
||||
if items_schema:
|
||||
for i, item in enumerate(data):
|
||||
errors.extend(_validate_schema(item, items_schema, f"{path}[{i}]"))
|
||||
|
||||
if "enum" in schema and data not in schema["enum"]:
|
||||
errors.append(f"{path or 'root'}: value '{data}' not in enum {schema['enum']}")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Prompt template rendering
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
def _render_prompt(template: str, variables: Dict[str, Any]) -> str:
|
||||
"""Render a prompt template with ``{{variable}}`` placeholders.
|
||||
|
||||
Uses simple regex substitution — no Jinja2 dependency needed.
|
||||
"""
|
||||
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
key = match.group(1).strip()
|
||||
value = variables.get(key, "")
|
||||
if isinstance(value, (dict, list)):
|
||||
return json.dumps(value, ensure_ascii=False, indent=2)
|
||||
return str(value)
|
||||
|
||||
return re.sub(r"\{\{(\w+)\}\}", replace, template)
|
||||
|
||||
|
||||
class AgentService:
|
||||
"""Orchestrate agent skill execution.
|
||||
|
||||
Usage::
|
||||
|
||||
service = await AgentService.get_instance()
|
||||
result = await service.execute_skill(
|
||||
skill_name="enrich_hf_metadata",
|
||||
input_data={"model_paths": ["/path/to/model.safetensors"]},
|
||||
progress_callback=AgentProgressReporter(),
|
||||
)
|
||||
"""
|
||||
|
||||
_instance: Optional["AgentService"] = None
|
||||
_lock: asyncio.Lock = asyncio.Lock()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
skill_registry: Optional[SkillRegistry] = None,
|
||||
llm_service: Optional[LLMService] = None,
|
||||
) -> None:
|
||||
self._registry = skill_registry
|
||||
self._llm_service = llm_service
|
||||
|
||||
@classmethod
|
||||
async def get_instance(cls) -> "AgentService":
|
||||
"""Return the lazily-initialised global ``AgentService``."""
|
||||
|
||||
if cls._instance is None:
|
||||
async with cls._lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = cls(
|
||||
skill_registry=await SkillRegistry.get_instance(),
|
||||
llm_service=await LLMService.get_instance(),
|
||||
)
|
||||
return cls._instance
|
||||
|
||||
@classmethod
|
||||
def reset_instance(cls) -> None:
|
||||
"""Reset the cached singleton — primarily for tests."""
|
||||
|
||||
cls._instance = None
|
||||
|
||||
async def _ensure_registry(self) -> SkillRegistry:
|
||||
if self._registry is None:
|
||||
self._registry = await SkillRegistry.get_instance()
|
||||
return self._registry
|
||||
|
||||
async def _ensure_llm(self) -> LLMService:
|
||||
if self._llm_service is None:
|
||||
self._llm_service = await LLMService.get_instance()
|
||||
return self._llm_service
|
||||
|
||||
async def list_skills(self) -> List[Dict[str, Any]]:
|
||||
"""Return a JSON-serialisable list of available skills."""
|
||||
|
||||
registry = await self._ensure_registry()
|
||||
return [
|
||||
{
|
||||
"name": s.name,
|
||||
"title": s.title,
|
||||
"description": s.description,
|
||||
"llm_required": s.llm_required,
|
||||
"model_type_filter": s.model_type_filter,
|
||||
}
|
||||
for s in registry.list_skills()
|
||||
]
|
||||
|
||||
async def execute_skill(
|
||||
self,
|
||||
*,
|
||||
skill_name: str,
|
||||
input_data: Dict[str, Any],
|
||||
progress_callback: Optional[AgentProgressReporter] = None,
|
||||
) -> SkillResult:
|
||||
"""Execute a pipeline (skill) on the given models.
|
||||
|
||||
Args:
|
||||
skill_name: Name of the pipeline to execute
|
||||
input_data: Input validated against the pipeline's ``input_schema``
|
||||
progress_callback: Optional WebSocket progress reporter
|
||||
|
||||
Returns:
|
||||
:class:`SkillResult` with success status and updated model info
|
||||
"""
|
||||
|
||||
registry = await self._ensure_registry()
|
||||
skill = registry.get_skill(skill_name)
|
||||
if skill is None:
|
||||
return SkillResult(
|
||||
success=False,
|
||||
errors=[f"Skill not found: {skill_name}"],
|
||||
summary=f"Skill '{skill_name}' does not exist",
|
||||
)
|
||||
|
||||
input_errors = _validate_schema(input_data, skill.input_schema)
|
||||
if input_errors:
|
||||
return SkillResult(
|
||||
success=False,
|
||||
errors=input_errors,
|
||||
summary=f"Invalid input: {'; '.join(input_errors)}",
|
||||
)
|
||||
|
||||
model_paths = input_data.get("model_paths", [])
|
||||
if not model_paths:
|
||||
return SkillResult(
|
||||
success=False,
|
||||
errors=["No model_paths provided"],
|
||||
summary="No models to process",
|
||||
)
|
||||
|
||||
total = len(model_paths)
|
||||
processed = 0
|
||||
success_count = 0
|
||||
skipped_count = 0
|
||||
updated_models: List[Dict[str, Any]] = []
|
||||
errors: List[str] = []
|
||||
post_processor = PostProcessor()
|
||||
|
||||
await self._emit_progress(
|
||||
progress_callback, skill_name, status="started",
|
||||
total=total, processed=0, success=0,
|
||||
)
|
||||
|
||||
llm = await self._ensure_llm()
|
||||
llm_configured = llm.is_configured() if skill.llm_required else True
|
||||
|
||||
# A collection repository holds many model files under one source id;
|
||||
# this memo keeps the README and the repository metadata from being
|
||||
# re-fetched once per file. It lives for this run only.
|
||||
source_cache = ModelSourceCache()
|
||||
|
||||
for model_path in model_paths:
|
||||
model_filename = os.path.basename(model_path)
|
||||
logger.info(
|
||||
"[%s] [%d/%d] %s",
|
||||
skill_name, processed + 1, total, model_filename,
|
||||
)
|
||||
updated_data: Dict[str, Any] = {}
|
||||
skip_model = False
|
||||
try:
|
||||
from ...metadata_ops import read_metadata
|
||||
metadata = await read_metadata(model_path)
|
||||
|
||||
# Fast-fail: enrich_hf_metadata needs an external model source
|
||||
# that exposes an accessible model card.
|
||||
if skill_name == "enrich_hf_metadata":
|
||||
skip_reason = self._enrichment_skip_reason(metadata)
|
||||
if skip_reason:
|
||||
logger.info(
|
||||
"[%s] SKIP %s — %s",
|
||||
skill_name, model_filename, skip_reason,
|
||||
)
|
||||
skipped_count += 1
|
||||
skip_model = True
|
||||
|
||||
if not skip_model:
|
||||
# The site's own data is deterministic and must land whether
|
||||
# or not an LLM is available: a user without a key still gets
|
||||
# the author summary, the example images and the tags.
|
||||
source_vars, source_context = await self._load_source_card(
|
||||
model_path, metadata, cache=source_cache,
|
||||
)
|
||||
resolved_base_model = ""
|
||||
if skill_name == "enrich_hf_metadata" and not (
|
||||
metadata.get("base_model") or ""
|
||||
).strip():
|
||||
resolved_base_model = await self._resolve_site_base_model(
|
||||
source_context,
|
||||
)
|
||||
|
||||
llm_response: Optional[Dict[str, Any]] = None
|
||||
if skill.llm_required and not llm_configured:
|
||||
# Without a provider the deterministic model-source data
|
||||
# still lands; the LLM-only fields simply stay untouched.
|
||||
logger.info(
|
||||
"[%s] No LLM configured for %s — applying %s data only",
|
||||
skill_name, model_filename,
|
||||
"model-source"
|
||||
if not source_context.is_empty()
|
||||
else "README",
|
||||
)
|
||||
elif skill.llm_required:
|
||||
prompt_vars = await self._build_prompt_context(
|
||||
skill_name, model_path, metadata, registry, llm,
|
||||
source_vars=source_vars,
|
||||
source_context=source_context,
|
||||
)
|
||||
prompt_template = registry.load_prompt(skill_name)
|
||||
rendered = _render_prompt(prompt_template, prompt_vars)
|
||||
llm_response = await llm.chat_completion_json(
|
||||
system_prompt=prompt_vars.get(
|
||||
"system_prompt",
|
||||
"You are a helpful assistant that extracts structured metadata.",
|
||||
),
|
||||
user_prompt=rendered,
|
||||
)
|
||||
if llm_response:
|
||||
logger.info(
|
||||
"[%s] [%d/%d] %s → base_model=%s confidence=%s",
|
||||
skill_name, processed + 1, total, model_filename,
|
||||
(llm_response.get("base_model") or "?")[:50],
|
||||
llm_response.get("confidence", "?"),
|
||||
)
|
||||
|
||||
model_result = await post_processor.process(
|
||||
skill_name=skill_name,
|
||||
model_path=model_path,
|
||||
llm_output=llm_response or {},
|
||||
metadata=metadata,
|
||||
readme_content=source_vars.get("readme_content_full", ""),
|
||||
source_context=source_context,
|
||||
resolved_base_model=resolved_base_model,
|
||||
)
|
||||
|
||||
if model_result.get("success", True):
|
||||
success_count += 1
|
||||
uf = model_result.get("updated_fields", [])
|
||||
if uf:
|
||||
updated_models.append({"path": model_path, "updated_fields": uf})
|
||||
updated_data = model_result.get("updates", {})
|
||||
if "preview_url" in updated_data and updated_data["preview_url"]:
|
||||
updated_data["preview_url"] = config.get_preview_static_url(
|
||||
updated_data["preview_url"]
|
||||
)
|
||||
else:
|
||||
errors.extend(
|
||||
model_result.get("errors", [model_result.get("error", "Unknown error")])
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Skill %s failed for %s: %s", skill_name, model_path, exc)
|
||||
errors.append(f"{model_path}: {exc}")
|
||||
|
||||
processed += 1
|
||||
await self._emit_progress(
|
||||
progress_callback, skill_name, status="processing",
|
||||
total=total, processed=processed, success=success_count,
|
||||
skipped=skipped_count,
|
||||
current_path=model_path,
|
||||
updated_data=updated_data,
|
||||
)
|
||||
|
||||
result = SkillResult(
|
||||
success=success_count > 0,
|
||||
updated_models=updated_models,
|
||||
errors=errors,
|
||||
summary=f"Processed {processed}/{total} models, {success_count} succeeded, {skipped_count} skipped",
|
||||
)
|
||||
|
||||
await self._emit_progress(
|
||||
progress_callback, skill_name, status="completed",
|
||||
total=total, processed=processed, success=success_count,
|
||||
skipped=skipped_count,
|
||||
updated_models=updated_models, errors=errors, summary=result.summary,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Base model grouping (keeps the prompt compact)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _enrichment_skip_reason(metadata: Dict[str, Any]) -> str:
|
||||
"""Return why ``enrich_hf_metadata`` cannot run, or ``""`` if it can.
|
||||
|
||||
Distinguishes the three cases the user can act on: no source linked,
|
||||
a source we don't know, and a known source whose model card is not
|
||||
reachable from the backend (TensorArt).
|
||||
"""
|
||||
|
||||
ref = resolve_source_ref(metadata)
|
||||
if ref is None:
|
||||
return "no model source linked (source_url missing)"
|
||||
source = get_source(ref.platform)
|
||||
if source is None:
|
||||
return f"unsupported model source platform '{ref.platform}'"
|
||||
if not source.supports_enrichment:
|
||||
return (
|
||||
f"{source.label} does not expose a model card to the backend; "
|
||||
"AI metadata enrichment is not available for this source"
|
||||
)
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _format_base_models(models: List[str]) -> str:
|
||||
"""Format the base model list as a flat, one-per-line list.
|
||||
|
||||
Attempts to group by family consistently degraded LLM extraction
|
||||
accuracy — the LLM finds individual model names harder to spot
|
||||
in comma-separated groups than in a simple ``- Name`` list.
|
||||
"""
|
||||
return "\n".join(f"- {m}" for m in models)
|
||||
|
||||
async def _load_source_card(
|
||||
self,
|
||||
model_path: str,
|
||||
metadata: Dict[str, Any],
|
||||
*,
|
||||
cache: Optional[ModelSourceCache] = None,
|
||||
) -> tuple[Dict[str, Any], ModelCardContext]:
|
||||
"""Fetch the model card and site-published extras for one model.
|
||||
|
||||
Runs for every source-backed enrichment regardless of LLM
|
||||
availability, because everything it returns is deterministic data that
|
||||
should be applied even without a configured provider.
|
||||
|
||||
*cache* is the per-run memo created by :meth:`execute_skill`. The
|
||||
README is repository-wide, so it is fetched once per source id; only
|
||||
successful reads are memoised, leaving a transient failure to be
|
||||
retried for the next file.
|
||||
"""
|
||||
|
||||
variables: Dict[str, Any] = {
|
||||
"asset_base_url": "",
|
||||
"source_description": "",
|
||||
"source_base_model": "",
|
||||
"source_official_tags": "",
|
||||
"source_example_images": "",
|
||||
"source_trigger_words": "",
|
||||
"readme_content": "(README not available)",
|
||||
"readme_content_full": "",
|
||||
}
|
||||
|
||||
ref = resolve_source_ref(metadata)
|
||||
source = get_source(ref.platform) if ref is not None else None
|
||||
if ref is None or source is None or not source.supports_enrichment:
|
||||
return variables, ModelCardContext()
|
||||
|
||||
raw_basename = os.path.splitext(os.path.basename(model_path))[0]
|
||||
variables["asset_base_url"] = source.asset_base_url(ref.source_id)
|
||||
|
||||
readme = await load_model_card(source, ref.source_id, cache)
|
||||
|
||||
# Sites such as ModelScope keep part of the model card outside the
|
||||
# README (author summary, curated tags, per-file example images). The
|
||||
# recorded hash identifies the file even after the user renames it.
|
||||
card_context = await source.fetch_model_card_context(
|
||||
ref.source_id,
|
||||
os.path.basename(model_path),
|
||||
sha256=(metadata.get("sha256") or "").strip(),
|
||||
cache=cache,
|
||||
)
|
||||
variables["source_description"] = card_context.description
|
||||
variables["source_base_model"] = card_context.base_model
|
||||
variables["source_official_tags"] = "\n".join(
|
||||
f"- {tag}" for tag in card_context.official_tags
|
||||
)
|
||||
variables["source_example_images"] = "\n".join(
|
||||
f"- {url}" for url in card_context.example_images
|
||||
)
|
||||
variables["source_trigger_words"] = ", ".join(card_context.trigger_words)
|
||||
|
||||
# Trim README to the section relevant to this model file
|
||||
# (collection repos often have multiple models in one README).
|
||||
if readme and raw_basename:
|
||||
trimmed = extract_relevant_section(readme, raw_basename)
|
||||
cleaned = clean_readme_for_llm(trimmed) if trimmed else ""
|
||||
else:
|
||||
cleaned = clean_readme_for_llm(readme) if readme else ""
|
||||
variables["readme_content"] = cleaned if cleaned else "(README not available)"
|
||||
variables["readme_content_full"] = readme or ""
|
||||
|
||||
return variables, card_context
|
||||
|
||||
async def _resolve_site_base_model(self, source_context: ModelCardContext) -> str:
|
||||
"""Resolve the site's base-model hints to a canonical name, or ``""``."""
|
||||
|
||||
return await resolve_site_base_model(source_context)
|
||||
|
||||
async def _build_prompt_context(
|
||||
self,
|
||||
skill_name: str,
|
||||
model_path: str,
|
||||
metadata: Dict[str, Any],
|
||||
registry: SkillRegistry,
|
||||
llm: Any,
|
||||
*,
|
||||
source_vars: Optional[Dict[str, Any]] = None,
|
||||
source_context: Optional[ModelCardContext] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Gather variables for the skill's prompt template.
|
||||
|
||||
Reads metadata, fetches the model card (unless a pre-fetched
|
||||
*source_vars* / *source_context* pair is supplied), lists available
|
||||
base models, loads user priority tags, and returns a dict that maps to
|
||||
``{{variable}}`` placeholders in ``prompt.md``.
|
||||
"""
|
||||
from ...metadata_ops import identify_model_type, list_base_models
|
||||
from ..settings_manager import SettingsManager
|
||||
|
||||
if source_vars is None or source_context is None:
|
||||
source_vars, source_context = await self._load_source_card(
|
||||
model_path, metadata,
|
||||
)
|
||||
|
||||
context: Dict[str, Any] = {
|
||||
"model_path": model_path,
|
||||
"model_basename": "",
|
||||
# Canonical external-source variables
|
||||
"source_url": "",
|
||||
"source_id": "",
|
||||
"source_platform": "",
|
||||
"source_label": "",
|
||||
"asset_base_url": "",
|
||||
# Site-provided card extras (see ModelSource.fetch_model_card_context)
|
||||
"source_description": "",
|
||||
"source_base_model": "",
|
||||
"source_official_tags": "",
|
||||
"source_example_images": "",
|
||||
"source_trigger_words": "",
|
||||
# Carrier for the structured context handed to the post-processor;
|
||||
# never rendered into the prompt.
|
||||
"source_context": ModelCardContext(),
|
||||
# Legacy Hugging Face aliases (kept so older prompt templates and
|
||||
# third-party skills keep rendering)
|
||||
"hf_url": "",
|
||||
"repo": "",
|
||||
"readme_content": "",
|
||||
"readme_content_full": "",
|
||||
"current_metadata": {},
|
||||
"base_models": [],
|
||||
"priority_tags": "",
|
||||
}
|
||||
|
||||
# Extract model basename (filename without extension) for the LLM
|
||||
# to use when locating the matching section in collection repos.
|
||||
raw_basename = os.path.splitext(os.path.basename(model_path))[0]
|
||||
context["model_basename"] = raw_basename or ""
|
||||
|
||||
context["current_metadata"] = {
|
||||
"file_name": metadata.get("file_name", ""),
|
||||
"base_model": metadata.get("base_model", ""),
|
||||
"tags": metadata.get("tags", []),
|
||||
"modelDescription": metadata.get("modelDescription", ""),
|
||||
"sha256": (metadata.get("sha256") or "")[:16] + "..." if metadata.get("sha256") else "",
|
||||
"size": metadata.get("size", 0),
|
||||
}
|
||||
|
||||
ref = resolve_source_ref(metadata)
|
||||
if ref is not None:
|
||||
context["source_url"] = ref.url
|
||||
context["source_id"] = ref.source_id
|
||||
context["source_platform"] = ref.platform
|
||||
context["source_label"] = source_label(ref.platform, ref.platform)
|
||||
if ref.platform == "huggingface":
|
||||
context["hf_url"] = ref.url
|
||||
context["repo"] = ref.source_id
|
||||
|
||||
source = get_source(ref.platform) if ref is not None else None
|
||||
if ref is not None and source is not None and source.supports_enrichment:
|
||||
# Values fetched once by _load_source_card and shared with the
|
||||
# post-processor, so the network is not hit twice per model.
|
||||
context["asset_base_url"] = source_vars["asset_base_url"]
|
||||
context["source_context"] = source_context
|
||||
context["source_description"] = source_vars["source_description"]
|
||||
context["source_base_model"] = source_vars["source_base_model"]
|
||||
context["source_official_tags"] = source_vars["source_official_tags"]
|
||||
context["source_example_images"] = source_vars["source_example_images"]
|
||||
context["source_trigger_words"] = source_vars["source_trigger_words"]
|
||||
context["readme_content"] = source_vars["readme_content"]
|
||||
context["readme_content_full"] = source_vars["readme_content_full"]
|
||||
|
||||
try:
|
||||
raw_models = await list_base_models()
|
||||
context["base_models"] = self._format_base_models(raw_models)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to list base models: %s", exc)
|
||||
context["base_models"] = "</not available>"
|
||||
|
||||
# Determine model type and load the corresponding priority_tags
|
||||
try:
|
||||
model_type = await identify_model_type(model_path)
|
||||
context["model_type"] = model_type
|
||||
settings = SettingsManager()
|
||||
priority_config = settings.get_priority_tag_config()
|
||||
context["priority_tags"] = priority_config.get(model_type, "")
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to load priority tags: %s", exc)
|
||||
context["model_type"] = "lora"
|
||||
context["priority_tags"] = ""
|
||||
|
||||
return context
|
||||
|
||||
@staticmethod
|
||||
def _extract_repo_from_url(hf_url: str) -> Optional[str]:
|
||||
"""Extract ``user/repo`` from a HuggingFace URL."""
|
||||
if not hf_url:
|
||||
return None
|
||||
m = re.match(r"https?://huggingface\.co/([^/]+/[^/]+)", hf_url)
|
||||
return m.group(1) if m else None
|
||||
|
||||
@staticmethod
|
||||
async def _fetch_readme(repo: str) -> str:
|
||||
"""Fetch a Hugging Face README (tries ``main``, then ``master``).
|
||||
|
||||
Kept for backward compatibility; new code should go through the
|
||||
model-source registry so every supported site works.
|
||||
"""
|
||||
from ..model_sources import HuggingFaceSource
|
||||
|
||||
return await HuggingFaceSource().fetch_model_card(repo)
|
||||
|
||||
async def _emit_progress(
|
||||
self,
|
||||
callback: Optional[AgentProgressReporter],
|
||||
skill_name: str,
|
||||
*,
|
||||
status: str,
|
||||
**extra: Any,
|
||||
) -> None:
|
||||
"""Send a progress update via WebSocket (if callback is set)."""
|
||||
payload: Dict[str, Any] = {"type": "agent_progress", "skill": skill_name, "status": status}
|
||||
payload.update(extra)
|
||||
if callback is not None:
|
||||
await callback.on_progress(payload)
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Map a site-reported base model onto this system's canonical vocabulary.
|
||||
|
||||
Model sites name base models in their own terms: ModelScope publishes
|
||||
``krea/Krea-2-Turbo`` and ``KREA_2_TURBO`` where this system expects the
|
||||
canonical ``Krea 2``. Turning one into the other is normally the LLM's job;
|
||||
this module resolves the cases that can be decided safely so the canonical
|
||||
field is still populated when the LLM returns nothing usable for it.
|
||||
|
||||
The resolver is deliberately strict, because a wrong base model written with
|
||||
apparent authority is worse than no value at all:
|
||||
|
||||
* it only ever returns a name that is already present in *known_names*;
|
||||
* matching is on the normalised form (lowercased, non-alphanumerics removed),
|
||||
so separators and casing are ignored but nothing is inferred;
|
||||
* a bounded set of published variant suffixes may be stripped, and only when
|
||||
the remainder still matches a known name exactly.
|
||||
|
||||
Anything it cannot decide returns ``""``, and the caller falls back to the LLM.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Iterable, Sequence
|
||||
|
||||
#: Variant suffixes sites append to a base-model *family* name. Stripping one
|
||||
#: is only attempted when the remainder matches a known name exactly, so an
|
||||
#: unrecognised suffix can never produce a bogus match.
|
||||
_VARIANT_SUFFIXES: tuple[str, ...] = (
|
||||
"turbo",
|
||||
"schnell",
|
||||
"lightning",
|
||||
"dev",
|
||||
"beta",
|
||||
"alpha",
|
||||
)
|
||||
|
||||
_NON_ALNUM = re.compile(r"[^a-z0-9]+")
|
||||
|
||||
|
||||
def _normalize(value: str) -> str:
|
||||
"""Return the comparison form of *value*.
|
||||
|
||||
Lowercases and drops every non-alphanumeric character, so ``KREA_2``,
|
||||
``Krea 2``, ``krea-2`` and ``krea.2`` all collapse to ``krea2``.
|
||||
"""
|
||||
|
||||
return _NON_ALNUM.sub("", (value or "").lower())
|
||||
|
||||
|
||||
def resolve_base_model(
|
||||
hints: Iterable[str], known_names: Sequence[str]
|
||||
) -> str:
|
||||
"""Return the canonical base model that *hints* refers to, or ``""``.
|
||||
|
||||
Args:
|
||||
hints: Site-reported names, best first (e.g. an architecture enum
|
||||
before a link-style repository id).
|
||||
known_names: The canonical vocabulary; only these are ever returned.
|
||||
|
||||
Returns:
|
||||
One of *known_names*, or ``""`` when nothing matches exactly.
|
||||
"""
|
||||
|
||||
normalized: dict[str, str] = {}
|
||||
for name in known_names:
|
||||
key = _normalize(name)
|
||||
if key and key not in normalized:
|
||||
normalized[key] = name
|
||||
if not normalized:
|
||||
return ""
|
||||
|
||||
ordered = [hint for hint in hints if hint]
|
||||
|
||||
# 1. Exact normalised match — the unambiguous case.
|
||||
for hint in ordered:
|
||||
candidate = _normalize(hint)
|
||||
if candidate in normalized:
|
||||
return normalized[candidate]
|
||||
|
||||
# 2. Drop one published variant suffix and retry exactly.
|
||||
for hint in ordered:
|
||||
candidate = _normalize(hint)
|
||||
for suffix in _VARIANT_SUFFIXES:
|
||||
if not candidate.endswith(suffix) or candidate == suffix:
|
||||
continue
|
||||
stem = candidate[: -len(suffix)]
|
||||
if stem in normalized:
|
||||
return normalized[stem]
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
__all__ = ["resolve_base_model"]
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user