mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-09-21 03:01:27 -03:00
Compare commits
210 Commits
v1.2.0
...
34ca14d7fc
| Author | SHA1 | Date | |
|---|---|---|---|
| 34ca14d7fc | |||
| f7b247f9e8 | |||
| 3005d2877e | |||
| ed2a17970f | |||
| 9584fa85c9 | |||
| 1fd7cc0123 | |||
| 39e7c1376c | |||
| 2a3c632dc5 | |||
| 8d46d26abe | |||
| d761ac77f7 | |||
| c8b9db5bf4 | |||
| bce7d1d30c | |||
| bccd494a56 | |||
| 3fd29f6943 | |||
| 838a374a56 | |||
| 6e31da7a70 | |||
| fc9088bfd6 | |||
| 675421ea84 | |||
| 2ff98ae089 | |||
| c972c755fc | |||
| ebe3df7d22 | |||
| be44a75b74 | |||
| fd1227d3b8 | |||
| 3a9e02137d | |||
| d8a2be8edc | |||
| 1c46b2e8c3 | |||
| 3c3ac49f2f | |||
| 1a1be95a64 | |||
| 7a36659a20 | |||
| cb18281b14 | |||
| 856c9a87ac | |||
| a7d65fe84a | |||
| 15bf079af2 | |||
| 65ba750634 | |||
| 17dcbd3d4f | |||
| e914a0e19d | |||
| 2ba04bb1bd | |||
| 1b7314591a | |||
| 2bfb987312 | |||
| df34efafbc | |||
| c2a2048c8b | |||
| 1e1921cabb | |||
| ee233548e5 | |||
| 574dfbbe55 | |||
| 1d3bcdfe47 | |||
| 74369940bf | |||
| d188cec306 | |||
| 641a61f804 | |||
| 3025c64fea | |||
| c52cfc7e7a | |||
| 4ed9f775f6 | |||
| 0b08ad283a | |||
| 08895f77ff | |||
| 74f889f160 | |||
| c51090ab16 | |||
| cdb044cb45 | |||
| c83b26b556 | |||
| a202c666bc | |||
| e05046af10 | |||
| 41ed03e5c6 | |||
| da071e8452 | |||
| a0bb6df2b8 | |||
| 6f5c444ec5 | |||
| 20f66a4fe1 | |||
| 879745da53 | |||
| 3afec0a0be | |||
| 06c270a6e1 | |||
| 87e93636dc | |||
| 074d1f2e51 | |||
| 40f922b0e8 | |||
| a7214b6cff | |||
| 8ca66e72eb | |||
| 90be5799e4 | |||
| 1a93b0eca2 | |||
| c2360a35ad | |||
| 030a32f8fa | |||
| 25e72b43ce | |||
| 41e9883daa | |||
| ae461ebc81 | |||
| 3ebf256c5d | |||
| 0905e2be6e | |||
| bd380bc1a1 | |||
| cb4fd3a0e6 | |||
| bbe0acac5c | |||
| 45e7c25308 | |||
| 86aa1d8059 | |||
| 74254756ef | |||
| 259e08e47c | |||
| 6647c45731 | |||
| b614a5c447 | |||
| b80830913c | |||
| e57e11897e | |||
| 8a16034135 | |||
| 7fc3b7e5be | |||
| b0c7a1baae | |||
| 6411d83d46 | |||
| 74a063b0e5 | |||
| 96376e5cce | |||
| e7c26bf722 | |||
| cef4129fc9 | |||
| 0a28500848 | |||
| fc3f3f3bdb | |||
| fa58297973 | |||
| 5d1a22fb8f | |||
| d2f50f26f1 | |||
| 4a6042d0b4 | |||
| 846206d958 | |||
| 0daf4924f0 | |||
| d38a3d091d | |||
| 94dd08646d | |||
| 658f88ca48 | |||
| f53352efb2 | |||
| 38809a9d1b | |||
| 395682509c | |||
| ef3e7d7bf4 | |||
| c85b6b64a1 | |||
| 34c87d4934 | |||
| 93472e5d67 | |||
| ae185ee714 | |||
| 795036275a | |||
| d43ab6e32f | |||
| 280181f92e | |||
| f8d98934ad | |||
| 303cca0d85 | |||
| c2f16784b3 | |||
| 5bc6d8286c | |||
| 3f8381ffee | |||
| 1ca99294c9 | |||
| 680f0a57f5 | |||
| 94e3f54571 | |||
| 5c2b2aedcc | |||
| ebc31fb963 | |||
| 9659df6ad9 | |||
| 04d131e9dc | |||
| 78fe6282c7 | |||
| 0c00ee22fc | |||
| 5fd4946b1f | |||
| f1d3ac0cdc | |||
| e2c45905f0 | |||
| b2c68e6a65 | |||
| eb0f6dd3b6 | |||
| 0bf87f9092 | |||
| 1da2433bb2 | |||
| 2d6cf545b9 | |||
| 6a259a14fa | |||
| 41e1fd1e1f | |||
| 95fb3c7fc9 | |||
| 8237e5f9ea | |||
| aa75986178 | |||
| b887922055 | |||
| 68fa0f29c7 | |||
| d9d362c9c9 | |||
| d0bc4be0dc | |||
| 420530f532 | |||
| 3001f0f0ef | |||
| b2a1307d23 | |||
| 64da845a58 | |||
| 27027c4497 | |||
| 86c85c08ec | |||
| 196c8ffc3e | |||
| cfc95ee02a | |||
| 479fa36997 | |||
| 3e1216e9bc | |||
| 007883b7d1 | |||
| dc9200a12c | |||
| d2f955266d | |||
| 8e724538bd | |||
| 6fcdeb799d | |||
| 97b9b1f62b | |||
| 4bf9a4b640 | |||
| c5088772e8 | |||
| 56acefbd6c | |||
| 5ab06c4aae | |||
| c11f4b5c68 | |||
| 86376284f4 | |||
| 2b8a2fc7d8 | |||
| f26e1b41c8 | |||
| c1671af99f | |||
| ac7707d0f6 | |||
| 381cd710a2 | |||
| ad0d18cb79 | |||
| 7980ee77d0 | |||
| 916b8bb327 | |||
| 87e3d4dea9 | |||
| 76a913f5e0 | |||
| d8c192e647 | |||
| c453437620 | |||
| 720fa6d909 | |||
| b4f71089f4 | |||
| 83e6657ead | |||
| 7ea6df4111 | |||
| d9ab92602a | |||
| 5ffadaed31 | |||
| 24f5f7df5d | |||
| daf01fb1d6 | |||
| 0f11b6def9 | |||
| 7df83f44b8 | |||
| 169fa7bed6 | |||
| 027b504fe8 | |||
| 186ef4da78 | |||
| dc674098e7 | |||
| 9087b4b07c | |||
| 8e45c22d7a | |||
| 191c4e03cd | |||
| ab4154c57d | |||
| 28e93d12ff | |||
| 75e63c758b | |||
| 823f71f269 | |||
| 042dd4088d | |||
| eaa791a9eb |
@@ -1,201 +1,146 @@
|
|||||||
---
|
---
|
||||||
name: lora-manager-e2e
|
name: lora-manager-e2e
|
||||||
description: End-to-end testing and validation for LoRa Manager features. Use when performing automated E2E validation of LoRa Manager standalone mode, including starting/restarting the server, using Chrome DevTools MCP to interact with the web UI at http://127.0.0.1:8188/loras, and verifying frontend-to-backend functionality. Covers workflow validation, UI interaction testing, and integration testing between the standalone Python backend and the browser frontend.
|
description: "End-to-end testing and validation for LoRa Manager features. Use ONLY for sandboxed E2E validation of LoRa Manager standalone mode: start the standalone server on a free port with --settings-path, drive the web UI (http://127.0.0.1:{PORT}/loras) via Chrome DevTools MCP, and verify frontend-to-backend integration. NOT for UI behavior checks that unit tests (Vitest/jsdom) can cover. Trigger keywords: E2E, standalone, Chrome DevTools MCP, lora-manager-e2e, sandbox."
|
||||||
---
|
---
|
||||||
|
|
||||||
# LoRa Manager E2E Testing
|
# LoRa Manager E2E Testing
|
||||||
|
|
||||||
This skill provides workflows and utilities for end-to-end testing of LoRa Manager using Chrome DevTools MCP.
|
End-to-end testing of LoRa Manager standalone mode using Chrome DevTools MCP.
|
||||||
|
|
||||||
## Prerequisites
|
## When to Use — and When NOT To
|
||||||
|
|
||||||
- LoRa Manager project cloned and dependencies installed (`pip install -r requirements.txt`)
|
E2E runs are slow and token-heavy. Reach for them only when the question genuinely
|
||||||
- Chrome browser available for debugging
|
spans server + browser (routing, scan persistence, websocket updates, EXIF writes).
|
||||||
- Chrome DevTools MCP connected
|
|
||||||
|
|
||||||
## Quick Start Workflow
|
- **Default to unit/component tests first**: `npm run test:js` (Vitest/jsdom) covers
|
||||||
|
DOM rendering, modal behavior, event handling and API-client calls deterministically
|
||||||
|
in seconds. Backend logic goes through `pytest`. A UI-behavior question answered by
|
||||||
|
jsdom MUST NOT be escalated to E2E.
|
||||||
|
- **Use E2E only when** the behavior cannot be observed without a live server and a
|
||||||
|
real browser, e.g. template rendering through the aiohttp server, scanner → SQLite
|
||||||
|
persistence → API → DOM round-trips, or real EXIF/image writes.
|
||||||
|
- If you start an E2E and realize a unit test would answer the question, stop and
|
||||||
|
switch.
|
||||||
|
|
||||||
### 1. Start LoRa Manager Standalone
|
**Browser driver is fixed: Chrome DevTools MCP.** Do not substitute kimi-webbridge —
|
||||||
|
it operates on the user's real browser (real tabs, real sessions, synthetic
|
||||||
|
`isTrusted=false` events), which breaks the isolation this skill requires and lacks
|
||||||
|
the console/network inspection E2E debugging relies on. kimi-webbridge is for
|
||||||
|
interactive browsing with the user's real login sessions, not for sandboxed E2E.
|
||||||
|
|
||||||
```python
|
## Conventions
|
||||||
# Use the provided script to start the server
|
|
||||||
python .agents/skills/lora-manager-e2e/scripts/start_server.py --port 8188
|
|
||||||
```
|
|
||||||
|
|
||||||
Or manually:
|
- **`{PORT}`**: default candidate `8188`, but it is **commonly occupied by a live
|
||||||
```bash
|
ComfyUI** — always check first (`ss -tlnp | grep ':{PORT}'`) and use a free port
|
||||||
cd /home/miao/workspace/ComfyUI/custom_nodes/ComfyUI-Lora-Manager
|
(e.g. `8199`). Substitute the chosen port everywhere below. Never kill a process
|
||||||
python standalone.py --port 8188
|
you did not start for this E2E.
|
||||||
```
|
- **`<repo-root>`**: the repository/worktree root; run all commands from there.
|
||||||
|
- **`<sandbox>`**: a throwaway dir, e.g. `/tmp/opencode/<plan>-e2e`.
|
||||||
|
|
||||||
Wait for server ready message before proceeding.
|
## SANDBOX (MANDATORY)
|
||||||
|
|
||||||
### 2. Open Chrome Debug Mode
|
> Every E2E run MUST target a throwaway sandbox, never real user data.
|
||||||
|
|
||||||
|
1. **Explicit settings directory**: always launch with `--settings-path <sandbox>/settings`.
|
||||||
|
This pins ALL runtime data (`settings.json`, `cache/`, `backups/`, `logs/`, `stats/`,
|
||||||
|
`wildcards/`) under the sandbox. **Never** create `<repo-root>/settings.json` — the repo
|
||||||
|
folder is usually the real ComfyUI plugin folder and a portable settings file there is
|
||||||
|
read by the real instance.
|
||||||
|
2. **Sandboxed library paths**: point `folder_paths` / `recipes_path` /
|
||||||
|
`example_images_path` at disposable dirs under `<sandbox>` — never the real library,
|
||||||
|
real recipe dir, or real settings:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"folder_paths": {
|
||||||
|
"loras": ["<sandbox>/models/loras"],
|
||||||
|
"checkpoints": ["<sandbox>/models/checkpoints"],
|
||||||
|
"unet": ["<sandbox>/models/checkpoints"],
|
||||||
|
"diffusers": []
|
||||||
|
},
|
||||||
|
"recipes_path": "<sandbox>/recipes",
|
||||||
|
"example_images_path": "<sandbox>/example_images"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **Real-data protection proof**: before starting and after finishing, snapshot the real
|
||||||
|
config and recipe library and confirm they are byte-identical; also confirm
|
||||||
|
`<repo-root>` gained no `settings.json` or `cache/`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sha256sum ~/.config/ComfyUI-LoRA-Manager/settings.json > <sandbox>/settings.before.sha256
|
||||||
|
ls ~/models/recipes/*.recipe.json 2>/dev/null | wc -l > <sandbox>/recipes-count.before.txt
|
||||||
|
# AFTER the run: record again and diff. Any change = the run leaked into real data.
|
||||||
|
```
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Chrome with remote debugging on port 9222
|
cd <repo-root>
|
||||||
google-chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:8188/loras
|
# 1. Sandbox
|
||||||
|
mkdir -p <sandbox>/settings <sandbox>/models/{loras,checkpoints} <sandbox>/{recipes,example_images}
|
||||||
|
# write <sandbox>/settings/settings.json per the SANDBOX example
|
||||||
|
# 2. Port
|
||||||
|
ss -tlnp | grep ':{PORT}' || echo "port {PORT} is free"
|
||||||
|
# 3. Server — MUST be fully detached (a plain background & dies with the shell);
|
||||||
|
# the helper enforces this and manages its own pidfile
|
||||||
|
python .agents/skills/lora-manager-e2e/scripts/start_server.py \
|
||||||
|
--port {PORT} --settings-path <sandbox>/settings --wait --timeout 30 --detach
|
||||||
|
ss -tlnp | grep ':{PORT}' # verify listening BEFORE proceeding
|
||||||
|
# 4. Chrome with remote debugging, then connect Chrome DevTools MCP (verify via list_pages)
|
||||||
|
google-chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:{PORT}/loras
|
||||||
```
|
```
|
||||||
|
|
||||||
### 3. Connect Chrome DevTools MCP
|
Then drive the UI with the MCP tools (`take_snapshot`, `click`, `fill`, `fill_form`,
|
||||||
|
`evaluate_script`, `wait_for`, `list_network_requests`, `list_console_messages`) —
|
||||||
|
see [references/mcp-cheatsheet.md](references/mcp-cheatsheet.md) for patterns.
|
||||||
|
|
||||||
Ensure the MCP server is connected to Chrome at `http://localhost:9222`.
|
Server restart after config/fixture changes:
|
||||||
|
|
||||||
### 4. Navigate and Interact
|
|
||||||
|
|
||||||
Use Chrome DevTools MCP tools to:
|
|
||||||
- Take snapshots: `take_snapshot`
|
|
||||||
- Click elements: `click`
|
|
||||||
- Fill forms: `fill` or `fill_form`
|
|
||||||
- Evaluate scripts: `evaluate_script`
|
|
||||||
- Wait for elements: `wait_for`
|
|
||||||
|
|
||||||
## Common E2E Test Patterns
|
|
||||||
|
|
||||||
### Pattern: Full Page Load Verification
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Navigate to LoRA list page
|
|
||||||
navigate_page(type="url", url="http://127.0.0.1:8188/loras")
|
|
||||||
|
|
||||||
# Wait for page to load
|
|
||||||
wait_for(text="LoRAs", timeout=10000)
|
|
||||||
|
|
||||||
# Take snapshot to verify UI state
|
|
||||||
snapshot = take_snapshot()
|
|
||||||
```
|
|
||||||
|
|
||||||
### Pattern: Restart Server for Configuration Changes
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Stop current server (if running)
|
|
||||||
# Start with new configuration
|
|
||||||
python .agents/skills/lora-manager-e2e/scripts/start_server.py --port 8188 --restart
|
|
||||||
|
|
||||||
# Wait and refresh browser
|
|
||||||
navigate_page(type="reload", ignoreCache=True)
|
|
||||||
wait_for(text="LoRAs", timeout=15000)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Pattern: Verify Backend API via Frontend
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Execute script in browser to call backend API
|
|
||||||
result = evaluate_script(function="""
|
|
||||||
async () => {
|
|
||||||
const response = await fetch('/loras/api/list');
|
|
||||||
const data = await response.json();
|
|
||||||
return { count: data.length, firstItem: data[0]?.name };
|
|
||||||
}
|
|
||||||
""")
|
|
||||||
```
|
|
||||||
|
|
||||||
### Pattern: Form Submission Flow
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Fill a form (e.g., search or filter)
|
|
||||||
fill_form(elements=[
|
|
||||||
{"uid": "search-input", "value": "character"},
|
|
||||||
])
|
|
||||||
|
|
||||||
# Click submit button
|
|
||||||
click(uid="search-button")
|
|
||||||
|
|
||||||
# Wait for results
|
|
||||||
wait_for(text="Results", timeout=5000)
|
|
||||||
|
|
||||||
# Verify results via snapshot
|
|
||||||
snapshot = take_snapshot()
|
|
||||||
```
|
|
||||||
|
|
||||||
### Pattern: Modal Dialog Interaction
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Open modal (e.g., add LoRA)
|
|
||||||
click(uid="add-lora-button")
|
|
||||||
|
|
||||||
# Wait for modal to appear
|
|
||||||
wait_for(text="Add LoRA", timeout=3000)
|
|
||||||
|
|
||||||
# Fill modal form
|
|
||||||
fill_form(elements=[
|
|
||||||
{"uid": "lora-name", "value": "Test LoRA"},
|
|
||||||
{"uid": "lora-path", "value": "/path/to/lora.safetensors"},
|
|
||||||
])
|
|
||||||
|
|
||||||
# Submit
|
|
||||||
click(uid="modal-submit-button")
|
|
||||||
|
|
||||||
# Wait for success message or close
|
|
||||||
wait_for(text="Success", timeout=5000)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Available Scripts
|
|
||||||
|
|
||||||
### scripts/start_server.py
|
|
||||||
|
|
||||||
Starts or restarts the LoRa Manager standalone server.
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python scripts/start_server.py [--port PORT] [--restart] [--wait]
|
python .agents/skills/lora-manager-e2e/scripts/start_server.py \
|
||||||
|
--port {PORT} --settings-path <sandbox>/settings --restart --wait --detach
|
||||||
|
# then reload the browser page (ignoreCache=True)
|
||||||
```
|
```
|
||||||
|
|
||||||
Options:
|
`--restart` only kills the E2E server the script itself started (via its pidfile) and
|
||||||
- `--port`: Server port (default: 8188)
|
aborts instead of killing unrelated processes on the port.
|
||||||
- `--restart`: Kill existing server before starting
|
|
||||||
- `--wait`: Wait for server to be ready before exiting
|
|
||||||
|
|
||||||
### scripts/wait_for_server.py
|
## Abort Rule
|
||||||
|
|
||||||
Polls server until ready or timeout.
|
A sandboxed E2E should finish in well under 30 minutes. If any phase exceeds ~2x its
|
||||||
|
expected duration (server readiness > 60 s, MCP connect > 2 min, a single scenario >
|
||||||
|
10 min), or any single tool call fails 3+ times in a row, **STOP** — do not retry
|
||||||
|
blindly. Report `BLOCKED` with the phase, last observed state (server PID,
|
||||||
|
`ss -tlnp` output, page snapshot, last API response) and suspected cause. A clean
|
||||||
|
BLOCKED report beats an hour of retries.
|
||||||
|
|
||||||
```bash
|
## Troubleshooting
|
||||||
python scripts/wait_for_server.py [--port PORT] [--timeout SECONDS]
|
|
||||||
```
|
|
||||||
|
|
||||||
## Test Scenarios Reference
|
- **"browser is already running" / `list_pages` fails**: a stale Chrome holds the
|
||||||
|
profile dir. Find it (`ps -ef | grep -i '[c]hrome.*user-data-dir'`), confirm it is a
|
||||||
See [references/test-scenarios.md](references/test-scenarios.md) for detailed test scenarios including:
|
leftover QA Chrome (not the live ComfyUI, not your current MCP browser), kill only
|
||||||
- LoRA list display and filtering
|
that PID, then retry `list_pages`.
|
||||||
- Model metadata editing
|
- **MCP refuses to write screenshots into the worktree**: save to `/tmp` via
|
||||||
- Recipe creation and management
|
`take_screenshot(filePath="/tmp/...")` and copy into the evidence dir from the shell.
|
||||||
- Settings configuration
|
|
||||||
- Import/export functionality
|
|
||||||
|
|
||||||
## Network Request Verification
|
|
||||||
|
|
||||||
Use `list_network_requests` and `get_network_request` to verify API calls:
|
|
||||||
|
|
||||||
```python
|
|
||||||
# List recent XHR/fetch requests
|
|
||||||
requests = list_network_requests(resourceTypes=["xhr", "fetch"])
|
|
||||||
|
|
||||||
# Get details of specific request
|
|
||||||
details = get_network_request(reqid=123)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Console Message Monitoring
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Check for errors or warnings
|
|
||||||
messages = list_console_messages(types=["error", "warn"])
|
|
||||||
```
|
|
||||||
|
|
||||||
## Performance Testing
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Start performance trace
|
|
||||||
performance_start_trace(reload=True, autoStop=False)
|
|
||||||
|
|
||||||
# Perform actions...
|
|
||||||
|
|
||||||
# Stop and analyze
|
|
||||||
results = performance_stop_trace()
|
|
||||||
```
|
|
||||||
|
|
||||||
## Cleanup
|
## Cleanup
|
||||||
|
|
||||||
Always ensure proper cleanup after tests:
|
1. Stop the standalone server: `kill <recorded-pid>` (only the PID you started), then
|
||||||
1. Stop the standalone server
|
confirm `ss -tlnp | grep ':{PORT}'` is empty.
|
||||||
2. Close browser pages (keep at least one open)
|
2. Close browser pages (keep at least one open).
|
||||||
3. Clear temporary data if needed
|
3. `rm -rf <sandbox>`; verify `<repo-root>` gained no `settings.json` or `cache/`.
|
||||||
|
4. Re-run the real-data protection check from the SANDBOX section and record the result.
|
||||||
|
|
||||||
|
## References & Scripts
|
||||||
|
|
||||||
|
- [references/mcp-cheatsheet.md](references/mcp-cheatsheet.md) — Chrome DevTools MCP
|
||||||
|
command patterns (navigation, waiting, snapshots, forms, network, console, performance).
|
||||||
|
- [references/test-scenarios.md](references/test-scenarios.md) — detailed test scenarios
|
||||||
|
(list display, metadata editing, recipes, settings, import/export).
|
||||||
|
- [references/recipe-rematch-fixtures.md](references/recipe-rematch-fixtures.md) —
|
||||||
|
fixture format, fresh-state reset and known gaps for recipe rematch/repair E2E runs.
|
||||||
|
- `scripts/start_server.py` — start/restart the standalone server
|
||||||
|
(`--port --settings-path --restart --wait --timeout --detach`); refuses to touch
|
||||||
|
unrelated processes on the port.
|
||||||
|
- `scripts/wait_for_server.py` — poll readiness (`--port --timeout`).
|
||||||
|
|||||||
@@ -2,11 +2,13 @@
|
|||||||
|
|
||||||
Quick reference for common MCP commands used in LoRa Manager E2E testing.
|
Quick reference for common MCP commands used in LoRa Manager E2E testing.
|
||||||
|
|
||||||
|
> **Port convention**: `{PORT}` is the port chosen for the E2E run (default candidate `8188`, but only if actually free — see the SKILL.md Port Selection section; use e.g. `8199` when `8188` is occupied by a live ComfyUI). Always run against the **sandboxed** standalone server, never a live instance.
|
||||||
|
|
||||||
## Navigation
|
## Navigation
|
||||||
|
|
||||||
```python
|
```python
|
||||||
# Navigate to LoRA list page
|
# Navigate to LoRA list page
|
||||||
navigate_page(type="url", url="http://127.0.0.1:8188/loras")
|
navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
||||||
|
|
||||||
# Reload page with cache clear
|
# Reload page with cache clear
|
||||||
navigate_page(type="reload", ignoreCache=True)
|
navigate_page(type="reload", ignoreCache=True)
|
||||||
@@ -179,7 +181,7 @@ pages = list_pages()
|
|||||||
select_page(pageId=0, bringToFront=True)
|
select_page(pageId=0, bringToFront=True)
|
||||||
|
|
||||||
# Create new page
|
# Create new page
|
||||||
new_page(url="http://127.0.0.1:8188/loras")
|
new_page(url="http://127.0.0.1:{PORT}/loras")
|
||||||
|
|
||||||
# Close page (keep at least one open!)
|
# Close page (keep at least one open!)
|
||||||
close_page(pageId=1)
|
close_page(pageId=1)
|
||||||
@@ -261,7 +263,7 @@ drag(from_uid="draggable-item", to_uid="drop-zone")
|
|||||||
### Verify LoRA Cards Loaded
|
### Verify LoRA Cards Loaded
|
||||||
|
|
||||||
```python
|
```python
|
||||||
navigate_page(type="url", url="http://127.0.0.1:8188/loras")
|
navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
||||||
wait_for(text="LoRAs", timeout=10000)
|
wait_for(text="LoRAs", timeout=10000)
|
||||||
|
|
||||||
# Check if cards loaded
|
# Check if cards loaded
|
||||||
@@ -322,3 +324,37 @@ navigate_page(type="reload")
|
|||||||
errors = list_console_messages(types=["error"])
|
errors = list_console_messages(types=["error"])
|
||||||
assert len(errors) == 0, f"Console errors: {errors}"
|
assert len(errors) == 0, f"Console errors: {errors}"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Stale profile lock ("browser is already running" / `list_pages` fails)
|
||||||
|
|
||||||
|
A Chrome profile held by a stale Chrome from a prior MCP session makes `list_pages`
|
||||||
|
fail with "browser is already running". Fix:
|
||||||
|
|
||||||
|
1. Find the stale Chrome that owns the profile dir (e.g. `~/.config/chrome-dev-profile`):
|
||||||
|
```bash
|
||||||
|
ps -ef | grep -i '[c]hrome.*user-data-dir'
|
||||||
|
```
|
||||||
|
2. Confirm it is a QA Chrome from a completed task (NOT the live ComfyUI server, NOT
|
||||||
|
your current MCP instance).
|
||||||
|
3. Kill ONLY that stale Chrome (`kill <stale-pid>`), then retry `list_pages`.
|
||||||
|
|
||||||
|
### Screenshot-write restrictions
|
||||||
|
|
||||||
|
The MCP may refuse to write into paths outside its configured workspace roots
|
||||||
|
(e.g. `.omo/evidence/screenshots/` under a worktree that canonicalizes to an unmapped
|
||||||
|
path). Save the screenshot to `/tmp` via the MCP, then copy it into the evidence dir:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# MCP: take_screenshot(filePath="/tmp/<plan>-e2e/recipe-b-after.png", format="png")
|
||||||
|
# Shell:
|
||||||
|
mkdir -p <repo-root>/.omo/evidence/screenshots
|
||||||
|
cp /tmp/<plan>-e2e/recipe-b-after.png <repo-root>/.omo/evidence/screenshots/
|
||||||
|
```
|
||||||
|
|
||||||
|
### Time budgets & abort rule
|
||||||
|
|
||||||
|
See SKILL.md "Time Budgets & Abort Guidance": if a phase exceeds ~2x its budget or a
|
||||||
|
tool call retries 3+ times in a row, STOP and report BLOCKED with the last observed
|
||||||
|
state (server PID + `ss -tlnp`, page snapshot, last API response). Do not loop.
|
||||||
|
|||||||
@@ -0,0 +1,72 @@
|
|||||||
|
# Recipe Rematch/Repair E2E — Fixtures, Fresh State, Known Gaps
|
||||||
|
|
||||||
|
Specialized guidance for recipe rematch/repair E2E runs, extracted from the SKILL.md
|
||||||
|
main flow. Read the SKILL.md SANDBOX section first — everything here assumes a
|
||||||
|
sandboxed run.
|
||||||
|
|
||||||
|
## Fixture Rules (validated by the task-8 E2E)
|
||||||
|
|
||||||
|
Seed the **sandboxed** `recipes_path` with hand-written fixture recipes:
|
||||||
|
|
||||||
|
1. **Filename constraint**: each file MUST be named `f"{id}.recipe.json"` **and** the
|
||||||
|
in-JSON `id` field MUST equal the filename. Discovery accepts any `*.recipe.json`,
|
||||||
|
but persistence resolves the path via `get_recipe_json_path` and
|
||||||
|
`_save_recipe_persistently` returns `False` on a mismatch → the fixture would be
|
||||||
|
counted as an error.
|
||||||
|
- `recipe-a.recipe.json` → in-JSON `"id": "recipe-a"`
|
||||||
|
2. **File format**: mirror an existing recipe JSON — top-level `id`, `file_path`,
|
||||||
|
`title`, `loras`, `fingerprint`, `gen_params`; lora entries per the persistence
|
||||||
|
conventions (`hash`, `file_name`, `modelVersionId`, `isDeleted`, ...).
|
||||||
|
3. **Companion image**: each recipe needs an image (e.g. a `.webp` generated with PIL)
|
||||||
|
referenced by `file_path`, used for EXIF verification
|
||||||
|
(`ExifUtils.append_recipe_metadata` writes a `"Recipe metadata: ..."` marker; a
|
||||||
|
freshly generated `.webp` with no marker is the clean "untouched" control).
|
||||||
|
4. **autov3 three-state contract**: for L3 (autov3-only, renamed-file) fixtures the
|
||||||
|
local model's `.metadata.json` sidecar MUST have the `autov3` key **ABSENT** (the
|
||||||
|
"unchecked" state), NOT `""` — `""` is the TERMINAL "checked but unavailable" state
|
||||||
|
that L3 deliberately skips. The scanner computes + persists `autov3` from the file
|
||||||
|
header during the normal library scan (`model_scanner.py` `_process_model_file`), so
|
||||||
|
the live L3 match resolves through the local autov3/hash cache; the
|
||||||
|
computed-autov3 branch for unchecked items is covered by the unit suite.
|
||||||
|
5. **Fixture design for a rematch run** (mirrors the task-8 E2E):
|
||||||
|
- `recipe-a`: lora entry `isDeleted=True`, `hash` = 12-char autov3 computed from the
|
||||||
|
local model (`calculate_autov3`, `py/utils/file_utils.py`), whose local model file
|
||||||
|
was RENAMED after the recipe was written so `file_name` differs (proves L3 match
|
||||||
|
without filename).
|
||||||
|
- `recipe-b`: parser-convention checkpoint entry (uses `id`, no `modelVersionId`)
|
||||||
|
matching a local checkpoint via L2 — the local checkpoint's `.metadata.json` MUST
|
||||||
|
carry civitai version data with that `id` so `version_index` contains it (L2
|
||||||
|
cannot match otherwise).
|
||||||
|
- `recipe-c`: healthy recipe (no deleted entries) → must remain untouched.
|
||||||
|
|
||||||
|
The scanner computes and persists model hashes during the library scan, so the sandbox
|
||||||
|
model dirs just need the model files + `.metadata.json` sidecars. With
|
||||||
|
`--settings-path`, all derived data lands under the sandbox settings dir (`cache/`,
|
||||||
|
`backups/`, `logs/`, `stats/`, `wildcards/`), and NO `cache/` appears in the repo root.
|
||||||
|
|
||||||
|
## Fresh State Between Entry-Point Runs
|
||||||
|
|
||||||
|
Each entry point (global / per-recipe / selection-bulk) must start from the same
|
||||||
|
deleted state. Between runs (keep a pristine copy in `<sandbox>/recipes-before/`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Reset fixtures to the before-state snapshot
|
||||||
|
cp <sandbox>/recipes-before/*.recipe.json <sandbox>/recipes/
|
||||||
|
# 2. Clear the recipe/FTS caches (with --settings-path these live under the sandbox
|
||||||
|
# settings dir, NOT <repo-root>/cache)
|
||||||
|
rm -f <sandbox>/settings/cache/recipe/*.sqlite
|
||||||
|
rm -rf <sandbox>/settings/cache/fts/*
|
||||||
|
# 3. Restart the server (fresh process, fresh scan)
|
||||||
|
python .agents/skills/lora-manager-e2e/scripts/start_server.py \
|
||||||
|
--port {PORT} --settings-path <sandbox>/settings --restart --wait --timeout 30 --detach
|
||||||
|
# 4. Re-verify the server is listening + reload the browser page
|
||||||
|
```
|
||||||
|
|
||||||
|
## Cancellation Testing (KNOWN GAP)
|
||||||
|
|
||||||
|
Testing the rematch-cancel path E2E requires a run long enough to cancel mid-flight. A
|
||||||
|
tiny 3-recipe fixture set completes in **seconds** — too fast to reliably cancel. The
|
||||||
|
cancel path is currently **unit-covered only** (`rematch_all_recipes` cancellation
|
||||||
|
tests); do not block an E2E run on cancel-path verification. If you must attempt it,
|
||||||
|
you would need an artificially large/deferred fixture set to create a cancellable
|
||||||
|
window — treat this as a research task, not part of the standard E2E.
|
||||||
@@ -2,6 +2,14 @@
|
|||||||
|
|
||||||
This document provides detailed test scenarios for end-to-end validation of LoRa Manager features.
|
This document provides detailed test scenarios for end-to-end validation of LoRa Manager features.
|
||||||
|
|
||||||
|
> **Run preconditions (from SKILL.md)**: every run uses the **sandboxed** standalone
|
||||||
|
> server on a free port `{PORT}` (default candidate `8188`, only if actually free — pick
|
||||||
|
> e.g. `8199` when `8188` is occupied by a live ComfyUI). Fixtures live in the sandboxed
|
||||||
|
> `recipes_path` as `f"{id}.recipe.json"` files with matching in-JSON `id`; the real user
|
||||||
|
> config and real library are never touched (record protection proof before/after).
|
||||||
|
> Abort if a phase exceeds ~2x its budget or a tool call retries 3+ times (SKILL.md
|
||||||
|
> "Time Budgets & Abort Guidance").
|
||||||
|
|
||||||
## Table of Contents
|
## Table of Contents
|
||||||
|
|
||||||
1. [LoRA List Page](#lora-list-page)
|
1. [LoRA List Page](#lora-list-page)
|
||||||
@@ -19,7 +27,7 @@ This document provides detailed test scenarios for end-to-end validation of LoRa
|
|||||||
**Objective**: Verify the LoRA list page loads correctly and displays models.
|
**Objective**: Verify the LoRA list page loads correctly and displays models.
|
||||||
|
|
||||||
**Steps**:
|
**Steps**:
|
||||||
1. Navigate to `http://127.0.0.1:8188/loras`
|
1. Navigate to `http://127.0.0.1:{PORT}/loras`
|
||||||
2. Wait for page title "LoRAs" to appear
|
2. Wait for page title "LoRAs" to appear
|
||||||
3. Take snapshot to verify:
|
3. Take snapshot to verify:
|
||||||
- Header with "LoRAs" title is visible
|
- Header with "LoRAs" title is visible
|
||||||
@@ -134,7 +142,7 @@ evaluate_script(function="""
|
|||||||
**Objective**: Verify recipes page loads and displays recipes.
|
**Objective**: Verify recipes page loads and displays recipes.
|
||||||
|
|
||||||
**Steps**:
|
**Steps**:
|
||||||
1. Navigate to `http://127.0.0.1:8188/recipes`
|
1. Navigate to `http://127.0.0.1:{PORT}/recipes`
|
||||||
2. Wait for "Recipes" title
|
2. Wait for "Recipes" title
|
||||||
3. Take snapshot
|
3. Take snapshot
|
||||||
|
|
||||||
@@ -176,7 +184,7 @@ evaluate_script(function="""
|
|||||||
**Objective**: Verify settings page displays correctly.
|
**Objective**: Verify settings page displays correctly.
|
||||||
|
|
||||||
**Steps**:
|
**Steps**:
|
||||||
1. Navigate to `http://127.0.0.1:8188/settings`
|
1. Navigate to `http://127.0.0.1:{PORT}/settings`
|
||||||
2. Wait for "Settings" title
|
2. Wait for "Settings" title
|
||||||
3. Take snapshot
|
3. Take snapshot
|
||||||
|
|
||||||
@@ -190,7 +198,7 @@ evaluate_script(function="""
|
|||||||
1. Navigate to settings page
|
1. Navigate to settings page
|
||||||
2. Change a setting (e.g., default view mode)
|
2. Change a setting (e.g., default view mode)
|
||||||
3. Save settings
|
3. Save settings
|
||||||
4. Restart server: `python scripts/start_server.py --restart --wait`
|
4. Restart server: `python scripts/start_server.py --port {PORT} --restart --wait --timeout 30 --detach`
|
||||||
5. Refresh browser page
|
5. Refresh browser page
|
||||||
6. Navigate to settings
|
6. Navigate to settings
|
||||||
|
|
||||||
|
|||||||
@@ -8,186 +8,208 @@ This script shows how to:
|
|||||||
3. Verify functionality end-to-end
|
3. Verify functionality end-to-end
|
||||||
|
|
||||||
Note: This is a template. Actual execution requires Chrome DevTools MCP.
|
Note: This is a template. Actual execution requires Chrome DevTools MCP.
|
||||||
|
|
||||||
|
Port: pick a FREE port for the run — 8188 is commonly occupied by a live
|
||||||
|
ComfyUI (see the skill's Port Selection section). Set PORT below to e.g. 8199
|
||||||
|
when 8188 is taken. Always run against a SANDBOXED standalone server.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import time
|
|
||||||
|
# Choose the E2E port. 8188 is only the default candidate; use 8199 (or any
|
||||||
|
# free port checked with `ss -tlnp`) when 8188 is occupied by a live ComfyUI.
|
||||||
|
PORT = "8188"
|
||||||
|
|
||||||
|
|
||||||
def run_test():
|
def run_test():
|
||||||
"""Run example E2E test flow."""
|
"""Run example E2E test flow."""
|
||||||
|
|
||||||
print("=" * 60)
|
print("=" * 60)
|
||||||
print("LoRa Manager E2E Test Example")
|
print("LoRa Manager E2E Test Example")
|
||||||
print("=" * 60)
|
print("=" * 60)
|
||||||
|
|
||||||
# Step 1: Start server
|
# Step 1: Start server (detached so it survives the shell)
|
||||||
print("\n[1/5] Starting LoRa Manager standalone server...")
|
print("\n[1/5] Starting LoRa Manager standalone server...")
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
[sys.executable, "start_server.py", "--port", "8188", "--wait", "--timeout", "30"],
|
[sys.executable, "start_server.py", "--port", PORT, "--wait", "--timeout", "30", "--detach"],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True
|
text=True,
|
||||||
)
|
)
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
print(f"Failed to start server: {result.stderr}")
|
print(f"Failed to start server: {result.stderr}")
|
||||||
return 1
|
return 1
|
||||||
print("Server ready!")
|
print("Server ready!")
|
||||||
|
|
||||||
# Step 2: Open Chrome (manual step - show command)
|
# Step 2: Open Chrome (manual step - show command)
|
||||||
print("\n[2/5] Open Chrome with debug mode:")
|
print("\n[2/5] Open Chrome with debug mode:")
|
||||||
print("google-chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:8188/loras")
|
print(
|
||||||
|
f"google-chrome --remote-debugging-port=9222 "
|
||||||
|
f"--user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:{PORT}/loras"
|
||||||
|
)
|
||||||
print("(In actual test, this would be automated via MCP)")
|
print("(In actual test, this would be automated via MCP)")
|
||||||
|
|
||||||
# Step 3: Navigate and verify page load
|
# Step 3: Navigate and verify page load
|
||||||
print("\n[3/5] Page Load Verification:")
|
print("\n[3/5] Page Load Verification:")
|
||||||
print("""
|
print(
|
||||||
|
f"""
|
||||||
MCP Commands to execute:
|
MCP Commands to execute:
|
||||||
1. navigate_page(type="url", url="http://127.0.0.1:8188/loras")
|
1. navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
||||||
2. wait_for(text="LoRAs", timeout=10000)
|
2. wait_for(text="LoRAs", timeout=10000)
|
||||||
3. snapshot = take_snapshot()
|
3. snapshot = take_snapshot()
|
||||||
""")
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
# Step 4: Test search functionality
|
# Step 4: Test search functionality
|
||||||
print("\n[4/5] Search Functionality Test:")
|
print("\n[4/5] Search Functionality Test:")
|
||||||
print("""
|
print(
|
||||||
|
"""
|
||||||
MCP Commands to execute:
|
MCP Commands to execute:
|
||||||
1. fill(uid="search-input", value="test")
|
1. fill(uid="search-input", value="test")
|
||||||
2. press_key(key="Enter")
|
2. press_key(key="Enter")
|
||||||
3. wait_for(text="Results", timeout=5000)
|
3. wait_for(text="Results", timeout=5000)
|
||||||
4. result = evaluate_script(function="""
|
4. result = evaluate_script(function=`
|
||||||
() => {
|
() => {
|
||||||
const cards = document.querySelectorAll('.lora-card');
|
const cards = document.querySelectorAll('.lora-card');
|
||||||
return { count: cards.length };
|
return { count: cards.length };
|
||||||
}
|
}
|
||||||
""")
|
`)
|
||||||
""")
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
# Step 5: Verify API
|
# Step 5: Verify API
|
||||||
print("\n[5/5] API Verification:")
|
print("\n[5/5] API Verification:")
|
||||||
print("""
|
print(
|
||||||
|
"""
|
||||||
MCP Commands to execute:
|
MCP Commands to execute:
|
||||||
1. api_result = evaluate_script(function="""
|
1. api_result = evaluate_script(function=`
|
||||||
async () => {
|
async () => {
|
||||||
const response = await fetch('/loras/api/list');
|
const response = await fetch('/loras/api/list');
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
return { count: data.length, status: response.status };
|
return { count: data.length, status: response.status };
|
||||||
}
|
}
|
||||||
""")
|
`)
|
||||||
2. Verify api_result['status'] == 200
|
2. Verify api_result['status'] == 200
|
||||||
""")
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
print("\n" + "=" * 60)
|
print("\n" + "=" * 60)
|
||||||
print("Test flow completed!")
|
print("Test flow completed!")
|
||||||
print("=" * 60)
|
print("=" * 60)
|
||||||
|
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def example_restart_flow():
|
def example_restart_flow():
|
||||||
"""Example: Testing configuration change that requires restart."""
|
"""Example: Testing configuration change that requires restart."""
|
||||||
|
|
||||||
print("\n" + "=" * 60)
|
print("\n" + "=" * 60)
|
||||||
print("Example: Server Restart Flow")
|
print("Example: Server Restart Flow")
|
||||||
print("=" * 60)
|
print("=" * 60)
|
||||||
|
|
||||||
print("""
|
print(
|
||||||
|
f"""
|
||||||
Scenario: Change setting and verify after restart
|
Scenario: Change setting and verify after restart
|
||||||
|
|
||||||
Steps:
|
Steps:
|
||||||
1. Navigate to settings page
|
1. Navigate to settings page
|
||||||
- navigate_page(type="url", url="http://127.0.0.1:8188/settings")
|
- navigate_page(type="url", url="http://127.0.0.1:{PORT}/settings")
|
||||||
|
|
||||||
2. Change a setting (e.g., theme)
|
2. Change a setting (e.g., theme)
|
||||||
- fill(uid="theme-select", value="dark")
|
- fill(uid="theme-select", value="dark")
|
||||||
- click(uid="save-settings-button")
|
- click(uid="save-settings-button")
|
||||||
|
|
||||||
3. Restart server
|
3. Restart server
|
||||||
- subprocess.run([python, "start_server.py", "--restart", "--wait"])
|
- subprocess.run([python, "start_server.py", "--port", "{PORT}", "--restart", "--wait", "--detach"])
|
||||||
|
|
||||||
4. Refresh browser
|
4. Refresh browser
|
||||||
- navigate_page(type="reload", ignoreCache=True)
|
- navigate_page(type="reload", ignoreCache=True)
|
||||||
- wait_for(text="LoRAs", timeout=15000)
|
- wait_for(text="LoRAs", timeout=15000)
|
||||||
|
|
||||||
5. Verify setting persisted
|
5. Verify setting persisted
|
||||||
- navigate_page(type="url", url="http://127.0.0.1:8188/settings")
|
- navigate_page(type="url", url="http://127.0.0.1:{PORT}/settings")
|
||||||
- theme = evaluate_script(function="() => document.querySelector('#theme-select').value")
|
- theme = evaluate_script(function="() => document.querySelector('#theme-select').value")
|
||||||
- assert theme == "dark"
|
- assert theme == "dark"
|
||||||
""")
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def example_modal_interaction():
|
def example_modal_interaction():
|
||||||
"""Example: Testing modal dialog interaction."""
|
"""Example: Testing modal dialog interaction."""
|
||||||
|
|
||||||
print("\n" + "=" * 60)
|
print("\n" + "=" * 60)
|
||||||
print("Example: Modal Dialog Interaction")
|
print("Example: Modal Dialog Interaction")
|
||||||
print("=" * 60)
|
print("=" * 60)
|
||||||
|
|
||||||
print("""
|
print(
|
||||||
|
"""
|
||||||
Scenario: Add new LoRA via modal
|
Scenario: Add new LoRA via modal
|
||||||
|
|
||||||
Steps:
|
Steps:
|
||||||
1. Open modal
|
1. Open modal
|
||||||
- click(uid="add-lora-button")
|
- click(uid="add-lora-button")
|
||||||
- wait_for(text="Add LoRA", timeout=3000)
|
- wait_for(text="Add LoRA", timeout=3000)
|
||||||
|
|
||||||
2. Fill form
|
2. Fill form
|
||||||
- fill_form(elements=[
|
- fill_form(elements=[
|
||||||
{"uid": "lora-name", "value": "Test Character"},
|
{"uid": "lora-name", "value": "Test Character"},
|
||||||
{"uid": "lora-path", "value": "/models/test.safetensors"},
|
{"uid": "lora-path", "value": "/models/test.safetensors"},
|
||||||
])
|
])
|
||||||
|
|
||||||
3. Submit
|
3. Submit
|
||||||
- click(uid="modal-submit-button")
|
- click(uid="modal-submit-button")
|
||||||
|
|
||||||
4. Verify success
|
4. Verify success
|
||||||
- wait_for(text="Successfully added", timeout=5000)
|
- wait_for(text="Successfully added", timeout=5000)
|
||||||
- snapshot = take_snapshot()
|
- snapshot = take_snapshot()
|
||||||
""")
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def example_network_monitoring():
|
def example_network_monitoring():
|
||||||
"""Example: Network request monitoring."""
|
"""Example: Network request monitoring."""
|
||||||
|
|
||||||
print("\n" + "=" * 60)
|
print("\n" + "=" * 60)
|
||||||
print("Example: Network Request Monitoring")
|
print("Example: Network Request Monitoring")
|
||||||
print("=" * 60)
|
print("=" * 60)
|
||||||
|
|
||||||
print("""
|
print(
|
||||||
|
f"""
|
||||||
Scenario: Verify API calls during user interaction
|
Scenario: Verify API calls during user interaction
|
||||||
|
|
||||||
Steps:
|
Steps:
|
||||||
1. Clear network log (implicit on navigation)
|
1. Clear network log (implicit on navigation)
|
||||||
- navigate_page(type="url", url="http://127.0.0.1:8188/loras")
|
- navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
||||||
|
|
||||||
2. Perform action that triggers API call
|
2. Perform action that triggers API call
|
||||||
- fill(uid="search-input", value="character")
|
- fill(uid="search-input", value="character")
|
||||||
- press_key(key="Enter")
|
- press_key(key="Enter")
|
||||||
|
|
||||||
3. List network requests
|
3. List network requests
|
||||||
- requests = list_network_requests(resourceTypes=["xhr", "fetch"])
|
- requests = list_network_requests(resourceTypes=["xhr", "fetch"])
|
||||||
|
|
||||||
4. Find search API call
|
4. Find search API call
|
||||||
- search_requests = [r for r in requests if "/api/search" in r.get("url", "")]
|
- search_requests = [r for r in requests if "/api/search" in r.get("url", "")]
|
||||||
- assert len(search_requests) > 0, "Search API was not called"
|
- assert len(search_requests) > 0, "Search API was not called"
|
||||||
|
|
||||||
5. Get request details
|
5. Get request details
|
||||||
- if search_requests:
|
- if search_requests:
|
||||||
details = get_network_request(reqid=search_requests[0]["reqid"])
|
details = get_network_request(reqid=search_requests[0]["reqid"])
|
||||||
- Verify request method, response status, etc.
|
- Verify request method, response status, etc.
|
||||||
""")
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
print("LoRa Manager E2E Test Examples\n")
|
print("LoRa Manager E2E Test Examples\n")
|
||||||
print("This script demonstrates E2E testing patterns.\n")
|
print("This script demonstrates E2E testing patterns.\n")
|
||||||
print("Note: Actual execution requires Chrome DevTools MCP connection.\n")
|
print("Note: Actual execution requires Chrome DevTools MCP connection.\n")
|
||||||
|
|
||||||
run_test()
|
run_test()
|
||||||
example_restart_flow()
|
example_restart_flow()
|
||||||
example_modal_interaction()
|
example_modal_interaction()
|
||||||
example_network_monitoring()
|
example_network_monitoring()
|
||||||
|
|
||||||
print("\n" + "=" * 60)
|
print("\n" + "=" * 60)
|
||||||
print("All examples shown!")
|
print("All examples shown!")
|
||||||
print("=" * 60)
|
print("=" * 60)
|
||||||
|
|||||||
@@ -1,15 +1,78 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""
|
"""
|
||||||
Start or restart LoRa Manager standalone server for E2E testing.
|
Start or restart LoRa Manager standalone server for E2E testing.
|
||||||
|
|
||||||
|
Backward-compatible CLI: --port, --restart, --wait, --timeout all work as before.
|
||||||
|
New options: --detach (setsid-style fully detached launch, survives shell death).
|
||||||
|
|
||||||
|
Safety rules implemented here:
|
||||||
|
- Never kill processes the script did not start. The script tracks the PIDs it
|
||||||
|
manages in a pidfile (/tmp/lora-manager-e2e-server-{PORT}.pid).
|
||||||
|
- If the port is held by an unrelated process (e.g. a live ComfyUI) the script
|
||||||
|
reports the conflict and exits early instead of killing it.
|
||||||
|
- --restart only kills managed PIDs; if unrelated processes still hold the port
|
||||||
|
afterwards, the script reports them and aborts.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import os
|
||||||
|
import signal
|
||||||
|
import socket
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import socket
|
|
||||||
import signal
|
PIDFILE_PREFIX = "/tmp/lora-manager-e2e-server"
|
||||||
import os
|
|
||||||
|
|
||||||
|
def pidfile_path(port: int) -> str:
|
||||||
|
"""Path of the pidfile that records PIDs this script started for a port."""
|
||||||
|
return f"{PIDFILE_PREFIX}-{port}.pid"
|
||||||
|
|
||||||
|
|
||||||
|
def read_managed_pids(port: int) -> list[int]:
|
||||||
|
"""Read PIDs this script previously managed for the port (may be stale)."""
|
||||||
|
path = pidfile_path(port)
|
||||||
|
if not os.path.exists(path):
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
with open(path, "r", encoding="utf-8") as fh:
|
||||||
|
return [int(line.strip()) for line in fh if line.strip().isdigit()]
|
||||||
|
except (OSError, ValueError):
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def write_managed_pids(port: int, pids: list[int]) -> None:
|
||||||
|
"""Record PIDs this script manages for the port."""
|
||||||
|
try:
|
||||||
|
with open(pidfile_path(port), "w", encoding="utf-8") as fh:
|
||||||
|
for pid in pids:
|
||||||
|
fh.write(f"{pid}\n")
|
||||||
|
except OSError as exc:
|
||||||
|
print(f"Warning: could not write pidfile for port {port}: {exc}")
|
||||||
|
|
||||||
|
|
||||||
|
def clear_managed_pids(port: int) -> None:
|
||||||
|
"""Remove the pidfile for the port (no longer managed)."""
|
||||||
|
path = pidfile_path(port)
|
||||||
|
try:
|
||||||
|
if os.path.exists(path):
|
||||||
|
os.remove(path)
|
||||||
|
except OSError as exc:
|
||||||
|
print(f"Warning: could not remove pidfile {path}: {exc}")
|
||||||
|
|
||||||
|
|
||||||
|
def process_alive(pid: int) -> bool:
|
||||||
|
"""Return True if a process with the given pid exists."""
|
||||||
|
try:
|
||||||
|
os.kill(pid, 0)
|
||||||
|
return True
|
||||||
|
except ProcessLookupError:
|
||||||
|
return False
|
||||||
|
except PermissionError:
|
||||||
|
return True # exists but owned by someone else
|
||||||
|
|
||||||
|
|
||||||
def find_server_process(port: int) -> list[int]:
|
def find_server_process(port: int) -> list[int]:
|
||||||
@@ -19,7 +82,7 @@ def find_server_process(port: int) -> list[int]:
|
|||||||
["lsof", "-ti", f":{port}"],
|
["lsof", "-ti", f":{port}"],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
check=False
|
check=False,
|
||||||
)
|
)
|
||||||
if result.returncode == 0 and result.stdout.strip():
|
if result.returncode == 0 and result.stdout.strip():
|
||||||
return [int(pid) for pid in result.stdout.strip().split("\n") if pid]
|
return [int(pid) for pid in result.stdout.strip().split("\n") if pid]
|
||||||
@@ -30,7 +93,7 @@ def find_server_process(port: int) -> list[int]:
|
|||||||
["netstat", "-tlnp"],
|
["netstat", "-tlnp"],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
check=False
|
check=False,
|
||||||
)
|
)
|
||||||
pids = []
|
pids = []
|
||||||
for line in result.stdout.split("\n"):
|
for line in result.stdout.split("\n"):
|
||||||
@@ -49,30 +112,48 @@ def find_server_process(port: int) -> list[int]:
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
def kill_server(port: int) -> None:
|
def describe_processes(pids: list[int]) -> str:
|
||||||
"""Kill processes using the specified port."""
|
"""Human-readable description of a pid list (pid + command line)."""
|
||||||
pids = find_server_process(port)
|
descriptions = []
|
||||||
for pid in pids:
|
for pid in pids:
|
||||||
|
cmdline = ""
|
||||||
|
try:
|
||||||
|
with open(f"/proc/{pid}/cmdline", "rb") as fh:
|
||||||
|
raw = fh.read().replace(b"\x00", b" ").decode("utf-8", "replace")
|
||||||
|
cmdline = raw.strip()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
descriptions.append(f"pid {pid}{' (' + cmdline + ')' if cmdline else ''}")
|
||||||
|
return ", ".join(descriptions) if descriptions else "none"
|
||||||
|
|
||||||
|
|
||||||
|
def kill_pids(pids: list[int], what: str) -> None:
|
||||||
|
"""Send SIGTERM (then SIGKILL) to the given PIDs, only after reporting."""
|
||||||
|
for pid in pids:
|
||||||
|
print(f"Sent SIGTERM to {what} pid {pid}")
|
||||||
try:
|
try:
|
||||||
os.kill(pid, signal.SIGTERM)
|
os.kill(pid, signal.SIGTERM)
|
||||||
print(f"Sent SIGTERM to process {pid}")
|
|
||||||
except ProcessLookupError:
|
except ProcessLookupError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# Wait for processes to terminate
|
# Wait for processes to terminate
|
||||||
time.sleep(1)
|
deadline = time.time() + 5
|
||||||
|
while time.time() < deadline:
|
||||||
|
if not any(process_alive(pid) for pid in pids):
|
||||||
|
break
|
||||||
|
time.sleep(0.2)
|
||||||
|
|
||||||
# Force kill if still running
|
# Force kill if still running
|
||||||
pids = find_server_process(port)
|
|
||||||
for pid in pids:
|
for pid in pids:
|
||||||
try:
|
if process_alive(pid):
|
||||||
os.kill(pid, signal.SIGKILL)
|
try:
|
||||||
print(f"Sent SIGKILL to process {pid}")
|
os.kill(pid, signal.SIGKILL)
|
||||||
except ProcessLookupError:
|
print(f"Sent SIGKILL to {what} pid {pid}")
|
||||||
pass
|
except ProcessLookupError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def is_server_ready(port: int, timeout: float = 0.5) -> bool:
|
def is_server_ready(port: int, timeout: float = 2.0) -> bool:
|
||||||
"""Check if server is accepting connections."""
|
"""Check if server is accepting connections."""
|
||||||
try:
|
try:
|
||||||
with socket.create_connection(("127.0.0.1", port), timeout=timeout):
|
with socket.create_connection(("127.0.0.1", port), timeout=timeout):
|
||||||
@@ -84,9 +165,15 @@ def is_server_ready(port: int, timeout: float = 0.5) -> bool:
|
|||||||
def wait_for_server(port: int, timeout: int = 30) -> bool:
|
def wait_for_server(port: int, timeout: int = 30) -> bool:
|
||||||
"""Wait for server to become ready."""
|
"""Wait for server to become ready."""
|
||||||
start = time.time()
|
start = time.time()
|
||||||
|
last_report = 0.0
|
||||||
while time.time() - start < timeout:
|
while time.time() - start < timeout:
|
||||||
if is_server_ready(port):
|
if is_server_ready(port):
|
||||||
return True
|
return True
|
||||||
|
# Report progress every ~5s so a slow boot is visible, not silent.
|
||||||
|
elapsed = time.time() - start
|
||||||
|
if elapsed - last_report >= 5:
|
||||||
|
print(f" ...still waiting ({int(elapsed)}s/{timeout}s)")
|
||||||
|
last_report = elapsed
|
||||||
time.sleep(0.5)
|
time.sleep(0.5)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -99,68 +186,169 @@ def main() -> int:
|
|||||||
"--port",
|
"--port",
|
||||||
type=int,
|
type=int,
|
||||||
default=8188,
|
default=8188,
|
||||||
help="Server port (default: 8188)"
|
help="Server port (default: 8188)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--restart",
|
"--restart",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
help="Kill existing server before starting"
|
help="Kill the E2E server previously managed by this script for the port "
|
||||||
|
"(tracked via pidfile) before starting; refuse to kill unrelated processes",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--wait",
|
"--wait",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
help="Wait for server to be ready before exiting"
|
help="Wait for server to be ready before exiting",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--timeout",
|
"--timeout",
|
||||||
type=int,
|
type=int,
|
||||||
default=30,
|
default=30,
|
||||||
help="Timeout for waiting (default: 30)"
|
help="Timeout for waiting (default: 30)",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--detach",
|
||||||
|
action="store_true",
|
||||||
|
help="Launch the server fully detached (setsid-style) so it survives shell "
|
||||||
|
"death. REQUIRED for E2E: a plain background process dies with the shell",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--settings-path",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
metavar="DIR",
|
||||||
|
help="Explicit settings directory passed to standalone.py (--settings-path, "
|
||||||
|
"equivalent to LORA_MANAGER_SETTINGS_DIR). settings.json, cache/, "
|
||||||
|
"wildcards/, backups/, logs/, stats/ all live under this directory instead "
|
||||||
|
"of the project root or the user config dir. Recommended for sandboxed E2E "
|
||||||
|
"so the real instance and the repo stay untouched",
|
||||||
|
)
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
# Get project root (parent of .agents directory)
|
# Get project root (parent of .agents directory)
|
||||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||||
skill_dir = os.path.dirname(script_dir)
|
skill_dir = os.path.dirname(script_dir)
|
||||||
project_root = os.path.dirname(os.path.dirname(os.path.dirname(skill_dir)))
|
project_root = os.path.dirname(os.path.dirname(os.path.dirname(skill_dir)))
|
||||||
|
|
||||||
# Restart if requested
|
managed_pids = read_managed_pids(args.port)
|
||||||
|
|
||||||
|
# Restart if requested: kill ONLY managed PIDs.
|
||||||
if args.restart:
|
if args.restart:
|
||||||
print(f"Killing existing server on port {args.port}...")
|
alive_managed = [pid for pid in managed_pids if process_alive(pid)]
|
||||||
kill_server(args.port)
|
if alive_managed:
|
||||||
|
print(
|
||||||
|
f"Killing E2E server previously started by this script on port "
|
||||||
|
f"{args.port} ({describe_processes(alive_managed)})..."
|
||||||
|
)
|
||||||
|
kill_pids(alive_managed, "managed E2E server")
|
||||||
|
else:
|
||||||
|
print(
|
||||||
|
f"No live managed E2E server for port {args.port} "
|
||||||
|
f"(pidfile: {pidfile_path(args.port)})"
|
||||||
|
)
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
|
# Refuse to kill anything the script did not manage.
|
||||||
# Check if already running
|
remaining = find_server_process(args.port)
|
||||||
if is_server_ready(args.port):
|
if remaining:
|
||||||
print(f"Server already running on port {args.port}")
|
print(
|
||||||
return 0
|
f"ERROR: port {args.port} is still held by process(es) this script "
|
||||||
|
f"did not start: {describe_processes(remaining)}."
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"These may be unrelated (e.g. a live ComfyUI). The script will NOT "
|
||||||
|
"kill them. Pick a different --port, or stop them manually if you "
|
||||||
|
"are certain they are stale E2E servers."
|
||||||
|
)
|
||||||
|
return 2
|
||||||
|
clear_managed_pids(args.port)
|
||||||
|
|
||||||
|
# Port conflict check before starting: never blind-kill.
|
||||||
|
port_pids = find_server_process(args.port)
|
||||||
|
if port_pids:
|
||||||
|
alive_managed = [pid for pid in port_pids if pid in managed_pids]
|
||||||
|
unmanaged = [pid for pid in port_pids if pid not in managed_pids]
|
||||||
|
if alive_managed and not unmanaged:
|
||||||
|
print(
|
||||||
|
f"Server already running on port {args.port} "
|
||||||
|
f"({describe_processes(alive_managed)}, started by this script). "
|
||||||
|
f"Use --restart to recycle it."
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
print(
|
||||||
|
f"ERROR: port {args.port} is already in use by process(es): "
|
||||||
|
f"{describe_processes(port_pids)}."
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"This is likely an unrelated process (e.g. a live ComfyUI holding 8188). "
|
||||||
|
"The script will NOT kill it. Pick a free port with --port, e.g. 8199."
|
||||||
|
)
|
||||||
|
return 2
|
||||||
|
|
||||||
# Start server
|
# Start server
|
||||||
print(f"Starting LoRa Manager standalone server on port {args.port}...")
|
print(f"Starting LoRa Manager standalone server on port {args.port}...")
|
||||||
cmd = [sys.executable, "standalone.py", "--port", str(args.port)]
|
cmd = [
|
||||||
|
sys.executable,
|
||||||
# Start in background
|
"standalone.py",
|
||||||
process = subprocess.Popen(
|
"--host",
|
||||||
cmd,
|
"127.0.0.1",
|
||||||
cwd=project_root,
|
"--port",
|
||||||
stdout=subprocess.PIPE,
|
str(args.port),
|
||||||
stderr=subprocess.PIPE,
|
]
|
||||||
start_new_session=True
|
if args.settings_path:
|
||||||
)
|
settings_dir = os.path.abspath(os.path.expanduser(args.settings_path))
|
||||||
|
if os.path.exists(settings_dir) and not os.path.isdir(settings_dir):
|
||||||
print(f"Server process started with PID {process.pid}")
|
print(
|
||||||
|
f"ERROR: --settings-path '{settings_dir}' exists but is not a directory."
|
||||||
|
)
|
||||||
|
return 2
|
||||||
|
os.makedirs(settings_dir, exist_ok=True)
|
||||||
|
cmd.extend(["--settings-path", settings_dir])
|
||||||
|
print(f"Settings directory: {settings_dir}")
|
||||||
|
|
||||||
|
if args.detach:
|
||||||
|
# Fully detached launch: new session (setsid), no controlling terminal,
|
||||||
|
# stdin from /dev/null, stdout/stderr to a log file. Survives the shell.
|
||||||
|
log_dir = os.path.join(script_dir, "logs")
|
||||||
|
os.makedirs(log_dir, exist_ok=True)
|
||||||
|
log_path = os.path.join(log_dir, f"server-{args.port}.log")
|
||||||
|
with open(log_path, "ab") as log_fh:
|
||||||
|
process = subprocess.Popen(
|
||||||
|
cmd,
|
||||||
|
cwd=project_root,
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
stdout=log_fh,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
start_new_session=True,
|
||||||
|
close_fds=True,
|
||||||
|
)
|
||||||
|
print(f"Detached server process started with PID {process.pid} (setsid)")
|
||||||
|
print(f"Log: {log_path}")
|
||||||
|
else:
|
||||||
|
# Plain background process (legacy behavior): dies with the shell.
|
||||||
|
process = subprocess.Popen(
|
||||||
|
cmd,
|
||||||
|
cwd=project_root,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
start_new_session=True,
|
||||||
|
)
|
||||||
|
print(f"Server process started with PID {process.pid}")
|
||||||
|
print(
|
||||||
|
"NOTE: not detached — this process dies when the launching shell exits. "
|
||||||
|
"For E2E use --detach."
|
||||||
|
)
|
||||||
|
|
||||||
|
write_managed_pids(args.port, [process.pid])
|
||||||
|
|
||||||
# Wait for ready if requested
|
# Wait for ready if requested
|
||||||
if args.wait:
|
if args.wait:
|
||||||
print(f"Waiting for server to be ready (timeout: {args.timeout}s)...")
|
print(f"Waiting for server to be ready (timeout: {args.timeout}s)...")
|
||||||
if wait_for_server(args.port, args.timeout):
|
if wait_for_server(args.port, args.timeout):
|
||||||
print(f"Server ready at http://127.0.0.1:{args.port}/loras")
|
print(f"Server ready at http://127.0.0.1:{args.port}/loras")
|
||||||
return 0
|
return 0
|
||||||
else:
|
print(f"Timeout waiting for server on port {args.port}")
|
||||||
print(f"Timeout waiting for server")
|
return 1
|
||||||
return 1
|
|
||||||
|
|
||||||
print(f"Server starting at http://127.0.0.1:{args.port}/loras")
|
print(f"Server starting at http://127.0.0.1:{args.port}/loras")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|||||||
@@ -1,15 +1,20 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""
|
"""
|
||||||
Wait for LoRa Manager server to become ready.
|
Wait for LoRa Manager server to become ready.
|
||||||
|
|
||||||
|
Timeout is configurable via --timeout (default 30s); the script polls the port
|
||||||
|
until the server accepts connections or the timeout expires.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import socket
|
import socket
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
|
|
||||||
|
|
||||||
def is_server_ready(port: int, timeout: float = 0.5) -> bool:
|
def is_server_ready(port: int, timeout: float = 2.0) -> bool:
|
||||||
"""Check if server is accepting connections."""
|
"""Check if server is accepting connections."""
|
||||||
try:
|
try:
|
||||||
with socket.create_connection(("127.0.0.1", port), timeout=timeout):
|
with socket.create_connection(("127.0.0.1", port), timeout=timeout):
|
||||||
@@ -21,9 +26,15 @@ def is_server_ready(port: int, timeout: float = 0.5) -> bool:
|
|||||||
def wait_for_server(port: int, timeout: int = 30) -> bool:
|
def wait_for_server(port: int, timeout: int = 30) -> bool:
|
||||||
"""Wait for server to become ready."""
|
"""Wait for server to become ready."""
|
||||||
start = time.time()
|
start = time.time()
|
||||||
|
last_report = 0.0
|
||||||
while time.time() - start < timeout:
|
while time.time() - start < timeout:
|
||||||
if is_server_ready(port):
|
if is_server_ready(port):
|
||||||
return True
|
return True
|
||||||
|
# Report progress every ~5s so a slow boot is visible, not silent.
|
||||||
|
elapsed = time.time() - start
|
||||||
|
if elapsed - last_report >= 5:
|
||||||
|
print(f" ...still waiting ({int(elapsed)}s/{timeout}s)")
|
||||||
|
last_report = elapsed
|
||||||
time.sleep(0.5)
|
time.sleep(0.5)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -36,25 +47,24 @@ def main() -> int:
|
|||||||
"--port",
|
"--port",
|
||||||
type=int,
|
type=int,
|
||||||
default=8188,
|
default=8188,
|
||||||
help="Server port (default: 8188)"
|
help="Server port (default: 8188)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--timeout",
|
"--timeout",
|
||||||
type=int,
|
type=int,
|
||||||
default=30,
|
default=30,
|
||||||
help="Timeout in seconds (default: 30)"
|
help="Timeout in seconds (default: 30)",
|
||||||
)
|
)
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
print(f"Waiting for server on port {args.port} (timeout: {args.timeout}s)...")
|
print(f"Waiting for server on port {args.port} (timeout: {args.timeout}s)...")
|
||||||
|
|
||||||
if wait_for_server(args.port, args.timeout):
|
if wait_for_server(args.port, args.timeout):
|
||||||
print(f"Server ready at http://127.0.0.1:{args.port}/loras")
|
print(f"Server ready at http://127.0.0.1:{args.port}/loras")
|
||||||
return 0
|
return 0
|
||||||
else:
|
print(f"Timeout: Server not ready after {args.timeout}s")
|
||||||
print(f"Timeout: Server not ready after {args.timeout}s")
|
return 1
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -9,7 +9,10 @@ description: Inspect ComfyUI LoRA Manager runtime configuration and local diagno
|
|||||||
|
|
||||||
- Treat runtime state as local user data. Prefer read-only inspection unless the user explicitly asks for mutation.
|
- Treat runtime state as local user data. Prefer read-only inspection unless the user explicitly asks for mutation.
|
||||||
- Never print secret-like settings values. Redact keys containing `key`, `token`, `secret`, `password`, `auth`, or `credential`, including `civitai_api_key`.
|
- Never print secret-like settings values. Redact keys containing `key`, `token`, `secret`, `password`, `auth`, or `credential`, including `civitai_api_key`.
|
||||||
- Resolve paths from the runtime configuration before guessing. In this environment the settings file is normally `/home/miao/.config/ComfyUI-LoRA-Manager/settings.json`, but portable settings can override this through the repository `settings.json`.
|
- Resolve paths from the runtime configuration before guessing. Settings-directory precedence (highest first):
|
||||||
|
1. **Explicit override** — env `LORA_MANAGER_SETTINGS_DIR` or standalone `--settings-path` (also accepted by the inspect script as `--settings-path DIR`). Pins EVERYTHING (`settings.json`, `cache/`, `wildcards/`, `backups/`, `logs/`, `stats/`) under the given directory; bypasses portable mode and the user config dir. Common when inspecting a sandboxed/E2E instance.
|
||||||
|
2. **Portable** — repository `<repo-root>/settings.json` with `"use_portable_settings": true` (or `LORA_MANAGER_PORTABLE=1`): settings dir = `<repo-root>`.
|
||||||
|
3. **Default** — `~/.config/ComfyUI-LoRA-Manager` on this machine (`platformdirs.user_config_dir("ComfyUI-LoRA-Manager", appauthor=False)`).
|
||||||
- Use the active library when selecting per-library caches and paths. Read `active_library` from settings; fall back to `default` if missing.
|
- Use the active library when selecting per-library caches and paths. Read `active_library` from settings; fall back to `default` if missing.
|
||||||
- Normalize and expand `~` before comparing paths. Symlinks are common in this repo.
|
- Normalize and expand `~` before comparing paths. Symlinks are common in this repo.
|
||||||
|
|
||||||
@@ -32,9 +35,17 @@ python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_conte
|
|||||||
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py sqlite --db /path/to/cache.sqlite --limit 3
|
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py sqlite --db /path/to/cache.sqlite --limit 3
|
||||||
```
|
```
|
||||||
|
|
||||||
|
To inspect a sandboxed/E2E instance that pins its settings directory:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# --settings-path DIR (or LORA_MANAGER_SETTINGS_DIR) works with every subcommand:
|
||||||
|
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py \
|
||||||
|
--settings-path /tmp/opencode/<plan>-e2e/settings summary
|
||||||
|
```
|
||||||
|
|
||||||
## Runtime Path Rules
|
## Runtime Path Rules
|
||||||
|
|
||||||
- Settings directory: use `py/utils/settings_paths.py`. Default platform path is `platformdirs.user_config_dir("ComfyUI-LoRA-Manager", appauthor=False)`.
|
- Settings directory: resolve via `py/utils/settings_paths.py` — `get_settings_dir()` honors the `LORA_MANAGER_SETTINGS_DIR` / programmatic override first, then portable mode, then `platformdirs.user_config_dir("ComfyUI-LoRA-Manager", appauthor=False)`. The inspect script mirrors this precedence in `resolve_settings_path()`.
|
||||||
- Settings file: `<settings_dir>/settings.json`.
|
- Settings file: `<settings_dir>/settings.json`.
|
||||||
- Cache root: `<settings_dir>/cache`.
|
- Cache root: `<settings_dir>/cache`.
|
||||||
- Canonical cache files:
|
- Canonical cache files:
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from typing import Any
|
|||||||
|
|
||||||
SECRET_PATTERN = re.compile(r"(key|token|secret|password|auth|credential)", re.IGNORECASE)
|
SECRET_PATTERN = re.compile(r"(key|token|secret|password|auth|credential)", re.IGNORECASE)
|
||||||
APP_NAME = "ComfyUI-LoRA-Manager"
|
APP_NAME = "ComfyUI-LoRA-Manager"
|
||||||
|
SETTINGS_DIR_ENV = "LORA_MANAGER_SETTINGS_DIR"
|
||||||
CACHE_SQLITE = {
|
CACHE_SQLITE = {
|
||||||
"model": ("model", "{library}.sqlite"),
|
"model": ("model", "{library}.sqlite"),
|
||||||
"recipe": ("recipe", "{library}.sqlite"),
|
"recipe": ("recipe", "{library}.sqlite"),
|
||||||
@@ -30,6 +31,15 @@ CACHE_JSON = {
|
|||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = argparse.ArgumentParser(description="Inspect LoRA Manager runtime state read-only.")
|
parser = argparse.ArgumentParser(description="Inspect LoRA Manager runtime state read-only.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--settings-path",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
metavar="DIR",
|
||||||
|
help="Explicit settings directory (same as LORA_MANAGER_SETTINGS_DIR / "
|
||||||
|
"standalone --settings-path). Overrides portable mode and the default "
|
||||||
|
"user config dir.",
|
||||||
|
)
|
||||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||||
|
|
||||||
subparsers.add_parser("summary", help="Print redacted settings and resolved paths.")
|
subparsers.add_parser("summary", help="Print redacted settings and resolved paths.")
|
||||||
@@ -44,6 +54,8 @@ def main() -> int:
|
|||||||
sqlite_parser.add_argument("--limit", type=int, default=3, help="Rows to sample from each user table.")
|
sqlite_parser.add_argument("--limit", type=int, default=3, help="Rows to sample from each user table.")
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
if args.settings_path:
|
||||||
|
os.environ[SETTINGS_DIR_ENV] = args.settings_path
|
||||||
context = build_context()
|
context = build_context()
|
||||||
|
|
||||||
if args.command == "summary":
|
if args.command == "summary":
|
||||||
@@ -78,6 +90,11 @@ def build_context() -> dict[str, Any]:
|
|||||||
|
|
||||||
|
|
||||||
def resolve_settings_path() -> Path:
|
def resolve_settings_path() -> Path:
|
||||||
|
# Explicit override: LORA_MANAGER_SETTINGS_DIR env or --settings-path.
|
||||||
|
explicit = os.environ.get(SETTINGS_DIR_ENV)
|
||||||
|
if explicit:
|
||||||
|
return Path(explicit).expanduser() / "settings.json"
|
||||||
|
|
||||||
repo_root = find_repo_root()
|
repo_root = find_repo_root()
|
||||||
portable = repo_root / "settings.json"
|
portable = repo_root / "settings.json"
|
||||||
if portable.exists():
|
if portable.exists():
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ model_cache/
|
|||||||
reasonix.toml
|
reasonix.toml
|
||||||
.reasonix/
|
.reasonix/
|
||||||
.codegraph/
|
.codegraph/
|
||||||
|
.playwright-mcp/
|
||||||
|
|
||||||
# Vue widgets development cache (but keep build output)
|
# Vue widgets development cache (but keep build output)
|
||||||
vue-widgets/node_modules/
|
vue-widgets/node_modules/
|
||||||
|
|||||||
@@ -0,0 +1,202 @@
|
|||||||
|
---
|
||||||
|
slug: undo-delete-staging
|
||||||
|
status: drafting
|
||||||
|
intent: clear
|
||||||
|
review_required: false
|
||||||
|
pending-action: write .omo/plans/undo-delete-staging.md
|
||||||
|
approach: "Option B: delayed physical deletion with Undo. Backend: same-volume rename to per-root staging dir (.lm-pending-delete/) [updated 2026-08: model staging moved to a SIBLING dir inside each deleted model's own folder — see 'Symlink fix (2026-08)' under Decisions] + manifest JSON (batch_id, expires_at, staged->original map) + purge (30s TTL timer + startup sweep + opportunistic) + undo-delete endpoint + settings toggle 'skip undo'. Small files (recipes: JSON+preview) copy to global staging under settings dir instead of rename. Frontend: extend toast system with action button + 30s countdown; delete flows (single model / recipe / bulk / duplicates) consume batch_id from delete response and show Undo toast; expired undo -> 'undo expired' toast. Plus confirm-modal friction (C-friction, NO type-to-confirm): delete button delay-activation 1.5s + modal shows file size 'will free X GB' + Cancel gets initial focus. i18n keys + sync_translation_keys.py."
|
||||||
|
---
|
||||||
|
|
||||||
|
# Draft: undo-delete-staging
|
||||||
|
|
||||||
|
## Components (topology ledger)
|
||||||
|
<!-- Lock the SHAPE before depth. One row per top-level component that can succeed or fail independently. -->
|
||||||
|
<!-- id | outcome (one line) | status: active|deferred | evidence path -->
|
||||||
|
- backend staging module (stage/purge/undo + manifest + per-volume dir resolution) | new module, active | pending exploration: model_lifecycle_service.py delete_model / delete_model_artifacts
|
||||||
|
- delete endpoints return batch_id (model/recipe/bulk/duplicates) | active | pending exploration: handlers + response shapes
|
||||||
|
- undo-delete HTTP endpoint + route registration | active | pending exploration: route registrar pattern
|
||||||
|
- purge scheduling (30s timer + startup sweep + opportunistic) | active | pending exploration: app on_startup hooks
|
||||||
|
- settings toggle "skip undo window" | active | pending exploration: settings service read pattern
|
||||||
|
- frontend toast extension (action button + countdown) | active | pending exploration: showToast impl
|
||||||
|
- frontend delete flows consume batch_id + Undo toast | active | pending exploration: call sites
|
||||||
|
- confirm-modal friction (delay-activate + size display + cancel focus) | active | pending exploration: modal focus behavior
|
||||||
|
- i18n keys + sync_translation_keys.py | active | known
|
||||||
|
|
||||||
|
## Open assumptions (announced defaults)
|
||||||
|
<!-- Record any default you adopt instead of asking, so the user can veto it at the gate. -->
|
||||||
|
<!-- assumption | adopted default | rationale | reversible? -->
|
||||||
|
- Undo window TTL = 30s | 30s balances space-freeing intent vs accident recovery | yes (constant)
|
||||||
|
- Staging dir name: `.lm-pending-delete/` under each model root; recipes: `{settings_dir}/.lm-pending-delete/` | hidden, same-volume [updated 2026-08: same-volume is now guaranteed by sibling staging inside the model's own folder, not by the root location], consistent | yes
|
||||||
|
- Staging failure falls back to existing hard delete | user intent is delete; staging is best-effort; hard delete likely fails identically under same conditions | yes
|
||||||
|
- Purge on startup uses expires_at (not purge-all) so a <30s restart with live tab can still undo | robust, matches client-side timer | yes
|
||||||
|
- Settings toggle label: "Delete permanently immediately (skip undo window)" | power users freeing space | yes
|
||||||
|
- C-friction: delete button enabled after 1.5s + modal shows freed size; NO type-to-confirm (user vetoed) | user explicitly rejected type-to-confirm | n/a
|
||||||
|
- Bulk/duplicates delete: one batch id for whole action, one undo restores all | simplest consistent semantics | yes
|
||||||
|
|
||||||
|
## Findings (cited - path:lines)
|
||||||
|
|
||||||
|
### Backend
|
||||||
|
- `delete_model_artifacts` (py/services/model_lifecycle_service.py:19-48) = physical delete via os.remove; patterns: main file + `{name}.metadata.json` + PREVIEW_EXTENSIONS (py/utils/constants.py:22-37). ALSO called by ModelScanner.bulk_delete_models (py/services/model_scanner.py:2221) - single swap point covers bulk models.
|
||||||
|
- `ModelLifecycleService.delete_model` (model_lifecycle_service.py:101-154): fetches `cached_entry` (111-116) - SNAPSHOT available for cache restore; after delete: cache.raw_data removal + resort + bump_cache_version (136-143), `_hash_index.remove_by_path` (145-146), `_sync_update_for_model` (148; update-service only, no recipe JSON rewrites - recipe refs are hash-based, re-resolve on restore), `_persist_current_cache` (150-152), returns `{"success": True, "deleted_files": [...]}` (154).
|
||||||
|
- Handler `delete_model` (py/routes/handlers/model_handlers.py:478-492): POST /api/lm/{prefix}/delete; response passthrough; `_broadcast_models_changed()` (57-74) after success; 400 `{"success":false,"error"}`; 500 plain text.
|
||||||
|
- Recipe delete: handler (recipe_handlers.py:1422-1438) DELETE /api/lm/recipe/{recipe_id} -> persistence_service.delete_recipe (py/services/recipes/persistence_service.py:193-209): os.remove(recipe_json_path) + os.remove(image_path) (204-206), recipe_scanner.remove_recipe (208), returns `{"success": true, "message": ...}`. PersistenceResult dataclass (20-25).
|
||||||
|
- Bulk models: POST /api/lm/{prefix}/bulk-delete (model_route_registrar.py:39) -> handler (model_handlers.py:974-994) -> lifecycle_service.bulk_delete_models (model_lifecycle_service.py:308-318) -> scanner.bulk_delete_models (model_scanner.py:2181-2269) which calls delete_model_artifacts per file (2221) + `_batch_update_cache_for_deleted_models` (2271-2335); response `{"success","status","total_deleted","total_attempted","cache_updated","results"}` (2254-2269).
|
||||||
|
- Bulk recipes: POST /api/lm/recipes/bulk-delete (recipe_route_registrar.py:50) -> handler (recipe_handlers.py:1554-1573) -> persistence_service.bulk_delete (persistence_service.py:439-482): per-id os.remove x2 (464-466), recipe_scanner.bulk_remove (472); response `{"success","deleted","failed","total_deleted","total_failed"}` (474-482).
|
||||||
|
- Duplicates: NO dedicated delete endpoints (find-only: GET /api/lm/{prefix}/find-duplicates model_route_registrar.py:59, GET /api/lm/recipes/find-duplicates recipe_route_registrar.py:49). Duplicate deletion reuses bulk-delete endpoints.
|
||||||
|
- Startup hooks: lora_manager.py:183-187 `app.on_startup.append(lambda app: cls._initialize_services())` (ComfyUI mode, app = PromptServer.instance.app at :78); standalone.py:370-374 same (StandaloneLoraManager.add_routes). Background tasks: `asyncio.create_task(name=...)` (lora_manager.py:224-239; recipe_handlers.py:793). Singleton+asyncio.Lock pattern: model_scanner.py:40-63.
|
||||||
|
- Settings: DEFAULT_SETTINGS (py/services/settings_manager.py:57-119), `get(key, default)` (1390-1392), get_settings_manager() (2215-2228), reset_settings_manager() (2231). Typed-bool getter example: get_skip_previously_downloaded_model_versions (1253-1262). Handlers: base_model_routes.py:70, base_recipe_routes.py:54.
|
||||||
|
- Model roots: ModelScanner.get_model_roots base NotImplementedError (model_scanner.py:1073-1075); impls lora_scanner.py:31-45, checkpoint_scanner.py:428-441, embedding_scanner.py:24-36. `_find_root_for_file(file_path)` (model_scanner.py:1108-1124) returns containing root - for per-root staging dir computation [updated 2026-08: staging no longer uses the containing root; batches are siblings inside the model's own folder]. Business-path rule (AGENTS.md): use os.path.abspath, never realpath, for staging/undo routing.
|
||||||
|
- Cache restore methods: ModelCache has raw_data + resort (conftest mocks: tests/conftest.py:144-154); ModelHashIndex.add_entry(sha256, file_path, autov3) (py/services/model_hash_index.py:16); RecipeScanner.add_recipe(recipe_data) (recipe_scanner.py:2136) -> recipe_cache.add_recipe (recipe_cache.py:64). No single-file incremental model rescan - use snapshot restore instead of rescan.
|
||||||
|
- Route registrar: model_route_registrar.py:177 add_route(method, path, handler), :180 add_prefixed_route - undo endpoint can be a non-prefixed route via add_route.
|
||||||
|
- Tests: tests/services/test_model_lifecycle_service.py (inline tmp_path files, per-test stub scanners ScannerForDelete/VersionAwareScanner etc); conftest MockScanner/MockCache/MockHashIndex (tests/conftest.py:134-212); integration fixtures tests/integration/conftest.py; lifecycle hook tests tests/routes/test_lora_manager_lifecycle.py:177-178, tests/standalone/test_standalone_server.py:83-84.
|
||||||
|
|
||||||
|
### Frontend
|
||||||
|
- 5 delete call sites:
|
||||||
|
a) Single model: static/js/utils/modalUtils.js confirmDelete (27-42) -> getModelApiClient().deleteModel(path); ignores return.
|
||||||
|
b) Recipe single: static/js/components/RecipeCard.js confirmDeleteRecipe (405-449) - RAW fetch DELETE /api/lm/recipe/{id}, checks only response.ok, showToast toast.recipes.deletedSuccessfully, state.virtualScroller.removeItemByFilePath.
|
||||||
|
c) Bulk: static/js/managers/BulkManager.js confirmBulkDelete (633-672) -> getActiveApiClient() (134-142) -> bulkDeleteModels(filePaths); reads result.cancelled/success/deleted_count/error.
|
||||||
|
d) Recipe duplicates: static/js/components/DuplicatesManager.js confirmDeleteDuplicates (457-494) - RAW fetch POST /api/lm/recipes/bulk-delete, reads data.success/data.total_deleted, exitDuplicateMode().
|
||||||
|
e) Model duplicates: static/js/components/ModelDuplicatesManager.js confirmDeleteDuplicates (710-776) - RAW fetch POST /api/lm/{type}/bulk-delete, reads data.total_deleted, then resetAndReload(true) + find-duplicates re-check.
|
||||||
|
Bonus: static/js/components/shared/ModelVersionsTab.js:1136-1144 client.deleteModel (ignores return).
|
||||||
|
- API clients: BaseModelApiClient.deleteModel (static/js/api/baseModelApi.js:184-216) returns true/false, shows its own toasts, does removeItemByFilePath inside; bulkDeleteModels (1591-1642) returns {success, deleted_count, failed_count, errors} or {success:false, cancelled:true}; RecipeSidebarApiClient.bulkDeleteModels (recipeApi.js:623-664) returns {success, deleted_count: total_deleted, ...}. Endpoint map apiConfig.js:56,64.
|
||||||
|
- Toast: showToast(key, params={}, type='info', fallback=null) (static/js/utils/uiHelpers.js:136-193) - textContent only, NO action/button support; durations 2000/5000ms; CSS static/css/components/toast.css (.toast flex gap:12px - button can be added). Closest action pattern: bannerService.registerBanner actions array + onRegister (static/js/managers/BannerService.js; used uiHelpers.js:18-57).
|
||||||
|
- i18n: locales/en.json delete keys (1303-1314 bulkDelete, 1945-1948 recipes, 1987-1991 models, 2124-2130 duplicates, 2166-2170 toast.api); t()/interpolate (static/js/i18n/index.js:193-248); translate wrapper (utils/i18nHelpers.js:13-23); sync script scripts/sync_translation_keys.py (en reference, [TODO: Translate] placeholders).
|
||||||
|
- Refresh after undo: recipes -> window.recipeManager.loadRecipes(true) (recipes.js:359; used by FilterManager.js:752 etc) or refreshRecipes (recipeApi.js:308); models -> resetAndReload(true) from modelApiFactory (used by ModelDuplicatesManager.js:740).
|
||||||
|
- Size for modal: card.dataset.file_size (ModelCard.js:467), formatFileSize (ModelModal.js:615).
|
||||||
|
- Tests: tests/frontend/utils/uiHelpers.dom.test.js (toast), api/recipeApi.bulk.test.js, components/duplicatesManager.test.js, components/modelDuplicatesManager.test.js, pages/*Page.test.js, i18n tests tests/i18n/test_i18n.py.
|
||||||
|
|
||||||
|
## Decisions (with rationale)
|
||||||
|
|
||||||
|
1. Same-volume rename staging for model files (atomic, no copy cost for multi-GB files); cross-volume rename forbidden. [CORRECTED 2026-08: "same-volume because under the containing root" was only true for plain directories — nested symlinked subdirs could cross volumes. Superseded by sibling staging: `.lm-pending-delete/<batch_id>/` inside the deleted model's own folder makes stage/undo same-device by construction; see "Symlink fix (2026-08)" below.]
|
||||||
|
2. Copy-to-global-staging for recipes (small files; avoids recipe JSON vs preview image cross-volume problem).
|
||||||
|
3. Manifest JSON files are the only state - no DB changes. Manifest includes model cached_entry snapshot for exact cache restore (no rescan needed).
|
||||||
|
4. Undo endpoint returns restored paths; expired batch -> 404-style error -> frontend 'undo expired' toast.
|
||||||
|
5. Skip-undo setting honored server-side (no batch_id in response -> no undo toast client-side).
|
||||||
|
6. Staging failure falls back to existing hard delete (best-effort undo, never blocks delete).
|
||||||
|
7. Undo window TTL = 30s constant (PENDING_DELETE_TTL_SECONDS); startup sweep uses expires_at (survives restart; browser-tab timer survives).
|
||||||
|
8. Purge triple-trigger: per-batch asyncio timer task + on_startup sweep + opportunistic purge at each stage/undo.
|
||||||
|
9. Frontend: new showActionToast (keep showToast signature untouched; extract shared createToastElement/appendToast internals); undo click -> shared handleUndoDelete(batchId, refreshFn); full list refresh after undo (recipes: window.recipeManager.loadRecipes(true); models: resetAndReload(true)).
|
||||||
|
10. C-friction wave (NO type-to-confirm - user vetoed): delete buttons delay-activate 1.5s after modal open, initial focus on Cancel, model delete modal gains "permanently deleted from disk" warning + file size display (card.dataset.file_size + formatFileSize).
|
||||||
|
11. Model cache restore on undo: append snapshot to cache.raw_data (dedupe by file_path) + resort + bump_cache_version + _persist_current_cache + _hash_index.add_entry + _broadcast_models_changed. Recipe restore: copy back files + recipe_scanner.add_recipe(recipe_data loaded from restored JSON).
|
||||||
|
|
||||||
|
### Symlink fix (2026-08)
|
||||||
|
|
||||||
|
Post-execution addendum (plan `.omo/plans/undo-delete-symlink-fix.md`, commits 5fd4946b / 0c00ee22):
|
||||||
|
|
||||||
|
12. Model staging moved from `<model_root>/.lm-pending-delete/<batch_id>/` to `<model_dir>/.lm-pending-delete/<batch_id>/` (sibling of the model artifacts, inside the deleted model's own folder). Stage/undo renames are same-device BY CONSTRUCTION — EXDEV is impossible even when the business path traverses nested symlinks to other volumes (the decision-1 "containing root" guarantee covered only plain directories). EXDEV remains possible only for cross-volume merges, which keep the batch_ids-array fallback. Accepted edge: deleting the model's whole FOLDER during the 30s window destroys that batch (undo returns 404). Batch discovery uses an in-memory registry (`_known_batch_dirs`) with a startup reconciliation scan (`purge_expired(scan_roots=True)`) covering restarts and crash leftovers. Recipe batches unchanged (copy-based settings-dir staging with the `_restore_file` EXDEV fallback).
|
||||||
|
|
||||||
|
## Scope IN
|
||||||
|
|
||||||
|
- Model single delete (model_handlers delete_model / model_lifecycle_service)
|
||||||
|
- Recipe delete (recipe_handlers delete_recipe / persistence_service)
|
||||||
|
- Bulk delete (models scanner + recipes persistence) + duplicates (reuse bulk endpoints)
|
||||||
|
- Undo endpoint POST /api/lm/undo-delete (models + recipes, one batch space)
|
||||||
|
- Purge: timer + startup sweep + opportunistic
|
||||||
|
- Settings toggle delete_undo_enabled + settings page checkbox
|
||||||
|
- Frontend: showActionToast + all 5 delete flows + shared undo handler
|
||||||
|
- C-friction modal changes (delay-activate + cancel focus + warning copy + size display)
|
||||||
|
- i18n keys + sync_translation_keys.py
|
||||||
|
- Backend + frontend tests
|
||||||
|
|
||||||
|
## Scope OUT (Must NOT have)
|
||||||
|
|
||||||
|
- NO type-to-confirm / hold-to-confirm friction (user vetoed)
|
||||||
|
- NO OS trash integration (send2trash) in this iteration
|
||||||
|
- NO persistent recycle-bin UI (no trash browsing page)
|
||||||
|
- NO changes to exclude/unexclude flow
|
||||||
|
- NO DB migrations
|
||||||
|
- NO new dependencies (no send2trash)
|
||||||
|
- NO changes to download flows
|
||||||
|
- NO recipe-JSON rewriting on model undo (hash-based refs re-resolve themselves)
|
||||||
|
|
||||||
|
## Open questions
|
||||||
|
|
||||||
|
None - all implementation details resolved by exploration. Design decisions settled in conversation (B+C, no type-to-confirm).
|
||||||
|
|
||||||
|
## Approval gate
|
||||||
|
status: approved
|
||||||
|
<!-- Approach approved -> rerun scaffold without --draft-only, run Metis gap analysis, APPEND todo batches, fill TL;DR last, run structural self-check, then Phase 4 handoff. -->
|
||||||
|
|
||||||
|
## Review round state (ulw-plan-review-round-state-contract)
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"transition": "replace",
|
||||||
|
"phase": "review_round_initialized",
|
||||||
|
"applies_when": ["retry_after_plan_change"],
|
||||||
|
"atomic": true,
|
||||||
|
"review_required": true,
|
||||||
|
"plan_path": ".omo/plans/undo-delete-staging.md",
|
||||||
|
"plan_sha256": "8cf7c9be38a76d8ef1fb832aba043d6d7e82b60465b1bf28c6eafa7045117adc",
|
||||||
|
"review_round_id": "rr-undo-del-20260811-006",
|
||||||
|
"round_status": "active",
|
||||||
|
"pending-action": "review .omo/plans/undo-delete-staging.md",
|
||||||
|
"review": {
|
||||||
|
"momus": { "status": "pending", "workspace_root": "/mnt/data/reinstall-backup-2026-04-12/data/workspace/ComfyUI/custom_nodes/ComfyUI-Lora-Manager", "runtime_home": null, "target": ".omo/plans/undo-delete-staging.md", "round_id": "rr-undo-del-20260811-006", "plan_sha256": "8cf7c9be38a76d8ef1fb832aba043d6d7e82b60465b1bf28c6eafa7045117adc", "launch_id": null, "session": null, "result": null },
|
||||||
|
"independent": { "status": "pending", "workspace_root": "/mnt/data/reinstall-backup-2026-04-12/data/workspace/ComfyUI/custom_nodes/ComfyUI-Lora-Manager", "runtime_home": null, "target": ".omo/plans/undo-delete-staging.md", "round_id": "rr-undo-del-20260811-006", "plan_sha256": "8cf7c9be38a76d8ef1fb832aba043d6d7e82b60465b1bf28c6eafa7045117adc", "launch_id": null, "session": null, "result": null }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Review results + fix/retry ledger
|
||||||
|
|
||||||
|
### Round 1 (rr-undo-del-20260811-001, plan sha256 6c52bf99...)
|
||||||
|
- momus: APPROVE (non-blocking notes: todo1+7 duplicate DEFAULT_SETTINGS key -> fixed todo 7 to verify-only; "batch_ids" plural in todos 8/9 acceptance -> fixed; purge OSError note -> folded into todo 1 purge semantics)
|
||||||
|
- independent (oracle): CHANGES_REQUESTED
|
||||||
|
- BLOCKING S1: scanner walks would index .lm-pending-delete staged files as ghost entries -> fixed: todo 1 now mandates scanner walk exclusion at model_scanner.py:706/:867/:1404/_process_model_file + acceptance (o) scanner-visibility test
|
||||||
|
- BLOCKING S2: manifest lacks model_type, undo could restore into wrong cache/hash index -> fixed: manifest now carries model_type + todo 5 resolves per-type scanner via registrar pattern + acceptance (b) checkpoint-batch test
|
||||||
|
- S3 merged-batch expires_at re-anchor -> fixed: merge_batches re-anchors now+TTL in todo 1 + todo 3/4 assertions
|
||||||
|
- S4 manifest-less dir policy -> fixed: quarantine to <batch_id>.orphaned, never delete (todo 1 + acceptance g)
|
||||||
|
- S5 partial-undo retry semantics -> fixed: per-entry restored flag write-through + retry test (acceptance e)
|
||||||
|
- S6 purge locked-file failure semantics -> fixed: skip file, keep batch, never rmtree past errors (todo 1 + acceptance i)
|
||||||
|
- T8 undo-after-restart test -> fixed: todo 5 acceptance (f)
|
||||||
|
- T9 recipe undo -> re-delete test -> fixed: todo 5 acceptance (h)
|
||||||
|
- T7 rescan-stale-entry test -> fixed: todo 5 acceptance (g)
|
||||||
|
- Route registration pinned to shared routes class per mode (NOT per-model-type registrar which registers 3x) -> fixed: todo 5 now creates py/routes/pending_delete_routes.py registered once in lora_manager.py:170-172 + standalone.py:356-358 + duplicate-route test (e)
|
||||||
|
- Version-index staleness on single-delete undo -> fixed: todo 5 follows bulk cache-update pattern incl. rebuild_version_index (model_scanner.py:2324)
|
||||||
|
- Cancelled-bulk batch_id frontend handling -> fixed: todo 9 shows action toast on cancelled+staged-subset
|
||||||
|
- Single-instance assumption -> added to Scope OUT
|
||||||
|
- Occupied-refusal loss UX -> accepted-intent documented in success criteria + modal copy
|
||||||
|
|
||||||
|
### Round 2 (rr-undo-del-20260811-002, plan sha256 f3d52235...)
|
||||||
|
- momus: APPROVE (all 12 round-1 fixes verified present; zero dead references; non-blocking nits only)
|
||||||
|
- independent (oracle): CHANGES_REQUESTED
|
||||||
|
- BLOCK-1: merge_batches file-movement semantics unspecified (silent data-loss vector) -> fixed: todo 1 now specifies move-into-winner-dir + entry re-point + loser-dirs-removed-only-when-empty + abort-on-move-failure (all batches intact) + merge inside service lock + acceptance (k) file-survival assertions + acceptance (l) merge-failure abort test
|
||||||
|
- BLOCK-2: same-file parallel edits within waves (todo 5 vs 6 on lora_manager.py; todo 8 vs 9 on baseModelApi.js) -> fixed: waves/matrix now serialize 5->6 and 8->9 with explicit reasons; matrix updated
|
||||||
|
- Recommended: checkpoint_scanner.py:331 exclusion -> fixed (todo 1 + acceptance p); S5 pre-check skips restored:true entries -> fixed (todo 1); _tags_count restore on undo -> fixed (todo 5 + acceptance j); undo-blind flows documented (ModelVersionsTab + misc_handlers:2456) -> fixed (todo 8 note + Scope OUT); merge-failure no-merge fallback contract (batch_ids array) -> fixed (todos 3/4/9)
|
||||||
|
|
||||||
|
### Round 3 (rr-undo-del-20260811-003, plan sha256 8f2dfd46...)
|
||||||
|
- momus: APPROVE (all round-2 fixes verified present + spot-checked refs; no new contradictions)
|
||||||
|
- independent (oracle): CHANGES_REQUESTED
|
||||||
|
- BLOCKING A: merged batches never timer-purged after re-anchor (winner's original timer no-ops at old expiry; no fresh timer for re-anchored expiry; idle server -> merged batch lingers, violating "30s purge" success criterion; affects EVERY bulk delete) -> fixed: todo 1 merge_batches now ARMS A FRESH PURGE TIMER for the winner with re-anchored expiry + acceptance (q) fresh-timer test + purge_expired must enumerate ALL scanner types' roots (explicit in todo 1)
|
||||||
|
- BLOCKING B: dependency matrix contradicted same-file policy for todos 8/9<->11 (5 shared files) and 12<->11 -> fixed: todo 11 now "Blocked by: 8, 9 (same files...)"; todo 12 blocked by 11 (sync after 11); wave text updated (11, then 12 AFTER 11); "Can parallelize with" columns corrected
|
||||||
|
- BLOCKING C: frontend batch_ids sequential-undo fallback has NO test + merge->undo loser-restore + merge->purge assertions missing -> fixed: todo 9 acceptance now tests the batch_ids fallback path; todo 1 acceptance now has (k2)/(k3)
|
||||||
|
- Notes folded: sub-second toast-tail expiry race accepted; EXDEV fallback = NORMAL path for cross-volume bulks [annotated 2026-08: after the sibling-staging fix, EXDEV can only arise during cross-volume MERGES, never during single stage/undo renames]
|
||||||
|
|
||||||
|
### Round 4 (rr-undo-del-20260811-004, plan sha256 179e7ff7...)
|
||||||
|
- momus: APPROVE (round-3 fixes verified; one non-blocking nit: todo 11 inline "Blocked by: —" stale -> fixed to "8, 9")
|
||||||
|
- independent (oracle): CHANGES_REQUESTED
|
||||||
|
- BLOCKING GAP-1 (NEW, introduced by round-3 fix): todo 8 handleUndoDelete always-refresh/always-toast contract contradicted todo 9's sequential loop "exactly ONE final refresh" -> fixed: handleUndoDelete(batchId, refreshFn, {showToast, refresh}) suppression options; todo 9 loop uses suppressed calls + one final refresh/toast; acceptance extended (loop failure mid-way -> stop + error toast + no final refresh; 404 body discrimination expired vs occupied)
|
||||||
|
- BLOCKING GAP-2: no cross-type purge enumeration test -> fixed: todo 1 acceptance (r) purges expired batches across lora root + checkpoint root + recipe staging dir in one call
|
||||||
|
- Non-blocking folded: GAP-3 404-copy discrimination -> fixed in todo 8 (d); GAP-4 merge partial-failure rollback direction (move back + restore manifests, extended (l) asserts sequential constituent undo still restores everything) -> fixed in todo 1; GAP-5 post-restart timer-loss residual gap documented -> fixed in todo 6; GAP-6 usage_stats.py:424 walk added to exclusion mandate + todo 5 acceptance (k) embeddings undo test
|
||||||
|
|
||||||
|
### Round 5 (rr-undo-del-20260811-005, plan sha256 dfaa39ea...)
|
||||||
|
- momus: APPROVE (all round-4 fixes verified; no new contradictions)
|
||||||
|
- independent (oracle): CHANGES_REQUESTED
|
||||||
|
- BLOCK-1: lock-ordering deadlock ambiguity (asyncio.Lock not re-entrant: opportunistic purge_expired called while stage/undo hold the lock would deadlock on first use) -> fixed: todo 1 now has explicit LOCK HIERARCHY (lock acquired ONLY by stage/merge/undo/purge_batch; purge_expired is lock-free and must be called BEFORE lock acquisition); todo 6 (c) updated with the same rule + acceptance (u) lock-no-deadlock test
|
||||||
|
- BLOCK-2: purge edge semantics unspecified -> fixed: purge_batch treats missing staged files (partially-restored batches) as already-purged (FileNotFoundError silent no-op); sweep skips `.orphaned`-suffixed dirs (quarantine is terminal); acceptance (s) partially-restored purge + (t) quarantine-terminal tests
|
||||||
|
- Non-blocking folded: todo 2/3 test-file collision -> todo 3's bulk tests moved to tests/services/test_model_scanner.py; todo 9 (d) DuplicatesManager refreshFn stated explicitly (recipes loadRecipes / models resetAndReload); modal-copy + bulk-count trade-offs acknowledged in success criteria; acceptance (r) extended with embeddings root
|
||||||
|
|
||||||
|
### Round 6 (rr-undo-del-20260811-006, plan sha256 8cf7c9be...)
|
||||||
|
- momus: APPROVE (all round-5 fixes verified; no new contradictions; references verified)
|
||||||
|
- independent (oracle): APPROVE — no blocking issues; all round-5 items fixed with working, tested solutions; no new race/data-loss/consistency defects
|
||||||
|
- Deferred optional improvements (non-blocking, recorded for executor awareness; plan file left untouched to preserve the approved digest):
|
||||||
|
1. Tag-count asymmetry: single delete_model never decrements _tags_count (lifecycle 101-154), bulk does (scanner 2297-2303); undo re-increment is exact for bulk, over-counts for single until rescan (cosmetic, self-healing). Optional fix riding in todo 2: decrement tags in the single-delete path to mirror bulk.
|
||||||
|
2. Todo 5 factual nit: ModelCache.resort() already rebuilds the version index — explicit rebuild in undo is belt-and-braces, no action needed.
|
||||||
|
3. Todo 8 premise nit: ModelVersionsTab call ignores deleteModel's return entirely — nothing breaks, no adaptation needed.
|
||||||
|
4. Todo 3's pytest command includes test_model_lifecycle_service.py which todo 2 edits in the same wave — run that file's tests after todo 2 lands.
|
||||||
|
5. merge_batches with a missing/quarantined constituent id: any sane fallback (abort -> batch_ids, or skip missing) acceptable — files stay staged either way.
|
||||||
|
|
||||||
|
## Review lifecycle
|
||||||
|
- rounds: 6 (rr-undo-del-20260811-001..006); final round both lanes APPROVE
|
||||||
|
- final live-plan validation: sha256 = 8cf7c9be38a76d8ef1fb832aba043d6d7e82b60465b1bf28c6eafa7045117adc — MATCHES approved round-6 digest
|
||||||
|
- status: APPROVED — ready for execution handoff ($start-work undo-delete-staging)
|
||||||
File diff suppressed because one or more lines are too long
@@ -2,6 +2,10 @@
|
|||||||
|
|
||||||
This file provides guidance for agentic coding assistants working in this repository.
|
This file provides guidance for agentic coding assistants working in this repository.
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
ComfyUI LoRA Manager is a comprehensive LoRA management system for ComfyUI that combines a Python backend with browser-based widgets. It provides model organization, downloading from CivitAI/CivArchive, recipe management, and one-click workflow integration.
|
||||||
|
|
||||||
## Development Commands
|
## Development Commands
|
||||||
|
|
||||||
### Backend Development
|
### Backend Development
|
||||||
@@ -28,16 +32,21 @@ COVERAGE_FILE=coverage/backend/.coverage pytest \
|
|||||||
--cov=py --cov=standalone \
|
--cov=py --cov=standalone \
|
||||||
--cov-report=term-missing \
|
--cov-report=term-missing \
|
||||||
--cov-report=html:coverage/backend/html \
|
--cov-report=html:coverage/backend/html \
|
||||||
--cov-report=xml:coverage/backend/coverage.xml
|
--cov-report=xml:coverage/backend/coverage.xml \
|
||||||
|
--cov-report=json:coverage/backend/coverage.json
|
||||||
```
|
```
|
||||||
|
|
||||||
### Frontend Development (Standalone Web UI)
|
### Frontend Development (LoRA Manager Web UI)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
# Install dependencies (root and Vue widgets)
|
||||||
npm install
|
npm install
|
||||||
|
cd vue-widgets && npm install && cd ..
|
||||||
|
|
||||||
npm test # Run all tests (JS + Vue)
|
npm test # Run all tests (JS + Vue)
|
||||||
npm run test:js # Run JS tests only
|
npm run test:js # Run JS tests only
|
||||||
npm run test:watch # Watch mode
|
npm run test:vue # Run Vue widget tests only
|
||||||
|
npm run test:watch # Watch mode (JS tests only)
|
||||||
npm run test:coverage # Generate coverage report
|
npm run test:coverage # Generate coverage report
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -54,88 +63,169 @@ npm run test:watch # Watch mode
|
|||||||
npm run test:coverage # Generate coverage report
|
npm run test:coverage # Generate coverage report
|
||||||
```
|
```
|
||||||
|
|
||||||
## Python Code Style
|
### Localization
|
||||||
|
|
||||||
### Imports & Formatting
|
```bash
|
||||||
|
# Sync translation keys after UI string updates
|
||||||
|
python scripts/sync_translation_keys.py
|
||||||
|
```
|
||||||
|
|
||||||
|
Locale files are in `locales/` (en, zh-CN, zh-TW, ja, ko, fr, de, es, ru, he).
|
||||||
|
|
||||||
|
After adding keys to `en.json` and syncing, **stop**: the `[TODO: Translate]` placeholders in
|
||||||
|
the other locales are the expected end state during feature development. Do NOT translate
|
||||||
|
proactively — translate only when the feature owner explicitly asks (see
|
||||||
|
`docs/i18n-translation-guidelines.md` §7).
|
||||||
|
|
||||||
|
**Before translating anything, read `docs/i18n-translation-guidelines.md`** — it defines the
|
||||||
|
term conventions (e.g. "Recipe" stays untranslated in French, 配方 in Chinese; model-type and
|
||||||
|
brand names are never translated), per-locale preferred renderings, placeholder rules, and
|
||||||
|
the known confusion hot-spots.
|
||||||
|
|
||||||
|
## Code Style
|
||||||
|
|
||||||
|
### Python
|
||||||
|
|
||||||
|
#### Imports & Formatting
|
||||||
|
|
||||||
- Use `from __future__ import annotations` for forward references
|
- Use `from __future__ import annotations` for forward references
|
||||||
- Group imports: standard library, third-party, local (blank line separated)
|
- Group imports: standard library, third-party, local (blank line separated)
|
||||||
|
- Use `TYPE_CHECKING` guard for type-checking-only imports
|
||||||
- Absolute imports within `py/`: `from ..services import X`
|
- Absolute imports within `py/`: `from ..services import X`
|
||||||
- PEP 8 with 4-space indentation, type hints required
|
- PEP 8 with 4-space indentation, type hints required
|
||||||
|
|
||||||
### Naming Conventions
|
#### Naming Conventions
|
||||||
|
|
||||||
- Files: `snake_case.py`, Classes: `PascalCase`, Functions/vars: `snake_case`
|
- Files: `snake_case.py`, Classes: `PascalCase`, Functions/vars: `snake_case`
|
||||||
- Constants: `UPPER_SNAKE_CASE`, Private: `_protected`, `__mangled`
|
- Constants: `UPPER_SNAKE_CASE`, Private: `_protected`, `__mangled`
|
||||||
|
|
||||||
### Error Handling & Async
|
#### Error Handling & Async
|
||||||
|
|
||||||
- Use `logging.getLogger(__name__)`, define custom exceptions in `py/services/errors.py`
|
- Use `logging.getLogger(__name__)`, define custom exceptions in `py/services/errors.py`
|
||||||
- `async def` for I/O, `@pytest.mark.asyncio` for async tests
|
- `async def` for I/O, `@pytest.mark.asyncio` for async tests
|
||||||
- Singleton with `asyncio.Lock`: see `ModelScanner.get_instance()`
|
- Singleton with `asyncio.Lock`: see `ModelScanner.get_instance()`
|
||||||
- Return `aiohttp.web.json_response` or `web.Response`
|
- Return `aiohttp.web.json_response` or `web.Response`
|
||||||
|
|
||||||
### Testing
|
### JavaScript/TypeScript
|
||||||
|
|
||||||
- `pytest` with `--import-mode=importlib`
|
#### Imports & Modules
|
||||||
- Fixtures in `tests/conftest.py`, use `tmp_path_factory` for isolation
|
|
||||||
- Mark tests needing real paths: `@pytest.mark.no_settings_dir_isolation`
|
|
||||||
- Mock ComfyUI dependencies via conftest patterns
|
|
||||||
|
|
||||||
## JavaScript/TypeScript Code Style
|
|
||||||
|
|
||||||
### Imports & Modules
|
|
||||||
|
|
||||||
- ES modules: `import { app } from "../../scripts/app.js"` for ComfyUI
|
- ES modules: `import { app } from "../../scripts/app.js"` for ComfyUI
|
||||||
- Vue: `import { ref, computed } from 'vue'`, type imports: `import type { Foo }`
|
- Vue: `import { ref, computed } from 'vue'`, type imports: `import type { Foo }`
|
||||||
- Export named functions: `export function foo() {}`
|
- Export named functions: `export function foo() {}`
|
||||||
|
|
||||||
### Naming & Formatting
|
#### Naming & Formatting
|
||||||
|
|
||||||
- camelCase for functions/vars/props, PascalCase for classes
|
- camelCase for functions/vars/props, PascalCase for classes
|
||||||
- Constants: `UPPER_SNAKE_CASE`, Files: `snake_case.js` or `kebab-case.js`
|
- Constants: `UPPER_SNAKE_CASE`, Files: `snake_case.js` or `kebab-case.js`
|
||||||
- 2-space indentation preferred (follow existing file conventions)
|
- 2-space indentation preferred (follow existing file conventions)
|
||||||
- Vue Single File Components: `<script setup lang="ts">` preferred
|
- Vue Single File Components: `<script setup lang="ts">` preferred
|
||||||
|
|
||||||
### Widget Development
|
#### Widget Development
|
||||||
|
|
||||||
|
- Prefer vanilla JS for `web/comfyui/` widgets; avoid framework dependencies (except the Vue widgets in `vue-widgets/`)
|
||||||
- ComfyUI: `app.registerExtension()`, `node.addDOMWidget(name, type, element, options)`
|
- ComfyUI: `app.registerExtension()`, `node.addDOMWidget(name, type, element, options)`
|
||||||
- Event handlers via `addEventListener` or widget callbacks
|
- Event handlers via `addEventListener` or widget callbacks
|
||||||
- Shared utilities: `web/comfyui/utils.js`
|
- Shared utilities: `web/comfyui/utils.js`
|
||||||
- Dual-mode rendering patterns (canvas vs Vue): see `docs/comfyui-dual-mode-widgets.md`
|
- Dual-mode rendering patterns (canvas vs Vue): see `docs/comfyui-dual-mode-widgets.md`
|
||||||
|
|
||||||
### Vue Composables Pattern
|
#### Vue Composables Pattern
|
||||||
|
|
||||||
- Use composition API: `useXxxState(widget)`, return reactive refs and methods
|
- Use composition API: `useXxxState(widget)`, return reactive refs and methods
|
||||||
- Guard restoration loops with flag: `let isRestoring = false`
|
- Guard restoration loops with flag: `let isRestoring = false`
|
||||||
- Build config from state: `const buildConfig = (): Config => { ... }`
|
- Build config from state: `const buildConfig = (): Config => { ... }`
|
||||||
|
|
||||||
## Architecture Patterns
|
## Architecture
|
||||||
|
|
||||||
|
### Dual Mode Operation
|
||||||
|
|
||||||
|
The system runs in two modes:
|
||||||
|
- **ComfyUI plugin mode**: Integrates with ComfyUI's PromptServer, uses `folder_paths` for model discovery
|
||||||
|
- **Standalone mode**: `standalone.py` mocks ComfyUI dependencies, reads paths from `settings.json`
|
||||||
|
- Detection: `os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"`
|
||||||
|
|
||||||
|
### Backend Entry Points
|
||||||
|
|
||||||
|
- `__init__.py` — ComfyUI plugin entry: registers nodes via `NODE_CLASS_MAPPINGS`, sets `WEB_DIRECTORY`, calls `LoraManager.add_routes()`
|
||||||
|
- `standalone.py` — Standalone server: mocks `folder_paths` and node modules, starts aiohttp server
|
||||||
|
- `py/lora_manager.py` — Main `LoraManager` class that registers all HTTP routes
|
||||||
|
|
||||||
### Service Layer
|
### Service Layer
|
||||||
|
|
||||||
- `ServiceRegistry` singleton for DI, services use `get_instance()` classmethod
|
- `ServiceRegistry` singleton for DI, services use `get_instance()` classmethod
|
||||||
|
- `BaseModelService` abstract base → `LoraService`, `CheckpointService`, `EmbeddingService`
|
||||||
|
- `ModelScanner` base → `LoraScanner`, `CheckpointScanner`, `EmbeddingScanner` for file discovery with hash-based deduplication
|
||||||
|
- `PersistentModelCache` (SQLite) for metadata persistence
|
||||||
|
- `MetadataSyncService` — background sync from CivitAI/CivArchive APIs
|
||||||
|
- `SettingsManager` — settings with schema migration support
|
||||||
|
- `WebSocketManager` — real-time progress broadcasting
|
||||||
|
- `ModelServiceFactory` — creates the right service for each model type
|
||||||
|
- Use cases in `py/services/use_cases/` orchestrate complex business logic (auto-organize, bulk refresh, downloads)
|
||||||
- Separate scanners (discovery) from services (business logic)
|
- Separate scanners (discovery) from services (business logic)
|
||||||
- Handlers in `py/routes/handlers/` are pure functions with deps as params
|
- Handlers in `py/routes/handlers/` are pure functions with deps as params
|
||||||
|
|
||||||
### Model Types & Routes
|
### Model Types & Routes
|
||||||
|
|
||||||
- `BaseModelService` base for LoRA, Checkpoint, Embedding
|
- API endpoints follow `/loras/*`, `/checkpoints/*`, `/embeddings/*` patterns
|
||||||
- `ModelScanner` for file discovery, hash deduplication
|
- Route registrars organize endpoints by domain: `ModelRouteRegistrar`, `RecipeRouteRegistrar`, etc.
|
||||||
- `PersistentModelCache` (SQLite) for persistence
|
- Request handlers in `py/routes/handlers/` implement route logic
|
||||||
- Route registrars: `ModelRouteRegistrar`, endpoints: `/loras/*`, `/checkpoints/*`, `/embeddings/*`
|
- All routes use aiohttp, return `web.json_response` or `web.Response`
|
||||||
- WebSocket via `WebSocketManager` for real-time updates
|
|
||||||
|
|
||||||
### Recipe System
|
### Recipe System
|
||||||
|
|
||||||
- Base: `py/recipes/base.py`, Enrichment: `RecipeEnrichmentService`
|
- Base: `py/recipes/base.py`, Enrichment: `RecipeEnrichmentService` in `py/recipes/enrichment.py`
|
||||||
- Parsers: `py/recipes/parsers/`
|
- Parsers: `py/recipes/parsers/` for PNG metadata, JSON, and workflow formats
|
||||||
|
|
||||||
|
### Custom Nodes
|
||||||
|
|
||||||
|
- Location: `py/nodes/`, all nodes registered in `__init__.py`
|
||||||
|
- Each node class has a `NAME` class attribute used as key in `NODE_CLASS_MAPPINGS`
|
||||||
|
- Standard ComfyUI node pattern: `INPUT_TYPES()` classmethod, `RETURN_TYPES`, `FUNCTION`
|
||||||
|
|
||||||
|
### Configuration
|
||||||
|
|
||||||
|
- `py/config.py` manages folder paths for models and handles symlink mappings
|
||||||
|
- Auto-saves paths to `settings.json` in ComfyUI mode
|
||||||
|
|
||||||
|
### Frontend UI Architecture
|
||||||
|
|
||||||
|
#### 1. LoRA Manager Web UI
|
||||||
|
- Location: `./static/` (JS/CSS) and `./templates/` (HTML)
|
||||||
|
- Tech: Vanilla JS + CSS, served by the hosting server (ComfyUI app in plugin mode, `standalone.py` in standalone mode)
|
||||||
|
- Tests: `tests/frontend/**/*.test.js` (vitest + jsdom)
|
||||||
|
|
||||||
|
#### 2. ComfyUI Custom Node Widgets
|
||||||
|
- Location: `./web/comfyui/` (Vanilla JS) + `./vue-widgets/` (Vue)
|
||||||
|
- Primary styles: `./web/comfyui/lm_styles.css` (NOT `./static/css/`)
|
||||||
|
- Vue widgets: Vue 3 + TypeScript + PrimeVue + vue-i18n, e.g. `LoraPoolWidget`, `LoraRandomizerWidget`, `LoraCyclerWidget`, `AutocompleteTextWidget`
|
||||||
|
- Vue builds to `./web/comfyui/vue-widgets/`; auto-built on ComfyUI startup via `py/vue_widget_builder.py`, typecheck via `vue-tsc`
|
||||||
|
- Widget registration: `app.registerExtension()` and `getCustomWidgets` hooks; `node.addDOMWidget(...)` embeds HTML in LiteGraph nodes
|
||||||
|
- See `docs/dom_widget_dev_guide.md` for the DOMWidget development guide
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
### Backend (pytest)
|
||||||
|
|
||||||
|
- Config in `pytest.ini`: `--import-mode=importlib`, testpaths=`tests`
|
||||||
|
- Fixtures in `tests/conftest.py` mock ComfyUI dependencies; use `tmp_path_factory` for isolation
|
||||||
|
- Markers: `@pytest.mark.asyncio`, `@pytest.mark.no_settings_dir_isolation` (tests needing real settings paths)
|
||||||
|
|
||||||
|
### Frontend (vitest)
|
||||||
|
|
||||||
|
- Vanilla JS tests: `tests/frontend/**/*.test.js` with jsdom; setup in `tests/frontend/setup.js`
|
||||||
|
- Vue widget tests: `vue-widgets/tests/**/*.test.ts` with jsdom + `@vue/test-utils`
|
||||||
|
|
||||||
|
## Key Integration Points
|
||||||
|
|
||||||
|
- **Settings:** Stored in the user config directory (via `platformdirs`) or portable mode (`"use_portable_settings": true`)
|
||||||
|
- **CivitAI/CivArchive:** API clients for metadata sync and model downloads; CivitAI API key stored in settings
|
||||||
|
- **Symlinks:** Config scans symlinks to map virtual→physical paths; fingerprinting prevents redundant rescans
|
||||||
|
- **WebSocket:** Broadcasts real-time progress for downloads, scans, and metadata sync
|
||||||
|
- **Model scanning flow:** Walk folders → compute hashes → deduplicate → extract safetensors metadata → cache in SQLite → background CivitAI sync → WebSocket broadcast
|
||||||
|
|
||||||
## Important Notes
|
## Important Notes
|
||||||
|
|
||||||
- ALWAYS use English for comments (per copilot-instructions.md)
|
- ALWAYS use English for comments (per copilot-instructions.md)
|
||||||
- Dual mode: ComfyUI plugin (folder_paths) vs standalone (settings.json)
|
|
||||||
- Detection: `os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"`
|
|
||||||
- Run `python scripts/sync_translation_keys.py` after adding UI strings to `locales/en.json`
|
- Run `python scripts/sync_translation_keys.py` after adding UI strings to `locales/en.json`
|
||||||
- Symlinks require normalized paths.
|
- Symlinks require normalized paths.
|
||||||
**Business paths vs real paths**: All stored paths and operation routing use the
|
**Business paths vs real paths**: All stored paths and operation routing use the
|
||||||
@@ -143,23 +233,4 @@ npm run test:coverage # Generate coverage report
|
|||||||
resolved. `os.path.realpath` is only for scanner dedup and the symlink cache.
|
resolved. `os.path.realpath` is only for scanner dedup and the symlink cache.
|
||||||
Any path passed to `os.remove`/`os.rename`/`shutil.move` or validated by a
|
Any path passed to `os.remove`/`os.rename`/`shutil.move` or validated by a
|
||||||
containment check MUST use the business path (i.e. `os.path.abspath`, not
|
containment check MUST use the business path (i.e. `os.path.abspath`, not
|
||||||
`realpath`).
|
`realpath`).
|
||||||
|
|
||||||
## Git / Commit Messages
|
|
||||||
|
|
||||||
- Follow the style of recent repository commits when writing commit messages
|
|
||||||
- Prefer the repo's existing `feat(...)`, `fix(...)`, `chore:` style where applicable
|
|
||||||
- If the user has provided a GitHub issue link or issue ID for the task, mention that issue in the commit message, for example `(#871)`
|
|
||||||
- When unrelated local changes exist, stage and commit only the files relevant to the requested task
|
|
||||||
|
|
||||||
## Frontend UI Architecture
|
|
||||||
|
|
||||||
### 1. Standalone Web UI
|
|
||||||
- Location: `./static/` and `./templates/`
|
|
||||||
- Tech: Vanilla JS + CSS, served by standalone server
|
|
||||||
- Tests via npm in root directory
|
|
||||||
|
|
||||||
### 2. ComfyUI Custom Node Widgets
|
|
||||||
- Location: `./web/comfyui/` (Vanilla JS) + `./vue-widgets/` (Vue)
|
|
||||||
- Primary styles: `./web/comfyui/lm_styles.css` (NOT `./static/css/`)
|
|
||||||
- Vue builds to `./web/comfyui/vue-widgets/`, typecheck via `vue-tsc`
|
|
||||||
@@ -1,189 +0,0 @@
|
|||||||
# CLAUDE.md
|
|
||||||
|
|
||||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
ComfyUI LoRA Manager is a comprehensive LoRA management system for ComfyUI that combines a Python backend with browser-based widgets. It provides model organization, downloading from CivitAI/CivArchive, recipe management, and one-click workflow integration.
|
|
||||||
|
|
||||||
## Development Commands
|
|
||||||
|
|
||||||
### Backend
|
|
||||||
|
|
||||||
```bash
|
|
||||||
pip install -r requirements.txt
|
|
||||||
pip install -r requirements-dev.txt
|
|
||||||
|
|
||||||
# Run standalone server (port 8188 by default)
|
|
||||||
python standalone.py --port 8188
|
|
||||||
|
|
||||||
# Run all backend tests
|
|
||||||
pytest
|
|
||||||
|
|
||||||
# Run specific test file or function
|
|
||||||
pytest tests/test_recipes.py
|
|
||||||
pytest tests/test_recipes.py::test_function_name
|
|
||||||
|
|
||||||
# Run backend tests with coverage
|
|
||||||
COVERAGE_FILE=coverage/backend/.coverage pytest \
|
|
||||||
--cov=py \
|
|
||||||
--cov=standalone \
|
|
||||||
--cov-report=term-missing \
|
|
||||||
--cov-report=html:coverage/backend/html \
|
|
||||||
--cov-report=xml:coverage/backend/coverage.xml \
|
|
||||||
--cov-report=json:coverage/backend/coverage.json
|
|
||||||
```
|
|
||||||
|
|
||||||
### Frontend
|
|
||||||
|
|
||||||
There are three test suites run by `npm test`: vanilla JS tests (vitest at root) and Vue widget tests (`vue-widgets/` vitest).
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npm install
|
|
||||||
cd vue-widgets && npm install && cd ..
|
|
||||||
|
|
||||||
# Run all frontend tests (JS + Vue)
|
|
||||||
npm test
|
|
||||||
|
|
||||||
# Run only vanilla JS tests
|
|
||||||
npm run test:js
|
|
||||||
|
|
||||||
# Run only Vue widget tests
|
|
||||||
npm run test:vue
|
|
||||||
|
|
||||||
# Watch mode (JS tests only)
|
|
||||||
npm run test:watch
|
|
||||||
|
|
||||||
# Frontend coverage
|
|
||||||
npm run test:coverage
|
|
||||||
|
|
||||||
# Build Vue widgets (output to web/comfyui/vue-widgets/)
|
|
||||||
cd vue-widgets && npm run build
|
|
||||||
|
|
||||||
# Vue widget dev mode (watch + rebuild)
|
|
||||||
cd vue-widgets && npm run dev
|
|
||||||
|
|
||||||
# Typecheck Vue widgets
|
|
||||||
cd vue-widgets && npm run typecheck
|
|
||||||
```
|
|
||||||
|
|
||||||
### Localization
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Sync translation keys after UI string updates
|
|
||||||
python scripts/sync_translation_keys.py
|
|
||||||
```
|
|
||||||
|
|
||||||
Locale files are in `locales/` (en, zh-CN, zh-TW, ja, ko, fr, de, es, ru, he).
|
|
||||||
|
|
||||||
## Architecture
|
|
||||||
|
|
||||||
### Dual Mode Operation
|
|
||||||
|
|
||||||
The system runs in two modes:
|
|
||||||
- **ComfyUI plugin mode**: Integrates with ComfyUI's PromptServer, uses `folder_paths` for model discovery
|
|
||||||
- **Standalone mode**: `standalone.py` mocks ComfyUI dependencies, reads paths from `settings.json`
|
|
||||||
- Detection: `os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"`
|
|
||||||
|
|
||||||
### Backend (Python)
|
|
||||||
|
|
||||||
**Entry points:**
|
|
||||||
- `__init__.py` — ComfyUI plugin entry: registers nodes via `NODE_CLASS_MAPPINGS`, sets `WEB_DIRECTORY`, calls `LoraManager.add_routes()`
|
|
||||||
- `standalone.py` — Standalone server: mocks `folder_paths` and node modules, starts aiohttp server
|
|
||||||
- `py/lora_manager.py` — Main `LoraManager` class that registers all HTTP routes
|
|
||||||
|
|
||||||
**Service layer** (`py/services/`):
|
|
||||||
- `ServiceRegistry` singleton for dependency injection; services follow `get_instance()` singleton pattern
|
|
||||||
- `BaseModelService` abstract base → `LoraService`, `CheckpointService`, `EmbeddingService`
|
|
||||||
- `ModelScanner` base → `LoraScanner`, `CheckpointScanner`, `EmbeddingScanner` for file discovery with hash-based deduplication
|
|
||||||
- `PersistentModelCache` — SQLite-based metadata cache
|
|
||||||
- `MetadataSyncService` — Background sync from CivitAI/CivArchive APIs
|
|
||||||
- `SettingsManager` — Settings with schema migration support
|
|
||||||
- `WebSocketManager` — Real-time progress broadcasting
|
|
||||||
- `ModelServiceFactory` — Creates the right service for each model type
|
|
||||||
- Use cases in `py/services/use_cases/` orchestrate complex business logic (auto-organize, bulk refresh, downloads)
|
|
||||||
|
|
||||||
**Routes** (`py/routes/`):
|
|
||||||
- Route registrars organize endpoints by domain: `ModelRouteRegistrar`, `RecipeRouteRegistrar`, etc.
|
|
||||||
- Request handlers in `py/routes/handlers/` implement route logic
|
|
||||||
- API endpoints follow `/loras/*`, `/checkpoints/*`, `/embeddings/*` patterns
|
|
||||||
- All routes use aiohttp, return `web.json_response` or `web.Response`
|
|
||||||
|
|
||||||
**Recipe system** (`py/recipes/`):
|
|
||||||
- `base.py` — Recipe metadata structure
|
|
||||||
- `enrichment.py` — Enriches recipes with model metadata
|
|
||||||
- `parsers/` — Parsers for PNG metadata, JSON, and workflow formats
|
|
||||||
|
|
||||||
**Custom nodes** (`py/nodes/`):
|
|
||||||
- Each node class has a `NAME` class attribute used as key in `NODE_CLASS_MAPPINGS`
|
|
||||||
- Standard ComfyUI node pattern: `INPUT_TYPES()` classmethod, `RETURN_TYPES`, `FUNCTION`
|
|
||||||
- All nodes registered in `__init__.py`
|
|
||||||
|
|
||||||
**Configuration** (`py/config.py`):
|
|
||||||
- Manages folder paths for models, handles symlink mappings
|
|
||||||
- Auto-saves paths to settings.json in ComfyUI mode
|
|
||||||
|
|
||||||
### Frontend — Two Distinct UI Systems
|
|
||||||
|
|
||||||
#### 1. Standalone Manager Web UI
|
|
||||||
- **Location:** `static/` (JS/CSS) and `templates/` (HTML)
|
|
||||||
- **Tech:** Vanilla JS + CSS, served by standalone server
|
|
||||||
- **Structure:** `static/js/core.js` (shared), `loras.js`, `checkpoints.js`, `embeddings.js`, `recipes.js`, `statistics.js`
|
|
||||||
- **Tests:** `tests/frontend/**/*.test.js` (vitest + jsdom)
|
|
||||||
|
|
||||||
#### 2. ComfyUI Custom Node Widgets
|
|
||||||
- **Vanilla JS widgets:** `web/comfyui/*.js` — ES modules extending ComfyUI's LiteGraph UI
|
|
||||||
- `loras_widget.js` / `loras_widget_events.js` — Main LoRA selection widget
|
|
||||||
- `autocomplete.js` — Trigger word and embedding autocomplete
|
|
||||||
- `preview_tooltip.js` — Model card preview tooltips
|
|
||||||
- `top_menu_extension.js` — "Launch LoRA Manager" menu item
|
|
||||||
- `utils.js` — Shared utilities and API helpers
|
|
||||||
- Widget styling in `web/comfyui/lm_styles.css` (NOT `static/css/`)
|
|
||||||
- **Vue widgets:** `vue-widgets/src/` → built to `web/comfyui/vue-widgets/`
|
|
||||||
- Vue 3 + TypeScript + PrimeVue + vue-i18n
|
|
||||||
- Vite build with CSS-injected-by-JS plugin
|
|
||||||
- Components: `LoraPoolWidget`, `LoraRandomizerWidget`, `LoraCyclerWidget`, `AutocompleteTextWidget`
|
|
||||||
- Auto-built on ComfyUI startup via `py/vue_widget_builder.py`
|
|
||||||
- Tests: `vue-widgets/tests/**/*.test.ts` (vitest)
|
|
||||||
|
|
||||||
**Widget registration pattern:**
|
|
||||||
- Widgets use `app.registerExtension()` and `getCustomWidgets` hooks
|
|
||||||
- `node.addDOMWidget(name, type, element, options)` embeds HTML in LiteGraph nodes
|
|
||||||
- See `docs/dom_widget_dev_guide.md` for DOMWidget development guide
|
|
||||||
|
|
||||||
## Code Style
|
|
||||||
|
|
||||||
**Python:**
|
|
||||||
- PEP 8, 4-space indentation, English comments only
|
|
||||||
- Use `from __future__ import annotations` for forward references
|
|
||||||
- Use `TYPE_CHECKING` guard for type-checking-only imports
|
|
||||||
- Loggers via `logging.getLogger(__name__)`
|
|
||||||
- Custom exceptions in `py/services/errors.py`
|
|
||||||
- Async patterns: `async def` for I/O, `@pytest.mark.asyncio` for async tests
|
|
||||||
- Singleton pattern with class-level `asyncio.Lock` (see `ModelScanner.get_instance()`)
|
|
||||||
|
|
||||||
**JavaScript:**
|
|
||||||
- ES modules, camelCase functions/variables, PascalCase classes
|
|
||||||
- Widget files use `*_widget.js` suffix
|
|
||||||
- Prefer vanilla JS for `web/comfyui/` widgets, avoid framework dependencies (except Vue widgets)
|
|
||||||
|
|
||||||
## Testing
|
|
||||||
|
|
||||||
**Backend (pytest):**
|
|
||||||
- Config in `pytest.ini`: `--import-mode=importlib`, testpaths=`tests`
|
|
||||||
- Fixtures in `tests/conftest.py` handle ComfyUI dependency mocking
|
|
||||||
- Markers: `@pytest.mark.asyncio`, `@pytest.mark.no_settings_dir_isolation`
|
|
||||||
- Uses `tmp_path_factory` for directory isolation
|
|
||||||
|
|
||||||
**Frontend (vitest):**
|
|
||||||
- Vanilla JS tests: `tests/frontend/**/*.test.js` with jsdom
|
|
||||||
- Vue widget tests: `vue-widgets/tests/**/*.test.ts` with jsdom + @vue/test-utils
|
|
||||||
- Setup in `tests/frontend/setup.js`
|
|
||||||
|
|
||||||
## Key Integration Points
|
|
||||||
|
|
||||||
- **Settings:** Stored in user directory (via `platformdirs`) or portable mode (`"use_portable_settings": true`)
|
|
||||||
- **CivitAI/CivArchive:** API clients for metadata sync and model downloads; CivitAI API key in settings
|
|
||||||
- **Symlink handling:** Config scans symlinks to map virtual→physical paths; fingerprinting prevents redundant rescans
|
|
||||||
- **WebSocket:** Broadcasts real-time progress for downloads, scans, and metadata sync
|
|
||||||
- **Model scanning flow:** Walk folders → compute hashes → deduplicate → extract safetensors metadata → cache in SQLite → background CivitAI sync → WebSocket broadcast
|
|
||||||
+327
-295
File diff suppressed because it is too large
Load Diff
@@ -54,7 +54,7 @@ The dedicated services encapsulate long-running work so handlers stay thin.
|
|||||||
| Use case | Entry point | Dependencies | Guarantees |
|
| Use case | Entry point | Dependencies | Guarantees |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `RecipeAnalysisService` | `analyze_uploaded_image`, `analyze_remote_image`, `analyze_local_image`, `analyze_widget_metadata` | `ExifUtils`, `RecipeParserFactory`, downloader factory, optional metadata collector/processor | Normalises missing/invalid payloads into `RecipeValidationError`; generates consistent fingerprint data to keep duplicate detection stable; temporary files are cleaned up after every analysis path. |
|
| `RecipeAnalysisService` | `analyze_uploaded_image`, `analyze_remote_image`, `analyze_local_image`, `analyze_widget_metadata` | `ExifUtils`, `RecipeParserFactory`, downloader factory, optional metadata collector/processor | Normalises missing/invalid payloads into `RecipeValidationError`; generates consistent fingerprint data to keep duplicate detection stable; temporary files are cleaned up after every analysis path. |
|
||||||
| `RecipePersistenceService` | `save_recipe`, `delete_recipe`, `update_recipe`, `reconnect_lora`, `bulk_delete`, `save_recipe_from_widget` | `ExifUtils`, recipe scanner, card preview sizing constants | Writes images/JSON metadata atomically; updates scanner caches and hash indices before returning; recalculates fingerprints whenever LoRA assignments change. |
|
| `RecipePersistenceService` | `save_recipe`, `delete_recipe`, `update_recipe`, `reconnect_lora`, `get_reconnect_suggestions`, `bulk_delete`, `save_recipe_from_widget` | `ExifUtils`, recipe scanner, card preview sizing constants | Writes images/JSON metadata atomically; updates scanner caches and hash indices before returning; recalculates fingerprints whenever LoRA assignments change. |
|
||||||
| `RecipeSharingService` | `share_recipe`, `prepare_download` | `tempfile`, recipe scanner | Copies originals to TTL-managed temp files; metadata lookups re-use the scanner; expired shares trigger cleanup and `RecipeNotFoundError`. |
|
| `RecipeSharingService` | `share_recipe`, `prepare_download` | `tempfile`, recipe scanner | Copies originals to TTL-managed temp files; metadata lookups re-use the scanner; expired shares trigger cleanup and `RecipeNotFoundError`. |
|
||||||
|
|
||||||
## Maintaining critical invariants
|
## Maintaining critical invariants
|
||||||
|
|||||||
@@ -0,0 +1,370 @@
|
|||||||
|
# i18n Translation Guidelines
|
||||||
|
|
||||||
|
This document is the canonical set of conventions for translating LoRA Manager UI strings.
|
||||||
|
It applies to **human translators and AI agents** alike. Read it before editing anything in
|
||||||
|
`locales/`.
|
||||||
|
|
||||||
|
Source of truth: `locales/en.json` (10 locales, 1810 leaf keys; all locales share the exact
|
||||||
|
same key structure).
|
||||||
|
|
||||||
|
Locales: `en`, `zh-CN`, `zh-TW`, `ja`, `ko`, `fr`, `de`, `es`, `ru`, `he` (RTL).
|
||||||
|
|
||||||
|
> **Status (2026-08 sweep):** a full audit was executed and the terminology, placeholder,
|
||||||
|
> stale-text, and untranslated-block fixes described in §2–§6 were applied across all locales
|
||||||
|
> (commits `3c3ac49f` … `fd1227d3`). The tables below are now the **normative target state**,
|
||||||
|
> not a to-do list — future edits should preserve these renderings and only add what is new.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Hard rules (do not violate)
|
||||||
|
|
||||||
|
### R1 — Key structure is sacred
|
||||||
|
- Only `locales/en.json` may add/remove/rename keys. All other locales must keep the exact
|
||||||
|
same nested key set. `tests/i18n/test_i18n.py` enforces this.
|
||||||
|
- When a new UI string is added to `en.json`, run
|
||||||
|
`python scripts/sync_translation_keys.py` (adds the missing keys to all locales with
|
||||||
|
`[TODO: Translate]` placeholder copies) — **then stop**. Do NOT translate proactively:
|
||||||
|
placeholders are the expected end state during feature development, and translations are
|
||||||
|
filled in only when the feature owner explicitly asks (workflow details in §7).
|
||||||
|
- Never reorder, re-indent, or reformat a locale file "for tidiness". The sync script
|
||||||
|
preserves formatting; manual reformatting creates noisy diffs.
|
||||||
|
|
||||||
|
### R2 — Placeholders and HTML must be preserved verbatim
|
||||||
|
- `{name}`-style placeholders must appear in the translation exactly as in `en.json`.
|
||||||
|
Do not invent placeholders the source string does not have — the caller may not pass them
|
||||||
|
(example bug: `zh-CN recipes.controls.import.downloadLocationPreview` added `{path}`; the
|
||||||
|
template renders this key with no parameters, so the literal text `{path}` shows in the UI).
|
||||||
|
- `{{...}}` in a locale value is an escaped literal brace — keep it identical.
|
||||||
|
- Keep embedded HTML tags (e.g. `<strong>...</strong>`, `<code>...</code>`) intact.
|
||||||
|
You may move the tag around the sentence if the target language needs different word order.
|
||||||
|
|
||||||
|
### R3 — Never translate or transliterate these
|
||||||
|
- Model types: **LoRA, Checkpoint, Embedding, Diffusion Model**
|
||||||
|
- Products/brands: **LoRA Manager, ComfyUI, CivitAI, CivArchive, HuggingFace, Ko-fi**
|
||||||
|
- Ecosystem names: **LyCORIS, DoRA**, trigger-adjacent jargon **Prompt, Workflow**
|
||||||
|
(these are used as-is in the target-language SD community; see §2 per-language policy)
|
||||||
|
- Theme names: **Nord, Midnight, Monokai, Dracula, Solarized**
|
||||||
|
|
||||||
|
### R4 — The "Recipe" convention (the most important domain term)
|
||||||
|
Product intent: a *Recipe* records a **LoRA combination + generation parameters**
|
||||||
|
(prompt, seed, sampler, …) that reproduces an image style. The metaphor is a **cooking
|
||||||
|
recipe** — "follow it and you get a similar dish". It is **not** a menu, not a dish list,
|
||||||
|
not a prescription.
|
||||||
|
|
||||||
|
Decision per language — translate only into a word whose everyday primary meaning is a
|
||||||
|
cooking recipe; where that word would mislead users, **keep the English "Recipe(s)"**:
|
||||||
|
|
||||||
|
| Locale | Use | Never use |
|
||||||
|
|---|---|---|
|
||||||
|
| fr | **Recipe / Recipes** (keep English) | recette(s) — cooking reading is secondary and it was explicitly judged misleading |
|
||||||
|
| zh-CN / zh-TW | 配方 | 食谱 (reads as "food cookbook") |
|
||||||
|
| ja | レシピ | — (leftover English "Recipe" in `initialization.recipes.title` / `toast.recipes.recipeSaved` → translate) |
|
||||||
|
| ko | 레시피 | — |
|
||||||
|
| de | Rezept / Rezepte | — (cooking meaning dominant; prescription reading acceptable) |
|
||||||
|
| es | receta / recetas | — (cooking meaning dominant) |
|
||||||
|
| ru | рецепт / рецепты | — (leftover English "Recipe" in `initialization.recipes.title` / `toast.recipes.recipeSaved` → translate) |
|
||||||
|
| he | מתכון / מתכונים | — (cooking meaning dominant) |
|
||||||
|
|
||||||
|
Whatever the choice, **one concept = one noun within a locale**. Currently violated in:
|
||||||
|
- `fr` — "Recipe" (~97 keys, incl. nav) mixed with "recette" (~58 keys)
|
||||||
|
- `zh-CN` / `zh-TW` — 配方 (126/122 keys) mixed with 食谱 / 食譜 (14/17 keys, all in the
|
||||||
|
*rematch* flow: `globalContextMenu.rematchRecipes.*`, `toast.recipes.rematch*`)
|
||||||
|
- `de` — "Rezept" (136 keys) mixed with leftover English "Recipe" (5 keys)
|
||||||
|
- `ja` / `ru` — leftover English "Recipe" in `initialization.recipes.title` ("Recipe Manager
|
||||||
|
zu initialisieren" / «Инициализация Recipe Manager») and `toast.recipes.recipeSaved`
|
||||||
|
|
||||||
|
### R5 — One term, one rendering (within each locale)
|
||||||
|
Same source word must not be translated several ways in one file. Known offender areas
|
||||||
|
(see §5 for the full fix list): recipe, Checkpoint, Embedding, prompt, base model, preset,
|
||||||
|
workflow, hash, metadata, tags, bulk. Every locale currently mixes variants of at least one
|
||||||
|
of these — pick the preferred form in the §2 tables and normalize.
|
||||||
|
|
||||||
|
### R6 — Register consistency
|
||||||
|
- `zh-CN` / `zh-TW`: pick 你 or 您 once. Do not mix (zh-CN has 44×你 + 5×您; zh-TW has
|
||||||
|
27×您 + 18×你).
|
||||||
|
- `de`: pick "du" or "Sie" once (currently 143×Sie + ~7×du).
|
||||||
|
- `es`: pick "tú" or "usted" once.
|
||||||
|
|
||||||
|
### R7 — Punctuation per script
|
||||||
|
- Full-width punctuation `:()` is correct **only in CJK locales** (zh-CN, zh-TW, ja, ko).
|
||||||
|
- Latin/Cyrillic/Hebrew locales must use ASCII `: ()` — full-width colons leaked in there
|
||||||
|
are machine-translation artifacts. Known: `fr toast.recipes.createError/createFailed`,
|
||||||
|
`es toast.recipes.createError/createFailed` (e.g. "…de la receta:" should be "…de la receta:").
|
||||||
|
- `fr` apostrophes must be U+2019 `'` / ASCII `'`, never a straight double quote:
|
||||||
|
`fr header.filter.allowSellingGeneratedContentTooltip` currently reads
|
||||||
|
`vendre d"images` → fix to `d'images`. Do not mix `'` and `'` in one file (fr has 299 vs 15).
|
||||||
|
- Ellipsis: use ASCII `...` (project style). Don't introduce `…`.
|
||||||
|
- Keep the sentence-ending period/omission consistent with the source string where the
|
||||||
|
language allows it.
|
||||||
|
- `he` is RTL: mix of Hebrew and Latin scripts is normal; keep Latin term ordering natural.
|
||||||
|
|
||||||
|
### R8 — No untranslated English leftovers
|
||||||
|
Full sentences left byte-identical to `en.json` are bugs (brand names and URL placeholders
|
||||||
|
are the exception). Every locale has them; see §6 for the per-locale checklist.
|
||||||
|
`[TODO: Translate]` placeholders are the sanctioned intermediate state during feature
|
||||||
|
development (see §7) — do not "fix" them unless the feature owner asked for translations.
|
||||||
|
|
||||||
|
### R9 — Mirror the source even when the source is wrong
|
||||||
|
If `en.json` itself contains an inconsistency (e.g. the `Civitai` vs `CivitAI` casing split,
|
||||||
|
or the `CivitArchive` typo in `modals.relinkCivitai.helpText.format4`), translate/transcribe
|
||||||
|
it as-is in your locale and instead **fix the source** in `en.json` (then propagate by
|
||||||
|
re-syncing and re-translating affected keys). Do not silently diverge in one locale only.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Per-language term maps
|
||||||
|
|
||||||
|
Preferred rendering per term. "Fix" means the locale currently contains the wrong variant
|
||||||
|
and must be normalized. `en` = keep the English word as-is.
|
||||||
|
|
||||||
|
### fr
|
||||||
|
|
||||||
|
| Term | Use | Fix |
|
||||||
|
|---|---|---|
|
||||||
|
| recipe | Recipe(s) | Replace all "recette(s)" (58 keys, e.g. `recipes.actions.deleteRecipeWithShortcut`, `toast.recipes.rematchComplete`) with "Recipe(s)" |
|
||||||
|
| Checkpoint | Checkpoint | `statistics.modelTypes.checkpoint` = "Point de contrôle" → "Checkpoint" |
|
||||||
|
| trigger words | mot(s)-clé(s) | unify: `modals.model.triggerWords.editWord` uses "mot déclencheur" — pick one |
|
||||||
|
| prompt / negative prompt | Prompt / prompt négatif | — |
|
||||||
|
| base model | modèle(s) de base | — |
|
||||||
|
| preset | préréglage | unify: `modals.model.usageTips.addPresetParameter` "prédéfini", `toast.presets.restored` "par défaut" |
|
||||||
|
| hash | hash | `conflictConfirm.message` "hachage" → "hash" |
|
||||||
|
| tags | tags | `settings.sections.priorityTags` "Étiquettes" → "Tags" |
|
||||||
|
| metadata | métadonnées | `loras.controls.refresh.fullTooltip` keeps English "metadata" |
|
||||||
|
| duplicates | doublon(s) | unify with "dupliqué(e)s" |
|
||||||
|
| bulk | groupé(e) | unify with "par lot / mode lot" variants |
|
||||||
|
|
||||||
|
### de
|
||||||
|
|
||||||
|
| Term | Use | Fix |
|
||||||
|
|---|---|---|
|
||||||
|
| recipe | Rezept/Rezepte | 5 leftover English "Recipe" keys → Rezept (e.g. `globalContextMenu.repairRecipes.label`, `toast.recipes.recipeSaved`) |
|
||||||
|
| base model | pick Basis-Modell or Basismodell | currently 27× hyphenated vs 15× closed |
|
||||||
|
| metadata | Metadaten | 4 keys use "Modelldaten" (`onboarding.steps.fetch.title/content`) → Metadaten |
|
||||||
|
| bulk | pick Massen- or Sammelmodus | `loras.controls.bulk.action` = "Massen" reads as "crowds" — use "Massenbearbeitung"/"Mehrfachauswahl" |
|
||||||
|
| register | Sie (formal) | 7 keys use "du/dein" (`settings.backup.managementHelp`, `modals.checkUpdates.message/tip`, `doctor.footer`, …) |
|
||||||
|
|
||||||
|
### es
|
||||||
|
|
||||||
|
| Term | Use | Fix |
|
||||||
|
|---|---|---|
|
||||||
|
| recipe | receta(s) | — |
|
||||||
|
| Checkpoint | Checkpoint | 5 statistics keys "Punto(s) de control" → "Checkpoints" (`statistics.metrics.checkpoints`, `statistics.insights.unusedCheckpoints.*`, `statistics.modelTypes.checkpoint`) |
|
||||||
|
| trigger words | palabra(s) de activación | 2 keys already use it; ~15 keys "palabra(s) clave" (reads as search keyword) → unify |
|
||||||
|
| base model | modelo base | — |
|
||||||
|
| preset | preajuste | 3 keys keep English "preset", 1 "preestablecido" → preajuste |
|
||||||
|
| workflow | pick flujo de trabajo or workflow | currently 21× "flujo de trabajo" vs 10× "workflow" |
|
||||||
|
| bulk | masivo / por lotes | unify; "Batch Import" → traducción |
|
||||||
|
| tags | etiquetas | — |
|
||||||
|
|
||||||
|
### ru
|
||||||
|
|
||||||
|
| Term | Use | Fix |
|
||||||
|
|---|---|---|
|
||||||
|
| recipe | рецепт(ы) | English leftovers: `initialization.recipes.title`, `recipes.batchImport.*`, `toast.recipes.recipeSaved` → translate |
|
||||||
|
| Checkpoint | Checkpoint (recommended) | 3 variants today: "Checkpoint" (17 keys), «Чекпойнт», «Контрольная точка» (statistics, 6 keys) — statistics MUST drop «Контрольная точка» |
|
||||||
|
| Embedding | Embedding | «Эмбеддинг» variant exists in `settings.priorityTags.modelTypes.embedding` — unify |
|
||||||
|
| prompt | промпт | 8 keys use «запрос» (reads as "database/HTTP request") → «промпт» |
|
||||||
|
| base model | базовая модель | — |
|
||||||
|
| preset | пресет | `header.theme.presets` "Предустановки" → пресеты |
|
||||||
|
| workflow | Workflow (recommended) | «рабочий процесс» used in 4 keys — unify |
|
||||||
|
| hash | pick хеш or хэш | both spellings co-occur |
|
||||||
|
| tag(s) | тег(и) | — |
|
||||||
|
| typos | — | `settings.misc.loraSyntaxFormatHelp`: «безпотерьного» → «беспотерьного» |
|
||||||
|
|
||||||
|
### he
|
||||||
|
|
||||||
|
| Term | Use | Fix |
|
||||||
|
|---|---|---|
|
||||||
|
| recipe | מתכון / מתכונים | — |
|
||||||
|
| Checkpoint | Checkpoint | 5 statistics keys «נקודת/נקודות ביקורת» (road/security checkpoint) → "Checkpoint(s)" (`statistics.metrics.checkpoints`, `statistics.modelTypes.checkpoint`, `statistics.insights.unusedCheckpoints.*`) |
|
||||||
|
| Embedding | Embedding | `statistics` keys use הטמעות → Embedding |
|
||||||
|
| prompt | pick הנחיה or פרומפט | 9 keys הנחיה vs 3 פרומפט — unify (recommend פרומפט, SD-community loanword) |
|
||||||
|
| preset | קביעה מראש | `header.filter.presetOverwriteConfirm` uses פריסט → unify |
|
||||||
|
| hash | pick one of האש / גיבוב / hash | 3 variants co-occur — unify (recommend hash or גיבוב) |
|
||||||
|
| metadata | pick מטא-דאטה or מטא-נתונים | 38 vs 17 keys — unify |
|
||||||
|
| model | מודל | 13 keys use דגם/דגמים — unify |
|
||||||
|
| bulk | pick one of 5 variants | 5 different renderings ("כמות גדולה", "המוני", "קבוצתי", "אצווה", …) — unify; `loras.controls.bulk.action` "כמות גדולה" reads as "large quantity" |
|
||||||
|
|
||||||
|
### ja
|
||||||
|
|
||||||
|
| Term | Use | Fix |
|
||||||
|
|---|---|---|
|
||||||
|
| recipe | レシピ | `initialization.recipes.title` keeps English "Recipe Manager" — translate to レシピマネージャー |
|
||||||
|
| Checkpoint | Checkpoint or チェックポイント (pick one) | 3 variants: Checkpoint (~14), checkpoint lowercase (4), チェックポイント (4, e.g. `settings.priorityTags.modelTypes.checkpoint`) |
|
||||||
|
| Embedding | Embedding | 4 keys lowercase "embedding" mid-sentence |
|
||||||
|
| bulk | 一括 | `modals.checkUpdates.tip` "バルクモード" → 一括モード |
|
||||||
|
| recipe counter | 件 or 個 | `repairRecipes.success` uses 件, `.cancelled` uses 個 — unify |
|
||||||
|
|
||||||
|
### ko
|
||||||
|
|
||||||
|
| Term | Use | Fix |
|
||||||
|
|---|---|---|
|
||||||
|
| recipe | 레시피 | — |
|
||||||
|
| Checkpoint | Checkpoint (recommended) | 4 keys transliterate 체크포인트 (`settings.priorityTags.modelTypes.checkpoint`, `toast.recipes.missingCheckpointPath/missingCheckpointInfo/downloadCheckpointFailed`) |
|
||||||
|
| Embedding | Embedding | 3 keys 임베딩 (`settings.priorityTags.modelTypes.embedding`, `uiHelpers.nodeSelector.embedding`) |
|
||||||
|
| base model | 베이스 모델 | 6 keys «기본 모델» read as "default model" → 베이스 모델 (`settings.downloadSkipBaseModels.*`, `toast.loras.downloadSkippedByBaseModel`) |
|
||||||
|
| workflow | pick 워크플로 or 워크플로우 | 26 vs 6 keys — unify |
|
||||||
|
| bulk | 일괄 | `modals.checkUpdates.tip` "벌크 모드" → 일괄 모드 |
|
||||||
|
| tag logic | — | `header.filter.tagLogicAny` = "모든 태그 일치 (OR)" is **inverted** (should be "하나 이상의 태그 일치") and identical to `tagLogicAll` |
|
||||||
|
| particle | — | `modelCard.sendToWorkflow.checkpointNotImplemented`: "Checkpoint을" → "Checkpoint를" |
|
||||||
|
|
||||||
|
### zh-CN / zh-TW
|
||||||
|
|
||||||
|
| Term | zh-CN | zh-TW |
|
||||||
|
|---|---|---|
|
||||||
|
| recipe | 配方 (fix 食谱 → 配方, 14 keys in rematch flow) | 配方 (fix 食譜 → 配方, 17 keys in rematch flow) |
|
||||||
|
| Checkpoint | Checkpoint (fix 检查点 → Checkpoint, 5 keys: `toast.recipes.missingCheckpointPath/missingCheckpointInfo/downloadCheckpointFailed`, `modelCard.actions.checkpointNameCopied`, `modelCard.sendToWorkflow.checkpointNotImplemented`) | Checkpoint (fix 檢查點 → Checkpoint, 4 keys: `modelCard.actions.copyCheckpointName`, `toast.recipes.missing*`×2, `toast.recipes.downloadCheckpointFailed`) |
|
||||||
|
| base model | 基础模型 (fix 基模型 → 基础模型, 3 keys in `modals.model.versions.filters.*`) | 基礎模型 ✓ consistent |
|
||||||
|
| prompt | 提示词 ✓ | 提示詞 ✓ |
|
||||||
|
| preset | 预设 ✓ | 預設 ✓ |
|
||||||
|
| workflow | 工作流 ✓ | 工作流 ✓ |
|
||||||
|
| trigger words | 触发词 ✓ | 觸發詞 ✓ |
|
||||||
|
| hash | 哈希 (哈希值 variant OK) | 雜湊 ✓ |
|
||||||
|
| register | 你 (fix 5×您 → 你) | 您 (fix 18×你 → 您) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Cross-cutting confusion hot-spots (must-fix list)
|
||||||
|
|
||||||
|
All items below were **resolved** in the 2026-08 sweep — treat them as a regression
|
||||||
|
watch-list: do not reintroduce these renderings.
|
||||||
|
|
||||||
|
1. **Checkpoint rendered as a literal security/road checkpoint** — fr, es, ru, he, zh-CN,
|
||||||
|
zh-TW all had 4–6 keys in the `statistics.*` domain reading as "control point"; reverted
|
||||||
|
to "Checkpoint".
|
||||||
|
2. **"recipe" variants that break the one-noun rule** — fr "recette" → "Recipe", zh
|
||||||
|
食谱/食譜 → 配方, de/ja/ru leftover English "Recipe" translated.
|
||||||
|
3. **ko `header.filter.tagLogicAny`** — was inverted ("모든 태그 일치 (OR)") and identical
|
||||||
|
to `tagLogicAll`; now "어느 하나의 태그와 일치 (OR)".
|
||||||
|
4. **ja `modals.model.versions.actions.viewLocalTooltip`** — was the stale "近日対応予定"
|
||||||
|
("coming soon"); all 9 locales now describe the actual action.
|
||||||
|
5. **Stale help texts** — `settings.downloadSkipBaseModels.help`,
|
||||||
|
`settings.aiProvider.apiBaseHelp`, `settings.hideEarlyAccessUpdates.help` retranslated
|
||||||
|
in all locales to the current `en.json` wording.
|
||||||
|
6. **en.json source bugs** (fixed in source, then mirrored):
|
||||||
|
- "Civitai" → "CivitAI" brand casing (values only; key names `relinkCivitai` etc. keep
|
||||||
|
their lowercase form and must not be renamed)
|
||||||
|
- `modals.relinkCivitai.helpText.format4` "CivitArchive" typo → "CivArchive"
|
||||||
|
- `zh-CN recipes.controls.import.downloadLocationPreview` invented `{path}` removed
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Placeholder contract deviations (current)
|
||||||
|
|
||||||
|
`{...}` token sets must match `en.json` per key. All deviations found in the 2026-08 sweep
|
||||||
|
were fixed, with one *intentional* exception:
|
||||||
|
|
||||||
|
**`toast.settings.mappingsUpdated`** — the caller passes a hardcoded English inflection
|
||||||
|
(`plural: count !== 1 ? 's' : ''`). Languages that cannot build a plural by appending that
|
||||||
|
`s` (zh-CN/zh-TW, ja, ko, de, ru, he) **drop `{plural}`** and render a count-friendly form
|
||||||
|
(`({count})` or a measure word); fr and es keep it (`mappage{plural}`, `mapeo{plural}`).
|
||||||
|
|
||||||
|
```python
|
||||||
|
# keep a copy of this rule next to the key if it ever moves:
|
||||||
|
# fr/es: "... ({count} mappage{plural})"
|
||||||
|
# de/ru/he: "... ({count})"
|
||||||
|
# zh-CN: "({count} 条映射)" / zh-TW: "({count} 個對應)" / ja: "({count} マッピング)"
|
||||||
|
```
|
||||||
|
|
||||||
|
Do NOT add `{...}` tokens the source lacks (the caller will not supply them, and the literal
|
||||||
|
text renders in the UI), and do NOT rename source tokens (`{typePlural}` stays `{typePlural}`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. One term, one rendering — offender matrix
|
||||||
|
|
||||||
|
Cross-locale summary of §2 inconsistencies. "✓" = already consistent. All ✗ cells were
|
||||||
|
resolved in the 2026-08 sweep; the row shows the single rendering now in force per locale.
|
||||||
|
|
||||||
|
| Term | fr | de | es | ru | he | ja | ko | zh-CN | zh-TW |
|
||||||
|
|---|---|---|---|---|---|---|---|---|---|
|
||||||
|
| recipe | Recipe | Rezept | receta | рецепт | מתכון | レシピ | 레시피 | 配方 | 配方 |
|
||||||
|
| Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint |
|
||||||
|
| Embedding | Embedding | Embedding | Embedding | Embedding | Embedding | Embedding | Embedding | Embedding | Embedding |
|
||||||
|
| prompt | Prompt | Prompt | prompt | промпт | פרומפט | プロンプト | 프롬프트 | 提示词 | 提示詞 |
|
||||||
|
| base model | modèle de base | Basismodell | modelo base | базовая модель | מודל בסיס | ベースモデル | 베이스 모델 | 基础模型 | 基礎模型 |
|
||||||
|
| preset | préréglage | Voreinstellung | preajuste | пресет | קביעה מראש | プリセット | 프리셋 | 预设 | 預設 |
|
||||||
|
| workflow | Workflow | Workflow | workflow | Workflow | workflow | ワークフロー | 워크플로 | 工作流 | 工作流 |
|
||||||
|
| hash | hash | Hash | hash | хеш | hash | ハッシュ | 해시 | 哈希 | 雜湊 |
|
||||||
|
| metadata | métadonnées | Metadaten | metadatos | метаданные | מטא-נתונים | メタデータ | 메타데이터 | 元数据 | 中繼資料 |
|
||||||
|
| tags | Tags | Tags | etiquetas | теги | תגיות | タグ | 태그 | 标签 | 標籤 |
|
||||||
|
| duplicates | en double | Duplikate | duplicados | дубликаты | כפילויות | 重複 | 중복 | 重复项 | 重複項 |
|
||||||
|
| bulk | groupé | Massen- | por lotes | пакетный | בכמות גדולה | 一括 | 일괄 | 批量 | 批量 |
|
||||||
|
|
||||||
|
Watch: ja/ko keep the model-type names **Checkpoint/Embedding** and `Diffusion Model` in
|
||||||
|
Latin (consistent with their model-type sections) — do not transliterate them as
|
||||||
|
チェックポイント/체크포인트.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Untranslated English leftovers (status)
|
||||||
|
|
||||||
|
Values byte-identical to `en.json` that are actual UI sentences are bugs (brand names and
|
||||||
|
URL placeholders are the exception). As of the 2026-08 sweep, **all previously untranslated
|
||||||
|
blocks are translated** in every locale: `recipes.batchImport.*` + `toast.recipes.batchImport*`
|
||||||
|
(fr/de/es/ru/he/ja/ko), `banners.communitySupport.*`, `modals.model.license.*`,
|
||||||
|
`globalContextMenu.fetchMissingLicenses.*`, the `doctor.*` issue/action/label subset,
|
||||||
|
`toast.settings.libraryLoadFailed` / `libraryActivateFailed`, `toast.api.moveFailed`,
|
||||||
|
`settings.extraFolderPaths.restartRequired`, `toast.recipes.recipeSaved`,
|
||||||
|
`sidebar.dragDrop.moveUnsupported`, `checkpoints.modelTypes.diffusion_model`
|
||||||
|
(ja/ko keep the English loanword), `initialization.recipes.title`.
|
||||||
|
|
||||||
|
The only values that remain intentionally identical to `en.json` are non-translatable:
|
||||||
|
URL/path placeholders (`https://…`, `C:/…`), numeric presets (`5 (1080p), 6 (2K), 8 (4K)`),
|
||||||
|
example token lists (`character, concept, style(toon|toon_style)`), service/provider names
|
||||||
|
(`CivitAI → CivArchive → Archive DB`), and the external playlist title
|
||||||
|
(`help.updateVlogs.playlistTitle`, de: translated to "LoRA Manager-Update-Playlist").
|
||||||
|
|
||||||
|
Rule for `uiHelpers.workflow.noPromptTargets`: the second line (`Mark as → Send Prompt
|
||||||
|
Target`) quotes literal ComfyUI context-menu items — keep those menu labels in English in
|
||||||
|
every locale because that is what the user actually sees in ComfyUI.
|
||||||
|
|
||||||
|
License labels (`modals.model.license.*`): the restriction labels are now translated in all
|
||||||
|
locales (the sibling `creditRequired` has always been translated).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Workflow for agents and translators
|
||||||
|
|
||||||
|
### Adding a new UI string
|
||||||
|
1. Add the key to `locales/en.json` only.
|
||||||
|
2. Run `python scripts/sync_translation_keys.py` — it inserts the key into the other 9
|
||||||
|
locales (as a `[TODO: Translate]` placeholder) preserving formatting.
|
||||||
|
3. **During feature development, stop here.** While the UI copy is still in flux, leave the
|
||||||
|
`[TODO: Translate]` placeholders as-is — translating churning strings into 9 locales is
|
||||||
|
wasted work. Placeholders are a normal intermediate state, not a bug.
|
||||||
|
4. Once the wording is final and the feature owner explicitly asks for translations,
|
||||||
|
translate **all** pending `[TODO: Translate]` keys in every locale (not just the latest
|
||||||
|
feature's), applying §1–§3 (placeholders verbatim, Recipe rule, term maps, register).
|
||||||
|
Find pending keys with: `grep -c "TODO: Translate" locales/*.json`
|
||||||
|
5. If the new string contains new terminology, extend §2 tables.
|
||||||
|
|
||||||
|
### Fixing a translation bug
|
||||||
|
1. Locate the key (dotted path) in the relevant locale file.
|
||||||
|
2. Check the corresponding `en.json` value and the actual caller (grep `static/js` or
|
||||||
|
`web/comfyui` for the key) to learn which placeholders are passed.
|
||||||
|
3. Fix trivially; for normalization sweeps (e.g. "recette" → "Recipe"), do it file-wide for
|
||||||
|
the offending keys only — do not touch unrelated lines.
|
||||||
|
4. If the bug is in `en.json` itself (R9), fix the source first, then re-sync and update all
|
||||||
|
locales.
|
||||||
|
|
||||||
|
### Verification
|
||||||
|
```bash
|
||||||
|
pytest tests/i18n/test_i18n.py # key parity + JSON validity + JS key references
|
||||||
|
python scripts/sync_translation_keys.py --dry-run # shows which keys would change; add --verbose for per-key detail
|
||||||
|
npm test # frontend tests incl. i18n helpers
|
||||||
|
```
|
||||||
|
|
||||||
|
`pytest tests/i18n` only checks structure. Quality conventions in this document are not
|
||||||
|
machine-enforced — a human/agent review pass is required.
|
||||||
|
|
||||||
|
### Anti-patterns checklist
|
||||||
|
- [ ] Placeholders `{x}` / `{{x}}` differ from `en.json`
|
||||||
|
- [ ] Same source term translated 2+ ways in the same file (see §5)
|
||||||
|
- [ ] "Checkpoint" became a literal checkpoint; "recipe" became menu/prescription/food-cookbook
|
||||||
|
- [ ] Brand names translated or transliterated (LoRA, CivitAI, ComfyUI, …)
|
||||||
|
- [ ] Latin locale using full-width `:()`; fr using `"` as apostrophe
|
||||||
|
- [ ] Mixed 你/您, du/Sie, tú/usted
|
||||||
|
- [ ] Full English sentences left behind (see §6)
|
||||||
|
- [ ] Register/typos/mojibake; source string is stale vs `en.json` (compare semantics, not
|
||||||
|
just words)
|
||||||
@@ -39,6 +39,7 @@ These fields are present in all model metadata files.
|
|||||||
| `metadata_source` | string\|null | ❌ No | ✅ Yes | Last provider that supplied metadata (see below) |
|
| `metadata_source` | string\|null | ❌ No | ✅ Yes | Last provider that supplied metadata (see below) |
|
||||||
| `last_checked_at` | float | ❌ No (default: `0`) | ✅ Yes | Unix timestamp of last metadata check |
|
| `last_checked_at` | float | ❌ No (default: `0`) | ✅ Yes | Unix timestamp of last metadata check |
|
||||||
| `hash_status` | string | ❌ No (default: `"completed"`) | ✅ Yes | Hash calculation status: `"pending"`, `"calculating"`, `"completed"`, `"failed"` |
|
| `hash_status` | string | ❌ No (default: `"completed"`) | ✅ Yes | Hash calculation status: `"pending"`, `"calculating"`, `"completed"`, `"failed"` |
|
||||||
|
| `autov3` | string\|null | ❌ No | ✅ Yes | CivitAI AutoV3 hash (first 12 chars, lowercase hex) sourced from the safetensors embedded metadata (`sshs_model_hash` / `modelspec.hash_sha256`). **Absent** = not yet checked (may be backfilled later); **`null`** = checked but unavailable (header has no recognized hash); **12-char hex string** = value |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -287,6 +288,7 @@ These fields are automatically synchronized with the filesystem:
|
|||||||
- `preview_url` — Updated if preview file is moved/removed
|
- `preview_url` — Updated if preview file is moved/removed
|
||||||
- `sha256` — Updated during hash calculation (when `hash_status="pending"`)
|
- `sha256` — Updated during hash calculation (when `hash_status="pending"`)
|
||||||
- `hash_status` — Updated during hash calculation
|
- `hash_status` — Updated during hash calculation
|
||||||
|
- `autov3` — Set when metadata is first created (from safetensors header); may be backfilled later for entries where it is absent
|
||||||
- `last_checked_at` — Timestamp of scan
|
- `last_checked_at` — Timestamp of scan
|
||||||
- `metadata_source` — Set based on metadata provider
|
- `metadata_source` — Set based on metadata provider
|
||||||
|
|
||||||
@@ -345,6 +347,7 @@ These fields can be edited by users at any time through the Lora Manager UI or b
|
|||||||
| `metadata_source` | `null` |
|
| `metadata_source` | `null` |
|
||||||
| `last_checked_at` | `0` |
|
| `last_checked_at` | `0` |
|
||||||
| `hash_status` | `"completed"` |
|
| `hash_status` | `"completed"` |
|
||||||
|
| `autov3` | absent (not checked) or `null` (checked, no value) |
|
||||||
| `usage_tips` | `"{}"` (LoRA only) |
|
| `usage_tips` | `"{}"` (LoRA only) |
|
||||||
| `model_type` | `"checkpoint"` or `"embedding"` (not present in LoRA models) |
|
| `model_type` | `"checkpoint"` or `"embedding"` (not present in LoRA models) |
|
||||||
|
|
||||||
@@ -354,6 +357,7 @@ These fields can be edited by users at any time through the Lora Manager UI or b
|
|||||||
|
|
||||||
| Version | Date | Changes |
|
| Version | Date | Changes |
|
||||||
|---------|------|---------|
|
|---------|------|---------|
|
||||||
|
| 1.1 | 2026-08 | Added `autov3` field (CivitAI AutoV3 hash with three-state semantics) |
|
||||||
| 1.0 | 2026-03 | Initial schema documentation |
|
| 1.0 | 2026-03 | Initial schema documentation |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
@@ -0,0 +1,206 @@
|
|||||||
|
# Plan: Multi-File Downloads Within a Single CivitAI Model Version
|
||||||
|
|
||||||
|
**Issue:** [#1058 — Cannot download multiple file variants from the same model version](https://github.com/willmiao/ComfyUI-Lora-Manager/issues/1058)
|
||||||
|
**Status:** v2 — revised after adversarial review (backend correctness + frontend/tests)
|
||||||
|
**Scope:** CivitAI/CivArchive downloads of `lora`, `checkpoint`, `embedding` model types. HuggingFace downloads are out of scope (already per-file).
|
||||||
|
|
||||||
|
> v2 changelog: incorporated 18 review findings. Key changes vs v1:
|
||||||
|
> shared file resolver + `resolved_version_id` for the gate (R1); `file_params` normalization at API boundary (R2); D2 hash-matching rule fixed for empty-hash cases (R6/R7); D3 extended to re-point `version_index` on removal (R4); D4 replaced with a child table (R3); `delete_model_version` interaction documented (R5); `ModelVersionsTab` surface added to phase 2 (F6); phase-2 multi-file loop requires a reload-deferred download variant (F7); queue-retry `file_params=NULL` known issue recorded (R9); test-fixture gaps and revised estimates (F10).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Problem Statement
|
||||||
|
|
||||||
|
A CivitAI model version can contain multiple downloadable weight files (e.g. fp16/fp32, safetensors/ckpt, different sizes). LoRA Manager already has a working file-selection pipeline (frontend file dialog → `fileParams` → backend file matching), but downloaded state is tracked at the **model-version** level. After any single file of a version is downloaded:
|
||||||
|
|
||||||
|
1. The version is marked **In Library** and the file-selection entry point disappears.
|
||||||
|
2. The backend rejects further download attempts for that version.
|
||||||
|
|
||||||
|
There is no way to download the remaining files of the same version through LoRA Manager.
|
||||||
|
|
||||||
|
## 2. Current State (verified against code; all references confirmed by review)
|
||||||
|
|
||||||
|
### 2.1 Download gating — backend (`py/services/download_manager.py`)
|
||||||
|
|
||||||
|
`_execute_original_download` enforces two version-level gates:
|
||||||
|
|
||||||
|
- **Library gate, early** (lines 1157–1184, before metadata fetch, fires when `model_version_id` given) and **late** (lines 1350–1376, fires only when `model_version_id is None`): `scanner.check_model_version_exists(version_id)` across lora/checkpoint/embedding scanners → hard error `"Model version already exists in ... library"`.
|
||||||
|
- **History gate** (lines 1238–1279): when `skip_previously_downloaded_model_versions` setting is on, `_has_been_downloaded(model_type, version_id)` → silent skip. History DB primary key is `(model_type, version_id)` (`py/services/downloaded_version_history_service.py:61`).
|
||||||
|
|
||||||
|
File selection works: `file_params {id, type, format, size, fp}` is matched against `version_info.files` (lines 1498–1569), **but only under `if file_params and model_version_id:` (line 1499)** — with `model_id`-only requests the selection silently falls back to the primary file (1571–1619). `file_params` currently carries no file `name` or hash.
|
||||||
|
|
||||||
|
### 2.2 Downloaded-state surfacing — backend (`py/routes/handlers/model_handlers.py`)
|
||||||
|
|
||||||
|
`get_civitai_versions` (lines 2148–2188) sets per-version `existsLocally` via `cache.version_index.get(version_id)` (plus a single `localPath` from that entry) and `hasBeenDownloaded` via the history service. No per-file granularity.
|
||||||
|
|
||||||
|
### 2.3 Frontend blockers (`static/js/managers/DownloadManager.js`)
|
||||||
|
|
||||||
|
Three independent gates prevent re-entering the file dialog:
|
||||||
|
|
||||||
|
1. **Line 598:** file-select badge rendered only when `modelFiles.length > 1 && !existsLocally`.
|
||||||
|
2. **Lines 666–681 (`updateNextButtonState`):** Next button disabled with "Already in Library" when `currentVersion.existsLocally`.
|
||||||
|
3. **Lines 784–787 (`proceedToLocation`):** toast + abort when `currentVersion.existsLocally`.
|
||||||
|
|
||||||
|
The badge path (`confirmFileSelection` lines 737–759 → `proceedToLocationContent` → `startDownload` single mode → `executeDownloadWithProgress` → POST `file_params`, `static/js/api/baseModelApi.js:1236–1250`) has **zero** `existsLocally` guards (all 12 occurrences enumerated; none on this path; `import/DownloadManager.js` has none either). The `.exists-locally` CSS class is purely visual (`download-modal.css:496–499`). **Making the badge visible again is sufficient to unlock the flow** for phase 1.
|
||||||
|
|
||||||
|
Post-download refresh is clean: the modal closes and `resetAndReload(true)` performs a full library refetch (`DownloadManager.js:1063`); dialog reopen resets state and refetches versions with no client-side cache. No same-session staleness.
|
||||||
|
|
||||||
|
### 2.4 Local identity of the downloaded file
|
||||||
|
|
||||||
|
`LoraMetadata/CheckpointMetadata/EmbeddingMetadata.from_civitai_info(version_info, file_info, ...)` (`py/utils/models.py:245–369`) persists:
|
||||||
|
|
||||||
|
- `sha256` = `file_info.hashes.SHA256` (lowercased, defaults to `""`) — a stable per-file identity;
|
||||||
|
- `civitai` = the full `version_info` payload (including the `files` list).
|
||||||
|
|
||||||
|
Metadata refresh (`metadata_sync_service.py:104–105`) replaces the `civitai` blob wholesale but never overwrites top-level `sha256`; `verify_duplicate_hashes` (481–526) corrects it to the on-disk hash. Top-level-sha256 matching is refresh-robust.
|
||||||
|
|
||||||
|
**Caveats (review R6/R7):**
|
||||||
|
- SHA256 is not guaranteed: CivArchive's transform only sets `hashes` when source data carries it (`civarchive_client.py:185–189`); `from_civitai_info` defaults to `""`.
|
||||||
|
- Name fallback is unreliable exactly when it matters: local `file_name` is extension-less (`models.py:264`) and `generate_unique_filename` rewrites it with a hash suffix on conflict (`download_manager.py:1125–1136`); checkpoints with `hash_status='pending'` keep empty sha256 until on-demand hashing (`model_scanner.py:1232–1240`).
|
||||||
|
|
||||||
|
### 2.5 Version index collision (pre-existing hazard)
|
||||||
|
|
||||||
|
`ModelCache.version_index` is single-valued (`model_cache.py:133`: `version_index[version_id] = item`). Two files of the same version in the library → second entry overwrites the first; `remove_from_version_index` (lines 151–181) drops the whole version key when the indexed entry is removed, even if a sibling file remains. ~10 read sites depend on this index (48 grep touch points total; readers include `recipe_scanner.py:2682–2726`, `recipe_format.py:37–40`, `misc_handlers.py:2440–2444`, `model_handlers.py`, `model_scanner.check_model_version_exists:2444`).
|
||||||
|
|
||||||
|
Review correction (F3): bulk paths `remove_models` (`model_scanner.py:2376`) and `update_single_model_cache` (`:1689`) call `rebuild_version_index()` right after, so a sibling re-enters the index in those flows — the hazard is narrower than v1 stated, but direct `remove_from_version_index` callers (e.g. `model_scanner.py:1018`) still drop the key, and the user-visible artifact in phase 1 is real: `localPath` in the dialog flips to whichever file was indexed last.
|
||||||
|
|
||||||
|
### 2.6 Entry points that send / don't send `file_params` (fully enumerated by review)
|
||||||
|
|
||||||
|
**Send `file_params` (user-initiated dialog flows only):** `DownloadManager.js:1611–1639` (single mode). API surface accepting arbitrary JSON `file_params`: GET `/api/lm/download-model-get` (`model_handlers.py:1634–1686`), POST `/api/lm/downloads/queue/add` (`model_handlers.py:1799–1832`).
|
||||||
|
|
||||||
|
**Never send `file_params` (keep version-level semantics):** batch download (`DownloadManager.js:1756–1766`; batch also filters out in-library versions at `:1648`), `downloadVersionWithDefaults` (`:1810–1830`), recipe import (`import/DownloadManager.js:269–276`), bulk missing-LoRA (`BulkMissingLoraDownloadManager.js:292–299`), `RecipeModal.js:1728–1736`, `ModelVersionsTab.js:1427`. `web/comfyui/` and `vue-widgets/src` contain **no** download triggers at all (grep-verified). `py/services/use_cases/` has only `download_model_use_case.py` (pass-through).
|
||||||
|
|
||||||
|
### 2.7 Paths that do NOT need changes (verified)
|
||||||
|
|
||||||
|
- **aria2 pause/resume** (`_resume_restored_aria2_download`, line 754+): resumes from persisted `resume_context`; never re-runs existence gates.
|
||||||
|
- **`download_coordinator.py:90`**: pure pass-through of `file_params`.
|
||||||
|
- **Update checker / plugin self-update** (`update_routes.py:496–501`): only closes the history DB handle.
|
||||||
|
- **History delete semantics**: `mark_as_deleted` sets `is_deleted_override=1` and `has_been_downloaded` then returns False (`downloaded_version_history_service.py:276`) — LM-initiated deletes already reset the history skip.
|
||||||
|
|
||||||
|
### 2.8 Related pre-existing issues (record, not necessarily fix)
|
||||||
|
|
||||||
|
- **Queue retry drops file selection** (R9): `download_queue_service.retry_from_history` / `retry_all_failed` re-queue with `file_params=NULL` (`download_queue_service.py:705, 758`) although the queue table has a `file_params` column (`:43`) — a retried non-primary download silently reverts to the primary file. Fix alongside phase 1 (small: persist and reuse the column).
|
||||||
|
- **`delete_model_version`** (`misc_handlers.py:2410–2487`): resolves the file via the single-valued `version_index` (2440–2444), deletes only that one file, and `mark_as_deleted` flags the **entire version** as deleted in history (2479) even when a sibling file remains in the library. See phase 2 item 6.1.5.
|
||||||
|
|
||||||
|
## 3. Goals / Non-Goals
|
||||||
|
|
||||||
|
**Goals**
|
||||||
|
|
||||||
|
- G1: A user can download any not-yet-downloaded file of a version already partially in the library (issue repro steps 6–8).
|
||||||
|
- G2: True duplicates stay blocked: downloading the *same* file of the same version twice is rejected.
|
||||||
|
- G3: Per-file downloaded state visible in the file dialog; multiple files selectable and downloadable in one pass.
|
||||||
|
- G4: No regression for version-level semantics relied on by batch download, recipe missing-LoRA detection, and `skip_previously_downloaded_model_versions`.
|
||||||
|
|
||||||
|
**Non-Goals**
|
||||||
|
|
||||||
|
- No change to recipe `inLibrary` semantics ("any file of the version present" remains sufficient).
|
||||||
|
- No change to the update-checker (version-level comparison).
|
||||||
|
- No primary-key rebuild of the history database.
|
||||||
|
- HuggingFace download flow untouched.
|
||||||
|
|
||||||
|
## 4. Design Decisions
|
||||||
|
|
||||||
|
- **D1 — Explicit file selection bypasses the history gate, version-level gates stay for everyone else.** The history skip exists to dedupe automated flows. A user explicitly picking a file is unambiguous intent; the file-level library gate (G2) still prevents real duplicates. **Guard conditions use normalized truthiness** (see D1a). All confirmed `file_params` senders are user-initiated dialog flows (2.6), and LM-initiated deletes already reset history (2.7), so the bypass only affects "downloaded but not LM-deleted" versions with the setting on — intended.
|
||||||
|
- **D1a — `file_params` normalization at the boundary (R2).** `download-model-get` and `downloads/queue/add` accept arbitrary JSON; `{}` is `not None` but falsy and would bypass gates while downloading the primary file. Normalize `file_params = file_params or None` in the coordinator/handlers, and treat the bypass as active only when a target file id is resolvable.
|
||||||
|
- **D2 — File identity matching rule (R6/R7):** hash-compare **only when both sides are non-empty** (lowercase SHA256 equality); name-compare when either side is empty. Never let `"" == ""` match. Name fallback caveats from 2.4 apply (renamed files, pending checkpoint hashes) — acceptable residual risk, worst case is a blocked re-download the user can retry after hashing completes.
|
||||||
|
- **D3 — Cache indexes: additive multi-index + removal re-pointing (R4).** Add `version_files_index: Dict[int, List[dict]]` maintained alongside `version_index` by the same add/remove/rebuild methods; existing readers of `version_index` untouched. Additionally fix `remove_from_version_index`: when the popped entry has a surviving sibling (per the multi-index), re-point `version_index[version_id]` to the sibling instead of dropping the key; same for the `model_id_index` descriptor. This closes the 2.5 hazard for existing readers (`check_model_version_exists`, `existsLocally`, recipe matching) without restructuring anything.
|
||||||
|
- **D4 — Per-file history via a child table (R3).** v1's additive-column approach is structurally impossible on a `(model_type, version_id)` PK (`ON CONFLICT DO UPDATE` would keep only the last file). Instead add `downloaded_version_files(model_type, version_id, file_id, file_name, downloaded_at, PRIMARY KEY(model_type, version_id, file_id))` — additive, no PK rebuild, honors the Non-Goal. Existing version-level table and queries unchanged. New per-file queries are opt-in. `_initialize_schema` uses `CREATE TABLE IF NOT EXISTS`, so the new table is created for existing DBs without any ALTER.
|
||||||
|
- **D5 — UI flow reuse, with an extracted inner download function for multi-file (F7).** Phase 1 unlocks the existing badge → file dialog → location → download pipeline. Phase 2 upgrades the dialog to multi-select; iterating `executeDownloadWithProgress` as-is would produce N full library reloads, N toasts, and competing failure-summary modals — so phase 2 extracts a reload-deferred, failure-aggregating inner variant and runs one reload + one summary at the end.
|
||||||
|
|
||||||
|
## 5. Implementation — Phase 1 (fix the issue; independently shippable)
|
||||||
|
|
||||||
|
### 5.1 Backend — `py/services/download_manager.py`
|
||||||
|
|
||||||
|
1. **Normalize `file_params`** at the boundary (D1a): `download_coordinator.schedule_download` and the two API handlers (`model_handlers.py:1649–1666`, `1810–1832`) apply `file_params = file_params or None`.
|
||||||
|
2. **Extract a shared file resolver** (R1): pull the matching logic at 1498–1569 into `_resolve_target_file(version_info, file_params) -> Optional[dict]`, used by **both** the new gate and the download-selection path. The selection path's condition (line 1499) switches from `model_version_id` to `resolved_version_id` (already computed at 1230–1236 from `version_info.id`), so gate and download always agree on the target file — including the `model_id`-only case.
|
||||||
|
3. **New helper** `_find_local_file_entry(version_id, target_file) -> Optional[dict]`: iterate the three scanners' cached `raw_data` (NOT `version_index` — single-valued); candidates = entries whose `civitai.id` normalizes to `version_id`; match per D2.
|
||||||
|
4. **Gate restructure in `_execute_original_download`**:
|
||||||
|
- Early scanner gate (1157–1184): add `file_params is None` guard; with normalized `file_params`, defer (file identity not resolvable before metadata fetch).
|
||||||
|
- After `version_info` fetch + `resolved_version_id` (~1229): when `file_params` present, resolve target file via the shared resolver; unresolvable → hard error "No matching file" (fail closed, prevents empty-dict bypass). Resolvable → `_find_local_file_entry`; hit → same hard error shape as today with the file name in the message.
|
||||||
|
- History gate (1238–1279): add `file_params is None` (D1). Base-model skip (1281–1324) unchanged — still applies.
|
||||||
|
- Late gate (1350–1376): add `file_params is None` guard (F2) — the post-fetch file-level check above already covers this case.
|
||||||
|
- Nothing between the early gate and the post-fetch point assumes the version is absent (review task 6: only provider selection + metadata fetch; no DB writes; `_persist_aria2_state` runs only when actually downloading at 1659).
|
||||||
|
5. **Queue retry fix** (2.8, small): persist `file_params` into the queue table on enqueue and reuse it in `retry_from_history` / `retry_all_failed`.
|
||||||
|
6. Logging: `[download]` lines for file-level allow/block, consistent with existing style.
|
||||||
|
|
||||||
|
**Estimated:** ~150–220 LOC + resolver extraction.
|
||||||
|
|
||||||
|
### 5.2 Frontend — `static/js/managers/DownloadManager.js`
|
||||||
|
|
||||||
|
1. Line 598: drop `&& !existsLocally` from the badge condition (badge shows whenever `modelFiles.length > 1`).
|
||||||
|
2. `fileParams` construction (1611–1616): add `name: this.selectedFile.name`.
|
||||||
|
3. Surface the backend "file already in library" hard error as a toast instead of only the batch-summary modal (R10/F12 nit; reuse existing error message field).
|
||||||
|
4. No changes to `updateNextButtonState` / `proceedToLocation` in phase 1; no template or CSS changes.
|
||||||
|
|
||||||
|
**Known phase-1 UX limitations (acknowledged, fixed in phase 2):** with all files downloaded the badge still renders and re-picking a downloaded file fails late (backend error after the location step); `localPath` may point at a sibling file; batch-preview "In Library" badge stays version-level and gives no hint of remaining files.
|
||||||
|
|
||||||
|
**Estimated:** ~10–30 LOC (confirmed realistic by review).
|
||||||
|
|
||||||
|
### 5.3 Phase 1 tests
|
||||||
|
|
||||||
|
Backend — extend `tests/services/test_download_manager_basic.py` (1694 lines; all fixture patterns exist):
|
||||||
|
|
||||||
|
- **Fixture gaps to add (F10):** `DummyScanner.get_cached_data()`/`raw_data` stub (~10 lines); `hashes.SHA256` in the metadata-provider payload's `files`.
|
||||||
|
- Cases: same version + different SHA256 in library + `file_params` → proceeds; same SHA256 → hard error; `file_params=None` + version in library → hard error (unchanged); history-skip on + `file_params` → not skipped; without → skipped (unchanged); empty-dict `file_params` normalized → version-level behavior; `model_id`-only + `file_params` → gate and selection resolve the same file; legacy metadata (empty local sha256) matched by name; target file with empty SHA256 → name fallback, no `""==""` false positive.
|
||||||
|
- Queue retry: `file_params` survives retry.
|
||||||
|
- Assert proceed/abort via the existing `_execute_download` mock pattern.
|
||||||
|
|
||||||
|
Frontend (`tests/frontend/`): badge renders for multi-file version with `existsLocally=true` (pattern from `downloadManager.history.test.js`).
|
||||||
|
|
||||||
|
**Estimated:** ~150–250 LOC (confirmed realistic).
|
||||||
|
|
||||||
|
## 6. Implementation — Phase 2 (per-file status + multi-select + index hardening)
|
||||||
|
|
||||||
|
### 6.1 Backend
|
||||||
|
|
||||||
|
1. **`py/services/model_cache.py`** (D3): add `version_files_index`; maintain in `add_to_version_index` / `remove_from_version_index` / `rebuild_version_index`; removal re-points `version_index[version_id]` (and the `model_id_index` descriptor) to a surviving sibling instead of dropping the key.
|
||||||
|
2. **`py/services/model_scanner.py`**: expose `get_files_for_version(version_id) -> List[dict]`.
|
||||||
|
3. **`py/routes/handlers/model_handlers.py` `get_civitai_versions`**: annotate each version with `downloadedFiles: [{fileId, fileName, filePath}]` via `version_files_index` + D2 matching against `version.files`.
|
||||||
|
4. **`py/services/downloaded_version_history_service.py`** (D4): new child table `downloaded_version_files`; `mark_downloaded` also upserts the child row when `file_id` known; `mark_as_deleted` clears the version's child rows only when no sibling remains in the library; new `get_downloaded_file_ids(model_type, version_id) -> set[int]`. `_record_downloaded_version_history` passes `file_info` through.
|
||||||
|
5. **`delete_model_version`** (`misc_handlers.py:2410–2487`, R5): resolve **all** local files of the version via `version_files_index`; delete all (current endpoint semantics are version-level) or — if kept per-file — only `mark_as_deleted` when no sibling remains. Decide at implementation time; minimum is documenting current behavior.
|
||||||
|
6. **`ModelVersionsTab` backend support**: none needed beyond item 3 (`downloadedFiles`); the tab consumes the same versions payload.
|
||||||
|
|
||||||
|
### 6.2 Frontend
|
||||||
|
|
||||||
|
1. **File dialog multi-select** — change surface (F8): option markup (`DownloadManager.js:712–724`), the single-select click handler (`727–734`), the `input[type="radio"]:checked` selector in `confirmFileSelection` (`738`); template `templates/components/modals/download_modal.html:48–60` (confirm-button label only); CSS `download-modal.css` — checkbox variant of `.file-option-radio input` (595–604) and a **new** `.file-option.disabled` style (does not exist). Files whose id ∈ `downloadedFiles` render disabled with an "In Library" tag.
|
||||||
|
2. **Mixed-type guard (F8):** multi-select is restricted to files sharing the same routing target (`_isDiffusionModel` is computed once from a single `selectedFile` at 798–803; e.g. "Model" + "UNet" files route to different roots). Disallow mixed-type multi-select (simplest, predictable); single-file selection unchanged.
|
||||||
|
3. **Multi-file download loop (D5/F7):** extract from `executeDownloadWithProgress` a reload-deferred, no-toast inner function; iterate per selected file with per-file progress; one `resetAndReload(true)` + one aggregated success/failure summary at the end (reuse `showDownloadBatchSummary`).
|
||||||
|
4. **`updateNextButtonState` / `proceedToLocation`:** for multi-file versions, Next routes into the file dialog; hard block only when *every* weight file is downloaded.
|
||||||
|
5. **`ModelVersionsTab.js` (F6):** the Download action (`:576` hidden when `isInLibrary`) — for multi-file versions with remaining files, show it and route into the download modal's file dialog; keep hidden when all files present.
|
||||||
|
6. **Batch preview (F5):** `batch-preview-local-badge` (`:1320`) gains a "partially downloaded" hint for multi-file versions with remaining files.
|
||||||
|
7. New i18n keys (`modals.download.fileSelection.inLibrary`, `downloadSelected`, partial-download tooltip, etc.) → run `python scripts/sync_translation_keys.py`.
|
||||||
|
|
||||||
|
### 6.3 Phase 2 tests
|
||||||
|
|
||||||
|
- `model_cache` (`tests/services/test_model_cache.py` already covers add/remove at 44–55): multi-valued index; sibling re-point on removal; rebuild.
|
||||||
|
- `get_civitai_versions`: `downloadedFiles` correctness (hash match, name fallback, no match, CivArchive no-hash payload).
|
||||||
|
- History service (`tests/services/test_downloaded_version_history_service.py` uses real SQLite on tmp_path): child-table creation on a legacy DB; per-file record/query; `mark_as_deleted` sibling semantics.
|
||||||
|
- Frontend: dialog checkbox rendering/disabled state and multi-file confirm — **greenfield behavior coverage** (F10: no existing test exercises `showFileSelectionStep`/`confirmFileSelection`; infra exists, patterns must be built).
|
||||||
|
|
||||||
|
## 7. Risks and Mitigations
|
||||||
|
|
||||||
|
| Risk | Impact | Mitigation |
|
||||||
|
|---|---|---|
|
||||||
|
| History-gate bypass (D1) causes unwanted re-downloads in automated flows | Large checkpoint files re-downloaded | Bypass only with normalized, resolvable `file_params` (D1a); all such senders are user-initiated dialog flows (2.6, verified); tests pin batch/recipe/bulk behavior. |
|
||||||
|
| Empty-hash matching edge cases (R6) | Duplicate download of the same file, or false block | D2 rule: hash only when both non-empty; name otherwise; never `""==""`. Residual risk documented (2.4). |
|
||||||
|
| Phase-1 late-failure UX (F12) | User picks a downloaded file, fails only after location step | Toast surfacing (5.2.3); phase 2 disables downloaded files up front. |
|
||||||
|
| Phase-2 index change corrupts existing behavior | Recipe matching, delete flows | Additive index + re-point only; `version_index` read semantics unchanged; `remove_models`/`update_single_model_cache` already rebuild (F3); tests. |
|
||||||
|
| `delete_model_version` marks whole version deleted while sibling remains (R5) | History wrongly suppresses re-download of the surviving sibling's version | Phase 2 item 6.1.5; documented until then. |
|
||||||
|
| History child-table migration failure on user installs | Service init crash | `CREATE TABLE IF NOT EXISTS` in `_initialize_schema`; failure degrades to version-level behavior (per-file queries return empty). |
|
||||||
|
| Batch-preview badge misleading for partial versions (F5) | Minor UX confusion | Acknowledged in phase 1; fixed in phase 2 item 6.2.6. |
|
||||||
|
| UI confusion: version shows "In Library" while files remain downloadable | Support burden | Phase 2: per-file disabled state + partial-download tooltip. |
|
||||||
|
| Hash-identical sibling files (repacked content) | Second file blocked | Acceptable: scanner hash dedup already collapses them. |
|
||||||
|
|
||||||
|
## 8. Rollout
|
||||||
|
|
||||||
|
1. **Commit 1** — `fix(download): allow downloading additional files of an in-library model version (#1058)` → Phase 1 (5.1–5.3).
|
||||||
|
2. **Commit 2** — `feat(download): per-file download status and multi-file selection (#1058)` → Phase 2 (6.1–6.3).
|
||||||
|
|
||||||
|
Phase 1 alone resolves the issue as reported; phase 2 can ship in a later release if review prefers smaller increments.
|
||||||
|
|
||||||
|
## 9. Effort Estimate (revised after review)
|
||||||
|
|
||||||
|
| Phase | Backend | Frontend | Tests | Risk |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| 1 | ~150–220 LOC (+ queue-retry fix ~30) | ~10–30 LOC | ~150–250 LOC | Low |
|
||||||
|
| 2 | ~250–350 LOC | ~250–350 LOC (multi-file loop refactor + ModelVersionsTab + batch badge) | ~250–350 LOC (dialog tests greenfield) | Medium |
|
||||||
@@ -0,0 +1,337 @@
|
|||||||
|
# Plan: Global Rate-Limit Abidance for Recipe Ingest & Metadata Fetching
|
||||||
|
|
||||||
|
**Issue:** [#1085 — Large Recipe Ingest Appears to not abide by vendor rate limits, possibly a few other errors?](https://github.com/willmiao/ComfyUI-Lora-Manager/issues/1085)
|
||||||
|
**Status:** v2 — reviewed; decisions recorded in §10. **Phase 1 implemented**
|
||||||
|
(2026-08-27, commit `c2a2048c`): coordinator + downloader gate + Fix C
|
||||||
|
failover semantics + helper double-wait fix + settings. **Phase 2
|
||||||
|
implemented** (2026-08-27): batch-import rate-limit failures map to
|
||||||
|
`SKIPPED` + `rate_limited` WebSocket flag + UI slowdown hint (toast + status
|
||||||
|
text, i18n keys synced); `download_to_memory` / `get_response_headers` /
|
||||||
|
`download_file` register 429 cooldowns. Changes vs v1: Fix C moved to
|
||||||
|
Phase 1, helper double-wait resolved in Phase 1, gate/guard ordering
|
||||||
|
specified.
|
||||||
|
**Scope:** HTTP API traffic to CivitAI (`civitai.red`) and CivArchive (`civarchive.com`) from metadata fetching (bulk refresh, metadata sync, recipe analysis/enrichment, usage-control lookups). Large binary downloads (model files / preview images via `download_file`) are out of scope for *pacing* (they are already single-connection transfers) but their 429 responses should still be *registered*.
|
||||||
|
|
||||||
|
> Context: a first batch of fixes for this issue was already committed as
|
||||||
|
> `ee233548` ("fix(recipes): enforce batch-import concurrency bound and harden
|
||||||
|
> ingest errors (#1085)"): the batch-import concurrency controller now shares a
|
||||||
|
> real semaphore (bounds 1–5 actually apply), the Comfy parser tolerates
|
||||||
|
> list/`None` `ckpt_name`, CivArchive treats empty error payloads as failures,
|
||||||
|
> and offline-cooldown short-circuits log at DEBUG. This plan covers the two
|
||||||
|
> remaining orchestration-level fixes:
|
||||||
|
> **Fix 2** — slow down globally when a vendor rate limit is hit (respect
|
||||||
|
> `Retry-After`, queue instead of hammering); **Fix 3** — stop immediately
|
||||||
|
> failing over to CivArchive when CivitAI is rate-limited.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Problem Statement
|
||||||
|
|
||||||
|
During a large recipe ingest (e.g. importing the example-images directory,
|
||||||
|
which can be thousands of images), the manager fires one metadata request per
|
||||||
|
checkpoint + per LoRA per image through the fallback provider chain
|
||||||
|
(`civitai_api → civarchive_api → sqlite`). Consequences observed in #1085:
|
||||||
|
|
||||||
|
1. **CivitAI gets hammered** → 429s. The consumer then *immediately* tries
|
||||||
|
CivArchive for the same lookup, so **CivArchive gets hammered too** before
|
||||||
|
it was ever naturally needed (its only real job is recovering metadata for
|
||||||
|
models deleted from CivitAI).
|
||||||
|
2. Requests are retried per-call after `Retry-After`, but **each concurrent
|
||||||
|
call sleeps independently** → thundering herd: thousands of coroutines wake
|
||||||
|
at the same moment and re-flood the vendor.
|
||||||
|
3. While CivArchive is in the `ConnectivityGuard` cooldown, every batch item
|
||||||
|
short-circuits and is marked `FAILED` — the batch import's success/failure
|
||||||
|
accounting is polluted by a transient vendor state (log spam was fixed in
|
||||||
|
`ee233548`; the item-failure accounting is not).
|
||||||
|
4. `ConnectivityGuard` (`py/services/connectivity_guard.py`) only treats
|
||||||
|
transport-level unreachability as offline; **HTTP 429 is invisible to it**,
|
||||||
|
so nothing ever intentionally paces request rate.
|
||||||
|
|
||||||
|
User expectation from the issue: *"once a vendor rate limit time out is hit,
|
||||||
|
you should trigger a slow down with intentional reduction in request rate"*.
|
||||||
|
|
||||||
|
## 2. Current State (verified against code)
|
||||||
|
|
||||||
|
### 2.1 Where 429s are surfaced
|
||||||
|
|
||||||
|
- `Downloader.make_request` (`py/services/downloader.py:1120-1132`): HTTP 429 →
|
||||||
|
returns `RateLimitError(message, retry_after=…)` parsed from `Retry-After`
|
||||||
|
(missing header defaults to `None`).
|
||||||
|
- `CivitaiClient._make_request` (`py/services/civitai_client.py:97-100`):
|
||||||
|
converts `RateLimitError` to a raise immediately; no waiting. Transient
|
||||||
|
5xx/connection errors are retried 3× with 1s/2s/4s backoff.
|
||||||
|
- `CivArchiveClient._make_request` (`py/services/civarchive_client.py`):
|
||||||
|
raises `RateLimitError` with `provider="civarchive_api"` when not set.
|
||||||
|
- `_RateLimitRetryHelper` (`py/services/model_metadata_provider.py:45-102`):
|
||||||
|
per-call retry loop — sleeps `retry_after` (capped at 1800 s; `≥120 s` ⇒ no
|
||||||
|
retry), then re-raises. Because every concurrent call runs its own helper,
|
||||||
|
they sleep in parallel and re-fire in parallel.
|
||||||
|
- `FallbackMetadataProvider` (`py/services/model_metadata_provider.py:488-508,
|
||||||
|
564-584` etc.): on a final `RateLimitError` from one provider it logs
|
||||||
|
"skipping to next provider" and **continues to the next network provider** —
|
||||||
|
this is the direct cause of the CivArchive flood.
|
||||||
|
- `MetadataSyncService.fetch_and_update_model`
|
||||||
|
(`py/services/metadata_sync_service.py:248-333`): manually iterates
|
||||||
|
`provider_attempts`; on `RateLimitError` it `continue`s to the next provider
|
||||||
|
(same failover problem), then reports `"Rate limited"` when nothing
|
||||||
|
succeeded.
|
||||||
|
- `Downloader.make_request` has a per-destination scope already available:
|
||||||
|
`_guard_destination(url)` returns the hostname (`downloader.py:1194-1199`),
|
||||||
|
used by `ConnectivityGuard`.
|
||||||
|
|
||||||
|
### 2.2 What pacing exists today
|
||||||
|
|
||||||
|
- `ConnectivityGuard`: per-destination cooldown (30 s base, ×2 per extra
|
||||||
|
failure batch, 300 s cap) triggered only by transport errors
|
||||||
|
(`connectivity_guard.py:168-197`).
|
||||||
|
- `AdaptiveConcurrencyController` (batch import, fixed in `ee233548`): shared
|
||||||
|
semaphore enforces 1–5 concurrent items; *duration*-based adjustment only —
|
||||||
|
it never sees HTTP statuses, so it cannot distinguish "slow because rate
|
||||||
|
limited" from "slow because big image".
|
||||||
|
- No token bucket, no minimum inter-request interval, no shared
|
||||||
|
`Retry-After` gate anywhere (`grep` for throttle/token-bucket/rate-limiter:
|
||||||
|
0 hits).
|
||||||
|
|
||||||
|
## 3. Requirements & Constraints
|
||||||
|
|
||||||
|
R1. **Respect `Retry-After`.** After a 429, no further request to that
|
||||||
|
destination may be sent before the vendor's retry window elapses.
|
||||||
|
R2. **No thundering herd.** Concurrent waiters must share one wake-up (gate),
|
||||||
|
not sleep independently.
|
||||||
|
R3. **No double load.** A CivitAI 429 must not trigger a CivArchive request
|
||||||
|
for the same lookup. CivArchive should only be consulted when CivitAI
|
||||||
|
legitimately has no answer (404 / "not found"), or when CivitAI is
|
||||||
|
unreachable long-term.
|
||||||
|
R4. **No spurious item failures.** A rate-limited request must not turn a
|
||||||
|
batch-import item into `FAILED`; it should wait (bounded) and retry, or at
|
||||||
|
worst be `SKIPPED` with a clear "rate limited" reason (re-runnable import).
|
||||||
|
R5. **Never hang forever.** All waiting is bounded by a configurable cap; on
|
||||||
|
expiry the caller receives the `RateLimitError` and can decide.
|
||||||
|
R6. **Keep legitimate failover.** Deleted-model recovery via CivArchive/sqlite
|
||||||
|
must keep working (404 paths unchanged).
|
||||||
|
R7. **Single choke point.** The pacing gate should live where every API call
|
||||||
|
passes (the `Downloader`), so bulk refresh, metadata sync, recipe
|
||||||
|
analysis, and usage-control lookups all benefit without per-feature work.
|
||||||
|
|
||||||
|
## 4. Approach Comparison
|
||||||
|
|
||||||
|
### A. Reactive gate — shared `Retry-After` deadman clock (recommended core)
|
||||||
|
|
||||||
|
A process-wide, per-destination coordinator records the *next-allowed-send*
|
||||||
|
timestamp from each 429 (`now + max(retry_after, backoff)`). Every request
|
||||||
|
through `Downloader.make_request` consults the gate *before sending* and *when
|
||||||
|
a 429 arrives*; waiters block on a shared `asyncio.Event` that fires when the
|
||||||
|
cooldown expires.
|
||||||
|
|
||||||
|
- Pros: single choke point (R7); herd-free (R2); honors server guidance (R1);
|
||||||
|
no guessing at vendor limits; covers all providers automatically; reuses
|
||||||
|
existing per-destination scoping.
|
||||||
|
- Cons: still experiences 429s before slowing down (reactive); long
|
||||||
|
`Retry-After` windows (CivArchive has been observed at ~1500 s) need a sane
|
||||||
|
wait cap + skip/retry UX.
|
||||||
|
|
||||||
|
### B. Preemptive pacing — minimum inter-request interval (recommended companion)
|
||||||
|
|
||||||
|
Per-destination token bucket (simplest form: capacity 1 — at least `N` seconds
|
||||||
|
between consecutive API requests; `N` configurable, default ~0.75 s ≈ 80
|
||||||
|
r/min ceiling).
|
||||||
|
|
||||||
|
- Pros: prevents most 429s before they happen — exactly the "intentional
|
||||||
|
reduction in request rate" the issue asks for; trivial to implement on top
|
||||||
|
of A's coordinator.
|
||||||
|
- Cons: adds latency to bulk operations (thousands of models × `N`); the *exact*
|
||||||
|
vendor limits are unknown (CivitAI anonymous vs keyed vs `civitai.red`
|
||||||
|
mirror differ), so the default must be conservative-but-not-crippling and
|
||||||
|
settings-tunable.
|
||||||
|
|
||||||
|
### C. Fallback semantics change — stop network→network failover on 429 (must-do, low risk)
|
||||||
|
|
||||||
|
`FallbackMetadataProvider` (and `MetadataSyncService.fetch_and_update_model`'s
|
||||||
|
manual loop) must treat a final `RateLimitError` as a **terminal, non-failover
|
||||||
|
result** for network providers. Local-only providers (sqlite archive DB) may
|
||||||
|
stay as a last resort (no vendor cost).
|
||||||
|
|
||||||
|
- Pros: directly removes the CivArchive flood; small, surgical change.
|
||||||
|
- Cons: none significant; requires care to keep 404-failover intact (R6).
|
||||||
|
|
||||||
|
### Rejected / deferred
|
||||||
|
|
||||||
|
- **Per-feature retry queues** (batch import pauses & resumes whole batches):
|
||||||
|
richer UX but much larger change (batch state machine, WebSocket states);
|
||||||
|
unnecessary once A+B make requests wait at the choke point. Defer unless
|
||||||
|
review finds the bounded-wait UX insufficient.
|
||||||
|
- **Full token bucket with burst credit**: overkill; capacity-1 interval is
|
||||||
|
enough given the shared semaphore already caps concurrency at 5.
|
||||||
|
- **Retrying in `connectivity_guard`**: wrong layer — the guard is about
|
||||||
|
transport reachability, not vendor quota.
|
||||||
|
|
||||||
|
## 5. Recommended Architecture
|
||||||
|
|
||||||
|
New singleton **`RateLimitCoordinator`** (`py/services/rate_limit_coordinator.py`,
|
||||||
|
mirroring `ConnectivityGuard`'s singleton + per-destination patterns):
|
||||||
|
|
||||||
|
```
|
||||||
|
state per destination (hostname):
|
||||||
|
next_allowed_send: float (monotonic) # from 429 Retry-After + backoff
|
||||||
|
consecutive_429: int # for backoff growth
|
||||||
|
last_send_at: float # for min-interval pacing
|
||||||
|
waiters: list[Future] | asyncio.Event # shared wake-up per cooldown cycle
|
||||||
|
```
|
||||||
|
|
||||||
|
API:
|
||||||
|
|
||||||
|
- `async wait_for_slot(destination, request_started_within_window: bool)`
|
||||||
|
— called by `Downloader.make_request` *before* sending (blocks until
|
||||||
|
`min(now >= next_allowed_send)` and inter-request interval elapses) and
|
||||||
|
re-armable after a 429.
|
||||||
|
- `register_rate_limit(destination, retry_after: float | None)`
|
||||||
|
— called on 429: `next_allowed_send = max(now + retry_after_or_backoff, current)`;
|
||||||
|
`consecutive_429 += 1`; backoff = `retry_after` honored, else exponential
|
||||||
|
`30 · 2^(n-1)` capped at 1800 s; creates/re-arms the shared wake-up event.
|
||||||
|
- `register_success(destination)` — resets `consecutive_429` (called from the
|
||||||
|
existing 200 path in `make_request`).
|
||||||
|
- `remaining_seconds(destination)`, `in_cooldown(destination)` — for tests and
|
||||||
|
diagnostics.
|
||||||
|
|
||||||
|
Enforcement points:
|
||||||
|
|
||||||
|
1. **`Downloader.make_request`** (`downloader.py:1102-1132`): ordering inside
|
||||||
|
the method is **connectivity-guard fail-fast first** (offline short-circuit
|
||||||
|
costs nothing to check), **then** `await coordinator.wait_for_slot(destination)`
|
||||||
|
before `session.request`. On 429: `coordinator.register_rate_limit(...)`,
|
||||||
|
then *wait for the gate and re-send* (loop, bounded by
|
||||||
|
`rate_limit_max_wait_seconds`, default 300; `retry_after ≥ cap` ⇒ fail
|
||||||
|
immediately). After the loop, return the `RateLimitError` to the caller
|
||||||
|
(unchanged contract) **with `exc.gate_handled = True` set** so downstream
|
||||||
|
retry helpers know the wait already happened. 200 path calls
|
||||||
|
`register_success`.
|
||||||
|
2. **`Downloader.download_to_memory` / `get_response_headers`** (phase 2):
|
||||||
|
register 429s (so API calls queue); waiting only in `make_request`
|
||||||
|
initially.
|
||||||
|
3. **`FallbackMetadataProvider`** (`model_metadata_provider.py`): remove
|
||||||
|
network→network failover on `RateLimitError` — re-raise; only sqlite stays
|
||||||
|
as a local last resort (implementation: per-method `except RateLimitError`
|
||||||
|
handler that marks the chain rate-limited and stops iterating).
|
||||||
|
4. **`MetadataSyncService.fetch_and_update_model`**
|
||||||
|
(`metadata_sync_service.py:248-333`): on `RateLimitError` from the default
|
||||||
|
provider, stop appending further network providers (sqlite may remain);
|
||||||
|
the existing `any_rate_limited` merge already produces `"Rate limited"`.
|
||||||
|
5. **Batch import** (`batch_import_service.py`): no structural change needed —
|
||||||
|
items now wait inside `make_request`; optionally (phase 2) map residual
|
||||||
|
rate-limit failures (after the wait cap) to `SKIPPED` with
|
||||||
|
`"rate limited (retry_after=…s); re-run the import later"` instead of
|
||||||
|
`FAILED`, and surface a `rate_limited` flag in the WebSocket progress
|
||||||
|
broadcast.
|
||||||
|
6. **`_RateLimitRetryHelper` retries** (`model_metadata_provider.py`):
|
||||||
|
**Phase 1** — when the raised `RateLimitError` carries `gate_handled = True`
|
||||||
|
(set by the downloader after honoring the gate), the helper skips its own
|
||||||
|
`retry_after` sleep and re-raises immediately, eliminating the double wait.
|
||||||
|
The wiring stays so a `RateLimitError` still propagates cleanly; full
|
||||||
|
demotion/removal can follow once the gate proves out.
|
||||||
|
|
||||||
|
Settings (`settings.json`, schema extension in `SettingsManager`):
|
||||||
|
|
||||||
|
| key | default | meaning |
|
||||||
|
|---|---|---|
|
||||||
|
| `rate_limit_gate_enabled` | `true` | master switch for the coordinator |
|
||||||
|
| `rate_limit_max_wait_seconds` | `300` | how long `make_request` waits on a 429 gate before returning the error |
|
||||||
|
| `rate_limit_min_interval_seconds` | `0.75` | minimum seconds between API requests per destination (pacing, R6-friendly conservative default) |
|
||||||
|
|
||||||
|
## 6. Changes by File
|
||||||
|
|
||||||
|
| File | Change |
|
||||||
|
|---|---|
|
||||||
|
| `py/services/rate_limit_coordinator.py` (new) | coordinator singleton + per-destination state + tests seam |
|
||||||
|
| `py/services/downloader.py` | gate pre-check + 429 register/wait/retry loop + `register_success`; log the 429 notice at INFO once per cooldown, then DEBUG |
|
||||||
|
| `py/services/model_metadata_provider.py` | `FallbackMetadataProvider`: stop network failover on `RateLimitError`; helper skips its sleep when the error is marked `gate_handled` |
|
||||||
|
| `py/services/metadata_sync_service.py` | `fetch_and_update_model`: same failover semantics; keep sqlite last resort |
|
||||||
|
| `py/services/batch_import_service.py` | (phase 2) rate-limit failures → `SKIPPED` + `rate_limited` progress flag |
|
||||||
|
| `py/services/settings_manager.py` | new settings keys + defaults |
|
||||||
|
| `tests/services/test_rate_limit_coordinator.py` (new) | gate unit tests |
|
||||||
|
| `tests/services/test_civitai_client.py` / `test_civarchive_client.py` | provider-level 429 behavior |
|
||||||
|
| `tests/services/test_metadata_service.py` | failover-chain tests |
|
||||||
|
| `tests/services/test_batch_import_service.py` | SKIPPED-on-rate-limit |
|
||||||
|
|
||||||
|
## 7. Impact, Risks, Open Questions
|
||||||
|
|
||||||
|
- **Behavior change**: with the gate in `make_request`, any request can block
|
||||||
|
up to the wait cap — UI actions that call the API (e.g. a model-details
|
||||||
|
fetch) may take longer during cooldowns. Mitigation: bounded cap + INFO log
|
||||||
|
+ the existing async request handling already tolerates slow responses.
|
||||||
|
**Decided (§10): interactive requests take the same bounded wait** — one
|
||||||
|
behavior, no call-source plumbing; cooldowns are usually short.
|
||||||
|
- **Gate waits occupy batch slots**: with the 1–5 batch semaphore, all slots
|
||||||
|
can park on a gate simultaneously, freezing visible progress for up to one
|
||||||
|
wait cap per wave. Bounded and acceptable; the phase-2 `SKIPPED` mapping +
|
||||||
|
WebSocket `rate_limited` flag (both confirmed in scope, §10) make the stall
|
||||||
|
visible and recoverable.
|
||||||
|
- **Rate limit reality check**: CivitAI anonymous vs keyed limits, and whether
|
||||||
|
`civitai.red` differs, is unverified. Default pacing `0.75 s/req` is a
|
||||||
|
conservative guess (R6). Open question for maintainer: preferred default
|
||||||
|
and whether an API-keyed ceiling should be higher.
|
||||||
|
- **Long CivArchive windows**: `Retry-After ~1500 s` observed in code
|
||||||
|
comments. **Decided (§10): keep the 300 s default cap** — such lookups
|
||||||
|
fail/skip rather than park a request path for 25 minutes; batch import maps
|
||||||
|
them to `SKIPPED` (phase 2) so the user can re-run later.
|
||||||
|
- **Double waiting**: `_RateLimitRetryHelper` + gate could stack waits.
|
||||||
|
**Resolved in Phase 1**: the downloader marks gate-honored errors with
|
||||||
|
`gate_handled = True` and the helper skips its own sleep for those.
|
||||||
|
- **Downloads**: `download_file` 429s return an error to download managers
|
||||||
|
unchanged (already handled); only *registration* is proposed, so future
|
||||||
|
API calls queue behind a large `Retry-After` from a download burst.
|
||||||
|
|
||||||
|
## 8. Test Plan
|
||||||
|
|
||||||
|
1. **Coordinator unit tests** (new file):
|
||||||
|
- 429 with `retry_after` → `wait_for_slot` blocks ~that long, then passes.
|
||||||
|
- N concurrent waiters all wake together (herd test, wall-clock ≈ one
|
||||||
|
window, not N windows).
|
||||||
|
- Consecutive 429s grow backoff; `register_success` resets.
|
||||||
|
- Missing `Retry-After` → default backoff path.
|
||||||
|
- Wait cap: request fails after `rate_limit_max_wait_seconds` with
|
||||||
|
`RateLimitError`.
|
||||||
|
2. **Downloader tests** (mock aiohttp session): 429 then 200 → `make_request`
|
||||||
|
returns success after gate delay; two back-to-back calls to the same
|
||||||
|
destination are spaced ≥ `min_interval`; different destinations are not
|
||||||
|
spaced.
|
||||||
|
3. **Provider tests**: `FallbackMetadataProvider.get_model_version_info` —
|
||||||
|
Civitai raises `RateLimitError` → CivArchive mock **not called**; 404 still
|
||||||
|
falls through to CivArchive; sqlite still tried after network 429.
|
||||||
|
4. **Sync-service test**: `fetch_and_update_model` with a rate-limited default
|
||||||
|
provider → result error contains `"Rate limited"` and sqlite attempt state
|
||||||
|
unchanged.
|
||||||
|
5. **Batch-import test**: analysis provider 429s first, then succeeds →
|
||||||
|
item ends `SUCCESS` (wait path), and post-cap 429 → `SKIPPED` with
|
||||||
|
rate-limit reason (phase 2).
|
||||||
|
6. Full regression: `pytest tests/services tests/routes tests/standalone`
|
||||||
|
(currently 1582 passing).
|
||||||
|
|
||||||
|
## 9. Implementation Phases
|
||||||
|
|
||||||
|
- **Phase 1 (this plan, reviewed):** `RateLimitCoordinator` +
|
||||||
|
`Downloader.make_request` integration (guard fail-fast → gate pre-check
|
||||||
|
pacing → 429 register/wait/retry loop with cap → `gate_handled` marking) +
|
||||||
|
settings + **Fix C failover semantics** (`FallbackMetadataProvider`,
|
||||||
|
`fetch_and_update_model` — moved up from phase 2: smallest diff, kills the
|
||||||
|
CivArchive flood immediately, independent of coordinator correctness) +
|
||||||
|
`_RateLimitRetryHelper` double-wait fix + coordinator/downloader/provider/
|
||||||
|
sync tests.
|
||||||
|
- **Phase 2:** batch-import `SKIPPED`-on-rate-limit + `rate_limited` WebSocket
|
||||||
|
progress flag + slowdown hint (confirmed, §10),
|
||||||
|
`download_to_memory`/HEAD 429 registration, batch tests.
|
||||||
|
- **Phase 3:** full regression + docs + commit referencing `(#1085)`.
|
||||||
|
|
||||||
|
## 10. Review Checklist — Decisions (2026-08-27)
|
||||||
|
|
||||||
|
- [x] Default pacing interval `0.75 s` — **accepted** as conservative default;
|
||||||
|
tunable via `rate_limit_min_interval_seconds`. Revisit if CivitAI
|
||||||
|
publishes keyed/anonymous ceilings.
|
||||||
|
- [x] Wait cap `300 s` — **accepted**; long-window CivArchive lookups fail →
|
||||||
|
batch import marks them `SKIPPED` with a rate-limit reason (phase 2).
|
||||||
|
- [x] Interactive API calls also wait (bounded) — **yes**, same behavior for
|
||||||
|
all callers.
|
||||||
|
- [x] Keep sqlite as last resort behind a network rate limit — **yes**
|
||||||
|
(local-only, no vendor cost).
|
||||||
|
- [x] UI hint — **yes**: WebSocket `rate_limited` flag + "rate limited —
|
||||||
|
slowing down" hint in batch-import progress (phase 2); INFO logging
|
||||||
|
regardless.
|
||||||
+2476
-2222
File diff suppressed because it is too large
Load Diff
+322
-68
@@ -67,11 +67,11 @@
|
|||||||
"steps": {
|
"steps": {
|
||||||
"fetch": {
|
"fetch": {
|
||||||
"title": "Fetch Models Metadata",
|
"title": "Fetch Models Metadata",
|
||||||
"content": "Click the <strong>Fetch</strong> button to download model metadata and preview images from Civitai."
|
"content": "Click the <strong>Fetch</strong> button to download model metadata and preview images from CivitAI."
|
||||||
},
|
},
|
||||||
"download": {
|
"download": {
|
||||||
"title": "Download New Models",
|
"title": "Download New Models",
|
||||||
"content": "Use the <strong>Download</strong> button to download models directly from Civitai URLs."
|
"content": "Use the <strong>Download</strong> button to download models directly from CivitAI URLs."
|
||||||
},
|
},
|
||||||
"bulk": {
|
"bulk": {
|
||||||
"title": "Bulk Operations",
|
"title": "Bulk Operations",
|
||||||
@@ -103,8 +103,8 @@
|
|||||||
"actions": {
|
"actions": {
|
||||||
"addToFavorites": "Add to favorites",
|
"addToFavorites": "Add to favorites",
|
||||||
"removeFromFavorites": "Remove from favorites",
|
"removeFromFavorites": "Remove from favorites",
|
||||||
"viewOnCivitai": "View on Civitai",
|
"viewOnCivitai": "View on CivitAI",
|
||||||
"notAvailableFromCivitai": "Not available from Civitai",
|
"notAvailableFromCivitai": "Not available from CivitAI",
|
||||||
"viewOnHuggingFace": "View on Hugging Face",
|
"viewOnHuggingFace": "View on Hugging Face",
|
||||||
"sendToWorkflow": "Send to ComfyUI (Click: Append, Shift+Click: Replace)",
|
"sendToWorkflow": "Send to ComfyUI (Click: Append, Shift+Click: Replace)",
|
||||||
"copyLoRASyntax": "Copy LoRA Syntax",
|
"copyLoRASyntax": "Copy LoRA Syntax",
|
||||||
@@ -137,7 +137,7 @@
|
|||||||
"exampleImages": {
|
"exampleImages": {
|
||||||
"checkError": "Error checking for example images",
|
"checkError": "Error checking for example images",
|
||||||
"missingHash": "Missing model hash information.",
|
"missingHash": "Missing model hash information.",
|
||||||
"noRemoteImagesAvailable": "No remote example images available for this model on Civitai"
|
"noRemoteImagesAvailable": "No remote example images available for this model on CivitAI"
|
||||||
},
|
},
|
||||||
"badges": {
|
"badges": {
|
||||||
"update": "Update",
|
"update": "Update",
|
||||||
@@ -186,6 +186,16 @@
|
|||||||
"cancelled": "Repair cancelled. {count} recipes were repaired.",
|
"cancelled": "Repair cancelled. {count} recipes were repaired.",
|
||||||
"error": "Recipe repair failed: {message}"
|
"error": "Recipe repair failed: {message}"
|
||||||
},
|
},
|
||||||
|
"rematchRecipes": {
|
||||||
|
"label": "Rematch recipes to local models",
|
||||||
|
"loading": "Rematching recipes to local models...",
|
||||||
|
"success": "Matched {entries} entries across {recipes} recipes",
|
||||||
|
"successErrors": "Matched {entries} entries across {recipes} recipes, {failures} failed",
|
||||||
|
"allFailed": "Rematch failed for {failures} of {total} recipes",
|
||||||
|
"noMatch": "No local match found for {entries} entries in {recipes} recipes",
|
||||||
|
"cancelled": "Rematch cancelled. {recipes} recipes updated ({entries} entries).",
|
||||||
|
"error": "Recipe rematch failed: {message}"
|
||||||
|
},
|
||||||
"manageExcludedModels": {
|
"manageExcludedModels": {
|
||||||
"label": "Manage Excluded Models"
|
"label": "Manage Excluded Models"
|
||||||
},
|
},
|
||||||
@@ -212,6 +222,7 @@
|
|||||||
"modelname": "Model Name",
|
"modelname": "Model Name",
|
||||||
"tags": "Tags",
|
"tags": "Tags",
|
||||||
"creator": "Creator",
|
"creator": "Creator",
|
||||||
|
"hash": "Hash",
|
||||||
"title": "Recipe Title",
|
"title": "Recipe Title",
|
||||||
"loraName": "LoRA Filename",
|
"loraName": "LoRA Filename",
|
||||||
"loraModel": "LoRA Model Name",
|
"loraModel": "LoRA Model Name",
|
||||||
@@ -249,7 +260,11 @@
|
|||||||
"any": "Any",
|
"any": "Any",
|
||||||
"all": "All",
|
"all": "All",
|
||||||
"tagLogicAny": "Match any tag (OR)",
|
"tagLogicAny": "Match any tag (OR)",
|
||||||
"tagLogicAll": "Match all tags (AND)"
|
"tagLogicAll": "Match all tags (AND)",
|
||||||
|
"loraAvailability": "Lora Availability",
|
||||||
|
"availabilityReady": "Ready to use",
|
||||||
|
"availabilityMissing": "Has missing",
|
||||||
|
"availabilityDeleted": "Has deleted"
|
||||||
},
|
},
|
||||||
"theme": {
|
"theme": {
|
||||||
"toggle": "Toggle theme",
|
"toggle": "Toggle theme",
|
||||||
@@ -275,15 +290,15 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"settings": {
|
"settings": {
|
||||||
"civitaiApiKey": "Civitai API Key",
|
"civitaiApiKey": "CivitAI API Key",
|
||||||
"civitaiApiKeyPlaceholder": "Enter your Civitai API key",
|
"civitaiApiKeyPlaceholder": "Enter your CivitAI API key",
|
||||||
"civitaiApiKeyHelp": "Used for authentication when downloading models from Civitai",
|
"civitaiApiKeyHelp": "Used for authentication when downloading models from CivitAI",
|
||||||
"civitaiApiKeyConfigured": "Configured",
|
"civitaiApiKeyConfigured": "Configured",
|
||||||
"civitaiApiKeyNotConfigured": "Not configured",
|
"civitaiApiKeyNotConfigured": "Not configured",
|
||||||
"civitaiApiKeySet": "Set up",
|
"civitaiApiKeySet": "Set up",
|
||||||
"civitaiHost": {
|
"civitaiHost": {
|
||||||
"label": "Civitai host",
|
"label": "CivitAI host",
|
||||||
"help": "Choose which Civitai site opens when using View on Civitai links.",
|
"help": "Choose which CivitAI site opens when using View on CivitAI links.",
|
||||||
"options": {
|
"options": {
|
||||||
"com": "civitai.com (SFW)",
|
"com": "civitai.com (SFW)",
|
||||||
"red": "civitai.red (unrestricted)"
|
"red": "civitai.red (unrestricted)"
|
||||||
@@ -304,8 +319,8 @@
|
|||||||
},
|
},
|
||||||
"aria2HelpLink": "Learn how to set up the aria2 download backend",
|
"aria2HelpLink": "Learn how to set up the aria2 download backend",
|
||||||
"civitaiHostBanner": {
|
"civitaiHostBanner": {
|
||||||
"title": "Civitai host preference available",
|
"title": "CivitAI host preference available",
|
||||||
"content": "Civitai now uses civitai.com for SFW content and civitai.red for unrestricted content. You can change which site opens by default in Settings.",
|
"content": "CivitAI now uses civitai.com for SFW content and civitai.red for unrestricted content. You can change which site opens by default in Settings.",
|
||||||
"openSettings": "Open Settings"
|
"openSettings": "Open Settings"
|
||||||
},
|
},
|
||||||
"openSettingsFileLocation": {
|
"openSettingsFileLocation": {
|
||||||
@@ -435,7 +450,7 @@
|
|||||||
},
|
},
|
||||||
"layoutSettings": {
|
"layoutSettings": {
|
||||||
"groupByModel": "Group by Model",
|
"groupByModel": "Group by Model",
|
||||||
"groupByModelHelp": "When enabled, only the latest version of each Civitai model is shown as a single card. Older versions are hidden.",
|
"groupByModelHelp": "When enabled, only the latest version of each CivitAI model is shown as a single card. Older versions are hidden.",
|
||||||
"displayDensity": "Display Density",
|
"displayDensity": "Display Density",
|
||||||
"displayDensityOptions": {
|
"displayDensityOptions": {
|
||||||
"default": "Default",
|
"default": "Default",
|
||||||
@@ -449,6 +464,12 @@
|
|||||||
"compact": "7 (1080p), 8 (2K), 10 (4K)"
|
"compact": "7 (1080p), 8 (2K), 10 (4K)"
|
||||||
},
|
},
|
||||||
"displayDensityWarning": "Warning: Higher densities may cause performance issues on systems with limited resources.",
|
"displayDensityWarning": "Warning: Higher densities may cause performance issues on systems with limited resources.",
|
||||||
|
"recipesLayout": "Recipes Layout",
|
||||||
|
"recipesLayoutHelp": "Choose how recipe cards are arranged: a uniform grid or a masonry (Pinterest-style) layout that preserves each image's aspect ratio.",
|
||||||
|
"recipesLayoutOptions": {
|
||||||
|
"grid": "Grid",
|
||||||
|
"masonry": "Masonry"
|
||||||
|
},
|
||||||
"showFolderSidebar": "Show Folder Sidebar",
|
"showFolderSidebar": "Show Folder Sidebar",
|
||||||
"showFolderSidebarHelp": "Toggle the folder navigation sidebar on model pages. When disabled, the sidebar and hover area stay hidden.",
|
"showFolderSidebarHelp": "Toggle the folder navigation sidebar on model pages. When disabled, the sidebar and hover area stay hidden.",
|
||||||
"cardInfoDisplay": "Card Info Display",
|
"cardInfoDisplay": "Card Info Display",
|
||||||
@@ -534,7 +555,7 @@
|
|||||||
},
|
},
|
||||||
"downloadPathTemplates": {
|
"downloadPathTemplates": {
|
||||||
"title": "Download Path Templates",
|
"title": "Download Path Templates",
|
||||||
"help": "Configure folder structures for different model types when downloading from Civitai.",
|
"help": "Configure folder structures for different model types when downloading from CivitAI.",
|
||||||
"availablePlaceholders": "Available placeholders:",
|
"availablePlaceholders": "Available placeholders:",
|
||||||
"templateOptions": {
|
"templateOptions": {
|
||||||
"flatStructure": "Flat Structure",
|
"flatStructure": "Flat Structure",
|
||||||
@@ -571,7 +592,7 @@
|
|||||||
"exampleImages": {
|
"exampleImages": {
|
||||||
"downloadLocation": "Download Location",
|
"downloadLocation": "Download Location",
|
||||||
"downloadLocationPlaceholder": "Enter folder path for example images",
|
"downloadLocationPlaceholder": "Enter folder path for example images",
|
||||||
"downloadLocationHelp": "Enter the folder path where example images from Civitai will be saved",
|
"downloadLocationHelp": "Enter the folder path where example images from CivitAI will be saved",
|
||||||
"autoDownload": "Auto Download Example Images",
|
"autoDownload": "Auto Download Example Images",
|
||||||
"autoDownloadHelp": "Automatically download example images for models that don't have them (requires download location to be set)",
|
"autoDownloadHelp": "Automatically download example images for models that don't have them (requires download location to be set)",
|
||||||
"openMode": "Open Example Images Action",
|
"openMode": "Open Example Images Action",
|
||||||
@@ -606,6 +627,10 @@
|
|||||||
"label": "Hide Early Access Updates",
|
"label": "Hide Early Access Updates",
|
||||||
"help": "When enabled, models with only early access updates will not show 'Update available' badge"
|
"help": "When enabled, models with only early access updates will not show 'Update available' badge"
|
||||||
},
|
},
|
||||||
|
"hidePaidUpdates": {
|
||||||
|
"label": "Hide Paid Updates",
|
||||||
|
"help": "When enabled, models with only paid updates will not show 'Update available' badge"
|
||||||
|
},
|
||||||
"licenseIcons": {
|
"licenseIcons": {
|
||||||
"useNewStyle": "Use updated license icons",
|
"useNewStyle": "Use updated license icons",
|
||||||
"useNewStyleHelp": "Display license permissions with colored indicators (new style) or restriction-only icons (classic style). Mirroring the current CivitAI design."
|
"useNewStyleHelp": "Display license permissions with colored indicators (new style) or restriction-only icons (classic style). Mirroring the current CivitAI design."
|
||||||
@@ -622,7 +647,7 @@
|
|||||||
},
|
},
|
||||||
"metadataArchive": {
|
"metadataArchive": {
|
||||||
"enableArchiveDb": "Enable Metadata Archive Database",
|
"enableArchiveDb": "Enable Metadata Archive Database",
|
||||||
"enableArchiveDbHelp": "Use a local database to access metadata for models that have been deleted from Civitai.",
|
"enableArchiveDbHelp": "Use a local database to access metadata for models that have been deleted from CivitAI.",
|
||||||
"status": "Status",
|
"status": "Status",
|
||||||
"statusAvailable": "Available",
|
"statusAvailable": "Available",
|
||||||
"statusUnavailable": "Not Available",
|
"statusUnavailable": "Not Available",
|
||||||
@@ -678,6 +703,7 @@
|
|||||||
"deepseek": "DeepSeek",
|
"deepseek": "DeepSeek",
|
||||||
"groq": "Groq",
|
"groq": "Groq",
|
||||||
"openrouter": "OpenRouter",
|
"openrouter": "OpenRouter",
|
||||||
|
"google": "Gemini",
|
||||||
"opencode-go": "OpenCode Go",
|
"opencode-go": "OpenCode Go",
|
||||||
"custom": "Custom (OpenAI-compatible)"
|
"custom": "Custom (OpenAI-compatible)"
|
||||||
},
|
},
|
||||||
@@ -714,7 +740,9 @@
|
|||||||
"versionsCount": "Local Versions",
|
"versionsCount": "Local Versions",
|
||||||
"versionsCountDesc": "Most versions first",
|
"versionsCountDesc": "Most versions first",
|
||||||
"versionsCountAsc": "Fewest versions first",
|
"versionsCountAsc": "Fewest versions first",
|
||||||
"versionIdDesc": "Newest version first"
|
"versionIdDesc": "Newest version first",
|
||||||
|
"random": "Random",
|
||||||
|
"randomAction": "Randomize (shuffle)"
|
||||||
},
|
},
|
||||||
"refresh": {
|
"refresh": {
|
||||||
"title": "Refresh model list",
|
"title": "Refresh model list",
|
||||||
@@ -722,7 +750,7 @@
|
|||||||
"fullTooltip": "Reload all model details from metadata files—use if the library looks out of date or after manual edits."
|
"fullTooltip": "Reload all model details from metadata files—use if the library looks out of date or after manual edits."
|
||||||
},
|
},
|
||||||
"fetch": {
|
"fetch": {
|
||||||
"title": "Fetch metadata from Civitai",
|
"title": "Fetch metadata from CivitAI",
|
||||||
"action": "Fetch"
|
"action": "Fetch"
|
||||||
},
|
},
|
||||||
"download": {
|
"download": {
|
||||||
@@ -759,6 +787,7 @@
|
|||||||
"copyAll": "Copy Selected Syntax",
|
"copyAll": "Copy Selected Syntax",
|
||||||
"refreshAll": "Refresh Selected Metadata",
|
"refreshAll": "Refresh Selected Metadata",
|
||||||
"repairMetadata": "Repair Metadata for Selected",
|
"repairMetadata": "Repair Metadata for Selected",
|
||||||
|
"rematchMetadata": "Rematch Selected to Local Models",
|
||||||
"reimportMetadata": "Re-import from Source",
|
"reimportMetadata": "Re-import from Source",
|
||||||
"checkUpdates": "Check Updates for Selected",
|
"checkUpdates": "Check Updates for Selected",
|
||||||
"moveAll": "Move Selected to Folder",
|
"moveAll": "Move Selected to Folder",
|
||||||
@@ -771,6 +800,8 @@
|
|||||||
"deleteAll": "Delete Selected",
|
"deleteAll": "Delete Selected",
|
||||||
"downloadMissingLoras": "Download Missing LoRAs",
|
"downloadMissingLoras": "Download Missing LoRAs",
|
||||||
"downloadExamples": "Download Example Images",
|
"downloadExamples": "Download Example Images",
|
||||||
|
"downloadMissingExamples": "Download Missing",
|
||||||
|
"reprocessExamples": "Re-process All",
|
||||||
"clear": "Clear Selection",
|
"clear": "Clear Selection",
|
||||||
"skipMetadataRefreshCount": "Skip ({count} models)",
|
"skipMetadataRefreshCount": "Skip ({count} models)",
|
||||||
"resumeMetadataRefreshCount": "Resume ({count} models)",
|
"resumeMetadataRefreshCount": "Resume ({count} models)",
|
||||||
@@ -794,10 +825,10 @@
|
|||||||
"enrichHfAgent": "Enrich HF Metadata (AI)"
|
"enrichHfAgent": "Enrich HF Metadata (AI)"
|
||||||
},
|
},
|
||||||
"contextMenu": {
|
"contextMenu": {
|
||||||
"refreshMetadata": "Refresh Civitai Data",
|
"refreshMetadata": "Refresh CivitAI Data",
|
||||||
"checkUpdates": "Check Updates",
|
"checkUpdates": "Check Updates",
|
||||||
"linkModel": "Link Model",
|
"linkModel": "Link Model",
|
||||||
"linkCivitai": "Link to Civitai",
|
"linkCivitai": "Link to CivitAI",
|
||||||
"linkHuggingFace": "Link to HuggingFace",
|
"linkHuggingFace": "Link to HuggingFace",
|
||||||
"copySyntax": "Copy LoRA Syntax",
|
"copySyntax": "Copy LoRA Syntax",
|
||||||
"copyFilename": "Copy Model Filename",
|
"copyFilename": "Copy Model Filename",
|
||||||
@@ -806,10 +837,13 @@
|
|||||||
"sendToWorkflowReplace": "Send to Workflow (Replace)",
|
"sendToWorkflowReplace": "Send to Workflow (Replace)",
|
||||||
"openExamples": "Open Examples Folder",
|
"openExamples": "Open Examples Folder",
|
||||||
"downloadExamples": "Download Example Images",
|
"downloadExamples": "Download Example Images",
|
||||||
|
"downloadMissingExamples": "Download Missing",
|
||||||
|
"reprocessExamples": "Re-process All",
|
||||||
"replacePreview": "Replace Preview",
|
"replacePreview": "Replace Preview",
|
||||||
"setContentRating": "Set Content Rating",
|
"setContentRating": "Set Content Rating",
|
||||||
"moveToFolder": "Move to Folder",
|
"moveToFolder": "Move to Folder",
|
||||||
"repairMetadata": "Repair metadata",
|
"repairMetadata": "Repair metadata",
|
||||||
|
"rematchMetadata": "Rematch to local models",
|
||||||
"reimportMetadata": "Re-import from Source",
|
"reimportMetadata": "Re-import from Source",
|
||||||
"excludeModel": "Exclude Model",
|
"excludeModel": "Exclude Model",
|
||||||
"restoreModel": "Restore Model",
|
"restoreModel": "Restore Model",
|
||||||
@@ -824,20 +858,130 @@
|
|||||||
"recipes": {
|
"recipes": {
|
||||||
"title": "LoRA Recipes",
|
"title": "LoRA Recipes",
|
||||||
"actions": {
|
"actions": {
|
||||||
"sendCheckpoint": "Send to ComfyUI"
|
"sendCheckpoint": "Send to ComfyUI",
|
||||||
|
"sendRecipe": "Send to ComfyUI",
|
||||||
|
"copyRecipeSyntax": "Copy Recipe Syntax",
|
||||||
|
"deleteRecipeWithShortcut": "Delete recipe (Del)"
|
||||||
|
},
|
||||||
|
"navigation": {
|
||||||
|
"label": "Recipe navigation",
|
||||||
|
"previousWithShortcut": "Previous recipe (←)",
|
||||||
|
"nextWithShortcut": "Next recipe (→)"
|
||||||
|
},
|
||||||
|
"modal": {
|
||||||
|
"metadata": {
|
||||||
|
"id": "ID"
|
||||||
|
},
|
||||||
|
"actions": {
|
||||||
|
"openFileLocation": "Open File Location",
|
||||||
|
"copyId": "Copy recipe ID"
|
||||||
|
},
|
||||||
|
"openFileLocation": {
|
||||||
|
"success": "File location opened successfully",
|
||||||
|
"failed": "Failed to open file location",
|
||||||
|
"copied": "Path copied to clipboard: {{path}}",
|
||||||
|
"clipboardFallback": "Path: {{path}}"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"workflow": {
|
||||||
|
"sendWorkflow": "Send Workflow to ComfyUI",
|
||||||
|
"sent": "Workflow sent to ComfyUI",
|
||||||
|
"sendFailed": "Failed to send workflow to ComfyUI",
|
||||||
|
"noWorkflow": "No embedded workflow found in this recipe"
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"ready": "Ready to use",
|
||||||
|
"missingCount": "{count} missing",
|
||||||
|
"deletedCount": "{count} deleted",
|
||||||
|
"downloadMissing": "Download {count} missing LoRAs",
|
||||||
|
"downloadMissingTooltip": "Click to download missing LoRAs"
|
||||||
|
},
|
||||||
|
"loraStatus": {
|
||||||
|
"none": "No LoRAs in this recipe",
|
||||||
|
"allAvailable": "All LoRAs available - Ready to use",
|
||||||
|
"missing": "{missing} of {total} LoRAs missing",
|
||||||
|
"missingAndUnavailable": "{missing} of {total} LoRAs missing, {unavailable} unavailable (deleted from source or unresolvable hash)",
|
||||||
|
"partial": "{unavailable} of {total} LoRAs unavailable (deleted from source or unresolvable hash) - skipped when recipe is used",
|
||||||
|
"noneUsable": "No usable LoRAs - {unavailable} of {total} deleted from source or unresolvable hash"
|
||||||
|
},
|
||||||
|
"resources": {
|
||||||
|
"inLibrary": "In Library",
|
||||||
|
"notInLibrary": "Not in Library",
|
||||||
|
"deleted": "Deleted",
|
||||||
|
"hashInvalid": "Unresolvable Hash",
|
||||||
|
"inLibraryTooltip": "This model exists in your local library",
|
||||||
|
"notInLibraryTooltip": "This model is not in your library",
|
||||||
|
"deletedTooltip": "This LoRA was deleted from the source and is no longer available for download",
|
||||||
|
"hashInvalidTooltip": "This LoRA hash cannot be resolved on CivitAI - the model may have been updated",
|
||||||
|
"noLorasAssociated": "No LoRAs associated with this recipe",
|
||||||
|
"noLorasWhyToggle": "Why no LoRAs?",
|
||||||
|
"noLorasImportMethod": "Import method",
|
||||||
|
"noLorasInferredNote": "Possible reason (inferred) — this recipe was imported before import diagnostics were recorded.",
|
||||||
|
"noLorasChannels": {
|
||||||
|
"batch_import_url": "Batch import (image URL)",
|
||||||
|
"batch_import_local": "Batch import (local file)",
|
||||||
|
"url": "Image URL import",
|
||||||
|
"local": "Local file import",
|
||||||
|
"upload": "Image upload",
|
||||||
|
"widget": "Saved from workflow",
|
||||||
|
"reimport_url": "Re-import (image URL)",
|
||||||
|
"reimport_local": "Re-import (local file)"
|
||||||
|
},
|
||||||
|
"noLorasReasons": {
|
||||||
|
"no_loras_used": "The generation metadata is complete and does not reference any LoRAs.",
|
||||||
|
"api_meta_no_lora_resources": "The source API returned no LoRA resource data for this image. LoRAs shown on the CivitAI page may come from internal data that the public API does not expose.",
|
||||||
|
"api_meta_missing": "The source API returned no generation metadata for this image.",
|
||||||
|
"no_embedded_metadata": "The image has no embedded generation metadata, so LoRA information could not be recovered.",
|
||||||
|
"workflow_metadata_limited": "The image's embedded metadata is a ComfyUI workflow; extracting LoRA information from workflows is limited.",
|
||||||
|
"video_no_metadata": "Video files do not carry embedded generation metadata.",
|
||||||
|
"metadata_unsupported": "The image contains metadata in a format that could not be parsed.",
|
||||||
|
"unknown": "The reason could not be determined from the stored recipe data."
|
||||||
|
},
|
||||||
|
"noLorasDetails": {
|
||||||
|
"apiMetaFields": "API metadata fields",
|
||||||
|
"modelVersionIds": "Model version IDs reported",
|
||||||
|
"embeddedMetadata": "Embedded metadata",
|
||||||
|
"present": "found",
|
||||||
|
"absent": "none"
|
||||||
|
},
|
||||||
|
"download": "Download",
|
||||||
|
"downloadLoraTooltip": "Download this LoRA",
|
||||||
|
"preparingDownload": "Preparing download...",
|
||||||
|
"reconnect": "Reconnect",
|
||||||
|
"reconnectTooltip": "Reconnect with a local LoRA",
|
||||||
|
"reconnectInstructions": "Enter LoRA syntax or name to reconnect:",
|
||||||
|
"reconnectExample": "Example: <lora:name:1> or just the name",
|
||||||
|
"reconnectPlaceholder": "Enter LoRA name or syntax",
|
||||||
|
"reconnectSuggestionsLoading": "Searching local library...",
|
||||||
|
"reconnectSuggestionsEmpty": "No matching LoRAs in your local library",
|
||||||
|
"reconnectMatchSameHash": "Same hash",
|
||||||
|
"reconnectMatchSameVersion": "Same model version",
|
||||||
|
"reconnectMatchSimilarFilename": "Similar filename",
|
||||||
|
"reconnectMatchSimilarName": "Similar name",
|
||||||
|
"undoReconnect": "Undo",
|
||||||
|
"undoReconnectTooltip": "Restore the association this entry had before reconnecting",
|
||||||
|
"undoReconnectTooltipNamed": "Restore to {name} (the association before reconnecting)",
|
||||||
|
"viewOnCivitai": "View on CivitAI",
|
||||||
|
"openLoraDetails": "View {name} in the LoRA library",
|
||||||
|
"openCheckpointDetails": "View {name} in the model library",
|
||||||
|
"checkpointDeletedTooltip": "This checkpoint was deleted from the source and can no longer be downloaded - reconnect it with a local model",
|
||||||
|
"checkpointHashInvalidTooltip": "This checkpoint hash cannot be resolved on CivitAI - the model may have been updated",
|
||||||
|
"reconnectCheckpoint": "Reconnect",
|
||||||
|
"reconnectCheckpointTooltip": "Reconnect with a local checkpoint",
|
||||||
|
"checkpointReconnectInstructions": "Enter checkpoint name to reconnect:",
|
||||||
|
"checkpointReconnectPlaceholder": "Enter checkpoint name",
|
||||||
|
"checkpointReconnectSuggestionsEmpty": "No matching checkpoints in your local library"
|
||||||
},
|
},
|
||||||
"controls": {
|
"controls": {
|
||||||
"import": {
|
"import": {
|
||||||
"action": "Import",
|
"action": "Import",
|
||||||
"title": "Import a recipe from image or URL",
|
"title": "Import a recipe from image or URL",
|
||||||
"urlLocalPath": "URL / Local Path",
|
"dropZoneLabel": "Upload image",
|
||||||
"uploadImage": "Upload Image",
|
"dropZoneHint": "Drag & drop an image here, paste from clipboard, or click to browse",
|
||||||
"urlSectionDescription": "Input a Civitai image URL from civitai.com or civitai.red, or a local file path, to import as a recipe.",
|
"orDivider": "or drag & drop / paste an image",
|
||||||
"imageUrlOrPath": "Image URL or File Path:",
|
"imageUrlOrPath": "Image URL or File Path:",
|
||||||
"urlPlaceholder": "https://civitai.com/images/... or https://civitai.red/images/... or C:/path/to/image.png",
|
"urlPlaceholder": "https://civitai.com/images/... or https://civitai.red/images/... or C:/path/to/image.png",
|
||||||
"fetchImage": "Fetch Image",
|
"fetchImage": "Fetch Image",
|
||||||
"uploadSectionDescription": "Upload an image with LoRA metadata to import as a recipe.",
|
|
||||||
"selectImage": "Select Image",
|
|
||||||
"recipeName": "Recipe Name",
|
"recipeName": "Recipe Name",
|
||||||
"recipeNamePlaceholder": "Enter recipe name",
|
"recipeNamePlaceholder": "Enter recipe name",
|
||||||
"tagsOptional": "Tags (optional)",
|
"tagsOptional": "Tags (optional)",
|
||||||
@@ -865,7 +1009,7 @@
|
|||||||
"downloadingLoras": "Downloading LoRAs...",
|
"downloadingLoras": "Downloading LoRAs...",
|
||||||
"savingRecipe": "Saving recipe...",
|
"savingRecipe": "Saving recipe...",
|
||||||
"startingDownload": "Starting download for LoRA {current}/{total}",
|
"startingDownload": "Starting download for LoRA {current}/{total}",
|
||||||
"deletedFromCivitai": "Deleted from Civitai",
|
"deletedFromCivitai": "Deleted from CivitAI",
|
||||||
"inLibrary": "In Library",
|
"inLibrary": "In Library",
|
||||||
"notInLibrary": "Not in Library",
|
"notInLibrary": "Not in Library",
|
||||||
"earlyAccessRequired": "This LoRA requires early access payment to download.",
|
"earlyAccessRequired": "This LoRA requires early access payment to download.",
|
||||||
@@ -882,6 +1026,8 @@
|
|||||||
"errors": {
|
"errors": {
|
||||||
"selectImageFile": "Please select an image file",
|
"selectImageFile": "Please select an image file",
|
||||||
"enterUrlOrPath": "Please enter a URL or file path",
|
"enterUrlOrPath": "Please enter a URL or file path",
|
||||||
|
"invalidUrl": "Please enter a valid URL",
|
||||||
|
"invalidInputFormat": "Please enter an image URL or a local image file path",
|
||||||
"selectLoraRoot": "Please select a LoRA root directory"
|
"selectLoraRoot": "Please select a LoRA root directory"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -895,7 +1041,9 @@
|
|||||||
"dateAsc": "Oldest",
|
"dateAsc": "Oldest",
|
||||||
"lorasCount": "LoRA Count",
|
"lorasCount": "LoRA Count",
|
||||||
"lorasCountDesc": "Most",
|
"lorasCountDesc": "Most",
|
||||||
"lorasCountAsc": "Least"
|
"lorasCountAsc": "Least",
|
||||||
|
"opened": "Recently Opened",
|
||||||
|
"openedDesc": "Recently opened"
|
||||||
},
|
},
|
||||||
"refresh": {
|
"refresh": {
|
||||||
"title": "Refresh recipe list",
|
"title": "Refresh recipe list",
|
||||||
@@ -906,12 +1054,26 @@
|
|||||||
"favorites": {
|
"favorites": {
|
||||||
"title": "Show Favorites Only",
|
"title": "Show Favorites Only",
|
||||||
"action": "Favorites"
|
"action": "Favorites"
|
||||||
|
},
|
||||||
|
"layout": {
|
||||||
|
"title": "Recipes Layout",
|
||||||
|
"grid": "Grid layout",
|
||||||
|
"masonry": "Masonry layout (Pinterest-style, preserves image aspect ratio)"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"duplicates": {
|
"duplicates": {
|
||||||
|
"finding": "Scanning for duplicate recipes...",
|
||||||
"found": "Found {count} duplicate groups",
|
"found": "Found {count} duplicate groups",
|
||||||
|
"noGroups": "No duplicate groups found with the current matching basis",
|
||||||
"keepLatest": "Keep Latest Versions",
|
"keepLatest": "Keep Latest Versions",
|
||||||
"deleteSelected": "Delete Selected"
|
"deleteSelected": "Delete Selected",
|
||||||
|
"includePromptLabel": "Include prompt in matching",
|
||||||
|
"basis": {
|
||||||
|
"loraCombo": "Matched by: LoRA combination",
|
||||||
|
"loraComboAndPrompt": "Matched by: LoRA combination + prompt",
|
||||||
|
"hintLoraCombo": "Recipes with the same LoRAs at identical strengths are grouped.",
|
||||||
|
"hintPromptIncluded": "Recipes are grouped only when they use the same LoRAs at identical strengths AND have the same prompt."
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"contextMenu": {
|
"contextMenu": {
|
||||||
"copyRecipe": {
|
"copyRecipe": {
|
||||||
@@ -970,6 +1132,8 @@
|
|||||||
"start": "Start Import",
|
"start": "Start Import",
|
||||||
"startImport": "Start Import",
|
"startImport": "Start Import",
|
||||||
"importing": "Importing...",
|
"importing": "Importing...",
|
||||||
|
"rateLimitedSlowdown": "Rate limited — slowing down...",
|
||||||
|
"rateLimitedHint": "Some items were skipped due to metadata provider rate limits. Re-run the import later to retry them.",
|
||||||
"progress": "Progress",
|
"progress": "Progress",
|
||||||
"total": "Total",
|
"total": "Total",
|
||||||
"success": "Success",
|
"success": "Success",
|
||||||
@@ -1173,7 +1337,7 @@
|
|||||||
"download": {
|
"download": {
|
||||||
"title": "Download Model from URL",
|
"title": "Download Model from URL",
|
||||||
"titleWithType": "Download {type} from URL",
|
"titleWithType": "Download {type} from URL",
|
||||||
"civitaiUrl": "Civitai URL(s):",
|
"civitaiUrl": "CivitAI URL(s):",
|
||||||
"placeholder": "https://civitai.com/models/...",
|
"placeholder": "https://civitai.com/models/...",
|
||||||
"urlHint": "Enter one CivitAI, CivArchive, or Hugging Face URL per line. Supports multiple URLs for batch download.",
|
"urlHint": "Enter one CivitAI, CivArchive, or Hugging Face URL per line. Supports multiple URLs for batch download.",
|
||||||
"selectHfFiles": "Select file(s) to download from this repository:",
|
"selectHfFiles": "Select file(s) to download from this repository:",
|
||||||
@@ -1199,14 +1363,16 @@
|
|||||||
"downloaded": "Downloaded",
|
"downloaded": "Downloaded",
|
||||||
"downloadedTooltip": "Previously downloaded, but it is not currently in your library.",
|
"downloadedTooltip": "Previously downloaded, but it is not currently in your library.",
|
||||||
"alreadyInLibrary": "Already in Library",
|
"alreadyInLibrary": "Already in Library",
|
||||||
|
"partiallyDownloaded": "Partially downloaded",
|
||||||
"autoOrganizedPath": "[Auto-organized by path template]",
|
"autoOrganizedPath": "[Auto-organized by path template]",
|
||||||
"fileSelection": {
|
"fileSelection": {
|
||||||
"title": "Select File Format",
|
"title": "Select File Format",
|
||||||
"files": "files",
|
"files": "files",
|
||||||
"select": "Select File"
|
"select": "Select File",
|
||||||
|
"inLibrary": "In Library"
|
||||||
},
|
},
|
||||||
"errors": {
|
"errors": {
|
||||||
"invalidUrl": "Invalid Civitai URL format",
|
"invalidUrl": "Invalid CivitAI URL format",
|
||||||
"noVersions": "No versions available for this model",
|
"noVersions": "No versions available for this model",
|
||||||
"mixedSources": "Cannot mix CivitAI and Hugging Face URLs in the same batch.",
|
"mixedSources": "Cannot mix CivitAI and Hugging Face URLs in the same batch.",
|
||||||
"noModelFiles": "No model files found in this repository."
|
"noModelFiles": "No model files found in this repository."
|
||||||
@@ -1244,8 +1410,13 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"deleteModel": {
|
"deleteModel": {
|
||||||
|
"freesSpace": "Frees {size}",
|
||||||
"title": "Delete Model",
|
"title": "Delete Model",
|
||||||
"message": "Are you sure you want to delete this model and all associated files?"
|
"message": "Are you sure you want to delete this model and all associated files?",
|
||||||
|
"recoverableWarning": "This will permanently delete the file after 20 seconds unless you undo."
|
||||||
|
},
|
||||||
|
"deleteRecipe": {
|
||||||
|
"recoverableWarning": "This action can be undone for 20 seconds."
|
||||||
},
|
},
|
||||||
"excludeModel": {
|
"excludeModel": {
|
||||||
"title": "Exclude Model",
|
"title": "Exclude Model",
|
||||||
@@ -1315,7 +1486,7 @@
|
|||||||
"title": "Local Example Images",
|
"title": "Local Example Images",
|
||||||
"message": "No local example images found for this model. View options:",
|
"message": "No local example images found for this model. View options:",
|
||||||
"downloadOption": {
|
"downloadOption": {
|
||||||
"title": "Download from Civitai",
|
"title": "Download from CivitAI",
|
||||||
"description": "Save remote examples locally for offline use and faster loading"
|
"description": "Save remote examples locally for offline use and faster loading"
|
||||||
},
|
},
|
||||||
"importOption": {
|
"importOption": {
|
||||||
@@ -1342,7 +1513,7 @@
|
|||||||
"confirmAction": "Save & Link"
|
"confirmAction": "Save & Link"
|
||||||
},
|
},
|
||||||
"relinkCivitai": {
|
"relinkCivitai": {
|
||||||
"title": "Re-link to Civitai",
|
"title": "Re-link to CivitAI",
|
||||||
"warning": "Warning:",
|
"warning": "Warning:",
|
||||||
"warningText": "This is a potentially destructive operation. Re-linking will:",
|
"warningText": "This is a potentially destructive operation. Re-linking will:",
|
||||||
"warningList": {
|
"warningList": {
|
||||||
@@ -1351,14 +1522,15 @@
|
|||||||
"unintendedConsequences": "May have other unintended consequences"
|
"unintendedConsequences": "May have other unintended consequences"
|
||||||
},
|
},
|
||||||
"proceedText": "Only proceed if you're sure this is what you want.",
|
"proceedText": "Only proceed if you're sure this is what you want.",
|
||||||
"urlLabel": "Civitai Model URL:",
|
"urlLabel": "CivitAI Model URL:",
|
||||||
"urlPlaceholder": "https://civitai.com/models/649516/model-name?modelVersionId=726676 or https://civitai.red/models/649516/model-name?modelVersionId=726676",
|
"urlPlaceholder": "https://civitai.com/models/12345/model-name?modelVersionId=67890 or https://civitai.red/models/12345/model-name?modelVersionId=67890",
|
||||||
"helpText": {
|
"helpText": {
|
||||||
"title": "Paste any Civitai model URL from civitai.com or civitai.red. Supported formats:",
|
"title": "Paste any CivitAI or CivitArchive model URL. Supported formats:",
|
||||||
"format1": "https://civitai.com/models/649516",
|
"format1": "https://civitai.com/models/12345",
|
||||||
"format2": "https://civitai.com/models/649516?modelVersionId=726676",
|
"format2": "https://civitai.com/models/12345?modelVersionId=67890",
|
||||||
"format3": "https://civitai.com/models/649516/model-name?modelVersionId=726676",
|
"format3": "https://civitai.com/models/12345/model-name?modelVersionId=67890",
|
||||||
"note": "Note: If no modelVersionId is provided, the latest version will be used."
|
"note": "Note: If no modelVersionId is provided, the latest version will be used.",
|
||||||
|
"format4": "https://civarchive.com/models/12345 (CivArchive)"
|
||||||
},
|
},
|
||||||
"confirmAction": "Confirm Re-link"
|
"confirmAction": "Confirm Re-link"
|
||||||
},
|
},
|
||||||
@@ -1368,14 +1540,16 @@
|
|||||||
"editFileName": "Edit file name",
|
"editFileName": "Edit file name",
|
||||||
"editBaseModel": "Edit base model",
|
"editBaseModel": "Edit base model",
|
||||||
"editVersionName": "Edit version name",
|
"editVersionName": "Edit version name",
|
||||||
"viewOnCivitai": "View on Civitai",
|
"viewOnCivitai": "View on CivitAI",
|
||||||
"viewOnCivitaiText": "View on Civitai",
|
"viewOnCivitaiText": "View on CivitAI",
|
||||||
"viewOnHuggingFace": "View on Hugging Face",
|
"viewOnHuggingFace": "View on Hugging Face",
|
||||||
"viewOnHuggingFaceText": "View on Hugging Face",
|
"viewOnHuggingFaceText": "View on Hugging Face",
|
||||||
"viewCreatorProfile": "View Creator Profile",
|
"viewCreatorProfile": "View Creator Profile",
|
||||||
"openFileLocation": "Open File Location",
|
"openFileLocation": "Open File Location",
|
||||||
"sendToWorkflow": "Send to ComfyUI",
|
"sendToWorkflow": "Send to ComfyUI",
|
||||||
"sendToWorkflowText": "Send to ComfyUI"
|
"sendToWorkflowText": "Send to ComfyUI",
|
||||||
|
"copyHash": "Copy hash",
|
||||||
|
"deleteModelWithShortcut": "Delete model (Del)"
|
||||||
},
|
},
|
||||||
"openFileLocation": {
|
"openFileLocation": {
|
||||||
"success": "File location opened successfully",
|
"success": "File location opened successfully",
|
||||||
@@ -1392,6 +1566,7 @@
|
|||||||
"location": "Location",
|
"location": "Location",
|
||||||
"baseModel": "Base Model",
|
"baseModel": "Base Model",
|
||||||
"size": "Size",
|
"size": "Size",
|
||||||
|
"hashes": "Hashes",
|
||||||
"unknown": "Unknown",
|
"unknown": "Unknown",
|
||||||
"usageTips": "Usage Tips",
|
"usageTips": "Usage Tips",
|
||||||
"additionalNotes": "Additional Notes",
|
"additionalNotes": "Additional Notes",
|
||||||
@@ -1468,7 +1643,7 @@
|
|||||||
},
|
},
|
||||||
"license": {
|
"license": {
|
||||||
"noImageSell": "No selling generated content",
|
"noImageSell": "No selling generated content",
|
||||||
"noRentCivit": "No Civitai generation",
|
"noRentCivit": "No CivitAI generation",
|
||||||
"noRent": "No generation services",
|
"noRent": "No generation services",
|
||||||
"noSell": "No selling models",
|
"noSell": "No selling models",
|
||||||
"creditRequired": "Creator credit required",
|
"creditRequired": "Creator credit required",
|
||||||
@@ -1483,6 +1658,30 @@
|
|||||||
"examples": "Loading examples...",
|
"examples": "Loading examples...",
|
||||||
"versions": "Loading versions..."
|
"versions": "Loading versions..."
|
||||||
},
|
},
|
||||||
|
"showcase": {
|
||||||
|
"hiddenBySfw": "{count} hidden by SFW-only setting",
|
||||||
|
"showExamples": "Show examples",
|
||||||
|
"showCount": "Show examples ({count})",
|
||||||
|
"hideExamples": "Hide examples",
|
||||||
|
"addExamples": "Add examples",
|
||||||
|
"previousExample": "Previous example",
|
||||||
|
"nextExample": "Next example",
|
||||||
|
"noExamples": "No example images available",
|
||||||
|
"addMoreExamples": "Add more examples",
|
||||||
|
"dragDrop": "Drag & drop images or videos here",
|
||||||
|
"or": "or",
|
||||||
|
"selectFiles": "Select Files",
|
||||||
|
"supportedFormats": "Supported formats: jpg, png, gif, webp, avif, jxl, mp4, webm",
|
||||||
|
"importing": "Importing files...",
|
||||||
|
"noSupportedFiles": "No supported files selected. Please select image or video files.",
|
||||||
|
"allFiltered": "All example images are filtered due to NSFW content settings",
|
||||||
|
"sfwOnlyEnabled": "Your settings are currently set to show only safe-for-work content",
|
||||||
|
"changeInSettings": "You can change this in Settings",
|
||||||
|
"nsfwMature": "Mature Content",
|
||||||
|
"nsfwR": "R-rated Content",
|
||||||
|
"nsfwX": "X-rated Content",
|
||||||
|
"nsfwXxx": "XXX-rated Content"
|
||||||
|
},
|
||||||
"versions": {
|
"versions": {
|
||||||
"heading": "Model versions",
|
"heading": "Model versions",
|
||||||
"copy": "Track and manage every version of this model in one place.",
|
"copy": "Track and manage every version of this model in one place.",
|
||||||
@@ -1509,24 +1708,28 @@
|
|||||||
"newer": "Newer Version",
|
"newer": "Newer Version",
|
||||||
"newerTooltip": "This version is newer than your latest local version",
|
"newerTooltip": "This version is newer than your latest local version",
|
||||||
"earlyAccess": "Early Access",
|
"earlyAccess": "Early Access",
|
||||||
"earlyAccessTooltip": "This version currently requires Civitai early access",
|
"earlyAccessTooltip": "This version currently requires CivitAI early access",
|
||||||
|
"paid": "Paid",
|
||||||
|
"paidTooltip": "This version requires payment to download",
|
||||||
"ignored": "Ignored",
|
"ignored": "Ignored",
|
||||||
"ignoredTooltip": "Update notifications are disabled for this version",
|
"ignoredTooltip": "Update notifications are disabled for this version",
|
||||||
"onSiteOnly": "On-Site Only",
|
"onSiteOnly": "On-Site Only",
|
||||||
"onSiteOnlyTooltip": "This version is only available for on-site generation on Civitai"
|
"onSiteOnlyTooltip": "This version is only available for on-site generation on CivitAI"
|
||||||
},
|
},
|
||||||
"actions": {
|
"actions": {
|
||||||
"download": "Download",
|
"download": "Download",
|
||||||
"downloadTooltip": "Download this version",
|
"downloadTooltip": "Download this version",
|
||||||
"downloadEarlyAccessTooltip": "Download this early access version from Civitai",
|
"downloadChooseFilesTooltip": "Choose which files to download",
|
||||||
"downloadNotAllowedTooltip": "This version is only available for on-site generation on Civitai",
|
"downloadEarlyAccessTooltip": "Download this early access version from CivitAI",
|
||||||
|
"downloadPaidTooltip": "Download this paid version from CivitAI",
|
||||||
|
"downloadNotAllowedTooltip": "This version is only available for on-site generation on CivitAI",
|
||||||
"delete": "Delete",
|
"delete": "Delete",
|
||||||
"deleteTooltip": "Delete this local version",
|
"deleteTooltip": "Delete this local version",
|
||||||
"ignore": "Ignore",
|
"ignore": "Ignore",
|
||||||
"unignore": "Unignore",
|
"unignore": "Unignore",
|
||||||
"ignoreTooltip": "Ignore update notifications for this version",
|
"ignoreTooltip": "Ignore update notifications for this version",
|
||||||
"unignoreTooltip": "Resume update notifications for this version",
|
"unignoreTooltip": "Resume update notifications for this version",
|
||||||
"viewVersionOnCivitai": "View version on Civitai",
|
"viewVersionOnCivitai": "View version on CivitAI",
|
||||||
"earlyAccessTooltip": "Requires early access purchase",
|
"earlyAccessTooltip": "Requires early access purchase",
|
||||||
"resumeModelUpdates": "Resume updates for this model",
|
"resumeModelUpdates": "Resume updates for this model",
|
||||||
"ignoreModelUpdates": "Ignore updates for this model",
|
"ignoreModelUpdates": "Ignore updates for this model",
|
||||||
@@ -1547,7 +1750,7 @@
|
|||||||
},
|
},
|
||||||
"empty": "No version history available for this model yet.",
|
"empty": "No version history available for this model yet.",
|
||||||
"error": "Failed to load versions.",
|
"error": "Failed to load versions.",
|
||||||
"missingModelId": "This model is missing a Civitai model id.",
|
"missingModelId": "This model is missing a CivitAI model id.",
|
||||||
"hfGroupInfo": "This is a HuggingFace model group. Open the library to see all versions in the grid.",
|
"hfGroupInfo": "This is a HuggingFace model group. Open the library to see all versions in the grid.",
|
||||||
"confirm": {
|
"confirm": {
|
||||||
"delete": "Delete this version from your library?"
|
"delete": "Delete this version from your library?"
|
||||||
@@ -1575,6 +1778,21 @@
|
|||||||
"downloadCsv": "Download CSV",
|
"downloadCsv": "Download CSV",
|
||||||
"columnModelName": "Model Name",
|
"columnModelName": "Model Name",
|
||||||
"columnError": "Error"
|
"columnError": "Error"
|
||||||
|
},
|
||||||
|
"downloadBatchSummary": {
|
||||||
|
"title": "Batch Download Summary",
|
||||||
|
"statSuccess": "Success",
|
||||||
|
"statFailed": "Failed",
|
||||||
|
"statTotal": "Total",
|
||||||
|
"successMessage": "All {count} models downloaded successfully",
|
||||||
|
"completedWithErrors": "Completed with errors",
|
||||||
|
"failed": "Download failed",
|
||||||
|
"failedItems": "Failed Items ({count})",
|
||||||
|
"columnName": "Model Name",
|
||||||
|
"columnError": "Error",
|
||||||
|
"close": "Close",
|
||||||
|
"copyReport": "Copy Report",
|
||||||
|
"retryFailed": "Retry Failed ({count})"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"modelTags": {
|
"modelTags": {
|
||||||
@@ -1616,14 +1834,14 @@
|
|||||||
"tips": {
|
"tips": {
|
||||||
"title": "Tips & Tricks",
|
"title": "Tips & Tricks",
|
||||||
"civitai": {
|
"civitai": {
|
||||||
"title": "Civitai Integration",
|
"title": "CivitAI Integration",
|
||||||
"description": "Connect your Civitai account: Visit Profile Avatar → Settings → API Keys → Add API Key, then paste it in Lora Manager settings.",
|
"description": "Connect your CivitAI account: Visit Profile Avatar → Settings → API Keys → Add API Key, then paste it in Lora Manager settings.",
|
||||||
"alt": "Civitai API Setup"
|
"alt": "CivitAI API Setup"
|
||||||
},
|
},
|
||||||
"download": {
|
"download": {
|
||||||
"title": "Easy Download",
|
"title": "Easy Download",
|
||||||
"description": "Use Civitai URLs to quickly download and install new models.",
|
"description": "Use CivitAI URLs to quickly download and install new models.",
|
||||||
"alt": "Civitai Download"
|
"alt": "CivitAI Download"
|
||||||
},
|
},
|
||||||
"recipes": {
|
"recipes": {
|
||||||
"title": "Save Recipes",
|
"title": "Save Recipes",
|
||||||
@@ -1673,6 +1891,7 @@
|
|||||||
"recipeReplaced": "Recipe replaced in workflow",
|
"recipeReplaced": "Recipe replaced in workflow",
|
||||||
"recipeFailedToSend": "Failed to send recipe to workflow",
|
"recipeFailedToSend": "Failed to send recipe to workflow",
|
||||||
"noMatchingNodes": "No compatible nodes available in the current workflow",
|
"noMatchingNodes": "No compatible nodes available in the current workflow",
|
||||||
|
"noPromptTargets": "No compatible prompt targets in the workflow.\nRight-click a node in ComfyUI → Mark as → Send Prompt Target",
|
||||||
"noTargetNodeSelected": "No target node selected",
|
"noTargetNodeSelected": "No target node selected",
|
||||||
"modelUpdated": "Model updated in workflow",
|
"modelUpdated": "Model updated in workflow",
|
||||||
"modelFailed": "Failed to update model node",
|
"modelFailed": "Failed to update model node",
|
||||||
@@ -1806,7 +2025,7 @@
|
|||||||
"submitGithubIssue": "Submit GitHub Issue",
|
"submitGithubIssue": "Submit GitHub Issue",
|
||||||
"joinDiscord": "Join Discord",
|
"joinDiscord": "Join Discord",
|
||||||
"youtubeChannel": "YouTube Channel",
|
"youtubeChannel": "YouTube Channel",
|
||||||
"civitaiProfile": "Civitai Profile",
|
"civitaiProfile": "CivitAI Profile",
|
||||||
"supportKofi": "Support on Ko-fi",
|
"supportKofi": "Support on Ko-fi",
|
||||||
"supportPatreon": "Support on Patreon"
|
"supportPatreon": "Support on Patreon"
|
||||||
},
|
},
|
||||||
@@ -1849,6 +2068,7 @@
|
|||||||
"downloadPartialSuccess": "Downloaded {completed} of {total} LoRAs",
|
"downloadPartialSuccess": "Downloaded {completed} of {total} LoRAs",
|
||||||
"downloadPartialWithAccess": "Downloaded {completed} of {total} LoRAs. {accessFailures} failed due to access restrictions. Check your API key in settings or early access status.",
|
"downloadPartialWithAccess": "Downloaded {completed} of {total} LoRAs. {accessFailures} failed due to access restrictions. Check your API key in settings or early access status.",
|
||||||
"pleaseSelectVersion": "Please select a version",
|
"pleaseSelectVersion": "Please select a version",
|
||||||
|
"pleaseSelectFile": "Please select at least one file",
|
||||||
"versionExists": "This version already exists in your library",
|
"versionExists": "This version already exists in your library",
|
||||||
"downloadCompleted": "Download completed successfully",
|
"downloadCompleted": "Download completed successfully",
|
||||||
"downloadSkippedByBaseModel": "Skipped download because base model {baseModel} is excluded",
|
"downloadSkippedByBaseModel": "Skipped download because base model {baseModel} is excluded",
|
||||||
@@ -1882,11 +2102,16 @@
|
|||||||
"createMissingData": "Missing required data to create recipe",
|
"createMissingData": "Missing required data to create recipe",
|
||||||
"created": "Recipe created successfully",
|
"created": "Recipe created successfully",
|
||||||
"noMissingLoras": "No missing LoRAs to download",
|
"noMissingLoras": "No missing LoRAs to download",
|
||||||
|
"noPreviousRecipe": "No previous recipe available",
|
||||||
|
"noNextRecipe": "No next recipe available",
|
||||||
"missingLorasInfoFailed": "Failed to get information for missing LoRAs",
|
"missingLorasInfoFailed": "Failed to get information for missing LoRAs",
|
||||||
"preparingForDownloadFailed": "Error preparing LoRAs for download",
|
"preparingForDownloadFailed": "Error preparing LoRAs for download",
|
||||||
"enterLoraName": "Please enter a LoRA name or syntax",
|
"enterLoraName": "Please enter a LoRA name or syntax",
|
||||||
"reconnectedSuccessfully": "LoRA reconnected successfully",
|
"reconnectedSuccessfully": "LoRA reconnected successfully",
|
||||||
|
"reconnectBaseModelMismatch": "Reconnected, but base models differ (recipe: {recipe}, LoRA: {lora}) — they are architecture-compatible",
|
||||||
"reconnectFailed": "Error reconnecting LoRA: {message}",
|
"reconnectFailed": "Error reconnecting LoRA: {message}",
|
||||||
|
"loraRestored": "LoRA restored to its previous association",
|
||||||
|
"loraRestoreFailed": "Error restoring LoRA: {message}",
|
||||||
"noPromptToSend": "No prompt to send",
|
"noPromptToSend": "No prompt to send",
|
||||||
"cannotSend": "Cannot send recipe: Missing recipe ID",
|
"cannotSend": "Cannot send recipe: Missing recipe ID",
|
||||||
"sendFailed": "Failed to send recipe to workflow",
|
"sendFailed": "Failed to send recipe to workflow",
|
||||||
@@ -1894,6 +2119,16 @@
|
|||||||
"missingCheckpointPath": "Checkpoint path not available",
|
"missingCheckpointPath": "Checkpoint path not available",
|
||||||
"missingCheckpointInfo": "Missing checkpoint information",
|
"missingCheckpointInfo": "Missing checkpoint information",
|
||||||
"downloadCheckpointFailed": "Failed to download checkpoint: {message}",
|
"downloadCheckpointFailed": "Failed to download checkpoint: {message}",
|
||||||
|
"enterCheckpointName": "Please enter a checkpoint name",
|
||||||
|
"checkpointReconnectedSuccessfully": "Checkpoint reconnected successfully",
|
||||||
|
"reconnectCheckpointBaseModelMismatch": "Reconnected, but base models differ (recipe: {recipe}, checkpoint: {checkpoint}) — they are architecture-compatible",
|
||||||
|
"checkpointReconnectFailed": "Error reconnecting checkpoint: {message}",
|
||||||
|
"checkpointRestored": "Checkpoint restored to its previous association",
|
||||||
|
"checkpointRestoreFailed": "Error restoring checkpoint: {message}",
|
||||||
|
"checkpointDownloadUnavailable": "This checkpoint cannot be downloaded without CivitAI identifiers - try reconnecting it with a local checkpoint",
|
||||||
|
"missingLoraDownloadInfo": "Missing download information for this LoRA",
|
||||||
|
"hashNotFoundOnCivitai": "This LoRA hash cannot be resolved on CivitAI - the model may have been updated or the hash is invalid",
|
||||||
|
"downloadLoraFailed": "Failed to download LoRA: {message}",
|
||||||
"cannotDelete": "Cannot delete recipe: Missing recipe ID",
|
"cannotDelete": "Cannot delete recipe: Missing recipe ID",
|
||||||
"deleteConfirmationError": "Error showing delete confirmation",
|
"deleteConfirmationError": "Error showing delete confirmation",
|
||||||
"deletedSuccessfully": "Recipe deleted successfully",
|
"deletedSuccessfully": "Recipe deleted successfully",
|
||||||
@@ -1917,18 +2152,28 @@
|
|||||||
"batchImportCancelFailed": "Failed to cancel batch import: {message}",
|
"batchImportCancelFailed": "Failed to cancel batch import: {message}",
|
||||||
"batchImportNoUrls": "Please enter at least one URL or file path",
|
"batchImportNoUrls": "Please enter at least one URL or file path",
|
||||||
"batchImportNoDirectory": "Please enter a directory path",
|
"batchImportNoDirectory": "Please enter a directory path",
|
||||||
|
"batchImportRateLimited": "Metadata provider rate limit reached — requests are being slowed and some items may be skipped. You can re-run the import later.",
|
||||||
"batchImportBrowseFailed": "Failed to browse directory: {message}",
|
"batchImportBrowseFailed": "Failed to browse directory: {message}",
|
||||||
"batchImportDirectorySelected": "Directory selected: {path}",
|
"batchImportDirectorySelected": "Directory selected: {path}",
|
||||||
"noRecipesSelected": "No recipes selected",
|
"noRecipesSelected": "No recipes selected",
|
||||||
"repairBulkComplete": "Repair complete: {repaired} repaired, {skipped} skipped (of {total})",
|
"repairBulkComplete": "Repair complete: {repaired} repaired, {skipped} skipped (of {total})",
|
||||||
"repairBulkSkipped": "No repair needed for any of the {total} selected recipes",
|
"repairBulkSkipped": "No repair needed for any of the {total} selected recipes",
|
||||||
"repairBulkFailed": "Failed to repair selected recipes: {message}",
|
"repairBulkFailed": "Failed to repair selected recipes: {message}",
|
||||||
|
"rematchComplete": "Matched {entries} entries across {recipes} recipes",
|
||||||
|
"rematchCompleteErrors": "Matched {entries} entries across {recipes} recipes, {failures} failed",
|
||||||
|
"rematchAllFailed": "Rematch failed for {failures} of {total} selected recipes",
|
||||||
|
"rematchUnmatched": "No local match found for {entries} entries in {recipes} recipes",
|
||||||
|
"rematchSkipped": "No rematch needed for any of the {total} selected recipes",
|
||||||
|
"rematchFailed": "Failed to rematch selected recipes: {message}",
|
||||||
"reimporting": "Re-importing recipe from source...",
|
"reimporting": "Re-importing recipe from source...",
|
||||||
"reimportSuccess": "Recipe re-imported successfully",
|
"reimportSuccess": "Recipe re-imported successfully",
|
||||||
"reimportBulkComplete": "Re-import complete: {completed} re-imported, {failed} failed (of {total})",
|
"reimportBulkComplete": "Re-import complete: {completed} re-imported, {failed} failed (of {total})",
|
||||||
"reimportBulkFailed": "Failed to re-import some recipes",
|
"reimportBulkFailed": "Failed to re-import some recipes",
|
||||||
"noMissingLorasInSelection": "No missing LoRAs found in selected recipes",
|
"noMissingLorasInSelection": "No missing LoRAs found in selected recipes",
|
||||||
"noLoraRootConfigured": "No LoRA root directory configured. Please set a default LoRA root in settings."
|
"noLoraRootConfigured": "No LoRA root directory configured. Please set a default LoRA root in settings.",
|
||||||
|
"workflowSent": "Workflow sent to ComfyUI",
|
||||||
|
"workflowSendFailed": "Failed to send workflow to ComfyUI: {error}",
|
||||||
|
"workflowNoWorkflow": "No embedded workflow found in this recipe"
|
||||||
},
|
},
|
||||||
"models": {
|
"models": {
|
||||||
"noModelsSelected": "No models selected",
|
"noModelsSelected": "No models selected",
|
||||||
@@ -1965,8 +2210,8 @@
|
|||||||
"bulkUpdatesChecking": "Checking selected {type}(s) for updates...",
|
"bulkUpdatesChecking": "Checking selected {type}(s) for updates...",
|
||||||
"bulkUpdatesSuccess": "Updates available for {count} selected {type}(s)",
|
"bulkUpdatesSuccess": "Updates available for {count} selected {type}(s)",
|
||||||
"bulkUpdatesNone": "No updates found for selected {type}(s)",
|
"bulkUpdatesNone": "No updates found for selected {type}(s)",
|
||||||
"bulkUpdatesMissing": "Selected {type}(s) are not linked to Civitai updates",
|
"bulkUpdatesMissing": "Selected {type}(s) are not linked to CivitAI updates",
|
||||||
"bulkUpdatesPartialMissing": "Skipped {missing} selected {type}(s) without Civitai links",
|
"bulkUpdatesPartialMissing": "Skipped {missing} selected {type}(s) without CivitAI links",
|
||||||
"bulkUpdatesFailed": "Failed to check updates for selected {type}(s): {message}",
|
"bulkUpdatesFailed": "Failed to check updates for selected {type}(s): {message}",
|
||||||
"invalidCharactersRemoved": "Invalid characters removed from filename",
|
"invalidCharactersRemoved": "Invalid characters removed from filename",
|
||||||
"filenameCannotBeEmpty": "File name cannot be empty",
|
"filenameCannotBeEmpty": "File name cannot be empty",
|
||||||
@@ -2031,7 +2276,6 @@
|
|||||||
"presetNameTooLong": "Preset name must be {max} characters or less",
|
"presetNameTooLong": "Preset name must be {max} characters or less",
|
||||||
"presetNameInvalidChars": "Preset name contains invalid characters",
|
"presetNameInvalidChars": "Preset name contains invalid characters",
|
||||||
"presetNameExists": "A preset with this name already exists",
|
"presetNameExists": "A preset with this name already exists",
|
||||||
"maxPresetsReached": "Maximum {max} presets allowed. Delete one to add more.",
|
|
||||||
"presetNotFound": "Preset not found",
|
"presetNotFound": "Preset not found",
|
||||||
"invalidPreset": "Invalid preset data",
|
"invalidPreset": "Invalid preset data",
|
||||||
"deletePresetFailed": "Failed to delete preset",
|
"deletePresetFailed": "Failed to delete preset",
|
||||||
@@ -2060,6 +2304,14 @@
|
|||||||
"updateFailed": "Failed to update trigger words",
|
"updateFailed": "Failed to update trigger words",
|
||||||
"copyFailed": "Copy failed"
|
"copyFailed": "Copy failed"
|
||||||
},
|
},
|
||||||
|
"undo": {
|
||||||
|
"action": "Undo",
|
||||||
|
"deleted": "Deleted {name}",
|
||||||
|
"deletedBulk": "Deleted {count} item(s)",
|
||||||
|
"expired": "Undo window expired. The item was permanently deleted.",
|
||||||
|
"failed": "Undo failed: {error}",
|
||||||
|
"restored": "Item restored"
|
||||||
|
},
|
||||||
"virtual": {
|
"virtual": {
|
||||||
"loadFailed": "Failed to load items",
|
"loadFailed": "Failed to load items",
|
||||||
"loadMoreFailed": "Failed to load more items",
|
"loadMoreFailed": "Failed to load more items",
|
||||||
@@ -2085,10 +2337,11 @@
|
|||||||
"contextMenu": {
|
"contextMenu": {
|
||||||
"contentRatingSet": "Content rating set to {level}",
|
"contentRatingSet": "Content rating set to {level}",
|
||||||
"contentRatingFailed": "Failed to set content rating: {message}",
|
"contentRatingFailed": "Failed to set content rating: {message}",
|
||||||
"relinkSuccess": "Model successfully re-linked to Civitai",
|
"relinkSuccess": "Model successfully re-linked to CivitAI",
|
||||||
"relinkFailed": "Error: {message}",
|
"relinkFailed": "Error: {message}",
|
||||||
"linkHfSuccess": "Model successfully linked to HuggingFace",
|
"linkHfSuccess": "Model successfully linked to HuggingFace",
|
||||||
"linkHfFailed": "Error: {message}",
|
"linkHfFailed": "Error: {message}",
|
||||||
|
"linkCivArchSuccess": "Model successfully re-linked via CivitArchive",
|
||||||
"fetchMetadataFirst": "Please fetch metadata from CivitAI first",
|
"fetchMetadataFirst": "Please fetch metadata from CivitAI first",
|
||||||
"noCivitaiInfo": "No CivitAI information available",
|
"noCivitaiInfo": "No CivitAI information available",
|
||||||
"missingHash": "Model hash not available"
|
"missingHash": "Model hash not available"
|
||||||
@@ -2123,6 +2376,7 @@
|
|||||||
"fileRenameFailed": "Failed to rename file: {error}",
|
"fileRenameFailed": "Failed to rename file: {error}",
|
||||||
"previewUpdated": "Preview updated successfully",
|
"previewUpdated": "Preview updated successfully",
|
||||||
"previewUploadFailed": "Failed to upload preview image",
|
"previewUploadFailed": "Failed to upload preview image",
|
||||||
|
"previewDropInvalid": "Unsupported file type: {name}. Drop an image or MP4 video instead.",
|
||||||
"refreshComplete": "{action} complete",
|
"refreshComplete": "{action} complete",
|
||||||
"refreshFailed": "Failed to {action} {type}s",
|
"refreshFailed": "Failed to {action} {type}s",
|
||||||
"metadataRefreshed": "Metadata refreshed successfully",
|
"metadataRefreshed": "Metadata refreshed successfully",
|
||||||
@@ -2177,7 +2431,7 @@
|
|||||||
},
|
},
|
||||||
"issues": {
|
"issues": {
|
||||||
"civitai_api_key": {
|
"civitai_api_key": {
|
||||||
"title": "Civitai API Key"
|
"title": "CivitAI API Key"
|
||||||
},
|
},
|
||||||
"cache_health": {
|
"cache_health": {
|
||||||
"title": "Model Cache Health"
|
"title": "Model Cache Health"
|
||||||
@@ -2231,9 +2485,9 @@
|
|||||||
},
|
},
|
||||||
"communitySupport": {
|
"communitySupport": {
|
||||||
"title": "Keep LoRA Manager Thriving with Your Support ❤️",
|
"title": "Keep LoRA Manager Thriving with Your Support ❤️",
|
||||||
"content": "LoRA Manager is a passion project maintained full-time by a solo developer. Your support on Ko-fi helps cover development costs, keeps new updates coming, and unlocks a license key for the LM Civitai Extension as a thank-you gift. Every contribution truly makes a difference.",
|
"content": "LoRA Manager is a passion project maintained full-time by a solo developer. Your support on Ko-fi helps cover development costs, keeps new updates coming, and unlocks a license key for the LM CivitAI Extension as a thank-you gift. Every contribution truly makes a difference.",
|
||||||
"supportCta": "Support on Ko-fi",
|
"supportCta": "Support on Ko-fi",
|
||||||
"learnMore": "LM Civitai Extension Tutorial"
|
"learnMore": "LM CivitAI Extension Tutorial"
|
||||||
},
|
},
|
||||||
"cacheHealth": {
|
"cacheHealth": {
|
||||||
"corrupted": {
|
"corrupted": {
|
||||||
@@ -2250,4 +2504,4 @@
|
|||||||
"retry": "Retry"
|
"retry": "Retry"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+2476
-2222
File diff suppressed because it is too large
Load Diff
+2476
-2222
File diff suppressed because it is too large
Load Diff
+2476
-2222
File diff suppressed because it is too large
Load Diff
+2476
-2222
File diff suppressed because it is too large
Load Diff
+2476
-2222
File diff suppressed because it is too large
Load Diff
+2476
-2222
File diff suppressed because it is too large
Load Diff
+2476
-2222
File diff suppressed because it is too large
Load Diff
+2476
-2222
File diff suppressed because it is too large
Load Diff
+15
-10
@@ -1,9 +1,13 @@
|
|||||||
|
# pyright: reportImportCycles=false
|
||||||
|
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||||
|
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||||
|
# import cycles. Breaking them would require an architectural refactor.
|
||||||
import os
|
import os
|
||||||
import platform
|
import platform
|
||||||
import posixpath
|
import posixpath
|
||||||
import threading
|
import threading
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import folder_paths # type: ignore
|
import folder_paths # pyright: ignore[reportMissingImports]
|
||||||
from typing import Any, Dict, Iterable, List, Mapping, Optional, Set, Tuple
|
from typing import Any, Dict, Iterable, List, Mapping, Optional, Set, Tuple
|
||||||
import logging
|
import logging
|
||||||
import json
|
import json
|
||||||
@@ -90,7 +94,7 @@ def _resolve_valid_default_root(
|
|||||||
|
|
||||||
|
|
||||||
def _normalize_folder_paths_for_comparison(
|
def _normalize_folder_paths_for_comparison(
|
||||||
folder_paths: Mapping[str, Iterable[str]],
|
folder_paths: Mapping[str, Any],
|
||||||
) -> Dict[str, Set[str]]:
|
) -> Dict[str, Set[str]]:
|
||||||
"""Normalize folder paths for comparison across libraries."""
|
"""Normalize folder paths for comparison across libraries."""
|
||||||
|
|
||||||
@@ -482,7 +486,7 @@ class Config:
|
|||||||
import ctypes
|
import ctypes
|
||||||
|
|
||||||
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
|
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
|
||||||
attrs = ctypes.windll.kernel32.GetFileAttributesW(str(path)) # type: ignore[attr-defined]
|
attrs = ctypes.windll.kernel32.GetFileAttributesW(str(path)) # pyright: ignore[reportAttributeAccessIssue]
|
||||||
return attrs != -1 and (attrs & FILE_ATTRIBUTE_REPARSE_POINT)
|
return attrs != -1 and (attrs & FILE_ATTRIBUTE_REPARSE_POINT)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error checking Windows reparse point: {e}")
|
logger.error(f"Error checking Windows reparse point: {e}")
|
||||||
@@ -491,7 +495,7 @@ class Config:
|
|||||||
logger.error(f"Error checking link status for {path}: {e}")
|
logger.error(f"Error checking link status for {path}: {e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def _entry_is_symlink(self, entry: os.DirEntry) -> bool:
|
def _entry_is_symlink(self, entry: os.DirEntry[str]) -> bool:
|
||||||
"""Check if a directory entry is a symlink, including Windows junctions."""
|
"""Check if a directory entry is a symlink, including Windows junctions."""
|
||||||
if entry.is_symlink():
|
if entry.is_symlink():
|
||||||
return True
|
return True
|
||||||
@@ -500,7 +504,7 @@ class Config:
|
|||||||
import ctypes
|
import ctypes
|
||||||
|
|
||||||
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
|
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
|
||||||
attrs = ctypes.windll.kernel32.GetFileAttributesW(entry.path) # type: ignore[attr-defined]
|
attrs = ctypes.windll.kernel32.GetFileAttributesW(entry.path) # pyright: ignore[reportAttributeAccessIssue]
|
||||||
return attrs != -1 and (attrs & FILE_ATTRIBUTE_REPARSE_POINT)
|
return attrs != -1 and (attrs & FILE_ATTRIBUTE_REPARSE_POINT)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
@@ -1126,8 +1130,8 @@ class Config:
|
|||||||
|
|
||||||
def _apply_library_paths(
|
def _apply_library_paths(
|
||||||
self,
|
self,
|
||||||
folder_paths: Mapping[str, Iterable[str]],
|
folder_paths: Mapping[str, Any],
|
||||||
extra_folder_paths: Optional[Mapping[str, Iterable[str]]] = None,
|
extra_folder_paths: Optional[Mapping[str, Any]] = None,
|
||||||
recipes_path: str = "",
|
recipes_path: str = "",
|
||||||
) -> None:
|
) -> None:
|
||||||
self._path_mappings.clear()
|
self._path_mappings.clear()
|
||||||
@@ -1432,12 +1436,13 @@ class Config:
|
|||||||
# ('_lm_config_cache') that is NEVER removed from sys.modules (its key does
|
# ('_lm_config_cache') that is NEVER removed from sys.modules (its key does
|
||||||
# NOT start with 'py.'), so it survives re-imports of py.* modules.
|
# NOT start with 'py.'), so it survives re-imports of py.* modules.
|
||||||
_CONFIG_SENTINEL = "_lm_config_cache"
|
_CONFIG_SENTINEL = "_lm_config_cache"
|
||||||
|
config: Config
|
||||||
if _CONFIG_SENTINEL in _sys.modules:
|
if _CONFIG_SENTINEL in _sys.modules:
|
||||||
# Re-import: reuse the existing singleton from the sentinel.
|
# Re-import: reuse the existing singleton from the sentinel.
|
||||||
config: Config = _sys.modules[_CONFIG_SENTINEL].config # type: ignore[valid-type]
|
config = _sys.modules[_CONFIG_SENTINEL].config
|
||||||
else:
|
else:
|
||||||
config: Config = Config()
|
config = Config()
|
||||||
# Register the sentinel so re-imports of py.config find us.
|
# Register the sentinel so re-imports of py.config find us.
|
||||||
_sentinel_mod = _types.ModuleType(_CONFIG_SENTINEL)
|
_sentinel_mod = _types.ModuleType(_CONFIG_SENTINEL)
|
||||||
_sentinel_mod.config = config
|
setattr(_sentinel_mod, "config", config)
|
||||||
_sys.modules[_CONFIG_SENTINEL] = _sentinel_mod
|
_sys.modules[_CONFIG_SENTINEL] = _sentinel_mod
|
||||||
|
|||||||
+18
-1
@@ -14,7 +14,7 @@ standalone_mode = (
|
|||||||
if not standalone_mode:
|
if not standalone_mode:
|
||||||
setup_logging()
|
setup_logging()
|
||||||
|
|
||||||
from server import PromptServer # type: ignore
|
from server import PromptServer # pyright: ignore[reportMissingImports]
|
||||||
|
|
||||||
from .config import config
|
from .config import config
|
||||||
from .services.model_service_factory import (
|
from .services.model_service_factory import (
|
||||||
@@ -25,10 +25,12 @@ from .routes.recipe_routes import RecipeRoutes
|
|||||||
from .routes.stats_routes import StatsRoutes
|
from .routes.stats_routes import StatsRoutes
|
||||||
from .routes.update_routes import UpdateRoutes
|
from .routes.update_routes import UpdateRoutes
|
||||||
from .routes.misc_routes import MiscRoutes
|
from .routes.misc_routes import MiscRoutes
|
||||||
|
from .routes.pending_delete_routes import PendingDeleteRoutes
|
||||||
from .routes.preview_routes import PreviewRoutes
|
from .routes.preview_routes import PreviewRoutes
|
||||||
from .routes.example_images_routes import ExampleImagesRoutes
|
from .routes.example_images_routes import ExampleImagesRoutes
|
||||||
from .services.service_registry import ServiceRegistry
|
from .services.service_registry import ServiceRegistry
|
||||||
from .services.settings_manager import get_settings_manager
|
from .services.settings_manager import get_settings_manager
|
||||||
|
from .services.pending_delete_service import get_pending_delete_service
|
||||||
from .utils.example_images_migration import ExampleImagesMigration
|
from .utils.example_images_migration import ExampleImagesMigration
|
||||||
from .services.websocket_manager import ws_manager
|
from .services.websocket_manager import ws_manager
|
||||||
from .services.example_images_cleanup_service import ExampleImagesCleanupService
|
from .services.example_images_cleanup_service import ExampleImagesCleanupService
|
||||||
@@ -170,6 +172,7 @@ class LoraManager:
|
|||||||
RecipeRoutes.setup_routes(app)
|
RecipeRoutes.setup_routes(app)
|
||||||
UpdateRoutes.setup_routes(app)
|
UpdateRoutes.setup_routes(app)
|
||||||
MiscRoutes.setup_routes(app)
|
MiscRoutes.setup_routes(app)
|
||||||
|
PendingDeleteRoutes.setup_routes(app)
|
||||||
ExampleImagesRoutes.setup_routes(app, ws_manager=ws_manager)
|
ExampleImagesRoutes.setup_routes(app, ws_manager=ws_manager)
|
||||||
PreviewRoutes.setup_routes(app)
|
PreviewRoutes.setup_routes(app)
|
||||||
|
|
||||||
@@ -245,6 +248,20 @@ class LoraManager:
|
|||||||
cls._run_post_initialization_tasks(init_tasks), name="post_init_tasks"
|
cls._run_post_initialization_tasks(init_tasks), name="post_init_tasks"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Startup sweep: purge pending-delete batches that expired during a
|
||||||
|
# previous run. Non-blocking (fire-and-forget); purge_expired only
|
||||||
|
# removes already-expired batches, so a staged undo that survived a
|
||||||
|
# restart stays restorable. scan_roots=True runs the reconciliation
|
||||||
|
# pass first so leftover batches (the in-process registry is empty
|
||||||
|
# after a restart) are re-discovered on disk. Covers both plugin
|
||||||
|
# and standalone modes (StandaloneLoraManager reuses this
|
||||||
|
# classmethod).
|
||||||
|
pending_delete_service = await get_pending_delete_service()
|
||||||
|
asyncio.create_task(
|
||||||
|
pending_delete_service.purge_expired(scan_roots=True),
|
||||||
|
name="pending_delete_startup_sweep",
|
||||||
|
)
|
||||||
|
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"LoRA Manager: All services initialized and background tasks scheduled"
|
"LoRA Manager: All services initialized and background tasks scheduled"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ if not standalone_mode:
|
|||||||
|
|
||||||
logger.info("ComfyUI Metadata Collector initialized")
|
logger.info("ComfyUI Metadata Collector initialized")
|
||||||
|
|
||||||
def get_metadata(prompt_id=None): # type: ignore[no-redef]
|
def get_metadata(prompt_id=None): # pyright: ignore[reportRedeclaration]
|
||||||
"""Helper function to get metadata from the registry"""
|
"""Helper function to get metadata from the registry"""
|
||||||
registry = MetadataRegistry()
|
registry = MetadataRegistry()
|
||||||
return registry.get_metadata(prompt_id)
|
return registry.get_metadata(prompt_id)
|
||||||
@@ -31,6 +31,6 @@ else:
|
|||||||
def init():
|
def init():
|
||||||
logger.info("ComfyUI Metadata Collector disabled in standalone mode")
|
logger.info("ComfyUI Metadata Collector disabled in standalone mode")
|
||||||
|
|
||||||
def get_metadata(prompt_id=None): # type: ignore[no-redef]
|
def get_metadata(prompt_id=None): # pyright: ignore[reportRedeclaration]
|
||||||
"""Dummy implementation for standalone mode"""
|
"""Dummy implementation for standalone mode"""
|
||||||
return {}
|
return {}
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ class MetadataHook:
|
|||||||
execution = None
|
execution = None
|
||||||
try:
|
try:
|
||||||
# Try direct import first
|
# Try direct import first
|
||||||
import execution # type: ignore
|
import execution # pyright: ignore[reportMissingImports]
|
||||||
except ImportError:
|
except ImportError:
|
||||||
# Try to locate from system modules
|
# Try to locate from system modules
|
||||||
for module_name in sys.modules:
|
for module_name in sys.modules:
|
||||||
|
|||||||
@@ -214,6 +214,24 @@ class MetadataProcessor:
|
|||||||
max_denoise = denoise
|
max_denoise = denoise
|
||||||
primary_sampler = sampler_info
|
primary_sampler = sampler_info
|
||||||
primary_sampler_id = node_id
|
primary_sampler_id = node_id
|
||||||
|
|
||||||
|
# Last resort: any registered sampler. Samplers without a denoise or
|
||||||
|
# add_noise parameter (e.g. multi-stage samplers like KreaTwoStageSampler)
|
||||||
|
# are not caught by the criteria above. Prefer execution order so the
|
||||||
|
# first executed sampler wins, matching the downstream_id branch.
|
||||||
|
if primary_sampler is None:
|
||||||
|
sampler_ids = [
|
||||||
|
node_id
|
||||||
|
for node_id, sampler_info in metadata.get(SAMPLING, {}).items()
|
||||||
|
if sampler_info.get(IS_SAMPLER, False)
|
||||||
|
]
|
||||||
|
if sampler_ids:
|
||||||
|
if downstream_id and "execution_order" in metadata:
|
||||||
|
for node_id in metadata["execution_order"]:
|
||||||
|
if node_id in sampler_ids:
|
||||||
|
return node_id, metadata[SAMPLING][node_id]
|
||||||
|
primary_sampler_id = sampler_ids[0]
|
||||||
|
primary_sampler = metadata[SAMPLING][sampler_ids[0]]
|
||||||
|
|
||||||
return primary_sampler_id, primary_sampler
|
return primary_sampler_id, primary_sampler
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import time
|
import time
|
||||||
from nodes import NODE_CLASS_MAPPINGS # type: ignore
|
from typing import Any
|
||||||
|
from nodes import NODE_CLASS_MAPPINGS # pyright: ignore[reportMissingImports, reportAttributeAccessIssue]
|
||||||
from .node_extractors import NODE_EXTRACTORS, GenericNodeExtractor
|
from .node_extractors import NODE_EXTRACTORS, GenericNodeExtractor
|
||||||
from .constants import METADATA_CATEGORIES, IMAGES, OVERWRITE
|
from .constants import METADATA_CATEGORIES, IMAGES, OVERWRITE
|
||||||
|
|
||||||
@@ -9,6 +10,15 @@ class MetadataRegistry:
|
|||||||
|
|
||||||
_instance = None
|
_instance = None
|
||||||
|
|
||||||
|
current_prompt_id: Any = None
|
||||||
|
current_prompt: Any = None
|
||||||
|
metadata: dict[str, Any] = {}
|
||||||
|
prompt_metadata: dict[str, Any] = {}
|
||||||
|
executed_nodes: set[str] = set()
|
||||||
|
node_cache: dict[str, Any] = {}
|
||||||
|
max_prompt_history: int = 3
|
||||||
|
metadata_categories: list[str] = METADATA_CATEGORIES
|
||||||
|
|
||||||
def __new__(cls):
|
def __new__(cls):
|
||||||
if cls._instance is None:
|
if cls._instance is None:
|
||||||
cls._instance = super().__new__(cls)
|
cls._instance = super().__new__(cls)
|
||||||
|
|||||||
@@ -2,7 +2,8 @@ import json
|
|||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from .constants import CLIP_SKIP_SENTINEL, MODELS, PROMPTS, SAMPLING, LORAS, SIZE, IMAGES, IS_SAMPLER, OVERWRITE, METADATA_OVERWRITE_FIELDS
|
from .constants import MODELS, PROMPTS, SAMPLING, LORAS, SIZE, IMAGES, IS_SAMPLER, OVERWRITE
|
||||||
|
from .overwrite_utils import collect_overwrite_params
|
||||||
|
|
||||||
|
|
||||||
def _store_checkpoint_metadata(metadata, node_id, model_name):
|
def _store_checkpoint_metadata(metadata, node_id, model_name):
|
||||||
@@ -39,7 +40,7 @@ class GenericNodeExtractor(NodeMetadataExtractor):
|
|||||||
* ``MODEL`` output: common input fields (ckpt_name, unet_name, etc.)
|
* ``MODEL`` output: common input fields (ckpt_name, unet_name, etc.)
|
||||||
are checked for a model file name and stored as checkpoint metadata.
|
are checked for a model file name and stored as checkpoint metadata.
|
||||||
* ``CONDITIONING`` output: common text input fields are checked for
|
* ``CONDITIONING`` output: common text input fields are checked for
|
||||||
prompt text and stored as prompt metadata.
|
prompt text, and conditioning inputs are tracked through transforms.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# Input field names that carry a model path in loader-style nodes.
|
# Input field names that carry a model path in loader-style nodes.
|
||||||
@@ -72,7 +73,7 @@ class GenericNodeExtractor(NodeMetadataExtractor):
|
|||||||
_store_checkpoint_metadata(metadata, node_id, name)
|
_store_checkpoint_metadata(metadata, node_id, name)
|
||||||
return
|
return
|
||||||
|
|
||||||
# — CONDITIONING encoder detection (CLIPTextEncode, Flux, custom) —
|
# — CONDITIONING encoder / transform detection —
|
||||||
if "CONDITIONING" in return_types or any("CONDITIONING" in str(t) for t in return_types):
|
if "CONDITIONING" in return_types or any("CONDITIONING" in str(t) for t in return_types):
|
||||||
text = None
|
text = None
|
||||||
for field in GenericNodeExtractor._TEXT_FIELDS:
|
for field in GenericNodeExtractor._TEXT_FIELDS:
|
||||||
@@ -80,12 +81,14 @@ class GenericNodeExtractor(NodeMetadataExtractor):
|
|||||||
if val and isinstance(val, str) and val.strip():
|
if val and isinstance(val, str) and val.strip():
|
||||||
text = val.strip()
|
text = val.strip()
|
||||||
break
|
break
|
||||||
if text:
|
|
||||||
prompt_data = metadata.setdefault(PROMPTS, {})
|
input_conditionings = _collect_conditioning_inputs(inputs)
|
||||||
prompt_data[node_id] = {
|
if text or input_conditionings:
|
||||||
"text": text,
|
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||||
"node_id": node_id,
|
if text:
|
||||||
}
|
prompt_metadata["text"] = text
|
||||||
|
if input_conditionings:
|
||||||
|
prompt_metadata["orig_conditionings"] = input_conditionings
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def update(node_id, outputs, metadata, return_types=None):
|
def update(node_id, outputs, metadata, return_types=None):
|
||||||
@@ -97,11 +100,26 @@ class GenericNodeExtractor(NodeMetadataExtractor):
|
|||||||
return
|
return
|
||||||
if node_id not in metadata.get(PROMPTS, {}):
|
if node_id not in metadata.get(PROMPTS, {}):
|
||||||
return
|
return
|
||||||
if outputs and isinstance(outputs, list) and len(outputs) > 0:
|
output_tuple = _first_output_tuple(outputs)
|
||||||
if isinstance(outputs[0], tuple) and len(outputs[0]) > 0:
|
if not output_tuple or len(output_tuple) < 1:
|
||||||
cond = outputs[0][0]
|
return
|
||||||
if cond is not None:
|
|
||||||
metadata[PROMPTS][node_id]["conditioning"] = cond
|
conditioning_index = _first_conditioning_index(return_types)
|
||||||
|
if conditioning_index is None or len(output_tuple) <= conditioning_index:
|
||||||
|
return
|
||||||
|
|
||||||
|
output_conditioning = output_tuple[conditioning_index]
|
||||||
|
if output_conditioning is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
prompt_metadata = metadata[PROMPTS][node_id]
|
||||||
|
prompt_metadata["conditioning"] = output_conditioning
|
||||||
|
_record_conditioning_source(
|
||||||
|
metadata,
|
||||||
|
node_id,
|
||||||
|
output_conditioning,
|
||||||
|
prompt_metadata.get("orig_conditionings", []),
|
||||||
|
)
|
||||||
|
|
||||||
class CheckpointLoaderExtractor(NodeMetadataExtractor):
|
class CheckpointLoaderExtractor(NodeMetadataExtractor):
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -416,6 +434,34 @@ def _first_output_tuple(outputs):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _first_conditioning_index(return_types):
|
||||||
|
"""Return the index of the first CONDITIONING output slot, or None."""
|
||||||
|
if not return_types:
|
||||||
|
return None
|
||||||
|
for index, return_type in enumerate(return_types):
|
||||||
|
if "CONDITIONING" in str(return_type):
|
||||||
|
return index
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _collect_conditioning_inputs(inputs):
|
||||||
|
"""Collect conditioning object inputs (``conditioning*`` keys).
|
||||||
|
|
||||||
|
Primitive values (None, str, int, float, bool) are excluded so scalar
|
||||||
|
fields like ``conditioning_strength`` are not mistaken for conditioning
|
||||||
|
objects during provenance tracking.
|
||||||
|
"""
|
||||||
|
if not inputs:
|
||||||
|
return []
|
||||||
|
return [
|
||||||
|
value
|
||||||
|
for input_name, value in inputs.items()
|
||||||
|
if input_name.startswith("conditioning")
|
||||||
|
and value is not None
|
||||||
|
and not isinstance(value, (str, int, float, bool))
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def _record_conditioning_source(
|
def _record_conditioning_source(
|
||||||
metadata, node_id, output_conditioning, input_conditionings
|
metadata, node_id, output_conditioning, input_conditionings
|
||||||
):
|
):
|
||||||
@@ -428,6 +474,14 @@ def _record_conditioning_source(
|
|||||||
if not sources:
|
if not sources:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# Identity-preserving selectors return one of their inputs unchanged:
|
||||||
|
# only that input contributed to the output, so record it alone instead
|
||||||
|
# of treating every input as a combination source.
|
||||||
|
for conditioning in sources:
|
||||||
|
if id(conditioning) == id(output_conditioning):
|
||||||
|
sources = [conditioning]
|
||||||
|
break
|
||||||
|
|
||||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||||
prompt_metadata.setdefault("conditioning_sources", []).append(
|
prompt_metadata.setdefault("conditioning_sources", []).append(
|
||||||
{
|
{
|
||||||
@@ -507,13 +561,7 @@ class ConditioningCombineExtractor(NodeMetadataExtractor):
|
|||||||
if not inputs:
|
if not inputs:
|
||||||
return
|
return
|
||||||
|
|
||||||
input_conditionings = []
|
input_conditionings = _collect_conditioning_inputs(inputs)
|
||||||
for input_name in inputs:
|
|
||||||
if (
|
|
||||||
input_name.startswith("conditioning")
|
|
||||||
and inputs[input_name] is not None
|
|
||||||
):
|
|
||||||
input_conditionings.append(inputs[input_name])
|
|
||||||
|
|
||||||
if input_conditionings:
|
if input_conditionings:
|
||||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||||
@@ -813,6 +861,65 @@ class TSCKSamplerAdvancedExtractor(KSamplerAdvancedExtractor, TSCSamplerBaseExtr
|
|||||||
|
|
||||||
# Update method is inherited from TSCSamplerBaseExtractor
|
# Update method is inherited from TSCSamplerBaseExtractor
|
||||||
|
|
||||||
|
class KreaTwoStageSamplerExtractor(BaseSamplerExtractor):
|
||||||
|
"""Extractor for Krea Two/Three Stage Samplers (Auryg/Krea-2-Two-Stage-Sampler).
|
||||||
|
|
||||||
|
The node samples in two (or three) stages with per-stage settings
|
||||||
|
(stage1_steps/stage2_steps, stage1_cfg/stage2_cfg, ...). The canonical
|
||||||
|
metadata fields consumed by ``extract_generation_params`` (steps, cfg,
|
||||||
|
sampler_name, scheduler) are derived from the base stage (stage 1; the
|
||||||
|
three-stage variant reuses stage 1 settings for stage 3), while the full
|
||||||
|
per-stage breakdown is preserved in the raw parameters.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# All per-stage parameter keys present on both node variants.
|
||||||
|
_STAGE_PARAM_KEYS = (
|
||||||
|
"stage1_steps", "stage1_cfg", "stage1_sampler_name", "stage1_scheduler",
|
||||||
|
"stage2_steps", "stage2_cfg", "stage2_sampler_name", "stage2_scheduler",
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def extract(node_id, inputs, outputs, metadata):
|
||||||
|
if not inputs:
|
||||||
|
return
|
||||||
|
|
||||||
|
BaseSamplerExtractor.extract_sampling_params(
|
||||||
|
node_id,
|
||||||
|
inputs,
|
||||||
|
metadata,
|
||||||
|
("seed", "handoff_percent", "stage3_handoff_percent")
|
||||||
|
+ KreaTwoStageSamplerExtractor._STAGE_PARAM_KEYS,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Derive the canonical fields expected by extract_generation_params.
|
||||||
|
sampling_params = metadata[SAMPLING][node_id]["parameters"]
|
||||||
|
if "stage1_steps" in sampling_params or "stage2_steps" in sampling_params:
|
||||||
|
sampling_params["steps"] = (
|
||||||
|
(sampling_params.get("stage1_steps") or 0)
|
||||||
|
+ (sampling_params.get("stage2_steps") or 0)
|
||||||
|
)
|
||||||
|
if "stage1_cfg" in sampling_params:
|
||||||
|
sampling_params["cfg"] = sampling_params["stage1_cfg"]
|
||||||
|
if "stage1_sampler_name" in sampling_params:
|
||||||
|
sampling_params["sampler_name"] = sampling_params["stage1_sampler_name"]
|
||||||
|
if "stage1_scheduler" in sampling_params:
|
||||||
|
sampling_params["scheduler"] = sampling_params["stage1_scheduler"]
|
||||||
|
|
||||||
|
BaseSamplerExtractor.extract_conditioning(node_id, inputs, metadata)
|
||||||
|
|
||||||
|
# Prefer the final generation resolution; latent dims are the fallback.
|
||||||
|
BaseSamplerExtractor.extract_latent_dimensions(node_id, inputs, metadata)
|
||||||
|
final_width = inputs.get("final_width")
|
||||||
|
final_height = inputs.get("final_height")
|
||||||
|
if final_width and final_height:
|
||||||
|
if SIZE not in metadata:
|
||||||
|
metadata[SIZE] = {}
|
||||||
|
metadata[SIZE][node_id] = {
|
||||||
|
"width": final_width,
|
||||||
|
"height": final_height,
|
||||||
|
"node_id": node_id,
|
||||||
|
}
|
||||||
|
|
||||||
class LoraLoaderExtractor(NodeMetadataExtractor):
|
class LoraLoaderExtractor(NodeMetadataExtractor):
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def extract(node_id, inputs, outputs, metadata):
|
def extract(node_id, inputs, outputs, metadata):
|
||||||
@@ -853,6 +960,37 @@ class ImageSizeExtractor(NodeMetadataExtractor):
|
|||||||
"node_id": node_id
|
"node_id": node_id
|
||||||
}
|
}
|
||||||
|
|
||||||
|
class KreaDualResolutionSelectorExtractor(NodeMetadataExtractor):
|
||||||
|
"""Extract base resolution from Krea Dual Resolution Selector outputs
|
||||||
|
(Auryg/Krea-2-Two-Stage-Sampler).
|
||||||
|
|
||||||
|
The node computes base/final dimensions at runtime from aspect ratio and
|
||||||
|
megapixel settings, so the values are only available in the update phase
|
||||||
|
(outputs: base_width, base_height, final_width, final_height, seed).
|
||||||
|
"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def extract(node_id, inputs, outputs, metadata):
|
||||||
|
# Dimensions are computed at runtime; nothing to do here.
|
||||||
|
pass
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def update(node_id, outputs, metadata):
|
||||||
|
output_tuple = _first_output_tuple(outputs)
|
||||||
|
if not output_tuple or len(output_tuple) < 2:
|
||||||
|
return
|
||||||
|
width, height = output_tuple[0], output_tuple[1]
|
||||||
|
if not isinstance(width, int) or not isinstance(height, int):
|
||||||
|
return
|
||||||
|
|
||||||
|
if SIZE not in metadata:
|
||||||
|
metadata[SIZE] = {}
|
||||||
|
metadata[SIZE][node_id] = {
|
||||||
|
"width": width,
|
||||||
|
"height": height,
|
||||||
|
"node_id": node_id,
|
||||||
|
}
|
||||||
|
|
||||||
class RgthreePowerLoraLoaderExtractor(NodeMetadataExtractor):
|
class RgthreePowerLoraLoaderExtractor(NodeMetadataExtractor):
|
||||||
"""Extract LoRA metadata from rgthree Power Lora Loader.
|
"""Extract LoRA metadata from rgthree Power Lora Loader.
|
||||||
|
|
||||||
@@ -1233,14 +1371,7 @@ class MetadataOverwriteExtractor(NodeMetadataExtractor):
|
|||||||
if not inputs:
|
if not inputs:
|
||||||
return
|
return
|
||||||
|
|
||||||
overwrite_params = {}
|
overwrite_params = collect_overwrite_params(inputs)
|
||||||
for key in METADATA_OVERWRITE_FIELDS:
|
|
||||||
value = inputs.get(key)
|
|
||||||
if key == "clip_skip":
|
|
||||||
if value != CLIP_SKIP_SENTINEL:
|
|
||||||
overwrite_params[key] = value
|
|
||||||
elif value: # truthy — only overwrite when user provided a real value
|
|
||||||
overwrite_params[key] = value
|
|
||||||
|
|
||||||
if overwrite_params:
|
if overwrite_params:
|
||||||
metadata.setdefault(OVERWRITE, {})
|
metadata.setdefault(OVERWRITE, {})
|
||||||
@@ -1261,6 +1392,8 @@ NODE_EXTRACTORS = {
|
|||||||
"ClownsharKSampler_Beta": SamplerExtractor,
|
"ClownsharKSampler_Beta": SamplerExtractor,
|
||||||
"TSC_KSampler": TSCKSamplerExtractor, # Efficient Nodes
|
"TSC_KSampler": TSCKSamplerExtractor, # Efficient Nodes
|
||||||
"TSC_KSamplerAdvanced": TSCKSamplerAdvancedExtractor, # Efficient Nodes
|
"TSC_KSamplerAdvanced": TSCKSamplerAdvancedExtractor, # Efficient Nodes
|
||||||
|
"KreaTwoStageSampler": KreaTwoStageSamplerExtractor, # Auryg/Krea-2-Two-Stage-Sampler
|
||||||
|
"KreaThreeStageSampler": KreaTwoStageSamplerExtractor, # Auryg/Krea-2-Two-Stage-Sampler
|
||||||
"KSamplerBasicPipe": KSamplerBasicPipeExtractor, # comfyui-impact-pack
|
"KSamplerBasicPipe": KSamplerBasicPipeExtractor, # comfyui-impact-pack
|
||||||
"KSamplerAdvancedBasicPipe": KSamplerAdvancedBasicPipeExtractor, # comfyui-impact-pack
|
"KSamplerAdvancedBasicPipe": KSamplerAdvancedBasicPipeExtractor, # comfyui-impact-pack
|
||||||
"KSampler_inspire_pipe": KSamplerBasicPipeExtractor, # comfyui-inspire-pack
|
"KSampler_inspire_pipe": KSamplerBasicPipeExtractor, # comfyui-inspire-pack
|
||||||
@@ -1312,6 +1445,7 @@ NODE_EXTRACTORS = {
|
|||||||
"GetNode": GetNodeExtractor,
|
"GetNode": GetNodeExtractor,
|
||||||
# Latent
|
# Latent
|
||||||
"EmptyLatentImage": ImageSizeExtractor,
|
"EmptyLatentImage": ImageSizeExtractor,
|
||||||
|
"KreaDualResolutionSelector": KreaDualResolutionSelectorExtractor, # Auryg/Krea-2-Two-Stage-Sampler
|
||||||
# Flux
|
# Flux
|
||||||
"FluxGuidance": FluxGuidanceExtractor, # Add FluxGuidance
|
"FluxGuidance": FluxGuidanceExtractor, # Add FluxGuidance
|
||||||
"CFGGuider": CFGGuiderExtractor, # Add CFGGuider
|
"CFGGuider": CFGGuiderExtractor, # Add CFGGuider
|
||||||
|
|||||||
@@ -0,0 +1,51 @@
|
|||||||
|
"""Shared helpers for Metadata Overwrite node metadata collection.
|
||||||
|
|
||||||
|
Used by both the MetadataOverwriteLM node (execution time) and the
|
||||||
|
MetadataOverwriteExtractor (hook time) so the conversion/filtering logic
|
||||||
|
cannot drift between the two paths.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
from ..utils.utils import model_patcher_to_name, sampler_object_to_name
|
||||||
|
from .constants import CLIP_SKIP_SENTINEL, METADATA_OVERWRITE_FIELDS
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def collect_overwrite_params(values: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
"""Convert node input values into non-default overwrite parameters.
|
||||||
|
|
||||||
|
For most fields, a falsy value (empty string, 0) means "not set" and is
|
||||||
|
skipped. clip_skip uses a dedicated sentinel (-25) so that a wired value
|
||||||
|
of 0 is preserved. The ``model`` field accepts either a manual string or
|
||||||
|
a wired MODEL (ModelPatcher) connection; in the latter case the source
|
||||||
|
model name is extracted from the patcher's ``cached_patcher_init`` and
|
||||||
|
stored as a ComfyUI-style relative path. The ``sampler`` field likewise
|
||||||
|
accepts a manual string or a wired SAMPLER (KSAMPLER) connection, from
|
||||||
|
which the sampler name is extracted via the sampler function's name.
|
||||||
|
"""
|
||||||
|
result: Dict[str, Any] = {}
|
||||||
|
for key in METADATA_OVERWRITE_FIELDS:
|
||||||
|
value = values.get(key)
|
||||||
|
if key == "model" and not isinstance(value, str):
|
||||||
|
value = model_patcher_to_name(value)
|
||||||
|
if value is None:
|
||||||
|
logger.warning(
|
||||||
|
"Could not extract model name from wired MODEL input "
|
||||||
|
"(no cached_patcher_init); model metadata overwrite skipped"
|
||||||
|
)
|
||||||
|
elif key == "sampler" and not isinstance(value, str):
|
||||||
|
value = sampler_object_to_name(value)
|
||||||
|
if value is None:
|
||||||
|
logger.warning(
|
||||||
|
"Could not extract sampler name from wired SAMPLER input "
|
||||||
|
"(unrecognized sampler function); sampler metadata overwrite skipped"
|
||||||
|
)
|
||||||
|
if key == "clip_skip":
|
||||||
|
if value != CLIP_SKIP_SENTINEL:
|
||||||
|
result[key] = value
|
||||||
|
elif value:
|
||||||
|
result[key] = value
|
||||||
|
return result
|
||||||
@@ -43,7 +43,7 @@ SCANNER_GETTER_NAMES = tuple(SCANNER_TYPE_MAP.keys())
|
|||||||
|
|
||||||
async def _find_model_entry(
|
async def _find_model_entry(
|
||||||
model_path: str,
|
model_path: str,
|
||||||
) -> tuple[object, object, str | None] | tuple[None, None, None]:
|
) -> tuple[Any, object, str | None] | tuple[None, None, None]:
|
||||||
"""Iterate all scanners and return the first (scanner, entry, getter_name)
|
"""Iterate all scanners and return the first (scanner, entry, getter_name)
|
||||||
that owns *model_path*. Returns ``(None, None, None)`` when no scanner
|
that owns *model_path*. Returns ``(None, None, None)`` when no scanner
|
||||||
claims it.
|
claims it.
|
||||||
@@ -73,7 +73,7 @@ async def _find_model_entry(
|
|||||||
|
|
||||||
async def _find_scanner_for_model(
|
async def _find_scanner_for_model(
|
||||||
model_path: str,
|
model_path: str,
|
||||||
) -> tuple[object, object] | tuple[None, None]:
|
) -> tuple[Any, object] | tuple[None, None]:
|
||||||
"""Find the (scanner, cache_entry) responsible for *model_path*."""
|
"""Find the (scanner, cache_entry) responsible for *model_path*."""
|
||||||
scanner, entry, _ = await _find_model_entry(model_path)
|
scanner, entry, _ = await _find_model_entry(model_path)
|
||||||
return scanner, entry
|
return scanner, entry
|
||||||
|
|||||||
@@ -46,6 +46,16 @@ async def api_json_error(
|
|||||||
if request.path.startswith("/api/lm/previews") and exc.status == 404:
|
if request.path.startswith("/api/lm/previews") and exc.status == 404:
|
||||||
logger_method = logger.debug
|
logger_method = logger.debug
|
||||||
|
|
||||||
|
# Download-progress 404 is routine too: in-memory tracking is removed
|
||||||
|
# once a download finishes/fails, so the extension's final polls 404.
|
||||||
|
# The extension relies on the 404 status itself (failure detection),
|
||||||
|
# so only the log level is lowered.
|
||||||
|
if (
|
||||||
|
request.path.startswith("/api/lm/download-progress/")
|
||||||
|
and exc.status == 404
|
||||||
|
):
|
||||||
|
logger_method = logger.debug
|
||||||
|
|
||||||
logger_method(
|
logger_method(
|
||||||
"API %s %s returned HTTP %d: %s",
|
"API %s %s returned HTTP %d: %s",
|
||||||
request.method,
|
request.method,
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
import logging
|
import logging
|
||||||
from typing import List, Tuple
|
import os
|
||||||
import comfy.sd # type: ignore
|
from typing import Any, List, Tuple
|
||||||
import folder_paths # type: ignore
|
import comfy.sd # pyright: ignore[reportMissingImports]
|
||||||
|
import folder_paths # pyright: ignore[reportMissingImports]
|
||||||
from ..utils.utils import get_checkpoint_info_absolute, _format_model_name_for_comfyui
|
from ..utils.utils import get_checkpoint_info_absolute, _format_model_name_for_comfyui
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -12,20 +13,42 @@ class CheckpointLoaderLM:
|
|||||||
|
|
||||||
Loads checkpoints from both standard ComfyUI folders and LoRA Manager's
|
Loads checkpoints from both standard ComfyUI folders and LoRA Manager's
|
||||||
extra folder paths, providing a unified interface for checkpoint loading.
|
extra folder paths, providing a unified interface for checkpoint loading.
|
||||||
|
The ckpt_name combo supports ComfyUI's control_after_generate, letting
|
||||||
|
users pick a random checkpoint on every run; the base_model input narrows
|
||||||
|
the random pool through a front-end extension that filters the combo
|
||||||
|
options.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
NAME = "Checkpoint Loader (LoraManager)"
|
NAME = "Checkpoint Loader (LoraManager)"
|
||||||
CATEGORY = "Lora Manager/loaders"
|
CATEGORY = "Lora Manager/loaders"
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(s):
|
def INPUT_TYPES(cls):
|
||||||
# Get list of checkpoint names from scanner (includes extra folder paths)
|
# Get list of checkpoint names from scanner (includes extra folder paths)
|
||||||
checkpoint_names = s._get_checkpoint_names()
|
checkpoint_names = cls._get_checkpoint_names()
|
||||||
|
base_models = cls._get_available_base_models()
|
||||||
return {
|
return {
|
||||||
"required": {
|
"required": {
|
||||||
"ckpt_name": (
|
"ckpt_name": (
|
||||||
checkpoint_names,
|
checkpoint_names,
|
||||||
{"tooltip": "The name of the checkpoint (model) to load."},
|
{
|
||||||
|
"tooltip": (
|
||||||
|
"The name of the checkpoint (model) to load. Use "
|
||||||
|
"control_after_generate to pick a random model on "
|
||||||
|
"every run."
|
||||||
|
),
|
||||||
|
"control_after_generate": "fixed",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
"base_model": (
|
||||||
|
base_models,
|
||||||
|
{
|
||||||
|
"default": "Any",
|
||||||
|
"tooltip": (
|
||||||
|
"Restrict the random selection pool to this base "
|
||||||
|
"model. 'Any' uses the full pool."
|
||||||
|
),
|
||||||
|
},
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -58,7 +81,10 @@ class CheckpointLoaderLM:
|
|||||||
for item in cache.raw_data:
|
for item in cache.raw_data:
|
||||||
if item.get("sub_type") == "checkpoint":
|
if item.get("sub_type") == "checkpoint":
|
||||||
file_path = item.get("file_path", "")
|
file_path = item.get("file_path", "")
|
||||||
if file_path:
|
# Only offer models that still exist on disk so ComfyUI
|
||||||
|
# flags missing checkpoints at queue time via
|
||||||
|
# "value not in list" (the scanner cache can be stale).
|
||||||
|
if file_path and os.path.exists(file_path):
|
||||||
# Format using relative path with OS-native separator
|
# Format using relative path with OS-native separator
|
||||||
formatted_name = _format_model_name_for_comfyui(
|
formatted_name = _format_model_name_for_comfyui(
|
||||||
file_path, model_roots
|
file_path, model_roots
|
||||||
@@ -89,15 +115,68 @@ class CheckpointLoaderLM:
|
|||||||
logger.error(f"Error getting checkpoint names: {e}")
|
logger.error(f"Error getting checkpoint names: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def load_checkpoint(self, ckpt_name: str) -> Tuple:
|
@classmethod
|
||||||
|
def _get_available_base_models(cls) -> List[str]:
|
||||||
|
"""Get distinct base_model values present among indexed checkpoints, for the random-selection filter."""
|
||||||
|
try:
|
||||||
|
from ..services.service_registry import ServiceRegistry
|
||||||
|
|
||||||
|
async def _get_base_models():
|
||||||
|
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||||
|
cache = await scanner.get_cached_data()
|
||||||
|
|
||||||
|
base_models = set()
|
||||||
|
for item in cache.raw_data:
|
||||||
|
if item.get("sub_type") != "checkpoint":
|
||||||
|
continue
|
||||||
|
base_model = item.get("base_model")
|
||||||
|
file_path = item.get("file_path", "")
|
||||||
|
if base_model and file_path and os.path.exists(file_path):
|
||||||
|
base_models.add(base_model)
|
||||||
|
|
||||||
|
return sorted(base_models)
|
||||||
|
|
||||||
|
return ["Any"] + cls._run_async(_get_base_models)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting available base models: {e}")
|
||||||
|
return ["Any"]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _run_async(coro_fn):
|
||||||
|
"""Run an async fetcher, handling the case where an event loop is already running."""
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
try:
|
||||||
|
asyncio.get_running_loop()
|
||||||
|
import concurrent.futures
|
||||||
|
|
||||||
|
def run_in_thread():
|
||||||
|
new_loop = asyncio.new_event_loop()
|
||||||
|
asyncio.set_event_loop(new_loop)
|
||||||
|
try:
|
||||||
|
return new_loop.run_until_complete(coro_fn())
|
||||||
|
finally:
|
||||||
|
new_loop.close()
|
||||||
|
|
||||||
|
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||||
|
future = executor.submit(run_in_thread)
|
||||||
|
return future.result()
|
||||||
|
except RuntimeError:
|
||||||
|
return asyncio.run(coro_fn())
|
||||||
|
|
||||||
|
def load_checkpoint(
|
||||||
|
self, ckpt_name: str, base_model: str = "Any"
|
||||||
|
) -> Tuple[Any, Any, Any]:
|
||||||
"""Load a checkpoint by name, supporting extra folder paths
|
"""Load a checkpoint by name, supporting extra folder paths
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
ckpt_name: The name of the checkpoint to load (relative path with extension)
|
ckpt_name: The name of the checkpoint to load (relative path with extension)
|
||||||
|
base_model: Only used by the front-end to filter the random pool
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (MODEL, CLIP, VAE)
|
Tuple of (MODEL, CLIP, VAE)
|
||||||
"""
|
"""
|
||||||
|
del base_model
|
||||||
# Get absolute path from cache using ComfyUI-style name
|
# Get absolute path from cache using ComfyUI-style name
|
||||||
ckpt_path, metadata = get_checkpoint_info_absolute(ckpt_name)
|
ckpt_path, metadata = get_checkpoint_info_absolute(ckpt_name)
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ from .utils import (
|
|||||||
any_type,
|
any_type,
|
||||||
apply_lora_syntax_format,
|
apply_lora_syntax_format,
|
||||||
get_loras_list,
|
get_loras_list,
|
||||||
|
validate_lora_entries,
|
||||||
)
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -38,15 +39,21 @@ class CreateHookLoraLM:
|
|||||||
),
|
),
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
|
"loras": ("LORAS", {}),
|
||||||
},
|
},
|
||||||
"optional": FlexibleOptionalInputType(any_type),
|
"optional": FlexibleOptionalInputType(any_type),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def VALIDATE_INPUTS(cls, loras=None):
|
||||||
|
"""Queue-time validation: reject missing local LoRAs before execution."""
|
||||||
|
return validate_lora_entries({"loras": loras}) or True
|
||||||
|
|
||||||
RETURN_TYPES = ("HOOKS", "STRING", "STRING")
|
RETURN_TYPES = ("HOOKS", "STRING", "STRING")
|
||||||
RETURN_NAMES = ("HOOKS", "trigger_words", "active_loras")
|
RETURN_NAMES = ("HOOKS", "trigger_words", "active_loras")
|
||||||
FUNCTION = "create_hook"
|
FUNCTION = "create_hook"
|
||||||
|
|
||||||
def create_hook(self, text: str, **kwargs):
|
def create_hook(self, text: str, loras, **kwargs):
|
||||||
"""Create a HookGroup from the selected LoRAs, chained with prev_hooks.
|
"""Create a HookGroup from the selected LoRAs, chained with prev_hooks.
|
||||||
|
|
||||||
Each active LoRA from the widget is loaded and wrapped in a WeightHook
|
Each active LoRA from the widget is loaded and wrapped in a WeightHook
|
||||||
@@ -57,8 +64,8 @@ class CreateHookLoraLM:
|
|||||||
del text # used by the frontend widget only
|
del text # used by the frontend widget only
|
||||||
|
|
||||||
# Lazy imports: comfy is not available in CI/test environment at module level
|
# Lazy imports: comfy is not available in CI/test environment at module level
|
||||||
import comfy.hooks # type: ignore # noqa: C0415
|
import comfy.hooks # pyright: ignore[reportMissingImports] # noqa: C0415
|
||||||
import comfy.utils # type: ignore # noqa: C0415
|
import comfy.utils # pyright: ignore[reportMissingImports] # noqa: C0415
|
||||||
|
|
||||||
prev_hooks: comfy.hooks.HookGroup | None = kwargs.get("prev_hooks")
|
prev_hooks: comfy.hooks.HookGroup | None = kwargs.get("prev_hooks")
|
||||||
|
|
||||||
@@ -67,7 +74,7 @@ class CreateHookLoraLM:
|
|||||||
all_trigger_words: list[str] = []
|
all_trigger_words: list[str] = []
|
||||||
active_loras: list[tuple[str, float, float]] = []
|
active_loras: list[tuple[str, float, float]] = []
|
||||||
|
|
||||||
for lora in get_loras_list(kwargs):
|
for lora in get_loras_list({"loras": loras}):
|
||||||
if not lora.get("active", False):
|
if not lora.get("active", False):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
|||||||
+14
-7
@@ -1,8 +1,8 @@
|
|||||||
import importlib
|
import importlib
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
import comfy.sd # type: ignore
|
import comfy.sd # pyright: ignore[reportMissingImports]
|
||||||
import comfy.utils # type: ignore
|
import comfy.utils # pyright: ignore[reportMissingImports]
|
||||||
|
|
||||||
from ..utils.utils import get_lora_info_absolute
|
from ..utils.utils import get_lora_info_absolute
|
||||||
from .utils import (
|
from .utils import (
|
||||||
@@ -14,6 +14,7 @@ from .utils import (
|
|||||||
get_loras_list,
|
get_loras_list,
|
||||||
nunchaku_load_lora,
|
nunchaku_load_lora,
|
||||||
parse_lora_syntax,
|
parse_lora_syntax,
|
||||||
|
validate_lora_entries,
|
||||||
)
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -48,9 +49,9 @@ def _collect_stack_entries(lora_stack):
|
|||||||
return entries
|
return entries
|
||||||
|
|
||||||
|
|
||||||
def _collect_widget_entries(kwargs):
|
def _collect_widget_entries(loras):
|
||||||
entries = []
|
entries = []
|
||||||
for lora in get_loras_list(kwargs):
|
for lora in get_loras_list({"loras": loras}):
|
||||||
if not lora.get("active", False):
|
if not lora.get("active", False):
|
||||||
continue
|
continue
|
||||||
lora_name = apply_lora_syntax_format(lora["name"])
|
lora_name = apply_lora_syntax_format(lora["name"])
|
||||||
@@ -138,20 +139,26 @@ class LoraLoaderLM:
|
|||||||
"placeholder": "Search LoRAs to add...",
|
"placeholder": "Search LoRAs to add...",
|
||||||
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
||||||
}),
|
}),
|
||||||
|
"loras": ("LORAS", {}),
|
||||||
},
|
},
|
||||||
"optional": FlexibleOptionalInputType(any_type),
|
"optional": FlexibleOptionalInputType(any_type),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def VALIDATE_INPUTS(cls, loras=None):
|
||||||
|
"""Queue-time validation: reject missing local LoRAs before execution."""
|
||||||
|
return validate_lora_entries({"loras": loras}) or True
|
||||||
|
|
||||||
RETURN_TYPES = ("MODEL", "CLIP", "STRING", "STRING")
|
RETURN_TYPES = ("MODEL", "CLIP", "STRING", "STRING")
|
||||||
RETURN_NAMES = ("MODEL", "CLIP", "trigger_words", "loaded_loras")
|
RETURN_NAMES = ("MODEL", "CLIP", "trigger_words", "loaded_loras")
|
||||||
FUNCTION = "load_loras"
|
FUNCTION = "load_loras"
|
||||||
|
|
||||||
def load_loras(self, model, text, **kwargs):
|
def load_loras(self, model, text, loras, **kwargs):
|
||||||
"""Loads multiple LoRAs based on the kwargs input and lora_stack."""
|
"""Loads multiple LoRAs based on the widget input and lora_stack."""
|
||||||
del text
|
del text
|
||||||
clip = kwargs.get("clip", None)
|
clip = kwargs.get("clip", None)
|
||||||
lora_entries = _collect_stack_entries(kwargs.get("lora_stack", None))
|
lora_entries = _collect_stack_entries(kwargs.get("lora_stack", None))
|
||||||
lora_entries.extend(_collect_widget_entries(kwargs))
|
lora_entries.extend(_collect_widget_entries(loras))
|
||||||
|
|
||||||
nunchaku_model_kind = detect_nunchaku_model_kind(model)
|
nunchaku_model_kind = detect_nunchaku_model_kind(model)
|
||||||
if nunchaku_model_kind == "flux":
|
if nunchaku_model_kind == "flux":
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ and tracks the last used combination for reuse.
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
from ..utils.utils import get_lora_info
|
from ..utils.utils import get_lora_info
|
||||||
|
from .utils import validate_lora_entries
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -31,6 +32,11 @@ class LoraRandomizerLM:
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def VALIDATE_INPUTS(cls, loras=None):
|
||||||
|
"""Queue-time validation: reject missing local LoRAs before execution."""
|
||||||
|
return validate_lora_entries({"loras": loras}) or True
|
||||||
|
|
||||||
RETURN_TYPES = ("LORA_STACK",)
|
RETURN_TYPES = ("LORA_STACK",)
|
||||||
RETURN_NAMES = ("LORA_STACK",)
|
RETURN_NAMES = ("LORA_STACK",)
|
||||||
|
|
||||||
|
|||||||
@@ -1,26 +1,102 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import inspect
|
||||||
|
import re
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
_STACK_INPUT_PATTERN = re.compile(r"^lora_stack(?:_([ab])|(\d+))$")
|
||||||
|
|
||||||
|
|
||||||
|
def _is_stack_input(name: str) -> bool:
|
||||||
|
return bool(_STACK_INPUT_PATTERN.match(name))
|
||||||
|
|
||||||
|
|
||||||
|
def _stack_slot_number(name: str) -> int:
|
||||||
|
"""Numeric slot used to order stack inputs; legacy a/b map to 1/2."""
|
||||||
|
match = _STACK_INPUT_PATTERN.match(name)
|
||||||
|
if not match:
|
||||||
|
return -1
|
||||||
|
letter, digits = match.group(1), match.group(2)
|
||||||
|
if digits is not None:
|
||||||
|
return int(digits)
|
||||||
|
return 1 if letter == "a" else 2
|
||||||
|
|
||||||
|
|
||||||
|
class _LoraStackOptionalInputs:
|
||||||
|
"""Lookup that preserves explicit optional inputs and dynamic lora_stack slots."""
|
||||||
|
|
||||||
|
def __init__(self, explicit_inputs: dict[str, tuple[str, dict[str, Any]]]) -> None:
|
||||||
|
self._explicit_inputs = explicit_inputs
|
||||||
|
|
||||||
|
def __contains__(self, item: object) -> bool:
|
||||||
|
if not isinstance(item, str):
|
||||||
|
return False
|
||||||
|
return item in self._explicit_inputs or _is_stack_input(item)
|
||||||
|
|
||||||
|
def __getitem__(self, key: str) -> tuple[str, dict[str, Any]]:
|
||||||
|
if key in self._explicit_inputs:
|
||||||
|
return self._explicit_inputs[key]
|
||||||
|
if _is_stack_input(key):
|
||||||
|
return (
|
||||||
|
"LORA_STACK",
|
||||||
|
{
|
||||||
|
"tooltip": "A LoRA stack to combine. Connect to add more inputs.",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
raise KeyError(key)
|
||||||
|
|
||||||
|
|
||||||
class LoraStackCombinerLM:
|
class LoraStackCombinerLM:
|
||||||
NAME = "Lora Stack Combiner (LoraManager)"
|
NAME = "Lora Stack Combiner (LoraManager)"
|
||||||
CATEGORY = "Lora Manager/stackers"
|
CATEGORY = "Lora Manager/stackers"
|
||||||
|
DESCRIPTION = (
|
||||||
|
"Combines multiple LoRA stacks into a single stack. "
|
||||||
|
"Supports dynamic inputs: connect a stack to add more inputs."
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(cls):
|
def INPUT_TYPES(cls):
|
||||||
|
optional_inputs: dict[str, tuple[str, dict[str, Any]]] = {
|
||||||
|
"lora_stack1": (
|
||||||
|
"LORA_STACK",
|
||||||
|
{
|
||||||
|
"tooltip": "A LoRA stack to combine. Connect to add more inputs.",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
"lora_stack2": (
|
||||||
|
"LORA_STACK",
|
||||||
|
{
|
||||||
|
"tooltip": "A LoRA stack to combine. Connect to add more inputs.",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
stack = inspect.stack()
|
||||||
|
if len(stack) > 2 and stack[2].function == "get_input_info":
|
||||||
|
optional_inputs = _LoraStackOptionalInputs(optional_inputs) # pyright: ignore[reportAssignmentType]
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"required": {
|
"required": {},
|
||||||
"lora_stack_a": ("LORA_STACK",),
|
"optional": optional_inputs,
|
||||||
"lora_stack_b": ("LORA_STACK",),
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
RETURN_TYPES = ("LORA_STACK",)
|
RETURN_TYPES = ("LORA_STACK",)
|
||||||
RETURN_NAMES = ("LORA_STACK",)
|
RETURN_NAMES = ("LORA_STACK",)
|
||||||
FUNCTION = "combine_stacks"
|
FUNCTION = "combine_stacks"
|
||||||
|
|
||||||
def combine_stacks(self, lora_stack_a, lora_stack_b):
|
def combine_stacks(self, lora_stack1=None, lora_stack2=None, **kwargs):
|
||||||
combined_stack = []
|
stacks = {
|
||||||
|
"lora_stack1": lora_stack1,
|
||||||
|
"lora_stack2": lora_stack2,
|
||||||
|
}
|
||||||
|
for key, value in kwargs.items():
|
||||||
|
if _is_stack_input(key) and value is not None:
|
||||||
|
stacks[key] = value
|
||||||
|
|
||||||
if lora_stack_a:
|
combined_stack = []
|
||||||
combined_stack.extend(lora_stack_a)
|
for key in sorted(stacks, key=_stack_slot_number):
|
||||||
if lora_stack_b:
|
stack = stacks[key]
|
||||||
combined_stack.extend(lora_stack_b)
|
if stack:
|
||||||
|
combined_stack.extend(stack)
|
||||||
|
|
||||||
return (combined_stack,)
|
return (combined_stack,)
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import os
|
import os
|
||||||
from ..utils.utils import get_lora_info
|
from ..utils.utils import get_lora_info
|
||||||
from .utils import FlexibleOptionalInputType, any_type, apply_lora_syntax_format, extract_lora_name, get_loras_list
|
from .utils import FlexibleOptionalInputType, any_type, apply_lora_syntax_format, extract_lora_name, get_loras_list, validate_lora_entries
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
@@ -18,16 +18,22 @@ class LoraStackerLM:
|
|||||||
"placeholder": "Search LoRAs to add...",
|
"placeholder": "Search LoRAs to add...",
|
||||||
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
||||||
}),
|
}),
|
||||||
|
"loras": ("LORAS", {}),
|
||||||
},
|
},
|
||||||
"optional": FlexibleOptionalInputType(any_type),
|
"optional": FlexibleOptionalInputType(any_type),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def VALIDATE_INPUTS(cls, loras=None):
|
||||||
|
"""Queue-time validation: reject missing local LoRAs before execution."""
|
||||||
|
return validate_lora_entries({"loras": loras}) or True
|
||||||
|
|
||||||
RETURN_TYPES = ("LORA_STACK", "STRING", "STRING")
|
RETURN_TYPES = ("LORA_STACK", "STRING", "STRING")
|
||||||
RETURN_NAMES = ("LORA_STACK", "trigger_words", "active_loras")
|
RETURN_NAMES = ("LORA_STACK", "trigger_words", "active_loras")
|
||||||
FUNCTION = "stack_loras"
|
FUNCTION = "stack_loras"
|
||||||
|
|
||||||
def stack_loras(self, text, **kwargs):
|
def stack_loras(self, text, loras, **kwargs):
|
||||||
"""Stacks multiple LoRAs based on the kwargs input without loading them."""
|
"""Stacks multiple LoRAs based on the widget input without loading them."""
|
||||||
stack = []
|
stack = []
|
||||||
active_loras = []
|
active_loras = []
|
||||||
all_trigger_words = []
|
all_trigger_words = []
|
||||||
@@ -42,8 +48,8 @@ class LoraStackerLM:
|
|||||||
_, trigger_words = get_lora_info(lora_name)
|
_, trigger_words = get_lora_info(lora_name)
|
||||||
all_trigger_words.extend(trigger_words)
|
all_trigger_words.extend(trigger_words)
|
||||||
|
|
||||||
# Process loras from kwargs with support for both old and new formats
|
# Process loras from the widget with support for both old and new formats
|
||||||
loras_list = get_loras_list(kwargs)
|
loras_list = get_loras_list({"loras": loras})
|
||||||
for lora in loras_list:
|
for lora in loras_list:
|
||||||
if not lora.get('active', False):
|
if not lora.get('active', False):
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -9,10 +9,8 @@ but users may wire 0 to express "no clip skip / default".
|
|||||||
|
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from ..metadata_collector.constants import (
|
from ..metadata_collector.constants import CLIP_SKIP_SENTINEL as _CLIP_SKIP_SENTINEL
|
||||||
CLIP_SKIP_SENTINEL as _CLIP_SKIP_SENTINEL,
|
from ..metadata_collector.overwrite_utils import collect_overwrite_params
|
||||||
METADATA_OVERWRITE_FIELDS,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class MetadataOverwriteLM:
|
class MetadataOverwriteLM:
|
||||||
@@ -73,10 +71,18 @@ class MetadataOverwriteLM:
|
|||||||
},
|
},
|
||||||
),
|
),
|
||||||
"sampler": (
|
"sampler": (
|
||||||
"STRING",
|
"STRING,SAMPLER",
|
||||||
{
|
{
|
||||||
"default": "",
|
"default": "",
|
||||||
"tooltip": "Sampler name. Only overwrites when non-empty.",
|
"widgetType": "STRING",
|
||||||
|
"tooltip": (
|
||||||
|
"Sampler name. Fill in the name manually or "
|
||||||
|
"connect a SAMPLER output (e.g. KSamplerSelect) "
|
||||||
|
"— the sampler name is then extracted "
|
||||||
|
"automatically. Note: ddim is recorded as "
|
||||||
|
"euler (ComfyUI internal representation). "
|
||||||
|
"Only overwrites when non-empty."
|
||||||
|
),
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
"scheduler": (
|
"scheduler": (
|
||||||
@@ -87,12 +93,16 @@ class MetadataOverwriteLM:
|
|||||||
},
|
},
|
||||||
),
|
),
|
||||||
"model": (
|
"model": (
|
||||||
"STRING",
|
"STRING,MODEL",
|
||||||
{
|
{
|
||||||
"default": "",
|
"default": "",
|
||||||
|
"widgetType": "STRING",
|
||||||
"tooltip": (
|
"tooltip": (
|
||||||
"The checkpoint or diffusion model (UNet) used "
|
"The checkpoint or diffusion model (UNet) used "
|
||||||
"for generation. Only overwrites when non-empty."
|
"for generation. Fill in the name manually or "
|
||||||
|
"connect a MODEL output — the model name is then "
|
||||||
|
"extracted automatically. Only overwrites when "
|
||||||
|
"non-empty."
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -158,13 +168,12 @@ class MetadataOverwriteLM:
|
|||||||
For most fields, a falsy value (empty string, 0) means "not set"
|
For most fields, a falsy value (empty string, 0) means "not set"
|
||||||
and is skipped. clip_skip uses a dedicated sentinel (-25) so that
|
and is skipped. clip_skip uses a dedicated sentinel (-25) so that
|
||||||
a wired value of 0 is preserved and reaches the metadata pipeline.
|
a wired value of 0 is preserved and reaches the metadata pipeline.
|
||||||
|
|
||||||
|
The ``model`` field accepts either a manual string or a wired MODEL
|
||||||
|
(ModelPatcher) connection; in the latter case the underlying model
|
||||||
|
name is extracted from the patcher's ``cached_patcher_init`` and
|
||||||
|
stored as a ComfyUI-style relative path. The ``sampler`` field
|
||||||
|
likewise accepts a manual string or a wired SAMPLER (KSAMPLER)
|
||||||
|
connection, from which the sampler name is extracted automatically.
|
||||||
"""
|
"""
|
||||||
result: dict[str, Any] = {}
|
return (collect_overwrite_params(kwargs),)
|
||||||
for key in METADATA_OVERWRITE_FIELDS:
|
|
||||||
value = kwargs.get(key)
|
|
||||||
if key == "clip_skip":
|
|
||||||
if value != _CLIP_SKIP_SENTINEL:
|
|
||||||
result[key] = value
|
|
||||||
elif value:
|
|
||||||
result[key] = value
|
|
||||||
return (result,)
|
|
||||||
|
|||||||
+12
-13
@@ -15,15 +15,15 @@ import os
|
|||||||
import re
|
import re
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Dict, List, Optional, Tuple, Union
|
from typing import Any, Dict, List, Optional, Tuple, Union, cast
|
||||||
|
|
||||||
import comfy.utils # type: ignore
|
import comfy.utils # pyright: ignore[reportMissingImports]
|
||||||
import folder_paths # type: ignore
|
import folder_paths # pyright: ignore[reportMissingImports]
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
from safetensors import safe_open
|
from safetensors import safe_open
|
||||||
|
|
||||||
from nunchaku.lora.flux.nunchaku_converter import (
|
from nunchaku.lora.flux.nunchaku_converter import ( # pyright: ignore[reportMissingTypeStubs]
|
||||||
pack_lowrank_weight,
|
pack_lowrank_weight,
|
||||||
unpack_lowrank_weight,
|
unpack_lowrank_weight,
|
||||||
)
|
)
|
||||||
@@ -87,10 +87,6 @@ def _rename_layer_underscore_layer_name(old_name: str) -> str:
|
|||||||
return new_name
|
return new_name
|
||||||
|
|
||||||
|
|
||||||
def _is_indexable_module(module):
|
|
||||||
return isinstance(module, (nn.ModuleList, nn.Sequential, list, tuple))
|
|
||||||
|
|
||||||
|
|
||||||
def _get_module_by_name(model: nn.Module, name: str) -> Optional[nn.Module]:
|
def _get_module_by_name(model: nn.Module, name: str) -> Optional[nn.Module]:
|
||||||
if not name:
|
if not name:
|
||||||
return model
|
return model
|
||||||
@@ -100,7 +96,7 @@ def _get_module_by_name(model: nn.Module, name: str) -> Optional[nn.Module]:
|
|||||||
continue
|
continue
|
||||||
if hasattr(module, part):
|
if hasattr(module, part):
|
||||||
module = getattr(module, part)
|
module = getattr(module, part)
|
||||||
elif part.isdigit() and _is_indexable_module(module):
|
elif part.isdigit() and isinstance(module, (nn.ModuleList, nn.Sequential, list, tuple)):
|
||||||
try:
|
try:
|
||||||
module = module[int(part)]
|
module = module[int(part)]
|
||||||
except (IndexError, TypeError):
|
except (IndexError, TypeError):
|
||||||
@@ -267,7 +263,9 @@ def _handle_proj_out_split(lora_dict: Dict[str, Dict[str, torch.Tensor]], base_k
|
|||||||
return result, consumed
|
return result, consumed
|
||||||
|
|
||||||
|
|
||||||
def _apply_lora_to_module(module: nn.Module, a_tensor: torch.Tensor, b_tensor: torch.Tensor, module_name: str, model: nn.Module) -> None:
|
def _apply_lora_to_module(module: Any, a_tensor: torch.Tensor, b_tensor: torch.Tensor, module_name: str, model: Any) -> None:
|
||||||
|
# These modules are dynamic torch containers; monkey-patched attributes
|
||||||
|
# below are set at runtime, so the module/model types are deliberately Any.
|
||||||
if not hasattr(module, "in_features") or not hasattr(module, "out_features"):
|
if not hasattr(module, "in_features") or not hasattr(module, "out_features"):
|
||||||
raise ValueError(f"{module_name}: unsupported module without in/out features")
|
raise ValueError(f"{module_name}: unsupported module without in/out features")
|
||||||
if a_tensor.shape[1] != module.in_features or b_tensor.shape[0] != module.out_features:
|
if a_tensor.shape[1] != module.in_features or b_tensor.shape[0] != module.out_features:
|
||||||
@@ -336,7 +334,7 @@ def _apply_lora_to_module(module: nn.Module, a_tensor: torch.Tensor, b_tensor: t
|
|||||||
raise ValueError(f"{module_name}: unsupported module type {type(module)}")
|
raise ValueError(f"{module_name}: unsupported module type {type(module)}")
|
||||||
|
|
||||||
|
|
||||||
def reset_lora_v2(model: nn.Module) -> None:
|
def reset_lora_v2(model: Any) -> None:
|
||||||
slots = getattr(model, "_lora_slots", None)
|
slots = getattr(model, "_lora_slots", None)
|
||||||
if not slots:
|
if not slots:
|
||||||
return
|
return
|
||||||
@@ -344,6 +342,7 @@ def reset_lora_v2(model: nn.Module) -> None:
|
|||||||
module = _get_module_by_name(model, name)
|
module = _get_module_by_name(model, name)
|
||||||
if module is None:
|
if module is None:
|
||||||
continue
|
continue
|
||||||
|
module = cast(Any, module)
|
||||||
module_type = info.get("type", "nunchaku")
|
module_type = info.get("type", "nunchaku")
|
||||||
if module_type == "nunchaku":
|
if module_type == "nunchaku":
|
||||||
base_rank = info["base_rank"]
|
base_rank = info["base_rank"]
|
||||||
@@ -371,7 +370,7 @@ def reset_lora_v2(model: nn.Module) -> None:
|
|||||||
def compose_loras_v2(model: nn.Module, lora_configs: List[Tuple[Union[str, Path, Dict[str, torch.Tensor]], float]], apply_awq_mod: bool = True) -> bool:
|
def compose_loras_v2(model: nn.Module, lora_configs: List[Tuple[Union[str, Path, Dict[str, torch.Tensor]], float]], apply_awq_mod: bool = True) -> bool:
|
||||||
del apply_awq_mod # retained for interface compatibility
|
del apply_awq_mod # retained for interface compatibility
|
||||||
reset_lora_v2(model)
|
reset_lora_v2(model)
|
||||||
aggregated_weights: Dict[str, List[Dict[str, object]]] = defaultdict(list)
|
aggregated_weights: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
|
||||||
saw_supported_format = False
|
saw_supported_format = False
|
||||||
unresolved_targets = 0
|
unresolved_targets = 0
|
||||||
|
|
||||||
@@ -471,7 +470,7 @@ def compose_loras_v2(model: nn.Module, lora_configs: List[Tuple[Union[str, Path,
|
|||||||
class ComfyQwenImageWrapperLM(nn.Module):
|
class ComfyQwenImageWrapperLM(nn.Module):
|
||||||
def __init__(self, model: nn.Module, config=None, apply_awq_mod: bool = True):
|
def __init__(self, model: nn.Module, config=None, apply_awq_mod: bool = True):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.model = model
|
self.model: Any = model
|
||||||
self.config = {} if config is None else config
|
self.config = {} if config is None else config
|
||||||
self.dtype = next(model.parameters()).dtype
|
self.dtype = next(model.parameters()).dtype
|
||||||
self.loras: List[Tuple[Union[str, Path, Dict[str, torch.Tensor]], float]] = []
|
self.loras: List[Tuple[Union[str, Path, Dict[str, torch.Tensor]], float]] = []
|
||||||
|
|||||||
+2
-2
@@ -67,7 +67,7 @@ class PromptLM:
|
|||||||
|
|
||||||
stack = inspect.stack()
|
stack = inspect.stack()
|
||||||
if len(stack) > 2 and stack[2].function == "get_input_info":
|
if len(stack) > 2 and stack[2].function == "get_input_info":
|
||||||
optional_inputs = _PromptOptionalInputs(optional_inputs) # type: ignore[assignment]
|
optional_inputs = _PromptOptionalInputs(optional_inputs) # pyright: ignore[reportAssignmentType]
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"required": {
|
"required": {
|
||||||
@@ -126,7 +126,7 @@ class PromptLM:
|
|||||||
else:
|
else:
|
||||||
prompt = expanded_text
|
prompt = expanded_text
|
||||||
|
|
||||||
from nodes import CLIPTextEncode # type: ignore
|
from nodes import CLIPTextEncode # pyright: ignore[reportMissingImports, reportAttributeAccessIssue]
|
||||||
|
|
||||||
conditioning = CLIPTextEncode().encode(clip, prompt)[0]
|
conditioning = CLIPTextEncode().encode(clip, prompt)[0]
|
||||||
return (conditioning, prompt)
|
return (conditioning, prompt)
|
||||||
|
|||||||
+30
-9
@@ -5,7 +5,7 @@ import time
|
|||||||
import uuid
|
import uuid
|
||||||
from typing import Any, Dict, Optional
|
from typing import Any, Dict, Optional
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import folder_paths # type: ignore
|
import folder_paths # pyright: ignore[reportMissingImports]
|
||||||
from ..services.service_registry import ServiceRegistry
|
from ..services.service_registry import ServiceRegistry
|
||||||
from ..metadata_collector.metadata_processor import MetadataProcessor
|
from ..metadata_collector.metadata_processor import MetadataProcessor
|
||||||
from ..metadata_collector import get_metadata
|
from ..metadata_collector import get_metadata
|
||||||
@@ -13,7 +13,7 @@ from ..utils.constants import CARD_PREVIEW_WIDTH
|
|||||||
from ..utils.exif_utils import ExifUtils
|
from ..utils.exif_utils import ExifUtils
|
||||||
from ..utils.utils import calculate_recipe_fingerprint, sanitize_folder_name
|
from ..utils.utils import calculate_recipe_fingerprint, sanitize_folder_name
|
||||||
from PIL import Image, PngImagePlugin
|
from PIL import Image, PngImagePlugin
|
||||||
import piexif
|
import piexif # pyright: ignore[reportMissingTypeStubs]
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
# Civitai-compatible sampler name mapping: ComfyUI internal → A1111 display name
|
# Civitai-compatible sampler name mapping: ComfyUI internal → A1111 display name
|
||||||
@@ -252,6 +252,13 @@ class SaveImageLM:
|
|||||||
"tooltip": "When enabled, embeds generation parameters into the saved image metadata. Disable to skip writing generation metadata.",
|
"tooltip": "When enabled, embeds generation parameters into the saved image metadata. Disable to skip writing generation metadata.",
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
|
"add_loras_to_prompt": (
|
||||||
|
"BOOLEAN",
|
||||||
|
{
|
||||||
|
"default": False,
|
||||||
|
"tooltip": "When enabled, appends the LoRA syntax line (e.g. <lora:name:strength>) after the positive prompt in the saved metadata.",
|
||||||
|
},
|
||||||
|
),
|
||||||
"add_counter_to_filename": (
|
"add_counter_to_filename": (
|
||||||
"BOOLEAN",
|
"BOOLEAN",
|
||||||
{
|
{
|
||||||
@@ -348,7 +355,7 @@ class SaveImageLM:
|
|||||||
type_lower = model_type.lower() if model_type else "other"
|
type_lower = model_type.lower() if model_type else "other"
|
||||||
return f"urn:air:{slug}:{type_lower}:civitai:{model_id}@{version_id}"
|
return f"urn:air:{slug}:{type_lower}:civitai:{model_id}@{version_id}"
|
||||||
|
|
||||||
def format_metadata(self, metadata_dict: dict) -> str:
|
def format_metadata(self, metadata_dict: dict[str, Any], add_loras_to_prompt: bool = False) -> str:
|
||||||
"""Format metadata as A1111-compatible parameters string with Hashes JSON and Civitai resources."""
|
"""Format metadata as A1111-compatible parameters string with Hashes JSON and Civitai resources."""
|
||||||
if not metadata_dict: return ""
|
if not metadata_dict: return ""
|
||||||
|
|
||||||
@@ -389,7 +396,7 @@ class SaveImageLM:
|
|||||||
ckpt_display_name = os.path.splitext(os.path.basename(checkpoint))[0]
|
ckpt_display_name = os.path.splitext(os.path.basename(checkpoint))[0]
|
||||||
|
|
||||||
# Resolve LoRA hash and Civitai data from local cache
|
# Resolve LoRA hash and Civitai data from local cache
|
||||||
loras_data: list[dict] = []
|
loras_data: list[dict[str, Any]] = []
|
||||||
for lora_name, strength in lora_entries:
|
for lora_name, strength in lora_entries:
|
||||||
lora_hash, lora_civitai, lora_base_model = self._resolve_model_cache_entry(
|
lora_hash, lora_civitai, lora_base_model = self._resolve_model_cache_entry(
|
||||||
"lora_scanner", lora_name
|
"lora_scanner", lora_name
|
||||||
@@ -411,9 +418,9 @@ class SaveImageLM:
|
|||||||
hashes[f"LORA:{lora['name']}"] = lora["hash"][:10].upper()
|
hashes[f"LORA:{lora['name']}"] = lora["hash"][:10].upper()
|
||||||
|
|
||||||
# Build Civitai resources JSON array
|
# Build Civitai resources JSON array
|
||||||
civitai_resources: list[dict] = []
|
civitai_resources: list[dict[str, Any]] = []
|
||||||
if ckpt_civitai.get("id", 0) > 0:
|
if ckpt_civitai.get("id", 0) > 0:
|
||||||
ckpt_resource: dict = {}
|
ckpt_resource: dict[str, Any] = {}
|
||||||
ckpt_type = (ckpt_civitai.get("model") or {}).get("type", "Checkpoint")
|
ckpt_type = (ckpt_civitai.get("model") or {}).get("type", "Checkpoint")
|
||||||
model_id = ckpt_civitai.get("modelId", 0)
|
model_id = ckpt_civitai.get("modelId", 0)
|
||||||
version_id = ckpt_civitai.get("id", 0)
|
version_id = ckpt_civitai.get("id", 0)
|
||||||
@@ -432,7 +439,7 @@ class SaveImageLM:
|
|||||||
lora_civitai = lora["civitai"]
|
lora_civitai = lora["civitai"]
|
||||||
if not lora_civitai or lora_civitai.get("id", 0) <= 0:
|
if not lora_civitai or lora_civitai.get("id", 0) <= 0:
|
||||||
continue
|
continue
|
||||||
lora_resource: dict = {"weight": lora["strength"]}
|
lora_resource: dict[str, Any] = {"weight": lora["strength"]}
|
||||||
lora_type = (lora_civitai.get("model") or {}).get("type", "LORA")
|
lora_type = (lora_civitai.get("model") or {}).get("type", "LORA")
|
||||||
model_id = lora_civitai.get("modelId", 0)
|
model_id = lora_civitai.get("modelId", 0)
|
||||||
version_id = lora_civitai.get("id", 0)
|
version_id = lora_civitai.get("id", 0)
|
||||||
@@ -458,7 +465,10 @@ class SaveImageLM:
|
|||||||
scheduler_name = scheduler_mapping.get(scheduler, scheduler) if scheduler else None
|
scheduler_name = scheduler_mapping.get(scheduler, scheduler) if scheduler else None
|
||||||
|
|
||||||
# Build output lines
|
# Build output lines
|
||||||
lines = [prompt] if prompt else [""]
|
prompt_line = prompt if prompt else ""
|
||||||
|
if add_loras_to_prompt and loras_text:
|
||||||
|
prompt_line = f"{prompt_line}\n{loras_text}" if prompt_line else loras_text
|
||||||
|
lines = [prompt_line] if prompt_line else [""]
|
||||||
if negative_prompt:
|
if negative_prompt:
|
||||||
lines.append(f"Negative prompt: {negative_prompt}")
|
lines.append(f"Negative prompt: {negative_prompt}")
|
||||||
|
|
||||||
@@ -768,6 +778,14 @@ class SaveImageLM:
|
|||||||
if checkpoint_entry:
|
if checkpoint_entry:
|
||||||
recipe_data["checkpoint"] = checkpoint_entry
|
recipe_data["checkpoint"] = checkpoint_entry
|
||||||
|
|
||||||
|
# The recipe image is the WebP produced above from the output file;
|
||||||
|
# reuse the same metadata extraction to record workflow presence.
|
||||||
|
try:
|
||||||
|
metadata = ExifUtils._load_structured_metadata(image_path)
|
||||||
|
recipe_data["has_workflow"] = bool(metadata.get("workflow"))
|
||||||
|
except Exception:
|
||||||
|
recipe_data["has_workflow"] = False
|
||||||
|
|
||||||
json_path = os.path.normpath(
|
json_path = os.path.normpath(
|
||||||
os.path.join(recipes_dir, f"{recipe_id}.recipe.json")
|
os.path.join(recipes_dir, f"{recipe_id}.recipe.json")
|
||||||
)
|
)
|
||||||
@@ -793,6 +811,7 @@ class SaveImageLM:
|
|||||||
save_with_metadata=True,
|
save_with_metadata=True,
|
||||||
add_counter_to_filename=True,
|
add_counter_to_filename=True,
|
||||||
save_as_recipe=False,
|
save_as_recipe=False,
|
||||||
|
add_loras_to_prompt=False,
|
||||||
):
|
):
|
||||||
"""Save images with metadata"""
|
"""Save images with metadata"""
|
||||||
results = []
|
results = []
|
||||||
@@ -801,7 +820,7 @@ class SaveImageLM:
|
|||||||
raw_metadata = get_metadata()
|
raw_metadata = get_metadata()
|
||||||
metadata_dict = MetadataProcessor.to_dict(raw_metadata, id)
|
metadata_dict = MetadataProcessor.to_dict(raw_metadata, id)
|
||||||
|
|
||||||
metadata = self.format_metadata(metadata_dict)
|
metadata = self.format_metadata(metadata_dict, add_loras_to_prompt)
|
||||||
|
|
||||||
# Process filename_prefix with pattern substitution
|
# Process filename_prefix with pattern substitution
|
||||||
filename_prefix = self.format_filename(filename_prefix, metadata_dict)
|
filename_prefix = self.format_filename(filename_prefix, metadata_dict)
|
||||||
@@ -943,6 +962,7 @@ class SaveImageLM:
|
|||||||
save_with_metadata=True,
|
save_with_metadata=True,
|
||||||
add_counter_to_filename=True,
|
add_counter_to_filename=True,
|
||||||
save_as_recipe=False,
|
save_as_recipe=False,
|
||||||
|
add_loras_to_prompt=False,
|
||||||
):
|
):
|
||||||
"""Process and save image with metadata"""
|
"""Process and save image with metadata"""
|
||||||
# Make sure the output directory exists
|
# Make sure the output directory exists
|
||||||
@@ -974,6 +994,7 @@ class SaveImageLM:
|
|||||||
save_with_metadata,
|
save_with_metadata,
|
||||||
add_counter_to_filename,
|
add_counter_to_filename,
|
||||||
save_as_recipe,
|
save_as_recipe,
|
||||||
|
add_loras_to_prompt,
|
||||||
)
|
)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
+107
-8
@@ -1,37 +1,74 @@
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
from typing import List, Tuple
|
from typing import Any, List, Tuple
|
||||||
import comfy.sd # type: ignore
|
import comfy.sd # pyright: ignore[reportMissingImports]
|
||||||
from ..utils.utils import get_checkpoint_info_absolute, _format_model_name_for_comfyui
|
from ..utils.utils import get_checkpoint_info_absolute, _format_model_name_for_comfyui
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _reload_gguf_unet(
|
||||||
|
unet_path: str, weight_dtype: str, disable_dynamic: bool = False
|
||||||
|
) -> object:
|
||||||
|
"""Reload a GGUF diffusion model from disk (cached_patcher_init factory).
|
||||||
|
|
||||||
|
Mirrors the GGUF branch of UNETLoaderLM.load_unet so ModelPatcher
|
||||||
|
deepclone/dynamic machinery can rebuild GGUF models with the correct
|
||||||
|
GGMLOps. ``disable_dynamic`` is accepted for signature compatibility
|
||||||
|
with core ComfyUI loaders.
|
||||||
|
"""
|
||||||
|
loader = UNETLoaderLM()
|
||||||
|
model, = loader._load_gguf_unet(unet_path, unet_path, weight_dtype)
|
||||||
|
return model
|
||||||
|
|
||||||
|
|
||||||
class UNETLoaderLM:
|
class UNETLoaderLM:
|
||||||
"""UNET Loader with support for extra folder paths
|
"""UNET Loader with support for extra folder paths
|
||||||
|
|
||||||
Loads diffusion models/UNets from both standard ComfyUI folders and LoRA Manager's
|
Loads diffusion models/UNets from both standard ComfyUI folders and LoRA Manager's
|
||||||
extra folder paths, providing a unified interface for UNET loading.
|
extra folder paths, providing a unified interface for UNET loading.
|
||||||
Supports both regular diffusion models and GGUF format models.
|
Supports both regular diffusion models and GGUF format models.
|
||||||
|
The unet_name combo supports ComfyUI's control_after_generate, letting
|
||||||
|
users pick a random diffusion model on every run; the base_model input
|
||||||
|
narrows the random pool through a front-end extension that filters the
|
||||||
|
combo options.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
NAME = "Unet Loader (LoraManager)"
|
NAME = "Unet Loader (LoraManager)"
|
||||||
CATEGORY = "Lora Manager/loaders"
|
CATEGORY = "Lora Manager/loaders"
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def INPUT_TYPES(s):
|
def INPUT_TYPES(cls):
|
||||||
# Get list of unet names from scanner (includes extra folder paths)
|
# Get list of unet names from scanner (includes extra folder paths)
|
||||||
unet_names = s._get_unet_names()
|
unet_names = cls._get_unet_names()
|
||||||
|
base_models = cls._get_available_base_models()
|
||||||
return {
|
return {
|
||||||
"required": {
|
"required": {
|
||||||
"unet_name": (
|
"unet_name": (
|
||||||
unet_names,
|
unet_names,
|
||||||
{"tooltip": "The name of the diffusion model to load."},
|
{
|
||||||
|
"tooltip": (
|
||||||
|
"The name of the diffusion model to load. Use "
|
||||||
|
"control_after_generate to pick a random model on "
|
||||||
|
"every run."
|
||||||
|
),
|
||||||
|
"control_after_generate": "fixed",
|
||||||
|
},
|
||||||
),
|
),
|
||||||
"weight_dtype": (
|
"weight_dtype": (
|
||||||
["default", "fp8_e4m3fn", "fp8_e4m3fn_fast", "fp8_e5m2"],
|
["default", "fp8_e4m3fn", "fp8_e4m3fn_fast", "fp8_e5m2"],
|
||||||
{"tooltip": "The dtype to use for the model weights."},
|
{"tooltip": "The dtype to use for the model weights."},
|
||||||
),
|
),
|
||||||
|
"base_model": (
|
||||||
|
base_models,
|
||||||
|
{
|
||||||
|
"default": "Any",
|
||||||
|
"tooltip": (
|
||||||
|
"Restrict the random selection pool to this base "
|
||||||
|
"model. 'Any' uses the full pool."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -59,7 +96,10 @@ class UNETLoaderLM:
|
|||||||
for item in cache.raw_data:
|
for item in cache.raw_data:
|
||||||
if item.get("sub_type") == "diffusion_model":
|
if item.get("sub_type") == "diffusion_model":
|
||||||
file_path = item.get("file_path", "")
|
file_path = item.get("file_path", "")
|
||||||
if file_path:
|
# Only offer models that still exist on disk so ComfyUI
|
||||||
|
# flags missing diffusion models at queue time via
|
||||||
|
# "value not in list" (the scanner cache can be stale).
|
||||||
|
if file_path and os.path.exists(file_path):
|
||||||
# Format using relative path with OS-native separator
|
# Format using relative path with OS-native separator
|
||||||
formatted_name = _format_model_name_for_comfyui(
|
formatted_name = _format_model_name_for_comfyui(
|
||||||
file_path, model_roots
|
file_path, model_roots
|
||||||
@@ -90,16 +130,69 @@ class UNETLoaderLM:
|
|||||||
logger.error(f"Error getting unet names: {e}")
|
logger.error(f"Error getting unet names: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def load_unet(self, unet_name: str, weight_dtype: str) -> Tuple:
|
@classmethod
|
||||||
|
def _get_available_base_models(cls) -> List[str]:
|
||||||
|
"""Get distinct base_model values present among indexed diffusion models, for the random-selection filter."""
|
||||||
|
try:
|
||||||
|
from ..services.service_registry import ServiceRegistry
|
||||||
|
|
||||||
|
async def _get_base_models():
|
||||||
|
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||||
|
cache = await scanner.get_cached_data()
|
||||||
|
|
||||||
|
base_models = set()
|
||||||
|
for item in cache.raw_data:
|
||||||
|
if item.get("sub_type") != "diffusion_model":
|
||||||
|
continue
|
||||||
|
base_model = item.get("base_model")
|
||||||
|
file_path = item.get("file_path", "")
|
||||||
|
if base_model and file_path and os.path.exists(file_path):
|
||||||
|
base_models.add(base_model)
|
||||||
|
|
||||||
|
return sorted(base_models)
|
||||||
|
|
||||||
|
return ["Any"] + cls._run_async(_get_base_models)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting available base models: {e}")
|
||||||
|
return ["Any"]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _run_async(coro_fn):
|
||||||
|
"""Run an async fetcher, handling the case where an event loop is already running."""
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
try:
|
||||||
|
asyncio.get_running_loop()
|
||||||
|
import concurrent.futures
|
||||||
|
|
||||||
|
def run_in_thread():
|
||||||
|
new_loop = asyncio.new_event_loop()
|
||||||
|
asyncio.set_event_loop(new_loop)
|
||||||
|
try:
|
||||||
|
return new_loop.run_until_complete(coro_fn())
|
||||||
|
finally:
|
||||||
|
new_loop.close()
|
||||||
|
|
||||||
|
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||||
|
future = executor.submit(run_in_thread)
|
||||||
|
return future.result()
|
||||||
|
except RuntimeError:
|
||||||
|
return asyncio.run(coro_fn())
|
||||||
|
|
||||||
|
def load_unet(
|
||||||
|
self, unet_name: str, weight_dtype: str, base_model: str = "Any"
|
||||||
|
) -> Tuple[Any, ...]:
|
||||||
"""Load a diffusion model by name, supporting extra folder paths
|
"""Load a diffusion model by name, supporting extra folder paths
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
unet_name: The name of the diffusion model to load (relative path with extension)
|
unet_name: The name of the diffusion model to load (relative path with extension)
|
||||||
weight_dtype: The dtype to use for model weights
|
weight_dtype: The dtype to use for model weights
|
||||||
|
base_model: Only used by the front-end to filter the random pool
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (MODEL,)
|
Tuple of (MODEL,)
|
||||||
"""
|
"""
|
||||||
|
del base_model
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
# Get absolute path from cache using ComfyUI-style name
|
# Get absolute path from cache using ComfyUI-style name
|
||||||
@@ -133,7 +226,7 @@ class UNETLoaderLM:
|
|||||||
|
|
||||||
def _load_gguf_unet(
|
def _load_gguf_unet(
|
||||||
self, unet_path: str, unet_name: str, weight_dtype: str
|
self, unet_path: str, unet_name: str, weight_dtype: str
|
||||||
) -> Tuple:
|
) -> Tuple[Any, ...]:
|
||||||
"""Load a GGUF format diffusion model
|
"""Load a GGUF format diffusion model
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -196,6 +289,12 @@ class UNETLoaderLM:
|
|||||||
# Wrap with GGUFModelPatcher
|
# Wrap with GGUFModelPatcher
|
||||||
model = GGUFModelPatcher.clone(model)
|
model = GGUFModelPatcher.clone(model)
|
||||||
|
|
||||||
|
# Register a reload factory so the MODEL carries its source path
|
||||||
|
# (cached_patcher_init) like core ComfyUI loaders do — required
|
||||||
|
# for model-name extraction downstream and for ModelPatcher
|
||||||
|
# deepclone/dynamic machinery.
|
||||||
|
model.cached_patcher_init = (_reload_gguf_unet, (unet_path, weight_dtype))
|
||||||
|
|
||||||
return (model,)
|
return (model,)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
+159
-3
@@ -1,3 +1,6 @@
|
|||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
class AnyType(str):
|
class AnyType(str):
|
||||||
"""A special class that is always equal in not equal comparisons. Credit to pythongosssss"""
|
"""A special class that is always equal in not equal comparisons. Credit to pythongosssss"""
|
||||||
|
|
||||||
@@ -6,7 +9,7 @@ class AnyType(str):
|
|||||||
|
|
||||||
|
|
||||||
# Credit to Regis Gaughan, III (rgthree)
|
# Credit to Regis Gaughan, III (rgthree)
|
||||||
class FlexibleOptionalInputType(dict):
|
class FlexibleOptionalInputType(dict[str, Any]):
|
||||||
"""A special class to make flexible nodes that pass data to our python handlers.
|
"""A special class to make flexible nodes that pass data to our python handlers.
|
||||||
|
|
||||||
Enables both flexible/dynamic input types (like for Any Switch) or a dynamic number of inputs
|
Enables both flexible/dynamic input types (like for Any Switch) or a dynamic number of inputs
|
||||||
@@ -23,6 +26,7 @@ class FlexibleOptionalInputType(dict):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, type):
|
def __init__(self, type):
|
||||||
|
super().__init__()
|
||||||
self.type = type
|
self.type = type
|
||||||
|
|
||||||
def __getitem__(self, key):
|
def __getitem__(self, key):
|
||||||
@@ -40,7 +44,8 @@ import re
|
|||||||
import logging
|
import logging
|
||||||
import copy
|
import copy
|
||||||
import sys
|
import sys
|
||||||
import folder_paths # type: ignore
|
import asyncio
|
||||||
|
import folder_paths # pyright: ignore[reportMissingImports]
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -70,7 +75,7 @@ def extract_lora_name(lora_path):
|
|||||||
return apply_lora_syntax_format(name_no_ext)
|
return apply_lora_syntax_format(name_no_ext)
|
||||||
|
|
||||||
|
|
||||||
def parse_lora_syntax(text: str) -> list[dict]:
|
def parse_lora_syntax(text: str) -> list[dict[str, Any]]:
|
||||||
"""Parse <lora:name:strength> syntax from text input into a list of dicts.
|
"""Parse <lora:name:strength> syntax from text input into a list of dicts.
|
||||||
|
|
||||||
Each entry contains: name, model_strength, clip_strength.
|
Each entry contains: name, model_strength, clip_strength.
|
||||||
@@ -107,6 +112,157 @@ def get_loras_list(kwargs):
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
_LORA_EXTENSIONS = (".safetensors", ".ckpt", ".pt", ".bin")
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_lora_extension(name: str) -> str:
|
||||||
|
"""Strip a known LoRA model extension from a name (case-insensitive)."""
|
||||||
|
lowered = name.lower()
|
||||||
|
for ext in _LORA_EXTENSIONS:
|
||||||
|
if lowered.endswith(ext):
|
||||||
|
return name[: -len(ext)]
|
||||||
|
return name
|
||||||
|
|
||||||
|
|
||||||
|
def _find_missing_loras(names: list[str]) -> list[str]:
|
||||||
|
"""Return the names that cannot be resolved to an existing local LoRA file.
|
||||||
|
|
||||||
|
Mirrors the matching semantics of ``get_lora_info_absolute``
|
||||||
|
(py/utils/utils.py): after stripping the extension, a name matches a cached
|
||||||
|
LoRA when it equals the cached file name or the ``folder/file`` path. As a
|
||||||
|
fallback, a name containing a folder that only matches by basename resolves
|
||||||
|
to the first basename match (same behavior as the runtime resolver). Raw
|
||||||
|
absolute paths that exist on disk are always considered available.
|
||||||
|
|
||||||
|
The scanner cache is fetched once for all names; the cache may be stale, so
|
||||||
|
resolved paths are additionally verified with ``os.path.isfile``.
|
||||||
|
"""
|
||||||
|
if not names:
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def _check() -> list[str]:
|
||||||
|
from ..services.service_registry import ServiceRegistry
|
||||||
|
|
||||||
|
scanner = await ServiceRegistry.get_lora_scanner()
|
||||||
|
# The scanner cache may not be hydrated yet (startup, library path
|
||||||
|
# change). An empty cache is not authoritative — treat it as "cannot
|
||||||
|
# verify" and skip validation instead of flagging every active LoRA
|
||||||
|
# as missing.
|
||||||
|
if getattr(scanner, "_cache", None) is None or getattr(
|
||||||
|
scanner, "_is_initializing", False
|
||||||
|
):
|
||||||
|
return []
|
||||||
|
cache = await scanner.get_cached_data()
|
||||||
|
|
||||||
|
lookup = {}
|
||||||
|
basename_candidates = {}
|
||||||
|
for item in cache.raw_data:
|
||||||
|
file_path = item.get("file_path")
|
||||||
|
if not file_path:
|
||||||
|
continue
|
||||||
|
file_name = item.get("file_name", "")
|
||||||
|
folder = item.get("folder", "")
|
||||||
|
file_name_no_ext = _strip_lora_extension(file_name)
|
||||||
|
path_name_no_ext = (
|
||||||
|
f"{folder}/{file_name_no_ext}".replace("\\", "/")
|
||||||
|
if folder
|
||||||
|
else file_name_no_ext
|
||||||
|
)
|
||||||
|
lookup.setdefault(file_name_no_ext, file_path)
|
||||||
|
lookup.setdefault(path_name_no_ext, file_path)
|
||||||
|
basename_candidates.setdefault(file_name_no_ext, []).append(
|
||||||
|
(folder, file_path)
|
||||||
|
)
|
||||||
|
|
||||||
|
missing = []
|
||||||
|
for name in names:
|
||||||
|
if not name:
|
||||||
|
continue
|
||||||
|
normalized = name.replace("\\", "/")
|
||||||
|
# Raw absolute paths (outside the library) are usable as-is.
|
||||||
|
if os.path.isfile(normalized):
|
||||||
|
continue
|
||||||
|
no_ext = _strip_lora_extension(normalized)
|
||||||
|
file_path = lookup.get(no_ext)
|
||||||
|
if file_path is None and "/" in no_ext:
|
||||||
|
# A name with a folder that matches only by basename resolves
|
||||||
|
# at runtime like get_lora_info_absolute's fallback does:
|
||||||
|
# prefer a candidate whose folder prefixes the name, else the
|
||||||
|
# first basename match.
|
||||||
|
folder, basename = no_ext.rsplit("/", 1)
|
||||||
|
candidates = basename_candidates.get(basename, [])
|
||||||
|
file_path = next(
|
||||||
|
(
|
||||||
|
fp
|
||||||
|
for fld, fp in candidates
|
||||||
|
if fld and no_ext.startswith(fld + "/")
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if file_path is None and candidates:
|
||||||
|
file_path = candidates[0][1]
|
||||||
|
if file_path is None or not os.path.isfile(file_path):
|
||||||
|
missing.append(name)
|
||||||
|
return missing
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Check if we're already in an event loop
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
# If we're in a running loop, run the async check in a separate thread
|
||||||
|
import concurrent.futures
|
||||||
|
|
||||||
|
def run_in_thread():
|
||||||
|
new_loop = asyncio.new_event_loop()
|
||||||
|
asyncio.set_event_loop(new_loop)
|
||||||
|
try:
|
||||||
|
return new_loop.run_until_complete(_check())
|
||||||
|
finally:
|
||||||
|
new_loop.close()
|
||||||
|
|
||||||
|
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||||
|
future = executor.submit(run_in_thread)
|
||||||
|
return future.result()
|
||||||
|
except RuntimeError:
|
||||||
|
# No event loop is running, we can use asyncio.run()
|
||||||
|
return asyncio.run(_check())
|
||||||
|
|
||||||
|
|
||||||
|
def validate_lora_entries(kwargs):
|
||||||
|
"""Validate active LoRA widget entries against the local library.
|
||||||
|
|
||||||
|
Used by node ``VALIDATE_INPUTS`` implementations so ComfyUI rejects the
|
||||||
|
prompt at queue time (``custom_validation_failed``) when an active entry
|
||||||
|
references a LoRA that is not available locally — mirroring how built-in
|
||||||
|
loader nodes flag missing models before execution starts.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
None when every active entry resolves to an existing local file,
|
||||||
|
otherwise a descriptive error string listing the missing LoRAs.
|
||||||
|
Verification failures (e.g. scanner not ready) are treated as valid
|
||||||
|
so queueing is never blocked by validation machinery itself.
|
||||||
|
"""
|
||||||
|
# Missing/empty loras input is always valid; skip get_loras_list so it
|
||||||
|
# does not log a warning for the None case on every queue.
|
||||||
|
if not kwargs.get("loras"):
|
||||||
|
return None
|
||||||
|
loras = get_loras_list(kwargs)
|
||||||
|
active_names = []
|
||||||
|
for lora in loras:
|
||||||
|
if not isinstance(lora, dict):
|
||||||
|
continue
|
||||||
|
if not lora.get("active", False):
|
||||||
|
continue
|
||||||
|
active_names.append(apply_lora_syntax_format(str(lora.get("name") or "")))
|
||||||
|
try:
|
||||||
|
missing = _find_missing_loras(active_names)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to validate LoRA entries against the local library")
|
||||||
|
return None
|
||||||
|
if not missing:
|
||||||
|
return None
|
||||||
|
return "Missing LoRA(s) in local library: " + ", ".join(missing)
|
||||||
|
|
||||||
|
|
||||||
def load_state_dict_in_safetensors(path, device="cpu", filter_prefix=""):
|
def load_state_dict_in_safetensors(path, device="cpu", filter_prefix=""):
|
||||||
"""Simplified version of load_state_dict_in_safetensors that just loads from a local path"""
|
"""Simplified version of load_state_dict_in_safetensors that just loads from a local path"""
|
||||||
import safetensors.torch
|
import safetensors.torch
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
from ..utils.utils import get_lora_info_absolute
|
from ..utils.utils import get_lora_info_absolute
|
||||||
from ..config import config
|
from ..config import config
|
||||||
from .utils import FlexibleOptionalInputType, any_type, get_loras_list
|
from .utils import FlexibleOptionalInputType, any_type, get_loras_list, validate_lora_entries
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -31,15 +31,21 @@ class WanVideoLoraSelectLM:
|
|||||||
"placeholder": "Search LoRAs to add...",
|
"placeholder": "Search LoRAs to add...",
|
||||||
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
||||||
}),
|
}),
|
||||||
|
"loras": ("LORAS", {}),
|
||||||
},
|
},
|
||||||
"optional": FlexibleOptionalInputType(any_type),
|
"optional": FlexibleOptionalInputType(any_type),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def VALIDATE_INPUTS(cls, loras=None):
|
||||||
|
"""Queue-time validation: reject missing local LoRAs before execution."""
|
||||||
|
return validate_lora_entries({"loras": loras}) or True
|
||||||
|
|
||||||
RETURN_TYPES = ("WANVIDLORA", "STRING", "STRING")
|
RETURN_TYPES = ("WANVIDLORA", "STRING", "STRING")
|
||||||
RETURN_NAMES = ("lora", "trigger_words", "active_loras")
|
RETURN_NAMES = ("lora", "trigger_words", "active_loras")
|
||||||
FUNCTION = "process_loras"
|
FUNCTION = "process_loras"
|
||||||
|
|
||||||
def process_loras(self, text, low_mem_load=False, merge_loras=True, **kwargs):
|
def process_loras(self, text, loras, low_mem_load=False, merge_loras=True, **kwargs):
|
||||||
loras_list = []
|
loras_list = []
|
||||||
all_trigger_words = []
|
all_trigger_words = []
|
||||||
active_loras = []
|
active_loras = []
|
||||||
@@ -57,8 +63,8 @@ class WanVideoLoraSelectLM:
|
|||||||
selected_blocks = blocks.get("selected_blocks", {})
|
selected_blocks = blocks.get("selected_blocks", {})
|
||||||
layer_filter = blocks.get("layer_filter", "")
|
layer_filter = blocks.get("layer_filter", "")
|
||||||
|
|
||||||
# Process loras from kwargs with support for both old and new formats
|
# Process loras from the widget with support for both old and new formats
|
||||||
loras_from_widget = get_loras_list(kwargs)
|
loras_from_widget = get_loras_list({"loras": loras})
|
||||||
for lora in loras_from_widget:
|
for lora in loras_from_widget:
|
||||||
if not lora.get('active', False):
|
if not lora.get('active', False):
|
||||||
continue
|
continue
|
||||||
|
|||||||
+65
-9
@@ -1,3 +1,7 @@
|
|||||||
|
# pyright: reportImportCycles=false
|
||||||
|
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||||
|
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||||
|
# import cycles. Breaking them would require an architectural refactor.
|
||||||
"""Base classes for recipe parsers."""
|
"""Base classes for recipe parsers."""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
@@ -7,7 +11,7 @@ import re
|
|||||||
from typing import Dict, List, Any, Optional, Tuple
|
from typing import Dict, List, Any, Optional, Tuple
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from ..config import config
|
from ..config import config
|
||||||
from ..utils.constants import VALID_LORA_TYPES, VALID_CHECKPOINT_SUB_TYPES
|
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES, VALID_LORA_TYPES, VALID_CHECKPOINT_SUB_TYPES
|
||||||
from ..utils.civitai_utils import rewrite_preview_url
|
from ..utils.civitai_utils import rewrite_preview_url
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -38,7 +42,41 @@ class RecipeMetadataParser(ABC):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def populate_lora_from_civitai(lora_entry: Dict[str, Any], civitai_info_tuple: Tuple[Dict[str, Any], Optional[str]],
|
def populate_lora_from_local(lora_entry: Dict[str, Any], local_lora: Dict[str, Any], base_model_counts=None) -> Dict[str, Any]:
|
||||||
|
"""Populate a recipe LoRA entry from the local scanner cache."""
|
||||||
|
local_path = local_lora.get('file_path') or ''
|
||||||
|
file_name = local_lora.get('file_name') or os.path.splitext(os.path.basename(local_path))[0]
|
||||||
|
base_model = local_lora.get('base_model') or ''
|
||||||
|
|
||||||
|
lora_entry['name'] = local_lora.get('model_name') or file_name or lora_entry.get('name', '')
|
||||||
|
lora_entry['file_name'] = file_name
|
||||||
|
lora_entry['hash'] = (local_lora.get('sha256') or lora_entry.get('hash') or '').lower()
|
||||||
|
lora_entry['localPath'] = local_path or None
|
||||||
|
lora_entry['size'] = local_lora.get('size', 0) or 0
|
||||||
|
lora_entry['baseModel'] = base_model
|
||||||
|
lora_entry['existsLocally'] = True
|
||||||
|
lora_entry['isDeleted'] = False
|
||||||
|
|
||||||
|
preview_url = local_lora.get('preview_url')
|
||||||
|
if preview_url:
|
||||||
|
lora_entry['thumbnailUrl'] = config.get_preview_static_url(preview_url)
|
||||||
|
|
||||||
|
civitai_info = local_lora.get('civitai') or {}
|
||||||
|
if isinstance(civitai_info, dict):
|
||||||
|
if civitai_info.get('id') is not None:
|
||||||
|
lora_entry['id'] = civitai_info['id']
|
||||||
|
if civitai_info.get('modelId') is not None:
|
||||||
|
lora_entry['modelId'] = civitai_info['modelId']
|
||||||
|
if civitai_info.get('name'):
|
||||||
|
lora_entry['version'] = civitai_info['name']
|
||||||
|
|
||||||
|
if base_model_counts is not None and base_model:
|
||||||
|
base_model_counts[base_model] = base_model_counts.get(base_model, 0) + 1
|
||||||
|
|
||||||
|
return lora_entry
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def populate_lora_from_civitai(lora_entry: Dict[str, Any], civitai_info_tuple: Tuple[Dict[str, Any] | None, str | None] | Dict[str, Any],
|
||||||
recipe_scanner=None, base_model_counts=None, hash_value=None) -> Optional[Dict[str, Any]]:
|
recipe_scanner=None, base_model_counts=None, hash_value=None) -> Optional[Dict[str, Any]]:
|
||||||
"""
|
"""
|
||||||
Populate a lora entry with information from Civitai API response
|
Populate a lora entry with information from Civitai API response
|
||||||
@@ -151,9 +189,9 @@ class RecipeMetadataParser(ABC):
|
|||||||
|
|
||||||
# Process file information if available
|
# Process file information if available
|
||||||
if 'files' in civitai_info:
|
if 'files' in civitai_info:
|
||||||
# Find the primary model file (type="Model" and primary=true) in the files list
|
# Find the primary model file (weights-type and primary=true) in the files list
|
||||||
model_file = next((file for file in civitai_info.get('files', [])
|
model_file = next((file for file in civitai_info.get('files', [])
|
||||||
if file.get('type') == 'Model' and file.get('primary') == True), None)
|
if file.get('type') in MODEL_WEIGHT_FILE_TYPES and file.get('primary') == True), None)
|
||||||
|
|
||||||
if model_file:
|
if model_file:
|
||||||
# Get size
|
# Get size
|
||||||
@@ -175,10 +213,18 @@ class RecipeMetadataParser(ABC):
|
|||||||
lora_entry['localPath'] = local_path
|
lora_entry['localPath'] = local_path
|
||||||
lora_entry['file_name'] = os.path.splitext(os.path.basename(local_path))[0]
|
lora_entry['file_name'] = os.path.splitext(os.path.basename(local_path))[0]
|
||||||
|
|
||||||
# Get thumbnail from local preview if available
|
# Get thumbnail from local preview if available.
|
||||||
|
# Match the cache item by local path first (get_path_by_hash
|
||||||
|
# cascade: 10-char autov2 / 12-char autov3), then by hash.
|
||||||
lora_cache = await lora_scanner.get_cached_data()
|
lora_cache = await lora_scanner.get_cached_data()
|
||||||
lora_item = next((item for item in lora_cache.raw_data
|
h = (lora_entry.get("hash") or "").lower()
|
||||||
if item['sha256'].lower() == lora_entry['hash'].lower()), None)
|
lora_item = next((item for item in lora_cache.raw_data
|
||||||
|
if (item.get("file_path") or "") == local_path), None)
|
||||||
|
if lora_item is None:
|
||||||
|
lora_item = next((item for item in lora_cache.raw_data
|
||||||
|
if (item.get("sha256") or "").lower() == h
|
||||||
|
or (item.get("autov3") or "").lower() == h
|
||||||
|
or (item.get("sha256") or "")[:10].lower() == h), None)
|
||||||
if lora_item and 'preview_url' in lora_item:
|
if lora_item and 'preview_url' in lora_item:
|
||||||
lora_entry['thumbnailUrl'] = config.get_preview_static_url(lora_item['preview_url'])
|
lora_entry['thumbnailUrl'] = config.get_preview_static_url(lora_item['preview_url'])
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -194,7 +240,7 @@ class RecipeMetadataParser(ABC):
|
|||||||
return lora_entry
|
return lora_entry
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def populate_checkpoint_from_civitai(checkpoint: Dict[str, Any], civitai_info: Dict[str, Any]) -> Dict[str, Any]:
|
async def populate_checkpoint_from_civitai(checkpoint: Dict[str, Any], civitai_info: Dict[str, Any] | Tuple[Dict[str, Any] | None, str | None] | None) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Populate checkpoint information from Civitai API response
|
Populate checkpoint information from Civitai API response
|
||||||
|
|
||||||
@@ -249,11 +295,21 @@ class RecipeMetadataParser(ABC):
|
|||||||
checkpoint['id'] = civitai_data.get('id', 0)
|
checkpoint['id'] = civitai_data.get('id', 0)
|
||||||
|
|
||||||
if 'files' in civitai_data:
|
if 'files' in civitai_data:
|
||||||
|
# Prefer the file CivitAI marked primary; fall back to any
|
||||||
|
# weights-type file (providers without primary flags).
|
||||||
model_file = next(
|
model_file = next(
|
||||||
(
|
(
|
||||||
file
|
file
|
||||||
for file in civitai_data.get('files', [])
|
for file in civitai_data.get('files', [])
|
||||||
if file.get('type') == 'Model'
|
if file.get('type') in MODEL_WEIGHT_FILE_TYPES
|
||||||
|
and file.get('primary') is True
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
) or next(
|
||||||
|
(
|
||||||
|
file
|
||||||
|
for file in civitai_data.get('files', [])
|
||||||
|
if file.get('type') in MODEL_WEIGHT_FILE_TYPES
|
||||||
),
|
),
|
||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,3 +1,7 @@
|
|||||||
|
# pyright: reportImportCycles=false
|
||||||
|
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||||
|
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||||
|
# import cycles. Breaking them would require an architectural refactor.
|
||||||
import logging
|
import logging
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
"""Factory for creating recipe metadata parsers."""
|
"""Factory for creating recipe metadata parsers."""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
from typing import Any
|
||||||
from .parsers import (
|
from .parsers import (
|
||||||
RecipeFormatParser,
|
RecipeFormatParser,
|
||||||
ComfyMetadataParser,
|
ComfyMetadataParser,
|
||||||
@@ -31,7 +32,8 @@ class RecipeParserFactory:
|
|||||||
# First, try CivitaiApiMetadataParser for dict input
|
# First, try CivitaiApiMetadataParser for dict input
|
||||||
if isinstance(metadata, dict):
|
if isinstance(metadata, dict):
|
||||||
try:
|
try:
|
||||||
if CivitaiApiMetadataParser().is_metadata_matching(metadata):
|
user_comment: Any = metadata
|
||||||
|
if CivitaiApiMetadataParser().is_metadata_matching(user_comment):
|
||||||
return CivitaiApiMetadataParser()
|
return CivitaiApiMetadataParser()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug(f"CivitaiApiMetadataParser check failed: {e}")
|
logger.debug(f"CivitaiApiMetadataParser check failed: {e}")
|
||||||
|
|||||||
+228
-69
@@ -8,6 +8,7 @@ from typing import Dict, Any
|
|||||||
from ..base import RecipeMetadataParser
|
from ..base import RecipeMetadataParser
|
||||||
from ..constants import GEN_PARAM_KEYS
|
from ..constants import GEN_PARAM_KEYS
|
||||||
from ...services.metadata_service import get_default_metadata_provider
|
from ...services.metadata_service import get_default_metadata_provider
|
||||||
|
from ...utils.constants import is_empty_placeholder_hash
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -52,7 +53,7 @@ class AutomaticMetadataParser(RecipeMetadataParser):
|
|||||||
negative_and_params = ""
|
negative_and_params = ""
|
||||||
|
|
||||||
# Initialize metadata
|
# Initialize metadata
|
||||||
metadata = {
|
metadata: Dict[str, Any] = {
|
||||||
"prompt": prompt,
|
"prompt": prompt,
|
||||||
"loras": []
|
"loras": []
|
||||||
}
|
}
|
||||||
@@ -146,15 +147,13 @@ class AutomaticMetadataParser(RecipeMetadataParser):
|
|||||||
# Initialize hashes dict if it doesn't exist
|
# Initialize hashes dict if it doesn't exist
|
||||||
if "hashes" not in metadata:
|
if "hashes" not in metadata:
|
||||||
metadata["hashes"] = {}
|
metadata["hashes"] = {}
|
||||||
# Add as lora type in the same format as
|
# Lora hashes carries the 12-char AutoV3
|
||||||
# regular hashes. Only override an
|
# hash (resolvable on CivitAI and the local
|
||||||
# existing entry if its value is empty
|
# autov3 index); the Hashes JSON value is
|
||||||
# (Lora hashes is the more reliable
|
# only the 10-char AutoV2 prefix, so on
|
||||||
# source when Hashes JSON has blanks).
|
# conflict the Lora hashes value wins.
|
||||||
key = f"lora:{lora_name}"
|
key = f"lora:{lora_name}"
|
||||||
existing = metadata["hashes"].get(key, "")
|
metadata["hashes"][key] = lora_hash
|
||||||
if not existing:
|
|
||||||
metadata["hashes"][key] = lora_hash
|
|
||||||
|
|
||||||
# Remove lora hashes from params section
|
# Remove lora hashes from params section
|
||||||
params_section = params_section.replace(lora_hashes_match.group(0), '')
|
params_section = params_section.replace(lora_hashes_match.group(0), '')
|
||||||
@@ -362,68 +361,228 @@ class AutomaticMetadataParser(RecipeMetadataParser):
|
|||||||
|
|
||||||
checkpoint = checkpoint_entry
|
checkpoint = checkpoint_entry
|
||||||
|
|
||||||
# If no LoRAs from Civitai resources or to supplement, extract from metadata["hashes"]
|
def normalize_lora_name(name, basename=False):
|
||||||
if not loras or len(loras) == 0:
|
normalized = str(name or '').replace('\\', '/')
|
||||||
# Extract lora weights from extranet tags in prompt (for later use)
|
if normalized.casefold().endswith('.safetensors'):
|
||||||
lora_weights = {}
|
normalized = normalized[:-12]
|
||||||
lora_matches = re.findall(self.EXTRANETS_REGEX, prompt)
|
if basename:
|
||||||
for lora_type, lora_name, lora_weight in lora_matches:
|
normalized = normalized.rsplit('/', 1)[-1]
|
||||||
key = f"{lora_type}:{lora_name}"
|
return normalized.casefold()
|
||||||
lora_weights[key] = round(float(lora_weight), 2)
|
|
||||||
|
def get_version_id(lora):
|
||||||
# Use hashes from metadata as the primary source
|
version_id = lora.get('id')
|
||||||
if metadata.get("hashes"):
|
if version_id in (None, '', 0, '0'):
|
||||||
for hash_key, lora_hash in metadata.get("hashes", {}).items():
|
version_id = lora.get('modelVersionId')
|
||||||
# Only process lora or hypernet types
|
if version_id in (None, '', 0, '0'):
|
||||||
if not hash_key.startswith(("lora:", "hypernet:")):
|
return None
|
||||||
|
return str(version_id)
|
||||||
|
|
||||||
|
prompt_loras = {}
|
||||||
|
for match in re.findall(self.EXTRANETS_REGEX, prompt):
|
||||||
|
lora_type, lora_name, _ = match
|
||||||
|
prompt_loras[(lora_type, normalize_lora_name(lora_name))] = match
|
||||||
|
|
||||||
|
prompt_by_basename = {}
|
||||||
|
for lora_type, lora_name, lora_weight in prompt_loras.values():
|
||||||
|
key = (lora_type, normalize_lora_name(lora_name, True))
|
||||||
|
prompt_by_basename.setdefault(key, []).append((lora_name, round(float(lora_weight), 2)))
|
||||||
|
|
||||||
|
hash_basenames = {
|
||||||
|
(hash_key.split(':', 1)[0], normalize_lora_name(hash_key.split(':', 1)[1], True))
|
||||||
|
for hash_key, hash_value in metadata.get("hashes", {}).items()
|
||||||
|
if hash_value and hash_key.startswith(("lora:", "hypernet:"))
|
||||||
|
}
|
||||||
|
recipe_base_model = checkpoint.get("baseModel") if checkpoint else None
|
||||||
|
if not recipe_base_model and len(base_model_counts) == 1:
|
||||||
|
recipe_base_model = next(iter(base_model_counts))
|
||||||
|
|
||||||
|
resource_lora_count = len(loras)
|
||||||
|
|
||||||
|
def make_lora_entry(lora_type, lora_name, weight, lora_hash=''):
|
||||||
|
return {
|
||||||
|
'name': lora_name,
|
||||||
|
'type': lora_type,
|
||||||
|
'weight': weight,
|
||||||
|
'hash': lora_hash,
|
||||||
|
'existsLocally': False,
|
||||||
|
'localPath': None,
|
||||||
|
'file_name': lora_name,
|
||||||
|
'thumbnailUrl': '/loras_static/images/no-preview.png',
|
||||||
|
'baseModel': '',
|
||||||
|
'size': 0,
|
||||||
|
'downloadUrl': '',
|
||||||
|
'isDeleted': False
|
||||||
|
}
|
||||||
|
|
||||||
|
def merge_or_append_civitai(civitai_entry, preserve_existing_weight=False):
|
||||||
|
civitai_id = get_version_id(civitai_entry)
|
||||||
|
civitai_hash = (civitai_entry.get('hash') or '').lower()
|
||||||
|
for index, existing in enumerate(loras):
|
||||||
|
existing_id = get_version_id(existing)
|
||||||
|
existing_hash = (existing.get('hash') or '').lower()
|
||||||
|
if not (
|
||||||
|
(civitai_id and existing_id == civitai_id)
|
||||||
|
or (civitai_hash and existing_hash == civitai_hash)
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
|
||||||
|
if preserve_existing_weight:
|
||||||
|
civitai_entry['weight'] = existing.get('weight', civitai_entry['weight'])
|
||||||
|
existing_base = existing.get('baseModel')
|
||||||
|
if not civitai_entry.get('baseModel'):
|
||||||
|
civitai_entry['baseModel'] = existing_base or ''
|
||||||
|
elif existing_base:
|
||||||
|
remaining = base_model_counts.get(existing_base, 0) - 1
|
||||||
|
if remaining > 0:
|
||||||
|
base_model_counts[existing_base] = remaining
|
||||||
|
else:
|
||||||
|
base_model_counts.pop(existing_base, None)
|
||||||
|
loras[index] = civitai_entry
|
||||||
|
return
|
||||||
|
loras.append(civitai_entry)
|
||||||
|
|
||||||
|
def merge_or_append_local(local_entry):
|
||||||
|
local_id = get_version_id(local_entry)
|
||||||
|
local_hash = (local_entry.get('hash') or '').lower()
|
||||||
|
for existing in loras:
|
||||||
|
existing_id = get_version_id(existing)
|
||||||
|
existing_hash = (existing.get('hash') or '').lower()
|
||||||
|
if not (
|
||||||
|
(local_id and existing_id == local_id)
|
||||||
|
or (local_hash and existing_hash == local_hash)
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
|
||||||
|
existing['weight'] = local_entry['weight']
|
||||||
|
existing['hash'] = local_entry['hash']
|
||||||
|
existing['file_name'] = local_entry['file_name']
|
||||||
|
existing['existsLocally'] = True
|
||||||
|
existing['localPath'] = local_entry['localPath']
|
||||||
|
existing['size'] = local_entry['size']
|
||||||
|
existing['isDeleted'] = False
|
||||||
|
if not existing.get('modelId') and local_entry.get('modelId'):
|
||||||
|
existing['modelId'] = local_entry['modelId']
|
||||||
|
if not existing.get('baseModel') and local_entry.get('baseModel'):
|
||||||
|
existing['baseModel'] = local_entry['baseModel']
|
||||||
|
base_model_counts[local_entry['baseModel']] = base_model_counts.get(local_entry['baseModel'], 0) + 1
|
||||||
|
thumbnail_url = local_entry.get('thumbnailUrl')
|
||||||
|
if thumbnail_url and not thumbnail_url.endswith('/images/no-preview.png'):
|
||||||
|
existing['thumbnailUrl'] = thumbnail_url
|
||||||
|
return
|
||||||
|
|
||||||
|
if local_entry.get('baseModel'):
|
||||||
|
base_model = local_entry['baseModel']
|
||||||
|
base_model_counts[base_model] = base_model_counts.get(base_model, 0) + 1
|
||||||
|
loras.append(local_entry)
|
||||||
|
|
||||||
|
resolved_prompt_basenames = set()
|
||||||
|
queried_local_basenames = set()
|
||||||
|
for lora_type, lora_name, lora_weight in prompt_loras.values():
|
||||||
|
weight = round(float(lora_weight), 2)
|
||||||
|
basename_key = (lora_type, normalize_lora_name(lora_name, True))
|
||||||
|
matching_resources = [
|
||||||
|
lora
|
||||||
|
for lora in loras[:resource_lora_count]
|
||||||
|
if lora.get('file_name')
|
||||||
|
and normalize_lora_name(lora['file_name'], True) == basename_key[1]
|
||||||
|
and (
|
||||||
|
(lora_type == 'hypernet' and str(lora.get('type', '')).casefold() in ('hypernet', 'hypernetwork'))
|
||||||
|
or (lora_type == 'lora' and str(lora.get('type', '')).casefold() not in ('hypernet', 'hypernetwork'))
|
||||||
|
)
|
||||||
|
]
|
||||||
|
if len(prompt_by_basename[basename_key]) == 1 and len(matching_resources) == 1:
|
||||||
|
matching_resources[0]['weight'] = weight
|
||||||
|
if basename_key not in hash_basenames:
|
||||||
|
resolved_prompt_basenames.add(basename_key)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if basename_key in hash_basenames:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not recipe_scanner or lora_type != 'lora':
|
||||||
|
continue
|
||||||
|
queried_local_basenames.add(basename_key)
|
||||||
|
local_lora = await recipe_scanner.get_local_lora(lora_name, recipe_base_model)
|
||||||
|
if not local_lora:
|
||||||
|
continue
|
||||||
|
|
||||||
|
local_entry = self.populate_lora_from_local(
|
||||||
|
make_lora_entry(lora_type, lora_name, weight),
|
||||||
|
local_lora,
|
||||||
|
)
|
||||||
|
merge_or_append_local(local_entry)
|
||||||
|
resolved_prompt_basenames.add(basename_key)
|
||||||
|
|
||||||
|
for hash_key, lora_hash in metadata.get("hashes", {}).items():
|
||||||
|
if not hash_key.startswith(("lora:", "hypernet:")):
|
||||||
|
continue
|
||||||
|
lora_type, lora_name = hash_key.split(':', 1)
|
||||||
|
basename_key = (lora_type, normalize_lora_name(lora_name, True))
|
||||||
|
if basename_key in resolved_prompt_basenames:
|
||||||
|
continue
|
||||||
|
|
||||||
|
prompt_entries = prompt_by_basename.get(basename_key, [])
|
||||||
|
weight = prompt_entries[0][1] if len(prompt_entries) == 1 else 1.0
|
||||||
|
lora_entry = make_lora_entry(lora_type, lora_name, weight, lora_hash)
|
||||||
|
|
||||||
|
if is_empty_placeholder_hash(lora_hash):
|
||||||
|
# The empty-hash placeholder (SHA256 of an empty byte
|
||||||
|
# string) is not a real hash: never look it up in the
|
||||||
|
# local hash index or on CivitAI. Match by filename;
|
||||||
|
# otherwise keep the item as unresolved (no hash, flagged
|
||||||
|
# hashInvalid so the UI shows the unresolvable-hash state
|
||||||
|
# and offers reconnect instead of download) rather than
|
||||||
|
# dropping it.
|
||||||
|
if recipe_scanner and lora_type == 'lora' and basename_key not in queried_local_basenames:
|
||||||
|
local_lora = await recipe_scanner.get_local_lora(lora_name, recipe_base_model)
|
||||||
|
if local_lora:
|
||||||
|
local_entry = self.populate_lora_from_local(lora_entry, local_lora)
|
||||||
|
merge_or_append_local(local_entry)
|
||||||
continue
|
continue
|
||||||
|
lora_entry['hash'] = ''
|
||||||
# Skip entries without a hash value — they can't be
|
lora_entry['hashInvalid'] = True
|
||||||
# resolved via CivitAI and would only produce a
|
if not resource_lora_count:
|
||||||
# useless "Deleted" entry in the recipe.
|
|
||||||
if not lora_hash:
|
|
||||||
continue
|
|
||||||
|
|
||||||
lora_type, lora_name = hash_key.split(':', 1)
|
|
||||||
|
|
||||||
# Get weight from extranet tags if available, else default to 1.0
|
|
||||||
weight = lora_weights.get(hash_key, 1.0)
|
|
||||||
|
|
||||||
# Initialize lora entry
|
|
||||||
lora_entry = {
|
|
||||||
'name': lora_name,
|
|
||||||
'type': lora_type, # 'lora' or 'hypernet'
|
|
||||||
'weight': weight,
|
|
||||||
'hash': lora_hash,
|
|
||||||
'existsLocally': False,
|
|
||||||
'localPath': None,
|
|
||||||
'file_name': lora_name,
|
|
||||||
'thumbnailUrl': '/loras_static/images/no-preview.png',
|
|
||||||
'baseModel': '',
|
|
||||||
'size': 0,
|
|
||||||
'downloadUrl': '',
|
|
||||||
'isDeleted': False
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try to get info from Civitai
|
|
||||||
if metadata_provider:
|
|
||||||
try:
|
|
||||||
civitai_info = await metadata_provider.get_model_by_hash(lora_hash)
|
|
||||||
|
|
||||||
populated_entry = await self.populate_lora_from_civitai(
|
|
||||||
lora_entry,
|
|
||||||
civitai_info,
|
|
||||||
recipe_scanner,
|
|
||||||
base_model_counts,
|
|
||||||
lora_hash
|
|
||||||
)
|
|
||||||
if populated_entry is None:
|
|
||||||
continue # Skip invalid LoRA types
|
|
||||||
lora_entry = populated_entry
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error fetching Civitai info for LoRA {lora_name}: {e}")
|
|
||||||
|
|
||||||
loras.append(lora_entry)
|
loras.append(lora_entry)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if lora_hash and recipe_scanner and lora_type == 'lora':
|
||||||
|
local_lora = await recipe_scanner.get_local_lora_by_hash(lora_hash)
|
||||||
|
if local_lora:
|
||||||
|
local_entry = self.populate_lora_from_local(lora_entry, local_lora)
|
||||||
|
merge_or_append_local(local_entry)
|
||||||
|
continue
|
||||||
|
|
||||||
|
hash_resolved = False
|
||||||
|
if lora_hash and metadata_provider:
|
||||||
|
try:
|
||||||
|
civitai_info = await metadata_provider.get_model_by_hash(lora_hash)
|
||||||
|
populated_entry = await self.populate_lora_from_civitai(
|
||||||
|
lora_entry,
|
||||||
|
civitai_info,
|
||||||
|
recipe_scanner,
|
||||||
|
base_model_counts,
|
||||||
|
lora_hash,
|
||||||
|
)
|
||||||
|
if populated_entry is None:
|
||||||
|
continue
|
||||||
|
lora_entry = populated_entry
|
||||||
|
hash_resolved = not lora_entry.get('isDeleted')
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error fetching Civitai info for LoRA {lora_name}: {e}")
|
||||||
|
|
||||||
|
if hash_resolved:
|
||||||
|
merge_or_append_civitai(lora_entry, preserve_existing_weight=not prompt_entries)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if recipe_scanner and lora_type == 'lora' and basename_key not in queried_local_basenames:
|
||||||
|
local_lora = await recipe_scanner.get_local_lora(lora_name, recipe_base_model)
|
||||||
|
if local_lora:
|
||||||
|
local_entry = self.populate_lora_from_local(lora_entry, local_lora)
|
||||||
|
merge_or_append_local(local_entry)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if lora_hash and not resource_lora_count:
|
||||||
|
loras.append(lora_entry)
|
||||||
|
|
||||||
# Try to get base model from resources or make educated guess
|
# Try to get base model from resources or make educated guess
|
||||||
base_model = None
|
base_model = None
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import json
|
|||||||
import logging
|
import logging
|
||||||
from typing import Dict, Any, Union
|
from typing import Dict, Any, Union
|
||||||
from ..base import RecipeMetadataParser
|
from ..base import RecipeMetadataParser
|
||||||
from ..constants import GEN_PARAM_KEYS
|
from ..constants import GEN_PARAM_KEYS, VALID_LORA_TYPES
|
||||||
from ...services.metadata_service import get_default_metadata_provider
|
from ...services.metadata_service import get_default_metadata_provider
|
||||||
from ...config import config
|
from ...config import config
|
||||||
|
|
||||||
@@ -14,15 +14,16 @@ logger = logging.getLogger(__name__)
|
|||||||
class CivitaiApiMetadataParser(RecipeMetadataParser):
|
class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||||
"""Parser for Civitai image metadata format"""
|
"""Parser for Civitai image metadata format"""
|
||||||
|
|
||||||
def is_metadata_matching(self, metadata) -> bool:
|
def is_metadata_matching(self, user_comment) -> bool:
|
||||||
"""Check if the metadata matches the Civitai image metadata format
|
"""Check if the metadata matches the Civitai image metadata format
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
metadata: The metadata from the image (dict)
|
user_comment: The metadata from the image (dict)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
bool: True if this parser can handle the metadata
|
bool: True if this parser can handle the metadata
|
||||||
"""
|
"""
|
||||||
|
metadata = user_comment
|
||||||
if not metadata or not isinstance(metadata, dict):
|
if not metadata or not isinstance(metadata, dict):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -73,7 +74,7 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
|||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
async def parse_metadata( # type: ignore[override]
|
async def parse_metadata( # pyright: ignore[reportIncompatibleMethodOverride]
|
||||||
self, user_comment, recipe_scanner=None, civitai_client=None,
|
self, user_comment, recipe_scanner=None, civitai_client=None,
|
||||||
local_cache: dict[str, Any] | None = None,
|
local_cache: dict[str, Any] | None = None,
|
||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
@@ -89,8 +90,7 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
|||||||
Returns:
|
Returns:
|
||||||
Dict containing parsed recipe data
|
Dict containing parsed recipe data
|
||||||
"""
|
"""
|
||||||
metadata: Dict[str, Any] = user_comment # type: ignore[assignment]
|
metadata: Dict[str, Any] = user_comment
|
||||||
metadata = user_comment
|
|
||||||
try:
|
try:
|
||||||
# Get metadata provider instead of using civitai_client directly
|
# Get metadata provider instead of using civitai_client directly
|
||||||
metadata_provider = await get_default_metadata_provider()
|
metadata_provider = await get_default_metadata_provider()
|
||||||
@@ -115,8 +115,29 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
|||||||
):
|
):
|
||||||
metadata = inner_meta
|
metadata = inner_meta
|
||||||
|
|
||||||
|
# Civitai's image API meta parser mangles the A1111 "Lora hashes"
|
||||||
|
# text field into a quote-wrapped dict entry:
|
||||||
|
# '"Daphne Blake Cosplay_v1": "e67ebd5e315f"'
|
||||||
|
# The 12-char AutoV3 it carries is more reliable than the stale
|
||||||
|
# 10-char AutoV2 value in the "hashes" dict, so recover it and
|
||||||
|
# let it override the conflicting entry.
|
||||||
|
if isinstance(metadata, dict):
|
||||||
|
for key, hash_value in list(metadata.items()):
|
||||||
|
if (
|
||||||
|
isinstance(key, str)
|
||||||
|
and key.startswith('"')
|
||||||
|
and isinstance(hash_value, str)
|
||||||
|
and hash_value.endswith('"')
|
||||||
|
):
|
||||||
|
clean_name = key.strip('"').strip()
|
||||||
|
clean_hash = hash_value.strip('"').strip()
|
||||||
|
if clean_name and clean_hash:
|
||||||
|
hashes_dict = metadata.get("hashes")
|
||||||
|
if isinstance(hashes_dict, dict):
|
||||||
|
hashes_dict[f"lora:{clean_name}"] = clean_hash
|
||||||
|
|
||||||
# Initialize result structure
|
# Initialize result structure
|
||||||
result = {
|
result: Dict[str, Any] = {
|
||||||
"base_model": None,
|
"base_model": None,
|
||||||
"loras": [],
|
"loras": [],
|
||||||
"model": None,
|
"model": None,
|
||||||
@@ -125,10 +146,10 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Track already added LoRAs to prevent duplicates
|
# Track already added LoRAs to prevent duplicates
|
||||||
added_loras = {} # key: model_version_id or hash, value: index in result["loras"]
|
added_loras: Dict[str, Any] = {} # key: model_version_id or hash, value: index in result["loras"]
|
||||||
|
|
||||||
# Extract hash information from hashes field for LoRA matching
|
# Extract hash information from hashes field for LoRA matching
|
||||||
lora_hashes = {}
|
lora_hashes: Dict[str, Any] = {}
|
||||||
if "hashes" in metadata and isinstance(metadata["hashes"], dict):
|
if "hashes" in metadata and isinstance(metadata["hashes"], dict):
|
||||||
for key, hash_value in metadata["hashes"].items():
|
for key, hash_value in metadata["hashes"].items():
|
||||||
key_str = str(key)
|
key_str = str(key)
|
||||||
@@ -184,7 +205,7 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
|||||||
if model_info:
|
if model_info:
|
||||||
result["base_model"] = model_info.get("baseModel", "")
|
result["base_model"] = model_info.get("baseModel", "")
|
||||||
|
|
||||||
base_model_counts = {}
|
base_model_counts: Dict[str, int] = {}
|
||||||
|
|
||||||
# Process standard resources array
|
# Process standard resources array
|
||||||
if "resources" in metadata and isinstance(metadata["resources"], list):
|
if "resources" in metadata and isinstance(metadata["resources"], list):
|
||||||
@@ -196,7 +217,7 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
|||||||
# identification because it has an explicit type field and hash,
|
# identification because it has an explicit type field and hash,
|
||||||
# unlike modelVersionIds which is a flat list with no type info.
|
# unlike modelVersionIds which is a flat list with no type info.
|
||||||
if resource_type == "model":
|
if resource_type == "model":
|
||||||
checkpoint_entry = {
|
checkpoint_entry: Dict[str, Any] = {
|
||||||
"id": 0,
|
"id": 0,
|
||||||
"modelId": 0,
|
"modelId": 0,
|
||||||
"name": resource.get("name", "Unknown Model"),
|
"name": resource.get("name", "Unknown Model"),
|
||||||
@@ -216,7 +237,8 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
|||||||
# Try to look up base model from the checkpoint hash
|
# Try to look up base model from the checkpoint hash
|
||||||
cp_hash = checkpoint_entry.get("hash")
|
cp_hash = checkpoint_entry.get("hash")
|
||||||
if cp_hash and metadata_provider:
|
if cp_hash and metadata_provider:
|
||||||
local_cached = local_cache.get(cp_hash) if local_cache else None
|
# local_cache keys are stored lowercase
|
||||||
|
local_cached = local_cache.get(cp_hash.lower()) if local_cache else None
|
||||||
if local_cached:
|
if local_cached:
|
||||||
self._populate_entry_from_cache(
|
self._populate_entry_from_cache(
|
||||||
checkpoint_entry, local_cached
|
checkpoint_entry, local_cached
|
||||||
@@ -294,8 +316,15 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
|||||||
|
|
||||||
# Try to get info from Civitai if hash is available
|
# Try to get info from Civitai if hash is available
|
||||||
if lora_hash and metadata_provider:
|
if lora_hash and metadata_provider:
|
||||||
local_cached = local_cache.get(lora_hash) if local_cache else None
|
# local_cache keys are stored lowercase
|
||||||
|
local_cached = local_cache.get(lora_hash.lower()) if local_cache else None
|
||||||
if local_cached:
|
if local_cached:
|
||||||
|
cached_type = self._cache_item_model_type(local_cached)
|
||||||
|
if cached_type and cached_type not in VALID_LORA_TYPES:
|
||||||
|
logger.debug(
|
||||||
|
f"Skipping non-LoRA cache item for hash {lora_hash}"
|
||||||
|
)
|
||||||
|
continue
|
||||||
self._populate_entry_from_cache(
|
self._populate_entry_from_cache(
|
||||||
lora_entry, local_cached
|
lora_entry, local_cached
|
||||||
)
|
)
|
||||||
@@ -304,6 +333,12 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
|||||||
added_loras[str(lora_entry["id"])] = len(
|
added_loras[str(lora_entry["id"])] = len(
|
||||||
result["loras"]
|
result["loras"]
|
||||||
)
|
)
|
||||||
|
# Mirror base.py:150-151 counts for API-path loras
|
||||||
|
bm = local_cached.get("base_model") or ""
|
||||||
|
if bm:
|
||||||
|
base_model_counts[bm] = base_model_counts.get(
|
||||||
|
bm, 0
|
||||||
|
) + 1
|
||||||
else:
|
else:
|
||||||
try:
|
try:
|
||||||
civitai_info = (
|
civitai_info = (
|
||||||
@@ -649,30 +684,47 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
|||||||
}
|
}
|
||||||
|
|
||||||
if metadata_provider:
|
if metadata_provider:
|
||||||
try:
|
# local_cache keys are stored lowercase
|
||||||
civitai_info = await metadata_provider.get_model_by_hash(
|
local_cached = local_cache.get(lora_hash.lower()) if local_cache else None
|
||||||
lora_hash
|
if local_cached:
|
||||||
)
|
cached_type = self._cache_item_model_type(local_cached)
|
||||||
|
if cached_type and cached_type not in VALID_LORA_TYPES:
|
||||||
populated_entry = await self.populate_lora_from_civitai(
|
logger.debug(
|
||||||
lora_entry,
|
f"Skipping non-LoRA cache item for hash {lora_hash}"
|
||||||
civitai_info,
|
)
|
||||||
recipe_scanner,
|
|
||||||
base_model_counts,
|
|
||||||
lora_hash,
|
|
||||||
)
|
|
||||||
|
|
||||||
if populated_entry is None:
|
|
||||||
continue
|
continue
|
||||||
|
self._populate_entry_from_cache(lora_entry, local_cached)
|
||||||
lora_entry = populated_entry
|
# Mirror base.py:150-151 counts for API-path loras
|
||||||
|
bm = local_cached.get("base_model") or ""
|
||||||
|
if bm:
|
||||||
|
base_model_counts[bm] = base_model_counts.get(bm, 0) + 1
|
||||||
if "id" in lora_entry and lora_entry["id"]:
|
if "id" in lora_entry and lora_entry["id"]:
|
||||||
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
||||||
except Exception as e:
|
else:
|
||||||
logger.error(
|
try:
|
||||||
f"Error fetching Civitai info for LoRA hash {lora_hash}: {e}"
|
civitai_info = await metadata_provider.get_model_by_hash(
|
||||||
)
|
lora_hash
|
||||||
|
)
|
||||||
|
|
||||||
|
populated_entry = await self.populate_lora_from_civitai(
|
||||||
|
lora_entry,
|
||||||
|
civitai_info,
|
||||||
|
recipe_scanner,
|
||||||
|
base_model_counts,
|
||||||
|
lora_hash,
|
||||||
|
)
|
||||||
|
|
||||||
|
if populated_entry is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
lora_entry = populated_entry
|
||||||
|
|
||||||
|
if "id" in lora_entry and lora_entry["id"]:
|
||||||
|
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(
|
||||||
|
f"Error fetching Civitai info for LoRA hash {lora_hash}: {e}"
|
||||||
|
)
|
||||||
|
|
||||||
added_loras[lora_hash] = len(result["loras"])
|
added_loras[lora_hash] = len(result["loras"])
|
||||||
result["loras"].append(lora_entry)
|
result["loras"].append(lora_entry)
|
||||||
@@ -711,32 +763,51 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
|||||||
|
|
||||||
# Try to get info from Civitai if hash is available
|
# Try to get info from Civitai if hash is available
|
||||||
if lora_entry["hash"] and metadata_provider:
|
if lora_entry["hash"] and metadata_provider:
|
||||||
try:
|
# local_cache keys are stored lowercase
|
||||||
civitai_info = await metadata_provider.get_model_by_hash(
|
local_cached = local_cache.get(lora_hash.lower()) if local_cache else None
|
||||||
lora_hash
|
if local_cached:
|
||||||
)
|
cached_type = self._cache_item_model_type(local_cached)
|
||||||
|
if cached_type and cached_type not in VALID_LORA_TYPES:
|
||||||
populated_entry = await self.populate_lora_from_civitai(
|
logger.debug(
|
||||||
lora_entry,
|
f"Skipping non-LoRA cache item for hash {lora_hash}"
|
||||||
civitai_info,
|
)
|
||||||
recipe_scanner,
|
|
||||||
base_model_counts,
|
|
||||||
lora_hash,
|
|
||||||
)
|
|
||||||
|
|
||||||
if populated_entry is None:
|
|
||||||
lora_index += 1
|
lora_index += 1
|
||||||
continue # Skip invalid LoRA types
|
continue # Skip non-LoRA cache items
|
||||||
|
self._populate_entry_from_cache(lora_entry, local_cached)
|
||||||
lora_entry = populated_entry
|
# Mirror base.py:150-151 counts for API-path loras
|
||||||
|
bm = local_cached.get("base_model") or ""
|
||||||
|
if bm:
|
||||||
|
base_model_counts[bm] = base_model_counts.get(bm, 0) + 1
|
||||||
# If we have a version ID from Civitai, track it for deduplication
|
# If we have a version ID from Civitai, track it for deduplication
|
||||||
if "id" in lora_entry and lora_entry["id"]:
|
if "id" in lora_entry and lora_entry["id"]:
|
||||||
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
||||||
except Exception as e:
|
else:
|
||||||
logger.error(
|
try:
|
||||||
f"Error fetching Civitai info for LoRA hash {lora_entry['hash']}: {e}"
|
civitai_info = await metadata_provider.get_model_by_hash(
|
||||||
)
|
lora_hash
|
||||||
|
)
|
||||||
|
|
||||||
|
populated_entry = await self.populate_lora_from_civitai(
|
||||||
|
lora_entry,
|
||||||
|
civitai_info,
|
||||||
|
recipe_scanner,
|
||||||
|
base_model_counts,
|
||||||
|
lora_hash,
|
||||||
|
)
|
||||||
|
|
||||||
|
if populated_entry is None:
|
||||||
|
lora_index += 1
|
||||||
|
continue # Skip invalid LoRA types
|
||||||
|
|
||||||
|
lora_entry = populated_entry
|
||||||
|
|
||||||
|
# If we have a version ID from Civitai, track it for deduplication
|
||||||
|
if "id" in lora_entry and lora_entry["id"]:
|
||||||
|
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(
|
||||||
|
f"Error fetching Civitai info for LoRA hash {lora_entry['hash']}: {e}"
|
||||||
|
)
|
||||||
|
|
||||||
# Track by hash if we have it
|
# Track by hash if we have it
|
||||||
if lora_hash:
|
if lora_hash:
|
||||||
@@ -795,3 +866,14 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
|||||||
base_model = cache_item.get("base_model", "")
|
base_model = cache_item.get("base_model", "")
|
||||||
if base_model:
|
if base_model:
|
||||||
entry["baseModel"] = base_model
|
entry["baseModel"] = base_model
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _cache_item_model_type(cache_item: dict[str, Any]) -> str:
|
||||||
|
"""Lowercased civitai.model.type of a cache item, or '' when unknown."""
|
||||||
|
civ = cache_item.get("civitai")
|
||||||
|
if not isinstance(civ, dict):
|
||||||
|
return ""
|
||||||
|
model_info = civ.get("model")
|
||||||
|
if not isinstance(model_info, dict):
|
||||||
|
return ""
|
||||||
|
return (model_info.get("type") or "").lower()
|
||||||
|
|||||||
+112
-75
@@ -31,41 +31,106 @@ class ComfyMetadataParser(RecipeMetadataParser):
|
|||||||
metadata_provider = await get_default_metadata_provider()
|
metadata_provider = await get_default_metadata_provider()
|
||||||
|
|
||||||
data = json.loads(user_comment)
|
data = json.loads(user_comment)
|
||||||
|
|
||||||
|
checkpoint_nodes = {k: v for k, v in data.items() if isinstance(v, dict) and v.get('class_type') == 'CheckpointLoaderSimple'}
|
||||||
|
checkpoint = None
|
||||||
|
checkpoint_id = None
|
||||||
|
checkpoint_version_id = None
|
||||||
|
if checkpoint_nodes:
|
||||||
|
checkpoint_node = next(iter(checkpoint_nodes.values()))
|
||||||
|
if 'inputs' in checkpoint_node and 'ckpt_name' in checkpoint_node['inputs']:
|
||||||
|
checkpoint_name = checkpoint_node['inputs']['ckpt_name']
|
||||||
|
# Some ComfyUI workflows serialize ckpt_name as a
|
||||||
|
# single-element list (e.g. ["model.safetensors"]) or leave
|
||||||
|
# the value unset (None). Neither is a string, so skip the
|
||||||
|
# CivitAI-URN lookup instead of crashing re.search with a
|
||||||
|
# TypeError that fails the whole image import.
|
||||||
|
if isinstance(checkpoint_name, list):
|
||||||
|
checkpoint_name = (
|
||||||
|
checkpoint_name[0] if checkpoint_name else None
|
||||||
|
)
|
||||||
|
if isinstance(checkpoint_name, str):
|
||||||
|
checkpoint_match = re.search(r'civitai:(\d+)@(\d+)', checkpoint_name)
|
||||||
|
if checkpoint_match:
|
||||||
|
checkpoint_id = checkpoint_match.group(1)
|
||||||
|
checkpoint_version_id = checkpoint_match.group(2)
|
||||||
|
checkpoint = {
|
||||||
|
'id': checkpoint_version_id,
|
||||||
|
'modelId': checkpoint_id,
|
||||||
|
'name': f"Checkpoint {checkpoint_id}",
|
||||||
|
'version': '',
|
||||||
|
'type': 'checkpoint'
|
||||||
|
}
|
||||||
|
if metadata_provider:
|
||||||
|
try:
|
||||||
|
civitai_info_tuple = await metadata_provider.get_model_version_info(checkpoint_version_id)
|
||||||
|
civitai_info, _ = civitai_info_tuple if isinstance(civitai_info_tuple, tuple) else (civitai_info_tuple, None)
|
||||||
|
checkpoint = await self.populate_checkpoint_from_civitai(checkpoint, civitai_info)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error fetching Civitai info for checkpoint: {e}")
|
||||||
|
|
||||||
|
recipe_base_model = checkpoint.get('baseModel') if checkpoint else None
|
||||||
loras = []
|
loras = []
|
||||||
|
lora_candidates = []
|
||||||
# Find all LoraLoader nodes
|
for node in data.values():
|
||||||
lora_nodes = {k: v for k, v in data.items() if isinstance(v, dict) and v.get('class_type') == 'LoraLoader'}
|
if not isinstance(node, dict):
|
||||||
|
|
||||||
# Process each LoraLoader node
|
|
||||||
for node_id, node in lora_nodes.items():
|
|
||||||
if 'inputs' not in node or 'lora_name' not in node['inputs']:
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
lora_name = node['inputs'].get('lora_name', '')
|
inputs = node.get('inputs')
|
||||||
|
if not isinstance(inputs, dict):
|
||||||
# Parse the URN to extract model ID and version ID
|
continue
|
||||||
# Format: "urn:air:sdxl:lora:civitai:1107767@1253442"
|
|
||||||
|
if node.get('class_type') == 'LoraLoader':
|
||||||
|
lora_name = inputs.get('lora_name', '')
|
||||||
|
if isinstance(lora_name, str) and lora_name:
|
||||||
|
lora_candidates.append((lora_name, inputs.get('strength_model', 1.0)))
|
||||||
|
continue
|
||||||
|
|
||||||
|
if node.get('class_type') != 'LoraLoaderLM':
|
||||||
|
continue
|
||||||
|
|
||||||
|
loras_data = inputs.get('loras', [])
|
||||||
|
if isinstance(loras_data, dict):
|
||||||
|
loras_data = loras_data.get('__value__', [])
|
||||||
|
if isinstance(loras_data, list) and len(loras_data) == 1 and isinstance(loras_data[0], list):
|
||||||
|
loras_data = loras_data[0]
|
||||||
|
if not isinstance(loras_data, list):
|
||||||
|
continue
|
||||||
|
|
||||||
|
for lora in loras_data:
|
||||||
|
if not isinstance(lora, dict) or not lora.get('active', False) or lora.get('_isDummy', False):
|
||||||
|
continue
|
||||||
|
lora_name = lora.get('name', '')
|
||||||
|
if isinstance(lora_name, str) and lora_name:
|
||||||
|
lora_candidates.append((lora_name, lora.get('strength', 1.0)))
|
||||||
|
|
||||||
|
for lora_name, weight in lora_candidates:
|
||||||
|
if isinstance(weight, str):
|
||||||
|
try:
|
||||||
|
weight = float(weight)
|
||||||
|
except ValueError:
|
||||||
|
weight = 1.0
|
||||||
lora_id_match = re.search(r'civitai:(\d+)@(\d+)', lora_name)
|
lora_id_match = re.search(r'civitai:(\d+)@(\d+)', lora_name)
|
||||||
if not lora_id_match:
|
if lora_id_match:
|
||||||
continue
|
model_id = lora_id_match.group(1)
|
||||||
|
model_version_id = lora_id_match.group(2)
|
||||||
model_id = lora_id_match.group(1)
|
entry_name = f"Lora {model_id}"
|
||||||
model_version_id = lora_id_match.group(2)
|
else:
|
||||||
|
model_id = 0
|
||||||
# Get strength from node inputs
|
model_version_id = 0
|
||||||
weight = node['inputs'].get('strength_model', 1.0)
|
entry_name = re.split(r'[\\/]', lora_name)[-1]
|
||||||
|
entry_name = re.sub(r'\.[^.]+$', '', entry_name)
|
||||||
# Initialize lora entry with default values
|
|
||||||
lora_entry = {
|
lora_entry = {
|
||||||
'id': model_version_id,
|
'id': model_version_id,
|
||||||
'modelId': model_id,
|
'modelId': model_id,
|
||||||
'name': f"Lora {model_id}", # Default name
|
'name': entry_name,
|
||||||
'version': '',
|
'version': '',
|
||||||
'type': 'lora',
|
'type': 'lora',
|
||||||
'weight': weight,
|
'weight': weight,
|
||||||
'existsLocally': False,
|
'existsLocally': False,
|
||||||
'localPath': None,
|
'localPath': None,
|
||||||
'file_name': '',
|
'file_name': entry_name,
|
||||||
'hash': '',
|
'hash': '',
|
||||||
'thumbnailUrl': '/loras_static/images/no-preview.png',
|
'thumbnailUrl': '/loras_static/images/no-preview.png',
|
||||||
'baseModel': '',
|
'baseModel': '',
|
||||||
@@ -73,59 +138,31 @@ class ComfyMetadataParser(RecipeMetadataParser):
|
|||||||
'downloadUrl': '',
|
'downloadUrl': '',
|
||||||
'isDeleted': False
|
'isDeleted': False
|
||||||
}
|
}
|
||||||
|
|
||||||
# Get additional info from Civitai if metadata provider is available
|
if lora_id_match:
|
||||||
if metadata_provider:
|
if metadata_provider:
|
||||||
try:
|
try:
|
||||||
civitai_info_tuple = await metadata_provider.get_model_version_info(model_version_id)
|
civitai_info_tuple = await metadata_provider.get_model_version_info(model_version_id)
|
||||||
# Populate lora entry with Civitai info
|
populated_entry = await self.populate_lora_from_civitai(
|
||||||
populated_entry = await self.populate_lora_from_civitai(
|
lora_entry,
|
||||||
lora_entry,
|
civitai_info_tuple,
|
||||||
civitai_info_tuple,
|
recipe_scanner
|
||||||
recipe_scanner
|
)
|
||||||
)
|
if populated_entry is None:
|
||||||
if populated_entry is None:
|
continue
|
||||||
continue # Skip invalid LoRA types
|
lora_entry = populated_entry
|
||||||
lora_entry = populated_entry
|
except Exception as e:
|
||||||
except Exception as e:
|
logger.error(f"Error fetching Civitai info for LoRA: {e}")
|
||||||
logger.error(f"Error fetching Civitai info for LoRA: {e}")
|
else:
|
||||||
|
if not recipe_scanner:
|
||||||
|
continue
|
||||||
|
local_lora = await recipe_scanner.get_local_lora(lora_name, recipe_base_model)
|
||||||
|
if not local_lora:
|
||||||
|
continue
|
||||||
|
lora_entry = self.populate_lora_from_local(lora_entry, local_lora)
|
||||||
|
|
||||||
loras.append(lora_entry)
|
loras.append(lora_entry)
|
||||||
|
|
||||||
# Find checkpoint info
|
|
||||||
checkpoint_nodes = {k: v for k, v in data.items() if isinstance(v, dict) and v.get('class_type') == 'CheckpointLoaderSimple'}
|
|
||||||
checkpoint = None
|
|
||||||
checkpoint_id = None
|
|
||||||
checkpoint_version_id = None
|
|
||||||
|
|
||||||
if checkpoint_nodes:
|
|
||||||
# Get the first checkpoint node
|
|
||||||
checkpoint_node = next(iter(checkpoint_nodes.values()))
|
|
||||||
if 'inputs' in checkpoint_node and 'ckpt_name' in checkpoint_node['inputs']:
|
|
||||||
checkpoint_name = checkpoint_node['inputs']['ckpt_name']
|
|
||||||
# Parse checkpoint URN
|
|
||||||
checkpoint_match = re.search(r'civitai:(\d+)@(\d+)', checkpoint_name)
|
|
||||||
if checkpoint_match:
|
|
||||||
checkpoint_id = checkpoint_match.group(1)
|
|
||||||
checkpoint_version_id = checkpoint_match.group(2)
|
|
||||||
checkpoint = {
|
|
||||||
'id': checkpoint_version_id,
|
|
||||||
'modelId': checkpoint_id,
|
|
||||||
'name': f"Checkpoint {checkpoint_id}",
|
|
||||||
'version': '',
|
|
||||||
'type': 'checkpoint'
|
|
||||||
}
|
|
||||||
|
|
||||||
# Get additional checkpoint info from Civitai
|
|
||||||
if metadata_provider:
|
|
||||||
try:
|
|
||||||
civitai_info_tuple = await metadata_provider.get_model_version_info(checkpoint_version_id)
|
|
||||||
civitai_info, _ = civitai_info_tuple if isinstance(civitai_info_tuple, tuple) else (civitai_info_tuple, None)
|
|
||||||
# Populate checkpoint with Civitai info
|
|
||||||
checkpoint = await self.populate_checkpoint_from_civitai(checkpoint, civitai_info)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error fetching Civitai info for checkpoint: {e}")
|
|
||||||
|
|
||||||
# Extract generation parameters
|
# Extract generation parameters
|
||||||
gen_params = {}
|
gen_params = {}
|
||||||
|
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ class MetaFormatParser(RecipeMetadataParser):
|
|||||||
prompt = parts[0].strip()
|
prompt = parts[0].strip()
|
||||||
|
|
||||||
# Initialize metadata
|
# Initialize metadata
|
||||||
metadata = {"prompt": prompt, "loras": []}
|
metadata: Dict[str, Any] = {"prompt": prompt, "loras": []}
|
||||||
|
|
||||||
# Extract negative prompt and parameters if available
|
# Extract negative prompt and parameters if available
|
||||||
if len(parts) > 1:
|
if len(parts) > 1:
|
||||||
|
|||||||
@@ -91,7 +91,15 @@ class RecipeFormatParser(RecipeMetadataParser):
|
|||||||
exists_locally = lora_scanner.has_hash(lora['hash'])
|
exists_locally = lora_scanner.has_hash(lora['hash'])
|
||||||
if exists_locally:
|
if exists_locally:
|
||||||
lora_cache = await lora_scanner.get_cached_data()
|
lora_cache = await lora_scanner.get_cached_data()
|
||||||
lora_item = next((item for item in lora_cache.raw_data if item['sha256'].lower() == lora['hash'].lower()), None)
|
# Cascade match: full sha256, stored autov3, or autov2 (sha256[:10]).
|
||||||
|
h = (lora.get('hash') or '').lower()
|
||||||
|
lora_item = next(
|
||||||
|
(item for item in lora_cache.raw_data
|
||||||
|
if (item.get("sha256") or "").lower() == h
|
||||||
|
or (item.get("autov3") or "").lower() == h
|
||||||
|
or (item.get("sha256") or "")[:10].lower() == h),
|
||||||
|
None
|
||||||
|
)
|
||||||
if lora_item:
|
if lora_item:
|
||||||
lora_entry['existsLocally'] = True
|
lora_entry['existsLocally'] = True
|
||||||
lora_entry['inLibrary'] = True
|
lora_entry['inLibrary'] = True
|
||||||
@@ -148,7 +156,7 @@ class RecipeFormatParser(RecipeMetadataParser):
|
|||||||
checkpoint_data = recipe_metadata.get('checkpoint') or {}
|
checkpoint_data = recipe_metadata.get('checkpoint') or {}
|
||||||
if isinstance(checkpoint_data, dict) and checkpoint_data:
|
if isinstance(checkpoint_data, dict) and checkpoint_data:
|
||||||
version_id = checkpoint_data.get('modelVersionId') or checkpoint_data.get('id')
|
version_id = checkpoint_data.get('modelVersionId') or checkpoint_data.get('id')
|
||||||
checkpoint_entry = {
|
checkpoint_entry: Dict[str, Any] = {
|
||||||
'id': version_id or 0,
|
'id': version_id or 0,
|
||||||
'modelId': checkpoint_data.get('modelId', 0),
|
'modelId': checkpoint_data.get('modelId', 0),
|
||||||
'name': checkpoint_data.get('name', 'Unknown Checkpoint'),
|
'name': checkpoint_data.get('name', 'Unknown Checkpoint'),
|
||||||
@@ -188,7 +196,7 @@ class RecipeFormatParser(RecipeMetadataParser):
|
|||||||
filtered_gen_params[key] = value
|
filtered_gen_params[key] = value
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'base_model': checkpoint['baseModel'] if checkpoint and checkpoint.get('baseModel') else recipe_metadata.get('base_model', ''),
|
'base_model': checkpoint['baseModel'] if checkpoint and checkpoint.get('baseModel') else (recipe_metadata.get('base_model') or None),
|
||||||
'loras': loras,
|
'loras': loras,
|
||||||
'gen_params': filtered_gen_params,
|
'gen_params': filtered_gen_params,
|
||||||
'tags': recipe_metadata.get('tags', []),
|
'tags': recipe_metadata.get('tags', []),
|
||||||
@@ -200,3 +208,24 @@ class RecipeFormatParser(RecipeMetadataParser):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error parsing recipe format metadata: {e}", exc_info=True)
|
logger.error(f"Error parsing recipe format metadata: {e}", exc_info=True)
|
||||||
return {"error": str(e), "loras": []}
|
return {"error": str(e), "loras": []}
|
||||||
|
|
||||||
|
|
||||||
|
def strip_recipe_metadata(metadata_text: str) -> str:
|
||||||
|
"""Strip the ``Recipe metadata: {...}`` block appended by LoRA Manager.
|
||||||
|
|
||||||
|
The saved recipe image carries the original generation metadata followed
|
||||||
|
by an appended recipe JSON block (see ``ExifUtils.append_recipe_metadata``).
|
||||||
|
Re-import wants to re-parse the original embedded metadata, so this returns
|
||||||
|
only the text before the appended marker. The input is returned unchanged
|
||||||
|
when no marker is present.
|
||||||
|
"""
|
||||||
|
if not metadata_text:
|
||||||
|
return metadata_text
|
||||||
|
match = re.search(
|
||||||
|
RecipeFormatParser.METADATA_MARKER,
|
||||||
|
metadata_text,
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
if not match:
|
||||||
|
return metadata_text
|
||||||
|
return metadata_text[: match.start()].strip()
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from typing import TYPE_CHECKING, Callable, Dict, Mapping
|
from typing import TYPE_CHECKING, Awaitable, Callable, Dict, Mapping
|
||||||
|
|
||||||
import jinja2
|
import jinja2
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
@@ -30,6 +30,7 @@ from ..services.websocket_progress_callback import (
|
|||||||
WebSocketProgressCallback,
|
WebSocketProgressCallback,
|
||||||
)
|
)
|
||||||
from ..utils.exif_utils import ExifUtils
|
from ..utils.exif_utils import ExifUtils
|
||||||
|
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES
|
||||||
from ..utils.metadata_manager import MetadataManager
|
from ..utils.metadata_manager import MetadataManager
|
||||||
from .model_route_registrar import COMMON_ROUTE_DEFINITIONS, ModelRouteRegistrar
|
from .model_route_registrar import COMMON_ROUTE_DEFINITIONS, ModelRouteRegistrar
|
||||||
from .handlers.model_handlers import (
|
from .handlers.model_handlers import (
|
||||||
@@ -84,7 +85,7 @@ class BaseModelRoutes(ABC):
|
|||||||
self.metadata_progress_callback = WebSocketBroadcastCallback()
|
self.metadata_progress_callback = WebSocketBroadcastCallback()
|
||||||
|
|
||||||
self._handler_set: ModelHandlerSet | None = None
|
self._handler_set: ModelHandlerSet | None = None
|
||||||
self._handler_mapping: Dict[str, Callable[[web.Request], web.StreamResponse]] | None = None
|
self._handler_mapping: Dict[str, Callable[[web.Request], Awaitable[web.Response]]] | None = None
|
||||||
|
|
||||||
self._preview_service = PreviewAssetService(
|
self._preview_service = PreviewAssetService(
|
||||||
metadata_manager=MetadataManager,
|
metadata_manager=MetadataManager,
|
||||||
@@ -131,7 +132,7 @@ class BaseModelRoutes(ABC):
|
|||||||
self._handler_set = None
|
self._handler_set = None
|
||||||
self._handler_mapping = None
|
self._handler_mapping = None
|
||||||
|
|
||||||
def _ensure_handler_mapping(self) -> Mapping[str, Callable[[web.Request], web.StreamResponse]]:
|
def _ensure_handler_mapping(self) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||||
if self._handler_mapping is None:
|
if self._handler_mapping is None:
|
||||||
handler_set = self._create_handler_set()
|
handler_set = self._create_handler_set()
|
||||||
self._handler_set = handler_set
|
self._handler_set = handler_set
|
||||||
@@ -220,7 +221,7 @@ class BaseModelRoutes(ABC):
|
|||||||
)
|
)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def route_handlers(self) -> Mapping[str, Callable[[web.Request], web.StreamResponse]]:
|
def route_handlers(self) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||||
return self._ensure_handler_mapping()
|
return self._ensure_handler_mapping()
|
||||||
|
|
||||||
def setup_routes(self, app: web.Application, prefix: str) -> None:
|
def setup_routes(self, app: web.Application, prefix: str) -> None:
|
||||||
@@ -237,7 +238,7 @@ class BaseModelRoutes(ABC):
|
|||||||
"""Setup model-specific routes."""
|
"""Setup model-specific routes."""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def _parse_specific_params(self, request: web.Request) -> Dict:
|
def _parse_specific_params(self, request: web.Request) -> Dict[str, Any]:
|
||||||
"""Parse model-specific parameters - to be overridden by subclasses."""
|
"""Parse model-specific parameters - to be overridden by subclasses."""
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
@@ -251,9 +252,9 @@ class BaseModelRoutes(ABC):
|
|||||||
|
|
||||||
def _find_model_file(self, files):
|
def _find_model_file(self, files):
|
||||||
"""Find the appropriate model file from the files list - can be overridden by subclasses."""
|
"""Find the appropriate model file from the files list - can be overridden by subclasses."""
|
||||||
return next((file for file in files if file.get("type") in ("Model", "Diffusion Model") and file.get("primary") is True), None)
|
return next((file for file in files if file.get("type") in MODEL_WEIGHT_FILE_TYPES and file.get("primary") is True), None)
|
||||||
|
|
||||||
def get_handler(self, name: str) -> Callable[[web.Request], web.StreamResponse]:
|
def get_handler(self, name: str) -> Callable[[web.Request], Awaitable[web.StreamResponse]]:
|
||||||
"""Expose handlers for subclasses or tests."""
|
"""Expose handlers for subclasses or tests."""
|
||||||
return self._ensure_handler_mapping()[name]
|
return self._ensure_handler_mapping()[name]
|
||||||
|
|
||||||
@@ -285,7 +286,7 @@ class BaseModelRoutes(ABC):
|
|||||||
)
|
)
|
||||||
return self.model_lifecycle_service
|
return self.model_lifecycle_service
|
||||||
|
|
||||||
def _make_handler_proxy(self, name: str) -> Callable[[web.Request], web.StreamResponse]:
|
def _make_handler_proxy(self, name: str) -> Callable[[web.Request], Awaitable[web.StreamResponse]]:
|
||||||
async def proxy(request: web.Request) -> web.StreamResponse:
|
async def proxy(request: web.Request) -> web.StreamResponse:
|
||||||
try:
|
try:
|
||||||
handler = self.get_handler(name)
|
handler = self.get_handler(name)
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
from typing import Callable, Mapping
|
from typing import Awaitable, Callable, Mapping
|
||||||
|
|
||||||
import jinja2
|
import jinja2
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
@@ -32,6 +32,7 @@ from .handlers.recipe_handlers import (
|
|||||||
RecipePageView,
|
RecipePageView,
|
||||||
RecipeQueryHandler,
|
RecipeQueryHandler,
|
||||||
RecipeSharingHandler,
|
RecipeSharingHandler,
|
||||||
|
RecipeWorkflowHandler,
|
||||||
)
|
)
|
||||||
from .recipe_route_registrar import ROUTE_DEFINITIONS
|
from .recipe_route_registrar import ROUTE_DEFINITIONS
|
||||||
|
|
||||||
@@ -61,7 +62,9 @@ class BaseRecipeRoutes:
|
|||||||
self._i18n_registered = False
|
self._i18n_registered = False
|
||||||
self._startup_hooks_registered = False
|
self._startup_hooks_registered = False
|
||||||
self._handler_set: RecipeHandlerSet | None = None
|
self._handler_set: RecipeHandlerSet | None = None
|
||||||
self._handler_mapping: dict[str, Callable] | None = None
|
self._handler_mapping: Mapping[
|
||||||
|
str, Callable[[web.Request], Awaitable[web.StreamResponse]]
|
||||||
|
] | None = None
|
||||||
|
|
||||||
async def attach_dependencies(self, app: web.Application | None = None) -> None:
|
async def attach_dependencies(self, app: web.Application | None = None) -> None:
|
||||||
"""Resolve shared services from the registry."""
|
"""Resolve shared services from the registry."""
|
||||||
@@ -84,7 +87,9 @@ class BaseRecipeRoutes:
|
|||||||
app.on_startup.append(self.attach_dependencies)
|
app.on_startup.append(self.attach_dependencies)
|
||||||
self._startup_hooks_registered = True
|
self._startup_hooks_registered = True
|
||||||
|
|
||||||
def to_route_mapping(self) -> Mapping[str, Callable]:
|
def to_route_mapping(
|
||||||
|
self,
|
||||||
|
) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||||
"""Return a mapping of handler name to coroutine for registrar binding."""
|
"""Return a mapping of handler name to coroutine for registrar binding."""
|
||||||
|
|
||||||
if self._handler_mapping is None:
|
if self._handler_mapping is None:
|
||||||
@@ -124,17 +129,17 @@ class BaseRecipeRoutes:
|
|||||||
or os.environ.get("HF_HUB_DISABLE_TELEMETRY", "0") == "0"
|
or os.environ.get("HF_HUB_DISABLE_TELEMETRY", "0") == "0"
|
||||||
)
|
)
|
||||||
if not standalone_mode:
|
if not standalone_mode:
|
||||||
from ..metadata_collector import get_metadata # type: ignore[import-not-found]
|
from ..metadata_collector import get_metadata # pyright: ignore[reportMissingImports]
|
||||||
from ..metadata_collector.metadata_processor import ( # type: ignore[import-not-found]
|
from ..metadata_collector.metadata_processor import ( # pyright: ignore[reportMissingImports]
|
||||||
MetadataProcessor,
|
MetadataProcessor,
|
||||||
)
|
)
|
||||||
from ..metadata_collector.metadata_registry import ( # type: ignore[import-not-found]
|
from ..metadata_collector.metadata_registry import ( # pyright: ignore[reportMissingImports]
|
||||||
MetadataRegistry,
|
MetadataRegistry,
|
||||||
)
|
)
|
||||||
else: # pragma: no cover - optional dependency path
|
else: # pragma: no cover - optional dependency path
|
||||||
get_metadata = None # type: ignore[assignment]
|
get_metadata = None # pyright: ignore[reportAssignmentType]
|
||||||
MetadataProcessor = None # type: ignore[assignment]
|
MetadataProcessor = None # pyright: ignore[reportAssignmentType]
|
||||||
MetadataRegistry = None # type: ignore[assignment]
|
MetadataRegistry = None # pyright: ignore[reportAssignmentType]
|
||||||
|
|
||||||
analysis_service = RecipeAnalysisService(
|
analysis_service = RecipeAnalysisService(
|
||||||
exif_utils=ExifUtils,
|
exif_utils=ExifUtils,
|
||||||
@@ -196,6 +201,18 @@ class BaseRecipeRoutes:
|
|||||||
sharing_service=sharing_service,
|
sharing_service=sharing_service,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Lazy import: standalone mode replaces the ``server`` module with a
|
||||||
|
# mock, so resolve PromptServer at handler-set build time instead of
|
||||||
|
# module import time. The handler's standalone check guards UX.
|
||||||
|
from server import PromptServer # pyright: ignore[reportMissingImports]
|
||||||
|
|
||||||
|
workflow = RecipeWorkflowHandler(
|
||||||
|
ensure_dependencies_ready=self.ensure_dependencies_ready,
|
||||||
|
recipe_scanner_getter=recipe_scanner_getter,
|
||||||
|
prompt_server=PromptServer,
|
||||||
|
logger=logger,
|
||||||
|
)
|
||||||
|
|
||||||
from ..services.websocket_manager import ws_manager
|
from ..services.websocket_manager import ws_manager
|
||||||
|
|
||||||
batch_import_service = BatchImportService(
|
batch_import_service = BatchImportService(
|
||||||
@@ -220,4 +237,5 @@ class BaseRecipeRoutes:
|
|||||||
analysis=analysis,
|
analysis=analysis,
|
||||||
sharing=sharing,
|
sharing=sharing,
|
||||||
batch_import=batch_import,
|
batch_import=batch_import,
|
||||||
|
workflow=workflow,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import logging
|
import logging
|
||||||
from typing import Dict, List, Set
|
import os
|
||||||
|
from typing import Any, Dict, List, Set
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
|
|
||||||
from .base_model_routes import BaseModelRoutes
|
from .base_model_routes import BaseModelRoutes
|
||||||
@@ -7,6 +8,7 @@ from .model_route_registrar import ModelRouteRegistrar
|
|||||||
from ..services.checkpoint_service import CheckpointService
|
from ..services.checkpoint_service import CheckpointService
|
||||||
from ..services.service_registry import ServiceRegistry
|
from ..services.service_registry import ServiceRegistry
|
||||||
from ..config import config
|
from ..config import config
|
||||||
|
from ..utils.utils import _format_model_name_for_comfyui
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -28,13 +30,13 @@ class CheckpointRoutes(BaseModelRoutes):
|
|||||||
# Attach service dependencies
|
# Attach service dependencies
|
||||||
self.attach_service(self.service)
|
self.attach_service(self.service)
|
||||||
|
|
||||||
def setup_routes(self, app: web.Application):
|
def setup_routes(self, app: web.Application, prefix: str = "checkpoints"):
|
||||||
"""Setup Checkpoint routes"""
|
"""Setup Checkpoint routes"""
|
||||||
# Schedule service initialization on app startup
|
# Schedule service initialization on app startup
|
||||||
app.on_startup.append(lambda _: self.initialize_services())
|
app.on_startup.append(lambda _: self.initialize_services())
|
||||||
|
|
||||||
# Setup common routes with 'checkpoints' prefix (includes page route)
|
# Setup common routes with 'checkpoints' prefix (includes page route)
|
||||||
super().setup_routes(app, 'checkpoints')
|
super().setup_routes(app, prefix)
|
||||||
|
|
||||||
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
||||||
"""Setup Checkpoint-specific routes"""
|
"""Setup Checkpoint-specific routes"""
|
||||||
@@ -44,7 +46,46 @@ class CheckpointRoutes(BaseModelRoutes):
|
|||||||
# Checkpoint roots and Unet roots
|
# Checkpoint roots and Unet roots
|
||||||
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/checkpoints_roots', prefix, self.get_checkpoints_roots)
|
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/checkpoints_roots', prefix, self.get_checkpoints_roots)
|
||||||
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/unet_roots', prefix, self.get_unet_roots)
|
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/unet_roots', prefix, self.get_unet_roots)
|
||||||
|
|
||||||
|
# Name/base_model pool for the Checkpoint/Unet Loader nodes' base_model filtering
|
||||||
|
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/loader-pool', prefix, self.get_loader_pool)
|
||||||
|
|
||||||
|
async def get_loader_pool(self, request: web.Request) -> web.Response:
|
||||||
|
"""Return ComfyUI-formatted model names with their base_model.
|
||||||
|
|
||||||
|
Backing data for the Checkpoint/Unet Loader nodes'
|
||||||
|
control_after_generate feature: the front-end filters the
|
||||||
|
ckpt_name/unet_name combo options by base_model using this pool, so
|
||||||
|
randomize mode picks within the narrowed set.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
sub_type = request.query.get("sub_type", "checkpoint")
|
||||||
|
if sub_type not in ("checkpoint", "diffusion_model"):
|
||||||
|
return web.json_response({"error": "invalid sub_type"}, status=400)
|
||||||
|
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||||
|
cache = await scanner.get_cached_data()
|
||||||
|
model_roots = scanner.get_model_roots()
|
||||||
|
items: List[Dict[str, str]] = []
|
||||||
|
for item in cache.raw_data:
|
||||||
|
if item.get("sub_type") != sub_type:
|
||||||
|
continue
|
||||||
|
file_path = item.get("file_path", "")
|
||||||
|
if not file_path or not os.path.exists(file_path):
|
||||||
|
continue
|
||||||
|
formatted_name = _format_model_name_for_comfyui(file_path, model_roots)
|
||||||
|
if formatted_name:
|
||||||
|
items.append(
|
||||||
|
{
|
||||||
|
"name": formatted_name,
|
||||||
|
"base_model": item.get("base_model", "") or "",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
items.sort(key=lambda x: x["name"])
|
||||||
|
return web.json_response({"items": items})
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting loader pool: {e}", exc_info=True)
|
||||||
|
return web.json_response({"error": str(e)}, status=500)
|
||||||
|
|
||||||
def _validate_civitai_model_type(self, model_type: str) -> bool:
|
def _validate_civitai_model_type(self, model_type: str) -> bool:
|
||||||
"""Validate CivitAI model type for Checkpoint"""
|
"""Validate CivitAI model type for Checkpoint"""
|
||||||
return model_type.lower() == 'checkpoint'
|
return model_type.lower() == 'checkpoint'
|
||||||
@@ -53,9 +94,9 @@ class CheckpointRoutes(BaseModelRoutes):
|
|||||||
"""Get expected model types string for error messages"""
|
"""Get expected model types string for error messages"""
|
||||||
return "Checkpoint"
|
return "Checkpoint"
|
||||||
|
|
||||||
def _parse_specific_params(self, request: web.Request) -> Dict:
|
def _parse_specific_params(self, request: web.Request) -> Dict[str, Any]:
|
||||||
"""Parse Checkpoint-specific parameters"""
|
"""Parse Checkpoint-specific parameters"""
|
||||||
params: Dict = {}
|
params: Dict[str, Any] = {}
|
||||||
|
|
||||||
if 'checkpoint_hash' in request.query:
|
if 'checkpoint_hash' in request.query:
|
||||||
params['hash_filters'] = {'single_hash': request.query['checkpoint_hash'].lower()}
|
params['hash_filters'] = {'single_hash': request.query['checkpoint_hash'].lower()}
|
||||||
@@ -70,7 +111,7 @@ class CheckpointRoutes(BaseModelRoutes):
|
|||||||
"""Get detailed information for a specific checkpoint by name"""
|
"""Get detailed information for a specific checkpoint by name"""
|
||||||
try:
|
try:
|
||||||
name = request.match_info.get('name', '')
|
name = request.match_info.get('name', '')
|
||||||
checkpoint_info = await self.service.get_model_info_by_name(name)
|
checkpoint_info = await self.service.get_model_info_by_name(name) # pyright: ignore[reportAttributeAccessIssue]
|
||||||
|
|
||||||
if checkpoint_info:
|
if checkpoint_info:
|
||||||
return web.json_response(checkpoint_info)
|
return web.json_response(checkpoint_info)
|
||||||
@@ -89,7 +130,7 @@ class CheckpointRoutes(BaseModelRoutes):
|
|||||||
roots.extend(config.checkpoints_roots or [])
|
roots.extend(config.checkpoints_roots or [])
|
||||||
roots.extend(config.extra_checkpoints_roots or [])
|
roots.extend(config.extra_checkpoints_roots or [])
|
||||||
# Remove duplicates while preserving order
|
# Remove duplicates while preserving order
|
||||||
seen: set = set()
|
seen: set[str] = set()
|
||||||
unique_roots: List[str] = []
|
unique_roots: List[str] = []
|
||||||
for root in roots:
|
for root in roots:
|
||||||
if root and root not in seen:
|
if root and root not in seen:
|
||||||
@@ -114,7 +155,7 @@ class CheckpointRoutes(BaseModelRoutes):
|
|||||||
roots.extend(config.unet_roots or [])
|
roots.extend(config.unet_roots or [])
|
||||||
roots.extend(config.extra_unet_roots or [])
|
roots.extend(config.extra_unet_roots or [])
|
||||||
# Remove duplicates while preserving order
|
# Remove duplicates while preserving order
|
||||||
seen: set = set()
|
seen: set[str] = set()
|
||||||
unique_roots: List[str] = []
|
unique_roots: List[str] = []
|
||||||
for root in roots:
|
for root in roots:
|
||||||
if root and root not in seen:
|
if root and root not in seen:
|
||||||
|
|||||||
@@ -26,13 +26,13 @@ class EmbeddingRoutes(BaseModelRoutes):
|
|||||||
# Attach service dependencies
|
# Attach service dependencies
|
||||||
self.attach_service(self.service)
|
self.attach_service(self.service)
|
||||||
|
|
||||||
def setup_routes(self, app: web.Application):
|
def setup_routes(self, app: web.Application, prefix: str = "embeddings"):
|
||||||
"""Setup Embedding routes"""
|
"""Setup Embedding routes"""
|
||||||
# Schedule service initialization on app startup
|
# Schedule service initialization on app startup
|
||||||
app.on_startup.append(lambda _: self.initialize_services())
|
app.on_startup.append(lambda _: self.initialize_services())
|
||||||
|
|
||||||
# Setup common routes with 'embeddings' prefix (includes page route)
|
# Setup common routes with 'embeddings' prefix (includes page route)
|
||||||
super().setup_routes(app, 'embeddings')
|
super().setup_routes(app, prefix)
|
||||||
|
|
||||||
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
||||||
"""Setup Embedding-specific routes"""
|
"""Setup Embedding-specific routes"""
|
||||||
@@ -51,7 +51,7 @@ class EmbeddingRoutes(BaseModelRoutes):
|
|||||||
"""Get detailed information for a specific embedding by name"""
|
"""Get detailed information for a specific embedding by name"""
|
||||||
try:
|
try:
|
||||||
name = request.match_info.get('name', '')
|
name = request.match_info.get('name', '')
|
||||||
embedding_info = await self.service.get_model_info_by_name(name)
|
embedding_info = await self.service.get_model_info_by_name(name) # pyright: ignore[reportAttributeAccessIssue]
|
||||||
|
|
||||||
if embedding_info:
|
if embedding_info:
|
||||||
return web.json_response(embedding_info)
|
return web.json_response(embedding_info)
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
from typing import Callable, Mapping
|
from typing import Any, Awaitable, Callable, Mapping
|
||||||
|
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
|
|
||||||
@@ -35,7 +35,7 @@ class ExampleImagesRoutes:
|
|||||||
*,
|
*,
|
||||||
ws_manager,
|
ws_manager,
|
||||||
download_manager: DownloadManager | None = None,
|
download_manager: DownloadManager | None = None,
|
||||||
processor=ExampleImagesProcessor,
|
processor: Any = ExampleImagesProcessor,
|
||||||
file_manager=ExampleImagesFileManager,
|
file_manager=ExampleImagesFileManager,
|
||||||
cleanup_service: ExampleImagesCleanupService | None = None,
|
cleanup_service: ExampleImagesCleanupService | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -46,7 +46,9 @@ class ExampleImagesRoutes:
|
|||||||
self._file_manager = file_manager
|
self._file_manager = file_manager
|
||||||
self._cleanup_service = cleanup_service or ExampleImagesCleanupService()
|
self._cleanup_service = cleanup_service or ExampleImagesCleanupService()
|
||||||
self._handler_set: ExampleImagesHandlerSet | None = None
|
self._handler_set: ExampleImagesHandlerSet | None = None
|
||||||
self._handler_mapping: Mapping[str, Callable[[web.Request], web.StreamResponse]] | None = None
|
self._handler_mapping: Mapping[
|
||||||
|
str, Callable[[web.Request], Awaitable[web.StreamResponse]]
|
||||||
|
] | None = None
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def setup_routes(cls, app: web.Application, *, ws_manager) -> None:
|
def setup_routes(cls, app: web.Application, *, ws_manager) -> None:
|
||||||
@@ -61,7 +63,9 @@ class ExampleImagesRoutes:
|
|||||||
registrar = ExampleImagesRouteRegistrar(app)
|
registrar = ExampleImagesRouteRegistrar(app)
|
||||||
registrar.register_routes(self.to_route_mapping())
|
registrar.register_routes(self.to_route_mapping())
|
||||||
|
|
||||||
def to_route_mapping(self) -> Mapping[str, Callable[[web.Request], web.StreamResponse]]:
|
def to_route_mapping(
|
||||||
|
self,
|
||||||
|
) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||||
"""Return the registrar-compatible mapping of handler names to callables."""
|
"""Return the registrar-compatible mapping of handler names to callables."""
|
||||||
|
|
||||||
if self._handler_mapping is None:
|
if self._handler_mapping is None:
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Callable, Mapping
|
from typing import Awaitable, Callable, Mapping
|
||||||
|
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
|
|
||||||
@@ -170,7 +170,7 @@ class ExampleImagesHandlerSet:
|
|||||||
management: ExampleImagesManagementHandler
|
management: ExampleImagesManagementHandler
|
||||||
files: ExampleImagesFileHandler
|
files: ExampleImagesFileHandler
|
||||||
|
|
||||||
def to_route_mapping(self) -> Mapping[str, Callable[[web.Request], web.StreamResponse]]:
|
def to_route_mapping(self) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||||
"""Flatten handler methods into the registrar mapping."""
|
"""Flatten handler methods into the registrar mapping."""
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -56,6 +56,7 @@ from ...utils.constants import (
|
|||||||
)
|
)
|
||||||
from .hf_handlers import HfHandler
|
from .hf_handlers import HfHandler
|
||||||
from .agent_handlers import AgentHandler
|
from .agent_handlers import AgentHandler
|
||||||
|
from .model_handlers import ModelCivitaiHandler
|
||||||
from ...utils.civitai_utils import rewrite_preview_url
|
from ...utils.civitai_utils import rewrite_preview_url
|
||||||
from ...utils.example_images_paths import (
|
from ...utils.example_images_paths import (
|
||||||
find_non_compliant_items_in_example_images_root,
|
find_non_compliant_items_in_example_images_root,
|
||||||
@@ -276,7 +277,7 @@ def _collect_comfyui_session_logs(
|
|||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
if log_entries is None:
|
if log_entries is None:
|
||||||
try:
|
try:
|
||||||
import app.logger as comfy_logger
|
import app.logger as comfy_logger # pyright: ignore[reportMissingImports]
|
||||||
|
|
||||||
log_entries = list(comfy_logger.get_logs() or [])
|
log_entries = list(comfy_logger.get_logs() or [])
|
||||||
except Exception as exc: # pragma: no cover - environment dependent
|
except Exception as exc: # pragma: no cover - environment dependent
|
||||||
@@ -422,10 +423,10 @@ class PromptServerProtocol(Protocol):
|
|||||||
"""Subset of PromptServer used by the handlers."""
|
"""Subset of PromptServer used by the handlers."""
|
||||||
|
|
||||||
instance: "PromptServerProtocol"
|
instance: "PromptServerProtocol"
|
||||||
sockets: dict # maps clientId (sid) → WebSocketResponse
|
sockets: dict[str, Any] # maps clientId (sid) → WebSocketResponse
|
||||||
|
|
||||||
def send_sync(
|
def send_sync(
|
||||||
self, event: str, payload: dict | None = None, sid: str | None = None
|
self, event: str, payload: dict[str, Any] | None = None, sid: str | None = None
|
||||||
) -> None: # pragma: no cover - protocol
|
) -> None: # pragma: no cover - protocol
|
||||||
...
|
...
|
||||||
|
|
||||||
@@ -443,7 +444,12 @@ class UsageStatsFactory(Protocol):
|
|||||||
class MetadataProviderProtocol(Protocol):
|
class MetadataProviderProtocol(Protocol):
|
||||||
async def get_model_versions(
|
async def get_model_versions(
|
||||||
self, model_id: int
|
self, model_id: int
|
||||||
) -> dict | None: # pragma: no cover - protocol
|
) -> dict[str, Any] | None: # pragma: no cover - protocol
|
||||||
|
...
|
||||||
|
|
||||||
|
async def get_user_models(
|
||||||
|
self, username: str, cursor: str | None = None
|
||||||
|
) -> Any: # pragma: no cover - protocol
|
||||||
...
|
...
|
||||||
|
|
||||||
|
|
||||||
@@ -466,16 +472,16 @@ class MetadataArchiveManagerProtocol(Protocol):
|
|||||||
class BackupServiceProtocol(Protocol):
|
class BackupServiceProtocol(Protocol):
|
||||||
async def create_snapshot(
|
async def create_snapshot(
|
||||||
self, *, snapshot_type: str = "manual", persist: bool = False
|
self, *, snapshot_type: str = "manual", persist: bool = False
|
||||||
) -> dict: # pragma: no cover - protocol
|
) -> dict[str, Any]: # pragma: no cover - protocol
|
||||||
...
|
...
|
||||||
|
|
||||||
async def restore_snapshot(self, archive_path: str) -> dict: # pragma: no cover - protocol
|
async def restore_snapshot(self, archive_path: str) -> dict[str, Any]: # pragma: no cover - protocol
|
||||||
...
|
...
|
||||||
|
|
||||||
def get_status(self) -> dict: # pragma: no cover - protocol
|
def get_status(self) -> dict[str, Any]: # pragma: no cover - protocol
|
||||||
...
|
...
|
||||||
|
|
||||||
def get_available_snapshots(self) -> list[dict]: # pragma: no cover - protocol
|
def get_available_snapshots(self) -> list[dict[str, Any]]: # pragma: no cover - protocol
|
||||||
...
|
...
|
||||||
|
|
||||||
|
|
||||||
@@ -491,7 +497,7 @@ class NodeRegistry:
|
|||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
self._lock = asyncio.Lock()
|
self._lock = asyncio.Lock()
|
||||||
# sid → {unique_id → node_info}
|
# sid → {unique_id → node_info}
|
||||||
self._tab_nodes: Dict[str, Dict[str, dict]] = {}
|
self._tab_nodes: Dict[str, Dict[str, dict[str, Any]]] = {}
|
||||||
self._ready = asyncio.Event()
|
self._ready = asyncio.Event()
|
||||||
self._waiting_clients: set[str] = set()
|
self._waiting_clients: set[str] = set()
|
||||||
|
|
||||||
@@ -504,7 +510,7 @@ class NodeRegistry:
|
|||||||
# Helpers to build one node dict (extracted so it's reused for each tab)
|
# Helpers to build one node dict (extracted so it's reused for each tab)
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _build_node_dict(node: dict) -> dict:
|
def _build_node_dict(node: dict[str, Any]) -> dict[str, Any]:
|
||||||
node_id = node["node_id"]
|
node_id = node["node_id"]
|
||||||
graph_id = str(node["graph_id"])
|
graph_id = str(node["graph_id"])
|
||||||
unique_id = f"{graph_id}:{node_id}"
|
unique_id = f"{graph_id}:{node_id}"
|
||||||
@@ -513,11 +519,11 @@ class NodeRegistry:
|
|||||||
bgcolor = node.get("bgcolor") or DEFAULT_NODE_COLOR
|
bgcolor = node.get("bgcolor") or DEFAULT_NODE_COLOR
|
||||||
|
|
||||||
raw_capabilities = node.get("capabilities")
|
raw_capabilities = node.get("capabilities")
|
||||||
capabilities: dict = {}
|
capabilities: dict[str, Any] = {}
|
||||||
if isinstance(raw_capabilities, dict):
|
if isinstance(raw_capabilities, dict):
|
||||||
capabilities = dict(raw_capabilities)
|
capabilities = dict(raw_capabilities)
|
||||||
|
|
||||||
raw_widget_names: list | None = node.get("widget_names")
|
raw_widget_names: list[Any] | None = node.get("widget_names")
|
||||||
if not isinstance(raw_widget_names, list):
|
if not isinstance(raw_widget_names, list):
|
||||||
capability_widget_names = capabilities.get("widget_names")
|
capability_widget_names = capabilities.get("widget_names")
|
||||||
raw_widget_names = (
|
raw_widget_names = (
|
||||||
@@ -565,9 +571,9 @@ class NodeRegistry:
|
|||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# Public API
|
# Public API
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
async def register_nodes(self, sid: str, nodes: list[dict]) -> None:
|
async def register_nodes(self, sid: str, nodes: list[dict[str, Any]]) -> None:
|
||||||
"""Register/replace the node list for a single ComfyUI tab (identified by *sid*)."""
|
"""Register/replace the node list for a single ComfyUI tab (identified by *sid*)."""
|
||||||
tab_nodes: dict[str, dict] = {}
|
tab_nodes: dict[str, dict[str, Any]] = {}
|
||||||
for node in nodes:
|
for node in nodes:
|
||||||
nd = self._build_node_dict(node)
|
nd = self._build_node_dict(node)
|
||||||
tab_nodes[nd["unique_id"]] = nd
|
tab_nodes[nd["unique_id"]] = nd
|
||||||
@@ -602,7 +608,7 @@ class NodeRegistry:
|
|||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
async def get_merged_registry(self, active_sids: set[str] | None = None) -> dict:
|
async def get_merged_registry(self, active_sids: set[str] | None = None) -> dict[str, Any]:
|
||||||
"""Return the union of all known tab nodes, pruning any tab that is no
|
"""Return the union of all known tab nodes, pruning any tab that is no
|
||||||
longer connected."""
|
longer connected."""
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
@@ -619,8 +625,8 @@ class NodeRegistry:
|
|||||||
len(stale_sids), stale_sids,
|
len(stale_sids), stale_sids,
|
||||||
)
|
)
|
||||||
|
|
||||||
merged: dict[str, dict] = {}
|
merged: dict[str, dict[str, Any]] = {}
|
||||||
tab_info: dict[str, dict] = {}
|
tab_info: dict[str, dict[str, Any]] = {}
|
||||||
for sid, nodes in self._tab_nodes.items():
|
for sid, nodes in self._tab_nodes.items():
|
||||||
tab_info[sid] = {
|
tab_info[sid] = {
|
||||||
"node_count": len(nodes),
|
"node_count": len(nodes),
|
||||||
@@ -643,9 +649,60 @@ class NodeRegistry:
|
|||||||
|
|
||||||
|
|
||||||
class HealthCheckHandler:
|
class HealthCheckHandler:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
scanner_getters: Mapping[str, Callable[[], Awaitable[Any]]] | None = None,
|
||||||
|
) -> None:
|
||||||
|
self._scanner_getters = scanner_getters or {
|
||||||
|
"lora": ServiceRegistry.get_lora_scanner,
|
||||||
|
"checkpoint": ServiceRegistry.get_checkpoint_scanner,
|
||||||
|
"embedding": ServiceRegistry.get_embedding_scanner,
|
||||||
|
"recipe": ServiceRegistry.get_recipe_scanner,
|
||||||
|
}
|
||||||
|
|
||||||
async def health_check(self, request: web.Request) -> web.Response:
|
async def health_check(self, request: web.Request) -> web.Response:
|
||||||
return web.json_response({"status": "ok"})
|
return web.json_response({"status": "ok"})
|
||||||
|
|
||||||
|
async def get_init_status(self, request: web.Request) -> web.Response:
|
||||||
|
"""Report aggregate scanner initialization status.
|
||||||
|
|
||||||
|
Used by the initialization page's polling fallback when the
|
||||||
|
/ws/init-progress WebSocket is unavailable. Omits pageType so every
|
||||||
|
page accepts the update and only reloads once all scanners are done.
|
||||||
|
"""
|
||||||
|
pending: list[str] = []
|
||||||
|
for name, getter in self._scanner_getters.items():
|
||||||
|
try:
|
||||||
|
scanner = await getter()
|
||||||
|
except Exception:
|
||||||
|
pending.append(name)
|
||||||
|
continue
|
||||||
|
cache_ready = getattr(scanner, "_cache", None) is not None
|
||||||
|
is_initializing = getattr(scanner, "is_initializing", None)
|
||||||
|
busy = (
|
||||||
|
is_initializing()
|
||||||
|
if callable(is_initializing)
|
||||||
|
else bool(getattr(scanner, "_is_initializing", False))
|
||||||
|
)
|
||||||
|
if busy or not cache_ready:
|
||||||
|
pending.append(name)
|
||||||
|
|
||||||
|
if pending:
|
||||||
|
return web.json_response(
|
||||||
|
{
|
||||||
|
"status": "initializing",
|
||||||
|
"stage": "processing",
|
||||||
|
"details": "Initializing: " + ", ".join(pending),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return web.json_response(
|
||||||
|
{
|
||||||
|
"status": "complete",
|
||||||
|
"progress": 100,
|
||||||
|
"details": "Initialization complete",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class SupportersHandler:
|
class SupportersHandler:
|
||||||
"""Handler for supporters data."""
|
"""Handler for supporters data."""
|
||||||
@@ -653,7 +710,7 @@ class SupportersHandler:
|
|||||||
def __init__(self, logger: logging.Logger | None = None) -> None:
|
def __init__(self, logger: logging.Logger | None = None) -> None:
|
||||||
self._logger = logger or logging.getLogger(__name__)
|
self._logger = logger or logging.getLogger(__name__)
|
||||||
|
|
||||||
def _load_supporters(self) -> dict:
|
def _load_supporters(self) -> dict[str, Any]:
|
||||||
"""Load supporters data from JSON file."""
|
"""Load supporters data from JSON file."""
|
||||||
try:
|
try:
|
||||||
current_file = os.path.abspath(__file__)
|
current_file = os.path.abspath(__file__)
|
||||||
@@ -1229,10 +1286,8 @@ class DoctorHandler:
|
|||||||
settings_snapshot = _sanitize_sensitive_data(
|
settings_snapshot = _sanitize_sensitive_data(
|
||||||
getattr(self._settings, "settings", {}) or {}
|
getattr(self._settings, "settings", {}) or {}
|
||||||
)
|
)
|
||||||
startup_messages_getter = getattr(self._settings, "get_startup_messages", None)
|
startup_messages_getter: Any = getattr(self._settings, "get_startup_messages", None)
|
||||||
startup_messages = (
|
startup_messages = list(startup_messages_getter()) if startup_messages_getter else []
|
||||||
list(startup_messages_getter()) if callable(startup_messages_getter) else []
|
|
||||||
)
|
|
||||||
|
|
||||||
environment = {
|
environment = {
|
||||||
"app_version": app_version,
|
"app_version": app_version,
|
||||||
@@ -1439,7 +1494,7 @@ class SettingsHandler:
|
|||||||
*,
|
*,
|
||||||
settings_service=None,
|
settings_service=None,
|
||||||
metadata_provider_updater: Callable[
|
metadata_provider_updater: Callable[
|
||||||
[], Awaitable[None]
|
[], Awaitable[Any]
|
||||||
] = update_metadata_providers,
|
] = update_metadata_providers,
|
||||||
downloader_factory: Callable[
|
downloader_factory: Callable[
|
||||||
[], Awaitable[DownloaderProtocol]
|
[], Awaitable[DownloaderProtocol]
|
||||||
@@ -1484,8 +1539,8 @@ class SettingsHandler:
|
|||||||
settings_file = getattr(self._settings, "settings_file", None)
|
settings_file = getattr(self._settings, "settings_file", None)
|
||||||
if settings_file:
|
if settings_file:
|
||||||
response_data["settings_file"] = settings_file
|
response_data["settings_file"] = settings_file
|
||||||
messages_getter = getattr(self._settings, "get_startup_messages", None)
|
messages_getter: Any = getattr(self._settings, "get_startup_messages", None)
|
||||||
messages = list(messages_getter()) if callable(messages_getter) else []
|
messages = list(messages_getter()) if messages_getter else []
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{
|
{
|
||||||
"success": True,
|
"success": True,
|
||||||
@@ -2005,11 +2060,11 @@ async def _noop_backup_service() -> None:
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ServiceRegistryAdapter:
|
class ServiceRegistryAdapter:
|
||||||
get_lora_scanner: Callable[[], Awaitable]
|
get_lora_scanner: Callable[[], Awaitable[Any]]
|
||||||
get_checkpoint_scanner: Callable[[], Awaitable]
|
get_checkpoint_scanner: Callable[[], Awaitable[Any]]
|
||||||
get_embedding_scanner: Callable[[], Awaitable]
|
get_embedding_scanner: Callable[[], Awaitable[Any]]
|
||||||
get_downloaded_version_history_service: Callable[[], Awaitable]
|
get_downloaded_version_history_service: Callable[[], Awaitable[Any]]
|
||||||
get_backup_service: Callable[[], Awaitable] = _noop_backup_service
|
get_backup_service: Callable[[], Awaitable[Any]] = _noop_backup_service
|
||||||
|
|
||||||
|
|
||||||
class ModelLibraryHandler:
|
class ModelLibraryHandler:
|
||||||
@@ -2050,14 +2105,71 @@ class ModelLibraryHandler:
|
|||||||
return await self._service_registry.get_downloaded_version_history_service()
|
return await self._service_registry.get_downloaded_version_history_service()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _with_downloaded_flag(versions: list[dict]) -> list[dict]:
|
def _with_downloaded_flag(versions: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||||
enriched: list[dict] = []
|
enriched: list[dict[str, Any]] = []
|
||||||
for version in versions:
|
for version in versions:
|
||||||
entry = dict(version)
|
entry = dict(version)
|
||||||
entry.setdefault("hasBeenDownloaded", True)
|
entry.setdefault("hasBeenDownloaded", True)
|
||||||
enriched.append(entry)
|
enriched.append(entry)
|
||||||
return enriched
|
return enriched
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def _get_downloaded_files(
|
||||||
|
scanner: Any, model_version_id: int
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""Return per-file downloaded state for a version in the library.
|
||||||
|
|
||||||
|
This handler has no CivitAI version payload, so the remote file list
|
||||||
|
is taken from the local entries' cached ``civitai`` metadata (the
|
||||||
|
full version payload persisted at download time, see
|
||||||
|
``BaseModelMetadata.from_civitai_info``) and matched with the same
|
||||||
|
D2 rule used by ``get_civitai_versions`` (#1058). Local entries that
|
||||||
|
cannot be matched to a known remote file (e.g. missing metadata or
|
||||||
|
renamed files) are still reported with ``fileId`` set to None.
|
||||||
|
Returns ``[{fileId, fileName, filePath}]``.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
cache = await scanner.get_cached_data()
|
||||||
|
except Exception: # pragma: no cover - defensive fallback
|
||||||
|
logger.debug(
|
||||||
|
"Failed to read cache for downloaded files of version %s",
|
||||||
|
model_version_id,
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
return []
|
||||||
|
|
||||||
|
files_getter = getattr(cache, "get_files_by_version_id", None)
|
||||||
|
local_entries = files_getter(model_version_id) if files_getter else []
|
||||||
|
if not local_entries:
|
||||||
|
return []
|
||||||
|
|
||||||
|
version_payload: Mapping[str, Any] = {}
|
||||||
|
for entry in local_entries:
|
||||||
|
civitai = entry.get("civitai") if isinstance(entry, Mapping) else None
|
||||||
|
if isinstance(civitai, Mapping) and isinstance(civitai.get("files"), list):
|
||||||
|
version_payload = civitai
|
||||||
|
break
|
||||||
|
|
||||||
|
downloaded = ModelCivitaiHandler._match_downloaded_files(
|
||||||
|
version_payload, local_entries
|
||||||
|
)
|
||||||
|
|
||||||
|
# Surface local files that D2 could not map to a known remote file
|
||||||
|
matched_paths = {item.get("filePath") for item in downloaded}
|
||||||
|
for entry in local_entries:
|
||||||
|
if not isinstance(entry, Mapping):
|
||||||
|
continue
|
||||||
|
if entry.get("file_path") in matched_paths:
|
||||||
|
continue
|
||||||
|
downloaded.append(
|
||||||
|
{
|
||||||
|
"fileId": None,
|
||||||
|
"fileName": entry.get("file_name"),
|
||||||
|
"filePath": entry.get("file_path"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return downloaded
|
||||||
|
|
||||||
async def check_model_exists(self, request: web.Request) -> web.Response:
|
async def check_model_exists(self, request: web.Request) -> web.Response:
|
||||||
try:
|
try:
|
||||||
model_id_str = request.query.get("modelId")
|
model_id_str = request.query.get("modelId")
|
||||||
@@ -2093,9 +2205,11 @@ class ModelLibraryHandler:
|
|||||||
|
|
||||||
exists = False
|
exists = False
|
||||||
model_type = None
|
model_type = None
|
||||||
|
matched_scanner = None
|
||||||
if await lora_scanner.check_model_version_exists(model_version_id):
|
if await lora_scanner.check_model_version_exists(model_version_id):
|
||||||
exists = True
|
exists = True
|
||||||
model_type = "lora"
|
model_type = "lora"
|
||||||
|
matched_scanner = lora_scanner
|
||||||
elif (
|
elif (
|
||||||
checkpoint_scanner
|
checkpoint_scanner
|
||||||
and await checkpoint_scanner.check_model_version_exists(
|
and await checkpoint_scanner.check_model_version_exists(
|
||||||
@@ -2104,6 +2218,7 @@ class ModelLibraryHandler:
|
|||||||
):
|
):
|
||||||
exists = True
|
exists = True
|
||||||
model_type = "checkpoint"
|
model_type = "checkpoint"
|
||||||
|
matched_scanner = checkpoint_scanner
|
||||||
elif (
|
elif (
|
||||||
embedding_scanner
|
embedding_scanner
|
||||||
and await embedding_scanner.check_model_version_exists(
|
and await embedding_scanner.check_model_version_exists(
|
||||||
@@ -2112,6 +2227,7 @@ class ModelLibraryHandler:
|
|||||||
):
|
):
|
||||||
exists = True
|
exists = True
|
||||||
model_type = "embedding"
|
model_type = "embedding"
|
||||||
|
matched_scanner = embedding_scanner
|
||||||
|
|
||||||
if exists:
|
if exists:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
@@ -2120,6 +2236,9 @@ class ModelLibraryHandler:
|
|||||||
"exists": True,
|
"exists": True,
|
||||||
"modelType": model_type,
|
"modelType": model_type,
|
||||||
"hasBeenDownloaded": False,
|
"hasBeenDownloaded": False,
|
||||||
|
"downloadedFiles": await self._get_downloaded_files(
|
||||||
|
matched_scanner, model_version_id
|
||||||
|
),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -2141,6 +2260,7 @@ class ModelLibraryHandler:
|
|||||||
"exists": False,
|
"exists": False,
|
||||||
"modelType": history_type,
|
"modelType": history_type,
|
||||||
"hasBeenDownloaded": has_been_downloaded,
|
"hasBeenDownloaded": has_been_downloaded,
|
||||||
|
"downloadedFiles": [],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -2244,7 +2364,7 @@ class ModelLibraryHandler:
|
|||||||
checkpoint_scanner = await self._service_registry.get_checkpoint_scanner()
|
checkpoint_scanner = await self._service_registry.get_checkpoint_scanner()
|
||||||
embedding_scanner = await self._service_registry.get_embedding_scanner()
|
embedding_scanner = await self._service_registry.get_embedding_scanner()
|
||||||
|
|
||||||
results: list[dict] = []
|
results: list[dict[str, Any]] = []
|
||||||
for model_id in model_ids:
|
for model_id in model_ids:
|
||||||
lora_versions = await lora_scanner.get_model_versions_by_id(model_id)
|
lora_versions = await lora_scanner.get_model_versions_by_id(model_id)
|
||||||
if lora_versions:
|
if lora_versions:
|
||||||
@@ -2353,7 +2473,7 @@ class ModelLibraryHandler:
|
|||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
model_version_id = int(data.get("modelVersionId"))
|
model_version_id = int(data.get("modelVersionId")) # pyright: ignore[reportArgumentType]
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{"success": False, "error": "Parameter modelVersionId must be an integer"},
|
{"success": False, "error": "Parameter modelVersionId must be an integer"},
|
||||||
@@ -2425,8 +2545,8 @@ class ModelLibraryHandler:
|
|||||||
embedding_scanner = await self._service_registry.get_embedding_scanner()
|
embedding_scanner = await self._service_registry.get_embedding_scanner()
|
||||||
|
|
||||||
found_type = None
|
found_type = None
|
||||||
file_path = None
|
|
||||||
found_cache = None
|
found_cache = None
|
||||||
|
entries: list = []
|
||||||
|
|
||||||
for model_type, scanner in (
|
for model_type, scanner in (
|
||||||
("lora", lora_scanner),
|
("lora", lora_scanner),
|
||||||
@@ -2437,27 +2557,43 @@ class ModelLibraryHandler:
|
|||||||
if cache and model_version_id in cache.version_index:
|
if cache and model_version_id in cache.version_index:
|
||||||
found_type = model_type
|
found_type = model_type
|
||||||
found_cache = cache
|
found_cache = cache
|
||||||
entry = cache.version_index[model_version_id]
|
# A version can have several local files (#1058); collect
|
||||||
file_path = entry.get("file_path")
|
# them all so the delete below covers every file.
|
||||||
|
files_getter = getattr(cache, "get_files_by_version_id", None)
|
||||||
|
if files_getter is not None:
|
||||||
|
entries = files_getter(model_version_id)
|
||||||
|
else:
|
||||||
|
entries = [cache.version_index[model_version_id]]
|
||||||
break
|
break
|
||||||
|
|
||||||
if not file_path:
|
file_paths = [
|
||||||
|
entry.get("file_path")
|
||||||
|
for entry in entries
|
||||||
|
if isinstance(entry, dict) and entry.get("file_path")
|
||||||
|
]
|
||||||
|
|
||||||
|
if not file_paths:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{"success": False, "error": "Model version not found in any scanner cache"},
|
{"success": False, "error": "Model version not found in any scanner cache"},
|
||||||
status=404,
|
status=404,
|
||||||
)
|
)
|
||||||
|
|
||||||
target_dir = os.path.dirname(file_path)
|
for file_path in file_paths:
|
||||||
base_name = os.path.basename(file_path)
|
target_dir = os.path.dirname(file_path)
|
||||||
file_name, extension = os.path.splitext(base_name)
|
base_name = os.path.basename(file_path)
|
||||||
await delete_model_artifacts(target_dir, file_name, main_extension=extension)
|
file_name, extension = os.path.splitext(base_name)
|
||||||
|
await delete_model_artifacts(target_dir, file_name, main_extension=extension)
|
||||||
|
|
||||||
if found_cache:
|
if found_cache:
|
||||||
|
removed_paths = set(file_paths)
|
||||||
found_cache.raw_data = [
|
found_cache.raw_data = [
|
||||||
item
|
item
|
||||||
for item in found_cache.raw_data
|
for item in found_cache.raw_data
|
||||||
if item.get("file_path") != file_path
|
if item.get("file_path") not in removed_paths
|
||||||
]
|
]
|
||||||
|
rebuild = getattr(found_cache, "rebuild_version_index", None)
|
||||||
|
if rebuild is not None:
|
||||||
|
rebuild()
|
||||||
await found_cache.resort()
|
await found_cache.resort()
|
||||||
|
|
||||||
scanner_map = {
|
scanner_map = {
|
||||||
@@ -2465,10 +2601,11 @@ class ModelLibraryHandler:
|
|||||||
"checkpoint": checkpoint_scanner,
|
"checkpoint": checkpoint_scanner,
|
||||||
"embedding": embedding_scanner,
|
"embedding": embedding_scanner,
|
||||||
}
|
}
|
||||||
scanner = scanner_map.get(found_type)
|
scanner = scanner_map.get(found_type or "")
|
||||||
if scanner:
|
if scanner:
|
||||||
persist = getattr(scanner, "_persist_current_cache", None)
|
scanner.bump_cache_version()
|
||||||
if callable(persist):
|
persist: Any = getattr(scanner, "_persist_current_cache", None)
|
||||||
|
if persist:
|
||||||
await persist()
|
await persist()
|
||||||
|
|
||||||
history_service = await self._get_download_history_service()
|
history_service = await self._get_download_history_service()
|
||||||
@@ -2479,6 +2616,7 @@ class ModelLibraryHandler:
|
|||||||
"success": True,
|
"success": True,
|
||||||
"modelType": found_type,
|
"modelType": found_type,
|
||||||
"modelVersionId": model_version_id,
|
"modelVersionId": model_version_id,
|
||||||
|
"deletedFiles": len(file_paths),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -2590,6 +2728,8 @@ class ModelLibraryHandler:
|
|||||||
status=400,
|
status=400,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
cursor = request.query.get("cursor")
|
||||||
|
|
||||||
metadata_provider = await self._metadata_provider_factory()
|
metadata_provider = await self._metadata_provider_factory()
|
||||||
if not metadata_provider:
|
if not metadata_provider:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
@@ -2598,7 +2738,7 @@ class ModelLibraryHandler:
|
|||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
models = await metadata_provider.get_user_models(username)
|
result = await metadata_provider.get_user_models(username, cursor)
|
||||||
except NotImplementedError:
|
except NotImplementedError:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{
|
{
|
||||||
@@ -2608,14 +2748,35 @@ class ModelLibraryHandler:
|
|||||||
status=501,
|
status=501,
|
||||||
)
|
)
|
||||||
|
|
||||||
if models is None:
|
if result is None:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{"success": False, "error": "Failed to fetch user models"},
|
{"success": False, "error": "Failed to fetch user models"},
|
||||||
status=502,
|
status=502,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if isinstance(result, dict):
|
||||||
|
models = result.get("items")
|
||||||
|
next_cursor = result.get("nextCursor")
|
||||||
|
else:
|
||||||
|
# Defensive: tolerate providers that still return a raw list
|
||||||
|
models = result
|
||||||
|
next_cursor = None
|
||||||
|
|
||||||
if not isinstance(models, list):
|
if not isinstance(models, list):
|
||||||
models = []
|
models = []
|
||||||
|
if next_cursor is not None and not isinstance(next_cursor, str):
|
||||||
|
next_cursor = str(next_cursor)
|
||||||
|
|
||||||
|
estimated_total = None
|
||||||
|
if cursor is None:
|
||||||
|
get_count = getattr(metadata_provider, "get_creator_model_count", None)
|
||||||
|
if get_count is not None:
|
||||||
|
try:
|
||||||
|
estimated_total = await get_count(username)
|
||||||
|
except Exception: # best-effort only
|
||||||
|
estimated_total = None
|
||||||
|
if not isinstance(estimated_total, int):
|
||||||
|
estimated_total = None
|
||||||
|
|
||||||
lora_scanner = await self._service_registry.get_lora_scanner()
|
lora_scanner = await self._service_registry.get_lora_scanner()
|
||||||
checkpoint_scanner = await self._service_registry.get_checkpoint_scanner()
|
checkpoint_scanner = await self._service_registry.get_checkpoint_scanner()
|
||||||
@@ -2626,15 +2787,16 @@ class ModelLibraryHandler:
|
|||||||
}
|
}
|
||||||
lora_type_aliases = {model_type.lower() for model_type in VALID_LORA_TYPES}
|
lora_type_aliases = {model_type.lower() for model_type in VALID_LORA_TYPES}
|
||||||
|
|
||||||
type_scanner_map: Dict[str, object | None] = {
|
type_scanner_map: Dict[str, Any] = {
|
||||||
**{alias: lora_scanner for alias in lora_type_aliases},
|
**{alias: lora_scanner for alias in lora_type_aliases},
|
||||||
"checkpoint": checkpoint_scanner,
|
"checkpoint": checkpoint_scanner,
|
||||||
"textualinversion": embedding_scanner,
|
"textualinversion": embedding_scanner,
|
||||||
}
|
}
|
||||||
|
|
||||||
versions: list[dict] = []
|
versions: list[dict[str, Any]] = []
|
||||||
history_service = await self._get_download_history_service()
|
history_service = await self._get_download_history_service()
|
||||||
model_ids: list[int] = []
|
model_ids: list[int] = []
|
||||||
|
model_count = 0
|
||||||
for model in models:
|
for model in models:
|
||||||
try:
|
try:
|
||||||
model_ids.append(int(model.get("id")))
|
model_ids.append(int(model.get("id")))
|
||||||
@@ -2668,6 +2830,8 @@ class ModelLibraryHandler:
|
|||||||
if model_type not in normalized_allowed_types:
|
if model_type not in normalized_allowed_types:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
model_count += 1
|
||||||
|
|
||||||
scanner = type_scanner_map.get(model_type)
|
scanner = type_scanner_map.get(model_type)
|
||||||
if scanner is None:
|
if scanner is None:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
@@ -2681,6 +2845,8 @@ class ModelLibraryHandler:
|
|||||||
tags_value = model.get("tags")
|
tags_value = model.get("tags")
|
||||||
tags = tags_value if isinstance(tags_value, list) else []
|
tags = tags_value if isinstance(tags_value, list) else []
|
||||||
model_id = model.get("id")
|
model_id = model.get("id")
|
||||||
|
if model_id is None:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
model_id_int = int(model_id)
|
model_id_int = int(model_id)
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
@@ -2696,6 +2862,8 @@ class ModelLibraryHandler:
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
version_id = version.get("id")
|
version_id = version.get("id")
|
||||||
|
if version_id is None:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
version_id_int = int(version_id)
|
version_id_int = int(version_id)
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
@@ -2733,7 +2901,15 @@ class ModelLibraryHandler:
|
|||||||
)
|
)
|
||||||
|
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{"success": True, "username": username, "versions": versions}
|
{
|
||||||
|
"success": True,
|
||||||
|
"username": username,
|
||||||
|
"versions": versions,
|
||||||
|
"modelCount": model_count,
|
||||||
|
"nextCursor": next_cursor,
|
||||||
|
"hasMore": next_cursor is not None,
|
||||||
|
"estimatedTotal": estimated_total,
|
||||||
|
}
|
||||||
)
|
)
|
||||||
except Exception as exc: # pragma: no cover - defensive logging
|
except Exception as exc: # pragma: no cover - defensive logging
|
||||||
logger.error("Failed to get Civitai user models: %s", exc, exc_info=True)
|
logger.error("Failed to get Civitai user models: %s", exc, exc_info=True)
|
||||||
@@ -2749,7 +2925,7 @@ class MetadataArchiveHandler:
|
|||||||
] = get_metadata_archive_manager,
|
] = get_metadata_archive_manager,
|
||||||
settings_service=None,
|
settings_service=None,
|
||||||
metadata_provider_updater: Callable[
|
metadata_provider_updater: Callable[
|
||||||
[], Awaitable[None]
|
[], Awaitable[Any]
|
||||||
] = update_metadata_providers,
|
] = update_metadata_providers,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._metadata_archive_manager_factory = metadata_archive_manager_factory
|
self._metadata_archive_manager_factory = metadata_archive_manager_factory
|
||||||
@@ -2896,7 +3072,7 @@ class BackupHandler:
|
|||||||
|
|
||||||
if request.content_type.startswith("multipart/"):
|
if request.content_type.startswith("multipart/"):
|
||||||
reader = await request.multipart()
|
reader = await request.multipart()
|
||||||
field = await reader.next()
|
field: Any = await reader.next()
|
||||||
uploaded = False
|
uploaded = False
|
||||||
while field is not None:
|
while field is not None:
|
||||||
if getattr(field, "filename", None):
|
if getattr(field, "filename", None):
|
||||||
@@ -3515,7 +3691,7 @@ class NodeRegistryHandler:
|
|||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
parsed_node_id = node_identifier
|
parsed_node_id = node_identifier
|
||||||
|
|
||||||
payload: dict = {
|
payload: dict[str, Any] = {
|
||||||
"id": parsed_node_id,
|
"id": parsed_node_id,
|
||||||
"value": value,
|
"value": value,
|
||||||
"mode": mode,
|
"mode": mode,
|
||||||
@@ -3639,7 +3815,7 @@ class NodeRegistryHandler:
|
|||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
parsed_node_id = node_identifier
|
parsed_node_id = node_identifier
|
||||||
|
|
||||||
payload: dict = {
|
payload: dict[str, Any] = {
|
||||||
"id": parsed_node_id,
|
"id": parsed_node_id,
|
||||||
"value": value,
|
"value": value,
|
||||||
"mode": mode,
|
"mode": mode,
|
||||||
@@ -3706,8 +3882,8 @@ class MiscHandlerSet:
|
|||||||
doctor: DoctorHandler,
|
doctor: DoctorHandler,
|
||||||
example_workflows: ExampleWorkflowsHandler,
|
example_workflows: ExampleWorkflowsHandler,
|
||||||
base_model: BaseModelHandlerSet,
|
base_model: BaseModelHandlerSet,
|
||||||
hf_handler: HfHandler | None = None,
|
hf_handler: Any = None,
|
||||||
agent_handler: AgentHandler | None = None,
|
agent_handler: Any = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.health = health
|
self.health = health
|
||||||
self.settings = settings
|
self.settings = settings
|
||||||
@@ -3734,6 +3910,7 @@ class MiscHandlerSet:
|
|||||||
) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||||
return {
|
return {
|
||||||
"health_check": self.health.health_check,
|
"health_check": self.health.health_check,
|
||||||
|
"get_init_status": self.health.get_init_status,
|
||||||
"get_settings": self.settings.get_settings,
|
"get_settings": self.settings.get_settings,
|
||||||
"update_settings": self.settings.update_settings,
|
"update_settings": self.settings.update_settings,
|
||||||
"get_doctor_diagnostics": self.doctor.get_doctor_diagnostics,
|
"get_doctor_diagnostics": self.doctor.get_doctor_diagnostics,
|
||||||
|
|||||||
@@ -51,6 +51,29 @@ LICENSE_FIELDS = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_broadcast_models_changed_tasks: set = set()
|
||||||
|
|
||||||
|
|
||||||
|
def _broadcast_models_changed() -> None:
|
||||||
|
"""Notify connected clients that the local model library changed.
|
||||||
|
|
||||||
|
The ComfyUI graph page listens for this event to invalidate its cached
|
||||||
|
model availability data (loras widget missing-model cues / error flags)
|
||||||
|
without waiting for the cache TTL to expire.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
from ...services.websocket_manager import ws_manager
|
||||||
|
|
||||||
|
task = asyncio.create_task(ws_manager.broadcast({"type": "models_changed"}))
|
||||||
|
# Keep a reference so the task is not garbage-collected mid-await.
|
||||||
|
_broadcast_models_changed_tasks.add(task)
|
||||||
|
task.add_done_callback(_broadcast_models_changed_tasks.discard)
|
||||||
|
except Exception:
|
||||||
|
logging.getLogger(__name__).debug(
|
||||||
|
"Failed to broadcast models_changed", exc_info=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class ModelPageView:
|
class ModelPageView:
|
||||||
"""Render the HTML view for model listings."""
|
"""Render the HTML view for model listings."""
|
||||||
|
|
||||||
@@ -71,7 +94,7 @@ class ModelPageView:
|
|||||||
self._server_i18n = server_i18n
|
self._server_i18n = server_i18n
|
||||||
self._logger = logger
|
self._logger = logger
|
||||||
|
|
||||||
def _load_supporters(self) -> dict:
|
def _load_supporters(self) -> dict[str, Any]:
|
||||||
"""Load supporters data from JSON file."""
|
"""Load supporters data from JSON file."""
|
||||||
try:
|
try:
|
||||||
current_file = os.path.abspath(__file__)
|
current_file = os.path.abspath(__file__)
|
||||||
@@ -152,7 +175,7 @@ class ModelPageView:
|
|||||||
self._template_env.filters["t"] = (
|
self._template_env.filters["t"] = (
|
||||||
self._server_i18n.create_template_filter()
|
self._server_i18n.create_template_filter()
|
||||||
)
|
)
|
||||||
self._template_env._i18n_filter_added = True # type: ignore[attr-defined]
|
self._template_env._i18n_filter_added = True # pyright: ignore[reportAttributeAccessIssue]
|
||||||
|
|
||||||
from ...services.llm_service import PROVIDER_PRESETS
|
from ...services.llm_service import PROVIDER_PRESETS
|
||||||
|
|
||||||
@@ -199,7 +222,7 @@ class ModelListingHandler:
|
|||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
service,
|
service,
|
||||||
parse_specific_params: Callable[[web.Request], Dict],
|
parse_specific_params: Callable[[web.Request], Dict[str, Any]],
|
||||||
logger: logging.Logger,
|
logger: logging.Logger,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._service = service
|
self._service = service
|
||||||
@@ -287,7 +310,7 @@ class ModelListingHandler:
|
|||||||
)
|
)
|
||||||
return web.json_response({"error": str(exc)}, status=500)
|
return web.json_response({"error": str(exc)}, status=500)
|
||||||
|
|
||||||
def _parse_common_params(self, request: web.Request) -> Dict:
|
def _parse_common_params(self, request: web.Request) -> Dict[str, Any]:
|
||||||
page = int(request.query.get("page", "1"))
|
page = int(request.query.get("page", "1"))
|
||||||
page_size = min(int(request.query.get("page_size", "20")), 100)
|
page_size = min(int(request.query.get("page_size", "20")), 100)
|
||||||
sort_by = request.query.get("sort_by", "name")
|
sort_by = request.query.get("sort_by", "name")
|
||||||
@@ -341,6 +364,7 @@ class ModelListingHandler:
|
|||||||
== "true",
|
== "true",
|
||||||
"tags": request.query.get("search_tags", "false").lower() == "true",
|
"tags": request.query.get("search_tags", "false").lower() == "true",
|
||||||
"creator": request.query.get("search_creator", "false").lower() == "true",
|
"creator": request.query.get("search_creator", "false").lower() == "true",
|
||||||
|
"hash": request.query.get("search_hash", "false").lower() == "true",
|
||||||
"recursive": request.query.get("recursive", "true").lower() == "true",
|
"recursive": request.query.get("recursive", "true").lower() == "true",
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -460,6 +484,7 @@ class ModelManagementHandler:
|
|||||||
return web.Response(text="Model path is required", status=400)
|
return web.Response(text="Model path is required", status=400)
|
||||||
|
|
||||||
result = await self._lifecycle_service.delete_model(file_path)
|
result = await self._lifecycle_service.delete_model(file_path)
|
||||||
|
_broadcast_models_changed()
|
||||||
return web.json_response(result)
|
return web.json_response(result)
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
return web.json_response({"success": False, "error": str(exc)}, status=400)
|
return web.json_response({"success": False, "error": str(exc)}, status=400)
|
||||||
@@ -609,6 +634,16 @@ class ModelManagementHandler:
|
|||||||
file_path = data.get("file_path")
|
file_path = data.get("file_path")
|
||||||
model_id = data.get("model_id")
|
model_id = data.get("model_id")
|
||||||
model_version_id = data.get("model_version_id")
|
model_version_id = data.get("model_version_id")
|
||||||
|
source = data.get("source")
|
||||||
|
|
||||||
|
if source not in (None, "", "civarchive"):
|
||||||
|
return web.json_response(
|
||||||
|
{
|
||||||
|
"success": False,
|
||||||
|
"error": f"Unsupported relink source: {source}",
|
||||||
|
},
|
||||||
|
status=400,
|
||||||
|
)
|
||||||
|
|
||||||
if not file_path or model_id is None:
|
if not file_path or model_id is None:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
@@ -624,20 +659,33 @@ class ModelManagementHandler:
|
|||||||
metadata_path
|
metadata_path
|
||||||
)
|
)
|
||||||
|
|
||||||
|
relink_kwargs = {
|
||||||
|
"file_path": file_path,
|
||||||
|
"metadata": local_metadata,
|
||||||
|
"model_id": int(model_id),
|
||||||
|
"model_version_id": int(model_version_id) if model_version_id else None,
|
||||||
|
}
|
||||||
|
if source == "civarchive":
|
||||||
|
relink_kwargs["provider_name"] = "civarchive_api"
|
||||||
|
|
||||||
updated_metadata = await self._metadata_sync.relink_metadata(
|
updated_metadata = await self._metadata_sync.relink_metadata(
|
||||||
file_path=file_path,
|
**relink_kwargs
|
||||||
metadata=local_metadata,
|
|
||||||
model_id=int(model_id),
|
|
||||||
model_version_id=int(model_version_id) if model_version_id else None,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
await self._service.scanner.update_single_model_cache(
|
await self._service.scanner.update_single_model_cache(
|
||||||
file_path, file_path, updated_metadata
|
file_path, file_path, updated_metadata
|
||||||
)
|
)
|
||||||
|
|
||||||
message = f"Model successfully re-linked to Civitai model {model_id}" + (
|
if source == "civarchive":
|
||||||
f" version {model_version_id}" if model_version_id else ""
|
message = (
|
||||||
)
|
f"Model successfully re-linked to CivArchive model {model_id}"
|
||||||
|
+ (f" version {model_version_id}" if model_version_id else "")
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
message = (
|
||||||
|
f"Model successfully re-linked to Civitai model {model_id}"
|
||||||
|
+ (f" version {model_version_id}" if model_version_id else "")
|
||||||
|
)
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{
|
{
|
||||||
"success": True,
|
"success": True,
|
||||||
@@ -645,6 +693,8 @@ class ModelManagementHandler:
|
|||||||
"hash": updated_metadata.get("sha256", ""),
|
"hash": updated_metadata.get("sha256", ""),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
except ValueError as exc:
|
||||||
|
return web.json_response({"success": False, "error": str(exc)}, status=400)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
if is_expected_offline_error(str(exc)):
|
if is_expected_offline_error(str(exc)):
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
@@ -658,7 +708,7 @@ class ModelManagementHandler:
|
|||||||
try:
|
try:
|
||||||
reader = await request.multipart()
|
reader = await request.multipart()
|
||||||
|
|
||||||
field = await reader.next()
|
field: Any = await reader.next()
|
||||||
if field is None or field.name != "preview_file":
|
if field is None or field.name != "preview_file":
|
||||||
raise ValueError("Expected 'preview_file' field")
|
raise ValueError("Expected 'preview_file' field")
|
||||||
content_type = field.headers.get("Content-Type", "image/png")
|
content_type = field.headers.get("Content-Type", "image/png")
|
||||||
@@ -700,7 +750,7 @@ class ModelManagementHandler:
|
|||||||
{
|
{
|
||||||
"success": True,
|
"success": True,
|
||||||
"preview_url": config.get_preview_static_url(
|
"preview_url": config.get_preview_static_url(
|
||||||
result["preview_path"]
|
str(result["preview_path"])
|
||||||
),
|
),
|
||||||
"preview_nsfw_level": result["preview_nsfw_level"],
|
"preview_nsfw_level": result["preview_nsfw_level"],
|
||||||
}
|
}
|
||||||
@@ -781,7 +831,7 @@ class ModelManagementHandler:
|
|||||||
|
|
||||||
result = await self._preview_service.replace_preview(
|
result = await self._preview_service.replace_preview(
|
||||||
model_path=model_path,
|
model_path=model_path,
|
||||||
preview_data=preview_data,
|
preview_data=preview_bytes,
|
||||||
content_type=content_type,
|
content_type=content_type,
|
||||||
original_filename=original_filename,
|
original_filename=original_filename,
|
||||||
nsfw_level=nsfw_level,
|
nsfw_level=nsfw_level,
|
||||||
@@ -793,7 +843,7 @@ class ModelManagementHandler:
|
|||||||
{
|
{
|
||||||
"success": True,
|
"success": True,
|
||||||
"preview_url": config.get_preview_static_url(
|
"preview_url": config.get_preview_static_url(
|
||||||
result["preview_path"]
|
str(result["preview_path"])
|
||||||
),
|
),
|
||||||
"preview_nsfw_level": result["preview_nsfw_level"],
|
"preview_nsfw_level": result["preview_nsfw_level"],
|
||||||
}
|
}
|
||||||
@@ -931,6 +981,8 @@ class ModelManagementHandler:
|
|||||||
file_path=file_path, new_file_name=new_file_name
|
file_path=file_path, new_file_name=new_file_name
|
||||||
)
|
)
|
||||||
|
|
||||||
|
_broadcast_models_changed()
|
||||||
|
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{
|
{
|
||||||
**result,
|
**result,
|
||||||
@@ -959,6 +1011,7 @@ class ModelManagementHandler:
|
|||||||
)
|
)
|
||||||
|
|
||||||
result = await self._lifecycle_service.bulk_delete_models(file_paths)
|
result = await self._lifecycle_service.bulk_delete_models(file_paths)
|
||||||
|
_broadcast_models_changed()
|
||||||
return web.json_response(result)
|
return web.json_response(result)
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
return web.json_response({"success": False, "error": str(exc)}, status=400)
|
return web.json_response({"success": False, "error": str(exc)}, status=400)
|
||||||
@@ -1002,6 +1055,11 @@ class ModelQueryHandler:
|
|||||||
self._service = service
|
self._service = service
|
||||||
self._logger = logger
|
self._logger = logger
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _parse_include_empty(request: web.Request) -> bool:
|
||||||
|
"""Parse the include_empty query flag (``1``/``true``)."""
|
||||||
|
return request.query.get("include_empty", "").lower() in ("1", "true")
|
||||||
|
|
||||||
async def get_top_tags(self, request: web.Request) -> web.Response:
|
async def get_top_tags(self, request: web.Request) -> web.Response:
|
||||||
try:
|
try:
|
||||||
limit = int(request.query.get("limit", "20"))
|
limit = int(request.query.get("limit", "20"))
|
||||||
@@ -1061,6 +1119,7 @@ class ModelQueryHandler:
|
|||||||
await self._service.scan_models(
|
await self._service.scan_models(
|
||||||
force_refresh=True, rebuild_cache=full_rebuild
|
force_refresh=True, rebuild_cache=full_rebuild
|
||||||
)
|
)
|
||||||
|
_broadcast_models_changed()
|
||||||
if self._service.scanner.is_cancelled():
|
if self._service.scanner.is_cancelled():
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{
|
{
|
||||||
@@ -1095,8 +1154,14 @@ class ModelQueryHandler:
|
|||||||
|
|
||||||
async def get_folders(self, request: web.Request) -> web.Response:
|
async def get_folders(self, request: web.Request) -> web.Response:
|
||||||
try:
|
try:
|
||||||
cache = await self._service.scanner.get_cached_data()
|
include_empty = self._parse_include_empty(request)
|
||||||
return web.json_response({"folders": cache.folders})
|
if include_empty:
|
||||||
|
# Live enumeration includes empty OS-created directories.
|
||||||
|
folders = await self._service.scanner.get_all_folders()
|
||||||
|
else:
|
||||||
|
cache = await self._service.scanner.get_cached_data()
|
||||||
|
folders = cache.folders
|
||||||
|
return web.json_response({"folders": folders})
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self._logger.error("Error getting folders: %s", exc)
|
self._logger.error("Error getting folders: %s", exc)
|
||||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||||
@@ -1121,7 +1186,9 @@ class ModelQueryHandler:
|
|||||||
{"success": False, "error": "model_root parameter is required"},
|
{"success": False, "error": "model_root parameter is required"},
|
||||||
status=400,
|
status=400,
|
||||||
)
|
)
|
||||||
folder_tree = await self._service.get_folder_tree(model_root)
|
folder_tree = await self._service.get_folder_tree(
|
||||||
|
model_root, include_empty=self._parse_include_empty(request)
|
||||||
|
)
|
||||||
return web.json_response({"success": True, "tree": folder_tree})
|
return web.json_response({"success": True, "tree": folder_tree})
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self._logger.error("Error getting folder tree: %s", exc)
|
self._logger.error("Error getting folder tree: %s", exc)
|
||||||
@@ -1129,7 +1196,9 @@ class ModelQueryHandler:
|
|||||||
|
|
||||||
async def get_unified_folder_tree(self, request: web.Request) -> web.Response:
|
async def get_unified_folder_tree(self, request: web.Request) -> web.Response:
|
||||||
try:
|
try:
|
||||||
unified_tree = await self._service.get_unified_folder_tree()
|
unified_tree = await self._service.get_unified_folder_tree(
|
||||||
|
include_empty=self._parse_include_empty(request)
|
||||||
|
)
|
||||||
return web.json_response({"success": True, "tree": unified_tree})
|
return web.json_response({"success": True, "tree": unified_tree})
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self._logger.error("Error getting unified folder tree: %s", exc)
|
self._logger.error("Error getting unified folder tree: %s", exc)
|
||||||
@@ -1488,8 +1557,73 @@ class ModelQueryHandler:
|
|||||||
search = request.query.get("search", "").strip()
|
search = request.query.get("search", "").strip()
|
||||||
limit = min(int(request.query.get("limit", "15")), 100)
|
limit = min(int(request.query.get("limit", "15")), 100)
|
||||||
offset = max(0, int(request.query.get("offset", "0")))
|
offset = max(0, int(request.query.get("offset", "0")))
|
||||||
|
|
||||||
|
folder = request.query.get("folder")
|
||||||
|
recursive = request.query.get("recursive", "true").lower() == "true"
|
||||||
|
base_models = list(request.query.getall("base_model", []))
|
||||||
|
model_types = list(request.query.getall("model_type", []))
|
||||||
|
|
||||||
|
tag_filters: Dict[str, str] = {}
|
||||||
|
for tag in request.query.getall("tag_include", []):
|
||||||
|
if tag:
|
||||||
|
tag_filters[tag] = "include"
|
||||||
|
for tag in request.query.getall("tag_exclude", []):
|
||||||
|
if tag:
|
||||||
|
tag_filters[tag] = "exclude"
|
||||||
|
|
||||||
|
auto_tag_filters: Dict[str, str] = {}
|
||||||
|
for tag in request.query.getall("auto_tag_include", []):
|
||||||
|
if tag:
|
||||||
|
auto_tag_filters[tag] = "include"
|
||||||
|
for tag in request.query.getall("auto_tag_exclude", []):
|
||||||
|
if tag:
|
||||||
|
auto_tag_filters[tag] = "exclude"
|
||||||
|
|
||||||
|
tag_logic = request.query.get("tag_logic", "any").lower()
|
||||||
|
if tag_logic not in ("any", "all"):
|
||||||
|
tag_logic = "any"
|
||||||
|
|
||||||
|
credit_required = request.query.get("credit_required")
|
||||||
|
if credit_required is not None:
|
||||||
|
credit_required = credit_required.lower() not in ("false", "0", "")
|
||||||
|
|
||||||
|
allow_selling_generated_content = request.query.get(
|
||||||
|
"allow_selling_generated_content"
|
||||||
|
)
|
||||||
|
if allow_selling_generated_content is not None:
|
||||||
|
allow_selling_generated_content = (
|
||||||
|
allow_selling_generated_content.lower() not in ("false", "0", "")
|
||||||
|
)
|
||||||
|
|
||||||
|
# The presence of the recursive param (always sent by the loras
|
||||||
|
# widget when filter mode is on) signals that the filter pipeline
|
||||||
|
# must run even when no concrete filter is set, so global settings
|
||||||
|
# like show_only_sfw stay consistent with the list endpoint.
|
||||||
|
apply_filters = (
|
||||||
|
"recursive" in request.query
|
||||||
|
or folder is not None
|
||||||
|
or bool(base_models)
|
||||||
|
or bool(model_types)
|
||||||
|
or bool(tag_filters)
|
||||||
|
or bool(auto_tag_filters)
|
||||||
|
or credit_required is not None
|
||||||
|
or allow_selling_generated_content is not None
|
||||||
|
)
|
||||||
|
|
||||||
matching_paths = await self._service.search_relative_paths(
|
matching_paths = await self._service.search_relative_paths(
|
||||||
search, limit, offset
|
search,
|
||||||
|
limit,
|
||||||
|
offset,
|
||||||
|
folder=folder,
|
||||||
|
recursive=recursive,
|
||||||
|
base_models=base_models,
|
||||||
|
model_types=model_types,
|
||||||
|
tags=tag_filters,
|
||||||
|
auto_tags=auto_tag_filters,
|
||||||
|
tag_logic=tag_logic,
|
||||||
|
credit_required=credit_required,
|
||||||
|
allow_selling_generated_content=allow_selling_generated_content,
|
||||||
|
apply_filters=apply_filters,
|
||||||
)
|
)
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{"success": True, "relative_paths": matching_paths}
|
{"success": True, "relative_paths": matching_paths}
|
||||||
@@ -1566,7 +1700,8 @@ class ModelDownloadHandler:
|
|||||||
import json
|
import json
|
||||||
|
|
||||||
try:
|
try:
|
||||||
data["file_params"] = json.loads(file_params_json)
|
# Normalize falsy payloads (e.g. {}) to None (#1058)
|
||||||
|
data["file_params"] = json.loads(file_params_json) or None
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
self._logger.warning(
|
self._logger.warning(
|
||||||
"Invalid file_params JSON: %s", file_params_json
|
"Invalid file_params JSON: %s", file_params_json
|
||||||
@@ -1718,7 +1853,8 @@ class ModelDownloadHandler:
|
|||||||
|
|
||||||
model_id = int(model_id_str) if model_id_str else None
|
model_id = int(model_id_str) if model_id_str else None
|
||||||
model_version_id = int(model_version_id_str) if model_version_id_str else None
|
model_version_id = int(model_version_id_str) if model_version_id_str else None
|
||||||
file_params = json.loads(file_params_json) if file_params_json else None
|
# Normalize falsy payloads (e.g. {}) to None (#1058)
|
||||||
|
file_params = (json.loads(file_params_json) if file_params_json else None) or None
|
||||||
|
|
||||||
service = await DownloadQueueService.get_instance()
|
service = await DownloadQueueService.get_instance()
|
||||||
item = await service.add_to_queue(
|
item = await service.add_to_queue(
|
||||||
@@ -1793,8 +1929,18 @@ class ModelDownloadHandler:
|
|||||||
try:
|
try:
|
||||||
status_filter = request.query.get("status") or None
|
status_filter = request.query.get("status") or None
|
||||||
service = await DownloadQueueService.get_instance()
|
service = await DownloadQueueService.get_instance()
|
||||||
cleared = await service.clear_queue(status_filter=status_filter)
|
cleared_ids = await service.clear_queue(status_filter=status_filter)
|
||||||
return web.json_response({"success": True, "cleared": cleared})
|
# Clearing the queue rows alone would orphan any in-memory tasks
|
||||||
|
# and persisted aria2 state for those downloads, leaving them
|
||||||
|
# polling the daemon invisibly. Tear that tracking down too.
|
||||||
|
try:
|
||||||
|
await self._download_coordinator.discard_cleared_downloads(cleared_ids)
|
||||||
|
except Exception:
|
||||||
|
self._logger.warning(
|
||||||
|
"Failed to discard in-memory state for cleared downloads",
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
return web.json_response({"success": True, "cleared": len(cleared_ids)})
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self._logger.error(
|
self._logger.error(
|
||||||
"Error clearing download queue: %s", exc, exc_info=True
|
"Error clearing download queue: %s", exc, exc_info=True
|
||||||
@@ -1877,9 +2023,11 @@ class ModelDownloadHandler:
|
|||||||
item_id=item_id, download_id=download_id
|
item_id=item_id, download_id=download_id
|
||||||
)
|
)
|
||||||
if item is None:
|
if item is None:
|
||||||
|
# Missing or non-retryable history entry is a business
|
||||||
|
# outcome, not a routing error: 200 lets the extension's
|
||||||
|
# apiFetch 404-fallback and error middleware stay quiet.
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{"success": False, "error": "History item not found or not retryable"},
|
{"success": False, "error": "History item not found or not retryable"}
|
||||||
status=404,
|
|
||||||
)
|
)
|
||||||
return web.json_response({"success": True, "item": item})
|
return web.json_response({"success": True, "item": item})
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -1930,8 +2078,12 @@ class ModelDownloadHandler:
|
|||||||
completed_at=completed_at,
|
completed_at=completed_at,
|
||||||
)
|
)
|
||||||
if item is None:
|
if item is None:
|
||||||
|
# A missing queue item (already completed, or never queued) is
|
||||||
|
# a normal business outcome, not a routing error. Return 200
|
||||||
|
# so the browser extension's apiFetch 404-fallback and the
|
||||||
|
# error middleware stay quiet.
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{"success": False, "error": "Download not found in queue"}, status=404
|
{"success": False, "error": "Download not found in queue"}
|
||||||
)
|
)
|
||||||
return web.json_response({"success": True, "item": item})
|
return web.json_response({"success": True, "item": item})
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -1973,9 +2125,10 @@ class ModelDownloadHandler:
|
|||||||
service = await DownloadQueueService.get_instance()
|
service = await DownloadQueueService.get_instance()
|
||||||
updated = await service.update_status(download_id, status)
|
updated = await service.update_status(download_id, status)
|
||||||
if not updated:
|
if not updated:
|
||||||
|
# Same rationale as complete_download_in_queue: a missing
|
||||||
|
# queue item is a business outcome, not a routing error.
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{"success": False, "error": "Download not found in queue"},
|
{"success": False, "error": "Download not found in queue"}
|
||||||
status=404,
|
|
||||||
)
|
)
|
||||||
return web.json_response({"success": True})
|
return web.json_response({"success": True})
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -1995,7 +2148,7 @@ class ModelCivitaiHandler:
|
|||||||
settings_service: SettingsManager,
|
settings_service: SettingsManager,
|
||||||
ws_manager: WebSocketManager,
|
ws_manager: WebSocketManager,
|
||||||
logger: logging.Logger,
|
logger: logging.Logger,
|
||||||
metadata_provider_factory: Callable[[], Awaitable],
|
metadata_provider_factory: Callable[[], Awaitable[Any]],
|
||||||
validate_model_type: Callable[[str], bool],
|
validate_model_type: Callable[[str], bool],
|
||||||
expected_model_types: Callable[[], str],
|
expected_model_types: Callable[[], str],
|
||||||
find_model_file: Callable[
|
find_model_file: Callable[
|
||||||
@@ -2060,7 +2213,7 @@ class ModelCivitaiHandler:
|
|||||||
downloaded_version_ids = set(
|
downloaded_version_ids = set(
|
||||||
await history_service.get_downloaded_version_ids(
|
await history_service.get_downloaded_version_ids(
|
||||||
self._service.model_type,
|
self._service.model_type,
|
||||||
model_id,
|
int(model_id),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
except Exception as exc: # pragma: no cover - defensive logging
|
except Exception as exc: # pragma: no cover - defensive logging
|
||||||
@@ -2094,6 +2247,19 @@ class ModelCivitaiHandler:
|
|||||||
else:
|
else:
|
||||||
version.pop("localPath", None)
|
version.pop("localPath", None)
|
||||||
|
|
||||||
|
# Per-file downloaded state so multi-file versions can show
|
||||||
|
# which individual files are already in the library (#1058)
|
||||||
|
local_entries: List[Any] = []
|
||||||
|
if version_id is not None and cache:
|
||||||
|
files_getter = getattr(cache, "get_files_by_version_id", None)
|
||||||
|
if files_getter is not None:
|
||||||
|
local_entries = files_getter(version_id)
|
||||||
|
elif cache_entry is not None:
|
||||||
|
local_entries = [cache_entry]
|
||||||
|
version["downloadedFiles"] = self._match_downloaded_files(
|
||||||
|
version, local_entries
|
||||||
|
)
|
||||||
|
|
||||||
model_file = (
|
model_file = (
|
||||||
self._find_model_file(version.get("files", []))
|
self._find_model_file(version.get("files", []))
|
||||||
if isinstance(version.get("files"), Iterable)
|
if isinstance(version.get("files"), Iterable)
|
||||||
@@ -2108,6 +2274,64 @@ class ModelCivitaiHandler:
|
|||||||
)
|
)
|
||||||
return web.Response(status=500, text=str(exc))
|
return web.Response(status=500, text=str(exc))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _match_downloaded_files(
|
||||||
|
version: Mapping[str, Any], local_entries: List[Any]
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Map local library entries back to individual files of a version.
|
||||||
|
|
||||||
|
Matching follows rule D2 (#1058): SHA256 is authoritative when the
|
||||||
|
local entry carries one; otherwise fall back to extension-less file
|
||||||
|
name equality. Returns ``[{fileId, fileName, filePath}]``.
|
||||||
|
"""
|
||||||
|
files = version.get("files")
|
||||||
|
if not isinstance(files, list) or not local_entries:
|
||||||
|
return []
|
||||||
|
|
||||||
|
by_hash: Dict[str, Mapping[str, Any]] = {}
|
||||||
|
by_name: Dict[str, Mapping[str, Any]] = {}
|
||||||
|
for file_info in files:
|
||||||
|
if not isinstance(file_info, Mapping):
|
||||||
|
continue
|
||||||
|
sha = str(
|
||||||
|
(file_info.get("hashes") or {}).get("SHA256") or ""
|
||||||
|
).strip().lower()
|
||||||
|
if sha:
|
||||||
|
by_hash.setdefault(sha, file_info)
|
||||||
|
name = str(file_info.get("name") or "").strip()
|
||||||
|
if name:
|
||||||
|
by_name.setdefault(os.path.splitext(name)[0], file_info)
|
||||||
|
|
||||||
|
downloaded: List[Dict[str, Any]] = []
|
||||||
|
seen_keys: set = set()
|
||||||
|
for entry in local_entries:
|
||||||
|
if not isinstance(entry, Mapping):
|
||||||
|
continue
|
||||||
|
matched: Optional[Mapping[str, Any]] = None
|
||||||
|
local_hash = str(entry.get("sha256") or "").strip().lower()
|
||||||
|
if local_hash:
|
||||||
|
matched = by_hash.get(local_hash)
|
||||||
|
if matched is None:
|
||||||
|
local_name = str(entry.get("file_name") or "").strip()
|
||||||
|
if local_name:
|
||||||
|
matched = by_name.get(local_name)
|
||||||
|
if matched is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
file_id = matched.get("id")
|
||||||
|
dedupe_key = file_id if file_id is not None else matched.get("name")
|
||||||
|
if dedupe_key in seen_keys:
|
||||||
|
continue
|
||||||
|
seen_keys.add(dedupe_key)
|
||||||
|
downloaded.append(
|
||||||
|
{
|
||||||
|
"fileId": file_id,
|
||||||
|
"fileName": matched.get("name"),
|
||||||
|
"filePath": entry.get("file_path"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return downloaded
|
||||||
|
|
||||||
async def get_civitai_model_by_version(self, request: web.Request) -> web.Response:
|
async def get_civitai_model_by_version(self, request: web.Request) -> web.Response:
|
||||||
try:
|
try:
|
||||||
model_version_id = request.match_info.get("modelVersionId")
|
model_version_id = request.match_info.get("modelVersionId")
|
||||||
@@ -2170,6 +2394,8 @@ class ModelMoveHandler:
|
|||||||
result = await self._move_service.move_model(
|
result = await self._move_service.move_model(
|
||||||
file_path, target_path, use_default_paths=use_default_paths
|
file_path, target_path, use_default_paths=use_default_paths
|
||||||
)
|
)
|
||||||
|
if result.get("success"):
|
||||||
|
_broadcast_models_changed()
|
||||||
status = 200 if result.get("success") else 500
|
status = 200 if result.get("success") else 500
|
||||||
return web.json_response(result, status=status)
|
return web.json_response(result, status=status)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -2189,6 +2415,8 @@ class ModelMoveHandler:
|
|||||||
result = await self._move_service.move_models_bulk(
|
result = await self._move_service.move_models_bulk(
|
||||||
file_paths, target_path, use_default_paths=use_default_paths
|
file_paths, target_path, use_default_paths=use_default_paths
|
||||||
)
|
)
|
||||||
|
if result.get("success"):
|
||||||
|
_broadcast_models_changed()
|
||||||
return web.json_response(result)
|
return web.json_response(result)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self._logger.error("Error moving models in bulk: %s", exc, exc_info=True)
|
self._logger.error("Error moving models in bulk: %s", exc, exc_info=True)
|
||||||
@@ -2234,6 +2462,7 @@ class ModelAutoOrganizeHandler:
|
|||||||
progress_callback=self._progress_callback,
|
progress_callback=self._progress_callback,
|
||||||
exclusion_patterns=exclusion_patterns,
|
exclusion_patterns=exclusion_patterns,
|
||||||
)
|
)
|
||||||
|
_broadcast_models_changed()
|
||||||
return web.json_response(result.to_dict())
|
return web.json_response(result.to_dict())
|
||||||
except AutoOrganizeInProgressError:
|
except AutoOrganizeInProgressError:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
@@ -2337,8 +2566,8 @@ class ModelUpdateHandler:
|
|||||||
self._logger.error("Failed to fetch license info: %s", exc, exc_info=True)
|
self._logger.error("Failed to fetch license info: %s", exc, exc_info=True)
|
||||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||||
|
|
||||||
updated: List[Dict[str, str]] = []
|
updated: List[Dict[str, Any]] = []
|
||||||
errors: List[Dict[str, str]] = []
|
errors: List[Dict[str, Any]] = []
|
||||||
for model_id in model_ids:
|
for model_id in model_ids:
|
||||||
license_payload = license_map.get(model_id)
|
license_payload = license_map.get(model_id)
|
||||||
if not license_payload:
|
if not license_payload:
|
||||||
@@ -2351,6 +2580,7 @@ class ModelUpdateHandler:
|
|||||||
model_section = civitai_section.get("model")
|
model_section = civitai_section.get("model")
|
||||||
if not isinstance(model_section, Mapping):
|
if not isinstance(model_section, Mapping):
|
||||||
model_section = {}
|
model_section = {}
|
||||||
|
model_section = dict(model_section)
|
||||||
model_section.update(resolved_payload)
|
model_section.update(resolved_payload)
|
||||||
civitai_section["model"] = model_section
|
civitai_section["model"] = model_section
|
||||||
metadata_payload["civitai"] = civitai_section
|
metadata_payload["civitai"] = civitai_section
|
||||||
@@ -2366,7 +2596,7 @@ class ModelUpdateHandler:
|
|||||||
)
|
)
|
||||||
errors.append({"filePath": metadata_path, "error": str(exc)})
|
errors.append({"filePath": metadata_path, "error": str(exc)})
|
||||||
|
|
||||||
response_payload = {"success": True, "updated": updated}
|
response_payload: Dict[str, Any] = {"success": True, "updated": updated}
|
||||||
missing_model_ids = [mid for mid in model_ids if mid not in license_map]
|
missing_model_ids = [mid for mid in model_ids if mid not in license_map]
|
||||||
if missing_model_ids:
|
if missing_model_ids:
|
||||||
response_payload["missingModelIds"] = missing_model_ids
|
response_payload["missingModelIds"] = missing_model_ids
|
||||||
@@ -2436,6 +2666,7 @@ class ModelUpdateHandler:
|
|||||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||||
|
|
||||||
hide_early_access = False
|
hide_early_access = False
|
||||||
|
hide_paid = False
|
||||||
if self._settings is not None:
|
if self._settings is not None:
|
||||||
try:
|
try:
|
||||||
hide_early_access = bool(
|
hide_early_access = bool(
|
||||||
@@ -2443,12 +2674,27 @@ class ModelUpdateHandler:
|
|||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
try:
|
||||||
|
hide_paid = bool(self._settings.get("hide_paid_updates", False))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
same_base_scope = self._uses_same_base_update_scope()
|
||||||
|
|
||||||
serialized_records = []
|
serialized_records = []
|
||||||
for record in records.values():
|
for record in records.values():
|
||||||
has_update_fn = getattr(record, "has_update", None)
|
has_update_fn = getattr(record, "has_update", None)
|
||||||
if callable(has_update_fn) and has_update_fn(
|
if not callable(has_update_fn):
|
||||||
hide_early_access=hide_early_access
|
continue
|
||||||
|
scoped_fn = (
|
||||||
|
getattr(record, "has_update_for_local_bases", None)
|
||||||
|
if same_base_scope
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
qualifies_fn = scoped_fn if callable(scoped_fn) else has_update_fn
|
||||||
|
if qualifies_fn(
|
||||||
|
hide_early_access=hide_early_access,
|
||||||
|
hide_paid=hide_paid,
|
||||||
):
|
):
|
||||||
serialized_records.append(self._serialize_record(record))
|
serialized_records.append(self._serialize_record(record))
|
||||||
|
|
||||||
@@ -2459,6 +2705,26 @@ class ModelUpdateHandler:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _uses_same_base_update_scope(self) -> bool:
|
||||||
|
"""Return True when update reporting must honor same-base scoping.
|
||||||
|
|
||||||
|
Mirrors ``BaseModelService._annotate_update_flags``: the Updates filter
|
||||||
|
evaluates updates per local base model when ``version_grouping`` is
|
||||||
|
``same_base`` (its default). The refresh summary counts with the same
|
||||||
|
scope so the "Found N update(s)" toast matches what the filter
|
||||||
|
displays. See issue #1083.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if self._settings is None:
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
strategy_value = self._settings.get("version_grouping")
|
||||||
|
except Exception:
|
||||||
|
return True
|
||||||
|
if isinstance(strategy_value, str) and strategy_value.strip():
|
||||||
|
return strategy_value.strip().lower() == "same_base"
|
||||||
|
return True
|
||||||
|
|
||||||
async def set_model_update_ignore(self, request: web.Request) -> web.Response:
|
async def set_model_update_ignore(self, request: web.Request) -> web.Response:
|
||||||
payload = await self._read_json(request)
|
payload = await self._read_json(request)
|
||||||
model_id = self._normalize_model_id(payload.get("modelId"))
|
model_id = self._normalize_model_id(payload.get("modelId"))
|
||||||
@@ -2602,10 +2868,16 @@ class ModelUpdateHandler:
|
|||||||
if not record or not record.versions:
|
if not record or not record.versions:
|
||||||
return record
|
return record
|
||||||
|
|
||||||
# Find versions that need enrichment
|
# Find versions that need enrichment. Permanent paid versions are not
|
||||||
|
# early access (mirror _is_early_access_active) and never carry an end
|
||||||
|
# time, so skip them to avoid pointless per-version API calls.
|
||||||
versions_needing_update = []
|
versions_needing_update = []
|
||||||
for version in record.versions:
|
for version in record.versions:
|
||||||
if version.is_early_access and not version.early_access_ends_at:
|
if (
|
||||||
|
version.is_early_access
|
||||||
|
and not version.early_access_ends_at
|
||||||
|
and not getattr(version, "is_paid", False)
|
||||||
|
):
|
||||||
versions_needing_update.append(version)
|
versions_needing_update.append(version)
|
||||||
|
|
||||||
if not versions_needing_update:
|
if not versions_needing_update:
|
||||||
@@ -2715,6 +2987,7 @@ class ModelUpdateHandler:
|
|||||||
civitai_payload = metadata_payload.get("civitai")
|
civitai_payload = metadata_payload.get("civitai")
|
||||||
if not isinstance(civitai_payload, Mapping):
|
if not isinstance(civitai_payload, Mapping):
|
||||||
civitai_payload = {}
|
civitai_payload = {}
|
||||||
|
civitai_payload = dict(civitai_payload)
|
||||||
|
|
||||||
model_payload = civitai_payload.get("model")
|
model_payload = civitai_payload.get("model")
|
||||||
if not isinstance(model_payload, Mapping):
|
if not isinstance(model_payload, Mapping):
|
||||||
@@ -2759,7 +3032,7 @@ class ModelUpdateHandler:
|
|||||||
|
|
||||||
return aggregated
|
return aggregated
|
||||||
|
|
||||||
def _extract_target_model_ids(self, payload: Dict) -> Optional[List[int]]:
|
def _extract_target_model_ids(self, payload: Dict[str, Any]) -> Optional[List[int]]:
|
||||||
if not isinstance(payload, Mapping):
|
if not isinstance(payload, Mapping):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -2787,7 +3060,7 @@ class ModelUpdateHandler:
|
|||||||
return {}
|
return {}
|
||||||
|
|
||||||
to_dict = getattr(metadata, "to_dict", None)
|
to_dict = getattr(metadata, "to_dict", None)
|
||||||
if callable(to_dict):
|
if to_dict:
|
||||||
try:
|
try:
|
||||||
return to_dict()
|
return to_dict()
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -2798,7 +3071,7 @@ class ModelUpdateHandler:
|
|||||||
|
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
async def _read_json(self, request: web.Request) -> Dict:
|
async def _read_json(self, request: web.Request) -> Dict[str, Any]:
|
||||||
if not request.can_read_body:
|
if not request.can_read_body:
|
||||||
return {}
|
return {}
|
||||||
try:
|
try:
|
||||||
@@ -2830,10 +3103,11 @@ class ModelUpdateHandler:
|
|||||||
record,
|
record,
|
||||||
*,
|
*,
|
||||||
version_context: Optional[Dict[int, Dict[str, Any]]] = None,
|
version_context: Optional[Dict[int, Dict[str, Any]]] = None,
|
||||||
) -> Dict:
|
) -> Dict[str, Any]:
|
||||||
context = version_context or {}
|
context = version_context or {}
|
||||||
# Check user setting for hiding early access versions
|
# Check user setting for hiding early access versions
|
||||||
hide_early_access = False
|
hide_early_access = False
|
||||||
|
hide_paid = False
|
||||||
if self._settings is not None:
|
if self._settings is not None:
|
||||||
try:
|
try:
|
||||||
hide_early_access = bool(
|
hide_early_access = bool(
|
||||||
@@ -2841,6 +3115,10 @@ class ModelUpdateHandler:
|
|||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
try:
|
||||||
|
hide_paid = bool(self._settings.get("hide_paid_updates", False))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
return {
|
return {
|
||||||
"modelType": record.model_type,
|
"modelType": record.model_type,
|
||||||
"modelId": record.model_id,
|
"modelId": record.model_id,
|
||||||
@@ -2849,7 +3127,10 @@ class ModelUpdateHandler:
|
|||||||
"inLibraryVersionIds": record.in_library_version_ids,
|
"inLibraryVersionIds": record.in_library_version_ids,
|
||||||
"lastCheckedAt": record.last_checked_at,
|
"lastCheckedAt": record.last_checked_at,
|
||||||
"shouldIgnore": record.should_ignore_model,
|
"shouldIgnore": record.should_ignore_model,
|
||||||
"hasUpdate": record.has_update(hide_early_access=hide_early_access),
|
"hasUpdate": record.has_update(
|
||||||
|
hide_early_access=hide_early_access,
|
||||||
|
hide_paid=hide_paid,
|
||||||
|
),
|
||||||
"versions": [
|
"versions": [
|
||||||
self._serialize_version(version, context.get(version.version_id))
|
self._serialize_version(version, context.get(version.version_id))
|
||||||
for version in record.versions
|
for version in record.versions
|
||||||
@@ -2859,7 +3140,7 @@ class ModelUpdateHandler:
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def _serialize_version(
|
def _serialize_version(
|
||||||
version, context: Optional[Dict[str, Any]]
|
version, context: Optional[Dict[str, Any]]
|
||||||
) -> Dict:
|
) -> Dict[str, Any]:
|
||||||
context = context or {}
|
context = context or {}
|
||||||
preview_override = context.get("preview_override")
|
preview_override = context.get("preview_override")
|
||||||
preview_url = (
|
preview_url = (
|
||||||
@@ -2868,8 +3149,11 @@ class ModelUpdateHandler:
|
|||||||
|
|
||||||
# Determine if version is currently in early access
|
# Determine if version is currently in early access
|
||||||
# Two-phase detection: use exact end time if available, otherwise fallback to basic flag
|
# Two-phase detection: use exact end time if available, otherwise fallback to basic flag
|
||||||
|
# Mirror _is_early_access_active: permanent paid versions (no end time) are NOT early access
|
||||||
is_early_access = False
|
is_early_access = False
|
||||||
if version.early_access_ends_at:
|
if getattr(version, "is_paid", False) and not version.early_access_ends_at:
|
||||||
|
is_early_access = False
|
||||||
|
elif version.early_access_ends_at:
|
||||||
try:
|
try:
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
@@ -2884,6 +3168,13 @@ class ModelUpdateHandler:
|
|||||||
# Fallback to basic EA flag from bulk API
|
# Fallback to basic EA flag from bulk API
|
||||||
is_early_access = True
|
is_early_access = True
|
||||||
|
|
||||||
|
paid_access_payload = None
|
||||||
|
if getattr(version, "paid_access", None):
|
||||||
|
try:
|
||||||
|
paid_access_payload = json.loads(version.paid_access)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
paid_access_payload = None
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"versionId": version.version_id,
|
"versionId": version.version_id,
|
||||||
"name": version.name,
|
"name": version.name,
|
||||||
@@ -2897,8 +3188,13 @@ class ModelUpdateHandler:
|
|||||||
"earlyAccessEndsAt": version.early_access_ends_at,
|
"earlyAccessEndsAt": version.early_access_ends_at,
|
||||||
"isEarlyAccess": is_early_access,
|
"isEarlyAccess": is_early_access,
|
||||||
"usageControl": version.usage_control,
|
"usageControl": version.usage_control,
|
||||||
|
"isPaid": bool(getattr(version, "is_paid", False)),
|
||||||
|
"paidAccess": paid_access_payload,
|
||||||
"filePath": context.get("file_path"),
|
"filePath": context.get("file_path"),
|
||||||
"fileName": context.get("file_name"),
|
"fileName": context.get("file_name"),
|
||||||
|
# Weight-file variant count (None when unknown); lets the UI hide
|
||||||
|
# the download affordance for single-file in-library versions.
|
||||||
|
"fileCount": getattr(version, "file_count", None),
|
||||||
}
|
}
|
||||||
|
|
||||||
async def _build_version_context(
|
async def _build_version_context(
|
||||||
|
|||||||
@@ -0,0 +1,323 @@
|
|||||||
|
"""Handler for the pending-delete undo endpoint.
|
||||||
|
|
||||||
|
Restores a staged delete batch (models or recipes) via
|
||||||
|
``PendingDeleteService.undo`` and then repairs the affected library caches:
|
||||||
|
the model cache entry is restored from the manifest's ``model_snapshot``
|
||||||
|
(including the version index and hash index), tag counts are re-incremented,
|
||||||
|
and the recipe cache is re-populated via ``RecipeScanner.add_recipe``.
|
||||||
|
|
||||||
|
The per-type scanner is resolved from the manifest's ``model_type`` page value
|
||||||
|
through the SAME ServiceRegistry getters the model route registrars use
|
||||||
|
(lora/checkpoint/embedding) - never a hardcoded lora scanner.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import inspect
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from typing import Any, Awaitable, Callable, Dict, List, Optional, Set, cast
|
||||||
|
|
||||||
|
from aiohttp import web
|
||||||
|
|
||||||
|
from ...services.pending_delete_service import get_pending_delete_service
|
||||||
|
from .model_handlers import _broadcast_models_changed
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Manifest ``model_type`` page values -> ServiceRegistry scanner getter names.
|
||||||
|
# The model route registrars resolve per-type scanners via these getters
|
||||||
|
# (lora_routes / checkpoint_routes / embedding_routes); undo must do the same
|
||||||
|
# so the CORRECT cache is restored for the deleted model's type.
|
||||||
|
_MODEL_TYPE_GETTER_NAMES: Dict[str, str] = {
|
||||||
|
"loras": "get_lora_scanner",
|
||||||
|
"checkpoints": "get_checkpoint_scanner",
|
||||||
|
"embeddings": "get_embedding_scanner",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Staged batch ids are ``uuid.uuid4().hex`` (32 lowercase hex chars). The id is
|
||||||
|
# joined into filesystem paths by ``_find_batch_dir``, so reject anything that
|
||||||
|
# does not match this exact shape (blocks path-traversal via batch_id).
|
||||||
|
_BATCH_ID_RE = re.compile(r"^[0-9a-f]{32}$")
|
||||||
|
|
||||||
|
|
||||||
|
class PendingDeleteHandler:
|
||||||
|
"""Handle undo requests for staged model/recipe deletions."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
service_factory: Callable[[], Awaitable[Any]] = get_pending_delete_service,
|
||||||
|
scanner_getter: Optional[Callable[[str], Awaitable[Any]]] = None,
|
||||||
|
recipe_scanner_getter: Optional[Callable[[], Awaitable[Any]]] = None,
|
||||||
|
) -> None:
|
||||||
|
self._service_factory: Callable[[], Awaitable[Any]] = service_factory
|
||||||
|
self._scanner_getter: Callable[[str], Awaitable[Any]] = (
|
||||||
|
scanner_getter or self._resolve_scanner
|
||||||
|
)
|
||||||
|
self._recipe_scanner_getter: Callable[[], Awaitable[Any]] = (
|
||||||
|
recipe_scanner_getter or self._resolve_recipe_scanner
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def _resolve_scanner(model_type: str) -> Any:
|
||||||
|
"""Resolve the per-type scanner for a manifest ``model_type``.
|
||||||
|
|
||||||
|
The getter is looked up on the ServiceRegistry module namespace at call
|
||||||
|
time so tests (and the registry stubs) can patch it.
|
||||||
|
"""
|
||||||
|
from ...services import service_registry
|
||||||
|
|
||||||
|
getter_name = _MODEL_TYPE_GETTER_NAMES.get(model_type)
|
||||||
|
if getter_name is None:
|
||||||
|
raise ValueError(f"Unknown model type: {model_type}")
|
||||||
|
getter = getattr(service_registry.ServiceRegistry, getter_name, None)
|
||||||
|
if not callable(getter):
|
||||||
|
raise ValueError(f"No scanner getter for model type: {model_type}")
|
||||||
|
scanner = await cast(Callable[[], Awaitable[Any]], getter)()
|
||||||
|
if scanner is None:
|
||||||
|
raise ValueError(f"No scanner registered for model type: {model_type}")
|
||||||
|
return scanner
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def _resolve_recipe_scanner() -> Any:
|
||||||
|
"""Resolve the recipe scanner via the ServiceRegistry module namespace."""
|
||||||
|
from ...services import service_registry
|
||||||
|
|
||||||
|
getter = getattr(service_registry.ServiceRegistry, "get_recipe_scanner", None)
|
||||||
|
if not callable(getter):
|
||||||
|
raise ValueError("Recipe scanner getter unavailable")
|
||||||
|
scanner = await cast(Callable[[], Awaitable[Any]], getter)()
|
||||||
|
if scanner is None:
|
||||||
|
raise ValueError("No recipe scanner registered")
|
||||||
|
return scanner
|
||||||
|
|
||||||
|
async def undo_delete(self, request: web.Request) -> web.Response:
|
||||||
|
"""Restore a staged batch and its library cache entry.
|
||||||
|
|
||||||
|
Body: ``{"batch_id": str}``. On success returns
|
||||||
|
``{"success": True, "restored": [<original paths>], "kind": kind}``.
|
||||||
|
Expired/unknown batches and occupied target paths -> 404.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
data = await request.json()
|
||||||
|
except Exception:
|
||||||
|
return web.json_response(
|
||||||
|
{"success": False, "error": "Invalid JSON body"}, status=400
|
||||||
|
)
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return web.json_response(
|
||||||
|
{"success": False, "error": "Invalid JSON body"}, status=400
|
||||||
|
)
|
||||||
|
batch_id = data.get("batch_id")
|
||||||
|
if not batch_id or not isinstance(batch_id, str):
|
||||||
|
return web.json_response(
|
||||||
|
{"success": False, "error": "batch_id is required"}, status=400
|
||||||
|
)
|
||||||
|
if not _BATCH_ID_RE.fullmatch(batch_id):
|
||||||
|
# batch_id is joined into a path by _find_batch_dir - restrict to
|
||||||
|
# the exact staged-id shape so traversal payloads get 400.
|
||||||
|
return web.json_response(
|
||||||
|
{"success": False, "error": "Invalid batch_id"}, status=400
|
||||||
|
)
|
||||||
|
|
||||||
|
service = await self._service_factory()
|
||||||
|
try:
|
||||||
|
# Read the manifest BEFORE undo: undo() removes the batch dir.
|
||||||
|
manifest = await self._read_staged_manifest(service, batch_id)
|
||||||
|
result = await service.undo(batch_id)
|
||||||
|
except ValueError as exc:
|
||||||
|
return web.json_response({"success": False, "error": str(exc)}, status=404)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error("Unexpected error undoing batch %s: %s", batch_id, exc, exc_info=True)
|
||||||
|
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||||
|
|
||||||
|
kind = result.get("kind")
|
||||||
|
try:
|
||||||
|
if kind == "model":
|
||||||
|
if manifest is not None:
|
||||||
|
await self._restore_model_cache(manifest)
|
||||||
|
else:
|
||||||
|
# undo() raises when the manifest is missing, so this only
|
||||||
|
# happens defensively - files are restored regardless.
|
||||||
|
logger.warning(
|
||||||
|
"Manifest missing after undo of %s; skipping cache restore",
|
||||||
|
batch_id,
|
||||||
|
)
|
||||||
|
_broadcast_models_changed()
|
||||||
|
elif kind == "recipe":
|
||||||
|
# Recipe undo is client-refresh only: re-add to the scanner
|
||||||
|
# cache, no models_changed broadcast.
|
||||||
|
if manifest is not None:
|
||||||
|
await self._restore_recipe_cache(result, manifest)
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
"Manifest missing after undo of %s; skipping cache restore",
|
||||||
|
batch_id,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
# Files are already restored; only the cache restoration failed.
|
||||||
|
logger.error(
|
||||||
|
"Cache restoration failed after undo of %s: %s",
|
||||||
|
batch_id,
|
||||||
|
exc,
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||||
|
|
||||||
|
return web.json_response(
|
||||||
|
{
|
||||||
|
"success": True,
|
||||||
|
"restored": result.get("restored", []),
|
||||||
|
"kind": kind,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def _read_staged_manifest(
|
||||||
|
service: Any, batch_id: str
|
||||||
|
) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Locate and read the batch manifest while it still exists on disk."""
|
||||||
|
batch_dir = await service._find_batch_dir(batch_id)
|
||||||
|
if not batch_dir:
|
||||||
|
return None
|
||||||
|
manifest_path = os.path.join(batch_dir, "manifest.json")
|
||||||
|
try:
|
||||||
|
with open(manifest_path, "r", encoding="utf-8") as handle:
|
||||||
|
payload = json.load(handle)
|
||||||
|
except (OSError, json.JSONDecodeError) as exc:
|
||||||
|
logger.debug("Failed to read manifest for batch %s: %s", batch_id, exc)
|
||||||
|
return None
|
||||||
|
return payload if isinstance(payload, dict) else None
|
||||||
|
|
||||||
|
async def _restore_model_cache(self, manifest: Dict[str, Any]) -> None:
|
||||||
|
"""Re-add every deleted model's cache entry from the manifest.
|
||||||
|
|
||||||
|
Each main-file entry carries the deleted model's ``snapshot`` (added at
|
||||||
|
stage time), so a merged bulk manifest holds ALL snapshots - undo must
|
||||||
|
restore every one, not just the top-level winner's. Old-format
|
||||||
|
manifests without entry snapshots fall back to the top-level
|
||||||
|
``model_snapshot`` (backward compat / single-delete path).
|
||||||
|
"""
|
||||||
|
model_type = manifest.get("model_type")
|
||||||
|
if not model_type or not isinstance(model_type, str):
|
||||||
|
raise ValueError(f"Manifest carries no model_type: {manifest.get('batch_id')}")
|
||||||
|
scanner = await self._scanner_getter(model_type)
|
||||||
|
|
||||||
|
# Collect one snapshot per distinct file_path from the entry snapshots.
|
||||||
|
snapshots: List[Dict[str, Any]] = []
|
||||||
|
seen: Set[str] = set()
|
||||||
|
for entry in manifest.get("entries") or []:
|
||||||
|
snapshot = entry.get("snapshot")
|
||||||
|
if not isinstance(snapshot, dict):
|
||||||
|
continue
|
||||||
|
file_path = snapshot.get("file_path")
|
||||||
|
if not file_path or not isinstance(file_path, str):
|
||||||
|
continue
|
||||||
|
if file_path in seen:
|
||||||
|
continue
|
||||||
|
seen.add(file_path)
|
||||||
|
snapshots.append(snapshot)
|
||||||
|
|
||||||
|
if not snapshots:
|
||||||
|
# Backward compat: pre-F3 manifests carry only the top-level
|
||||||
|
# model_snapshot (single-delete path, unchanged behavior).
|
||||||
|
top = manifest.get("model_snapshot")
|
||||||
|
if isinstance(top, dict) and top.get("file_path"):
|
||||||
|
snapshots = [top]
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
"Manifest %s has no restorable model snapshot; skipping cache restore",
|
||||||
|
manifest.get("batch_id"),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
cache = await scanner.get_cached_data()
|
||||||
|
if cache is None:
|
||||||
|
logger.warning(
|
||||||
|
"Scanner cache unavailable for %s; skipping cache restore", model_type
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
for snapshot in snapshots:
|
||||||
|
file_path = str(snapshot["file_path"])
|
||||||
|
# A rescan between delete and undo may have re-added a stale entry
|
||||||
|
# for this path - drop it so exactly one (the snapshot) remains.
|
||||||
|
cache.raw_data = [
|
||||||
|
item for item in cache.raw_data if item.get("file_path") != file_path
|
||||||
|
]
|
||||||
|
|
||||||
|
# Restore tag counts (mirror of the bulk-delete decrement in
|
||||||
|
# _batch_update_cache_for_deleted_models: undo re-increments).
|
||||||
|
tags = snapshot.get("tags")
|
||||||
|
if isinstance(tags, list):
|
||||||
|
for tag in tags:
|
||||||
|
if not isinstance(tag, str) or not tag:
|
||||||
|
continue
|
||||||
|
scanner._tags_count[tag] = scanner._tags_count.get(tag, 0) + 1
|
||||||
|
|
||||||
|
cache.raw_data.append(dict(snapshot))
|
||||||
|
|
||||||
|
# Re-register the path in the hash index (add_entry guards a
|
||||||
|
# missing sha256 internally; still guard defensively here).
|
||||||
|
sha256 = snapshot.get("sha256") or ""
|
||||||
|
autov3 = snapshot.get("autov3")
|
||||||
|
hash_index = getattr(scanner, "_hash_index", None)
|
||||||
|
if hash_index is not None and sha256 and file_path:
|
||||||
|
hash_index.add_entry(sha256, file_path, autov3)
|
||||||
|
|
||||||
|
# Follow the bulk-delete cache-update pattern ONCE after all entries,
|
||||||
|
# including the explicit version-index rebuild so the version index
|
||||||
|
# does not go stale.
|
||||||
|
cache.rebuild_version_index()
|
||||||
|
await cache.resort()
|
||||||
|
|
||||||
|
scanner.bump_cache_version()
|
||||||
|
|
||||||
|
persist = getattr(scanner, "_persist_current_cache", None)
|
||||||
|
if callable(persist):
|
||||||
|
result = persist()
|
||||||
|
if inspect.isawaitable(result):
|
||||||
|
await result
|
||||||
|
|
||||||
|
async def _restore_recipe_cache(
|
||||||
|
self, result: Dict[str, Any], manifest: Dict[str, Any]
|
||||||
|
) -> None:
|
||||||
|
"""Re-add a restored recipe via ``RecipeScanner.add_recipe``.
|
||||||
|
|
||||||
|
The recipe JSON embeds the full recipe_data (incl. id/file_path);
|
||||||
|
``add_recipe`` only READS the ``_json_path_map`` so the forced frontend
|
||||||
|
refresh self-heals any transient path-map gap.
|
||||||
|
"""
|
||||||
|
restored = result.get("restored") or []
|
||||||
|
json_path = next(
|
||||||
|
(p for p in restored if isinstance(p, str) and p.endswith(".json")),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if not json_path or not os.path.exists(json_path):
|
||||||
|
# Defensive fallback to the manifest's recipe_snapshot file_path.
|
||||||
|
snapshot = manifest.get("recipe_snapshot") or {}
|
||||||
|
fallback = snapshot.get("file_path")
|
||||||
|
if fallback and os.path.exists(fallback):
|
||||||
|
json_path = fallback
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
"Restored recipe JSON not found in %s; skipping cache restore",
|
||||||
|
restored,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
with open(json_path, "r", encoding="utf-8") as handle:
|
||||||
|
recipe_data = json.load(handle)
|
||||||
|
except (OSError, json.JSONDecodeError) as exc:
|
||||||
|
logger.warning("Failed to load restored recipe JSON %s: %s", json_path, exc)
|
||||||
|
return
|
||||||
|
if not isinstance(recipe_data, dict):
|
||||||
|
return
|
||||||
|
recipe_scanner = await self._recipe_scanner_getter()
|
||||||
|
await recipe_scanner.add_recipe(recipe_data)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["PendingDeleteHandler"]
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,8 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
from typing import Dict
|
from typing import Any, Dict
|
||||||
from server import PromptServer # type: ignore
|
from server import PromptServer # pyright: ignore[reportMissingImports]
|
||||||
|
|
||||||
from .base_model_routes import BaseModelRoutes
|
from .base_model_routes import BaseModelRoutes
|
||||||
from .model_route_registrar import ModelRouteRegistrar
|
from .model_route_registrar import ModelRouteRegistrar
|
||||||
@@ -31,13 +31,13 @@ class LoraRoutes(BaseModelRoutes):
|
|||||||
# Attach service dependencies
|
# Attach service dependencies
|
||||||
self.attach_service(self.service)
|
self.attach_service(self.service)
|
||||||
|
|
||||||
def setup_routes(self, app: web.Application):
|
def setup_routes(self, app: web.Application, prefix: str = "loras"):
|
||||||
"""Setup LoRA routes"""
|
"""Setup LoRA routes"""
|
||||||
# Schedule service initialization on app startup
|
# Schedule service initialization on app startup
|
||||||
app.on_startup.append(lambda _: self.initialize_services())
|
app.on_startup.append(lambda _: self.initialize_services())
|
||||||
|
|
||||||
# Setup common routes with 'loras' prefix (includes page route)
|
# Setup common routes with 'loras' prefix (includes page route)
|
||||||
super().setup_routes(app, "loras")
|
super().setup_routes(app, prefix)
|
||||||
|
|
||||||
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
||||||
"""Setup LoRA-specific routes"""
|
"""Setup LoRA-specific routes"""
|
||||||
@@ -73,7 +73,7 @@ class LoraRoutes(BaseModelRoutes):
|
|||||||
"POST", "/api/lm/{prefix}/get_trigger_words", prefix, self.get_trigger_words
|
"POST", "/api/lm/{prefix}/get_trigger_words", prefix, self.get_trigger_words
|
||||||
)
|
)
|
||||||
|
|
||||||
def _parse_specific_params(self, request: web.Request) -> Dict:
|
def _parse_specific_params(self, request: web.Request) -> Dict[str, Any]:
|
||||||
"""Parse LoRA-specific parameters"""
|
"""Parse LoRA-specific parameters"""
|
||||||
params = {}
|
params = {}
|
||||||
|
|
||||||
@@ -119,25 +119,6 @@ class LoraRoutes(BaseModelRoutes):
|
|||||||
logger.error(f"Error getting letter counts: {e}")
|
logger.error(f"Error getting letter counts: {e}")
|
||||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||||
|
|
||||||
async def get_lora_notes(self, request: web.Request) -> web.Response:
|
|
||||||
"""Get notes for a specific LoRA file"""
|
|
||||||
try:
|
|
||||||
lora_name = request.query.get("name")
|
|
||||||
if not lora_name:
|
|
||||||
return web.Response(text="Lora file name is required", status=400)
|
|
||||||
|
|
||||||
notes = await self.service.get_lora_notes(lora_name)
|
|
||||||
if notes is not None:
|
|
||||||
return web.json_response({"success": True, "notes": notes})
|
|
||||||
else:
|
|
||||||
return web.json_response(
|
|
||||||
{"success": False, "error": "LoRA not found in cache"}, status=404
|
|
||||||
)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error getting lora notes: {e}", exc_info=True)
|
|
||||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
|
||||||
|
|
||||||
async def get_lora_trigger_words(self, request: web.Request) -> web.Response:
|
async def get_lora_trigger_words(self, request: web.Request) -> web.Response:
|
||||||
"""Get trigger words for a specific LoRA file"""
|
"""Get trigger words for a specific LoRA file"""
|
||||||
try:
|
try:
|
||||||
@@ -168,52 +149,6 @@ class LoraRoutes(BaseModelRoutes):
|
|||||||
logger.error(f"Error getting lora usage tips by path: {e}", exc_info=True)
|
logger.error(f"Error getting lora usage tips by path: {e}", exc_info=True)
|
||||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||||
|
|
||||||
async def get_lora_preview_url(self, request: web.Request) -> web.Response:
|
|
||||||
"""Get the static preview URL for a LoRA file"""
|
|
||||||
try:
|
|
||||||
lora_name = request.query.get("name")
|
|
||||||
if not lora_name:
|
|
||||||
return web.Response(text="Lora file name is required", status=400)
|
|
||||||
|
|
||||||
preview_url = await self.service.get_lora_preview_url(lora_name)
|
|
||||||
if preview_url:
|
|
||||||
return web.json_response({"success": True, "preview_url": preview_url})
|
|
||||||
else:
|
|
||||||
return web.json_response(
|
|
||||||
{
|
|
||||||
"success": False,
|
|
||||||
"error": "No preview URL found for the specified lora",
|
|
||||||
},
|
|
||||||
status=404,
|
|
||||||
)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error getting lora preview URL: {e}", exc_info=True)
|
|
||||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
|
||||||
|
|
||||||
async def get_lora_civitai_url(self, request: web.Request) -> web.Response:
|
|
||||||
"""Get the Civitai URL for a LoRA file"""
|
|
||||||
try:
|
|
||||||
lora_name = request.query.get("name")
|
|
||||||
if not lora_name:
|
|
||||||
return web.Response(text="Lora file name is required", status=400)
|
|
||||||
|
|
||||||
result = await self.service.get_lora_civitai_url(lora_name)
|
|
||||||
if result["civitai_url"]:
|
|
||||||
return web.json_response({"success": True, **result})
|
|
||||||
else:
|
|
||||||
return web.json_response(
|
|
||||||
{
|
|
||||||
"success": False,
|
|
||||||
"error": "No Civitai data found for the specified lora",
|
|
||||||
},
|
|
||||||
status=404,
|
|
||||||
)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error getting lora Civitai URL: {e}", exc_info=True)
|
|
||||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
|
||||||
|
|
||||||
async def get_random_loras(self, request: web.Request) -> web.Response:
|
async def get_random_loras(self, request: web.Request) -> web.Response:
|
||||||
"""Get random LoRAs based on filters and strength ranges"""
|
"""Get random LoRAs based on filters and strength ranges"""
|
||||||
try:
|
try:
|
||||||
@@ -337,7 +272,7 @@ class LoraRoutes(BaseModelRoutes):
|
|||||||
graph_identifier = entry.get("graph_id")
|
graph_identifier = entry.get("graph_id")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
parsed_node_id = int(node_identifier)
|
parsed_node_id = int(node_identifier) # pyright: ignore[reportArgumentType]
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
parsed_node_id = node_identifier
|
parsed_node_id = node_identifier
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ miscellaneous endpoints share a consistent registration flow.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Callable, Iterable, Mapping
|
from typing import Any, Callable, Iterable, Mapping
|
||||||
|
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
|
|
||||||
@@ -32,6 +32,7 @@ MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
|||||||
RouteDefinition("GET", "/api/lm/settings/libraries", "get_settings_libraries"),
|
RouteDefinition("GET", "/api/lm/settings/libraries", "get_settings_libraries"),
|
||||||
RouteDefinition("POST", "/api/lm/settings/libraries/activate", "activate_library"),
|
RouteDefinition("POST", "/api/lm/settings/libraries/activate", "activate_library"),
|
||||||
RouteDefinition("GET", "/api/lm/health-check", "health_check"),
|
RouteDefinition("GET", "/api/lm/health-check", "health_check"),
|
||||||
|
RouteDefinition("GET", "/api/lm/init-status", "get_init_status"),
|
||||||
RouteDefinition("GET", "/api/lm/supporters", "get_supporters"),
|
RouteDefinition("GET", "/api/lm/supporters", "get_supporters"),
|
||||||
RouteDefinition("GET", "/api/lm/wildcards/search", "search_wildcards"),
|
RouteDefinition("GET", "/api/lm/wildcards/search", "search_wildcards"),
|
||||||
RouteDefinition("POST", "/api/lm/wildcards/open-location", "open_wildcards_location"),
|
RouteDefinition("POST", "/api/lm/wildcards/open-location", "open_wildcards_location"),
|
||||||
@@ -147,7 +148,7 @@ class MiscRouteRegistrar:
|
|||||||
handler_lookup[definition.handler_name],
|
handler_lookup[definition.handler_name],
|
||||||
)
|
)
|
||||||
|
|
||||||
def _bind(self, method: str, path: str, handler: Callable) -> None:
|
def _bind(self, method: str, path: str, handler: Callable[..., Any]) -> None:
|
||||||
add_method_name = self._METHOD_MAP[method.upper()]
|
add_method_name = self._METHOD_MAP[method.upper()]
|
||||||
add_method = getattr(self._app.router, add_method_name)
|
add_method = getattr(self._app.router, add_method_name)
|
||||||
add_method(path, handler)
|
add_method(path, handler)
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import os
|
|||||||
from typing import Awaitable, Callable, Mapping
|
from typing import Awaitable, Callable, Mapping
|
||||||
|
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
from server import PromptServer # type: ignore
|
from server import PromptServer # pyright: ignore[reportMissingImports]
|
||||||
|
|
||||||
from ..services.metadata_service import (
|
from ..services.metadata_service import (
|
||||||
get_metadata_archive_manager,
|
get_metadata_archive_manager,
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Callable, Iterable, Mapping
|
from typing import Any, Callable, Iterable, Mapping
|
||||||
|
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
|
|
||||||
@@ -174,15 +174,15 @@ class ModelRouteRegistrar:
|
|||||||
handler_lookup[definition.handler_name],
|
handler_lookup[definition.handler_name],
|
||||||
)
|
)
|
||||||
|
|
||||||
def add_route(self, method: str, path: str, handler: Callable) -> None:
|
def add_route(self, method: str, path: str, handler: Callable[..., Any]) -> None:
|
||||||
self._bind_route(method, path, handler)
|
self._bind_route(method, path, handler)
|
||||||
|
|
||||||
def add_prefixed_route(
|
def add_prefixed_route(
|
||||||
self, method: str, path_template: str, prefix: str, handler: Callable
|
self, method: str, path_template: str, prefix: str, handler: Callable[..., Any]
|
||||||
) -> None:
|
) -> None:
|
||||||
self._bind_route(method, path_template.replace("{prefix}", prefix), handler)
|
self._bind_route(method, path_template.replace("{prefix}", prefix), handler)
|
||||||
|
|
||||||
def _bind_route(self, method: str, path: str, handler: Callable) -> None:
|
def _bind_route(self, method: str, path: str, handler: Callable[..., Any]) -> None:
|
||||||
add_method_name = self._METHOD_MAP[method.upper()]
|
add_method_name = self._METHOD_MAP[method.upper()]
|
||||||
add_method = getattr(self._app.router, add_method_name)
|
add_method = getattr(self._app.router, add_method_name)
|
||||||
add_method(path, handler)
|
add_method(path, handler)
|
||||||
|
|||||||
@@ -0,0 +1,25 @@
|
|||||||
|
"""Route controller for the pending-delete undo endpoint."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from aiohttp import web
|
||||||
|
|
||||||
|
from .handlers.pending_delete_handler import PendingDeleteHandler
|
||||||
|
|
||||||
|
|
||||||
|
class PendingDeleteRoutes:
|
||||||
|
"""Shared route controller mirroring MiscRoutes/UpdateRoutes.
|
||||||
|
|
||||||
|
Registered ONCE per mode (py/lora_manager.py, standalone.py); NEVER through
|
||||||
|
the per-model-type ModelRouteRegistrar, which is instantiated per model
|
||||||
|
type and would register this non-prefixed route three times.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def setup_routes(app: web.Application) -> None:
|
||||||
|
"""Register the shared undo-delete endpoint."""
|
||||||
|
handler = PendingDeleteHandler()
|
||||||
|
_ = app.router.add_post("/api/lm/undo-delete", handler.undo_delete)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["PendingDeleteRoutes"]
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Callable, Mapping
|
from typing import Any, Callable, Mapping
|
||||||
|
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
|
|
||||||
@@ -43,9 +43,37 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
|||||||
),
|
),
|
||||||
RouteDefinition("GET", "/api/lm/recipe/{recipe_id}/syntax", "get_recipe_syntax"),
|
RouteDefinition("GET", "/api/lm/recipe/{recipe_id}/syntax", "get_recipe_syntax"),
|
||||||
RouteDefinition("PUT", "/api/lm/recipe/{recipe_id}/update", "update_recipe"),
|
RouteDefinition("PUT", "/api/lm/recipe/{recipe_id}/update", "update_recipe"),
|
||||||
|
RouteDefinition(
|
||||||
|
"POST", "/api/lm/recipe/{recipe_id}/opened", "record_recipe_open"
|
||||||
|
),
|
||||||
RouteDefinition("POST", "/api/lm/recipe/move", "move_recipe"),
|
RouteDefinition("POST", "/api/lm/recipe/move", "move_recipe"),
|
||||||
RouteDefinition("POST", "/api/lm/recipes/move-bulk", "move_recipes_bulk"),
|
RouteDefinition("POST", "/api/lm/recipes/move-bulk", "move_recipes_bulk"),
|
||||||
RouteDefinition("POST", "/api/lm/recipe/lora/reconnect", "reconnect_lora"),
|
RouteDefinition("POST", "/api/lm/recipe/lora/reconnect", "reconnect_lora"),
|
||||||
|
RouteDefinition("POST", "/api/lm/recipe/lora/restore", "restore_lora"),
|
||||||
|
RouteDefinition(
|
||||||
|
"GET",
|
||||||
|
"/api/lm/recipe/{recipe_id}/lora/{lora_index}/reconnect-suggestions",
|
||||||
|
"get_reconnect_suggestions",
|
||||||
|
),
|
||||||
|
RouteDefinition(
|
||||||
|
"POST", "/api/lm/recipe/lora/mark-hash-invalid", "mark_lora_hash_invalid"
|
||||||
|
),
|
||||||
|
RouteDefinition(
|
||||||
|
"POST", "/api/lm/recipe/checkpoint/reconnect", "reconnect_checkpoint"
|
||||||
|
),
|
||||||
|
RouteDefinition(
|
||||||
|
"POST", "/api/lm/recipe/checkpoint/restore", "restore_checkpoint"
|
||||||
|
),
|
||||||
|
RouteDefinition(
|
||||||
|
"GET",
|
||||||
|
"/api/lm/recipe/{recipe_id}/checkpoint/reconnect-suggestions",
|
||||||
|
"get_checkpoint_reconnect_suggestions",
|
||||||
|
),
|
||||||
|
RouteDefinition(
|
||||||
|
"POST",
|
||||||
|
"/api/lm/recipe/checkpoint/mark-hash-invalid",
|
||||||
|
"mark_checkpoint_hash_invalid",
|
||||||
|
),
|
||||||
RouteDefinition("GET", "/api/lm/recipes/find-duplicates", "find_duplicates"),
|
RouteDefinition("GET", "/api/lm/recipes/find-duplicates", "find_duplicates"),
|
||||||
RouteDefinition("POST", "/api/lm/recipes/bulk-delete", "bulk_delete"),
|
RouteDefinition("POST", "/api/lm/recipes/bulk-delete", "bulk_delete"),
|
||||||
RouteDefinition(
|
RouteDefinition(
|
||||||
@@ -61,6 +89,11 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
|||||||
RouteDefinition("POST", "/api/lm/recipe/{recipe_id}/repair", "repair_recipe"),
|
RouteDefinition("POST", "/api/lm/recipe/{recipe_id}/repair", "repair_recipe"),
|
||||||
RouteDefinition("POST", "/api/lm/recipes/repair-bulk", "repair_recipes_bulk"),
|
RouteDefinition("POST", "/api/lm/recipes/repair-bulk", "repair_recipes_bulk"),
|
||||||
RouteDefinition("GET", "/api/lm/recipes/repair-progress", "get_repair_progress"),
|
RouteDefinition("GET", "/api/lm/recipes/repair-progress", "get_repair_progress"),
|
||||||
|
RouteDefinition("POST", "/api/lm/recipes/rematch", "rematch_recipes"),
|
||||||
|
RouteDefinition("POST", "/api/lm/recipes/rematch-bulk", "rematch_recipes_bulk"),
|
||||||
|
RouteDefinition("POST", "/api/lm/recipe/{recipe_id}/rematch", "rematch_recipe"),
|
||||||
|
RouteDefinition("POST", "/api/lm/recipes/cancel-rematch", "cancel_rematch"),
|
||||||
|
RouteDefinition("GET", "/api/lm/recipes/rematch-progress", "get_rematch_progress"),
|
||||||
RouteDefinition("POST", "/api/lm/recipes/batch-import/start", "start_batch_import"),
|
RouteDefinition("POST", "/api/lm/recipes/batch-import/start", "start_batch_import"),
|
||||||
RouteDefinition(
|
RouteDefinition(
|
||||||
"GET", "/api/lm/recipes/batch-import/progress", "get_batch_import_progress"
|
"GET", "/api/lm/recipes/batch-import/progress", "get_batch_import_progress"
|
||||||
@@ -82,6 +115,9 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
|||||||
RouteDefinition(
|
RouteDefinition(
|
||||||
"POST", "/api/lm/recipe/{recipe_id}/reimport", "reimport_recipe"
|
"POST", "/api/lm/recipe/{recipe_id}/reimport", "reimport_recipe"
|
||||||
),
|
),
|
||||||
|
RouteDefinition(
|
||||||
|
"POST", "/api/lm/recipe/{recipe_id}/send-workflow", "send_recipe_workflow"
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -105,7 +141,7 @@ class RecipeRouteRegistrar:
|
|||||||
handler = handler_lookup[definition.handler_name]
|
handler = handler_lookup[definition.handler_name]
|
||||||
self._bind_route(definition.method, definition.path, handler)
|
self._bind_route(definition.method, definition.path, handler)
|
||||||
|
|
||||||
def _bind_route(self, method: str, path: str, handler: Callable) -> None:
|
def _bind_route(self, method: str, path: str, handler: Callable[..., Any]) -> None:
|
||||||
add_method_name = self._METHOD_MAP[method.upper()]
|
add_method_name = self._METHOD_MAP[method.upper()]
|
||||||
add_method = getattr(self._app.router, add_method_name)
|
add_method = getattr(self._app.router, add_method_name)
|
||||||
add_method(path, handler)
|
add_method(path, handler)
|
||||||
|
|||||||
+11
-10
@@ -40,10 +40,11 @@ class StatsRoutes:
|
|||||||
"""Route handlers for Statistics page and API endpoints"""
|
"""Route handlers for Statistics page and API endpoints"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.lora_scanner = None
|
self.lora_scanner: Any = None
|
||||||
self.checkpoint_scanner = None
|
self.checkpoint_scanner: Any = None
|
||||||
self.embedding_scanner = None
|
self.embedding_scanner: Any = None
|
||||||
self.usage_stats = None
|
self.usage_stats: Any = None
|
||||||
|
self._i18n_filter_added = False
|
||||||
self.template_env = jinja2.Environment(
|
self.template_env = jinja2.Environment(
|
||||||
loader=jinja2.FileSystemLoader(config.templates_path),
|
loader=jinja2.FileSystemLoader(config.templates_path),
|
||||||
autoescape=True
|
autoescape=True
|
||||||
@@ -95,9 +96,9 @@ class StatsRoutes:
|
|||||||
server_i18n.set_locale(user_language)
|
server_i18n.set_locale(user_language)
|
||||||
|
|
||||||
# 为模板环境添加i18n过滤器
|
# 为模板环境添加i18n过滤器
|
||||||
if not hasattr(self.template_env, '_i18n_filter_added'):
|
if not self._i18n_filter_added:
|
||||||
self.template_env.filters['t'] = server_i18n.create_template_filter()
|
self.template_env.filters['t'] = server_i18n.create_template_filter()
|
||||||
self.template_env._i18n_filter_added = True
|
self._i18n_filter_added = True
|
||||||
|
|
||||||
template = self.template_env.get_template('statistics.html')
|
template = self.template_env.get_template('statistics.html')
|
||||||
rendered = template.render(
|
rendered = template.render(
|
||||||
@@ -549,7 +550,7 @@ class StatsRoutes:
|
|||||||
'error': str(e)
|
'error': str(e)
|
||||||
}, status=500)
|
}, status=500)
|
||||||
|
|
||||||
def _count_unused_models(self, models: List[Dict], usage_data: Dict) -> int:
|
def _count_unused_models(self, models: List[Dict[str, Any]], usage_data: Dict[str, Any]) -> int:
|
||||||
"""Count models that have never been used"""
|
"""Count models that have never been used"""
|
||||||
used_hashes = set(usage_data.keys())
|
used_hashes = set(usage_data.keys())
|
||||||
unused_count = 0
|
unused_count = 0
|
||||||
@@ -560,7 +561,7 @@ class StatsRoutes:
|
|||||||
|
|
||||||
return unused_count
|
return unused_count
|
||||||
|
|
||||||
def _get_top_used_models(self, usage_data: Dict, model_map: Dict, limit: int) -> List[Dict]:
|
def _get_top_used_models(self, usage_data: Dict[str, Any], model_map: Dict[str, Any], limit: int) -> List[Dict[str, Any]]:
|
||||||
"""Get top used models with their metadata"""
|
"""Get top used models with their metadata"""
|
||||||
sorted_usage = sorted(usage_data.items(), key=lambda x: x[1].get('total', 0), reverse=True)
|
sorted_usage = sorted(usage_data.items(), key=lambda x: x[1].get('total', 0), reverse=True)
|
||||||
|
|
||||||
@@ -578,7 +579,7 @@ class StatsRoutes:
|
|||||||
|
|
||||||
return top_models
|
return top_models
|
||||||
|
|
||||||
def _get_usage_timeline(self, usage_data: Dict, days: int) -> List[Dict]:
|
def _get_usage_timeline(self, usage_data: Dict[str, Any], days: int) -> List[Dict[str, Any]]:
|
||||||
"""Get usage timeline for the past N days"""
|
"""Get usage timeline for the past N days"""
|
||||||
timeline = []
|
timeline = []
|
||||||
today = datetime.now()
|
today = datetime.now()
|
||||||
@@ -614,7 +615,7 @@ class StatsRoutes:
|
|||||||
|
|
||||||
return list(reversed(timeline)) # Oldest to newest
|
return list(reversed(timeline)) # Oldest to newest
|
||||||
|
|
||||||
def _format_size(self, size_bytes: int) -> str:
|
def _format_size(self, size_bytes: float) -> str:
|
||||||
"""Format file size in human readable format"""
|
"""Format file size in human readable format"""
|
||||||
for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
|
for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
|
||||||
if size_bytes < 1024.0:
|
if size_bytes < 1024.0:
|
||||||
|
|||||||
+16
-13
@@ -6,7 +6,7 @@ import shutil
|
|||||||
import tempfile
|
import tempfile
|
||||||
import asyncio
|
import asyncio
|
||||||
from aiohttp import web, ClientError
|
from aiohttp import web, ClientError
|
||||||
from typing import Dict, List
|
from typing import Any, Dict, List, cast
|
||||||
|
|
||||||
from ..utils.settings_paths import ensure_settings_file
|
from ..utils.settings_paths import ensure_settings_file
|
||||||
from ..services.downloader import get_downloader
|
from ..services.downloader import get_downloader
|
||||||
@@ -467,9 +467,10 @@ class UpdateRoutes:
|
|||||||
if not success:
|
if not success:
|
||||||
logger.error(f"Failed to fetch release info: {data}")
|
logger.error(f"Failed to fetch release info: {data}")
|
||||||
return False, ""
|
return False, ""
|
||||||
|
|
||||||
zip_url = data.get("zipball_url")
|
release_payload = cast(dict[str, Any], data)
|
||||||
version = data.get("tag_name", "unknown")
|
zip_url = release_payload.get("zipball_url", "")
|
||||||
|
version = release_payload.get("tag_name", "unknown")
|
||||||
|
|
||||||
# Download ZIP to temporary file
|
# Download ZIP to temporary file
|
||||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".zip") as tmp_zip:
|
with tempfile.NamedTemporaryFile(delete=False, suffix=".zip") as tmp_zip:
|
||||||
@@ -580,9 +581,10 @@ class UpdateRoutes:
|
|||||||
logger.warning("Failed to fetch GitHub commit: %s", data)
|
logger.warning("Failed to fetch GitHub commit: %s", data)
|
||||||
return "main", [], 0, ""
|
return "main", [], 0, ""
|
||||||
|
|
||||||
commit_sha = data.get('sha', '')[:7]
|
commit_payload = cast(dict[str, Any], data)
|
||||||
commit_message = data.get('commit', {}).get('message', '')
|
commit_sha = commit_payload.get('sha', '')[:7]
|
||||||
commit_date = data.get('commit', {}).get('committer', {}).get('date', '')[:10]
|
commit_message = commit_payload.get('commit', {}).get('message', '')
|
||||||
|
commit_date = commit_payload.get('commit', {}).get('committer', {}).get('date', '')[:10]
|
||||||
|
|
||||||
version = f"main-{commit_sha}"
|
version = f"main-{commit_sha}"
|
||||||
changelog = [commit_message] if commit_message else []
|
changelog = [commit_message] if commit_message else []
|
||||||
@@ -598,10 +600,11 @@ class UpdateRoutes:
|
|||||||
custom_headers={'Accept': 'application/vnd.github+json'}
|
custom_headers={'Accept': 'application/vnd.github+json'}
|
||||||
)
|
)
|
||||||
if c_ok:
|
if c_ok:
|
||||||
if c_data.get('status') in ('ahead', 'diverged'):
|
compare_payload = cast(dict[str, Any], c_data)
|
||||||
behind_by = c_data.get('ahead_by', 0)
|
if compare_payload.get('status') in ('ahead', 'diverged'):
|
||||||
|
behind_by = compare_payload.get('ahead_by', 0)
|
||||||
else:
|
else:
|
||||||
behind_by = c_data.get('behind_by', 0)
|
behind_by = compare_payload.get('behind_by', 0)
|
||||||
|
|
||||||
return version, changelog, behind_by, commit_date
|
return version, changelog, behind_by, commit_date
|
||||||
|
|
||||||
@@ -706,7 +709,7 @@ class UpdateRoutes:
|
|||||||
logger.info(f"Successfully updated to {new_version}")
|
logger.info(f"Successfully updated to {new_version}")
|
||||||
return True, new_version
|
return True, new_version
|
||||||
|
|
||||||
except git.exc.GitError as e:
|
except git.exc.GitError as e: # pyright: ignore[reportAttributeAccessIssue]
|
||||||
logger.error(f"Git error during update: {e}")
|
logger.error(f"Git error during update: {e}")
|
||||||
return False, ""
|
return False, ""
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -767,7 +770,7 @@ class UpdateRoutes:
|
|||||||
return git_info
|
return git_info
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def _get_remote_version() -> tuple[str, List[str], List[Dict]]:
|
async def _get_remote_version() -> tuple[str, List[str], List[Dict[str, Any]]]:
|
||||||
"""
|
"""
|
||||||
Fetch remote version from GitHub
|
Fetch remote version from GitHub
|
||||||
Returns:
|
Returns:
|
||||||
@@ -789,7 +792,7 @@ class UpdateRoutes:
|
|||||||
|
|
||||||
# Parse releases
|
# Parse releases
|
||||||
releases = []
|
releases = []
|
||||||
for i, release in enumerate(data):
|
for i, release in enumerate(cast(list[dict[str, Any]], data)):
|
||||||
version = release.get('tag_name', '')
|
version = release.get('tag_name', '')
|
||||||
if not version.startswith('v'):
|
if not version.startswith('v'):
|
||||||
version = f"v{version}"
|
version = f"v{version}"
|
||||||
|
|||||||
@@ -117,7 +117,7 @@ def _render_prompt(template: str, variables: Dict[str, Any]) -> str:
|
|||||||
Uses simple regex substitution — no Jinja2 dependency needed.
|
Uses simple regex substitution — no Jinja2 dependency needed.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def replace(match: re.Match) -> str:
|
def replace(match: re.Match[str]) -> str:
|
||||||
key = match.group(1).strip()
|
key = match.group(1).strip()
|
||||||
value = variables.get(key, "")
|
value = variables.get(key, "")
|
||||||
if isinstance(value, (dict, list)):
|
if isinstance(value, (dict, list)):
|
||||||
|
|||||||
@@ -295,7 +295,7 @@ class PostProcessor:
|
|||||||
normalises every tag to lowercase for case-insensitive dedup.
|
normalises every tag to lowercase for case-insensitive dedup.
|
||||||
"""
|
"""
|
||||||
merged: List[str] = []
|
merged: List[str] = []
|
||||||
seen: set = set()
|
seen: set[str] = set()
|
||||||
for tag in list(existing) + list(new):
|
for tag in list(existing) + list(new):
|
||||||
t = tag.strip().lower()
|
t = tag.strip().lower()
|
||||||
if t and t not in seen:
|
if t and t not in seen:
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ _FRONTMATTER_RE = re.compile(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _parse_skill_file(path: Path) -> tuple[dict, str]:
|
def _parse_skill_file(path: Path) -> tuple[dict[str, Any], str]:
|
||||||
"""Read a prompt definition file (``prompt.md`` or legacy ``SKILL.md``) and
|
"""Read a prompt definition file (``prompt.md`` or legacy ``SKILL.md``) and
|
||||||
return (frontmatter_dict, body_text).
|
return (frontmatter_dict, body_text).
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import html as html_module
|
import html as html_module
|
||||||
import re
|
import re
|
||||||
from typing import List, Tuple
|
from typing import Any, List, Tuple
|
||||||
|
|
||||||
|
|
||||||
_REPO_URL_PATTERN = re.compile(r"https?://huggingface\.co/([^/]+/[^/]+)")
|
_REPO_URL_PATTERN = re.compile(r"https?://huggingface\.co/([^/]+/[^/]+)")
|
||||||
@@ -18,10 +18,10 @@ _REPO_URL_PATTERN = re.compile(r"https?://huggingface\.co/([^/]+/[^/]+)")
|
|||||||
def extract_simple_markdown_images(
|
def extract_simple_markdown_images(
|
||||||
markdown_text: str,
|
markdown_text: str,
|
||||||
repo: str,
|
repo: str,
|
||||||
existing_urls: set | None = None,
|
existing_urls: set[str] | None = None,
|
||||||
default_width: int = 512,
|
default_width: int = 512,
|
||||||
default_height: int = 512,
|
default_height: int = 512,
|
||||||
) -> list[dict]:
|
) -> list[dict[str, Any]]:
|
||||||
"""Extract standalone markdown images from the README body.
|
"""Extract standalone markdown images from the README body.
|
||||||
|
|
||||||
Matches ```` on lines that are NOT part of a markdown table
|
Matches ```` on lines that are NOT part of a markdown table
|
||||||
@@ -36,8 +36,8 @@ def extract_simple_markdown_images(
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
base_url = f"https://huggingface.co/{repo}/resolve/main"
|
base_url = f"https://huggingface.co/{repo}/resolve/main"
|
||||||
images: list[dict] = []
|
images: list[dict[str, Any]] = []
|
||||||
seen_urls: set = set(existing_urls) if existing_urls else set()
|
seen_urls: set[str] = set(existing_urls) if existing_urls else set()
|
||||||
|
|
||||||
# Collect lines that are NOT inside fenced code blocks
|
# Collect lines that are NOT inside fenced code blocks
|
||||||
lines = markdown_text.split("\n")
|
lines = markdown_text.split("\n")
|
||||||
@@ -86,10 +86,10 @@ def extract_simple_markdown_images(
|
|||||||
def extract_html_img_tags(
|
def extract_html_img_tags(
|
||||||
markdown_text: str,
|
markdown_text: str,
|
||||||
repo: str,
|
repo: str,
|
||||||
existing_urls: set | None = None,
|
existing_urls: set[str] | None = None,
|
||||||
default_width: int = 512,
|
default_width: int = 512,
|
||||||
default_height: int = 512,
|
default_height: int = 512,
|
||||||
) -> list[dict]:
|
) -> list[dict[str, Any]]:
|
||||||
"""Extract image URLs from HTML ``<img src=\"...\">`` tags in the README.
|
"""Extract image URLs from HTML ``<img src=\"...\">`` tags in the README.
|
||||||
|
|
||||||
Many HF collection repos (e.g. ``deadman44/Z-Image_LoRA``) use raw HTML
|
Many HF collection repos (e.g. ``deadman44/Z-Image_LoRA``) use raw HTML
|
||||||
@@ -103,8 +103,8 @@ def extract_html_img_tags(
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
base_url = f"https://huggingface.co/{repo}/resolve/main"
|
base_url = f"https://huggingface.co/{repo}/resolve/main"
|
||||||
images: list[dict] = []
|
images: list[dict[str, Any]] = []
|
||||||
seen_urls: set = set(existing_urls) if existing_urls else set()
|
seen_urls: set[str] = set(existing_urls) if existing_urls else set()
|
||||||
|
|
||||||
for m in re.finditer(
|
for m in re.finditer(
|
||||||
r'<img\s[^>]*src=\"([^\"]+)\"',
|
r'<img\s[^>]*src=\"([^\"]+)\"',
|
||||||
@@ -175,7 +175,7 @@ def extract_gallery_images(
|
|||||||
repo: str,
|
repo: str,
|
||||||
default_width: int = 512,
|
default_width: int = 512,
|
||||||
default_height: int = 512,
|
default_height: int = 512,
|
||||||
) -> List[dict]:
|
) -> List[dict[str, Any]]:
|
||||||
"""Extract widget/gallery images from the YAML frontmatter of a HF README.
|
"""Extract widget/gallery images from the YAML frontmatter of a HF README.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -196,7 +196,7 @@ def extract_gallery_images(
|
|||||||
if not frontmatter:
|
if not frontmatter:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
images: List[dict] = []
|
images: List[dict[str, Any]] = []
|
||||||
base_url = f"https://huggingface.co/{repo}/resolve/main"
|
base_url = f"https://huggingface.co/{repo}/resolve/main"
|
||||||
w = default_width or 512
|
w = default_width or 512
|
||||||
h = default_height or 512
|
h = default_height or 512
|
||||||
@@ -258,7 +258,7 @@ def extract_gallery_images(
|
|||||||
text = raw_text
|
text = raw_text
|
||||||
|
|
||||||
if url:
|
if url:
|
||||||
image: dict = {
|
image: dict[str, Any] = {
|
||||||
"url": url,
|
"url": url,
|
||||||
"type": "image",
|
"type": "image",
|
||||||
"nsfwLevel": 0,
|
"nsfwLevel": 0,
|
||||||
@@ -276,10 +276,10 @@ def extract_gallery_images(
|
|||||||
def extract_gallery_table_images(
|
def extract_gallery_table_images(
|
||||||
markdown_text: str,
|
markdown_text: str,
|
||||||
repo: str,
|
repo: str,
|
||||||
existing_urls: set | None = None,
|
existing_urls: set[str] | None = None,
|
||||||
default_width: int = 512,
|
default_width: int = 512,
|
||||||
default_height: int = 512,
|
default_height: int = 512,
|
||||||
) -> list[dict]:
|
) -> list[dict[str, Any]]:
|
||||||
"""Extract images from ``| Preview | Prompt |`` markdown gallery tables.
|
"""Extract images from ``| Preview | Prompt |`` markdown gallery tables.
|
||||||
|
|
||||||
Many HF READMEs include a sample-gallery table in the body (outside
|
Many HF READMEs include a sample-gallery table in the body (outside
|
||||||
@@ -295,8 +295,8 @@ def extract_gallery_table_images(
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
base_url = f"https://huggingface.co/{repo}/resolve/main"
|
base_url = f"https://huggingface.co/{repo}/resolve/main"
|
||||||
images: list[dict] = []
|
images: list[dict[str, Any]] = []
|
||||||
seen_urls: set = set(existing_urls) if existing_urls else set()
|
seen_urls: set[str] = set(existing_urls) if existing_urls else set()
|
||||||
lines = markdown_text.split("\n")
|
lines = markdown_text.split("\n")
|
||||||
n = len(lines)
|
n = len(lines)
|
||||||
i = 0
|
i = 0
|
||||||
@@ -514,7 +514,7 @@ def _strip_standalone_images(text: str) -> str:
|
|||||||
URL was stripped entirely, making it impossible for the LLM to return
|
URL was stripped entirely, making it impossible for the LLM to return
|
||||||
a ``preview_url`` for repos that use HTML ``<img>`` tags exclusively.
|
a ``preview_url`` for repos that use HTML ``<img>`` tags exclusively.
|
||||||
"""
|
"""
|
||||||
def _img_to_md(match: re.Match) -> str:
|
def _img_to_md(match: re.Match[str]) -> str:
|
||||||
"""Convert an ``<img>`` tag to markdown image syntax ````."""
|
"""Convert an ``<img>`` tag to markdown image syntax ````."""
|
||||||
tag = match.group(0)
|
tag = match.group(0)
|
||||||
src_m = re.search(r'src="([^"]+)"', tag) or re.search(r"src='([^']+)'", tag)
|
src_m = re.search(r'src="([^"]+)"', tag) or re.search(r"src='([^']+)'", tag)
|
||||||
@@ -942,7 +942,7 @@ def _strip_badge_images(text: str) -> str:
|
|||||||
"twitter", "colab", "gradio", "space",
|
"twitter", "colab", "gradio", "space",
|
||||||
)
|
)
|
||||||
|
|
||||||
def _should_remove(m: re.Match) -> str:
|
def _should_remove(m: re.Match[str]) -> str:
|
||||||
alt = (m.group(1) or "").lower()
|
alt = (m.group(1) or "").lower()
|
||||||
for kw in badge_keywords:
|
for kw in badge_keywords:
|
||||||
if kw in alt:
|
if kw in alt:
|
||||||
|
|||||||
+233
-28
@@ -1,3 +1,7 @@
|
|||||||
|
# pyright: reportImportCycles=false
|
||||||
|
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||||
|
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||||
|
# import cycles. Breaking them would require an architectural refactor.
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
@@ -7,6 +11,7 @@ import os
|
|||||||
import secrets
|
import secrets
|
||||||
import shutil
|
import shutil
|
||||||
import socket
|
import socket
|
||||||
|
import time
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -20,10 +25,43 @@ from .settings_manager import get_settings_manager
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Maximum times the download poll loop will re-schedule a transfer after it
|
||||||
|
# is lost (daemon restart / RPC outage) before failing the download.
|
||||||
|
MAX_TRANSFER_RECOVERY_ATTEMPTS = 2
|
||||||
|
|
||||||
|
# stderr lines matching these markers indicate a disk write failure inside
|
||||||
|
# aria2 (piece cache flush or raw file write). They are promoted to INFO so
|
||||||
|
# the root cause (disk full, permission denied, file locked by another
|
||||||
|
# process, ...) is visible in the default logs; all other stderr output stays
|
||||||
|
# at DEBUG to avoid noise.
|
||||||
|
_DISK_WRITE_ERROR_MARKERS = (
|
||||||
|
# aria2 wrapper messages (write disk cache flush path)
|
||||||
|
"write disk cache flush failure",
|
||||||
|
"error when trying to flush write cache",
|
||||||
|
"failed to write into the file",
|
||||||
|
"failed to open the file",
|
||||||
|
"failed to seek the file",
|
||||||
|
# underlying root-cause phrases reported via "cause: ..." (POSIX + Windows)
|
||||||
|
"no space left on device",
|
||||||
|
"not enough space on the disk",
|
||||||
|
"input/output error",
|
||||||
|
"permission denied",
|
||||||
|
"access is denied",
|
||||||
|
"disk quota exceeded",
|
||||||
|
"used by another process",
|
||||||
|
"sharing violation",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Minimum interval between INFO-level reports of the same stderr line so a
|
||||||
|
# repeated failure (e.g. aria2 retrying against a full disk) does not spam
|
||||||
|
# the log.
|
||||||
|
STDERR_ERROR_REPORT_INTERVAL = 60.0
|
||||||
|
|
||||||
|
|
||||||
def _try_certifi_ca_path() -> str | None:
|
def _try_certifi_ca_path() -> str | None:
|
||||||
"""Return the certifi CA bundle path if available, else None."""
|
"""Return the certifi CA bundle path if available, else None."""
|
||||||
try:
|
try:
|
||||||
import certifi # type: ignore[import-untyped]
|
import certifi # pyright: ignore[reportMissingTypeStubs]
|
||||||
|
|
||||||
path = certifi.where()
|
path = certifi.where()
|
||||||
if os.path.isfile(path):
|
if os.path.isfile(path):
|
||||||
@@ -44,6 +82,17 @@ CIVITAI_DOWNLOAD_URL_PREFIXES = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_no_uri_available_error(message: str) -> bool:
|
||||||
|
"""Return True for aria2's "No URI available" transfer failure.
|
||||||
|
|
||||||
|
aria2 reports this when every URI for the transfer has become unusable.
|
||||||
|
For CivitAI downloads this typically means the temporary signed URL
|
||||||
|
expired mid-download; the transfer can be recovered by resolving a fresh
|
||||||
|
signed URL and re-scheduling with ``continue=true``.
|
||||||
|
"""
|
||||||
|
return "no uri available" in message.lower()
|
||||||
|
|
||||||
|
|
||||||
class Aria2Error(RuntimeError):
|
class Aria2Error(RuntimeError):
|
||||||
"""Raised when aria2 integration fails."""
|
"""Raised when aria2 integration fails."""
|
||||||
|
|
||||||
@@ -81,10 +130,12 @@ class Aria2Downloader:
|
|||||||
self._rpc_session: Optional[aiohttp.ClientSession] = None
|
self._rpc_session: Optional[aiohttp.ClientSession] = None
|
||||||
self._rpc_session_lock = asyncio.Lock()
|
self._rpc_session_lock = asyncio.Lock()
|
||||||
self._process_lock = asyncio.Lock()
|
self._process_lock = asyncio.Lock()
|
||||||
|
self._register_lock = asyncio.Lock()
|
||||||
self._transfers: Dict[str, Aria2Transfer] = {}
|
self._transfers: Dict[str, Aria2Transfer] = {}
|
||||||
self._poll_interval = 0.5
|
self._poll_interval = 0.5
|
||||||
self._state_store = Aria2TransferStateStore()
|
self._state_store = Aria2TransferStateStore()
|
||||||
self._stderr_reader_task: Optional[asyncio.Task] = None
|
self._stderr_reader_task: Optional[asyncio.Task[Any]] = None
|
||||||
|
self._stderr_error_report: Dict[str, float] = {}
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def is_running(self) -> bool:
|
def is_running(self) -> bool:
|
||||||
@@ -99,26 +150,61 @@ class Aria2Downloader:
|
|||||||
progress_callback=None,
|
progress_callback=None,
|
||||||
headers: Optional[Dict[str, str]] = None,
|
headers: Optional[Dict[str, str]] = None,
|
||||||
) -> Tuple[bool, str]:
|
) -> Tuple[bool, str]:
|
||||||
"""Download a file using aria2 RPC and wait for completion."""
|
"""Download a file using aria2 RPC and wait for completion.
|
||||||
|
|
||||||
|
The poll loop is self-healing: when the in-memory transfer entry
|
||||||
|
disappears (e.g. another download restarted the daemon and
|
||||||
|
``close()`` cleared ``_transfers``) or the RPC becomes unreachable,
|
||||||
|
the transfer is re-scheduled with ``continue=true`` so the download
|
||||||
|
resumes from the on-disk ``.aria2`` control file. The same
|
||||||
|
re-scheduling happens when aria2 fails with "No URI available"
|
||||||
|
(typically an expired CivitAI signed URL): a fresh URL is resolved
|
||||||
|
and the partial download continues. Recovery is bounded by
|
||||||
|
``MAX_TRANSFER_RECOVERY_ATTEMPTS``.
|
||||||
|
"""
|
||||||
|
|
||||||
await self._ensure_process()
|
await self._ensure_process()
|
||||||
save_path = os.path.abspath(save_path)
|
save_path = os.path.abspath(save_path)
|
||||||
transfer = self._transfers.get(download_id)
|
|
||||||
if transfer is None or os.path.abspath(transfer.save_path) != save_path:
|
|
||||||
gid = await self._schedule_download(
|
|
||||||
url,
|
|
||||||
save_path,
|
|
||||||
download_id=download_id,
|
|
||||||
headers=headers,
|
|
||||||
)
|
|
||||||
transfer = Aria2Transfer(gid=gid, save_path=save_path)
|
|
||||||
self._transfers[download_id] = transfer
|
|
||||||
|
|
||||||
|
async with self._register_lock:
|
||||||
|
transfer = self._transfers.get(download_id)
|
||||||
|
if transfer is None or os.path.abspath(transfer.save_path) != save_path:
|
||||||
|
transfer = await self._register_transfer(
|
||||||
|
url,
|
||||||
|
save_path,
|
||||||
|
download_id=download_id,
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
|
||||||
|
recovery_attempts = 0
|
||||||
try:
|
try:
|
||||||
while True:
|
while True:
|
||||||
status = await self._get_status_with_retry(download_id)
|
try:
|
||||||
|
status = await self._get_status_with_retry(download_id)
|
||||||
|
except Aria2Error:
|
||||||
|
status = None
|
||||||
|
|
||||||
if status is None:
|
if status is None:
|
||||||
return False, "aria2 download not found"
|
if recovery_attempts >= MAX_TRANSFER_RECOVERY_ATTEMPTS:
|
||||||
|
return False, "aria2 download not found"
|
||||||
|
recovery_attempts += 1
|
||||||
|
logger.warning(
|
||||||
|
"aria2 transfer %s lost; re-scheduling with resume "
|
||||||
|
"(attempt %d/%d)",
|
||||||
|
download_id,
|
||||||
|
recovery_attempts,
|
||||||
|
MAX_TRANSFER_RECOVERY_ATTEMPTS,
|
||||||
|
)
|
||||||
|
await asyncio.sleep(1.0)
|
||||||
|
await self._ensure_process()
|
||||||
|
async with self._register_lock:
|
||||||
|
transfer = await self._register_transfer(
|
||||||
|
url,
|
||||||
|
save_path,
|
||||||
|
download_id=download_id,
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
snapshot = self._build_progress_snapshot(status)
|
snapshot = self._build_progress_snapshot(status)
|
||||||
if progress_callback is not None:
|
if progress_callback is not None:
|
||||||
@@ -129,13 +215,44 @@ class Aria2Downloader:
|
|||||||
completed_path = self._resolve_completed_path(status, save_path)
|
completed_path = self._resolve_completed_path(status, save_path)
|
||||||
return True, completed_path
|
return True, completed_path
|
||||||
if state == "error":
|
if state == "error":
|
||||||
return False, status.get("errorMessage") or "aria2 download failed"
|
error_message = status.get("errorMessage") or "aria2 download failed"
|
||||||
|
if (
|
||||||
|
_is_no_uri_available_error(error_message)
|
||||||
|
and recovery_attempts < MAX_TRANSFER_RECOVERY_ATTEMPTS
|
||||||
|
):
|
||||||
|
# The signed URL (e.g. CivitAI's) expired before the
|
||||||
|
# transfer finished. Re-registering resolves a fresh
|
||||||
|
# URL and resumes from the on-disk partial payload and
|
||||||
|
# .aria2 control file via ``continue=true``.
|
||||||
|
recovery_attempts += 1
|
||||||
|
logger.warning(
|
||||||
|
"aria2 transfer %s failed with %r; refreshing the "
|
||||||
|
"URL and resuming the partial download "
|
||||||
|
"(attempt %d/%d)",
|
||||||
|
download_id,
|
||||||
|
error_message,
|
||||||
|
recovery_attempts,
|
||||||
|
MAX_TRANSFER_RECOVERY_ATTEMPTS,
|
||||||
|
)
|
||||||
|
await asyncio.sleep(1.0)
|
||||||
|
await self._ensure_process()
|
||||||
|
async with self._register_lock:
|
||||||
|
transfer = await self._register_transfer(
|
||||||
|
url,
|
||||||
|
save_path,
|
||||||
|
download_id=download_id,
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
return False, error_message
|
||||||
if state == "removed":
|
if state == "removed":
|
||||||
return False, "Download was cancelled"
|
return False, "Download was cancelled"
|
||||||
|
|
||||||
await asyncio.sleep(self._poll_interval)
|
await asyncio.sleep(self._poll_interval)
|
||||||
finally:
|
finally:
|
||||||
self._transfers.pop(download_id, None)
|
current = self._transfers.get(download_id)
|
||||||
|
if current is not None and current.gid == transfer.gid:
|
||||||
|
self._transfers.pop(download_id, None)
|
||||||
|
|
||||||
async def _get_status_with_retry(
|
async def _get_status_with_retry(
|
||||||
self, download_id: str, *, max_retries: int = 4, retry_delay: float = 3.0
|
self, download_id: str, *, max_retries: int = 4, retry_delay: float = 3.0
|
||||||
@@ -143,8 +260,9 @@ class Aria2Downloader:
|
|||||||
"""Call get_status with retry for transient RPC failures.
|
"""Call get_status with retry for transient RPC failures.
|
||||||
|
|
||||||
Only retries on :exc:`Aria2Error` (RPC-level failure). Returns
|
Only retries on :exc:`Aria2Error` (RPC-level failure). Returns
|
||||||
``None`` immediately when the download_id is not tracked (a missing
|
``None`` immediately when the transfer is not tracked or its GID is
|
||||||
transfer is not a transient condition, so retrying is pointless).
|
gone from the daemon (a missing transfer is not a transient
|
||||||
|
condition, so retrying is pointless).
|
||||||
|
|
||||||
A single failed RPC call should not immediately fail the download,
|
A single failed RPC call should not immediately fail the download,
|
||||||
because aria2 may be temporarily busy (e.g. finalizing multiple
|
because aria2 may be temporarily busy (e.g. finalizing multiple
|
||||||
@@ -190,7 +308,7 @@ class Aria2Downloader:
|
|||||||
download_id,
|
download_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
options: Dict[str, str] = {
|
options: Dict[str, Any] = {
|
||||||
"dir": save_dir,
|
"dir": save_dir,
|
||||||
"out": out_name,
|
"out": out_name,
|
||||||
"continue": "true",
|
"continue": "true",
|
||||||
@@ -238,8 +356,33 @@ class Aria2Downloader:
|
|||||||
)
|
)
|
||||||
return gid
|
return gid
|
||||||
|
|
||||||
|
async def _register_transfer(
|
||||||
|
self,
|
||||||
|
url: str,
|
||||||
|
save_path: str,
|
||||||
|
*,
|
||||||
|
download_id: str,
|
||||||
|
headers: Optional[Dict[str, str]] = None,
|
||||||
|
) -> Aria2Transfer:
|
||||||
|
"""Schedule a download and track it in the in-memory transfer registry."""
|
||||||
|
gid = await self._schedule_download(
|
||||||
|
url,
|
||||||
|
save_path,
|
||||||
|
download_id=download_id,
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
transfer = Aria2Transfer(gid=gid, save_path=os.path.abspath(save_path))
|
||||||
|
self._transfers[download_id] = transfer
|
||||||
|
return transfer
|
||||||
|
|
||||||
async def get_status(self, download_id: str) -> Optional[Dict[str, Any]]:
|
async def get_status(self, download_id: str) -> Optional[Dict[str, Any]]:
|
||||||
"""Return the raw aria2 status payload for a known download."""
|
"""Return the raw aria2 status payload for a known download.
|
||||||
|
|
||||||
|
Returns ``None`` when the download_id is not tracked or the daemon no
|
||||||
|
longer knows the transfer's GID (daemon restart / forceRemove). A
|
||||||
|
forgotten GID is permanent, not transient, so the caller's recovery
|
||||||
|
path handles it instead of burning retry attempts on a dead GID.
|
||||||
|
"""
|
||||||
|
|
||||||
transfer = self._transfers.get(download_id)
|
transfer = self._transfers.get(download_id)
|
||||||
if transfer is None:
|
if transfer is None:
|
||||||
@@ -255,8 +398,17 @@ class Aria2Downloader:
|
|||||||
"files",
|
"files",
|
||||||
]
|
]
|
||||||
try:
|
try:
|
||||||
status = await self._rpc_call("aria2.tellStatus", [transfer.gid, keys])
|
status = await self._rpc_call(
|
||||||
|
"aria2.tellStatus", [transfer.gid, keys], log_errors=False
|
||||||
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
if "not found" in str(exc).lower():
|
||||||
|
logger.debug(
|
||||||
|
"aria2 GID %s for download %s is gone; treating as lost transfer",
|
||||||
|
transfer.gid,
|
||||||
|
download_id,
|
||||||
|
)
|
||||||
|
return None
|
||||||
raise Aria2Error(f"Failed to query aria2 download status: {exc}") from exc
|
raise Aria2Error(f"Failed to query aria2 download status: {exc}") from exc
|
||||||
|
|
||||||
if isinstance(status, dict):
|
if isinstance(status, dict):
|
||||||
@@ -274,7 +426,9 @@ class Aria2Downloader:
|
|||||||
"files",
|
"files",
|
||||||
]
|
]
|
||||||
try:
|
try:
|
||||||
status = await self._rpc_call("aria2.tellStatus", [gid, keys])
|
status = await self._rpc_call(
|
||||||
|
"aria2.tellStatus", [gid, keys], log_errors=False
|
||||||
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
message = str(exc)
|
message = str(exc)
|
||||||
if "cannot be found" in message.lower() or "not found" in message.lower():
|
if "cannot be found" in message.lower() or "not found" in message.lower():
|
||||||
@@ -341,8 +495,19 @@ class Aria2Downloader:
|
|||||||
try:
|
try:
|
||||||
await self._rpc_call("aria2.forceRemove", [transfer.gid])
|
await self._rpc_call("aria2.forceRemove", [transfer.gid])
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return {"success": False, "error": str(exc)}
|
if "not found" not in str(exc).lower():
|
||||||
|
return {"success": False, "error": str(exc)}
|
||||||
|
# The daemon already forgot this GID (restart / prior removal),
|
||||||
|
# so the transfer is effectively cancelled.
|
||||||
|
logger.debug(
|
||||||
|
"aria2 GID %s for download %s already gone during cancel",
|
||||||
|
transfer.gid,
|
||||||
|
download_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Drop the in-memory entry as well so a concurrent poll loop does
|
||||||
|
# not mistake the removal for a lost transfer and re-register it.
|
||||||
|
self._transfers.pop(download_id, None)
|
||||||
await self._state_store.remove(download_id)
|
await self._state_store.remove(download_id)
|
||||||
return {"success": True, "message": "Download cancelled successfully"}
|
return {"success": True, "message": "Download cancelled successfully"}
|
||||||
|
|
||||||
@@ -385,16 +550,51 @@ class Aria2Downloader:
|
|||||||
blocks, which freezes the entire ``aria2c`` process — including its
|
blocks, which freezes the entire ``aria2c`` process — including its
|
||||||
RPC handler. This background task reads lines from stderr as they
|
RPC handler. This background task reads lines from stderr as they
|
||||||
arrive and forwards them to Python's logger.
|
arrive and forwards them to Python's logger.
|
||||||
|
|
||||||
|
Lines that indicate a disk write failure (e.g. the "cause: No space
|
||||||
|
left on device" line that follows "Write disk cache flush failure")
|
||||||
|
are promoted to INFO so the root cause is visible without enabling
|
||||||
|
debug logging; every other line stays at DEBUG to avoid noise.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
assert self._process is not None and self._process.stderr is not None
|
assert self._process is not None and self._process.stderr is not None
|
||||||
async for line in self._process.stderr:
|
async for line in self._process.stderr:
|
||||||
text = line.decode("utf-8", errors="replace").rstrip()
|
text = line.decode("utf-8", errors="replace").rstrip()
|
||||||
if text:
|
if text:
|
||||||
logger.debug("aria2 stderr: %s", text)
|
if self._is_disk_write_error(text):
|
||||||
|
self._report_stderr_error(text)
|
||||||
|
else:
|
||||||
|
logger.debug("aria2 stderr: %s", text)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_disk_write_error(text: str) -> bool:
|
||||||
|
lowered = text.lower()
|
||||||
|
return any(marker in lowered for marker in _DISK_WRITE_ERROR_MARKERS)
|
||||||
|
|
||||||
|
def _report_stderr_error(self, text: str) -> None:
|
||||||
|
"""INFO-log a disk write failure line, rate-limited per line text.
|
||||||
|
|
||||||
|
aria2 re-emits the same error chain on every poll/retry while the
|
||||||
|
underlying condition persists; only the first occurrence within
|
||||||
|
``STDERR_ERROR_REPORT_INTERVAL`` seconds is promoted to INFO.
|
||||||
|
"""
|
||||||
|
now = time.monotonic()
|
||||||
|
last = self._stderr_error_report.get(text)
|
||||||
|
if last is not None and now - last < STDERR_ERROR_REPORT_INTERVAL:
|
||||||
|
logger.debug("aria2 stderr (repeated disk write error): %s", text)
|
||||||
|
return
|
||||||
|
# Drop entries older than the window so the map stays bounded even
|
||||||
|
# during a long disk-full episode (piece indexes change per line).
|
||||||
|
self._stderr_error_report = {
|
||||||
|
line: timestamp
|
||||||
|
for line, timestamp in self._stderr_error_report.items()
|
||||||
|
if now - timestamp < STDERR_ERROR_REPORT_INTERVAL
|
||||||
|
}
|
||||||
|
self._stderr_error_report[text] = now
|
||||||
|
logger.info("aria2 disk write failure: %s", text)
|
||||||
|
|
||||||
async def _dispatch_progress(self, callback, snapshot: DownloadProgress) -> None:
|
async def _dispatch_progress(self, callback, snapshot: DownloadProgress) -> None:
|
||||||
try:
|
try:
|
||||||
result = callback(snapshot, snapshot)
|
result = callback(snapshot, snapshot)
|
||||||
@@ -597,7 +797,9 @@ class Aria2Downloader:
|
|||||||
|
|
||||||
return isinstance(result, dict)
|
return isinstance(result, dict)
|
||||||
|
|
||||||
async def _rpc_call(self, method: str, params: list[Any]) -> Any:
|
async def _rpc_call(
|
||||||
|
self, method: str, params: list[Any], *, log_errors: bool = True
|
||||||
|
) -> Any:
|
||||||
if not self._rpc_url:
|
if not self._rpc_url:
|
||||||
raise Aria2Error("aria2 RPC endpoint is not initialized")
|
raise Aria2Error("aria2 RPC endpoint is not initialized")
|
||||||
|
|
||||||
@@ -628,7 +830,10 @@ class Aria2Downloader:
|
|||||||
error = body["error"] or {}
|
error = body["error"] or {}
|
||||||
code = error.get("code") if isinstance(error, dict) else None
|
code = error.get("code") if isinstance(error, dict) else None
|
||||||
message = error.get("message") if isinstance(error, dict) else str(error)
|
message = error.get("message") if isinstance(error, dict) else str(error)
|
||||||
logger.error(
|
# Probing calls (e.g. tellStatus for a GID the daemon may have
|
||||||
|
# forgotten) pass log_errors=False: an expected "not found" must
|
||||||
|
# not spam the log at ERROR level.
|
||||||
|
(logger.error if log_errors else logger.debug)(
|
||||||
"aria2 RPC %s failed with HTTP %s, code=%s, message=%s",
|
"aria2 RPC %s failed with HTTP %s, code=%s, message=%s",
|
||||||
method,
|
method,
|
||||||
response.status,
|
response.status,
|
||||||
@@ -643,7 +848,7 @@ class Aria2Downloader:
|
|||||||
raise Aria2Error(status_message or "Unknown aria2 RPC error")
|
raise Aria2Error(status_message or "Unknown aria2 RPC error")
|
||||||
|
|
||||||
if response.status != 200:
|
if response.status != 200:
|
||||||
logger.error(
|
(logger.error if log_errors else logger.debug)(
|
||||||
"aria2 RPC %s returned unexpected HTTP status %s without error payload: %s",
|
"aria2 RPC %s returned unexpected HTTP status %s without error payload: %s",
|
||||||
method,
|
method,
|
||||||
response.status,
|
response.status,
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from filename, base_model, and CivitAI version name — no manual tagging requir
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import re
|
import re
|
||||||
from typing import Dict, List, Set
|
from typing import Any, Dict, List, Set
|
||||||
|
|
||||||
# ── Tag category definitions ──────────────────────────────────────────
|
# ── Tag category definitions ──────────────────────────────────────────
|
||||||
# Each category maps a display label to a regex pattern.
|
# Each category maps a display label to a regex pattern.
|
||||||
@@ -52,7 +52,7 @@ AUTO_TAG_GROUPS = {
|
|||||||
DEFAULT_ENABLED_GROUPS = {"mode", "video"}
|
DEFAULT_ENABLED_GROUPS = {"mode", "video"}
|
||||||
|
|
||||||
|
|
||||||
def _collect_sources(model_data: Dict) -> List[str]:
|
def _collect_sources(model_data: Dict[str, Any]) -> List[str]:
|
||||||
"""Collect all text sources from model data for tag matching."""
|
"""Collect all text sources from model data for tag matching."""
|
||||||
sources: List[str] = []
|
sources: List[str] = []
|
||||||
|
|
||||||
@@ -73,7 +73,7 @@ def _collect_sources(model_data: Dict) -> List[str]:
|
|||||||
return sources
|
return sources
|
||||||
|
|
||||||
|
|
||||||
def extract_auto_tags(model_data: Dict) -> List[str]:
|
def extract_auto_tags(model_data: Dict[str, Any]) -> List[str]:
|
||||||
"""Extract auto-detected tags from model metadata.
|
"""Extract auto-detected tags from model metadata.
|
||||||
|
|
||||||
Uses a two-layer approach:
|
Uses a two-layer approach:
|
||||||
|
|||||||
@@ -0,0 +1,144 @@
|
|||||||
|
# pyright: reportImportCycles=false
|
||||||
|
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||||
|
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||||
|
# import cycles. Breaking them would require an architectural refactor.
|
||||||
|
"""Backfill the AutoV3 checked state for models loaded from a persisted snapshot.
|
||||||
|
|
||||||
|
The SQLite persistent cache predates the AutoV3 feature, so entries hydrated
|
||||||
|
from it have a NULL ``autov3`` column (the "not checked yet" state). This
|
||||||
|
service computes the embedded AutoV3 hash for each such model — once per
|
||||||
|
process — and persists it through the scanner's single write path
|
||||||
|
(:meth:`ModelScanner.update_autov3_for_model`), marking every visited row so a
|
||||||
|
subsequent run finds nothing left to do.
|
||||||
|
|
||||||
|
Three-state contract honored here:
|
||||||
|
|
||||||
|
- ``NULL`` (sqlite) / absent (dict) = not checked yet → backfill computes it
|
||||||
|
- ``''`` (sqlite/dict) / JSON null = checked, no value available → never recompute
|
||||||
|
- 12-char lowercase hex = value → never recompute
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
|
if TYPE_CHECKING: # pragma: no cover - type-check only; runtime imports are local
|
||||||
|
from .model_scanner import ModelScanner
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_autov3(file_path: str) -> str:
|
||||||
|
"""Resolve the AutoV3 hash for a model file.
|
||||||
|
|
||||||
|
Prefers the Civitai AutoV3 reported for the file whose SHA256 matches
|
||||||
|
(the authoritative value for recipe matching); falls back to the embedded
|
||||||
|
safetensors header hash. Returns ``''`` when neither is available.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
metadata_path = f"{os.path.splitext(file_path)[0]}.metadata.json"
|
||||||
|
if os.path.exists(metadata_path):
|
||||||
|
with open(metadata_path, "r", encoding="utf-8") as handle:
|
||||||
|
payload = json.load(handle)
|
||||||
|
if isinstance(payload, dict):
|
||||||
|
from ..utils.models import autov3_from_civitai_files # local import avoids cycles
|
||||||
|
|
||||||
|
sha256 = (payload.get("sha256") or "").lower()
|
||||||
|
civitai_autov3 = autov3_from_civitai_files(payload.get("civitai"), sha256)
|
||||||
|
if civitai_autov3:
|
||||||
|
return civitai_autov3
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
from ..utils.file_utils import calculate_autov3 # local import avoids cycles
|
||||||
|
|
||||||
|
return calculate_autov3(file_path) or ""
|
||||||
|
|
||||||
|
|
||||||
|
class Autov3BackfillService:
|
||||||
|
"""Compute and persist AutoV3 hashes for models missing a checked state."""
|
||||||
|
|
||||||
|
_instance: Optional["Autov3BackfillService"] = None
|
||||||
|
_instance_lock = threading.Lock()
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
# Re-entrancy guard per model type: scanners for different model types
|
||||||
|
# initialize concurrently (lora_manager.py), so a global guard would
|
||||||
|
# silently skip every type but the first to start. Each model type
|
||||||
|
# runs its own backfill; a duplicate trigger for the same type no-ops.
|
||||||
|
self._running_types: set[str] = set()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_instance(cls) -> "Autov3BackfillService":
|
||||||
|
"""Return the process-wide singleton instance."""
|
||||||
|
if cls._instance is None:
|
||||||
|
with cls._instance_lock:
|
||||||
|
if cls._instance is None:
|
||||||
|
cls._instance = cls()
|
||||||
|
return cls._instance
|
||||||
|
|
||||||
|
async def backfill(self, scanner: "ModelScanner") -> int:
|
||||||
|
"""Compute AutoV3 for every un-checked model of ``scanner.model_type``.
|
||||||
|
|
||||||
|
Each candidate file is read once via :func:`~py.utils.file_utils.calculate_autov3`
|
||||||
|
(cheap: safetensors header only) and the result is persisted through
|
||||||
|
``scanner.update_autov3_for_model``. Files that no longer exist on
|
||||||
|
disk are skipped — they are intentionally NOT marked, because scanner
|
||||||
|
cleanup removes the stale row later.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The number of models successfully updated. Never raises; on any
|
||||||
|
failure a warning is logged and ``0`` is returned. A duplicate
|
||||||
|
trigger for a model type that is already being backfilled returns
|
||||||
|
``0`` immediately; different model types run concurrently.
|
||||||
|
"""
|
||||||
|
model_type = scanner.model_type
|
||||||
|
if model_type in self._running_types:
|
||||||
|
return 0
|
||||||
|
self._running_types.add(model_type)
|
||||||
|
try:
|
||||||
|
# Local imports avoid import cycles at module load time.
|
||||||
|
from .persistent_model_cache import get_persistent_cache
|
||||||
|
from ..utils.file_utils import calculate_autov3
|
||||||
|
|
||||||
|
persistent = getattr(scanner, "_persistent_cache", None) or get_persistent_cache()
|
||||||
|
paths = persistent.get_models_missing_autov3(model_type)
|
||||||
|
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
count = 0
|
||||||
|
for path in paths:
|
||||||
|
# A file that no longer exists must not be marked; scanner
|
||||||
|
# cleanup removes the stale row later. The existence check and
|
||||||
|
# hash resolution run in the executor so the loop stays
|
||||||
|
# responsive to API requests while the backfill iterates a
|
||||||
|
# large library.
|
||||||
|
if not await loop.run_in_executor(None, os.path.exists, path):
|
||||||
|
continue
|
||||||
|
autov3 = await loop.run_in_executor(None, _resolve_autov3, path)
|
||||||
|
if await scanner.update_autov3_for_model(model_type, path, autov3):
|
||||||
|
count += 1
|
||||||
|
|
||||||
|
if paths:
|
||||||
|
logger.info(
|
||||||
|
"AutoV3 backfill: updated %d/%d models for %s",
|
||||||
|
count,
|
||||||
|
len(paths),
|
||||||
|
model_type,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# Steady state after the first run: nothing left to backfill.
|
||||||
|
logger.debug("AutoV3 backfill: nothing to process for %s", model_type)
|
||||||
|
return count
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(
|
||||||
|
"AutoV3 backfill failed for %s: %s",
|
||||||
|
getattr(scanner, "model_type", "?"),
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
finally:
|
||||||
|
self._running_types.discard(model_type)
|
||||||
@@ -1,3 +1,7 @@
|
|||||||
|
# pyright: reportImportCycles=false
|
||||||
|
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||||
|
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||||
|
# import cycles. Breaking them would require an architectural refactor.
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
import asyncio
|
import asyncio
|
||||||
import re
|
import re
|
||||||
from typing import Any, Dict, List, Optional, Type, Union, TYPE_CHECKING
|
import random
|
||||||
|
from typing import Any, Awaitable, Dict, List, Optional, Type, Union, TYPE_CHECKING, cast
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
@@ -69,24 +70,24 @@ class BaseModelService(ABC):
|
|||||||
page: int,
|
page: int,
|
||||||
page_size: int,
|
page_size: int,
|
||||||
sort_by: str = "name",
|
sort_by: str = "name",
|
||||||
folder: str = None,
|
folder: str | None = None,
|
||||||
folder_include: list = None,
|
folder_include: list[str] | None = None,
|
||||||
folder_exclude: list = None,
|
folder_exclude: list[str] | None = None,
|
||||||
search: str = None,
|
search: str | None = None,
|
||||||
fuzzy_search: bool = False,
|
fuzzy_search: bool = False,
|
||||||
base_models: list = None,
|
base_models: list[str] | None = None,
|
||||||
model_types: list = None,
|
model_types: list[str] | None = None,
|
||||||
tags: Optional[Dict[str, str]] = None,
|
tags: Optional[Dict[str, str]] = None,
|
||||||
auto_tags: Optional[Dict[str, str]] = None,
|
auto_tags: Optional[Dict[str, str]] = None,
|
||||||
search_options: dict = None,
|
search_options: dict[str, Any] | None = None,
|
||||||
hash_filters: dict = None,
|
hash_filters: dict[str, Any] | None = None,
|
||||||
favorites_only: bool = False,
|
favorites_only: bool = False,
|
||||||
update_available_only: bool = False,
|
update_available_only: bool = False,
|
||||||
credit_required: Optional[bool] = None,
|
credit_required: Optional[bool] = None,
|
||||||
allow_selling_generated_content: Optional[bool] = None,
|
allow_selling_generated_content: Optional[bool] = None,
|
||||||
tag_logic: str = "any",
|
tag_logic: str = "any",
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict:
|
) -> Dict[str, Any]:
|
||||||
"""Get paginated and filtered model data"""
|
"""Get paginated and filtered model data"""
|
||||||
overall_start = time.perf_counter()
|
overall_start = time.perf_counter()
|
||||||
|
|
||||||
@@ -177,8 +178,8 @@ class BaseModelService(ABC):
|
|||||||
ufs = self.settings.get("version_grouping", "same_base")
|
ufs = self.settings.get("version_grouping", "same_base")
|
||||||
group_by_base = ufs == "same_base"
|
group_by_base = ufs == "same_base"
|
||||||
|
|
||||||
model_groups: Dict[Any, List[Dict]] = {}
|
model_groups: Dict[Any, List[Dict[str, Any]]] = {}
|
||||||
ungrouped_standalone: List[Dict] = []
|
ungrouped_standalone: List[Dict[str, Any]] = []
|
||||||
for item in sorted_data:
|
for item in sorted_data:
|
||||||
mid = self._extract_group_key(item)
|
mid = self._extract_group_key(item)
|
||||||
if mid is None:
|
if mid is None:
|
||||||
@@ -248,7 +249,7 @@ class BaseModelService(ABC):
|
|||||||
filter_duration = time.perf_counter() - t1
|
filter_duration = time.perf_counter() - t1
|
||||||
post_filter_count = len(filtered_data)
|
post_filter_count = len(filtered_data)
|
||||||
|
|
||||||
annotated_for_filter: Optional[List[Dict]] = None
|
annotated_for_filter: Optional[List[Dict[str, Any]]] = None
|
||||||
t2 = time.perf_counter()
|
t2 = time.perf_counter()
|
||||||
if update_available_only:
|
if update_available_only:
|
||||||
annotated_for_filter = await self._annotate_update_flags(filtered_data)
|
annotated_for_filter = await self._annotate_update_flags(filtered_data)
|
||||||
@@ -295,11 +296,11 @@ class BaseModelService(ABC):
|
|||||||
page: int,
|
page: int,
|
||||||
page_size: int,
|
page_size: int,
|
||||||
sort_by: str = "name",
|
sort_by: str = "name",
|
||||||
search: str = None,
|
search: str | None = None,
|
||||||
fuzzy_search: bool = False,
|
fuzzy_search: bool = False,
|
||||||
search_options: dict = None,
|
search_options: dict[str, Any] | None = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict:
|
) -> Dict[str, Any]:
|
||||||
"""Get paginated excluded model data."""
|
"""Get paginated excluded model data."""
|
||||||
excluded_paths = list(self.scanner.get_excluded_models())
|
excluded_paths = list(self.scanner.get_excluded_models())
|
||||||
excluded_entries: List[Dict[str, Any]] = []
|
excluded_entries: List[Dict[str, Any]] = []
|
||||||
@@ -325,7 +326,7 @@ class BaseModelService(ABC):
|
|||||||
]
|
]
|
||||||
persist_current_cache = getattr(self.scanner, "_persist_current_cache", None)
|
persist_current_cache = getattr(self.scanner, "_persist_current_cache", None)
|
||||||
if callable(persist_current_cache):
|
if callable(persist_current_cache):
|
||||||
await persist_current_cache()
|
await cast(Awaitable[Any], persist_current_cache())
|
||||||
|
|
||||||
excluded_entries = self._sort_entries(excluded_entries, sort_by)
|
excluded_entries = self._sort_entries(excluded_entries, sort_by)
|
||||||
|
|
||||||
@@ -390,6 +391,12 @@ class BaseModelService(ABC):
|
|||||||
(item.get("model_name") or item.get("file_name") or "").lower(),
|
(item.get("model_name") or item.get("file_name") or "").lower(),
|
||||||
item.get("file_path", "").lower(),
|
item.get("file_path", "").lower(),
|
||||||
)
|
)
|
||||||
|
elif key_name == "random":
|
||||||
|
# Seeded random shuffle: same seed -> same order (stable pagination)
|
||||||
|
rng = random.Random(sort_params.seed or "random")
|
||||||
|
result = list(data)
|
||||||
|
rng.shuffle(result)
|
||||||
|
return result
|
||||||
elif key_name == "size":
|
elif key_name == "size":
|
||||||
key_fn = lambda item: (
|
key_fn = lambda item: (
|
||||||
int(item.get("size", 0) or 0),
|
int(item.get("size", 0) or 0),
|
||||||
@@ -437,39 +444,50 @@ class BaseModelService(ABC):
|
|||||||
return entry
|
return entry
|
||||||
|
|
||||||
async def _apply_hash_filters(
|
async def _apply_hash_filters(
|
||||||
self, data: List[Dict], hash_filters: Dict
|
self, data: List[Dict[str, Any]], hash_filters: Dict[str, Any]
|
||||||
) -> List[Dict]:
|
) -> List[Dict[str, Any]]:
|
||||||
"""Apply hash-based filtering"""
|
"""Apply hash-based filtering (SHA256 and AutoV3)."""
|
||||||
|
|
||||||
|
def matches_hash_set(item: Dict[str, Any], hash_set: set[str]) -> bool:
|
||||||
|
"""Check whether an item matches any hash in the set.
|
||||||
|
|
||||||
|
Compares the item's ``sha256`` field and its non-empty ``autov3``
|
||||||
|
field, both case-insensitively.
|
||||||
|
"""
|
||||||
|
if item.get("sha256", "").lower() in hash_set:
|
||||||
|
return True
|
||||||
|
autov3 = item.get("autov3", "")
|
||||||
|
return bool(autov3) and autov3.lower() in hash_set
|
||||||
|
|
||||||
single_hash = hash_filters.get("single_hash")
|
single_hash = hash_filters.get("single_hash")
|
||||||
multiple_hashes = hash_filters.get("multiple_hashes")
|
multiple_hashes = hash_filters.get("multiple_hashes")
|
||||||
|
|
||||||
if single_hash:
|
if single_hash:
|
||||||
# Filter by single hash
|
# Filter by single hash (SHA256 or AutoV3)
|
||||||
single_hash = single_hash.lower()
|
|
||||||
return [
|
return [
|
||||||
item for item in data if item.get("sha256", "").lower() == single_hash
|
item for item in data if matches_hash_set(item, {single_hash.lower()})
|
||||||
]
|
]
|
||||||
elif multiple_hashes:
|
elif multiple_hashes:
|
||||||
# Filter by multiple hashes
|
# Filter by multiple hashes (SHA256 or AutoV3)
|
||||||
hash_set = set(hash.lower() for hash in multiple_hashes)
|
hash_set = {hash.lower() for hash in multiple_hashes}
|
||||||
return [item for item in data if item.get("sha256", "").lower() in hash_set]
|
return [item for item in data if matches_hash_set(item, hash_set)]
|
||||||
|
|
||||||
return data
|
return data
|
||||||
|
|
||||||
async def _apply_common_filters(
|
async def _apply_common_filters(
|
||||||
self,
|
self,
|
||||||
data: List[Dict],
|
data: List[Dict[str, Any]],
|
||||||
folder: str = None,
|
folder: str | None = None,
|
||||||
folder_include: list = None,
|
folder_include: list[str] | None = None,
|
||||||
folder_exclude: list = None,
|
folder_exclude: list[str] | None = None,
|
||||||
base_models: list = None,
|
base_models: list[str] | None = None,
|
||||||
model_types: list = None,
|
model_types: list[str] | None = None,
|
||||||
tags: Optional[Dict[str, str]] = None,
|
tags: Optional[Dict[str, str]] = None,
|
||||||
auto_tags: Optional[Dict[str, str]] = None,
|
auto_tags: Optional[Dict[str, str]] = None,
|
||||||
favorites_only: bool = False,
|
favorites_only: bool = False,
|
||||||
search_options: dict = None,
|
search_options: dict[str, Any] | None = None,
|
||||||
tag_logic: str = "any",
|
tag_logic: str = "any",
|
||||||
) -> List[Dict]:
|
) -> List[Dict[str, Any]]:
|
||||||
"""Apply common filters that work across all model types"""
|
"""Apply common filters that work across all model types"""
|
||||||
normalized_options = self.search_strategy.normalize_options(search_options)
|
normalized_options = self.search_strategy.normalize_options(search_options)
|
||||||
criteria = FilterCriteria(
|
criteria = FilterCriteria(
|
||||||
@@ -488,24 +506,24 @@ class BaseModelService(ABC):
|
|||||||
|
|
||||||
async def _apply_search_filters(
|
async def _apply_search_filters(
|
||||||
self,
|
self,
|
||||||
data: List[Dict],
|
data: List[Dict[str, Any]],
|
||||||
search: str,
|
search: str,
|
||||||
fuzzy_search: bool,
|
fuzzy_search: bool,
|
||||||
search_options: dict,
|
search_options: dict[str, Any] | None,
|
||||||
) -> List[Dict]:
|
) -> List[Dict[str, Any]]:
|
||||||
"""Apply search filtering"""
|
"""Apply search filtering"""
|
||||||
normalized_options = self.search_strategy.normalize_options(search_options)
|
normalized_options = self.search_strategy.normalize_options(search_options)
|
||||||
return self.search_strategy.apply(
|
return self.search_strategy.apply(
|
||||||
data, search, normalized_options, fuzzy_search
|
data, search, normalized_options, fuzzy_search
|
||||||
)
|
)
|
||||||
|
|
||||||
async def _apply_specific_filters(self, data: List[Dict], **kwargs) -> List[Dict]:
|
async def _apply_specific_filters(self, data: List[Dict[str, Any]], **kwargs) -> List[Dict[str, Any]]:
|
||||||
"""Apply model-specific filters - to be overridden by subclasses if needed"""
|
"""Apply model-specific filters - to be overridden by subclasses if needed"""
|
||||||
return data
|
return data
|
||||||
|
|
||||||
async def _apply_credit_required_filter(
|
async def _apply_credit_required_filter(
|
||||||
self, data: List[Dict], credit_required: bool
|
self, data: List[Dict[str, Any]], credit_required: bool
|
||||||
) -> List[Dict]:
|
) -> List[Dict[str, Any]]:
|
||||||
"""Apply credit required filtering based on license_flags.
|
"""Apply credit required filtering based on license_flags.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -535,8 +553,8 @@ class BaseModelService(ABC):
|
|||||||
return filtered_data
|
return filtered_data
|
||||||
|
|
||||||
async def _apply_allow_selling_filter(
|
async def _apply_allow_selling_filter(
|
||||||
self, data: List[Dict], allow_selling: bool
|
self, data: List[Dict[str, Any]], allow_selling: bool
|
||||||
) -> List[Dict]:
|
) -> List[Dict[str, Any]]:
|
||||||
"""Apply allow selling generated content filtering based on license_flags.
|
"""Apply allow selling generated content filtering based on license_flags.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -568,8 +586,8 @@ class BaseModelService(ABC):
|
|||||||
|
|
||||||
async def _annotate_update_flags(
|
async def _annotate_update_flags(
|
||||||
self,
|
self,
|
||||||
items: List[Dict],
|
items: List[Dict[str, Any]],
|
||||||
) -> List[Dict]:
|
) -> List[Dict[str, Any]]:
|
||||||
"""Attach an update_available flag to each response item.
|
"""Attach an update_available flag to each response item.
|
||||||
|
|
||||||
Items without a civitai model id default to False.
|
Items without a civitai model id default to False.
|
||||||
@@ -584,7 +602,7 @@ class BaseModelService(ABC):
|
|||||||
item["update_available"] = False
|
item["update_available"] = False
|
||||||
return annotated
|
return annotated
|
||||||
|
|
||||||
id_to_items: Dict[int, List[Dict]] = {}
|
id_to_items: Dict[int, List[Dict[str, Any]]] = {}
|
||||||
ordered_ids: List[int] = []
|
ordered_ids: List[int] = []
|
||||||
for item in annotated:
|
for item in annotated:
|
||||||
model_id = self._extract_model_id(item)
|
model_id = self._extract_model_id(item)
|
||||||
@@ -615,15 +633,25 @@ class BaseModelService(ABC):
|
|||||||
except Exception:
|
except Exception:
|
||||||
hide_early_access = False
|
hide_early_access = False
|
||||||
|
|
||||||
|
# Check user setting for hiding permanent paid updates
|
||||||
|
hide_paid = False
|
||||||
|
try:
|
||||||
|
hide_paid = bool(self.settings.get("hide_paid_updates", False))
|
||||||
|
except Exception:
|
||||||
|
hide_paid = False
|
||||||
|
|
||||||
records = None
|
records = None
|
||||||
resolved: Optional[Dict[int, bool]] = None
|
resolved: Optional[Dict[int, bool]] = None
|
||||||
if same_base_mode:
|
if same_base_mode:
|
||||||
record_method = getattr(self.update_service, "get_records_bulk", None)
|
record_method = getattr(self.update_service, "get_records_bulk", None)
|
||||||
if callable(record_method):
|
if callable(record_method):
|
||||||
try:
|
try:
|
||||||
records = await record_method(self.model_type, ordered_ids)
|
records = await cast(Awaitable[Any], record_method(self.model_type, ordered_ids))
|
||||||
resolved = {
|
resolved = {
|
||||||
model_id: record.has_update(hide_early_access=hide_early_access)
|
model_id: record.has_update(
|
||||||
|
hide_early_access=hide_early_access,
|
||||||
|
hide_paid=hide_paid,
|
||||||
|
)
|
||||||
for model_id, record in records.items()
|
for model_id, record in records.items()
|
||||||
}
|
}
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -641,11 +669,12 @@ class BaseModelService(ABC):
|
|||||||
bulk_method = getattr(self.update_service, "has_updates_bulk", None)
|
bulk_method = getattr(self.update_service, "has_updates_bulk", None)
|
||||||
if callable(bulk_method):
|
if callable(bulk_method):
|
||||||
try:
|
try:
|
||||||
resolved = await bulk_method(
|
resolved = await cast(Awaitable[Any], bulk_method(
|
||||||
self.model_type,
|
self.model_type,
|
||||||
ordered_ids,
|
ordered_ids,
|
||||||
hide_early_access=hide_early_access,
|
hide_early_access=hide_early_access,
|
||||||
)
|
hide_paid=hide_paid,
|
||||||
|
))
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.error(
|
logger.error(
|
||||||
"Failed to resolve update status in bulk for %s models (%s): %s",
|
"Failed to resolve update status in bulk for %s models (%s): %s",
|
||||||
@@ -659,7 +688,10 @@ class BaseModelService(ABC):
|
|||||||
if resolved is None:
|
if resolved is None:
|
||||||
tasks = [
|
tasks = [
|
||||||
self.update_service.has_update(
|
self.update_service.has_update(
|
||||||
self.model_type, model_id, hide_early_access=hide_early_access
|
self.model_type,
|
||||||
|
model_id,
|
||||||
|
hide_early_access=hide_early_access,
|
||||||
|
hide_paid=hide_paid,
|
||||||
)
|
)
|
||||||
for model_id in ordered_ids
|
for model_id in ordered_ids
|
||||||
]
|
]
|
||||||
@@ -699,6 +731,7 @@ class BaseModelService(ABC):
|
|||||||
threshold_version,
|
threshold_version,
|
||||||
base_model,
|
base_model,
|
||||||
hide_early_access=hide_early_access,
|
hide_early_access=hide_early_access,
|
||||||
|
hide_paid=hide_paid,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
flag = default_flag
|
flag = default_flag
|
||||||
@@ -707,7 +740,7 @@ class BaseModelService(ABC):
|
|||||||
return annotated
|
return annotated
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _extract_hf_group_key(item: Dict) -> Optional[str]:
|
def _extract_hf_group_key(item: Dict[str, Any]) -> Optional[str]:
|
||||||
"""Extract `hf:{owner}/{repo}` from item's ``hf_url``, or None."""
|
"""Extract `hf:{owner}/{repo}` from item's ``hf_url``, or None."""
|
||||||
hf_url = item.get("hf_url") if isinstance(item, dict) else None
|
hf_url = item.get("hf_url") if isinstance(item, dict) else None
|
||||||
if not hf_url or not isinstance(hf_url, str):
|
if not hf_url or not isinstance(hf_url, str):
|
||||||
@@ -720,7 +753,7 @@ class BaseModelService(ABC):
|
|||||||
return f"hf:{m.group(1)}"
|
return f"hf:{m.group(1)}"
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _extract_group_key(item: Dict) -> Union[int, str, None]:
|
def _extract_group_key(item: Dict[str, Any]) -> Union[int, str, None]:
|
||||||
"""Return the group identity key: CivitAI modelId (int) or HF repo (str).
|
"""Return the group identity key: CivitAI modelId (int) or HF repo (str).
|
||||||
|
|
||||||
Preference order:
|
Preference order:
|
||||||
@@ -734,7 +767,7 @@ class BaseModelService(ABC):
|
|||||||
return BaseModelService._extract_hf_group_key(item)
|
return BaseModelService._extract_hf_group_key(item)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _extract_model_id(item: Dict) -> Optional[int]:
|
def _extract_model_id(item: Dict[str, Any]) -> Optional[int]:
|
||||||
civitai = item.get("civitai") if isinstance(item, dict) else None
|
civitai = item.get("civitai") if isinstance(item, dict) else None
|
||||||
if not isinstance(civitai, dict):
|
if not isinstance(civitai, dict):
|
||||||
return None
|
return None
|
||||||
@@ -747,7 +780,7 @@ class BaseModelService(ABC):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _extract_version_id(item: Dict) -> Optional[int]:
|
def _extract_version_id(item: Dict[str, Any]) -> Optional[int]:
|
||||||
civitai = item.get("civitai") if isinstance(item, dict) else None
|
civitai = item.get("civitai") if isinstance(item, dict) else None
|
||||||
if not isinstance(civitai, dict):
|
if not isinstance(civitai, dict):
|
||||||
return None
|
return None
|
||||||
@@ -760,7 +793,7 @@ class BaseModelService(ABC):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _extract_base_model(item: Dict) -> Optional[str]:
|
def _extract_base_model(item: Dict[str, Any]) -> Optional[str]:
|
||||||
value = item.get("base_model")
|
value = item.get("base_model")
|
||||||
if value is None:
|
if value is None:
|
||||||
return None
|
return None
|
||||||
@@ -812,7 +845,7 @@ class BaseModelService(ABC):
|
|||||||
|
|
||||||
return highest_by_base
|
return highest_by_base
|
||||||
|
|
||||||
def _paginate(self, data: List[Dict], page: int, page_size: int) -> Dict:
|
def _paginate(self, data: List[Dict[str, Any]], page: int, page_size: int) -> Dict[str, Any]:
|
||||||
"""Apply pagination to filtered data"""
|
"""Apply pagination to filtered data"""
|
||||||
total_items = len(data)
|
total_items = len(data)
|
||||||
start_idx = (page - 1) * page_size
|
start_idx = (page - 1) * page_size
|
||||||
@@ -827,7 +860,7 @@ class BaseModelService(ABC):
|
|||||||
}
|
}
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def format_response(self, model_data: Dict) -> Optional[Dict]:
|
async def format_response(self, model_data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||||
"""Format model data for API response - must be implemented by subclasses.
|
"""Format model data for API response - must be implemented by subclasses.
|
||||||
|
|
||||||
Subclasses should return None for corrupted entries so the handler
|
Subclasses should return None for corrupted entries so the handler
|
||||||
@@ -836,17 +869,17 @@ class BaseModelService(ABC):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
# Common service methods that delegate to scanner
|
# Common service methods that delegate to scanner
|
||||||
async def get_top_tags(self, limit: int = 20) -> List[Dict]:
|
async def get_top_tags(self, limit: int = 20) -> List[Dict[str, Any]]:
|
||||||
"""Get top tags sorted by frequency"""
|
"""Get top tags sorted by frequency"""
|
||||||
return await self.scanner.get_top_tags(limit)
|
return await self.scanner.get_top_tags(limit)
|
||||||
|
|
||||||
async def search_tags(
|
async def search_tags(
|
||||||
self, query: str, limit: int = 50
|
self, query: str, limit: int = 50
|
||||||
) -> List[Dict]:
|
) -> List[Dict[str, Any]]:
|
||||||
"""Search tags by substring, sorted by frequency"""
|
"""Search tags by substring, sorted by frequency"""
|
||||||
return await self.scanner.search_tags(query, limit)
|
return await self.scanner.search_tags(query, limit)
|
||||||
|
|
||||||
async def get_base_models(self, limit: int = 20) -> List[Dict]:
|
async def get_base_models(self, limit: int = 20) -> List[Dict[str, Any]]:
|
||||||
"""Get base models sorted by frequency"""
|
"""Get base models sorted by frequency"""
|
||||||
return await self.scanner.get_base_models(limit)
|
return await self.scanner.get_base_models(limit)
|
||||||
|
|
||||||
@@ -913,7 +946,7 @@ class BaseModelService(ABC):
|
|||||||
"""Get model root directories"""
|
"""Get model root directories"""
|
||||||
return self.scanner.get_model_roots()
|
return self.scanner.get_model_roots()
|
||||||
|
|
||||||
def filter_civitai_data(self, data: Dict, minimal: bool = False) -> Dict:
|
def filter_civitai_data(self, data: Dict[str, Any], minimal: bool = False) -> Dict[str, Any]:
|
||||||
"""Filter relevant fields from CivitAI data"""
|
"""Filter relevant fields from CivitAI data"""
|
||||||
if not data:
|
if not data:
|
||||||
return {}
|
return {}
|
||||||
@@ -939,14 +972,25 @@ class BaseModelService(ABC):
|
|||||||
)
|
)
|
||||||
return {k: data[k] for k in fields if k in data}
|
return {k: data[k] for k in fields if k in data}
|
||||||
|
|
||||||
async def get_folder_tree(self, model_root: str) -> Dict:
|
async def _get_tree_folders(self, cache, include_empty: bool) -> List[str]:
|
||||||
|
"""Return the folder list backing folder tree responses.
|
||||||
|
|
||||||
|
With ``include_empty`` the directories are enumerated live from the
|
||||||
|
filesystem (including empty ones) via the scanner; otherwise the
|
||||||
|
models-only ``cache.folders`` list is used unchanged.
|
||||||
|
"""
|
||||||
|
if include_empty:
|
||||||
|
return await self.scanner.get_all_folders()
|
||||||
|
return cache.folders
|
||||||
|
|
||||||
|
async def get_folder_tree(self, model_root: str, include_empty: bool = False) -> Dict[str, Any]:
|
||||||
"""Get hierarchical folder tree for a specific model root"""
|
"""Get hierarchical folder tree for a specific model root"""
|
||||||
cache = await self.scanner.get_cached_data()
|
cache = await self.scanner.get_cached_data()
|
||||||
|
|
||||||
# Build tree structure from folders
|
# Build tree structure from folders
|
||||||
tree = {}
|
tree = {}
|
||||||
|
|
||||||
for folder in cache.folders:
|
for folder in await self._get_tree_folders(cache, include_empty):
|
||||||
# Check if this folder belongs to the specified model root
|
# Check if this folder belongs to the specified model root
|
||||||
folder_belongs_to_root = False
|
folder_belongs_to_root = False
|
||||||
for root in self.scanner.get_model_roots():
|
for root in self.scanner.get_model_roots():
|
||||||
@@ -968,7 +1012,7 @@ class BaseModelService(ABC):
|
|||||||
|
|
||||||
return tree
|
return tree
|
||||||
|
|
||||||
async def get_unified_folder_tree(self) -> Dict:
|
async def get_unified_folder_tree(self, include_empty: bool = False) -> Dict[str, Any]:
|
||||||
"""Get unified folder tree across all model roots"""
|
"""Get unified folder tree across all model roots"""
|
||||||
cache = await self.scanner.get_cached_data()
|
cache = await self.scanner.get_cached_data()
|
||||||
|
|
||||||
@@ -978,7 +1022,7 @@ class BaseModelService(ABC):
|
|||||||
# Get all model roots for path normalization
|
# Get all model roots for path normalization
|
||||||
model_roots = self.scanner.get_model_roots()
|
model_roots = self.scanner.get_model_roots()
|
||||||
|
|
||||||
for folder in cache.folders:
|
for folder in await self._get_tree_folders(cache, include_empty):
|
||||||
if not folder: # Skip empty folders
|
if not folder: # Skip empty folders
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -997,7 +1041,7 @@ class BaseModelService(ABC):
|
|||||||
|
|
||||||
return unified_tree
|
return unified_tree
|
||||||
|
|
||||||
async def get_model_notes(self, model_name: str) -> Optional[dict]:
|
async def get_model_notes(self, model_name: str) -> Optional[dict[str, Any]]:
|
||||||
"""Get notes and file_path for a specific model file.
|
"""Get notes and file_path for a specific model file.
|
||||||
|
|
||||||
Supports both simple names (``OWSMianne_ANIMA_V1``) and full-path
|
Supports both simple names (``OWSMianne_ANIMA_V1``) and full-path
|
||||||
@@ -1129,7 +1173,7 @@ class BaseModelService(ABC):
|
|||||||
|
|
||||||
return {"civitai_url": None, "model_id": None, "version_id": None}
|
return {"civitai_url": None, "model_id": None, "version_id": None}
|
||||||
|
|
||||||
async def get_model_metadata(self, file_path: str) -> Optional[Dict]:
|
async def get_model_metadata(self, file_path: str) -> Optional[Dict[str, Any]]:
|
||||||
"""Load full metadata for a single model.
|
"""Load full metadata for a single model.
|
||||||
|
|
||||||
Listing/search endpoints return lightweight cache entries; this method performs
|
Listing/search endpoints return lightweight cache entries; this method performs
|
||||||
@@ -1225,7 +1269,7 @@ class BaseModelService(ABC):
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _relative_path_sort_key(relative_path: str, include_terms: List[str]) -> tuple:
|
def _relative_path_sort_key(relative_path: str, include_terms: List[str]) -> tuple[int, int, int, str]:
|
||||||
"""Sort paths by how well they satisfy the include tokens.
|
"""Sort paths by how well they satisfy the include tokens.
|
||||||
|
|
||||||
Sorts based on path without extension for consistent ordering.
|
Sorts based on path without extension for consistent ordering.
|
||||||
@@ -1252,19 +1296,87 @@ class BaseModelService(ABC):
|
|||||||
)
|
)
|
||||||
|
|
||||||
async def search_relative_paths(
|
async def search_relative_paths(
|
||||||
self, search_term: str, limit: int = 15, offset: int = 0
|
self,
|
||||||
|
search_term: str,
|
||||||
|
limit: int = 15,
|
||||||
|
offset: int = 0,
|
||||||
|
*,
|
||||||
|
folder: Optional[str] = None,
|
||||||
|
folder_include: Optional[list[str]] = None,
|
||||||
|
folder_exclude: Optional[list[str]] = None,
|
||||||
|
base_models: Optional[list[str]] = None,
|
||||||
|
model_types: Optional[list[str]] = None,
|
||||||
|
tags: Optional[dict[str, str]] = None,
|
||||||
|
auto_tags: Optional[dict[str, str]] = None,
|
||||||
|
tag_logic: str = "any",
|
||||||
|
credit_required: Optional[bool] = None,
|
||||||
|
allow_selling_generated_content: Optional[bool] = None,
|
||||||
|
recursive: bool = True,
|
||||||
|
apply_filters: bool = False,
|
||||||
) -> List[str]:
|
) -> List[str]:
|
||||||
"""Search model relative file paths for autocomplete functionality"""
|
"""Search model relative file paths for autocomplete functionality.
|
||||||
|
|
||||||
|
Optional filter kwargs mirror the filters used by the list endpoint
|
||||||
|
(/api/lm/{prefix}/list). When no filter kwargs are provided the
|
||||||
|
behavior is identical to plain token-based path matching.
|
||||||
|
"""
|
||||||
cache = await self.scanner.get_cached_data()
|
cache = await self.scanner.get_cached_data()
|
||||||
include_terms, exclude_terms = self._parse_search_tokens(search_term)
|
include_terms, exclude_terms = self._parse_search_tokens(search_term)
|
||||||
|
|
||||||
|
data = cache.raw_data
|
||||||
|
has_filters = any(
|
||||||
|
[
|
||||||
|
apply_filters,
|
||||||
|
folder is not None,
|
||||||
|
folder_include,
|
||||||
|
folder_exclude,
|
||||||
|
base_models,
|
||||||
|
model_types,
|
||||||
|
tags,
|
||||||
|
auto_tags,
|
||||||
|
credit_required is not None,
|
||||||
|
allow_selling_generated_content is not None,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
if has_filters:
|
||||||
|
# Auto-tags are not stored in the scanner cache — they are computed
|
||||||
|
# on the fly. Pre-compute them only when an auto-tag filter is
|
||||||
|
# active to avoid mutating cache entries unnecessarily.
|
||||||
|
if auto_tags:
|
||||||
|
from .auto_tag_service import extract_auto_tags
|
||||||
|
|
||||||
|
for item in data:
|
||||||
|
if not item.get("auto_tags"):
|
||||||
|
item["auto_tags"] = extract_auto_tags(item)
|
||||||
|
|
||||||
|
criteria = FilterCriteria(
|
||||||
|
folder=folder,
|
||||||
|
folder_include=folder_include,
|
||||||
|
folder_exclude=folder_exclude,
|
||||||
|
base_models=base_models,
|
||||||
|
model_types=model_types,
|
||||||
|
tags=tags,
|
||||||
|
auto_tags=auto_tags,
|
||||||
|
search_options={"recursive": recursive},
|
||||||
|
tag_logic=tag_logic,
|
||||||
|
)
|
||||||
|
data = self.filter_set.apply(data, criteria)
|
||||||
|
if credit_required is not None:
|
||||||
|
data = await self._apply_credit_required_filter(
|
||||||
|
data, credit_required
|
||||||
|
)
|
||||||
|
if allow_selling_generated_content is not None:
|
||||||
|
data = await self._apply_allow_selling_filter(
|
||||||
|
data, allow_selling_generated_content
|
||||||
|
)
|
||||||
|
|
||||||
matching_paths = []
|
matching_paths = []
|
||||||
|
|
||||||
# Get model roots for path calculation
|
# Get model roots for path calculation
|
||||||
model_roots = self.scanner.get_model_roots()
|
model_roots = self.scanner.get_model_roots()
|
||||||
|
|
||||||
# Collect all matching paths first (needed for proper sorting and offset)
|
# Collect all matching paths first (needed for proper sorting and offset)
|
||||||
for model in cache.raw_data:
|
for model in data:
|
||||||
file_path = model.get("file_path", "")
|
file_path = model.get("file_path", "")
|
||||||
if not file_path:
|
if not file_path:
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -20,6 +20,11 @@ from .recipes import (
|
|||||||
RecipeDownloadError,
|
RecipeDownloadError,
|
||||||
RecipeNotFoundError,
|
RecipeNotFoundError,
|
||||||
)
|
)
|
||||||
|
from .recipes.import_info import (
|
||||||
|
CHANNEL_BATCH_IMPORT_LOCAL,
|
||||||
|
CHANNEL_BATCH_IMPORT_URL,
|
||||||
|
build_import_info,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class ImportItemType(Enum):
|
class ImportItemType(Enum):
|
||||||
@@ -71,6 +76,9 @@ class BatchImportProgress:
|
|||||||
tags: List[str] = field(default_factory=list)
|
tags: List[str] = field(default_factory=list)
|
||||||
skip_no_metadata: bool = False
|
skip_no_metadata: bool = False
|
||||||
skip_duplicates: bool = False
|
skip_duplicates: bool = False
|
||||||
|
# Set once any item is skipped due to vendor rate limiting (#1085); lets
|
||||||
|
# the UI surface a "slowing down / try again later" hint.
|
||||||
|
rate_limited: bool = False
|
||||||
|
|
||||||
def to_dict(self) -> Dict[str, Any]:
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
return {
|
return {
|
||||||
@@ -82,6 +90,7 @@ class BatchImportProgress:
|
|||||||
"skipped": self.skipped,
|
"skipped": self.skipped,
|
||||||
"current_item": self.current_item,
|
"current_item": self.current_item,
|
||||||
"status": self.status,
|
"status": self.status,
|
||||||
|
"rate_limited": self.rate_limited,
|
||||||
"started_at": self.started_at,
|
"started_at": self.started_at,
|
||||||
"finished_at": self.finished_at,
|
"finished_at": self.finished_at,
|
||||||
"progress_percent": round((self.completed / self.total) * 100, 1)
|
"progress_percent": round((self.completed / self.total) * 100, 1)
|
||||||
@@ -118,6 +127,10 @@ class AdaptiveConcurrencyController:
|
|||||||
self._task_durations: List[float] = []
|
self._task_durations: List[float] = []
|
||||||
self._recent_errors = 0
|
self._recent_errors = 0
|
||||||
self._recent_successes = 0
|
self._recent_successes = 0
|
||||||
|
# Batch-wide shared semaphore; created lazily on first use so the
|
||||||
|
# controller can also be constructed outside a running event loop.
|
||||||
|
self._semaphore: Optional[asyncio.Semaphore] = None
|
||||||
|
self._semaphore_capacity = initial_concurrency
|
||||||
|
|
||||||
def record_result(self, duration: float, success: bool) -> None:
|
def record_result(self, duration: float, success: bool) -> None:
|
||||||
self._task_durations.append(duration)
|
self._task_durations.append(duration)
|
||||||
@@ -146,7 +159,37 @@ class AdaptiveConcurrencyController:
|
|||||||
self._recent_successes = 0
|
self._recent_successes = 0
|
||||||
|
|
||||||
def get_semaphore(self) -> asyncio.Semaphore:
|
def get_semaphore(self) -> asyncio.Semaphore:
|
||||||
return asyncio.Semaphore(self.current_concurrency)
|
"""Return the batch-wide shared semaphore.
|
||||||
|
|
||||||
|
The same semaphore instance is returned for every item of a batch so
|
||||||
|
the configured concurrency bounds are actually enforced. Previously a
|
||||||
|
fresh semaphore was created per call, letting every item run
|
||||||
|
concurrently and hammering remote metadata providers without any
|
||||||
|
limit.
|
||||||
|
"""
|
||||||
|
if self._semaphore is None:
|
||||||
|
self._semaphore = asyncio.Semaphore(self.current_concurrency)
|
||||||
|
self._semaphore_capacity = self.current_concurrency
|
||||||
|
return self._semaphore
|
||||||
|
|
||||||
|
async def apply_concurrency(self) -> None:
|
||||||
|
"""Synchronize the shared semaphore capacity with ``current_concurrency``.
|
||||||
|
|
||||||
|
Call after ``record_result`` (once per completed item). Growing the
|
||||||
|
capacity is immediate (release). Shrinking requires acquiring a permit
|
||||||
|
and holding it, which is best-effort while other tasks are still
|
||||||
|
running — the capacity converges on subsequent calls.
|
||||||
|
"""
|
||||||
|
semaphore = self.get_semaphore()
|
||||||
|
while self._semaphore_capacity < self.current_concurrency:
|
||||||
|
semaphore.release()
|
||||||
|
self._semaphore_capacity += 1
|
||||||
|
while self._semaphore_capacity > self.current_concurrency:
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(semaphore.acquire(), timeout=0.01)
|
||||||
|
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||||
|
break
|
||||||
|
self._semaphore_capacity -= 1
|
||||||
|
|
||||||
|
|
||||||
class BatchImportService:
|
class BatchImportService:
|
||||||
@@ -184,6 +227,7 @@ class BatchImportService:
|
|||||||
def cancel_import(self, operation_id: str) -> bool:
|
def cancel_import(self, operation_id: str) -> bool:
|
||||||
if operation_id in self._active_operations:
|
if operation_id in self._active_operations:
|
||||||
self._cancellation_flags[operation_id] = True
|
self._cancellation_flags[operation_id] = True
|
||||||
|
self._logger.info("Cancel requested for batch import operation %s", operation_id)
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -273,6 +317,14 @@ class BatchImportService:
|
|||||||
self._active_operations[operation_id] = progress
|
self._active_operations[operation_id] = progress
|
||||||
self._cancellation_flags[operation_id] = False
|
self._cancellation_flags[operation_id] = False
|
||||||
|
|
||||||
|
self._logger.info(
|
||||||
|
"Starting batch import operation %s: %d item(s) (%d URL(s), %d local path(s))",
|
||||||
|
operation_id,
|
||||||
|
len(import_items),
|
||||||
|
sum(1 for it in import_items if it.item_type == ImportItemType.URL),
|
||||||
|
sum(1 for it in import_items if it.item_type == ImportItemType.LOCAL_PATH),
|
||||||
|
)
|
||||||
|
|
||||||
asyncio.create_task(
|
asyncio.create_task(
|
||||||
self._run_batch_import(
|
self._run_batch_import(
|
||||||
operation_id=operation_id,
|
operation_id=operation_id,
|
||||||
@@ -295,6 +347,12 @@ class BatchImportService:
|
|||||||
skip_duplicates: bool = False,
|
skip_duplicates: bool = False,
|
||||||
) -> str:
|
) -> str:
|
||||||
image_paths = await self._discover_images(directory, recursive)
|
image_paths = await self._discover_images(directory, recursive)
|
||||||
|
self._logger.info(
|
||||||
|
"Batch import directory scan: %d image(s) discovered in %s (recursive=%s)",
|
||||||
|
len(image_paths),
|
||||||
|
directory,
|
||||||
|
recursive,
|
||||||
|
)
|
||||||
|
|
||||||
items = [{"source": path, "type": "local_path"} for path in image_paths]
|
items = [{"source": path, "type": "local_path"} for path in image_paths]
|
||||||
|
|
||||||
@@ -334,6 +392,13 @@ class BatchImportService:
|
|||||||
ext = os.path.splitext(filename)[1].lower()
|
ext = os.path.splitext(filename)[1].lower()
|
||||||
return ext in self.SUPPORTED_EXTENSIONS
|
return ext in self.SUPPORTED_EXTENSIONS
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_rate_limit_error(error: Optional[str]) -> bool:
|
||||||
|
"""Return True when an error payload represents vendor rate limiting."""
|
||||||
|
if not error:
|
||||||
|
return False
|
||||||
|
return "rate limit" in error.lower()
|
||||||
|
|
||||||
async def _run_batch_import(
|
async def _run_batch_import(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -379,6 +444,9 @@ class BatchImportService:
|
|||||||
self._concurrency_controller.record_result(
|
self._concurrency_controller.record_result(
|
||||||
duration, result.get("success", False)
|
duration, result.get("success", False)
|
||||||
)
|
)
|
||||||
|
# Keep the shared batch semaphore in sync with the adaptively
|
||||||
|
# adjusted concurrency so the bounds actually take effect.
|
||||||
|
await self._concurrency_controller.apply_concurrency()
|
||||||
|
|
||||||
if result.get("success"):
|
if result.get("success"):
|
||||||
item.status = ImportStatus.SUCCESS
|
item.status = ImportStatus.SUCCESS
|
||||||
@@ -389,6 +457,17 @@ class BatchImportService:
|
|||||||
item.status = ImportStatus.SKIPPED
|
item.status = ImportStatus.SKIPPED
|
||||||
item.error_message = result.get("error")
|
item.error_message = result.get("error")
|
||||||
progress.skipped += 1
|
progress.skipped += 1
|
||||||
|
elif self._is_rate_limit_error(result.get("error")):
|
||||||
|
# Vendor rate limit is a transient, external condition —
|
||||||
|
# do not pollute the failure count with it (#1085). The
|
||||||
|
# import can simply be re-run later.
|
||||||
|
item.status = ImportStatus.SKIPPED
|
||||||
|
item.error_message = (
|
||||||
|
f"Rate limited by metadata provider; "
|
||||||
|
f"re-run the import later ({result.get('error')})"
|
||||||
|
)
|
||||||
|
progress.skipped += 1
|
||||||
|
progress.rate_limited = True
|
||||||
else:
|
else:
|
||||||
item.status = ImportStatus.FAILED
|
item.status = ImportStatus.FAILED
|
||||||
item.error_message = result.get("error")
|
item.error_message = result.get("error")
|
||||||
@@ -396,13 +475,36 @@ class BatchImportService:
|
|||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self._logger.error(f"Error importing {item.source}: {e}")
|
self._logger.error(f"Error importing {item.source}: {e}")
|
||||||
item.status = ImportStatus.FAILED
|
|
||||||
item.error_message = str(e)
|
|
||||||
item.duration = time.time() - start_time
|
item.duration = time.time() - start_time
|
||||||
progress.failed += 1
|
if self._is_rate_limit_error(str(e)):
|
||||||
|
item.status = ImportStatus.SKIPPED
|
||||||
|
item.error_message = (
|
||||||
|
f"Rate limited by metadata provider; "
|
||||||
|
f"re-run the import later ({e})"
|
||||||
|
)
|
||||||
|
progress.skipped += 1
|
||||||
|
progress.rate_limited = True
|
||||||
|
else:
|
||||||
|
item.status = ImportStatus.FAILED
|
||||||
|
item.error_message = str(e)
|
||||||
|
progress.failed += 1
|
||||||
self._concurrency_controller.record_result(item.duration, False)
|
self._concurrency_controller.record_result(item.duration, False)
|
||||||
|
await self._concurrency_controller.apply_concurrency()
|
||||||
|
|
||||||
progress.completed += 1
|
progress.completed += 1
|
||||||
|
self._logger.info(
|
||||||
|
"Batch import %s: item %d/%d status=%s source=%s%s",
|
||||||
|
operation_id,
|
||||||
|
progress.completed,
|
||||||
|
progress.total,
|
||||||
|
item.status.value,
|
||||||
|
(
|
||||||
|
os.path.basename(item.source)
|
||||||
|
if item.item_type == ImportItemType.LOCAL_PATH
|
||||||
|
else item.source[:50]
|
||||||
|
),
|
||||||
|
(f" error={item.error_message}" if item.error_message else ""),
|
||||||
|
)
|
||||||
await self._broadcast_progress(progress)
|
await self._broadcast_progress(progress)
|
||||||
|
|
||||||
tasks = [process_item(item) for item in progress.items]
|
tasks = [process_item(item) for item in progress.items]
|
||||||
@@ -415,6 +517,15 @@ class BatchImportService:
|
|||||||
|
|
||||||
progress.finished_at = time.time()
|
progress.finished_at = time.time()
|
||||||
progress.current_item = ""
|
progress.current_item = ""
|
||||||
|
self._logger.info(
|
||||||
|
"Batch import %s finished: status=%s total=%d success=%d failed=%d skipped=%d",
|
||||||
|
operation_id,
|
||||||
|
progress.status,
|
||||||
|
progress.total,
|
||||||
|
progress.success,
|
||||||
|
progress.failed,
|
||||||
|
progress.skipped,
|
||||||
|
)
|
||||||
await self._broadcast_progress(progress)
|
await self._broadcast_progress(progress)
|
||||||
|
|
||||||
await asyncio.sleep(5)
|
await asyncio.sleep(5)
|
||||||
@@ -518,6 +629,17 @@ class BatchImportService:
|
|||||||
"loras": loras,
|
"loras": loras,
|
||||||
"gen_params": payload.get("gen_params", {}),
|
"gen_params": payload.get("gen_params", {}),
|
||||||
"source_path": item.source,
|
"source_path": item.source,
|
||||||
|
# Record why this import ended up with no LoRAs so the
|
||||||
|
# recipe modal can explain it (collapsed by default).
|
||||||
|
"import_info": build_import_info(
|
||||||
|
(
|
||||||
|
CHANNEL_BATCH_IMPORT_URL
|
||||||
|
if item.item_type == ImportItemType.URL
|
||||||
|
else CHANNEL_BATCH_IMPORT_LOCAL
|
||||||
|
),
|
||||||
|
payload.get("diagnostics"),
|
||||||
|
loras,
|
||||||
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
if payload.get("checkpoint"):
|
if payload.get("checkpoint"):
|
||||||
@@ -595,3 +717,6 @@ class BatchImportService:
|
|||||||
def _cleanup_operation(self, operation_id: str) -> None:
|
def _cleanup_operation(self, operation_id: str) -> None:
|
||||||
if operation_id in self._cancellation_flags:
|
if operation_id in self._cancellation_flags:
|
||||||
del self._cancellation_flags[operation_id]
|
del self._cancellation_flags[operation_id]
|
||||||
|
if operation_id in self._active_operations:
|
||||||
|
del self._active_operations[operation_id]
|
||||||
|
self._logger.info("Batch import operation %s cleaned up", operation_id)
|
||||||
|
|||||||
@@ -59,6 +59,7 @@ class CacheEntryValidator:
|
|||||||
'notes': ('', False),
|
'notes': ('', False),
|
||||||
'usage_tips': ('', False),
|
'usage_tips': ('', False),
|
||||||
'hash_status': ('completed', False),
|
'hash_status': ('completed', False),
|
||||||
|
'autov3': (None, False),
|
||||||
}
|
}
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -119,8 +120,13 @@ class CacheEntryValidator:
|
|||||||
if is_required:
|
if is_required:
|
||||||
errors.append(f"Required field '{field_name}' is missing or None")
|
errors.append(f"Required field '{field_name}' is missing or None")
|
||||||
if auto_repair:
|
if auto_repair:
|
||||||
working_entry[field_name] = cls._get_default_copy(default_value)
|
# A missing optional field whose default is None is already
|
||||||
repaired = True
|
# semantically equal to its default (e.g. autov3: absent
|
||||||
|
# means "not checked") — writing None back is a no-op, not
|
||||||
|
# a repair.
|
||||||
|
if default_value is not None:
|
||||||
|
working_entry[field_name] = cls._get_default_copy(default_value)
|
||||||
|
repaired = True
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Validate field type and value
|
# Validate field type and value
|
||||||
@@ -175,6 +181,15 @@ class CacheEntryValidator:
|
|||||||
# that invalidates the entry, but we also don't mark it repaired.
|
# that invalidates the entry, but we also don't mark it repaired.
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
# Normalize autov3 to lowercase if needed (optional field, never stripped).
|
||||||
|
autov3 = working_entry.get('autov3')
|
||||||
|
if isinstance(autov3, str) and autov3:
|
||||||
|
normalized_autov3 = autov3.lower()
|
||||||
|
if normalized_autov3 != autov3:
|
||||||
|
if auto_repair:
|
||||||
|
working_entry['autov3'] = normalized_autov3
|
||||||
|
repaired = True
|
||||||
|
|
||||||
# Determine if entry is valid
|
# Determine if entry is valid
|
||||||
# Entry is valid if no critical required field errors remain after repair
|
# Entry is valid if no critical required field errors remain after repair
|
||||||
# Critical fields are file_path and sha256
|
# Critical fields are file_path and sha256
|
||||||
@@ -242,6 +257,19 @@ class CacheEntryValidator:
|
|||||||
"""
|
"""
|
||||||
expected_type = type(default_value)
|
expected_type = type(default_value)
|
||||||
|
|
||||||
|
# Special case: autov3 is optional with a three-state contract.
|
||||||
|
# None = not checked, "" = checked but unavailable, otherwise a
|
||||||
|
# 12-character hex string (case-insensitive here; normalized to
|
||||||
|
# lowercase separately).
|
||||||
|
if field_name == 'autov3':
|
||||||
|
if value is None or value == "":
|
||||||
|
return None
|
||||||
|
if not isinstance(value, str):
|
||||||
|
return f"Field 'autov3' should be string or None, got {type(value).__name__}"
|
||||||
|
if len(value) != 12 or any(c not in '0123456789abcdefABCDEF' for c in value):
|
||||||
|
return "Field 'autov3' should be a 12-character hex string"
|
||||||
|
return None
|
||||||
|
|
||||||
# Special handling for numeric types
|
# Special handling for numeric types
|
||||||
if expected_type == int:
|
if expected_type == int:
|
||||||
if not isinstance(value, (int, float)):
|
if not isinstance(value, (int, float)):
|
||||||
|
|||||||
@@ -1,3 +1,7 @@
|
|||||||
|
# pyright: reportImportCycles=false
|
||||||
|
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||||
|
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||||
|
# import cycles. Breaking them would require an architectural refactor.
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
@@ -6,10 +10,10 @@ from datetime import datetime
|
|||||||
from typing import Any, Dict, List, Optional
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
from ..utils.models import CheckpointMetadata
|
from ..utils.models import CheckpointMetadata
|
||||||
from ..utils.file_utils import find_preview_file, normalize_path
|
from ..utils.file_utils import find_preview_file, normalize_path, calculate_autov3
|
||||||
from ..utils.metadata_manager import MetadataManager
|
from ..utils.metadata_manager import MetadataManager
|
||||||
from ..config import config
|
from ..config import config
|
||||||
from .model_scanner import ModelScanner
|
from .model_scanner import ModelScanner, _is_excluded_dir
|
||||||
from .model_hash_index import ModelHashIndex
|
from .model_hash_index import ModelHashIndex
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -62,6 +66,11 @@ class CheckpointScanner(ModelScanner):
|
|||||||
# Find preview image
|
# Find preview image
|
||||||
preview_url = find_preview_file(base_name, dir_path)
|
preview_url = find_preview_file(base_name, dir_path)
|
||||||
|
|
||||||
|
# AutoV3 reads only the safetensors header, so it is cheap even for
|
||||||
|
# large checkpoints; record the checked state at creation time ("" =
|
||||||
|
# checked but unavailable).
|
||||||
|
autov3 = calculate_autov3(real_path)
|
||||||
|
|
||||||
# Create metadata WITHOUT calculating hash
|
# Create metadata WITHOUT calculating hash
|
||||||
metadata = CheckpointMetadata(
|
metadata = CheckpointMetadata(
|
||||||
file_name=base_name,
|
file_name=base_name,
|
||||||
@@ -77,6 +86,7 @@ class CheckpointScanner(ModelScanner):
|
|||||||
sub_type="checkpoint",
|
sub_type="checkpoint",
|
||||||
from_civitai=False, # Mark as local model since no hash yet
|
from_civitai=False, # Mark as local model since no hash yet
|
||||||
hash_status="pending", # Mark hash as pending
|
hash_status="pending", # Mark hash as pending
|
||||||
|
autov3=autov3 or "",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Save the created metadata
|
# Save the created metadata
|
||||||
@@ -120,7 +130,11 @@ class CheckpointScanner(ModelScanner):
|
|||||||
# that queries get_hash_by_filename first) will miss on every
|
# that queries get_hash_by_filename first) will miss on every
|
||||||
# lookup and keep calling back into this method, creating a
|
# lookup and keep calling back into this method, creating a
|
||||||
# tight loop that never populates the index.
|
# tight loop that never populates the index.
|
||||||
self._hash_index.add_entry(metadata.sha256.lower(), file_path)
|
self._hash_index.add_entry(
|
||||||
|
metadata.sha256.lower(),
|
||||||
|
file_path,
|
||||||
|
getattr(metadata, "autov3", None) or None,
|
||||||
|
)
|
||||||
return metadata.sha256
|
return metadata.sha256
|
||||||
|
|
||||||
async with self._hash_calculation_lock:
|
async with self._hash_calculation_lock:
|
||||||
@@ -132,7 +146,11 @@ class CheckpointScanner(ModelScanner):
|
|||||||
and metadata.hash_status == "completed"
|
and metadata.hash_status == "completed"
|
||||||
and metadata.sha256
|
and metadata.sha256
|
||||||
):
|
):
|
||||||
self._hash_index.add_entry(metadata.sha256.lower(), file_path)
|
self._hash_index.add_entry(
|
||||||
|
metadata.sha256.lower(),
|
||||||
|
file_path,
|
||||||
|
getattr(metadata, "autov3", None) or None,
|
||||||
|
)
|
||||||
return metadata.sha256
|
return metadata.sha256
|
||||||
|
|
||||||
task = self._hash_calculation_tasks.get(real_path)
|
task = self._hash_calculation_tasks.get(real_path)
|
||||||
@@ -185,7 +203,11 @@ class CheckpointScanner(ModelScanner):
|
|||||||
if metadata.hash_status == "completed" and metadata.sha256:
|
if metadata.hash_status == "completed" and metadata.sha256:
|
||||||
# Populate the in-memory hash index even for pre-computed
|
# Populate the in-memory hash index even for pre-computed
|
||||||
# hashes, mirroring the fix in calculate_hash_for_model.
|
# hashes, mirroring the fix in calculate_hash_for_model.
|
||||||
self._hash_index.add_entry(metadata.sha256.lower(), file_path)
|
self._hash_index.add_entry(
|
||||||
|
metadata.sha256.lower(),
|
||||||
|
file_path,
|
||||||
|
getattr(metadata, "autov3", None) or None,
|
||||||
|
)
|
||||||
return metadata.sha256
|
return metadata.sha256
|
||||||
|
|
||||||
# Update status to calculating
|
# Update status to calculating
|
||||||
@@ -202,7 +224,11 @@ class CheckpointScanner(ModelScanner):
|
|||||||
await MetadataManager.save_metadata(file_path, metadata)
|
await MetadataManager.save_metadata(file_path, metadata)
|
||||||
|
|
||||||
# Update hash index
|
# Update hash index
|
||||||
self._hash_index.add_entry(sha256.lower(), file_path)
|
self._hash_index.add_entry(
|
||||||
|
sha256.lower(),
|
||||||
|
file_path,
|
||||||
|
getattr(metadata, "autov3", None) or None,
|
||||||
|
)
|
||||||
|
|
||||||
# Update the in-memory cache entry so that subsequent
|
# Update the in-memory cache entry so that subsequent
|
||||||
# _persist_current_cache / _save_persistent_cache calls
|
# _persist_current_cache / _save_persistent_cache calls
|
||||||
@@ -216,6 +242,7 @@ class CheckpointScanner(ModelScanner):
|
|||||||
if entry.get("file_path") == file_path:
|
if entry.get("file_path") == file_path:
|
||||||
entry["sha256"] = sha256.lower()
|
entry["sha256"] = sha256.lower()
|
||||||
entry["hash_status"] = "completed"
|
entry["hash_status"] = "completed"
|
||||||
|
self.bump_cache_version()
|
||||||
break
|
break
|
||||||
|
|
||||||
logger.info(f"Hash calculated for checkpoint: {file_path}")
|
logger.info(f"Hash calculated for checkpoint: {file_path}")
|
||||||
@@ -301,7 +328,8 @@ class CheckpointScanner(ModelScanner):
|
|||||||
if not os.path.exists(root_path):
|
if not os.path.exists(root_path):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
for dirpath, _dirnames, filenames in os.walk(root_path):
|
for dirpath, dirnames, filenames in os.walk(root_path):
|
||||||
|
dirnames[:] = [d for d in dirnames if not _is_excluded_dir(d)]
|
||||||
for filename in filenames:
|
for filename in filenames:
|
||||||
if not filename.endswith(".metadata.json"):
|
if not filename.endswith(".metadata.json"):
|
||||||
continue
|
continue
|
||||||
@@ -405,7 +433,7 @@ class CheckpointScanner(ModelScanner):
|
|||||||
roots.extend(config.extra_checkpoints_roots or [])
|
roots.extend(config.extra_checkpoints_roots or [])
|
||||||
roots.extend(config.extra_unet_roots or [])
|
roots.extend(config.extra_unet_roots or [])
|
||||||
# Remove duplicates while preserving order
|
# Remove duplicates while preserving order
|
||||||
seen: set = set()
|
seen: set[str] = set()
|
||||||
unique_roots: List[str] = []
|
unique_roots: List[str] = []
|
||||||
for root in roots:
|
for root in roots:
|
||||||
if root not in seen:
|
if root not in seen:
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import os
|
import os
|
||||||
import logging
|
import logging
|
||||||
from typing import Dict, Optional
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
from .base_model_service import BaseModelService
|
from .base_model_service import BaseModelService
|
||||||
from .auto_tag_service import extract_auto_tags
|
from .auto_tag_service import extract_auto_tags
|
||||||
@@ -21,58 +21,59 @@ class CheckpointService(BaseModelService):
|
|||||||
"""
|
"""
|
||||||
super().__init__("checkpoint", scanner, CheckpointMetadata, update_service=update_service)
|
super().__init__("checkpoint", scanner, CheckpointMetadata, update_service=update_service)
|
||||||
|
|
||||||
async def format_response(self, checkpoint_data: Dict) -> Optional[Dict]:
|
async def format_response(self, model_data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||||
"""Format Checkpoint data for API response.
|
"""Format Checkpoint data for API response.
|
||||||
|
|
||||||
Returns None when the entry is missing critical fields (corrupted cache
|
Returns None when the entry is missing critical fields (corrupted cache
|
||||||
row), so the handler layer can filter it out. See issue #730.
|
row), so the handler layer can filter it out. See issue #730.
|
||||||
"""
|
"""
|
||||||
# Guard against corrupted cache entries missing critical fields
|
# Guard against corrupted cache entries missing critical fields
|
||||||
file_path = checkpoint_data.get("file_path")
|
file_path = model_data.get("file_path")
|
||||||
if not file_path or not isinstance(file_path, str):
|
if not file_path or not isinstance(file_path, str):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Skipping corrupted checkpoint entry (missing file_path): %s",
|
"Skipping corrupted checkpoint entry (missing file_path): %s",
|
||||||
checkpoint_data.get("file_name", "<unknown>"),
|
model_data.get("file_name", "<unknown>"),
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# Get sub_type from cache entry (new canonical field)
|
# Get sub_type from cache entry (new canonical field)
|
||||||
sub_type = checkpoint_data.get("sub_type", "checkpoint")
|
sub_type = model_data.get("sub_type", "checkpoint")
|
||||||
|
|
||||||
file_name = checkpoint_data.get("file_name") or ""
|
file_name = model_data.get("file_name") or ""
|
||||||
model_name = checkpoint_data.get("model_name") or file_name
|
model_name = model_data.get("model_name") or file_name
|
||||||
folder = checkpoint_data.get("folder") or ""
|
folder = model_data.get("folder") or ""
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"model_name": model_name,
|
"model_name": model_name,
|
||||||
"file_name": file_name,
|
"file_name": file_name,
|
||||||
"preview_url": config.get_preview_static_url(checkpoint_data.get("preview_url", "")),
|
"preview_url": config.get_preview_static_url(model_data.get("preview_url", "")),
|
||||||
"preview_nsfw_level": checkpoint_data.get("preview_nsfw_level", 0),
|
"preview_nsfw_level": model_data.get("preview_nsfw_level", 0),
|
||||||
"base_model": checkpoint_data.get("base_model", ""),
|
"base_model": model_data.get("base_model", ""),
|
||||||
"folder": folder,
|
"folder": folder,
|
||||||
"sha256": checkpoint_data.get("sha256", ""),
|
"sha256": model_data.get("sha256", ""),
|
||||||
|
"autov3": model_data.get("autov3"),
|
||||||
"file_path": file_path.replace(os.sep, "/"),
|
"file_path": file_path.replace(os.sep, "/"),
|
||||||
"file_size": checkpoint_data.get("size", 0),
|
"file_size": model_data.get("size", 0),
|
||||||
"modified": checkpoint_data.get("modified", ""),
|
"modified": model_data.get("modified", ""),
|
||||||
"tags": checkpoint_data.get("tags", []),
|
"tags": model_data.get("tags", []),
|
||||||
"from_civitai": checkpoint_data.get("from_civitai", True),
|
"from_civitai": model_data.get("from_civitai", True),
|
||||||
"usage_count": checkpoint_data.get("usage_count", 0),
|
"usage_count": model_data.get("usage_count", 0),
|
||||||
"notes": checkpoint_data.get("notes", ""),
|
"notes": model_data.get("notes", ""),
|
||||||
"sub_type": sub_type,
|
"sub_type": sub_type,
|
||||||
"favorite": checkpoint_data.get("favorite", False),
|
"favorite": model_data.get("favorite", False),
|
||||||
"exclude": bool(checkpoint_data.get("exclude", False)),
|
"exclude": bool(model_data.get("exclude", False)),
|
||||||
"update_available": bool(checkpoint_data.get("update_available", False)),
|
"update_available": bool(model_data.get("update_available", False)),
|
||||||
"skip_metadata_refresh": bool(checkpoint_data.get("skip_metadata_refresh", False)),
|
"skip_metadata_refresh": bool(model_data.get("skip_metadata_refresh", False)),
|
||||||
"civitai": self.filter_civitai_data(checkpoint_data.get("civitai", {}), minimal=True),
|
"civitai": self.filter_civitai_data(model_data.get("civitai", {}), minimal=True),
|
||||||
"auto_tags": checkpoint_data.get("auto_tags") or extract_auto_tags(checkpoint_data),
|
"auto_tags": model_data.get("auto_tags") or extract_auto_tags(model_data),
|
||||||
"version_count": checkpoint_data.get("version_count"),
|
"version_count": model_data.get("version_count"),
|
||||||
"hf_url": checkpoint_data.get("hf_url", ""),
|
"hf_url": model_data.get("hf_url", ""),
|
||||||
}
|
}
|
||||||
|
|
||||||
def find_duplicate_hashes(self) -> Dict:
|
def find_duplicate_hashes(self) -> Dict[str, Any]:
|
||||||
"""Find Checkpoints with duplicate SHA256 hashes"""
|
"""Find Checkpoints with duplicate SHA256 hashes"""
|
||||||
return self.scanner._hash_index.get_duplicate_hashes()
|
return self.scanner._hash_index.get_duplicate_hashes()
|
||||||
|
|
||||||
def find_duplicate_filenames(self) -> Dict:
|
def find_duplicate_filenames(self) -> Dict[str, Any]:
|
||||||
"""Find Checkpoints with conflicting filenames"""
|
"""Find Checkpoints with conflicting filenames"""
|
||||||
return self.scanner._hash_index.get_duplicate_filenames()
|
return self.scanner._hash_index.get_duplicate_filenames()
|
||||||
|
|||||||
@@ -1,8 +1,13 @@
|
|||||||
|
# pyright: reportImportCycles=false
|
||||||
|
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||||
|
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||||
|
# import cycles. Breaking them would require an architectural refactor.
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import asyncio
|
import asyncio
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
from typing import Optional, Dict, Tuple, List
|
from typing import Any, Optional, Dict, Tuple, List, cast
|
||||||
|
from .connectivity_guard import is_expected_offline_error
|
||||||
from .model_metadata_provider import CivArchiveModelMetadataProvider, ModelMetadataProviderManager
|
from .model_metadata_provider import CivArchiveModelMetadataProvider, ModelMetadataProviderManager
|
||||||
from .downloader import get_downloader
|
from .downloader import get_downloader
|
||||||
from .errors import RateLimitError
|
from .errors import RateLimitError
|
||||||
@@ -37,12 +42,16 @@ class CivArchiveClient:
|
|||||||
async def _request_json(
|
async def _request_json(
|
||||||
self,
|
self,
|
||||||
path: str,
|
path: str,
|
||||||
params: Optional[Dict[str, str]] = None
|
params: Optional[Dict[str, Any]] = None
|
||||||
) -> Tuple[Optional[Dict], Optional[str]]:
|
) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||||
"""Call CivArchive API and return JSON payload"""
|
"""Call CivArchive API and return JSON payload"""
|
||||||
success, payload = await self._make_request(path, params=params)
|
success, payload = await self._make_request(path, params=params)
|
||||||
if not success:
|
if not success:
|
||||||
error = payload if isinstance(payload, str) else "Request failed"
|
# Normalize empty-string failure payloads (e.g. a throttled
|
||||||
|
# connection dropped without a message) so callers never see a
|
||||||
|
# falsy error alongside a None payload — that combination used to
|
||||||
|
# crash downstream None.get() calls.
|
||||||
|
error = payload if isinstance(payload, str) and payload else "Request failed"
|
||||||
return None, error
|
return None, error
|
||||||
if not isinstance(payload, dict):
|
if not isinstance(payload, dict):
|
||||||
return None, "Invalid response structure"
|
return None, "Invalid response structure"
|
||||||
@@ -52,12 +61,12 @@ class CivArchiveClient:
|
|||||||
self,
|
self,
|
||||||
path: str,
|
path: str,
|
||||||
*,
|
*,
|
||||||
params: Optional[Dict[str, str]] = None,
|
params: Optional[Dict[str, Any]] = None,
|
||||||
) -> Tuple[bool, Dict | str]:
|
) -> Tuple[bool, Dict[str, Any] | str]:
|
||||||
"""Wrapper around downloader.make_request that surfaces rate limits."""
|
"""Wrapper around downloader.make_request that surfaces rate limits."""
|
||||||
|
|
||||||
downloader = await get_downloader()
|
downloader = await get_downloader()
|
||||||
kwargs: Dict[str, Dict[str, str]] = {}
|
kwargs: Dict[str, Dict[str, Any]] = {}
|
||||||
if params:
|
if params:
|
||||||
safe_params = {str(key): str(value) for key, value in params.items() if value is not None}
|
safe_params = {str(key): str(value) for key, value in params.items() if value is not None}
|
||||||
if safe_params:
|
if safe_params:
|
||||||
@@ -73,10 +82,11 @@ class CivArchiveClient:
|
|||||||
if payload.provider is None:
|
if payload.provider is None:
|
||||||
payload.provider = "civarchive_api"
|
payload.provider = "civarchive_api"
|
||||||
raise payload
|
raise payload
|
||||||
return success, payload
|
# RateLimitError is always raised above, so the returned payload is a dict or str.
|
||||||
|
return success, cast(Dict[str, Any] | str, payload)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _normalize_payload(payload: Dict) -> Dict:
|
def _normalize_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
"""Unwrap CivArchive responses that wrap content under a data key"""
|
"""Unwrap CivArchive responses that wrap content under a data key"""
|
||||||
if not isinstance(payload, dict):
|
if not isinstance(payload, dict):
|
||||||
return {}
|
return {}
|
||||||
@@ -86,12 +96,12 @@ class CivArchiveClient:
|
|||||||
return payload
|
return payload
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _split_context(payload: Dict) -> Tuple[Dict, Dict, List[Dict]]:
|
def _split_context(payload: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any], List[Dict[str, Any]]]:
|
||||||
"""Separate version payload from surrounding model context"""
|
"""Separate version payload from surrounding model context"""
|
||||||
data = CivArchiveClient._normalize_payload(payload)
|
data = CivArchiveClient._normalize_payload(payload)
|
||||||
context: Dict = {}
|
context: Dict[str, Any] = {}
|
||||||
fallback_files: List[Dict] = []
|
fallback_files: List[Dict[str, Any]] = []
|
||||||
version: Dict = {}
|
version: Dict[str, Any] = {}
|
||||||
|
|
||||||
for key, value in data.items():
|
for key, value in data.items():
|
||||||
if key in {"version", "model"}:
|
if key in {"version", "model"}:
|
||||||
@@ -115,7 +125,7 @@ class CivArchiveClient:
|
|||||||
return context, version, fallback_files
|
return context, version, fallback_files
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _ensure_list(value) -> List:
|
def _ensure_list(value: Any) -> List[Any]:
|
||||||
if isinstance(value, list):
|
if isinstance(value, list):
|
||||||
return value
|
return value
|
||||||
if value is None:
|
if value is None:
|
||||||
@@ -123,7 +133,7 @@ class CivArchiveClient:
|
|||||||
return [value]
|
return [value]
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _build_model_info(context: Dict) -> Dict:
|
def _build_model_info(context: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
tags = context.get("tags")
|
tags = context.get("tags")
|
||||||
if not isinstance(tags, list):
|
if not isinstance(tags, list):
|
||||||
tags = list(tags) if isinstance(tags, (set, tuple)) else ([] if tags is None else [tags])
|
tags = list(tags) if isinstance(tags, (set, tuple)) else ([] if tags is None else [tags])
|
||||||
@@ -136,7 +146,7 @@ class CivArchiveClient:
|
|||||||
}
|
}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _build_creator_info(context: Dict) -> Dict:
|
def _build_creator_info(context: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
username = context.get("creator_username") or context.get("username") or ""
|
username = context.get("creator_username") or context.get("username") or ""
|
||||||
image = context.get("creator_image") or context.get("creator_avatar") or ""
|
image = context.get("creator_image") or context.get("creator_avatar") or ""
|
||||||
creator: Dict[str, Optional[str]] = {
|
creator: Dict[str, Optional[str]] = {
|
||||||
@@ -150,7 +160,7 @@ class CivArchiveClient:
|
|||||||
return creator
|
return creator
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _transform_file_entry(file_data: Dict) -> Dict:
|
def _transform_file_entry(file_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
mirrors = file_data.get("mirrors") or []
|
mirrors = file_data.get("mirrors") or []
|
||||||
if not isinstance(mirrors, list):
|
if not isinstance(mirrors, list):
|
||||||
mirrors = [mirrors]
|
mirrors = [mirrors]
|
||||||
@@ -165,7 +175,7 @@ class CivArchiveClient:
|
|||||||
if not name and available_mirror:
|
if not name and available_mirror:
|
||||||
name = available_mirror.get("filename")
|
name = available_mirror.get("filename")
|
||||||
|
|
||||||
transformed: Dict = {
|
transformed: Dict[str, Any] = {
|
||||||
"id": file_data.get("id"),
|
"id": file_data.get("id"),
|
||||||
"sizeKB": file_data.get("sizeKB"),
|
"sizeKB": file_data.get("sizeKB"),
|
||||||
"name": name,
|
"name": name,
|
||||||
@@ -216,23 +226,23 @@ class CivArchiveClient:
|
|||||||
|
|
||||||
def _transform_files(
|
def _transform_files(
|
||||||
self,
|
self,
|
||||||
files: Optional[List[Dict]],
|
files: Optional[List[Dict[str, Any]]],
|
||||||
fallback_files: Optional[List[Dict]] = None
|
fallback_files: Optional[List[Dict[str, Any]]] = None
|
||||||
) -> List[Dict]:
|
) -> List[Dict[str, Any]]:
|
||||||
candidates: List[Dict] = []
|
candidates: List[Dict[str, Any]] = []
|
||||||
if isinstance(files, list) and files:
|
if isinstance(files, list) and files:
|
||||||
candidates = files
|
candidates = files
|
||||||
elif isinstance(fallback_files, list):
|
elif isinstance(fallback_files, list):
|
||||||
candidates = fallback_files
|
candidates = fallback_files
|
||||||
|
|
||||||
transformed_files: List[Dict] = []
|
transformed_files: List[Dict[str, Any]] = []
|
||||||
for file_data in candidates:
|
for file_data in candidates:
|
||||||
if isinstance(file_data, dict):
|
if isinstance(file_data, dict):
|
||||||
transformed_files.append(self._transform_file_entry(file_data))
|
transformed_files.append(self._transform_file_entry(file_data))
|
||||||
|
|
||||||
# Sort: .safetensors first, .ckpt second, others last
|
# Sort: .safetensors first, .ckpt second, others last
|
||||||
# so the backend fallback (no file_params) prefers safetensors
|
# so the backend fallback (no file_params) prefers safetensors
|
||||||
def _sort_key(f: Dict) -> int:
|
def _sort_key(f: Dict[str, Any]) -> int:
|
||||||
fname = f.get("name") or ""
|
fname = f.get("name") or ""
|
||||||
if isinstance(fname, str):
|
if isinstance(fname, str):
|
||||||
lower = fname.lower()
|
lower = fname.lower()
|
||||||
@@ -247,10 +257,10 @@ class CivArchiveClient:
|
|||||||
|
|
||||||
def _transform_version(
|
def _transform_version(
|
||||||
self,
|
self,
|
||||||
context: Dict,
|
context: Dict[str, Any],
|
||||||
version: Dict,
|
version: Dict[str, Any],
|
||||||
fallback_files: Optional[List[Dict]] = None
|
fallback_files: Optional[List[Dict[str, Any]]] = None
|
||||||
) -> Optional[Dict]:
|
) -> Optional[Dict[str, Any]]:
|
||||||
if not version:
|
if not version:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -291,8 +301,10 @@ class CivArchiveClient:
|
|||||||
|
|
||||||
return version_copy
|
return version_copy
|
||||||
|
|
||||||
async def _resolve_version_from_files(self, payload: Dict) -> Optional[Dict]:
|
async def _resolve_version_from_files(self, payload: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||||
"""Fallback to fetch version data when only file metadata is available"""
|
"""Fallback to fetch version data when only file metadata is available"""
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
return None
|
||||||
data = self._normalize_payload(payload)
|
data = self._normalize_payload(payload)
|
||||||
files = data.get("files") or payload.get("files") or []
|
files = data.get("files") or payload.get("files") or []
|
||||||
if not isinstance(files, list):
|
if not isinstance(files, list):
|
||||||
@@ -323,21 +335,24 @@ class CivArchiveClient:
|
|||||||
return resolved
|
return resolved
|
||||||
return None
|
return None
|
||||||
|
|
||||||
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict], Optional[str]]:
|
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||||
"""Find model by SHA256 hash value using CivArchive API"""
|
"""Find model by SHA256 hash value using CivArchive API"""
|
||||||
try:
|
try:
|
||||||
payload, error = await self._request_json(f"/sha256/{model_hash.lower()}")
|
payload, error = await self._request_json(f"/sha256/{model_hash.lower()}")
|
||||||
if error:
|
# Treat a missing payload as an error even when the error string is
|
||||||
if "not found" in error.lower():
|
# falsy; passing None into the split/transform helpers below used to
|
||||||
|
# crash with "'NoneType' object has no attribute 'get'".
|
||||||
|
if error is not None or payload is None:
|
||||||
|
if error and "not found" in error.lower():
|
||||||
return None, "Model not found"
|
return None, "Model not found"
|
||||||
return None, error
|
return None, error or "Request failed"
|
||||||
|
|
||||||
context, version_data, fallback_files = self._split_context(payload)
|
context, version_data, fallback_files = self._split_context(cast(Dict[str, Any], payload))
|
||||||
transformed = self._transform_version(context, version_data, fallback_files)
|
transformed = self._transform_version(context, version_data, fallback_files)
|
||||||
if transformed:
|
if transformed:
|
||||||
return transformed, None
|
return transformed, None
|
||||||
|
|
||||||
resolved = await self._resolve_version_from_files(payload)
|
resolved = await self._resolve_version_from_files(cast(Dict[str, Any], payload))
|
||||||
if resolved:
|
if resolved:
|
||||||
return resolved, None
|
return resolved, None
|
||||||
|
|
||||||
@@ -347,24 +362,38 @@ class CivArchiveClient:
|
|||||||
except RateLimitError:
|
except RateLimitError:
|
||||||
raise
|
raise
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error fetching CivArchive model by hash {model_hash[:10]}: {e}")
|
if is_expected_offline_error(str(e)):
|
||||||
|
logger.debug(
|
||||||
|
"Skipping CivArchive model by hash %s while offline: %s",
|
||||||
|
model_hash[:10],
|
||||||
|
e,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.error(f"Error fetching CivArchive model by hash {model_hash[:10]}: {e}")
|
||||||
return None, str(e)
|
return None, str(e)
|
||||||
|
|
||||||
async def get_model_versions(self, model_id: str) -> Optional[Dict]:
|
async def get_model_versions(self, model_id: str) -> Optional[Dict[str, Any]]:
|
||||||
"""Get all versions of a model using CivArchive API"""
|
"""Get all versions of a model using CivArchive API"""
|
||||||
try:
|
try:
|
||||||
payload, error = await self._request_json(f"/models/{model_id}")
|
payload, error = await self._request_json(f"/models/{model_id}")
|
||||||
if error or payload is None:
|
if error or payload is None:
|
||||||
if error and "not found" in error.lower():
|
if error and "not found" in error.lower():
|
||||||
return None
|
return None
|
||||||
logger.error(f"Error fetching CivArchive model versions for {model_id}: {error}")
|
if is_expected_offline_error(error):
|
||||||
|
logger.debug(
|
||||||
|
"Skipping CivArchive model versions fetch for %s while offline: %s",
|
||||||
|
model_id,
|
||||||
|
error,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.error(f"Error fetching CivArchive model versions for {model_id}: {error}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
data = self._normalize_payload(payload)
|
data = self._normalize_payload(payload)
|
||||||
context, version_data, fallback_files = self._split_context(payload)
|
context, version_data, fallback_files = self._split_context(payload)
|
||||||
|
|
||||||
versions_meta = data.get("versions") or []
|
versions_meta = data.get("versions") or []
|
||||||
transformed_versions: List[Dict] = []
|
transformed_versions: List[Dict[str, Any]] = []
|
||||||
for meta in versions_meta:
|
for meta in versions_meta:
|
||||||
if not isinstance(meta, dict):
|
if not isinstance(meta, dict):
|
||||||
continue
|
continue
|
||||||
@@ -381,7 +410,7 @@ class CivArchiveClient:
|
|||||||
if primary_version:
|
if primary_version:
|
||||||
transformed_versions.insert(0, primary_version)
|
transformed_versions.insert(0, primary_version)
|
||||||
|
|
||||||
ordered_versions: List[Dict] = []
|
ordered_versions: List[Dict[str, Any]] = []
|
||||||
seen_ids = set()
|
seen_ids = set()
|
||||||
for version in transformed_versions:
|
for version in transformed_versions:
|
||||||
version_id = version.get("id")
|
version_id = version.get("id")
|
||||||
@@ -402,7 +431,7 @@ class CivArchiveClient:
|
|||||||
logger.error(f"Error fetching CivArchive model versions for {model_id}: {e}")
|
logger.error(f"Error fetching CivArchive model versions for {model_id}: {e}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
async def get_model_version(self, model_id: int = None, version_id: int = None) -> Optional[Dict]:
|
async def get_model_version(self, model_id: int | str | None = None, version_id: int | str | None = None) -> Optional[Dict[str, Any]]:
|
||||||
"""Get specific model version using CivArchive API
|
"""Get specific model version using CivArchive API
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -421,7 +450,19 @@ class CivArchiveClient:
|
|||||||
if error or payload is None:
|
if error or payload is None:
|
||||||
if error and "not found" in error.lower():
|
if error and "not found" in error.lower():
|
||||||
return None
|
return None
|
||||||
logger.error(f"Error fetching CivArchive model version via API {model_id}/{version_id}: {error}")
|
# The connectivity guard short-circuits requests during its
|
||||||
|
# offline cooldown; that is an expected, transient state, so
|
||||||
|
# log it as DEBUG instead of spamming one ERROR per request
|
||||||
|
# (batch imports can hit this thousands of times).
|
||||||
|
if is_expected_offline_error(error):
|
||||||
|
logger.debug(
|
||||||
|
"Skipping CivArchive model version fetch %s/%s while offline: %s",
|
||||||
|
model_id,
|
||||||
|
version_id,
|
||||||
|
error,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.error(f"Error fetching CivArchive model version via API {model_id}/{version_id}: {error}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
context, version_data, fallback_files = self._split_context(payload)
|
context, version_data, fallback_files = self._split_context(payload)
|
||||||
@@ -459,7 +500,7 @@ class CivArchiveClient:
|
|||||||
logger.error(f"Error fetching CivArchive model version via API {model_id}/{version_id}: {e}")
|
logger.error(f"Error fetching CivArchive model version via API {model_id}/{version_id}: {e}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict], Optional[str]]:
|
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||||
""" Fetch model version metadata using a known bogus model lookup
|
""" Fetch model version metadata using a known bogus model lookup
|
||||||
CivArchive lacks a direct version lookup API, this uses a workaround (which we handle in the main model request now)
|
CivArchive lacks a direct version lookup API, this uses a workaround (which we handle in the main model request now)
|
||||||
|
|
||||||
|
|||||||
@@ -283,7 +283,7 @@ class CivitaiBaseModelService:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
if isinstance(result, str):
|
if isinstance(result, str):
|
||||||
data = json.loads(result)
|
data: Any = json.loads(result)
|
||||||
else:
|
else:
|
||||||
data = result
|
data = result
|
||||||
|
|
||||||
|
|||||||
+144
-42
@@ -1,9 +1,14 @@
|
|||||||
|
# pyright: reportImportCycles=false
|
||||||
|
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||||
|
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||||
|
# import cycles. Breaking them would require an architectural refactor.
|
||||||
import asyncio
|
import asyncio
|
||||||
import copy
|
import copy
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import time
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
from typing import Any, Optional, Dict, Tuple, List, Sequence
|
from typing import Any, Optional, Dict, Tuple, List, Sequence, cast
|
||||||
from .connectivity_guard import (
|
from .connectivity_guard import (
|
||||||
OFFLINE_FRIENDLY_MESSAGE,
|
OFFLINE_FRIENDLY_MESSAGE,
|
||||||
is_expected_offline_error,
|
is_expected_offline_error,
|
||||||
@@ -16,9 +21,16 @@ from .model_metadata_provider import (
|
|||||||
from .downloader import get_downloader
|
from .downloader import get_downloader
|
||||||
from .errors import RateLimitError, ResourceNotFoundError
|
from .errors import RateLimitError, ResourceNotFoundError
|
||||||
from ..utils.civitai_utils import resolve_license_payload
|
from ..utils.civitai_utils import resolve_license_payload
|
||||||
|
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES, is_empty_placeholder_hash
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Best-effort cache for creator model counts, keyed by lowercase username.
|
||||||
|
# Values are (monotonic timestamp, count or None); None results are cached
|
||||||
|
# too so repeated failures don't hammer the API.
|
||||||
|
_CREATOR_COUNT_CACHE_TTL_SECONDS = 600
|
||||||
|
_creator_model_count_cache: Dict[str, Tuple[float, Optional[int]]] = {}
|
||||||
|
|
||||||
|
|
||||||
class CivitaiClient:
|
class CivitaiClient:
|
||||||
_instance = None
|
_instance = None
|
||||||
@@ -51,7 +63,7 @@ class CivitaiClient:
|
|||||||
# Uses OrderedDict with LRU eviction at MAX_CACHE_ENTRIES to prevent
|
# Uses OrderedDict with LRU eviction at MAX_CACHE_ENTRIES to prevent
|
||||||
# unbounded growth in long-running server processes.
|
# unbounded growth in long-running server processes.
|
||||||
self._version_info_cache: OrderedDict[
|
self._version_info_cache: OrderedDict[
|
||||||
str, Tuple[Optional[Dict], Optional[str]]
|
str, Tuple[Optional[Dict[str, Any]], Optional[str]]
|
||||||
] = OrderedDict()
|
] = OrderedDict()
|
||||||
self._MAX_CACHE_ENTRIES = 500
|
self._MAX_CACHE_ENTRIES = 500
|
||||||
|
|
||||||
@@ -65,7 +77,7 @@ class CivitaiClient:
|
|||||||
*,
|
*,
|
||||||
use_auth: bool = False,
|
use_auth: bool = False,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Tuple[bool, Dict | str]:
|
) -> Tuple[bool, Dict[str, Any] | str]:
|
||||||
"""Wrapper around downloader.make_request that surfaces rate limits,
|
"""Wrapper around downloader.make_request that surfaces rate limits,
|
||||||
with retry for transient server errors (5xx, Cloudflare 524, network flakiness)."""
|
with retry for transient server errors (5xx, Cloudflare 524, network flakiness)."""
|
||||||
|
|
||||||
@@ -79,7 +91,8 @@ class CivitaiClient:
|
|||||||
**kwargs,
|
**kwargs,
|
||||||
)
|
)
|
||||||
if success:
|
if success:
|
||||||
return True, result
|
# RateLimitError is raised below; a successful result is dict or str.
|
||||||
|
return True, cast(Dict[str, Any] | str, result)
|
||||||
|
|
||||||
if isinstance(result, RateLimitError):
|
if isinstance(result, RateLimitError):
|
||||||
if result.provider is None:
|
if result.provider is None:
|
||||||
@@ -119,7 +132,7 @@ class CivitaiClient:
|
|||||||
return False, "Unexpected error in _make_request"
|
return False, "Unexpected error in _make_request"
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _remove_comfy_metadata(model_version: Optional[Dict]) -> None:
|
def _remove_comfy_metadata(model_version: Optional[Dict[str, Any]]) -> None:
|
||||||
"""Remove Comfy-specific metadata from model version images."""
|
"""Remove Comfy-specific metadata from model version images."""
|
||||||
if not isinstance(model_version, dict):
|
if not isinstance(model_version, dict):
|
||||||
return
|
return
|
||||||
@@ -166,7 +179,12 @@ class CivitaiClient:
|
|||||||
|
|
||||||
async def get_model_by_hash(
|
async def get_model_by_hash(
|
||||||
self, model_hash: str
|
self, model_hash: str
|
||||||
) -> Tuple[Optional[Dict], Optional[str]]:
|
) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||||
|
if is_empty_placeholder_hash(model_hash):
|
||||||
|
# The empty-hash placeholder (SHA256 of an empty byte string)
|
||||||
|
# matches no real file; CivitAI's by-hash index can contain
|
||||||
|
# polluted entries for it, so never resolve it.
|
||||||
|
return None, "Model not found"
|
||||||
try:
|
try:
|
||||||
success, version = await self._make_request(
|
success, version = await self._make_request(
|
||||||
"GET",
|
"GET",
|
||||||
@@ -213,7 +231,7 @@ class CivitaiClient:
|
|||||||
# Ensure directory exists
|
# Ensure directory exists
|
||||||
os.makedirs(os.path.dirname(save_path), exist_ok=True)
|
os.makedirs(os.path.dirname(save_path), exist_ok=True)
|
||||||
with open(save_path, "wb") as f:
|
with open(save_path, "wb") as f:
|
||||||
f.write(content)
|
f.write(content if isinstance(content, bytes) else content.encode("utf-8"))
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -268,7 +286,7 @@ class CivitaiClient:
|
|||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
async def get_model_versions(self, model_id: str) -> Optional[Dict]:
|
async def get_model_versions(self, model_id: str) -> Optional[Dict[str, Any]]:
|
||||||
"""Get all versions of a model with local availability info"""
|
"""Get all versions of a model with local availability info"""
|
||||||
try:
|
try:
|
||||||
success, result = await self._make_request(
|
success, result = await self._make_request(
|
||||||
@@ -276,7 +294,7 @@ class CivitaiClient:
|
|||||||
f"{self.base_url}/models/{model_id}",
|
f"{self.base_url}/models/{model_id}",
|
||||||
use_auth=True,
|
use_auth=True,
|
||||||
)
|
)
|
||||||
if success:
|
if success and isinstance(result, dict):
|
||||||
# Also return model type along with versions
|
# Also return model type along with versions
|
||||||
return {
|
return {
|
||||||
"modelVersions": result.get("modelVersions", []),
|
"modelVersions": result.get("modelVersions", []),
|
||||||
@@ -310,7 +328,7 @@ class CivitaiClient:
|
|||||||
|
|
||||||
async def get_model_versions_bulk(
|
async def get_model_versions_bulk(
|
||||||
self, model_ids: Sequence[int]
|
self, model_ids: Sequence[int]
|
||||||
) -> Optional[Dict[int, Dict]]:
|
) -> Optional[Dict[int, Dict[str, Any]]]:
|
||||||
"""Fetch model metadata for multiple ids using the batch API."""
|
"""Fetch model metadata for multiple ids using the batch API."""
|
||||||
|
|
||||||
deduped: Dict[int, None] = {}
|
deduped: Dict[int, None] = {}
|
||||||
@@ -340,13 +358,13 @@ class CivitaiClient:
|
|||||||
if not isinstance(items, list):
|
if not isinstance(items, list):
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
payload: Dict[int, Dict] = {}
|
payload: Dict[int, Dict[str, Any]] = {}
|
||||||
for item in items:
|
for item in items:
|
||||||
if not isinstance(item, dict):
|
if not isinstance(item, dict):
|
||||||
continue
|
continue
|
||||||
model_id = item.get("id")
|
model_id = item.get("id")
|
||||||
try:
|
try:
|
||||||
normalized_id = int(model_id)
|
normalized_id = int(cast(Any, model_id))
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
continue
|
continue
|
||||||
payload[normalized_id] = {
|
payload[normalized_id] = {
|
||||||
@@ -366,8 +384,8 @@ class CivitaiClient:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
async def get_model_version(
|
async def get_model_version(
|
||||||
self, model_id: int = None, version_id: int = None
|
self, model_id: int | None = None, version_id: int | None = None
|
||||||
) -> Optional[Dict]:
|
) -> Optional[Dict[str, Any]]:
|
||||||
"""Get specific model version with additional metadata."""
|
"""Get specific model version with additional metadata."""
|
||||||
try:
|
try:
|
||||||
if model_id is None and version_id is not None:
|
if model_id is None and version_id is not None:
|
||||||
@@ -385,7 +403,7 @@ class CivitaiClient:
|
|||||||
logger.error(f"Error fetching model version: {e}")
|
logger.error(f"Error fetching model version: {e}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
async def _get_version_by_id_only(self, version_id: int) -> Optional[Dict]:
|
async def _get_version_by_id_only(self, version_id: int) -> Optional[Dict[str, Any]]:
|
||||||
version = await self._fetch_version_by_id(version_id)
|
version = await self._fetch_version_by_id(version_id)
|
||||||
if version is None:
|
if version is None:
|
||||||
return None
|
return None
|
||||||
@@ -404,7 +422,7 @@ class CivitaiClient:
|
|||||||
|
|
||||||
async def _get_version_with_model_id(
|
async def _get_version_with_model_id(
|
||||||
self, model_id: int, version_id: Optional[int]
|
self, model_id: int, version_id: Optional[int]
|
||||||
) -> Optional[Dict]:
|
) -> Optional[Dict[str, Any]]:
|
||||||
model_data = await self._fetch_model_data(model_id)
|
model_data = await self._fetch_model_data(model_id)
|
||||||
if not model_data:
|
if not model_data:
|
||||||
return None
|
return None
|
||||||
@@ -457,20 +475,20 @@ class CivitaiClient:
|
|||||||
self._remove_comfy_metadata(version)
|
self._remove_comfy_metadata(version)
|
||||||
return version
|
return version
|
||||||
|
|
||||||
async def _fetch_model_data(self, model_id: int) -> Optional[Dict]:
|
async def _fetch_model_data(self, model_id: int) -> Optional[Dict[str, Any]]:
|
||||||
success, data = await self._make_request(
|
success, data = await self._make_request(
|
||||||
"GET",
|
"GET",
|
||||||
f"{self.base_url}/models/{model_id}",
|
f"{self.base_url}/models/{model_id}",
|
||||||
use_auth=True,
|
use_auth=True,
|
||||||
)
|
)
|
||||||
if success:
|
if success and isinstance(data, dict):
|
||||||
return data
|
return data
|
||||||
if is_expected_offline_error(data):
|
if is_expected_offline_error(data):
|
||||||
return None
|
return None
|
||||||
logger.warning(f"Failed to fetch model data for model {model_id}")
|
logger.warning(f"Failed to fetch model data for model {model_id}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
async def _fetch_version_by_id(self, version_id: Optional[int]) -> Optional[Dict]:
|
async def _fetch_version_by_id(self, version_id: Optional[int]) -> Optional[Dict[str, Any]]:
|
||||||
if version_id is None:
|
if version_id is None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -479,7 +497,7 @@ class CivitaiClient:
|
|||||||
f"{self.base_url}/model-versions/{version_id}",
|
f"{self.base_url}/model-versions/{version_id}",
|
||||||
use_auth=True,
|
use_auth=True,
|
||||||
)
|
)
|
||||||
if success:
|
if success and isinstance(version, dict):
|
||||||
return version
|
return version
|
||||||
if is_expected_offline_error(version):
|
if is_expected_offline_error(version):
|
||||||
return None
|
return None
|
||||||
@@ -487,16 +505,18 @@ class CivitaiClient:
|
|||||||
logger.warning(f"Failed to fetch version by id {version_id}")
|
logger.warning(f"Failed to fetch version by id {version_id}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
async def _fetch_version_by_hash(self, model_hash: Optional[str]) -> Optional[Dict]:
|
async def _fetch_version_by_hash(self, model_hash: Optional[str]) -> Optional[Dict[str, Any]]:
|
||||||
if not model_hash:
|
if not model_hash:
|
||||||
return None
|
return None
|
||||||
|
if is_empty_placeholder_hash(model_hash):
|
||||||
|
return None
|
||||||
|
|
||||||
success, version = await self._make_request(
|
success, version = await self._make_request(
|
||||||
"GET",
|
"GET",
|
||||||
f"{self.base_url}/model-versions/by-hash/{model_hash}",
|
f"{self.base_url}/model-versions/by-hash/{model_hash}",
|
||||||
use_auth=True,
|
use_auth=True,
|
||||||
)
|
)
|
||||||
if success:
|
if success and isinstance(version, dict):
|
||||||
return version
|
return version
|
||||||
if is_expected_offline_error(version):
|
if is_expected_offline_error(version):
|
||||||
return None
|
return None
|
||||||
@@ -505,8 +525,8 @@ class CivitaiClient:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
def _select_target_version(
|
def _select_target_version(
|
||||||
self, model_data: Dict, model_id: int, version_id: Optional[int]
|
self, model_data: Dict[str, Any], model_id: int, version_id: Optional[int]
|
||||||
) -> Optional[Dict]:
|
) -> Optional[Dict[str, Any]]:
|
||||||
model_versions = model_data.get("modelVersions", [])
|
model_versions = model_data.get("modelVersions", [])
|
||||||
if not model_versions:
|
if not model_versions:
|
||||||
logger.warning(f"No model versions found for model {model_id}")
|
logger.warning(f"No model versions found for model {model_id}")
|
||||||
@@ -525,18 +545,24 @@ class CivitaiClient:
|
|||||||
|
|
||||||
return model_versions[0]
|
return model_versions[0]
|
||||||
|
|
||||||
def _extract_primary_model_hash(self, version_entry: Dict) -> Optional[str]:
|
def _extract_primary_model_hash(self, version_entry: Dict[str, Any]) -> Optional[str]:
|
||||||
|
# Prefer the generic "Model" file (most reliable version identity);
|
||||||
|
# fall back to any other weights-type primary.
|
||||||
for file_info in version_entry.get("files", []):
|
for file_info in version_entry.get("files", []):
|
||||||
if file_info.get("type") == "Model" and file_info.get("primary"):
|
if file_info.get("type") == "Model" and file_info.get("primary"):
|
||||||
hashes = file_info.get("hashes", {})
|
model_hash = (file_info.get("hashes", {}) or {}).get("SHA256")
|
||||||
model_hash = hashes.get("SHA256")
|
if model_hash:
|
||||||
|
return model_hash
|
||||||
|
for file_info in version_entry.get("files", []):
|
||||||
|
if file_info.get("type") in MODEL_WEIGHT_FILE_TYPES and file_info.get("primary"):
|
||||||
|
model_hash = (file_info.get("hashes", {}) or {}).get("SHA256")
|
||||||
if model_hash:
|
if model_hash:
|
||||||
return model_hash
|
return model_hash
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _build_version_from_model_data(
|
def _build_version_from_model_data(
|
||||||
self, version_entry: Dict, model_id: int, model_data: Dict
|
self, version_entry: Dict[str, Any], model_id: int, model_data: Dict[str, Any]
|
||||||
) -> Dict:
|
) -> Dict[str, Any]:
|
||||||
version = copy.deepcopy(version_entry)
|
version = copy.deepcopy(version_entry)
|
||||||
version.pop("index", None)
|
version.pop("index", None)
|
||||||
version["modelId"] = model_id
|
version["modelId"] = model_id
|
||||||
@@ -548,7 +574,7 @@ class CivitaiClient:
|
|||||||
}
|
}
|
||||||
return version
|
return version
|
||||||
|
|
||||||
def _enrich_version_with_model_data(self, version: Dict, model_data: Dict) -> None:
|
def _enrich_version_with_model_data(self, version: Dict[str, Any], model_data: Dict[str, Any]) -> None:
|
||||||
model_info = version.get("model")
|
model_info = version.get("model")
|
||||||
if not isinstance(model_info, dict):
|
if not isinstance(model_info, dict):
|
||||||
model_info = {}
|
model_info = {}
|
||||||
@@ -564,7 +590,7 @@ class CivitaiClient:
|
|||||||
|
|
||||||
async def get_model_version_info(
|
async def get_model_version_info(
|
||||||
self, version_id: str
|
self, version_id: str
|
||||||
) -> Tuple[Optional[Dict], Optional[str]]:
|
) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||||
"""Fetch model version metadata from Civitai
|
"""Fetch model version metadata from Civitai
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -589,7 +615,7 @@ class CivitaiClient:
|
|||||||
logger.debug("Resolving Civitai model version info: %s", url)
|
logger.debug("Resolving Civitai model version info: %s", url)
|
||||||
success, result = await self._make_request("GET", url, use_auth=True)
|
success, result = await self._make_request("GET", url, use_auth=True)
|
||||||
|
|
||||||
if success:
|
if success and isinstance(result, dict):
|
||||||
logger.debug("Successfully fetched model version info for: %s", version_id)
|
logger.debug("Successfully fetched model version info for: %s", version_id)
|
||||||
self._remove_comfy_metadata(result)
|
self._remove_comfy_metadata(result)
|
||||||
self._version_info_cache[version_id] = (result, None)
|
self._version_info_cache[version_id] = (result, None)
|
||||||
@@ -619,7 +645,7 @@ class CivitaiClient:
|
|||||||
|
|
||||||
async def get_image_info(
|
async def get_image_info(
|
||||||
self, image_id: str, source_url: str | None = None
|
self, image_id: str, source_url: str | None = None
|
||||||
) -> Optional[Dict]:
|
) -> Optional[Dict[str, Any]]:
|
||||||
"""Fetch image information from Civitai API
|
"""Fetch image information from Civitai API
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -652,7 +678,7 @@ class CivitaiClient:
|
|||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
if result and "items" in result and isinstance(result["items"], list):
|
if isinstance(result, dict) and "items" in result and isinstance(result["items"], list):
|
||||||
items = result["items"]
|
items = result["items"]
|
||||||
|
|
||||||
for item in items:
|
for item in items:
|
||||||
@@ -692,7 +718,7 @@ class CivitaiClient:
|
|||||||
|
|
||||||
async def get_model_versions_by_hashes(
|
async def get_model_versions_by_hashes(
|
||||||
self, hashes: List[str]
|
self, hashes: List[str]
|
||||||
) -> Optional[List[Dict]]:
|
) -> Optional[List[Dict[str, Any]]]:
|
||||||
"""Fetch full version details for up to 100 SHA256 hashes via the batch endpoint.
|
"""Fetch full version details for up to 100 SHA256 hashes via the batch endpoint.
|
||||||
|
|
||||||
Uses POST /api/v1/model-versions/by-hash which returns full version
|
Uses POST /api/v1/model-versions/by-hash which returns full version
|
||||||
@@ -709,7 +735,7 @@ class CivitaiClient:
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
BATCH_SIZE = 100
|
BATCH_SIZE = 100
|
||||||
all_versions: List[Dict] = []
|
all_versions: List[Dict[str, Any]] = []
|
||||||
|
|
||||||
for start in range(0, len(hashes), BATCH_SIZE):
|
for start in range(0, len(hashes), BATCH_SIZE):
|
||||||
batch = hashes[start : start + BATCH_SIZE]
|
batch = hashes[start : start + BATCH_SIZE]
|
||||||
@@ -729,7 +755,7 @@ class CivitaiClient:
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
if isinstance(result, list):
|
if isinstance(result, list):
|
||||||
all_versions.extend(result)
|
all_versions.extend(cast(Any, result))
|
||||||
else:
|
else:
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Unexpected by-hash response type: %s", type(result)
|
"Unexpected by-hash response type: %s", type(result)
|
||||||
@@ -743,17 +769,34 @@ class CivitaiClient:
|
|||||||
|
|
||||||
return all_versions if all_versions else None
|
return all_versions if all_versions else None
|
||||||
|
|
||||||
async def get_user_models(self, username: str) -> Optional[List[Dict]]:
|
async def get_user_models(
|
||||||
"""Fetch all models for a specific Civitai user."""
|
self, username: str, cursor: Optional[str] = None
|
||||||
|
) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Fetch one page (up to 100 models) for a specific Civitai user.
|
||||||
|
|
||||||
|
Returns ``{"items": [...], "nextCursor": <str|None>}`` on success,
|
||||||
|
or None on failure. Pass ``cursor`` (from a previous response's
|
||||||
|
``nextCursor``) to fetch subsequent pages.
|
||||||
|
"""
|
||||||
if not username:
|
if not username:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
params: Dict[str, Any] = {
|
||||||
|
"username": username,
|
||||||
|
"nsfw": "true",
|
||||||
|
"limit": 100,
|
||||||
|
"sort": "Newest",
|
||||||
|
"period": "AllTime",
|
||||||
|
}
|
||||||
|
if cursor:
|
||||||
|
params["cursor"] = cursor
|
||||||
|
|
||||||
try:
|
try:
|
||||||
success, result = await self._make_request(
|
success, result = await self._make_request(
|
||||||
"GET",
|
"GET",
|
||||||
f"{self.base_url}/models",
|
f"{self.base_url}/models",
|
||||||
use_auth=True,
|
use_auth=True,
|
||||||
params={"username": username, "nsfw": "true"},
|
params=params,
|
||||||
)
|
)
|
||||||
|
|
||||||
if not success:
|
if not success:
|
||||||
@@ -765,7 +808,7 @@ class CivitaiClient:
|
|||||||
|
|
||||||
items = result.get("items") if isinstance(result, dict) else None
|
items = result.get("items") if isinstance(result, dict) else None
|
||||||
if not isinstance(items, list):
|
if not isinstance(items, list):
|
||||||
return []
|
items = []
|
||||||
|
|
||||||
for model in items:
|
for model in items:
|
||||||
versions = model.get("modelVersions")
|
versions = model.get("modelVersions")
|
||||||
@@ -774,9 +817,68 @@ class CivitaiClient:
|
|||||||
for version in versions:
|
for version in versions:
|
||||||
self._remove_comfy_metadata(version)
|
self._remove_comfy_metadata(version)
|
||||||
|
|
||||||
return items
|
next_cursor: Optional[str] = None
|
||||||
|
metadata = result.get("metadata") if isinstance(result, dict) else None
|
||||||
|
if isinstance(metadata, dict):
|
||||||
|
raw_cursor = metadata.get("nextCursor")
|
||||||
|
if raw_cursor is not None:
|
||||||
|
next_cursor = str(raw_cursor)
|
||||||
|
|
||||||
|
return {"items": items, "nextCursor": next_cursor}
|
||||||
except RateLimitError:
|
except RateLimitError:
|
||||||
raise
|
raise
|
||||||
except Exception as exc: # pragma: no cover - defensive logging
|
except Exception as exc: # pragma: no cover - defensive logging
|
||||||
logger.error("Error fetching models for %s: %s", username, exc)
|
logger.error("Error fetching models for %s: %s", username, exc)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
async def get_creator_model_count(self, username: str) -> Optional[int]:
|
||||||
|
"""Best-effort lookup of a creator's published model count.
|
||||||
|
|
||||||
|
Uses the ``/creators`` endpoint (a contains-match query), picking the
|
||||||
|
entry whose username matches exactly (case-insensitive). Returns None
|
||||||
|
on any failure; never raises. Results (including None) are cached
|
||||||
|
for ``_CREATOR_COUNT_CACHE_TTL_SECONDS``.
|
||||||
|
"""
|
||||||
|
if not username:
|
||||||
|
return None
|
||||||
|
|
||||||
|
cache_key = username.lower()
|
||||||
|
cached = _creator_model_count_cache.get(cache_key)
|
||||||
|
if cached is not None:
|
||||||
|
cached_at, cached_count = cached
|
||||||
|
if time.monotonic() - cached_at < _CREATOR_COUNT_CACHE_TTL_SECONDS:
|
||||||
|
return cached_count
|
||||||
|
|
||||||
|
count: Optional[int] = None
|
||||||
|
try:
|
||||||
|
success, result = await self._make_request(
|
||||||
|
"GET",
|
||||||
|
f"{self.base_url}/creators",
|
||||||
|
use_auth=True,
|
||||||
|
params={"query": username, "limit": 10},
|
||||||
|
)
|
||||||
|
|
||||||
|
if success and isinstance(result, dict):
|
||||||
|
creators = result.get("items")
|
||||||
|
if isinstance(creators, list):
|
||||||
|
for creator in creators:
|
||||||
|
if not isinstance(creator, dict):
|
||||||
|
continue
|
||||||
|
creator_name = creator.get("username")
|
||||||
|
if not isinstance(creator_name, str):
|
||||||
|
continue
|
||||||
|
if creator_name.lower() != cache_key:
|
||||||
|
continue
|
||||||
|
model_count = creator.get("modelCount")
|
||||||
|
if isinstance(model_count, (int, float)) and not isinstance(
|
||||||
|
model_count, bool
|
||||||
|
):
|
||||||
|
count = int(model_count)
|
||||||
|
break
|
||||||
|
except Exception as exc: # best-effort only, never propagate
|
||||||
|
logger.debug(
|
||||||
|
"Failed to fetch creator model count for %s: %s", username, exc
|
||||||
|
)
|
||||||
|
|
||||||
|
_creator_model_count_cache[cache_key] = (time.monotonic(), count)
|
||||||
|
return count
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
from typing import Any, Awaitable, Callable, Dict, Optional
|
from typing import Any, Awaitable, Callable, Dict, Iterable, Optional
|
||||||
|
|
||||||
from .downloader import DownloadProgress
|
from .downloader import DownloadProgress
|
||||||
|
|
||||||
@@ -18,7 +18,7 @@ class DownloadCoordinator:
|
|||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
ws_manager,
|
ws_manager,
|
||||||
download_manager_factory: Callable[[], Awaitable],
|
download_manager_factory: Callable[[], Awaitable[Any]],
|
||||||
) -> None:
|
) -> None:
|
||||||
self._ws_manager = ws_manager
|
self._ws_manager = ws_manager
|
||||||
self._download_manager_factory = download_manager_factory
|
self._download_manager_factory = download_manager_factory
|
||||||
@@ -83,10 +83,13 @@ class DownloadCoordinator:
|
|||||||
save_dir=payload.get("model_root"),
|
save_dir=payload.get("model_root"),
|
||||||
relative_path=payload.get("relative_path", ""),
|
relative_path=payload.get("relative_path", ""),
|
||||||
use_default_paths=payload.get("use_default_paths", False),
|
use_default_paths=payload.get("use_default_paths", False),
|
||||||
|
use_save_dir_as_root=payload.get("use_save_dir_as_root", False),
|
||||||
progress_callback=progress_callback,
|
progress_callback=progress_callback,
|
||||||
download_id=download_id,
|
download_id=download_id,
|
||||||
source=payload.get("source"),
|
source=payload.get("source"),
|
||||||
file_params=payload.get("file_params"),
|
# Normalize falsy file_params (e.g. {}) to None so download gates
|
||||||
|
# treat it as "no explicit file selection" (#1058).
|
||||||
|
file_params=payload.get("file_params") or None,
|
||||||
)
|
)
|
||||||
|
|
||||||
result["download_id"] = download_id
|
result["download_id"] = download_id
|
||||||
@@ -183,6 +186,14 @@ class DownloadCoordinator:
|
|||||||
download_manager = await self._download_manager_factory()
|
download_manager = await self._download_manager_factory()
|
||||||
return await download_manager.get_active_downloads()
|
return await download_manager.get_active_downloads()
|
||||||
|
|
||||||
|
async def discard_cleared_downloads(self, download_ids: Iterable[str]) -> int:
|
||||||
|
"""Tear down in-memory/aria2 tracking for queue-cleared downloads."""
|
||||||
|
|
||||||
|
if not download_ids:
|
||||||
|
return 0
|
||||||
|
download_manager = await self._download_manager_factory()
|
||||||
|
return await download_manager.discard_cleared_downloads(download_ids)
|
||||||
|
|
||||||
def _parse_optional_int(self, value: Any, field: str) -> Optional[int]:
|
def _parse_optional_int(self, value: Any, field: str) -> Optional[int]:
|
||||||
"""Parse an optional integer from user input."""
|
"""Parse an optional integer from user input."""
|
||||||
|
|
||||||
|
|||||||
+660
-206
File diff suppressed because it is too large
Load Diff
@@ -6,12 +6,21 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import time
|
import time
|
||||||
from typing import Any, Optional
|
from typing import Any, List, Optional
|
||||||
|
|
||||||
from ..utils.cache_paths import get_cache_base_dir
|
from ..utils.cache_paths import get_cache_base_dir
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# SQL fragment extracting the CivitAI file id from the JSON ``file_params``
|
||||||
|
# column (#1058). ``json_valid`` guards against NULL and legacy/unparseable
|
||||||
|
# values, yielding NULL for rows without a file identity; NULL keys group
|
||||||
|
# together so such rows keep the old version-level dedup behavior.
|
||||||
|
_FILE_ID_SQL = (
|
||||||
|
"CASE WHEN json_valid(file_params) "
|
||||||
|
"THEN json_extract(file_params, '$.id') END"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _resolve_database_path() -> str:
|
def _resolve_database_path() -> str:
|
||||||
base_dir = get_cache_base_dir(create=True)
|
base_dir = get_cache_base_dir(create=True)
|
||||||
@@ -64,6 +73,7 @@ class DownloadQueueService:
|
|||||||
model_name TEXT NOT NULL DEFAULT '',
|
model_name TEXT NOT NULL DEFAULT '',
|
||||||
version_name TEXT DEFAULT '',
|
version_name TEXT DEFAULT '',
|
||||||
thumbnail_url TEXT DEFAULT '',
|
thumbnail_url TEXT DEFAULT '',
|
||||||
|
file_params TEXT,
|
||||||
status TEXT NOT NULL,
|
status TEXT NOT NULL,
|
||||||
error TEXT,
|
error TEXT,
|
||||||
file_path TEXT,
|
file_path TEXT,
|
||||||
@@ -120,6 +130,18 @@ class DownloadQueueService:
|
|||||||
with self._connect() as conn:
|
with self._connect() as conn:
|
||||||
conn.executescript(self._SCHEMA_TABLES)
|
conn.executescript(self._SCHEMA_TABLES)
|
||||||
|
|
||||||
|
# Databases created by older versions lack
|
||||||
|
# download_history.file_params; add it so retry-from-history can
|
||||||
|
# restore the originally selected file (#1058).
|
||||||
|
history_columns = {
|
||||||
|
row["name"]
|
||||||
|
for row in conn.execute("PRAGMA table_info(download_history)")
|
||||||
|
}
|
||||||
|
if "file_params" not in history_columns:
|
||||||
|
conn.execute(
|
||||||
|
"ALTER TABLE download_history ADD COLUMN file_params TEXT"
|
||||||
|
)
|
||||||
|
|
||||||
# Creating the unique index on download_history.download_id can
|
# Creating the unique index on download_history.download_id can
|
||||||
# fail if pre-existing rows have duplicate values (e.g. from a
|
# fail if pre-existing rows have duplicate values (e.g. from a
|
||||||
# previous version that lacked the index). Deduplicate first so
|
# previous version that lacked the index). Deduplicate first so
|
||||||
@@ -368,23 +390,31 @@ class DownloadQueueService:
|
|||||||
conn.commit()
|
conn.commit()
|
||||||
return True
|
return True
|
||||||
|
|
||||||
async def clear_queue(self, status_filter: Optional[str] = None) -> int:
|
async def clear_queue(self, status_filter: Optional[str] = None) -> List[str]:
|
||||||
"""Remove items from the queue.
|
"""Remove items from the queue.
|
||||||
|
|
||||||
When *status_filter* is provided only items with that status are
|
When *status_filter* is provided only items with that status are
|
||||||
deleted. Returns the number of deleted rows.
|
deleted. Returns the ``download_id`` values of the deleted rows so
|
||||||
|
callers can also tear down any in-memory tracking for them.
|
||||||
"""
|
"""
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
conn = self._get_conn()
|
conn = self._get_conn()
|
||||||
if status_filter is not None:
|
if status_filter is not None:
|
||||||
cursor = conn.execute(
|
rows = conn.execute(
|
||||||
|
"SELECT download_id FROM download_queue WHERE status = ?",
|
||||||
|
(status_filter,),
|
||||||
|
).fetchall()
|
||||||
|
conn.execute(
|
||||||
"DELETE FROM download_queue WHERE status = ?",
|
"DELETE FROM download_queue WHERE status = ?",
|
||||||
(status_filter,),
|
(status_filter,),
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
cursor = conn.execute("DELETE FROM download_queue")
|
rows = conn.execute(
|
||||||
|
"SELECT download_id FROM download_queue"
|
||||||
|
).fetchall()
|
||||||
|
conn.execute("DELETE FROM download_queue")
|
||||||
conn.commit()
|
conn.commit()
|
||||||
return cursor.rowcount
|
return [row["download_id"] for row in rows]
|
||||||
|
|
||||||
async def complete_download(
|
async def complete_download(
|
||||||
self,
|
self,
|
||||||
@@ -418,6 +448,12 @@ class DownloadQueueService:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
now = completed_at if completed_at is not None else time.time()
|
now = completed_at if completed_at is not None else time.time()
|
||||||
|
# Guard against legacy databases whose download_queue table
|
||||||
|
# predates the file_params column.
|
||||||
|
queue_columns = set(row.keys())
|
||||||
|
file_params_json = (
|
||||||
|
row["file_params"] if "file_params" in queue_columns else None
|
||||||
|
)
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"DELETE FROM download_queue WHERE download_id = ?",
|
"DELETE FROM download_queue WHERE download_id = ?",
|
||||||
(download_id,),
|
(download_id,),
|
||||||
@@ -426,9 +462,9 @@ class DownloadQueueService:
|
|||||||
"""
|
"""
|
||||||
INSERT OR IGNORE INTO download_history (
|
INSERT OR IGNORE INTO download_history (
|
||||||
download_id, model_id, model_version_id, model_name,
|
download_id, model_id, model_version_id, model_name,
|
||||||
version_name, thumbnail_url, status, error, file_path,
|
version_name, thumbnail_url, file_params, status, error,
|
||||||
bytes_downloaded, total_bytes, completed_at
|
file_path, bytes_downloaded, total_bytes, completed_at
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
""",
|
""",
|
||||||
(
|
(
|
||||||
row["download_id"],
|
row["download_id"],
|
||||||
@@ -437,6 +473,7 @@ class DownloadQueueService:
|
|||||||
row["model_name"],
|
row["model_name"],
|
||||||
row["version_name"],
|
row["version_name"],
|
||||||
row["thumbnail_url"],
|
row["thumbnail_url"],
|
||||||
|
file_params_json,
|
||||||
status,
|
status,
|
||||||
error,
|
error,
|
||||||
file_path,
|
file_path,
|
||||||
@@ -503,6 +540,7 @@ class DownloadQueueService:
|
|||||||
bytes_downloaded: int = 0,
|
bytes_downloaded: int = 0,
|
||||||
total_bytes: Optional[int] = None,
|
total_bytes: Optional[int] = None,
|
||||||
is_already_exists: int = 0,
|
is_already_exists: int = 0,
|
||||||
|
file_params: Optional[dict[str, Any]] = None,
|
||||||
) -> int:
|
) -> int:
|
||||||
"""Insert a record into the download history.
|
"""Insert a record into the download history.
|
||||||
|
|
||||||
@@ -510,6 +548,7 @@ class DownloadQueueService:
|
|||||||
inserted row.
|
inserted row.
|
||||||
"""
|
"""
|
||||||
now = time.time()
|
now = time.time()
|
||||||
|
file_params_json = json.dumps(file_params) if file_params is not None else None
|
||||||
|
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
conn = self._get_conn()
|
conn = self._get_conn()
|
||||||
@@ -517,9 +556,10 @@ class DownloadQueueService:
|
|||||||
"""
|
"""
|
||||||
INSERT INTO download_history (
|
INSERT INTO download_history (
|
||||||
download_id, model_id, model_version_id, model_name,
|
download_id, model_id, model_version_id, model_name,
|
||||||
version_name, thumbnail_url, status, error, file_path,
|
version_name, thumbnail_url, file_params, status, error,
|
||||||
bytes_downloaded, total_bytes, completed_at, is_already_exists
|
file_path, bytes_downloaded, total_bytes, completed_at,
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
is_already_exists
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
""",
|
""",
|
||||||
(
|
(
|
||||||
download_id,
|
download_id,
|
||||||
@@ -528,6 +568,7 @@ class DownloadQueueService:
|
|||||||
model_name,
|
model_name,
|
||||||
version_name,
|
version_name,
|
||||||
thumbnail_url,
|
thumbnail_url,
|
||||||
|
file_params_json,
|
||||||
status,
|
status,
|
||||||
error,
|
error,
|
||||||
file_path,
|
file_path,
|
||||||
@@ -702,7 +743,7 @@ class DownloadQueueService:
|
|||||||
download_id, model_id, model_version_id, model_name,
|
download_id, model_id, model_version_id, model_name,
|
||||||
version_name, thumbnail_url, source, file_params,
|
version_name, thumbnail_url, source, file_params,
|
||||||
status, priority, added_at
|
status, priority, added_at
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, NULL, 'queued', 0, ?)
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?)
|
||||||
""",
|
""",
|
||||||
(
|
(
|
||||||
new_id,
|
new_id,
|
||||||
@@ -712,6 +753,7 @@ class DownloadQueueService:
|
|||||||
row["version_name"],
|
row["version_name"],
|
||||||
row["thumbnail_url"],
|
row["thumbnail_url"],
|
||||||
"retry",
|
"retry",
|
||||||
|
row["file_params"],
|
||||||
now,
|
now,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -755,7 +797,7 @@ class DownloadQueueService:
|
|||||||
download_id, model_id, model_version_id, model_name,
|
download_id, model_id, model_version_id, model_name,
|
||||||
version_name, thumbnail_url, source, file_params,
|
version_name, thumbnail_url, source, file_params,
|
||||||
status, priority, added_at
|
status, priority, added_at
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, NULL, 'queued', 0, ?)
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?)
|
||||||
""",
|
""",
|
||||||
(
|
(
|
||||||
new_id,
|
new_id,
|
||||||
@@ -765,6 +807,7 @@ class DownloadQueueService:
|
|||||||
row["version_name"],
|
row["version_name"],
|
||||||
row["thumbnail_url"],
|
row["thumbnail_url"],
|
||||||
"retry",
|
"retry",
|
||||||
|
row["file_params"],
|
||||||
now,
|
now,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -840,33 +883,44 @@ class DownloadQueueService:
|
|||||||
async with self._lock:
|
async with self._lock:
|
||||||
conn = self._get_conn()
|
conn = self._get_conn()
|
||||||
|
|
||||||
# 1. History: for each (model_id, model_version_id, status) triplet
|
# 1. History: for each (model_id, model_version_id, file_id,
|
||||||
# keep only the row with the highest id (most recently inserted).
|
# status) group keep only the row with the highest id (most
|
||||||
conn.execute("""
|
# recently inserted). file_id comes from file_params (#1058)
|
||||||
|
# so distinct files of the same version never collapse.
|
||||||
|
conn.execute(f"""
|
||||||
DELETE FROM download_history
|
DELETE FROM download_history
|
||||||
WHERE id NOT IN (
|
WHERE id NOT IN (
|
||||||
SELECT MAX(id)
|
SELECT MAX(id)
|
||||||
FROM download_history
|
FROM download_history
|
||||||
GROUP BY model_id, model_version_id, status
|
GROUP BY model_id, model_version_id, status,
|
||||||
|
{_FILE_ID_SQL}
|
||||||
)
|
)
|
||||||
""")
|
""")
|
||||||
result["removed_history"] = conn.execute(
|
result["removed_history"] = conn.execute(
|
||||||
"SELECT changes()"
|
"SELECT changes()"
|
||||||
).fetchone()[0]
|
).fetchone()[0]
|
||||||
|
|
||||||
# 2. Cross-status dedup: for each (model_id, model_version_id),
|
# 2. Cross-status dedup: for each (model_id, model_version_id,
|
||||||
# keep only the entry with the highest-priority terminal status.
|
# file_id), keep only the entry with the highest-priority
|
||||||
|
# terminal status.
|
||||||
# Priority: completed (3) > failed (2) > canceled (1).
|
# Priority: completed (3) > failed (2) > canceled (1).
|
||||||
# This prevents the same model version from having both a
|
# This prevents the same file of a model version from having
|
||||||
# 'failed' and a 'canceled' entry (or a 'completed' alongside
|
# both a 'failed' and a 'canceled' entry (or a 'completed'
|
||||||
# either) after the bug-created duplicates are removed.
|
# alongside either) after the bug-created duplicates are
|
||||||
conn.execute("""
|
# removed. ``IS`` matches NULL file ids against each other so
|
||||||
|
# rows without file identity keep the old behavior.
|
||||||
|
conn.execute(f"""
|
||||||
DELETE FROM download_history
|
DELETE FROM download_history
|
||||||
WHERE id NOT IN (
|
WHERE id NOT IN (
|
||||||
SELECT dh.id
|
SELECT dh.id
|
||||||
FROM download_history dh
|
FROM (
|
||||||
|
SELECT id, model_id, model_version_id, status,
|
||||||
|
{_FILE_ID_SQL} AS file_id
|
||||||
|
FROM download_history
|
||||||
|
) dh
|
||||||
INNER JOIN (
|
INNER JOIN (
|
||||||
SELECT model_id, model_version_id,
|
SELECT model_id, model_version_id,
|
||||||
|
{_FILE_ID_SQL} AS file_id,
|
||||||
MAX(CASE status
|
MAX(CASE status
|
||||||
WHEN 'completed' THEN 3
|
WHEN 'completed' THEN 3
|
||||||
WHEN 'failed' THEN 2
|
WHEN 'failed' THEN 2
|
||||||
@@ -874,17 +928,18 @@ class DownloadQueueService:
|
|||||||
ELSE 0
|
ELSE 0
|
||||||
END) AS best_prio
|
END) AS best_prio
|
||||||
FROM download_history
|
FROM download_history
|
||||||
GROUP BY model_id, model_version_id
|
GROUP BY model_id, model_version_id, {_FILE_ID_SQL}
|
||||||
) best
|
) best
|
||||||
ON dh.model_id = best.model_id
|
ON dh.model_id = best.model_id
|
||||||
AND dh.model_version_id = best.model_version_id
|
AND dh.model_version_id = best.model_version_id
|
||||||
|
AND dh.file_id IS best.file_id
|
||||||
AND CASE dh.status
|
AND CASE dh.status
|
||||||
WHEN 'completed' THEN 3
|
WHEN 'completed' THEN 3
|
||||||
WHEN 'failed' THEN 2
|
WHEN 'failed' THEN 2
|
||||||
WHEN 'canceled' THEN 1
|
WHEN 'canceled' THEN 1
|
||||||
ELSE 0
|
ELSE 0
|
||||||
END = best.best_prio
|
END = best.best_prio
|
||||||
GROUP BY dh.model_id, dh.model_version_id
|
GROUP BY dh.model_id, dh.model_version_id, dh.file_id
|
||||||
HAVING dh.id = MAX(dh.id)
|
HAVING dh.id = MAX(dh.id)
|
||||||
)
|
)
|
||||||
""")
|
""")
|
||||||
@@ -892,15 +947,17 @@ class DownloadQueueService:
|
|||||||
"SELECT changes()"
|
"SELECT changes()"
|
||||||
).fetchone()[0]
|
).fetchone()[0]
|
||||||
|
|
||||||
# 3. Queue: for each (model_id, model_version_id) keep only the
|
# 3. Queue: for each (model_id, model_version_id, file_id) keep
|
||||||
# row with the latest added_at (most recently enqueued).
|
# only the row with the latest added_at (most recently
|
||||||
conn.execute("""
|
# enqueued). file_id comes from file_params (#1058) so
|
||||||
|
# distinct files of the same version never collapse.
|
||||||
|
conn.execute(f"""
|
||||||
DELETE FROM download_queue
|
DELETE FROM download_queue
|
||||||
WHERE rowid NOT IN (
|
WHERE rowid NOT IN (
|
||||||
SELECT MAX(rowid)
|
SELECT MAX(rowid)
|
||||||
FROM download_queue
|
FROM download_queue
|
||||||
WHERE status IN ('queued', 'downloading', 'paused', 'waiting')
|
WHERE status IN ('queued', 'downloading', 'paused', 'waiting')
|
||||||
GROUP BY model_id, model_version_id
|
GROUP BY model_id, model_version_id, {_FILE_ID_SQL}
|
||||||
)
|
)
|
||||||
AND status IN ('queued', 'downloading', 'paused', 'waiting')
|
AND status IN ('queued', 'downloading', 'paused', 'waiting')
|
||||||
""")
|
""")
|
||||||
|
|||||||
@@ -1,3 +1,7 @@
|
|||||||
|
# pyright: reportImportCycles=false
|
||||||
|
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||||
|
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||||
|
# import cycles. Breaking them would require an architectural refactor.
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
@@ -58,6 +62,14 @@ class DownloadedVersionHistoryService:
|
|||||||
);
|
);
|
||||||
CREATE INDEX IF NOT EXISTS idx_downloaded_model_versions_model
|
CREATE INDEX IF NOT EXISTS idx_downloaded_model_versions_model
|
||||||
ON downloaded_model_versions(model_type, model_id);
|
ON downloaded_model_versions(model_type, model_id);
|
||||||
|
CREATE TABLE IF NOT EXISTS downloaded_version_files (
|
||||||
|
model_type TEXT NOT NULL,
|
||||||
|
version_id INTEGER NOT NULL,
|
||||||
|
file_id INTEGER NOT NULL,
|
||||||
|
file_name TEXT,
|
||||||
|
downloaded_at REAL NOT NULL,
|
||||||
|
PRIMARY KEY (model_type, version_id, file_id)
|
||||||
|
);
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, db_path: str | None = None, *, settings_manager=None) -> None:
|
def __init__(self, db_path: str | None = None, *, settings_manager=None) -> None:
|
||||||
@@ -127,10 +139,13 @@ class DownloadedVersionHistoryService:
|
|||||||
source: str = "manual",
|
source: str = "manual",
|
||||||
file_path: str | None = None,
|
file_path: str | None = None,
|
||||||
library_name: str | None = None,
|
library_name: str | None = None,
|
||||||
|
file_id: int | None = None,
|
||||||
|
file_name: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
normalized_type = _normalize_model_type(model_type)
|
normalized_type = _normalize_model_type(model_type)
|
||||||
normalized_version_id = _normalize_int(version_id)
|
normalized_version_id = _normalize_int(version_id)
|
||||||
normalized_model_id = _normalize_int(model_id)
|
normalized_model_id = _normalize_int(model_id)
|
||||||
|
normalized_file_id = _normalize_int(file_id)
|
||||||
if normalized_type is None or normalized_version_id is None:
|
if normalized_type is None or normalized_version_id is None:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -164,6 +179,25 @@ class DownloadedVersionHistoryService:
|
|||||||
active_library_name,
|
active_library_name,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
if normalized_file_id is not None:
|
||||||
|
# Per-file history for multi-file versions (#1058)
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO downloaded_version_files (
|
||||||
|
model_type, version_id, file_id, file_name, downloaded_at
|
||||||
|
) VALUES (?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(model_type, version_id, file_id) DO UPDATE SET
|
||||||
|
file_name = COALESCE(excluded.file_name, downloaded_version_files.file_name),
|
||||||
|
downloaded_at = excluded.downloaded_at
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
normalized_type,
|
||||||
|
normalized_version_id,
|
||||||
|
normalized_file_id,
|
||||||
|
file_name,
|
||||||
|
timestamp,
|
||||||
|
),
|
||||||
|
)
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|
||||||
async def mark_downloaded_bulk(
|
async def mark_downloaded_bulk(
|
||||||
@@ -202,24 +236,33 @@ class DownloadedVersionHistoryService:
|
|||||||
return
|
return
|
||||||
|
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
conn = self._get_conn()
|
# The connection is created with check_same_thread=False and all
|
||||||
conn.executemany(
|
# access is serialized by self._lock, so the executemany upsert +
|
||||||
"""
|
# commit can run in the default executor without blocking the
|
||||||
INSERT INTO downloaded_model_versions (
|
# event loop on large hydration payloads.
|
||||||
model_type, version_id, model_id, first_seen_at, last_seen_at,
|
loop = asyncio.get_running_loop()
|
||||||
source, last_file_path, last_library_name, is_deleted_override
|
await loop.run_in_executor(None, self._mark_downloaded_bulk_sync, payload)
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0)
|
|
||||||
ON CONFLICT(model_type, version_id) DO UPDATE SET
|
def _mark_downloaded_bulk_sync(self, payload: Sequence[tuple[object, ...]]) -> None:
|
||||||
model_id = COALESCE(excluded.model_id, downloaded_model_versions.model_id),
|
"""Synchronous executemany upsert + commit; runs in a worker thread."""
|
||||||
last_seen_at = excluded.last_seen_at,
|
conn = self._get_conn()
|
||||||
source = excluded.source,
|
conn.executemany(
|
||||||
last_file_path = COALESCE(excluded.last_file_path, downloaded_model_versions.last_file_path),
|
"""
|
||||||
last_library_name = COALESCE(excluded.last_library_name, downloaded_model_versions.last_library_name),
|
INSERT INTO downloaded_model_versions (
|
||||||
is_deleted_override = 0
|
model_type, version_id, model_id, first_seen_at, last_seen_at,
|
||||||
""",
|
source, last_file_path, last_library_name, is_deleted_override
|
||||||
payload,
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0)
|
||||||
)
|
ON CONFLICT(model_type, version_id) DO UPDATE SET
|
||||||
conn.commit()
|
model_id = COALESCE(excluded.model_id, downloaded_model_versions.model_id),
|
||||||
|
last_seen_at = excluded.last_seen_at,
|
||||||
|
source = excluded.source,
|
||||||
|
last_file_path = COALESCE(excluded.last_file_path, downloaded_model_versions.last_file_path),
|
||||||
|
last_library_name = COALESCE(excluded.last_library_name, downloaded_model_versions.last_library_name),
|
||||||
|
is_deleted_override = 0
|
||||||
|
""",
|
||||||
|
payload,
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
async def mark_as_deleted(self, model_type: str, version_id: int) -> None:
|
async def mark_as_deleted(self, model_type: str, version_id: int) -> None:
|
||||||
normalized_type = _normalize_model_type(model_type)
|
normalized_type = _normalize_model_type(model_type)
|
||||||
@@ -251,8 +294,63 @@ class DownloadedVersionHistoryService:
|
|||||||
self._get_active_library_name(),
|
self._get_active_library_name(),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
# Whole-version deletion also clears the per-file records (#1058)
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
DELETE FROM downloaded_version_files
|
||||||
|
WHERE model_type = ? AND version_id = ?
|
||||||
|
""",
|
||||||
|
(normalized_type, normalized_version_id),
|
||||||
|
)
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|
||||||
|
async def mark_file_deleted(
|
||||||
|
self, model_type: str, version_id: int, file_id: int
|
||||||
|
) -> None:
|
||||||
|
"""Drop a single file record of a version, keeping siblings (#1058)."""
|
||||||
|
normalized_type = _normalize_model_type(model_type)
|
||||||
|
normalized_version_id = _normalize_int(version_id)
|
||||||
|
normalized_file_id = _normalize_int(file_id)
|
||||||
|
if (
|
||||||
|
normalized_type is None
|
||||||
|
or normalized_version_id is None
|
||||||
|
or normalized_file_id is None
|
||||||
|
):
|
||||||
|
return
|
||||||
|
|
||||||
|
async with self._lock:
|
||||||
|
conn = self._get_conn()
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
DELETE FROM downloaded_version_files
|
||||||
|
WHERE model_type = ? AND version_id = ? AND file_id = ?
|
||||||
|
""",
|
||||||
|
(normalized_type, normalized_version_id, normalized_file_id),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
async def get_downloaded_file_ids(
|
||||||
|
self, model_type: str, version_id: int
|
||||||
|
) -> list[int]:
|
||||||
|
"""Return the CivitAI file ids recorded as downloaded for a version."""
|
||||||
|
normalized_type = _normalize_model_type(model_type)
|
||||||
|
normalized_version_id = _normalize_int(version_id)
|
||||||
|
if normalized_type is None or normalized_version_id is None:
|
||||||
|
return []
|
||||||
|
|
||||||
|
async with self._lock:
|
||||||
|
conn = self._get_conn()
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT file_id
|
||||||
|
FROM downloaded_version_files
|
||||||
|
WHERE model_type = ? AND version_id = ?
|
||||||
|
ORDER BY file_id ASC
|
||||||
|
""",
|
||||||
|
(normalized_type, normalized_version_id),
|
||||||
|
).fetchall()
|
||||||
|
return [int(row["file_id"]) for row in rows]
|
||||||
|
|
||||||
async def has_been_downloaded(self, model_type: str, version_id: int) -> bool:
|
async def has_been_downloaded(self, model_type: str, version_id: int) -> bool:
|
||||||
normalized_type = _normalize_model_type(model_type)
|
normalized_type = _normalize_model_type(model_type)
|
||||||
normalized_version_id = _normalize_int(version_id)
|
normalized_version_id = _normalize_int(version_id)
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user