mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-09-21 11:11:26 -03:00
Compare commits
185 Commits
v1.2.1
..
84146b62fd
| Author | SHA1 | Date | |
|---|---|---|---|
| 84146b62fd | |||
| adeb40bfff | |||
| 8a21837ca2 | |||
| 3302147a43 | |||
| 4d87ae7637 | |||
| b1a653f18f | |||
| 6fe0543d2e | |||
| 931dfbe1d3 | |||
| b5c1331911 | |||
| 37f2cba72d | |||
| 0e789cb38c | |||
| f3b3393a16 | |||
| 480a3f4ea5 | |||
| 69a62d739c | |||
| 28fbb86dce | |||
| f88fe2665c | |||
| 3592eab48c | |||
| 1dbdf5b00c | |||
| fc3b2d7c13 | |||
| f2a7297cb9 | |||
| 57729375b6 | |||
| fa7ce725c1 | |||
| 27da7b3ca3 | |||
| 3070838a42 | |||
| fe160134d0 | |||
| 6d3f82976f | |||
| 91b2735dad | |||
| 3112869a21 | |||
| aa630bf85b | |||
| e0052cd237 | |||
| 3cdc5ba7a2 | |||
| 04485e384f | |||
| a03dc4002f | |||
| cc9d3bff42 | |||
| 2672b3331b | |||
| 4963bf2b2e | |||
| 51cad6f852 | |||
| 1b5cbbbaa0 | |||
| e747946f7a | |||
| 53fa22f39c | |||
| 82b34097fb | |||
| a7995db009 | |||
| 5ae4aef30e | |||
| 08023f0cd9 | |||
| 6e2185c182 | |||
| 41302e75ba | |||
| a17399d667 | |||
| e2d85a0a21 | |||
| 303833bbae | |||
| f86b7b55d6 | |||
| 782bb53784 | |||
| 139231e225 | |||
| 121d8d5cea | |||
| ec147bd677 | |||
| 93fc28b499 | |||
| 7afed1a14b | |||
| e6f5142e48 | |||
| 87f05fb66c | |||
| cf64e5baa8 | |||
| 634ea7f299 | |||
| 6ba64ebb3c | |||
| 03569c62df | |||
| a61840b366 | |||
| 726fc178f1 | |||
| 8260bd022d | |||
| b309becdf9 | |||
| 1e375bb8d9 | |||
| 14da8a6f17 | |||
| da71985c3e | |||
| 7c4c8b8f30 | |||
| 77109b3cf8 | |||
| 00095a5398 | |||
| 6b41c3bbb4 | |||
| b37238d790 | |||
| bc33e32c6f | |||
| 49704d801c | |||
| 34ca14d7fc | |||
| f7b247f9e8 | |||
| 3005d2877e | |||
| ed2a17970f | |||
| 9584fa85c9 | |||
| 1fd7cc0123 | |||
| 39e7c1376c | |||
| 2a3c632dc5 | |||
| 8d46d26abe | |||
| d761ac77f7 | |||
| c8b9db5bf4 | |||
| bce7d1d30c | |||
| bccd494a56 | |||
| 3fd29f6943 | |||
| 838a374a56 | |||
| 6e31da7a70 | |||
| fc9088bfd6 | |||
| 675421ea84 | |||
| 2ff98ae089 | |||
| c972c755fc | |||
| ebe3df7d22 | |||
| be44a75b74 | |||
| fd1227d3b8 | |||
| 3a9e02137d | |||
| d8a2be8edc | |||
| 1c46b2e8c3 | |||
| 3c3ac49f2f | |||
| 1a1be95a64 | |||
| 7a36659a20 | |||
| cb18281b14 | |||
| 856c9a87ac | |||
| a7d65fe84a | |||
| 15bf079af2 | |||
| 65ba750634 | |||
| 17dcbd3d4f | |||
| e914a0e19d | |||
| 2ba04bb1bd | |||
| 1b7314591a | |||
| 2bfb987312 | |||
| df34efafbc | |||
| c2a2048c8b | |||
| 1e1921cabb | |||
| ee233548e5 | |||
| 574dfbbe55 | |||
| 1d3bcdfe47 | |||
| 74369940bf | |||
| d188cec306 | |||
| 641a61f804 | |||
| 3025c64fea | |||
| c52cfc7e7a | |||
| 4ed9f775f6 | |||
| 0b08ad283a | |||
| 08895f77ff | |||
| 74f889f160 | |||
| c51090ab16 | |||
| cdb044cb45 | |||
| c83b26b556 | |||
| a202c666bc | |||
| e05046af10 | |||
| 41ed03e5c6 | |||
| da071e8452 | |||
| a0bb6df2b8 | |||
| 6f5c444ec5 | |||
| 20f66a4fe1 | |||
| 879745da53 | |||
| 3afec0a0be | |||
| 06c270a6e1 | |||
| 87e93636dc | |||
| 074d1f2e51 | |||
| 40f922b0e8 | |||
| a7214b6cff | |||
| 8ca66e72eb | |||
| 90be5799e4 | |||
| 1a93b0eca2 | |||
| c2360a35ad | |||
| 030a32f8fa | |||
| 25e72b43ce | |||
| 41e9883daa | |||
| ae461ebc81 | |||
| 3ebf256c5d | |||
| 0905e2be6e | |||
| bd380bc1a1 | |||
| cb4fd3a0e6 | |||
| bbe0acac5c | |||
| 45e7c25308 | |||
| 86aa1d8059 | |||
| 74254756ef | |||
| 259e08e47c | |||
| 6647c45731 | |||
| b614a5c447 | |||
| b80830913c | |||
| e57e11897e | |||
| 8a16034135 | |||
| 7fc3b7e5be | |||
| b0c7a1baae | |||
| 6411d83d46 | |||
| 74a063b0e5 | |||
| 96376e5cce | |||
| e7c26bf722 | |||
| cef4129fc9 | |||
| 0a28500848 | |||
| fc3f3f3bdb | |||
| fa58297973 | |||
| 5d1a22fb8f | |||
| d2f50f26f1 | |||
| 4a6042d0b4 | |||
| 846206d958 | |||
| 0daf4924f0 | |||
| d38a3d091d |
@@ -1,373 +0,0 @@
|
|||||||
---
|
|
||||||
name: lora-manager-e2e
|
|
||||||
description: End-to-end testing and validation for LoRa Manager features. Use when performing automated E2E validation of LoRa Manager standalone mode in a SANDBOXED, disposable configuration: check the port, start/restart the standalone server on a free port, use Chrome DevTools MCP to interact with the web UI (http://127.0.0.1:{PORT}/loras), and verify frontend-to-backend functionality. Covers workflow validation, UI interaction testing, and integration testing between the standalone Python backend and the browser frontend. Trigger keywords: E2E, standalone, Chrome DevTools MCP, lora-manager-e2e, sandbox.
|
|
||||||
---
|
|
||||||
|
|
||||||
# LoRa Manager E2E Testing
|
|
||||||
|
|
||||||
This skill provides workflows and utilities for end-to-end testing of LoRa Manager using Chrome DevTools MCP.
|
|
||||||
|
|
||||||
## Conventions Used in This Document
|
|
||||||
|
|
||||||
- **`{PORT}`**: The server port. The default candidate is `8188`, but **`8188` is commonly occupied by a live ComfyUI process** and MUST NOT be assumed to be free. Always check availability first (see [Port Selection](#port-selection)) and use a free port (e.g. `8199`) for the E2E run. Substitute the actual port for every `{PORT}` in the commands below.
|
|
||||||
- **`<repo-root>`**: The repository/worktree root. Always run commands from the repo or worktree root; never assume a specific absolute path (paths such as `/home/<user>/...` differ per machine). The E2E scripts resolve the project root themselves, but fixture/settings paths are relative to `<repo-root>`.
|
|
||||||
|
|
||||||
## SANDBOX (MANDATORY)
|
|
||||||
|
|
||||||
> **Read this section before running anything.** Every E2E run MUST target a throwaway sandbox, never the real user data. A fresh subagent that skips this section WILL permanently mutate real user recipes.
|
|
||||||
|
|
||||||
1. **Portable settings**: create `<repo-root>/settings.json` (gitignored) with `"use_portable_settings": true` plus sandboxed `folder_paths` (lora/checkpoint roots) and `recipes_path`. This keeps the configuration inside the repo instead of the real user config dir (`~/.config/ComfyUI-LoRA-Manager/settings.json`).
|
|
||||||
2. **Sandboxed paths**: point `folder_paths` / `recipes_path` / `example_images_path` at disposable dirs — e.g. under `/tmp/opencode/<plan-name>-e2e/` (or worktree-local dirs). NEVER point the E2E at the real library (`~/models/...`), real recipe dir, or real settings.
|
|
||||||
3. **Never touch the real config**: the real user config at `~/.config/ComfyUI-LoRA-Manager/settings.json` and the real recipe dir must remain byte-identical before and after the run.
|
|
||||||
4. **Record real-data protection proof** before starting and after finishing:
|
|
||||||
```bash
|
|
||||||
# BEFORE: snapshot real config + recipe library state
|
|
||||||
sha256sum ~/.config/ComfyUI-LoRA-Manager/settings.json > /tmp/opencode/<plan>-e2e/settings.before.sha256
|
|
||||||
ls ~/models/recipes/*.recipe.json 2>/dev/null | wc -l > /tmp/opencode/<plan>-e2e/recipes-count.before.txt
|
|
||||||
find ~/models/recipes -name '*.recipe.json' -newermt "$(date -Iseconds)" | head # expect empty after run
|
|
||||||
# AFTER: record again, then diff the two snapshots. Any change = the run leaked into real data.
|
|
||||||
```
|
|
||||||
Also confirm `<repo-root>/git status` stays clean for `settings.json`/`cache/` (both are gitignored).
|
|
||||||
|
|
||||||
### Portable Settings Example
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"use_portable_settings": true,
|
|
||||||
"folder_paths": {
|
|
||||||
"loras": ["/tmp/opencode/<plan>-e2e/models/loras"],
|
|
||||||
"checkpoints": ["/tmp/opencode/<plan>-e2e/models/checkpoints"],
|
|
||||||
"unet": ["/tmp/opencode/<plan>-e2e/models/checkpoints"],
|
|
||||||
"diffusers": []
|
|
||||||
},
|
|
||||||
"recipes_path": "/tmp/opencode/<plan>-e2e/recipes",
|
|
||||||
"example_images_path": "/tmp/opencode/<plan>-e2e/example_images"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The scanner computes and persists model hashes during the library scan, so the sandbox model dirs just need the model files + `.metadata.json` sidecars (see [Fixture + Fresh-State Guidance](#fixture--fresh-state-guidance)).
|
|
||||||
|
|
||||||
## Time Budgets & Abort Guidance
|
|
||||||
|
|
||||||
A fresh subagent should complete a sandboxed standalone E2E **in well under 30 minutes**. Budget each phase:
|
|
||||||
|
|
||||||
| Phase | Expected duration | Abort if |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| Port check + sandbox setup | < 2 min | — |
|
|
||||||
| Server start (detached) + readiness | < 30 s | > 60 s (2x) → stop |
|
|
||||||
| Chrome DevTools MCP connect | < 1 min | > 2 min → stop |
|
|
||||||
| Per entry-point run (after fixtures ready) | < 5 min | > 10 min (2x) → stop |
|
|
||||||
| Fixture reset + cache clear between runs | < 1 min | > 2 min → stop |
|
|
||||||
|
|
||||||
**Abort rule**: if a phase exceeds ~2x its budget, OR any single tool call fails/retries 3+ times in a row, **STOP**. Do not loop or retry blindly. Report `BLOCKED` with: the phase, the last observed state (server PID + `ss -tlnp` output, page snapshot, last API response), and the suspected cause. Record the partial state as evidence; a clean BLOCKED report is more valuable than an hour of retries.
|
|
||||||
|
|
||||||
## Prerequisites
|
|
||||||
|
|
||||||
- LoRa Manager project cloned and dependencies installed (`pip install -r requirements.txt`) — run everything from `<repo-root>`
|
|
||||||
- Chrome browser available for debugging
|
|
||||||
- Chrome DevTools MCP connected
|
|
||||||
- `ss` (or `lsof`/`netstat`) available for port checks: `ss -tlnp`
|
|
||||||
|
|
||||||
## Port Selection
|
|
||||||
|
|
||||||
`8188` is only the *default candidate*. Verify it is actually free before every run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Is anything listening on 8188?
|
|
||||||
ss -tlnp | grep ':8188' || echo "8188 is free"
|
|
||||||
```
|
|
||||||
|
|
||||||
- If a process holds `8188` (e.g. a live ComfyUI — pid 6575 on this machine), pick a different free port, e.g. `8199`:
|
|
||||||
```bash
|
|
||||||
ss -tlnp | grep ':8199' || echo "8199 is free"
|
|
||||||
```
|
|
||||||
- **Never** kill a process you did not start for this E2E. The live ComfyUI is off-limits. Pick a free port instead.
|
|
||||||
- Use your chosen port for **all** subsequent commands (server, Chrome launch, browser URLs).
|
|
||||||
|
|
||||||
## Quick Start Workflow (sandboxed)
|
|
||||||
|
|
||||||
### 1. Prepare the sandbox
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd <repo-root> # ALWAYS run from the repo/worktree root
|
|
||||||
mkdir -p /tmp/opencode/<plan>-e2e/models/{loras,checkpoints}
|
|
||||||
mkdir -p /tmp/opencode/<plan>-e2e/{recipes,example_images,recipes-before}
|
|
||||||
# write <repo-root>/settings.json per the portable-settings example above
|
|
||||||
# record real-data protection proof (see SANDBOX section)
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Check port availability
|
|
||||||
|
|
||||||
```bash
|
|
||||||
ss -tlnp | grep ':{PORT}' || echo "port {PORT} is free"
|
|
||||||
```
|
|
||||||
|
|
||||||
If `{PORT}` is occupied by an unrelated process, pick a free one and use it everywhere below. When in doubt use `8199`.
|
|
||||||
|
|
||||||
### 3. Start LoRa Manager Standalone (detached)
|
|
||||||
|
|
||||||
The standalone server **dies with the shell unless launched fully detached** — a plain background `&` from the bash tool is killed when the tool call returns. Launch via the helper script:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python .agents/skills/lora-manager-e2e/scripts/start_server.py --port {PORT} --wait --timeout 30 --detach
|
|
||||||
```
|
|
||||||
|
|
||||||
Or manually (equivalent detached form):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
setsid nohup python standalone.py --port {PORT} --host 127.0.0.1 < /dev/null \
|
|
||||||
>> /tmp/opencode/<plan>-e2e/server.log 2>&1 &
|
|
||||||
echo "started" # record the printed/pidfile PID for cleanup
|
|
||||||
```
|
|
||||||
|
|
||||||
Verify it is listening **before** proceeding (readiness poll is not a substitute for this):
|
|
||||||
|
|
||||||
```bash
|
|
||||||
ss -tlnp | grep ':{PORT}'
|
|
||||||
```
|
|
||||||
|
|
||||||
Record the server PID for cleanup: the helper script writes it to `/tmp/lora-manager-e2e-server-{PORT}.pid`; a manual `setsid` launch has no pidfile, so capture it explicitly (e.g. from `ss -tlnp`).
|
|
||||||
|
|
||||||
### 4. Open Chrome Debug Mode
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Chrome with remote debugging on port 9222 (note the {PORT} URL)
|
|
||||||
google-chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:{PORT}/loras
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5. Connect Chrome DevTools MCP
|
|
||||||
|
|
||||||
Ensure the MCP server is connected to Chrome at `http://localhost:9222`. Verify with `list_pages` — if it fails with "browser is already running", see [Chrome DevTools MCP Troubleshooting](#chrome-devtools-mcp-troubleshooting).
|
|
||||||
|
|
||||||
### 6. Navigate and Interact
|
|
||||||
|
|
||||||
Use Chrome DevTools MCP tools to:
|
|
||||||
- Take snapshots: `take_snapshot`
|
|
||||||
- Click elements: `click`
|
|
||||||
- Fill forms: `fill` or `fill_form`
|
|
||||||
- Evaluate scripts: `evaluate_script`
|
|
||||||
- Wait for elements: `wait_for`
|
|
||||||
|
|
||||||
## Common E2E Test Patterns
|
|
||||||
|
|
||||||
### Pattern: Full Page Load Verification
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Navigate to LoRA list page
|
|
||||||
navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
|
||||||
|
|
||||||
# Wait for page to load
|
|
||||||
wait_for(text="LoRAs", timeout=10000)
|
|
||||||
|
|
||||||
# Take snapshot to verify UI state
|
|
||||||
snapshot = take_snapshot()
|
|
||||||
```
|
|
||||||
|
|
||||||
### Pattern: Restart Server for Configuration Changes
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Stop current server (if running), start with new configuration.
|
|
||||||
# --restart only kills the E2E server this script started before (via its pidfile);
|
|
||||||
# it refuses to blindly kill unrelated processes on the port.
|
|
||||||
python .agents/skills/lora-manager-e2e/scripts/start_server.py --port {PORT} --restart --wait --detach
|
|
||||||
|
|
||||||
# Wait and refresh browser
|
|
||||||
navigate_page(type="reload", ignoreCache=True)
|
|
||||||
wait_for(text="LoRAs", timeout=15000)
|
|
||||||
```
|
|
||||||
|
|
||||||
### Pattern: Verify Backend API via Frontend
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Execute script in browser to call backend API
|
|
||||||
result = evaluate_script(function="""
|
|
||||||
async () => {
|
|
||||||
const response = await fetch('/loras/api/list');
|
|
||||||
const data = await response.json();
|
|
||||||
return { count: data.length, firstItem: data[0]?.name };
|
|
||||||
}
|
|
||||||
""")
|
|
||||||
```
|
|
||||||
|
|
||||||
### Pattern: Form Submission Flow
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Fill a form (e.g., search or filter)
|
|
||||||
fill_form(elements=[
|
|
||||||
{"uid": "search-input", "value": "character"},
|
|
||||||
])
|
|
||||||
|
|
||||||
# Click submit button
|
|
||||||
click(uid="search-button")
|
|
||||||
|
|
||||||
# Wait for results
|
|
||||||
wait_for(text="Results", timeout=5000)
|
|
||||||
|
|
||||||
# Verify results via snapshot
|
|
||||||
snapshot = take_snapshot()
|
|
||||||
```
|
|
||||||
|
|
||||||
### Pattern: Modal Dialog Interaction
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Open modal (e.g., add LoRA)
|
|
||||||
click(uid="add-lora-button")
|
|
||||||
|
|
||||||
# Wait for modal to appear
|
|
||||||
wait_for(text="Add LoRA", timeout=3000)
|
|
||||||
|
|
||||||
# Fill modal form
|
|
||||||
fill_form(elements=[
|
|
||||||
{"uid": "lora-name", "value": "Test LoRA"},
|
|
||||||
{"uid": "lora-path", "value": "/path/to/lora.safetensors"},
|
|
||||||
])
|
|
||||||
|
|
||||||
# Submit
|
|
||||||
click(uid="modal-submit-button")
|
|
||||||
|
|
||||||
# Wait for success message or close
|
|
||||||
wait_for(text="Success", timeout=5000)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Fixture + Fresh-State Guidance
|
|
||||||
|
|
||||||
For rematch/repair E2E runs, seed the **sandboxed** `recipes_path` with hand-written fixture recipes. Rules (validated by the task-8 E2E):
|
|
||||||
|
|
||||||
1. **Filename constraint**: each file MUST be named `f"{id}.recipe.json"` **and** the in-JSON `id` field MUST equal the filename. Discovery accepts any `*.recipe.json`, but persistence resolves the path via `get_recipe_json_path` and `_save_recipe_persistently` returns `False` on a mismatch → the fixture would be counted as an error.
|
|
||||||
- `recipe-a.recipe.json` → in-JSON `"id": "recipe-a"`
|
|
||||||
2. **File format**: mirror an existing recipe JSON — top-level `id`, `file_path`, `title`, `loras`, `fingerprint`, `gen_params`; lora entries per the persistence conventions (`hash`, `file_name`, `modelVersionId`, `isDeleted`, ...).
|
|
||||||
3. **Companion image**: each recipe needs an image (e.g. a `.webp` generated with PIL) referenced by `file_path`, used for EXIF verification (`ExifUtils.append_recipe_metadata` writes a `"Recipe metadata: ..."` marker; a freshly generated `.webp` with no marker is the clean "untouched" control).
|
|
||||||
4. **autov3 three-state contract**: for L3 (autov3-only, renamed-file) fixtures the local model's `.metadata.json` sidecar MUST have the `autov3` key **ABSENT** (the "unchecked" state), NOT `""` — `""` is the TERMINAL "checked but unavailable" state that L3 deliberately skips. The scanner computes + persists `autov3` from the file header during the normal library scan (`model_scanner.py` `_process_model_file`), so the live L3 match resolves through the local autov3/hash cache; the computed-autov3 branch for unchecked items is covered by the unit suite.
|
|
||||||
5. **Fixture design for a rematch run** (mirrors the task-8 E2E):
|
|
||||||
- `recipe-a`: lora entry `isDeleted=True`, `hash` = 12-char autov3 computed from the local model (`calculate_autov3`, `py/utils/file_utils.py`), whose local model file was RENAMED after the recipe was written so `file_name` differs (proves L3 match without filename).
|
|
||||||
- `recipe-b`: parser-convention checkpoint entry (uses `id`, no `modelVersionId`) matching a local checkpoint via L2 — the local checkpoint's `.metadata.json` MUST carry civitai version data with that `id` so `version_index` contains it (L2 cannot match otherwise).
|
|
||||||
- `recipe-c`: healthy recipe (no deleted entries) → must remain untouched.
|
|
||||||
|
|
||||||
### Fresh state between entry-point runs
|
|
||||||
|
|
||||||
Each entry point (global / per-recipe / selection-bulk) must start from the same deleted state. Between runs:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. Reset fixtures to the before-state snapshot (copy back from recipes-before/)
|
|
||||||
cp /tmp/opencode/<plan>-e2e/recipes-before/*.recipe.json /tmp/opencode/<plan>-e2e/recipes/
|
|
||||||
# 2. Clear the recipe/FTS caches so the stale in-memory/library state is gone
|
|
||||||
rm -f <repo-root>/cache/recipe/*.sqlite
|
|
||||||
rm -rf <repo-root>/cache/fts/*
|
|
||||||
# 3. Restart the server (fresh process, fresh scan)
|
|
||||||
python .agents/skills/lora-manager-e2e/scripts/start_server.py --port {PORT} --restart --wait --timeout 30 --detach
|
|
||||||
# 4. Re-verify server listening + reload the browser page
|
|
||||||
```
|
|
||||||
|
|
||||||
## Server Lifecycle
|
|
||||||
|
|
||||||
- **Detached launch is mandatory**: the standalone server dies with the shell unless launched via `setsid` (or the helper script's `--detach`). Use `setsid nohup python standalone.py --port {PORT} --host 127.0.0.1 ... < /dev/null &`.
|
|
||||||
- **Verify with `ss -tlnp`** after every (re)start; do not proceed on a blind "server starting" message.
|
|
||||||
- **Never kill pre-existing processes** — only kill the E2E server PID you started (`start_server.py --restart` kills only PIDs it manages via its pidfile). The live ComfyUI or a stale QA Chrome must never be killed as part of cleanup unless explicitly identified as such (see Chrome troubleshooting).
|
|
||||||
- **Record your PID for cleanup**: note the PID printed/pidfile, and stop exactly that PID at the end (`kill <PID>`, then confirm with `ss -tlnp` that `{PORT}` is released).
|
|
||||||
|
|
||||||
## Chrome DevTools MCP Troubleshooting
|
|
||||||
|
|
||||||
### Stale profile lock ("browser is already running" / `list_pages` fails)
|
|
||||||
|
|
||||||
A Chrome profile can be held by a stale Chrome from a prior MCP session, which makes `list_pages` fail with "browser is already running":
|
|
||||||
|
|
||||||
1. Identify the stale Chrome — it owns the profile dir in `--user-data-dir` (e.g. `~/.config/chrome-dev-profile`). Find its process:
|
|
||||||
```bash
|
|
||||||
ps -ef | grep -i '[c]hrome.*user-data-dir'
|
|
||||||
```
|
|
||||||
2. Confirm it is a QA Chrome from a completed task (its parent is an old MCP/browser process, it is NOT the live ComfyUI server, and it is NOT your current MCP instance).
|
|
||||||
3. Kill ONLY that stale Chrome:
|
|
||||||
```bash
|
|
||||||
kill <stale-chrome-pid>
|
|
||||||
```
|
|
||||||
Never kill the live server or unrelated processes.
|
|
||||||
4. Retry `list_pages`. The current MCP will spawn a fresh browser.
|
|
||||||
|
|
||||||
### Screenshot-write restrictions
|
|
||||||
|
|
||||||
The chrome-devtools MCP may refuse to write into paths outside its configured workspace roots (e.g. the worktree `.omo/evidence/...` canonicalizing to an unmapped path). Workaround:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. Save the screenshot to /tmp via the MCP
|
|
||||||
# take_screenshot(filePath="/tmp/<plan>-e2e/recipe-b-after.png", format="png")
|
|
||||||
# 2. Copy it into the evidence dir from the shell
|
|
||||||
mkdir -p <repo-root>/.omo/evidence/screenshots
|
|
||||||
cp /tmp/<plan>-e2e/recipe-b-after.png <repo-root>/.omo/evidence/screenshots/
|
|
||||||
```
|
|
||||||
|
|
||||||
## Cancellation Testing (KNOWN GAP)
|
|
||||||
|
|
||||||
Testing the rematch-cancel path E2E requires a run long enough to cancel mid-flight. A tiny 3-recipe fixture set completes in **seconds** — too fast to reliably cancel. The cancel path is currently **unit-covered only** (`rematch_all_recipes` cancellation tests); do not block an E2E run on cancel-path verification. If you must attempt it, you would need an artificially large/deferred fixture set to create a cancellable window — treat this as a research task, not part of the standard E2E.
|
|
||||||
|
|
||||||
## Available Scripts
|
|
||||||
|
|
||||||
### scripts/start_server.py
|
|
||||||
|
|
||||||
Starts or restarts the LoRa Manager standalone server for E2E testing.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python scripts/start_server.py [--port PORT] [--restart] [--wait] [--timeout SECONDS] [--detach]
|
|
||||||
```
|
|
||||||
|
|
||||||
Options:
|
|
||||||
- `--port`: Server port (default: 8188). The script exits early with a clear message if the port is already in use by an unrelated process.
|
|
||||||
- `--restart`: Kill the E2E server this script previously managed (tracked via `/tmp/lora-manager-e2e-server-{PORT}.pid`) before starting. If unrelated processes still hold the port after that, the script reports them and aborts instead of killing them.
|
|
||||||
- `--wait`: Wait for the server to be ready before exiting.
|
|
||||||
- `--timeout`: Readiness wait timeout in seconds (default: 30).
|
|
||||||
- `--detach`: Launch the server fully detached (`setsid`-style, survives shell death — REQUIRED for E2E). Default off: a normal background process that dies with the shell.
|
|
||||||
|
|
||||||
### scripts/wait_for_server.py
|
|
||||||
|
|
||||||
Polls the server until ready or timeout.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python scripts/wait_for_server.py [--port PORT] [--timeout SECONDS]
|
|
||||||
```
|
|
||||||
|
|
||||||
## Test Scenarios Reference
|
|
||||||
|
|
||||||
See [references/test-scenarios.md](references/test-scenarios.md) for detailed test scenarios including:
|
|
||||||
- LoRA list display and filtering
|
|
||||||
- Model metadata editing
|
|
||||||
- Recipe creation and management
|
|
||||||
- Settings configuration
|
|
||||||
- Import/export functionality
|
|
||||||
|
|
||||||
## Network Request Verification
|
|
||||||
|
|
||||||
Use `list_network_requests` and `get_network_request` to verify API calls:
|
|
||||||
|
|
||||||
```python
|
|
||||||
# List recent XHR/fetch requests
|
|
||||||
requests = list_network_requests(resourceTypes=["xhr", "fetch"])
|
|
||||||
|
|
||||||
# Get details of specific request
|
|
||||||
details = get_network_request(reqid=123)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Console Message Monitoring
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Check for errors or warnings
|
|
||||||
messages = list_console_messages(types=["error", "warn"])
|
|
||||||
```
|
|
||||||
|
|
||||||
## Performance Testing
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Start performance trace
|
|
||||||
performance_start_trace(reload=True, autoStop=False)
|
|
||||||
|
|
||||||
# Perform actions...
|
|
||||||
|
|
||||||
# Stop and analyze
|
|
||||||
results = performance_stop_trace()
|
|
||||||
```
|
|
||||||
|
|
||||||
## Cleanup
|
|
||||||
|
|
||||||
Always ensure proper cleanup after tests:
|
|
||||||
1. Stop the standalone server: `kill <recorded-pid>` (only the PID you started), then confirm `ss -tlnp | grep ':{PORT}'` is empty.
|
|
||||||
2. Close browser pages (keep at least one open).
|
|
||||||
3. Remove the sandbox: `rm -rf /tmp/opencode/<plan>-e2e` and `<repo-root>/settings.json` + `<repo-root>/cache` (both gitignored).
|
|
||||||
4. Re-run the real-data protection check from the SANDBOX section and record the result in your evidence.
|
|
||||||
@@ -1,360 +0,0 @@
|
|||||||
# Chrome DevTools MCP Cheatsheet for LoRa Manager
|
|
||||||
|
|
||||||
Quick reference for common MCP commands used in LoRa Manager E2E testing.
|
|
||||||
|
|
||||||
> **Port convention**: `{PORT}` is the port chosen for the E2E run (default candidate `8188`, but only if actually free — see the SKILL.md Port Selection section; use e.g. `8199` when `8188` is occupied by a live ComfyUI). Always run against the **sandboxed** standalone server, never a live instance.
|
|
||||||
|
|
||||||
## Navigation
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Navigate to LoRA list page
|
|
||||||
navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
|
||||||
|
|
||||||
# Reload page with cache clear
|
|
||||||
navigate_page(type="reload", ignoreCache=True)
|
|
||||||
|
|
||||||
# Go back/forward
|
|
||||||
navigate_page(type="back")
|
|
||||||
navigate_page(type="forward")
|
|
||||||
```
|
|
||||||
|
|
||||||
## Waiting
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Wait for text to appear
|
|
||||||
wait_for(text="LoRAs", timeout=10000)
|
|
||||||
|
|
||||||
# Wait for specific element (via evaluate_script)
|
|
||||||
evaluate_script(function="""
|
|
||||||
() => {
|
|
||||||
return new Promise((resolve) => {
|
|
||||||
const check = () => {
|
|
||||||
if (document.querySelector('.lora-card')) {
|
|
||||||
resolve(true);
|
|
||||||
} else {
|
|
||||||
setTimeout(check, 100);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
check();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
""")
|
|
||||||
```
|
|
||||||
|
|
||||||
## Taking Snapshots
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Full page snapshot
|
|
||||||
snapshot = take_snapshot()
|
|
||||||
|
|
||||||
# Verbose snapshot (more details)
|
|
||||||
snapshot = take_snapshot(verbose=True)
|
|
||||||
|
|
||||||
# Save to file
|
|
||||||
take_snapshot(filePath="test-snapshots/page-load.json")
|
|
||||||
```
|
|
||||||
|
|
||||||
## Element Interaction
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Click element
|
|
||||||
click(uid="element-uid-from-snapshot")
|
|
||||||
|
|
||||||
# Double click
|
|
||||||
click(uid="element-uid", dblClick=True)
|
|
||||||
|
|
||||||
# Fill input
|
|
||||||
fill(uid="search-input", value="test query")
|
|
||||||
|
|
||||||
# Fill multiple inputs
|
|
||||||
fill_form(elements=[
|
|
||||||
{"uid": "input-1", "value": "value 1"},
|
|
||||||
{"uid": "input-2", "value": "value 2"},
|
|
||||||
])
|
|
||||||
|
|
||||||
# Hover
|
|
||||||
hover(uid="lora-card-1")
|
|
||||||
|
|
||||||
# Upload file
|
|
||||||
upload_file(uid="file-input", filePath="/path/to/file.safetensors")
|
|
||||||
```
|
|
||||||
|
|
||||||
## Keyboard Input
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Press key
|
|
||||||
press_key(key="Enter")
|
|
||||||
press_key(key="Escape")
|
|
||||||
press_key(key="Tab")
|
|
||||||
|
|
||||||
# Keyboard shortcuts
|
|
||||||
press_key(key="Control+A") # Select all
|
|
||||||
press_key(key="Control+F") # Find
|
|
||||||
```
|
|
||||||
|
|
||||||
## JavaScript Evaluation
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Simple evaluation
|
|
||||||
result = evaluate_script(function="() => document.title")
|
|
||||||
|
|
||||||
# Async evaluation
|
|
||||||
result = evaluate_script(function="""
|
|
||||||
async () => {
|
|
||||||
const response = await fetch('/loras/api/list');
|
|
||||||
return await response.json();
|
|
||||||
}
|
|
||||||
""")
|
|
||||||
|
|
||||||
# Check element existence
|
|
||||||
exists = evaluate_script(function="""
|
|
||||||
() => document.querySelector('.lora-card') !== null
|
|
||||||
""")
|
|
||||||
|
|
||||||
# Get element count
|
|
||||||
count = evaluate_script(function="""
|
|
||||||
() => document.querySelectorAll('.lora-card').length
|
|
||||||
""")
|
|
||||||
```
|
|
||||||
|
|
||||||
## Network Monitoring
|
|
||||||
|
|
||||||
```python
|
|
||||||
# List all network requests
|
|
||||||
requests = list_network_requests()
|
|
||||||
|
|
||||||
# Filter by resource type
|
|
||||||
xhr_requests = list_network_requests(resourceTypes=["xhr", "fetch"])
|
|
||||||
|
|
||||||
# Get specific request details
|
|
||||||
details = get_network_request(reqid=123)
|
|
||||||
|
|
||||||
# Include preserved requests from previous navigations
|
|
||||||
all_requests = list_network_requests(includePreservedRequests=True)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Console Monitoring
|
|
||||||
|
|
||||||
```python
|
|
||||||
# List all console messages
|
|
||||||
messages = list_console_messages()
|
|
||||||
|
|
||||||
# Filter by type
|
|
||||||
errors = list_console_messages(types=["error", "warn"])
|
|
||||||
|
|
||||||
# Include preserved messages
|
|
||||||
all_messages = list_console_messages(includePreservedMessages=True)
|
|
||||||
|
|
||||||
# Get specific message
|
|
||||||
details = get_console_message(msgid=1)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Performance Testing
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Start trace with page reload
|
|
||||||
performance_start_trace(reload=True, autoStop=False)
|
|
||||||
|
|
||||||
# Start trace without reload
|
|
||||||
performance_start_trace(reload=False, autoStop=True, filePath="trace.json.gz")
|
|
||||||
|
|
||||||
# Stop trace
|
|
||||||
results = performance_stop_trace()
|
|
||||||
|
|
||||||
# Stop and save
|
|
||||||
performance_stop_trace(filePath="trace-results.json.gz")
|
|
||||||
|
|
||||||
# Analyze specific insight
|
|
||||||
insight = performance_analyze_insight(
|
|
||||||
insightSetId="results.insightSets[0].id",
|
|
||||||
insightName="LCPBreakdown"
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Page Management
|
|
||||||
|
|
||||||
```python
|
|
||||||
# List open pages
|
|
||||||
pages = list_pages()
|
|
||||||
|
|
||||||
# Select a page
|
|
||||||
select_page(pageId=0, bringToFront=True)
|
|
||||||
|
|
||||||
# Create new page
|
|
||||||
new_page(url="http://127.0.0.1:{PORT}/loras")
|
|
||||||
|
|
||||||
# Close page (keep at least one open!)
|
|
||||||
close_page(pageId=1)
|
|
||||||
|
|
||||||
# Resize page
|
|
||||||
resize_page(width=1920, height=1080)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Screenshots
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Full page screenshot
|
|
||||||
take_screenshot(fullPage=True)
|
|
||||||
|
|
||||||
# Viewport screenshot
|
|
||||||
take_screenshot()
|
|
||||||
|
|
||||||
# Element screenshot
|
|
||||||
take_screenshot(uid="lora-card-1")
|
|
||||||
|
|
||||||
# Save to file
|
|
||||||
take_screenshot(filePath="screenshots/page.png", format="png")
|
|
||||||
|
|
||||||
# JPEG with quality
|
|
||||||
take_screenshot(filePath="screenshots/page.jpg", format="jpeg", quality=90)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Dialog Handling
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Accept dialog
|
|
||||||
handle_dialog(action="accept")
|
|
||||||
|
|
||||||
# Accept with text input
|
|
||||||
handle_dialog(action="accept", promptText="user input")
|
|
||||||
|
|
||||||
# Dismiss dialog
|
|
||||||
handle_dialog(action="dismiss")
|
|
||||||
```
|
|
||||||
|
|
||||||
## Device Emulation
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Mobile viewport
|
|
||||||
emulate(viewport={"width": 375, "height": 667, "isMobile": True, "hasTouch": True})
|
|
||||||
|
|
||||||
# Tablet viewport
|
|
||||||
emulate(viewport={"width": 768, "height": 1024, "isMobile": True, "hasTouch": True})
|
|
||||||
|
|
||||||
# Desktop viewport
|
|
||||||
emulate(viewport={"width": 1920, "height": 1080})
|
|
||||||
|
|
||||||
# Network throttling
|
|
||||||
emulate(networkConditions="Slow 3G")
|
|
||||||
emulate(networkConditions="Fast 4G")
|
|
||||||
|
|
||||||
# CPU throttling
|
|
||||||
emulate(cpuThrottlingRate=4) # 4x slowdown
|
|
||||||
|
|
||||||
# Geolocation
|
|
||||||
emulate(geolocation={"latitude": 37.7749, "longitude": -122.4194})
|
|
||||||
|
|
||||||
# User agent
|
|
||||||
emulate(userAgent="Mozilla/5.0 (Custom)")
|
|
||||||
|
|
||||||
# Reset emulation
|
|
||||||
emulate(viewport=None, networkConditions="No emulation", userAgent=None)
|
|
||||||
```
|
|
||||||
|
|
||||||
## Drag and Drop
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Drag element to another
|
|
||||||
drag(from_uid="draggable-item", to_uid="drop-zone")
|
|
||||||
```
|
|
||||||
|
|
||||||
## Common LoRa Manager Test Patterns
|
|
||||||
|
|
||||||
### Verify LoRA Cards Loaded
|
|
||||||
|
|
||||||
```python
|
|
||||||
navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
|
||||||
wait_for(text="LoRAs", timeout=10000)
|
|
||||||
|
|
||||||
# Check if cards loaded
|
|
||||||
result = evaluate_script(function="""
|
|
||||||
() => {
|
|
||||||
const cards = document.querySelectorAll('.lora-card');
|
|
||||||
return {
|
|
||||||
count: cards.length,
|
|
||||||
hasData: cards.length > 0
|
|
||||||
};
|
|
||||||
}
|
|
||||||
""")
|
|
||||||
```
|
|
||||||
|
|
||||||
### Search and Verify Results
|
|
||||||
|
|
||||||
```python
|
|
||||||
fill(uid="search-input", value="character")
|
|
||||||
press_key(key="Enter")
|
|
||||||
wait_for(timeout=2000) # Wait for debounce
|
|
||||||
|
|
||||||
# Check results
|
|
||||||
result = evaluate_script(function="""
|
|
||||||
() => {
|
|
||||||
const cards = document.querySelectorAll('.lora-card');
|
|
||||||
const names = Array.from(cards).map(c => c.dataset.name || c.textContent);
|
|
||||||
return { count: cards.length, names };
|
|
||||||
}
|
|
||||||
""")
|
|
||||||
```
|
|
||||||
|
|
||||||
### Check API Response
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Trigger API call
|
|
||||||
evaluate_script(function="""
|
|
||||||
() => window.loraApiCallPromise = fetch('/loras/api/list').then(r => r.json())
|
|
||||||
""")
|
|
||||||
|
|
||||||
# Wait and get result
|
|
||||||
import time
|
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
result = evaluate_script(function="""
|
|
||||||
async () => await window.loraApiCallPromise
|
|
||||||
""")
|
|
||||||
```
|
|
||||||
|
|
||||||
### Monitor Console for Errors
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Before test: clear console (navigate reloads)
|
|
||||||
navigate_page(type="reload")
|
|
||||||
|
|
||||||
# ... perform actions ...
|
|
||||||
|
|
||||||
# Check for errors
|
|
||||||
errors = list_console_messages(types=["error"])
|
|
||||||
assert len(errors) == 0, f"Console errors: {errors}"
|
|
||||||
```
|
|
||||||
|
|
||||||
## Troubleshooting
|
|
||||||
|
|
||||||
### Stale profile lock ("browser is already running" / `list_pages` fails)
|
|
||||||
|
|
||||||
A Chrome profile held by a stale Chrome from a prior MCP session makes `list_pages`
|
|
||||||
fail with "browser is already running". Fix:
|
|
||||||
|
|
||||||
1. Find the stale Chrome that owns the profile dir (e.g. `~/.config/chrome-dev-profile`):
|
|
||||||
```bash
|
|
||||||
ps -ef | grep -i '[c]hrome.*user-data-dir'
|
|
||||||
```
|
|
||||||
2. Confirm it is a QA Chrome from a completed task (NOT the live ComfyUI server, NOT
|
|
||||||
your current MCP instance).
|
|
||||||
3. Kill ONLY that stale Chrome (`kill <stale-pid>`), then retry `list_pages`.
|
|
||||||
|
|
||||||
### Screenshot-write restrictions
|
|
||||||
|
|
||||||
The MCP may refuse to write into paths outside its configured workspace roots
|
|
||||||
(e.g. `.omo/evidence/screenshots/` under a worktree that canonicalizes to an unmapped
|
|
||||||
path). Save the screenshot to `/tmp` via the MCP, then copy it into the evidence dir:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# MCP: take_screenshot(filePath="/tmp/<plan>-e2e/recipe-b-after.png", format="png")
|
|
||||||
# Shell:
|
|
||||||
mkdir -p <repo-root>/.omo/evidence/screenshots
|
|
||||||
cp /tmp/<plan>-e2e/recipe-b-after.png <repo-root>/.omo/evidence/screenshots/
|
|
||||||
```
|
|
||||||
|
|
||||||
### Time budgets & abort rule
|
|
||||||
|
|
||||||
See SKILL.md "Time Budgets & Abort Guidance": if a phase exceeds ~2x its budget or a
|
|
||||||
tool call retries 3+ times in a row, STOP and report BLOCKED with the last observed
|
|
||||||
state (server PID + `ss -tlnp`, page snapshot, last API response). Do not loop.
|
|
||||||
@@ -1,280 +0,0 @@
|
|||||||
# LoRa Manager E2E Test Scenarios
|
|
||||||
|
|
||||||
This document provides detailed test scenarios for end-to-end validation of LoRa Manager features.
|
|
||||||
|
|
||||||
> **Run preconditions (from SKILL.md)**: every run uses the **sandboxed** standalone
|
|
||||||
> server on a free port `{PORT}` (default candidate `8188`, only if actually free — pick
|
|
||||||
> e.g. `8199` when `8188` is occupied by a live ComfyUI). Fixtures live in the sandboxed
|
|
||||||
> `recipes_path` as `f"{id}.recipe.json"` files with matching in-JSON `id`; the real user
|
|
||||||
> config and real library are never touched (record protection proof before/after).
|
|
||||||
> Abort if a phase exceeds ~2x its budget or a tool call retries 3+ times (SKILL.md
|
|
||||||
> "Time Budgets & Abort Guidance").
|
|
||||||
|
|
||||||
## Table of Contents
|
|
||||||
|
|
||||||
1. [LoRA List Page](#lora-list-page)
|
|
||||||
2. [Model Details](#model-details)
|
|
||||||
3. [Recipes](#recipes)
|
|
||||||
4. [Settings](#settings)
|
|
||||||
5. [Import/Export](#importexport)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## LoRA List Page
|
|
||||||
|
|
||||||
### Scenario: Page Load and Display
|
|
||||||
|
|
||||||
**Objective**: Verify the LoRA list page loads correctly and displays models.
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Navigate to `http://127.0.0.1:{PORT}/loras`
|
|
||||||
2. Wait for page title "LoRAs" to appear
|
|
||||||
3. Take snapshot to verify:
|
|
||||||
- Header with "LoRAs" title is visible
|
|
||||||
- Search/filter controls are present
|
|
||||||
- Grid/list view toggle exists
|
|
||||||
- LoRA cards are displayed (if models exist)
|
|
||||||
- Pagination controls (if applicable)
|
|
||||||
|
|
||||||
**Expected Result**: Page loads without errors, UI elements are present.
|
|
||||||
|
|
||||||
### Scenario: Search Functionality
|
|
||||||
|
|
||||||
**Objective**: Verify search filters LoRA models correctly.
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Ensure at least one LoRA exists with known name (e.g., "test-character")
|
|
||||||
2. Navigate to LoRA list page
|
|
||||||
3. Enter search term in search box: "test"
|
|
||||||
4. Press Enter or click search button
|
|
||||||
5. Wait for results to update
|
|
||||||
|
|
||||||
**Expected Result**: Only LoRAs matching search term are displayed.
|
|
||||||
|
|
||||||
**Verification Script**:
|
|
||||||
```python
|
|
||||||
# After search, verify filtered results
|
|
||||||
evaluate_script(function="""
|
|
||||||
() => {
|
|
||||||
const cards = document.querySelectorAll('.lora-card');
|
|
||||||
const names = Array.from(cards).map(c => c.dataset.name);
|
|
||||||
return { count: cards.length, names };
|
|
||||||
}
|
|
||||||
""")
|
|
||||||
```
|
|
||||||
|
|
||||||
### Scenario: Filter by Tags
|
|
||||||
|
|
||||||
**Objective**: Verify tag filtering works correctly.
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Navigate to LoRA list page
|
|
||||||
2. Click on a tag (e.g., "character", "style")
|
|
||||||
3. Wait for filtered results
|
|
||||||
|
|
||||||
**Expected Result**: Only LoRAs with selected tag are displayed.
|
|
||||||
|
|
||||||
### Scenario: View Mode Toggle
|
|
||||||
|
|
||||||
**Objective**: Verify grid/list view toggle works.
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Navigate to LoRA list page
|
|
||||||
2. Click list view button
|
|
||||||
3. Verify list layout
|
|
||||||
4. Click grid view button
|
|
||||||
5. Verify grid layout
|
|
||||||
|
|
||||||
**Expected Result**: View mode changes correctly, layout updates.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Model Details
|
|
||||||
|
|
||||||
### Scenario: Open Model Details
|
|
||||||
|
|
||||||
**Objective**: Verify clicking a LoRA opens its details.
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Navigate to LoRA list page
|
|
||||||
2. Click on a LoRA card
|
|
||||||
3. Wait for details panel/modal to open
|
|
||||||
|
|
||||||
**Expected Result**: Details panel shows:
|
|
||||||
- Model name
|
|
||||||
- Preview image
|
|
||||||
- Metadata (trigger words, tags, etc.)
|
|
||||||
- Action buttons (edit, delete, etc.)
|
|
||||||
|
|
||||||
### Scenario: Edit Model Metadata
|
|
||||||
|
|
||||||
**Objective**: Verify metadata editing works end-to-end.
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Open a LoRA's details
|
|
||||||
2. Click "Edit" button
|
|
||||||
3. Modify trigger words field
|
|
||||||
4. Add/remove tags
|
|
||||||
5. Save changes
|
|
||||||
6. Refresh page
|
|
||||||
7. Reopen the same LoRA
|
|
||||||
|
|
||||||
**Expected Result**: Changes persist after refresh.
|
|
||||||
|
|
||||||
### Scenario: Delete Model
|
|
||||||
|
|
||||||
**Objective**: Verify model deletion works.
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Open a LoRA's details
|
|
||||||
2. Click "Delete" button
|
|
||||||
3. Confirm deletion in dialog
|
|
||||||
4. Wait for removal
|
|
||||||
|
|
||||||
**Expected Result**: Model removed from list, success message shown.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Recipes
|
|
||||||
|
|
||||||
### Scenario: Recipe List Display
|
|
||||||
|
|
||||||
**Objective**: Verify recipes page loads and displays recipes.
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Navigate to `http://127.0.0.1:{PORT}/recipes`
|
|
||||||
2. Wait for "Recipes" title
|
|
||||||
3. Take snapshot
|
|
||||||
|
|
||||||
**Expected Result**: Recipe list displayed with cards/items.
|
|
||||||
|
|
||||||
### Scenario: Create New Recipe
|
|
||||||
|
|
||||||
**Objective**: Verify recipe creation workflow.
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Navigate to recipes page
|
|
||||||
2. Click "New Recipe" button
|
|
||||||
3. Fill recipe form:
|
|
||||||
- Name: "Test Recipe"
|
|
||||||
- Description: "E2E test recipe"
|
|
||||||
- Add LoRA models
|
|
||||||
4. Save recipe
|
|
||||||
5. Verify recipe appears in list
|
|
||||||
|
|
||||||
**Expected Result**: New recipe created and displayed.
|
|
||||||
|
|
||||||
### Scenario: Apply Recipe
|
|
||||||
|
|
||||||
**Objective**: Verify applying a recipe to ComfyUI.
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Open a recipe
|
|
||||||
2. Click "Apply" or "Load in ComfyUI"
|
|
||||||
3. Verify action completes
|
|
||||||
|
|
||||||
**Expected Result**: Recipe applied successfully.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Settings
|
|
||||||
|
|
||||||
### Scenario: Settings Page Load
|
|
||||||
|
|
||||||
**Objective**: Verify settings page displays correctly.
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Navigate to `http://127.0.0.1:{PORT}/settings`
|
|
||||||
2. Wait for "Settings" title
|
|
||||||
3. Take snapshot
|
|
||||||
|
|
||||||
**Expected Result**: Settings form with various options displayed.
|
|
||||||
|
|
||||||
### Scenario: Change Setting and Restart
|
|
||||||
|
|
||||||
**Objective**: Verify settings persist after restart.
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Navigate to settings page
|
|
||||||
2. Change a setting (e.g., default view mode)
|
|
||||||
3. Save settings
|
|
||||||
4. Restart server: `python scripts/start_server.py --port {PORT} --restart --wait --timeout 30 --detach`
|
|
||||||
5. Refresh browser page
|
|
||||||
6. Navigate to settings
|
|
||||||
|
|
||||||
**Expected Result**: Changed setting value persists.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Import/Export
|
|
||||||
|
|
||||||
### Scenario: Export Models List
|
|
||||||
|
|
||||||
**Objective**: Verify export functionality.
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Navigate to LoRA list
|
|
||||||
2. Click "Export" button
|
|
||||||
3. Select format (JSON/CSV)
|
|
||||||
4. Download file
|
|
||||||
|
|
||||||
**Expected Result**: File downloaded with correct data.
|
|
||||||
|
|
||||||
### Scenario: Import Models
|
|
||||||
|
|
||||||
**Objective**: Verify import functionality.
|
|
||||||
|
|
||||||
**Steps**:
|
|
||||||
1. Prepare import file
|
|
||||||
2. Navigate to import page
|
|
||||||
3. Upload file
|
|
||||||
4. Verify import results
|
|
||||||
|
|
||||||
**Expected Result**: Models imported successfully, confirmation shown.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## API Integration Tests
|
|
||||||
|
|
||||||
### Scenario: Verify API Endpoints
|
|
||||||
|
|
||||||
**Objective**: Verify backend API responds correctly.
|
|
||||||
|
|
||||||
**Test via browser console**:
|
|
||||||
```javascript
|
|
||||||
// List LoRAs
|
|
||||||
fetch('/loras/api/list').then(r => r.json()).then(console.log)
|
|
||||||
|
|
||||||
// Get LoRA details
|
|
||||||
fetch('/loras/api/detail/<id>').then(r => r.json()).then(console.log)
|
|
||||||
|
|
||||||
// Search LoRAs
|
|
||||||
fetch('/loras/api/search?q=test').then(r => r.json()).then(console.log)
|
|
||||||
```
|
|
||||||
|
|
||||||
**Expected Result**: APIs return valid JSON with expected structure.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Console Error Monitoring
|
|
||||||
|
|
||||||
During all tests, monitor browser console for errors:
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Check for JavaScript errors
|
|
||||||
messages = list_console_messages(types=["error"])
|
|
||||||
assert len(messages) == 0, f"Console errors found: {messages}"
|
|
||||||
```
|
|
||||||
|
|
||||||
## Network Request Verification
|
|
||||||
|
|
||||||
Verify key API calls are made:
|
|
||||||
|
|
||||||
```python
|
|
||||||
# List XHR requests
|
|
||||||
requests = list_network_requests(resourceTypes=["xhr", "fetch"])
|
|
||||||
|
|
||||||
# Look for specific endpoints
|
|
||||||
lora_list_requests = [r for r in requests if "/api/list" in r.get("url", "")]
|
|
||||||
assert len(lora_list_requests) > 0, "LoRA list API not called"
|
|
||||||
```
|
|
||||||
@@ -1,215 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
Example E2E test demonstrating LoRa Manager testing workflow.
|
|
||||||
|
|
||||||
This script shows how to:
|
|
||||||
1. Start the standalone server
|
|
||||||
2. Use Chrome DevTools MCP to interact with the UI
|
|
||||||
3. Verify functionality end-to-end
|
|
||||||
|
|
||||||
Note: This is a template. Actual execution requires Chrome DevTools MCP.
|
|
||||||
|
|
||||||
Port: pick a FREE port for the run — 8188 is commonly occupied by a live
|
|
||||||
ComfyUI (see the skill's Port Selection section). Set PORT below to e.g. 8199
|
|
||||||
when 8188 is taken. Always run against a SANDBOXED standalone server.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
|
|
||||||
# Choose the E2E port. 8188 is only the default candidate; use 8199 (or any
|
|
||||||
# free port checked with `ss -tlnp`) when 8188 is occupied by a live ComfyUI.
|
|
||||||
PORT = "8188"
|
|
||||||
|
|
||||||
|
|
||||||
def run_test():
|
|
||||||
"""Run example E2E test flow."""
|
|
||||||
|
|
||||||
print("=" * 60)
|
|
||||||
print("LoRa Manager E2E Test Example")
|
|
||||||
print("=" * 60)
|
|
||||||
|
|
||||||
# Step 1: Start server (detached so it survives the shell)
|
|
||||||
print("\n[1/5] Starting LoRa Manager standalone server...")
|
|
||||||
result = subprocess.run(
|
|
||||||
[sys.executable, "start_server.py", "--port", PORT, "--wait", "--timeout", "30", "--detach"],
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
)
|
|
||||||
if result.returncode != 0:
|
|
||||||
print(f"Failed to start server: {result.stderr}")
|
|
||||||
return 1
|
|
||||||
print("Server ready!")
|
|
||||||
|
|
||||||
# Step 2: Open Chrome (manual step - show command)
|
|
||||||
print("\n[2/5] Open Chrome with debug mode:")
|
|
||||||
print(
|
|
||||||
f"google-chrome --remote-debugging-port=9222 "
|
|
||||||
f"--user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:{PORT}/loras"
|
|
||||||
)
|
|
||||||
print("(In actual test, this would be automated via MCP)")
|
|
||||||
|
|
||||||
# Step 3: Navigate and verify page load
|
|
||||||
print("\n[3/5] Page Load Verification:")
|
|
||||||
print(
|
|
||||||
f"""
|
|
||||||
MCP Commands to execute:
|
|
||||||
1. navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
|
||||||
2. wait_for(text="LoRAs", timeout=10000)
|
|
||||||
3. snapshot = take_snapshot()
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
# Step 4: Test search functionality
|
|
||||||
print("\n[4/5] Search Functionality Test:")
|
|
||||||
print(
|
|
||||||
"""
|
|
||||||
MCP Commands to execute:
|
|
||||||
1. fill(uid="search-input", value="test")
|
|
||||||
2. press_key(key="Enter")
|
|
||||||
3. wait_for(text="Results", timeout=5000)
|
|
||||||
4. result = evaluate_script(function=`
|
|
||||||
() => {
|
|
||||||
const cards = document.querySelectorAll('.lora-card');
|
|
||||||
return { count: cards.length };
|
|
||||||
}
|
|
||||||
`)
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
# Step 5: Verify API
|
|
||||||
print("\n[5/5] API Verification:")
|
|
||||||
print(
|
|
||||||
"""
|
|
||||||
MCP Commands to execute:
|
|
||||||
1. api_result = evaluate_script(function=`
|
|
||||||
async () => {
|
|
||||||
const response = await fetch('/loras/api/list');
|
|
||||||
const data = await response.json();
|
|
||||||
return { count: data.length, status: response.status };
|
|
||||||
}
|
|
||||||
`)
|
|
||||||
2. Verify api_result['status'] == 200
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
print("\n" + "=" * 60)
|
|
||||||
print("Test flow completed!")
|
|
||||||
print("=" * 60)
|
|
||||||
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
def example_restart_flow():
|
|
||||||
"""Example: Testing configuration change that requires restart."""
|
|
||||||
|
|
||||||
print("\n" + "=" * 60)
|
|
||||||
print("Example: Server Restart Flow")
|
|
||||||
print("=" * 60)
|
|
||||||
|
|
||||||
print(
|
|
||||||
f"""
|
|
||||||
Scenario: Change setting and verify after restart
|
|
||||||
|
|
||||||
Steps:
|
|
||||||
1. Navigate to settings page
|
|
||||||
- navigate_page(type="url", url="http://127.0.0.1:{PORT}/settings")
|
|
||||||
|
|
||||||
2. Change a setting (e.g., theme)
|
|
||||||
- fill(uid="theme-select", value="dark")
|
|
||||||
- click(uid="save-settings-button")
|
|
||||||
|
|
||||||
3. Restart server
|
|
||||||
- subprocess.run([python, "start_server.py", "--port", "{PORT}", "--restart", "--wait", "--detach"])
|
|
||||||
|
|
||||||
4. Refresh browser
|
|
||||||
- navigate_page(type="reload", ignoreCache=True)
|
|
||||||
- wait_for(text="LoRAs", timeout=15000)
|
|
||||||
|
|
||||||
5. Verify setting persisted
|
|
||||||
- navigate_page(type="url", url="http://127.0.0.1:{PORT}/settings")
|
|
||||||
- theme = evaluate_script(function="() => document.querySelector('#theme-select').value")
|
|
||||||
- assert theme == "dark"
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def example_modal_interaction():
|
|
||||||
"""Example: Testing modal dialog interaction."""
|
|
||||||
|
|
||||||
print("\n" + "=" * 60)
|
|
||||||
print("Example: Modal Dialog Interaction")
|
|
||||||
print("=" * 60)
|
|
||||||
|
|
||||||
print(
|
|
||||||
"""
|
|
||||||
Scenario: Add new LoRA via modal
|
|
||||||
|
|
||||||
Steps:
|
|
||||||
1. Open modal
|
|
||||||
- click(uid="add-lora-button")
|
|
||||||
- wait_for(text="Add LoRA", timeout=3000)
|
|
||||||
|
|
||||||
2. Fill form
|
|
||||||
- fill_form(elements=[
|
|
||||||
{"uid": "lora-name", "value": "Test Character"},
|
|
||||||
{"uid": "lora-path", "value": "/models/test.safetensors"},
|
|
||||||
])
|
|
||||||
|
|
||||||
3. Submit
|
|
||||||
- click(uid="modal-submit-button")
|
|
||||||
|
|
||||||
4. Verify success
|
|
||||||
- wait_for(text="Successfully added", timeout=5000)
|
|
||||||
- snapshot = take_snapshot()
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def example_network_monitoring():
|
|
||||||
"""Example: Network request monitoring."""
|
|
||||||
|
|
||||||
print("\n" + "=" * 60)
|
|
||||||
print("Example: Network Request Monitoring")
|
|
||||||
print("=" * 60)
|
|
||||||
|
|
||||||
print(
|
|
||||||
f"""
|
|
||||||
Scenario: Verify API calls during user interaction
|
|
||||||
|
|
||||||
Steps:
|
|
||||||
1. Clear network log (implicit on navigation)
|
|
||||||
- navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
|
||||||
|
|
||||||
2. Perform action that triggers API call
|
|
||||||
- fill(uid="search-input", value="character")
|
|
||||||
- press_key(key="Enter")
|
|
||||||
|
|
||||||
3. List network requests
|
|
||||||
- requests = list_network_requests(resourceTypes=["xhr", "fetch"])
|
|
||||||
|
|
||||||
4. Find search API call
|
|
||||||
- search_requests = [r for r in requests if "/api/search" in r.get("url", "")]
|
|
||||||
- assert len(search_requests) > 0, "Search API was not called"
|
|
||||||
|
|
||||||
5. Get request details
|
|
||||||
- if search_requests:
|
|
||||||
details = get_network_request(reqid=search_requests[0]["reqid"])
|
|
||||||
- Verify request method, response status, etc.
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
print("LoRa Manager E2E Test Examples\n")
|
|
||||||
print("This script demonstrates E2E testing patterns.\n")
|
|
||||||
print("Note: Actual execution requires Chrome DevTools MCP connection.\n")
|
|
||||||
|
|
||||||
run_test()
|
|
||||||
example_restart_flow()
|
|
||||||
example_modal_interaction()
|
|
||||||
example_network_monitoring()
|
|
||||||
|
|
||||||
print("\n" + "=" * 60)
|
|
||||||
print("All examples shown!")
|
|
||||||
print("=" * 60)
|
|
||||||
@@ -9,7 +9,10 @@ description: Inspect ComfyUI LoRA Manager runtime configuration and local diagno
|
|||||||
|
|
||||||
- Treat runtime state as local user data. Prefer read-only inspection unless the user explicitly asks for mutation.
|
- Treat runtime state as local user data. Prefer read-only inspection unless the user explicitly asks for mutation.
|
||||||
- Never print secret-like settings values. Redact keys containing `key`, `token`, `secret`, `password`, `auth`, or `credential`, including `civitai_api_key`.
|
- Never print secret-like settings values. Redact keys containing `key`, `token`, `secret`, `password`, `auth`, or `credential`, including `civitai_api_key`.
|
||||||
- Resolve paths from the runtime configuration before guessing. In this environment the settings file is normally `/home/miao/.config/ComfyUI-LoRA-Manager/settings.json`, but portable settings can override this through the repository `settings.json`.
|
- Resolve paths from the runtime configuration before guessing. Settings-directory precedence (highest first):
|
||||||
|
1. **Explicit override** — env `LORA_MANAGER_SETTINGS_DIR` or standalone `--settings-path` (also accepted by the inspect script as `--settings-path DIR`). Pins EVERYTHING (`settings.json`, `cache/`, `wildcards/`, `backups/`, `logs/`, `stats/`) under the given directory; bypasses portable mode and the user config dir. Common when inspecting a sandboxed/E2E instance.
|
||||||
|
2. **Portable** — repository `<repo-root>/settings.json` with `"use_portable_settings": true` (or `LORA_MANAGER_PORTABLE=1`): settings dir = `<repo-root>`.
|
||||||
|
3. **Default** — `~/.config/ComfyUI-LoRA-Manager` on this machine (`platformdirs.user_config_dir("ComfyUI-LoRA-Manager", appauthor=False)`).
|
||||||
- Use the active library when selecting per-library caches and paths. Read `active_library` from settings; fall back to `default` if missing.
|
- Use the active library when selecting per-library caches and paths. Read `active_library` from settings; fall back to `default` if missing.
|
||||||
- Normalize and expand `~` before comparing paths. Symlinks are common in this repo.
|
- Normalize and expand `~` before comparing paths. Symlinks are common in this repo.
|
||||||
|
|
||||||
@@ -32,9 +35,17 @@ python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_conte
|
|||||||
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py sqlite --db /path/to/cache.sqlite --limit 3
|
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py sqlite --db /path/to/cache.sqlite --limit 3
|
||||||
```
|
```
|
||||||
|
|
||||||
|
To inspect a sandboxed/E2E instance that pins its settings directory:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# --settings-path DIR (or LORA_MANAGER_SETTINGS_DIR) works with every subcommand:
|
||||||
|
python .agents/skills/lora-manager-runtime-context/scripts/inspect_runtime_context.py \
|
||||||
|
--settings-path /tmp/opencode/<plan>-e2e/settings summary
|
||||||
|
```
|
||||||
|
|
||||||
## Runtime Path Rules
|
## Runtime Path Rules
|
||||||
|
|
||||||
- Settings directory: use `py/utils/settings_paths.py`. Default platform path is `platformdirs.user_config_dir("ComfyUI-LoRA-Manager", appauthor=False)`.
|
- Settings directory: resolve via `py/utils/settings_paths.py` — `get_settings_dir()` honors the `LORA_MANAGER_SETTINGS_DIR` / programmatic override first, then portable mode, then `platformdirs.user_config_dir("ComfyUI-LoRA-Manager", appauthor=False)`. The inspect script mirrors this precedence in `resolve_settings_path()`.
|
||||||
- Settings file: `<settings_dir>/settings.json`.
|
- Settings file: `<settings_dir>/settings.json`.
|
||||||
- Cache root: `<settings_dir>/cache`.
|
- Cache root: `<settings_dir>/cache`.
|
||||||
- Canonical cache files:
|
- Canonical cache files:
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from typing import Any
|
|||||||
|
|
||||||
SECRET_PATTERN = re.compile(r"(key|token|secret|password|auth|credential)", re.IGNORECASE)
|
SECRET_PATTERN = re.compile(r"(key|token|secret|password|auth|credential)", re.IGNORECASE)
|
||||||
APP_NAME = "ComfyUI-LoRA-Manager"
|
APP_NAME = "ComfyUI-LoRA-Manager"
|
||||||
|
SETTINGS_DIR_ENV = "LORA_MANAGER_SETTINGS_DIR"
|
||||||
CACHE_SQLITE = {
|
CACHE_SQLITE = {
|
||||||
"model": ("model", "{library}.sqlite"),
|
"model": ("model", "{library}.sqlite"),
|
||||||
"recipe": ("recipe", "{library}.sqlite"),
|
"recipe": ("recipe", "{library}.sqlite"),
|
||||||
@@ -30,6 +31,15 @@ CACHE_JSON = {
|
|||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = argparse.ArgumentParser(description="Inspect LoRA Manager runtime state read-only.")
|
parser = argparse.ArgumentParser(description="Inspect LoRA Manager runtime state read-only.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--settings-path",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
metavar="DIR",
|
||||||
|
help="Explicit settings directory (same as LORA_MANAGER_SETTINGS_DIR / "
|
||||||
|
"standalone --settings-path). Overrides portable mode and the default "
|
||||||
|
"user config dir.",
|
||||||
|
)
|
||||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||||
|
|
||||||
subparsers.add_parser("summary", help="Print redacted settings and resolved paths.")
|
subparsers.add_parser("summary", help="Print redacted settings and resolved paths.")
|
||||||
@@ -44,6 +54,8 @@ def main() -> int:
|
|||||||
sqlite_parser.add_argument("--limit", type=int, default=3, help="Rows to sample from each user table.")
|
sqlite_parser.add_argument("--limit", type=int, default=3, help="Rows to sample from each user table.")
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
if args.settings_path:
|
||||||
|
os.environ[SETTINGS_DIR_ENV] = args.settings_path
|
||||||
context = build_context()
|
context = build_context()
|
||||||
|
|
||||||
if args.command == "summary":
|
if args.command == "summary":
|
||||||
@@ -78,6 +90,11 @@ def build_context() -> dict[str, Any]:
|
|||||||
|
|
||||||
|
|
||||||
def resolve_settings_path() -> Path:
|
def resolve_settings_path() -> Path:
|
||||||
|
# Explicit override: LORA_MANAGER_SETTINGS_DIR env or --settings-path.
|
||||||
|
explicit = os.environ.get(SETTINGS_DIR_ENV)
|
||||||
|
if explicit:
|
||||||
|
return Path(explicit).expanduser() / "settings.json"
|
||||||
|
|
||||||
repo_root = find_repo_root()
|
repo_root = find_repo_root()
|
||||||
portable = repo_root / "settings.json"
|
portable = repo_root / "settings.json"
|
||||||
if portable.exists():
|
if portable.exists():
|
||||||
|
|||||||
@@ -2,6 +2,10 @@
|
|||||||
|
|
||||||
This file provides guidance for agentic coding assistants working in this repository.
|
This file provides guidance for agentic coding assistants working in this repository.
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
ComfyUI LoRA Manager is a comprehensive LoRA management system for ComfyUI that combines a Python backend with browser-based widgets. It provides model organization, downloading from CivitAI/CivArchive, recipe management, and one-click workflow integration.
|
||||||
|
|
||||||
## Development Commands
|
## Development Commands
|
||||||
|
|
||||||
### Backend Development
|
### Backend Development
|
||||||
@@ -28,16 +32,21 @@ COVERAGE_FILE=coverage/backend/.coverage pytest \
|
|||||||
--cov=py --cov=standalone \
|
--cov=py --cov=standalone \
|
||||||
--cov-report=term-missing \
|
--cov-report=term-missing \
|
||||||
--cov-report=html:coverage/backend/html \
|
--cov-report=html:coverage/backend/html \
|
||||||
--cov-report=xml:coverage/backend/coverage.xml
|
--cov-report=xml:coverage/backend/coverage.xml \
|
||||||
|
--cov-report=json:coverage/backend/coverage.json
|
||||||
```
|
```
|
||||||
|
|
||||||
### Frontend Development (LoRA Manager Web UI)
|
### Frontend Development (LoRA Manager Web UI)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
# Install dependencies (root and Vue widgets)
|
||||||
npm install
|
npm install
|
||||||
|
cd vue-widgets && npm install && cd ..
|
||||||
|
|
||||||
npm test # Run all tests (JS + Vue)
|
npm test # Run all tests (JS + Vue)
|
||||||
npm run test:js # Run JS tests only
|
npm run test:js # Run JS tests only
|
||||||
npm run test:watch # Watch mode
|
npm run test:vue # Run Vue widget tests only
|
||||||
|
npm run test:watch # Watch mode (JS tests only)
|
||||||
npm run test:coverage # Generate coverage report
|
npm run test:coverage # Generate coverage report
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -54,88 +63,201 @@ npm run test:watch # Watch mode
|
|||||||
npm run test:coverage # Generate coverage report
|
npm run test:coverage # Generate coverage report
|
||||||
```
|
```
|
||||||
|
|
||||||
## Python Code Style
|
### Localization
|
||||||
|
|
||||||
### Imports & Formatting
|
```bash
|
||||||
|
# Sync translation keys after UI string updates
|
||||||
|
python scripts/sync_translation_keys.py
|
||||||
|
```
|
||||||
|
|
||||||
|
Locale files are in `locales/` (en, zh-CN, zh-TW, ja, ko, fr, de, es, ru, he).
|
||||||
|
|
||||||
|
After adding keys to `en.json` and syncing, **stop**: the `[TODO: Translate]` placeholders in
|
||||||
|
the other locales are the expected end state during feature development. Do NOT translate
|
||||||
|
proactively — translate only when the feature owner explicitly asks (see
|
||||||
|
`docs/i18n-translation-guidelines.md` §7).
|
||||||
|
|
||||||
|
**Before translating anything, read `docs/i18n-translation-guidelines.md`** — it defines the
|
||||||
|
term conventions (e.g. "Recipe" stays untranslated in French, 配方 in Chinese; model-type and
|
||||||
|
brand names are never translated), per-locale preferred renderings, placeholder rules, and
|
||||||
|
the known confusion hot-spots.
|
||||||
|
|
||||||
|
## Code Style
|
||||||
|
|
||||||
|
### Python
|
||||||
|
|
||||||
|
#### Imports & Formatting
|
||||||
|
|
||||||
- Use `from __future__ import annotations` for forward references
|
- Use `from __future__ import annotations` for forward references
|
||||||
- Group imports: standard library, third-party, local (blank line separated)
|
- Group imports: standard library, third-party, local (blank line separated)
|
||||||
|
- Use `TYPE_CHECKING` guard for type-checking-only imports
|
||||||
- Absolute imports within `py/`: `from ..services import X`
|
- Absolute imports within `py/`: `from ..services import X`
|
||||||
- PEP 8 with 4-space indentation, type hints required
|
- PEP 8 with 4-space indentation, type hints required
|
||||||
|
|
||||||
### Naming Conventions
|
#### Naming Conventions
|
||||||
|
|
||||||
- Files: `snake_case.py`, Classes: `PascalCase`, Functions/vars: `snake_case`
|
- Files: `snake_case.py`, Classes: `PascalCase`, Functions/vars: `snake_case`
|
||||||
- Constants: `UPPER_SNAKE_CASE`, Private: `_protected`, `__mangled`
|
- Constants: `UPPER_SNAKE_CASE`, Private: `_protected`, `__mangled`
|
||||||
|
|
||||||
### Error Handling & Async
|
#### Error Handling & Async
|
||||||
|
|
||||||
- Use `logging.getLogger(__name__)`, define custom exceptions in `py/services/errors.py`
|
- Use `logging.getLogger(__name__)`, define custom exceptions in `py/services/errors.py`
|
||||||
- `async def` for I/O, `@pytest.mark.asyncio` for async tests
|
- `async def` for I/O, `@pytest.mark.asyncio` for async tests
|
||||||
- Singleton with `asyncio.Lock`: see `ModelScanner.get_instance()`
|
- Singleton with `asyncio.Lock`: see `ModelScanner.get_instance()`
|
||||||
- Return `aiohttp.web.json_response` or `web.Response`
|
- Return `aiohttp.web.json_response` or `web.Response`
|
||||||
|
|
||||||
### Testing
|
### JavaScript/TypeScript
|
||||||
|
|
||||||
- `pytest` with `--import-mode=importlib`
|
#### Imports & Modules
|
||||||
- Fixtures in `tests/conftest.py`, use `tmp_path_factory` for isolation
|
|
||||||
- Mark tests needing real paths: `@pytest.mark.no_settings_dir_isolation`
|
|
||||||
- Mock ComfyUI dependencies via conftest patterns
|
|
||||||
|
|
||||||
## JavaScript/TypeScript Code Style
|
|
||||||
|
|
||||||
### Imports & Modules
|
|
||||||
|
|
||||||
- ES modules: `import { app } from "../../scripts/app.js"` for ComfyUI
|
- ES modules: `import { app } from "../../scripts/app.js"` for ComfyUI
|
||||||
- Vue: `import { ref, computed } from 'vue'`, type imports: `import type { Foo }`
|
- Vue: `import { ref, computed } from 'vue'`, type imports: `import type { Foo }`
|
||||||
- Export named functions: `export function foo() {}`
|
- Export named functions: `export function foo() {}`
|
||||||
|
|
||||||
### Naming & Formatting
|
#### Naming & Formatting
|
||||||
|
|
||||||
- camelCase for functions/vars/props, PascalCase for classes
|
- camelCase for functions/vars/props, PascalCase for classes
|
||||||
- Constants: `UPPER_SNAKE_CASE`, Files: `snake_case.js` or `kebab-case.js`
|
- Constants: `UPPER_SNAKE_CASE`, Files: `snake_case.js` or `kebab-case.js`
|
||||||
- 2-space indentation preferred (follow existing file conventions)
|
- 2-space indentation preferred (follow existing file conventions)
|
||||||
- Vue Single File Components: `<script setup lang="ts">` preferred
|
- Vue Single File Components: `<script setup lang="ts">` preferred
|
||||||
|
|
||||||
### Widget Development
|
#### Widget Development
|
||||||
|
|
||||||
|
- Prefer vanilla JS for `web/comfyui/` widgets; avoid framework dependencies (except the Vue widgets in `vue-widgets/`)
|
||||||
- ComfyUI: `app.registerExtension()`, `node.addDOMWidget(name, type, element, options)`
|
- ComfyUI: `app.registerExtension()`, `node.addDOMWidget(name, type, element, options)`
|
||||||
- Event handlers via `addEventListener` or widget callbacks
|
- Event handlers via `addEventListener` or widget callbacks
|
||||||
- Shared utilities: `web/comfyui/utils.js`
|
- Shared utilities: `web/comfyui/utils.js`
|
||||||
- Dual-mode rendering patterns (canvas vs Vue): see `docs/comfyui-dual-mode-widgets.md`
|
- Dual-mode rendering patterns (canvas vs Vue): see `docs/comfyui-dual-mode-widgets.md`
|
||||||
|
|
||||||
### Vue Composables Pattern
|
#### Vue Composables Pattern
|
||||||
|
|
||||||
- Use composition API: `useXxxState(widget)`, return reactive refs and methods
|
- Use composition API: `useXxxState(widget)`, return reactive refs and methods
|
||||||
- Guard restoration loops with flag: `let isRestoring = false`
|
- Guard restoration loops with flag: `let isRestoring = false`
|
||||||
- Build config from state: `const buildConfig = (): Config => { ... }`
|
- Build config from state: `const buildConfig = (): Config => { ... }`
|
||||||
|
|
||||||
## Architecture Patterns
|
## Architecture
|
||||||
|
|
||||||
|
### Dual Mode Operation
|
||||||
|
|
||||||
|
The system runs in two modes:
|
||||||
|
- **ComfyUI plugin mode**: Integrates with ComfyUI's PromptServer, uses `folder_paths` for model discovery
|
||||||
|
- **Standalone mode**: `standalone.py` mocks ComfyUI dependencies, reads paths from `settings.json`
|
||||||
|
- Detection: `os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"`
|
||||||
|
|
||||||
|
### Backend Entry Points
|
||||||
|
|
||||||
|
- `__init__.py` — ComfyUI plugin entry: registers nodes via `NODE_CLASS_MAPPINGS`, sets `WEB_DIRECTORY`, calls `LoraManager.add_routes()`
|
||||||
|
- `standalone.py` — Standalone server: mocks `folder_paths` and node modules, starts aiohttp server
|
||||||
|
- `py/lora_manager.py` — Main `LoraManager` class that registers all HTTP routes
|
||||||
|
|
||||||
### Service Layer
|
### Service Layer
|
||||||
|
|
||||||
- `ServiceRegistry` singleton for DI, services use `get_instance()` classmethod
|
- `ServiceRegistry` singleton for DI, services use `get_instance()` classmethod
|
||||||
|
- `BaseModelService` abstract base → `LoraService`, `CheckpointService`, `EmbeddingService`
|
||||||
|
- `ModelScanner` base → `LoraScanner`, `CheckpointScanner`, `EmbeddingScanner` for file discovery with hash-based deduplication
|
||||||
|
- `PersistentModelCache` (SQLite) for metadata persistence
|
||||||
|
- `MetadataSyncService` — background sync from CivitAI/CivArchive APIs
|
||||||
|
- `SettingsManager` — settings with schema migration support
|
||||||
|
- `WebSocketManager` — real-time progress broadcasting
|
||||||
|
- `ModelServiceFactory` — creates the right service for each model type
|
||||||
|
- Use cases in `py/services/use_cases/` orchestrate complex business logic (auto-organize, bulk refresh, downloads)
|
||||||
- Separate scanners (discovery) from services (business logic)
|
- Separate scanners (discovery) from services (business logic)
|
||||||
- Handlers in `py/routes/handlers/` are pure functions with deps as params
|
- Handlers in `py/routes/handlers/` are pure functions with deps as params
|
||||||
|
|
||||||
### Model Types & Routes
|
### Model Types & Routes
|
||||||
|
|
||||||
- `BaseModelService` base for LoRA, Checkpoint, Embedding
|
- API endpoints follow `/loras/*`, `/checkpoints/*`, `/embeddings/*`, `/other/*` patterns
|
||||||
- `ModelScanner` for file discovery, hash deduplication
|
- Route registrars organize endpoints by domain: `ModelRouteRegistrar`, `RecipeRouteRegistrar`, etc.
|
||||||
- `PersistentModelCache` (SQLite) for persistence
|
- Request handlers in `py/routes/handlers/` implement route logic
|
||||||
- Route registrars: `ModelRouteRegistrar`, endpoints: `/loras/*`, `/checkpoints/*`, `/embeddings/*`
|
- All routes use aiohttp, return `web.json_response` or `web.Response`
|
||||||
- WebSocket via `WebSocketManager` for real-time updates
|
- Endpoints consumed by the companion browser extension (lm-civitai-extension)
|
||||||
|
MUST also accept `GET` with query-string params: the extension is GET-only by
|
||||||
|
convention (see its AGENTS.md), even for state-changing operations such as
|
||||||
|
`GET /api/lm/recipe/{recipe_id}/reimport`
|
||||||
|
|
||||||
### Recipe System
|
### Recipe System
|
||||||
|
|
||||||
- Base: `py/recipes/base.py`, Enrichment: `RecipeEnrichmentService`
|
- Base: `py/recipes/base.py`, Enrichment: `RecipeEnrichmentService` in `py/recipes/enrichment.py`
|
||||||
- Parsers: `py/recipes/parsers/`
|
- Parsers: `py/recipes/parsers/` for PNG metadata, JSON, and workflow formats
|
||||||
|
|
||||||
|
### Custom Nodes
|
||||||
|
|
||||||
|
- Location: `py/nodes/`, all nodes registered in `__init__.py`
|
||||||
|
- Each node class has a `NAME` class attribute used as key in `NODE_CLASS_MAPPINGS`
|
||||||
|
- Standard ComfyUI node pattern: `INPUT_TYPES()` classmethod, `RETURN_TYPES`, `FUNCTION`
|
||||||
|
|
||||||
|
### Configuration
|
||||||
|
|
||||||
|
- `py/config.py` manages folder paths for models and handles symlink mappings
|
||||||
|
- Auto-saves paths to `settings.json` in ComfyUI mode
|
||||||
|
- `settings.json.example` is intentionally minimal (see Important Notes); all
|
||||||
|
other defaults live in `DEFAULT_SETTINGS` (`py/services/settings_manager.py`)
|
||||||
|
|
||||||
|
### Frontend UI Architecture
|
||||||
|
|
||||||
|
#### 1. LoRA Manager Web UI
|
||||||
|
- Location: `./static/` (JS/CSS) and `./templates/` (HTML)
|
||||||
|
- Tech: Vanilla JS + CSS, served by the hosting server (ComfyUI app in plugin mode, `standalone.py` in standalone mode)
|
||||||
|
- Tests: `tests/frontend/**/*.test.js` (vitest + jsdom)
|
||||||
|
|
||||||
|
#### 2. ComfyUI Custom Node Widgets
|
||||||
|
- Location: `./web/comfyui/` (Vanilla JS) + `./vue-widgets/` (Vue)
|
||||||
|
- Primary styles: `./web/comfyui/lm_styles.css` (NOT `./static/css/`)
|
||||||
|
- Vue widgets: Vue 3 + TypeScript + PrimeVue + vue-i18n, e.g. `LoraPoolWidget`, `LoraRandomizerWidget`, `LoraCyclerWidget`, `AutocompleteTextWidget`
|
||||||
|
- Vue builds to `./web/comfyui/vue-widgets/`; auto-built on ComfyUI startup via `py/vue_widget_builder.py`, typecheck via `vue-tsc`
|
||||||
|
- Widget registration: `app.registerExtension()` and `getCustomWidgets` hooks; `node.addDOMWidget(...)` embeds HTML in LiteGraph nodes
|
||||||
|
- See `docs/dom_widget_dev_guide.md` for the DOMWidget development guide
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
### Backend (pytest)
|
||||||
|
|
||||||
|
- Config in `pytest.ini`: `--import-mode=importlib`, testpaths=`tests`
|
||||||
|
- Fixtures in `tests/conftest.py` mock ComfyUI dependencies; use `tmp_path_factory` for isolation
|
||||||
|
- Markers: `@pytest.mark.asyncio`, `@pytest.mark.no_settings_dir_isolation` (tests needing real settings paths)
|
||||||
|
|
||||||
|
### Frontend (vitest)
|
||||||
|
|
||||||
|
- Vanilla JS tests: `tests/frontend/**/*.test.js` with jsdom; setup in `tests/frontend/setup.js`
|
||||||
|
- Vue widget tests: `vue-widgets/tests/**/*.test.ts` with jsdom + `@vue/test-utils`
|
||||||
|
|
||||||
|
### UI Verification (manual default)
|
||||||
|
|
||||||
|
UI/layout changes are verified by the user by eye — do NOT spin up a sandbox,
|
||||||
|
standalone server, or browser automation to "prove" a visual fix. Ask the user to
|
||||||
|
look instead. The full browser E2E ceremony (server + Chrome DevTools MCP +
|
||||||
|
screenshots) is slow, token-heavy, and fragile; reserve it for genuine
|
||||||
|
server+browser integration bugs, and only when the user explicitly agrees.
|
||||||
|
|
||||||
|
If a cross-layer issue ever needs a live server, the sandboxed helpers live in
|
||||||
|
`scripts/e2e/` (`start_server.py`, `wait_for_server.py`). Non-negotiable rules:
|
||||||
|
|
||||||
|
- Always launch with `--settings-path <sandbox>/settings` and sandboxed
|
||||||
|
`folder_paths` under `/tmp` — the repo folder is the real plugin folder and a
|
||||||
|
`settings.json` there is read by the live instance. Never touch real config or
|
||||||
|
real model libraries.
|
||||||
|
- Never kill a process you did not start; `start_server.py` tracks its own PIDs
|
||||||
|
via pidfile and refuses to touch unrelated processes on the port.
|
||||||
|
- Abort after ~30 minutes or 3 consecutive tool failures; report `BLOCKED` with
|
||||||
|
observed state instead of retrying blindly. Clean up sandbox and server after.
|
||||||
|
|
||||||
|
## Key Integration Points
|
||||||
|
|
||||||
|
- **Settings:** Stored in the user config directory (via `platformdirs`) or portable mode (`"use_portable_settings": true`)
|
||||||
|
- **CivitAI/CivArchive:** API clients for metadata sync and model downloads; CivitAI API key stored in settings
|
||||||
|
- **Symlinks:** Config scans symlinks to map virtual→physical paths; fingerprinting prevents redundant rescans
|
||||||
|
- **WebSocket:** Broadcasts real-time progress for downloads, scans, and metadata sync
|
||||||
|
- **Model scanning flow:** Walk folders → compute hashes → deduplicate → extract safetensors metadata → cache in SQLite → background CivitAI sync → WebSocket broadcast
|
||||||
|
|
||||||
## Important Notes
|
## Important Notes
|
||||||
|
|
||||||
- ALWAYS use English for comments (per copilot-instructions.md)
|
- ALWAYS use English for comments (per copilot-instructions.md)
|
||||||
- Dual mode: ComfyUI plugin (folder_paths) vs standalone (settings.json)
|
- **`settings.json.example` must stay minimal**: only `use_portable_settings`,
|
||||||
- Detection: `os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"`
|
`civitai_api_key`, and the four core `folder_paths` keys (`loras`,
|
||||||
|
`checkpoints`, `unet`, `embeddings`). Do NOT add optional/default keys
|
||||||
|
(model-category folders, `default_*_root`, `auto_organize_exclusions`, etc.)
|
||||||
|
to this file unless the user explicitly asks for it. Defaults belong in
|
||||||
|
`DEFAULT_SETTINGS` in `py/services/settings_manager.py`.
|
||||||
- Run `python scripts/sync_translation_keys.py` after adding UI strings to `locales/en.json`
|
- Run `python scripts/sync_translation_keys.py` after adding UI strings to `locales/en.json`
|
||||||
- Symlinks require normalized paths.
|
- Symlinks require normalized paths.
|
||||||
**Business paths vs real paths**: All stored paths and operation routing use the
|
**Business paths vs real paths**: All stored paths and operation routing use the
|
||||||
@@ -143,23 +265,4 @@ npm run test:coverage # Generate coverage report
|
|||||||
resolved. `os.path.realpath` is only for scanner dedup and the symlink cache.
|
resolved. `os.path.realpath` is only for scanner dedup and the symlink cache.
|
||||||
Any path passed to `os.remove`/`os.rename`/`shutil.move` or validated by a
|
Any path passed to `os.remove`/`os.rename`/`shutil.move` or validated by a
|
||||||
containment check MUST use the business path (i.e. `os.path.abspath`, not
|
containment check MUST use the business path (i.e. `os.path.abspath`, not
|
||||||
`realpath`).
|
`realpath`).
|
||||||
|
|
||||||
## Git / Commit Messages
|
|
||||||
|
|
||||||
- Follow the style of recent repository commits when writing commit messages
|
|
||||||
- Prefer the repo's existing `feat(...)`, `fix(...)`, `chore:` style where applicable
|
|
||||||
- If the user has provided a GitHub issue link or issue ID for the task, mention that issue in the commit message, for example `(#871)`
|
|
||||||
- When unrelated local changes exist, stage and commit only the files relevant to the requested task
|
|
||||||
|
|
||||||
## Frontend UI Architecture
|
|
||||||
|
|
||||||
### 1. LoRA Manager Web UI
|
|
||||||
- Location: `./static/` and `./templates/`
|
|
||||||
- Tech: Vanilla JS + CSS, served by the hosting server (ComfyUI app in plugin mode, `standalone.py` in standalone mode)
|
|
||||||
- Tests via npm in root directory
|
|
||||||
|
|
||||||
### 2. ComfyUI Custom Node Widgets
|
|
||||||
- Location: `./web/comfyui/` (Vanilla JS) + `./vue-widgets/` (Vue)
|
|
||||||
- Primary styles: `./web/comfyui/lm_styles.css` (NOT `./static/css/`)
|
|
||||||
- Vue builds to `./web/comfyui/vue-widgets/`, typecheck via `vue-tsc`
|
|
||||||
@@ -1,189 +0,0 @@
|
|||||||
# CLAUDE.md
|
|
||||||
|
|
||||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
ComfyUI LoRA Manager is a comprehensive LoRA management system for ComfyUI that combines a Python backend with browser-based widgets. It provides model organization, downloading from CivitAI/CivArchive, recipe management, and one-click workflow integration.
|
|
||||||
|
|
||||||
## Development Commands
|
|
||||||
|
|
||||||
### Backend
|
|
||||||
|
|
||||||
```bash
|
|
||||||
pip install -r requirements.txt
|
|
||||||
pip install -r requirements-dev.txt
|
|
||||||
|
|
||||||
# Run standalone server (port 8188 by default)
|
|
||||||
python standalone.py --port 8188
|
|
||||||
|
|
||||||
# Run all backend tests
|
|
||||||
pytest
|
|
||||||
|
|
||||||
# Run specific test file or function
|
|
||||||
pytest tests/test_recipes.py
|
|
||||||
pytest tests/test_recipes.py::test_function_name
|
|
||||||
|
|
||||||
# Run backend tests with coverage
|
|
||||||
COVERAGE_FILE=coverage/backend/.coverage pytest \
|
|
||||||
--cov=py \
|
|
||||||
--cov=standalone \
|
|
||||||
--cov-report=term-missing \
|
|
||||||
--cov-report=html:coverage/backend/html \
|
|
||||||
--cov-report=xml:coverage/backend/coverage.xml \
|
|
||||||
--cov-report=json:coverage/backend/coverage.json
|
|
||||||
```
|
|
||||||
|
|
||||||
### Frontend
|
|
||||||
|
|
||||||
There are three test suites run by `npm test`: vanilla JS tests (vitest at root) and Vue widget tests (`vue-widgets/` vitest).
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npm install
|
|
||||||
cd vue-widgets && npm install && cd ..
|
|
||||||
|
|
||||||
# Run all frontend tests (JS + Vue)
|
|
||||||
npm test
|
|
||||||
|
|
||||||
# Run only vanilla JS tests
|
|
||||||
npm run test:js
|
|
||||||
|
|
||||||
# Run only Vue widget tests
|
|
||||||
npm run test:vue
|
|
||||||
|
|
||||||
# Watch mode (JS tests only)
|
|
||||||
npm run test:watch
|
|
||||||
|
|
||||||
# Frontend coverage
|
|
||||||
npm run test:coverage
|
|
||||||
|
|
||||||
# Build Vue widgets (output to web/comfyui/vue-widgets/)
|
|
||||||
cd vue-widgets && npm run build
|
|
||||||
|
|
||||||
# Vue widget dev mode (watch + rebuild)
|
|
||||||
cd vue-widgets && npm run dev
|
|
||||||
|
|
||||||
# Typecheck Vue widgets
|
|
||||||
cd vue-widgets && npm run typecheck
|
|
||||||
```
|
|
||||||
|
|
||||||
### Localization
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Sync translation keys after UI string updates
|
|
||||||
python scripts/sync_translation_keys.py
|
|
||||||
```
|
|
||||||
|
|
||||||
Locale files are in `locales/` (en, zh-CN, zh-TW, ja, ko, fr, de, es, ru, he).
|
|
||||||
|
|
||||||
## Architecture
|
|
||||||
|
|
||||||
### Dual Mode Operation
|
|
||||||
|
|
||||||
The system runs in two modes:
|
|
||||||
- **ComfyUI plugin mode**: Integrates with ComfyUI's PromptServer, uses `folder_paths` for model discovery
|
|
||||||
- **Standalone mode**: `standalone.py` mocks ComfyUI dependencies, reads paths from `settings.json`
|
|
||||||
- Detection: `os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"`
|
|
||||||
|
|
||||||
### Backend (Python)
|
|
||||||
|
|
||||||
**Entry points:**
|
|
||||||
- `__init__.py` — ComfyUI plugin entry: registers nodes via `NODE_CLASS_MAPPINGS`, sets `WEB_DIRECTORY`, calls `LoraManager.add_routes()`
|
|
||||||
- `standalone.py` — Standalone server: mocks `folder_paths` and node modules, starts aiohttp server
|
|
||||||
- `py/lora_manager.py` — Main `LoraManager` class that registers all HTTP routes
|
|
||||||
|
|
||||||
**Service layer** (`py/services/`):
|
|
||||||
- `ServiceRegistry` singleton for dependency injection; services follow `get_instance()` singleton pattern
|
|
||||||
- `BaseModelService` abstract base → `LoraService`, `CheckpointService`, `EmbeddingService`
|
|
||||||
- `ModelScanner` base → `LoraScanner`, `CheckpointScanner`, `EmbeddingScanner` for file discovery with hash-based deduplication
|
|
||||||
- `PersistentModelCache` — SQLite-based metadata cache
|
|
||||||
- `MetadataSyncService` — Background sync from CivitAI/CivArchive APIs
|
|
||||||
- `SettingsManager` — Settings with schema migration support
|
|
||||||
- `WebSocketManager` — Real-time progress broadcasting
|
|
||||||
- `ModelServiceFactory` — Creates the right service for each model type
|
|
||||||
- Use cases in `py/services/use_cases/` orchestrate complex business logic (auto-organize, bulk refresh, downloads)
|
|
||||||
|
|
||||||
**Routes** (`py/routes/`):
|
|
||||||
- Route registrars organize endpoints by domain: `ModelRouteRegistrar`, `RecipeRouteRegistrar`, etc.
|
|
||||||
- Request handlers in `py/routes/handlers/` implement route logic
|
|
||||||
- API endpoints follow `/loras/*`, `/checkpoints/*`, `/embeddings/*` patterns
|
|
||||||
- All routes use aiohttp, return `web.json_response` or `web.Response`
|
|
||||||
|
|
||||||
**Recipe system** (`py/recipes/`):
|
|
||||||
- `base.py` — Recipe metadata structure
|
|
||||||
- `enrichment.py` — Enriches recipes with model metadata
|
|
||||||
- `parsers/` — Parsers for PNG metadata, JSON, and workflow formats
|
|
||||||
|
|
||||||
**Custom nodes** (`py/nodes/`):
|
|
||||||
- Each node class has a `NAME` class attribute used as key in `NODE_CLASS_MAPPINGS`
|
|
||||||
- Standard ComfyUI node pattern: `INPUT_TYPES()` classmethod, `RETURN_TYPES`, `FUNCTION`
|
|
||||||
- All nodes registered in `__init__.py`
|
|
||||||
|
|
||||||
**Configuration** (`py/config.py`):
|
|
||||||
- Manages folder paths for models, handles symlink mappings
|
|
||||||
- Auto-saves paths to settings.json in ComfyUI mode
|
|
||||||
|
|
||||||
### Frontend — Two Distinct UI Systems
|
|
||||||
|
|
||||||
#### 1. Standalone Manager Web UI
|
|
||||||
- **Location:** `static/` (JS/CSS) and `templates/` (HTML)
|
|
||||||
- **Tech:** Vanilla JS + CSS, served by standalone server
|
|
||||||
- **Structure:** `static/js/core.js` (shared), `loras.js`, `checkpoints.js`, `embeddings.js`, `recipes.js`, `statistics.js`
|
|
||||||
- **Tests:** `tests/frontend/**/*.test.js` (vitest + jsdom)
|
|
||||||
|
|
||||||
#### 2. ComfyUI Custom Node Widgets
|
|
||||||
- **Vanilla JS widgets:** `web/comfyui/*.js` — ES modules extending ComfyUI's LiteGraph UI
|
|
||||||
- `loras_widget.js` / `loras_widget_events.js` — Main LoRA selection widget
|
|
||||||
- `autocomplete.js` — Trigger word and embedding autocomplete
|
|
||||||
- `preview_tooltip.js` — Model card preview tooltips
|
|
||||||
- `top_menu_extension.js` — "Launch LoRA Manager" menu item
|
|
||||||
- `utils.js` — Shared utilities and API helpers
|
|
||||||
- Widget styling in `web/comfyui/lm_styles.css` (NOT `static/css/`)
|
|
||||||
- **Vue widgets:** `vue-widgets/src/` → built to `web/comfyui/vue-widgets/`
|
|
||||||
- Vue 3 + TypeScript + PrimeVue + vue-i18n
|
|
||||||
- Vite build with CSS-injected-by-JS plugin
|
|
||||||
- Components: `LoraPoolWidget`, `LoraRandomizerWidget`, `LoraCyclerWidget`, `AutocompleteTextWidget`
|
|
||||||
- Auto-built on ComfyUI startup via `py/vue_widget_builder.py`
|
|
||||||
- Tests: `vue-widgets/tests/**/*.test.ts` (vitest)
|
|
||||||
|
|
||||||
**Widget registration pattern:**
|
|
||||||
- Widgets use `app.registerExtension()` and `getCustomWidgets` hooks
|
|
||||||
- `node.addDOMWidget(name, type, element, options)` embeds HTML in LiteGraph nodes
|
|
||||||
- See `docs/dom_widget_dev_guide.md` for DOMWidget development guide
|
|
||||||
|
|
||||||
## Code Style
|
|
||||||
|
|
||||||
**Python:**
|
|
||||||
- PEP 8, 4-space indentation, English comments only
|
|
||||||
- Use `from __future__ import annotations` for forward references
|
|
||||||
- Use `TYPE_CHECKING` guard for type-checking-only imports
|
|
||||||
- Loggers via `logging.getLogger(__name__)`
|
|
||||||
- Custom exceptions in `py/services/errors.py`
|
|
||||||
- Async patterns: `async def` for I/O, `@pytest.mark.asyncio` for async tests
|
|
||||||
- Singleton pattern with class-level `asyncio.Lock` (see `ModelScanner.get_instance()`)
|
|
||||||
|
|
||||||
**JavaScript:**
|
|
||||||
- ES modules, camelCase functions/variables, PascalCase classes
|
|
||||||
- Widget files use `*_widget.js` suffix
|
|
||||||
- Prefer vanilla JS for `web/comfyui/` widgets, avoid framework dependencies (except Vue widgets)
|
|
||||||
|
|
||||||
## Testing
|
|
||||||
|
|
||||||
**Backend (pytest):**
|
|
||||||
- Config in `pytest.ini`: `--import-mode=importlib`, testpaths=`tests`
|
|
||||||
- Fixtures in `tests/conftest.py` handle ComfyUI dependency mocking
|
|
||||||
- Markers: `@pytest.mark.asyncio`, `@pytest.mark.no_settings_dir_isolation`
|
|
||||||
- Uses `tmp_path_factory` for directory isolation
|
|
||||||
|
|
||||||
**Frontend (vitest):**
|
|
||||||
- Vanilla JS tests: `tests/frontend/**/*.test.js` with jsdom
|
|
||||||
- Vue widget tests: `vue-widgets/tests/**/*.test.ts` with jsdom + @vue/test-utils
|
|
||||||
- Setup in `tests/frontend/setup.js`
|
|
||||||
|
|
||||||
## Key Integration Points
|
|
||||||
|
|
||||||
- **Settings:** Stored in user directory (via `platformdirs`) or portable mode (`"use_portable_settings": true`)
|
|
||||||
- **CivitAI/CivArchive:** API clients for metadata sync and model downloads; CivitAI API key in settings
|
|
||||||
- **Symlink handling:** Config scans symlinks to map virtual→physical paths; fingerprinting prevents redundant rescans
|
|
||||||
- **WebSocket:** Broadcasts real-time progress for downloads, scans, and metadata sync
|
|
||||||
- **Model scanning flow:** Walk folders → compute hashes → deduplicate → extract safetensors metadata → cache in SQLite → background CivitAI sync → WebSocket broadcast
|
|
||||||
-10
@@ -3,8 +3,6 @@ try: # pragma: no cover - import fallback for pytest collection
|
|||||||
from .py.nodes.lora_loader import LoraLoaderLM, LoraTextLoaderLM
|
from .py.nodes.lora_loader import LoraLoaderLM, LoraTextLoaderLM
|
||||||
from .py.nodes.checkpoint_loader import CheckpointLoaderLM
|
from .py.nodes.checkpoint_loader import CheckpointLoaderLM
|
||||||
from .py.nodes.unet_loader import UNETLoaderLM
|
from .py.nodes.unet_loader import UNETLoaderLM
|
||||||
from .py.nodes.random_checkpoint_loader import RandomCheckpointLoaderLM
|
|
||||||
from .py.nodes.random_unet_loader import RandomUNETLoaderLM
|
|
||||||
from .py.nodes.trigger_word_toggle import TriggerWordToggleLM
|
from .py.nodes.trigger_word_toggle import TriggerWordToggleLM
|
||||||
from .py.nodes.prompt import PromptLM
|
from .py.nodes.prompt import PromptLM
|
||||||
from .py.nodes.text import TextLM
|
from .py.nodes.text import TextLM
|
||||||
@@ -42,12 +40,6 @@ except (
|
|||||||
"py.nodes.checkpoint_loader"
|
"py.nodes.checkpoint_loader"
|
||||||
).CheckpointLoaderLM
|
).CheckpointLoaderLM
|
||||||
UNETLoaderLM = importlib.import_module("py.nodes.unet_loader").UNETLoaderLM
|
UNETLoaderLM = importlib.import_module("py.nodes.unet_loader").UNETLoaderLM
|
||||||
RandomCheckpointLoaderLM = importlib.import_module(
|
|
||||||
"py.nodes.random_checkpoint_loader"
|
|
||||||
).RandomCheckpointLoaderLM
|
|
||||||
RandomUNETLoaderLM = importlib.import_module(
|
|
||||||
"py.nodes.random_unet_loader"
|
|
||||||
).RandomUNETLoaderLM
|
|
||||||
TriggerWordToggleLM = importlib.import_module(
|
TriggerWordToggleLM = importlib.import_module(
|
||||||
"py.nodes.trigger_word_toggle"
|
"py.nodes.trigger_word_toggle"
|
||||||
).TriggerWordToggleLM
|
).TriggerWordToggleLM
|
||||||
@@ -87,8 +79,6 @@ NODE_CLASS_MAPPINGS = {
|
|||||||
LoraTextLoaderLM.NAME: LoraTextLoaderLM,
|
LoraTextLoaderLM.NAME: LoraTextLoaderLM,
|
||||||
CheckpointLoaderLM.NAME: CheckpointLoaderLM,
|
CheckpointLoaderLM.NAME: CheckpointLoaderLM,
|
||||||
UNETLoaderLM.NAME: UNETLoaderLM,
|
UNETLoaderLM.NAME: UNETLoaderLM,
|
||||||
RandomCheckpointLoaderLM.NAME: RandomCheckpointLoaderLM,
|
|
||||||
RandomUNETLoaderLM.NAME: RandomUNETLoaderLM,
|
|
||||||
TriggerWordToggleLM.NAME: TriggerWordToggleLM,
|
TriggerWordToggleLM.NAME: TriggerWordToggleLM,
|
||||||
LoraStackerLM.NAME: LoraStackerLM,
|
LoraStackerLM.NAME: LoraStackerLM,
|
||||||
LoraStackCombinerLM.NAME: LoraStackCombinerLM,
|
LoraStackCombinerLM.NAME: LoraStackCombinerLM,
|
||||||
|
|||||||
+427
-395
File diff suppressed because it is too large
Load Diff
@@ -54,7 +54,7 @@ The dedicated services encapsulate long-running work so handlers stay thin.
|
|||||||
| Use case | Entry point | Dependencies | Guarantees |
|
| Use case | Entry point | Dependencies | Guarantees |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `RecipeAnalysisService` | `analyze_uploaded_image`, `analyze_remote_image`, `analyze_local_image`, `analyze_widget_metadata` | `ExifUtils`, `RecipeParserFactory`, downloader factory, optional metadata collector/processor | Normalises missing/invalid payloads into `RecipeValidationError`; generates consistent fingerprint data to keep duplicate detection stable; temporary files are cleaned up after every analysis path. |
|
| `RecipeAnalysisService` | `analyze_uploaded_image`, `analyze_remote_image`, `analyze_local_image`, `analyze_widget_metadata` | `ExifUtils`, `RecipeParserFactory`, downloader factory, optional metadata collector/processor | Normalises missing/invalid payloads into `RecipeValidationError`; generates consistent fingerprint data to keep duplicate detection stable; temporary files are cleaned up after every analysis path. |
|
||||||
| `RecipePersistenceService` | `save_recipe`, `delete_recipe`, `update_recipe`, `reconnect_lora`, `bulk_delete`, `save_recipe_from_widget` | `ExifUtils`, recipe scanner, card preview sizing constants | Writes images/JSON metadata atomically; updates scanner caches and hash indices before returning; recalculates fingerprints whenever LoRA assignments change. |
|
| `RecipePersistenceService` | `save_recipe`, `delete_recipe`, `update_recipe`, `reconnect_lora`, `get_reconnect_suggestions`, `bulk_delete`, `save_recipe_from_widget` | `ExifUtils`, recipe scanner, card preview sizing constants | Writes images/JSON metadata atomically; updates scanner caches and hash indices before returning; recalculates fingerprints whenever LoRA assignments change. |
|
||||||
| `RecipeSharingService` | `share_recipe`, `prepare_download` | `tempfile`, recipe scanner | Copies originals to TTL-managed temp files; metadata lookups re-use the scanner; expired shares trigger cleanup and `RecipeNotFoundError`. |
|
| `RecipeSharingService` | `share_recipe`, `prepare_download` | `tempfile`, recipe scanner | Copies originals to TTL-managed temp files; metadata lookups re-use the scanner; expired shares trigger cleanup and `RecipeNotFoundError`. |
|
||||||
|
|
||||||
## Maintaining critical invariants
|
## Maintaining critical invariants
|
||||||
|
|||||||
@@ -0,0 +1,441 @@
|
|||||||
|
# i18n Translation Guidelines
|
||||||
|
|
||||||
|
This document is the canonical set of conventions for translating LoRA Manager UI strings.
|
||||||
|
It applies to **human translators and AI agents** alike. Read it before editing anything in
|
||||||
|
`locales/`.
|
||||||
|
|
||||||
|
Source of truth: `locales/en.json` (10 locales, 1982 leaf keys; all locales share the exact
|
||||||
|
same key structure).
|
||||||
|
|
||||||
|
Locales: `en`, `zh-CN`, `zh-TW`, `ja`, `ko`, `fr`, `de`, `es`, `ru`, `he` (RTL).
|
||||||
|
|
||||||
|
> **Status (2026-08 sweep):** a full audit was executed and the terminology, placeholder,
|
||||||
|
> stale-text, and untranslated-block fixes described in §2–§6 were applied across all locales
|
||||||
|
> (commits `3c3ac49f` … `fd1227d3`). The tables below are now the **normative target state**,
|
||||||
|
> not a to-do list — future edits should preserve these renderings and only add what is new.
|
||||||
|
>
|
||||||
|
> **Status (2026-09, Other Models):** the `other` model type (VAE / Upscaler / Text Encoder /
|
||||||
|
> CLIP Vision / ControlNet) and the Other Models opt-in toggles added 36 new keys; all of them
|
||||||
|
> are now translated in all 9 locales (terminology in §2 "Other Models feature"). There are no
|
||||||
|
> remaining `[TODO: Translate]` placeholders in any locale.
|
||||||
|
>
|
||||||
|
> **Status (2026-09, revision):** `other.disabled.description`, `banners.otherModels.content` and
|
||||||
|
> `settings.folderSettings.enableOtherModelsHelp` were refreshed in `en.json` to name all five
|
||||||
|
> sub_types (they had listed four, which read as "these are what enabling manages") and
|
||||||
|
> re-translated in all 9 locales in the same pass. `clip_vision` and `controlnet` are now both
|
||||||
|
> opt-in, so the first two describe **capability** and the third the **master switch**, not the
|
||||||
|
> default set — keep all three enumerating the full five (`VAE / upscaler / text encoder /
|
||||||
|
> CLIP vision / ControlNet` in `en`; locale slash-list casing follows each file's existing
|
||||||
|
> `VAE / Upscaler / Text Encoder / …` style, de compounds as `CLIP-Vision- und ControlNet-Ordner`).
|
||||||
|
>
|
||||||
|
> **Status (2026-09, "no folders found" state):** the Other Models page gained an *enabled but
|
||||||
|
> nothing to scan* empty state with 6 new keys (`other.noPaths.*`); translated in all 9 locales
|
||||||
|
> in the same pass. The `folder_paths` JSON snippet shown in that state lives in
|
||||||
|
> `templates/other.html`, **not** in the locale files, so it is never translated — only the
|
||||||
|
> surrounding prose is. Terminology added in §2.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Hard rules (do not violate)
|
||||||
|
|
||||||
|
### R1 — Key structure is sacred
|
||||||
|
- Only `locales/en.json` may add/remove/rename keys. All other locales must keep the exact
|
||||||
|
same nested key set. `tests/i18n/test_i18n.py` enforces this.
|
||||||
|
- When a new UI string is added to `en.json`, run
|
||||||
|
`python scripts/sync_translation_keys.py` (adds the missing keys to all locales with
|
||||||
|
`[TODO: Translate]` placeholder copies) — **then stop**. Do NOT translate proactively:
|
||||||
|
placeholders are the expected end state during feature development, and translations are
|
||||||
|
filled in only when the feature owner explicitly asks (workflow details in §7).
|
||||||
|
- Never reorder, re-indent, or reformat a locale file "for tidiness". The sync script
|
||||||
|
preserves formatting; manual reformatting creates noisy diffs.
|
||||||
|
|
||||||
|
### R2 — Placeholders and HTML must be preserved verbatim
|
||||||
|
- `{name}`-style placeholders must appear in the translation exactly as in `en.json`.
|
||||||
|
Do not invent placeholders the source string does not have — the caller may not pass them
|
||||||
|
(example bug: `zh-CN recipes.controls.import.downloadLocationPreview` added `{path}`; the
|
||||||
|
template renders this key with no parameters, so the literal text `{path}` shows in the UI).
|
||||||
|
- `{{...}}` in a locale value is an escaped literal brace — keep it identical.
|
||||||
|
- Keep embedded HTML tags (e.g. `<strong>...</strong>`, `<code>...</code>`) intact.
|
||||||
|
You may move the tag around the sentence if the target language needs different word order.
|
||||||
|
|
||||||
|
### R3 — Never translate or transliterate these
|
||||||
|
- Model types: **LoRA, Checkpoint, Embedding, Diffusion Model**
|
||||||
|
- Products/brands: **LoRA Manager, ComfyUI, CivitAI, CivArchive, HuggingFace, Ko-fi**
|
||||||
|
- Ecosystem names: **LyCORIS, DoRA**, trigger-adjacent jargon **Prompt, Workflow**
|
||||||
|
(these are used as-is in the target-language SD community; see §2 per-language policy)
|
||||||
|
- Theme names: **Nord, Midnight, Monokai, Dracula, Solarized**
|
||||||
|
|
||||||
|
### R4 — The "Recipe" convention (the most important domain term)
|
||||||
|
Product intent: a *Recipe* records a **LoRA combination + generation parameters**
|
||||||
|
(prompt, seed, sampler, …) that reproduces an image style. The metaphor is a **cooking
|
||||||
|
recipe** — "follow it and you get a similar dish". It is **not** a menu, not a dish list,
|
||||||
|
not a prescription.
|
||||||
|
|
||||||
|
Decision per language — translate only into a word whose everyday primary meaning is a
|
||||||
|
cooking recipe; where that word would mislead users, **keep the English "Recipe(s)"**:
|
||||||
|
|
||||||
|
| Locale | Use | Never use |
|
||||||
|
|---|---|---|
|
||||||
|
| fr | **Recipe / Recipes** (keep English) | recette(s) — cooking reading is secondary and it was explicitly judged misleading |
|
||||||
|
| zh-CN / zh-TW | 配方 | 食谱 (reads as "food cookbook") |
|
||||||
|
| ja | レシピ | — (leftover English "Recipe" in `initialization.recipes.title` / `toast.recipes.recipeSaved` → translate) |
|
||||||
|
| ko | 레시피 | — |
|
||||||
|
| de | Rezept / Rezepte | — (cooking meaning dominant; prescription reading acceptable) |
|
||||||
|
| es | receta / recetas | — (cooking meaning dominant) |
|
||||||
|
| ru | рецепт / рецепты | — (leftover English "Recipe" in `initialization.recipes.title` / `toast.recipes.recipeSaved` → translate) |
|
||||||
|
| he | מתכון / מתכונים | — (cooking meaning dominant) |
|
||||||
|
|
||||||
|
Whatever the choice, **one concept = one noun within a locale**. Currently violated in:
|
||||||
|
- `fr` — "Recipe" (~97 keys, incl. nav) mixed with "recette" (~58 keys)
|
||||||
|
- `zh-CN` / `zh-TW` — 配方 (126/122 keys) mixed with 食谱 / 食譜 (14/17 keys, all in the
|
||||||
|
*rematch* flow: `globalContextMenu.rematchRecipes.*`, `toast.recipes.rematch*`)
|
||||||
|
- `de` — "Rezept" (136 keys) mixed with leftover English "Recipe" (5 keys)
|
||||||
|
- `ja` / `ru` — leftover English "Recipe" in `initialization.recipes.title` ("Recipe Manager
|
||||||
|
zu initialisieren" / «Инициализация Recipe Manager») and `toast.recipes.recipeSaved`
|
||||||
|
|
||||||
|
### R5 — One term, one rendering (within each locale)
|
||||||
|
Same source word must not be translated several ways in one file. Known offender areas
|
||||||
|
(see §5 for the full fix list): recipe, Checkpoint, Embedding, prompt, base model, preset,
|
||||||
|
workflow, hash, metadata, tags, bulk. Every locale currently mixes variants of at least one
|
||||||
|
of these — pick the preferred form in the §2 tables and normalize.
|
||||||
|
|
||||||
|
### R6 — Register consistency
|
||||||
|
- `zh-CN` / `zh-TW`: pick 你 or 您 once. Do not mix (zh-CN has 44×你 + 5×您; zh-TW has
|
||||||
|
27×您 + 18×你).
|
||||||
|
- `de`: pick "du" or "Sie" once (currently 143×Sie + ~7×du).
|
||||||
|
- `es`: pick "tú" or "usted" once.
|
||||||
|
|
||||||
|
### R7 — Punctuation per script
|
||||||
|
- Full-width punctuation `:()` is correct **only in CJK locales** (zh-CN, zh-TW, ja, ko).
|
||||||
|
- Latin/Cyrillic/Hebrew locales must use ASCII `: ()` — full-width colons leaked in there
|
||||||
|
are machine-translation artifacts. Known: `fr toast.recipes.createError/createFailed`,
|
||||||
|
`es toast.recipes.createError/createFailed` (e.g. "…de la receta:" should be "…de la receta:").
|
||||||
|
- `fr` apostrophes must be U+2019 `'` / ASCII `'`, never a straight double quote:
|
||||||
|
`fr header.filter.allowSellingGeneratedContentTooltip` currently reads
|
||||||
|
`vendre d"images` → fix to `d'images`. Do not mix `'` and `'` in one file (fr has 299 vs 15).
|
||||||
|
- Ellipsis: use ASCII `...` (project style). Don't introduce `…`.
|
||||||
|
- Keep the sentence-ending period/omission consistent with the source string where the
|
||||||
|
language allows it.
|
||||||
|
- `he` is RTL: mix of Hebrew and Latin scripts is normal; keep Latin term ordering natural.
|
||||||
|
|
||||||
|
### R8 — No untranslated English leftovers
|
||||||
|
Full sentences left byte-identical to `en.json` are bugs (brand names and URL placeholders
|
||||||
|
are the exception). Every locale has them; see §6 for the per-locale checklist.
|
||||||
|
`[TODO: Translate]` placeholders are the sanctioned intermediate state during feature
|
||||||
|
development (see §7) — do not "fix" them unless the feature owner asked for translations.
|
||||||
|
|
||||||
|
### R9 — Mirror the source even when the source is wrong
|
||||||
|
If `en.json` itself contains an inconsistency (e.g. the `Civitai` vs `CivitAI` casing split,
|
||||||
|
or the `CivitArchive` typo in `modals.relinkCivitai.helpText.format4`), translate/transcribe
|
||||||
|
it as-is in your locale and instead **fix the source** in `en.json` (then propagate by
|
||||||
|
re-syncing and re-translating affected keys). Do not silently diverge in one locale only.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Per-language term maps
|
||||||
|
|
||||||
|
Preferred rendering per term. "Fix" means the locale currently contains the wrong variant
|
||||||
|
and must be normalized. `en` = keep the English word as-is.
|
||||||
|
|
||||||
|
### fr
|
||||||
|
|
||||||
|
| Term | Use | Fix |
|
||||||
|
|---|---|---|
|
||||||
|
| recipe | Recipe(s) | Replace all "recette(s)" (58 keys, e.g. `recipes.actions.deleteRecipeWithShortcut`, `toast.recipes.rematchComplete`) with "Recipe(s)" |
|
||||||
|
| Checkpoint | Checkpoint | `statistics.modelTypes.checkpoint` = "Point de contrôle" → "Checkpoint" |
|
||||||
|
| trigger words | mot(s)-clé(s) | unify: `modals.model.triggerWords.editWord` uses "mot déclencheur" — pick one |
|
||||||
|
| prompt / negative prompt | Prompt / prompt négatif | — |
|
||||||
|
| base model | modèle(s) de base | — |
|
||||||
|
| preset | préréglage | unify: `modals.model.usageTips.addPresetParameter` "prédéfini", `toast.presets.restored` "par défaut" |
|
||||||
|
| hash | hash | `conflictConfirm.message` "hachage" → "hash" |
|
||||||
|
| tags | tags | `settings.sections.priorityTags` "Étiquettes" → "Tags" |
|
||||||
|
| metadata | métadonnées | `loras.controls.refresh.fullTooltip` keeps English "metadata" |
|
||||||
|
| duplicates | doublon(s) | unify with "dupliqué(e)s" |
|
||||||
|
| bulk | groupé(e) | unify with "par lot / mode lot" variants |
|
||||||
|
|
||||||
|
### de
|
||||||
|
|
||||||
|
| Term | Use | Fix |
|
||||||
|
|---|---|---|
|
||||||
|
| recipe | Rezept/Rezepte | leftover English "Recipe" keys → Rezept (e.g. `toast.recipes.recipeSaved`) |
|
||||||
|
| base model | pick Basis-Modell or Basismodell | currently 27× hyphenated vs 15× closed |
|
||||||
|
| metadata | Metadaten | 4 keys use "Modelldaten" (`onboarding.steps.fetch.title/content`) → Metadaten |
|
||||||
|
| bulk | pick Massen- or Sammelmodus | `loras.controls.bulk.action` = "Massen" reads as "crowds" — use "Massenbearbeitung"/"Mehrfachauswahl" |
|
||||||
|
| register | Sie (formal) | 7 keys use "du/dein" (`settings.backup.managementHelp`, `modals.checkUpdates.message/tip`, `doctor.footer`, …) |
|
||||||
|
|
||||||
|
### es
|
||||||
|
|
||||||
|
| Term | Use | Fix |
|
||||||
|
|---|---|---|
|
||||||
|
| recipe | receta(s) | — |
|
||||||
|
| Checkpoint | Checkpoint | 5 statistics keys "Punto(s) de control" → "Checkpoints" (`statistics.metrics.checkpoints`, `statistics.insights.unusedCheckpoints.*`, `statistics.modelTypes.checkpoint`) |
|
||||||
|
| trigger words | palabra(s) de activación | 2 keys already use it; ~15 keys "palabra(s) clave" (reads as search keyword) → unify |
|
||||||
|
| base model | modelo base | — |
|
||||||
|
| preset | preajuste | 3 keys keep English "preset", 1 "preestablecido" → preajuste |
|
||||||
|
| workflow | pick flujo de trabajo or workflow | currently 21× "flujo de trabajo" vs 10× "workflow" |
|
||||||
|
| bulk | masivo / por lotes | unify; "Batch Import" → traducción |
|
||||||
|
| tags | etiquetas | — |
|
||||||
|
|
||||||
|
### ru
|
||||||
|
|
||||||
|
| Term | Use | Fix |
|
||||||
|
|---|---|---|
|
||||||
|
| recipe | рецепт(ы) | English leftovers: `initialization.recipes.title`, `recipes.batchImport.*`, `toast.recipes.recipeSaved` → translate |
|
||||||
|
| Checkpoint | Checkpoint (recommended) | 3 variants today: "Checkpoint" (17 keys), «Чекпойнт», «Контрольная точка» (statistics, 6 keys) — statistics MUST drop «Контрольная точка» |
|
||||||
|
| Embedding | Embedding | «Эмбеддинг» variant exists in `settings.priorityTags.modelTypes.embedding` — unify |
|
||||||
|
| prompt | промпт | 8 keys use «запрос» (reads as "database/HTTP request") → «промпт» |
|
||||||
|
| base model | базовая модель | — |
|
||||||
|
| preset | пресет | `header.theme.presets` "Предустановки" → пресеты |
|
||||||
|
| workflow | Workflow (recommended) | «рабочий процесс» used in 4 keys — unify |
|
||||||
|
| hash | pick хеш or хэш | both spellings co-occur |
|
||||||
|
| tag(s) | тег(и) | — |
|
||||||
|
| typos | — | `settings.misc.loraSyntaxFormatHelp`: «безпотерьного» → «беспотерьного» |
|
||||||
|
|
||||||
|
### he
|
||||||
|
|
||||||
|
| Term | Use | Fix |
|
||||||
|
|---|---|---|
|
||||||
|
| recipe | מתכון / מתכונים | — |
|
||||||
|
| Checkpoint | Checkpoint | 5 statistics keys «נקודת/נקודות ביקורת» (road/security checkpoint) → "Checkpoint(s)" (`statistics.metrics.checkpoints`, `statistics.modelTypes.checkpoint`, `statistics.insights.unusedCheckpoints.*`) |
|
||||||
|
| Embedding | Embedding | `statistics` keys use הטמעות → Embedding |
|
||||||
|
| prompt | pick הנחיה or פרומפט | 9 keys הנחיה vs 3 פרומפט — unify (recommend פרומפט, SD-community loanword) |
|
||||||
|
| preset | קביעה מראש | `header.filter.presetOverwriteConfirm` uses פריסט → unify |
|
||||||
|
| hash | pick one of האש / גיבוב / hash | 3 variants co-occur — unify (recommend hash or גיבוב) |
|
||||||
|
| metadata | pick מטא-דאטה or מטא-נתונים | 38 vs 17 keys — unify |
|
||||||
|
| model | מודל | 13 keys use דגם/דגמים — unify |
|
||||||
|
| bulk | pick one of 5 variants | 5 different renderings ("כמות גדולה", "המוני", "קבוצתי", "אצווה", …) — unify; `loras.controls.bulk.action` "כמות גדולה" reads as "large quantity" |
|
||||||
|
|
||||||
|
### ja
|
||||||
|
|
||||||
|
| Term | Use | Fix |
|
||||||
|
|---|---|---|
|
||||||
|
| recipe | レシピ | `initialization.recipes.title` keeps English "Recipe Manager" — translate to レシピマネージャー |
|
||||||
|
| Checkpoint | Checkpoint or チェックポイント (pick one) | 3 variants: Checkpoint (~14), checkpoint lowercase (4), チェックポイント (4, e.g. `settings.priorityTags.modelTypes.checkpoint`) |
|
||||||
|
| Embedding | Embedding | 4 keys lowercase "embedding" mid-sentence |
|
||||||
|
| bulk | 一括 | `modals.checkUpdates.tip` "バルクモード" → 一括モード |
|
||||||
|
| recipe counter | 件 or 個 | `globalContextMenu.rematchRecipes.success` uses 件, `.cancelled` uses 個 — unify |
|
||||||
|
|
||||||
|
### ko
|
||||||
|
|
||||||
|
| Term | Use | Fix |
|
||||||
|
|---|---|---|
|
||||||
|
| recipe | 레시피 | — |
|
||||||
|
| Checkpoint | Checkpoint (recommended) | 4 keys transliterate 체크포인트 (`settings.priorityTags.modelTypes.checkpoint`, `toast.recipes.missingCheckpointPath/missingCheckpointInfo/downloadCheckpointFailed`) |
|
||||||
|
| Embedding | Embedding | 3 keys 임베딩 (`settings.priorityTags.modelTypes.embedding`, `uiHelpers.nodeSelector.embedding`) |
|
||||||
|
| base model | 베이스 모델 | 6 keys «기본 모델» read as "default model" → 베이스 모델 (`settings.downloadSkipBaseModels.*`, `toast.loras.downloadSkippedByBaseModel`) |
|
||||||
|
| workflow | pick 워크플로 or 워크플로우 | 26 vs 6 keys — unify |
|
||||||
|
| bulk | 일괄 | `modals.checkUpdates.tip` "벌크 모드" → 일괄 모드 |
|
||||||
|
| tag logic | — | `header.filter.tagLogicAny` = "모든 태그 일치 (OR)" is **inverted** (should be "하나 이상의 태그 일치") and identical to `tagLogicAll` |
|
||||||
|
| particle | — | `modelCard.sendToWorkflow.checkpointNotImplemented`: "Checkpoint을" → "Checkpoint를" |
|
||||||
|
|
||||||
|
### zh-CN / zh-TW
|
||||||
|
|
||||||
|
| Term | zh-CN | zh-TW |
|
||||||
|
|---|---|---|
|
||||||
|
| recipe | 配方 (fix 食谱 → 配方, 14 keys in rematch flow) | 配方 (fix 食譜 → 配方, 17 keys in rematch flow) |
|
||||||
|
| Checkpoint | Checkpoint (fix 检查点 → Checkpoint, 5 keys: `toast.recipes.missingCheckpointPath/missingCheckpointInfo/downloadCheckpointFailed`, `modelCard.actions.checkpointNameCopied`, `modelCard.sendToWorkflow.checkpointNotImplemented`) | Checkpoint (fix 檢查點 → Checkpoint, 4 keys: `modelCard.actions.copyCheckpointName`, `toast.recipes.missing*`×2, `toast.recipes.downloadCheckpointFailed`) |
|
||||||
|
| base model | 基础模型 (fix 基模型 → 基础模型, 3 keys in `modals.model.versions.filters.*`) | 基礎模型 ✓ consistent |
|
||||||
|
| prompt | 提示词 ✓ | 提示詞 ✓ |
|
||||||
|
| preset | 预设 ✓ | 預設 ✓ |
|
||||||
|
| workflow | 工作流 ✓ | 工作流 ✓ |
|
||||||
|
| trigger words | 触发词 ✓ | 觸發詞 ✓ |
|
||||||
|
| hash | 哈希 (哈希值 variant OK) | 雜湊 ✓ |
|
||||||
|
| register | 你 (fix 5×您 → 你) | 您 (fix 18×你 → 您) |
|
||||||
|
|
||||||
|
### Other Models feature (VAE / Upscaler / Text Encoder / CLIP Vision / ControlNet)
|
||||||
|
|
||||||
|
The `other` model type exposes five sub_types. They are **model-type names**, so they follow
|
||||||
|
R3 and stay in Latin in every locale. The `settings.folderSettings.subType*` values are
|
||||||
|
therefore **intentionally byte-identical to `en.json`** (same precedent as
|
||||||
|
`settings.priorityTags.modelTypes` / `checkpoints.modelTypes.checkpoint`) — a §6 sweep must
|
||||||
|
not "fix" them.
|
||||||
|
|
||||||
|
| Term | Rendering | Note |
|
||||||
|
|---|---|---|
|
||||||
|
| VAE | `VAE` everywhere | acronym, always upper-case |
|
||||||
|
| Upscaler | `Upscaler` everywhere | CivitAI `ModelType` name |
|
||||||
|
| Text Encoder | `Text Encoder` everywhere | de compounds as `Text-Encoder-Stammordner` |
|
||||||
|
| CLIP Vision | `CLIP Vision` everywhere | de compounds as `CLIP-Vision-Stammordner` |
|
||||||
|
| ControlNet | `ControlNet` everywhere | brand casing, capital N |
|
||||||
|
|
||||||
|
In prose these names sit next to localized nouns the same way `Diffusion Model` does
|
||||||
|
(zh `VAE 根目录`, ja `VAEルート`, ko `VAE 루트`, ru `Корневая папка VAE`).
|
||||||
|
|
||||||
|
**"Other Models" is the page/feature name, not a model type — translate it:**
|
||||||
|
|
||||||
|
| Locale | `other.title` | `header.navigation.other` |
|
||||||
|
|---|---|---|
|
||||||
|
| fr | Autres modèles | Autres |
|
||||||
|
| zh-CN | 其他模型 | 其他 |
|
||||||
|
| zh-TW | 其他模型 | 其他 |
|
||||||
|
| ja | その他のモデル | その他 |
|
||||||
|
| ko | 기타 모델 | 기타 |
|
||||||
|
| de | Weitere Modelle | Andere |
|
||||||
|
| es | Otros modelos | Otros |
|
||||||
|
| ru | Другие модели | Другое |
|
||||||
|
| he | מודלים אחרים | אחרים |
|
||||||
|
|
||||||
|
`settings.folderSettings.otherSubTypes` ("Managed Types") must name **model** types, matching
|
||||||
|
each locale's `header.filter.modelTypes` rendering (zh `管理的模型类型`, ja `管理するモデルタイプ`,
|
||||||
|
de `Verwaltete Modelltypen`, …).
|
||||||
|
|
||||||
|
The "no folders found" empty state (`other.noPaths.*`) uses two phrases that must stay
|
||||||
|
consistent whenever that copy is edited. `folder key` means the `folder_paths` key name
|
||||||
|
(`vae`, `upscale_models`, … — Latin per the table above); `on disk` means the folder must
|
||||||
|
physically exist:
|
||||||
|
|
||||||
|
| Phrase | Rendering |
|
||||||
|
|---|---|
|
||||||
|
| folder key | zh-CN 文件夹键 · zh-TW 資料夾鍵 · ja フォルダーキー · ko 폴더 키 · fr clé de dossier · de Ordnerschlüssel · es clave de carpeta · ru ключ папки · he מפתח תיקייה |
|
||||||
|
| on disk | zh-CN 在磁盘上 · zh-TW 在磁碟上 · ja ディスク上 · ko 디스크에 · fr sur le disque · de auf dem Datenträger · es en el disco · ru на диске · he בדיסק |
|
||||||
|
|
||||||
|
`settings.json` and `ComfyUI` stay verbatim in every locale; "reload this page" / "restart
|
||||||
|
LoRA Manager" reuse each locale's existing restart wording (`settings.extraFolderPaths.*`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Cross-cutting confusion hot-spots (must-fix list)
|
||||||
|
|
||||||
|
All items below were **resolved** in the 2026-08 sweep — treat them as a regression
|
||||||
|
watch-list: do not reintroduce these renderings.
|
||||||
|
|
||||||
|
1. **Checkpoint rendered as a literal security/road checkpoint** — fr, es, ru, he, zh-CN,
|
||||||
|
zh-TW all had 4–6 keys in the `statistics.*` domain reading as "control point"; reverted
|
||||||
|
to "Checkpoint".
|
||||||
|
2. **"recipe" variants that break the one-noun rule** — fr "recette" → "Recipe", zh
|
||||||
|
食谱/食譜 → 配方, de/ja/ru leftover English "Recipe" translated.
|
||||||
|
3. **ko `header.filter.tagLogicAny`** — was inverted ("모든 태그 일치 (OR)") and identical
|
||||||
|
to `tagLogicAll`; now "어느 하나의 태그와 일치 (OR)".
|
||||||
|
4. **ja `modals.model.versions.actions.viewLocalTooltip`** — was the stale "近日対応予定"
|
||||||
|
("coming soon"); all 9 locales now describe the actual action.
|
||||||
|
5. **Stale help texts** — `settings.downloadSkipBaseModels.help`,
|
||||||
|
`settings.aiProvider.apiBaseHelp`, `settings.hideEarlyAccessUpdates.help` retranslated
|
||||||
|
in all locales to the current `en.json` wording.
|
||||||
|
6. **en.json source bugs** (fixed in source, then mirrored):
|
||||||
|
- "Civitai" → "CivitAI" brand casing (values only; key names `relinkCivitai` etc. keep
|
||||||
|
their lowercase form and must not be renamed)
|
||||||
|
- `modals.relinkCivitai.helpText.format4` "CivitArchive" typo → "CivArchive"
|
||||||
|
- `zh-CN recipes.controls.import.downloadLocationPreview` invented `{path}` removed
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Placeholder contract deviations (current)
|
||||||
|
|
||||||
|
`{...}` token sets must match `en.json` per key. All deviations found in the 2026-08 sweep
|
||||||
|
were fixed, with one *intentional* exception:
|
||||||
|
|
||||||
|
**`toast.settings.mappingsUpdated`** — the caller passes a hardcoded English inflection
|
||||||
|
(`plural: count !== 1 ? 's' : ''`). Languages that cannot build a plural by appending that
|
||||||
|
`s` (zh-CN/zh-TW, ja, ko, de, ru, he) **drop `{plural}`** and render a count-friendly form
|
||||||
|
(`({count})` or a measure word); fr and es keep it (`mappage{plural}`, `mapeo{plural}`).
|
||||||
|
|
||||||
|
```python
|
||||||
|
# keep a copy of this rule next to the key if it ever moves:
|
||||||
|
# fr/es: "... ({count} mappage{plural})"
|
||||||
|
# de/ru/he: "... ({count})"
|
||||||
|
# zh-CN: "({count} 条映射)" / zh-TW: "({count} 個對應)" / ja: "({count} マッピング)"
|
||||||
|
```
|
||||||
|
|
||||||
|
Do NOT add `{...}` tokens the source lacks (the caller will not supply them, and the literal
|
||||||
|
text renders in the UI), and do NOT rename source tokens (`{typePlural}` stays `{typePlural}`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. One term, one rendering — offender matrix
|
||||||
|
|
||||||
|
Cross-locale summary of §2 inconsistencies. "✓" = already consistent. All ✗ cells were
|
||||||
|
resolved in the 2026-08 sweep; the row shows the single rendering now in force per locale.
|
||||||
|
|
||||||
|
| Term | fr | de | es | ru | he | ja | ko | zh-CN | zh-TW |
|
||||||
|
|---|---|---|---|---|---|---|---|---|---|
|
||||||
|
| recipe | Recipe | Rezept | receta | рецепт | מתכון | レシピ | 레시피 | 配方 | 配方 |
|
||||||
|
| Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint |
|
||||||
|
| Embedding | Embedding | Embedding | Embedding | Embedding | Embedding | Embedding | Embedding | Embedding | Embedding |
|
||||||
|
| prompt | Prompt | Prompt | prompt | промпт | פרומפט | プロンプト | 프롬프트 | 提示词 | 提示詞 |
|
||||||
|
| base model | modèle de base | Basismodell | modelo base | базовая модель | מודל בסיס | ベースモデル | 베이스 모델 | 基础模型 | 基礎模型 |
|
||||||
|
| preset | préréglage | Voreinstellung | preajuste | пресет | קביעה מראש | プリセット | 프리셋 | 预设 | 預設 |
|
||||||
|
| workflow | Workflow | Workflow | workflow | Workflow | workflow | ワークフロー | 워크플로 | 工作流 | 工作流 |
|
||||||
|
| hash | hash | Hash | hash | хеш | hash | ハッシュ | 해시 | 哈希 | 雜湊 |
|
||||||
|
| metadata | métadonnées | Metadaten | metadatos | метаданные | מטא-נתונים | メタデータ | 메타데이터 | 元数据 | 中繼資料 |
|
||||||
|
| tags | Tags | Tags | etiquetas | теги | תגיות | タグ | 태그 | 标签 | 標籤 |
|
||||||
|
| duplicates | en double | Duplikate | duplicados | дубликаты | כפילויות | 重複 | 중복 | 重复项 | 重複項 |
|
||||||
|
| bulk | groupé | Massen- | por lotes | пакетный | בכמות גדולה | 一括 | 일괄 | 批量 | 批量 |
|
||||||
|
|
||||||
|
Watch: ja/ko keep the model-type names **Checkpoint/Embedding** and `Diffusion Model` in
|
||||||
|
Latin (consistent with their model-type sections) — do not transliterate them as
|
||||||
|
チェックポイント/체크포인트.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Untranslated English leftovers (status)
|
||||||
|
|
||||||
|
Values byte-identical to `en.json` that are actual UI sentences are bugs (brand names and
|
||||||
|
URL placeholders are the exception). As of the 2026-08 sweep, **all previously untranslated
|
||||||
|
blocks are translated** in every locale: `recipes.batchImport.*` + `toast.recipes.batchImport*`
|
||||||
|
(fr/de/es/ru/he/ja/ko), `banners.communitySupport.*`, `modals.model.license.*`,
|
||||||
|
`globalContextMenu.fetchMissingLicenses.*`, the `doctor.*` issue/action/label subset,
|
||||||
|
`toast.settings.libraryLoadFailed` / `libraryActivateFailed`, `toast.api.moveFailed`,
|
||||||
|
`settings.extraFolderPaths.restartRequired`, `toast.recipes.recipeSaved`,
|
||||||
|
`sidebar.dragDrop.moveUnsupported`, `checkpoints.modelTypes.diffusion_model`
|
||||||
|
(ja/ko keep the English loanword), `initialization.recipes.title`.
|
||||||
|
|
||||||
|
The only values that remain intentionally identical to `en.json` are non-translatable:
|
||||||
|
URL/path placeholders (`https://…`, `C:/…`), numeric presets (`5 (1080p), 6 (2K), 8 (4K)`),
|
||||||
|
example token lists (`character, concept, style(toon|toon_style)`), service/provider names
|
||||||
|
(`CivitAI → CivArchive → Archive DB`), model-type names (`settings.priorityTags.modelTypes.*`,
|
||||||
|
`settings.folderSettings.subTypeVae` … `subTypeControlnet` — see §2), and the external playlist
|
||||||
|
title (`help.updateVlogs.playlistTitle`, de: translated to "LoRA Manager-Update-Playlist").
|
||||||
|
|
||||||
|
Rule for `uiHelpers.workflow.noPromptTargets`: the second line (`Mark as → Send Prompt
|
||||||
|
Target`) quotes literal ComfyUI context-menu items — keep those menu labels in English in
|
||||||
|
every locale because that is what the user actually sees in ComfyUI.
|
||||||
|
|
||||||
|
License labels (`modals.model.license.*`): the restriction labels are now translated in all
|
||||||
|
locales (the sibling `creditRequired` has always been translated).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Workflow for agents and translators
|
||||||
|
|
||||||
|
### Adding a new UI string
|
||||||
|
1. Add the key to `locales/en.json` only.
|
||||||
|
2. Run `python scripts/sync_translation_keys.py` — it inserts the key into the other 9
|
||||||
|
locales (as a `[TODO: Translate]` placeholder) preserving formatting.
|
||||||
|
3. **During feature development, stop here.** While the UI copy is still in flux, leave the
|
||||||
|
`[TODO: Translate]` placeholders as-is — translating churning strings into 9 locales is
|
||||||
|
wasted work. Placeholders are a normal intermediate state, not a bug.
|
||||||
|
4. Once the wording is final and the feature owner explicitly asks for translations,
|
||||||
|
translate **all** pending `[TODO: Translate]` keys in every locale (not just the latest
|
||||||
|
feature's), applying §1–§3 (placeholders verbatim, Recipe rule, term maps, register).
|
||||||
|
Find pending keys with: `grep -c "TODO: Translate" locales/*.json`
|
||||||
|
5. If the new string contains new terminology, extend §2 tables.
|
||||||
|
|
||||||
|
### Fixing a translation bug
|
||||||
|
1. Locate the key (dotted path) in the relevant locale file.
|
||||||
|
2. Check the corresponding `en.json` value and the actual caller (grep `static/js` or
|
||||||
|
`web/comfyui` for the key) to learn which placeholders are passed.
|
||||||
|
3. Fix trivially; for normalization sweeps (e.g. "recette" → "Recipe"), do it file-wide for
|
||||||
|
the offending keys only — do not touch unrelated lines.
|
||||||
|
4. If the bug is in `en.json` itself (R9), fix the source first, then re-sync and update all
|
||||||
|
locales.
|
||||||
|
|
||||||
|
### Verification
|
||||||
|
```bash
|
||||||
|
pytest tests/i18n/test_i18n.py # key parity + JSON validity + JS key references
|
||||||
|
python scripts/sync_translation_keys.py --dry-run # shows which keys would change; add --verbose for per-key detail
|
||||||
|
npm test # frontend tests incl. i18n helpers
|
||||||
|
```
|
||||||
|
|
||||||
|
`pytest tests/i18n` only checks structure. Quality conventions in this document are not
|
||||||
|
machine-enforced — a human/agent review pass is required.
|
||||||
|
|
||||||
|
### Anti-patterns checklist
|
||||||
|
- [ ] Placeholders `{x}` / `{{x}}` differ from `en.json`
|
||||||
|
- [ ] Same source term translated 2+ ways in the same file (see §5)
|
||||||
|
- [ ] "Checkpoint" became a literal checkpoint; "recipe" became menu/prescription/food-cookbook
|
||||||
|
- [ ] Brand names translated or transliterated (LoRA, CivitAI, ComfyUI, …)
|
||||||
|
- [ ] Latin locale using full-width `:()`; fr using `"` as apostrophe
|
||||||
|
- [ ] Mixed 你/您, du/Sie, tú/usted
|
||||||
|
- [ ] Full English sentences left behind (see §6)
|
||||||
|
- [ ] Register/typos/mojibake; source string is stale vs `en.json` (compare semantics, not
|
||||||
|
just words)
|
||||||
@@ -0,0 +1,206 @@
|
|||||||
|
# Plan: Multi-File Downloads Within a Single CivitAI Model Version
|
||||||
|
|
||||||
|
**Issue:** [#1058 — Cannot download multiple file variants from the same model version](https://github.com/willmiao/ComfyUI-Lora-Manager/issues/1058)
|
||||||
|
**Status:** v2 — revised after adversarial review (backend correctness + frontend/tests)
|
||||||
|
**Scope:** CivitAI/CivArchive downloads of `lora`, `checkpoint`, `embedding` model types. HuggingFace downloads are out of scope (already per-file).
|
||||||
|
|
||||||
|
> v2 changelog: incorporated 18 review findings. Key changes vs v1:
|
||||||
|
> shared file resolver + `resolved_version_id` for the gate (R1); `file_params` normalization at API boundary (R2); D2 hash-matching rule fixed for empty-hash cases (R6/R7); D3 extended to re-point `version_index` on removal (R4); D4 replaced with a child table (R3); `delete_model_version` interaction documented (R5); `ModelVersionsTab` surface added to phase 2 (F6); phase-2 multi-file loop requires a reload-deferred download variant (F7); queue-retry `file_params=NULL` known issue recorded (R9); test-fixture gaps and revised estimates (F10).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Problem Statement
|
||||||
|
|
||||||
|
A CivitAI model version can contain multiple downloadable weight files (e.g. fp16/fp32, safetensors/ckpt, different sizes). LoRA Manager already has a working file-selection pipeline (frontend file dialog → `fileParams` → backend file matching), but downloaded state is tracked at the **model-version** level. After any single file of a version is downloaded:
|
||||||
|
|
||||||
|
1. The version is marked **In Library** and the file-selection entry point disappears.
|
||||||
|
2. The backend rejects further download attempts for that version.
|
||||||
|
|
||||||
|
There is no way to download the remaining files of the same version through LoRA Manager.
|
||||||
|
|
||||||
|
## 2. Current State (verified against code; all references confirmed by review)
|
||||||
|
|
||||||
|
### 2.1 Download gating — backend (`py/services/download_manager.py`)
|
||||||
|
|
||||||
|
`_execute_original_download` enforces two version-level gates:
|
||||||
|
|
||||||
|
- **Library gate, early** (lines 1157–1184, before metadata fetch, fires when `model_version_id` given) and **late** (lines 1350–1376, fires only when `model_version_id is None`): `scanner.check_model_version_exists(version_id)` across lora/checkpoint/embedding scanners → hard error `"Model version already exists in ... library"`.
|
||||||
|
- **History gate** (lines 1238–1279): when `skip_previously_downloaded_model_versions` setting is on, `_has_been_downloaded(model_type, version_id)` → silent skip. History DB primary key is `(model_type, version_id)` (`py/services/downloaded_version_history_service.py:61`).
|
||||||
|
|
||||||
|
File selection works: `file_params {id, type, format, size, fp}` is matched against `version_info.files` (lines 1498–1569), **but only under `if file_params and model_version_id:` (line 1499)** — with `model_id`-only requests the selection silently falls back to the primary file (1571–1619). `file_params` currently carries no file `name` or hash.
|
||||||
|
|
||||||
|
### 2.2 Downloaded-state surfacing — backend (`py/routes/handlers/model_handlers.py`)
|
||||||
|
|
||||||
|
`get_civitai_versions` (lines 2148–2188) sets per-version `existsLocally` via `cache.version_index.get(version_id)` (plus a single `localPath` from that entry) and `hasBeenDownloaded` via the history service. No per-file granularity.
|
||||||
|
|
||||||
|
### 2.3 Frontend blockers (`static/js/managers/DownloadManager.js`)
|
||||||
|
|
||||||
|
Three independent gates prevent re-entering the file dialog:
|
||||||
|
|
||||||
|
1. **Line 598:** file-select badge rendered only when `modelFiles.length > 1 && !existsLocally`.
|
||||||
|
2. **Lines 666–681 (`updateNextButtonState`):** Next button disabled with "Already in Library" when `currentVersion.existsLocally`.
|
||||||
|
3. **Lines 784–787 (`proceedToLocation`):** toast + abort when `currentVersion.existsLocally`.
|
||||||
|
|
||||||
|
The badge path (`confirmFileSelection` lines 737–759 → `proceedToLocationContent` → `startDownload` single mode → `executeDownloadWithProgress` → POST `file_params`, `static/js/api/baseModelApi.js:1236–1250`) has **zero** `existsLocally` guards (all 12 occurrences enumerated; none on this path; `import/DownloadManager.js` has none either). The `.exists-locally` CSS class is purely visual (`download-modal.css:496–499`). **Making the badge visible again is sufficient to unlock the flow** for phase 1.
|
||||||
|
|
||||||
|
Post-download refresh is clean: the modal closes and `resetAndReload(true)` performs a full library refetch (`DownloadManager.js:1063`); dialog reopen resets state and refetches versions with no client-side cache. No same-session staleness.
|
||||||
|
|
||||||
|
### 2.4 Local identity of the downloaded file
|
||||||
|
|
||||||
|
`LoraMetadata/CheckpointMetadata/EmbeddingMetadata.from_civitai_info(version_info, file_info, ...)` (`py/utils/models.py:245–369`) persists:
|
||||||
|
|
||||||
|
- `sha256` = `file_info.hashes.SHA256` (lowercased, defaults to `""`) — a stable per-file identity;
|
||||||
|
- `civitai` = the full `version_info` payload (including the `files` list).
|
||||||
|
|
||||||
|
Metadata refresh (`metadata_sync_service.py:104–105`) replaces the `civitai` blob wholesale but never overwrites top-level `sha256`; `verify_duplicate_hashes` (481–526) corrects it to the on-disk hash. Top-level-sha256 matching is refresh-robust.
|
||||||
|
|
||||||
|
**Caveats (review R6/R7):**
|
||||||
|
- SHA256 is not guaranteed: CivArchive's transform only sets `hashes` when source data carries it (`civarchive_client.py:185–189`); `from_civitai_info` defaults to `""`.
|
||||||
|
- Name fallback is unreliable exactly when it matters: local `file_name` is extension-less (`models.py:264`) and `generate_unique_filename` rewrites it with a hash suffix on conflict (`download_manager.py:1125–1136`); checkpoints with `hash_status='pending'` keep empty sha256 until on-demand hashing (`model_scanner.py:1232–1240`).
|
||||||
|
|
||||||
|
### 2.5 Version index collision (pre-existing hazard)
|
||||||
|
|
||||||
|
`ModelCache.version_index` is single-valued (`model_cache.py:133`: `version_index[version_id] = item`). Two files of the same version in the library → second entry overwrites the first; `remove_from_version_index` (lines 151–181) drops the whole version key when the indexed entry is removed, even if a sibling file remains. ~10 read sites depend on this index (48 grep touch points total; readers include `recipe_scanner.py:2682–2726`, `recipe_format.py:37–40`, `misc_handlers.py:2440–2444`, `model_handlers.py`, `model_scanner.check_model_version_exists:2444`).
|
||||||
|
|
||||||
|
Review correction (F3): bulk paths `remove_models` (`model_scanner.py:2376`) and `update_single_model_cache` (`:1689`) call `rebuild_version_index()` right after, so a sibling re-enters the index in those flows — the hazard is narrower than v1 stated, but direct `remove_from_version_index` callers (e.g. `model_scanner.py:1018`) still drop the key, and the user-visible artifact in phase 1 is real: `localPath` in the dialog flips to whichever file was indexed last.
|
||||||
|
|
||||||
|
### 2.6 Entry points that send / don't send `file_params` (fully enumerated by review)
|
||||||
|
|
||||||
|
**Send `file_params` (user-initiated dialog flows only):** `DownloadManager.js:1611–1639` (single mode). API surface accepting arbitrary JSON `file_params`: GET `/api/lm/download-model-get` (`model_handlers.py:1634–1686`), POST `/api/lm/downloads/queue/add` (`model_handlers.py:1799–1832`).
|
||||||
|
|
||||||
|
**Never send `file_params` (keep version-level semantics):** batch download (`DownloadManager.js:1756–1766`; batch also filters out in-library versions at `:1648`), `downloadVersionWithDefaults` (`:1810–1830`), recipe import (`import/DownloadManager.js:269–276`), bulk missing-LoRA (`BulkMissingLoraDownloadManager.js:292–299`), `RecipeModal.js:1728–1736`, `ModelVersionsTab.js:1427`. `web/comfyui/` and `vue-widgets/src` contain **no** download triggers at all (grep-verified). `py/services/use_cases/` has only `download_model_use_case.py` (pass-through).
|
||||||
|
|
||||||
|
### 2.7 Paths that do NOT need changes (verified)
|
||||||
|
|
||||||
|
- **aria2 pause/resume** (`_resume_restored_aria2_download`, line 754+): resumes from persisted `resume_context`; never re-runs existence gates.
|
||||||
|
- **`download_coordinator.py:90`**: pure pass-through of `file_params`.
|
||||||
|
- **Update checker / plugin self-update** (`update_routes.py:496–501`): only closes the history DB handle.
|
||||||
|
- **History delete semantics**: `mark_as_deleted` sets `is_deleted_override=1` and `has_been_downloaded` then returns False (`downloaded_version_history_service.py:276`) — LM-initiated deletes already reset the history skip.
|
||||||
|
|
||||||
|
### 2.8 Related pre-existing issues (record, not necessarily fix)
|
||||||
|
|
||||||
|
- **Queue retry drops file selection** (R9): `download_queue_service.retry_from_history` / `retry_all_failed` re-queue with `file_params=NULL` (`download_queue_service.py:705, 758`) although the queue table has a `file_params` column (`:43`) — a retried non-primary download silently reverts to the primary file. Fix alongside phase 1 (small: persist and reuse the column).
|
||||||
|
- **`delete_model_version`** (`misc_handlers.py:2410–2487`): resolves the file via the single-valued `version_index` (2440–2444), deletes only that one file, and `mark_as_deleted` flags the **entire version** as deleted in history (2479) even when a sibling file remains in the library. See phase 2 item 6.1.5.
|
||||||
|
|
||||||
|
## 3. Goals / Non-Goals
|
||||||
|
|
||||||
|
**Goals**
|
||||||
|
|
||||||
|
- G1: A user can download any not-yet-downloaded file of a version already partially in the library (issue repro steps 6–8).
|
||||||
|
- G2: True duplicates stay blocked: downloading the *same* file of the same version twice is rejected.
|
||||||
|
- G3: Per-file downloaded state visible in the file dialog; multiple files selectable and downloadable in one pass.
|
||||||
|
- G4: No regression for version-level semantics relied on by batch download, recipe missing-LoRA detection, and `skip_previously_downloaded_model_versions`.
|
||||||
|
|
||||||
|
**Non-Goals**
|
||||||
|
|
||||||
|
- No change to recipe `inLibrary` semantics ("any file of the version present" remains sufficient).
|
||||||
|
- No change to the update-checker (version-level comparison).
|
||||||
|
- No primary-key rebuild of the history database.
|
||||||
|
- HuggingFace download flow untouched.
|
||||||
|
|
||||||
|
## 4. Design Decisions
|
||||||
|
|
||||||
|
- **D1 — Explicit file selection bypasses the history gate, version-level gates stay for everyone else.** The history skip exists to dedupe automated flows. A user explicitly picking a file is unambiguous intent; the file-level library gate (G2) still prevents real duplicates. **Guard conditions use normalized truthiness** (see D1a). All confirmed `file_params` senders are user-initiated dialog flows (2.6), and LM-initiated deletes already reset history (2.7), so the bypass only affects "downloaded but not LM-deleted" versions with the setting on — intended.
|
||||||
|
- **D1a — `file_params` normalization at the boundary (R2).** `download-model-get` and `downloads/queue/add` accept arbitrary JSON; `{}` is `not None` but falsy and would bypass gates while downloading the primary file. Normalize `file_params = file_params or None` in the coordinator/handlers, and treat the bypass as active only when a target file id is resolvable.
|
||||||
|
- **D2 — File identity matching rule (R6/R7):** hash-compare **only when both sides are non-empty** (lowercase SHA256 equality); name-compare when either side is empty. Never let `"" == ""` match. Name fallback caveats from 2.4 apply (renamed files, pending checkpoint hashes) — acceptable residual risk, worst case is a blocked re-download the user can retry after hashing completes.
|
||||||
|
- **D3 — Cache indexes: additive multi-index + removal re-pointing (R4).** Add `version_files_index: Dict[int, List[dict]]` maintained alongside `version_index` by the same add/remove/rebuild methods; existing readers of `version_index` untouched. Additionally fix `remove_from_version_index`: when the popped entry has a surviving sibling (per the multi-index), re-point `version_index[version_id]` to the sibling instead of dropping the key; same for the `model_id_index` descriptor. This closes the 2.5 hazard for existing readers (`check_model_version_exists`, `existsLocally`, recipe matching) without restructuring anything.
|
||||||
|
- **D4 — Per-file history via a child table (R3).** v1's additive-column approach is structurally impossible on a `(model_type, version_id)` PK (`ON CONFLICT DO UPDATE` would keep only the last file). Instead add `downloaded_version_files(model_type, version_id, file_id, file_name, downloaded_at, PRIMARY KEY(model_type, version_id, file_id))` — additive, no PK rebuild, honors the Non-Goal. Existing version-level table and queries unchanged. New per-file queries are opt-in. `_initialize_schema` uses `CREATE TABLE IF NOT EXISTS`, so the new table is created for existing DBs without any ALTER.
|
||||||
|
- **D5 — UI flow reuse, with an extracted inner download function for multi-file (F7).** Phase 1 unlocks the existing badge → file dialog → location → download pipeline. Phase 2 upgrades the dialog to multi-select; iterating `executeDownloadWithProgress` as-is would produce N full library reloads, N toasts, and competing failure-summary modals — so phase 2 extracts a reload-deferred, failure-aggregating inner variant and runs one reload + one summary at the end.
|
||||||
|
|
||||||
|
## 5. Implementation — Phase 1 (fix the issue; independently shippable)
|
||||||
|
|
||||||
|
### 5.1 Backend — `py/services/download_manager.py`
|
||||||
|
|
||||||
|
1. **Normalize `file_params`** at the boundary (D1a): `download_coordinator.schedule_download` and the two API handlers (`model_handlers.py:1649–1666`, `1810–1832`) apply `file_params = file_params or None`.
|
||||||
|
2. **Extract a shared file resolver** (R1): pull the matching logic at 1498–1569 into `_resolve_target_file(version_info, file_params) -> Optional[dict]`, used by **both** the new gate and the download-selection path. The selection path's condition (line 1499) switches from `model_version_id` to `resolved_version_id` (already computed at 1230–1236 from `version_info.id`), so gate and download always agree on the target file — including the `model_id`-only case.
|
||||||
|
3. **New helper** `_find_local_file_entry(version_id, target_file) -> Optional[dict]`: iterate the three scanners' cached `raw_data` (NOT `version_index` — single-valued); candidates = entries whose `civitai.id` normalizes to `version_id`; match per D2.
|
||||||
|
4. **Gate restructure in `_execute_original_download`**:
|
||||||
|
- Early scanner gate (1157–1184): add `file_params is None` guard; with normalized `file_params`, defer (file identity not resolvable before metadata fetch).
|
||||||
|
- After `version_info` fetch + `resolved_version_id` (~1229): when `file_params` present, resolve target file via the shared resolver; unresolvable → hard error "No matching file" (fail closed, prevents empty-dict bypass). Resolvable → `_find_local_file_entry`; hit → same hard error shape as today with the file name in the message.
|
||||||
|
- History gate (1238–1279): add `file_params is None` (D1). Base-model skip (1281–1324) unchanged — still applies.
|
||||||
|
- Late gate (1350–1376): add `file_params is None` guard (F2) — the post-fetch file-level check above already covers this case.
|
||||||
|
- Nothing between the early gate and the post-fetch point assumes the version is absent (review task 6: only provider selection + metadata fetch; no DB writes; `_persist_aria2_state` runs only when actually downloading at 1659).
|
||||||
|
5. **Queue retry fix** (2.8, small): persist `file_params` into the queue table on enqueue and reuse it in `retry_from_history` / `retry_all_failed`.
|
||||||
|
6. Logging: `[download]` lines for file-level allow/block, consistent with existing style.
|
||||||
|
|
||||||
|
**Estimated:** ~150–220 LOC + resolver extraction.
|
||||||
|
|
||||||
|
### 5.2 Frontend — `static/js/managers/DownloadManager.js`
|
||||||
|
|
||||||
|
1. Line 598: drop `&& !existsLocally` from the badge condition (badge shows whenever `modelFiles.length > 1`).
|
||||||
|
2. `fileParams` construction (1611–1616): add `name: this.selectedFile.name`.
|
||||||
|
3. Surface the backend "file already in library" hard error as a toast instead of only the batch-summary modal (R10/F12 nit; reuse existing error message field).
|
||||||
|
4. No changes to `updateNextButtonState` / `proceedToLocation` in phase 1; no template or CSS changes.
|
||||||
|
|
||||||
|
**Known phase-1 UX limitations (acknowledged, fixed in phase 2):** with all files downloaded the badge still renders and re-picking a downloaded file fails late (backend error after the location step); `localPath` may point at a sibling file; batch-preview "In Library" badge stays version-level and gives no hint of remaining files.
|
||||||
|
|
||||||
|
**Estimated:** ~10–30 LOC (confirmed realistic by review).
|
||||||
|
|
||||||
|
### 5.3 Phase 1 tests
|
||||||
|
|
||||||
|
Backend — extend `tests/services/test_download_manager_basic.py` (1694 lines; all fixture patterns exist):
|
||||||
|
|
||||||
|
- **Fixture gaps to add (F10):** `DummyScanner.get_cached_data()`/`raw_data` stub (~10 lines); `hashes.SHA256` in the metadata-provider payload's `files`.
|
||||||
|
- Cases: same version + different SHA256 in library + `file_params` → proceeds; same SHA256 → hard error; `file_params=None` + version in library → hard error (unchanged); history-skip on + `file_params` → not skipped; without → skipped (unchanged); empty-dict `file_params` normalized → version-level behavior; `model_id`-only + `file_params` → gate and selection resolve the same file; legacy metadata (empty local sha256) matched by name; target file with empty SHA256 → name fallback, no `""==""` false positive.
|
||||||
|
- Queue retry: `file_params` survives retry.
|
||||||
|
- Assert proceed/abort via the existing `_execute_download` mock pattern.
|
||||||
|
|
||||||
|
Frontend (`tests/frontend/`): badge renders for multi-file version with `existsLocally=true` (pattern from `downloadManager.history.test.js`).
|
||||||
|
|
||||||
|
**Estimated:** ~150–250 LOC (confirmed realistic).
|
||||||
|
|
||||||
|
## 6. Implementation — Phase 2 (per-file status + multi-select + index hardening)
|
||||||
|
|
||||||
|
### 6.1 Backend
|
||||||
|
|
||||||
|
1. **`py/services/model_cache.py`** (D3): add `version_files_index`; maintain in `add_to_version_index` / `remove_from_version_index` / `rebuild_version_index`; removal re-points `version_index[version_id]` (and the `model_id_index` descriptor) to a surviving sibling instead of dropping the key.
|
||||||
|
2. **`py/services/model_scanner.py`**: expose `get_files_for_version(version_id) -> List[dict]`.
|
||||||
|
3. **`py/routes/handlers/model_handlers.py` `get_civitai_versions`**: annotate each version with `downloadedFiles: [{fileId, fileName, filePath}]` via `version_files_index` + D2 matching against `version.files`.
|
||||||
|
4. **`py/services/downloaded_version_history_service.py`** (D4): new child table `downloaded_version_files`; `mark_downloaded` also upserts the child row when `file_id` known; `mark_as_deleted` clears the version's child rows only when no sibling remains in the library; new `get_downloaded_file_ids(model_type, version_id) -> set[int]`. `_record_downloaded_version_history` passes `file_info` through.
|
||||||
|
5. **`delete_model_version`** (`misc_handlers.py:2410–2487`, R5): resolve **all** local files of the version via `version_files_index`; delete all (current endpoint semantics are version-level) or — if kept per-file — only `mark_as_deleted` when no sibling remains. Decide at implementation time; minimum is documenting current behavior.
|
||||||
|
6. **`ModelVersionsTab` backend support**: none needed beyond item 3 (`downloadedFiles`); the tab consumes the same versions payload.
|
||||||
|
|
||||||
|
### 6.2 Frontend
|
||||||
|
|
||||||
|
1. **File dialog multi-select** — change surface (F8): option markup (`DownloadManager.js:712–724`), the single-select click handler (`727–734`), the `input[type="radio"]:checked` selector in `confirmFileSelection` (`738`); template `templates/components/modals/download_modal.html:48–60` (confirm-button label only); CSS `download-modal.css` — checkbox variant of `.file-option-radio input` (595–604) and a **new** `.file-option.disabled` style (does not exist). Files whose id ∈ `downloadedFiles` render disabled with an "In Library" tag.
|
||||||
|
2. **Mixed-type guard (F8):** multi-select is restricted to files sharing the same routing target (`_isDiffusionModel` is computed once from a single `selectedFile` at 798–803; e.g. "Model" + "UNet" files route to different roots). Disallow mixed-type multi-select (simplest, predictable); single-file selection unchanged.
|
||||||
|
3. **Multi-file download loop (D5/F7):** extract from `executeDownloadWithProgress` a reload-deferred, no-toast inner function; iterate per selected file with per-file progress; one `resetAndReload(true)` + one aggregated success/failure summary at the end (reuse `showDownloadBatchSummary`).
|
||||||
|
4. **`updateNextButtonState` / `proceedToLocation`:** for multi-file versions, Next routes into the file dialog; hard block only when *every* weight file is downloaded.
|
||||||
|
5. **`ModelVersionsTab.js` (F6):** the Download action (`:576` hidden when `isInLibrary`) — for multi-file versions with remaining files, show it and route into the download modal's file dialog; keep hidden when all files present.
|
||||||
|
6. **Batch preview (F5):** `batch-preview-local-badge` (`:1320`) gains a "partially downloaded" hint for multi-file versions with remaining files.
|
||||||
|
7. New i18n keys (`modals.download.fileSelection.inLibrary`, `downloadSelected`, partial-download tooltip, etc.) → run `python scripts/sync_translation_keys.py`.
|
||||||
|
|
||||||
|
### 6.3 Phase 2 tests
|
||||||
|
|
||||||
|
- `model_cache` (`tests/services/test_model_cache.py` already covers add/remove at 44–55): multi-valued index; sibling re-point on removal; rebuild.
|
||||||
|
- `get_civitai_versions`: `downloadedFiles` correctness (hash match, name fallback, no match, CivArchive no-hash payload).
|
||||||
|
- History service (`tests/services/test_downloaded_version_history_service.py` uses real SQLite on tmp_path): child-table creation on a legacy DB; per-file record/query; `mark_as_deleted` sibling semantics.
|
||||||
|
- Frontend: dialog checkbox rendering/disabled state and multi-file confirm — **greenfield behavior coverage** (F10: no existing test exercises `showFileSelectionStep`/`confirmFileSelection`; infra exists, patterns must be built).
|
||||||
|
|
||||||
|
## 7. Risks and Mitigations
|
||||||
|
|
||||||
|
| Risk | Impact | Mitigation |
|
||||||
|
|---|---|---|
|
||||||
|
| History-gate bypass (D1) causes unwanted re-downloads in automated flows | Large checkpoint files re-downloaded | Bypass only with normalized, resolvable `file_params` (D1a); all such senders are user-initiated dialog flows (2.6, verified); tests pin batch/recipe/bulk behavior. |
|
||||||
|
| Empty-hash matching edge cases (R6) | Duplicate download of the same file, or false block | D2 rule: hash only when both non-empty; name otherwise; never `""==""`. Residual risk documented (2.4). |
|
||||||
|
| Phase-1 late-failure UX (F12) | User picks a downloaded file, fails only after location step | Toast surfacing (5.2.3); phase 2 disables downloaded files up front. |
|
||||||
|
| Phase-2 index change corrupts existing behavior | Recipe matching, delete flows | Additive index + re-point only; `version_index` read semantics unchanged; `remove_models`/`update_single_model_cache` already rebuild (F3); tests. |
|
||||||
|
| `delete_model_version` marks whole version deleted while sibling remains (R5) | History wrongly suppresses re-download of the surviving sibling's version | Phase 2 item 6.1.5; documented until then. |
|
||||||
|
| History child-table migration failure on user installs | Service init crash | `CREATE TABLE IF NOT EXISTS` in `_initialize_schema`; failure degrades to version-level behavior (per-file queries return empty). |
|
||||||
|
| Batch-preview badge misleading for partial versions (F5) | Minor UX confusion | Acknowledged in phase 1; fixed in phase 2 item 6.2.6. |
|
||||||
|
| UI confusion: version shows "In Library" while files remain downloadable | Support burden | Phase 2: per-file disabled state + partial-download tooltip. |
|
||||||
|
| Hash-identical sibling files (repacked content) | Second file blocked | Acceptable: scanner hash dedup already collapses them. |
|
||||||
|
|
||||||
|
## 8. Rollout
|
||||||
|
|
||||||
|
1. **Commit 1** — `fix(download): allow downloading additional files of an in-library model version (#1058)` → Phase 1 (5.1–5.3).
|
||||||
|
2. **Commit 2** — `feat(download): per-file download status and multi-file selection (#1058)` → Phase 2 (6.1–6.3).
|
||||||
|
|
||||||
|
Phase 1 alone resolves the issue as reported; phase 2 can ship in a later release if review prefers smaller increments.
|
||||||
|
|
||||||
|
## 9. Effort Estimate (revised after review)
|
||||||
|
|
||||||
|
| Phase | Backend | Frontend | Tests | Risk |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| 1 | ~150–220 LOC (+ queue-retry fix ~30) | ~10–30 LOC | ~150–250 LOC | Low |
|
||||||
|
| 2 | ~250–350 LOC | ~250–350 LOC (multi-file loop refactor + ModelVersionsTab + batch badge) | ~250–350 LOC (dialog tests greenfield) | Medium |
|
||||||
@@ -0,0 +1,337 @@
|
|||||||
|
# Plan: Global Rate-Limit Abidance for Recipe Ingest & Metadata Fetching
|
||||||
|
|
||||||
|
**Issue:** [#1085 — Large Recipe Ingest Appears to not abide by vendor rate limits, possibly a few other errors?](https://github.com/willmiao/ComfyUI-Lora-Manager/issues/1085)
|
||||||
|
**Status:** v2 — reviewed; decisions recorded in §10. **Phase 1 implemented**
|
||||||
|
(2026-08-27, commit `c2a2048c`): coordinator + downloader gate + Fix C
|
||||||
|
failover semantics + helper double-wait fix + settings. **Phase 2
|
||||||
|
implemented** (2026-08-27): batch-import rate-limit failures map to
|
||||||
|
`SKIPPED` + `rate_limited` WebSocket flag + UI slowdown hint (toast + status
|
||||||
|
text, i18n keys synced); `download_to_memory` / `get_response_headers` /
|
||||||
|
`download_file` register 429 cooldowns. Changes vs v1: Fix C moved to
|
||||||
|
Phase 1, helper double-wait resolved in Phase 1, gate/guard ordering
|
||||||
|
specified.
|
||||||
|
**Scope:** HTTP API traffic to CivitAI (`civitai.red`) and CivArchive (`civarchive.com`) from metadata fetching (bulk refresh, metadata sync, recipe analysis/enrichment, usage-control lookups). Large binary downloads (model files / preview images via `download_file`) are out of scope for *pacing* (they are already single-connection transfers) but their 429 responses should still be *registered*.
|
||||||
|
|
||||||
|
> Context: a first batch of fixes for this issue was already committed as
|
||||||
|
> `ee233548` ("fix(recipes): enforce batch-import concurrency bound and harden
|
||||||
|
> ingest errors (#1085)"): the batch-import concurrency controller now shares a
|
||||||
|
> real semaphore (bounds 1–5 actually apply), the Comfy parser tolerates
|
||||||
|
> list/`None` `ckpt_name`, CivArchive treats empty error payloads as failures,
|
||||||
|
> and offline-cooldown short-circuits log at DEBUG. This plan covers the two
|
||||||
|
> remaining orchestration-level fixes:
|
||||||
|
> **Fix 2** — slow down globally when a vendor rate limit is hit (respect
|
||||||
|
> `Retry-After`, queue instead of hammering); **Fix 3** — stop immediately
|
||||||
|
> failing over to CivArchive when CivitAI is rate-limited.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Problem Statement
|
||||||
|
|
||||||
|
During a large recipe ingest (e.g. importing the example-images directory,
|
||||||
|
which can be thousands of images), the manager fires one metadata request per
|
||||||
|
checkpoint + per LoRA per image through the fallback provider chain
|
||||||
|
(`civitai_api → civarchive_api → sqlite`). Consequences observed in #1085:
|
||||||
|
|
||||||
|
1. **CivitAI gets hammered** → 429s. The consumer then *immediately* tries
|
||||||
|
CivArchive for the same lookup, so **CivArchive gets hammered too** before
|
||||||
|
it was ever naturally needed (its only real job is recovering metadata for
|
||||||
|
models deleted from CivitAI).
|
||||||
|
2. Requests are retried per-call after `Retry-After`, but **each concurrent
|
||||||
|
call sleeps independently** → thundering herd: thousands of coroutines wake
|
||||||
|
at the same moment and re-flood the vendor.
|
||||||
|
3. While CivArchive is in the `ConnectivityGuard` cooldown, every batch item
|
||||||
|
short-circuits and is marked `FAILED` — the batch import's success/failure
|
||||||
|
accounting is polluted by a transient vendor state (log spam was fixed in
|
||||||
|
`ee233548`; the item-failure accounting is not).
|
||||||
|
4. `ConnectivityGuard` (`py/services/connectivity_guard.py`) only treats
|
||||||
|
transport-level unreachability as offline; **HTTP 429 is invisible to it**,
|
||||||
|
so nothing ever intentionally paces request rate.
|
||||||
|
|
||||||
|
User expectation from the issue: *"once a vendor rate limit time out is hit,
|
||||||
|
you should trigger a slow down with intentional reduction in request rate"*.
|
||||||
|
|
||||||
|
## 2. Current State (verified against code)
|
||||||
|
|
||||||
|
### 2.1 Where 429s are surfaced
|
||||||
|
|
||||||
|
- `Downloader.make_request` (`py/services/downloader.py:1120-1132`): HTTP 429 →
|
||||||
|
returns `RateLimitError(message, retry_after=…)` parsed from `Retry-After`
|
||||||
|
(missing header defaults to `None`).
|
||||||
|
- `CivitaiClient._make_request` (`py/services/civitai_client.py:97-100`):
|
||||||
|
converts `RateLimitError` to a raise immediately; no waiting. Transient
|
||||||
|
5xx/connection errors are retried 3× with 1s/2s/4s backoff.
|
||||||
|
- `CivArchiveClient._make_request` (`py/services/civarchive_client.py`):
|
||||||
|
raises `RateLimitError` with `provider="civarchive_api"` when not set.
|
||||||
|
- `_RateLimitRetryHelper` (`py/services/model_metadata_provider.py:45-102`):
|
||||||
|
per-call retry loop — sleeps `retry_after` (capped at 1800 s; `≥120 s` ⇒ no
|
||||||
|
retry), then re-raises. Because every concurrent call runs its own helper,
|
||||||
|
they sleep in parallel and re-fire in parallel.
|
||||||
|
- `FallbackMetadataProvider` (`py/services/model_metadata_provider.py:488-508,
|
||||||
|
564-584` etc.): on a final `RateLimitError` from one provider it logs
|
||||||
|
"skipping to next provider" and **continues to the next network provider** —
|
||||||
|
this is the direct cause of the CivArchive flood.
|
||||||
|
- `MetadataSyncService.fetch_and_update_model`
|
||||||
|
(`py/services/metadata_sync_service.py:248-333`): manually iterates
|
||||||
|
`provider_attempts`; on `RateLimitError` it `continue`s to the next provider
|
||||||
|
(same failover problem), then reports `"Rate limited"` when nothing
|
||||||
|
succeeded.
|
||||||
|
- `Downloader.make_request` has a per-destination scope already available:
|
||||||
|
`_guard_destination(url)` returns the hostname (`downloader.py:1194-1199`),
|
||||||
|
used by `ConnectivityGuard`.
|
||||||
|
|
||||||
|
### 2.2 What pacing exists today
|
||||||
|
|
||||||
|
- `ConnectivityGuard`: per-destination cooldown (30 s base, ×2 per extra
|
||||||
|
failure batch, 300 s cap) triggered only by transport errors
|
||||||
|
(`connectivity_guard.py:168-197`).
|
||||||
|
- `AdaptiveConcurrencyController` (batch import, fixed in `ee233548`): shared
|
||||||
|
semaphore enforces 1–5 concurrent items; *duration*-based adjustment only —
|
||||||
|
it never sees HTTP statuses, so it cannot distinguish "slow because rate
|
||||||
|
limited" from "slow because big image".
|
||||||
|
- No token bucket, no minimum inter-request interval, no shared
|
||||||
|
`Retry-After` gate anywhere (`grep` for throttle/token-bucket/rate-limiter:
|
||||||
|
0 hits).
|
||||||
|
|
||||||
|
## 3. Requirements & Constraints
|
||||||
|
|
||||||
|
R1. **Respect `Retry-After`.** After a 429, no further request to that
|
||||||
|
destination may be sent before the vendor's retry window elapses.
|
||||||
|
R2. **No thundering herd.** Concurrent waiters must share one wake-up (gate),
|
||||||
|
not sleep independently.
|
||||||
|
R3. **No double load.** A CivitAI 429 must not trigger a CivArchive request
|
||||||
|
for the same lookup. CivArchive should only be consulted when CivitAI
|
||||||
|
legitimately has no answer (404 / "not found"), or when CivitAI is
|
||||||
|
unreachable long-term.
|
||||||
|
R4. **No spurious item failures.** A rate-limited request must not turn a
|
||||||
|
batch-import item into `FAILED`; it should wait (bounded) and retry, or at
|
||||||
|
worst be `SKIPPED` with a clear "rate limited" reason (re-runnable import).
|
||||||
|
R5. **Never hang forever.** All waiting is bounded by a configurable cap; on
|
||||||
|
expiry the caller receives the `RateLimitError` and can decide.
|
||||||
|
R6. **Keep legitimate failover.** Deleted-model recovery via CivArchive/sqlite
|
||||||
|
must keep working (404 paths unchanged).
|
||||||
|
R7. **Single choke point.** The pacing gate should live where every API call
|
||||||
|
passes (the `Downloader`), so bulk refresh, metadata sync, recipe
|
||||||
|
analysis, and usage-control lookups all benefit without per-feature work.
|
||||||
|
|
||||||
|
## 4. Approach Comparison
|
||||||
|
|
||||||
|
### A. Reactive gate — shared `Retry-After` deadman clock (recommended core)
|
||||||
|
|
||||||
|
A process-wide, per-destination coordinator records the *next-allowed-send*
|
||||||
|
timestamp from each 429 (`now + max(retry_after, backoff)`). Every request
|
||||||
|
through `Downloader.make_request` consults the gate *before sending* and *when
|
||||||
|
a 429 arrives*; waiters block on a shared `asyncio.Event` that fires when the
|
||||||
|
cooldown expires.
|
||||||
|
|
||||||
|
- Pros: single choke point (R7); herd-free (R2); honors server guidance (R1);
|
||||||
|
no guessing at vendor limits; covers all providers automatically; reuses
|
||||||
|
existing per-destination scoping.
|
||||||
|
- Cons: still experiences 429s before slowing down (reactive); long
|
||||||
|
`Retry-After` windows (CivArchive has been observed at ~1500 s) need a sane
|
||||||
|
wait cap + skip/retry UX.
|
||||||
|
|
||||||
|
### B. Preemptive pacing — minimum inter-request interval (recommended companion)
|
||||||
|
|
||||||
|
Per-destination token bucket (simplest form: capacity 1 — at least `N` seconds
|
||||||
|
between consecutive API requests; `N` configurable, default ~0.75 s ≈ 80
|
||||||
|
r/min ceiling).
|
||||||
|
|
||||||
|
- Pros: prevents most 429s before they happen — exactly the "intentional
|
||||||
|
reduction in request rate" the issue asks for; trivial to implement on top
|
||||||
|
of A's coordinator.
|
||||||
|
- Cons: adds latency to bulk operations (thousands of models × `N`); the *exact*
|
||||||
|
vendor limits are unknown (CivitAI anonymous vs keyed vs `civitai.red`
|
||||||
|
mirror differ), so the default must be conservative-but-not-crippling and
|
||||||
|
settings-tunable.
|
||||||
|
|
||||||
|
### C. Fallback semantics change — stop network→network failover on 429 (must-do, low risk)
|
||||||
|
|
||||||
|
`FallbackMetadataProvider` (and `MetadataSyncService.fetch_and_update_model`'s
|
||||||
|
manual loop) must treat a final `RateLimitError` as a **terminal, non-failover
|
||||||
|
result** for network providers. Local-only providers (sqlite archive DB) may
|
||||||
|
stay as a last resort (no vendor cost).
|
||||||
|
|
||||||
|
- Pros: directly removes the CivArchive flood; small, surgical change.
|
||||||
|
- Cons: none significant; requires care to keep 404-failover intact (R6).
|
||||||
|
|
||||||
|
### Rejected / deferred
|
||||||
|
|
||||||
|
- **Per-feature retry queues** (batch import pauses & resumes whole batches):
|
||||||
|
richer UX but much larger change (batch state machine, WebSocket states);
|
||||||
|
unnecessary once A+B make requests wait at the choke point. Defer unless
|
||||||
|
review finds the bounded-wait UX insufficient.
|
||||||
|
- **Full token bucket with burst credit**: overkill; capacity-1 interval is
|
||||||
|
enough given the shared semaphore already caps concurrency at 5.
|
||||||
|
- **Retrying in `connectivity_guard`**: wrong layer — the guard is about
|
||||||
|
transport reachability, not vendor quota.
|
||||||
|
|
||||||
|
## 5. Recommended Architecture
|
||||||
|
|
||||||
|
New singleton **`RateLimitCoordinator`** (`py/services/rate_limit_coordinator.py`,
|
||||||
|
mirroring `ConnectivityGuard`'s singleton + per-destination patterns):
|
||||||
|
|
||||||
|
```
|
||||||
|
state per destination (hostname):
|
||||||
|
next_allowed_send: float (monotonic) # from 429 Retry-After + backoff
|
||||||
|
consecutive_429: int # for backoff growth
|
||||||
|
last_send_at: float # for min-interval pacing
|
||||||
|
waiters: list[Future] | asyncio.Event # shared wake-up per cooldown cycle
|
||||||
|
```
|
||||||
|
|
||||||
|
API:
|
||||||
|
|
||||||
|
- `async wait_for_slot(destination, request_started_within_window: bool)`
|
||||||
|
— called by `Downloader.make_request` *before* sending (blocks until
|
||||||
|
`min(now >= next_allowed_send)` and inter-request interval elapses) and
|
||||||
|
re-armable after a 429.
|
||||||
|
- `register_rate_limit(destination, retry_after: float | None)`
|
||||||
|
— called on 429: `next_allowed_send = max(now + retry_after_or_backoff, current)`;
|
||||||
|
`consecutive_429 += 1`; backoff = `retry_after` honored, else exponential
|
||||||
|
`30 · 2^(n-1)` capped at 1800 s; creates/re-arms the shared wake-up event.
|
||||||
|
- `register_success(destination)` — resets `consecutive_429` (called from the
|
||||||
|
existing 200 path in `make_request`).
|
||||||
|
- `remaining_seconds(destination)`, `in_cooldown(destination)` — for tests and
|
||||||
|
diagnostics.
|
||||||
|
|
||||||
|
Enforcement points:
|
||||||
|
|
||||||
|
1. **`Downloader.make_request`** (`downloader.py:1102-1132`): ordering inside
|
||||||
|
the method is **connectivity-guard fail-fast first** (offline short-circuit
|
||||||
|
costs nothing to check), **then** `await coordinator.wait_for_slot(destination)`
|
||||||
|
before `session.request`. On 429: `coordinator.register_rate_limit(...)`,
|
||||||
|
then *wait for the gate and re-send* (loop, bounded by
|
||||||
|
`rate_limit_max_wait_seconds`, default 300; `retry_after ≥ cap` ⇒ fail
|
||||||
|
immediately). After the loop, return the `RateLimitError` to the caller
|
||||||
|
(unchanged contract) **with `exc.gate_handled = True` set** so downstream
|
||||||
|
retry helpers know the wait already happened. 200 path calls
|
||||||
|
`register_success`.
|
||||||
|
2. **`Downloader.download_to_memory` / `get_response_headers`** (phase 2):
|
||||||
|
register 429s (so API calls queue); waiting only in `make_request`
|
||||||
|
initially.
|
||||||
|
3. **`FallbackMetadataProvider`** (`model_metadata_provider.py`): remove
|
||||||
|
network→network failover on `RateLimitError` — re-raise; only sqlite stays
|
||||||
|
as a local last resort (implementation: per-method `except RateLimitError`
|
||||||
|
handler that marks the chain rate-limited and stops iterating).
|
||||||
|
4. **`MetadataSyncService.fetch_and_update_model`**
|
||||||
|
(`metadata_sync_service.py:248-333`): on `RateLimitError` from the default
|
||||||
|
provider, stop appending further network providers (sqlite may remain);
|
||||||
|
the existing `any_rate_limited` merge already produces `"Rate limited"`.
|
||||||
|
5. **Batch import** (`batch_import_service.py`): no structural change needed —
|
||||||
|
items now wait inside `make_request`; optionally (phase 2) map residual
|
||||||
|
rate-limit failures (after the wait cap) to `SKIPPED` with
|
||||||
|
`"rate limited (retry_after=…s); re-run the import later"` instead of
|
||||||
|
`FAILED`, and surface a `rate_limited` flag in the WebSocket progress
|
||||||
|
broadcast.
|
||||||
|
6. **`_RateLimitRetryHelper` retries** (`model_metadata_provider.py`):
|
||||||
|
**Phase 1** — when the raised `RateLimitError` carries `gate_handled = True`
|
||||||
|
(set by the downloader after honoring the gate), the helper skips its own
|
||||||
|
`retry_after` sleep and re-raises immediately, eliminating the double wait.
|
||||||
|
The wiring stays so a `RateLimitError` still propagates cleanly; full
|
||||||
|
demotion/removal can follow once the gate proves out.
|
||||||
|
|
||||||
|
Settings (`settings.json`, schema extension in `SettingsManager`):
|
||||||
|
|
||||||
|
| key | default | meaning |
|
||||||
|
|---|---|---|
|
||||||
|
| `rate_limit_gate_enabled` | `true` | master switch for the coordinator |
|
||||||
|
| `rate_limit_max_wait_seconds` | `300` | how long `make_request` waits on a 429 gate before returning the error |
|
||||||
|
| `rate_limit_min_interval_seconds` | `0.75` | minimum seconds between API requests per destination (pacing, R6-friendly conservative default) |
|
||||||
|
|
||||||
|
## 6. Changes by File
|
||||||
|
|
||||||
|
| File | Change |
|
||||||
|
|---|---|
|
||||||
|
| `py/services/rate_limit_coordinator.py` (new) | coordinator singleton + per-destination state + tests seam |
|
||||||
|
| `py/services/downloader.py` | gate pre-check + 429 register/wait/retry loop + `register_success`; log the 429 notice at INFO once per cooldown, then DEBUG |
|
||||||
|
| `py/services/model_metadata_provider.py` | `FallbackMetadataProvider`: stop network failover on `RateLimitError`; helper skips its sleep when the error is marked `gate_handled` |
|
||||||
|
| `py/services/metadata_sync_service.py` | `fetch_and_update_model`: same failover semantics; keep sqlite last resort |
|
||||||
|
| `py/services/batch_import_service.py` | (phase 2) rate-limit failures → `SKIPPED` + `rate_limited` progress flag |
|
||||||
|
| `py/services/settings_manager.py` | new settings keys + defaults |
|
||||||
|
| `tests/services/test_rate_limit_coordinator.py` (new) | gate unit tests |
|
||||||
|
| `tests/services/test_civitai_client.py` / `test_civarchive_client.py` | provider-level 429 behavior |
|
||||||
|
| `tests/services/test_metadata_service.py` | failover-chain tests |
|
||||||
|
| `tests/services/test_batch_import_service.py` | SKIPPED-on-rate-limit |
|
||||||
|
|
||||||
|
## 7. Impact, Risks, Open Questions
|
||||||
|
|
||||||
|
- **Behavior change**: with the gate in `make_request`, any request can block
|
||||||
|
up to the wait cap — UI actions that call the API (e.g. a model-details
|
||||||
|
fetch) may take longer during cooldowns. Mitigation: bounded cap + INFO log
|
||||||
|
+ the existing async request handling already tolerates slow responses.
|
||||||
|
**Decided (§10): interactive requests take the same bounded wait** — one
|
||||||
|
behavior, no call-source plumbing; cooldowns are usually short.
|
||||||
|
- **Gate waits occupy batch slots**: with the 1–5 batch semaphore, all slots
|
||||||
|
can park on a gate simultaneously, freezing visible progress for up to one
|
||||||
|
wait cap per wave. Bounded and acceptable; the phase-2 `SKIPPED` mapping +
|
||||||
|
WebSocket `rate_limited` flag (both confirmed in scope, §10) make the stall
|
||||||
|
visible and recoverable.
|
||||||
|
- **Rate limit reality check**: CivitAI anonymous vs keyed limits, and whether
|
||||||
|
`civitai.red` differs, is unverified. Default pacing `0.75 s/req` is a
|
||||||
|
conservative guess (R6). Open question for maintainer: preferred default
|
||||||
|
and whether an API-keyed ceiling should be higher.
|
||||||
|
- **Long CivArchive windows**: `Retry-After ~1500 s` observed in code
|
||||||
|
comments. **Decided (§10): keep the 300 s default cap** — such lookups
|
||||||
|
fail/skip rather than park a request path for 25 minutes; batch import maps
|
||||||
|
them to `SKIPPED` (phase 2) so the user can re-run later.
|
||||||
|
- **Double waiting**: `_RateLimitRetryHelper` + gate could stack waits.
|
||||||
|
**Resolved in Phase 1**: the downloader marks gate-honored errors with
|
||||||
|
`gate_handled = True` and the helper skips its own sleep for those.
|
||||||
|
- **Downloads**: `download_file` 429s return an error to download managers
|
||||||
|
unchanged (already handled); only *registration* is proposed, so future
|
||||||
|
API calls queue behind a large `Retry-After` from a download burst.
|
||||||
|
|
||||||
|
## 8. Test Plan
|
||||||
|
|
||||||
|
1. **Coordinator unit tests** (new file):
|
||||||
|
- 429 with `retry_after` → `wait_for_slot` blocks ~that long, then passes.
|
||||||
|
- N concurrent waiters all wake together (herd test, wall-clock ≈ one
|
||||||
|
window, not N windows).
|
||||||
|
- Consecutive 429s grow backoff; `register_success` resets.
|
||||||
|
- Missing `Retry-After` → default backoff path.
|
||||||
|
- Wait cap: request fails after `rate_limit_max_wait_seconds` with
|
||||||
|
`RateLimitError`.
|
||||||
|
2. **Downloader tests** (mock aiohttp session): 429 then 200 → `make_request`
|
||||||
|
returns success after gate delay; two back-to-back calls to the same
|
||||||
|
destination are spaced ≥ `min_interval`; different destinations are not
|
||||||
|
spaced.
|
||||||
|
3. **Provider tests**: `FallbackMetadataProvider.get_model_version_info` —
|
||||||
|
Civitai raises `RateLimitError` → CivArchive mock **not called**; 404 still
|
||||||
|
falls through to CivArchive; sqlite still tried after network 429.
|
||||||
|
4. **Sync-service test**: `fetch_and_update_model` with a rate-limited default
|
||||||
|
provider → result error contains `"Rate limited"` and sqlite attempt state
|
||||||
|
unchanged.
|
||||||
|
5. **Batch-import test**: analysis provider 429s first, then succeeds →
|
||||||
|
item ends `SUCCESS` (wait path), and post-cap 429 → `SKIPPED` with
|
||||||
|
rate-limit reason (phase 2).
|
||||||
|
6. Full regression: `pytest tests/services tests/routes tests/standalone`
|
||||||
|
(currently 1582 passing).
|
||||||
|
|
||||||
|
## 9. Implementation Phases
|
||||||
|
|
||||||
|
- **Phase 1 (this plan, reviewed):** `RateLimitCoordinator` +
|
||||||
|
`Downloader.make_request` integration (guard fail-fast → gate pre-check
|
||||||
|
pacing → 429 register/wait/retry loop with cap → `gate_handled` marking) +
|
||||||
|
settings + **Fix C failover semantics** (`FallbackMetadataProvider`,
|
||||||
|
`fetch_and_update_model` — moved up from phase 2: smallest diff, kills the
|
||||||
|
CivArchive flood immediately, independent of coordinator correctness) +
|
||||||
|
`_RateLimitRetryHelper` double-wait fix + coordinator/downloader/provider/
|
||||||
|
sync tests.
|
||||||
|
- **Phase 2:** batch-import `SKIPPED`-on-rate-limit + `rate_limited` WebSocket
|
||||||
|
progress flag + slowdown hint (confirmed, §10),
|
||||||
|
`download_to_memory`/HEAD 429 registration, batch tests.
|
||||||
|
- **Phase 3:** full regression + docs + commit referencing `(#1085)`.
|
||||||
|
|
||||||
|
## 10. Review Checklist — Decisions (2026-08-27)
|
||||||
|
|
||||||
|
- [x] Default pacing interval `0.75 s` — **accepted** as conservative default;
|
||||||
|
tunable via `rate_limit_min_interval_seconds`. Revisit if CivitAI
|
||||||
|
publishes keyed/anonymous ceilings.
|
||||||
|
- [x] Wait cap `300 s` — **accepted**; long-window CivArchive lookups fail →
|
||||||
|
batch import marks them `SKIPPED` with a rate-limit reason (phase 2).
|
||||||
|
- [x] Interactive API calls also wait (bounded) — **yes**, same behavior for
|
||||||
|
all callers.
|
||||||
|
- [x] Keep sqlite as last resort behind a network rate limit — **yes**
|
||||||
|
(local-only, no vendor cost).
|
||||||
|
- [x] UI hint — **yes**: WebSocket `rate_limited` flag + "rate limited —
|
||||||
|
slowing down" hint in batch-import progress (phase 2); INFO logging
|
||||||
|
regardless.
|
||||||
@@ -0,0 +1,363 @@
|
|||||||
|
# Plan: "Other Models" Page — Unified Management for VAE / Upscaler / Text Encoder / etc.
|
||||||
|
|
||||||
|
**Status:** v2 — **Phase 1 implemented** (2026-09-12, commits `27da7b3c` backend + `fa7ce725` frontend; verified live against a running ComfyUI instance: scan/hash/sub_type-derivation/fetch/previews all green). **Phase 2 implemented** (2026-09-12, per §9 design; full pytest + vitest green). **Phase 3 implemented** (§11: opt-in management toggles; default off). **i18n done** (2026-09-13): all 36 new keys translated in the 9 non-English locales — the `[TODO: Translate]` placeholders left by the sync script during development are gone (see `docs/i18n-translation-guidelines.md` §2, "Other Models feature"). **Default set revised (pre-release):** only `vae` / `upscaler` / `text_encoder` are managed by default — `clip_vision` and `controlnet` are both opt-in (§2, §11.1.1).
|
||||||
|
**Scope (Phase 1):** scan + manage (list, search, filter, tags, folders, preview, rename, move, delete/exclude, CivitAI metadata fetch) for a new model type `other`, exposed as a new web page. **Phase 2 (§9):** one-click download from CivitAI for these types.
|
||||||
|
|
||||||
|
## 1. Goal
|
||||||
|
|
||||||
|
Today the manager supports three model types:
|
||||||
|
|
||||||
|
| page | model_type | sub_types |
|
||||||
|
|---|---|---|
|
||||||
|
| `/loras` | `lora` | `lora`, `locon`, `dora` |
|
||||||
|
| `/checkpoints` | `checkpoint` | `checkpoint`, `diffusion_model` |
|
||||||
|
| `/embeddings` | `embedding` | `embedding` |
|
||||||
|
|
||||||
|
Add a fourth page that manages "everything else" — VAE, upscalers, text encoders / CLIP, CLIP vision, optionally ControlNet — with a folder→sub_type mapping table so new ComfyUI folder categories can be added later by configuration, not code.
|
||||||
|
|
||||||
|
## 2. Locked Decisions
|
||||||
|
|
||||||
|
1. **Architecture: one scanner + one service + one page, sub_type derived by location.**
|
||||||
|
Replicates the checkpoint pattern (`CheckpointScanner` aggregates `checkpoints` + `unet` roots and derives `checkpoint` vs `diffusion_model` from the root containing the file, `py/services/checkpoint_scanner.py:384-415`). One `OtherScanner` aggregates all enabled folder roots; `resolve_sub_type_for_path()` maps each root to a sub_type. No per-category scanners.
|
||||||
|
|
||||||
|
2. **Naming: internal `model_type = "other"`, route prefix `/other`, page id `other`.**
|
||||||
|
- `misc` is rejected: `py/routes/misc_routes.py` already owns that name for system/settings routes (`/api/lm/settings`, `/api/lm/doctor/*`).
|
||||||
|
- `components` is rejected: `templates/components/` and `static/js/components/` directories would make `components.html` / `components.js` confusing neighbors.
|
||||||
|
- `other` matches CivitAI's `Other` fallback type semantics. The **display name** is an i18n string (`other.title`, e.g. "Other Models") and can be renamed later without touching code.
|
||||||
|
|
||||||
|
3. **sub_type values:** snake_case, aligned with CivitAI `ModelType` semantics:
|
||||||
|
|
||||||
|
| sub_type | ComfyUI `folder_paths` key(s) | CivitAI ModelType | enabled by default |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `vae` | `vae` | `VAE` | yes |
|
||||||
|
| `upscaler` | `upscale_models` | `Upscaler` | yes |
|
||||||
|
| `text_encoder` | `text_encoders`, `clip` (legacy) | `TextEncoder` (CLIP is retired upstream) | yes |
|
||||||
|
| `clip_vision` | `clip_vision` | `CLIPVision` | no (mapping present, opt-in) |
|
||||||
|
| `controlnet` | `controlnet` | `Controlnet` | no (mapping present, opt-in) |
|
||||||
|
|
||||||
|
New folder categories = one line in the mapping table (see §4.1).
|
||||||
|
|
||||||
|
**Why only three are on by default** (revised in Phase 3, before release):
|
||||||
|
VAE, upscalers and text encoders are dependency-style assets every pipeline
|
||||||
|
needs, and "which one am I actually using" is the recurring problem they
|
||||||
|
solve. `clip_vision` and `controlnet` are workflow-driven instead
|
||||||
|
(IPAdapter/SVD image conditioning; per-workflow ControlNet variants), and
|
||||||
|
ControlNet libraries routinely run to dozens of files, so both are treated
|
||||||
|
symmetrically as opt-in. Enumerating all five as "the default set" was not
|
||||||
|
defensible on demand breadth alone.
|
||||||
|
|
||||||
|
4. **Phase 1 = scan/manage only.** Downloads from CivitAI (`download_manager.py` type mapping, default-root settings keys, download routing) are Phase 2 (§9). CivitAI **metadata fetch** for existing files IS in Phase 1 (hash-based lookup is type-agnostic; only the type-validation hook needs new values).
|
||||||
|
|
||||||
|
5. **Out of scope (default off, revisit later):** usage statistics buckets, recipe matching (`recipe_scanner.py` only merges lora+checkpoint scanners), statistics page, embeddings re-classification (stays its own page — merging would be a breaking change).
|
||||||
|
|
||||||
|
## 3. Why This Works With Minimal Churn
|
||||||
|
|
||||||
|
- `ModelScanner` (`py/services/model_scanner.py:93`) is specialized entirely via constructor params (`model_type`, `model_class`, `file_extensions`) + optional hooks (`adjust_metadata`, `adjust_cached_entry`, `resolve_sub_type_for_path`, `model_scanner.py:1429-1443`).
|
||||||
|
- `BaseModelService` subclasses can be one method (`EmbeddingService` implements only `format_response`, `py/services/embedding_service.py:12`).
|
||||||
|
- Routes: `ModelServiceFactory.register_model_type()` (`py/services/model_service_factory.py:120-136`) + `COMMON_ROUTE_DEFINITIONS` (`py/routes/model_route_registrar.py:23-149`) generate the full `/api/lm/{prefix}/*` surface (~50 endpoints) plus the `GET /{prefix}` page route.
|
||||||
|
- `PersistentModelCache` (`py/services/persistent_model_cache.py:526-606`) is a single `models` table keyed `(model_type, file_path)` with `model_type` as free text — **zero schema change**.
|
||||||
|
- Frontend `apiConfig.js` (`static/js/api/apiConfig.js:51`) generates all endpoints from the model-type string; `ModelCard.js:670-675` renders the sub_type badge from data; the checkpoints page already demonstrates the "one page, multiple sub_types" filter (`header.html:298`).
|
||||||
|
|
||||||
|
## 4. Backend Changes
|
||||||
|
|
||||||
|
### 4.1 New constants — `py/utils/constants.py`
|
||||||
|
|
||||||
|
```python
|
||||||
|
# folder_paths key -> sub_type; single source of truth for extensibility
|
||||||
|
OTHER_MODEL_FOLDER_SUBTYPES = {
|
||||||
|
"vae": "vae",
|
||||||
|
"upscale_models": "upscaler",
|
||||||
|
"text_encoders": "text_encoder",
|
||||||
|
"clip": "text_encoder", # legacy ComfyUI key
|
||||||
|
"clip_vision": "clip_vision",
|
||||||
|
"controlnet": "controlnet",
|
||||||
|
}
|
||||||
|
DEFAULT_OTHER_MODEL_FOLDERS = ("vae", "upscale_models", "text_encoders", "clip", "clip_vision")
|
||||||
|
VALID_OTHER_SUB_TYPES = ["vae", "upscaler", "text_encoder", "clip_vision", "controlnet"]
|
||||||
|
# CivitAI model.type values accepted for this page (fetch-metadata validation)
|
||||||
|
VALID_OTHER_CIVITAI_TYPES = {"vae", "upscaler", "textencoder", "clipvision", "controlnet", "other"}
|
||||||
|
```
|
||||||
|
|
||||||
|
Also extend `CIVITAI_USER_MODEL_TYPES` (`constants.py:90`) if user-model queries should include these types.
|
||||||
|
|
||||||
|
### 4.2 New files (mirror the embedding/checkpoint implementations)
|
||||||
|
|
||||||
|
1. **`py/utils/models.py`** — add `OtherModelMetadata(BaseModelMetadata)`: default `sub_type="vae"` placeholder overridden by scanner hook; `from_civitai_info` mapping CivitAI types → our sub_types (`TextEncoder`→`text_encoder`, `CLIPVision`→`clip_vision`, `Upscaler`→`upscaler`, `VAE`→`vae`, `Controlnet`→`controlnet`, else `other`-ish fallback to folder-derived sub_type).
|
||||||
|
2. **`py/services/other_scanner.py`** — `OtherScanner(ModelScanner)`:
|
||||||
|
- `model_type="other"`, extensions: reuse the checkpoint set (`safetensors/pt/pt2/bin/pth/pkl/sft/gguf`).
|
||||||
|
- `get_model_roots()`: iterate `OTHER_MODEL_FOLDER_SUBTYPES` ∩ enabled keys, pull each from `config` (§4.3); dedupe; build `root → sub_type` map (normalized abspaths; multiple keys may share a sub_type).
|
||||||
|
- Implement all three hooks like `CheckpointScanner` (`checkpoint_scanner.py:384-415`): `resolve_sub_type_for_path` by longest-prefix root match, `adjust_metadata`, `adjust_cached_entry` (sub_type is re-derived on cache load, never persisted).
|
||||||
|
- **Lazy hashing, checkpoint-style**: text encoders (T5-XXL ≈ 10 GB) make eager sha256 painful. Copy the `hash_status="pending"` + singleflight `calculate_hash_for_model` pattern from `CheckpointScanner`.
|
||||||
|
3. **`py/services/other_model_service.py`** — `OtherModelService(BaseModelService)`, `format_response` only (no usage_count, like `EmbeddingService`).
|
||||||
|
4. **`py/routes/other_routes.py`** — `OtherRoutes(BaseModelRoutes)`, `template_name="other.html"`, hooks:
|
||||||
|
- `_validate_civitai_model_type` → `VALID_OTHER_CIVITAI_TYPES`
|
||||||
|
- `_get_expected_model_types`, `_parse_specific_params` (no type-specific download params in Phase 1)
|
||||||
|
- `initialize_services()` on `app.on_startup` pulling `ServiceRegistry.get_other_scanner()`.
|
||||||
|
|
||||||
|
### 4.3 `py/config.py`
|
||||||
|
|
||||||
|
- New `other_roots` property: for each enabled key in `OTHER_MODEL_FOLDER_SUBTYPES`, `folder_paths.get_folder_paths(key)` (plugin mode) — standalone mode needs nothing new: `MockFolderPaths` (`standalone.py:66-105`) already serves arbitrary keys from `settings.json.folder_paths`.
|
||||||
|
- Follow the existing per-type recipe: an `_prepare_other_paths()` (dedupe + symlink registration; also **cross-scanner overlap detection** — warn if an `other` root is already covered by checkpoints/unet/embedding roots, mirroring the checkpoint/unet overlap check).
|
||||||
|
- Wire into: `_apply_library_paths`, `_symlink_roots()`, `_rebuild_preview_roots()` (hard requirement — preview images are served per registered root), `save_folder_paths_to_settings()`.
|
||||||
|
|
||||||
|
### 4.4 Existing-file edits (the "type string scatter" — each is a small branch/entry)
|
||||||
|
|
||||||
|
| file | change |
|
||||||
|
|---|---|
|
||||||
|
| `py/services/model_service_factory.py:120` | register `("other", OtherModelService, OtherRoutes)` in `register_default_model_types()` |
|
||||||
|
| `py/services/service_registry.py` | add `get_other_scanner()` (mirror `:297` `get_embedding_scanner`) |
|
||||||
|
| `py/services/model_scanner.py:67` | `PAGE_TYPE_MAP['other'] = 'other'` (WebSocket progress) |
|
||||||
|
| `py/services/base_model_service.py:896-906` | `get_model_types()` branch → `VALID_OTHER_SUB_TYPES` |
|
||||||
|
| `py/lora_manager.py` | `_initialize_services` scanner task list (`:219-242`), `_cleanup` cancel list (`:463`), `_cleanup_backup_files` roots (`:327-330`) |
|
||||||
|
| `py/routes/handlers/misc_handlers.py` | `scanner_getters` (`:657-661`) + `scanner_factories` (`:757-759`) so Doctor / init-status / refresh-all see the new scanner |
|
||||||
|
| `py/services/pending_delete_service.py` | `_PAGE_TYPE` map (`:57-61`) + scanner getter list (`:983-985`) |
|
||||||
|
| `py/metadata_ops/__init__.py:36-38` | `SCANNER_TYPE_MAP['other']` |
|
||||||
|
| `settings.json.example` | document optional `folder_paths` keys: `vae`, `upscale_models`, `text_encoders`, `clip_vision` |
|
||||||
|
|
||||||
|
**Explicitly NOT touched in Phase 1:** `py/services/download_manager.py`, `py/services/download_routing.py`, `py/services/settings_manager.py` default-root keys, `py/routes/stats_routes.py`, `py/utils/usage_stats.py`, `py/services/recipe_scanner.py`, `py/metadata_collector/`, `py/nodes/`.
|
||||||
|
|
||||||
|
**Zero-change confirmations (verified):** `PersistentModelCache`, `ModelUpdateService`, `DownloadedVersionHistoryService`, `MetadataSyncService` + provider chain (type-agnostic hash lookups), `ModelFileService` / `ModelMoveService` / `ModelLifecycleService` (scanner + model_type injected), `ModelCache` / `ModelHashIndex`, `AutoV3BackfillService`.
|
||||||
|
|
||||||
|
## 5. Frontend Changes
|
||||||
|
|
||||||
|
1. **`static/js/api/apiConfig.js`** — `MODEL_TYPES.OTHER = 'other'`; `MODEL_CONFIG.other` entry (displayName, singularName, `supportsMove`, `supportsBulkOperations`; no letter filter); endpoints come free from `getApiEndpoints()` (`:51`).
|
||||||
|
2. **`static/js/api/otherApi.js`** — thin `OtherApiClient extends BaseModelApiClient` (mirror `embeddingApi.js`); register in `modelApiFactory.js`.
|
||||||
|
3. **`static/js/other.js`** — page entry (mirror `embeddings.js`): `appCore.initialize()` + `createPageControls('other')` + `initializePageFeatures()` + `ModelDuplicatesManager` + `initActiveFiltersSync('other')`.
|
||||||
|
4. **Controls & context menu** — `OtherControls extends PageControls` and `OtherContextMenu` (start from the embedding variants — the smallest); add branches in the two factories (`components/controls/index.js:15`, `components/ContextMenu/index.js:15`). Context-menu template block lives in `templates/other.html` (`{% block additional_components %}`, the checkpoints/embeddings pattern — do NOT touch the shared `context_menu.html`).
|
||||||
|
5. **`templates/other.html`** — copy `embeddings.html`: same content blocks (controls + breadcrumb + duplicates banner + folder sidebar + `#modelGrid`), `data-page="other"`, main script `/loras_static/js/other.js`.
|
||||||
|
6. **`templates/components/header.html`** — nav entry (`:23-43`, active when `request.path.startswith('/other')`); enable the `modelTypes` sub_type filter panel for `other` (`:298-305` pattern from checkpoints); check search-options panel conditions (`:199-224`).
|
||||||
|
7. **`static/js/utils/constants.js`** — `MODEL_SUBTYPE_ABBREVIATIONS` (`:115`): `vae→VAE`, `upscaler→UPS`, `text_encoder→TE`, `clip_vision→CV`, `controlnet→CN`; matching `MODEL_SUBTYPE_DISPLAY_NAMES` (`:99`). (Unknown fallback already uppercases 4 chars, but explicit mappings read better.)
|
||||||
|
8. **`static/js/core.js:110` `getPageType()`** — verify `data-page="other"` flows through `state.pages` generically; add only if the page list is enumerated anywhere.
|
||||||
|
9. No change to `web/comfyui/top_menu_extension.js` (it opens `/loras`; page-to-page nav is the header bar).
|
||||||
|
|
||||||
|
## 6. i18n
|
||||||
|
|
||||||
|
- `locales/en.json`: add `other.title` (e.g. "Other Models") + minimal `other.contextMenu.*` / `other.modelTypes.*` keys; reuse `modelCard.*`, `loras.contextMenu.*`, `common.*` wherever possible (the established pattern — checkpoints/embeddings already reuse lora keys).
|
||||||
|
- Run `python scripts/sync_translation_keys.py`; leave `[TODO: Translate]` placeholders in other locales (per `docs/i18n-translation-guidelines.md` §7 — do not translate proactively).
|
||||||
|
|
||||||
|
## 7. Testing
|
||||||
|
|
||||||
|
Follow existing conventions (`pytest.ini`, `tests/frontend/` vitest):
|
||||||
|
|
||||||
|
1. **Backend (pytest, async where needed):**
|
||||||
|
- `OtherScanner` root aggregation + `resolve_sub_type_for_path` (file under `vae/` root → `vae`; `text_encoders` and legacy `clip` both → `text_encoder`; disabled `controlnet` root not scanned).
|
||||||
|
- Cache round-trip: sub_type re-derived via `adjust_cached_entry` (not persisted).
|
||||||
|
- Lazy hash: `hash_status="pending"` default; `calculate_hash_for_model` singleflight.
|
||||||
|
- `OtherRoutes` registration smoke test: `/api/lm/other/...` endpoints exist; `_validate_civitai_model_type` accepts `vae`/`upscaler`/`textencoder`, rejects `lora`.
|
||||||
|
- Config: `other_roots` in both modes (mock `folder_paths`, and standalone `settings.json.folder_paths`).
|
||||||
|
2. **Frontend (vitest + jsdom, `tests/frontend/`):**
|
||||||
|
- `apiConfig`: `getApiEndpoints('other')` URL shapes; `modelApiFactory` returns the Other client.
|
||||||
|
- `ModelCard` badge rendering for new sub_types.
|
||||||
|
- `createPageControls('other')` / `createPageContextMenu('other')` factories.
|
||||||
|
3. **Manual UI verification by the user** (per AGENTS.md — no sandbox/browser automation): page loads, scans a real library, sub_type filter + badges, context menu actions.
|
||||||
|
|
||||||
|
## 8. Execution Order
|
||||||
|
|
||||||
|
1. `constants.py` + `OtherModelMetadata` + `config.py` roots
|
||||||
|
2. `OtherScanner` (+ registry, factory, `PAGE_TYPE_MAP`) → scanner unit tests green
|
||||||
|
3. `OtherModelService` + `OtherRoutes` + handler/registrar wiring + `lora_manager.py` lifecycle → route tests green
|
||||||
|
4. Doctor/pending-delete/metadata-ops scatter entries
|
||||||
|
5. Template + header nav + frontend API/controls/context-menu/card badges → vitest green
|
||||||
|
6. i18n keys + sync script
|
||||||
|
7. `pytest` + `npm test` full runs; hand to user for manual UI check
|
||||||
|
|
||||||
|
## 9. Phase 2 Detailed Design — CivitAI Downloads for `other`
|
||||||
|
|
||||||
|
Designed 2026-09-12 against the Phase-1 code on this branch; decisions marked **[locked]** follow the same recommendations the feature owner approved for Phase 1.
|
||||||
|
|
||||||
|
### 9.1 Download pipeline touch points
|
||||||
|
|
||||||
|
Flow: `POST /api/lm/download-model` (`py/routes/model_route_registrar.py:104`; GET variant `:105` for the browser extension) → `ModelDownloadHandler.download_model` (`model_handlers.py:1740`) → `DownloadModelUseCase.execute` → `DownloadCoordinator.schedule_download` → `DownloadManager.download_from_civitai` (`download_manager.py:386`) → `_execute_original_download` (`:1415`). Inside, seven scatter points need an `other` branch:
|
||||||
|
|
||||||
|
1. **Type map** (`:1496-1507`): accept `model.type.lower() in VALID_OTHER_CIVITAI_TYPES` → `model_type = "other"` (reuses the Phase-1 set, incl. `"other"` itself).
|
||||||
|
2. **Early version-exists gate** (`:1436-1463`): add `other_scanner.check_model_version_exists`.
|
||||||
|
3. **File-level exists gate** (`:1640-1655` → `_find_local_file_entry` `:320-346` → `_get_scanner_for_model_type` `:230-236`): add explicit `other` branch. **Trap**: the function currently falls through to the lora scanner for unknown types — `"other"` would silently dedupe against loras. Also narrow the fall-through to `"lora"` only / raise on unknown.
|
||||||
|
4. **Version-level fallback gate** (`:1656-1688`): add `elif model_type == "other"`.
|
||||||
|
5. **Default-root selection** (`:1690-1727`): for `other`, first resolve sub_type (§9.2), then read `default_other_roots[sub_type]` (§9.3); if sub_type is undecidable or no default root configured → error guiding the user to pick a folder explicitly.
|
||||||
|
6. **Metadata class selection** (`:1909-1928`) + `_build_metadata_for_resume` (`:969-981`): add `OtherModelMetadata.from_civitai_info` branches.
|
||||||
|
7. **Post-download cache write** (`_execute_download_pipeline` `:2622-2679`): add `other` scanner branch; `adjust_metadata` re-derives sub_type from the on-disk root automatically. `_get_supported_extensions_for_type` (`:2720-2744`): `other` reuses the checkpoint extension set.
|
||||||
|
|
||||||
|
Hooks: `_record_downloaded_version_history` (model_type is free text — zero change); `_sync_downloaded_version` (`:1984` → scanner dispatch `:2130-2135`) add `other`; `py/utils/example_images_download_manager.py` scanner dispatch at `:411-421`, `:591-601`, `:1089+` — add `other` at all three (silent no-scanner otherwise).
|
||||||
|
|
||||||
|
Path templates: `get_download_path_template("other")` is unset, so `other` resolves to a **flat** layout (empty template) — downloads land directly under the resolved sub_type root. This is deliberate: other-model roots are already split per sub_type (`default_other_roots`), and `priority_tags` has no `other` entry, so `{first_tag}` would fall back to an arbitrary CivitAI tag and scatter files into unstable folders. Users who want nesting can still set `download_path_templates["other"]` in `settings.json`. See `DEFAULT_DOWNLOAD_PATH_TEMPLATES` (`py/utils/constants.py`) and `DEFAULT_PATH_TEMPLATES` (`static/js/utils/constants.js`).
|
||||||
|
|
||||||
|
### 9.2 File-level routing (model.type / file.type → sub_type) **[locked]**
|
||||||
|
|
||||||
|
Table-driven, mirroring Phase 1. New in `py/utils/constants.py`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
CIVITAI_FILE_TYPE_TO_OTHER_SUB_TYPE = {
|
||||||
|
"VAE": "vae", "Upscaler": "upscaler", "Text Encoder": "text_encoder",
|
||||||
|
"Vision Encoder": "clip_vision", "CLIPVision": "clip_vision",
|
||||||
|
"ControlNet": "controlnet",
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`download_routing.py` gains `resolve_other_download_sub_type(civitai_model_type, file_types, selected_file_type=None)` with fixed priority:
|
||||||
|
|
||||||
|
1. **Explicit user file pick** (`file_params` from #1058's `_resolve_target_file`) — if the picked file's type maps, it wins even when model.type is `Checkpoint`.
|
||||||
|
2. **model.type** via the existing `CIVITAI_TYPE_TO_OTHER_SUB_TYPE` (`constants.py:120-127`).
|
||||||
|
3. **file.type fallback** — only when model.type maps to nothing (e.g. model.type `Other` or retired `CLIP`). MUST NOT override a mapped model.type: checkpoint models routinely bundle VAE/Text Encoder component files, and unconditional file-type routing would misroute them.
|
||||||
|
4. Still undecidable → `None`; `use_default_paths` errors and the UI offers all other roots for manual selection.
|
||||||
|
|
||||||
|
HTTP: extend `DownloadRoutingHandler.get_download_routing` (`download_routing_handlers.py:23`) with an `other` branch returning `{root_kind: "other", sub_type: ...}`; add `GET /api/lm/other/roots_by_subtype` in `OtherRoutes.setup_specific_routes` (data from `config._prepare_other_paths`'s per-key roots, aggregating `text_encoders` + legacy `clip` under `text_encoder`).
|
||||||
|
|
||||||
|
### 9.3 Settings: single dict key `default_other_roots` **[locked]**
|
||||||
|
|
||||||
|
Rejected: four flat keys (`default_vae_root`…) — each flat key costs ~13 touch points in `settings_manager.py` (defaults `:82-85`, `_check_and_auto_set` `:890-895`, `set()` `:1621-1628`, `_update_active_library_entry` `:738-805`, upsert/create signatures `:1953-2132`, `_build_library_payload` `:552-612`, `_sync_active_library_to_root` `:519-547`, three library constructors, frontend `DEFAULT_SETTINGS_BASE`), repeated per future sub_type.
|
||||||
|
|
||||||
|
Chosen: one mapping key `default_other_roots: {sub_type: path}`, copying the `extra_folder_paths` precedent (generic Mapping handling at `:533-535`, `:573-578`, `:763-767`). `_check_and_auto_set` generalizes to per-sub_type candidates (union over that sub_type's folder keys — `text_encoder` → `text_encoders` + `clip`). `set()` validates keys against `VALID_OTHER_SUB_TYPES`.
|
||||||
|
|
||||||
|
Also fix the Phase-1 omission: add `"other_scanner"` to `_notify_library_change` (`:2150-2156`) and `_notify_model_name_display_change` (`:1795-1800`) — otherwise switching libraries leaves the other page stale.
|
||||||
|
|
||||||
|
### 9.4 Settings UI
|
||||||
|
|
||||||
|
- `templates/components/modals/settings/library.html:34-40`: sub_type selectors after the existing four `setting_select`s (Jinja loop; controlnet selector only when `enabled_other_folders` includes it). Dict-subkey save helper `saveOtherRootSetting(subType, value)` alongside the flat `saveSelectSetting`.
|
||||||
|
- `static/js/managers/SettingsManager.js:1547-1697`: `loadOtherRoots()` mirroring `loadUnetRoots()`, fed by `/api/lm/other/roots_by_subtype`; current values from `state.global.settings.default_other_roots`. `state/index.js:24` `DEFAULT_SETTINGS_BASE` += `default_other_roots: {}`.
|
||||||
|
- Optional: one `other` row in the download-path-template block (`library.html:153-211`).
|
||||||
|
- i18n: `settings.folderSettings.*` keys into `locales/en.json` + sync script; other locales keep `[TODO: Translate]`.
|
||||||
|
- Settings GET (`misc_handlers.py:1528-1536`) already returns all non-sensitive keys — new key reaches the frontend for free.
|
||||||
|
|
||||||
|
### 9.5 Frontend download entry
|
||||||
|
|
||||||
|
- `templates/components/controls.html:83`: drop the `page_id != 'other'` exclusion on the download button (keyboard shortcut D self-enables via `PageControls.js:196-198`).
|
||||||
|
- `OtherControls.js:22-55`: add `showDownloadModal: () => downloadManager.showDownloadModal()` (mirror `EmbeddingsControls.js:43-45`).
|
||||||
|
- `DownloadManager.js` `proceedToLocationContent` (`:955-1017`): add `_resolveOtherSubType()` (mirror `_resolveIsDiffusionModel` `:1026`): selected file type → `/api/lm/download/routing` → `otherApiClient.fetchModelRoots(subType)` (new); default-root preselect reads `default_other_roots[subType]` instead of `` `default_${singularType}_root` `` (`:974`). Undecidable → list all other roots (`/api/lm/other/roots`) for manual pick; an explicit save_dir skips backend default-root logic, so the two paths cannot disagree.
|
||||||
|
- `ModelVersionsTab` download buttons are modelType-generic and already work via `getModelApiClient('other')`; context menu has no CivitAI download entry — no change.
|
||||||
|
- Version-list type validation (`get_civitai_versions` → `_validate_civitai_model_type`) already accepts `VALID_OTHER_CIVITAI_TYPES` from Phase 1.
|
||||||
|
|
||||||
|
### 9.6 CivitAI type mapping decisions **[locked]**
|
||||||
|
|
||||||
|
- Download accepts exactly `VALID_OTHER_CIVITAI_TYPES` (`VAE, Upscaler, TextEncoder, CLIP, CLIPVision, Controlnet, Other`) — reuse the Phase-1 tables; do NOT create new ones.
|
||||||
|
- Extend `CIVITAI_USER_MODEL_TYPES` (`constants.py:133-137`) with the 7 aliases, and point them at the other scanner / `"other"` history bucket in `misc_handlers.py` (`type_scanner_map` `:2793-2797`, `downloaded_version_map` `:2821-2827`) — otherwise creator pages silently filter these models while downloads claim support.
|
||||||
|
- Fix (small Phase-1 bug): `OtherModelMetadata.from_civitai_info` (`py/utils/models.py:343`) reads `version_info.get("type")`, but the type lives at `version["model"]["type"]` — the mapping never fires and always degrades to the placeholder. Read `version_info.get("model", {}).get("type")` instead. (`CheckpointMetadata:290` has the same shape; leave it alone here.)
|
||||||
|
|
||||||
|
### 9.7 Tests
|
||||||
|
|
||||||
|
Existing base: `tests/services/test_download_manager_basic.py` (incl. `test_download_rejects_unsupported_model_type` `:1336`), `test_download_manager_error.py`, `test_download_manager_concurrent.py`, `tests/integration/test_download_flow.py`, `tests/services/test_settings_manager.py`; frontend `tests/frontend/managers/downloadManager.routing.test.js`, `settingsManager.library.test.js`.
|
||||||
|
|
||||||
|
Add: (1) `resolve_other_download_sub_type` unit tests — every priority tier, bundled-component anti-misrouting, undecidable → None, civarchive-shaped payload; (2) download_manager — six model.types accepted → other scanner (mock), unknown still rejected, no lora-scanner fall-through, per-sub_type default roots + unconfigured error, resume metadata, extension set; (3) settings_manager — `default_other_roots` defaults/auto-set (incl. text_encoder dual-key union)/library sync/upsert passthrough/illegal sub_type rejection; (4) routes — `/api/lm/download/routing` other branch, `roots_by_subtype` shape; (5) example-images dispatch accepts `other` (3 sites); (6) vitest — `_resolveOtherSubType` + root select + default preselect, `loadOtherRoots`; (7) user-models existsLocally for VAE.
|
||||||
|
|
||||||
|
### 9.8 Phase 2 file list
|
||||||
|
|
||||||
|
Backend: `py/utils/constants.py`, `py/services/download_routing.py`, `py/routes/handlers/download_routing_handlers.py`, `py/services/download_manager.py`, `py/utils/example_images_download_manager.py`, `py/services/settings_manager.py`, `py/utils/models.py`, `py/routes/other_routes.py`, `py/routes/handlers/misc_handlers.py`, `settings.json.example`.
|
||||||
|
Frontend/templates: `templates/components/controls.html`, `static/js/components/controls/OtherControls.js`, `static/js/managers/DownloadManager.js`, `static/js/api/otherApi.js`, `templates/components/modals/settings/library.html`, `static/js/managers/SettingsManager.js`, `static/js/state/index.js`, `locales/en.json` + sync.
|
||||||
|
|
||||||
|
## 10. Risks / Open Questions
|
||||||
|
|
||||||
|
- **Root overlap**: a user may point `text_encoders` at a directory already scanned as checkpoints/unet. Realpath dedup inside one scanner won't catch cross-scanner overlap → the `_prepare_other_paths` overlap warning (§4.3) is the mitigation; duplicate cards across pages are cosmetic, not corrupting (cache keyed by `(model_type, file_path)`).
|
||||||
|
- **Huge text encoders + lazy hash**: CivitAI fetch for a pending-hash model must trigger on-demand hash like checkpoints do — verify that flow (`calculate_hash_for_model`) is reachable from the `other` routes' fetch-metadata handler.
|
||||||
|
- **Retired CivitAI types**: `CLIP`/`CLIPVision` are retired upstream (grandfathered for existing models); metadata fetch must tolerate both retired and current types — `VALID_OTHER_CIVITAI_TYPES` includes them deliberately.
|
||||||
|
- **Standalone users** must add the new `folder_paths` keys to `settings.json` themselves; document in `settings.json.example` and the feature doc.
|
||||||
|
- **Page display name** is i18n-only; if "Other Models" tests poorly, rename `other.title` without code changes.
|
||||||
|
|
||||||
|
### Phase 2 risks
|
||||||
|
|
||||||
|
- **Bundled component files**: checkpoint models routinely ship VAE/Text Encoder component files — file.type routing must stay a fallback (or explicit user pick), never an override (§9.2 priority is load-bearing; test it).
|
||||||
|
- **`_get_scanner_for_model_type` lora fall-through** (`download_manager.py:236`): without an explicit `other` branch, dedupe checks run against the lora scanner — the most insidious trap in Phase 2.
|
||||||
|
- **text_encoder dual folder keys** (`text_encoders` + legacy `clip`): default-root candidates, `roots_by_subtype`, and auto-set must all merge both keys; miss one and the default-root dropdown comes up empty.
|
||||||
|
- **Undecidable sub_type** (model.type `Other` + unknown file types): must error and ask, never silently default to the vae folder.
|
||||||
|
- **Lazy hash after download**: downloads carry CivitAI SHA256 (no recompute needed) — ensure the post-download cache write doesn't leave `hash_status="pending"`, or the next metadata fetch re-hashes a 10 GB file.
|
||||||
|
- **CivArchive source**: same `_execute_original_download` path, same payload shape — cover it once in tests.
|
||||||
|
|
||||||
|
## 11. Phase 3 — Opt-in Management Toggles (implemented)
|
||||||
|
|
||||||
|
Designed 2026-09-13 against the Phase-1/2 code. Other Models is **opt-in**: after
|
||||||
|
Phase 3 the feature ships disabled, so no other-model folder is scanned and the
|
||||||
|
page shows an "enable" empty state until the user turns it on.
|
||||||
|
|
||||||
|
### 11.1 Settings (global, not per-library)
|
||||||
|
|
||||||
|
| key | type | default | meaning |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `enable_other_models` | bool | `false` | master switch |
|
||||||
|
| `enabled_other_sub_types` | list[str] | `["vae","upscaler","text_encoder"]` | allow-list; `clip_vision` and `controlnet` are opt-in (see §2) |
|
||||||
|
|
||||||
|
`enabled_other_folders` (the unreleased, additive, no-UI backend key) was removed
|
||||||
|
and replaced by the sub_type-level allow-list; there is no migration because the
|
||||||
|
feature never shipped. `text_encoder` expands to `text_encoders` + legacy `clip`
|
||||||
|
via `OTHER_SUB_TYPE_FOLDER_KEYS`.
|
||||||
|
|
||||||
|
The default allow-list lives on five surfaces that must stay in sync:
|
||||||
|
`DEFAULT_ENABLED_OTHER_SUB_TYPES` (`py/utils/constants.py`), `DEFAULT_SETTINGS`
|
||||||
|
(`py/services/settings_manager.py`), the two `DEFAULT_SETTINGS_BASE` /
|
||||||
|
`createDefaultSettings` lists (`static/js/state/index.js`), the
|
||||||
|
`updateOtherModelsControls()` fallback (`static/js/managers/SettingsManager.js`)
|
||||||
|
and the server-rendered Jinja fallback
|
||||||
|
(`templates/components/modals/settings/library.html`).
|
||||||
|
|
||||||
|
### 11.1.1 Legacy key handling in `Config._init_other_paths`
|
||||||
|
|
||||||
|
ComfyUI's `folder_paths` rewrites legacy names before every access (`clip` →
|
||||||
|
`text_encoders`, `unet` → `diffusion_models`) and registers both legacy
|
||||||
|
directories under the canonical key, so `get_folder_paths("clip")` returns
|
||||||
|
exactly the same list as `get_folder_paths("text_encoders")`. Querying both keys
|
||||||
|
made the overlap guard fire twice with `please fix your path configuration` for a
|
||||||
|
configuration the user cannot fix. `Config._collapse_legacy_folder_keys()` now
|
||||||
|
drops a key when the host exposes `map_legacy` and resolves it to another queried
|
||||||
|
key, and `_prepare_other_paths()` downgrades a same-`sub_type` duplicate to
|
||||||
|
`debug` (a cross-`sub_type` collision still warns). In standalone mode
|
||||||
|
`MockFolderPaths` has no `map_legacy` and its keys are independent
|
||||||
|
`settings.json` entries, so every key is still queried there.
|
||||||
|
|
||||||
|
`settings.json.example` intentionally stays minimal (only `use_portable_settings`,
|
||||||
|
`civitai_api_key`, and the four core `folder_paths` keys: `loras`, `checkpoints`,
|
||||||
|
`unet`, `embeddings`). Optional keys — including the other-model folder paths and
|
||||||
|
`enable_other_models` — are NOT documented there; they live in `DEFAULT_SETTINGS`
|
||||||
|
and reach the user's `settings.json` on demand. This supersedes the Phase-1/Phase-2
|
||||||
|
notes that proposed adding the other-model folder keys to the example.
|
||||||
|
|
||||||
|
### 11.2 Behaviour matrix
|
||||||
|
|
||||||
|
| state | scan | nav / `/other` | other downloads | `default_other_roots` | Doctor / refresh-all |
|
||||||
|
|---|---|---|---|---|---|
|
||||||
|
| master off | nothing (`other_roots == []`) | nav entry hidden (`nav-item--hidden`); `/other` still renders the disabled empty state + Enable button; one-time dismissible announcement banner on first visit | rejected | preserved, never auto-set | scanner skipped |
|
||||||
|
| sub_type off | that sub_type's folder keys excluded | page keeps working, type disappears from data | auto-routing refused (manual folder still allowed) | preserved, not preselected | normal |
|
||||||
|
| all on (after enabling) | Phase-1/2 behaviour | normal | normal | normal | normal |
|
||||||
|
|
||||||
|
### 11.3 Backend touch points
|
||||||
|
|
||||||
|
- `py/utils/constants.py` — `DEFAULT_ENABLED_OTHER_SUB_TYPES`, `OTHER_SUB_TYPE_FOLDER_KEYS`, `normalize_other_sub_types`.
|
||||||
|
- `py/config.py` — `_get_enabled_other_folder_keys()` is the single scan gate (master switch + allow-list); new `refresh_other_roots()` rebuilds roots + preview roots on toggle.
|
||||||
|
- `py/services/settings_manager.py` — new defaults, `set()` normalization, `is_other_models_enabled()` / `get_enabled_other_sub_types()` / `is_other_sub_type_enabled()`, and `_apply_other_model_settings_change()` which reapplies config and calls `other_scanner.on_library_changed(reconcile=True)`.
|
||||||
|
- `py/services/model_scanner.py` — `_should_keep_cached_entry()` hydration hook (default keep) plus `on_library_changed(reconcile=...)` / `initialize_in_background(reconcile=...)`; the hook filters `raw_data` and the hash/autov3 index rows.
|
||||||
|
- `py/services/other_scanner.py` — drops persisted entries whose folder is no longer a managed root (sub_type is location-derived, so config is the source of truth).
|
||||||
|
- `py/routes/other_routes.py` — `_validate_civitai_model_type` rejects everything while off / mapped-but-disabled sub_types; `_get_page_context_provider()` injects `other_disabled` into the template.
|
||||||
|
- `py/routes/handlers/model_handlers.py` + `base_model_routes.py` — optional `page_context_provider` hook on `ModelPageView`.
|
||||||
|
- `py/routes/handlers/download_routing_handlers.py` — returns `{sub_type: None, disabled: true, reason}` instead of guessing.
|
||||||
|
- `py/services/download_manager.py` — rejects other-type downloads while off; disabled sub_type refuses default-path routing with a "pick a folder" error.
|
||||||
|
- `py/routes/handlers/misc_handlers.py` — Doctor / init-status / refresh-all skip the other scanner while off (`_active_scanner_factories` / `_active_scanner_getters`).
|
||||||
|
- `py/services/pending_delete_service.py` — deliberately untouched: the scanner stays registered so staged deletes still merge.
|
||||||
|
|
||||||
|
### 11.4 Frontend
|
||||||
|
|
||||||
|
Discoverability: the nav entry is hidden while the feature is off, and three
|
||||||
|
lightweight surfaces replace it — a one-time announcement banner, the download
|
||||||
|
toast, and the settings toggle itself.
|
||||||
|
|
||||||
|
- `templates/components/header.html` + `static/css/components/header.css` — `nav-item--hidden` class (server-rendered when off, client-toggled after enabling) and the `fa-shapes` icon.
|
||||||
|
- `templates/other.html` — `other_disabled` branch in `content` + `main_script`; page-scoped CSS for the empty state.
|
||||||
|
- `static/js/other_disabled.js` — boots `appCore` (shared header) and delegates to the shared enable helper.
|
||||||
|
- `static/js/utils/otherModels.js` — shared `enableOtherModels()` (POST settings + reload) and `openOtherModelsSettings()` (settings modal on the Library section); used by the disabled page, the banner and the download modal.
|
||||||
|
- `static/js/managers/BannerService.js` — `other-models-announcement` banner (only when off and not dismissed; `priority: 0`, dismissal persisted via `dismissed_banners`) with Enable / Open Settings actions; `removeOtherModelsAnnouncement()` drops it without persisting a dismissal.
|
||||||
|
- `templates/components/modals/settings/library.html` + `SettingsManager.updateOtherModelsControls()` / `saveEnabledOtherSubTypes()` / `updateOtherModelsNavVisibility()` — master toggle + five sub_type checkboxes; unchecked/disabled sub_types have their default-root select disabled.
|
||||||
|
- `static/js/managers/DownloadManager.js` — a disabled routing answer surfaces a `showActionToast` with an "Enable Other Models" action (opening settings) and falls back to manual selection.
|
||||||
|
- i18n: `settings.folderSettings.*`, `other.disabled.*` and `banners.otherModels.*` keys in `locales/en.json` + `scripts/sync_translation_keys.py` (other locales keep `[TODO: Translate]`).
|
||||||
|
|
||||||
|
### 11.5 Cache consistency
|
||||||
|
|
||||||
|
- Disabling purges rows from the in-memory view at hydration time (the
|
||||||
|
`_should_keep_cached_entry` hook) and from SQLite on the reconcile triggered by
|
||||||
|
the toggle; the `.metadata.json` sidecars survive, so re-enabling rescans
|
||||||
|
without recomputing hashes (critical for multi-GB text encoders).
|
||||||
|
- Enabling triggers a reconcile so newly managed roots are scanned immediately.
|
||||||
|
- Editing `settings.json` while the server is stopped is still covered by the
|
||||||
|
hydration hook, so disabled types never appear after a restart.
|
||||||
|
|
||||||
|
### 11.6 Tests
|
||||||
|
|
||||||
|
Backend: opt-in fixtures added to the other-related suites; new coverage for
|
||||||
|
"default off scans nothing", per-sub_type gating, routing/download rejection,
|
||||||
|
`_should_keep_cached_entry`, settings normalization and `other_disabled` page
|
||||||
|
context. Frontend: `updateOtherModelsControls` / `saveEnabledOtherSubTypes` and
|
||||||
|
the disabled-page enable flow.
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
# CivitAI image imports can end up with 0 LoRAs
|
||||||
|
|
||||||
|
## Symptom
|
||||||
|
|
||||||
|
Importing a CivitAI image URL can produce a recipe with **zero LoRA
|
||||||
|
entries**, even though the image page lists LoRAs in its resource panel.
|
||||||
|
|
||||||
|
Reported example: `https://civitai.red/images/140818889` was imported as a
|
||||||
|
local recipe with 0 LoRAs, while the page shows 3 LoRAs. Some images (e.g.
|
||||||
|
NSFW / higher browsing level) additionally require a login to view, so their
|
||||||
|
data is not publicly reachable at all.
|
||||||
|
|
||||||
|
## Root cause
|
||||||
|
|
||||||
|
URL imports use only two data sources:
|
||||||
|
|
||||||
|
1. **CivitAI REST image API** — `GET /api/v1/images?imageId=<id>&nsfw=X&withMeta=true` → `meta`
|
||||||
|
2. **Embedded image metadata** — EXIF/XMP read from the downloaded bytes
|
||||||
|
|
||||||
|
For the same image both sources can be empty, and the one source that does
|
||||||
|
contain the data is never queried. Verified for image 140818889:
|
||||||
|
|
||||||
|
| Source | What it returned |
|
||||||
|
|---|---|
|
||||||
|
| REST image API | `meta` holds only a prompt; `modelVersionIds: []`; no `resources`/`hashes`; `baseModel: null` |
|
||||||
|
| Downloaded image | PNG with **no EXIF/XMP** (the CDN URL ends in `.jpeg`, the body is PNG) |
|
||||||
|
| Image page HTML | `__NEXT_DATA__` embeds the trpc `image.getGenerationData` result → full `resources` list: 3 LoRAs, each with `modelId`, `modelVersionId`, `modelName`, `modelType`, `versionName`, `baseModel` |
|
||||||
|
|
||||||
|
Key points:
|
||||||
|
|
||||||
|
- The page's resource panel is fed by an **internal, non-public trpc
|
||||||
|
endpoint**, not by the public REST image API.
|
||||||
|
- That internal endpoint is **login-gated** for some content — the
|
||||||
|
"requires login" symptom.
|
||||||
|
- Even with the version IDs in hand, `/model-versions/{id}` for these
|
||||||
|
(Krea) versions returns **no `sha256`**, so an exact local-file hash match
|
||||||
|
is impossible; only model/version identity is recoverable.
|
||||||
|
|
||||||
|
## Conclusion / status
|
||||||
|
|
||||||
|
0-LoRA imports are a data-source gap: public REST meta and image EXIF are
|
||||||
|
both empty, while the only complete source (page generation data) is
|
||||||
|
internal, sometimes login-gated, and not used by the importer.
|
||||||
|
|
||||||
|
Such imports **cannot be reliably auto-repaired/completed** by the backend
|
||||||
|
alone. The old "Repair Metadata" feature only re-fetched the same incomplete
|
||||||
|
REST meta and could not fix them; it was deprecated and has been removed.
|
||||||
|
|
||||||
|
**Fixed via the companion browser extension.** When the extension is
|
||||||
|
installed with a valid license, it scrapes the image page's internal trpc
|
||||||
|
generation data with the user's session and calls the payload-capable
|
||||||
|
re-import endpoint (`POST /api/lm/recipe/{recipe_id}/reimport` with
|
||||||
|
`image_url`/`name`/`resources`/`gen_params`/`base_model`/`tags` query
|
||||||
|
params), which rebuilds the recipe from the caller-supplied metadata. The
|
||||||
|
web UI delegates re-import of CivitAI-image-sourced recipes to the extension
|
||||||
|
automatically (probe + `lm:reimport*` DOM events); without the extension,
|
||||||
|
re-import silently falls back to the native path, which remains limited by
|
||||||
|
the data-source gap documented above.
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
# Reconcile 的 Windows 大小写回退分支 - 待验证清单
|
||||||
|
|
||||||
|
> **状态**: 待 Windows 环境验证 | **创建日期**: 2026-09-11
|
||||||
|
> **相关文件**: `py/services/model_scanner.py` (`ModelScanner._reconcile_cache`)
|
||||||
|
> **相关历史**: #871 (`76ee59cd`, 路径重叠去重)、#1108 (按文件夹扫描的需求)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 背景
|
||||||
|
|
||||||
|
Refresh 按钮走的是 `_reconcile_cache()`(快速增量对账)。2026-09-11 做了一轮性能优化,把两处"预防性"的
|
||||||
|
realpath 全量遍历改成按需触发(详见下方"已完成")。优化后,一次零变更 Refresh 在 5 万文件库上从
|
||||||
|
~1400 ms 降到 ~120 ms。
|
||||||
|
|
||||||
|
清理过程中发现**唯一一处遗留的可疑点**:Windows 专属的大小写不敏感回退分支。它无法在 Linux 上验证,
|
||||||
|
因此单独记录,留待 Windows 机器上确认。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 待验证分支(现状)
|
||||||
|
|
||||||
|
`py/services/model_scanner.py` 中 `_reconcile_cache()` 的 walk 循环内:
|
||||||
|
|
||||||
|
```python
|
||||||
|
# Try case-insensitive match on Windows
|
||||||
|
if os.name == 'nt':
|
||||||
|
lower_path = file_path.lower()
|
||||||
|
matched = False
|
||||||
|
for cached_path in cached_paths: # 每个未命中文件都全量扫一遍缓存
|
||||||
|
if cached_path.lower() == lower_path:
|
||||||
|
found_paths.add(cached_path)
|
||||||
|
matched = True
|
||||||
|
break
|
||||||
|
if matched:
|
||||||
|
continue
|
||||||
|
```
|
||||||
|
|
||||||
|
它排在精确匹配(`file_path in cached_paths`)和 realpath 别名匹配之后,只有**未命中**的文件才会走到。
|
||||||
|
|
||||||
|
### 为什么可疑
|
||||||
|
|
||||||
|
1. **可能不可达**:Windows 上 `os.path.realpath()` 会返回磁盘上的真实大小写,因此"缓存路径大小写与磁盘
|
||||||
|
不一致"的情形,理论上已经被上一步的 realpath 别名匹配覆盖。若如此,这段就是纯冗余代码。
|
||||||
|
2. **一旦可达就是 O(N×M)**:每个未命中文件都要遍历全部 `cached_paths` 做小写比较。若某种路径写法让
|
||||||
|
整个库都变成"未命中"(例如缓存里的盘符/大小写形式与 walk 结果系统性不一致),一次 Refresh 会退化
|
||||||
|
成 文件数 × 缓存条目数 次字符串比较,比真实 IO 还贵。
|
||||||
|
3. **没有测试覆盖**:`tests/services/test_model_scanner.py` 没有任何针对该分支的用例(它在 Linux 上
|
||||||
|
被 `os.name == 'nt'` 短路,无法覆盖)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 待办
|
||||||
|
|
||||||
|
- [ ] **验证可达性**:在 Windows 上构造"缓存路径与磁盘真实大小写不一致"的场景,确认 realpath 别名匹配
|
||||||
|
是否已经命中,即上面的 `if os.name == 'nt'` 分支是否还有进入的必要。
|
||||||
|
- [ ] **若不可达 / 冗余**:删除该分支,并在删除处留注释说明 realpath 已覆盖大小写归一(附验证记录)。
|
||||||
|
- [ ] **若可达**:保留语义但改成 O(1)——预先构建一次 `lower_path -> cached_path` 映射(与
|
||||||
|
`cached_real_paths` 同样按需、懒构建),把内层全量扫描换成一次字典查询。
|
||||||
|
- [ ] **补一个 Windows-only 的回归测试**(`pytest.mark.skipif(os.name != "nt", ...)`),锁定最终结论。
|
||||||
|
- [ ] 把验证结论回填到本文件,并同步更新状态行。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 验证方法(Windows)
|
||||||
|
|
||||||
|
1. **构造不一致的大小写**:让缓存里的 `file_path` 与磁盘实际路径大小写不同(例如改过盘符/目录大小写,
|
||||||
|
或从另一台机器迁移了 `settings.json` 与持久化缓存),然后在 UI 点 Refresh。
|
||||||
|
2. **看后端日志判据**:
|
||||||
|
- 若 realpath 已覆盖 → 日志应显示 `Cache reconciliation completed in X seconds. Added 0, removed 0 models.`,
|
||||||
|
且**没有** `Found N new files to process` / `Processing <path>`。
|
||||||
|
- 若回退分支在起作用 → 同样应该是 `Added 0, removed 0`(因为 `found_paths` 被补上),这是"分支可达"
|
||||||
|
的证据;反之若出现大量 `Processing ...` 并重新 hash,说明连回退分支也没命中,问题更严重
|
||||||
|
(缓存路径被当成了新文件 + 旧条目被删)。
|
||||||
|
3. **跑测试**:`python -m pytest tests/services/test_model_scanner.py -k reconcile`(该文件在 Windows 上会
|
||||||
|
真实执行 `os.name == 'nt'` 分支)。
|
||||||
|
4. **量化**:如果需要,可在 `_reconcile_cache` 里临时插桩统计该分支的进入次数与内层迭代次数,确认是否为 0。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 已完成(本轮优化,供对照)
|
||||||
|
|
||||||
|
同一次清理里已经落地并验证的部分(Linux,5 万文件库):
|
||||||
|
|
||||||
|
- `cached_real_paths` 别名映射改为**首次未命中时**懒构建(原来每次 Refresh 都对全部缓存条目算一次 realpath)。
|
||||||
|
- 每个文件的 `realpath` 移到精确命中检查**之后**(原来对每个文件都算,命中即丢弃)。
|
||||||
|
- `get_model_roots()` 在新增文件处理阶段只快照一次(原来每个新文件重读一次)。
|
||||||
|
- 全量去重 pass 加了 O(1) 前置判断(`cached_size_before != len(cached_paths) or total_added > 0`),
|
||||||
|
零变更且缓存干净时跳过;快照本身含重复路径时仍会自愈。
|
||||||
|
|
||||||
|
结果:零变更 Refresh 5 万文件 **~1400 ms → ~120 ms**;根目录顺序/符号链接别名翻转场景仍是
|
||||||
|
`re-processed=0`(不重新读 metadata、不重新 hash)。测试:`tests/services/test_model_scanner.py`
|
||||||
|
47 项、全量后端 2567 项全部通过。
|
||||||
+550
-233
File diff suppressed because it is too large
Load Diff
+414
-97
@@ -50,6 +50,27 @@
|
|||||||
"mb": "MB",
|
"mb": "MB",
|
||||||
"gb": "GB",
|
"gb": "GB",
|
||||||
"tb": "TB"
|
"tb": "TB"
|
||||||
|
},
|
||||||
|
"scanProgress": {
|
||||||
|
"refreshing": "Refreshing {type}s...",
|
||||||
|
"fullRebuilding": "Full rebuild {type}s...",
|
||||||
|
"actionRefresh": "Refresh",
|
||||||
|
"actionFullRebuild": "Full rebuild",
|
||||||
|
"actionRefreshLower": "refresh",
|
||||||
|
"actionRebuildLower": "rebuild",
|
||||||
|
"stages": {
|
||||||
|
"scan_folders": "Scanning folders...",
|
||||||
|
"count_models": "Found {total} files",
|
||||||
|
"process_models": "Processing models",
|
||||||
|
"reconcile_scan": "Checking for changes...",
|
||||||
|
"process_new": "Processing new models",
|
||||||
|
"finalizing": "Finalizing..."
|
||||||
|
},
|
||||||
|
"eta": {
|
||||||
|
"lessThanMinute": "Less than a minute remaining",
|
||||||
|
"minutes": "~{minutes} min remaining",
|
||||||
|
"hours": "~{hours} hr {minutes} min remaining"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"onboarding": {
|
"onboarding": {
|
||||||
@@ -67,15 +88,15 @@
|
|||||||
"steps": {
|
"steps": {
|
||||||
"fetch": {
|
"fetch": {
|
||||||
"title": "Fetch Models Metadata",
|
"title": "Fetch Models Metadata",
|
||||||
"content": "Click the <strong>Fetch</strong> button to download model metadata and preview images from Civitai."
|
"content": "Click the <strong>Fetch</strong> button to download model metadata and preview images from CivitAI."
|
||||||
},
|
},
|
||||||
"download": {
|
"download": {
|
||||||
"title": "Download New Models",
|
"title": "Download New Models",
|
||||||
"content": "Use the <strong>Download</strong> button to download models directly from Civitai URLs."
|
"content": "Use the <strong>Download</strong> button to download models directly from CivitAI URLs."
|
||||||
},
|
},
|
||||||
"bulk": {
|
"bulk": {
|
||||||
"title": "Bulk Operations",
|
"title": "Bulk Operations",
|
||||||
"content": "Enter bulk mode by clicking this button or pressing <span class=\"onboarding-shortcut\">B</span>. Select multiple models and perform batch operations. Use <span class=\"onboarding-shortcut\">Ctrl+A</span> to select all visible models."
|
"content": "Enter bulk mode by clicking this button or pressing <span class=\"onboarding-shortcut\">B</span> to select multiple models and perform batch operations.<br>• <span class=\"onboarding-shortcut\">Ctrl/Cmd+A</span> select all visible models, <span class=\"onboarding-shortcut\">Shift+Click</span> select a range.<br>• <span class=\"onboarding-shortcut\">Esc</span> or clicking an empty area exits bulk mode."
|
||||||
},
|
},
|
||||||
"searchOptions": {
|
"searchOptions": {
|
||||||
"title": "Search Options",
|
"title": "Search Options",
|
||||||
@@ -95,7 +116,19 @@
|
|||||||
},
|
},
|
||||||
"contextMenu": {
|
"contextMenu": {
|
||||||
"title": "Context Menu",
|
"title": "Context Menu",
|
||||||
"content": "<strong>Right-click</strong> any model card for a context menu with additional actions."
|
"content": "<strong>Right-click</strong> any model card for a context menu with card actions like moving, deleting, or editing metadata."
|
||||||
|
},
|
||||||
|
"marqueeSelect": {
|
||||||
|
"title": "Drag to Select",
|
||||||
|
"content": "Hold the <strong>left mouse button</strong> on an empty area of the grid and drag to draw a marquee that selects multiple cards at once."
|
||||||
|
},
|
||||||
|
"dragToSidebar": {
|
||||||
|
"title": "Organize by Dragging",
|
||||||
|
"content": "Drag a model card onto a folder in the sidebar to move the file there. This also works with multiple selected cards in bulk mode."
|
||||||
|
},
|
||||||
|
"contextMenus": {
|
||||||
|
"title": "More Context Menus",
|
||||||
|
"content": "In bulk mode, <strong>right-click a selected card</strong> for bulk actions. <strong>Right-click an empty area</strong> of the page for global actions like update checks and managing excluded models."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -103,8 +136,8 @@
|
|||||||
"actions": {
|
"actions": {
|
||||||
"addToFavorites": "Add to favorites",
|
"addToFavorites": "Add to favorites",
|
||||||
"removeFromFavorites": "Remove from favorites",
|
"removeFromFavorites": "Remove from favorites",
|
||||||
"viewOnCivitai": "View on Civitai",
|
"viewOnCivitai": "View on CivitAI",
|
||||||
"notAvailableFromCivitai": "Not available from Civitai",
|
"notAvailableFromCivitai": "Not available from CivitAI",
|
||||||
"viewOnHuggingFace": "View on Hugging Face",
|
"viewOnHuggingFace": "View on Hugging Face",
|
||||||
"sendToWorkflow": "Send to ComfyUI (Click: Append, Shift+Click: Replace)",
|
"sendToWorkflow": "Send to ComfyUI (Click: Append, Shift+Click: Replace)",
|
||||||
"copyLoRASyntax": "Copy LoRA Syntax",
|
"copyLoRASyntax": "Copy LoRA Syntax",
|
||||||
@@ -116,6 +149,7 @@
|
|||||||
"copyCheckpointName": "Copy checkpoint name",
|
"copyCheckpointName": "Copy checkpoint name",
|
||||||
"copyEmbeddingName": "Copy embedding name",
|
"copyEmbeddingName": "Copy embedding name",
|
||||||
"embeddingNameCopied": "Embedding syntax copied",
|
"embeddingNameCopied": "Embedding syntax copied",
|
||||||
|
"modelNameCopied": "Model name copied",
|
||||||
"sendCheckpointToWorkflow": "Send to ComfyUI",
|
"sendCheckpointToWorkflow": "Send to ComfyUI",
|
||||||
"sendEmbeddingToWorkflow": "Send to ComfyUI"
|
"sendEmbeddingToWorkflow": "Send to ComfyUI"
|
||||||
},
|
},
|
||||||
@@ -137,7 +171,7 @@
|
|||||||
"exampleImages": {
|
"exampleImages": {
|
||||||
"checkError": "Error checking for example images",
|
"checkError": "Error checking for example images",
|
||||||
"missingHash": "Missing model hash information.",
|
"missingHash": "Missing model hash information.",
|
||||||
"noRemoteImagesAvailable": "No remote example images available for this model on Civitai"
|
"noRemoteImagesAvailable": "No remote example images available for this model on CivitAI"
|
||||||
},
|
},
|
||||||
"badges": {
|
"badges": {
|
||||||
"update": "Update",
|
"update": "Update",
|
||||||
@@ -179,20 +213,10 @@
|
|||||||
"none": "All {typePlural} already have license metadata",
|
"none": "All {typePlural} already have license metadata",
|
||||||
"error": "Failed to refresh license metadata for {typePlural}: {message}"
|
"error": "Failed to refresh license metadata for {typePlural}: {message}"
|
||||||
},
|
},
|
||||||
"repairRecipes": {
|
|
||||||
"label": "Repair recipes data",
|
|
||||||
"loading": "Repairing recipe data...",
|
|
||||||
"success": "Successfully repaired {count} recipes.",
|
|
||||||
"cancelled": "Repair cancelled. {count} recipes were repaired.",
|
|
||||||
"error": "Recipe repair failed: {message}"
|
|
||||||
},
|
|
||||||
"rematchRecipes": {
|
"rematchRecipes": {
|
||||||
"label": "Rematch recipes to local models",
|
"label": "Rematch recipes to local models",
|
||||||
"loading": "Rematching recipes to local models...",
|
"loading": "Rematching recipes to local models...",
|
||||||
"success": "Matched {entries} entries across {recipes} recipes",
|
"success": "Matched {entries} entries across {recipes} recipes",
|
||||||
"successErrors": "Matched {entries} entries across {recipes} recipes, {failures} failed",
|
|
||||||
"allFailed": "Rematch failed for {failures} of {total} recipes",
|
|
||||||
"noMatch": "No local match found for {entries} entries in {recipes} recipes",
|
|
||||||
"cancelled": "Rematch cancelled. {recipes} recipes updated ({entries} entries).",
|
"cancelled": "Rematch cancelled. {recipes} recipes updated ({entries} entries).",
|
||||||
"error": "Recipe rematch failed: {message}"
|
"error": "Recipe rematch failed: {message}"
|
||||||
},
|
},
|
||||||
@@ -210,6 +234,7 @@
|
|||||||
"recipes": "Recipes",
|
"recipes": "Recipes",
|
||||||
"checkpoints": "Checkpoints",
|
"checkpoints": "Checkpoints",
|
||||||
"embeddings": "Embeddings",
|
"embeddings": "Embeddings",
|
||||||
|
"other": "Other",
|
||||||
"statistics": "Stats"
|
"statistics": "Stats"
|
||||||
},
|
},
|
||||||
"search": {
|
"search": {
|
||||||
@@ -222,6 +247,7 @@
|
|||||||
"modelname": "Model Name",
|
"modelname": "Model Name",
|
||||||
"tags": "Tags",
|
"tags": "Tags",
|
||||||
"creator": "Creator",
|
"creator": "Creator",
|
||||||
|
"hash": "Hash",
|
||||||
"title": "Recipe Title",
|
"title": "Recipe Title",
|
||||||
"loraName": "LoRA Filename",
|
"loraName": "LoRA Filename",
|
||||||
"loraModel": "LoRA Model Name",
|
"loraModel": "LoRA Model Name",
|
||||||
@@ -259,7 +285,11 @@
|
|||||||
"any": "Any",
|
"any": "Any",
|
||||||
"all": "All",
|
"all": "All",
|
||||||
"tagLogicAny": "Match any tag (OR)",
|
"tagLogicAny": "Match any tag (OR)",
|
||||||
"tagLogicAll": "Match all tags (AND)"
|
"tagLogicAll": "Match all tags (AND)",
|
||||||
|
"loraAvailability": "Lora Availability",
|
||||||
|
"availabilityReady": "Ready to use",
|
||||||
|
"availabilityMissing": "Has missing",
|
||||||
|
"availabilityDeleted": "Has deleted"
|
||||||
},
|
},
|
||||||
"theme": {
|
"theme": {
|
||||||
"toggle": "Toggle theme",
|
"toggle": "Toggle theme",
|
||||||
@@ -285,15 +315,15 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"settings": {
|
"settings": {
|
||||||
"civitaiApiKey": "Civitai API Key",
|
"civitaiApiKey": "CivitAI API Key",
|
||||||
"civitaiApiKeyPlaceholder": "Enter your Civitai API key",
|
"civitaiApiKeyPlaceholder": "Enter your CivitAI API key",
|
||||||
"civitaiApiKeyHelp": "Used for authentication when downloading models from Civitai",
|
"civitaiApiKeyHelp": "Used for authentication when downloading models from CivitAI",
|
||||||
"civitaiApiKeyConfigured": "Configured",
|
"civitaiApiKeyConfigured": "Configured",
|
||||||
"civitaiApiKeyNotConfigured": "Not configured",
|
"civitaiApiKeyNotConfigured": "Not configured",
|
||||||
"civitaiApiKeySet": "Set up",
|
"civitaiApiKeySet": "Set up",
|
||||||
"civitaiHost": {
|
"civitaiHost": {
|
||||||
"label": "Civitai host",
|
"label": "CivitAI host",
|
||||||
"help": "Choose which Civitai site opens when using View on Civitai links.",
|
"help": "Choose which CivitAI site opens when using View on CivitAI links.",
|
||||||
"options": {
|
"options": {
|
||||||
"com": "civitai.com (SFW)",
|
"com": "civitai.com (SFW)",
|
||||||
"red": "civitai.red (unrestricted)"
|
"red": "civitai.red (unrestricted)"
|
||||||
@@ -314,8 +344,8 @@
|
|||||||
},
|
},
|
||||||
"aria2HelpLink": "Learn how to set up the aria2 download backend",
|
"aria2HelpLink": "Learn how to set up the aria2 download backend",
|
||||||
"civitaiHostBanner": {
|
"civitaiHostBanner": {
|
||||||
"title": "Civitai host preference available",
|
"title": "CivitAI host preference available",
|
||||||
"content": "Civitai now uses civitai.com for SFW content and civitai.red for unrestricted content. You can change which site opens by default in Settings.",
|
"content": "CivitAI now uses civitai.com for SFW content and civitai.red for unrestricted content. You can change which site opens by default in Settings.",
|
||||||
"openSettings": "Open Settings"
|
"openSettings": "Open Settings"
|
||||||
},
|
},
|
||||||
"openSettingsFileLocation": {
|
"openSettingsFileLocation": {
|
||||||
@@ -445,7 +475,9 @@
|
|||||||
},
|
},
|
||||||
"layoutSettings": {
|
"layoutSettings": {
|
||||||
"groupByModel": "Group by Model",
|
"groupByModel": "Group by Model",
|
||||||
"groupByModelHelp": "When enabled, only the latest version of each Civitai model is shown as a single card. Older versions are hidden.",
|
"groupByModelHelp": "When enabled, only the latest version of each CivitAI model is shown as a single card. Older versions are hidden.",
|
||||||
|
"stickyControls": "Keep Action Bar Visible",
|
||||||
|
"stickyControlsHelp": "When enabled, the action bar (Refresh, Download, etc.) stays pinned at the top while scrolling, together with the breadcrumb navigation.",
|
||||||
"displayDensity": "Display Density",
|
"displayDensity": "Display Density",
|
||||||
"displayDensityOptions": {
|
"displayDensityOptions": {
|
||||||
"default": "Default",
|
"default": "Default",
|
||||||
@@ -503,6 +535,25 @@
|
|||||||
"defaultUnetRootHelp": "Set default diffusion model (UNET) root directory for downloads, imports and moves",
|
"defaultUnetRootHelp": "Set default diffusion model (UNET) root directory for downloads, imports and moves",
|
||||||
"defaultEmbeddingRoot": "Embedding Root",
|
"defaultEmbeddingRoot": "Embedding Root",
|
||||||
"defaultEmbeddingRootHelp": "Set default embedding root directory for downloads, imports and moves",
|
"defaultEmbeddingRootHelp": "Set default embedding root directory for downloads, imports and moves",
|
||||||
|
"defaultVaeRoot": "VAE Root",
|
||||||
|
"defaultVaeRootHelp": "Set default VAE root directory for downloads, imports and moves",
|
||||||
|
"defaultUpscalerRoot": "Upscaler Root",
|
||||||
|
"defaultUpscalerRootHelp": "Set default upscaler root directory for downloads, imports and moves",
|
||||||
|
"defaultTextEncoderRoot": "Text Encoder Root",
|
||||||
|
"defaultTextEncoderRootHelp": "Set default text encoder root directory for downloads, imports and moves",
|
||||||
|
"defaultClipVisionRoot": "CLIP Vision Root",
|
||||||
|
"defaultClipVisionRootHelp": "Set default CLIP vision root directory for downloads, imports and moves",
|
||||||
|
"defaultControlnetRoot": "ControlNet Root",
|
||||||
|
"defaultControlnetRootHelp": "Set default ControlNet root directory for downloads, imports and moves",
|
||||||
|
"enableOtherModels": "Other Models Management",
|
||||||
|
"enableOtherModelsHelp": "When off, VAE / upscaler / text encoder / CLIP vision / ControlNet folders are not scanned, the Other Models page stays disabled, and these model types cannot be downloaded.",
|
||||||
|
"otherSubTypes": "Managed Types",
|
||||||
|
"otherSubTypesHelp": "Choose which other-model categories are scanned and shown on the Other Models page.",
|
||||||
|
"subTypeVae": "VAE",
|
||||||
|
"subTypeUpscaler": "Upscaler",
|
||||||
|
"subTypeTextEncoder": "Text Encoder",
|
||||||
|
"subTypeClipVision": "CLIP Vision",
|
||||||
|
"subTypeControlnet": "ControlNet",
|
||||||
"recipesPath": "Recipes Storage Path",
|
"recipesPath": "Recipes Storage Path",
|
||||||
"recipesPathHelp": "Optional custom directory for stored recipes. Leave empty to use the first LoRA root's recipes folder.",
|
"recipesPathHelp": "Optional custom directory for stored recipes. Leave empty to use the first LoRA root's recipes folder.",
|
||||||
"recipesPathPlaceholder": "/path/to/recipes",
|
"recipesPathPlaceholder": "/path/to/recipes",
|
||||||
@@ -550,7 +601,7 @@
|
|||||||
},
|
},
|
||||||
"downloadPathTemplates": {
|
"downloadPathTemplates": {
|
||||||
"title": "Download Path Templates",
|
"title": "Download Path Templates",
|
||||||
"help": "Configure folder structures for different model types when downloading from Civitai.",
|
"help": "Configure folder structures for different model types when downloading from CivitAI.",
|
||||||
"availablePlaceholders": "Available placeholders:",
|
"availablePlaceholders": "Available placeholders:",
|
||||||
"templateOptions": {
|
"templateOptions": {
|
||||||
"flatStructure": "Flat Structure",
|
"flatStructure": "Flat Structure",
|
||||||
@@ -587,7 +638,7 @@
|
|||||||
"exampleImages": {
|
"exampleImages": {
|
||||||
"downloadLocation": "Download Location",
|
"downloadLocation": "Download Location",
|
||||||
"downloadLocationPlaceholder": "Enter folder path for example images",
|
"downloadLocationPlaceholder": "Enter folder path for example images",
|
||||||
"downloadLocationHelp": "Enter the folder path where example images from Civitai will be saved",
|
"downloadLocationHelp": "Enter the folder path where example images from CivitAI will be saved",
|
||||||
"autoDownload": "Auto Download Example Images",
|
"autoDownload": "Auto Download Example Images",
|
||||||
"autoDownloadHelp": "Automatically download example images for models that don't have them (requires download location to be set)",
|
"autoDownloadHelp": "Automatically download example images for models that don't have them (requires download location to be set)",
|
||||||
"openMode": "Open Example Images Action",
|
"openMode": "Open Example Images Action",
|
||||||
@@ -642,7 +693,7 @@
|
|||||||
},
|
},
|
||||||
"metadataArchive": {
|
"metadataArchive": {
|
||||||
"enableArchiveDb": "Enable Metadata Archive Database",
|
"enableArchiveDb": "Enable Metadata Archive Database",
|
||||||
"enableArchiveDbHelp": "Use a local database to access metadata for models that have been deleted from Civitai.",
|
"enableArchiveDbHelp": "Use a local database to access metadata for models that have been deleted from CivitAI.",
|
||||||
"status": "Status",
|
"status": "Status",
|
||||||
"statusAvailable": "Available",
|
"statusAvailable": "Available",
|
||||||
"statusUnavailable": "Not Available",
|
"statusUnavailable": "Not Available",
|
||||||
@@ -745,7 +796,7 @@
|
|||||||
"fullTooltip": "Reload all model details from metadata files—use if the library looks out of date or after manual edits."
|
"fullTooltip": "Reload all model details from metadata files—use if the library looks out of date or after manual edits."
|
||||||
},
|
},
|
||||||
"fetch": {
|
"fetch": {
|
||||||
"title": "Fetch metadata from Civitai",
|
"title": "Fetch metadata from CivitAI",
|
||||||
"action": "Fetch"
|
"action": "Fetch"
|
||||||
},
|
},
|
||||||
"download": {
|
"download": {
|
||||||
@@ -781,7 +832,6 @@
|
|||||||
"setContentRating": "Set Content Rating for Selected",
|
"setContentRating": "Set Content Rating for Selected",
|
||||||
"copyAll": "Copy Selected Syntax",
|
"copyAll": "Copy Selected Syntax",
|
||||||
"refreshAll": "Refresh Selected Metadata",
|
"refreshAll": "Refresh Selected Metadata",
|
||||||
"repairMetadata": "Repair Metadata for Selected",
|
|
||||||
"rematchMetadata": "Rematch Selected to Local Models",
|
"rematchMetadata": "Rematch Selected to Local Models",
|
||||||
"reimportMetadata": "Re-import from Source",
|
"reimportMetadata": "Re-import from Source",
|
||||||
"checkUpdates": "Check Updates for Selected",
|
"checkUpdates": "Check Updates for Selected",
|
||||||
@@ -820,10 +870,10 @@
|
|||||||
"enrichHfAgent": "Enrich HF Metadata (AI)"
|
"enrichHfAgent": "Enrich HF Metadata (AI)"
|
||||||
},
|
},
|
||||||
"contextMenu": {
|
"contextMenu": {
|
||||||
"refreshMetadata": "Refresh Civitai Data",
|
"refreshMetadata": "Refresh CivitAI Data",
|
||||||
"checkUpdates": "Check Updates",
|
"checkUpdates": "Check Updates",
|
||||||
"linkModel": "Link Model",
|
"linkModel": "Link Model",
|
||||||
"linkCivitai": "Link to Civitai",
|
"linkCivitai": "Link to CivitAI",
|
||||||
"linkHuggingFace": "Link to HuggingFace",
|
"linkHuggingFace": "Link to HuggingFace",
|
||||||
"copySyntax": "Copy LoRA Syntax",
|
"copySyntax": "Copy LoRA Syntax",
|
||||||
"copyFilename": "Copy Model Filename",
|
"copyFilename": "Copy Model Filename",
|
||||||
@@ -837,7 +887,6 @@
|
|||||||
"replacePreview": "Replace Preview",
|
"replacePreview": "Replace Preview",
|
||||||
"setContentRating": "Set Content Rating",
|
"setContentRating": "Set Content Rating",
|
||||||
"moveToFolder": "Move to Folder",
|
"moveToFolder": "Move to Folder",
|
||||||
"repairMetadata": "Repair metadata",
|
|
||||||
"rematchMetadata": "Rematch to local models",
|
"rematchMetadata": "Rematch to local models",
|
||||||
"reimportMetadata": "Re-import from Source",
|
"reimportMetadata": "Re-import from Source",
|
||||||
"excludeModel": "Exclude Model",
|
"excludeModel": "Exclude Model",
|
||||||
@@ -853,20 +902,130 @@
|
|||||||
"recipes": {
|
"recipes": {
|
||||||
"title": "LoRA Recipes",
|
"title": "LoRA Recipes",
|
||||||
"actions": {
|
"actions": {
|
||||||
"sendCheckpoint": "Send to ComfyUI"
|
"sendCheckpoint": "Send to ComfyUI",
|
||||||
|
"sendRecipe": "Send to ComfyUI",
|
||||||
|
"copyRecipeSyntax": "Copy Recipe Syntax",
|
||||||
|
"deleteRecipeWithShortcut": "Delete recipe (Del)"
|
||||||
|
},
|
||||||
|
"navigation": {
|
||||||
|
"label": "Recipe navigation",
|
||||||
|
"previousWithShortcut": "Previous recipe (←)",
|
||||||
|
"nextWithShortcut": "Next recipe (→)"
|
||||||
|
},
|
||||||
|
"modal": {
|
||||||
|
"metadata": {
|
||||||
|
"id": "ID"
|
||||||
|
},
|
||||||
|
"actions": {
|
||||||
|
"openFileLocation": "Open File Location",
|
||||||
|
"copyId": "Copy recipe ID"
|
||||||
|
},
|
||||||
|
"openFileLocation": {
|
||||||
|
"success": "File location opened successfully",
|
||||||
|
"failed": "Failed to open file location",
|
||||||
|
"copied": "Path copied to clipboard: {{path}}",
|
||||||
|
"clipboardFallback": "Path: {{path}}"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"workflow": {
|
||||||
|
"sendWorkflow": "Send Workflow to ComfyUI",
|
||||||
|
"sent": "Workflow sent to ComfyUI",
|
||||||
|
"sendFailed": "Failed to send workflow to ComfyUI",
|
||||||
|
"noWorkflow": "No embedded workflow found in this recipe"
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"ready": "Ready to use",
|
||||||
|
"missingCount": "{count} missing",
|
||||||
|
"deletedCount": "{count} deleted",
|
||||||
|
"downloadMissing": "Download {count} missing LoRAs",
|
||||||
|
"downloadMissingTooltip": "Click to download missing LoRAs"
|
||||||
|
},
|
||||||
|
"loraStatus": {
|
||||||
|
"none": "No LoRAs in this recipe",
|
||||||
|
"allAvailable": "All LoRAs available - Ready to use",
|
||||||
|
"missing": "{missing} of {total} LoRAs missing",
|
||||||
|
"missingAndUnavailable": "{missing} of {total} LoRAs missing, {unavailable} unavailable (deleted from source or unresolvable hash)",
|
||||||
|
"partial": "{unavailable} of {total} LoRAs unavailable (deleted from source or unresolvable hash) - skipped when recipe is used",
|
||||||
|
"noneUsable": "No usable LoRAs - {unavailable} of {total} deleted from source or unresolvable hash"
|
||||||
|
},
|
||||||
|
"resources": {
|
||||||
|
"inLibrary": "In Library",
|
||||||
|
"notInLibrary": "Not in Library",
|
||||||
|
"deleted": "Deleted",
|
||||||
|
"hashInvalid": "Unresolvable Hash",
|
||||||
|
"inLibraryTooltip": "This model exists in your local library",
|
||||||
|
"notInLibraryTooltip": "This model is not in your library",
|
||||||
|
"deletedTooltip": "This LoRA was deleted from the source and is no longer available for download",
|
||||||
|
"hashInvalidTooltip": "This LoRA hash cannot be resolved on CivitAI - the model may have been updated",
|
||||||
|
"noLorasAssociated": "No LoRAs associated with this recipe",
|
||||||
|
"noLorasWhyToggle": "Why no LoRAs?",
|
||||||
|
"noLorasImportMethod": "Import method",
|
||||||
|
"noLorasInferredNote": "Possible reason (inferred) — this recipe was imported before import diagnostics were recorded.",
|
||||||
|
"noLorasChannels": {
|
||||||
|
"batch_import_url": "Batch import (image URL)",
|
||||||
|
"batch_import_local": "Batch import (local file)",
|
||||||
|
"url": "Image URL import",
|
||||||
|
"local": "Local file import",
|
||||||
|
"upload": "Image upload",
|
||||||
|
"widget": "Saved from workflow",
|
||||||
|
"reimport_url": "Re-import (image URL)",
|
||||||
|
"reimport_local": "Re-import (local file)"
|
||||||
|
},
|
||||||
|
"noLorasReasons": {
|
||||||
|
"no_loras_used": "The generation metadata is complete and does not reference any LoRAs.",
|
||||||
|
"api_meta_no_lora_resources": "The source API returned no LoRA resource data for this image. LoRAs shown on the CivitAI page may come from internal data that the public API does not expose.",
|
||||||
|
"api_meta_missing": "The source API returned no generation metadata for this image.",
|
||||||
|
"no_embedded_metadata": "The image has no embedded generation metadata, so LoRA information could not be recovered.",
|
||||||
|
"workflow_metadata_limited": "The image's embedded metadata is a ComfyUI workflow; extracting LoRA information from workflows is limited.",
|
||||||
|
"video_no_metadata": "Video files do not carry embedded generation metadata.",
|
||||||
|
"metadata_unsupported": "The image contains metadata in a format that could not be parsed.",
|
||||||
|
"unknown": "The reason could not be determined from the stored recipe data."
|
||||||
|
},
|
||||||
|
"noLorasDetails": {
|
||||||
|
"apiMetaFields": "API metadata fields",
|
||||||
|
"modelVersionIds": "Model version IDs reported",
|
||||||
|
"embeddedMetadata": "Embedded metadata",
|
||||||
|
"present": "found",
|
||||||
|
"absent": "none"
|
||||||
|
},
|
||||||
|
"download": "Download",
|
||||||
|
"downloadLoraTooltip": "Download this LoRA",
|
||||||
|
"preparingDownload": "Preparing download...",
|
||||||
|
"reconnect": "Reconnect",
|
||||||
|
"reconnectTooltip": "Reconnect with a local LoRA",
|
||||||
|
"reconnectInstructions": "Enter LoRA syntax or name to reconnect:",
|
||||||
|
"reconnectExample": "Example: <lora:name:1> or just the name",
|
||||||
|
"reconnectPlaceholder": "Enter LoRA name or syntax",
|
||||||
|
"reconnectSuggestionsLoading": "Searching local library...",
|
||||||
|
"reconnectSuggestionsEmpty": "No matching LoRAs in your local library",
|
||||||
|
"reconnectMatchSameHash": "Same hash",
|
||||||
|
"reconnectMatchSameVersion": "Same model version",
|
||||||
|
"reconnectMatchSimilarFilename": "Similar filename",
|
||||||
|
"reconnectMatchSimilarName": "Similar name",
|
||||||
|
"undoReconnect": "Undo",
|
||||||
|
"undoReconnectTooltip": "Restore the association this entry had before reconnecting",
|
||||||
|
"undoReconnectTooltipNamed": "Restore to {name} (the association before reconnecting)",
|
||||||
|
"viewOnCivitai": "View on CivitAI",
|
||||||
|
"openLoraDetails": "View {name} in the LoRA library",
|
||||||
|
"openCheckpointDetails": "View {name} in the model library",
|
||||||
|
"checkpointDeletedTooltip": "This checkpoint was deleted from the source and can no longer be downloaded - reconnect it with a local model",
|
||||||
|
"checkpointHashInvalidTooltip": "This checkpoint hash cannot be resolved on CivitAI - the model may have been updated",
|
||||||
|
"reconnectCheckpoint": "Reconnect",
|
||||||
|
"reconnectCheckpointTooltip": "Reconnect with a local checkpoint",
|
||||||
|
"checkpointReconnectInstructions": "Enter checkpoint name to reconnect:",
|
||||||
|
"checkpointReconnectPlaceholder": "Enter checkpoint name",
|
||||||
|
"checkpointReconnectSuggestionsEmpty": "No matching checkpoints in your local library"
|
||||||
},
|
},
|
||||||
"controls": {
|
"controls": {
|
||||||
"import": {
|
"import": {
|
||||||
"action": "Import",
|
"action": "Import",
|
||||||
"title": "Import a recipe from image or URL",
|
"title": "Import a recipe from image or URL",
|
||||||
"urlLocalPath": "URL / Local Path",
|
"dropZoneLabel": "Upload image",
|
||||||
"uploadImage": "Upload Image",
|
"dropZoneHint": "Drag & drop an image here, paste from clipboard, or click to browse",
|
||||||
"urlSectionDescription": "Input a Civitai image URL from civitai.com or civitai.red, or a local file path, to import as a recipe.",
|
"orDivider": "or drag & drop / paste an image",
|
||||||
"imageUrlOrPath": "Image URL or File Path:",
|
"imageUrlOrPath": "Image URL or File Path:",
|
||||||
"urlPlaceholder": "https://civitai.com/images/... or https://civitai.red/images/... or C:/path/to/image.png",
|
"urlPlaceholder": "https://civitai.com/images/... or https://civitai.red/images/... or C:/path/to/image.png",
|
||||||
"fetchImage": "Fetch Image",
|
"fetchImage": "Fetch Image",
|
||||||
"uploadSectionDescription": "Upload an image with LoRA metadata to import as a recipe.",
|
|
||||||
"selectImage": "Select Image",
|
|
||||||
"recipeName": "Recipe Name",
|
"recipeName": "Recipe Name",
|
||||||
"recipeNamePlaceholder": "Enter recipe name",
|
"recipeNamePlaceholder": "Enter recipe name",
|
||||||
"tagsOptional": "Tags (optional)",
|
"tagsOptional": "Tags (optional)",
|
||||||
@@ -894,7 +1053,7 @@
|
|||||||
"downloadingLoras": "Downloading LoRAs...",
|
"downloadingLoras": "Downloading LoRAs...",
|
||||||
"savingRecipe": "Saving recipe...",
|
"savingRecipe": "Saving recipe...",
|
||||||
"startingDownload": "Starting download for LoRA {current}/{total}",
|
"startingDownload": "Starting download for LoRA {current}/{total}",
|
||||||
"deletedFromCivitai": "Deleted from Civitai",
|
"deletedFromCivitai": "Deleted from CivitAI",
|
||||||
"inLibrary": "In Library",
|
"inLibrary": "In Library",
|
||||||
"notInLibrary": "Not in Library",
|
"notInLibrary": "Not in Library",
|
||||||
"earlyAccessRequired": "This LoRA requires early access payment to download.",
|
"earlyAccessRequired": "This LoRA requires early access payment to download.",
|
||||||
@@ -911,6 +1070,8 @@
|
|||||||
"errors": {
|
"errors": {
|
||||||
"selectImageFile": "Please select an image file",
|
"selectImageFile": "Please select an image file",
|
||||||
"enterUrlOrPath": "Please enter a URL or file path",
|
"enterUrlOrPath": "Please enter a URL or file path",
|
||||||
|
"invalidUrl": "Please enter a valid URL",
|
||||||
|
"invalidInputFormat": "Please enter an image URL or a local image file path",
|
||||||
"selectLoraRoot": "Please select a LoRA root directory"
|
"selectLoraRoot": "Please select a LoRA root directory"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -945,6 +1106,7 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"duplicates": {
|
"duplicates": {
|
||||||
|
"finding": "Scanning for duplicate recipes...",
|
||||||
"found": "Found {count} duplicate groups",
|
"found": "Found {count} duplicate groups",
|
||||||
"noGroups": "No duplicate groups found with the current matching basis",
|
"noGroups": "No duplicate groups found with the current matching basis",
|
||||||
"keepLatest": "Keep Latest Versions",
|
"keepLatest": "Keep Latest Versions",
|
||||||
@@ -977,13 +1139,6 @@
|
|||||||
"getInfoFailed": "Failed to get information for missing LoRAs",
|
"getInfoFailed": "Failed to get information for missing LoRAs",
|
||||||
"prepareError": "Error preparing LoRAs for download: {message}"
|
"prepareError": "Error preparing LoRAs for download: {message}"
|
||||||
},
|
},
|
||||||
"repair": {
|
|
||||||
"starting": "Repairing recipe metadata...",
|
|
||||||
"success": "Recipe metadata repaired successfully",
|
|
||||||
"skipped": "Recipe already at latest version, no repair needed",
|
|
||||||
"failed": "Failed to repair recipe: {message}",
|
|
||||||
"missingId": "Cannot repair recipe: Missing recipe ID"
|
|
||||||
},
|
|
||||||
"reimport": {
|
"reimport": {
|
||||||
"starting": "Re-importing recipe from source...",
|
"starting": "Re-importing recipe from source...",
|
||||||
"success": "Recipe re-imported successfully",
|
"success": "Recipe re-imported successfully",
|
||||||
@@ -1014,6 +1169,8 @@
|
|||||||
"start": "Start Import",
|
"start": "Start Import",
|
||||||
"startImport": "Start Import",
|
"startImport": "Start Import",
|
||||||
"importing": "Importing...",
|
"importing": "Importing...",
|
||||||
|
"rateLimitedSlowdown": "Rate limited — slowing down...",
|
||||||
|
"rateLimitedHint": "Some items were skipped due to metadata provider rate limits. Re-run the import later to retry them.",
|
||||||
"progress": "Progress",
|
"progress": "Progress",
|
||||||
"total": "Total",
|
"total": "Total",
|
||||||
"success": "Success",
|
"success": "Success",
|
||||||
@@ -1065,6 +1222,26 @@
|
|||||||
"embeddings": {
|
"embeddings": {
|
||||||
"title": "Embedding Models"
|
"title": "Embedding Models"
|
||||||
},
|
},
|
||||||
|
"other": {
|
||||||
|
"title": "Other Models",
|
||||||
|
"disabled": {
|
||||||
|
"title": "Other Models management is off",
|
||||||
|
"description": "Enable it to scan and manage VAE, upscaler, text encoder, CLIP vision and ControlNet files, and to download them from CivitAI.",
|
||||||
|
"enableButton": "Enable Other Models",
|
||||||
|
"hint": "You can change the managed model types later in Settings > Library.",
|
||||||
|
"enableFailed": "Failed to enable Other Models",
|
||||||
|
"downloadBlocked": "Other Models management is disabled for this model type. Enable it in Settings > Library to download this file.",
|
||||||
|
"enableAction": "Enable Other Models"
|
||||||
|
},
|
||||||
|
"noPaths": {
|
||||||
|
"title": "No other-model folders found",
|
||||||
|
"descriptionStandalone": "Other Models management is on, but none of the configured model folders exist on disk. Add the folder paths below to settings.json and restart LoRA Manager.",
|
||||||
|
"hintStandalone": "Only the folder keys listed above are scanned; keys you do not need can be omitted.",
|
||||||
|
"descriptionComfyUI": "Other Models management is on, but none of the configured model folders exist on disk. Add the matching model folders to your ComfyUI model paths, then reload this page.",
|
||||||
|
"hintComfyUI": "Other models are read from ComfyUI's vae, upscale_models, text_encoders, clip_vision and controlnet folders.",
|
||||||
|
"openSettings": "Open Settings"
|
||||||
|
}
|
||||||
|
},
|
||||||
"sidebar": {
|
"sidebar": {
|
||||||
"modelRoot": "Root",
|
"modelRoot": "Root",
|
||||||
"collapseAll": "Collapse All Folders",
|
"collapseAll": "Collapse All Folders",
|
||||||
@@ -1217,7 +1394,7 @@
|
|||||||
"download": {
|
"download": {
|
||||||
"title": "Download Model from URL",
|
"title": "Download Model from URL",
|
||||||
"titleWithType": "Download {type} from URL",
|
"titleWithType": "Download {type} from URL",
|
||||||
"civitaiUrl": "Civitai URL(s):",
|
"civitaiUrl": "CivitAI URL(s):",
|
||||||
"placeholder": "https://civitai.com/models/...",
|
"placeholder": "https://civitai.com/models/...",
|
||||||
"urlHint": "Enter one CivitAI, CivArchive, or Hugging Face URL per line. Supports multiple URLs for batch download.",
|
"urlHint": "Enter one CivitAI, CivArchive, or Hugging Face URL per line. Supports multiple URLs for batch download.",
|
||||||
"selectHfFiles": "Select file(s) to download from this repository:",
|
"selectHfFiles": "Select file(s) to download from this repository:",
|
||||||
@@ -1243,14 +1420,16 @@
|
|||||||
"downloaded": "Downloaded",
|
"downloaded": "Downloaded",
|
||||||
"downloadedTooltip": "Previously downloaded, but it is not currently in your library.",
|
"downloadedTooltip": "Previously downloaded, but it is not currently in your library.",
|
||||||
"alreadyInLibrary": "Already in Library",
|
"alreadyInLibrary": "Already in Library",
|
||||||
|
"partiallyDownloaded": "Partially downloaded",
|
||||||
"autoOrganizedPath": "[Auto-organized by path template]",
|
"autoOrganizedPath": "[Auto-organized by path template]",
|
||||||
"fileSelection": {
|
"fileSelection": {
|
||||||
"title": "Select File Format",
|
"title": "Select File Format",
|
||||||
"files": "files",
|
"files": "files",
|
||||||
"select": "Select File"
|
"select": "Select File",
|
||||||
|
"inLibrary": "In Library"
|
||||||
},
|
},
|
||||||
"errors": {
|
"errors": {
|
||||||
"invalidUrl": "Invalid Civitai URL format",
|
"invalidUrl": "Invalid CivitAI URL format",
|
||||||
"noVersions": "No versions available for this model",
|
"noVersions": "No versions available for this model",
|
||||||
"mixedSources": "Cannot mix CivitAI and Hugging Face URLs in the same batch.",
|
"mixedSources": "Cannot mix CivitAI and Hugging Face URLs in the same batch.",
|
||||||
"noModelFiles": "No model files found in this repository."
|
"noModelFiles": "No model files found in this repository."
|
||||||
@@ -1360,11 +1539,46 @@
|
|||||||
"note": "Files will be downloaded using default path templates. This may take a while depending on the number of LoRAs.",
|
"note": "Files will be downloaded using default path templates. This may take a while depending on the number of LoRAs.",
|
||||||
"downloadButton": "Download {count} LoRA(s)"
|
"downloadButton": "Download {count} LoRA(s)"
|
||||||
},
|
},
|
||||||
|
"rematchOptions": {
|
||||||
|
"title": "Rematch Recipes",
|
||||||
|
"messageGlobal": "All recipes will be scanned against your local model library.",
|
||||||
|
"messageSingle": "This recipe will be scanned against your local model library.",
|
||||||
|
"messageBulk": "{count} selected recipe(s) will be scanned against your local model library.",
|
||||||
|
"relaxedLabel": "Also reconnect missing models by file name",
|
||||||
|
"relaxedDescription": "These models could also be fixed by downloading — download is more accurate. Matches may link a different version; they'll be listed for review and can be undone.",
|
||||||
|
"confirmButton": "Rematch"
|
||||||
|
},
|
||||||
|
"rematchResults": {
|
||||||
|
"undo": "Undo",
|
||||||
|
"undone": "Undone",
|
||||||
|
"undoFailed": "Failed to undo rematch: {message}"
|
||||||
|
},
|
||||||
|
"rematchSummary": {
|
||||||
|
"title": "Rematch Summary",
|
||||||
|
"successMessage": "Matched {entries} entries",
|
||||||
|
"failed": "Rematch failed",
|
||||||
|
"completedWithWarnings": "Rematch completed — review recommended",
|
||||||
|
"cancelledNote": "Run cancelled before completion — counts are partial.",
|
||||||
|
"statMatched": "Matched entries",
|
||||||
|
"statReview": "Needs review",
|
||||||
|
"statUnresolved": "Unresolved",
|
||||||
|
"statErrors": "Errors",
|
||||||
|
"reviewSection": "Filename matches to review ({count})",
|
||||||
|
"columnRecipe": "Recipe",
|
||||||
|
"columnEntry": "Entry",
|
||||||
|
"columnFile": "Matched file",
|
||||||
|
"columnUndo": "Undo",
|
||||||
|
"copyReport": "Copy Report",
|
||||||
|
"close": "Close",
|
||||||
|
"scope_global": "All recipes",
|
||||||
|
"scope_bulk": "Selected recipes",
|
||||||
|
"scope_single": "Single recipe"
|
||||||
|
},
|
||||||
"exampleAccess": {
|
"exampleAccess": {
|
||||||
"title": "Local Example Images",
|
"title": "Local Example Images",
|
||||||
"message": "No local example images found for this model. View options:",
|
"message": "No local example images found for this model. View options:",
|
||||||
"downloadOption": {
|
"downloadOption": {
|
||||||
"title": "Download from Civitai",
|
"title": "Download from CivitAI",
|
||||||
"description": "Save remote examples locally for offline use and faster loading"
|
"description": "Save remote examples locally for offline use and faster loading"
|
||||||
},
|
},
|
||||||
"importOption": {
|
"importOption": {
|
||||||
@@ -1391,7 +1605,7 @@
|
|||||||
"confirmAction": "Save & Link"
|
"confirmAction": "Save & Link"
|
||||||
},
|
},
|
||||||
"relinkCivitai": {
|
"relinkCivitai": {
|
||||||
"title": "Re-link to Civitai",
|
"title": "Re-link to CivitAI",
|
||||||
"warning": "Warning:",
|
"warning": "Warning:",
|
||||||
"warningText": "This is a potentially destructive operation. Re-linking will:",
|
"warningText": "This is a potentially destructive operation. Re-linking will:",
|
||||||
"warningList": {
|
"warningList": {
|
||||||
@@ -1400,14 +1614,15 @@
|
|||||||
"unintendedConsequences": "May have other unintended consequences"
|
"unintendedConsequences": "May have other unintended consequences"
|
||||||
},
|
},
|
||||||
"proceedText": "Only proceed if you're sure this is what you want.",
|
"proceedText": "Only proceed if you're sure this is what you want.",
|
||||||
"urlLabel": "Civitai Model URL:",
|
"urlLabel": "CivitAI Model URL:",
|
||||||
"urlPlaceholder": "https://civitai.com/models/649516/model-name?modelVersionId=726676 or https://civitai.red/models/649516/model-name?modelVersionId=726676",
|
"urlPlaceholder": "https://civitai.com/models/12345/model-name?modelVersionId=67890 or https://civitai.red/models/12345/model-name?modelVersionId=67890",
|
||||||
"helpText": {
|
"helpText": {
|
||||||
"title": "Paste any Civitai model URL from civitai.com or civitai.red. Supported formats:",
|
"title": "Paste any CivitAI or CivitArchive model URL. Supported formats:",
|
||||||
"format1": "https://civitai.com/models/649516",
|
"format1": "https://civitai.com/models/12345",
|
||||||
"format2": "https://civitai.com/models/649516?modelVersionId=726676",
|
"format2": "https://civitai.com/models/12345?modelVersionId=67890",
|
||||||
"format3": "https://civitai.com/models/649516/model-name?modelVersionId=726676",
|
"format3": "https://civitai.com/models/12345/model-name?modelVersionId=67890",
|
||||||
"note": "Note: If no modelVersionId is provided, the latest version will be used."
|
"note": "Note: If no modelVersionId is provided, the latest version will be used.",
|
||||||
|
"format4": "https://civarchive.com/models/12345 (CivArchive)"
|
||||||
},
|
},
|
||||||
"confirmAction": "Confirm Re-link"
|
"confirmAction": "Confirm Re-link"
|
||||||
},
|
},
|
||||||
@@ -1417,14 +1632,16 @@
|
|||||||
"editFileName": "Edit file name",
|
"editFileName": "Edit file name",
|
||||||
"editBaseModel": "Edit base model",
|
"editBaseModel": "Edit base model",
|
||||||
"editVersionName": "Edit version name",
|
"editVersionName": "Edit version name",
|
||||||
"viewOnCivitai": "View on Civitai",
|
"viewOnCivitai": "View on CivitAI",
|
||||||
"viewOnCivitaiText": "View on Civitai",
|
"viewOnCivitaiText": "View on CivitAI",
|
||||||
"viewOnHuggingFace": "View on Hugging Face",
|
"viewOnHuggingFace": "View on Hugging Face",
|
||||||
"viewOnHuggingFaceText": "View on Hugging Face",
|
"viewOnHuggingFaceText": "View on Hugging Face",
|
||||||
"viewCreatorProfile": "View Creator Profile",
|
"viewCreatorProfile": "View Creator Profile",
|
||||||
"openFileLocation": "Open File Location",
|
"openFileLocation": "Open File Location",
|
||||||
"sendToWorkflow": "Send to ComfyUI",
|
"sendToWorkflow": "Send to ComfyUI",
|
||||||
"sendToWorkflowText": "Send to ComfyUI"
|
"sendToWorkflowText": "Send to ComfyUI",
|
||||||
|
"copyHash": "Copy hash",
|
||||||
|
"deleteModelWithShortcut": "Delete model (Del)"
|
||||||
},
|
},
|
||||||
"openFileLocation": {
|
"openFileLocation": {
|
||||||
"success": "File location opened successfully",
|
"success": "File location opened successfully",
|
||||||
@@ -1441,6 +1658,7 @@
|
|||||||
"location": "Location",
|
"location": "Location",
|
||||||
"baseModel": "Base Model",
|
"baseModel": "Base Model",
|
||||||
"size": "Size",
|
"size": "Size",
|
||||||
|
"hashes": "Hashes",
|
||||||
"unknown": "Unknown",
|
"unknown": "Unknown",
|
||||||
"usageTips": "Usage Tips",
|
"usageTips": "Usage Tips",
|
||||||
"additionalNotes": "Additional Notes",
|
"additionalNotes": "Additional Notes",
|
||||||
@@ -1467,7 +1685,11 @@
|
|||||||
"clipSkip": "Clip Skip",
|
"clipSkip": "Clip Skip",
|
||||||
"valuePlaceholder": "Value",
|
"valuePlaceholder": "Value",
|
||||||
"add": "Add",
|
"add": "Add",
|
||||||
"invalidRange": "Invalid range format. Use x.x-y.y"
|
"invalidRange": "Invalid range format. Use x.x-y.y",
|
||||||
|
"invalidValue": "Please enter a valid number",
|
||||||
|
"saveFailed": "Failed to save preset parameter",
|
||||||
|
"added": "Preset parameter added",
|
||||||
|
"updated": "Preset parameter updated"
|
||||||
},
|
},
|
||||||
"triggerWords": {
|
"triggerWords": {
|
||||||
"label": "Trigger Words",
|
"label": "Trigger Words",
|
||||||
@@ -1478,7 +1700,7 @@
|
|||||||
"addPlaceholder": "Type to add or click suggestions below",
|
"addPlaceholder": "Type to add or click suggestions below",
|
||||||
"editWord": "Edit trigger word",
|
"editWord": "Edit trigger word",
|
||||||
"editPlaceholder": "Edit trigger word",
|
"editPlaceholder": "Edit trigger word",
|
||||||
"copyWord": "Copy trigger word",
|
"copyOrEditWord": "Click to copy, double-click to edit",
|
||||||
"deleteWord": "Delete trigger word",
|
"deleteWord": "Delete trigger word",
|
||||||
"suggestions": {
|
"suggestions": {
|
||||||
"noSuggestions": "No suggestions available",
|
"noSuggestions": "No suggestions available",
|
||||||
@@ -1517,7 +1739,7 @@
|
|||||||
},
|
},
|
||||||
"license": {
|
"license": {
|
||||||
"noImageSell": "No selling generated content",
|
"noImageSell": "No selling generated content",
|
||||||
"noRentCivit": "No Civitai generation",
|
"noRentCivit": "No CivitAI generation",
|
||||||
"noRent": "No generation services",
|
"noRent": "No generation services",
|
||||||
"noSell": "No selling models",
|
"noSell": "No selling models",
|
||||||
"creditRequired": "Creator credit required",
|
"creditRequired": "Creator credit required",
|
||||||
@@ -1532,6 +1754,30 @@
|
|||||||
"examples": "Loading examples...",
|
"examples": "Loading examples...",
|
||||||
"versions": "Loading versions..."
|
"versions": "Loading versions..."
|
||||||
},
|
},
|
||||||
|
"showcase": {
|
||||||
|
"hiddenBySfw": "{count} hidden by SFW-only setting",
|
||||||
|
"showExamples": "Show examples",
|
||||||
|
"showCount": "Show examples ({count})",
|
||||||
|
"hideExamples": "Hide examples",
|
||||||
|
"addExamples": "Add examples",
|
||||||
|
"previousExample": "Previous example ([)",
|
||||||
|
"nextExample": "Next example (])",
|
||||||
|
"noExamples": "No example images available",
|
||||||
|
"addMoreExamples": "Add more examples",
|
||||||
|
"dragDrop": "Drag & drop images or videos here",
|
||||||
|
"or": "or",
|
||||||
|
"selectFiles": "Select Files",
|
||||||
|
"supportedFormats": "Supported formats: jpg, png, gif, webp, avif, jxl, mp4, webm",
|
||||||
|
"importing": "Importing files...",
|
||||||
|
"noSupportedFiles": "No supported files selected. Please select image or video files.",
|
||||||
|
"allFiltered": "All example images are filtered due to NSFW content settings",
|
||||||
|
"sfwOnlyEnabled": "Your settings are currently set to show only safe-for-work content",
|
||||||
|
"changeInSettings": "You can change this in Settings",
|
||||||
|
"nsfwMature": "Mature Content",
|
||||||
|
"nsfwR": "R-rated Content",
|
||||||
|
"nsfwX": "X-rated Content",
|
||||||
|
"nsfwXxx": "XXX-rated Content"
|
||||||
|
},
|
||||||
"versions": {
|
"versions": {
|
||||||
"heading": "Model versions",
|
"heading": "Model versions",
|
||||||
"copy": "Track and manage every version of this model in one place.",
|
"copy": "Track and manage every version of this model in one place.",
|
||||||
@@ -1558,27 +1804,28 @@
|
|||||||
"newer": "Newer Version",
|
"newer": "Newer Version",
|
||||||
"newerTooltip": "This version is newer than your latest local version",
|
"newerTooltip": "This version is newer than your latest local version",
|
||||||
"earlyAccess": "Early Access",
|
"earlyAccess": "Early Access",
|
||||||
"earlyAccessTooltip": "This version currently requires Civitai early access",
|
"earlyAccessTooltip": "This version currently requires CivitAI early access",
|
||||||
"paid": "Paid",
|
"paid": "Paid",
|
||||||
"paidTooltip": "This version requires payment to download",
|
"paidTooltip": "This version requires payment to download",
|
||||||
"ignored": "Ignored",
|
"ignored": "Ignored",
|
||||||
"ignoredTooltip": "Update notifications are disabled for this version",
|
"ignoredTooltip": "Update notifications are disabled for this version",
|
||||||
"onSiteOnly": "On-Site Only",
|
"onSiteOnly": "On-Site Only",
|
||||||
"onSiteOnlyTooltip": "This version is only available for on-site generation on Civitai"
|
"onSiteOnlyTooltip": "This version is only available for on-site generation on CivitAI"
|
||||||
},
|
},
|
||||||
"actions": {
|
"actions": {
|
||||||
"download": "Download",
|
"download": "Download",
|
||||||
"downloadTooltip": "Download this version",
|
"downloadTooltip": "Download this version",
|
||||||
"downloadEarlyAccessTooltip": "Download this early access version from Civitai",
|
"downloadChooseFilesTooltip": "Choose which files to download",
|
||||||
"downloadPaidTooltip": "Download this paid version from Civitai",
|
"downloadEarlyAccessTooltip": "Download this early access version from CivitAI",
|
||||||
"downloadNotAllowedTooltip": "This version is only available for on-site generation on Civitai",
|
"downloadPaidTooltip": "Download this paid version from CivitAI",
|
||||||
|
"downloadNotAllowedTooltip": "This version is only available for on-site generation on CivitAI",
|
||||||
"delete": "Delete",
|
"delete": "Delete",
|
||||||
"deleteTooltip": "Delete this local version",
|
"deleteTooltip": "Delete this local version",
|
||||||
"ignore": "Ignore",
|
"ignore": "Ignore",
|
||||||
"unignore": "Unignore",
|
"unignore": "Unignore",
|
||||||
"ignoreTooltip": "Ignore update notifications for this version",
|
"ignoreTooltip": "Ignore update notifications for this version",
|
||||||
"unignoreTooltip": "Resume update notifications for this version",
|
"unignoreTooltip": "Resume update notifications for this version",
|
||||||
"viewVersionOnCivitai": "View version on Civitai",
|
"viewVersionOnCivitai": "View version on CivitAI",
|
||||||
"earlyAccessTooltip": "Requires early access purchase",
|
"earlyAccessTooltip": "Requires early access purchase",
|
||||||
"resumeModelUpdates": "Resume updates for this model",
|
"resumeModelUpdates": "Resume updates for this model",
|
||||||
"ignoreModelUpdates": "Ignore updates for this model",
|
"ignoreModelUpdates": "Ignore updates for this model",
|
||||||
@@ -1599,7 +1846,7 @@
|
|||||||
},
|
},
|
||||||
"empty": "No version history available for this model yet.",
|
"empty": "No version history available for this model yet.",
|
||||||
"error": "Failed to load versions.",
|
"error": "Failed to load versions.",
|
||||||
"missingModelId": "This model is missing a Civitai model id.",
|
"missingModelId": "This model is missing a CivitAI model id.",
|
||||||
"hfGroupInfo": "This is a HuggingFace model group. Open the library to see all versions in the grid.",
|
"hfGroupInfo": "This is a HuggingFace model group. Open the library to see all versions in the grid.",
|
||||||
"confirm": {
|
"confirm": {
|
||||||
"delete": "Delete this version from your library?"
|
"delete": "Delete this version from your library?"
|
||||||
@@ -1672,6 +1919,10 @@
|
|||||||
"title": "Initializing Embedding Manager",
|
"title": "Initializing Embedding Manager",
|
||||||
"message": "Scanning and building embedding cache. This may take a few minutes..."
|
"message": "Scanning and building embedding cache. This may take a few minutes..."
|
||||||
},
|
},
|
||||||
|
"other": {
|
||||||
|
"title": "Initializing Other Models Manager",
|
||||||
|
"message": "Scanning and building model cache. This may take a few minutes..."
|
||||||
|
},
|
||||||
"recipes": {
|
"recipes": {
|
||||||
"title": "Initializing Recipe Manager",
|
"title": "Initializing Recipe Manager",
|
||||||
"message": "Loading and processing recipes. This may take a few minutes..."
|
"message": "Loading and processing recipes. This may take a few minutes..."
|
||||||
@@ -1683,14 +1934,14 @@
|
|||||||
"tips": {
|
"tips": {
|
||||||
"title": "Tips & Tricks",
|
"title": "Tips & Tricks",
|
||||||
"civitai": {
|
"civitai": {
|
||||||
"title": "Civitai Integration",
|
"title": "CivitAI Integration",
|
||||||
"description": "Connect your Civitai account: Visit Profile Avatar → Settings → API Keys → Add API Key, then paste it in Lora Manager settings.",
|
"description": "Connect your CivitAI account: Visit Profile Avatar → Settings → API Keys → Add API Key, then paste it in Lora Manager settings.",
|
||||||
"alt": "Civitai API Setup"
|
"alt": "CivitAI API Setup"
|
||||||
},
|
},
|
||||||
"download": {
|
"download": {
|
||||||
"title": "Easy Download",
|
"title": "Easy Download",
|
||||||
"description": "Use Civitai URLs to quickly download and install new models.",
|
"description": "Use CivitAI URLs to quickly download and install new models.",
|
||||||
"alt": "Civitai Download"
|
"alt": "CivitAI Download"
|
||||||
},
|
},
|
||||||
"recipes": {
|
"recipes": {
|
||||||
"title": "Save Recipes",
|
"title": "Save Recipes",
|
||||||
@@ -1778,10 +2029,52 @@
|
|||||||
"tabs": {
|
"tabs": {
|
||||||
"gettingStarted": "Getting Started",
|
"gettingStarted": "Getting Started",
|
||||||
"updateVlogs": "Update Vlogs",
|
"updateVlogs": "Update Vlogs",
|
||||||
"documentation": "Documentation"
|
"documentation": "Documentation",
|
||||||
|
"shortcuts": "Shortcuts"
|
||||||
},
|
},
|
||||||
"gettingStarted": {
|
"gettingStarted": {
|
||||||
"title": "Getting Started with LoRA Manager"
|
"title": "Getting Started with LoRA Manager",
|
||||||
|
"replayTutorial": "Replay Tutorial"
|
||||||
|
},
|
||||||
|
"shortcuts": {
|
||||||
|
"title": "Keyboard & Mouse Shortcuts",
|
||||||
|
"groups": {
|
||||||
|
"general": "General",
|
||||||
|
"actions": "Actions",
|
||||||
|
"selection": "Selection & Bulk Mode",
|
||||||
|
"navigation": "Navigation",
|
||||||
|
"modelModal": "Model / Recipe Modal",
|
||||||
|
"mediaViewer": "Media Viewer / Showcase"
|
||||||
|
},
|
||||||
|
"keys": {
|
||||||
|
"click": "Click",
|
||||||
|
"drag": "Drag",
|
||||||
|
"rightClick": "Right-click",
|
||||||
|
"letter": "Letter",
|
||||||
|
"swipe": "Swipe"
|
||||||
|
},
|
||||||
|
"entries": {
|
||||||
|
"focusSearch": "Focus search",
|
||||||
|
"closeModal": "Close modal / panel",
|
||||||
|
"openShortcuts": "Open this shortcuts panel",
|
||||||
|
"refresh": "Refresh model list",
|
||||||
|
"fetchMetadata": "Fetch metadata from CivitAI (model pages only)",
|
||||||
|
"downloadModel": "Download a model (model pages only)",
|
||||||
|
"toggleBulkMode": "Toggle bulk mode",
|
||||||
|
"selectAll": "Select all visible models",
|
||||||
|
"rangeSelect": "Range select",
|
||||||
|
"marqueeSelect": "Marquee-select cards (on empty grid area)",
|
||||||
|
"exitBulkMode": "Exit bulk mode",
|
||||||
|
"bulkActions": "On selected card: bulk actions menu",
|
||||||
|
"globalActions": "On empty page area: global actions menu (update check, manage excluded models)",
|
||||||
|
"scrollPages": "Scroll pages",
|
||||||
|
"jumpAlphabet": "Jump alphabet bar",
|
||||||
|
"prevNext": "Previous / next model",
|
||||||
|
"deleteEntry": "Delete",
|
||||||
|
"cycleMedia": "Cycle media ([ / ] in showcase gallery)",
|
||||||
|
"swipeTouch": "Cycle media on touch devices",
|
||||||
|
"closeViewer": "Close viewer"
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"updateVlogs": {
|
"updateVlogs": {
|
||||||
"title": "Latest Updates",
|
"title": "Latest Updates",
|
||||||
@@ -1798,7 +2091,8 @@
|
|||||||
"settings": "Settings & Configuration",
|
"settings": "Settings & Configuration",
|
||||||
"extensions": "Extensions",
|
"extensions": "Extensions",
|
||||||
"newBadge": "NEW"
|
"newBadge": "NEW"
|
||||||
}
|
},
|
||||||
|
"newContentBadge": "New"
|
||||||
},
|
},
|
||||||
"update": {
|
"update": {
|
||||||
"title": "Check for Updates",
|
"title": "Check for Updates",
|
||||||
@@ -1874,7 +2168,7 @@
|
|||||||
"submitGithubIssue": "Submit GitHub Issue",
|
"submitGithubIssue": "Submit GitHub Issue",
|
||||||
"joinDiscord": "Join Discord",
|
"joinDiscord": "Join Discord",
|
||||||
"youtubeChannel": "YouTube Channel",
|
"youtubeChannel": "YouTube Channel",
|
||||||
"civitaiProfile": "Civitai Profile",
|
"civitaiProfile": "CivitAI Profile",
|
||||||
"supportKofi": "Support on Ko-fi",
|
"supportKofi": "Support on Ko-fi",
|
||||||
"supportPatreon": "Support on Patreon"
|
"supportPatreon": "Support on Patreon"
|
||||||
},
|
},
|
||||||
@@ -1917,6 +2211,7 @@
|
|||||||
"downloadPartialSuccess": "Downloaded {completed} of {total} LoRAs",
|
"downloadPartialSuccess": "Downloaded {completed} of {total} LoRAs",
|
||||||
"downloadPartialWithAccess": "Downloaded {completed} of {total} LoRAs. {accessFailures} failed due to access restrictions. Check your API key in settings or early access status.",
|
"downloadPartialWithAccess": "Downloaded {completed} of {total} LoRAs. {accessFailures} failed due to access restrictions. Check your API key in settings or early access status.",
|
||||||
"pleaseSelectVersion": "Please select a version",
|
"pleaseSelectVersion": "Please select a version",
|
||||||
|
"pleaseSelectFile": "Please select at least one file",
|
||||||
"versionExists": "This version already exists in your library",
|
"versionExists": "This version already exists in your library",
|
||||||
"downloadCompleted": "Download completed successfully",
|
"downloadCompleted": "Download completed successfully",
|
||||||
"downloadSkippedByBaseModel": "Skipped download because base model {baseModel} is excluded",
|
"downloadSkippedByBaseModel": "Skipped download because base model {baseModel} is excluded",
|
||||||
@@ -1950,11 +2245,17 @@
|
|||||||
"createMissingData": "Missing required data to create recipe",
|
"createMissingData": "Missing required data to create recipe",
|
||||||
"created": "Recipe created successfully",
|
"created": "Recipe created successfully",
|
||||||
"noMissingLoras": "No missing LoRAs to download",
|
"noMissingLoras": "No missing LoRAs to download",
|
||||||
|
"unresolvableMarkedForReconnect": "{count} unresolvable entr(ies) marked — they can now be reconnected to a local LoRA.",
|
||||||
|
"noPreviousRecipe": "No previous recipe available",
|
||||||
|
"noNextRecipe": "No next recipe available",
|
||||||
"missingLorasInfoFailed": "Failed to get information for missing LoRAs",
|
"missingLorasInfoFailed": "Failed to get information for missing LoRAs",
|
||||||
"preparingForDownloadFailed": "Error preparing LoRAs for download",
|
"preparingForDownloadFailed": "Error preparing LoRAs for download",
|
||||||
"enterLoraName": "Please enter a LoRA name or syntax",
|
"enterLoraName": "Please enter a LoRA name or syntax",
|
||||||
"reconnectedSuccessfully": "LoRA reconnected successfully",
|
"reconnectedSuccessfully": "LoRA reconnected successfully",
|
||||||
|
"reconnectBaseModelMismatch": "Reconnected, but base models differ (recipe: {recipe}, LoRA: {lora}) — they are architecture-compatible",
|
||||||
"reconnectFailed": "Error reconnecting LoRA: {message}",
|
"reconnectFailed": "Error reconnecting LoRA: {message}",
|
||||||
|
"loraRestored": "LoRA restored to its previous association",
|
||||||
|
"loraRestoreFailed": "Error restoring LoRA: {message}",
|
||||||
"noPromptToSend": "No prompt to send",
|
"noPromptToSend": "No prompt to send",
|
||||||
"cannotSend": "Cannot send recipe: Missing recipe ID",
|
"cannotSend": "Cannot send recipe: Missing recipe ID",
|
||||||
"sendFailed": "Failed to send recipe to workflow",
|
"sendFailed": "Failed to send recipe to workflow",
|
||||||
@@ -1962,6 +2263,16 @@
|
|||||||
"missingCheckpointPath": "Checkpoint path not available",
|
"missingCheckpointPath": "Checkpoint path not available",
|
||||||
"missingCheckpointInfo": "Missing checkpoint information",
|
"missingCheckpointInfo": "Missing checkpoint information",
|
||||||
"downloadCheckpointFailed": "Failed to download checkpoint: {message}",
|
"downloadCheckpointFailed": "Failed to download checkpoint: {message}",
|
||||||
|
"enterCheckpointName": "Please enter a checkpoint name",
|
||||||
|
"checkpointReconnectedSuccessfully": "Checkpoint reconnected successfully",
|
||||||
|
"reconnectCheckpointBaseModelMismatch": "Reconnected, but base models differ (recipe: {recipe}, checkpoint: {checkpoint}) — they are architecture-compatible",
|
||||||
|
"checkpointReconnectFailed": "Error reconnecting checkpoint: {message}",
|
||||||
|
"checkpointRestored": "Checkpoint restored to its previous association",
|
||||||
|
"checkpointRestoreFailed": "Error restoring checkpoint: {message}",
|
||||||
|
"checkpointDownloadUnavailable": "This checkpoint cannot be downloaded without CivitAI identifiers - try reconnecting it with a local checkpoint",
|
||||||
|
"missingLoraDownloadInfo": "Missing download information for this LoRA",
|
||||||
|
"hashNotFoundOnCivitai": "This LoRA hash cannot be resolved on CivitAI - the model may have been updated or the hash is invalid",
|
||||||
|
"downloadLoraFailed": "Failed to download LoRA: {message}",
|
||||||
"cannotDelete": "Cannot delete recipe: Missing recipe ID",
|
"cannotDelete": "Cannot delete recipe: Missing recipe ID",
|
||||||
"deleteConfirmationError": "Error showing delete confirmation",
|
"deleteConfirmationError": "Error showing delete confirmation",
|
||||||
"deletedSuccessfully": "Recipe deleted successfully",
|
"deletedSuccessfully": "Recipe deleted successfully",
|
||||||
@@ -1985,24 +2296,22 @@
|
|||||||
"batchImportCancelFailed": "Failed to cancel batch import: {message}",
|
"batchImportCancelFailed": "Failed to cancel batch import: {message}",
|
||||||
"batchImportNoUrls": "Please enter at least one URL or file path",
|
"batchImportNoUrls": "Please enter at least one URL or file path",
|
||||||
"batchImportNoDirectory": "Please enter a directory path",
|
"batchImportNoDirectory": "Please enter a directory path",
|
||||||
|
"batchImportRateLimited": "Metadata provider rate limit reached — requests are being slowed and some items may be skipped. You can re-run the import later.",
|
||||||
"batchImportBrowseFailed": "Failed to browse directory: {message}",
|
"batchImportBrowseFailed": "Failed to browse directory: {message}",
|
||||||
"batchImportDirectorySelected": "Directory selected: {path}",
|
"batchImportDirectorySelected": "Directory selected: {path}",
|
||||||
"noRecipesSelected": "No recipes selected",
|
"noRecipesSelected": "No recipes selected",
|
||||||
"repairBulkComplete": "Repair complete: {repaired} repaired, {skipped} skipped (of {total})",
|
|
||||||
"repairBulkSkipped": "No repair needed for any of the {total} selected recipes",
|
|
||||||
"repairBulkFailed": "Failed to repair selected recipes: {message}",
|
|
||||||
"rematchComplete": "Matched {entries} entries across {recipes} recipes",
|
|
||||||
"rematchCompleteErrors": "Matched {entries} entries across {recipes} recipes, {failures} failed",
|
|
||||||
"rematchAllFailed": "Rematch failed for {failures} of {total} selected recipes",
|
|
||||||
"rematchUnmatched": "No local match found for {entries} entries in {recipes} recipes",
|
|
||||||
"rematchSkipped": "No rematch needed for any of the {total} selected recipes",
|
"rematchSkipped": "No rematch needed for any of the {total} selected recipes",
|
||||||
"rematchFailed": "Failed to rematch selected recipes: {message}",
|
"rematchFailed": "Failed to rematch selected recipes: {message}",
|
||||||
"reimporting": "Re-importing recipe from source...",
|
"reimporting": "Re-importing recipe from source...",
|
||||||
|
"reimportingViaExtension": "Re-importing recipe {current}/{total} via browser extension...",
|
||||||
"reimportSuccess": "Recipe re-imported successfully",
|
"reimportSuccess": "Recipe re-imported successfully",
|
||||||
"reimportBulkComplete": "Re-import complete: {completed} re-imported, {failed} failed (of {total})",
|
"reimportBulkComplete": "Re-import complete: {completed} re-imported, {failed} failed (of {total})",
|
||||||
"reimportBulkFailed": "Failed to re-import some recipes",
|
"reimportBulkFailed": "Failed to re-import some recipes",
|
||||||
"noMissingLorasInSelection": "No missing LoRAs found in selected recipes",
|
"noMissingLorasInSelection": "No missing LoRAs found in selected recipes",
|
||||||
"noLoraRootConfigured": "No LoRA root directory configured. Please set a default LoRA root in settings."
|
"noLoraRootConfigured": "No LoRA root directory configured. Please set a default LoRA root in settings.",
|
||||||
|
"workflowSent": "Workflow sent to ComfyUI",
|
||||||
|
"workflowSendFailed": "Failed to send workflow to ComfyUI: {error}",
|
||||||
|
"workflowNoWorkflow": "No embedded workflow found in this recipe"
|
||||||
},
|
},
|
||||||
"models": {
|
"models": {
|
||||||
"noModelsSelected": "No models selected",
|
"noModelsSelected": "No models selected",
|
||||||
@@ -2039,8 +2348,8 @@
|
|||||||
"bulkUpdatesChecking": "Checking selected {type}(s) for updates...",
|
"bulkUpdatesChecking": "Checking selected {type}(s) for updates...",
|
||||||
"bulkUpdatesSuccess": "Updates available for {count} selected {type}(s)",
|
"bulkUpdatesSuccess": "Updates available for {count} selected {type}(s)",
|
||||||
"bulkUpdatesNone": "No updates found for selected {type}(s)",
|
"bulkUpdatesNone": "No updates found for selected {type}(s)",
|
||||||
"bulkUpdatesMissing": "Selected {type}(s) are not linked to Civitai updates",
|
"bulkUpdatesMissing": "Selected {type}(s) are not linked to CivitAI updates",
|
||||||
"bulkUpdatesPartialMissing": "Skipped {missing} selected {type}(s) without Civitai links",
|
"bulkUpdatesPartialMissing": "Skipped {missing} selected {type}(s) without CivitAI links",
|
||||||
"bulkUpdatesFailed": "Failed to check updates for selected {type}(s): {message}",
|
"bulkUpdatesFailed": "Failed to check updates for selected {type}(s): {message}",
|
||||||
"invalidCharactersRemoved": "Invalid characters removed from filename",
|
"invalidCharactersRemoved": "Invalid characters removed from filename",
|
||||||
"filenameCannotBeEmpty": "File name cannot be empty",
|
"filenameCannotBeEmpty": "File name cannot be empty",
|
||||||
@@ -2069,6 +2378,7 @@
|
|||||||
"checkpointRootsFailed": "Failed to load checkpoint roots: {message}",
|
"checkpointRootsFailed": "Failed to load checkpoint roots: {message}",
|
||||||
"unetRootsFailed": "Failed to load diffusion model roots: {message}",
|
"unetRootsFailed": "Failed to load diffusion model roots: {message}",
|
||||||
"embeddingRootsFailed": "Failed to load embedding roots: {message}",
|
"embeddingRootsFailed": "Failed to load embedding roots: {message}",
|
||||||
|
"otherRootsFailed": "Failed to load other model roots: {message}",
|
||||||
"mappingsUpdated": "Base model path mappings updated ({count} mapping{plural})",
|
"mappingsUpdated": "Base model path mappings updated ({count} mapping{plural})",
|
||||||
"mappingsCleared": "Base model path mappings cleared",
|
"mappingsCleared": "Base model path mappings cleared",
|
||||||
"mappingSaveFailed": "Failed to save base model mappings: {message}",
|
"mappingSaveFailed": "Failed to save base model mappings: {message}",
|
||||||
@@ -2166,10 +2476,11 @@
|
|||||||
"contextMenu": {
|
"contextMenu": {
|
||||||
"contentRatingSet": "Content rating set to {level}",
|
"contentRatingSet": "Content rating set to {level}",
|
||||||
"contentRatingFailed": "Failed to set content rating: {message}",
|
"contentRatingFailed": "Failed to set content rating: {message}",
|
||||||
"relinkSuccess": "Model successfully re-linked to Civitai",
|
"relinkSuccess": "Model successfully re-linked to CivitAI",
|
||||||
"relinkFailed": "Error: {message}",
|
"relinkFailed": "Error: {message}",
|
||||||
"linkHfSuccess": "Model successfully linked to HuggingFace",
|
"linkHfSuccess": "Model successfully linked to HuggingFace",
|
||||||
"linkHfFailed": "Error: {message}",
|
"linkHfFailed": "Error: {message}",
|
||||||
|
"linkCivArchSuccess": "Model successfully re-linked via CivitArchive",
|
||||||
"fetchMetadataFirst": "Please fetch metadata from CivitAI first",
|
"fetchMetadataFirst": "Please fetch metadata from CivitAI first",
|
||||||
"noCivitaiInfo": "No CivitAI information available",
|
"noCivitaiInfo": "No CivitAI information available",
|
||||||
"missingHash": "Model hash not available"
|
"missingHash": "Model hash not available"
|
||||||
@@ -2259,7 +2570,7 @@
|
|||||||
},
|
},
|
||||||
"issues": {
|
"issues": {
|
||||||
"civitai_api_key": {
|
"civitai_api_key": {
|
||||||
"title": "Civitai API Key"
|
"title": "CivitAI API Key"
|
||||||
},
|
},
|
||||||
"cache_health": {
|
"cache_health": {
|
||||||
"title": "Model Cache Health"
|
"title": "Model Cache Health"
|
||||||
@@ -2313,9 +2624,9 @@
|
|||||||
},
|
},
|
||||||
"communitySupport": {
|
"communitySupport": {
|
||||||
"title": "Keep LoRA Manager Thriving with Your Support ❤️",
|
"title": "Keep LoRA Manager Thriving with Your Support ❤️",
|
||||||
"content": "LoRA Manager is a passion project maintained full-time by a solo developer. Your support on Ko-fi helps cover development costs, keeps new updates coming, and unlocks a license key for the LM Civitai Extension as a thank-you gift. Every contribution truly makes a difference.",
|
"content": "LoRA Manager is a passion project maintained full-time by a solo developer. Your support on Ko-fi helps cover development costs, keeps new updates coming, and unlocks a license key for the LM CivitAI Extension as a thank-you gift. Every contribution truly makes a difference.",
|
||||||
"supportCta": "Support on Ko-fi",
|
"supportCta": "Support on Ko-fi",
|
||||||
"learnMore": "LM Civitai Extension Tutorial"
|
"learnMore": "LM CivitAI Extension Tutorial"
|
||||||
},
|
},
|
||||||
"cacheHealth": {
|
"cacheHealth": {
|
||||||
"corrupted": {
|
"corrupted": {
|
||||||
@@ -2330,6 +2641,12 @@
|
|||||||
"rebuilding": "Rebuilding cache...",
|
"rebuilding": "Rebuilding cache...",
|
||||||
"rebuildFailed": "Failed to rebuild cache: {error}",
|
"rebuildFailed": "Failed to rebuild cache: {error}",
|
||||||
"retry": "Retry"
|
"retry": "Retry"
|
||||||
|
},
|
||||||
|
"otherModels": {
|
||||||
|
"title": "Other Models Management is available",
|
||||||
|
"content": "Scan and manage VAE, upscaler, text encoder, CLIP vision and ControlNet files — and download them from CivitAI — from one dedicated page.",
|
||||||
|
"enable": "Enable Other Models",
|
||||||
|
"openSettings": "Open Settings"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+558
-241
File diff suppressed because it is too large
Load Diff
+562
-245
File diff suppressed because it is too large
Load Diff
+583
-266
File diff suppressed because it is too large
Load Diff
+516
-199
File diff suppressed because it is too large
Load Diff
+520
-203
File diff suppressed because it is too large
Load Diff
+537
-220
File diff suppressed because it is too large
Load Diff
+452
-135
File diff suppressed because it is too large
Load Diff
+464
-147
File diff suppressed because it is too large
Load Diff
+276
-1
@@ -17,6 +17,9 @@ import types as _types
|
|||||||
import time
|
import time
|
||||||
|
|
||||||
from .utils.cache_paths import CacheType, get_cache_file_path, get_legacy_cache_paths
|
from .utils.cache_paths import CacheType, get_cache_file_path, get_legacy_cache_paths
|
||||||
|
from .utils.constants import (
|
||||||
|
OTHER_MODEL_FOLDER_SUBTYPES,
|
||||||
|
)
|
||||||
from .utils.settings_paths import (
|
from .utils.settings_paths import (
|
||||||
ensure_settings_file,
|
ensure_settings_file,
|
||||||
get_settings_dir,
|
get_settings_dir,
|
||||||
@@ -172,6 +175,13 @@ class Config:
|
|||||||
self.embeddings_roots = None
|
self.embeddings_roots = None
|
||||||
self.base_models_roots = self._init_checkpoint_paths()
|
self.base_models_roots = self._init_checkpoint_paths()
|
||||||
self.embeddings_roots = self._init_embedding_paths()
|
self.embeddings_roots = self._init_embedding_paths()
|
||||||
|
# Other-model roots (VAE, upscalers, text encoders, ...): flat deduped
|
||||||
|
# list plus a normalized root -> sub_type map and per-folder_paths-key
|
||||||
|
# roots for settings persistence.
|
||||||
|
self.other_roots: Optional[List[str]] = None
|
||||||
|
self.other_root_subtypes: Dict[str, str] = {}
|
||||||
|
self.other_folder_roots: Dict[str, List[str]] = {}
|
||||||
|
self.other_roots = self._init_other_paths()
|
||||||
# Extra paths (only for LoRA Manager, not shared with ComfyUI)
|
# Extra paths (only for LoRA Manager, not shared with ComfyUI)
|
||||||
self.extra_loras_roots: List[str] = []
|
self.extra_loras_roots: List[str] = []
|
||||||
self.extra_checkpoints_roots: List[str] = []
|
self.extra_checkpoints_roots: List[str] = []
|
||||||
@@ -336,6 +346,10 @@ class Config:
|
|||||||
"unet": list(self.unet_roots or []),
|
"unet": list(self.unet_roots or []),
|
||||||
"embeddings": list(self.embeddings_roots or []),
|
"embeddings": list(self.embeddings_roots or []),
|
||||||
}
|
}
|
||||||
|
# Persist the other-model roots under their original folder_paths
|
||||||
|
# keys so library switching round-trips them.
|
||||||
|
for key, roots in (self.other_folder_roots or {}).items():
|
||||||
|
target_folder_paths[key] = list(roots)
|
||||||
|
|
||||||
normalized_target_paths = _normalize_folder_paths_for_comparison(
|
normalized_target_paths = _normalize_folder_paths_for_comparison(
|
||||||
target_folder_paths
|
target_folder_paths
|
||||||
@@ -522,6 +536,7 @@ class Config:
|
|||||||
roots.extend(self.loras_roots or [])
|
roots.extend(self.loras_roots or [])
|
||||||
roots.extend(self.base_models_roots or [])
|
roots.extend(self.base_models_roots or [])
|
||||||
roots.extend(self.embeddings_roots or [])
|
roots.extend(self.embeddings_roots or [])
|
||||||
|
roots.extend(self.other_roots or [])
|
||||||
# Include extra paths for scanning symlinks
|
# Include extra paths for scanning symlinks
|
||||||
roots.extend(self.extra_loras_roots or [])
|
roots.extend(self.extra_loras_roots or [])
|
||||||
roots.extend(self.extra_checkpoints_roots or [])
|
roots.extend(self.extra_checkpoints_roots or [])
|
||||||
@@ -862,6 +877,8 @@ class Config:
|
|||||||
preview_roots.update(self._expand_preview_root(root))
|
preview_roots.update(self._expand_preview_root(root))
|
||||||
for root in self.embeddings_roots or []:
|
for root in self.embeddings_roots or []:
|
||||||
preview_roots.update(self._expand_preview_root(root))
|
preview_roots.update(self._expand_preview_root(root))
|
||||||
|
for root in self.other_roots or []:
|
||||||
|
preview_roots.update(self._expand_preview_root(root))
|
||||||
# Include extra paths for preview access
|
# Include extra paths for preview access
|
||||||
for root in self.extra_loras_roots or []:
|
for root in self.extra_loras_roots or []:
|
||||||
preview_roots.update(self._expand_preview_root(root))
|
preview_roots.update(self._expand_preview_root(root))
|
||||||
@@ -882,7 +899,7 @@ class Config:
|
|||||||
path for path in preview_roots if path.is_absolute()
|
path for path in preview_roots if path.is_absolute()
|
||||||
}
|
}
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Preview roots rebuilt: %d paths from %d lora roots (%d extra), %d checkpoint roots (%d extra), %d embedding roots (%d extra), %d symlink mappings",
|
"Preview roots rebuilt: %d paths from %d lora roots (%d extra), %d checkpoint roots (%d extra), %d embedding roots (%d extra), %d other roots, %d symlink mappings",
|
||||||
len(self._preview_root_paths),
|
len(self._preview_root_paths),
|
||||||
len(self.loras_roots or []),
|
len(self.loras_roots or []),
|
||||||
len(self.extra_loras_roots or []),
|
len(self.extra_loras_roots or []),
|
||||||
@@ -890,6 +907,7 @@ class Config:
|
|||||||
len(self.extra_checkpoints_roots or []),
|
len(self.extra_checkpoints_roots or []),
|
||||||
len(self.embeddings_roots or []),
|
len(self.embeddings_roots or []),
|
||||||
len(self.extra_embeddings_roots or []),
|
len(self.extra_embeddings_roots or []),
|
||||||
|
len(self.other_roots or []),
|
||||||
len(self._path_mappings),
|
len(self._path_mappings),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1128,6 +1146,155 @@ class Config:
|
|||||||
|
|
||||||
return unique_paths
|
return unique_paths
|
||||||
|
|
||||||
|
def _get_enabled_other_folder_keys(self) -> List[str]:
|
||||||
|
"""Return the OTHER_MODEL_FOLDER_SUBTYPES keys that are enabled.
|
||||||
|
|
||||||
|
Other Models management is opt-in: while ``enable_other_models`` is
|
||||||
|
off (the default) no other-model folder is scanned at all. When it is
|
||||||
|
on, only the folder keys of the enabled sub_types are scanned
|
||||||
|
(text_encoder merges ``text_encoders`` with the legacy ``clip`` key).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
from .services.settings_manager import get_settings_manager
|
||||||
|
|
||||||
|
enabled_sub_types = get_settings_manager().get_enabled_other_sub_types()
|
||||||
|
except Exception:
|
||||||
|
enabled_sub_types = []
|
||||||
|
if not enabled_sub_types:
|
||||||
|
return []
|
||||||
|
allowed = set(enabled_sub_types)
|
||||||
|
return [
|
||||||
|
key
|
||||||
|
for key, sub_type in OTHER_MODEL_FOLDER_SUBTYPES.items()
|
||||||
|
if sub_type in allowed
|
||||||
|
]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _collapse_legacy_folder_keys(keys: List[str]) -> List[str]:
|
||||||
|
"""Drop folder keys the host already normalizes onto another queried key.
|
||||||
|
|
||||||
|
ComfyUI's ``folder_paths`` rewrites legacy names before every access
|
||||||
|
(``clip`` -> ``text_encoders``, ``unet`` -> ``diffusion_models``), and
|
||||||
|
registers both legacy directories under the canonical key, so
|
||||||
|
``get_folder_paths("clip")`` returns exactly the same list as
|
||||||
|
``get_folder_paths("text_encoders")``. Querying both therefore reports
|
||||||
|
every text-encoder folder twice and trips the overlap guard with a
|
||||||
|
conflict the user cannot fix.
|
||||||
|
|
||||||
|
When the host exposes ``map_legacy`` the alias is provably redundant and
|
||||||
|
is skipped (an empty canonical list implies an empty alias list).
|
||||||
|
Without it - the standalone mock, whose keys are independent
|
||||||
|
``settings.json`` entries - every key is kept, because a ``clip``-only
|
||||||
|
configuration is then genuinely distinct.
|
||||||
|
"""
|
||||||
|
map_legacy = getattr(folder_paths, "map_legacy", None)
|
||||||
|
if not callable(map_legacy):
|
||||||
|
return list(keys)
|
||||||
|
|
||||||
|
queried = set(keys)
|
||||||
|
collapsed: List[str] = []
|
||||||
|
for key in keys:
|
||||||
|
try:
|
||||||
|
canonical = map_legacy(key)
|
||||||
|
except Exception:
|
||||||
|
canonical = key
|
||||||
|
if canonical != key and canonical in queried:
|
||||||
|
logger.debug(
|
||||||
|
"Skipping legacy folder key '%s'; the host resolves it to "
|
||||||
|
"'%s', which is queried as well.",
|
||||||
|
key,
|
||||||
|
canonical,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
collapsed.append(key)
|
||||||
|
return collapsed
|
||||||
|
|
||||||
|
def _prepare_other_paths(
|
||||||
|
self, folder_path_map: Mapping[str, Iterable[str]]
|
||||||
|
) -> Tuple[List[str], Dict[str, str], Dict[str, List[str]]]:
|
||||||
|
"""Prepare other-model paths from a folder_paths-key -> raw paths map.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (all_unique_roots, business_root -> sub_type map,
|
||||||
|
folder_paths key -> business roots). This method does NOT modify
|
||||||
|
instance variables - callers must set them.
|
||||||
|
"""
|
||||||
|
unique_paths: List[str] = []
|
||||||
|
sub_type_map: Dict[str, str] = {}
|
||||||
|
per_key_roots: Dict[str, List[str]] = {}
|
||||||
|
# real path -> (business path, sub_type) of the category that claimed it
|
||||||
|
seen_real_paths: Dict[str, Tuple[str, str]] = {}
|
||||||
|
|
||||||
|
# Cross-scanner overlap detection: warn when an "other" root is
|
||||||
|
# already covered by the checkpoints/unet or embeddings scanners.
|
||||||
|
# Kept (not dropped) on purpose - duplicate cards across pages are
|
||||||
|
# cosmetic, while dropping would silently unmanage the files.
|
||||||
|
covered_real_paths = {
|
||||||
|
os.path.normpath(os.path.realpath(path)).replace(os.sep, "/"): path
|
||||||
|
for path in [
|
||||||
|
*(self.base_models_roots or []),
|
||||||
|
*(self.embeddings_roots or []),
|
||||||
|
]
|
||||||
|
if isinstance(path, str) and path.strip() and os.path.exists(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
for key, sub_type in OTHER_MODEL_FOLDER_SUBTYPES.items():
|
||||||
|
raw_paths = folder_path_map.get(key)
|
||||||
|
if not raw_paths:
|
||||||
|
continue
|
||||||
|
path_map = self._dedupe_existing_paths(raw_paths)
|
||||||
|
key_roots: List[str] = []
|
||||||
|
for real_path, business_path in sorted(
|
||||||
|
path_map.items(), key=lambda item: item[1].lower()
|
||||||
|
):
|
||||||
|
seen = seen_real_paths.get(real_path)
|
||||||
|
if seen is not None:
|
||||||
|
seen_business_path, seen_sub_type = seen
|
||||||
|
if seen_sub_type == sub_type:
|
||||||
|
# Same category reached through a second folder_paths
|
||||||
|
# key (legacy alias, or a sub_type spanning two keys).
|
||||||
|
# Expected, so never a "fix your configuration" warning.
|
||||||
|
logger.debug(
|
||||||
|
"Ignoring duplicate folder '%s' for category '%s' "
|
||||||
|
"(already covered by '%s').",
|
||||||
|
business_path,
|
||||||
|
sub_type,
|
||||||
|
seen_business_path,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
"Detected the same folder '%s' under multiple other-model "
|
||||||
|
"categories ('%s' is already mapped as '%s'). Keeping the "
|
||||||
|
"first category; please fix your path configuration.",
|
||||||
|
business_path,
|
||||||
|
seen_business_path,
|
||||||
|
seen_sub_type,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
seen_real_paths[real_path] = (business_path, sub_type)
|
||||||
|
unique_paths.append(business_path)
|
||||||
|
key_roots.append(business_path)
|
||||||
|
sub_type_map[business_path] = sub_type
|
||||||
|
|
||||||
|
if real_path != business_path:
|
||||||
|
self.add_path_mapping(business_path, real_path)
|
||||||
|
|
||||||
|
covered_by = covered_real_paths.get(real_path)
|
||||||
|
if covered_by:
|
||||||
|
logger.warning(
|
||||||
|
"Detected an other-model root ('%s', category '%s') that "
|
||||||
|
"overlaps an existing checkpoints/embeddings root ('%s'). "
|
||||||
|
"The same files will appear on both pages; please review "
|
||||||
|
"your path configuration.",
|
||||||
|
business_path,
|
||||||
|
key,
|
||||||
|
covered_by,
|
||||||
|
)
|
||||||
|
if key_roots:
|
||||||
|
per_key_roots[key] = key_roots
|
||||||
|
|
||||||
|
return unique_paths, sub_type_map, per_key_roots
|
||||||
|
|
||||||
def _apply_library_paths(
|
def _apply_library_paths(
|
||||||
self,
|
self,
|
||||||
folder_paths: Mapping[str, Any],
|
folder_paths: Mapping[str, Any],
|
||||||
@@ -1151,6 +1318,16 @@ class Config:
|
|||||||
) = self._prepare_checkpoint_paths(checkpoint_paths, unet_paths)
|
) = self._prepare_checkpoint_paths(checkpoint_paths, unet_paths)
|
||||||
self.embeddings_roots = self._prepare_embedding_paths(embedding_paths)
|
self.embeddings_roots = self._prepare_embedding_paths(embedding_paths)
|
||||||
|
|
||||||
|
other_path_map = {
|
||||||
|
key: folder_paths.get(key, []) or []
|
||||||
|
for key in self._get_enabled_other_folder_keys()
|
||||||
|
}
|
||||||
|
(
|
||||||
|
self.other_roots,
|
||||||
|
self.other_root_subtypes,
|
||||||
|
self.other_folder_roots,
|
||||||
|
) = self._prepare_other_paths(other_path_map)
|
||||||
|
|
||||||
# Process extra paths (only for LoRA Manager, not shared with ComfyUI)
|
# Process extra paths (only for LoRA Manager, not shared with ComfyUI)
|
||||||
extra_paths = extra_folder_paths or {}
|
extra_paths = extra_folder_paths or {}
|
||||||
extra_lora_paths = extra_paths.get("loras", []) or []
|
extra_lora_paths = extra_paths.get("loras", []) or []
|
||||||
@@ -1267,6 +1444,104 @@ class Config:
|
|||||||
logger.warning(f"Error initializing embedding paths: {e}")
|
logger.warning(f"Error initializing embedding paths: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
def _init_other_paths(self) -> List[str]:
|
||||||
|
"""Initialize and validate other-model paths from ComfyUI settings.
|
||||||
|
|
||||||
|
Iterates the enabled OTHER_MODEL_FOLDER_SUBTYPES keys and pulls each
|
||||||
|
from ``folder_paths.get_folder_paths(key)`` (in standalone mode the
|
||||||
|
mock serves arbitrary keys from ``settings.json.folder_paths``).
|
||||||
|
Legacy aliases the host normalizes onto a canonical key (``clip`` ->
|
||||||
|
``text_encoders``) are collapsed first so the same folders are not
|
||||||
|
reported twice.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
folder_path_map: Dict[str, List[str]] = {}
|
||||||
|
for key in self._collapse_legacy_folder_keys(
|
||||||
|
self._get_enabled_other_folder_keys()
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
folder_path_map[key] = folder_paths.get_folder_paths(key)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("Error reading folder paths for '%s': %s", key, exc)
|
||||||
|
|
||||||
|
(
|
||||||
|
unique_paths,
|
||||||
|
self.other_root_subtypes,
|
||||||
|
self.other_folder_roots,
|
||||||
|
) = self._prepare_other_paths(folder_path_map)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"Found other model roots:"
|
||||||
|
+ ("\n - " + "\n - ".join(unique_paths) if unique_paths else "[]")
|
||||||
|
)
|
||||||
|
|
||||||
|
if not unique_paths:
|
||||||
|
logger.info("No valid other-model folders found in configuration")
|
||||||
|
return []
|
||||||
|
|
||||||
|
return unique_paths
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Error initializing other model paths: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
def refresh_other_roots(self) -> None:
|
||||||
|
"""Rebuild other-model roots after the management toggles changed.
|
||||||
|
|
||||||
|
Called when ``enable_other_models`` / ``enabled_other_sub_types`` are
|
||||||
|
updated so the scanner immediately reflects the new folder set without
|
||||||
|
a full application restart.
|
||||||
|
"""
|
||||||
|
self.other_roots = self._init_other_paths()
|
||||||
|
self._rebuild_preview_roots()
|
||||||
|
|
||||||
|
def get_other_models_availability(self) -> Dict[str, Any]:
|
||||||
|
"""Report the other-model folders the host can actually expose.
|
||||||
|
|
||||||
|
Independent of the opt-in ``enable_other_models`` toggle: this answers
|
||||||
|
"could Other Models management work here at all?". ComfyUI mode almost
|
||||||
|
always has these folder keys registered, while standalone mode only
|
||||||
|
knows the keys present in ``settings.json.folder_paths`` - so the UI
|
||||||
|
uses this to decide whether announcing the feature would be actionable.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
``{"available": bool, "sub_types": {sub_type: [existing roots]}}``.
|
||||||
|
A folder only counts when it exists on disk; an empty folder still
|
||||||
|
counts because CivitAI downloads can target it.
|
||||||
|
"""
|
||||||
|
sub_types: Dict[str, List[str]] = {}
|
||||||
|
try:
|
||||||
|
keys = self._collapse_legacy_folder_keys(
|
||||||
|
list(OTHER_MODEL_FOLDER_SUBTYPES.keys())
|
||||||
|
)
|
||||||
|
except Exception: # pragma: no cover - defensive
|
||||||
|
keys = list(OTHER_MODEL_FOLDER_SUBTYPES.keys())
|
||||||
|
|
||||||
|
for key in keys:
|
||||||
|
sub_type = OTHER_MODEL_FOLDER_SUBTYPES.get(key)
|
||||||
|
if not sub_type:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
raw_paths = folder_paths.get_folder_paths(key)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("Error probing folder paths for '%s': %s", key, exc)
|
||||||
|
continue
|
||||||
|
|
||||||
|
bucket = sub_types.setdefault(sub_type, [])
|
||||||
|
for root in sorted(
|
||||||
|
self._dedupe_existing_paths(raw_paths or []).values(),
|
||||||
|
key=lambda path: path.lower(),
|
||||||
|
):
|
||||||
|
if root not in bucket:
|
||||||
|
bucket.append(root)
|
||||||
|
|
||||||
|
available_sub_types = {
|
||||||
|
sub_type: roots for sub_type, roots in sub_types.items() if roots
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"available": bool(available_sub_types),
|
||||||
|
"sub_types": available_sub_types,
|
||||||
|
}
|
||||||
|
|
||||||
def get_preview_static_url(self, preview_path: str) -> str:
|
def get_preview_static_url(self, preview_path: str) -> str:
|
||||||
if not preview_path:
|
if not preview_path:
|
||||||
return ""
|
return ""
|
||||||
|
|||||||
+7
-1
@@ -219,6 +219,7 @@ class LoraManager:
|
|||||||
lora_scanner = await ServiceRegistry.get_lora_scanner()
|
lora_scanner = await ServiceRegistry.get_lora_scanner()
|
||||||
checkpoint_scanner = await ServiceRegistry.get_checkpoint_scanner()
|
checkpoint_scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||||
embedding_scanner = await ServiceRegistry.get_embedding_scanner()
|
embedding_scanner = await ServiceRegistry.get_embedding_scanner()
|
||||||
|
other_scanner = await ServiceRegistry.get_other_scanner()
|
||||||
|
|
||||||
# Initialize recipe scanner if needed
|
# Initialize recipe scanner if needed
|
||||||
recipe_scanner = await ServiceRegistry.get_recipe_scanner()
|
recipe_scanner = await ServiceRegistry.get_recipe_scanner()
|
||||||
@@ -236,6 +237,10 @@ class LoraManager:
|
|||||||
embedding_scanner.initialize_in_background(),
|
embedding_scanner.initialize_in_background(),
|
||||||
name="embedding_cache_init",
|
name="embedding_cache_init",
|
||||||
),
|
),
|
||||||
|
asyncio.create_task(
|
||||||
|
other_scanner.initialize_in_background(),
|
||||||
|
name="other_cache_init",
|
||||||
|
),
|
||||||
asyncio.create_task(
|
asyncio.create_task(
|
||||||
recipe_scanner.initialize_in_background(), name="recipe_cache_init"
|
recipe_scanner.initialize_in_background(), name="recipe_cache_init"
|
||||||
),
|
),
|
||||||
@@ -328,6 +333,7 @@ class LoraManager:
|
|||||||
all_roots.update(config.loras_roots)
|
all_roots.update(config.loras_roots)
|
||||||
all_roots.update(config.base_models_roots or [])
|
all_roots.update(config.base_models_roots or [])
|
||||||
all_roots.update(config.embeddings_roots or [])
|
all_roots.update(config.embeddings_roots or [])
|
||||||
|
all_roots.update(config.other_roots or [])
|
||||||
|
|
||||||
total_deleted = 0
|
total_deleted = 0
|
||||||
total_size_freed = 0
|
total_size_freed = 0
|
||||||
@@ -460,7 +466,7 @@ class LoraManager:
|
|||||||
# Cancel any in-flight scanner initialization tasks so thread-pool
|
# Cancel any in-flight scanner initialization tasks so thread-pool
|
||||||
# workers (e.g. _initialize_cache_sync) can break out of their loops
|
# workers (e.g. _initialize_cache_sync) can break out of their loops
|
||||||
# when the server shuts down (e.g. Ctrl+C on WSL).
|
# when the server shuts down (e.g. Ctrl+C on WSL).
|
||||||
for name in ("lora_scanner", "checkpoint_scanner", "embedding_scanner"):
|
for name in ("lora_scanner", "checkpoint_scanner", "embedding_scanner", "other_scanner"):
|
||||||
scanner = ServiceRegistry.get_service_sync(name)
|
scanner = ServiceRegistry.get_service_sync(name)
|
||||||
if scanner is not None and hasattr(scanner, "cancel_task"):
|
if scanner is not None and hasattr(scanner, "cancel_task"):
|
||||||
scanner.cancel_task()
|
scanner.cancel_task()
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ SCANNER_TYPE_MAP: dict[str, str] = {
|
|||||||
"get_lora_scanner": "lora",
|
"get_lora_scanner": "lora",
|
||||||
"get_checkpoint_scanner": "checkpoint",
|
"get_checkpoint_scanner": "checkpoint",
|
||||||
"get_embedding_scanner": "embedding",
|
"get_embedding_scanner": "embedding",
|
||||||
|
"get_other_scanner": "other",
|
||||||
}
|
}
|
||||||
|
|
||||||
SCANNER_GETTER_NAMES = tuple(SCANNER_TYPE_MAP.keys())
|
SCANNER_GETTER_NAMES = tuple(SCANNER_TYPE_MAP.keys())
|
||||||
@@ -80,8 +81,8 @@ async def _find_scanner_for_model(
|
|||||||
|
|
||||||
|
|
||||||
async def identify_model_type(model_path: str) -> str:
|
async def identify_model_type(model_path: str) -> str:
|
||||||
"""Determine the model type (``\"lora\"``, ``\"checkpoint\"``, or
|
"""Determine the model type (``\"lora\"``, ``\"checkpoint\"``,
|
||||||
``\"embedding\"``) for *model_path*.
|
``\"embedding\"``, or ``\"other\"``) for *model_path*.
|
||||||
|
|
||||||
Falls back to ``\"lora\"`` when unknown.
|
Falls back to ``\"lora\"`` when unknown.
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -46,6 +46,16 @@ async def api_json_error(
|
|||||||
if request.path.startswith("/api/lm/previews") and exc.status == 404:
|
if request.path.startswith("/api/lm/previews") and exc.status == 404:
|
||||||
logger_method = logger.debug
|
logger_method = logger.debug
|
||||||
|
|
||||||
|
# Download-progress 404 is routine too: in-memory tracking is removed
|
||||||
|
# once a download finishes/fails, so the extension's final polls 404.
|
||||||
|
# The extension relies on the 404 status itself (failure detection),
|
||||||
|
# so only the log level is lowered.
|
||||||
|
if (
|
||||||
|
request.path.startswith("/api/lm/download-progress/")
|
||||||
|
and exc.status == 404
|
||||||
|
):
|
||||||
|
logger_method = logger.debug
|
||||||
|
|
||||||
logger_method(
|
logger_method(
|
||||||
"API %s %s returned HTTP %d: %s",
|
"API %s %s returned HTTP %d: %s",
|
||||||
request.method,
|
request.method,
|
||||||
|
|||||||
@@ -13,6 +13,10 @@ class CheckpointLoaderLM:
|
|||||||
|
|
||||||
Loads checkpoints from both standard ComfyUI folders and LoRA Manager's
|
Loads checkpoints from both standard ComfyUI folders and LoRA Manager's
|
||||||
extra folder paths, providing a unified interface for checkpoint loading.
|
extra folder paths, providing a unified interface for checkpoint loading.
|
||||||
|
The ckpt_name combo supports ComfyUI's control_after_generate, letting
|
||||||
|
users pick a random checkpoint on every run; the base_model input narrows
|
||||||
|
the random pool through a front-end extension that filters the combo
|
||||||
|
options.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
NAME = "Checkpoint Loader (LoraManager)"
|
NAME = "Checkpoint Loader (LoraManager)"
|
||||||
@@ -22,11 +26,29 @@ class CheckpointLoaderLM:
|
|||||||
def INPUT_TYPES(cls):
|
def INPUT_TYPES(cls):
|
||||||
# Get list of checkpoint names from scanner (includes extra folder paths)
|
# Get list of checkpoint names from scanner (includes extra folder paths)
|
||||||
checkpoint_names = cls._get_checkpoint_names()
|
checkpoint_names = cls._get_checkpoint_names()
|
||||||
|
base_models = cls._get_available_base_models()
|
||||||
return {
|
return {
|
||||||
"required": {
|
"required": {
|
||||||
"ckpt_name": (
|
"ckpt_name": (
|
||||||
checkpoint_names,
|
checkpoint_names,
|
||||||
{"tooltip": "The name of the checkpoint (model) to load."},
|
{
|
||||||
|
"tooltip": (
|
||||||
|
"The name of the checkpoint (model) to load. Use "
|
||||||
|
"control_after_generate to pick a random model on "
|
||||||
|
"every run."
|
||||||
|
),
|
||||||
|
"control_after_generate": "fixed",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
"base_model": (
|
||||||
|
base_models,
|
||||||
|
{
|
||||||
|
"default": "Any",
|
||||||
|
"tooltip": (
|
||||||
|
"Restrict the random selection pool to this base "
|
||||||
|
"model. 'Any' uses the full pool."
|
||||||
|
),
|
||||||
|
},
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -93,15 +115,68 @@ class CheckpointLoaderLM:
|
|||||||
logger.error(f"Error getting checkpoint names: {e}")
|
logger.error(f"Error getting checkpoint names: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def load_checkpoint(self, ckpt_name: str) -> Tuple[Any, Any, Any]:
|
@classmethod
|
||||||
|
def _get_available_base_models(cls) -> List[str]:
|
||||||
|
"""Get distinct base_model values present among indexed checkpoints, for the random-selection filter."""
|
||||||
|
try:
|
||||||
|
from ..services.service_registry import ServiceRegistry
|
||||||
|
|
||||||
|
async def _get_base_models():
|
||||||
|
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||||
|
cache = await scanner.get_cached_data()
|
||||||
|
|
||||||
|
base_models = set()
|
||||||
|
for item in cache.raw_data:
|
||||||
|
if item.get("sub_type") != "checkpoint":
|
||||||
|
continue
|
||||||
|
base_model = item.get("base_model")
|
||||||
|
file_path = item.get("file_path", "")
|
||||||
|
if base_model and file_path and os.path.exists(file_path):
|
||||||
|
base_models.add(base_model)
|
||||||
|
|
||||||
|
return sorted(base_models)
|
||||||
|
|
||||||
|
return ["Any"] + cls._run_async(_get_base_models)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting available base models: {e}")
|
||||||
|
return ["Any"]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _run_async(coro_fn):
|
||||||
|
"""Run an async fetcher, handling the case where an event loop is already running."""
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
try:
|
||||||
|
asyncio.get_running_loop()
|
||||||
|
import concurrent.futures
|
||||||
|
|
||||||
|
def run_in_thread():
|
||||||
|
new_loop = asyncio.new_event_loop()
|
||||||
|
asyncio.set_event_loop(new_loop)
|
||||||
|
try:
|
||||||
|
return new_loop.run_until_complete(coro_fn())
|
||||||
|
finally:
|
||||||
|
new_loop.close()
|
||||||
|
|
||||||
|
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||||
|
future = executor.submit(run_in_thread)
|
||||||
|
return future.result()
|
||||||
|
except RuntimeError:
|
||||||
|
return asyncio.run(coro_fn())
|
||||||
|
|
||||||
|
def load_checkpoint(
|
||||||
|
self, ckpt_name: str, base_model: str = "Any"
|
||||||
|
) -> Tuple[Any, Any, Any]:
|
||||||
"""Load a checkpoint by name, supporting extra folder paths
|
"""Load a checkpoint by name, supporting extra folder paths
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
ckpt_name: The name of the checkpoint to load (relative path with extension)
|
ckpt_name: The name of the checkpoint to load (relative path with extension)
|
||||||
|
base_model: Only used by the front-end to filter the random pool
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (MODEL, CLIP, VAE)
|
Tuple of (MODEL, CLIP, VAE)
|
||||||
"""
|
"""
|
||||||
|
del base_model
|
||||||
# Get absolute path from cache using ComfyUI-style name
|
# Get absolute path from cache using ComfyUI-style name
|
||||||
ckpt_path, metadata = get_checkpoint_info_absolute(ckpt_name)
|
ckpt_path, metadata = get_checkpoint_info_absolute(ckpt_name)
|
||||||
|
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ class CreateHookLoraLM:
|
|||||||
),
|
),
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
|
"loras": ("LORAS", {}),
|
||||||
},
|
},
|
||||||
"optional": FlexibleOptionalInputType(any_type),
|
"optional": FlexibleOptionalInputType(any_type),
|
||||||
}
|
}
|
||||||
@@ -52,7 +53,7 @@ class CreateHookLoraLM:
|
|||||||
RETURN_NAMES = ("HOOKS", "trigger_words", "active_loras")
|
RETURN_NAMES = ("HOOKS", "trigger_words", "active_loras")
|
||||||
FUNCTION = "create_hook"
|
FUNCTION = "create_hook"
|
||||||
|
|
||||||
def create_hook(self, text: str, **kwargs):
|
def create_hook(self, text: str, loras, **kwargs):
|
||||||
"""Create a HookGroup from the selected LoRAs, chained with prev_hooks.
|
"""Create a HookGroup from the selected LoRAs, chained with prev_hooks.
|
||||||
|
|
||||||
Each active LoRA from the widget is loaded and wrapped in a WeightHook
|
Each active LoRA from the widget is loaded and wrapped in a WeightHook
|
||||||
@@ -73,7 +74,7 @@ class CreateHookLoraLM:
|
|||||||
all_trigger_words: list[str] = []
|
all_trigger_words: list[str] = []
|
||||||
active_loras: list[tuple[str, float, float]] = []
|
active_loras: list[tuple[str, float, float]] = []
|
||||||
|
|
||||||
for lora in get_loras_list(kwargs):
|
for lora in get_loras_list({"loras": loras}):
|
||||||
if not lora.get("active", False):
|
if not lora.get("active", False):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
|||||||
@@ -49,9 +49,9 @@ def _collect_stack_entries(lora_stack):
|
|||||||
return entries
|
return entries
|
||||||
|
|
||||||
|
|
||||||
def _collect_widget_entries(kwargs):
|
def _collect_widget_entries(loras):
|
||||||
entries = []
|
entries = []
|
||||||
for lora in get_loras_list(kwargs):
|
for lora in get_loras_list({"loras": loras}):
|
||||||
if not lora.get("active", False):
|
if not lora.get("active", False):
|
||||||
continue
|
continue
|
||||||
lora_name = apply_lora_syntax_format(lora["name"])
|
lora_name = apply_lora_syntax_format(lora["name"])
|
||||||
@@ -139,6 +139,7 @@ class LoraLoaderLM:
|
|||||||
"placeholder": "Search LoRAs to add...",
|
"placeholder": "Search LoRAs to add...",
|
||||||
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
||||||
}),
|
}),
|
||||||
|
"loras": ("LORAS", {}),
|
||||||
},
|
},
|
||||||
"optional": FlexibleOptionalInputType(any_type),
|
"optional": FlexibleOptionalInputType(any_type),
|
||||||
}
|
}
|
||||||
@@ -152,12 +153,12 @@ class LoraLoaderLM:
|
|||||||
RETURN_NAMES = ("MODEL", "CLIP", "trigger_words", "loaded_loras")
|
RETURN_NAMES = ("MODEL", "CLIP", "trigger_words", "loaded_loras")
|
||||||
FUNCTION = "load_loras"
|
FUNCTION = "load_loras"
|
||||||
|
|
||||||
def load_loras(self, model, text, **kwargs):
|
def load_loras(self, model, text, loras, **kwargs):
|
||||||
"""Loads multiple LoRAs based on the kwargs input and lora_stack."""
|
"""Loads multiple LoRAs based on the widget input and lora_stack."""
|
||||||
del text
|
del text
|
||||||
clip = kwargs.get("clip", None)
|
clip = kwargs.get("clip", None)
|
||||||
lora_entries = _collect_stack_entries(kwargs.get("lora_stack", None))
|
lora_entries = _collect_stack_entries(kwargs.get("lora_stack", None))
|
||||||
lora_entries.extend(_collect_widget_entries(kwargs))
|
lora_entries.extend(_collect_widget_entries(loras))
|
||||||
|
|
||||||
nunchaku_model_kind = detect_nunchaku_model_kind(model)
|
nunchaku_model_kind = detect_nunchaku_model_kind(model)
|
||||||
if nunchaku_model_kind == "flux":
|
if nunchaku_model_kind == "flux":
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ class LoraStackerLM:
|
|||||||
"placeholder": "Search LoRAs to add...",
|
"placeholder": "Search LoRAs to add...",
|
||||||
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
||||||
}),
|
}),
|
||||||
|
"loras": ("LORAS", {}),
|
||||||
},
|
},
|
||||||
"optional": FlexibleOptionalInputType(any_type),
|
"optional": FlexibleOptionalInputType(any_type),
|
||||||
}
|
}
|
||||||
@@ -31,8 +32,8 @@ class LoraStackerLM:
|
|||||||
RETURN_NAMES = ("LORA_STACK", "trigger_words", "active_loras")
|
RETURN_NAMES = ("LORA_STACK", "trigger_words", "active_loras")
|
||||||
FUNCTION = "stack_loras"
|
FUNCTION = "stack_loras"
|
||||||
|
|
||||||
def stack_loras(self, text, **kwargs):
|
def stack_loras(self, text, loras, **kwargs):
|
||||||
"""Stacks multiple LoRAs based on the kwargs input without loading them."""
|
"""Stacks multiple LoRAs based on the widget input without loading them."""
|
||||||
stack = []
|
stack = []
|
||||||
active_loras = []
|
active_loras = []
|
||||||
all_trigger_words = []
|
all_trigger_words = []
|
||||||
@@ -47,8 +48,8 @@ class LoraStackerLM:
|
|||||||
_, trigger_words = get_lora_info(lora_name)
|
_, trigger_words = get_lora_info(lora_name)
|
||||||
all_trigger_words.extend(trigger_words)
|
all_trigger_words.extend(trigger_words)
|
||||||
|
|
||||||
# Process loras from kwargs with support for both old and new formats
|
# Process loras from the widget with support for both old and new formats
|
||||||
loras_list = get_loras_list(kwargs)
|
loras_list = get_loras_list({"loras": loras})
|
||||||
for lora in loras_list:
|
for lora in loras_list:
|
||||||
if not lora.get('active', False):
|
if not lora.get('active', False):
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -1,214 +0,0 @@
|
|||||||
import logging
|
|
||||||
import os
|
|
||||||
import random
|
|
||||||
from typing import Any, List, Optional, Tuple
|
|
||||||
import comfy.sd # pyright: ignore[reportMissingImports]
|
|
||||||
import folder_paths # pyright: ignore[reportMissingImports]
|
|
||||||
from ..utils.utils import get_checkpoint_info_absolute, _format_model_name_for_comfyui
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class RandomCheckpointLoaderLM:
|
|
||||||
"""Checkpoint Loader that can randomly pick a checkpoint from the pool
|
|
||||||
|
|
||||||
Loads checkpoints from both standard ComfyUI folders and LoRA Manager's
|
|
||||||
extra folder paths. When select_at_random is enabled, ignores ckpt_name
|
|
||||||
and picks a random checkpoint (optionally filtered by base_model) on
|
|
||||||
every run.
|
|
||||||
"""
|
|
||||||
|
|
||||||
NAME = "Random Checkpoint Loader (LoraManager)"
|
|
||||||
CATEGORY = "Lora Manager/loaders"
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def INPUT_TYPES(cls):
|
|
||||||
# Get list of checkpoint names from scanner (includes extra folder paths)
|
|
||||||
checkpoint_names = cls._get_checkpoint_names()
|
|
||||||
base_models = cls._get_available_base_models()
|
|
||||||
return {
|
|
||||||
"required": {
|
|
||||||
"ckpt_name": (
|
|
||||||
checkpoint_names,
|
|
||||||
{"tooltip": "The name of the checkpoint (model) to load."},
|
|
||||||
),
|
|
||||||
"select_at_random": (
|
|
||||||
"BOOLEAN",
|
|
||||||
{
|
|
||||||
"default": False,
|
|
||||||
"tooltip": (
|
|
||||||
"Ignore ckpt_name and pick a random checkpoint from the "
|
|
||||||
"pool (optionally filtered by base_model) on every run."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
),
|
|
||||||
"base_model": (
|
|
||||||
base_models,
|
|
||||||
{
|
|
||||||
"default": "Any",
|
|
||||||
"tooltip": "Restrict random selection to this base model. 'Any' uses the full pool.",
|
|
||||||
},
|
|
||||||
),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
RETURN_TYPES = ("MODEL", "CLIP", "VAE", "STRING")
|
|
||||||
RETURN_NAMES = ("MODEL", "CLIP", "VAE", "model_name")
|
|
||||||
OUTPUT_TOOLTIPS = (
|
|
||||||
"The model used for denoising latents.",
|
|
||||||
"The CLIP model used for encoding text prompts.",
|
|
||||||
"The VAE model used for encoding and decoding images to and from latent space.",
|
|
||||||
"The name of the checkpoint that was loaded (useful when select_at_random is enabled).",
|
|
||||||
)
|
|
||||||
FUNCTION = "load_checkpoint"
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def IS_CHANGED(cls, ckpt_name, select_at_random=False, base_model="Any"):
|
|
||||||
# Force re-execution on every run while randomizing, since the widget
|
|
||||||
# values themselves don't change between queue runs.
|
|
||||||
if select_at_random:
|
|
||||||
return float("nan")
|
|
||||||
return ckpt_name
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _run_async(coro_fn):
|
|
||||||
"""Run an async fetcher, handling the case where an event loop is already running."""
|
|
||||||
import asyncio
|
|
||||||
|
|
||||||
try:
|
|
||||||
asyncio.get_running_loop()
|
|
||||||
import concurrent.futures
|
|
||||||
|
|
||||||
def run_in_thread():
|
|
||||||
new_loop = asyncio.new_event_loop()
|
|
||||||
asyncio.set_event_loop(new_loop)
|
|
||||||
try:
|
|
||||||
return new_loop.run_until_complete(coro_fn())
|
|
||||||
finally:
|
|
||||||
new_loop.close()
|
|
||||||
|
|
||||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|
||||||
future = executor.submit(run_in_thread)
|
|
||||||
return future.result()
|
|
||||||
except RuntimeError:
|
|
||||||
return asyncio.run(coro_fn())
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _get_checkpoint_names(cls, base_model: Optional[str] = None) -> List[str]:
|
|
||||||
"""Get list of checkpoint names from scanner cache in ComfyUI format (relative path with extension)
|
|
||||||
|
|
||||||
Args:
|
|
||||||
base_model: If given (and not "Any"), only include checkpoints matching this base model.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
from ..services.service_registry import ServiceRegistry
|
|
||||||
|
|
||||||
async def _get_names():
|
|
||||||
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
|
||||||
cache = await scanner.get_cached_data()
|
|
||||||
|
|
||||||
# Get all model roots for calculating relative paths
|
|
||||||
model_roots = scanner.get_model_roots()
|
|
||||||
|
|
||||||
# Filter only checkpoint type (not diffusion_model) and format names
|
|
||||||
names = []
|
|
||||||
for item in cache.raw_data:
|
|
||||||
if item.get("sub_type") != "checkpoint":
|
|
||||||
continue
|
|
||||||
if (
|
|
||||||
base_model
|
|
||||||
and base_model != "Any"
|
|
||||||
and item.get("base_model") != base_model
|
|
||||||
):
|
|
||||||
continue
|
|
||||||
file_path = item.get("file_path", "")
|
|
||||||
# Only offer models that still exist on disk so ComfyUI
|
|
||||||
# flags missing checkpoints at queue time via
|
|
||||||
# "value not in list" (the scanner cache can be stale).
|
|
||||||
if file_path and os.path.exists(file_path):
|
|
||||||
# Format using relative path with OS-native separator
|
|
||||||
formatted_name = _format_model_name_for_comfyui(
|
|
||||||
file_path, model_roots
|
|
||||||
)
|
|
||||||
if formatted_name:
|
|
||||||
names.append(formatted_name)
|
|
||||||
|
|
||||||
return sorted(names)
|
|
||||||
|
|
||||||
return cls._run_async(_get_names)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error getting checkpoint names: {e}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _get_available_base_models(cls) -> List[str]:
|
|
||||||
"""Get distinct base_model values present among indexed checkpoints, for the random-selection filter."""
|
|
||||||
try:
|
|
||||||
from ..services.service_registry import ServiceRegistry
|
|
||||||
|
|
||||||
async def _get_base_models():
|
|
||||||
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
|
||||||
cache = await scanner.get_cached_data()
|
|
||||||
|
|
||||||
base_models = set()
|
|
||||||
for item in cache.raw_data:
|
|
||||||
if item.get("sub_type") != "checkpoint":
|
|
||||||
continue
|
|
||||||
base_model = item.get("base_model")
|
|
||||||
file_path = item.get("file_path", "")
|
|
||||||
if base_model and file_path and os.path.exists(file_path):
|
|
||||||
base_models.add(base_model)
|
|
||||||
|
|
||||||
return sorted(base_models)
|
|
||||||
|
|
||||||
return ["Any"] + cls._run_async(_get_base_models)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error getting available base models: {e}")
|
|
||||||
return ["Any"]
|
|
||||||
|
|
||||||
def load_checkpoint(
|
|
||||||
self,
|
|
||||||
ckpt_name: str,
|
|
||||||
select_at_random: bool = False,
|
|
||||||
base_model: str = "Any",
|
|
||||||
) -> Tuple[Any, Any, Any, str]:
|
|
||||||
"""Load a checkpoint by name, supporting extra folder paths
|
|
||||||
|
|
||||||
Args:
|
|
||||||
ckpt_name: The name of the checkpoint to load (relative path with extension)
|
|
||||||
select_at_random: If True, ignore ckpt_name and pick randomly from the pool
|
|
||||||
base_model: Restricts random selection to this base model ("Any" = no filter)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Tuple of (MODEL, CLIP, VAE, model_name)
|
|
||||||
"""
|
|
||||||
if select_at_random:
|
|
||||||
pool = self._get_checkpoint_names(base_model)
|
|
||||||
if not pool:
|
|
||||||
raise FileNotFoundError(
|
|
||||||
f"No checkpoints found for base model '{base_model}'. "
|
|
||||||
"Pick a different base model or disable 'select_at_random'."
|
|
||||||
)
|
|
||||||
ckpt_name = random.choice(pool)
|
|
||||||
logger.info(
|
|
||||||
f"[RandomCheckpointLoaderLM] Randomly selected checkpoint: {ckpt_name}"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Get absolute path from cache using ComfyUI-style name
|
|
||||||
ckpt_path, metadata = get_checkpoint_info_absolute(ckpt_name)
|
|
||||||
|
|
||||||
if metadata is None:
|
|
||||||
raise FileNotFoundError(
|
|
||||||
f"Checkpoint '{ckpt_name}' not found in LoRA Manager cache. "
|
|
||||||
"Make sure the checkpoint is indexed and try again."
|
|
||||||
)
|
|
||||||
|
|
||||||
# Load regular checkpoint using ComfyUI's API
|
|
||||||
logger.info(f"Loading checkpoint from: {ckpt_path}")
|
|
||||||
out = comfy.sd.load_checkpoint_guess_config(
|
|
||||||
ckpt_path,
|
|
||||||
output_vae=True,
|
|
||||||
output_clip=True,
|
|
||||||
embedding_directory=folder_paths.get_folder_paths("embeddings"),
|
|
||||||
)
|
|
||||||
return out[:3] + (ckpt_name,)
|
|
||||||
@@ -1,326 +0,0 @@
|
|||||||
import logging
|
|
||||||
import os
|
|
||||||
import random
|
|
||||||
from typing import Any, List, Optional, Tuple
|
|
||||||
import comfy.sd # pyright: ignore[reportMissingImports]
|
|
||||||
from ..utils.utils import get_checkpoint_info_absolute, _format_model_name_for_comfyui
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
def _reload_gguf_unet(
|
|
||||||
unet_path: str, weight_dtype: str, disable_dynamic: bool = False
|
|
||||||
) -> object:
|
|
||||||
"""Reload a GGUF diffusion model from disk (cached_patcher_init factory).
|
|
||||||
|
|
||||||
Mirrors the GGUF branch of RandomUNETLoaderLM.load_unet so ModelPatcher
|
|
||||||
deepclone/dynamic machinery can rebuild GGUF models with the correct
|
|
||||||
GGMLOps. ``disable_dynamic`` is accepted for signature compatibility
|
|
||||||
with core ComfyUI loaders.
|
|
||||||
"""
|
|
||||||
loader = RandomUNETLoaderLM()
|
|
||||||
model, _unet_name = loader._load_gguf_unet(unet_path, unet_path, weight_dtype)
|
|
||||||
return model
|
|
||||||
|
|
||||||
|
|
||||||
class RandomUNETLoaderLM:
|
|
||||||
"""UNET Loader that can randomly pick a diffusion model from the pool
|
|
||||||
|
|
||||||
Loads diffusion models/UNets from both standard ComfyUI folders and LoRA
|
|
||||||
Manager's extra folder paths. Supports both regular diffusion models and
|
|
||||||
GGUF format models. When select_at_random is enabled, ignores unet_name
|
|
||||||
and picks a random diffusion model (optionally filtered by base_model)
|
|
||||||
on every run.
|
|
||||||
"""
|
|
||||||
|
|
||||||
NAME = "Random Unet Loader (LoraManager)"
|
|
||||||
CATEGORY = "Lora Manager/loaders"
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def INPUT_TYPES(cls):
|
|
||||||
# Get list of unet names from scanner (includes extra folder paths)
|
|
||||||
unet_names = cls._get_unet_names()
|
|
||||||
base_models = cls._get_available_base_models()
|
|
||||||
return {
|
|
||||||
"required": {
|
|
||||||
"unet_name": (
|
|
||||||
unet_names,
|
|
||||||
{"tooltip": "The name of the diffusion model to load."},
|
|
||||||
),
|
|
||||||
"weight_dtype": (
|
|
||||||
["default", "fp8_e4m3fn", "fp8_e4m3fn_fast", "fp8_e5m2"],
|
|
||||||
{"tooltip": "The dtype to use for the model weights."},
|
|
||||||
),
|
|
||||||
"select_at_random": (
|
|
||||||
"BOOLEAN",
|
|
||||||
{
|
|
||||||
"default": False,
|
|
||||||
"tooltip": (
|
|
||||||
"Ignore unet_name and pick a random diffusion model from "
|
|
||||||
"the pool (optionally filtered by base_model) on every run."
|
|
||||||
),
|
|
||||||
},
|
|
||||||
),
|
|
||||||
"base_model": (
|
|
||||||
base_models,
|
|
||||||
{
|
|
||||||
"default": "Any",
|
|
||||||
"tooltip": "Restrict random selection to this base model. 'Any' uses the full pool.",
|
|
||||||
},
|
|
||||||
),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
RETURN_TYPES = ("MODEL", "STRING")
|
|
||||||
RETURN_NAMES = ("MODEL", "model_name")
|
|
||||||
OUTPUT_TOOLTIPS = (
|
|
||||||
"The model used for denoising latents.",
|
|
||||||
"The name of the diffusion model that was loaded (useful when select_at_random is enabled).",
|
|
||||||
)
|
|
||||||
FUNCTION = "load_unet"
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def IS_CHANGED(
|
|
||||||
cls, unet_name, weight_dtype, select_at_random=False, base_model="Any"
|
|
||||||
):
|
|
||||||
# Force re-execution on every run while randomizing, since the widget
|
|
||||||
# values themselves don't change between queue runs.
|
|
||||||
if select_at_random:
|
|
||||||
return float("nan")
|
|
||||||
return unet_name
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _run_async(coro_fn):
|
|
||||||
"""Run an async fetcher, handling the case where an event loop is already running."""
|
|
||||||
import asyncio
|
|
||||||
|
|
||||||
try:
|
|
||||||
asyncio.get_running_loop()
|
|
||||||
import concurrent.futures
|
|
||||||
|
|
||||||
def run_in_thread():
|
|
||||||
new_loop = asyncio.new_event_loop()
|
|
||||||
asyncio.set_event_loop(new_loop)
|
|
||||||
try:
|
|
||||||
return new_loop.run_until_complete(coro_fn())
|
|
||||||
finally:
|
|
||||||
new_loop.close()
|
|
||||||
|
|
||||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|
||||||
future = executor.submit(run_in_thread)
|
|
||||||
return future.result()
|
|
||||||
except RuntimeError:
|
|
||||||
return asyncio.run(coro_fn())
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _get_unet_names(cls, base_model: Optional[str] = None) -> List[str]:
|
|
||||||
"""Get list of diffusion model names from scanner cache in ComfyUI format (relative path with extension)
|
|
||||||
|
|
||||||
Args:
|
|
||||||
base_model: If given (and not "Any"), only include models matching this base model.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
from ..services.service_registry import ServiceRegistry
|
|
||||||
|
|
||||||
async def _get_names():
|
|
||||||
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
|
||||||
cache = await scanner.get_cached_data()
|
|
||||||
|
|
||||||
# Get all model roots for calculating relative paths
|
|
||||||
model_roots = scanner.get_model_roots()
|
|
||||||
|
|
||||||
# Filter only diffusion_model type and format names
|
|
||||||
names = []
|
|
||||||
for item in cache.raw_data:
|
|
||||||
if item.get("sub_type") != "diffusion_model":
|
|
||||||
continue
|
|
||||||
if (
|
|
||||||
base_model
|
|
||||||
and base_model != "Any"
|
|
||||||
and item.get("base_model") != base_model
|
|
||||||
):
|
|
||||||
continue
|
|
||||||
file_path = item.get("file_path", "")
|
|
||||||
# Only offer models that still exist on disk so ComfyUI
|
|
||||||
# flags missing diffusion models at queue time via
|
|
||||||
# "value not in list" (the scanner cache can be stale).
|
|
||||||
if file_path and os.path.exists(file_path):
|
|
||||||
# Format using relative path with OS-native separator
|
|
||||||
formatted_name = _format_model_name_for_comfyui(
|
|
||||||
file_path, model_roots
|
|
||||||
)
|
|
||||||
if formatted_name:
|
|
||||||
names.append(formatted_name)
|
|
||||||
|
|
||||||
return sorted(names)
|
|
||||||
|
|
||||||
return cls._run_async(_get_names)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error getting unet names: {e}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _get_available_base_models(cls) -> List[str]:
|
|
||||||
"""Get distinct base_model values present among indexed diffusion models, for the random-selection filter."""
|
|
||||||
try:
|
|
||||||
from ..services.service_registry import ServiceRegistry
|
|
||||||
|
|
||||||
async def _get_base_models():
|
|
||||||
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
|
||||||
cache = await scanner.get_cached_data()
|
|
||||||
|
|
||||||
base_models = set()
|
|
||||||
for item in cache.raw_data:
|
|
||||||
if item.get("sub_type") != "diffusion_model":
|
|
||||||
continue
|
|
||||||
base_model = item.get("base_model")
|
|
||||||
file_path = item.get("file_path", "")
|
|
||||||
if base_model and file_path and os.path.exists(file_path):
|
|
||||||
base_models.add(base_model)
|
|
||||||
|
|
||||||
return sorted(base_models)
|
|
||||||
|
|
||||||
return ["Any"] + cls._run_async(_get_base_models)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error getting available base models: {e}")
|
|
||||||
return ["Any"]
|
|
||||||
|
|
||||||
def load_unet(
|
|
||||||
self,
|
|
||||||
unet_name: str,
|
|
||||||
weight_dtype: str,
|
|
||||||
select_at_random: bool = False,
|
|
||||||
base_model: str = "Any",
|
|
||||||
) -> Tuple[Any, ...]:
|
|
||||||
"""Load a diffusion model by name, supporting extra folder paths
|
|
||||||
|
|
||||||
Args:
|
|
||||||
unet_name: The name of the diffusion model to load (relative path with extension)
|
|
||||||
weight_dtype: The dtype to use for model weights
|
|
||||||
select_at_random: If True, ignore unet_name and pick randomly from the pool
|
|
||||||
base_model: Restricts random selection to this base model ("Any" = no filter)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Tuple of (MODEL, model_name)
|
|
||||||
"""
|
|
||||||
import torch
|
|
||||||
|
|
||||||
if select_at_random:
|
|
||||||
pool = self._get_unet_names(base_model)
|
|
||||||
if not pool:
|
|
||||||
raise FileNotFoundError(
|
|
||||||
f"No diffusion models found for base model '{base_model}'. "
|
|
||||||
"Pick a different base model or disable 'select_at_random'."
|
|
||||||
)
|
|
||||||
unet_name = random.choice(pool)
|
|
||||||
logger.info(
|
|
||||||
f"[RandomUNETLoaderLM] Randomly selected diffusion model: {unet_name}"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Get absolute path from cache using ComfyUI-style name
|
|
||||||
unet_path, metadata = get_checkpoint_info_absolute(unet_name)
|
|
||||||
|
|
||||||
if metadata is None:
|
|
||||||
raise FileNotFoundError(
|
|
||||||
f"Diffusion model '{unet_name}' not found in LoRA Manager cache. "
|
|
||||||
"Make sure the model is indexed and try again."
|
|
||||||
)
|
|
||||||
|
|
||||||
# Check if it's a GGUF model
|
|
||||||
if unet_path.endswith(".gguf"):
|
|
||||||
return self._load_gguf_unet(unet_path, unet_name, weight_dtype)
|
|
||||||
|
|
||||||
# Load regular diffusion model using ComfyUI's API
|
|
||||||
logger.info(f"Loading diffusion model from: {unet_path}")
|
|
||||||
|
|
||||||
# Build model options based on weight_dtype
|
|
||||||
model_options = {}
|
|
||||||
if weight_dtype == "fp8_e4m3fn":
|
|
||||||
model_options["dtype"] = torch.float8_e4m3fn
|
|
||||||
elif weight_dtype == "fp8_e4m3fn_fast":
|
|
||||||
model_options["dtype"] = torch.float8_e4m3fn
|
|
||||||
model_options["fp8_optimizations"] = True
|
|
||||||
elif weight_dtype == "fp8_e5m2":
|
|
||||||
model_options["dtype"] = torch.float8_e5m2
|
|
||||||
|
|
||||||
model = comfy.sd.load_diffusion_model(unet_path, model_options=model_options)
|
|
||||||
return (model, unet_name)
|
|
||||||
|
|
||||||
def _load_gguf_unet(
|
|
||||||
self, unet_path: str, unet_name: str, weight_dtype: str
|
|
||||||
) -> Tuple[Any, ...]:
|
|
||||||
"""Load a GGUF format diffusion model
|
|
||||||
|
|
||||||
Args:
|
|
||||||
unet_path: Absolute path to the GGUF file
|
|
||||||
unet_name: Name of the model for error messages
|
|
||||||
weight_dtype: The dtype to use for model weights
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Tuple of (MODEL, model_name)
|
|
||||||
"""
|
|
||||||
import torch
|
|
||||||
from .gguf_import_helper import get_gguf_modules
|
|
||||||
|
|
||||||
# Get ComfyUI-GGUF modules using helper (handles various import scenarios)
|
|
||||||
try:
|
|
||||||
loader_module, ops_module, nodes_module = get_gguf_modules()
|
|
||||||
gguf_sd_loader = getattr(loader_module, "gguf_sd_loader")
|
|
||||||
GGMLOps = getattr(ops_module, "GGMLOps")
|
|
||||||
GGUFModelPatcher = getattr(nodes_module, "GGUFModelPatcher")
|
|
||||||
except RuntimeError as e:
|
|
||||||
raise RuntimeError(f"Cannot load GGUF model '{unet_name}'. {str(e)}")
|
|
||||||
|
|
||||||
logger.info(f"Loading GGUF diffusion model from: {unet_path}")
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Load GGUF state dict
|
|
||||||
sd, extra = gguf_sd_loader(unet_path)
|
|
||||||
|
|
||||||
# Prepare kwargs for metadata if supported
|
|
||||||
kwargs = {}
|
|
||||||
import inspect
|
|
||||||
|
|
||||||
valid_params = inspect.signature(
|
|
||||||
comfy.sd.load_diffusion_model_state_dict
|
|
||||||
).parameters
|
|
||||||
if "metadata" in valid_params:
|
|
||||||
kwargs["metadata"] = extra.get("metadata", {})
|
|
||||||
|
|
||||||
# Setup custom operations with GGUF support
|
|
||||||
ops = GGMLOps()
|
|
||||||
|
|
||||||
# Handle weight_dtype for GGUF models
|
|
||||||
if weight_dtype in ("default", None):
|
|
||||||
ops.Linear.dequant_dtype = None
|
|
||||||
elif weight_dtype in ["target"]:
|
|
||||||
ops.Linear.dequant_dtype = weight_dtype
|
|
||||||
else:
|
|
||||||
ops.Linear.dequant_dtype = getattr(torch, weight_dtype, None)
|
|
||||||
|
|
||||||
# Load the model
|
|
||||||
model = comfy.sd.load_diffusion_model_state_dict(
|
|
||||||
sd, model_options={"custom_operations": ops}, **kwargs
|
|
||||||
)
|
|
||||||
|
|
||||||
if model is None:
|
|
||||||
raise RuntimeError(
|
|
||||||
f"Could not detect model type for GGUF diffusion model: {unet_path}"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Wrap with GGUFModelPatcher
|
|
||||||
model = GGUFModelPatcher.clone(model)
|
|
||||||
|
|
||||||
# Register a reload factory so the MODEL carries its source path
|
|
||||||
# (cached_patcher_init) like core ComfyUI loaders do — required
|
|
||||||
# for model-name extraction downstream and for ModelPatcher
|
|
||||||
# deepclone/dynamic machinery.
|
|
||||||
model.cached_patcher_init = (_reload_gguf_unet, (unet_path, weight_dtype))
|
|
||||||
|
|
||||||
return (model, unet_name)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error loading GGUF diffusion model '{unet_name}': {e}")
|
|
||||||
raise RuntimeError(
|
|
||||||
f"Failed to load GGUF diffusion model '{unet_name}': {str(e)}"
|
|
||||||
)
|
|
||||||
@@ -778,6 +778,14 @@ class SaveImageLM:
|
|||||||
if checkpoint_entry:
|
if checkpoint_entry:
|
||||||
recipe_data["checkpoint"] = checkpoint_entry
|
recipe_data["checkpoint"] = checkpoint_entry
|
||||||
|
|
||||||
|
# The recipe image is the WebP produced above from the output file;
|
||||||
|
# reuse the same metadata extraction to record workflow presence.
|
||||||
|
try:
|
||||||
|
metadata = ExifUtils._load_structured_metadata(image_path)
|
||||||
|
recipe_data["has_workflow"] = bool(metadata.get("workflow"))
|
||||||
|
except Exception:
|
||||||
|
recipe_data["has_workflow"] = False
|
||||||
|
|
||||||
json_path = os.path.normpath(
|
json_path = os.path.normpath(
|
||||||
os.path.join(recipes_dir, f"{recipe_id}.recipe.json")
|
os.path.join(recipes_dir, f"{recipe_id}.recipe.json")
|
||||||
)
|
)
|
||||||
|
|||||||
+77
-2
@@ -28,6 +28,10 @@ class UNETLoaderLM:
|
|||||||
Loads diffusion models/UNets from both standard ComfyUI folders and LoRA Manager's
|
Loads diffusion models/UNets from both standard ComfyUI folders and LoRA Manager's
|
||||||
extra folder paths, providing a unified interface for UNET loading.
|
extra folder paths, providing a unified interface for UNET loading.
|
||||||
Supports both regular diffusion models and GGUF format models.
|
Supports both regular diffusion models and GGUF format models.
|
||||||
|
The unet_name combo supports ComfyUI's control_after_generate, letting
|
||||||
|
users pick a random diffusion model on every run; the base_model input
|
||||||
|
narrows the random pool through a front-end extension that filters the
|
||||||
|
combo options.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
NAME = "Unet Loader (LoraManager)"
|
NAME = "Unet Loader (LoraManager)"
|
||||||
@@ -37,16 +41,34 @@ class UNETLoaderLM:
|
|||||||
def INPUT_TYPES(cls):
|
def INPUT_TYPES(cls):
|
||||||
# Get list of unet names from scanner (includes extra folder paths)
|
# Get list of unet names from scanner (includes extra folder paths)
|
||||||
unet_names = cls._get_unet_names()
|
unet_names = cls._get_unet_names()
|
||||||
|
base_models = cls._get_available_base_models()
|
||||||
return {
|
return {
|
||||||
"required": {
|
"required": {
|
||||||
"unet_name": (
|
"unet_name": (
|
||||||
unet_names,
|
unet_names,
|
||||||
{"tooltip": "The name of the diffusion model to load."},
|
{
|
||||||
|
"tooltip": (
|
||||||
|
"The name of the diffusion model to load. Use "
|
||||||
|
"control_after_generate to pick a random model on "
|
||||||
|
"every run."
|
||||||
|
),
|
||||||
|
"control_after_generate": "fixed",
|
||||||
|
},
|
||||||
),
|
),
|
||||||
"weight_dtype": (
|
"weight_dtype": (
|
||||||
["default", "fp8_e4m3fn", "fp8_e4m3fn_fast", "fp8_e5m2"],
|
["default", "fp8_e4m3fn", "fp8_e4m3fn_fast", "fp8_e5m2"],
|
||||||
{"tooltip": "The dtype to use for the model weights."},
|
{"tooltip": "The dtype to use for the model weights."},
|
||||||
),
|
),
|
||||||
|
"base_model": (
|
||||||
|
base_models,
|
||||||
|
{
|
||||||
|
"default": "Any",
|
||||||
|
"tooltip": (
|
||||||
|
"Restrict the random selection pool to this base "
|
||||||
|
"model. 'Any' uses the full pool."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -108,16 +130,69 @@ class UNETLoaderLM:
|
|||||||
logger.error(f"Error getting unet names: {e}")
|
logger.error(f"Error getting unet names: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def load_unet(self, unet_name: str, weight_dtype: str) -> Tuple[Any, ...]:
|
@classmethod
|
||||||
|
def _get_available_base_models(cls) -> List[str]:
|
||||||
|
"""Get distinct base_model values present among indexed diffusion models, for the random-selection filter."""
|
||||||
|
try:
|
||||||
|
from ..services.service_registry import ServiceRegistry
|
||||||
|
|
||||||
|
async def _get_base_models():
|
||||||
|
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||||
|
cache = await scanner.get_cached_data()
|
||||||
|
|
||||||
|
base_models = set()
|
||||||
|
for item in cache.raw_data:
|
||||||
|
if item.get("sub_type") != "diffusion_model":
|
||||||
|
continue
|
||||||
|
base_model = item.get("base_model")
|
||||||
|
file_path = item.get("file_path", "")
|
||||||
|
if base_model and file_path and os.path.exists(file_path):
|
||||||
|
base_models.add(base_model)
|
||||||
|
|
||||||
|
return sorted(base_models)
|
||||||
|
|
||||||
|
return ["Any"] + cls._run_async(_get_base_models)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting available base models: {e}")
|
||||||
|
return ["Any"]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _run_async(coro_fn):
|
||||||
|
"""Run an async fetcher, handling the case where an event loop is already running."""
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
try:
|
||||||
|
asyncio.get_running_loop()
|
||||||
|
import concurrent.futures
|
||||||
|
|
||||||
|
def run_in_thread():
|
||||||
|
new_loop = asyncio.new_event_loop()
|
||||||
|
asyncio.set_event_loop(new_loop)
|
||||||
|
try:
|
||||||
|
return new_loop.run_until_complete(coro_fn())
|
||||||
|
finally:
|
||||||
|
new_loop.close()
|
||||||
|
|
||||||
|
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||||
|
future = executor.submit(run_in_thread)
|
||||||
|
return future.result()
|
||||||
|
except RuntimeError:
|
||||||
|
return asyncio.run(coro_fn())
|
||||||
|
|
||||||
|
def load_unet(
|
||||||
|
self, unet_name: str, weight_dtype: str, base_model: str = "Any"
|
||||||
|
) -> Tuple[Any, ...]:
|
||||||
"""Load a diffusion model by name, supporting extra folder paths
|
"""Load a diffusion model by name, supporting extra folder paths
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
unet_name: The name of the diffusion model to load (relative path with extension)
|
unet_name: The name of the diffusion model to load (relative path with extension)
|
||||||
weight_dtype: The dtype to use for model weights
|
weight_dtype: The dtype to use for model weights
|
||||||
|
base_model: Only used by the front-end to filter the random pool
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (MODEL,)
|
Tuple of (MODEL,)
|
||||||
"""
|
"""
|
||||||
|
del base_model
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
# Get absolute path from cache using ComfyUI-style name
|
# Get absolute path from cache using ComfyUI-style name
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ class WanVideoLoraSelectLM:
|
|||||||
"placeholder": "Search LoRAs to add...",
|
"placeholder": "Search LoRAs to add...",
|
||||||
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
"tooltip": "Format: <lora:lora_name:strength> separated by spaces or punctuation",
|
||||||
}),
|
}),
|
||||||
|
"loras": ("LORAS", {}),
|
||||||
},
|
},
|
||||||
"optional": FlexibleOptionalInputType(any_type),
|
"optional": FlexibleOptionalInputType(any_type),
|
||||||
}
|
}
|
||||||
@@ -44,7 +45,7 @@ class WanVideoLoraSelectLM:
|
|||||||
RETURN_NAMES = ("lora", "trigger_words", "active_loras")
|
RETURN_NAMES = ("lora", "trigger_words", "active_loras")
|
||||||
FUNCTION = "process_loras"
|
FUNCTION = "process_loras"
|
||||||
|
|
||||||
def process_loras(self, text, low_mem_load=False, merge_loras=True, **kwargs):
|
def process_loras(self, text, loras, low_mem_load=False, merge_loras=True, **kwargs):
|
||||||
loras_list = []
|
loras_list = []
|
||||||
all_trigger_words = []
|
all_trigger_words = []
|
||||||
active_loras = []
|
active_loras = []
|
||||||
@@ -62,8 +63,8 @@ class WanVideoLoraSelectLM:
|
|||||||
selected_blocks = blocks.get("selected_blocks", {})
|
selected_blocks = blocks.get("selected_blocks", {})
|
||||||
layer_filter = blocks.get("layer_filter", "")
|
layer_filter = blocks.get("layer_filter", "")
|
||||||
|
|
||||||
# Process loras from kwargs with support for both old and new formats
|
# Process loras from the widget with support for both old and new formats
|
||||||
loras_from_widget = get_loras_list(kwargs)
|
loras_from_widget = get_loras_list({"loras": loras})
|
||||||
for lora in loras_from_widget:
|
for lora in loras_from_widget:
|
||||||
if not lora.get('active', False):
|
if not lora.get('active', False):
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -41,6 +41,40 @@ class RecipeMetadataParser(ABC):
|
|||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def populate_lora_from_local(lora_entry: Dict[str, Any], local_lora: Dict[str, Any], base_model_counts=None) -> Dict[str, Any]:
|
||||||
|
"""Populate a recipe LoRA entry from the local scanner cache."""
|
||||||
|
local_path = local_lora.get('file_path') or ''
|
||||||
|
file_name = local_lora.get('file_name') or os.path.splitext(os.path.basename(local_path))[0]
|
||||||
|
base_model = local_lora.get('base_model') or ''
|
||||||
|
|
||||||
|
lora_entry['name'] = local_lora.get('model_name') or file_name or lora_entry.get('name', '')
|
||||||
|
lora_entry['file_name'] = file_name
|
||||||
|
lora_entry['hash'] = (local_lora.get('sha256') or lora_entry.get('hash') or '').lower()
|
||||||
|
lora_entry['localPath'] = local_path or None
|
||||||
|
lora_entry['size'] = local_lora.get('size', 0) or 0
|
||||||
|
lora_entry['baseModel'] = base_model
|
||||||
|
lora_entry['existsLocally'] = True
|
||||||
|
lora_entry['isDeleted'] = False
|
||||||
|
|
||||||
|
preview_url = local_lora.get('preview_url')
|
||||||
|
if preview_url:
|
||||||
|
lora_entry['thumbnailUrl'] = config.get_preview_static_url(preview_url)
|
||||||
|
|
||||||
|
civitai_info = local_lora.get('civitai') or {}
|
||||||
|
if isinstance(civitai_info, dict):
|
||||||
|
if civitai_info.get('id') is not None:
|
||||||
|
lora_entry['id'] = civitai_info['id']
|
||||||
|
if civitai_info.get('modelId') is not None:
|
||||||
|
lora_entry['modelId'] = civitai_info['modelId']
|
||||||
|
if civitai_info.get('name'):
|
||||||
|
lora_entry['version'] = civitai_info['name']
|
||||||
|
|
||||||
|
if base_model_counts is not None and base_model:
|
||||||
|
base_model_counts[base_model] = base_model_counts.get(base_model, 0) + 1
|
||||||
|
|
||||||
|
return lora_entry
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def populate_lora_from_civitai(lora_entry: Dict[str, Any], civitai_info_tuple: Tuple[Dict[str, Any] | None, str | None] | Dict[str, Any],
|
async def populate_lora_from_civitai(lora_entry: Dict[str, Any], civitai_info_tuple: Tuple[Dict[str, Any] | None, str | None] | Dict[str, Any],
|
||||||
recipe_scanner=None, base_model_counts=None, hash_value=None) -> Optional[Dict[str, Any]]:
|
recipe_scanner=None, base_model_counts=None, hash_value=None) -> Optional[Dict[str, Any]]:
|
||||||
|
|||||||
+227
-68
@@ -8,6 +8,7 @@ from typing import Dict, Any
|
|||||||
from ..base import RecipeMetadataParser
|
from ..base import RecipeMetadataParser
|
||||||
from ..constants import GEN_PARAM_KEYS
|
from ..constants import GEN_PARAM_KEYS
|
||||||
from ...services.metadata_service import get_default_metadata_provider
|
from ...services.metadata_service import get_default_metadata_provider
|
||||||
|
from ...utils.constants import is_empty_placeholder_hash
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -146,15 +147,13 @@ class AutomaticMetadataParser(RecipeMetadataParser):
|
|||||||
# Initialize hashes dict if it doesn't exist
|
# Initialize hashes dict if it doesn't exist
|
||||||
if "hashes" not in metadata:
|
if "hashes" not in metadata:
|
||||||
metadata["hashes"] = {}
|
metadata["hashes"] = {}
|
||||||
# Add as lora type in the same format as
|
# Lora hashes carries the 12-char AutoV3
|
||||||
# regular hashes. Only override an
|
# hash (resolvable on CivitAI and the local
|
||||||
# existing entry if its value is empty
|
# autov3 index); the Hashes JSON value is
|
||||||
# (Lora hashes is the more reliable
|
# only the 10-char AutoV2 prefix, so on
|
||||||
# source when Hashes JSON has blanks).
|
# conflict the Lora hashes value wins.
|
||||||
key = f"lora:{lora_name}"
|
key = f"lora:{lora_name}"
|
||||||
existing = metadata["hashes"].get(key, "")
|
metadata["hashes"][key] = lora_hash
|
||||||
if not existing:
|
|
||||||
metadata["hashes"][key] = lora_hash
|
|
||||||
|
|
||||||
# Remove lora hashes from params section
|
# Remove lora hashes from params section
|
||||||
params_section = params_section.replace(lora_hashes_match.group(0), '')
|
params_section = params_section.replace(lora_hashes_match.group(0), '')
|
||||||
@@ -362,68 +361,228 @@ class AutomaticMetadataParser(RecipeMetadataParser):
|
|||||||
|
|
||||||
checkpoint = checkpoint_entry
|
checkpoint = checkpoint_entry
|
||||||
|
|
||||||
# If no LoRAs from Civitai resources or to supplement, extract from metadata["hashes"]
|
def normalize_lora_name(name, basename=False):
|
||||||
if not loras or len(loras) == 0:
|
normalized = str(name or '').replace('\\', '/')
|
||||||
# Extract lora weights from extranet tags in prompt (for later use)
|
if normalized.casefold().endswith('.safetensors'):
|
||||||
lora_weights = {}
|
normalized = normalized[:-12]
|
||||||
lora_matches = re.findall(self.EXTRANETS_REGEX, prompt)
|
if basename:
|
||||||
for lora_type, lora_name, lora_weight in lora_matches:
|
normalized = normalized.rsplit('/', 1)[-1]
|
||||||
key = f"{lora_type}:{lora_name}"
|
return normalized.casefold()
|
||||||
lora_weights[key] = round(float(lora_weight), 2)
|
|
||||||
|
def get_version_id(lora):
|
||||||
# Use hashes from metadata as the primary source
|
version_id = lora.get('id')
|
||||||
if metadata.get("hashes"):
|
if version_id in (None, '', 0, '0'):
|
||||||
for hash_key, lora_hash in metadata.get("hashes", {}).items():
|
version_id = lora.get('modelVersionId')
|
||||||
# Only process lora or hypernet types
|
if version_id in (None, '', 0, '0'):
|
||||||
if not hash_key.startswith(("lora:", "hypernet:")):
|
return None
|
||||||
|
return str(version_id)
|
||||||
|
|
||||||
|
prompt_loras = {}
|
||||||
|
for match in re.findall(self.EXTRANETS_REGEX, prompt):
|
||||||
|
lora_type, lora_name, _ = match
|
||||||
|
prompt_loras[(lora_type, normalize_lora_name(lora_name))] = match
|
||||||
|
|
||||||
|
prompt_by_basename = {}
|
||||||
|
for lora_type, lora_name, lora_weight in prompt_loras.values():
|
||||||
|
key = (lora_type, normalize_lora_name(lora_name, True))
|
||||||
|
prompt_by_basename.setdefault(key, []).append((lora_name, round(float(lora_weight), 2)))
|
||||||
|
|
||||||
|
hash_basenames = {
|
||||||
|
(hash_key.split(':', 1)[0], normalize_lora_name(hash_key.split(':', 1)[1], True))
|
||||||
|
for hash_key, hash_value in metadata.get("hashes", {}).items()
|
||||||
|
if hash_value and hash_key.startswith(("lora:", "hypernet:"))
|
||||||
|
}
|
||||||
|
recipe_base_model = checkpoint.get("baseModel") if checkpoint else None
|
||||||
|
if not recipe_base_model and len(base_model_counts) == 1:
|
||||||
|
recipe_base_model = next(iter(base_model_counts))
|
||||||
|
|
||||||
|
resource_lora_count = len(loras)
|
||||||
|
|
||||||
|
def make_lora_entry(lora_type, lora_name, weight, lora_hash=''):
|
||||||
|
return {
|
||||||
|
'name': lora_name,
|
||||||
|
'type': lora_type,
|
||||||
|
'weight': weight,
|
||||||
|
'hash': lora_hash,
|
||||||
|
'existsLocally': False,
|
||||||
|
'localPath': None,
|
||||||
|
'file_name': lora_name,
|
||||||
|
'thumbnailUrl': '/loras_static/images/no-preview.png',
|
||||||
|
'baseModel': '',
|
||||||
|
'size': 0,
|
||||||
|
'downloadUrl': '',
|
||||||
|
'isDeleted': False
|
||||||
|
}
|
||||||
|
|
||||||
|
def merge_or_append_civitai(civitai_entry, preserve_existing_weight=False):
|
||||||
|
civitai_id = get_version_id(civitai_entry)
|
||||||
|
civitai_hash = (civitai_entry.get('hash') or '').lower()
|
||||||
|
for index, existing in enumerate(loras):
|
||||||
|
existing_id = get_version_id(existing)
|
||||||
|
existing_hash = (existing.get('hash') or '').lower()
|
||||||
|
if not (
|
||||||
|
(civitai_id and existing_id == civitai_id)
|
||||||
|
or (civitai_hash and existing_hash == civitai_hash)
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
|
||||||
|
if preserve_existing_weight:
|
||||||
|
civitai_entry['weight'] = existing.get('weight', civitai_entry['weight'])
|
||||||
|
existing_base = existing.get('baseModel')
|
||||||
|
if not civitai_entry.get('baseModel'):
|
||||||
|
civitai_entry['baseModel'] = existing_base or ''
|
||||||
|
elif existing_base:
|
||||||
|
remaining = base_model_counts.get(existing_base, 0) - 1
|
||||||
|
if remaining > 0:
|
||||||
|
base_model_counts[existing_base] = remaining
|
||||||
|
else:
|
||||||
|
base_model_counts.pop(existing_base, None)
|
||||||
|
loras[index] = civitai_entry
|
||||||
|
return
|
||||||
|
loras.append(civitai_entry)
|
||||||
|
|
||||||
|
def merge_or_append_local(local_entry):
|
||||||
|
local_id = get_version_id(local_entry)
|
||||||
|
local_hash = (local_entry.get('hash') or '').lower()
|
||||||
|
for existing in loras:
|
||||||
|
existing_id = get_version_id(existing)
|
||||||
|
existing_hash = (existing.get('hash') or '').lower()
|
||||||
|
if not (
|
||||||
|
(local_id and existing_id == local_id)
|
||||||
|
or (local_hash and existing_hash == local_hash)
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
|
||||||
|
existing['weight'] = local_entry['weight']
|
||||||
|
existing['hash'] = local_entry['hash']
|
||||||
|
existing['file_name'] = local_entry['file_name']
|
||||||
|
existing['existsLocally'] = True
|
||||||
|
existing['localPath'] = local_entry['localPath']
|
||||||
|
existing['size'] = local_entry['size']
|
||||||
|
existing['isDeleted'] = False
|
||||||
|
if not existing.get('modelId') and local_entry.get('modelId'):
|
||||||
|
existing['modelId'] = local_entry['modelId']
|
||||||
|
if not existing.get('baseModel') and local_entry.get('baseModel'):
|
||||||
|
existing['baseModel'] = local_entry['baseModel']
|
||||||
|
base_model_counts[local_entry['baseModel']] = base_model_counts.get(local_entry['baseModel'], 0) + 1
|
||||||
|
thumbnail_url = local_entry.get('thumbnailUrl')
|
||||||
|
if thumbnail_url and not thumbnail_url.endswith('/images/no-preview.png'):
|
||||||
|
existing['thumbnailUrl'] = thumbnail_url
|
||||||
|
return
|
||||||
|
|
||||||
|
if local_entry.get('baseModel'):
|
||||||
|
base_model = local_entry['baseModel']
|
||||||
|
base_model_counts[base_model] = base_model_counts.get(base_model, 0) + 1
|
||||||
|
loras.append(local_entry)
|
||||||
|
|
||||||
|
resolved_prompt_basenames = set()
|
||||||
|
queried_local_basenames = set()
|
||||||
|
for lora_type, lora_name, lora_weight in prompt_loras.values():
|
||||||
|
weight = round(float(lora_weight), 2)
|
||||||
|
basename_key = (lora_type, normalize_lora_name(lora_name, True))
|
||||||
|
matching_resources = [
|
||||||
|
lora
|
||||||
|
for lora in loras[:resource_lora_count]
|
||||||
|
if lora.get('file_name')
|
||||||
|
and normalize_lora_name(lora['file_name'], True) == basename_key[1]
|
||||||
|
and (
|
||||||
|
(lora_type == 'hypernet' and str(lora.get('type', '')).casefold() in ('hypernet', 'hypernetwork'))
|
||||||
|
or (lora_type == 'lora' and str(lora.get('type', '')).casefold() not in ('hypernet', 'hypernetwork'))
|
||||||
|
)
|
||||||
|
]
|
||||||
|
if len(prompt_by_basename[basename_key]) == 1 and len(matching_resources) == 1:
|
||||||
|
matching_resources[0]['weight'] = weight
|
||||||
|
if basename_key not in hash_basenames:
|
||||||
|
resolved_prompt_basenames.add(basename_key)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if basename_key in hash_basenames:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not recipe_scanner or lora_type != 'lora':
|
||||||
|
continue
|
||||||
|
queried_local_basenames.add(basename_key)
|
||||||
|
local_lora = await recipe_scanner.get_local_lora(lora_name, recipe_base_model)
|
||||||
|
if not local_lora:
|
||||||
|
continue
|
||||||
|
|
||||||
|
local_entry = self.populate_lora_from_local(
|
||||||
|
make_lora_entry(lora_type, lora_name, weight),
|
||||||
|
local_lora,
|
||||||
|
)
|
||||||
|
merge_or_append_local(local_entry)
|
||||||
|
resolved_prompt_basenames.add(basename_key)
|
||||||
|
|
||||||
|
for hash_key, lora_hash in metadata.get("hashes", {}).items():
|
||||||
|
if not hash_key.startswith(("lora:", "hypernet:")):
|
||||||
|
continue
|
||||||
|
lora_type, lora_name = hash_key.split(':', 1)
|
||||||
|
basename_key = (lora_type, normalize_lora_name(lora_name, True))
|
||||||
|
if basename_key in resolved_prompt_basenames:
|
||||||
|
continue
|
||||||
|
|
||||||
|
prompt_entries = prompt_by_basename.get(basename_key, [])
|
||||||
|
weight = prompt_entries[0][1] if len(prompt_entries) == 1 else 1.0
|
||||||
|
lora_entry = make_lora_entry(lora_type, lora_name, weight, lora_hash)
|
||||||
|
|
||||||
|
if is_empty_placeholder_hash(lora_hash):
|
||||||
|
# The empty-hash placeholder (SHA256 of an empty byte
|
||||||
|
# string) is not a real hash: never look it up in the
|
||||||
|
# local hash index or on CivitAI. Match by filename;
|
||||||
|
# otherwise keep the item as unresolved (no hash, flagged
|
||||||
|
# hashInvalid so the UI shows the unresolvable-hash state
|
||||||
|
# and offers reconnect instead of download) rather than
|
||||||
|
# dropping it.
|
||||||
|
if recipe_scanner and lora_type == 'lora' and basename_key not in queried_local_basenames:
|
||||||
|
local_lora = await recipe_scanner.get_local_lora(lora_name, recipe_base_model)
|
||||||
|
if local_lora:
|
||||||
|
local_entry = self.populate_lora_from_local(lora_entry, local_lora)
|
||||||
|
merge_or_append_local(local_entry)
|
||||||
continue
|
continue
|
||||||
|
lora_entry['hash'] = ''
|
||||||
# Skip entries without a hash value — they can't be
|
lora_entry['hashInvalid'] = True
|
||||||
# resolved via CivitAI and would only produce a
|
if not resource_lora_count:
|
||||||
# useless "Deleted" entry in the recipe.
|
|
||||||
if not lora_hash:
|
|
||||||
continue
|
|
||||||
|
|
||||||
lora_type, lora_name = hash_key.split(':', 1)
|
|
||||||
|
|
||||||
# Get weight from extranet tags if available, else default to 1.0
|
|
||||||
weight = lora_weights.get(hash_key, 1.0)
|
|
||||||
|
|
||||||
# Initialize lora entry
|
|
||||||
lora_entry = {
|
|
||||||
'name': lora_name,
|
|
||||||
'type': lora_type, # 'lora' or 'hypernet'
|
|
||||||
'weight': weight,
|
|
||||||
'hash': lora_hash,
|
|
||||||
'existsLocally': False,
|
|
||||||
'localPath': None,
|
|
||||||
'file_name': lora_name,
|
|
||||||
'thumbnailUrl': '/loras_static/images/no-preview.png',
|
|
||||||
'baseModel': '',
|
|
||||||
'size': 0,
|
|
||||||
'downloadUrl': '',
|
|
||||||
'isDeleted': False
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try to get info from Civitai
|
|
||||||
if metadata_provider:
|
|
||||||
try:
|
|
||||||
civitai_info = await metadata_provider.get_model_by_hash(lora_hash)
|
|
||||||
|
|
||||||
populated_entry = await self.populate_lora_from_civitai(
|
|
||||||
lora_entry,
|
|
||||||
civitai_info,
|
|
||||||
recipe_scanner,
|
|
||||||
base_model_counts,
|
|
||||||
lora_hash
|
|
||||||
)
|
|
||||||
if populated_entry is None:
|
|
||||||
continue # Skip invalid LoRA types
|
|
||||||
lora_entry = populated_entry
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error fetching Civitai info for LoRA {lora_name}: {e}")
|
|
||||||
|
|
||||||
loras.append(lora_entry)
|
loras.append(lora_entry)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if lora_hash and recipe_scanner and lora_type == 'lora':
|
||||||
|
local_lora = await recipe_scanner.get_local_lora_by_hash(lora_hash)
|
||||||
|
if local_lora:
|
||||||
|
local_entry = self.populate_lora_from_local(lora_entry, local_lora)
|
||||||
|
merge_or_append_local(local_entry)
|
||||||
|
continue
|
||||||
|
|
||||||
|
hash_resolved = False
|
||||||
|
if lora_hash and metadata_provider:
|
||||||
|
try:
|
||||||
|
civitai_info = await metadata_provider.get_model_by_hash(lora_hash)
|
||||||
|
populated_entry = await self.populate_lora_from_civitai(
|
||||||
|
lora_entry,
|
||||||
|
civitai_info,
|
||||||
|
recipe_scanner,
|
||||||
|
base_model_counts,
|
||||||
|
lora_hash,
|
||||||
|
)
|
||||||
|
if populated_entry is None:
|
||||||
|
continue
|
||||||
|
lora_entry = populated_entry
|
||||||
|
hash_resolved = not lora_entry.get('isDeleted')
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error fetching Civitai info for LoRA {lora_name}: {e}")
|
||||||
|
|
||||||
|
if hash_resolved:
|
||||||
|
merge_or_append_civitai(lora_entry, preserve_existing_weight=not prompt_entries)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if recipe_scanner and lora_type == 'lora' and basename_key not in queried_local_basenames:
|
||||||
|
local_lora = await recipe_scanner.get_local_lora(lora_name, recipe_base_model)
|
||||||
|
if local_lora:
|
||||||
|
local_entry = self.populate_lora_from_local(lora_entry, local_lora)
|
||||||
|
merge_or_append_local(local_entry)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if lora_hash and not resource_lora_count:
|
||||||
|
loras.append(lora_entry)
|
||||||
|
|
||||||
# Try to get base model from resources or make educated guess
|
# Try to get base model from resources or make educated guess
|
||||||
base_model = None
|
base_model = None
|
||||||
|
|||||||
@@ -115,6 +115,27 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
|||||||
):
|
):
|
||||||
metadata = inner_meta
|
metadata = inner_meta
|
||||||
|
|
||||||
|
# Civitai's image API meta parser mangles the A1111 "Lora hashes"
|
||||||
|
# text field into a quote-wrapped dict entry:
|
||||||
|
# '"Daphne Blake Cosplay_v1": "e67ebd5e315f"'
|
||||||
|
# The 12-char AutoV3 it carries is more reliable than the stale
|
||||||
|
# 10-char AutoV2 value in the "hashes" dict, so recover it and
|
||||||
|
# let it override the conflicting entry.
|
||||||
|
if isinstance(metadata, dict):
|
||||||
|
for key, hash_value in list(metadata.items()):
|
||||||
|
if (
|
||||||
|
isinstance(key, str)
|
||||||
|
and key.startswith('"')
|
||||||
|
and isinstance(hash_value, str)
|
||||||
|
and hash_value.endswith('"')
|
||||||
|
):
|
||||||
|
clean_name = key.strip('"').strip()
|
||||||
|
clean_hash = hash_value.strip('"').strip()
|
||||||
|
if clean_name and clean_hash:
|
||||||
|
hashes_dict = metadata.get("hashes")
|
||||||
|
if isinstance(hashes_dict, dict):
|
||||||
|
hashes_dict[f"lora:{clean_name}"] = clean_hash
|
||||||
|
|
||||||
# Initialize result structure
|
# Initialize result structure
|
||||||
result: Dict[str, Any] = {
|
result: Dict[str, Any] = {
|
||||||
"base_model": None,
|
"base_model": None,
|
||||||
|
|||||||
+112
-75
@@ -31,41 +31,106 @@ class ComfyMetadataParser(RecipeMetadataParser):
|
|||||||
metadata_provider = await get_default_metadata_provider()
|
metadata_provider = await get_default_metadata_provider()
|
||||||
|
|
||||||
data = json.loads(user_comment)
|
data = json.loads(user_comment)
|
||||||
|
|
||||||
|
checkpoint_nodes = {k: v for k, v in data.items() if isinstance(v, dict) and v.get('class_type') == 'CheckpointLoaderSimple'}
|
||||||
|
checkpoint = None
|
||||||
|
checkpoint_id = None
|
||||||
|
checkpoint_version_id = None
|
||||||
|
if checkpoint_nodes:
|
||||||
|
checkpoint_node = next(iter(checkpoint_nodes.values()))
|
||||||
|
if 'inputs' in checkpoint_node and 'ckpt_name' in checkpoint_node['inputs']:
|
||||||
|
checkpoint_name = checkpoint_node['inputs']['ckpt_name']
|
||||||
|
# Some ComfyUI workflows serialize ckpt_name as a
|
||||||
|
# single-element list (e.g. ["model.safetensors"]) or leave
|
||||||
|
# the value unset (None). Neither is a string, so skip the
|
||||||
|
# CivitAI-URN lookup instead of crashing re.search with a
|
||||||
|
# TypeError that fails the whole image import.
|
||||||
|
if isinstance(checkpoint_name, list):
|
||||||
|
checkpoint_name = (
|
||||||
|
checkpoint_name[0] if checkpoint_name else None
|
||||||
|
)
|
||||||
|
if isinstance(checkpoint_name, str):
|
||||||
|
checkpoint_match = re.search(r'civitai:(\d+)@(\d+)', checkpoint_name)
|
||||||
|
if checkpoint_match:
|
||||||
|
checkpoint_id = checkpoint_match.group(1)
|
||||||
|
checkpoint_version_id = checkpoint_match.group(2)
|
||||||
|
checkpoint = {
|
||||||
|
'id': checkpoint_version_id,
|
||||||
|
'modelId': checkpoint_id,
|
||||||
|
'name': f"Checkpoint {checkpoint_id}",
|
||||||
|
'version': '',
|
||||||
|
'type': 'checkpoint'
|
||||||
|
}
|
||||||
|
if metadata_provider:
|
||||||
|
try:
|
||||||
|
civitai_info_tuple = await metadata_provider.get_model_version_info(checkpoint_version_id)
|
||||||
|
civitai_info, _ = civitai_info_tuple if isinstance(civitai_info_tuple, tuple) else (civitai_info_tuple, None)
|
||||||
|
checkpoint = await self.populate_checkpoint_from_civitai(checkpoint, civitai_info)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error fetching Civitai info for checkpoint: {e}")
|
||||||
|
|
||||||
|
recipe_base_model = checkpoint.get('baseModel') if checkpoint else None
|
||||||
loras = []
|
loras = []
|
||||||
|
lora_candidates = []
|
||||||
# Find all LoraLoader nodes
|
for node in data.values():
|
||||||
lora_nodes = {k: v for k, v in data.items() if isinstance(v, dict) and v.get('class_type') == 'LoraLoader'}
|
if not isinstance(node, dict):
|
||||||
|
|
||||||
# Process each LoraLoader node
|
|
||||||
for node_id, node in lora_nodes.items():
|
|
||||||
if 'inputs' not in node or 'lora_name' not in node['inputs']:
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
lora_name = node['inputs'].get('lora_name', '')
|
inputs = node.get('inputs')
|
||||||
|
if not isinstance(inputs, dict):
|
||||||
# Parse the URN to extract model ID and version ID
|
continue
|
||||||
# Format: "urn:air:sdxl:lora:civitai:1107767@1253442"
|
|
||||||
|
if node.get('class_type') == 'LoraLoader':
|
||||||
|
lora_name = inputs.get('lora_name', '')
|
||||||
|
if isinstance(lora_name, str) and lora_name:
|
||||||
|
lora_candidates.append((lora_name, inputs.get('strength_model', 1.0)))
|
||||||
|
continue
|
||||||
|
|
||||||
|
if node.get('class_type') != 'LoraLoaderLM':
|
||||||
|
continue
|
||||||
|
|
||||||
|
loras_data = inputs.get('loras', [])
|
||||||
|
if isinstance(loras_data, dict):
|
||||||
|
loras_data = loras_data.get('__value__', [])
|
||||||
|
if isinstance(loras_data, list) and len(loras_data) == 1 and isinstance(loras_data[0], list):
|
||||||
|
loras_data = loras_data[0]
|
||||||
|
if not isinstance(loras_data, list):
|
||||||
|
continue
|
||||||
|
|
||||||
|
for lora in loras_data:
|
||||||
|
if not isinstance(lora, dict) or not lora.get('active', False) or lora.get('_isDummy', False):
|
||||||
|
continue
|
||||||
|
lora_name = lora.get('name', '')
|
||||||
|
if isinstance(lora_name, str) and lora_name:
|
||||||
|
lora_candidates.append((lora_name, lora.get('strength', 1.0)))
|
||||||
|
|
||||||
|
for lora_name, weight in lora_candidates:
|
||||||
|
if isinstance(weight, str):
|
||||||
|
try:
|
||||||
|
weight = float(weight)
|
||||||
|
except ValueError:
|
||||||
|
weight = 1.0
|
||||||
lora_id_match = re.search(r'civitai:(\d+)@(\d+)', lora_name)
|
lora_id_match = re.search(r'civitai:(\d+)@(\d+)', lora_name)
|
||||||
if not lora_id_match:
|
if lora_id_match:
|
||||||
continue
|
model_id = lora_id_match.group(1)
|
||||||
|
model_version_id = lora_id_match.group(2)
|
||||||
model_id = lora_id_match.group(1)
|
entry_name = f"Lora {model_id}"
|
||||||
model_version_id = lora_id_match.group(2)
|
else:
|
||||||
|
model_id = 0
|
||||||
# Get strength from node inputs
|
model_version_id = 0
|
||||||
weight = node['inputs'].get('strength_model', 1.0)
|
entry_name = re.split(r'[\\/]', lora_name)[-1]
|
||||||
|
entry_name = re.sub(r'\.[^.]+$', '', entry_name)
|
||||||
# Initialize lora entry with default values
|
|
||||||
lora_entry = {
|
lora_entry = {
|
||||||
'id': model_version_id,
|
'id': model_version_id,
|
||||||
'modelId': model_id,
|
'modelId': model_id,
|
||||||
'name': f"Lora {model_id}", # Default name
|
'name': entry_name,
|
||||||
'version': '',
|
'version': '',
|
||||||
'type': 'lora',
|
'type': 'lora',
|
||||||
'weight': weight,
|
'weight': weight,
|
||||||
'existsLocally': False,
|
'existsLocally': False,
|
||||||
'localPath': None,
|
'localPath': None,
|
||||||
'file_name': '',
|
'file_name': entry_name,
|
||||||
'hash': '',
|
'hash': '',
|
||||||
'thumbnailUrl': '/loras_static/images/no-preview.png',
|
'thumbnailUrl': '/loras_static/images/no-preview.png',
|
||||||
'baseModel': '',
|
'baseModel': '',
|
||||||
@@ -73,59 +138,31 @@ class ComfyMetadataParser(RecipeMetadataParser):
|
|||||||
'downloadUrl': '',
|
'downloadUrl': '',
|
||||||
'isDeleted': False
|
'isDeleted': False
|
||||||
}
|
}
|
||||||
|
|
||||||
# Get additional info from Civitai if metadata provider is available
|
if lora_id_match:
|
||||||
if metadata_provider:
|
if metadata_provider:
|
||||||
try:
|
try:
|
||||||
civitai_info_tuple = await metadata_provider.get_model_version_info(model_version_id)
|
civitai_info_tuple = await metadata_provider.get_model_version_info(model_version_id)
|
||||||
# Populate lora entry with Civitai info
|
populated_entry = await self.populate_lora_from_civitai(
|
||||||
populated_entry = await self.populate_lora_from_civitai(
|
lora_entry,
|
||||||
lora_entry,
|
civitai_info_tuple,
|
||||||
civitai_info_tuple,
|
recipe_scanner
|
||||||
recipe_scanner
|
)
|
||||||
)
|
if populated_entry is None:
|
||||||
if populated_entry is None:
|
continue
|
||||||
continue # Skip invalid LoRA types
|
lora_entry = populated_entry
|
||||||
lora_entry = populated_entry
|
except Exception as e:
|
||||||
except Exception as e:
|
logger.error(f"Error fetching Civitai info for LoRA: {e}")
|
||||||
logger.error(f"Error fetching Civitai info for LoRA: {e}")
|
else:
|
||||||
|
if not recipe_scanner:
|
||||||
|
continue
|
||||||
|
local_lora = await recipe_scanner.get_local_lora(lora_name, recipe_base_model)
|
||||||
|
if not local_lora:
|
||||||
|
continue
|
||||||
|
lora_entry = self.populate_lora_from_local(lora_entry, local_lora)
|
||||||
|
|
||||||
loras.append(lora_entry)
|
loras.append(lora_entry)
|
||||||
|
|
||||||
# Find checkpoint info
|
|
||||||
checkpoint_nodes = {k: v for k, v in data.items() if isinstance(v, dict) and v.get('class_type') == 'CheckpointLoaderSimple'}
|
|
||||||
checkpoint = None
|
|
||||||
checkpoint_id = None
|
|
||||||
checkpoint_version_id = None
|
|
||||||
|
|
||||||
if checkpoint_nodes:
|
|
||||||
# Get the first checkpoint node
|
|
||||||
checkpoint_node = next(iter(checkpoint_nodes.values()))
|
|
||||||
if 'inputs' in checkpoint_node and 'ckpt_name' in checkpoint_node['inputs']:
|
|
||||||
checkpoint_name = checkpoint_node['inputs']['ckpt_name']
|
|
||||||
# Parse checkpoint URN
|
|
||||||
checkpoint_match = re.search(r'civitai:(\d+)@(\d+)', checkpoint_name)
|
|
||||||
if checkpoint_match:
|
|
||||||
checkpoint_id = checkpoint_match.group(1)
|
|
||||||
checkpoint_version_id = checkpoint_match.group(2)
|
|
||||||
checkpoint = {
|
|
||||||
'id': checkpoint_version_id,
|
|
||||||
'modelId': checkpoint_id,
|
|
||||||
'name': f"Checkpoint {checkpoint_id}",
|
|
||||||
'version': '',
|
|
||||||
'type': 'checkpoint'
|
|
||||||
}
|
|
||||||
|
|
||||||
# Get additional checkpoint info from Civitai
|
|
||||||
if metadata_provider:
|
|
||||||
try:
|
|
||||||
civitai_info_tuple = await metadata_provider.get_model_version_info(checkpoint_version_id)
|
|
||||||
civitai_info, _ = civitai_info_tuple if isinstance(civitai_info_tuple, tuple) else (civitai_info_tuple, None)
|
|
||||||
# Populate checkpoint with Civitai info
|
|
||||||
checkpoint = await self.populate_checkpoint_from_civitai(checkpoint, civitai_info)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error fetching Civitai info for checkpoint: {e}")
|
|
||||||
|
|
||||||
# Extract generation parameters
|
# Extract generation parameters
|
||||||
gen_params = {}
|
gen_params = {}
|
||||||
|
|
||||||
|
|||||||
@@ -196,7 +196,7 @@ class RecipeFormatParser(RecipeMetadataParser):
|
|||||||
filtered_gen_params[key] = value
|
filtered_gen_params[key] = value
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'base_model': checkpoint['baseModel'] if checkpoint and checkpoint.get('baseModel') else recipe_metadata.get('base_model', ''),
|
'base_model': checkpoint['baseModel'] if checkpoint and checkpoint.get('baseModel') else (recipe_metadata.get('base_model') or None),
|
||||||
'loras': loras,
|
'loras': loras,
|
||||||
'gen_params': filtered_gen_params,
|
'gen_params': filtered_gen_params,
|
||||||
'tags': recipe_metadata.get('tags', []),
|
'tags': recipe_metadata.get('tags', []),
|
||||||
@@ -208,3 +208,24 @@ class RecipeFormatParser(RecipeMetadataParser):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error parsing recipe format metadata: {e}", exc_info=True)
|
logger.error(f"Error parsing recipe format metadata: {e}", exc_info=True)
|
||||||
return {"error": str(e), "loras": []}
|
return {"error": str(e), "loras": []}
|
||||||
|
|
||||||
|
|
||||||
|
def strip_recipe_metadata(metadata_text: str) -> str:
|
||||||
|
"""Strip the ``Recipe metadata: {...}`` block appended by LoRA Manager.
|
||||||
|
|
||||||
|
The saved recipe image carries the original generation metadata followed
|
||||||
|
by an appended recipe JSON block (see ``ExifUtils.append_recipe_metadata``).
|
||||||
|
Re-import wants to re-parse the original embedded metadata, so this returns
|
||||||
|
only the text before the appended marker. The input is returned unchanged
|
||||||
|
when no marker is present.
|
||||||
|
"""
|
||||||
|
if not metadata_text:
|
||||||
|
return metadata_text
|
||||||
|
match = re.search(
|
||||||
|
RecipeFormatParser.METADATA_MARKER,
|
||||||
|
metadata_text,
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
if not match:
|
||||||
|
return metadata_text
|
||||||
|
return metadata_text[: match.start()].strip()
|
||||||
|
|||||||
@@ -149,6 +149,7 @@ class BaseModelRoutes(ABC):
|
|||||||
settings_service=self._settings,
|
settings_service=self._settings,
|
||||||
server_i18n=self._server_i18n,
|
server_i18n=self._server_i18n,
|
||||||
logger=logger,
|
logger=logger,
|
||||||
|
page_context_provider=self._get_page_context_provider(),
|
||||||
)
|
)
|
||||||
listing = ModelListingHandler(
|
listing = ModelListingHandler(
|
||||||
service=service,
|
service=service,
|
||||||
@@ -250,6 +251,10 @@ class BaseModelRoutes(ABC):
|
|||||||
"""Get expected model types string for error messages - to be overridden by subclasses."""
|
"""Get expected model types string for error messages - to be overridden by subclasses."""
|
||||||
return "any model type"
|
return "any model type"
|
||||||
|
|
||||||
|
def _get_page_context_provider(self):
|
||||||
|
"""Optional hook returning extra template context for the page view."""
|
||||||
|
return None
|
||||||
|
|
||||||
def _find_model_file(self, files):
|
def _find_model_file(self, files):
|
||||||
"""Find the appropriate model file from the files list - can be overridden by subclasses."""
|
"""Find the appropriate model file from the files list - can be overridden by subclasses."""
|
||||||
return next((file for file in files if file.get("type") in MODEL_WEIGHT_FILE_TYPES and file.get("primary") is True), None)
|
return next((file for file in files if file.get("type") in MODEL_WEIGHT_FILE_TYPES and file.get("primary") is True), None)
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ from .handlers.recipe_handlers import (
|
|||||||
RecipePageView,
|
RecipePageView,
|
||||||
RecipeQueryHandler,
|
RecipeQueryHandler,
|
||||||
RecipeSharingHandler,
|
RecipeSharingHandler,
|
||||||
|
RecipeWorkflowHandler,
|
||||||
)
|
)
|
||||||
from .recipe_route_registrar import ROUTE_DEFINITIONS
|
from .recipe_route_registrar import ROUTE_DEFINITIONS
|
||||||
|
|
||||||
@@ -200,6 +201,18 @@ class BaseRecipeRoutes:
|
|||||||
sharing_service=sharing_service,
|
sharing_service=sharing_service,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Lazy import: standalone mode replaces the ``server`` module with a
|
||||||
|
# mock, so resolve PromptServer at handler-set build time instead of
|
||||||
|
# module import time. The handler's standalone check guards UX.
|
||||||
|
from server import PromptServer # pyright: ignore[reportMissingImports]
|
||||||
|
|
||||||
|
workflow = RecipeWorkflowHandler(
|
||||||
|
ensure_dependencies_ready=self.ensure_dependencies_ready,
|
||||||
|
recipe_scanner_getter=recipe_scanner_getter,
|
||||||
|
prompt_server=PromptServer,
|
||||||
|
logger=logger,
|
||||||
|
)
|
||||||
|
|
||||||
from ..services.websocket_manager import ws_manager
|
from ..services.websocket_manager import ws_manager
|
||||||
|
|
||||||
batch_import_service = BatchImportService(
|
batch_import_service = BatchImportService(
|
||||||
@@ -224,4 +237,5 @@ class BaseRecipeRoutes:
|
|||||||
analysis=analysis,
|
analysis=analysis,
|
||||||
sharing=sharing,
|
sharing=sharing,
|
||||||
batch_import=batch_import,
|
batch_import=batch_import,
|
||||||
|
workflow=workflow,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
from typing import Any, Dict, List, Set
|
from typing import Any, Dict, List, Set
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
|
|
||||||
@@ -7,6 +8,7 @@ from .model_route_registrar import ModelRouteRegistrar
|
|||||||
from ..services.checkpoint_service import CheckpointService
|
from ..services.checkpoint_service import CheckpointService
|
||||||
from ..services.service_registry import ServiceRegistry
|
from ..services.service_registry import ServiceRegistry
|
||||||
from ..config import config
|
from ..config import config
|
||||||
|
from ..utils.utils import _format_model_name_for_comfyui
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -44,7 +46,46 @@ class CheckpointRoutes(BaseModelRoutes):
|
|||||||
# Checkpoint roots and Unet roots
|
# Checkpoint roots and Unet roots
|
||||||
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/checkpoints_roots', prefix, self.get_checkpoints_roots)
|
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/checkpoints_roots', prefix, self.get_checkpoints_roots)
|
||||||
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/unet_roots', prefix, self.get_unet_roots)
|
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/unet_roots', prefix, self.get_unet_roots)
|
||||||
|
|
||||||
|
# Name/base_model pool for the Checkpoint/Unet Loader nodes' base_model filtering
|
||||||
|
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/loader-pool', prefix, self.get_loader_pool)
|
||||||
|
|
||||||
|
async def get_loader_pool(self, request: web.Request) -> web.Response:
|
||||||
|
"""Return ComfyUI-formatted model names with their base_model.
|
||||||
|
|
||||||
|
Backing data for the Checkpoint/Unet Loader nodes'
|
||||||
|
control_after_generate feature: the front-end filters the
|
||||||
|
ckpt_name/unet_name combo options by base_model using this pool, so
|
||||||
|
randomize mode picks within the narrowed set.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
sub_type = request.query.get("sub_type", "checkpoint")
|
||||||
|
if sub_type not in ("checkpoint", "diffusion_model"):
|
||||||
|
return web.json_response({"error": "invalid sub_type"}, status=400)
|
||||||
|
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||||
|
cache = await scanner.get_cached_data()
|
||||||
|
model_roots = scanner.get_model_roots()
|
||||||
|
items: List[Dict[str, str]] = []
|
||||||
|
for item in cache.raw_data:
|
||||||
|
if item.get("sub_type") != sub_type:
|
||||||
|
continue
|
||||||
|
file_path = item.get("file_path", "")
|
||||||
|
if not file_path or not os.path.exists(file_path):
|
||||||
|
continue
|
||||||
|
formatted_name = _format_model_name_for_comfyui(file_path, model_roots)
|
||||||
|
if formatted_name:
|
||||||
|
items.append(
|
||||||
|
{
|
||||||
|
"name": formatted_name,
|
||||||
|
"base_model": item.get("base_model", "") or "",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
items.sort(key=lambda x: x["name"])
|
||||||
|
return web.json_response({"items": items})
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting loader pool: {e}", exc_info=True)
|
||||||
|
return web.json_response({"error": str(e)}, status=500)
|
||||||
|
|
||||||
def _validate_civitai_model_type(self, model_type: str) -> bool:
|
def _validate_civitai_model_type(self, model_type: str) -> bool:
|
||||||
"""Validate CivitAI model type for Checkpoint"""
|
"""Validate CivitAI model type for Checkpoint"""
|
||||||
return model_type.lower() == 'checkpoint'
|
return model_type.lower() == 'checkpoint'
|
||||||
|
|||||||
@@ -0,0 +1,110 @@
|
|||||||
|
"""HTTP handler for download target routing decisions."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from aiohttp import web
|
||||||
|
|
||||||
|
from ...services.download_routing import (
|
||||||
|
is_diffusion_model_download,
|
||||||
|
resolve_other_download_sub_type,
|
||||||
|
)
|
||||||
|
from ...utils.constants import VALID_OTHER_CIVITAI_TYPES
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadRoutingHandler:
|
||||||
|
"""Expose the download-time checkpoint/diffusion-model routing decision.
|
||||||
|
|
||||||
|
The web UI calls this when the user reaches the download location step
|
||||||
|
so the root dropdown offers the same root set (checkpoint vs unet) that
|
||||||
|
the download manager would pick for ``use_default_paths``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
async def get_download_routing(self, request: web.Request) -> web.Response:
|
||||||
|
try:
|
||||||
|
payload = await request.json()
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
return web.json_response(
|
||||||
|
{"success": False, "error": "Invalid JSON payload"}, status=400
|
||||||
|
)
|
||||||
|
|
||||||
|
model_type = payload.get("model_type", "")
|
||||||
|
base_model = payload.get("base_model") or ""
|
||||||
|
file_types = payload.get("file_types") or []
|
||||||
|
selected_file_type = payload.get("selected_file_type")
|
||||||
|
|
||||||
|
if not isinstance(model_type, str) or not model_type:
|
||||||
|
return web.json_response(
|
||||||
|
{"success": False, "error": "model_type is required"}, status=400
|
||||||
|
)
|
||||||
|
if not isinstance(base_model, str) or not isinstance(file_types, list):
|
||||||
|
return web.json_response(
|
||||||
|
{
|
||||||
|
"success": False,
|
||||||
|
"error": "base_model must be a string and file_types a list",
|
||||||
|
},
|
||||||
|
status=400,
|
||||||
|
)
|
||||||
|
if selected_file_type is not None and not isinstance(selected_file_type, str):
|
||||||
|
return web.json_response(
|
||||||
|
{"success": False, "error": "selected_file_type must be a string"},
|
||||||
|
status=400,
|
||||||
|
)
|
||||||
|
|
||||||
|
if model_type.lower() in VALID_OTHER_CIVITAI_TYPES:
|
||||||
|
from ...services.settings_manager import get_settings_manager
|
||||||
|
|
||||||
|
settings = get_settings_manager()
|
||||||
|
if not settings.is_other_models_enabled():
|
||||||
|
# Opt-in feature is off: never auto-route, the UI falls back to
|
||||||
|
# manual folder selection and the download manager rejects it.
|
||||||
|
return web.json_response(
|
||||||
|
{
|
||||||
|
"success": True,
|
||||||
|
"root_kind": "other",
|
||||||
|
"sub_type": None,
|
||||||
|
"disabled": True,
|
||||||
|
"reason": "other_models_disabled",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
sub_type = resolve_other_download_sub_type(
|
||||||
|
model_type,
|
||||||
|
file_types=(str(t) for t in file_types),
|
||||||
|
selected_file_type=selected_file_type,
|
||||||
|
)
|
||||||
|
if sub_type and not settings.is_other_sub_type_enabled(sub_type):
|
||||||
|
return web.json_response(
|
||||||
|
{
|
||||||
|
"success": True,
|
||||||
|
"root_kind": "other",
|
||||||
|
"sub_type": None,
|
||||||
|
"disabled": True,
|
||||||
|
"reason": "other_sub_type_disabled",
|
||||||
|
"requested_sub_type": sub_type,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return web.json_response(
|
||||||
|
{
|
||||||
|
"success": True,
|
||||||
|
"root_kind": "other",
|
||||||
|
"sub_type": sub_type,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
is_diffusion = is_diffusion_model_download(
|
||||||
|
model_type,
|
||||||
|
file_types=(str(t) for t in file_types),
|
||||||
|
base_model=base_model,
|
||||||
|
)
|
||||||
|
return web.json_response(
|
||||||
|
{
|
||||||
|
"success": True,
|
||||||
|
"is_diffusion_model": is_diffusion,
|
||||||
|
"root_kind": "unet" if is_diffusion else model_type,
|
||||||
|
}
|
||||||
|
)
|
||||||
@@ -122,12 +122,8 @@ async def _save_hf_metadata(dest_path: str, repo: str, model_root: str) -> None:
|
|||||||
metadata._unknown_fields["hf_url"] = hf_url
|
metadata._unknown_fields["hf_url"] = hf_url
|
||||||
metadata.from_civitai = False # HF models are not from CivitAI
|
metadata.from_civitai = False # HF models are not from CivitAI
|
||||||
|
|
||||||
metadata_dict = metadata.to_dict()
|
|
||||||
if "trainedWords" in metadata_dict and not metadata_dict["trainedWords"]:
|
|
||||||
del metadata_dict["trainedWords"]
|
|
||||||
|
|
||||||
# 3. Save metadata atomically
|
# 3. Save metadata atomically
|
||||||
await MetadataManager.save_metadata(dest_path, metadata_dict)
|
await MetadataManager.save_metadata(dest_path, metadata)
|
||||||
logger.info("Saved HF metadata (with hf_url) for %s", dest_path)
|
logger.info("Saved HF metadata (with hf_url) for %s", dest_path)
|
||||||
|
|
||||||
# 4. Determine relative folder path for cache
|
# 4. Determine relative folder path for cache
|
||||||
@@ -244,7 +240,10 @@ class HfHandler:
|
|||||||
})
|
})
|
||||||
|
|
||||||
existing["hf_url"] = hf_url
|
existing["hf_url"] = hf_url
|
||||||
existing["from_civitai"] = False
|
# NOTE: deliberately do NOT touch `from_civitai` here. It records
|
||||||
|
# where the metadata came from, and the UI must show the CivitAI
|
||||||
|
# link whenever CivitAI data is present — linking HuggingFace must
|
||||||
|
# not hide it (#1094). HF provenance is tracked via `hf_url`.
|
||||||
await MetadataManager.save_metadata(file_path, existing)
|
await MetadataManager.save_metadata(file_path, existing)
|
||||||
|
|
||||||
await _add_to_scanner_cache(file_path, existing)
|
await _add_to_scanner_cache(file_path, existing)
|
||||||
|
|||||||
@@ -53,9 +53,12 @@ from ...utils.constants import (
|
|||||||
PREVIEW_EXTENSIONS,
|
PREVIEW_EXTENSIONS,
|
||||||
SUPPORTED_MEDIA_EXTENSIONS,
|
SUPPORTED_MEDIA_EXTENSIONS,
|
||||||
VALID_LORA_TYPES,
|
VALID_LORA_TYPES,
|
||||||
|
VALID_OTHER_CIVITAI_TYPES,
|
||||||
)
|
)
|
||||||
from .hf_handlers import HfHandler
|
from .hf_handlers import HfHandler
|
||||||
from .agent_handlers import AgentHandler
|
from .agent_handlers import AgentHandler
|
||||||
|
from .download_routing_handlers import DownloadRoutingHandler
|
||||||
|
from .model_handlers import ModelCivitaiHandler
|
||||||
from ...utils.civitai_utils import rewrite_preview_url
|
from ...utils.civitai_utils import rewrite_preview_url
|
||||||
from ...utils.example_images_paths import (
|
from ...utils.example_images_paths import (
|
||||||
find_non_compliant_items_in_example_images_root,
|
find_non_compliant_items_in_example_images_root,
|
||||||
@@ -648,9 +651,72 @@ class NodeRegistry:
|
|||||||
|
|
||||||
|
|
||||||
class HealthCheckHandler:
|
class HealthCheckHandler:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
scanner_getters: Mapping[str, Callable[[], Awaitable[Any]]] | None = None,
|
||||||
|
) -> None:
|
||||||
|
self._scanner_getters = scanner_getters or {
|
||||||
|
"lora": ServiceRegistry.get_lora_scanner,
|
||||||
|
"checkpoint": ServiceRegistry.get_checkpoint_scanner,
|
||||||
|
"embedding": ServiceRegistry.get_embedding_scanner,
|
||||||
|
"other": ServiceRegistry.get_other_scanner,
|
||||||
|
"recipe": ServiceRegistry.get_recipe_scanner,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _active_scanner_getters(
|
||||||
|
self,
|
||||||
|
) -> Mapping[str, Callable[[], Awaitable[Any]]]:
|
||||||
|
"""Drop the opt-in other scanner while Other Models is disabled."""
|
||||||
|
getters = self._scanner_getters
|
||||||
|
if "other" not in getters:
|
||||||
|
return getters
|
||||||
|
if get_settings_manager().is_other_models_enabled():
|
||||||
|
return getters
|
||||||
|
return {name: getter for name, getter in getters.items() if name != "other"}
|
||||||
|
|
||||||
async def health_check(self, request: web.Request) -> web.Response:
|
async def health_check(self, request: web.Request) -> web.Response:
|
||||||
return web.json_response({"status": "ok"})
|
return web.json_response({"status": "ok"})
|
||||||
|
|
||||||
|
async def get_init_status(self, request: web.Request) -> web.Response:
|
||||||
|
"""Report aggregate scanner initialization status.
|
||||||
|
|
||||||
|
Used by the initialization page's polling fallback when the
|
||||||
|
/ws/init-progress WebSocket is unavailable. Omits pageType so every
|
||||||
|
page accepts the update and only reloads once all scanners are done.
|
||||||
|
"""
|
||||||
|
pending: list[str] = []
|
||||||
|
for name, getter in self._active_scanner_getters().items():
|
||||||
|
try:
|
||||||
|
scanner = await getter()
|
||||||
|
except Exception:
|
||||||
|
pending.append(name)
|
||||||
|
continue
|
||||||
|
cache_ready = getattr(scanner, "_cache", None) is not None
|
||||||
|
is_initializing = getattr(scanner, "is_initializing", None)
|
||||||
|
busy = (
|
||||||
|
is_initializing()
|
||||||
|
if callable(is_initializing)
|
||||||
|
else bool(getattr(scanner, "_is_initializing", False))
|
||||||
|
)
|
||||||
|
if busy or not cache_ready:
|
||||||
|
pending.append(name)
|
||||||
|
|
||||||
|
if pending:
|
||||||
|
return web.json_response(
|
||||||
|
{
|
||||||
|
"status": "initializing",
|
||||||
|
"stage": "processing",
|
||||||
|
"details": "Initializing: " + ", ".join(pending),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return web.json_response(
|
||||||
|
{
|
||||||
|
"status": "complete",
|
||||||
|
"progress": 100,
|
||||||
|
"details": "Initialization complete",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class SupportersHandler:
|
class SupportersHandler:
|
||||||
"""Handler for supporters data."""
|
"""Handler for supporters data."""
|
||||||
@@ -704,10 +770,19 @@ class DoctorHandler:
|
|||||||
("lora", "LoRAs", ServiceRegistry.get_lora_scanner),
|
("lora", "LoRAs", ServiceRegistry.get_lora_scanner),
|
||||||
("checkpoint", "Checkpoints", ServiceRegistry.get_checkpoint_scanner),
|
("checkpoint", "Checkpoints", ServiceRegistry.get_checkpoint_scanner),
|
||||||
("embedding", "Embeddings", ServiceRegistry.get_embedding_scanner),
|
("embedding", "Embeddings", ServiceRegistry.get_embedding_scanner),
|
||||||
|
("other", "Other Models", ServiceRegistry.get_other_scanner),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
self._app_version_getter = app_version_getter
|
self._app_version_getter = app_version_getter
|
||||||
|
|
||||||
|
def _active_scanner_factories(
|
||||||
|
self,
|
||||||
|
) -> Sequence[tuple[str, str, Callable[[], Awaitable[Any]]]]:
|
||||||
|
"""Drop the opt-in other scanner while Other Models is disabled."""
|
||||||
|
if self._settings.is_other_models_enabled():
|
||||||
|
return self._scanner_factories
|
||||||
|
return tuple(entry for entry in self._scanner_factories if entry[0] != "other")
|
||||||
|
|
||||||
async def get_doctor_diagnostics(self, request: web.Request) -> web.Response:
|
async def get_doctor_diagnostics(self, request: web.Request) -> web.Response:
|
||||||
try:
|
try:
|
||||||
client_version = (request.query.get("clientVersion") or "").strip()
|
client_version = (request.query.get("clientVersion") or "").strip()
|
||||||
@@ -755,7 +830,7 @@ class DoctorHandler:
|
|||||||
repaired: list[dict[str, Any]] = []
|
repaired: list[dict[str, Any]] = []
|
||||||
failures: list[dict[str, str]] = []
|
failures: list[dict[str, str]] = []
|
||||||
|
|
||||||
for model_type, label, factory in self._scanner_factories:
|
for model_type, label, factory in self._active_scanner_factories():
|
||||||
try:
|
try:
|
||||||
scanner = await factory()
|
scanner = await factory()
|
||||||
await scanner.get_cached_data(force_refresh=True, rebuild_cache=True)
|
await scanner.get_cached_data(force_refresh=True, rebuild_cache=True)
|
||||||
@@ -787,7 +862,7 @@ class DoctorHandler:
|
|||||||
renamed: list[dict[str, Any]] = []
|
renamed: list[dict[str, Any]] = []
|
||||||
|
|
||||||
try:
|
try:
|
||||||
for model_type, label, factory in self._scanner_factories:
|
for model_type, label, factory in self._active_scanner_factories():
|
||||||
try:
|
try:
|
||||||
scanner = await factory()
|
scanner = await factory()
|
||||||
hash_index = getattr(scanner, "_hash_index", None)
|
hash_index = getattr(scanner, "_hash_index", None)
|
||||||
@@ -1019,7 +1094,7 @@ class DoctorHandler:
|
|||||||
overall_status = "ok"
|
overall_status = "ok"
|
||||||
summary = "All model caches look healthy."
|
summary = "All model caches look healthy."
|
||||||
|
|
||||||
for model_type, label, factory in self._scanner_factories:
|
for model_type, label, factory in self._active_scanner_factories():
|
||||||
try:
|
try:
|
||||||
scanner = await factory()
|
scanner = await factory()
|
||||||
persisted = None
|
persisted = None
|
||||||
@@ -1104,7 +1179,7 @@ class DoctorHandler:
|
|||||||
total_conflict_groups = 0
|
total_conflict_groups = 0
|
||||||
total_conflict_files = 0
|
total_conflict_files = 0
|
||||||
|
|
||||||
for model_type, label, factory in self._scanner_factories:
|
for model_type, label, factory in self._active_scanner_factories():
|
||||||
# Duplicate filename detection targets LoRAs which use basename-only
|
# Duplicate filename detection targets LoRAs which use basename-only
|
||||||
# syntax (<lora:name:strength>). Checkpoints/embeddings reference
|
# syntax (<lora:name:strength>). Checkpoints/embeddings reference
|
||||||
# models via relative paths with extensions, so conflicts there would
|
# models via relative paths with extensions, so conflicts there would
|
||||||
@@ -1484,6 +1559,22 @@ class SettingsHandler:
|
|||||||
response_data["civitai_api_key_set"] = bool(raw_key)
|
response_data["civitai_api_key_set"] = bool(raw_key)
|
||||||
raw_llm_key = self._settings.get("llm_api_key")
|
raw_llm_key = self._settings.get("llm_api_key")
|
||||||
response_data["llm_api_key_set"] = bool(raw_llm_key)
|
response_data["llm_api_key_set"] = bool(raw_llm_key)
|
||||||
|
# Derived capability flag (not persisted): whether the host exposes
|
||||||
|
# any other-model folder at all. Standalone installs only know the
|
||||||
|
# folder_paths keys present in settings.json, so the announcement
|
||||||
|
# banner uses this to avoid promising a page that cannot list
|
||||||
|
# anything.
|
||||||
|
try:
|
||||||
|
availability = config.get_other_models_availability()
|
||||||
|
response_data["other_models_paths_available"] = bool(
|
||||||
|
availability.get("available")
|
||||||
|
)
|
||||||
|
except Exception as availability_error: # pragma: no cover - defensive
|
||||||
|
logger.debug(
|
||||||
|
"Could not resolve Other Models availability: %s",
|
||||||
|
availability_error,
|
||||||
|
)
|
||||||
|
response_data["other_models_paths_available"] = None
|
||||||
settings_file = getattr(self._settings, "settings_file", None)
|
settings_file = getattr(self._settings, "settings_file", None)
|
||||||
if settings_file:
|
if settings_file:
|
||||||
response_data["settings_file"] = settings_file
|
response_data["settings_file"] = settings_file
|
||||||
@@ -2013,6 +2104,7 @@ class ServiceRegistryAdapter:
|
|||||||
get_embedding_scanner: Callable[[], Awaitable[Any]]
|
get_embedding_scanner: Callable[[], Awaitable[Any]]
|
||||||
get_downloaded_version_history_service: Callable[[], Awaitable[Any]]
|
get_downloaded_version_history_service: Callable[[], Awaitable[Any]]
|
||||||
get_backup_service: Callable[[], Awaitable[Any]] = _noop_backup_service
|
get_backup_service: Callable[[], Awaitable[Any]] = _noop_backup_service
|
||||||
|
get_other_scanner: Callable[[], Awaitable[Any]] = ServiceRegistry.get_other_scanner
|
||||||
|
|
||||||
|
|
||||||
class ModelLibraryHandler:
|
class ModelLibraryHandler:
|
||||||
@@ -2037,6 +2129,8 @@ class ModelLibraryHandler:
|
|||||||
return "checkpoint"
|
return "checkpoint"
|
||||||
if normalized in {"embedding", "textualinversion"}:
|
if normalized in {"embedding", "textualinversion"}:
|
||||||
return "embedding"
|
return "embedding"
|
||||||
|
if normalized in VALID_OTHER_CIVITAI_TYPES:
|
||||||
|
return "other"
|
||||||
return None
|
return None
|
||||||
|
|
||||||
async def _get_scanner_for_type(self, model_type: str | None):
|
async def _get_scanner_for_type(self, model_type: str | None):
|
||||||
@@ -2047,6 +2141,13 @@ class ModelLibraryHandler:
|
|||||||
return normalized_type, await self._service_registry.get_checkpoint_scanner()
|
return normalized_type, await self._service_registry.get_checkpoint_scanner()
|
||||||
if normalized_type == "embedding":
|
if normalized_type == "embedding":
|
||||||
return normalized_type, await self._service_registry.get_embedding_scanner()
|
return normalized_type, await self._service_registry.get_embedding_scanner()
|
||||||
|
if normalized_type == "other":
|
||||||
|
# Opt-in feature: the other scanner only resolves while the master
|
||||||
|
# switch is on, so callers keep returning the legacy "required"
|
||||||
|
# error (400) when it is off.
|
||||||
|
if not get_settings_manager().is_other_models_enabled():
|
||||||
|
return None, None
|
||||||
|
return normalized_type, await self._service_registry.get_other_scanner()
|
||||||
return None, None
|
return None, None
|
||||||
|
|
||||||
async def _get_download_history_service(self):
|
async def _get_download_history_service(self):
|
||||||
@@ -2061,6 +2162,63 @@ class ModelLibraryHandler:
|
|||||||
enriched.append(entry)
|
enriched.append(entry)
|
||||||
return enriched
|
return enriched
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def _get_downloaded_files(
|
||||||
|
scanner: Any, model_version_id: int
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""Return per-file downloaded state for a version in the library.
|
||||||
|
|
||||||
|
This handler has no CivitAI version payload, so the remote file list
|
||||||
|
is taken from the local entries' cached ``civitai`` metadata (the
|
||||||
|
full version payload persisted at download time, see
|
||||||
|
``BaseModelMetadata.from_civitai_info``) and matched with the same
|
||||||
|
D2 rule used by ``get_civitai_versions`` (#1058). Local entries that
|
||||||
|
cannot be matched to a known remote file (e.g. missing metadata or
|
||||||
|
renamed files) are still reported with ``fileId`` set to None.
|
||||||
|
Returns ``[{fileId, fileName, filePath}]``.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
cache = await scanner.get_cached_data()
|
||||||
|
except Exception: # pragma: no cover - defensive fallback
|
||||||
|
logger.debug(
|
||||||
|
"Failed to read cache for downloaded files of version %s",
|
||||||
|
model_version_id,
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
return []
|
||||||
|
|
||||||
|
files_getter = getattr(cache, "get_files_by_version_id", None)
|
||||||
|
local_entries = files_getter(model_version_id) if files_getter else []
|
||||||
|
if not local_entries:
|
||||||
|
return []
|
||||||
|
|
||||||
|
version_payload: Mapping[str, Any] = {}
|
||||||
|
for entry in local_entries:
|
||||||
|
civitai = entry.get("civitai") if isinstance(entry, Mapping) else None
|
||||||
|
if isinstance(civitai, Mapping) and isinstance(civitai.get("files"), list):
|
||||||
|
version_payload = civitai
|
||||||
|
break
|
||||||
|
|
||||||
|
downloaded = ModelCivitaiHandler._match_downloaded_files(
|
||||||
|
version_payload, local_entries
|
||||||
|
)
|
||||||
|
|
||||||
|
# Surface local files that D2 could not map to a known remote file
|
||||||
|
matched_paths = {item.get("filePath") for item in downloaded}
|
||||||
|
for entry in local_entries:
|
||||||
|
if not isinstance(entry, Mapping):
|
||||||
|
continue
|
||||||
|
if entry.get("file_path") in matched_paths:
|
||||||
|
continue
|
||||||
|
downloaded.append(
|
||||||
|
{
|
||||||
|
"fileId": None,
|
||||||
|
"fileName": entry.get("file_name"),
|
||||||
|
"filePath": entry.get("file_path"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return downloaded
|
||||||
|
|
||||||
async def check_model_exists(self, request: web.Request) -> web.Response:
|
async def check_model_exists(self, request: web.Request) -> web.Response:
|
||||||
try:
|
try:
|
||||||
model_id_str = request.query.get("modelId")
|
model_id_str = request.query.get("modelId")
|
||||||
@@ -2081,6 +2239,11 @@ class ModelLibraryHandler:
|
|||||||
lora_scanner = await self._service_registry.get_lora_scanner()
|
lora_scanner = await self._service_registry.get_lora_scanner()
|
||||||
checkpoint_scanner = await self._service_registry.get_checkpoint_scanner()
|
checkpoint_scanner = await self._service_registry.get_checkpoint_scanner()
|
||||||
embedding_scanner = await self._service_registry.get_embedding_scanner()
|
embedding_scanner = await self._service_registry.get_embedding_scanner()
|
||||||
|
# Opt-in: probe the other scanner only while Other Models is enabled,
|
||||||
|
# so the disabled behaviour stays byte-identical to the legacy one.
|
||||||
|
other_scanner = None
|
||||||
|
if get_settings_manager().is_other_models_enabled():
|
||||||
|
other_scanner = await self._service_registry.get_other_scanner()
|
||||||
|
|
||||||
if model_version_id_str:
|
if model_version_id_str:
|
||||||
try:
|
try:
|
||||||
@@ -2096,9 +2259,11 @@ class ModelLibraryHandler:
|
|||||||
|
|
||||||
exists = False
|
exists = False
|
||||||
model_type = None
|
model_type = None
|
||||||
|
matched_scanner = None
|
||||||
if await lora_scanner.check_model_version_exists(model_version_id):
|
if await lora_scanner.check_model_version_exists(model_version_id):
|
||||||
exists = True
|
exists = True
|
||||||
model_type = "lora"
|
model_type = "lora"
|
||||||
|
matched_scanner = lora_scanner
|
||||||
elif (
|
elif (
|
||||||
checkpoint_scanner
|
checkpoint_scanner
|
||||||
and await checkpoint_scanner.check_model_version_exists(
|
and await checkpoint_scanner.check_model_version_exists(
|
||||||
@@ -2107,6 +2272,7 @@ class ModelLibraryHandler:
|
|||||||
):
|
):
|
||||||
exists = True
|
exists = True
|
||||||
model_type = "checkpoint"
|
model_type = "checkpoint"
|
||||||
|
matched_scanner = checkpoint_scanner
|
||||||
elif (
|
elif (
|
||||||
embedding_scanner
|
embedding_scanner
|
||||||
and await embedding_scanner.check_model_version_exists(
|
and await embedding_scanner.check_model_version_exists(
|
||||||
@@ -2115,6 +2281,14 @@ class ModelLibraryHandler:
|
|||||||
):
|
):
|
||||||
exists = True
|
exists = True
|
||||||
model_type = "embedding"
|
model_type = "embedding"
|
||||||
|
matched_scanner = embedding_scanner
|
||||||
|
elif (
|
||||||
|
other_scanner
|
||||||
|
and await other_scanner.check_model_version_exists(model_version_id)
|
||||||
|
):
|
||||||
|
exists = True
|
||||||
|
model_type = "other"
|
||||||
|
matched_scanner = other_scanner
|
||||||
|
|
||||||
if exists:
|
if exists:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
@@ -2123,13 +2297,16 @@ class ModelLibraryHandler:
|
|||||||
"exists": True,
|
"exists": True,
|
||||||
"modelType": model_type,
|
"modelType": model_type,
|
||||||
"hasBeenDownloaded": False,
|
"hasBeenDownloaded": False,
|
||||||
|
"downloadedFiles": await self._get_downloaded_files(
|
||||||
|
matched_scanner, model_version_id
|
||||||
|
),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
history_service = await self._get_download_history_service()
|
history_service = await self._get_download_history_service()
|
||||||
has_been_downloaded = False
|
has_been_downloaded = False
|
||||||
history_type = None
|
history_type = None
|
||||||
for candidate_type in ("lora", "checkpoint", "embedding"):
|
for candidate_type in ("lora", "checkpoint", "embedding", "other"):
|
||||||
if await history_service.has_been_downloaded(
|
if await history_service.has_been_downloaded(
|
||||||
candidate_type,
|
candidate_type,
|
||||||
model_version_id,
|
model_version_id,
|
||||||
@@ -2144,12 +2321,14 @@ class ModelLibraryHandler:
|
|||||||
"exists": False,
|
"exists": False,
|
||||||
"modelType": history_type,
|
"modelType": history_type,
|
||||||
"hasBeenDownloaded": has_been_downloaded,
|
"hasBeenDownloaded": has_been_downloaded,
|
||||||
|
"downloadedFiles": [],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
lora_versions = await lora_scanner.get_model_versions_by_id(model_id)
|
lora_versions = await lora_scanner.get_model_versions_by_id(model_id)
|
||||||
checkpoint_versions = []
|
checkpoint_versions = []
|
||||||
embedding_versions = []
|
embedding_versions = []
|
||||||
|
other_versions = []
|
||||||
if not lora_versions and checkpoint_scanner:
|
if not lora_versions and checkpoint_scanner:
|
||||||
checkpoint_versions = await checkpoint_scanner.get_model_versions_by_id(
|
checkpoint_versions = await checkpoint_scanner.get_model_versions_by_id(
|
||||||
model_id
|
model_id
|
||||||
@@ -2158,6 +2337,13 @@ class ModelLibraryHandler:
|
|||||||
embedding_versions = await embedding_scanner.get_model_versions_by_id(
|
embedding_versions = await embedding_scanner.get_model_versions_by_id(
|
||||||
model_id
|
model_id
|
||||||
)
|
)
|
||||||
|
if (
|
||||||
|
not lora_versions
|
||||||
|
and not checkpoint_versions
|
||||||
|
and not embedding_versions
|
||||||
|
and other_scanner
|
||||||
|
):
|
||||||
|
other_versions = await other_scanner.get_model_versions_by_id(model_id)
|
||||||
|
|
||||||
model_type = None
|
model_type = None
|
||||||
versions = []
|
versions = []
|
||||||
@@ -2189,9 +2375,18 @@ class ModelLibraryHandler:
|
|||||||
"downloadedVersionIds": [],
|
"downloadedVersionIds": [],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
if other_versions:
|
||||||
|
return web.json_response(
|
||||||
|
{
|
||||||
|
"success": True,
|
||||||
|
"modelType": "other",
|
||||||
|
"versions": self._with_downloaded_flag(other_versions),
|
||||||
|
"downloadedVersionIds": [],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
history_service = await self._get_download_history_service()
|
history_service = await self._get_download_history_service()
|
||||||
for candidate_type in ("lora", "checkpoint", "embedding"):
|
for candidate_type in ("lora", "checkpoint", "embedding", "other"):
|
||||||
candidate_downloaded_version_ids = (
|
candidate_downloaded_version_ids = (
|
||||||
await history_service.get_downloaded_version_ids(
|
await history_service.get_downloaded_version_ids(
|
||||||
candidate_type,
|
candidate_type,
|
||||||
@@ -2246,6 +2441,11 @@ class ModelLibraryHandler:
|
|||||||
lora_scanner = await self._service_registry.get_lora_scanner()
|
lora_scanner = await self._service_registry.get_lora_scanner()
|
||||||
checkpoint_scanner = await self._service_registry.get_checkpoint_scanner()
|
checkpoint_scanner = await self._service_registry.get_checkpoint_scanner()
|
||||||
embedding_scanner = await self._service_registry.get_embedding_scanner()
|
embedding_scanner = await self._service_registry.get_embedding_scanner()
|
||||||
|
# Opt-in: keep the other probe last so model cards for lora /
|
||||||
|
# checkpoint / embedding ids are unaffected by the extra scanner.
|
||||||
|
other_scanner = None
|
||||||
|
if get_settings_manager().is_other_models_enabled():
|
||||||
|
other_scanner = await self._service_registry.get_other_scanner()
|
||||||
|
|
||||||
results: list[dict[str, Any]] = []
|
results: list[dict[str, Any]] = []
|
||||||
for model_id in model_ids:
|
for model_id in model_ids:
|
||||||
@@ -2281,6 +2481,17 @@ class ModelLibraryHandler:
|
|||||||
})
|
})
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
if other_scanner:
|
||||||
|
other_versions = await other_scanner.get_model_versions_by_id(model_id)
|
||||||
|
if other_versions:
|
||||||
|
results.append({
|
||||||
|
"modelId": model_id,
|
||||||
|
"modelType": "other",
|
||||||
|
"versions": self._with_downloaded_flag(other_versions),
|
||||||
|
"downloadedVersionIds": [],
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
|
||||||
results.append({
|
results.append({
|
||||||
"modelId": model_id,
|
"modelId": model_id,
|
||||||
"modelType": None,
|
"modelType": None,
|
||||||
@@ -2428,8 +2639,8 @@ class ModelLibraryHandler:
|
|||||||
embedding_scanner = await self._service_registry.get_embedding_scanner()
|
embedding_scanner = await self._service_registry.get_embedding_scanner()
|
||||||
|
|
||||||
found_type = None
|
found_type = None
|
||||||
file_path = None
|
|
||||||
found_cache = None
|
found_cache = None
|
||||||
|
entries: list = []
|
||||||
|
|
||||||
for model_type, scanner in (
|
for model_type, scanner in (
|
||||||
("lora", lora_scanner),
|
("lora", lora_scanner),
|
||||||
@@ -2440,27 +2651,43 @@ class ModelLibraryHandler:
|
|||||||
if cache and model_version_id in cache.version_index:
|
if cache and model_version_id in cache.version_index:
|
||||||
found_type = model_type
|
found_type = model_type
|
||||||
found_cache = cache
|
found_cache = cache
|
||||||
entry = cache.version_index[model_version_id]
|
# A version can have several local files (#1058); collect
|
||||||
file_path = entry.get("file_path")
|
# them all so the delete below covers every file.
|
||||||
|
files_getter = getattr(cache, "get_files_by_version_id", None)
|
||||||
|
if files_getter is not None:
|
||||||
|
entries = files_getter(model_version_id)
|
||||||
|
else:
|
||||||
|
entries = [cache.version_index[model_version_id]]
|
||||||
break
|
break
|
||||||
|
|
||||||
if not file_path:
|
file_paths = [
|
||||||
|
entry.get("file_path")
|
||||||
|
for entry in entries
|
||||||
|
if isinstance(entry, dict) and entry.get("file_path")
|
||||||
|
]
|
||||||
|
|
||||||
|
if not file_paths:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{"success": False, "error": "Model version not found in any scanner cache"},
|
{"success": False, "error": "Model version not found in any scanner cache"},
|
||||||
status=404,
|
status=404,
|
||||||
)
|
)
|
||||||
|
|
||||||
target_dir = os.path.dirname(file_path)
|
for file_path in file_paths:
|
||||||
base_name = os.path.basename(file_path)
|
target_dir = os.path.dirname(file_path)
|
||||||
file_name, extension = os.path.splitext(base_name)
|
base_name = os.path.basename(file_path)
|
||||||
await delete_model_artifacts(target_dir, file_name, main_extension=extension)
|
file_name, extension = os.path.splitext(base_name)
|
||||||
|
await delete_model_artifacts(target_dir, file_name, main_extension=extension)
|
||||||
|
|
||||||
if found_cache:
|
if found_cache:
|
||||||
|
removed_paths = set(file_paths)
|
||||||
found_cache.raw_data = [
|
found_cache.raw_data = [
|
||||||
item
|
item
|
||||||
for item in found_cache.raw_data
|
for item in found_cache.raw_data
|
||||||
if item.get("file_path") != file_path
|
if item.get("file_path") not in removed_paths
|
||||||
]
|
]
|
||||||
|
rebuild = getattr(found_cache, "rebuild_version_index", None)
|
||||||
|
if rebuild is not None:
|
||||||
|
rebuild()
|
||||||
await found_cache.resort()
|
await found_cache.resort()
|
||||||
|
|
||||||
scanner_map = {
|
scanner_map = {
|
||||||
@@ -2483,6 +2710,7 @@ class ModelLibraryHandler:
|
|||||||
"success": True,
|
"success": True,
|
||||||
"modelType": found_type,
|
"modelType": found_type,
|
||||||
"modelVersionId": model_version_id,
|
"modelVersionId": model_version_id,
|
||||||
|
"deletedFiles": len(file_paths),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -2652,12 +2880,32 @@ class ModelLibraryHandler:
|
|||||||
model_type.lower() for model_type in CIVITAI_USER_MODEL_TYPES
|
model_type.lower() for model_type in CIVITAI_USER_MODEL_TYPES
|
||||||
}
|
}
|
||||||
lora_type_aliases = {model_type.lower() for model_type in VALID_LORA_TYPES}
|
lora_type_aliases = {model_type.lower() for model_type in VALID_LORA_TYPES}
|
||||||
|
other_type_aliases = {
|
||||||
|
model_type.lower() for model_type in VALID_OTHER_CIVITAI_TYPES
|
||||||
|
}
|
||||||
|
|
||||||
|
# Acquire the other scanner lazily so adapters without it only
|
||||||
|
# fail when the payload actually contains other-type models.
|
||||||
|
# While the opt-in feature is off the scanner still exists (its
|
||||||
|
# cache is empty), so other types simply report inLibrary=False.
|
||||||
|
needs_other_scanner = any(
|
||||||
|
isinstance(model, dict)
|
||||||
|
and str(model.get("type", "")).lower() in other_type_aliases
|
||||||
|
for model in models
|
||||||
|
)
|
||||||
|
other_scanner = None
|
||||||
|
if needs_other_scanner:
|
||||||
|
other_scanner = await self._service_registry.get_other_scanner()
|
||||||
|
|
||||||
type_scanner_map: Dict[str, Any] = {
|
type_scanner_map: Dict[str, Any] = {
|
||||||
**{alias: lora_scanner for alias in lora_type_aliases},
|
**{alias: lora_scanner for alias in lora_type_aliases},
|
||||||
"checkpoint": checkpoint_scanner,
|
"checkpoint": checkpoint_scanner,
|
||||||
"textualinversion": embedding_scanner,
|
"textualinversion": embedding_scanner,
|
||||||
}
|
}
|
||||||
|
if other_scanner is not None:
|
||||||
|
type_scanner_map.update(
|
||||||
|
{alias: other_scanner for alias in other_type_aliases}
|
||||||
|
)
|
||||||
|
|
||||||
versions: list[dict[str, Any]] = []
|
versions: list[dict[str, Any]] = []
|
||||||
history_service = await self._get_download_history_service()
|
history_service = await self._get_download_history_service()
|
||||||
@@ -2681,12 +2929,17 @@ class ModelLibraryHandler:
|
|||||||
"embedding",
|
"embedding",
|
||||||
model_ids,
|
model_ids,
|
||||||
)
|
)
|
||||||
|
other_downloaded = await history_service.get_downloaded_version_ids_bulk(
|
||||||
|
"other",
|
||||||
|
model_ids,
|
||||||
|
)
|
||||||
downloaded_version_map: Dict[str, Dict[int, set[int]]] = {
|
downloaded_version_map: Dict[str, Dict[int, set[int]]] = {
|
||||||
"lora": lora_downloaded,
|
"lora": lora_downloaded,
|
||||||
"locon": lora_downloaded,
|
"locon": lora_downloaded,
|
||||||
"dora": lora_downloaded,
|
"dora": lora_downloaded,
|
||||||
"checkpoint": checkpoint_downloaded,
|
"checkpoint": checkpoint_downloaded,
|
||||||
"textualinversion": embedding_downloaded,
|
"textualinversion": embedding_downloaded,
|
||||||
|
**{alias: other_downloaded for alias in VALID_OTHER_CIVITAI_TYPES},
|
||||||
}
|
}
|
||||||
for model in models:
|
for model in models:
|
||||||
if not isinstance(model, dict):
|
if not isinstance(model, dict):
|
||||||
@@ -3750,6 +4003,7 @@ class MiscHandlerSet:
|
|||||||
base_model: BaseModelHandlerSet,
|
base_model: BaseModelHandlerSet,
|
||||||
hf_handler: Any = None,
|
hf_handler: Any = None,
|
||||||
agent_handler: Any = None,
|
agent_handler: Any = None,
|
||||||
|
download_routing: Any = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.health = health
|
self.health = health
|
||||||
self.settings = settings
|
self.settings = settings
|
||||||
@@ -3770,12 +4024,14 @@ class MiscHandlerSet:
|
|||||||
self.base_model = base_model
|
self.base_model = base_model
|
||||||
self.hf_handler = hf_handler
|
self.hf_handler = hf_handler
|
||||||
self.agent_handler = agent_handler
|
self.agent_handler = agent_handler
|
||||||
|
self.download_routing = download_routing
|
||||||
|
|
||||||
def to_route_mapping(
|
def to_route_mapping(
|
||||||
self,
|
self,
|
||||||
) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||||
return {
|
return {
|
||||||
"health_check": self.health.health_check,
|
"health_check": self.health.health_check,
|
||||||
|
"get_init_status": self.health.get_init_status,
|
||||||
"get_settings": self.settings.get_settings,
|
"get_settings": self.settings.get_settings,
|
||||||
"update_settings": self.settings.update_settings,
|
"update_settings": self.settings.update_settings,
|
||||||
"get_doctor_diagnostics": self.doctor.get_doctor_diagnostics,
|
"get_doctor_diagnostics": self.doctor.get_doctor_diagnostics,
|
||||||
@@ -3827,6 +4083,8 @@ class MiscHandlerSet:
|
|||||||
"get_agent_skills": self.agent_handler.get_agent_skills,
|
"get_agent_skills": self.agent_handler.get_agent_skills,
|
||||||
"execute_agent_skill": self.agent_handler.execute_agent_skill,
|
"execute_agent_skill": self.agent_handler.execute_agent_skill,
|
||||||
"cancel_agent_skill": self.agent_handler.cancel_agent_skill,
|
"cancel_agent_skill": self.agent_handler.cancel_agent_skill,
|
||||||
|
# Download routing handler
|
||||||
|
"get_download_routing": self.download_routing.get_download_routing,
|
||||||
# Base model handlers
|
# Base model handlers
|
||||||
"get_base_models": self.base_model.get_base_models,
|
"get_base_models": self.base_model.get_base_models,
|
||||||
"refresh_base_models": self.base_model.refresh_base_models,
|
"refresh_base_models": self.base_model.refresh_base_models,
|
||||||
@@ -3840,6 +4098,7 @@ def build_service_registry_adapter() -> ServiceRegistryAdapter:
|
|||||||
get_lora_scanner=ServiceRegistry.get_lora_scanner,
|
get_lora_scanner=ServiceRegistry.get_lora_scanner,
|
||||||
get_checkpoint_scanner=ServiceRegistry.get_checkpoint_scanner,
|
get_checkpoint_scanner=ServiceRegistry.get_checkpoint_scanner,
|
||||||
get_embedding_scanner=ServiceRegistry.get_embedding_scanner,
|
get_embedding_scanner=ServiceRegistry.get_embedding_scanner,
|
||||||
|
get_other_scanner=ServiceRegistry.get_other_scanner,
|
||||||
get_downloaded_version_history_service=ServiceRegistry.get_downloaded_version_history_service,
|
get_downloaded_version_history_service=ServiceRegistry.get_downloaded_version_history_service,
|
||||||
get_backup_service=ServiceRegistry.get_backup_service,
|
get_backup_service=ServiceRegistry.get_backup_service,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -15,6 +15,10 @@ from aiohttp import web
|
|||||||
import jinja2
|
import jinja2
|
||||||
|
|
||||||
from ...config import config
|
from ...config import config
|
||||||
|
from ...services.active_filters_store import (
|
||||||
|
ActiveFiltersStore,
|
||||||
|
active_filters_to_query_kwargs,
|
||||||
|
)
|
||||||
from ...services.download_coordinator import DownloadCoordinator
|
from ...services.download_coordinator import DownloadCoordinator
|
||||||
from ...services.connectivity_guard import (
|
from ...services.connectivity_guard import (
|
||||||
OFFLINE_FRIENDLY_MESSAGE,
|
OFFLINE_FRIENDLY_MESSAGE,
|
||||||
@@ -86,6 +90,7 @@ class ModelPageView:
|
|||||||
settings_service: SettingsManager,
|
settings_service: SettingsManager,
|
||||||
server_i18n,
|
server_i18n,
|
||||||
logger: logging.Logger,
|
logger: logging.Logger,
|
||||||
|
page_context_provider: Callable[[web.Request], Dict[str, Any]] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._template_env = template_env
|
self._template_env = template_env
|
||||||
self._template_name = template_name
|
self._template_name = template_name
|
||||||
@@ -93,6 +98,7 @@ class ModelPageView:
|
|||||||
self._settings = settings_service
|
self._settings = settings_service
|
||||||
self._server_i18n = server_i18n
|
self._server_i18n = server_i18n
|
||||||
self._logger = logger
|
self._logger = logger
|
||||||
|
self._page_context_provider = page_context_provider
|
||||||
|
|
||||||
def _load_supporters(self) -> dict[str, Any]:
|
def _load_supporters(self) -> dict[str, Any]:
|
||||||
"""Load supporters data from JSON file."""
|
"""Load supporters data from JSON file."""
|
||||||
@@ -206,6 +212,16 @@ class ModelPageView:
|
|||||||
self._logger.error("Error loading cache data: %s", cache_error)
|
self._logger.error("Error loading cache data: %s", cache_error)
|
||||||
template_context["is_initializing"] = True
|
template_context["is_initializing"] = True
|
||||||
|
|
||||||
|
if self._page_context_provider is not None:
|
||||||
|
try:
|
||||||
|
extra_context = self._page_context_provider(request)
|
||||||
|
if isinstance(extra_context, dict):
|
||||||
|
template_context.update(extra_context)
|
||||||
|
except Exception as context_error: # pragma: no cover - logging path
|
||||||
|
self._logger.error(
|
||||||
|
"Error building page context: %s", context_error
|
||||||
|
)
|
||||||
|
|
||||||
rendered = self._template_env.get_template(self._template_name).render(
|
rendered = self._template_env.get_template(self._template_name).render(
|
||||||
**template_context
|
**template_context
|
||||||
)
|
)
|
||||||
@@ -364,6 +380,7 @@ class ModelListingHandler:
|
|||||||
== "true",
|
== "true",
|
||||||
"tags": request.query.get("search_tags", "false").lower() == "true",
|
"tags": request.query.get("search_tags", "false").lower() == "true",
|
||||||
"creator": request.query.get("search_creator", "false").lower() == "true",
|
"creator": request.query.get("search_creator", "false").lower() == "true",
|
||||||
|
"hash": request.query.get("search_hash", "false").lower() == "true",
|
||||||
"recursive": request.query.get("recursive", "true").lower() == "true",
|
"recursive": request.query.get("recursive", "true").lower() == "true",
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -633,6 +650,16 @@ class ModelManagementHandler:
|
|||||||
file_path = data.get("file_path")
|
file_path = data.get("file_path")
|
||||||
model_id = data.get("model_id")
|
model_id = data.get("model_id")
|
||||||
model_version_id = data.get("model_version_id")
|
model_version_id = data.get("model_version_id")
|
||||||
|
source = data.get("source")
|
||||||
|
|
||||||
|
if source not in (None, "", "civarchive"):
|
||||||
|
return web.json_response(
|
||||||
|
{
|
||||||
|
"success": False,
|
||||||
|
"error": f"Unsupported relink source: {source}",
|
||||||
|
},
|
||||||
|
status=400,
|
||||||
|
)
|
||||||
|
|
||||||
if not file_path or model_id is None:
|
if not file_path or model_id is None:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
@@ -648,20 +675,33 @@ class ModelManagementHandler:
|
|||||||
metadata_path
|
metadata_path
|
||||||
)
|
)
|
||||||
|
|
||||||
|
relink_kwargs = {
|
||||||
|
"file_path": file_path,
|
||||||
|
"metadata": local_metadata,
|
||||||
|
"model_id": int(model_id),
|
||||||
|
"model_version_id": int(model_version_id) if model_version_id else None,
|
||||||
|
}
|
||||||
|
if source == "civarchive":
|
||||||
|
relink_kwargs["provider_name"] = "civarchive_api"
|
||||||
|
|
||||||
updated_metadata = await self._metadata_sync.relink_metadata(
|
updated_metadata = await self._metadata_sync.relink_metadata(
|
||||||
file_path=file_path,
|
**relink_kwargs
|
||||||
metadata=local_metadata,
|
|
||||||
model_id=int(model_id),
|
|
||||||
model_version_id=int(model_version_id) if model_version_id else None,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
await self._service.scanner.update_single_model_cache(
|
await self._service.scanner.update_single_model_cache(
|
||||||
file_path, file_path, updated_metadata
|
file_path, file_path, updated_metadata
|
||||||
)
|
)
|
||||||
|
|
||||||
message = f"Model successfully re-linked to Civitai model {model_id}" + (
|
if source == "civarchive":
|
||||||
f" version {model_version_id}" if model_version_id else ""
|
message = (
|
||||||
)
|
f"Model successfully re-linked to CivArchive model {model_id}"
|
||||||
|
+ (f" version {model_version_id}" if model_version_id else "")
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
message = (
|
||||||
|
f"Model successfully re-linked to Civitai model {model_id}"
|
||||||
|
+ (f" version {model_version_id}" if model_version_id else "")
|
||||||
|
)
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{
|
{
|
||||||
"success": True,
|
"success": True,
|
||||||
@@ -669,6 +709,8 @@ class ModelManagementHandler:
|
|||||||
"hash": updated_metadata.get("sha256", ""),
|
"hash": updated_metadata.get("sha256", ""),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
except ValueError as exc:
|
||||||
|
return web.json_response({"success": False, "error": str(exc)}, status=400)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
if is_expected_offline_error(str(exc)):
|
if is_expected_offline_error(str(exc)):
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
@@ -1029,6 +1071,11 @@ class ModelQueryHandler:
|
|||||||
self._service = service
|
self._service = service
|
||||||
self._logger = logger
|
self._logger = logger
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _parse_include_empty(request: web.Request) -> bool:
|
||||||
|
"""Parse the include_empty query flag (``1``/``true``)."""
|
||||||
|
return request.query.get("include_empty", "").lower() in ("1", "true")
|
||||||
|
|
||||||
async def get_top_tags(self, request: web.Request) -> web.Response:
|
async def get_top_tags(self, request: web.Request) -> web.Response:
|
||||||
try:
|
try:
|
||||||
limit = int(request.query.get("limit", "20"))
|
limit = int(request.query.get("limit", "20"))
|
||||||
@@ -1123,8 +1170,14 @@ class ModelQueryHandler:
|
|||||||
|
|
||||||
async def get_folders(self, request: web.Request) -> web.Response:
|
async def get_folders(self, request: web.Request) -> web.Response:
|
||||||
try:
|
try:
|
||||||
cache = await self._service.scanner.get_cached_data()
|
include_empty = self._parse_include_empty(request)
|
||||||
return web.json_response({"folders": cache.folders})
|
if include_empty:
|
||||||
|
# Live enumeration includes empty OS-created directories.
|
||||||
|
folders = await self._service.scanner.get_all_folders()
|
||||||
|
else:
|
||||||
|
cache = await self._service.scanner.get_cached_data()
|
||||||
|
folders = cache.folders
|
||||||
|
return web.json_response({"folders": folders})
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self._logger.error("Error getting folders: %s", exc)
|
self._logger.error("Error getting folders: %s", exc)
|
||||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||||
@@ -1149,7 +1202,9 @@ class ModelQueryHandler:
|
|||||||
{"success": False, "error": "model_root parameter is required"},
|
{"success": False, "error": "model_root parameter is required"},
|
||||||
status=400,
|
status=400,
|
||||||
)
|
)
|
||||||
folder_tree = await self._service.get_folder_tree(model_root)
|
folder_tree = await self._service.get_folder_tree(
|
||||||
|
model_root, include_empty=self._parse_include_empty(request)
|
||||||
|
)
|
||||||
return web.json_response({"success": True, "tree": folder_tree})
|
return web.json_response({"success": True, "tree": folder_tree})
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self._logger.error("Error getting folder tree: %s", exc)
|
self._logger.error("Error getting folder tree: %s", exc)
|
||||||
@@ -1157,7 +1212,9 @@ class ModelQueryHandler:
|
|||||||
|
|
||||||
async def get_unified_folder_tree(self, request: web.Request) -> web.Response:
|
async def get_unified_folder_tree(self, request: web.Request) -> web.Response:
|
||||||
try:
|
try:
|
||||||
unified_tree = await self._service.get_unified_folder_tree()
|
unified_tree = await self._service.get_unified_folder_tree(
|
||||||
|
include_empty=self._parse_include_empty(request)
|
||||||
|
)
|
||||||
return web.json_response({"success": True, "tree": unified_tree})
|
return web.json_response({"success": True, "tree": unified_tree})
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self._logger.error("Error getting unified folder tree: %s", exc)
|
self._logger.error("Error getting unified folder tree: %s", exc)
|
||||||
@@ -1554,12 +1611,50 @@ class ModelQueryHandler:
|
|||||||
allow_selling_generated_content.lower() not in ("false", "0", "")
|
allow_selling_generated_content.lower() not in ("false", "0", "")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# When requested, merge the manager page's active filters stored
|
||||||
|
# server-side. Explicit query parameters take precedence over the
|
||||||
|
# stored values.
|
||||||
|
use_active_filters = (
|
||||||
|
request.query.get("use_active_filters", "").lower() in ("1", "true")
|
||||||
|
)
|
||||||
|
if use_active_filters:
|
||||||
|
stored = ActiveFiltersStore.get_instance().get_filters(
|
||||||
|
self._service.model_type
|
||||||
|
)
|
||||||
|
injected = active_filters_to_query_kwargs(stored)
|
||||||
|
if folder is None and "folder" in injected:
|
||||||
|
folder = injected["folder"]
|
||||||
|
if "recursive" not in request.query and "recursive" in injected:
|
||||||
|
recursive = injected["recursive"]
|
||||||
|
if not base_models and injected.get("base_models"):
|
||||||
|
base_models = injected["base_models"]
|
||||||
|
if not model_types and injected.get("model_types"):
|
||||||
|
model_types = injected["model_types"]
|
||||||
|
if not tag_filters and injected.get("tags"):
|
||||||
|
tag_filters = injected["tags"]
|
||||||
|
if not auto_tag_filters and injected.get("auto_tags"):
|
||||||
|
auto_tag_filters = injected["auto_tags"]
|
||||||
|
if "tag_logic" not in request.query and injected.get("tag_logic"):
|
||||||
|
injected_logic = str(injected["tag_logic"]).lower()
|
||||||
|
if injected_logic in ("any", "all"):
|
||||||
|
tag_logic = injected_logic
|
||||||
|
if credit_required is None and "credit_required" in injected:
|
||||||
|
credit_required = injected["credit_required"]
|
||||||
|
if (
|
||||||
|
allow_selling_generated_content is None
|
||||||
|
and "allow_selling_generated_content" in injected
|
||||||
|
):
|
||||||
|
allow_selling_generated_content = injected[
|
||||||
|
"allow_selling_generated_content"
|
||||||
|
]
|
||||||
|
|
||||||
# The presence of the recursive param (always sent by the loras
|
# The presence of the recursive param (always sent by the loras
|
||||||
# widget when filter mode is on) signals that the filter pipeline
|
# widget when filter mode is on) signals that the filter pipeline
|
||||||
# must run even when no concrete filter is set, so global settings
|
# must run even when no concrete filter is set, so global settings
|
||||||
# like show_only_sfw stay consistent with the list endpoint.
|
# like show_only_sfw stay consistent with the list endpoint.
|
||||||
apply_filters = (
|
apply_filters = (
|
||||||
"recursive" in request.query
|
use_active_filters
|
||||||
|
or "recursive" in request.query
|
||||||
or folder is not None
|
or folder is not None
|
||||||
or bool(base_models)
|
or bool(base_models)
|
||||||
or bool(model_types)
|
or bool(model_types)
|
||||||
@@ -1593,6 +1688,50 @@ class ModelQueryHandler:
|
|||||||
)
|
)
|
||||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||||
|
|
||||||
|
async def update_active_filters(self, request: web.Request) -> web.Response:
|
||||||
|
"""Store the manager page's active filters for this model type."""
|
||||||
|
try:
|
||||||
|
payload = await request.json()
|
||||||
|
except Exception:
|
||||||
|
return web.json_response(
|
||||||
|
{"success": False, "error": "Invalid JSON body"}, status=400
|
||||||
|
)
|
||||||
|
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
return web.json_response(
|
||||||
|
{"success": False, "error": "Body must be a JSON object"}, status=400
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
ActiveFiltersStore.get_instance().set_filters(
|
||||||
|
self._service.model_type, payload
|
||||||
|
)
|
||||||
|
return web.json_response({"success": True})
|
||||||
|
except Exception as exc:
|
||||||
|
self._logger.error(
|
||||||
|
"Error updating active filters for %s: %s",
|
||||||
|
self._service.model_type,
|
||||||
|
exc,
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||||
|
|
||||||
|
async def get_active_filters(self, request: web.Request) -> web.Response:
|
||||||
|
"""Return the stored active filters for this model type."""
|
||||||
|
try:
|
||||||
|
filters = ActiveFiltersStore.get_instance().get_filters(
|
||||||
|
self._service.model_type
|
||||||
|
)
|
||||||
|
return web.json_response({"success": True, "filters": filters})
|
||||||
|
except Exception as exc:
|
||||||
|
self._logger.error(
|
||||||
|
"Error getting active filters for %s: %s",
|
||||||
|
self._service.model_type,
|
||||||
|
exc,
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||||
|
|
||||||
|
|
||||||
class ModelDownloadHandler:
|
class ModelDownloadHandler:
|
||||||
"""Coordinate downloads and progress reporting."""
|
"""Coordinate downloads and progress reporting."""
|
||||||
@@ -1659,7 +1798,8 @@ class ModelDownloadHandler:
|
|||||||
import json
|
import json
|
||||||
|
|
||||||
try:
|
try:
|
||||||
data["file_params"] = json.loads(file_params_json)
|
# Normalize falsy payloads (e.g. {}) to None (#1058)
|
||||||
|
data["file_params"] = json.loads(file_params_json) or None
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
self._logger.warning(
|
self._logger.warning(
|
||||||
"Invalid file_params JSON: %s", file_params_json
|
"Invalid file_params JSON: %s", file_params_json
|
||||||
@@ -1811,7 +1951,8 @@ class ModelDownloadHandler:
|
|||||||
|
|
||||||
model_id = int(model_id_str) if model_id_str else None
|
model_id = int(model_id_str) if model_id_str else None
|
||||||
model_version_id = int(model_version_id_str) if model_version_id_str else None
|
model_version_id = int(model_version_id_str) if model_version_id_str else None
|
||||||
file_params = json.loads(file_params_json) if file_params_json else None
|
# Normalize falsy payloads (e.g. {}) to None (#1058)
|
||||||
|
file_params = (json.loads(file_params_json) if file_params_json else None) or None
|
||||||
|
|
||||||
service = await DownloadQueueService.get_instance()
|
service = await DownloadQueueService.get_instance()
|
||||||
item = await service.add_to_queue(
|
item = await service.add_to_queue(
|
||||||
@@ -1886,8 +2027,18 @@ class ModelDownloadHandler:
|
|||||||
try:
|
try:
|
||||||
status_filter = request.query.get("status") or None
|
status_filter = request.query.get("status") or None
|
||||||
service = await DownloadQueueService.get_instance()
|
service = await DownloadQueueService.get_instance()
|
||||||
cleared = await service.clear_queue(status_filter=status_filter)
|
cleared_ids = await service.clear_queue(status_filter=status_filter)
|
||||||
return web.json_response({"success": True, "cleared": cleared})
|
# Clearing the queue rows alone would orphan any in-memory tasks
|
||||||
|
# and persisted aria2 state for those downloads, leaving them
|
||||||
|
# polling the daemon invisibly. Tear that tracking down too.
|
||||||
|
try:
|
||||||
|
await self._download_coordinator.discard_cleared_downloads(cleared_ids)
|
||||||
|
except Exception:
|
||||||
|
self._logger.warning(
|
||||||
|
"Failed to discard in-memory state for cleared downloads",
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
return web.json_response({"success": True, "cleared": len(cleared_ids)})
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self._logger.error(
|
self._logger.error(
|
||||||
"Error clearing download queue: %s", exc, exc_info=True
|
"Error clearing download queue: %s", exc, exc_info=True
|
||||||
@@ -1970,9 +2121,11 @@ class ModelDownloadHandler:
|
|||||||
item_id=item_id, download_id=download_id
|
item_id=item_id, download_id=download_id
|
||||||
)
|
)
|
||||||
if item is None:
|
if item is None:
|
||||||
|
# Missing or non-retryable history entry is a business
|
||||||
|
# outcome, not a routing error: 200 lets the extension's
|
||||||
|
# apiFetch 404-fallback and error middleware stay quiet.
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{"success": False, "error": "History item not found or not retryable"},
|
{"success": False, "error": "History item not found or not retryable"}
|
||||||
status=404,
|
|
||||||
)
|
)
|
||||||
return web.json_response({"success": True, "item": item})
|
return web.json_response({"success": True, "item": item})
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -2023,8 +2176,12 @@ class ModelDownloadHandler:
|
|||||||
completed_at=completed_at,
|
completed_at=completed_at,
|
||||||
)
|
)
|
||||||
if item is None:
|
if item is None:
|
||||||
|
# A missing queue item (already completed, or never queued) is
|
||||||
|
# a normal business outcome, not a routing error. Return 200
|
||||||
|
# so the browser extension's apiFetch 404-fallback and the
|
||||||
|
# error middleware stay quiet.
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{"success": False, "error": "Download not found in queue"}, status=404
|
{"success": False, "error": "Download not found in queue"}
|
||||||
)
|
)
|
||||||
return web.json_response({"success": True, "item": item})
|
return web.json_response({"success": True, "item": item})
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -2066,9 +2223,10 @@ class ModelDownloadHandler:
|
|||||||
service = await DownloadQueueService.get_instance()
|
service = await DownloadQueueService.get_instance()
|
||||||
updated = await service.update_status(download_id, status)
|
updated = await service.update_status(download_id, status)
|
||||||
if not updated:
|
if not updated:
|
||||||
|
# Same rationale as complete_download_in_queue: a missing
|
||||||
|
# queue item is a business outcome, not a routing error.
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{"success": False, "error": "Download not found in queue"},
|
{"success": False, "error": "Download not found in queue"}
|
||||||
status=404,
|
|
||||||
)
|
)
|
||||||
return web.json_response({"success": True})
|
return web.json_response({"success": True})
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -2187,6 +2345,19 @@ class ModelCivitaiHandler:
|
|||||||
else:
|
else:
|
||||||
version.pop("localPath", None)
|
version.pop("localPath", None)
|
||||||
|
|
||||||
|
# Per-file downloaded state so multi-file versions can show
|
||||||
|
# which individual files are already in the library (#1058)
|
||||||
|
local_entries: List[Any] = []
|
||||||
|
if version_id is not None and cache:
|
||||||
|
files_getter = getattr(cache, "get_files_by_version_id", None)
|
||||||
|
if files_getter is not None:
|
||||||
|
local_entries = files_getter(version_id)
|
||||||
|
elif cache_entry is not None:
|
||||||
|
local_entries = [cache_entry]
|
||||||
|
version["downloadedFiles"] = self._match_downloaded_files(
|
||||||
|
version, local_entries
|
||||||
|
)
|
||||||
|
|
||||||
model_file = (
|
model_file = (
|
||||||
self._find_model_file(version.get("files", []))
|
self._find_model_file(version.get("files", []))
|
||||||
if isinstance(version.get("files"), Iterable)
|
if isinstance(version.get("files"), Iterable)
|
||||||
@@ -2201,6 +2372,64 @@ class ModelCivitaiHandler:
|
|||||||
)
|
)
|
||||||
return web.Response(status=500, text=str(exc))
|
return web.Response(status=500, text=str(exc))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _match_downloaded_files(
|
||||||
|
version: Mapping[str, Any], local_entries: List[Any]
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Map local library entries back to individual files of a version.
|
||||||
|
|
||||||
|
Matching follows rule D2 (#1058): SHA256 is authoritative when the
|
||||||
|
local entry carries one; otherwise fall back to extension-less file
|
||||||
|
name equality. Returns ``[{fileId, fileName, filePath}]``.
|
||||||
|
"""
|
||||||
|
files = version.get("files")
|
||||||
|
if not isinstance(files, list) or not local_entries:
|
||||||
|
return []
|
||||||
|
|
||||||
|
by_hash: Dict[str, Mapping[str, Any]] = {}
|
||||||
|
by_name: Dict[str, Mapping[str, Any]] = {}
|
||||||
|
for file_info in files:
|
||||||
|
if not isinstance(file_info, Mapping):
|
||||||
|
continue
|
||||||
|
sha = str(
|
||||||
|
(file_info.get("hashes") or {}).get("SHA256") or ""
|
||||||
|
).strip().lower()
|
||||||
|
if sha:
|
||||||
|
by_hash.setdefault(sha, file_info)
|
||||||
|
name = str(file_info.get("name") or "").strip()
|
||||||
|
if name:
|
||||||
|
by_name.setdefault(os.path.splitext(name)[0], file_info)
|
||||||
|
|
||||||
|
downloaded: List[Dict[str, Any]] = []
|
||||||
|
seen_keys: set = set()
|
||||||
|
for entry in local_entries:
|
||||||
|
if not isinstance(entry, Mapping):
|
||||||
|
continue
|
||||||
|
matched: Optional[Mapping[str, Any]] = None
|
||||||
|
local_hash = str(entry.get("sha256") or "").strip().lower()
|
||||||
|
if local_hash:
|
||||||
|
matched = by_hash.get(local_hash)
|
||||||
|
if matched is None:
|
||||||
|
local_name = str(entry.get("file_name") or "").strip()
|
||||||
|
if local_name:
|
||||||
|
matched = by_name.get(local_name)
|
||||||
|
if matched is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
file_id = matched.get("id")
|
||||||
|
dedupe_key = file_id if file_id is not None else matched.get("name")
|
||||||
|
if dedupe_key in seen_keys:
|
||||||
|
continue
|
||||||
|
seen_keys.add(dedupe_key)
|
||||||
|
downloaded.append(
|
||||||
|
{
|
||||||
|
"fileId": file_id,
|
||||||
|
"fileName": matched.get("name"),
|
||||||
|
"filePath": entry.get("file_path"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return downloaded
|
||||||
|
|
||||||
async def get_civitai_model_by_version(self, request: web.Request) -> web.Response:
|
async def get_civitai_model_by_version(self, request: web.Request) -> web.Response:
|
||||||
try:
|
try:
|
||||||
model_version_id = request.match_info.get("modelVersionId")
|
model_version_id = request.match_info.get("modelVersionId")
|
||||||
@@ -2548,10 +2777,20 @@ class ModelUpdateHandler:
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
same_base_scope = self._uses_same_base_update_scope()
|
||||||
|
|
||||||
serialized_records = []
|
serialized_records = []
|
||||||
for record in records.values():
|
for record in records.values():
|
||||||
has_update_fn = getattr(record, "has_update", None)
|
has_update_fn = getattr(record, "has_update", None)
|
||||||
if callable(has_update_fn) and has_update_fn(
|
if not callable(has_update_fn):
|
||||||
|
continue
|
||||||
|
scoped_fn = (
|
||||||
|
getattr(record, "has_update_for_local_bases", None)
|
||||||
|
if same_base_scope
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
qualifies_fn = scoped_fn if callable(scoped_fn) else has_update_fn
|
||||||
|
if qualifies_fn(
|
||||||
hide_early_access=hide_early_access,
|
hide_early_access=hide_early_access,
|
||||||
hide_paid=hide_paid,
|
hide_paid=hide_paid,
|
||||||
):
|
):
|
||||||
@@ -2564,6 +2803,26 @@ class ModelUpdateHandler:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _uses_same_base_update_scope(self) -> bool:
|
||||||
|
"""Return True when update reporting must honor same-base scoping.
|
||||||
|
|
||||||
|
Mirrors ``BaseModelService._annotate_update_flags``: the Updates filter
|
||||||
|
evaluates updates per local base model when ``version_grouping`` is
|
||||||
|
``same_base`` (its default). The refresh summary counts with the same
|
||||||
|
scope so the "Found N update(s)" toast matches what the filter
|
||||||
|
displays. See issue #1083.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if self._settings is None:
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
strategy_value = self._settings.get("version_grouping")
|
||||||
|
except Exception:
|
||||||
|
return True
|
||||||
|
if isinstance(strategy_value, str) and strategy_value.strip():
|
||||||
|
return strategy_value.strip().lower() == "same_base"
|
||||||
|
return True
|
||||||
|
|
||||||
async def set_model_update_ignore(self, request: web.Request) -> web.Response:
|
async def set_model_update_ignore(self, request: web.Request) -> web.Response:
|
||||||
payload = await self._read_json(request)
|
payload = await self._read_json(request)
|
||||||
model_id = self._normalize_model_id(payload.get("modelId"))
|
model_id = self._normalize_model_id(payload.get("modelId"))
|
||||||
@@ -3031,6 +3290,9 @@ class ModelUpdateHandler:
|
|||||||
"paidAccess": paid_access_payload,
|
"paidAccess": paid_access_payload,
|
||||||
"filePath": context.get("file_path"),
|
"filePath": context.get("file_path"),
|
||||||
"fileName": context.get("file_name"),
|
"fileName": context.get("file_name"),
|
||||||
|
# Weight-file variant count (None when unknown); lets the UI hide
|
||||||
|
# the download affordance for single-file in-library versions.
|
||||||
|
"fileCount": getattr(version, "file_count", None),
|
||||||
}
|
}
|
||||||
|
|
||||||
async def _build_version_context(
|
async def _build_version_context(
|
||||||
@@ -3175,6 +3437,8 @@ class ModelHandlerSet:
|
|||||||
"get_model_metadata": self.query.get_model_metadata,
|
"get_model_metadata": self.query.get_model_metadata,
|
||||||
"get_model_description": self.query.get_model_description,
|
"get_model_description": self.query.get_model_description,
|
||||||
"get_relative_paths": self.query.get_relative_paths,
|
"get_relative_paths": self.query.get_relative_paths,
|
||||||
|
"update_active_filters": self.query.update_active_filters,
|
||||||
|
"get_active_filters": self.query.get_active_filters,
|
||||||
"refresh_model_updates": self.updates.refresh_model_updates,
|
"refresh_model_updates": self.updates.refresh_model_updates,
|
||||||
"fetch_missing_civitai_license_data": self.updates.fetch_missing_civitai_license_data,
|
"fetch_missing_civitai_license_data": self.updates.fetch_missing_civitai_license_data,
|
||||||
"set_model_update_ignore": self.updates.set_model_update_ignore,
|
"set_model_update_ignore": self.updates.set_model_update_ignore,
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ _MODEL_TYPE_GETTER_NAMES: Dict[str, str] = {
|
|||||||
"loras": "get_lora_scanner",
|
"loras": "get_lora_scanner",
|
||||||
"checkpoints": "get_checkpoint_scanner",
|
"checkpoints": "get_checkpoint_scanner",
|
||||||
"embeddings": "get_embedding_scanner",
|
"embeddings": "get_embedding_scanner",
|
||||||
|
"other": "get_other_scanner",
|
||||||
}
|
}
|
||||||
|
|
||||||
# Staged batch ids are ``uuid.uuid4().hex`` (32 lowercase hex chars). The id is
|
# Staged batch ids are ``uuid.uuid4().hex`` (32 lowercase hex chars). The id is
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -32,6 +32,7 @@ MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
|||||||
RouteDefinition("GET", "/api/lm/settings/libraries", "get_settings_libraries"),
|
RouteDefinition("GET", "/api/lm/settings/libraries", "get_settings_libraries"),
|
||||||
RouteDefinition("POST", "/api/lm/settings/libraries/activate", "activate_library"),
|
RouteDefinition("POST", "/api/lm/settings/libraries/activate", "activate_library"),
|
||||||
RouteDefinition("GET", "/api/lm/health-check", "health_check"),
|
RouteDefinition("GET", "/api/lm/health-check", "health_check"),
|
||||||
|
RouteDefinition("GET", "/api/lm/init-status", "get_init_status"),
|
||||||
RouteDefinition("GET", "/api/lm/supporters", "get_supporters"),
|
RouteDefinition("GET", "/api/lm/supporters", "get_supporters"),
|
||||||
RouteDefinition("GET", "/api/lm/wildcards/search", "search_wildcards"),
|
RouteDefinition("GET", "/api/lm/wildcards/search", "search_wildcards"),
|
||||||
RouteDefinition("POST", "/api/lm/wildcards/open-location", "open_wildcards_location"),
|
RouteDefinition("POST", "/api/lm/wildcards/open-location", "open_wildcards_location"),
|
||||||
@@ -102,6 +103,10 @@ MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
|||||||
RouteDefinition(
|
RouteDefinition(
|
||||||
"GET", "/api/lm/hf-repo-files", "get_hf_repo_files"
|
"GET", "/api/lm/hf-repo-files", "get_hf_repo_files"
|
||||||
),
|
),
|
||||||
|
# Download target routing decision (checkpoint vs diffusion model roots)
|
||||||
|
RouteDefinition(
|
||||||
|
"POST", "/api/lm/download/routing", "get_download_routing"
|
||||||
|
),
|
||||||
RouteDefinition(
|
RouteDefinition(
|
||||||
"POST", "/api/lm/download-hf-model", "download_hf_model"
|
"POST", "/api/lm/download-hf-model", "download_hf_model"
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -41,6 +41,7 @@ from .handlers.misc_handlers import (
|
|||||||
from .handlers.base_model_handlers import BaseModelHandlerSet
|
from .handlers.base_model_handlers import BaseModelHandlerSet
|
||||||
from .handlers.hf_handlers import HfHandler
|
from .handlers.hf_handlers import HfHandler
|
||||||
from .handlers.agent_handlers import AgentHandler
|
from .handlers.agent_handlers import AgentHandler
|
||||||
|
from .handlers.download_routing_handlers import DownloadRoutingHandler
|
||||||
from .misc_route_registrar import MiscRouteRegistrar
|
from .misc_route_registrar import MiscRouteRegistrar
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -140,6 +141,7 @@ class MiscRoutes:
|
|||||||
base_model = BaseModelHandlerSet()
|
base_model = BaseModelHandlerSet()
|
||||||
hf_handler = HfHandler()
|
hf_handler = HfHandler()
|
||||||
agent_handler = AgentHandler()
|
agent_handler = AgentHandler()
|
||||||
|
download_routing = DownloadRoutingHandler()
|
||||||
|
|
||||||
return self._handler_set_factory(
|
return self._handler_set_factory(
|
||||||
health=health,
|
health=health,
|
||||||
@@ -161,6 +163,7 @@ class MiscRoutes:
|
|||||||
base_model=base_model,
|
base_model=base_model,
|
||||||
hf_handler=hf_handler,
|
hf_handler=hf_handler,
|
||||||
agent_handler=agent_handler,
|
agent_handler=agent_handler,
|
||||||
|
download_routing=download_routing,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -68,6 +68,8 @@ COMMON_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
|||||||
"GET", "/api/lm/{prefix}/model-description", "get_model_description"
|
"GET", "/api/lm/{prefix}/model-description", "get_model_description"
|
||||||
),
|
),
|
||||||
RouteDefinition("GET", "/api/lm/{prefix}/relative-paths", "get_relative_paths"),
|
RouteDefinition("GET", "/api/lm/{prefix}/relative-paths", "get_relative_paths"),
|
||||||
|
RouteDefinition("PUT", "/api/lm/{prefix}/active-filters", "update_active_filters"),
|
||||||
|
RouteDefinition("GET", "/api/lm/{prefix}/active-filters", "get_active_filters"),
|
||||||
RouteDefinition(
|
RouteDefinition(
|
||||||
"GET", "/api/lm/{prefix}/civitai/versions/{model_id}", "get_civitai_versions"
|
"GET", "/api/lm/{prefix}/civitai/versions/{model_id}", "get_civitai_versions"
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -0,0 +1,139 @@
|
|||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
from aiohttp import web
|
||||||
|
|
||||||
|
from .base_model_routes import BaseModelRoutes
|
||||||
|
from .model_route_registrar import ModelRouteRegistrar
|
||||||
|
from ..config import config
|
||||||
|
from ..services.other_model_service import OtherModelService
|
||||||
|
from ..services.service_registry import ServiceRegistry
|
||||||
|
from ..utils.constants import (
|
||||||
|
CIVITAI_TYPE_TO_OTHER_SUB_TYPE,
|
||||||
|
OTHER_MODEL_FOLDER_SUBTYPES,
|
||||||
|
VALID_OTHER_CIVITAI_TYPES,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class OtherRoutes(BaseModelRoutes):
|
||||||
|
"""Other-model-specific route controller (VAE, upscaler, text encoder, ...)"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
"""Initialize Other-model routes with OtherModel service"""
|
||||||
|
super().__init__()
|
||||||
|
self.template_name = "other.html"
|
||||||
|
|
||||||
|
async def initialize_services(self):
|
||||||
|
"""Initialize services from ServiceRegistry"""
|
||||||
|
other_scanner = await ServiceRegistry.get_other_scanner()
|
||||||
|
update_service = await ServiceRegistry.get_model_update_service()
|
||||||
|
self.service = OtherModelService(other_scanner, update_service=update_service)
|
||||||
|
self.set_model_update_service(update_service)
|
||||||
|
|
||||||
|
# Attach service dependencies
|
||||||
|
self.attach_service(self.service)
|
||||||
|
|
||||||
|
def setup_routes(self, app: web.Application, prefix: str = "other"):
|
||||||
|
"""Setup Other-model routes"""
|
||||||
|
# Schedule service initialization on app startup
|
||||||
|
app.on_startup.append(lambda _: self.initialize_services())
|
||||||
|
|
||||||
|
# Setup common routes with 'other' prefix (includes page route)
|
||||||
|
super().setup_routes(app, prefix)
|
||||||
|
|
||||||
|
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
||||||
|
"""Setup Other-model-specific routes"""
|
||||||
|
# Other-model info by name
|
||||||
|
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/info/{name}', prefix, self.get_other_model_info)
|
||||||
|
# Other-model roots grouped by sub_type (text_encoders + legacy clip
|
||||||
|
# are aggregated under text_encoder)
|
||||||
|
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/roots_by_subtype', prefix, self.get_roots_by_subtype)
|
||||||
|
|
||||||
|
def _validate_civitai_model_type(self, model_type: str) -> bool:
|
||||||
|
"""Validate CivitAI model type for other models.
|
||||||
|
|
||||||
|
Accepts retired CivitAI types (CLIP, CLIPVision) as well — grandfathered
|
||||||
|
models on CivitAI still carry them. Types whose sub_type is currently
|
||||||
|
disabled (or every type while the opt-in feature is off) are rejected.
|
||||||
|
"""
|
||||||
|
normalized = (model_type or "").strip().lower()
|
||||||
|
if normalized not in VALID_OTHER_CIVITAI_TYPES:
|
||||||
|
return False
|
||||||
|
if not self._settings.is_other_models_enabled():
|
||||||
|
return False
|
||||||
|
|
||||||
|
sub_type = CIVITAI_TYPE_TO_OTHER_SUB_TYPE.get(normalized)
|
||||||
|
if sub_type is None:
|
||||||
|
# CivitAI "Other" has no sub_type of its own; it is only usable
|
||||||
|
# while at least one sub_type is enabled.
|
||||||
|
return bool(self._settings.get_enabled_other_sub_types())
|
||||||
|
return self._settings.is_other_sub_type_enabled(sub_type)
|
||||||
|
|
||||||
|
def _get_page_context_provider(self):
|
||||||
|
"""Expose the opt-in feature state to the Other Models page template."""
|
||||||
|
return self._page_context_for_other
|
||||||
|
|
||||||
|
def _page_context_for_other(self, request: web.Request) -> Dict[str, Any]:
|
||||||
|
if not self._settings.is_other_models_enabled():
|
||||||
|
return {"other_disabled": True, "other_no_paths": False}
|
||||||
|
|
||||||
|
# Enabled but nothing to scan: folder paths for the managed sub_types
|
||||||
|
# resolved to no existing folder. Render an actionable empty state
|
||||||
|
# instead of an apparently broken empty grid.
|
||||||
|
standalone_mode = os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"
|
||||||
|
return {
|
||||||
|
"other_disabled": False,
|
||||||
|
"other_no_paths": not bool(config.other_roots),
|
||||||
|
"standalone_mode": standalone_mode,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _get_expected_model_types(self) -> str:
|
||||||
|
"""Get expected model types string for error messages"""
|
||||||
|
return "VAE, Upscaler, TextEncoder, CLIPVision, Controlnet, or Other"
|
||||||
|
|
||||||
|
def _parse_specific_params(self, request: web.Request) -> Dict[str, Any]:
|
||||||
|
"""Parse other-model-specific parameters (none in Phase 1)."""
|
||||||
|
return {}
|
||||||
|
|
||||||
|
async def get_roots_by_subtype(self, request: web.Request) -> web.Response:
|
||||||
|
"""Return other-model roots grouped by sub_type.
|
||||||
|
|
||||||
|
Aggregates the per-folder_paths-key roots from config
|
||||||
|
(``text_encoders`` and the legacy ``clip`` key both land under
|
||||||
|
``text_encoder``).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
roots_by_subtype: Dict[str, List[str]] = {}
|
||||||
|
for key, roots in (config.other_folder_roots or {}).items():
|
||||||
|
sub_type = OTHER_MODEL_FOLDER_SUBTYPES.get(key)
|
||||||
|
if not sub_type:
|
||||||
|
continue
|
||||||
|
bucket = roots_by_subtype.setdefault(sub_type, [])
|
||||||
|
for root in roots:
|
||||||
|
if root and root not in bucket:
|
||||||
|
bucket.append(root)
|
||||||
|
return web.json_response(
|
||||||
|
{"success": True, "roots_by_subtype": roots_by_subtype}
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error getting other roots by sub_type: {e}", exc_info=True)
|
||||||
|
return web.json_response(
|
||||||
|
{"success": False, "error": str(e)}, status=500
|
||||||
|
)
|
||||||
|
|
||||||
|
async def get_other_model_info(self, request: web.Request) -> web.Response:
|
||||||
|
"""Get detailed information for a specific other model by name"""
|
||||||
|
try:
|
||||||
|
name = request.match_info.get('name', '')
|
||||||
|
model_info = await self.service.get_model_info_by_name(name) # pyright: ignore[reportAttributeAccessIssue]
|
||||||
|
|
||||||
|
if model_info:
|
||||||
|
return web.json_response(model_info)
|
||||||
|
else:
|
||||||
|
return web.json_response({"error": "Model not found"}, status=404)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error in get_other_model_info: {e}", exc_info=True)
|
||||||
|
return web.json_response({"error": str(e)}, status=500)
|
||||||
@@ -49,6 +49,31 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
|||||||
RouteDefinition("POST", "/api/lm/recipe/move", "move_recipe"),
|
RouteDefinition("POST", "/api/lm/recipe/move", "move_recipe"),
|
||||||
RouteDefinition("POST", "/api/lm/recipes/move-bulk", "move_recipes_bulk"),
|
RouteDefinition("POST", "/api/lm/recipes/move-bulk", "move_recipes_bulk"),
|
||||||
RouteDefinition("POST", "/api/lm/recipe/lora/reconnect", "reconnect_lora"),
|
RouteDefinition("POST", "/api/lm/recipe/lora/reconnect", "reconnect_lora"),
|
||||||
|
RouteDefinition("POST", "/api/lm/recipe/lora/restore", "restore_lora"),
|
||||||
|
RouteDefinition(
|
||||||
|
"GET",
|
||||||
|
"/api/lm/recipe/{recipe_id}/lora/{lora_index}/reconnect-suggestions",
|
||||||
|
"get_reconnect_suggestions",
|
||||||
|
),
|
||||||
|
RouteDefinition(
|
||||||
|
"POST", "/api/lm/recipe/lora/mark-hash-invalid", "mark_lora_hash_invalid"
|
||||||
|
),
|
||||||
|
RouteDefinition(
|
||||||
|
"POST", "/api/lm/recipe/checkpoint/reconnect", "reconnect_checkpoint"
|
||||||
|
),
|
||||||
|
RouteDefinition(
|
||||||
|
"POST", "/api/lm/recipe/checkpoint/restore", "restore_checkpoint"
|
||||||
|
),
|
||||||
|
RouteDefinition(
|
||||||
|
"GET",
|
||||||
|
"/api/lm/recipe/{recipe_id}/checkpoint/reconnect-suggestions",
|
||||||
|
"get_checkpoint_reconnect_suggestions",
|
||||||
|
),
|
||||||
|
RouteDefinition(
|
||||||
|
"POST",
|
||||||
|
"/api/lm/recipe/checkpoint/mark-hash-invalid",
|
||||||
|
"mark_checkpoint_hash_invalid",
|
||||||
|
),
|
||||||
RouteDefinition("GET", "/api/lm/recipes/find-duplicates", "find_duplicates"),
|
RouteDefinition("GET", "/api/lm/recipes/find-duplicates", "find_duplicates"),
|
||||||
RouteDefinition("POST", "/api/lm/recipes/bulk-delete", "bulk_delete"),
|
RouteDefinition("POST", "/api/lm/recipes/bulk-delete", "bulk_delete"),
|
||||||
RouteDefinition(
|
RouteDefinition(
|
||||||
@@ -59,11 +84,6 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
|||||||
"GET", "/api/lm/recipes/for-checkpoint", "get_recipes_for_checkpoint"
|
"GET", "/api/lm/recipes/for-checkpoint", "get_recipes_for_checkpoint"
|
||||||
),
|
),
|
||||||
RouteDefinition("GET", "/api/lm/recipes/scan", "scan_recipes"),
|
RouteDefinition("GET", "/api/lm/recipes/scan", "scan_recipes"),
|
||||||
RouteDefinition("POST", "/api/lm/recipes/repair", "repair_recipes"),
|
|
||||||
RouteDefinition("POST", "/api/lm/recipes/cancel-repair", "cancel_repair"),
|
|
||||||
RouteDefinition("POST", "/api/lm/recipe/{recipe_id}/repair", "repair_recipe"),
|
|
||||||
RouteDefinition("POST", "/api/lm/recipes/repair-bulk", "repair_recipes_bulk"),
|
|
||||||
RouteDefinition("GET", "/api/lm/recipes/repair-progress", "get_repair_progress"),
|
|
||||||
RouteDefinition("POST", "/api/lm/recipes/rematch", "rematch_recipes"),
|
RouteDefinition("POST", "/api/lm/recipes/rematch", "rematch_recipes"),
|
||||||
RouteDefinition("POST", "/api/lm/recipes/rematch-bulk", "rematch_recipes_bulk"),
|
RouteDefinition("POST", "/api/lm/recipes/rematch-bulk", "rematch_recipes_bulk"),
|
||||||
RouteDefinition("POST", "/api/lm/recipe/{recipe_id}/rematch", "rematch_recipe"),
|
RouteDefinition("POST", "/api/lm/recipe/{recipe_id}/rematch", "rematch_recipe"),
|
||||||
@@ -90,6 +110,14 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
|||||||
RouteDefinition(
|
RouteDefinition(
|
||||||
"POST", "/api/lm/recipe/{recipe_id}/reimport", "reimport_recipe"
|
"POST", "/api/lm/recipe/{recipe_id}/reimport", "reimport_recipe"
|
||||||
),
|
),
|
||||||
|
# The companion browser extension only ever issues GET requests, so the
|
||||||
|
# payload-based re-import variant must also be reachable via GET.
|
||||||
|
RouteDefinition(
|
||||||
|
"GET", "/api/lm/recipe/{recipe_id}/reimport", "reimport_recipe"
|
||||||
|
),
|
||||||
|
RouteDefinition(
|
||||||
|
"POST", "/api/lm/recipe/{recipe_id}/send-workflow", "send_recipe_workflow"
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,135 @@
|
|||||||
|
"""In-memory store for the LoRA Manager page's active filters.
|
||||||
|
|
||||||
|
The manager page keeps its filter state in localStorage for its own
|
||||||
|
restoration, but the ComfyUI node autocomplete runs in a potentially
|
||||||
|
different browser/origin (or Electron shell) where that storage is not
|
||||||
|
shared. This store mirrors the active filters server-side so the
|
||||||
|
``/api/lm/{prefix}/relative-paths`` endpoint can inject them into
|
||||||
|
autocomplete searches regardless of which client set them.
|
||||||
|
|
||||||
|
State is process-local and intentionally not persisted; the manager page
|
||||||
|
re-pushes its restored state on load.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Keys copied from the manager page's persisted filter snapshot.
|
||||||
|
_FILTER_KEYS = (
|
||||||
|
"baseModel",
|
||||||
|
"tags",
|
||||||
|
"autoTags",
|
||||||
|
"modelTypes",
|
||||||
|
"tagLogic",
|
||||||
|
"license",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ActiveFiltersStore:
|
||||||
|
"""Process-local store of active filters, keyed by model type."""
|
||||||
|
|
||||||
|
_instance: Optional["ActiveFiltersStore"] = None
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self._filters: Dict[str, Dict[str, Any]] = {}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_instance(cls) -> "ActiveFiltersStore":
|
||||||
|
if cls._instance is None:
|
||||||
|
cls._instance = cls()
|
||||||
|
return cls._instance
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def reset_instance(cls) -> None:
|
||||||
|
"""Drop the singleton (test isolation)."""
|
||||||
|
cls._instance = None
|
||||||
|
|
||||||
|
def set_filters(self, model_type: str, payload: Dict[str, Any]) -> None:
|
||||||
|
"""Replace the stored active filters for a model type.
|
||||||
|
|
||||||
|
Only recognized keys are kept; everything else is discarded.
|
||||||
|
"""
|
||||||
|
filters = payload.get("filters")
|
||||||
|
sanitized: Dict[str, Any] = {
|
||||||
|
"activeFolder": payload.get("activeFolder"),
|
||||||
|
"recursiveSearch": bool(payload.get("recursiveSearch", True)),
|
||||||
|
"filters": (
|
||||||
|
{key: filters[key] for key in _FILTER_KEYS if key in filters}
|
||||||
|
if isinstance(filters, dict)
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
}
|
||||||
|
self._filters[model_type] = sanitized
|
||||||
|
|
||||||
|
def get_filters(self, model_type: str) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Return the stored payload for a model type, or None if unset."""
|
||||||
|
return self._filters.get(model_type)
|
||||||
|
|
||||||
|
def clear(self, model_type: str) -> None:
|
||||||
|
self._filters.pop(model_type, None)
|
||||||
|
|
||||||
|
|
||||||
|
def active_filters_to_query_kwargs(payload: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
||||||
|
"""Map a stored active-filters payload to ``search_relative_paths`` kwargs.
|
||||||
|
|
||||||
|
Mirrors the query-param mapping that the ComfyUI autocomplete used to
|
||||||
|
build client-side from localStorage (web/comfyui/autocomplete.js).
|
||||||
|
"""
|
||||||
|
kwargs: Dict[str, Any] = {}
|
||||||
|
if not payload:
|
||||||
|
return kwargs
|
||||||
|
|
||||||
|
active_folder = payload.get("activeFolder")
|
||||||
|
recursive = payload.get("recursiveSearch", True)
|
||||||
|
|
||||||
|
if active_folder and active_folder != "null":
|
||||||
|
kwargs["folder"] = active_folder
|
||||||
|
elif not recursive:
|
||||||
|
# Root folder with recursion disabled mirrors the page list,
|
||||||
|
# which matches only root-level files via folder=''.
|
||||||
|
kwargs["folder"] = ""
|
||||||
|
|
||||||
|
filters = payload.get("filters")
|
||||||
|
if isinstance(filters, dict):
|
||||||
|
base_models = filters.get("baseModel")
|
||||||
|
if isinstance(base_models, list):
|
||||||
|
kwargs["base_models"] = [m for m in base_models if m]
|
||||||
|
|
||||||
|
for source_key, target_key in (("tags", "tags"), ("autoTags", "auto_tags")):
|
||||||
|
states = filters.get(source_key)
|
||||||
|
if isinstance(states, dict):
|
||||||
|
mapped = {
|
||||||
|
tag: state
|
||||||
|
for tag, state in states.items()
|
||||||
|
if state in ("include", "exclude")
|
||||||
|
}
|
||||||
|
if mapped:
|
||||||
|
kwargs[target_key] = mapped
|
||||||
|
|
||||||
|
model_types = filters.get("modelTypes")
|
||||||
|
if isinstance(model_types, list):
|
||||||
|
kwargs["model_types"] = [t for t in model_types if t]
|
||||||
|
|
||||||
|
tag_logic = filters.get("tagLogic")
|
||||||
|
if tag_logic:
|
||||||
|
kwargs["tag_logic"] = tag_logic
|
||||||
|
|
||||||
|
license_filter = filters.get("license")
|
||||||
|
if isinstance(license_filter, dict):
|
||||||
|
no_credit = license_filter.get("noCredit")
|
||||||
|
if no_credit == "include":
|
||||||
|
kwargs["credit_required"] = False
|
||||||
|
elif no_credit == "exclude":
|
||||||
|
kwargs["credit_required"] = True
|
||||||
|
allow_selling = license_filter.get("allowSelling")
|
||||||
|
if allow_selling == "include":
|
||||||
|
kwargs["allow_selling_generated_content"] = True
|
||||||
|
elif allow_selling == "exclude":
|
||||||
|
kwargs["allow_selling_generated_content"] = False
|
||||||
|
|
||||||
|
kwargs["recursive"] = recursive
|
||||||
|
return kwargs
|
||||||
@@ -407,7 +407,6 @@ class AgentService:
|
|||||||
"base_model": metadata.get("base_model", ""),
|
"base_model": metadata.get("base_model", ""),
|
||||||
"tags": metadata.get("tags", []),
|
"tags": metadata.get("tags", []),
|
||||||
"modelDescription": metadata.get("modelDescription", ""),
|
"modelDescription": metadata.get("modelDescription", ""),
|
||||||
"trainedWords": metadata.get("trainedWords", []),
|
|
||||||
"sha256": (metadata.get("sha256") or "")[:16] + "..." if metadata.get("sha256") else "",
|
"sha256": (metadata.get("sha256") or "")[:16] + "..." if metadata.get("sha256") else "",
|
||||||
"size": metadata.get("size", 0),
|
"size": metadata.get("size", 0),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -92,7 +92,10 @@ class PostProcessor:
|
|||||||
preview_downloaded = False
|
preview_downloaded = False
|
||||||
|
|
||||||
# -- Determine whether this is an HF-sourced model -----------------
|
# -- Determine whether this is an HF-sourced model -----------------
|
||||||
is_hf_model = not metadata.get("from_civitai", True)
|
# Key off `hf_url` directly: `from_civitai` records provenance and can
|
||||||
|
# be true for a model that is also linked to HuggingFace (both sources
|
||||||
|
# coexist, see #1094), so it must not gate HF enrichment.
|
||||||
|
is_hf_model = bool(metadata.get("hf_url", ""))
|
||||||
|
|
||||||
# -- Collect updates -----------------------------------------------
|
# -- Collect updates -----------------------------------------------
|
||||||
updates: Dict[str, Any] = {}
|
updates: Dict[str, Any] = {}
|
||||||
|
|||||||
+170
-23
@@ -82,6 +82,17 @@ CIVITAI_DOWNLOAD_URL_PREFIXES = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_no_uri_available_error(message: str) -> bool:
|
||||||
|
"""Return True for aria2's "No URI available" transfer failure.
|
||||||
|
|
||||||
|
aria2 reports this when every URI for the transfer has become unusable.
|
||||||
|
For CivitAI downloads this typically means the temporary signed URL
|
||||||
|
expired mid-download; the transfer can be recovered by resolving a fresh
|
||||||
|
signed URL and re-scheduling with ``continue=true``.
|
||||||
|
"""
|
||||||
|
return "no uri available" in message.lower()
|
||||||
|
|
||||||
|
|
||||||
class Aria2Error(RuntimeError):
|
class Aria2Error(RuntimeError):
|
||||||
"""Raised when aria2 integration fails."""
|
"""Raised when aria2 integration fails."""
|
||||||
|
|
||||||
@@ -145,8 +156,16 @@ class Aria2Downloader:
|
|||||||
disappears (e.g. another download restarted the daemon and
|
disappears (e.g. another download restarted the daemon and
|
||||||
``close()`` cleared ``_transfers``) or the RPC becomes unreachable,
|
``close()`` cleared ``_transfers``) or the RPC becomes unreachable,
|
||||||
the transfer is re-scheduled with ``continue=true`` so the download
|
the transfer is re-scheduled with ``continue=true`` so the download
|
||||||
resumes from the on-disk ``.aria2`` control file. Recovery is bounded
|
resumes from the on-disk ``.aria2`` control file. The same
|
||||||
by ``MAX_TRANSFER_RECOVERY_ATTEMPTS``.
|
re-scheduling happens when aria2 fails with "No URI available"
|
||||||
|
(typically an expired CivitAI signed URL): a fresh URL is resolved
|
||||||
|
and the partial download continues. Recovery is bounded by
|
||||||
|
``MAX_TRANSFER_RECOVERY_ATTEMPTS``.
|
||||||
|
|
||||||
|
Cancellation never leaks daemon transfers: the gid is tracked in
|
||||||
|
``_transfers`` before any post-``addUri`` await, and a gid accepted
|
||||||
|
by the daemon while the caller is being cancelled is removed again
|
||||||
|
before the ``CancelledError`` propagates.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
await self._ensure_process()
|
await self._ensure_process()
|
||||||
@@ -201,14 +220,47 @@ class Aria2Downloader:
|
|||||||
completed_path = self._resolve_completed_path(status, save_path)
|
completed_path = self._resolve_completed_path(status, save_path)
|
||||||
return True, completed_path
|
return True, completed_path
|
||||||
if state == "error":
|
if state == "error":
|
||||||
return False, status.get("errorMessage") or "aria2 download failed"
|
error_message = status.get("errorMessage") or "aria2 download failed"
|
||||||
|
if (
|
||||||
|
_is_no_uri_available_error(error_message)
|
||||||
|
and recovery_attempts < MAX_TRANSFER_RECOVERY_ATTEMPTS
|
||||||
|
):
|
||||||
|
# The signed URL (e.g. CivitAI's) expired before the
|
||||||
|
# transfer finished. Re-registering resolves a fresh
|
||||||
|
# URL and resumes from the on-disk partial payload and
|
||||||
|
# .aria2 control file via ``continue=true``.
|
||||||
|
recovery_attempts += 1
|
||||||
|
logger.warning(
|
||||||
|
"aria2 transfer %s failed with %r; refreshing the "
|
||||||
|
"URL and resuming the partial download "
|
||||||
|
"(attempt %d/%d)",
|
||||||
|
download_id,
|
||||||
|
error_message,
|
||||||
|
recovery_attempts,
|
||||||
|
MAX_TRANSFER_RECOVERY_ATTEMPTS,
|
||||||
|
)
|
||||||
|
await asyncio.sleep(1.0)
|
||||||
|
await self._ensure_process()
|
||||||
|
async with self._register_lock:
|
||||||
|
transfer = await self._register_transfer(
|
||||||
|
url,
|
||||||
|
save_path,
|
||||||
|
download_id=download_id,
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
return False, error_message
|
||||||
if state == "removed":
|
if state == "removed":
|
||||||
return False, "Download was cancelled"
|
return False, "Download was cancelled"
|
||||||
|
|
||||||
await asyncio.sleep(self._poll_interval)
|
await asyncio.sleep(self._poll_interval)
|
||||||
finally:
|
finally:
|
||||||
current = self._transfers.get(download_id)
|
current = self._transfers.get(download_id)
|
||||||
if current is not None and current.gid == transfer.gid:
|
if (
|
||||||
|
transfer is not None
|
||||||
|
and current is not None
|
||||||
|
and current.gid == transfer.gid
|
||||||
|
):
|
||||||
self._transfers.pop(download_id, None)
|
self._transfers.pop(download_id, None)
|
||||||
|
|
||||||
async def _get_status_with_retry(
|
async def _get_status_with_retry(
|
||||||
@@ -217,8 +269,9 @@ class Aria2Downloader:
|
|||||||
"""Call get_status with retry for transient RPC failures.
|
"""Call get_status with retry for transient RPC failures.
|
||||||
|
|
||||||
Only retries on :exc:`Aria2Error` (RPC-level failure). Returns
|
Only retries on :exc:`Aria2Error` (RPC-level failure). Returns
|
||||||
``None`` immediately when the download_id is not tracked (a missing
|
``None`` immediately when the transfer is not tracked or its GID is
|
||||||
transfer is not a transient condition, so retrying is pointless).
|
gone from the daemon (a missing transfer is not a transient
|
||||||
|
condition, so retrying is pointless).
|
||||||
|
|
||||||
A single failed RPC call should not immediately fail the download,
|
A single failed RPC call should not immediately fail the download,
|
||||||
because aria2 may be temporarily busy (e.g. finalizing multiple
|
because aria2 may be temporarily busy (e.g. finalizing multiple
|
||||||
@@ -295,21 +348,43 @@ class Aria2Downloader:
|
|||||||
resolved_url != url,
|
resolved_url != url,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Shield the addUri RPC from cancellation: the daemon may accept the
|
||||||
|
# download even when the caller is cancelled while the request is in
|
||||||
|
# flight. On cancellation, wait for the RPC result so the freshly
|
||||||
|
# created gid can be removed instead of leaking an untracked
|
||||||
|
# download that keeps running in the daemon.
|
||||||
|
add_task = asyncio.ensure_future(
|
||||||
|
self._rpc_call("aria2.addUri", [[resolved_url], options])
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
gid = await self._rpc_call("aria2.addUri", [[resolved_url], options])
|
gid = await asyncio.shield(add_task)
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
leaked_gid: Any = None
|
||||||
|
try:
|
||||||
|
leaked_gid = await add_task
|
||||||
|
except Exception:
|
||||||
|
leaked_gid = None
|
||||||
|
if isinstance(leaked_gid, str) and leaked_gid:
|
||||||
|
logger.info(
|
||||||
|
"Removing aria2 gid %s accepted while download %s was "
|
||||||
|
"being cancelled",
|
||||||
|
leaked_gid,
|
||||||
|
download_id,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
await self._rpc_call("aria2.forceRemove", [leaked_gid])
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(
|
||||||
|
"Failed to remove leaked aria2 gid %s for download %s: %s",
|
||||||
|
leaked_gid,
|
||||||
|
download_id,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise Aria2Error(f"Failed to schedule aria2 download: {exc}") from exc
|
raise Aria2Error(f"Failed to schedule aria2 download: {exc}") from exc
|
||||||
|
|
||||||
logger.debug("aria2 accepted download %s with gid %s", download_id, gid)
|
logger.debug("aria2 accepted download %s with gid %s", download_id, gid)
|
||||||
await self._state_store.upsert(
|
|
||||||
download_id,
|
|
||||||
{
|
|
||||||
"gid": gid,
|
|
||||||
"save_path": save_path,
|
|
||||||
"status": "downloading",
|
|
||||||
"url": url,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
return gid
|
return gid
|
||||||
|
|
||||||
async def _register_transfer(
|
async def _register_transfer(
|
||||||
@@ -328,11 +403,56 @@ class Aria2Downloader:
|
|||||||
headers=headers,
|
headers=headers,
|
||||||
)
|
)
|
||||||
transfer = Aria2Transfer(gid=gid, save_path=os.path.abspath(save_path))
|
transfer = Aria2Transfer(gid=gid, save_path=os.path.abspath(save_path))
|
||||||
|
# Register the transfer before any further await: once the daemon
|
||||||
|
# holds the gid, cancel_download() must be able to find it. An await
|
||||||
|
# in between would open a window where a concurrent cancel reports
|
||||||
|
# "Download task not found" and the daemon keeps downloading
|
||||||
|
# untracked.
|
||||||
self._transfers[download_id] = transfer
|
self._transfers[download_id] = transfer
|
||||||
|
try:
|
||||||
|
await self._state_store.upsert(
|
||||||
|
download_id,
|
||||||
|
{
|
||||||
|
"gid": gid,
|
||||||
|
"save_path": transfer.save_path,
|
||||||
|
"status": "downloading",
|
||||||
|
"url": url,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
# The task was cancelled while persisting state and the
|
||||||
|
# coordinator's cancel ran before the transfer was registered
|
||||||
|
# above. Remove the daemon transfer unless it was deliberately
|
||||||
|
# paused (skip_download preserves paused transfers for resume).
|
||||||
|
status = None
|
||||||
|
try:
|
||||||
|
status = await self.get_status(download_id)
|
||||||
|
except Exception:
|
||||||
|
status = None
|
||||||
|
if status is not None and status.get("status") != "paused":
|
||||||
|
try:
|
||||||
|
await self._rpc_call("aria2.forceRemove", [gid])
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(
|
||||||
|
"Failed to remove aria2 gid %s for cancelled download %s: %s",
|
||||||
|
gid,
|
||||||
|
download_id,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
current = self._transfers.get(download_id)
|
||||||
|
if current is not None and current.gid == gid:
|
||||||
|
self._transfers.pop(download_id, None)
|
||||||
|
raise
|
||||||
return transfer
|
return transfer
|
||||||
|
|
||||||
async def get_status(self, download_id: str) -> Optional[Dict[str, Any]]:
|
async def get_status(self, download_id: str) -> Optional[Dict[str, Any]]:
|
||||||
"""Return the raw aria2 status payload for a known download."""
|
"""Return the raw aria2 status payload for a known download.
|
||||||
|
|
||||||
|
Returns ``None`` when the download_id is not tracked or the daemon no
|
||||||
|
longer knows the transfer's GID (daemon restart / forceRemove). A
|
||||||
|
forgotten GID is permanent, not transient, so the caller's recovery
|
||||||
|
path handles it instead of burning retry attempts on a dead GID.
|
||||||
|
"""
|
||||||
|
|
||||||
transfer = self._transfers.get(download_id)
|
transfer = self._transfers.get(download_id)
|
||||||
if transfer is None:
|
if transfer is None:
|
||||||
@@ -348,8 +468,17 @@ class Aria2Downloader:
|
|||||||
"files",
|
"files",
|
||||||
]
|
]
|
||||||
try:
|
try:
|
||||||
status = await self._rpc_call("aria2.tellStatus", [transfer.gid, keys])
|
status = await self._rpc_call(
|
||||||
|
"aria2.tellStatus", [transfer.gid, keys], log_errors=False
|
||||||
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
if "not found" in str(exc).lower():
|
||||||
|
logger.debug(
|
||||||
|
"aria2 GID %s for download %s is gone; treating as lost transfer",
|
||||||
|
transfer.gid,
|
||||||
|
download_id,
|
||||||
|
)
|
||||||
|
return None
|
||||||
raise Aria2Error(f"Failed to query aria2 download status: {exc}") from exc
|
raise Aria2Error(f"Failed to query aria2 download status: {exc}") from exc
|
||||||
|
|
||||||
if isinstance(status, dict):
|
if isinstance(status, dict):
|
||||||
@@ -367,7 +496,9 @@ class Aria2Downloader:
|
|||||||
"files",
|
"files",
|
||||||
]
|
]
|
||||||
try:
|
try:
|
||||||
status = await self._rpc_call("aria2.tellStatus", [gid, keys])
|
status = await self._rpc_call(
|
||||||
|
"aria2.tellStatus", [gid, keys], log_errors=False
|
||||||
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
message = str(exc)
|
message = str(exc)
|
||||||
if "cannot be found" in message.lower() or "not found" in message.lower():
|
if "cannot be found" in message.lower() or "not found" in message.lower():
|
||||||
@@ -434,8 +565,19 @@ class Aria2Downloader:
|
|||||||
try:
|
try:
|
||||||
await self._rpc_call("aria2.forceRemove", [transfer.gid])
|
await self._rpc_call("aria2.forceRemove", [transfer.gid])
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
return {"success": False, "error": str(exc)}
|
if "not found" not in str(exc).lower():
|
||||||
|
return {"success": False, "error": str(exc)}
|
||||||
|
# The daemon already forgot this GID (restart / prior removal),
|
||||||
|
# so the transfer is effectively cancelled.
|
||||||
|
logger.debug(
|
||||||
|
"aria2 GID %s for download %s already gone during cancel",
|
||||||
|
transfer.gid,
|
||||||
|
download_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Drop the in-memory entry as well so a concurrent poll loop does
|
||||||
|
# not mistake the removal for a lost transfer and re-register it.
|
||||||
|
self._transfers.pop(download_id, None)
|
||||||
await self._state_store.remove(download_id)
|
await self._state_store.remove(download_id)
|
||||||
return {"success": True, "message": "Download cancelled successfully"}
|
return {"success": True, "message": "Download cancelled successfully"}
|
||||||
|
|
||||||
@@ -725,7 +867,9 @@ class Aria2Downloader:
|
|||||||
|
|
||||||
return isinstance(result, dict)
|
return isinstance(result, dict)
|
||||||
|
|
||||||
async def _rpc_call(self, method: str, params: list[Any]) -> Any:
|
async def _rpc_call(
|
||||||
|
self, method: str, params: list[Any], *, log_errors: bool = True
|
||||||
|
) -> Any:
|
||||||
if not self._rpc_url:
|
if not self._rpc_url:
|
||||||
raise Aria2Error("aria2 RPC endpoint is not initialized")
|
raise Aria2Error("aria2 RPC endpoint is not initialized")
|
||||||
|
|
||||||
@@ -756,7 +900,10 @@ class Aria2Downloader:
|
|||||||
error = body["error"] or {}
|
error = body["error"] or {}
|
||||||
code = error.get("code") if isinstance(error, dict) else None
|
code = error.get("code") if isinstance(error, dict) else None
|
||||||
message = error.get("message") if isinstance(error, dict) else str(error)
|
message = error.get("message") if isinstance(error, dict) else str(error)
|
||||||
logger.error(
|
# Probing calls (e.g. tellStatus for a GID the daemon may have
|
||||||
|
# forgotten) pass log_errors=False: an expected "not found" must
|
||||||
|
# not spam the log at ERROR level.
|
||||||
|
(logger.error if log_errors else logger.debug)(
|
||||||
"aria2 RPC %s failed with HTTP %s, code=%s, message=%s",
|
"aria2 RPC %s failed with HTTP %s, code=%s, message=%s",
|
||||||
method,
|
method,
|
||||||
response.status,
|
response.status,
|
||||||
@@ -771,7 +918,7 @@ class Aria2Downloader:
|
|||||||
raise Aria2Error(status_message or "Unknown aria2 RPC error")
|
raise Aria2Error(status_message or "Unknown aria2 RPC error")
|
||||||
|
|
||||||
if response.status != 200:
|
if response.status != 200:
|
||||||
logger.error(
|
(logger.error if log_errors else logger.debug)(
|
||||||
"aria2 RPC %s returned unexpected HTTP status %s without error payload: %s",
|
"aria2 RPC %s returned unexpected HTTP status %s without error payload: %s",
|
||||||
method,
|
method,
|
||||||
response.status,
|
response.status,
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from ..utils.constants import VALID_LORA_SUB_TYPES, VALID_CHECKPOINT_SUB_TYPES
|
from ..utils.constants import VALID_LORA_SUB_TYPES, VALID_CHECKPOINT_SUB_TYPES, VALID_OTHER_SUB_TYPES
|
||||||
from ..utils.models import BaseModelMetadata
|
from ..utils.models import BaseModelMetadata
|
||||||
from ..utils.metadata_manager import MetadataManager
|
from ..utils.metadata_manager import MetadataManager
|
||||||
from ..utils.usage_stats import UsageStats
|
from ..utils.usage_stats import UsageStats
|
||||||
@@ -904,6 +904,11 @@ class BaseModelService(ABC):
|
|||||||
and normalized_type not in VALID_CHECKPOINT_SUB_TYPES
|
and normalized_type not in VALID_CHECKPOINT_SUB_TYPES
|
||||||
):
|
):
|
||||||
continue
|
continue
|
||||||
|
if (
|
||||||
|
self.model_type == "other"
|
||||||
|
and normalized_type not in VALID_OTHER_SUB_TYPES
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
|
||||||
type_counts[normalized_type] = type_counts.get(normalized_type, 0) + 1
|
type_counts[normalized_type] = type_counts.get(normalized_type, 0) + 1
|
||||||
|
|
||||||
@@ -972,14 +977,25 @@ class BaseModelService(ABC):
|
|||||||
)
|
)
|
||||||
return {k: data[k] for k in fields if k in data}
|
return {k: data[k] for k in fields if k in data}
|
||||||
|
|
||||||
async def get_folder_tree(self, model_root: str) -> Dict[str, Any]:
|
async def _get_tree_folders(self, cache, include_empty: bool) -> List[str]:
|
||||||
|
"""Return the folder list backing folder tree responses.
|
||||||
|
|
||||||
|
With ``include_empty`` the directories are enumerated live from the
|
||||||
|
filesystem (including empty ones) via the scanner; otherwise the
|
||||||
|
models-only ``cache.folders`` list is used unchanged.
|
||||||
|
"""
|
||||||
|
if include_empty:
|
||||||
|
return await self.scanner.get_all_folders()
|
||||||
|
return cache.folders
|
||||||
|
|
||||||
|
async def get_folder_tree(self, model_root: str, include_empty: bool = False) -> Dict[str, Any]:
|
||||||
"""Get hierarchical folder tree for a specific model root"""
|
"""Get hierarchical folder tree for a specific model root"""
|
||||||
cache = await self.scanner.get_cached_data()
|
cache = await self.scanner.get_cached_data()
|
||||||
|
|
||||||
# Build tree structure from folders
|
# Build tree structure from folders
|
||||||
tree = {}
|
tree = {}
|
||||||
|
|
||||||
for folder in cache.folders:
|
for folder in await self._get_tree_folders(cache, include_empty):
|
||||||
# Check if this folder belongs to the specified model root
|
# Check if this folder belongs to the specified model root
|
||||||
folder_belongs_to_root = False
|
folder_belongs_to_root = False
|
||||||
for root in self.scanner.get_model_roots():
|
for root in self.scanner.get_model_roots():
|
||||||
@@ -1001,7 +1017,7 @@ class BaseModelService(ABC):
|
|||||||
|
|
||||||
return tree
|
return tree
|
||||||
|
|
||||||
async def get_unified_folder_tree(self) -> Dict[str, Any]:
|
async def get_unified_folder_tree(self, include_empty: bool = False) -> Dict[str, Any]:
|
||||||
"""Get unified folder tree across all model roots"""
|
"""Get unified folder tree across all model roots"""
|
||||||
cache = await self.scanner.get_cached_data()
|
cache = await self.scanner.get_cached_data()
|
||||||
|
|
||||||
@@ -1011,7 +1027,7 @@ class BaseModelService(ABC):
|
|||||||
# Get all model roots for path normalization
|
# Get all model roots for path normalization
|
||||||
model_roots = self.scanner.get_model_roots()
|
model_roots = self.scanner.get_model_roots()
|
||||||
|
|
||||||
for folder in cache.folders:
|
for folder in await self._get_tree_folders(cache, include_empty):
|
||||||
if not folder: # Skip empty folders
|
if not folder: # Skip empty folders
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -1284,6 +1300,27 @@ class BaseModelService(ABC):
|
|||||||
path_for_sorting,
|
path_for_sorting,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _relative_path_folder_group_sort_key(
|
||||||
|
relative_path: str, include_terms: List[str]
|
||||||
|
) -> tuple:
|
||||||
|
"""Group paths by folder, then sort by relevance within each group.
|
||||||
|
|
||||||
|
Folders are ordered alphabetically (case-insensitive) by their full
|
||||||
|
folder path, with root-level files (empty folder) first. Within a
|
||||||
|
folder, paths keep the relevance ordering of
|
||||||
|
``_relative_path_sort_key``. This keeps same-folder entries together
|
||||||
|
in the autocomplete dropdown instead of interleaving them by filename.
|
||||||
|
"""
|
||||||
|
path_for_sorting = BaseModelService._remove_model_extension(
|
||||||
|
relative_path.lower()
|
||||||
|
)
|
||||||
|
folder = path_for_sorting.rpartition(os.sep)[0]
|
||||||
|
|
||||||
|
return (folder,) + BaseModelService._relative_path_sort_key(
|
||||||
|
relative_path, include_terms
|
||||||
|
)
|
||||||
|
|
||||||
async def search_relative_paths(
|
async def search_relative_paths(
|
||||||
self,
|
self,
|
||||||
search_term: str,
|
search_term: str,
|
||||||
@@ -1393,9 +1430,13 @@ class BaseModelService(ABC):
|
|||||||
):
|
):
|
||||||
matching_paths.append(relative_path)
|
matching_paths.append(relative_path)
|
||||||
|
|
||||||
# Sort by relevance (prefix and earliest hits first, then by length and alphabetically)
|
# Group by folder (root first, then alphabetically) and sort by
|
||||||
|
# relevance (prefix and earliest hits, then length and alphabetically)
|
||||||
|
# within each folder group.
|
||||||
matching_paths.sort(
|
matching_paths.sort(
|
||||||
key=lambda relative: self._relative_path_sort_key(relative, include_terms)
|
key=lambda relative: self._relative_path_folder_group_sort_key(
|
||||||
|
relative, include_terms
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# Apply offset and limit
|
# Apply offset and limit
|
||||||
|
|||||||
@@ -20,6 +20,11 @@ from .recipes import (
|
|||||||
RecipeDownloadError,
|
RecipeDownloadError,
|
||||||
RecipeNotFoundError,
|
RecipeNotFoundError,
|
||||||
)
|
)
|
||||||
|
from .recipes.import_info import (
|
||||||
|
CHANNEL_BATCH_IMPORT_LOCAL,
|
||||||
|
CHANNEL_BATCH_IMPORT_URL,
|
||||||
|
build_import_info,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class ImportItemType(Enum):
|
class ImportItemType(Enum):
|
||||||
@@ -71,6 +76,9 @@ class BatchImportProgress:
|
|||||||
tags: List[str] = field(default_factory=list)
|
tags: List[str] = field(default_factory=list)
|
||||||
skip_no_metadata: bool = False
|
skip_no_metadata: bool = False
|
||||||
skip_duplicates: bool = False
|
skip_duplicates: bool = False
|
||||||
|
# Set once any item is skipped due to vendor rate limiting (#1085); lets
|
||||||
|
# the UI surface a "slowing down / try again later" hint.
|
||||||
|
rate_limited: bool = False
|
||||||
|
|
||||||
def to_dict(self) -> Dict[str, Any]:
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
return {
|
return {
|
||||||
@@ -82,6 +90,7 @@ class BatchImportProgress:
|
|||||||
"skipped": self.skipped,
|
"skipped": self.skipped,
|
||||||
"current_item": self.current_item,
|
"current_item": self.current_item,
|
||||||
"status": self.status,
|
"status": self.status,
|
||||||
|
"rate_limited": self.rate_limited,
|
||||||
"started_at": self.started_at,
|
"started_at": self.started_at,
|
||||||
"finished_at": self.finished_at,
|
"finished_at": self.finished_at,
|
||||||
"progress_percent": round((self.completed / self.total) * 100, 1)
|
"progress_percent": round((self.completed / self.total) * 100, 1)
|
||||||
@@ -118,6 +127,10 @@ class AdaptiveConcurrencyController:
|
|||||||
self._task_durations: List[float] = []
|
self._task_durations: List[float] = []
|
||||||
self._recent_errors = 0
|
self._recent_errors = 0
|
||||||
self._recent_successes = 0
|
self._recent_successes = 0
|
||||||
|
# Batch-wide shared semaphore; created lazily on first use so the
|
||||||
|
# controller can also be constructed outside a running event loop.
|
||||||
|
self._semaphore: Optional[asyncio.Semaphore] = None
|
||||||
|
self._semaphore_capacity = initial_concurrency
|
||||||
|
|
||||||
def record_result(self, duration: float, success: bool) -> None:
|
def record_result(self, duration: float, success: bool) -> None:
|
||||||
self._task_durations.append(duration)
|
self._task_durations.append(duration)
|
||||||
@@ -146,7 +159,37 @@ class AdaptiveConcurrencyController:
|
|||||||
self._recent_successes = 0
|
self._recent_successes = 0
|
||||||
|
|
||||||
def get_semaphore(self) -> asyncio.Semaphore:
|
def get_semaphore(self) -> asyncio.Semaphore:
|
||||||
return asyncio.Semaphore(self.current_concurrency)
|
"""Return the batch-wide shared semaphore.
|
||||||
|
|
||||||
|
The same semaphore instance is returned for every item of a batch so
|
||||||
|
the configured concurrency bounds are actually enforced. Previously a
|
||||||
|
fresh semaphore was created per call, letting every item run
|
||||||
|
concurrently and hammering remote metadata providers without any
|
||||||
|
limit.
|
||||||
|
"""
|
||||||
|
if self._semaphore is None:
|
||||||
|
self._semaphore = asyncio.Semaphore(self.current_concurrency)
|
||||||
|
self._semaphore_capacity = self.current_concurrency
|
||||||
|
return self._semaphore
|
||||||
|
|
||||||
|
async def apply_concurrency(self) -> None:
|
||||||
|
"""Synchronize the shared semaphore capacity with ``current_concurrency``.
|
||||||
|
|
||||||
|
Call after ``record_result`` (once per completed item). Growing the
|
||||||
|
capacity is immediate (release). Shrinking requires acquiring a permit
|
||||||
|
and holding it, which is best-effort while other tasks are still
|
||||||
|
running — the capacity converges on subsequent calls.
|
||||||
|
"""
|
||||||
|
semaphore = self.get_semaphore()
|
||||||
|
while self._semaphore_capacity < self.current_concurrency:
|
||||||
|
semaphore.release()
|
||||||
|
self._semaphore_capacity += 1
|
||||||
|
while self._semaphore_capacity > self.current_concurrency:
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(semaphore.acquire(), timeout=0.01)
|
||||||
|
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||||
|
break
|
||||||
|
self._semaphore_capacity -= 1
|
||||||
|
|
||||||
|
|
||||||
class BatchImportService:
|
class BatchImportService:
|
||||||
@@ -184,6 +227,7 @@ class BatchImportService:
|
|||||||
def cancel_import(self, operation_id: str) -> bool:
|
def cancel_import(self, operation_id: str) -> bool:
|
||||||
if operation_id in self._active_operations:
|
if operation_id in self._active_operations:
|
||||||
self._cancellation_flags[operation_id] = True
|
self._cancellation_flags[operation_id] = True
|
||||||
|
self._logger.info("Cancel requested for batch import operation %s", operation_id)
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -273,6 +317,14 @@ class BatchImportService:
|
|||||||
self._active_operations[operation_id] = progress
|
self._active_operations[operation_id] = progress
|
||||||
self._cancellation_flags[operation_id] = False
|
self._cancellation_flags[operation_id] = False
|
||||||
|
|
||||||
|
self._logger.info(
|
||||||
|
"Starting batch import operation %s: %d item(s) (%d URL(s), %d local path(s))",
|
||||||
|
operation_id,
|
||||||
|
len(import_items),
|
||||||
|
sum(1 for it in import_items if it.item_type == ImportItemType.URL),
|
||||||
|
sum(1 for it in import_items if it.item_type == ImportItemType.LOCAL_PATH),
|
||||||
|
)
|
||||||
|
|
||||||
asyncio.create_task(
|
asyncio.create_task(
|
||||||
self._run_batch_import(
|
self._run_batch_import(
|
||||||
operation_id=operation_id,
|
operation_id=operation_id,
|
||||||
@@ -295,6 +347,12 @@ class BatchImportService:
|
|||||||
skip_duplicates: bool = False,
|
skip_duplicates: bool = False,
|
||||||
) -> str:
|
) -> str:
|
||||||
image_paths = await self._discover_images(directory, recursive)
|
image_paths = await self._discover_images(directory, recursive)
|
||||||
|
self._logger.info(
|
||||||
|
"Batch import directory scan: %d image(s) discovered in %s (recursive=%s)",
|
||||||
|
len(image_paths),
|
||||||
|
directory,
|
||||||
|
recursive,
|
||||||
|
)
|
||||||
|
|
||||||
items = [{"source": path, "type": "local_path"} for path in image_paths]
|
items = [{"source": path, "type": "local_path"} for path in image_paths]
|
||||||
|
|
||||||
@@ -334,6 +392,13 @@ class BatchImportService:
|
|||||||
ext = os.path.splitext(filename)[1].lower()
|
ext = os.path.splitext(filename)[1].lower()
|
||||||
return ext in self.SUPPORTED_EXTENSIONS
|
return ext in self.SUPPORTED_EXTENSIONS
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_rate_limit_error(error: Optional[str]) -> bool:
|
||||||
|
"""Return True when an error payload represents vendor rate limiting."""
|
||||||
|
if not error:
|
||||||
|
return False
|
||||||
|
return "rate limit" in error.lower()
|
||||||
|
|
||||||
async def _run_batch_import(
|
async def _run_batch_import(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -379,6 +444,9 @@ class BatchImportService:
|
|||||||
self._concurrency_controller.record_result(
|
self._concurrency_controller.record_result(
|
||||||
duration, result.get("success", False)
|
duration, result.get("success", False)
|
||||||
)
|
)
|
||||||
|
# Keep the shared batch semaphore in sync with the adaptively
|
||||||
|
# adjusted concurrency so the bounds actually take effect.
|
||||||
|
await self._concurrency_controller.apply_concurrency()
|
||||||
|
|
||||||
if result.get("success"):
|
if result.get("success"):
|
||||||
item.status = ImportStatus.SUCCESS
|
item.status = ImportStatus.SUCCESS
|
||||||
@@ -389,6 +457,17 @@ class BatchImportService:
|
|||||||
item.status = ImportStatus.SKIPPED
|
item.status = ImportStatus.SKIPPED
|
||||||
item.error_message = result.get("error")
|
item.error_message = result.get("error")
|
||||||
progress.skipped += 1
|
progress.skipped += 1
|
||||||
|
elif self._is_rate_limit_error(result.get("error")):
|
||||||
|
# Vendor rate limit is a transient, external condition —
|
||||||
|
# do not pollute the failure count with it (#1085). The
|
||||||
|
# import can simply be re-run later.
|
||||||
|
item.status = ImportStatus.SKIPPED
|
||||||
|
item.error_message = (
|
||||||
|
f"Rate limited by metadata provider; "
|
||||||
|
f"re-run the import later ({result.get('error')})"
|
||||||
|
)
|
||||||
|
progress.skipped += 1
|
||||||
|
progress.rate_limited = True
|
||||||
else:
|
else:
|
||||||
item.status = ImportStatus.FAILED
|
item.status = ImportStatus.FAILED
|
||||||
item.error_message = result.get("error")
|
item.error_message = result.get("error")
|
||||||
@@ -396,13 +475,36 @@ class BatchImportService:
|
|||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self._logger.error(f"Error importing {item.source}: {e}")
|
self._logger.error(f"Error importing {item.source}: {e}")
|
||||||
item.status = ImportStatus.FAILED
|
|
||||||
item.error_message = str(e)
|
|
||||||
item.duration = time.time() - start_time
|
item.duration = time.time() - start_time
|
||||||
progress.failed += 1
|
if self._is_rate_limit_error(str(e)):
|
||||||
|
item.status = ImportStatus.SKIPPED
|
||||||
|
item.error_message = (
|
||||||
|
f"Rate limited by metadata provider; "
|
||||||
|
f"re-run the import later ({e})"
|
||||||
|
)
|
||||||
|
progress.skipped += 1
|
||||||
|
progress.rate_limited = True
|
||||||
|
else:
|
||||||
|
item.status = ImportStatus.FAILED
|
||||||
|
item.error_message = str(e)
|
||||||
|
progress.failed += 1
|
||||||
self._concurrency_controller.record_result(item.duration, False)
|
self._concurrency_controller.record_result(item.duration, False)
|
||||||
|
await self._concurrency_controller.apply_concurrency()
|
||||||
|
|
||||||
progress.completed += 1
|
progress.completed += 1
|
||||||
|
self._logger.info(
|
||||||
|
"Batch import %s: item %d/%d status=%s source=%s%s",
|
||||||
|
operation_id,
|
||||||
|
progress.completed,
|
||||||
|
progress.total,
|
||||||
|
item.status.value,
|
||||||
|
(
|
||||||
|
os.path.basename(item.source)
|
||||||
|
if item.item_type == ImportItemType.LOCAL_PATH
|
||||||
|
else item.source[:50]
|
||||||
|
),
|
||||||
|
(f" error={item.error_message}" if item.error_message else ""),
|
||||||
|
)
|
||||||
await self._broadcast_progress(progress)
|
await self._broadcast_progress(progress)
|
||||||
|
|
||||||
tasks = [process_item(item) for item in progress.items]
|
tasks = [process_item(item) for item in progress.items]
|
||||||
@@ -415,6 +517,15 @@ class BatchImportService:
|
|||||||
|
|
||||||
progress.finished_at = time.time()
|
progress.finished_at = time.time()
|
||||||
progress.current_item = ""
|
progress.current_item = ""
|
||||||
|
self._logger.info(
|
||||||
|
"Batch import %s finished: status=%s total=%d success=%d failed=%d skipped=%d",
|
||||||
|
operation_id,
|
||||||
|
progress.status,
|
||||||
|
progress.total,
|
||||||
|
progress.success,
|
||||||
|
progress.failed,
|
||||||
|
progress.skipped,
|
||||||
|
)
|
||||||
await self._broadcast_progress(progress)
|
await self._broadcast_progress(progress)
|
||||||
|
|
||||||
await asyncio.sleep(5)
|
await asyncio.sleep(5)
|
||||||
@@ -518,6 +629,17 @@ class BatchImportService:
|
|||||||
"loras": loras,
|
"loras": loras,
|
||||||
"gen_params": payload.get("gen_params", {}),
|
"gen_params": payload.get("gen_params", {}),
|
||||||
"source_path": item.source,
|
"source_path": item.source,
|
||||||
|
# Record why this import ended up with no LoRAs so the
|
||||||
|
# recipe modal can explain it (collapsed by default).
|
||||||
|
"import_info": build_import_info(
|
||||||
|
(
|
||||||
|
CHANNEL_BATCH_IMPORT_URL
|
||||||
|
if item.item_type == ImportItemType.URL
|
||||||
|
else CHANNEL_BATCH_IMPORT_LOCAL
|
||||||
|
),
|
||||||
|
payload.get("diagnostics"),
|
||||||
|
loras,
|
||||||
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
if payload.get("checkpoint"):
|
if payload.get("checkpoint"):
|
||||||
@@ -595,3 +717,6 @@ class BatchImportService:
|
|||||||
def _cleanup_operation(self, operation_id: str) -> None:
|
def _cleanup_operation(self, operation_id: str) -> None:
|
||||||
if operation_id in self._cancellation_flags:
|
if operation_id in self._cancellation_flags:
|
||||||
del self._cancellation_flags[operation_id]
|
del self._cancellation_flags[operation_id]
|
||||||
|
if operation_id in self._active_operations:
|
||||||
|
del self._active_operations[operation_id]
|
||||||
|
self._logger.info("Batch import operation %s cleaned up", operation_id)
|
||||||
|
|||||||
@@ -410,6 +410,10 @@ class CheckpointScanner(ModelScanner):
|
|||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def resolve_sub_type_for_path(self, file_path: Optional[str]) -> Optional[str]:
|
||||||
|
"""Resolve sub_type from the configured root that contains the file."""
|
||||||
|
return self._resolve_sub_type(self._find_root_for_file(file_path))
|
||||||
|
|
||||||
def adjust_metadata(self, metadata, file_path, root_path):
|
def adjust_metadata(self, metadata, file_path, root_path):
|
||||||
"""Adjust metadata during scanning to set sub_type."""
|
"""Adjust metadata during scanning to set sub_type."""
|
||||||
sub_type = self._resolve_sub_type(root_path)
|
sub_type = self._resolve_sub_type(root_path)
|
||||||
@@ -419,9 +423,7 @@ class CheckpointScanner(ModelScanner):
|
|||||||
|
|
||||||
def adjust_cached_entry(self, entry: Dict[str, Any]) -> Dict[str, Any]:
|
def adjust_cached_entry(self, entry: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
"""Adjust entries loaded from the persisted cache to ensure sub_type is set."""
|
"""Adjust entries loaded from the persisted cache to ensure sub_type is set."""
|
||||||
sub_type = self._resolve_sub_type(
|
sub_type = self.resolve_sub_type_for_path(entry.get("file_path"))
|
||||||
self._find_root_for_file(entry.get("file_path"))
|
|
||||||
)
|
|
||||||
if sub_type:
|
if sub_type:
|
||||||
entry["sub_type"] = sub_type
|
entry["sub_type"] = sub_type
|
||||||
return entry
|
return entry
|
||||||
|
|||||||
@@ -51,6 +51,7 @@ class CheckpointService(BaseModelService):
|
|||||||
"base_model": model_data.get("base_model", ""),
|
"base_model": model_data.get("base_model", ""),
|
||||||
"folder": folder,
|
"folder": folder,
|
||||||
"sha256": model_data.get("sha256", ""),
|
"sha256": model_data.get("sha256", ""),
|
||||||
|
"autov3": model_data.get("autov3"),
|
||||||
"file_path": file_path.replace(os.sep, "/"),
|
"file_path": file_path.replace(os.sep, "/"),
|
||||||
"file_size": model_data.get("size", 0),
|
"file_size": model_data.get("size", 0),
|
||||||
"modified": model_data.get("modified", ""),
|
"modified": model_data.get("modified", ""),
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import logging
|
|||||||
import asyncio
|
import asyncio
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
from typing import Any, Optional, Dict, Tuple, List, cast
|
from typing import Any, Optional, Dict, Tuple, List, cast
|
||||||
|
from .connectivity_guard import is_expected_offline_error
|
||||||
from .model_metadata_provider import CivArchiveModelMetadataProvider, ModelMetadataProviderManager
|
from .model_metadata_provider import CivArchiveModelMetadataProvider, ModelMetadataProviderManager
|
||||||
from .downloader import get_downloader
|
from .downloader import get_downloader
|
||||||
from .errors import RateLimitError
|
from .errors import RateLimitError
|
||||||
@@ -46,7 +47,11 @@ class CivArchiveClient:
|
|||||||
"""Call CivArchive API and return JSON payload"""
|
"""Call CivArchive API and return JSON payload"""
|
||||||
success, payload = await self._make_request(path, params=params)
|
success, payload = await self._make_request(path, params=params)
|
||||||
if not success:
|
if not success:
|
||||||
error = payload if isinstance(payload, str) else "Request failed"
|
# Normalize empty-string failure payloads (e.g. a throttled
|
||||||
|
# connection dropped without a message) so callers never see a
|
||||||
|
# falsy error alongside a None payload — that combination used to
|
||||||
|
# crash downstream None.get() calls.
|
||||||
|
error = payload if isinstance(payload, str) and payload else "Request failed"
|
||||||
return None, error
|
return None, error
|
||||||
if not isinstance(payload, dict):
|
if not isinstance(payload, dict):
|
||||||
return None, "Invalid response structure"
|
return None, "Invalid response structure"
|
||||||
@@ -298,6 +303,8 @@ class CivArchiveClient:
|
|||||||
|
|
||||||
async def _resolve_version_from_files(self, payload: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
async def _resolve_version_from_files(self, payload: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||||
"""Fallback to fetch version data when only file metadata is available"""
|
"""Fallback to fetch version data when only file metadata is available"""
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
return None
|
||||||
data = self._normalize_payload(payload)
|
data = self._normalize_payload(payload)
|
||||||
files = data.get("files") or payload.get("files") or []
|
files = data.get("files") or payload.get("files") or []
|
||||||
if not isinstance(files, list):
|
if not isinstance(files, list):
|
||||||
@@ -332,10 +339,13 @@ class CivArchiveClient:
|
|||||||
"""Find model by SHA256 hash value using CivArchive API"""
|
"""Find model by SHA256 hash value using CivArchive API"""
|
||||||
try:
|
try:
|
||||||
payload, error = await self._request_json(f"/sha256/{model_hash.lower()}")
|
payload, error = await self._request_json(f"/sha256/{model_hash.lower()}")
|
||||||
if error:
|
# Treat a missing payload as an error even when the error string is
|
||||||
if "not found" in error.lower():
|
# falsy; passing None into the split/transform helpers below used to
|
||||||
|
# crash with "'NoneType' object has no attribute 'get'".
|
||||||
|
if error is not None or payload is None:
|
||||||
|
if error and "not found" in error.lower():
|
||||||
return None, "Model not found"
|
return None, "Model not found"
|
||||||
return None, error
|
return None, error or "Request failed"
|
||||||
|
|
||||||
context, version_data, fallback_files = self._split_context(cast(Dict[str, Any], payload))
|
context, version_data, fallback_files = self._split_context(cast(Dict[str, Any], payload))
|
||||||
transformed = self._transform_version(context, version_data, fallback_files)
|
transformed = self._transform_version(context, version_data, fallback_files)
|
||||||
@@ -352,7 +362,14 @@ class CivArchiveClient:
|
|||||||
except RateLimitError:
|
except RateLimitError:
|
||||||
raise
|
raise
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error fetching CivArchive model by hash {model_hash[:10]}: {e}")
|
if is_expected_offline_error(str(e)):
|
||||||
|
logger.debug(
|
||||||
|
"Skipping CivArchive model by hash %s while offline: %s",
|
||||||
|
model_hash[:10],
|
||||||
|
e,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.error(f"Error fetching CivArchive model by hash {model_hash[:10]}: {e}")
|
||||||
return None, str(e)
|
return None, str(e)
|
||||||
|
|
||||||
async def get_model_versions(self, model_id: str) -> Optional[Dict[str, Any]]:
|
async def get_model_versions(self, model_id: str) -> Optional[Dict[str, Any]]:
|
||||||
@@ -362,7 +379,14 @@ class CivArchiveClient:
|
|||||||
if error or payload is None:
|
if error or payload is None:
|
||||||
if error and "not found" in error.lower():
|
if error and "not found" in error.lower():
|
||||||
return None
|
return None
|
||||||
logger.error(f"Error fetching CivArchive model versions for {model_id}: {error}")
|
if is_expected_offline_error(error):
|
||||||
|
logger.debug(
|
||||||
|
"Skipping CivArchive model versions fetch for %s while offline: %s",
|
||||||
|
model_id,
|
||||||
|
error,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.error(f"Error fetching CivArchive model versions for {model_id}: {error}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
data = self._normalize_payload(payload)
|
data = self._normalize_payload(payload)
|
||||||
@@ -426,7 +450,19 @@ class CivArchiveClient:
|
|||||||
if error or payload is None:
|
if error or payload is None:
|
||||||
if error and "not found" in error.lower():
|
if error and "not found" in error.lower():
|
||||||
return None
|
return None
|
||||||
logger.error(f"Error fetching CivArchive model version via API {model_id}/{version_id}: {error}")
|
# The connectivity guard short-circuits requests during its
|
||||||
|
# offline cooldown; that is an expected, transient state, so
|
||||||
|
# log it as DEBUG instead of spamming one ERROR per request
|
||||||
|
# (batch imports can hit this thousands of times).
|
||||||
|
if is_expected_offline_error(error):
|
||||||
|
logger.debug(
|
||||||
|
"Skipping CivArchive model version fetch %s/%s while offline: %s",
|
||||||
|
model_id,
|
||||||
|
version_id,
|
||||||
|
error,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.error(f"Error fetching CivArchive model version via API {model_id}/{version_id}: {error}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
context, version_data, fallback_files = self._split_context(payload)
|
context, version_data, fallback_files = self._split_context(payload)
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ from .model_metadata_provider import (
|
|||||||
from .downloader import get_downloader
|
from .downloader import get_downloader
|
||||||
from .errors import RateLimitError, ResourceNotFoundError
|
from .errors import RateLimitError, ResourceNotFoundError
|
||||||
from ..utils.civitai_utils import resolve_license_payload
|
from ..utils.civitai_utils import resolve_license_payload
|
||||||
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES
|
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES, is_empty_placeholder_hash
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -180,6 +180,11 @@ class CivitaiClient:
|
|||||||
async def get_model_by_hash(
|
async def get_model_by_hash(
|
||||||
self, model_hash: str
|
self, model_hash: str
|
||||||
) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||||
|
if is_empty_placeholder_hash(model_hash):
|
||||||
|
# The empty-hash placeholder (SHA256 of an empty byte string)
|
||||||
|
# matches no real file; CivitAI's by-hash index can contain
|
||||||
|
# polluted entries for it, so never resolve it.
|
||||||
|
return None, "Model not found"
|
||||||
try:
|
try:
|
||||||
success, version = await self._make_request(
|
success, version = await self._make_request(
|
||||||
"GET",
|
"GET",
|
||||||
@@ -500,9 +505,55 @@ class CivitaiClient:
|
|||||||
logger.warning(f"Failed to fetch version by id {version_id}")
|
logger.warning(f"Failed to fetch version by id {version_id}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
async def get_version_file_mini(
|
||||||
|
self, version_id: int, file_id: int
|
||||||
|
) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Fetch raw stored file info via the model-versions/mini endpoint.
|
||||||
|
|
||||||
|
The public REST API rewrites ``files[].name`` to
|
||||||
|
``"{model}_{version}"`` for non-LoRA model types, so every
|
||||||
|
precision variant of a multi-file version shares one name (#1100).
|
||||||
|
The mini endpoint returns the raw ``ModelFile.name`` in
|
||||||
|
``fileName``. ``file_id`` is mandatory: without it mini picks a
|
||||||
|
file via its own primary-file logic, which can disagree with the
|
||||||
|
REST ``primary`` flag.
|
||||||
|
|
||||||
|
Returns the mini payload dict on success, None on any failure.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
success, data = await self._make_request(
|
||||||
|
"GET",
|
||||||
|
f"{self.base_url}/model-versions/mini/{version_id}",
|
||||||
|
params={"modelFileId": file_id},
|
||||||
|
use_auth=True,
|
||||||
|
)
|
||||||
|
if success and isinstance(data, dict):
|
||||||
|
return data
|
||||||
|
if is_expected_offline_error(data):
|
||||||
|
return None
|
||||||
|
logger.debug(
|
||||||
|
"Mini endpoint lookup failed for version %s file %s: %s",
|
||||||
|
version_id,
|
||||||
|
file_id,
|
||||||
|
data,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
except RateLimitError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug(
|
||||||
|
"Error fetching mini info for version %s file %s: %s",
|
||||||
|
version_id,
|
||||||
|
file_id,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
async def _fetch_version_by_hash(self, model_hash: Optional[str]) -> Optional[Dict[str, Any]]:
|
async def _fetch_version_by_hash(self, model_hash: Optional[str]) -> Optional[Dict[str, Any]]:
|
||||||
if not model_hash:
|
if not model_hash:
|
||||||
return None
|
return None
|
||||||
|
if is_empty_placeholder_hash(model_hash):
|
||||||
|
return None
|
||||||
|
|
||||||
success, version = await self._make_request(
|
success, version = await self._make_request(
|
||||||
"GET",
|
"GET",
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
from typing import Any, Awaitable, Callable, Dict, Optional
|
from typing import Any, Awaitable, Callable, Dict, Iterable, Optional
|
||||||
|
|
||||||
from .downloader import DownloadProgress
|
from .downloader import DownloadProgress
|
||||||
|
|
||||||
@@ -87,7 +87,9 @@ class DownloadCoordinator:
|
|||||||
progress_callback=progress_callback,
|
progress_callback=progress_callback,
|
||||||
download_id=download_id,
|
download_id=download_id,
|
||||||
source=payload.get("source"),
|
source=payload.get("source"),
|
||||||
file_params=payload.get("file_params"),
|
# Normalize falsy file_params (e.g. {}) to None so download gates
|
||||||
|
# treat it as "no explicit file selection" (#1058).
|
||||||
|
file_params=payload.get("file_params") or None,
|
||||||
)
|
)
|
||||||
|
|
||||||
result["download_id"] = download_id
|
result["download_id"] = download_id
|
||||||
@@ -184,6 +186,14 @@ class DownloadCoordinator:
|
|||||||
download_manager = await self._download_manager_factory()
|
download_manager = await self._download_manager_factory()
|
||||||
return await download_manager.get_active_downloads()
|
return await download_manager.get_active_downloads()
|
||||||
|
|
||||||
|
async def discard_cleared_downloads(self, download_ids: Iterable[str]) -> int:
|
||||||
|
"""Tear down in-memory/aria2 tracking for queue-cleared downloads."""
|
||||||
|
|
||||||
|
if not download_ids:
|
||||||
|
return 0
|
||||||
|
download_manager = await self._download_manager_factory()
|
||||||
|
return await download_manager.discard_cleared_downloads(download_ids)
|
||||||
|
|
||||||
def _parse_optional_int(self, value: Any, field: str) -> Optional[int]:
|
def _parse_optional_int(self, value: Any, field: str) -> Optional[int]:
|
||||||
"""Parse an optional integer from user input."""
|
"""Parse an optional integer from user input."""
|
||||||
|
|
||||||
|
|||||||
+598
-107
@@ -2,6 +2,7 @@
|
|||||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||||
# import cycles. Breaking them would require an architectural refactor.
|
# import cycles. Breaking them would require an architectural refactor.
|
||||||
|
import contextlib
|
||||||
import copy
|
import copy
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
@@ -12,16 +13,22 @@ import shutil
|
|||||||
import zipfile
|
import zipfile
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
|
from dataclasses import dataclass, field
|
||||||
import uuid
|
import uuid
|
||||||
from typing import Any, Dict, List, Optional, Set, Tuple, cast
|
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, cast
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
from ..utils.models import LoraMetadata, CheckpointMetadata, EmbeddingMetadata
|
from ..utils.models import (
|
||||||
|
LoraMetadata,
|
||||||
|
CheckpointMetadata,
|
||||||
|
EmbeddingMetadata,
|
||||||
|
OtherModelMetadata,
|
||||||
|
)
|
||||||
from ..utils.constants import (
|
from ..utils.constants import (
|
||||||
CARD_PREVIEW_WIDTH,
|
CARD_PREVIEW_WIDTH,
|
||||||
DIFFUSION_MODEL_BASE_MODELS,
|
|
||||||
MODEL_WEIGHT_FILE_TYPES,
|
MODEL_WEIGHT_FILE_TYPES,
|
||||||
SUPPORTED_DOWNLOAD_SKIP_BASE_MODELS,
|
SUPPORTED_DOWNLOAD_SKIP_BASE_MODELS,
|
||||||
VALID_LORA_TYPES,
|
VALID_LORA_TYPES,
|
||||||
|
VALID_OTHER_CIVITAI_TYPES,
|
||||||
)
|
)
|
||||||
from ..utils.civitai_utils import normalize_civitai_download_url, rewrite_preview_url
|
from ..utils.civitai_utils import normalize_civitai_download_url, rewrite_preview_url
|
||||||
from ..utils.file_utils import calculate_sha256, calculate_autov3
|
from ..utils.file_utils import calculate_sha256, calculate_autov3
|
||||||
@@ -30,9 +37,11 @@ from ..utils.utils import sanitize_folder_name
|
|||||||
from ..utils.exif_utils import ExifUtils
|
from ..utils.exif_utils import ExifUtils
|
||||||
from ..utils.metadata_manager import MetadataManager
|
from ..utils.metadata_manager import MetadataManager
|
||||||
from .service_registry import ServiceRegistry
|
from .service_registry import ServiceRegistry
|
||||||
|
from .download_routing import is_diffusion_model_download, resolve_other_download_sub_type
|
||||||
from .settings_manager import get_settings_manager
|
from .settings_manager import get_settings_manager
|
||||||
from .metadata_service import get_default_metadata_provider, get_metadata_provider
|
from .metadata_service import get_default_metadata_provider, get_metadata_provider
|
||||||
from .downloader import get_downloader, DownloadProgress, DownloadStreamControl
|
from .downloader import get_downloader, DownloadProgress, DownloadStreamControl
|
||||||
|
from .errors import RateLimitError
|
||||||
from .aria2_downloader import Aria2Error, get_aria2_downloader
|
from .aria2_downloader import Aria2Error, get_aria2_downloader
|
||||||
from .aria2_transfer_state import Aria2TransferStateStore
|
from .aria2_transfer_state import Aria2TransferStateStore
|
||||||
from .download_queue_service import DownloadQueueService
|
from .download_queue_service import DownloadQueueService
|
||||||
@@ -53,6 +62,12 @@ CIVITAI_DOWNLOAD_URL_PREFIXES = (
|
|||||||
NON_DOWNLOADABLE_PRIMARY_TYPES = ("Config", "Archive", "Workflow", "Training Data")
|
NON_DOWNLOADABLE_PRIMARY_TYPES = ("Config", "Archive", "Workflow", "Training Data")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _PathSlot:
|
||||||
|
lock: asyncio.Lock = field(default_factory=asyncio.Lock)
|
||||||
|
refs: int = 0
|
||||||
|
|
||||||
|
|
||||||
class DownloadManager:
|
class DownloadManager:
|
||||||
_instance = None
|
_instance = None
|
||||||
_lock = asyncio.Lock()
|
_lock = asyncio.Lock()
|
||||||
@@ -82,6 +97,11 @@ class DownloadManager:
|
|||||||
self._aria2_state_store = Aria2TransferStateStore()
|
self._aria2_state_store = Aria2TransferStateStore()
|
||||||
self._restored_persisted_downloads = False
|
self._restored_persisted_downloads = False
|
||||||
self._restore_lock = asyncio.Lock()
|
self._restore_lock = asyncio.Lock()
|
||||||
|
# Refcounted per-target-path locks: two downloads resolving to the
|
||||||
|
# same save_path (e.g. model versions sharing one filename) must not
|
||||||
|
# overlap, or one task's failure cleanup can delete the other's file.
|
||||||
|
self._path_slot_guard: asyncio.Lock = asyncio.Lock()
|
||||||
|
self._path_slots: dict[str, _PathSlot] = {}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _get_model_download_backend() -> str:
|
def _get_model_download_backend() -> str:
|
||||||
@@ -213,6 +233,171 @@ class DownloadManager:
|
|||||||
)
|
)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
async def _get_scanner_for_model_type(self, model_type: str):
|
||||||
|
"""Return the scanner responsible for the given model type.
|
||||||
|
|
||||||
|
Every supported type resolves explicitly — an unknown type must never
|
||||||
|
fall through to the lora scanner (an "other" download would silently
|
||||||
|
dedupe against the lora library).
|
||||||
|
"""
|
||||||
|
if model_type == "checkpoint":
|
||||||
|
return await self._get_checkpoint_scanner()
|
||||||
|
if model_type == "embedding":
|
||||||
|
return await ServiceRegistry.get_embedding_scanner()
|
||||||
|
if model_type == "other":
|
||||||
|
return await ServiceRegistry.get_other_scanner()
|
||||||
|
if model_type == "lora":
|
||||||
|
return await self._get_lora_scanner()
|
||||||
|
raise ValueError(f'Unknown model type "{model_type}"')
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _resolve_target_file(
|
||||||
|
files: Any, file_params: Dict[str, Any] | None
|
||||||
|
) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Resolve the target file within a version's file list from file_params.
|
||||||
|
|
||||||
|
Shared by the existence gate and the actual file selection so both
|
||||||
|
always agree on which file a download refers to (#1058). Returns None
|
||||||
|
when file_params is None or no file matches.
|
||||||
|
"""
|
||||||
|
if not file_params or not isinstance(files, list):
|
||||||
|
return None
|
||||||
|
|
||||||
|
target_file_id = file_params.get("id")
|
||||||
|
target_type = file_params.get("type", "Model")
|
||||||
|
target_format = file_params.get("format")
|
||||||
|
target_size = file_params.get("size")
|
||||||
|
target_fp = file_params.get("fp")
|
||||||
|
is_primary = file_params.get("isPrimary", False)
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
"[download] file_params received: id=%s, type=%s, format=%s, size=%s, fp=%s, "
|
||||||
|
"isPrimary=%s, total_files=%d",
|
||||||
|
target_file_id, target_type, target_format, target_size, target_fp,
|
||||||
|
is_primary, len(files),
|
||||||
|
)
|
||||||
|
|
||||||
|
file_info: Optional[Dict[str, Any]] = None
|
||||||
|
|
||||||
|
if target_file_id:
|
||||||
|
target_id_str = str(target_file_id)
|
||||||
|
for f in files:
|
||||||
|
if not isinstance(f, dict):
|
||||||
|
continue
|
||||||
|
f_id = f.get("id")
|
||||||
|
if str(f_id) == target_id_str:
|
||||||
|
file_info = f
|
||||||
|
logger.debug(
|
||||||
|
"[download] MATCH by ID: id=%s name='%s'",
|
||||||
|
f_id, f.get("name"),
|
||||||
|
)
|
||||||
|
break
|
||||||
|
if not file_info:
|
||||||
|
logger.debug("[download] No file found with id=%s", target_file_id)
|
||||||
|
|
||||||
|
elif is_primary:
|
||||||
|
file_info = next(
|
||||||
|
(
|
||||||
|
f
|
||||||
|
for f in files
|
||||||
|
if isinstance(f, dict)
|
||||||
|
and f.get("primary")
|
||||||
|
and f.get("type") in MODEL_WEIGHT_FILE_TYPES
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# Lenient metadata match: only compare fields present on both sides
|
||||||
|
for f in files:
|
||||||
|
if not isinstance(f, dict):
|
||||||
|
continue
|
||||||
|
f_type = f.get("type", "")
|
||||||
|
if f_type != target_type:
|
||||||
|
continue
|
||||||
|
|
||||||
|
f_meta = f.get("metadata", {})
|
||||||
|
f_format = f_meta.get("format") or f.get("format")
|
||||||
|
f_size = f_meta.get("size") or f.get("size")
|
||||||
|
f_fp = f_meta.get("fp") or f.get("fp")
|
||||||
|
|
||||||
|
if target_format and f_format != target_format:
|
||||||
|
continue
|
||||||
|
if target_size and f_size and f_size != target_size:
|
||||||
|
continue
|
||||||
|
if target_fp and f_fp and f_fp != target_fp:
|
||||||
|
continue
|
||||||
|
|
||||||
|
file_info = f
|
||||||
|
break
|
||||||
|
|
||||||
|
return file_info
|
||||||
|
|
||||||
|
async def _find_local_file_entry(
|
||||||
|
self,
|
||||||
|
model_type: str,
|
||||||
|
model_version_id: int,
|
||||||
|
target_file: Dict[str, Any],
|
||||||
|
) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Find a local library entry for a specific file of a model version.
|
||||||
|
|
||||||
|
Matches per design rule D2 (#1058): SHA256 is only compared when both
|
||||||
|
sides carry a non-empty hash; otherwise fall back to (extension-less)
|
||||||
|
file name equality. Never let two empty hashes compare equal.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
normalized_version_id = int(model_version_id)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
scanner = await self._get_scanner_for_model_type(model_type)
|
||||||
|
cache = await scanner.get_cached_data()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug(
|
||||||
|
"Failed to scan local entries for version %s file check: %s",
|
||||||
|
model_version_id,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
raw_data = getattr(cache, "raw_data", None) if cache else None
|
||||||
|
if not raw_data:
|
||||||
|
return None
|
||||||
|
|
||||||
|
target_hash = str(
|
||||||
|
(target_file.get("hashes") or {}).get("SHA256") or ""
|
||||||
|
).strip().lower()
|
||||||
|
target_name = str(target_file.get("name") or "").strip()
|
||||||
|
target_base = os.path.splitext(target_name)[0] if target_name else ""
|
||||||
|
|
||||||
|
for item in raw_data:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
civitai_data = item.get("civitai")
|
||||||
|
if not isinstance(civitai_data, dict):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
item_version_id = int(civitai_data.get("id"))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
if item_version_id != normalized_version_id:
|
||||||
|
continue
|
||||||
|
|
||||||
|
local_hash = str(item.get("sha256") or "").strip().lower()
|
||||||
|
if target_hash and local_hash:
|
||||||
|
if local_hash == target_hash:
|
||||||
|
return item
|
||||||
|
# Both sides carry hashes that differ: this is a different
|
||||||
|
# file of the same version — do not fall back to name match.
|
||||||
|
continue
|
||||||
|
|
||||||
|
if target_base:
|
||||||
|
local_name = str(item.get("file_name") or "").strip()
|
||||||
|
if local_name == target_base:
|
||||||
|
return item
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
async def download_from_civitai(
|
async def download_from_civitai(
|
||||||
self,
|
self,
|
||||||
model_id: int | None = None,
|
model_id: int | None = None,
|
||||||
@@ -242,6 +427,10 @@ class DownloadManager:
|
|||||||
Returns:
|
Returns:
|
||||||
Dict with download result
|
Dict with download result
|
||||||
"""
|
"""
|
||||||
|
# Normalize falsy file_params (e.g. an empty dict from API JSON
|
||||||
|
# parsing) to None so gate conditions behave consistently (#1058).
|
||||||
|
file_params = file_params or None
|
||||||
|
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"[download] download_from_civitai called: model_id=%s, model_version_id=%s, "
|
"[download] download_from_civitai called: model_id=%s, model_version_id=%s, "
|
||||||
"source=%s, file_params=%s",
|
"source=%s, file_params=%s",
|
||||||
@@ -544,6 +733,47 @@ class DownloadManager:
|
|||||||
await asyncio.sleep(delay)
|
await asyncio.sleep(delay)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _reconcile_failed_aria2_partial(save_path: str) -> None:
|
||||||
|
"""Reconcile on-disk partial state after a failed aria2 transfer.
|
||||||
|
|
||||||
|
The payload and its ``.aria2`` control file form a resumable pair and
|
||||||
|
are preserved together so a retry (with a refreshed URL when needed)
|
||||||
|
can resume via aria2's ``continue=true``. A control file without its
|
||||||
|
payload cannot resume anything, so the orphan is reported and removed.
|
||||||
|
"""
|
||||||
|
control_path = f"{save_path}.aria2"
|
||||||
|
payload_exists = os.path.exists(save_path)
|
||||||
|
control_exists = os.path.exists(control_path)
|
||||||
|
|
||||||
|
if payload_exists and not control_exists:
|
||||||
|
# If the .aria2 control file is missing, aria2 considers the
|
||||||
|
# download complete. A transient RPC failure may have made us
|
||||||
|
# think the download failed even though the file is fully on disk.
|
||||||
|
# Keep the file so a retry can find it already complete.
|
||||||
|
logger.warning(
|
||||||
|
"aria2 download reported failure but .aria2 file is absent "
|
||||||
|
"for %s — the file is likely complete. Preserving it for retry.",
|
||||||
|
save_path,
|
||||||
|
)
|
||||||
|
elif payload_exists and control_exists:
|
||||||
|
logger.info(
|
||||||
|
"Preserving aria2 partial download for resume: %s", save_path
|
||||||
|
)
|
||||||
|
elif control_exists:
|
||||||
|
logger.warning(
|
||||||
|
"Orphaned aria2 control file without payload: %s — removing it",
|
||||||
|
control_path,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
os.remove(control_path)
|
||||||
|
except OSError as exc:
|
||||||
|
logger.warning(
|
||||||
|
"Failed to remove orphaned aria2 control file %s: %s",
|
||||||
|
control_path,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
|
||||||
async def _cleanup_cancelled_download_files(
|
async def _cleanup_cancelled_download_files(
|
||||||
self,
|
self,
|
||||||
download_id: str,
|
download_id: str,
|
||||||
@@ -715,6 +945,42 @@ class DownloadManager:
|
|||||||
|
|
||||||
return download_urls
|
return download_urls
|
||||||
|
|
||||||
|
async def _fetch_raw_file_name(
|
||||||
|
self,
|
||||||
|
metadata_provider,
|
||||||
|
version_id: Optional[int],
|
||||||
|
file_id: Any,
|
||||||
|
) -> Optional[str]:
|
||||||
|
"""Best-effort lookup of the raw stored filename via the CivitAI
|
||||||
|
model-versions/mini endpoint (#1100). Returns None on any failure so
|
||||||
|
the caller can fall back to the (possibly rewritten) REST name."""
|
||||||
|
if version_id is None or file_id is None:
|
||||||
|
return None
|
||||||
|
fetch = getattr(metadata_provider, "get_version_file_mini", None)
|
||||||
|
if fetch is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
mini_info = await fetch(int(version_id), int(file_id))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
except RateLimitError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug(
|
||||||
|
"Mini endpoint lookup failed for version %s file %s: %s",
|
||||||
|
version_id,
|
||||||
|
file_id,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
if not isinstance(mini_info, dict):
|
||||||
|
return None
|
||||||
|
raw_name = mini_info.get("fileName")
|
||||||
|
if not isinstance(raw_name, str) or not raw_name.strip():
|
||||||
|
return None
|
||||||
|
# Defensive: never let a path component slip into the filename.
|
||||||
|
return os.path.basename(raw_name.strip()) or None
|
||||||
|
|
||||||
def _build_metadata_for_resume(
|
def _build_metadata_for_resume(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -727,6 +993,8 @@ class DownloadManager:
|
|||||||
return CheckpointMetadata.from_civitai_info(version_info, file_info, save_path)
|
return CheckpointMetadata.from_civitai_info(version_info, file_info, save_path)
|
||||||
if model_type == "embedding":
|
if model_type == "embedding":
|
||||||
return EmbeddingMetadata.from_civitai_info(version_info, file_info, save_path)
|
return EmbeddingMetadata.from_civitai_info(version_info, file_info, save_path)
|
||||||
|
if model_type == "other":
|
||||||
|
return OtherModelMetadata.from_civitai_info(version_info, file_info, save_path)
|
||||||
return LoraMetadata.from_civitai_info(version_info, file_info, save_path)
|
return LoraMetadata.from_civitai_info(version_info, file_info, save_path)
|
||||||
|
|
||||||
def _resolve_save_path_from_persisted_record(self, record: Dict[str, Any]) -> Optional[str]:
|
def _resolve_save_path_from_persisted_record(self, record: Dict[str, Any]) -> Optional[str]:
|
||||||
@@ -816,6 +1084,7 @@ class DownloadManager:
|
|||||||
version_info,
|
version_info,
|
||||||
record.get("model_version_id"),
|
record.get("model_version_id"),
|
||||||
record.get("save_path") or record.get("file_path"),
|
record.get("save_path") or record.get("file_path"),
|
||||||
|
file_info=file_info,
|
||||||
)
|
)
|
||||||
await self._sync_downloaded_version(
|
await self._sync_downloaded_version(
|
||||||
model_type,
|
model_type,
|
||||||
@@ -939,6 +1208,11 @@ class DownloadManager:
|
|||||||
|
|
||||||
save_path = self._resolve_save_path_from_persisted_record(record)
|
save_path = self._resolve_save_path_from_persisted_record(record)
|
||||||
if save_path is None:
|
if save_path is None:
|
||||||
|
# No resolvable target path (e.g. a queued download whose
|
||||||
|
# paths were never resolved before shutdown): the record
|
||||||
|
# can never be restored, so drop it instead of letting it
|
||||||
|
# accumulate in the state store forever.
|
||||||
|
await self._aria2_state_store.remove(download_id)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if (
|
if (
|
||||||
@@ -1047,6 +1321,24 @@ class DownloadManager:
|
|||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
if not os.path.exists(save_path) and os.path.exists(control_path):
|
||||||
|
# A control file without its payload cannot resume
|
||||||
|
# anything; report it and clean up the orphan.
|
||||||
|
logger.warning(
|
||||||
|
"Orphaned aria2 control file without payload for %s: "
|
||||||
|
"%s — removing it",
|
||||||
|
download_id,
|
||||||
|
control_path,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
os.remove(control_path)
|
||||||
|
except OSError as exc:
|
||||||
|
logger.warning(
|
||||||
|
"Failed to remove orphaned aria2 control file %s: %s",
|
||||||
|
control_path,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
|
||||||
await self._aria2_state_store.remove(download_id)
|
await self._aria2_state_store.remove(download_id)
|
||||||
|
|
||||||
self._restored_persisted_downloads = True
|
self._restored_persisted_downloads = True
|
||||||
@@ -1152,13 +1444,18 @@ class DownloadManager:
|
|||||||
use_save_dir_as_root: bool = False,
|
use_save_dir_as_root: bool = False,
|
||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
"""Wrapper for original download_from_civitai implementation"""
|
"""Wrapper for original download_from_civitai implementation"""
|
||||||
|
file_params = file_params or None
|
||||||
try:
|
try:
|
||||||
# Check if model version already exists in library
|
# Check if model version already exists in library.
|
||||||
if model_version_id is not None:
|
# With an explicit file selection (file_params) the version-level
|
||||||
|
# check is deferred until after the metadata fetch, when the target
|
||||||
|
# file can be resolved and checked individually (#1058).
|
||||||
|
if model_version_id is not None and file_params is None:
|
||||||
# Check both scanners
|
# Check both scanners
|
||||||
lora_scanner = await self._get_lora_scanner()
|
lora_scanner = await self._get_lora_scanner()
|
||||||
checkpoint_scanner = await self._get_checkpoint_scanner()
|
checkpoint_scanner = await self._get_checkpoint_scanner()
|
||||||
embedding_scanner = await ServiceRegistry.get_embedding_scanner()
|
embedding_scanner = await ServiceRegistry.get_embedding_scanner()
|
||||||
|
other_scanner = await ServiceRegistry.get_other_scanner()
|
||||||
|
|
||||||
# Check lora scanner first
|
# Check lora scanner first
|
||||||
if await lora_scanner.check_model_version_exists(model_version_id):
|
if await lora_scanner.check_model_version_exists(model_version_id):
|
||||||
@@ -1183,6 +1480,13 @@ class DownloadManager:
|
|||||||
"error": "Model version already exists in embedding library",
|
"error": "Model version already exists in embedding library",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Check other scanner
|
||||||
|
if await other_scanner.check_model_version_exists(model_version_id):
|
||||||
|
return {
|
||||||
|
"success": False,
|
||||||
|
"error": "Model version already exists in other library",
|
||||||
|
}
|
||||||
|
|
||||||
# Use CivArchive provider directly when source is 'civarchive'
|
# Use CivArchive provider directly when source is 'civarchive'
|
||||||
# This prioritizes CivArchive metadata (with mirror availability info) over Civitai
|
# This prioritizes CivArchive metadata (with mirror availability info) over Civitai
|
||||||
if source == "civarchive":
|
if source == "civarchive":
|
||||||
@@ -1221,6 +1525,20 @@ class DownloadManager:
|
|||||||
model_type = "lora"
|
model_type = "lora"
|
||||||
elif model_type_from_info == "textualinversion":
|
elif model_type_from_info == "textualinversion":
|
||||||
model_type = "embedding"
|
model_type = "embedding"
|
||||||
|
elif model_type_from_info in VALID_OTHER_CIVITAI_TYPES:
|
||||||
|
if not get_settings_manager().is_other_models_enabled():
|
||||||
|
return {
|
||||||
|
"success": False,
|
||||||
|
"error": (
|
||||||
|
"Other Models management is disabled. Enable it in "
|
||||||
|
"Settings > Library before downloading VAE, upscaler, "
|
||||||
|
"text encoder or CLIP files."
|
||||||
|
),
|
||||||
|
# Machine-readable failure code consumed by the companion
|
||||||
|
# browser extension (docs/other-models-support.md C4).
|
||||||
|
"reason": "other_models_disabled",
|
||||||
|
}
|
||||||
|
model_type = "other"
|
||||||
else:
|
else:
|
||||||
return {
|
return {
|
||||||
"success": False,
|
"success": False,
|
||||||
@@ -1235,8 +1553,26 @@ class DownloadManager:
|
|||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
resolved_version_id = None
|
resolved_version_id = None
|
||||||
|
|
||||||
|
# Resolve the explicitly selected file (if any) up front so the
|
||||||
|
# existence gates and the actual file selection below always agree
|
||||||
|
# on the target file (#1058).
|
||||||
|
target_file: Optional[Dict[str, Any]] = None
|
||||||
|
if file_params is not None:
|
||||||
|
target_file = self._resolve_target_file(
|
||||||
|
version_info.get("files") or [], file_params
|
||||||
|
)
|
||||||
|
if target_file is None:
|
||||||
|
logger.warning(
|
||||||
|
"[download] file_params provided but no file matched; "
|
||||||
|
"falling back to version-level checks and primary file "
|
||||||
|
"selection (model_version_id=%s)",
|
||||||
|
resolved_version_id,
|
||||||
|
)
|
||||||
|
explicit_file = target_file is not None
|
||||||
|
|
||||||
if (
|
if (
|
||||||
get_settings_manager().get_skip_previously_downloaded_model_versions()
|
not explicit_file
|
||||||
|
and get_settings_manager().get_skip_previously_downloaded_model_versions()
|
||||||
and resolved_version_id is not None
|
and resolved_version_id is not None
|
||||||
and await self._has_been_downloaded(model_type, resolved_version_id)
|
and await self._has_been_downloaded(model_type, resolved_version_id)
|
||||||
):
|
):
|
||||||
@@ -1324,31 +1660,46 @@ class DownloadManager:
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Check if this checkpoint should be treated as a diffusion model
|
# Check if this checkpoint should be treated as a diffusion model
|
||||||
# Priority: (1) any file has type "UNet" or "Diffusion Model",
|
# (shared with the download routing endpoint so the UI location
|
||||||
# (2) baseModel is in DIFFUSION_MODEL_BASE_MODELS
|
# step and the actual download agree on the target roots).
|
||||||
is_diffusion_model = False
|
is_diffusion_model = is_diffusion_model_download(
|
||||||
if model_type == "checkpoint":
|
model_type,
|
||||||
# Check file types first (more direct signal from CivitAI)
|
file_types=(f.get("type", "") for f in version_info.get("files", [])),
|
||||||
version_files = version_info.get("files", [])
|
base_model=base_model_value,
|
||||||
for f in version_files:
|
)
|
||||||
f_type = f.get("type", "")
|
|
||||||
if f_type in ("UNet", "Diffusion Model"):
|
|
||||||
is_diffusion_model = True
|
|
||||||
logger.info(
|
|
||||||
f"File type '{f_type}' detected, routing checkpoint to unet folder"
|
|
||||||
)
|
|
||||||
break
|
|
||||||
|
|
||||||
# Fallback to baseModel name check
|
# Existence check after the metadata fetch (#1058):
|
||||||
if not is_diffusion_model and base_model_value in DIFFUSION_MODEL_BASE_MODELS:
|
# - An explicit file selection only blocks when THIS file is
|
||||||
is_diffusion_model = True
|
# already in the library; other files of the same version
|
||||||
logger.info(
|
# remain downloadable.
|
||||||
f"baseModel '{base_model_value}' is a known diffusion model, routing to unet folder"
|
# - Without file_params (or when file_params failed to resolve),
|
||||||
|
# keep version-level protection. The case "model_version_id
|
||||||
|
# given + no file_params" was already covered by the early
|
||||||
|
# gate above.
|
||||||
|
if explicit_file and resolved_version_id is not None:
|
||||||
|
existing_entry = await self._find_local_file_entry(
|
||||||
|
model_type, resolved_version_id, target_file
|
||||||
|
)
|
||||||
|
if existing_entry is not None:
|
||||||
|
error_message = (
|
||||||
|
f"File '{target_file.get('name')}' from model version "
|
||||||
|
f"{resolved_version_id} already exists in {model_type} library"
|
||||||
)
|
)
|
||||||
|
logger.info("[download] %s", error_message)
|
||||||
# Case 2: model_version_id was None, check after getting version_info
|
return {"success": False, "error": error_message}
|
||||||
if model_version_id is None:
|
logger.info(
|
||||||
version_id = version_info.get("id")
|
"[download] File '%s' of model version %s not in %s library — "
|
||||||
|
"download allowed (other files of this version may exist locally)",
|
||||||
|
target_file.get("name"), resolved_version_id, model_type,
|
||||||
|
)
|
||||||
|
elif file_params is not None or model_version_id is None:
|
||||||
|
# Case 2: model_version_id was None, or file_params did not
|
||||||
|
# resolve to a concrete file — check at version level.
|
||||||
|
version_id = (
|
||||||
|
resolved_version_id
|
||||||
|
if resolved_version_id is not None
|
||||||
|
else version_info.get("id")
|
||||||
|
)
|
||||||
|
|
||||||
if model_type == "lora":
|
if model_type == "lora":
|
||||||
# Check lora scanner
|
# Check lora scanner
|
||||||
@@ -1374,6 +1725,13 @@ class DownloadManager:
|
|||||||
"success": False,
|
"success": False,
|
||||||
"error": "Model version already exists in embedding library",
|
"error": "Model version already exists in embedding library",
|
||||||
}
|
}
|
||||||
|
elif model_type == "other":
|
||||||
|
other_scanner = await ServiceRegistry.get_other_scanner()
|
||||||
|
if await other_scanner.check_model_version_exists(version_id):
|
||||||
|
return {
|
||||||
|
"success": False,
|
||||||
|
"error": "Model version already exists in other library",
|
||||||
|
}
|
||||||
|
|
||||||
# Handle use_default_paths
|
# Handle use_default_paths
|
||||||
if use_default_paths:
|
if use_default_paths:
|
||||||
@@ -1413,6 +1771,60 @@ class DownloadManager:
|
|||||||
"error": "Default embedding root path not set in settings",
|
"error": "Default embedding root path not set in settings",
|
||||||
}
|
}
|
||||||
save_dir = default_path
|
save_dir = default_path
|
||||||
|
elif model_type == "other":
|
||||||
|
other_sub_type = resolve_other_download_sub_type(
|
||||||
|
model_type_from_info,
|
||||||
|
file_types=(
|
||||||
|
f.get("type", "")
|
||||||
|
for f in version_info.get("files", [])
|
||||||
|
if isinstance(f, dict)
|
||||||
|
),
|
||||||
|
selected_file_type=(
|
||||||
|
target_file.get("type") if explicit_file else None
|
||||||
|
),
|
||||||
|
)
|
||||||
|
default_other_roots = (
|
||||||
|
settings_manager.get("default_other_roots") or {}
|
||||||
|
)
|
||||||
|
if other_sub_type and not settings_manager.is_other_sub_type_enabled(
|
||||||
|
other_sub_type
|
||||||
|
):
|
||||||
|
return {
|
||||||
|
"success": False,
|
||||||
|
"error": (
|
||||||
|
f"Other-model sub-type '{other_sub_type}' is "
|
||||||
|
f"disabled in settings. Please pick a destination "
|
||||||
|
f"folder explicitly instead of using default paths."
|
||||||
|
),
|
||||||
|
"reason": "other_sub_type_disabled",
|
||||||
|
}
|
||||||
|
default_path = (
|
||||||
|
default_other_roots.get(other_sub_type)
|
||||||
|
if other_sub_type
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
if not isinstance(default_path, str) or not default_path:
|
||||||
|
if other_sub_type:
|
||||||
|
detail = (
|
||||||
|
f"No default root configured for other-model "
|
||||||
|
f"sub-type '{other_sub_type}'"
|
||||||
|
)
|
||||||
|
reason = "other_no_default_root"
|
||||||
|
else:
|
||||||
|
detail = (
|
||||||
|
"Could not determine the other-model sub-type "
|
||||||
|
"from the model metadata"
|
||||||
|
)
|
||||||
|
reason = "other_sub_type_undecidable"
|
||||||
|
return {
|
||||||
|
"success": False,
|
||||||
|
"error": (
|
||||||
|
f"{detail}. Please pick a destination folder "
|
||||||
|
f"explicitly instead of using default paths."
|
||||||
|
),
|
||||||
|
"reason": reason,
|
||||||
|
}
|
||||||
|
save_dir = default_path
|
||||||
|
|
||||||
# Calculate relative path using template
|
# Calculate relative path using template
|
||||||
relative_path = self._calculate_relative_path(version_info, model_type)
|
relative_path = self._calculate_relative_path(version_info, model_type)
|
||||||
@@ -1495,73 +1907,16 @@ class DownloadManager:
|
|||||||
files = version_info.get("files", [])
|
files = version_info.get("files", [])
|
||||||
file_info = None
|
file_info = None
|
||||||
|
|
||||||
# If file_params is provided, try to find matching file
|
# If file_params is provided, reuse the file resolved right after
|
||||||
if file_params and model_version_id:
|
# the metadata fetch so the existence gate and this selection
|
||||||
target_file_id = file_params.get("id")
|
# always agree on the target file (#1058).
|
||||||
target_type = file_params.get("type", "Model")
|
if file_params is not None:
|
||||||
target_format = file_params.get("format")
|
file_info = target_file
|
||||||
target_size = file_params.get("size")
|
|
||||||
target_fp = file_params.get("fp")
|
|
||||||
is_primary = file_params.get("isPrimary", False)
|
|
||||||
|
|
||||||
logger.debug(
|
|
||||||
"[download] file_params received: id=%s, type=%s, format=%s, size=%s, fp=%s, isPrimary=%s, "
|
|
||||||
"model_version_id=%s, total_files=%d",
|
|
||||||
target_file_id, target_type, target_format, target_size, target_fp, is_primary,
|
|
||||||
model_version_id, len(files),
|
|
||||||
)
|
|
||||||
|
|
||||||
if target_file_id:
|
|
||||||
target_id_str = str(target_file_id)
|
|
||||||
for f in files:
|
|
||||||
f_id = f.get("id")
|
|
||||||
if str(f_id) == target_id_str:
|
|
||||||
file_info = f
|
|
||||||
logger.debug(
|
|
||||||
"[download] MATCH by ID: id=%s name='%s'",
|
|
||||||
f_id, f.get("name"),
|
|
||||||
)
|
|
||||||
break
|
|
||||||
if not file_info:
|
|
||||||
logger.debug("[download] No file found with id=%s", target_file_id)
|
|
||||||
|
|
||||||
elif is_primary:
|
|
||||||
file_info = next(
|
|
||||||
(
|
|
||||||
f
|
|
||||||
for f in files
|
|
||||||
if f.get("primary")
|
|
||||||
and f.get("type") in MODEL_WEIGHT_FILE_TYPES
|
|
||||||
),
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
# Lenient metadata match: only compare fields present on both sides
|
|
||||||
for f in files:
|
|
||||||
f_type = f.get("type", "")
|
|
||||||
if f_type != target_type:
|
|
||||||
continue
|
|
||||||
|
|
||||||
f_meta = f.get("metadata", {})
|
|
||||||
f_format = f_meta.get("format") or f.get("format")
|
|
||||||
f_size = f_meta.get("size") or f.get("size")
|
|
||||||
f_fp = f_meta.get("fp") or f.get("fp")
|
|
||||||
|
|
||||||
if target_format and f_format != target_format:
|
|
||||||
continue
|
|
||||||
if target_size and f_size and f_size != target_size:
|
|
||||||
continue
|
|
||||||
if target_fp and f_fp and f_fp != target_fp:
|
|
||||||
continue
|
|
||||||
|
|
||||||
file_info = f
|
|
||||||
break
|
|
||||||
|
|
||||||
if not file_info:
|
if not file_info:
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"[download] No match found via file_params — falling back to primary file lookup",
|
"[download] No match found via file_params — falling back to primary file lookup",
|
||||||
)
|
)
|
||||||
elif not file_params:
|
else:
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"[download] No file_params provided (null/None) — will use primary file lookup. "
|
"[download] No file_params provided (null/None) — will use primary file lookup. "
|
||||||
"model_version_id=%s, total_files=%d",
|
"model_version_id=%s, total_files=%d",
|
||||||
@@ -1626,6 +1981,24 @@ class DownloadManager:
|
|||||||
if not download_urls:
|
if not download_urls:
|
||||||
return {"success": False, "error": "No mirror URL found"}
|
return {"success": False, "error": "No mirror URL found"}
|
||||||
|
|
||||||
|
# The public REST API rewrites files[].name to
|
||||||
|
# "{model}_{version}" for non-LoRA model types, so every
|
||||||
|
# precision variant of a multi-file version shares one name and
|
||||||
|
# lands on disk with a random short-hash suffix. The mini
|
||||||
|
# endpoint returns the raw stored filename (#1100). CivArchive
|
||||||
|
# already serves raw names.
|
||||||
|
if source != "civarchive":
|
||||||
|
raw_file_name = await self._fetch_raw_file_name(
|
||||||
|
metadata_provider, resolved_version_id, file_info.get("id")
|
||||||
|
)
|
||||||
|
if raw_file_name and raw_file_name != file_info.get("name"):
|
||||||
|
logger.info(
|
||||||
|
"[download] Using raw stored filename '%s' instead of REST name '%s'",
|
||||||
|
raw_file_name,
|
||||||
|
file_info.get("name"),
|
||||||
|
)
|
||||||
|
file_info = {**file_info, "name": raw_file_name}
|
||||||
|
|
||||||
# 3. Prepare download
|
# 3. Prepare download
|
||||||
file_name = file_info.get("name", "")
|
file_name = file_info.get("name", "")
|
||||||
if not file_name:
|
if not file_name:
|
||||||
@@ -1648,6 +2021,11 @@ class DownloadManager:
|
|||||||
version_info, file_info, save_path
|
version_info, file_info, save_path
|
||||||
)
|
)
|
||||||
logger.info(f"Creating EmbeddingMetadata for {file_name}")
|
logger.info(f"Creating EmbeddingMetadata for {file_name}")
|
||||||
|
elif model_type == "other":
|
||||||
|
metadata = OtherModelMetadata.from_civitai_info(
|
||||||
|
version_info, file_info, save_path
|
||||||
|
)
|
||||||
|
logger.info(f"Creating OtherModelMetadata for {file_name}")
|
||||||
else:
|
else:
|
||||||
return {
|
return {
|
||||||
"success": False,
|
"success": False,
|
||||||
@@ -1706,6 +2084,7 @@ class DownloadManager:
|
|||||||
version_info,
|
version_info,
|
||||||
model_version_id,
|
model_version_id,
|
||||||
save_path,
|
save_path,
|
||||||
|
file_info=file_info,
|
||||||
)
|
)
|
||||||
await self._sync_downloaded_version(
|
await self._sync_downloaded_version(
|
||||||
model_type,
|
model_type,
|
||||||
@@ -1748,6 +2127,7 @@ class DownloadManager:
|
|||||||
version_info: Dict[str, Any],
|
version_info: Dict[str, Any],
|
||||||
fallback_version_id=None,
|
fallback_version_id=None,
|
||||||
file_path: str | None = None,
|
file_path: str | None = None,
|
||||||
|
file_info: Dict[str, Any] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
try:
|
try:
|
||||||
history_service = await ServiceRegistry.get_downloaded_version_history_service()
|
history_service = await ServiceRegistry.get_downloaded_version_history_service()
|
||||||
@@ -1773,6 +2153,15 @@ class DownloadManager:
|
|||||||
if version_id is None:
|
if version_id is None:
|
||||||
version_id = fallback_version_id
|
version_id = fallback_version_id
|
||||||
|
|
||||||
|
# Per-file identity for multi-file versions (#1058)
|
||||||
|
file_id = None
|
||||||
|
file_name = None
|
||||||
|
if isinstance(file_info, dict):
|
||||||
|
file_id = file_info.get("id")
|
||||||
|
raw_file_name = file_info.get("name")
|
||||||
|
if isinstance(raw_file_name, str) and raw_file_name.strip():
|
||||||
|
file_name = raw_file_name.strip()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
await history_service.mark_downloaded(
|
await history_service.mark_downloaded(
|
||||||
model_type,
|
model_type,
|
||||||
@@ -1780,6 +2169,8 @@ class DownloadManager:
|
|||||||
model_id=int(cast(Any, resolved_model_id)) if resolved_model_id is not None else None,
|
model_id=int(cast(Any, resolved_model_id)) if resolved_model_id is not None else None,
|
||||||
source="download",
|
source="download",
|
||||||
file_path=file_path,
|
file_path=file_path,
|
||||||
|
file_id=file_id,
|
||||||
|
file_name=file_name,
|
||||||
)
|
)
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
logger.debug(
|
logger.debug(
|
||||||
@@ -1847,6 +2238,8 @@ class DownloadManager:
|
|||||||
scanner = await self._get_checkpoint_scanner()
|
scanner = await self._get_checkpoint_scanner()
|
||||||
elif model_type == "embedding":
|
elif model_type == "embedding":
|
||||||
scanner = await ServiceRegistry.get_embedding_scanner()
|
scanner = await ServiceRegistry.get_embedding_scanner()
|
||||||
|
elif model_type == "other":
|
||||||
|
scanner = await ServiceRegistry.get_other_scanner()
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.debug("Failed to acquire scanner for %s models: %s", model_type, exc)
|
logger.debug("Failed to acquire scanner for %s models: %s", model_type, exc)
|
||||||
|
|
||||||
@@ -1959,6 +2352,28 @@ class DownloadManager:
|
|||||||
|
|
||||||
return formatted_path
|
return formatted_path
|
||||||
|
|
||||||
|
@contextlib.asynccontextmanager
|
||||||
|
async def _exclusive_target_slot(self, target_key: str):
|
||||||
|
async with self._path_slot_guard:
|
||||||
|
slot = self._path_slots.get(target_key)
|
||||||
|
if slot is None:
|
||||||
|
slot = _PathSlot()
|
||||||
|
self._path_slots[target_key] = slot
|
||||||
|
slot.refs += 1
|
||||||
|
try:
|
||||||
|
async with slot.lock:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
async with self._path_slot_guard:
|
||||||
|
slot.refs -= 1
|
||||||
|
if slot.refs <= 0:
|
||||||
|
_ = self._path_slots.pop(target_key, None)
|
||||||
|
|
||||||
|
def _target_slot_key(self, save_dir: str, metadata) -> str:
|
||||||
|
return os.path.abspath(
|
||||||
|
os.path.join(save_dir, os.path.basename(metadata.file_path))
|
||||||
|
)
|
||||||
|
|
||||||
async def _execute_download(
|
async def _execute_download(
|
||||||
self,
|
self,
|
||||||
download_urls: List[str],
|
download_urls: List[str],
|
||||||
@@ -1970,6 +2385,33 @@ class DownloadManager:
|
|||||||
model_type: str = "lora",
|
model_type: str = "lora",
|
||||||
download_id: str | None = None,
|
download_id: str | None = None,
|
||||||
transfer_backend: Optional[str] = None,
|
transfer_backend: Optional[str] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Execute the download serialized against other downloads targeting the same path."""
|
||||||
|
target_key = self._target_slot_key(save_dir, metadata)
|
||||||
|
async with self._exclusive_target_slot(target_key):
|
||||||
|
return await self._execute_download_pipeline(
|
||||||
|
download_urls=download_urls,
|
||||||
|
save_dir=save_dir,
|
||||||
|
metadata=metadata,
|
||||||
|
version_info=version_info,
|
||||||
|
relative_path=relative_path,
|
||||||
|
progress_callback=progress_callback,
|
||||||
|
model_type=model_type,
|
||||||
|
download_id=download_id,
|
||||||
|
transfer_backend=transfer_backend,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _execute_download_pipeline(
|
||||||
|
self,
|
||||||
|
download_urls: List[str],
|
||||||
|
save_dir: str,
|
||||||
|
metadata,
|
||||||
|
version_info: Dict[str, Any],
|
||||||
|
relative_path: str,
|
||||||
|
progress_callback=None,
|
||||||
|
model_type: str = "lora",
|
||||||
|
download_id: str | None = None,
|
||||||
|
transfer_backend: Optional[str] = None,
|
||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
"""Execute the actual download process including preview images and model files"""
|
"""Execute the actual download process including preview images and model files"""
|
||||||
metadata_entries: List[Any] = []
|
metadata_entries: List[Any] = []
|
||||||
@@ -2188,20 +2630,8 @@ class DownloadManager:
|
|||||||
break
|
break
|
||||||
|
|
||||||
last_error = result
|
last_error = result
|
||||||
# For aria2: if the .aria2 control file is missing, aria2 considers
|
if transfer_backend == "aria2":
|
||||||
# the download complete. A transient RPC failure may have made us
|
self._reconcile_failed_aria2_partial(save_path)
|
||||||
# think the download failed even though the file is fully on disk.
|
|
||||||
# Keep the file so a retry can find it already complete.
|
|
||||||
if (
|
|
||||||
transfer_backend == "aria2"
|
|
||||||
and os.path.exists(save_path)
|
|
||||||
and not os.path.exists(f"{save_path}.aria2")
|
|
||||||
):
|
|
||||||
logger.warning(
|
|
||||||
"aria2 download reported failure but .aria2 file is absent "
|
|
||||||
"for %s — the file is likely complete. Preserving it for retry.",
|
|
||||||
save_path,
|
|
||||||
)
|
|
||||||
elif os.path.exists(save_path):
|
elif os.path.exists(save_path):
|
||||||
try:
|
try:
|
||||||
os.remove(save_path)
|
os.remove(save_path)
|
||||||
@@ -2306,6 +2736,9 @@ class DownloadManager:
|
|||||||
elif model_type == "embedding":
|
elif model_type == "embedding":
|
||||||
scanner = await ServiceRegistry.get_embedding_scanner()
|
scanner = await ServiceRegistry.get_embedding_scanner()
|
||||||
logger.info(f"Updating embedding cache for {actual_file_paths[0]}")
|
logger.info(f"Updating embedding cache for {actual_file_paths[0]}")
|
||||||
|
elif model_type == "other":
|
||||||
|
scanner = await ServiceRegistry.get_other_scanner()
|
||||||
|
logger.info(f"Updating other-model cache for {actual_file_paths[0]}")
|
||||||
|
|
||||||
adjust_cached_entry = (
|
adjust_cached_entry = (
|
||||||
getattr(scanner, "adjust_cached_entry", None)
|
getattr(scanner, "adjust_cached_entry", None)
|
||||||
@@ -2395,7 +2828,7 @@ class DownloadManager:
|
|||||||
return {"success": False, "error": str(e)}
|
return {"success": False, "error": str(e)}
|
||||||
|
|
||||||
def _get_supported_extensions_for_type(self, model_type: str) -> Set[str]:
|
def _get_supported_extensions_for_type(self, model_type: str) -> Set[str]:
|
||||||
if model_type == "checkpoint":
|
if model_type in ("checkpoint", "other"):
|
||||||
return {
|
return {
|
||||||
".ckpt",
|
".ckpt",
|
||||||
".pt",
|
".pt",
|
||||||
@@ -2729,6 +3162,64 @@ class DownloadManager:
|
|||||||
# Preserve aria2 state store entry so the partial download
|
# Preserve aria2 state store entry so the partial download
|
||||||
# info survives restarts and can be resumed later
|
# info survives restarts and can be resumed later
|
||||||
|
|
||||||
|
async def discard_cleared_downloads(self, download_ids: Iterable[str]) -> int:
|
||||||
|
"""Stop in-memory tracking for downloads cleared from the queue.
|
||||||
|
|
||||||
|
Cancels asyncio tasks, removes live aria2 transfers and drops the
|
||||||
|
persisted aria2 state so cleared downloads cannot keep polling the
|
||||||
|
daemon or be resurrected as ghost entries on the next restart.
|
||||||
|
Partial files on disk are preserved; unlike ``cancel_download`` no
|
||||||
|
files are deleted.
|
||||||
|
|
||||||
|
Returns the number of downloads that had any in-memory or persisted
|
||||||
|
tracking removed.
|
||||||
|
"""
|
||||||
|
discarded = 0
|
||||||
|
aria2_downloader = None
|
||||||
|
|
||||||
|
for download_id in download_ids:
|
||||||
|
task = self._download_tasks.get(download_id)
|
||||||
|
info = self._active_downloads.get(download_id)
|
||||||
|
persisted = await self._aria2_state_store.get(download_id)
|
||||||
|
if task is None and info is None and persisted is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
discarded += 1
|
||||||
|
|
||||||
|
if task is not None:
|
||||||
|
task.cancel()
|
||||||
|
|
||||||
|
pause_control = self._pause_events.pop(download_id, None)
|
||||||
|
if pause_control is not None:
|
||||||
|
pause_control.resume()
|
||||||
|
|
||||||
|
if task is not None:
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(asyncio.shield(task), timeout=2.0)
|
||||||
|
except (asyncio.CancelledError, asyncio.TimeoutError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
self._download_tasks.pop(download_id, None)
|
||||||
|
self._active_downloads.pop(download_id, None)
|
||||||
|
|
||||||
|
backend = (info or persisted or {}).get("transfer_backend") or "python"
|
||||||
|
if backend == "aria2":
|
||||||
|
if aria2_downloader is None:
|
||||||
|
aria2_downloader = await get_aria2_downloader()
|
||||||
|
if await aria2_downloader.has_transfer(download_id):
|
||||||
|
try:
|
||||||
|
await aria2_downloader.cancel_download(download_id)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(
|
||||||
|
"Failed to remove aria2 transfer for cleared download %s: %s",
|
||||||
|
download_id,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
|
||||||
|
await self._aria2_state_store.remove(download_id)
|
||||||
|
|
||||||
|
return discarded
|
||||||
|
|
||||||
async def pause_download(self, download_id: str) -> Dict[str, Any]:
|
async def pause_download(self, download_id: str) -> Dict[str, Any]:
|
||||||
"""Pause an active download without losing progress."""
|
"""Pause an active download without losing progress."""
|
||||||
|
|
||||||
|
|||||||
@@ -6,12 +6,21 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import time
|
import time
|
||||||
from typing import Any, Optional
|
from typing import Any, List, Optional
|
||||||
|
|
||||||
from ..utils.cache_paths import get_cache_base_dir
|
from ..utils.cache_paths import get_cache_base_dir
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# SQL fragment extracting the CivitAI file id from the JSON ``file_params``
|
||||||
|
# column (#1058). ``json_valid`` guards against NULL and legacy/unparseable
|
||||||
|
# values, yielding NULL for rows without a file identity; NULL keys group
|
||||||
|
# together so such rows keep the old version-level dedup behavior.
|
||||||
|
_FILE_ID_SQL = (
|
||||||
|
"CASE WHEN json_valid(file_params) "
|
||||||
|
"THEN json_extract(file_params, '$.id') END"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _resolve_database_path() -> str:
|
def _resolve_database_path() -> str:
|
||||||
base_dir = get_cache_base_dir(create=True)
|
base_dir = get_cache_base_dir(create=True)
|
||||||
@@ -64,6 +73,7 @@ class DownloadQueueService:
|
|||||||
model_name TEXT NOT NULL DEFAULT '',
|
model_name TEXT NOT NULL DEFAULT '',
|
||||||
version_name TEXT DEFAULT '',
|
version_name TEXT DEFAULT '',
|
||||||
thumbnail_url TEXT DEFAULT '',
|
thumbnail_url TEXT DEFAULT '',
|
||||||
|
file_params TEXT,
|
||||||
status TEXT NOT NULL,
|
status TEXT NOT NULL,
|
||||||
error TEXT,
|
error TEXT,
|
||||||
file_path TEXT,
|
file_path TEXT,
|
||||||
@@ -120,6 +130,18 @@ class DownloadQueueService:
|
|||||||
with self._connect() as conn:
|
with self._connect() as conn:
|
||||||
conn.executescript(self._SCHEMA_TABLES)
|
conn.executescript(self._SCHEMA_TABLES)
|
||||||
|
|
||||||
|
# Databases created by older versions lack
|
||||||
|
# download_history.file_params; add it so retry-from-history can
|
||||||
|
# restore the originally selected file (#1058).
|
||||||
|
history_columns = {
|
||||||
|
row["name"]
|
||||||
|
for row in conn.execute("PRAGMA table_info(download_history)")
|
||||||
|
}
|
||||||
|
if "file_params" not in history_columns:
|
||||||
|
conn.execute(
|
||||||
|
"ALTER TABLE download_history ADD COLUMN file_params TEXT"
|
||||||
|
)
|
||||||
|
|
||||||
# Creating the unique index on download_history.download_id can
|
# Creating the unique index on download_history.download_id can
|
||||||
# fail if pre-existing rows have duplicate values (e.g. from a
|
# fail if pre-existing rows have duplicate values (e.g. from a
|
||||||
# previous version that lacked the index). Deduplicate first so
|
# previous version that lacked the index). Deduplicate first so
|
||||||
@@ -368,23 +390,31 @@ class DownloadQueueService:
|
|||||||
conn.commit()
|
conn.commit()
|
||||||
return True
|
return True
|
||||||
|
|
||||||
async def clear_queue(self, status_filter: Optional[str] = None) -> int:
|
async def clear_queue(self, status_filter: Optional[str] = None) -> List[str]:
|
||||||
"""Remove items from the queue.
|
"""Remove items from the queue.
|
||||||
|
|
||||||
When *status_filter* is provided only items with that status are
|
When *status_filter* is provided only items with that status are
|
||||||
deleted. Returns the number of deleted rows.
|
deleted. Returns the ``download_id`` values of the deleted rows so
|
||||||
|
callers can also tear down any in-memory tracking for them.
|
||||||
"""
|
"""
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
conn = self._get_conn()
|
conn = self._get_conn()
|
||||||
if status_filter is not None:
|
if status_filter is not None:
|
||||||
cursor = conn.execute(
|
rows = conn.execute(
|
||||||
|
"SELECT download_id FROM download_queue WHERE status = ?",
|
||||||
|
(status_filter,),
|
||||||
|
).fetchall()
|
||||||
|
conn.execute(
|
||||||
"DELETE FROM download_queue WHERE status = ?",
|
"DELETE FROM download_queue WHERE status = ?",
|
||||||
(status_filter,),
|
(status_filter,),
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
cursor = conn.execute("DELETE FROM download_queue")
|
rows = conn.execute(
|
||||||
|
"SELECT download_id FROM download_queue"
|
||||||
|
).fetchall()
|
||||||
|
conn.execute("DELETE FROM download_queue")
|
||||||
conn.commit()
|
conn.commit()
|
||||||
return cursor.rowcount
|
return [row["download_id"] for row in rows]
|
||||||
|
|
||||||
async def complete_download(
|
async def complete_download(
|
||||||
self,
|
self,
|
||||||
@@ -418,6 +448,12 @@ class DownloadQueueService:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
now = completed_at if completed_at is not None else time.time()
|
now = completed_at if completed_at is not None else time.time()
|
||||||
|
# Guard against legacy databases whose download_queue table
|
||||||
|
# predates the file_params column.
|
||||||
|
queue_columns = set(row.keys())
|
||||||
|
file_params_json = (
|
||||||
|
row["file_params"] if "file_params" in queue_columns else None
|
||||||
|
)
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"DELETE FROM download_queue WHERE download_id = ?",
|
"DELETE FROM download_queue WHERE download_id = ?",
|
||||||
(download_id,),
|
(download_id,),
|
||||||
@@ -426,9 +462,9 @@ class DownloadQueueService:
|
|||||||
"""
|
"""
|
||||||
INSERT OR IGNORE INTO download_history (
|
INSERT OR IGNORE INTO download_history (
|
||||||
download_id, model_id, model_version_id, model_name,
|
download_id, model_id, model_version_id, model_name,
|
||||||
version_name, thumbnail_url, status, error, file_path,
|
version_name, thumbnail_url, file_params, status, error,
|
||||||
bytes_downloaded, total_bytes, completed_at
|
file_path, bytes_downloaded, total_bytes, completed_at
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
""",
|
""",
|
||||||
(
|
(
|
||||||
row["download_id"],
|
row["download_id"],
|
||||||
@@ -437,6 +473,7 @@ class DownloadQueueService:
|
|||||||
row["model_name"],
|
row["model_name"],
|
||||||
row["version_name"],
|
row["version_name"],
|
||||||
row["thumbnail_url"],
|
row["thumbnail_url"],
|
||||||
|
file_params_json,
|
||||||
status,
|
status,
|
||||||
error,
|
error,
|
||||||
file_path,
|
file_path,
|
||||||
@@ -503,6 +540,7 @@ class DownloadQueueService:
|
|||||||
bytes_downloaded: int = 0,
|
bytes_downloaded: int = 0,
|
||||||
total_bytes: Optional[int] = None,
|
total_bytes: Optional[int] = None,
|
||||||
is_already_exists: int = 0,
|
is_already_exists: int = 0,
|
||||||
|
file_params: Optional[dict[str, Any]] = None,
|
||||||
) -> int:
|
) -> int:
|
||||||
"""Insert a record into the download history.
|
"""Insert a record into the download history.
|
||||||
|
|
||||||
@@ -510,6 +548,7 @@ class DownloadQueueService:
|
|||||||
inserted row.
|
inserted row.
|
||||||
"""
|
"""
|
||||||
now = time.time()
|
now = time.time()
|
||||||
|
file_params_json = json.dumps(file_params) if file_params is not None else None
|
||||||
|
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
conn = self._get_conn()
|
conn = self._get_conn()
|
||||||
@@ -517,9 +556,10 @@ class DownloadQueueService:
|
|||||||
"""
|
"""
|
||||||
INSERT INTO download_history (
|
INSERT INTO download_history (
|
||||||
download_id, model_id, model_version_id, model_name,
|
download_id, model_id, model_version_id, model_name,
|
||||||
version_name, thumbnail_url, status, error, file_path,
|
version_name, thumbnail_url, file_params, status, error,
|
||||||
bytes_downloaded, total_bytes, completed_at, is_already_exists
|
file_path, bytes_downloaded, total_bytes, completed_at,
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
is_already_exists
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
""",
|
""",
|
||||||
(
|
(
|
||||||
download_id,
|
download_id,
|
||||||
@@ -528,6 +568,7 @@ class DownloadQueueService:
|
|||||||
model_name,
|
model_name,
|
||||||
version_name,
|
version_name,
|
||||||
thumbnail_url,
|
thumbnail_url,
|
||||||
|
file_params_json,
|
||||||
status,
|
status,
|
||||||
error,
|
error,
|
||||||
file_path,
|
file_path,
|
||||||
@@ -702,7 +743,7 @@ class DownloadQueueService:
|
|||||||
download_id, model_id, model_version_id, model_name,
|
download_id, model_id, model_version_id, model_name,
|
||||||
version_name, thumbnail_url, source, file_params,
|
version_name, thumbnail_url, source, file_params,
|
||||||
status, priority, added_at
|
status, priority, added_at
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, NULL, 'queued', 0, ?)
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?)
|
||||||
""",
|
""",
|
||||||
(
|
(
|
||||||
new_id,
|
new_id,
|
||||||
@@ -712,6 +753,7 @@ class DownloadQueueService:
|
|||||||
row["version_name"],
|
row["version_name"],
|
||||||
row["thumbnail_url"],
|
row["thumbnail_url"],
|
||||||
"retry",
|
"retry",
|
||||||
|
row["file_params"],
|
||||||
now,
|
now,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -755,7 +797,7 @@ class DownloadQueueService:
|
|||||||
download_id, model_id, model_version_id, model_name,
|
download_id, model_id, model_version_id, model_name,
|
||||||
version_name, thumbnail_url, source, file_params,
|
version_name, thumbnail_url, source, file_params,
|
||||||
status, priority, added_at
|
status, priority, added_at
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, NULL, 'queued', 0, ?)
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?)
|
||||||
""",
|
""",
|
||||||
(
|
(
|
||||||
new_id,
|
new_id,
|
||||||
@@ -765,6 +807,7 @@ class DownloadQueueService:
|
|||||||
row["version_name"],
|
row["version_name"],
|
||||||
row["thumbnail_url"],
|
row["thumbnail_url"],
|
||||||
"retry",
|
"retry",
|
||||||
|
row["file_params"],
|
||||||
now,
|
now,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -840,33 +883,44 @@ class DownloadQueueService:
|
|||||||
async with self._lock:
|
async with self._lock:
|
||||||
conn = self._get_conn()
|
conn = self._get_conn()
|
||||||
|
|
||||||
# 1. History: for each (model_id, model_version_id, status) triplet
|
# 1. History: for each (model_id, model_version_id, file_id,
|
||||||
# keep only the row with the highest id (most recently inserted).
|
# status) group keep only the row with the highest id (most
|
||||||
conn.execute("""
|
# recently inserted). file_id comes from file_params (#1058)
|
||||||
|
# so distinct files of the same version never collapse.
|
||||||
|
conn.execute(f"""
|
||||||
DELETE FROM download_history
|
DELETE FROM download_history
|
||||||
WHERE id NOT IN (
|
WHERE id NOT IN (
|
||||||
SELECT MAX(id)
|
SELECT MAX(id)
|
||||||
FROM download_history
|
FROM download_history
|
||||||
GROUP BY model_id, model_version_id, status
|
GROUP BY model_id, model_version_id, status,
|
||||||
|
{_FILE_ID_SQL}
|
||||||
)
|
)
|
||||||
""")
|
""")
|
||||||
result["removed_history"] = conn.execute(
|
result["removed_history"] = conn.execute(
|
||||||
"SELECT changes()"
|
"SELECT changes()"
|
||||||
).fetchone()[0]
|
).fetchone()[0]
|
||||||
|
|
||||||
# 2. Cross-status dedup: for each (model_id, model_version_id),
|
# 2. Cross-status dedup: for each (model_id, model_version_id,
|
||||||
# keep only the entry with the highest-priority terminal status.
|
# file_id), keep only the entry with the highest-priority
|
||||||
|
# terminal status.
|
||||||
# Priority: completed (3) > failed (2) > canceled (1).
|
# Priority: completed (3) > failed (2) > canceled (1).
|
||||||
# This prevents the same model version from having both a
|
# This prevents the same file of a model version from having
|
||||||
# 'failed' and a 'canceled' entry (or a 'completed' alongside
|
# both a 'failed' and a 'canceled' entry (or a 'completed'
|
||||||
# either) after the bug-created duplicates are removed.
|
# alongside either) after the bug-created duplicates are
|
||||||
conn.execute("""
|
# removed. ``IS`` matches NULL file ids against each other so
|
||||||
|
# rows without file identity keep the old behavior.
|
||||||
|
conn.execute(f"""
|
||||||
DELETE FROM download_history
|
DELETE FROM download_history
|
||||||
WHERE id NOT IN (
|
WHERE id NOT IN (
|
||||||
SELECT dh.id
|
SELECT dh.id
|
||||||
FROM download_history dh
|
FROM (
|
||||||
|
SELECT id, model_id, model_version_id, status,
|
||||||
|
{_FILE_ID_SQL} AS file_id
|
||||||
|
FROM download_history
|
||||||
|
) dh
|
||||||
INNER JOIN (
|
INNER JOIN (
|
||||||
SELECT model_id, model_version_id,
|
SELECT model_id, model_version_id,
|
||||||
|
{_FILE_ID_SQL} AS file_id,
|
||||||
MAX(CASE status
|
MAX(CASE status
|
||||||
WHEN 'completed' THEN 3
|
WHEN 'completed' THEN 3
|
||||||
WHEN 'failed' THEN 2
|
WHEN 'failed' THEN 2
|
||||||
@@ -874,17 +928,18 @@ class DownloadQueueService:
|
|||||||
ELSE 0
|
ELSE 0
|
||||||
END) AS best_prio
|
END) AS best_prio
|
||||||
FROM download_history
|
FROM download_history
|
||||||
GROUP BY model_id, model_version_id
|
GROUP BY model_id, model_version_id, {_FILE_ID_SQL}
|
||||||
) best
|
) best
|
||||||
ON dh.model_id = best.model_id
|
ON dh.model_id = best.model_id
|
||||||
AND dh.model_version_id = best.model_version_id
|
AND dh.model_version_id = best.model_version_id
|
||||||
|
AND dh.file_id IS best.file_id
|
||||||
AND CASE dh.status
|
AND CASE dh.status
|
||||||
WHEN 'completed' THEN 3
|
WHEN 'completed' THEN 3
|
||||||
WHEN 'failed' THEN 2
|
WHEN 'failed' THEN 2
|
||||||
WHEN 'canceled' THEN 1
|
WHEN 'canceled' THEN 1
|
||||||
ELSE 0
|
ELSE 0
|
||||||
END = best.best_prio
|
END = best.best_prio
|
||||||
GROUP BY dh.model_id, dh.model_version_id
|
GROUP BY dh.model_id, dh.model_version_id, dh.file_id
|
||||||
HAVING dh.id = MAX(dh.id)
|
HAVING dh.id = MAX(dh.id)
|
||||||
)
|
)
|
||||||
""")
|
""")
|
||||||
@@ -892,15 +947,17 @@ class DownloadQueueService:
|
|||||||
"SELECT changes()"
|
"SELECT changes()"
|
||||||
).fetchone()[0]
|
).fetchone()[0]
|
||||||
|
|
||||||
# 3. Queue: for each (model_id, model_version_id) keep only the
|
# 3. Queue: for each (model_id, model_version_id, file_id) keep
|
||||||
# row with the latest added_at (most recently enqueued).
|
# only the row with the latest added_at (most recently
|
||||||
conn.execute("""
|
# enqueued). file_id comes from file_params (#1058) so
|
||||||
|
# distinct files of the same version never collapse.
|
||||||
|
conn.execute(f"""
|
||||||
DELETE FROM download_queue
|
DELETE FROM download_queue
|
||||||
WHERE rowid NOT IN (
|
WHERE rowid NOT IN (
|
||||||
SELECT MAX(rowid)
|
SELECT MAX(rowid)
|
||||||
FROM download_queue
|
FROM download_queue
|
||||||
WHERE status IN ('queued', 'downloading', 'paused', 'waiting')
|
WHERE status IN ('queued', 'downloading', 'paused', 'waiting')
|
||||||
GROUP BY model_id, model_version_id
|
GROUP BY model_id, model_version_id, {_FILE_ID_SQL}
|
||||||
)
|
)
|
||||||
AND status IN ('queued', 'downloading', 'paused', 'waiting')
|
AND status IN ('queued', 'downloading', 'paused', 'waiting')
|
||||||
""")
|
""")
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
"""Shared download routing logic.
|
||||||
|
|
||||||
|
Decides whether a download initiated from the checkpoint library should be
|
||||||
|
routed to the unet/diffusion-model roots instead of the checkpoint roots.
|
||||||
|
Used by both the download manager (at download time) and the download
|
||||||
|
routing HTTP endpoint (when the user picks a location in the UI), so the
|
||||||
|
two can never disagree.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Iterable, Optional
|
||||||
|
|
||||||
|
from ..utils.constants import (
|
||||||
|
CIVITAI_FILE_TYPE_TO_OTHER_SUB_TYPE,
|
||||||
|
CIVITAI_TYPE_TO_OTHER_SUB_TYPE,
|
||||||
|
DIFFUSION_MODEL_BASE_MODELS,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# File types reported by the CivitAI API that indicate a raw diffusion
|
||||||
|
# model (loaded via UNETLoader in ComfyUI) rather than a full checkpoint.
|
||||||
|
DIFFUSION_FILE_TYPES = frozenset({"UNet", "Diffusion Model"})
|
||||||
|
|
||||||
|
|
||||||
|
def is_diffusion_model_download(
|
||||||
|
model_type: str,
|
||||||
|
file_types: Iterable[str] = (),
|
||||||
|
base_model: str = "",
|
||||||
|
) -> bool:
|
||||||
|
"""Return True when a download should be routed to the unet roots.
|
||||||
|
|
||||||
|
Only applies to downloads initiated from the checkpoint library.
|
||||||
|
Priority: (1) any file has type "UNet" or "Diffusion Model" (the more
|
||||||
|
direct signal from CivitAI), (2) baseModel is a known diffusion model.
|
||||||
|
"""
|
||||||
|
if model_type != "checkpoint":
|
||||||
|
return False
|
||||||
|
|
||||||
|
for file_type in file_types:
|
||||||
|
if file_type in DIFFUSION_FILE_TYPES:
|
||||||
|
logger.info(
|
||||||
|
"File type '%s' detected, routing checkpoint to unet folder",
|
||||||
|
file_type,
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
if base_model in DIFFUSION_MODEL_BASE_MODELS:
|
||||||
|
logger.info(
|
||||||
|
"baseModel '%s' is a known diffusion model, routing to unet folder",
|
||||||
|
base_model,
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_other_download_sub_type(
|
||||||
|
civitai_model_type: str,
|
||||||
|
file_types: Iterable[str] = (),
|
||||||
|
selected_file_type: Optional[str] = None,
|
||||||
|
) -> Optional[str]:
|
||||||
|
"""Resolve the "other"-page sub_type for a download.
|
||||||
|
|
||||||
|
Fixed priority (locked design, docs/plans/other-models-page.md §9.2):
|
||||||
|
|
||||||
|
1. Explicit user file pick — when the picked file's type maps, it wins
|
||||||
|
even when model.type maps to something else.
|
||||||
|
2. model.type via CIVITAI_TYPE_TO_OTHER_SUB_TYPE.
|
||||||
|
3. file.type fallback — only when model.type maps to nothing. Must NOT
|
||||||
|
override a mapped model.type: checkpoint models routinely bundle
|
||||||
|
VAE/Text Encoder component files.
|
||||||
|
4. Still undecidable -> None (caller must ask the user for a folder).
|
||||||
|
"""
|
||||||
|
if selected_file_type:
|
||||||
|
mapped = CIVITAI_FILE_TYPE_TO_OTHER_SUB_TYPE.get(selected_file_type)
|
||||||
|
if mapped:
|
||||||
|
logger.info(
|
||||||
|
"Explicit file pick type '%s' routes other download to '%s'",
|
||||||
|
selected_file_type,
|
||||||
|
mapped,
|
||||||
|
)
|
||||||
|
return mapped
|
||||||
|
|
||||||
|
normalized_model_type = (civitai_model_type or "").strip().lower()
|
||||||
|
mapped = CIVITAI_TYPE_TO_OTHER_SUB_TYPE.get(normalized_model_type)
|
||||||
|
if mapped:
|
||||||
|
return mapped
|
||||||
|
|
||||||
|
for file_type in file_types:
|
||||||
|
mapped = CIVITAI_FILE_TYPE_TO_OTHER_SUB_TYPE.get(file_type)
|
||||||
|
if mapped:
|
||||||
|
logger.info(
|
||||||
|
"model.type '%s' unmapped; file type '%s' routes other download to '%s'",
|
||||||
|
civitai_model_type,
|
||||||
|
file_type,
|
||||||
|
mapped,
|
||||||
|
)
|
||||||
|
return mapped
|
||||||
|
|
||||||
|
return None
|
||||||
@@ -62,6 +62,14 @@ class DownloadedVersionHistoryService:
|
|||||||
);
|
);
|
||||||
CREATE INDEX IF NOT EXISTS idx_downloaded_model_versions_model
|
CREATE INDEX IF NOT EXISTS idx_downloaded_model_versions_model
|
||||||
ON downloaded_model_versions(model_type, model_id);
|
ON downloaded_model_versions(model_type, model_id);
|
||||||
|
CREATE TABLE IF NOT EXISTS downloaded_version_files (
|
||||||
|
model_type TEXT NOT NULL,
|
||||||
|
version_id INTEGER NOT NULL,
|
||||||
|
file_id INTEGER NOT NULL,
|
||||||
|
file_name TEXT,
|
||||||
|
downloaded_at REAL NOT NULL,
|
||||||
|
PRIMARY KEY (model_type, version_id, file_id)
|
||||||
|
);
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, db_path: str | None = None, *, settings_manager=None) -> None:
|
def __init__(self, db_path: str | None = None, *, settings_manager=None) -> None:
|
||||||
@@ -131,10 +139,13 @@ class DownloadedVersionHistoryService:
|
|||||||
source: str = "manual",
|
source: str = "manual",
|
||||||
file_path: str | None = None,
|
file_path: str | None = None,
|
||||||
library_name: str | None = None,
|
library_name: str | None = None,
|
||||||
|
file_id: int | None = None,
|
||||||
|
file_name: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
normalized_type = _normalize_model_type(model_type)
|
normalized_type = _normalize_model_type(model_type)
|
||||||
normalized_version_id = _normalize_int(version_id)
|
normalized_version_id = _normalize_int(version_id)
|
||||||
normalized_model_id = _normalize_int(model_id)
|
normalized_model_id = _normalize_int(model_id)
|
||||||
|
normalized_file_id = _normalize_int(file_id)
|
||||||
if normalized_type is None or normalized_version_id is None:
|
if normalized_type is None or normalized_version_id is None:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -168,6 +179,25 @@ class DownloadedVersionHistoryService:
|
|||||||
active_library_name,
|
active_library_name,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
if normalized_file_id is not None:
|
||||||
|
# Per-file history for multi-file versions (#1058)
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO downloaded_version_files (
|
||||||
|
model_type, version_id, file_id, file_name, downloaded_at
|
||||||
|
) VALUES (?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(model_type, version_id, file_id) DO UPDATE SET
|
||||||
|
file_name = COALESCE(excluded.file_name, downloaded_version_files.file_name),
|
||||||
|
downloaded_at = excluded.downloaded_at
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
normalized_type,
|
||||||
|
normalized_version_id,
|
||||||
|
normalized_file_id,
|
||||||
|
file_name,
|
||||||
|
timestamp,
|
||||||
|
),
|
||||||
|
)
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|
||||||
async def mark_downloaded_bulk(
|
async def mark_downloaded_bulk(
|
||||||
@@ -206,24 +236,33 @@ class DownloadedVersionHistoryService:
|
|||||||
return
|
return
|
||||||
|
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
conn = self._get_conn()
|
# The connection is created with check_same_thread=False and all
|
||||||
conn.executemany(
|
# access is serialized by self._lock, so the executemany upsert +
|
||||||
"""
|
# commit can run in the default executor without blocking the
|
||||||
INSERT INTO downloaded_model_versions (
|
# event loop on large hydration payloads.
|
||||||
model_type, version_id, model_id, first_seen_at, last_seen_at,
|
loop = asyncio.get_running_loop()
|
||||||
source, last_file_path, last_library_name, is_deleted_override
|
await loop.run_in_executor(None, self._mark_downloaded_bulk_sync, payload)
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0)
|
|
||||||
ON CONFLICT(model_type, version_id) DO UPDATE SET
|
def _mark_downloaded_bulk_sync(self, payload: Sequence[tuple[object, ...]]) -> None:
|
||||||
model_id = COALESCE(excluded.model_id, downloaded_model_versions.model_id),
|
"""Synchronous executemany upsert + commit; runs in a worker thread."""
|
||||||
last_seen_at = excluded.last_seen_at,
|
conn = self._get_conn()
|
||||||
source = excluded.source,
|
conn.executemany(
|
||||||
last_file_path = COALESCE(excluded.last_file_path, downloaded_model_versions.last_file_path),
|
"""
|
||||||
last_library_name = COALESCE(excluded.last_library_name, downloaded_model_versions.last_library_name),
|
INSERT INTO downloaded_model_versions (
|
||||||
is_deleted_override = 0
|
model_type, version_id, model_id, first_seen_at, last_seen_at,
|
||||||
""",
|
source, last_file_path, last_library_name, is_deleted_override
|
||||||
payload,
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0)
|
||||||
)
|
ON CONFLICT(model_type, version_id) DO UPDATE SET
|
||||||
conn.commit()
|
model_id = COALESCE(excluded.model_id, downloaded_model_versions.model_id),
|
||||||
|
last_seen_at = excluded.last_seen_at,
|
||||||
|
source = excluded.source,
|
||||||
|
last_file_path = COALESCE(excluded.last_file_path, downloaded_model_versions.last_file_path),
|
||||||
|
last_library_name = COALESCE(excluded.last_library_name, downloaded_model_versions.last_library_name),
|
||||||
|
is_deleted_override = 0
|
||||||
|
""",
|
||||||
|
payload,
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
async def mark_as_deleted(self, model_type: str, version_id: int) -> None:
|
async def mark_as_deleted(self, model_type: str, version_id: int) -> None:
|
||||||
normalized_type = _normalize_model_type(model_type)
|
normalized_type = _normalize_model_type(model_type)
|
||||||
@@ -255,8 +294,63 @@ class DownloadedVersionHistoryService:
|
|||||||
self._get_active_library_name(),
|
self._get_active_library_name(),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
# Whole-version deletion also clears the per-file records (#1058)
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
DELETE FROM downloaded_version_files
|
||||||
|
WHERE model_type = ? AND version_id = ?
|
||||||
|
""",
|
||||||
|
(normalized_type, normalized_version_id),
|
||||||
|
)
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|
||||||
|
async def mark_file_deleted(
|
||||||
|
self, model_type: str, version_id: int, file_id: int
|
||||||
|
) -> None:
|
||||||
|
"""Drop a single file record of a version, keeping siblings (#1058)."""
|
||||||
|
normalized_type = _normalize_model_type(model_type)
|
||||||
|
normalized_version_id = _normalize_int(version_id)
|
||||||
|
normalized_file_id = _normalize_int(file_id)
|
||||||
|
if (
|
||||||
|
normalized_type is None
|
||||||
|
or normalized_version_id is None
|
||||||
|
or normalized_file_id is None
|
||||||
|
):
|
||||||
|
return
|
||||||
|
|
||||||
|
async with self._lock:
|
||||||
|
conn = self._get_conn()
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
DELETE FROM downloaded_version_files
|
||||||
|
WHERE model_type = ? AND version_id = ? AND file_id = ?
|
||||||
|
""",
|
||||||
|
(normalized_type, normalized_version_id, normalized_file_id),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
async def get_downloaded_file_ids(
|
||||||
|
self, model_type: str, version_id: int
|
||||||
|
) -> list[int]:
|
||||||
|
"""Return the CivitAI file ids recorded as downloaded for a version."""
|
||||||
|
normalized_type = _normalize_model_type(model_type)
|
||||||
|
normalized_version_id = _normalize_int(version_id)
|
||||||
|
if normalized_type is None or normalized_version_id is None:
|
||||||
|
return []
|
||||||
|
|
||||||
|
async with self._lock:
|
||||||
|
conn = self._get_conn()
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT file_id
|
||||||
|
FROM downloaded_version_files
|
||||||
|
WHERE model_type = ? AND version_id = ?
|
||||||
|
ORDER BY file_id ASC
|
||||||
|
""",
|
||||||
|
(normalized_type, normalized_version_id),
|
||||||
|
).fetchall()
|
||||||
|
return [int(row["file_id"]) for row in rows]
|
||||||
|
|
||||||
async def has_been_downloaded(self, model_type: str, version_id: int) -> bool:
|
async def has_been_downloaded(self, model_type: str, version_id: int) -> bool:
|
||||||
normalized_type = _normalize_model_type(model_type)
|
normalized_type = _normalize_model_type(model_type)
|
||||||
normalized_version_id = _normalize_int(version_id)
|
normalized_version_id = _normalize_int(version_id)
|
||||||
|
|||||||
+145
-57
@@ -32,6 +32,7 @@ from .connectivity_guard import (
|
|||||||
ConnectivityGuard,
|
ConnectivityGuard,
|
||||||
)
|
)
|
||||||
from .errors import RateLimitError
|
from .errors import RateLimitError
|
||||||
|
from .rate_limit_coordinator import RateLimitCoordinator
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -156,6 +157,25 @@ class DownloadStalledError(Exception):
|
|||||||
"""Raised when download progress stalls beyond the configured timeout."""
|
"""Raised when download progress stalls beyond the configured timeout."""
|
||||||
|
|
||||||
|
|
||||||
|
def _disable_netrc_auth(session: aiohttp.ClientSession) -> None:
|
||||||
|
"""Prevent the session from loading credentials from netrc files.
|
||||||
|
|
||||||
|
``trust_env=True`` is kept so system-level proxies still work, but aiohttp
|
||||||
|
would also auto-apply netrc entries (e.g. ``machine civitai.red``) as
|
||||||
|
BasicAuth. aiohttp refuses to combine those with the explicit
|
||||||
|
``Authorization: Bearer`` header set for CivitAI requests, raising
|
||||||
|
"Cannot combine AUTHORIZATION header with AUTH argument or credentials
|
||||||
|
encoded in URL" before the request is even sent. Subclassing ClientSession
|
||||||
|
is discouraged by aiohttp (emits a DeprecationWarning), so the private
|
||||||
|
hook is patched on the instance instead.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _no_netrc_auth(*args: Any, **kwargs: Any) -> Optional[aiohttp.BasicAuth]:
|
||||||
|
return None
|
||||||
|
|
||||||
|
setattr(session, "_get_netrc_auth", _no_netrc_auth)
|
||||||
|
|
||||||
|
|
||||||
class Downloader:
|
class Downloader:
|
||||||
"""Unified downloader for all HTTP/HTTPS downloads in the application."""
|
"""Unified downloader for all HTTP/HTTPS downloads in the application."""
|
||||||
|
|
||||||
@@ -370,6 +390,7 @@ class Downloader:
|
|||||||
trust_env=not app_proxy_active,
|
trust_env=not app_proxy_active,
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
|
_disable_netrc_auth(self._session)
|
||||||
|
|
||||||
# Store proxy URL for per-request use. Stays None for SOCKS because the
|
# Store proxy URL for per-request use. Stays None for SOCKS because the
|
||||||
# ProxyConnector already tunnels everything; passing proxy= for SOCKS
|
# ProxyConnector already tunnels everything; passing proxy= for SOCKS
|
||||||
@@ -575,6 +596,21 @@ class Downloader:
|
|||||||
False,
|
False,
|
||||||
"File not found - the download link may be invalid or expired.",
|
"File not found - the download link may be invalid or expired.",
|
||||||
)
|
)
|
||||||
|
elif response.status == 429:
|
||||||
|
# Register the vendor's cooldown so API calls through
|
||||||
|
# make_request queue behind it (#1085). The download
|
||||||
|
# itself fails as before; retry policy stays with the
|
||||||
|
# caller (download manager).
|
||||||
|
retry_after = self._extract_retry_after(response.headers)
|
||||||
|
coordinator = await RateLimitCoordinator.get_instance()
|
||||||
|
if coordinator.enabled:
|
||||||
|
coordinator.register_rate_limit(
|
||||||
|
self._guard_destination(url), retry_after
|
||||||
|
)
|
||||||
|
logger.warning(
|
||||||
|
f"Rate limited (429) for {url}, retry_after={retry_after}"
|
||||||
|
)
|
||||||
|
return False, f"Download rate limited (429), retry after {retry_after}s"
|
||||||
else:
|
else:
|
||||||
logger.error(
|
logger.error(
|
||||||
f"Download failed for {url} with status {response.status}"
|
f"Download failed for {url} with status {response.status}"
|
||||||
@@ -952,6 +988,11 @@ class Downloader:
|
|||||||
elif response.status == 429:
|
elif response.status == 429:
|
||||||
raw_retry_after = response.headers.get("Retry-After")
|
raw_retry_after = response.headers.get("Retry-After")
|
||||||
retry_after = _parse_retry_after(raw_retry_after or "")
|
retry_after = _parse_retry_after(raw_retry_after or "")
|
||||||
|
# Register the vendor's cooldown so API calls through
|
||||||
|
# make_request queue behind it (#1085).
|
||||||
|
coordinator = await RateLimitCoordinator.get_instance()
|
||||||
|
if coordinator.enabled:
|
||||||
|
coordinator.register_rate_limit(destination, retry_after)
|
||||||
if raw_retry_after:
|
if raw_retry_after:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Rate limited (429) for %s, Retry-After: %ss", url, retry_after
|
"Rate limited (429) for %s, Retry-After: %ss", url, retry_after
|
||||||
@@ -1021,6 +1062,14 @@ class Downloader:
|
|||||||
if response.status == 200:
|
if response.status == 200:
|
||||||
guard.register_success(destination)
|
guard.register_success(destination)
|
||||||
return True, dict(response.headers)
|
return True, dict(response.headers)
|
||||||
|
elif response.status == 429:
|
||||||
|
# Register the vendor's cooldown so API calls through
|
||||||
|
# make_request queue behind it (#1085).
|
||||||
|
retry_after = self._extract_retry_after(response.headers)
|
||||||
|
coordinator = await RateLimitCoordinator.get_instance()
|
||||||
|
if coordinator.enabled:
|
||||||
|
coordinator.register_rate_limit(destination, retry_after)
|
||||||
|
return False, f"Head request rate limited (429), retry after {retry_after}s"
|
||||||
else:
|
else:
|
||||||
return False, f"Head request failed with status {response.status}"
|
return False, f"Head request failed with status {response.status}"
|
||||||
|
|
||||||
@@ -1054,74 +1103,113 @@ class Downloader:
|
|||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple[bool, Union[Dict, str]]: (success, response data or error message)
|
Tuple[bool, Union[Dict, str]]: (success, response data or error message)
|
||||||
|
|
||||||
|
When the rate-limit gate is enabled (``rate_limit_gate_enabled``),
|
||||||
|
requests are paced per destination and 429 responses are honored by
|
||||||
|
waiting out the ``Retry-After`` window (bounded by
|
||||||
|
``rate_limit_max_wait_seconds``) before re-sending. A ``RateLimitError``
|
||||||
|
returned after gate involvement is marked with ``gate_handled = True``
|
||||||
|
so downstream retry helpers do not wait a second time.
|
||||||
"""
|
"""
|
||||||
guard = await ConnectivityGuard.get_instance()
|
guard = await ConnectivityGuard.get_instance()
|
||||||
destination = self._guard_destination(url)
|
destination = self._guard_destination(url)
|
||||||
|
# Fail fast on transport-level outages before pacing: there is no
|
||||||
|
# point waiting out a vendor cooldown while the network is down.
|
||||||
if guard.should_block_request(destination):
|
if guard.should_block_request(destination):
|
||||||
return False, OFFLINE_COOLDOWN_ERROR
|
return False, OFFLINE_COOLDOWN_ERROR
|
||||||
|
|
||||||
try:
|
coordinator = await RateLimitCoordinator.get_instance()
|
||||||
session = await self.session
|
gate_enabled = coordinator.enabled
|
||||||
# Debug log for proxy mode at request time
|
# Safety bound on the wait-and-resend loop; each 429 normally exits
|
||||||
if self.proxy_url:
|
# via the wait cap in wait_for_slot, this covers pathological 429s
|
||||||
logger.debug(f"[make_request] Using app-level proxy: {self.proxy_url}")
|
# with tiny Retry-After values.
|
||||||
else:
|
max_resend_attempts = 5
|
||||||
logger.debug(
|
attempt = 0
|
||||||
"[make_request] Using system-level proxy (trust_env) if configured."
|
|
||||||
)
|
|
||||||
|
|
||||||
# Prepare headers
|
while True:
|
||||||
headers = self._get_auth_headers(use_auth)
|
if gate_enabled:
|
||||||
if custom_headers:
|
try:
|
||||||
headers.update(custom_headers)
|
await coordinator.wait_for_slot(destination)
|
||||||
|
except RateLimitError as exc:
|
||||||
|
exc.gate_handled = True
|
||||||
|
return False, exc
|
||||||
|
|
||||||
# Add proxy to kwargs if not already present
|
try:
|
||||||
if "proxy" not in kwargs:
|
session = await self.session
|
||||||
kwargs["proxy"] = self.proxy_url
|
# Debug log for proxy mode at request time
|
||||||
|
if self.proxy_url:
|
||||||
async with session.request(
|
logger.debug(f"[make_request] Using app-level proxy: {self.proxy_url}")
|
||||||
method, url, headers=headers, **kwargs
|
|
||||||
) as response:
|
|
||||||
if response.status == 200:
|
|
||||||
guard.register_success(destination)
|
|
||||||
# Try to parse as JSON, fall back to text
|
|
||||||
try:
|
|
||||||
data = await response.json()
|
|
||||||
return True, data
|
|
||||||
except:
|
|
||||||
text = await response.text()
|
|
||||||
return True, text
|
|
||||||
elif response.status == 401:
|
|
||||||
return False, "Unauthorized access - invalid or missing API key"
|
|
||||||
elif response.status == 403:
|
|
||||||
return False, "Access forbidden"
|
|
||||||
elif response.status == 404:
|
|
||||||
return False, "Resource not found"
|
|
||||||
elif response.status == 429:
|
|
||||||
retry_after = self._extract_retry_after(response.headers)
|
|
||||||
error_msg = "Request rate limited"
|
|
||||||
logger.warning(
|
|
||||||
"Rate limit encountered for %s %s; retry_after=%s",
|
|
||||||
method,
|
|
||||||
url,
|
|
||||||
retry_after,
|
|
||||||
)
|
|
||||||
return False, RateLimitError(
|
|
||||||
error_msg,
|
|
||||||
retry_after=retry_after,
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
return False, f"Request failed with status {response.status}"
|
logger.debug(
|
||||||
|
"[make_request] Using system-level proxy (trust_env) if configured."
|
||||||
|
)
|
||||||
|
|
||||||
except Exception as e:
|
# Prepare headers
|
||||||
if guard.is_network_unreachable_error(e):
|
headers = self._get_auth_headers(use_auth)
|
||||||
guard.register_network_failure(e, destination)
|
if custom_headers:
|
||||||
if guard.should_block_request(destination):
|
headers.update(custom_headers)
|
||||||
return False, OFFLINE_COOLDOWN_ERROR
|
|
||||||
logger.debug("Network unavailable for %s %s: %s", method, url, e)
|
# Add proxy to kwargs if not already present
|
||||||
|
if "proxy" not in kwargs:
|
||||||
|
kwargs["proxy"] = self.proxy_url
|
||||||
|
|
||||||
|
async with session.request(
|
||||||
|
method, url, headers=headers, **kwargs
|
||||||
|
) as response:
|
||||||
|
if response.status == 200:
|
||||||
|
guard.register_success(destination)
|
||||||
|
if gate_enabled:
|
||||||
|
coordinator.register_success(destination)
|
||||||
|
# Try to parse as JSON, fall back to text
|
||||||
|
try:
|
||||||
|
data = await response.json()
|
||||||
|
return True, data
|
||||||
|
except:
|
||||||
|
text = await response.text()
|
||||||
|
return True, text
|
||||||
|
elif response.status == 401:
|
||||||
|
return False, "Unauthorized access - invalid or missing API key"
|
||||||
|
elif response.status == 403:
|
||||||
|
return False, "Access forbidden"
|
||||||
|
elif response.status == 404:
|
||||||
|
return False, "Resource not found"
|
||||||
|
elif response.status == 429:
|
||||||
|
retry_after = self._extract_retry_after(response.headers)
|
||||||
|
error_msg = "Request rate limited"
|
||||||
|
if not gate_enabled:
|
||||||
|
logger.warning(
|
||||||
|
"Rate limit encountered for %s %s; retry_after=%s",
|
||||||
|
method,
|
||||||
|
url,
|
||||||
|
retry_after,
|
||||||
|
)
|
||||||
|
return False, RateLimitError(
|
||||||
|
error_msg,
|
||||||
|
retry_after=retry_after,
|
||||||
|
)
|
||||||
|
# The coordinator logs the cooldown notice (INFO once
|
||||||
|
# per window, DEBUG on extension).
|
||||||
|
coordinator.register_rate_limit(destination, retry_after)
|
||||||
|
attempt += 1
|
||||||
|
if attempt >= max_resend_attempts:
|
||||||
|
error = RateLimitError(error_msg, retry_after=retry_after)
|
||||||
|
error.gate_handled = True
|
||||||
|
return False, error
|
||||||
|
# Loop back: wait_for_slot blocks until the cooldown
|
||||||
|
# elapses (or raises once the wait exceeds the cap).
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
return False, f"Request failed with status {response.status}"
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
if guard.is_network_unreachable_error(e):
|
||||||
|
guard.register_network_failure(e, destination)
|
||||||
|
if guard.should_block_request(destination):
|
||||||
|
return False, OFFLINE_COOLDOWN_ERROR
|
||||||
|
logger.debug("Network unavailable for %s %s: %s", method, url, e)
|
||||||
|
return False, str(e)
|
||||||
|
logger.error(f"Error making {method} request to {url}: {e}")
|
||||||
return False, str(e)
|
return False, str(e)
|
||||||
logger.error(f"Error making {method} request to {url}: {e}")
|
|
||||||
return False, str(e)
|
|
||||||
|
|
||||||
async def close(self):
|
async def close(self):
|
||||||
"""Close the HTTP session"""
|
"""Close the HTTP session"""
|
||||||
|
|||||||
@@ -51,6 +51,7 @@ class EmbeddingService(BaseModelService):
|
|||||||
"base_model": model_data.get("base_model", ""),
|
"base_model": model_data.get("base_model", ""),
|
||||||
"folder": folder,
|
"folder": folder,
|
||||||
"sha256": model_data.get("sha256", ""),
|
"sha256": model_data.get("sha256", ""),
|
||||||
|
"autov3": model_data.get("autov3"),
|
||||||
"file_path": file_path.replace(os.sep, "/"),
|
"file_path": file_path.replace(os.sep, "/"),
|
||||||
"file_size": model_data.get("size", 0),
|
"file_size": model_data.get("size", 0),
|
||||||
"modified": model_data.get("modified", ""),
|
"modified": model_data.get("modified", ""),
|
||||||
|
|||||||
+92
-49
@@ -11,6 +11,7 @@ from __future__ import annotations
|
|||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import time
|
||||||
from typing import Any, Dict, List, Optional
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
@@ -32,8 +33,26 @@ _catalog_cache: Optional[Dict[str, List[str]]] = None
|
|||||||
# ``{provider_id: {model_id: max_output_tokens}}``.
|
# ``{provider_id: {model_id: max_output_tokens}}``.
|
||||||
_model_output_limits: Dict[str, Dict[str, int]] = {}
|
_model_output_limits: Dict[str, Dict[str, int]] = {}
|
||||||
|
|
||||||
|
# Monotonic timestamp of the last failed catalog fetch (None = no failure
|
||||||
|
# yet). Failed fetches are negatively cached: further calls return the
|
||||||
|
# empty fallback without hitting the network until the cooldown elapses,
|
||||||
|
# so users on broken networks don't stall on every settings-modal open.
|
||||||
|
_catalog_last_failure: Optional[float] = None
|
||||||
|
_CATALOG_FAILURE_COOLDOWN = 600.0 # seconds
|
||||||
|
|
||||||
|
# Serializes catalog fetches so concurrent callers don't duplicate requests.
|
||||||
|
_catalog_lock = asyncio.Lock()
|
||||||
|
|
||||||
_CATALOG_TIMEOUT = aiohttp.ClientTimeout(total=30)
|
_CATALOG_TIMEOUT = aiohttp.ClientTimeout(total=30)
|
||||||
|
|
||||||
|
# Cloudflare serves brotli when the client advertises it, and brotli is a
|
||||||
|
# required dependency here — a corrupted br stream can crash the native
|
||||||
|
# decoder with a Windows access violation (issue #1099). Request gzip
|
||||||
|
# instead; zlib decompression is not affected and corrupt gzip data only
|
||||||
|
# raises ContentEncodingError (an aiohttp.ClientError subclass), which the
|
||||||
|
# exception handlers below already catch.
|
||||||
|
_NO_BROTLI_HEADERS = {"Accept-Encoding": "gzip, deflate"}
|
||||||
|
|
||||||
|
|
||||||
async def _load_model_catalog() -> Dict[str, List[str]]:
|
async def _load_model_catalog() -> Dict[str, List[str]]:
|
||||||
"""Fetch and parse the model catalog.
|
"""Fetch and parse the model catalog.
|
||||||
@@ -46,61 +65,85 @@ async def _load_model_catalog() -> Dict[str, List[str]]:
|
|||||||
value has a ``models`` sub-dict keyed by model ID. The result is cached
|
value has a ``models`` sub-dict keyed by model ID. The result is cached
|
||||||
in memory after the first successful fetch.
|
in memory after the first successful fetch.
|
||||||
Subsequent calls return the cached data immediately.
|
Subsequent calls return the cached data immediately.
|
||||||
|
|
||||||
|
Failed fetches are negatively cached: further calls return an empty
|
||||||
|
dict without hitting the network until ``_CATALOG_FAILURE_COOLDOWN``
|
||||||
|
has elapsed, so a broken network does not stall every settings-modal
|
||||||
|
open. Concurrent callers are serialized behind :data:`_catalog_lock`
|
||||||
|
so only one request is ever in flight.
|
||||||
"""
|
"""
|
||||||
global _catalog_cache, _model_output_limits
|
global _catalog_cache, _model_output_limits, _catalog_last_failure
|
||||||
if _catalog_cache is not None:
|
if _catalog_cache is not None:
|
||||||
return _catalog_cache
|
return _catalog_cache
|
||||||
|
|
||||||
try:
|
async with _catalog_lock:
|
||||||
async with aiohttp.ClientSession(timeout=_CATALOG_TIMEOUT) as session:
|
# Re-check under the lock: another caller may have fetched (or
|
||||||
async with session.get(_MODEL_CATALOG_URL) as resp:
|
# failed) while we were waiting.
|
||||||
if resp.status != 200:
|
if _catalog_cache is not None:
|
||||||
logger.warning("Model catalog returned HTTP %s", resp.status)
|
return _catalog_cache
|
||||||
return _catalog_cache or {}
|
if (
|
||||||
data = await resp.json()
|
_catalog_last_failure is not None
|
||||||
except (aiohttp.ClientError, asyncio.TimeoutError, json.JSONDecodeError) as exc:
|
and time.monotonic() - _catalog_last_failure < _CATALOG_FAILURE_COOLDOWN
|
||||||
logger.warning("Failed to fetch model catalog: %s", exc)
|
):
|
||||||
return _catalog_cache or {}
|
logger.debug(
|
||||||
|
"Skipping model catalog fetch: last attempt failed %.0fs ago",
|
||||||
|
time.monotonic() - _catalog_last_failure,
|
||||||
|
)
|
||||||
|
return {}
|
||||||
|
|
||||||
if not isinstance(data, dict):
|
try:
|
||||||
logger.warning("Model catalog is not a dict, got %s", type(data).__name__)
|
async with aiohttp.ClientSession(timeout=_CATALOG_TIMEOUT) as session:
|
||||||
return _catalog_cache or {}
|
async with session.get(_MODEL_CATALOG_URL, headers=_NO_BROTLI_HEADERS) as resp:
|
||||||
|
if resp.status != 200:
|
||||||
|
logger.warning("Model catalog returned HTTP %s", resp.status)
|
||||||
|
_catalog_last_failure = time.monotonic()
|
||||||
|
return {}
|
||||||
|
data = await resp.json()
|
||||||
|
except (aiohttp.ClientError, asyncio.TimeoutError, json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||||
|
logger.warning("Failed to fetch model catalog: %s", exc)
|
||||||
|
_catalog_last_failure = time.monotonic()
|
||||||
|
return {}
|
||||||
|
|
||||||
result: Dict[str, List[str]] = {}
|
if not isinstance(data, dict):
|
||||||
output_limits: Dict[str, Dict[str, int]] = {}
|
logger.warning("Model catalog is not a dict, got %s", type(data).__name__)
|
||||||
for provider_id, provider_info in data.items():
|
_catalog_last_failure = time.monotonic()
|
||||||
if not isinstance(provider_info, dict):
|
return {}
|
||||||
continue
|
|
||||||
models_dict = provider_info.get("models")
|
result: Dict[str, List[str]] = {}
|
||||||
if not isinstance(models_dict, dict):
|
output_limits: Dict[str, Dict[str, int]] = {}
|
||||||
continue
|
for provider_id, provider_info in data.items():
|
||||||
model_ids: List[str] = []
|
if not isinstance(provider_info, dict):
|
||||||
provider_limits: Dict[str, int] = {}
|
|
||||||
for mid, model_info in models_dict.items():
|
|
||||||
if not isinstance(mid, str):
|
|
||||||
continue
|
continue
|
||||||
model_ids.append(mid)
|
models_dict = provider_info.get("models")
|
||||||
if isinstance(model_info, dict):
|
if not isinstance(models_dict, dict):
|
||||||
limit = model_info.get("limit")
|
continue
|
||||||
if isinstance(limit, dict):
|
model_ids: List[str] = []
|
||||||
output = limit.get("output")
|
provider_limits: Dict[str, int] = {}
|
||||||
if isinstance(output, (int, float)) and output > 0:
|
for mid, model_info in models_dict.items():
|
||||||
provider_limits[mid] = int(output)
|
if not isinstance(mid, str):
|
||||||
if model_ids:
|
continue
|
||||||
result[provider_id] = model_ids
|
model_ids.append(mid)
|
||||||
if provider_limits:
|
if isinstance(model_info, dict):
|
||||||
output_limits[provider_id] = provider_limits
|
limit = model_info.get("limit")
|
||||||
|
if isinstance(limit, dict):
|
||||||
|
output = limit.get("output")
|
||||||
|
if isinstance(output, (int, float)) and output > 0:
|
||||||
|
provider_limits[mid] = int(output)
|
||||||
|
if model_ids:
|
||||||
|
result[provider_id] = model_ids
|
||||||
|
if provider_limits:
|
||||||
|
output_limits[provider_id] = provider_limits
|
||||||
|
|
||||||
_catalog_cache = result
|
_catalog_cache = result
|
||||||
_model_output_limits = output_limits
|
_model_output_limits = output_limits
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Loaded model catalog: %d providers, %d total models "
|
"Loaded model catalog: %d providers, %d total models "
|
||||||
"(%d providers have output limits)",
|
"(%d providers have output limits)",
|
||||||
len(result),
|
len(result),
|
||||||
sum(len(m) for m in result.values()),
|
sum(len(m) for m in result.values()),
|
||||||
len(output_limits),
|
len(output_limits),
|
||||||
)
|
)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def _get_model_max_output(provider: str, model: str) -> Optional[int]:
|
def _get_model_max_output(provider: str, model: str) -> Optional[int]:
|
||||||
@@ -126,12 +169,12 @@ async def fetch_ollama_models(api_base: str) -> List[str]:
|
|||||||
url = f"{api_base.rstrip('/')}/models"
|
url = f"{api_base.rstrip('/')}/models"
|
||||||
try:
|
try:
|
||||||
async with aiohttp.ClientSession(timeout=_OLLAMA_API_TIMEOUT) as session:
|
async with aiohttp.ClientSession(timeout=_OLLAMA_API_TIMEOUT) as session:
|
||||||
async with session.get(url) as resp:
|
async with session.get(url, headers=_NO_BROTLI_HEADERS) as resp:
|
||||||
if resp.status != 200:
|
if resp.status != 200:
|
||||||
logger.debug("Ollama API returned HTTP %s from %s", resp.status, api_base)
|
logger.debug("Ollama API returned HTTP %s from %s", resp.status, api_base)
|
||||||
return []
|
return []
|
||||||
data = await resp.json()
|
data = await resp.json()
|
||||||
except (aiohttp.ClientError, asyncio.TimeoutError, json.JSONDecodeError) as exc:
|
except (aiohttp.ClientError, asyncio.TimeoutError, json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||||
logger.debug("Ollama not reachable at %s: %s", api_base, exc)
|
logger.debug("Ollama not reachable at %s: %s", api_base, exc)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|||||||
@@ -58,6 +58,7 @@ class LoraService(BaseModelService):
|
|||||||
"base_model": model_data.get("base_model", ""),
|
"base_model": model_data.get("base_model", ""),
|
||||||
"folder": folder,
|
"folder": folder,
|
||||||
"sha256": model_data.get("sha256", ""),
|
"sha256": model_data.get("sha256", ""),
|
||||||
|
"autov3": model_data.get("autov3"),
|
||||||
"file_path": file_path.replace(os.sep, "/"),
|
"file_path": file_path.replace(os.sep, "/"),
|
||||||
"file_size": model_data.get("size", 0),
|
"file_size": model_data.get("size", 0),
|
||||||
"modified": model_data.get("modified", ""),
|
"modified": model_data.get("modified", ""),
|
||||||
|
|||||||
@@ -245,16 +245,23 @@ class MetadataSyncService:
|
|||||||
civitai_api_not_found = False
|
civitai_api_not_found = False
|
||||||
any_rate_limited = False
|
any_rate_limited = False
|
||||||
|
|
||||||
|
skip_network_providers = False
|
||||||
for provider_name, provider in provider_attempts:
|
for provider_name, provider in provider_attempts:
|
||||||
|
if skip_network_providers and provider_name != "sqlite":
|
||||||
|
# A network provider was already rate-limited; failing
|
||||||
|
# over to another network provider just spreads the flood
|
||||||
|
# (#1085). The local sqlite archive stays as last resort.
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
civitai_metadata_candidate, error = await provider.get_model_by_hash(sha256)
|
civitai_metadata_candidate, error = await provider.get_model_by_hash(sha256)
|
||||||
except RateLimitError as exc:
|
except RateLimitError as exc:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
|
||||||
provider_name or provider.__class__.__name__,
|
provider_name or provider.__class__.__name__,
|
||||||
exc.retry_after or 0,
|
exc.retry_after or 0,
|
||||||
)
|
)
|
||||||
any_rate_limited = True
|
any_rate_limited = True
|
||||||
|
skip_network_providers = True
|
||||||
continue
|
continue
|
||||||
except Exception as exc: # pragma: no cover - defensive logging
|
except Exception as exc: # pragma: no cover - defensive logging
|
||||||
logger.error("Provider %s failed for hash %s: %s", provider_name, sha256, exc)
|
logger.error("Provider %s failed for hash %s: %s", provider_name, sha256, exc)
|
||||||
@@ -419,14 +426,37 @@ class MetadataSyncService:
|
|||||||
metadata: Dict[str, Any],
|
metadata: Dict[str, Any],
|
||||||
model_id: int,
|
model_id: int,
|
||||||
model_version_id: Optional[int],
|
model_version_id: Optional[int],
|
||||||
|
provider_name: Optional[str] = None,
|
||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
"""Relink a local metadata record to a specific CivitAI model version."""
|
"""Relink a local metadata record to a specific CivitAI model version.
|
||||||
|
|
||||||
|
When ``provider_name`` is given, the named provider is resolved via the
|
||||||
|
metadata provider selector instead of the default fallback chain. A
|
||||||
|
missing/disabled provider surfaces a user-friendly error instead of the
|
||||||
|
raw selector exception.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if provider_name:
|
||||||
|
try:
|
||||||
|
provider = await self._get_provider(provider_name)
|
||||||
|
except ValueError as exc:
|
||||||
|
logger.warning(
|
||||||
|
"Unable to resolve metadata provider %s: %s", provider_name, exc
|
||||||
|
)
|
||||||
|
raise ValueError(
|
||||||
|
"CivitArchive is not available or not enabled. "
|
||||||
|
"Enable the CivitArchive API in settings to relink via CivArchive."
|
||||||
|
) from exc
|
||||||
|
else:
|
||||||
|
provider = await self._get_default_provider()
|
||||||
|
|
||||||
provider = await self._get_default_provider()
|
|
||||||
civitai_metadata = await provider.get_model_version(model_id, model_version_id)
|
civitai_metadata = await provider.get_model_version(model_id, model_version_id)
|
||||||
if not civitai_metadata:
|
if not civitai_metadata:
|
||||||
|
provider_label = (
|
||||||
|
"CivitArchive" if provider_name == "civarchive_api" else "CivitAI"
|
||||||
|
)
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Model version not found on CivitAI for ID: {model_id}"
|
f"Model version not found on {provider_label} for ID: {model_id}"
|
||||||
+ (f" with version: {model_version_id}" if model_version_id else "")
|
+ (f" with version: {model_version_id}" if model_version_id else "")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -33,8 +33,17 @@ class ModelCache:
|
|||||||
|
|
||||||
raw_data: List[Dict[str, Any]]
|
raw_data: List[Dict[str, Any]]
|
||||||
folders: List[str]
|
folders: List[str]
|
||||||
|
# Every directory under the model roots (including empty ones), as
|
||||||
|
# recorded by the last scan/hydration. ``None`` means "never recorded"
|
||||||
|
# (e.g. a persisted snapshot predating this field) and triggers a
|
||||||
|
# background filesystem backfill in the scanner.
|
||||||
|
all_folders: Optional[List[str]] = None
|
||||||
version_index: Dict[int, Dict[str, Any]] = field(default_factory=dict)
|
version_index: Dict[int, Dict[str, Any]] = field(default_factory=dict)
|
||||||
model_id_index: Dict[int, List[Dict[str, Any]]] = field(default_factory=dict)
|
model_id_index: Dict[int, List[Dict[str, Any]]] = field(default_factory=dict)
|
||||||
|
# Multi-valued companion to version_index: every local file entry of a
|
||||||
|
# CivitAI model version, so versions with several downloaded files stay
|
||||||
|
# consistent (#1058).
|
||||||
|
version_files_index: Dict[int, List[Dict[str, Any]]] = field(default_factory=dict)
|
||||||
name_display_mode: str = "model_name"
|
name_display_mode: str = "model_name"
|
||||||
_lock: Any = field(init=False, repr=False, default=None)
|
_lock: Any = field(init=False, repr=False, default=None)
|
||||||
# Cache for last sort: (sort_key, order, seed) -> sorted list
|
# Cache for last sort: (sort_key, order, seed) -> sorted list
|
||||||
@@ -116,6 +125,7 @@ class ModelCache:
|
|||||||
|
|
||||||
self.version_index = {}
|
self.version_index = {}
|
||||||
self.model_id_index = {}
|
self.model_id_index = {}
|
||||||
|
self.version_files_index = {}
|
||||||
for item in self.raw_data:
|
for item in self.raw_data:
|
||||||
self.add_to_version_index(item)
|
self.add_to_version_index(item)
|
||||||
|
|
||||||
@@ -132,6 +142,17 @@ class ModelCache:
|
|||||||
|
|
||||||
self.version_index[version_id] = item
|
self.version_index[version_id] = item
|
||||||
|
|
||||||
|
# Register in the multi-valued index, deduplicated by file_path (#1058)
|
||||||
|
files = self.version_files_index.setdefault(version_id, [])
|
||||||
|
for entry in files:
|
||||||
|
if entry is item or (
|
||||||
|
isinstance(entry, dict)
|
||||||
|
and entry.get('file_path') == item.get('file_path')
|
||||||
|
):
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
files.append(item)
|
||||||
|
|
||||||
model_id = self._normalize_version_id(civitai_data.get('modelId'))
|
model_id = self._normalize_version_id(civitai_data.get('modelId'))
|
||||||
if model_id is None:
|
if model_id is None:
|
||||||
return
|
return
|
||||||
@@ -159,12 +180,37 @@ class ModelCache:
|
|||||||
if version_id is None:
|
if version_id is None:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# Drop only this file's entry from the multi-valued index (#1058)
|
||||||
|
files = self.version_files_index.get(version_id)
|
||||||
|
if files:
|
||||||
|
remaining = [
|
||||||
|
entry
|
||||||
|
for entry in files
|
||||||
|
if not (
|
||||||
|
entry is item
|
||||||
|
or (
|
||||||
|
isinstance(entry, dict)
|
||||||
|
and entry.get('file_path') == item.get('file_path')
|
||||||
|
)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
if remaining:
|
||||||
|
self.version_files_index[version_id] = remaining
|
||||||
|
else:
|
||||||
|
self.version_files_index.pop(version_id, None)
|
||||||
|
|
||||||
|
# A surviving sibling file keeps the version present in the indexes
|
||||||
|
sibling = (self.version_files_index.get(version_id) or [None])[0]
|
||||||
|
|
||||||
existing = self.version_index.get(version_id)
|
existing = self.version_index.get(version_id)
|
||||||
if existing is item or (
|
if existing is item or (
|
||||||
isinstance(existing, dict)
|
isinstance(existing, dict)
|
||||||
and existing.get('file_path') == item.get('file_path')
|
and existing.get('file_path') == item.get('file_path')
|
||||||
):
|
):
|
||||||
self.version_index.pop(version_id, None)
|
if sibling is not None:
|
||||||
|
self.version_index[version_id] = sibling
|
||||||
|
else:
|
||||||
|
self.version_index.pop(version_id, None)
|
||||||
|
|
||||||
model_id = self._normalize_version_id(civitai_data.get('modelId'))
|
model_id = self._normalize_version_id(civitai_data.get('modelId'))
|
||||||
if model_id is None:
|
if model_id is None:
|
||||||
@@ -174,6 +220,20 @@ class ModelCache:
|
|||||||
if not versions:
|
if not versions:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
if sibling is not None:
|
||||||
|
# Update the descriptor to reflect the surviving sibling file
|
||||||
|
descriptor = self._build_version_descriptor(
|
||||||
|
sibling,
|
||||||
|
sibling.get('civitai') if isinstance(sibling, dict) else {},
|
||||||
|
version_id,
|
||||||
|
)
|
||||||
|
for index, existing_desc in enumerate(versions):
|
||||||
|
if existing_desc.get('versionId') == version_id:
|
||||||
|
if descriptor is not None:
|
||||||
|
versions[index] = descriptor
|
||||||
|
break
|
||||||
|
return
|
||||||
|
|
||||||
filtered = [v for v in versions if v.get('versionId') != version_id]
|
filtered = [v for v in versions if v.get('versionId') != version_id]
|
||||||
if filtered:
|
if filtered:
|
||||||
self.model_id_index[model_id] = filtered
|
self.model_id_index[model_id] = filtered
|
||||||
@@ -206,6 +266,15 @@ class ModelCache:
|
|||||||
versions = self.model_id_index.get(normalized_id, [])
|
versions = self.model_id_index.get(normalized_id, [])
|
||||||
return [dict(version) for version in versions]
|
return [dict(version) for version in versions]
|
||||||
|
|
||||||
|
def get_files_by_version_id(self, version_id: Any) -> List[Dict[str, Any]]:
|
||||||
|
"""Return every local file entry for a CivitAI model version (#1058)."""
|
||||||
|
|
||||||
|
normalized_id = self._normalize_version_id(version_id)
|
||||||
|
if normalized_id is None:
|
||||||
|
return []
|
||||||
|
|
||||||
|
return list(self.version_files_index.get(normalized_id, []))
|
||||||
|
|
||||||
async def resort(self):
|
async def resort(self):
|
||||||
"""Resort cached data according to last sort mode if set"""
|
"""Resort cached data according to last sort mode if set"""
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
from typing import Dict, Optional, Set, List
|
from typing import Dict, Optional, Set, List
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
from ..utils.constants import is_empty_placeholder_hash
|
||||||
|
|
||||||
class ModelHashIndex:
|
class ModelHashIndex:
|
||||||
"""Index for looking up models by hash or filename"""
|
"""Index for looking up models by hash or filename"""
|
||||||
|
|
||||||
@@ -81,6 +83,8 @@ class ModelHashIndex:
|
|||||||
# mapping. First-time registrations stay O(1).
|
# mapping. First-time registrations stay O(1).
|
||||||
if autov3:
|
if autov3:
|
||||||
autov3 = autov3.lower()
|
autov3 = autov3.lower()
|
||||||
|
if is_empty_placeholder_hash(autov3):
|
||||||
|
autov3 = None
|
||||||
if is_re_registration and (existing_hash != sha256 or autov3):
|
if is_re_registration and (existing_hash != sha256 or autov3):
|
||||||
stale_autov3_keys = [
|
stale_autov3_keys = [
|
||||||
key for key, mapped_path in self._autov3_to_path.items()
|
key for key, mapped_path in self._autov3_to_path.items()
|
||||||
@@ -93,7 +97,7 @@ class ModelHashIndex:
|
|||||||
|
|
||||||
def add_autov3(self, autov3: str, file_path: str) -> None:
|
def add_autov3(self, autov3: str, file_path: str) -> None:
|
||||||
"""Add or update an AutoV3-only index entry (used when only AutoV3 is known)"""
|
"""Add or update an AutoV3-only index entry (used when only AutoV3 is known)"""
|
||||||
if not autov3:
|
if not autov3 or is_empty_placeholder_hash(autov3):
|
||||||
return
|
return
|
||||||
autov3 = autov3.lower()
|
autov3 = autov3.lower()
|
||||||
self._autov3_to_path[autov3] = file_path
|
self._autov3_to_path[autov3] = file_path
|
||||||
@@ -250,6 +254,8 @@ class ModelHashIndex:
|
|||||||
|
|
||||||
def has_hash(self, hash_value: str) -> bool:
|
def has_hash(self, hash_value: str) -> bool:
|
||||||
"""Check if hash exists in index (SHA256, AutoV2, or AutoV3)"""
|
"""Check if hash exists in index (SHA256, AutoV2, or AutoV3)"""
|
||||||
|
if is_empty_placeholder_hash(hash_value):
|
||||||
|
return False
|
||||||
normalized = hash_value.lower()
|
normalized = hash_value.lower()
|
||||||
if normalized in self._hash_to_path:
|
if normalized in self._hash_to_path:
|
||||||
return True
|
return True
|
||||||
@@ -261,6 +267,8 @@ class ModelHashIndex:
|
|||||||
|
|
||||||
def get_path(self, hash_value: str) -> Optional[str]:
|
def get_path(self, hash_value: str) -> Optional[str]:
|
||||||
"""Get file path for a hash (SHA256, AutoV2, or AutoV3)"""
|
"""Get file path for a hash (SHA256, AutoV2, or AutoV3)"""
|
||||||
|
if is_empty_placeholder_hash(hash_value):
|
||||||
|
return None
|
||||||
normalized = hash_value.lower()
|
normalized = hash_value.lower()
|
||||||
path = self._hash_to_path.get(normalized)
|
path = self._hash_to_path.get(normalized)
|
||||||
if path is not None:
|
if path is not None:
|
||||||
|
|||||||
@@ -66,6 +66,14 @@ class _RateLimitRetryHelper:
|
|||||||
except RateLimitError as exc:
|
except RateLimitError as exc:
|
||||||
attempt += 1
|
attempt += 1
|
||||||
|
|
||||||
|
# The downloader's rate-limit gate already applied the wait
|
||||||
|
# policy for this request (waited out the vendor window or
|
||||||
|
# deliberately refused because it exceeds the cap). Sleeping
|
||||||
|
# again here would double the wait — just propagate.
|
||||||
|
if getattr(exc, "gate_handled", False):
|
||||||
|
exc.provider = exc.provider or label
|
||||||
|
raise
|
||||||
|
|
||||||
# Determine effective retry limit based on rate-limit magnitude
|
# Determine effective retry limit based on rate-limit magnitude
|
||||||
effective_retry_limit = self._retry_limit # default: 3
|
effective_retry_limit = self._retry_limit # default: 3
|
||||||
if exc.retry_after is not None and exc.retry_after >= 120.0:
|
if exc.retry_after is not None and exc.retry_after >= 120.0:
|
||||||
@@ -101,6 +109,12 @@ class _RateLimitRetryHelper:
|
|||||||
|
|
||||||
return min(self._max_delay, max(0.0, base_delay))
|
return min(self._max_delay, max(0.0, base_delay))
|
||||||
|
|
||||||
|
|
||||||
|
# Labels of providers that are free to consult even while a network provider
|
||||||
|
# is rate-limited (local lookups, no vendor cost).
|
||||||
|
_LOCAL_PROVIDER_LABELS = frozenset({"sqlite"})
|
||||||
|
|
||||||
|
|
||||||
class ModelMetadataProvider(ABC):
|
class ModelMetadataProvider(ABC):
|
||||||
"""Base abstract class for all model metadata providers"""
|
"""Base abstract class for all model metadata providers"""
|
||||||
|
|
||||||
@@ -155,6 +169,17 @@ class ModelMetadataProvider(ABC):
|
|||||||
"""Published model count for the user; None when unsupported."""
|
"""Published model count for the user; None when unsupported."""
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
async def get_version_file_mini(
|
||||||
|
self, version_id: int, file_id: int
|
||||||
|
) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Fetch raw stored file info via CivitAI's model-versions/mini endpoint.
|
||||||
|
|
||||||
|
Only the CivitAI provider implements this (#1100); other providers
|
||||||
|
already serve raw file names (CivArchive) or cannot resolve this
|
||||||
|
lookup (SQLite), so the default is None.
|
||||||
|
"""
|
||||||
|
return None
|
||||||
|
|
||||||
class CivitaiModelMetadataProvider(ModelMetadataProvider):
|
class CivitaiModelMetadataProvider(ModelMetadataProvider):
|
||||||
"""Provider that uses Civitai API for metadata"""
|
"""Provider that uses Civitai API for metadata"""
|
||||||
|
|
||||||
@@ -189,6 +214,11 @@ class CivitaiModelMetadataProvider(ModelMetadataProvider):
|
|||||||
async def get_creator_model_count(self, username: str) -> Optional[int]:
|
async def get_creator_model_count(self, username: str) -> Optional[int]:
|
||||||
return await self.client.get_creator_model_count(username)
|
return await self.client.get_creator_model_count(username)
|
||||||
|
|
||||||
|
async def get_version_file_mini(
|
||||||
|
self, version_id: int, file_id: int
|
||||||
|
) -> Optional[Dict[str, Any]]:
|
||||||
|
return await self.client.get_version_file_mini(version_id, file_id)
|
||||||
|
|
||||||
class CivArchiveModelMetadataProvider(ModelMetadataProvider):
|
class CivArchiveModelMetadataProvider(ModelMetadataProvider):
|
||||||
"""Provider that uses CivArchive API for metadata"""
|
"""Provider that uses CivArchive API for metadata"""
|
||||||
|
|
||||||
@@ -451,7 +481,14 @@ class SQLiteModelMetadataProvider(ModelMetadataProvider):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
class FallbackMetadataProvider(ModelMetadataProvider):
|
class FallbackMetadataProvider(ModelMetadataProvider):
|
||||||
"""Try providers in order, return first successful result."""
|
"""Try providers in order, return first successful result.
|
||||||
|
|
||||||
|
Rate-limit policy (#1085): once a *network* provider raises
|
||||||
|
``RateLimitError``, the chain stops consulting further network providers —
|
||||||
|
failing over would just spread the flood to the next vendor. Local-only
|
||||||
|
providers (see ``_LOCAL_PROVIDER_LABELS``) are still allowed as a last
|
||||||
|
resort because they cost the vendor nothing.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -486,7 +523,10 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
)
|
)
|
||||||
|
|
||||||
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||||
|
rate_limited = False
|
||||||
for provider, label in self._iter_providers():
|
for provider, label in self._iter_providers():
|
||||||
|
if rate_limited and label not in _LOCAL_PROVIDER_LABELS:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
result, error = await self._call_with_rate_limit(
|
result, error = await self._call_with_rate_limit(
|
||||||
label,
|
label,
|
||||||
@@ -496,8 +536,9 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
if result:
|
if result:
|
||||||
return result, error
|
return result, error
|
||||||
except RateLimitError as exc:
|
except RateLimitError as exc:
|
||||||
|
rate_limited = True
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
|
||||||
label,
|
label,
|
||||||
exc.retry_after or 0,
|
exc.retry_after or 0,
|
||||||
)
|
)
|
||||||
@@ -505,11 +546,18 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug("Provider %s failed for get_model_by_hash: %s", label, e)
|
logger.debug("Provider %s failed for get_model_by_hash: %s", label, e)
|
||||||
continue
|
continue
|
||||||
|
if rate_limited:
|
||||||
|
# Distinct from "Model not found": callers must not mistake a
|
||||||
|
# rate-limited lookup for a confirmed deletion.
|
||||||
|
return None, "Rate limited"
|
||||||
return None, "Model not found"
|
return None, "Model not found"
|
||||||
|
|
||||||
async def get_model_versions(self, model_id: str) -> Optional[Dict[str, Any]]:
|
async def get_model_versions(self, model_id: str) -> Optional[Dict[str, Any]]:
|
||||||
not_found_confirmed = False
|
not_found_confirmed = False
|
||||||
|
rate_limited = False
|
||||||
for provider, label in self._iter_providers():
|
for provider, label in self._iter_providers():
|
||||||
|
if rate_limited and label not in _LOCAL_PROVIDER_LABELS:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
result = await self._call_with_rate_limit(
|
result = await self._call_with_rate_limit(
|
||||||
label,
|
label,
|
||||||
@@ -519,8 +567,9 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
if result:
|
if result:
|
||||||
return result
|
return result
|
||||||
except RateLimitError as exc:
|
except RateLimitError as exc:
|
||||||
|
rate_limited = True
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
|
||||||
label,
|
label,
|
||||||
exc.retry_after or 0,
|
exc.retry_after or 0,
|
||||||
)
|
)
|
||||||
@@ -539,7 +588,10 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
async def get_model_version(self, model_id: Optional[int] = None, version_id: Optional[int] = None) -> Optional[Dict[str, Any]]:
|
async def get_model_version(self, model_id: Optional[int] = None, version_id: Optional[int] = None) -> Optional[Dict[str, Any]]:
|
||||||
|
rate_limited = False
|
||||||
for provider, label in self._iter_providers():
|
for provider, label in self._iter_providers():
|
||||||
|
if rate_limited and label not in _LOCAL_PROVIDER_LABELS:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
result = await self._call_with_rate_limit(
|
result = await self._call_with_rate_limit(
|
||||||
label,
|
label,
|
||||||
@@ -550,8 +602,9 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
if result:
|
if result:
|
||||||
return result
|
return result
|
||||||
except RateLimitError as exc:
|
except RateLimitError as exc:
|
||||||
|
rate_limited = True
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
|
||||||
label,
|
label,
|
||||||
exc.retry_after or 0,
|
exc.retry_after or 0,
|
||||||
)
|
)
|
||||||
@@ -562,7 +615,10 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||||
|
rate_limited = False
|
||||||
for provider, label in self._iter_providers():
|
for provider, label in self._iter_providers():
|
||||||
|
if rate_limited and label not in _LOCAL_PROVIDER_LABELS:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
result, error = await self._call_with_rate_limit(
|
result, error = await self._call_with_rate_limit(
|
||||||
label,
|
label,
|
||||||
@@ -572,8 +628,9 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
if result:
|
if result:
|
||||||
return result, error
|
return result, error
|
||||||
except RateLimitError as exc:
|
except RateLimitError as exc:
|
||||||
|
rate_limited = True
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
|
||||||
label,
|
label,
|
||||||
exc.retry_after or 0,
|
exc.retry_after or 0,
|
||||||
)
|
)
|
||||||
@@ -581,12 +638,17 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug("Provider %s failed for get_model_version_info: %s", label, e)
|
logger.debug("Provider %s failed for get_model_version_info: %s", label, e)
|
||||||
continue
|
continue
|
||||||
|
if rate_limited:
|
||||||
|
return None, "Rate limited"
|
||||||
return None, "No provider could retrieve the data"
|
return None, "No provider could retrieve the data"
|
||||||
|
|
||||||
async def get_model_versions_by_hashes(
|
async def get_model_versions_by_hashes(
|
||||||
self, hashes: List[str]
|
self, hashes: List[str]
|
||||||
) -> Optional[List[Dict[str, Any]]]:
|
) -> Optional[List[Dict[str, Any]]]:
|
||||||
|
rate_limited = False
|
||||||
for provider, label in self._iter_providers():
|
for provider, label in self._iter_providers():
|
||||||
|
if rate_limited and label not in _LOCAL_PROVIDER_LABELS:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
result = await self._call_with_rate_limit(
|
result = await self._call_with_rate_limit(
|
||||||
label,
|
label,
|
||||||
@@ -598,8 +660,9 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
except NotImplementedError:
|
except NotImplementedError:
|
||||||
continue
|
continue
|
||||||
except RateLimitError as exc:
|
except RateLimitError as exc:
|
||||||
|
rate_limited = True
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
|
||||||
label,
|
label,
|
||||||
exc.retry_after or 0,
|
exc.retry_after or 0,
|
||||||
)
|
)
|
||||||
@@ -614,7 +677,10 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
||||||
|
rate_limited = False
|
||||||
for provider, label in self._iter_providers():
|
for provider, label in self._iter_providers():
|
||||||
|
if rate_limited and label not in _LOCAL_PROVIDER_LABELS:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
result = await self._call_with_rate_limit(
|
result = await self._call_with_rate_limit(
|
||||||
label,
|
label,
|
||||||
@@ -625,8 +691,9 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
if result is not None:
|
if result is not None:
|
||||||
return result
|
return result
|
||||||
except RateLimitError as exc:
|
except RateLimitError as exc:
|
||||||
|
rate_limited = True
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Provider %s is rate-limited (retry_after=%.0fs); skipping to next provider",
|
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
|
||||||
label,
|
label,
|
||||||
exc.retry_after or 0,
|
exc.retry_after or 0,
|
||||||
)
|
)
|
||||||
@@ -649,6 +716,37 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
|||||||
continue
|
continue
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
async def get_version_file_mini(
|
||||||
|
self, version_id: int, file_id: int
|
||||||
|
) -> Optional[Dict[str, Any]]:
|
||||||
|
rate_limited = False
|
||||||
|
for provider, label in self._iter_providers():
|
||||||
|
if rate_limited and label not in _LOCAL_PROVIDER_LABELS:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
result = await self._call_with_rate_limit(
|
||||||
|
label,
|
||||||
|
provider.get_version_file_mini,
|
||||||
|
version_id,
|
||||||
|
file_id,
|
||||||
|
)
|
||||||
|
if result:
|
||||||
|
return result
|
||||||
|
except RateLimitError as exc:
|
||||||
|
rate_limited = True
|
||||||
|
logger.warning(
|
||||||
|
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
|
||||||
|
label,
|
||||||
|
exc.retry_after or 0,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug(
|
||||||
|
"Provider %s failed for get_version_file_mini: %s", label, e
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
return None
|
||||||
|
|
||||||
def _iter_providers(self):
|
def _iter_providers(self):
|
||||||
return zip(self.providers, self._provider_labels)
|
return zip(self.providers, self._provider_labels)
|
||||||
|
|
||||||
@@ -740,6 +838,16 @@ class RateLimitRetryingProvider(ModelMetadataProvider):
|
|||||||
async def get_creator_model_count(self, username: str) -> Optional[int]:
|
async def get_creator_model_count(self, username: str) -> Optional[int]:
|
||||||
return await self._provider.get_creator_model_count(username)
|
return await self._provider.get_creator_model_count(username)
|
||||||
|
|
||||||
|
async def get_version_file_mini(
|
||||||
|
self, version_id: int, file_id: int
|
||||||
|
) -> Optional[Dict[str, Any]]:
|
||||||
|
return await self._rate_limit_helper.run(
|
||||||
|
self._label,
|
||||||
|
self._provider.get_version_file_mini,
|
||||||
|
version_id,
|
||||||
|
file_id,
|
||||||
|
)
|
||||||
|
|
||||||
class ModelMetadataProviderManager:
|
class ModelMetadataProviderManager:
|
||||||
"""Manager for selecting and using model metadata providers"""
|
"""Manager for selecting and using model metadata providers"""
|
||||||
|
|
||||||
|
|||||||
@@ -432,6 +432,7 @@ class SearchStrategy:
|
|||||||
"tags": False,
|
"tags": False,
|
||||||
"recursive": True,
|
"recursive": True,
|
||||||
"creator": False,
|
"creator": False,
|
||||||
|
"hash": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -494,8 +495,28 @@ class SearchStrategy:
|
|||||||
results.append(item)
|
results.append(item)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# Hash search is always exact (never fuzzy): match the full
|
||||||
|
# sha256, its autov2 prefix (first 10 chars), or the autov3 hash.
|
||||||
|
if options.get("hash", False):
|
||||||
|
hash_query = search_lower.strip()
|
||||||
|
if hash_query and self._matches_hash(item, hash_query):
|
||||||
|
results.append(item)
|
||||||
|
continue
|
||||||
|
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
def _matches_hash(self, item: Dict[str, Any], hash_query: str) -> bool:
|
||||||
|
"""Exact-match the normalized query against the item's known hashes."""
|
||||||
|
sha256 = item.get("sha256")
|
||||||
|
sha256_lower = sha256.lower() if isinstance(sha256, str) else ""
|
||||||
|
if sha256_lower and hash_query in (sha256_lower, sha256_lower[:10]):
|
||||||
|
return True
|
||||||
|
# autov3 is None when unchecked and "" when checked but unavailable
|
||||||
|
autov3 = item.get("autov3")
|
||||||
|
if isinstance(autov3, str) and autov3 and hash_query == autov3.lower():
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
def _matches(
|
def _matches(
|
||||||
self, candidate: str, search_term: str, search_lower: str, fuzzy: bool
|
self, candidate: str, search_term: str, search_lower: str, fuzzy: bool
|
||||||
) -> bool:
|
) -> bool:
|
||||||
|
|||||||
+649
-117
File diff suppressed because it is too large
Load Diff
@@ -118,19 +118,24 @@ class ModelServiceFactory:
|
|||||||
|
|
||||||
|
|
||||||
def register_default_model_types():
|
def register_default_model_types():
|
||||||
"""Register the default model types (LoRA, Checkpoint, and Embedding)"""
|
"""Register the default model types (LoRA, Checkpoint, Embedding, and Other)"""
|
||||||
from ..services.lora_service import LoraService
|
from ..services.lora_service import LoraService
|
||||||
from ..services.checkpoint_service import CheckpointService
|
from ..services.checkpoint_service import CheckpointService
|
||||||
from ..services.embedding_service import EmbeddingService
|
from ..services.embedding_service import EmbeddingService
|
||||||
|
from ..services.other_model_service import OtherModelService
|
||||||
from ..routes.lora_routes import LoraRoutes
|
from ..routes.lora_routes import LoraRoutes
|
||||||
from ..routes.checkpoint_routes import CheckpointRoutes
|
from ..routes.checkpoint_routes import CheckpointRoutes
|
||||||
from ..routes.embedding_routes import EmbeddingRoutes
|
from ..routes.embedding_routes import EmbeddingRoutes
|
||||||
|
from ..routes.other_routes import OtherRoutes
|
||||||
|
|
||||||
# Register LoRA model type
|
# Register LoRA model type
|
||||||
ModelServiceFactory.register_model_type('lora', LoraService, LoraRoutes)
|
ModelServiceFactory.register_model_type('lora', LoraService, LoraRoutes)
|
||||||
|
|
||||||
# Register Checkpoint model type
|
# Register Checkpoint model type
|
||||||
ModelServiceFactory.register_model_type('checkpoint', CheckpointService, CheckpointRoutes)
|
ModelServiceFactory.register_model_type('checkpoint', CheckpointService, CheckpointRoutes)
|
||||||
|
|
||||||
# Register Embedding model type
|
# Register Embedding model type
|
||||||
ModelServiceFactory.register_model_type('embedding', EmbeddingService, EmbeddingRoutes)
|
ModelServiceFactory.register_model_type('embedding', EmbeddingService, EmbeddingRoutes)
|
||||||
|
|
||||||
|
# Register Other model type (VAE, upscaler, text encoder, ...)
|
||||||
|
ModelServiceFactory.register_model_type('other', OtherModelService, OtherRoutes)
|
||||||
@@ -13,11 +13,12 @@ import sqlite3
|
|||||||
import time
|
import time
|
||||||
from dataclasses import dataclass, replace
|
from dataclasses import dataclass, replace
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence
|
from typing import Any, Dict, Iterable, Iterator, List, Mapping, Optional, Sequence
|
||||||
|
|
||||||
from .errors import RateLimitError, ResourceNotFoundError
|
from .errors import RateLimitError, ResourceNotFoundError
|
||||||
from .settings_manager import get_settings_manager
|
from .settings_manager import get_settings_manager
|
||||||
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
||||||
|
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES
|
||||||
from ..utils.civitai_utils import rewrite_preview_url
|
from ..utils.civitai_utils import rewrite_preview_url
|
||||||
from ..utils.preview_selection import resolve_mature_threshold, select_preview_media
|
from ..utils.preview_selection import resolve_mature_threshold, select_preview_media
|
||||||
|
|
||||||
@@ -77,6 +78,10 @@ class ModelVersionRecord:
|
|||||||
usage_control: Optional[str] = None # "Download", "Generation", "InternalGeneration"
|
usage_control: Optional[str] = None # "Download", "Generation", "InternalGeneration"
|
||||||
paid_access: Optional[str] = None # JSON string of the CivitAI paidAccess DTO
|
paid_access: Optional[str] = None # JSON string of the CivitAI paidAccess DTO
|
||||||
is_paid: bool = False # True when paidAccess.permanent is True (permanent paid gate)
|
is_paid: bool = False # True when paidAccess.permanent is True (permanent paid gate)
|
||||||
|
# Number of downloadable weight files for the version (None when unknown,
|
||||||
|
# e.g. records persisted before this field existed or locally-synthesized
|
||||||
|
# entries). Mirrors the frontend isModelWeightFile() filter.
|
||||||
|
file_count: Optional[int] = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -245,6 +250,51 @@ class ModelUpdateRecord:
|
|||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
def has_update_for_local_bases(
|
||||||
|
self,
|
||||||
|
hide_early_access: bool = False,
|
||||||
|
hide_non_downloadable: bool = True,
|
||||||
|
hide_paid: bool = False,
|
||||||
|
) -> bool:
|
||||||
|
"""Return True when any locally-held base model scope has an update.
|
||||||
|
|
||||||
|
Aggregates :meth:`has_update_for_base` across every distinct base model
|
||||||
|
present among in-library versions. This mirrors the per-item evaluation
|
||||||
|
performed by ``BaseModelService._annotate_update_flags`` when the
|
||||||
|
``version_grouping`` setting is ``same_base``, so callers reporting
|
||||||
|
"how many models have updates" stay aligned with what the Updates
|
||||||
|
filter displays. Use this instead of :meth:`has_update` for such
|
||||||
|
summaries; see issue #1083.
|
||||||
|
|
||||||
|
When no local base model is known (nothing held locally, or versions
|
||||||
|
never seen in any remote listing), falls back to :meth:`has_update` so
|
||||||
|
a model the item-level filter may still flag is not silently dropped
|
||||||
|
from summaries.
|
||||||
|
"""
|
||||||
|
|
||||||
|
bases = {
|
||||||
|
_normalize_base_model(version.base_model)
|
||||||
|
for version in self.versions
|
||||||
|
if version.is_in_library
|
||||||
|
}
|
||||||
|
bases.discard(None)
|
||||||
|
if not bases:
|
||||||
|
return self.has_update(
|
||||||
|
hide_early_access=hide_early_access,
|
||||||
|
hide_non_downloadable=hide_non_downloadable,
|
||||||
|
hide_paid=hide_paid,
|
||||||
|
)
|
||||||
|
return any(
|
||||||
|
self.has_update_for_base(
|
||||||
|
None,
|
||||||
|
base,
|
||||||
|
hide_early_access=hide_early_access,
|
||||||
|
hide_non_downloadable=hide_non_downloadable,
|
||||||
|
hide_paid=hide_paid,
|
||||||
|
)
|
||||||
|
for base in bases
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class ModelUpdateService:
|
class ModelUpdateService:
|
||||||
"""Persist and query remote model version metadata."""
|
"""Persist and query remote model version metadata."""
|
||||||
@@ -273,6 +323,7 @@ class ModelUpdateService:
|
|||||||
usage_control TEXT,
|
usage_control TEXT,
|
||||||
paid_access TEXT,
|
paid_access TEXT,
|
||||||
is_paid INTEGER NOT NULL DEFAULT 0,
|
is_paid INTEGER NOT NULL DEFAULT 0,
|
||||||
|
file_count INTEGER,
|
||||||
PRIMARY KEY (model_id, version_id),
|
PRIMARY KEY (model_id, version_id),
|
||||||
FOREIGN KEY(model_id) REFERENCES model_update_status(model_id) ON DELETE CASCADE
|
FOREIGN KEY(model_id) REFERENCES model_update_status(model_id) ON DELETE CASCADE
|
||||||
);
|
);
|
||||||
@@ -520,6 +571,10 @@ class ModelUpdateService:
|
|||||||
"ALTER TABLE model_update_versions "
|
"ALTER TABLE model_update_versions "
|
||||||
"ADD COLUMN is_paid INTEGER NOT NULL DEFAULT 0"
|
"ADD COLUMN is_paid INTEGER NOT NULL DEFAULT 0"
|
||||||
),
|
),
|
||||||
|
"file_count": (
|
||||||
|
"ALTER TABLE model_update_versions "
|
||||||
|
"ADD COLUMN file_count INTEGER"
|
||||||
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
for column, statement in migrations.items():
|
for column, statement in migrations.items():
|
||||||
@@ -623,6 +678,7 @@ class ModelUpdateService:
|
|||||||
is_early_access INTEGER NOT NULL DEFAULT 0,
|
is_early_access INTEGER NOT NULL DEFAULT 0,
|
||||||
paid_access TEXT,
|
paid_access TEXT,
|
||||||
is_paid INTEGER NOT NULL DEFAULT 0,
|
is_paid INTEGER NOT NULL DEFAULT 0,
|
||||||
|
file_count INTEGER,
|
||||||
PRIMARY KEY (model_id, version_id),
|
PRIMARY KEY (model_id, version_id),
|
||||||
FOREIGN KEY(model_id) REFERENCES model_update_status(model_id) ON DELETE CASCADE
|
FOREIGN KEY(model_id) REFERENCES model_update_status(model_id) ON DELETE CASCADE
|
||||||
)
|
)
|
||||||
@@ -644,6 +700,7 @@ class ModelUpdateService:
|
|||||||
"is_early_access",
|
"is_early_access",
|
||||||
"paid_access",
|
"paid_access",
|
||||||
"is_paid",
|
"is_paid",
|
||||||
|
"file_count",
|
||||||
]
|
]
|
||||||
defaults = {
|
defaults = {
|
||||||
"sort_index": "0",
|
"sort_index": "0",
|
||||||
@@ -658,6 +715,7 @@ class ModelUpdateService:
|
|||||||
"is_early_access": "0",
|
"is_early_access": "0",
|
||||||
"paid_access": "NULL",
|
"paid_access": "NULL",
|
||||||
"is_paid": "0",
|
"is_paid": "0",
|
||||||
|
"file_count": "NULL",
|
||||||
}
|
}
|
||||||
|
|
||||||
select_parts = []
|
select_parts = []
|
||||||
@@ -773,6 +831,11 @@ class ModelUpdateService:
|
|||||||
target_model_ids=target_filter,
|
target_model_ids=target_filter,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
local_base_models = await self._collect_local_version_bases(
|
||||||
|
scanner,
|
||||||
|
target_model_ids=target_filter,
|
||||||
|
)
|
||||||
|
|
||||||
results: Dict[int, ModelUpdateRecord] = {}
|
results: Dict[int, ModelUpdateRecord] = {}
|
||||||
prefetched: Dict[int, Mapping[Any, Any]] = {}
|
prefetched: Dict[int, Mapping[Any, Any]] = {}
|
||||||
|
|
||||||
@@ -825,6 +888,7 @@ class ModelUpdateService:
|
|||||||
force_refresh=force_refresh,
|
force_refresh=force_refresh,
|
||||||
prefetched_response=prefetched.get(model_id),
|
prefetched_response=prefetched.get(model_id),
|
||||||
all_local_version_ids=all_vids,
|
all_local_version_ids=all_vids,
|
||||||
|
local_base_models=local_base_models,
|
||||||
)
|
)
|
||||||
if scanner.is_cancelled():
|
if scanner.is_cancelled():
|
||||||
logger.info(f"{model_type.capitalize()} Update Service: Refresh cancelled by user")
|
logger.info(f"{model_type.capitalize()} Update Service: Refresh cancelled by user")
|
||||||
@@ -859,12 +923,14 @@ class ModelUpdateService:
|
|||||||
|
|
||||||
local_versions = await self._collect_local_versions(scanner)
|
local_versions = await self._collect_local_versions(scanner)
|
||||||
version_ids = local_versions.get(model_id, [])
|
version_ids = local_versions.get(model_id, [])
|
||||||
|
local_base_models = await self._collect_local_version_bases(scanner)
|
||||||
return await self._refresh_single_model(
|
return await self._refresh_single_model(
|
||||||
model_type,
|
model_type,
|
||||||
model_id,
|
model_id,
|
||||||
version_ids,
|
version_ids,
|
||||||
metadata_provider,
|
metadata_provider,
|
||||||
force_refresh=force_refresh,
|
force_refresh=force_refresh,
|
||||||
|
local_base_models=local_base_models,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def update_in_library_versions(
|
async def update_in_library_versions(
|
||||||
@@ -1040,6 +1106,7 @@ class ModelUpdateService:
|
|||||||
force_refresh: bool = False,
|
force_refresh: bool = False,
|
||||||
prefetched_response: Optional[Mapping[str, Any]] = None,
|
prefetched_response: Optional[Mapping[str, Any]] = None,
|
||||||
all_local_version_ids: Optional[Sequence[int]] = None,
|
all_local_version_ids: Optional[Sequence[int]] = None,
|
||||||
|
local_base_models: Optional[Mapping[int, str]] = None,
|
||||||
) -> Optional[ModelUpdateRecord]:
|
) -> Optional[ModelUpdateRecord]:
|
||||||
normalized_local = self._normalize_sequence(local_versions)
|
normalized_local = self._normalize_sequence(local_versions)
|
||||||
# When folder-filtering, this carries the cross-folder version set
|
# When folder-filtering, this carries the cross-folder version set
|
||||||
@@ -1164,6 +1231,7 @@ class ModelUpdateService:
|
|||||||
existing,
|
existing,
|
||||||
now,
|
now,
|
||||||
all_local_version_ids=normalized_all,
|
all_local_version_ids=normalized_all,
|
||||||
|
local_base_models=local_base_models,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
record = self._merge_with_local_versions(
|
record = self._merge_with_local_versions(
|
||||||
@@ -1370,27 +1438,17 @@ class ModelUpdateService:
|
|||||||
await self._enrich_version_entries(metadata_provider, aggregated)
|
await self._enrich_version_entries(metadata_provider, aggregated)
|
||||||
return aggregated
|
return aggregated
|
||||||
|
|
||||||
async def _collect_local_versions(
|
@staticmethod
|
||||||
self,
|
def _iter_local_civitai_items(
|
||||||
scanner,
|
cache,
|
||||||
*,
|
*,
|
||||||
target_model_ids: Optional[Sequence[int]] = None,
|
target_set: Optional[set[int]] = None,
|
||||||
folder_path: Optional[str] = None,
|
normalized_folder: Optional[str] = None,
|
||||||
) -> Dict[int, List[int]]:
|
) -> Iterator[tuple[int, int, Any]]:
|
||||||
cache = await scanner.get_cached_data()
|
"""Yield ``(modelId, versionId, base_model)`` for each scannable item."""
|
||||||
mapping: Dict[int, set[int]] = {}
|
|
||||||
if not cache or not getattr(cache, "raw_data", None):
|
if not cache or not getattr(cache, "raw_data", None):
|
||||||
return {}
|
return
|
||||||
|
|
||||||
target_set = None
|
|
||||||
if target_model_ids:
|
|
||||||
target_set = set(target_model_ids)
|
|
||||||
if not target_set:
|
|
||||||
return {}
|
|
||||||
|
|
||||||
normalized_folder = None
|
|
||||||
if folder_path is not None:
|
|
||||||
normalized_folder = folder_path.replace("\\", "/").strip("/")
|
|
||||||
|
|
||||||
for item in cache.raw_data:
|
for item in cache.raw_data:
|
||||||
# Apply folder filter first (cheapest check)
|
# Apply folder filter first (cheapest check)
|
||||||
@@ -1410,10 +1468,75 @@ class ModelUpdateService:
|
|||||||
continue
|
continue
|
||||||
if target_set is not None and model_id not in target_set:
|
if target_set is not None and model_id not in target_set:
|
||||||
continue
|
continue
|
||||||
|
yield model_id, version_id, item.get("base_model")
|
||||||
|
|
||||||
|
def _prepare_collection_filters(
|
||||||
|
self,
|
||||||
|
target_model_ids: Optional[Sequence[int]],
|
||||||
|
folder_path: Optional[str],
|
||||||
|
) -> tuple[Optional[set[int]], Optional[str]]:
|
||||||
|
target_set: Optional[set[int]] = None
|
||||||
|
if target_model_ids:
|
||||||
|
target_set = set(target_model_ids)
|
||||||
|
|
||||||
|
normalized_folder = None
|
||||||
|
if folder_path is not None:
|
||||||
|
normalized_folder = folder_path.replace("\\", "/").strip("/")
|
||||||
|
return target_set, normalized_folder
|
||||||
|
|
||||||
|
async def _collect_local_versions(
|
||||||
|
self,
|
||||||
|
scanner,
|
||||||
|
*,
|
||||||
|
target_model_ids: Optional[Sequence[int]] = None,
|
||||||
|
folder_path: Optional[str] = None,
|
||||||
|
) -> Dict[int, List[int]]:
|
||||||
|
cache = await scanner.get_cached_data()
|
||||||
|
mapping: Dict[int, set[int]] = {}
|
||||||
|
target_set, normalized_folder = self._prepare_collection_filters(
|
||||||
|
target_model_ids, folder_path
|
||||||
|
)
|
||||||
|
|
||||||
|
if target_model_ids and not target_set:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
for model_id, version_id, _base_model in self._iter_local_civitai_items(
|
||||||
|
cache, target_set=target_set, normalized_folder=normalized_folder
|
||||||
|
):
|
||||||
mapping.setdefault(model_id, set()).add(version_id)
|
mapping.setdefault(model_id, set()).add(version_id)
|
||||||
|
|
||||||
return {model_id: sorted(ids) for model_id, ids in mapping.items()}
|
return {model_id: sorted(ids) for model_id, ids in mapping.items()}
|
||||||
|
|
||||||
|
async def _collect_local_version_bases(
|
||||||
|
self,
|
||||||
|
scanner,
|
||||||
|
*,
|
||||||
|
target_model_ids: Optional[Sequence[int]] = None,
|
||||||
|
) -> Dict[int, str]:
|
||||||
|
"""Map version id -> base model from cache items.
|
||||||
|
|
||||||
|
Deliberately unfiltered by folder: synthesized in-library entries must
|
||||||
|
carry a base regardless of which folder triggered the refresh.
|
||||||
|
"""
|
||||||
|
|
||||||
|
cache = await scanner.get_cached_data()
|
||||||
|
bases: Dict[int, str] = {}
|
||||||
|
target_set, _normalized_folder = self._prepare_collection_filters(
|
||||||
|
target_model_ids, None
|
||||||
|
)
|
||||||
|
|
||||||
|
if target_model_ids and not target_set:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
for _model_id, version_id, base_model in self._iter_local_civitai_items(
|
||||||
|
cache, target_set=target_set
|
||||||
|
):
|
||||||
|
normalized_base = _normalize_string(base_model)
|
||||||
|
if normalized_base:
|
||||||
|
bases[version_id] = normalized_base
|
||||||
|
|
||||||
|
return bases
|
||||||
|
|
||||||
def _merge_with_local_versions(
|
def _merge_with_local_versions(
|
||||||
self,
|
self,
|
||||||
existing: Optional[ModelUpdateRecord],
|
existing: Optional[ModelUpdateRecord],
|
||||||
@@ -1493,6 +1616,7 @@ class ModelUpdateService:
|
|||||||
timestamp: float,
|
timestamp: float,
|
||||||
*,
|
*,
|
||||||
all_local_version_ids: Optional[Sequence[int]] = None,
|
all_local_version_ids: Optional[Sequence[int]] = None,
|
||||||
|
local_base_models: Optional[Mapping[int, str]] = None,
|
||||||
) -> ModelUpdateRecord:
|
) -> ModelUpdateRecord:
|
||||||
local_set = set(local_versions)
|
local_set = set(local_versions)
|
||||||
# When folder-filtering, also consider versions in other folders
|
# When folder-filtering, also consider versions in other folders
|
||||||
@@ -1504,6 +1628,7 @@ class ModelUpdateService:
|
|||||||
)
|
)
|
||||||
ignore_map = {version.version_id: version.should_ignore for version in existing.versions} if existing else {}
|
ignore_map = {version.version_id: version.should_ignore for version in existing.versions} if existing else {}
|
||||||
preview_map = {version.version_id: version.preview_url for version in existing.versions} if existing else {}
|
preview_map = {version.version_id: version.preview_url for version in existing.versions} if existing else {}
|
||||||
|
file_count_map = {version.version_id: version.file_count for version in existing.versions} if existing else {}
|
||||||
sort_map = {version.version_id: version.sort_index for version in existing.versions} if existing else {}
|
sort_map = {version.version_id: version.sort_index for version in existing.versions} if existing else {}
|
||||||
existing_map = {version.version_id: version for version in existing.versions} if existing else {}
|
existing_map = {version.version_id: version for version in existing.versions} if existing else {}
|
||||||
|
|
||||||
@@ -1528,11 +1653,17 @@ class ModelUpdateService:
|
|||||||
usage_control=remote_version.usage_control,
|
usage_control=remote_version.usage_control,
|
||||||
paid_access=remote_version.paid_access,
|
paid_access=remote_version.paid_access,
|
||||||
is_paid=remote_version.is_paid,
|
is_paid=remote_version.is_paid,
|
||||||
|
file_count=(
|
||||||
|
remote_version.file_count
|
||||||
|
if remote_version.file_count is not None
|
||||||
|
else file_count_map.get(version_id)
|
||||||
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
missing_local = local_set - seen_ids
|
missing_local = local_set - seen_ids
|
||||||
if missing_local:
|
if missing_local:
|
||||||
|
item_base_models = local_base_models or {}
|
||||||
for version_id in sorted(missing_local):
|
for version_id in sorted(missing_local):
|
||||||
existing_version = existing_map.get(version_id)
|
existing_version = existing_map.get(version_id)
|
||||||
if existing_version:
|
if existing_version:
|
||||||
@@ -1547,7 +1678,7 @@ class ModelUpdateService:
|
|||||||
ModelVersionRecord(
|
ModelVersionRecord(
|
||||||
version_id=version_id,
|
version_id=version_id,
|
||||||
name=None,
|
name=None,
|
||||||
base_model=None,
|
base_model=item_base_models.get(version_id),
|
||||||
released_at=None,
|
released_at=None,
|
||||||
size_bytes=None,
|
size_bytes=None,
|
||||||
preview_url=None,
|
preview_url=None,
|
||||||
@@ -1620,6 +1751,7 @@ class ModelUpdateService:
|
|||||||
base_model = _normalize_string(entry.get("baseModel"))
|
base_model = _normalize_string(entry.get("baseModel"))
|
||||||
released_at = _normalize_string(entry.get("publishedAt") or entry.get("createdAt"))
|
released_at = _normalize_string(entry.get("publishedAt") or entry.get("createdAt"))
|
||||||
size_bytes = self._extract_size_bytes(entry.get("files"))
|
size_bytes = self._extract_size_bytes(entry.get("files"))
|
||||||
|
file_count = self._extract_file_count(entry.get("files"))
|
||||||
preview_url = self._extract_preview_url(entry.get("images"))
|
preview_url = self._extract_preview_url(entry.get("images"))
|
||||||
early_access_ends_at = _normalize_string(entry.get("earlyAccessEndsAt"))
|
early_access_ends_at = _normalize_string(entry.get("earlyAccessEndsAt"))
|
||||||
|
|
||||||
@@ -1655,6 +1787,7 @@ class ModelUpdateService:
|
|||||||
usage_control=usage_control,
|
usage_control=usage_control,
|
||||||
paid_access=paid_access_json,
|
paid_access=paid_access_json,
|
||||||
is_paid=is_paid,
|
is_paid=is_paid,
|
||||||
|
file_count=file_count,
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -1683,6 +1816,25 @@ class ModelUpdateService:
|
|||||||
return None
|
return None
|
||||||
return {"permanent": permanent, "endsAt": ends_at}
|
return {"permanent": permanent, "endsAt": ends_at}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _extract_file_count(files) -> Optional[int]:
|
||||||
|
"""Count downloadable weight files in a version entry's ``files`` list.
|
||||||
|
|
||||||
|
Returns None when the payload carries no files array (unknown), so
|
||||||
|
callers can distinguish "no weight files" from "no data".
|
||||||
|
"""
|
||||||
|
|
||||||
|
if not isinstance(files, list):
|
||||||
|
return None
|
||||||
|
count = 0
|
||||||
|
for entry in files:
|
||||||
|
if not isinstance(entry, Mapping):
|
||||||
|
continue
|
||||||
|
entry_type = entry.get("type")
|
||||||
|
if isinstance(entry_type, str) and entry_type in MODEL_WEIGHT_FILE_TYPES:
|
||||||
|
count += 1
|
||||||
|
return count
|
||||||
|
|
||||||
def _extract_size_bytes(self, files) -> Optional[int]:
|
def _extract_size_bytes(self, files) -> Optional[int]:
|
||||||
if not isinstance(files, Iterable):
|
if not isinstance(files, Iterable):
|
||||||
return None
|
return None
|
||||||
@@ -1795,7 +1947,7 @@ class ModelUpdateService:
|
|||||||
f"""
|
f"""
|
||||||
SELECT model_id, version_id, sort_index, name, base_model, released_at,
|
SELECT model_id, version_id, sort_index, name, base_model, released_at,
|
||||||
size_bytes, preview_url, is_in_library, should_ignore, early_access_ends_at,
|
size_bytes, preview_url, is_in_library, should_ignore, early_access_ends_at,
|
||||||
is_early_access, usage_control, paid_access, is_paid
|
is_early_access, usage_control, paid_access, is_paid, file_count
|
||||||
FROM model_update_versions
|
FROM model_update_versions
|
||||||
WHERE model_id IN ({placeholders})
|
WHERE model_id IN ({placeholders})
|
||||||
ORDER BY model_id ASC, sort_index ASC, version_id ASC
|
ORDER BY model_id ASC, sort_index ASC, version_id ASC
|
||||||
@@ -1826,6 +1978,7 @@ class ModelUpdateService:
|
|||||||
usage_control=row["usage_control"],
|
usage_control=row["usage_control"],
|
||||||
paid_access=row["paid_access"],
|
paid_access=row["paid_access"],
|
||||||
is_paid=bool(row["is_paid"]),
|
is_paid=bool(row["is_paid"]),
|
||||||
|
file_count=_normalize_int(row["file_count"]),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1888,8 +2041,8 @@ class ModelUpdateService:
|
|||||||
INSERT INTO model_update_versions (
|
INSERT INTO model_update_versions (
|
||||||
version_id, model_id, sort_index, name, base_model, released_at,
|
version_id, model_id, sort_index, name, base_model, released_at,
|
||||||
size_bytes, preview_url, is_in_library, should_ignore, early_access_ends_at,
|
size_bytes, preview_url, is_in_library, should_ignore, early_access_ends_at,
|
||||||
is_early_access, usage_control, paid_access, is_paid
|
is_early_access, usage_control, paid_access, is_paid, file_count
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
""",
|
""",
|
||||||
(
|
(
|
||||||
version.version_id,
|
version.version_id,
|
||||||
@@ -1907,6 +2060,7 @@ class ModelUpdateService:
|
|||||||
version.usage_control,
|
version.usage_control,
|
||||||
paid_access_value,
|
paid_access_value,
|
||||||
1 if version.is_paid else 0,
|
1 if version.is_paid else 0,
|
||||||
|
version.file_count,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
import os
|
||||||
|
import logging
|
||||||
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
|
from .base_model_service import BaseModelService
|
||||||
|
from .auto_tag_service import extract_auto_tags
|
||||||
|
from ..utils.models import OtherModelMetadata
|
||||||
|
from ..config import config
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class OtherModelService(BaseModelService):
|
||||||
|
"""Other-model-specific service implementation (VAE, upscaler, text encoder, ...)"""
|
||||||
|
|
||||||
|
def __init__(self, scanner, update_service=None):
|
||||||
|
"""Initialize Other-model service
|
||||||
|
|
||||||
|
Args:
|
||||||
|
scanner: Other-model scanner instance
|
||||||
|
update_service: Optional service for remote update tracking.
|
||||||
|
"""
|
||||||
|
super().__init__("other", scanner, OtherModelMetadata, update_service=update_service)
|
||||||
|
|
||||||
|
async def format_response(self, model_data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Format other-model data for API response.
|
||||||
|
|
||||||
|
Returns None when the entry is missing critical fields (corrupted cache
|
||||||
|
row), so the handler layer can filter it out. See issue #730.
|
||||||
|
"""
|
||||||
|
# Guard against corrupted cache entries missing critical fields
|
||||||
|
file_path = model_data.get("file_path")
|
||||||
|
if not file_path or not isinstance(file_path, str):
|
||||||
|
logger.warning(
|
||||||
|
"Skipping corrupted other-model entry (missing file_path): %s",
|
||||||
|
model_data.get("file_name", "<unknown>"),
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Get sub_type from cache entry (new canonical field)
|
||||||
|
sub_type = model_data.get("sub_type", "vae")
|
||||||
|
|
||||||
|
file_name = model_data.get("file_name") or ""
|
||||||
|
model_name = model_data.get("model_name") or file_name
|
||||||
|
folder = model_data.get("folder") or ""
|
||||||
|
|
||||||
|
return {
|
||||||
|
"model_name": model_name,
|
||||||
|
"file_name": file_name,
|
||||||
|
"preview_url": config.get_preview_static_url(model_data.get("preview_url", "")),
|
||||||
|
"preview_nsfw_level": model_data.get("preview_nsfw_level", 0),
|
||||||
|
"base_model": model_data.get("base_model", ""),
|
||||||
|
"folder": folder,
|
||||||
|
"sha256": model_data.get("sha256", ""),
|
||||||
|
"autov3": model_data.get("autov3"),
|
||||||
|
"file_path": file_path.replace(os.sep, "/"),
|
||||||
|
"file_size": model_data.get("size", 0),
|
||||||
|
"modified": model_data.get("modified", ""),
|
||||||
|
"tags": model_data.get("tags", []),
|
||||||
|
"from_civitai": model_data.get("from_civitai", True),
|
||||||
|
"notes": model_data.get("notes", ""),
|
||||||
|
"sub_type": sub_type,
|
||||||
|
"favorite": model_data.get("favorite", False),
|
||||||
|
"exclude": bool(model_data.get("exclude", False)),
|
||||||
|
"update_available": bool(model_data.get("update_available", False)),
|
||||||
|
"skip_metadata_refresh": bool(model_data.get("skip_metadata_refresh", False)),
|
||||||
|
"civitai": self.filter_civitai_data(model_data.get("civitai", {}), minimal=True),
|
||||||
|
"auto_tags": model_data.get("auto_tags") or extract_auto_tags(model_data),
|
||||||
|
"version_count": model_data.get("version_count"),
|
||||||
|
"hf_url": model_data.get("hf_url", ""),
|
||||||
|
}
|
||||||
|
|
||||||
|
def find_duplicate_hashes(self) -> Dict[str, Any]:
|
||||||
|
"""Find other models with duplicate SHA256 hashes"""
|
||||||
|
return self.scanner._hash_index.get_duplicate_hashes()
|
||||||
|
|
||||||
|
def find_duplicate_filenames(self) -> Dict[str, Any]:
|
||||||
|
"""Find other models with conflicting filenames"""
|
||||||
|
return self.scanner._hash_index.get_duplicate_filenames()
|
||||||
@@ -0,0 +1,478 @@
|
|||||||
|
# pyright: reportImportCycles=false
|
||||||
|
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||||
|
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||||
|
# import cycles. Breaking them would require an architectural refactor.
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from ..utils.models import OtherModelMetadata
|
||||||
|
from ..utils.file_utils import find_preview_file, normalize_path, calculate_autov3
|
||||||
|
from ..utils.metadata_manager import MetadataManager
|
||||||
|
from ..config import config
|
||||||
|
from .model_scanner import ModelScanner, _is_excluded_dir
|
||||||
|
from .model_hash_index import ModelHashIndex
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class OtherScanner(ModelScanner):
|
||||||
|
"""Service for scanning and managing "other" model files.
|
||||||
|
|
||||||
|
Aggregates every enabled folder_paths category from
|
||||||
|
OTHER_MODEL_FOLDER_SUBTYPES (VAE, upscalers, text encoders, CLIP vision,
|
||||||
|
opt-in ControlNet) into one scanner; sub_type is derived from the root
|
||||||
|
containing the file (mirrors CheckpointScanner's checkpoints/unet split).
|
||||||
|
|
||||||
|
Hashing is lazy (checkpoint-style): text encoders can be ~10 GB, so the
|
||||||
|
initial scan records hash_status="pending" and the SHA256 is computed
|
||||||
|
on-demand via calculate_hash_for_model (e.g. when fetching CivitAI
|
||||||
|
metadata).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
# Same extension set as CheckpointScanner (ComfyUI's
|
||||||
|
# supported_pt_extensions plus ".gguf").
|
||||||
|
file_extensions = {
|
||||||
|
".ckpt",
|
||||||
|
".pt",
|
||||||
|
".pt2",
|
||||||
|
".bin",
|
||||||
|
".pth",
|
||||||
|
".safetensors",
|
||||||
|
".pkl",
|
||||||
|
".sft",
|
||||||
|
".gguf",
|
||||||
|
}
|
||||||
|
super().__init__(
|
||||||
|
model_type="other",
|
||||||
|
model_class=OtherModelMetadata,
|
||||||
|
file_extensions=file_extensions,
|
||||||
|
hash_index=ModelHashIndex(),
|
||||||
|
)
|
||||||
|
if not hasattr(self, "_hash_calculation_lock"):
|
||||||
|
self._hash_calculation_lock = asyncio.Lock()
|
||||||
|
self._hash_calculation_tasks: dict[str, asyncio.Task[Optional[str]]] = {}
|
||||||
|
|
||||||
|
async def _create_default_metadata(
|
||||||
|
self, file_path: str
|
||||||
|
) -> Optional[OtherModelMetadata]:
|
||||||
|
"""Create default metadata without calculating hash (lazy hash).
|
||||||
|
|
||||||
|
Other models include multi-GB text encoders, so hash calculation is
|
||||||
|
deferred until on-demand (e.g. CivitAI metadata fetch).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
real_path = os.path.realpath(file_path)
|
||||||
|
if not os.path.exists(real_path):
|
||||||
|
logger.error(f"File not found: {file_path}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
base_name = os.path.splitext(os.path.basename(file_path))[0]
|
||||||
|
dir_path = os.path.dirname(file_path)
|
||||||
|
|
||||||
|
# Find preview image
|
||||||
|
preview_url = find_preview_file(base_name, dir_path)
|
||||||
|
|
||||||
|
# AutoV3 reads only the safetensors header, so it is cheap even for
|
||||||
|
# large files; record the checked state at creation time ("" =
|
||||||
|
# checked but unavailable).
|
||||||
|
autov3 = calculate_autov3(real_path)
|
||||||
|
|
||||||
|
# Create metadata WITHOUT calculating hash
|
||||||
|
metadata = OtherModelMetadata(
|
||||||
|
file_name=base_name,
|
||||||
|
model_name=base_name,
|
||||||
|
file_path=normalize_path(file_path),
|
||||||
|
size=os.path.getsize(real_path),
|
||||||
|
modified=datetime.now().timestamp(),
|
||||||
|
sha256="", # Empty hash - will be calculated on-demand
|
||||||
|
base_model="Unknown",
|
||||||
|
preview_url=normalize_path(preview_url),
|
||||||
|
tags=[],
|
||||||
|
modelDescription="",
|
||||||
|
sub_type=self.resolve_sub_type_for_path(file_path) or "vae",
|
||||||
|
from_civitai=False, # Mark as local model since no hash yet
|
||||||
|
hash_status="pending", # Mark hash as pending
|
||||||
|
autov3=autov3 or "",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Save the created metadata
|
||||||
|
logger.info(f"Creating other-model metadata (hash pending) for {file_path}")
|
||||||
|
await MetadataManager.save_metadata(file_path, metadata)
|
||||||
|
|
||||||
|
return metadata
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(
|
||||||
|
f"Error creating default other-model metadata for {file_path}: {e}"
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def calculate_hash_for_model(self, file_path: str) -> Optional[str]:
|
||||||
|
"""Calculate hash for a model on-demand with per-file singleflight.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
file_path: Path to the model file
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
SHA256 hash string, or None if calculation failed
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
real_path = os.path.realpath(file_path)
|
||||||
|
if not os.path.exists(real_path):
|
||||||
|
logger.error(f"File not found for hash calculation: {file_path}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
metadata, _ = await MetadataManager.load_metadata(
|
||||||
|
file_path, self.model_class
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
metadata is not None
|
||||||
|
and metadata.hash_status == "completed"
|
||||||
|
and metadata.sha256
|
||||||
|
):
|
||||||
|
# Ensure the in-memory hash index is populated even when
|
||||||
|
# the hash was already computed and persisted to the metadata
|
||||||
|
# file. Without this, usage tracking (and any other caller
|
||||||
|
# that queries get_hash_by_filename first) will miss on every
|
||||||
|
# lookup and keep calling back into this method, creating a
|
||||||
|
# tight loop that never populates the index.
|
||||||
|
self._hash_index.add_entry(
|
||||||
|
metadata.sha256.lower(),
|
||||||
|
file_path,
|
||||||
|
getattr(metadata, "autov3", None) or None,
|
||||||
|
)
|
||||||
|
return metadata.sha256
|
||||||
|
|
||||||
|
async with self._hash_calculation_lock:
|
||||||
|
metadata, _ = await MetadataManager.load_metadata(
|
||||||
|
file_path, self.model_class
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
metadata is not None
|
||||||
|
and metadata.hash_status == "completed"
|
||||||
|
and metadata.sha256
|
||||||
|
):
|
||||||
|
self._hash_index.add_entry(
|
||||||
|
metadata.sha256.lower(),
|
||||||
|
file_path,
|
||||||
|
getattr(metadata, "autov3", None) or None,
|
||||||
|
)
|
||||||
|
return metadata.sha256
|
||||||
|
|
||||||
|
task = self._hash_calculation_tasks.get(real_path)
|
||||||
|
if task is None:
|
||||||
|
task = asyncio.create_task(
|
||||||
|
self._run_hash_calculation_task(file_path, real_path)
|
||||||
|
)
|
||||||
|
self._hash_calculation_tasks[real_path] = task
|
||||||
|
|
||||||
|
return await asyncio.shield(task)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error calculating hash for {file_path}: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def _run_hash_calculation_task(
|
||||||
|
self, file_path: str, real_path: str
|
||||||
|
) -> Optional[str]:
|
||||||
|
"""Run a hash calculation task and remove it from the in-flight map."""
|
||||||
|
try:
|
||||||
|
return await self._calculate_hash_for_model_uncached(file_path, real_path)
|
||||||
|
finally:
|
||||||
|
task = asyncio.current_task()
|
||||||
|
async with self._hash_calculation_lock:
|
||||||
|
if self._hash_calculation_tasks.get(real_path) is task:
|
||||||
|
del self._hash_calculation_tasks[real_path]
|
||||||
|
|
||||||
|
async def _calculate_hash_for_model_uncached(
|
||||||
|
self, file_path: str, real_path: str
|
||||||
|
) -> Optional[str]:
|
||||||
|
"""Calculate hash for a model without checking in-flight tasks."""
|
||||||
|
from ..utils.file_utils import calculate_sha256
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Load current metadata
|
||||||
|
metadata, should_skip = await MetadataManager.load_metadata(
|
||||||
|
file_path, self.model_class
|
||||||
|
)
|
||||||
|
if metadata is None:
|
||||||
|
if should_skip:
|
||||||
|
logger.error(f"Invalid metadata found for {file_path}")
|
||||||
|
return None
|
||||||
|
created_metadata = await self._create_default_metadata(file_path)
|
||||||
|
if created_metadata is None:
|
||||||
|
logger.error(f"No metadata found for {file_path}")
|
||||||
|
return None
|
||||||
|
metadata = created_metadata
|
||||||
|
|
||||||
|
# Check if hash is already calculated
|
||||||
|
if metadata.hash_status == "completed" and metadata.sha256:
|
||||||
|
# Populate the in-memory hash index even for pre-computed
|
||||||
|
# hashes, mirroring the fix in calculate_hash_for_model.
|
||||||
|
self._hash_index.add_entry(
|
||||||
|
metadata.sha256.lower(),
|
||||||
|
file_path,
|
||||||
|
getattr(metadata, "autov3", None) or None,
|
||||||
|
)
|
||||||
|
return metadata.sha256
|
||||||
|
|
||||||
|
# Update status to calculating
|
||||||
|
metadata.hash_status = "calculating"
|
||||||
|
await MetadataManager.save_metadata(file_path, metadata)
|
||||||
|
|
||||||
|
# Calculate hash
|
||||||
|
logger.info(f"Calculating hash for other model: {file_path}")
|
||||||
|
sha256 = await calculate_sha256(real_path)
|
||||||
|
|
||||||
|
# Update metadata with hash
|
||||||
|
metadata.sha256 = sha256
|
||||||
|
metadata.hash_status = "completed"
|
||||||
|
await MetadataManager.save_metadata(file_path, metadata)
|
||||||
|
|
||||||
|
# Update hash index
|
||||||
|
self._hash_index.add_entry(
|
||||||
|
sha256.lower(),
|
||||||
|
file_path,
|
||||||
|
getattr(metadata, "autov3", None) or None,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Update the in-memory cache entry so that subsequent
|
||||||
|
# _persist_current_cache / _save_persistent_cache calls
|
||||||
|
# write the hash back to the SQLite models table. Without
|
||||||
|
# this the hash only lives in the metadata file and the
|
||||||
|
# in-memory hash index, both of which are lost across
|
||||||
|
# restarts, causing the same re-computation loop on the
|
||||||
|
# next session.
|
||||||
|
if self._cache is not None and self._cache.raw_data:
|
||||||
|
for entry in self._cache.raw_data:
|
||||||
|
if entry.get("file_path") == file_path:
|
||||||
|
entry["sha256"] = sha256.lower()
|
||||||
|
entry["hash_status"] = "completed"
|
||||||
|
self.bump_cache_version()
|
||||||
|
break
|
||||||
|
|
||||||
|
logger.info(f"Hash calculated for other model: {file_path}")
|
||||||
|
return sha256
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error calculating hash for {file_path}: {e}")
|
||||||
|
# Update status to failed
|
||||||
|
try:
|
||||||
|
metadata, _ = await MetadataManager.load_metadata(
|
||||||
|
file_path, self.model_class
|
||||||
|
)
|
||||||
|
if metadata:
|
||||||
|
metadata.hash_status = "failed"
|
||||||
|
await MetadataManager.save_metadata(file_path, metadata)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def calculate_all_pending_hashes(
|
||||||
|
self, progress_callback=None
|
||||||
|
) -> Dict[str, int]:
|
||||||
|
"""Calculate hashes for all other models with pending hash status.
|
||||||
|
|
||||||
|
If cache is not initialized, scans filesystem directly for metadata files
|
||||||
|
with hash_status != 'completed'.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
progress_callback: Optional callback(progress, total, current_file)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dict with 'completed', 'failed', 'total' counts
|
||||||
|
"""
|
||||||
|
# Try to get from cache first
|
||||||
|
cache = await self.get_cached_data()
|
||||||
|
|
||||||
|
if cache and cache.raw_data:
|
||||||
|
# Use cache if available
|
||||||
|
pending_models = [
|
||||||
|
item
|
||||||
|
for item in cache.raw_data
|
||||||
|
if item.get("hash_status") != "completed" or not item.get("sha256")
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
# Cache not initialized, scan filesystem directly
|
||||||
|
pending_models = await self._find_pending_models_from_filesystem()
|
||||||
|
|
||||||
|
if not pending_models:
|
||||||
|
return {"completed": 0, "failed": 0, "total": 0}
|
||||||
|
|
||||||
|
total = len(pending_models)
|
||||||
|
completed = 0
|
||||||
|
failed = 0
|
||||||
|
|
||||||
|
for i, model_data in enumerate(pending_models):
|
||||||
|
file_path = model_data.get("file_path")
|
||||||
|
if not file_path:
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
sha256 = await self.calculate_hash_for_model(file_path)
|
||||||
|
if sha256:
|
||||||
|
completed += 1
|
||||||
|
else:
|
||||||
|
failed += 1
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error calculating hash for {file_path}: {e}")
|
||||||
|
failed += 1
|
||||||
|
|
||||||
|
if progress_callback:
|
||||||
|
try:
|
||||||
|
await progress_callback(i + 1, total, file_path)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return {"completed": completed, "failed": failed, "total": total}
|
||||||
|
|
||||||
|
async def _find_pending_models_from_filesystem(self) -> List[Dict[str, Any]]:
|
||||||
|
"""Scan filesystem for other-model metadata files with pending hash status."""
|
||||||
|
pending_models = []
|
||||||
|
|
||||||
|
for root_path in self.get_model_roots():
|
||||||
|
if not os.path.exists(root_path):
|
||||||
|
continue
|
||||||
|
|
||||||
|
for dirpath, dirnames, filenames in os.walk(root_path):
|
||||||
|
dirnames[:] = [d for d in dirnames if not _is_excluded_dir(d)]
|
||||||
|
for filename in filenames:
|
||||||
|
if not filename.endswith(".metadata.json"):
|
||||||
|
continue
|
||||||
|
|
||||||
|
metadata_path = os.path.join(dirpath, filename)
|
||||||
|
try:
|
||||||
|
with open(metadata_path, "r", encoding="utf-8") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
|
||||||
|
# Check if hash is pending
|
||||||
|
hash_status = data.get("hash_status", "completed")
|
||||||
|
sha256 = data.get("sha256", "")
|
||||||
|
|
||||||
|
if hash_status != "completed" or not sha256:
|
||||||
|
# Find corresponding model file
|
||||||
|
model_name = filename.replace(".metadata.json", "")
|
||||||
|
model_path = None
|
||||||
|
|
||||||
|
# Look for model file with matching name
|
||||||
|
for ext in self.file_extensions:
|
||||||
|
potential_path = os.path.join(dirpath, model_name + ext)
|
||||||
|
if os.path.exists(potential_path):
|
||||||
|
model_path = potential_path
|
||||||
|
break
|
||||||
|
|
||||||
|
if model_path:
|
||||||
|
pending_models.append(
|
||||||
|
{
|
||||||
|
"file_path": model_path.replace(os.sep, "/"),
|
||||||
|
"hash_status": hash_status,
|
||||||
|
"sha256": sha256,
|
||||||
|
**{
|
||||||
|
k: v
|
||||||
|
for k, v in data.items()
|
||||||
|
if k
|
||||||
|
not in [
|
||||||
|
"file_path",
|
||||||
|
"hash_status",
|
||||||
|
"sha256",
|
||||||
|
]
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
except (json.JSONDecodeError, Exception) as e:
|
||||||
|
logger.debug(
|
||||||
|
f"Error reading metadata file {metadata_path}: {e}"
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
return pending_models
|
||||||
|
|
||||||
|
def _root_sub_type_map(self) -> Dict[str, str]:
|
||||||
|
"""Return the configured business root -> sub_type map."""
|
||||||
|
root_map = getattr(config, "other_root_subtypes", None)
|
||||||
|
return root_map if isinstance(root_map, dict) else {}
|
||||||
|
|
||||||
|
def _resolve_sub_type(self, root_path: Optional[str]) -> Optional[str]:
|
||||||
|
"""Resolve the sub_type for a configured root path."""
|
||||||
|
if not root_path:
|
||||||
|
return None
|
||||||
|
|
||||||
|
normalized_root = self._normalize_path_value(root_path)
|
||||||
|
for root, sub_type in self._root_sub_type_map().items():
|
||||||
|
if self._normalize_path_value(root) == normalized_root:
|
||||||
|
return sub_type
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
def resolve_sub_type_for_path(self, file_path: Optional[str]) -> Optional[str]:
|
||||||
|
"""Resolve sub_type from the configured root that contains the file.
|
||||||
|
|
||||||
|
Uses the longest-prefix match so nested roots (e.g. a controlnet root
|
||||||
|
inside a vae root) resolve to the most specific category.
|
||||||
|
"""
|
||||||
|
normalized_path = self._normalize_path_value(file_path)
|
||||||
|
if not normalized_path:
|
||||||
|
return None
|
||||||
|
|
||||||
|
best_length = 0
|
||||||
|
best_sub_type: Optional[str] = None
|
||||||
|
for root, sub_type in self._root_sub_type_map().items():
|
||||||
|
normalized_root = self._normalize_path_value(root)
|
||||||
|
if not normalized_root:
|
||||||
|
continue
|
||||||
|
if (
|
||||||
|
normalized_path == normalized_root
|
||||||
|
or normalized_path.startswith(f"{normalized_root}/")
|
||||||
|
) and len(normalized_root) > best_length:
|
||||||
|
best_length = len(normalized_root)
|
||||||
|
best_sub_type = sub_type
|
||||||
|
|
||||||
|
return best_sub_type
|
||||||
|
|
||||||
|
def adjust_metadata(self, metadata, file_path, root_path):
|
||||||
|
"""Adjust metadata during scanning to set sub_type."""
|
||||||
|
sub_type = self._resolve_sub_type(root_path) or self.resolve_sub_type_for_path(
|
||||||
|
file_path
|
||||||
|
)
|
||||||
|
if sub_type:
|
||||||
|
metadata.sub_type = sub_type
|
||||||
|
return metadata
|
||||||
|
|
||||||
|
def adjust_cached_entry(self, entry: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
"""Adjust entries loaded from the persisted cache to ensure sub_type is set.
|
||||||
|
|
||||||
|
sub_type is location-derived: it is re-derived on cache load, never
|
||||||
|
trusted from the persisted snapshot.
|
||||||
|
"""
|
||||||
|
sub_type = self.resolve_sub_type_for_path(entry.get("file_path"))
|
||||||
|
if sub_type:
|
||||||
|
entry["sub_type"] = sub_type
|
||||||
|
return entry
|
||||||
|
|
||||||
|
def _should_keep_cached_entry(self, entry: Dict[str, Any]) -> bool:
|
||||||
|
"""Drop persisted entries whose folder is no longer a managed root.
|
||||||
|
|
||||||
|
sub_type is location-derived and config only maps enabled roots, so a
|
||||||
|
file under a disabled sub_type - or under any other root while the
|
||||||
|
feature is off - resolves to None here and is filtered out while the
|
||||||
|
persisted cache is hydrated.
|
||||||
|
"""
|
||||||
|
return self.resolve_sub_type_for_path(entry.get("file_path")) is not None
|
||||||
|
|
||||||
|
def get_model_roots(self) -> List[str]:
|
||||||
|
"""Get other-model root directories"""
|
||||||
|
roots: List[str] = []
|
||||||
|
roots.extend(config.other_roots or [])
|
||||||
|
# Remove duplicates while preserving order
|
||||||
|
seen: set[str] = set()
|
||||||
|
unique_roots: List[str] = []
|
||||||
|
for root in roots:
|
||||||
|
if root and root not in seen:
|
||||||
|
seen.add(root)
|
||||||
|
unique_roots.append(root)
|
||||||
|
return unique_roots
|
||||||
@@ -59,6 +59,7 @@ _MODEL_TYPE_PAGE_MAP = {
|
|||||||
"lora": "loras",
|
"lora": "loras",
|
||||||
"checkpoint": "checkpoints",
|
"checkpoint": "checkpoints",
|
||||||
"embedding": "embeddings",
|
"embedding": "embeddings",
|
||||||
|
"other": "other",
|
||||||
}
|
}
|
||||||
|
|
||||||
# Module-level alias so tests can spy on timer task creation without patching
|
# Module-level alias so tests can spy on timer task creation without patching
|
||||||
@@ -274,17 +275,21 @@ class PendingDeleteService:
|
|||||||
async def merge_batches(self, batch_ids: Sequence[str]) -> Optional[str]:
|
async def merge_batches(self, batch_ids: Sequence[str]) -> Optional[str]:
|
||||||
"""Merge several batches into the first batch's manifest.
|
"""Merge several batches into the first batch's manifest.
|
||||||
|
|
||||||
Winner is ``batch_ids[0]``. The staged files of losing batches are
|
Winner is ``batch_ids[0]``. Merging is MANIFEST-ONLY: staged files
|
||||||
MOVED (os.rename) into the winner's batch dir and their ``staged``
|
are NEVER moved, so the merge is a pure metadata operation with zero
|
||||||
paths rewritten in the merged manifest BEFORE any loser dir is
|
data IO and is inherently cross-volume safe (no EXDEV, no rollback).
|
||||||
removed. ``expires_at`` is re-anchored to ``now + TTL`` at merge time
|
Every loser's entries are appended to the winner's manifest with
|
||||||
and a FRESH purge timer is armed for the winner.
|
their ``staged`` paths unchanged (files keep living in the loser's
|
||||||
|
own batch dir - the sibling-of-model staging location), each loser
|
||||||
|
dir is recorded in the winner manifest's ``merged_sources``, and each
|
||||||
|
loser manifest is stamped ``merged_into`` so its own purge timer, a
|
||||||
|
post-restart sweep or a direct undo call no-op. ``expires_at`` is
|
||||||
|
re-anchored to ``now + TTL`` at merge time and a FRESH purge timer is
|
||||||
|
armed for the winner.
|
||||||
|
|
||||||
On any move failure every already-moved file is moved BACK and the
|
Returns the winner id, or ``None`` when the winner batch cannot be
|
||||||
original batch dirs/manifests are left intact; ``None`` is returned so
|
resolved (callers then fall back to the ``batch_ids`` array
|
||||||
callers fall back to the ``batch_ids`` array contract. Cross-volume
|
contract).
|
||||||
merges hit EXDEV here - expected and fine (the fallback is the normal
|
|
||||||
path for those bulks).
|
|
||||||
"""
|
"""
|
||||||
if not batch_ids:
|
if not batch_ids:
|
||||||
return None
|
return None
|
||||||
@@ -298,77 +303,68 @@ class PendingDeleteService:
|
|||||||
if winner_manifest is None:
|
if winner_manifest is None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# Track (entry, original_staged_path, loser_dir) for rollback.
|
# Build the merged manifest in memory: loser entries are appended
|
||||||
moved: List[Tuple[Dict[str, Any], str, str]] = []
|
# with their staged paths UNCHANGED - no file moves, no IO, no
|
||||||
processed_losers: List[Tuple[str, str]] = [] # (loser_id, loser_dir)
|
# EXDEV. Loser dirs remain as physical storage until the merged
|
||||||
|
# batch is undone or purged.
|
||||||
try:
|
merged_sources: List[str] = []
|
||||||
for loser_id in batch_ids[1:]:
|
seen_loser_dirs: Set[str] = set()
|
||||||
loser_dir = await self._find_batch_dir(loser_id)
|
for loser_id in batch_ids[1:]:
|
||||||
if not loser_dir or os.path.normpath(loser_dir) == os.path.normpath(
|
loser_dir = await self._find_batch_dir(loser_id)
|
||||||
winner_dir
|
if not loser_dir or os.path.normpath(loser_dir) == os.path.normpath(
|
||||||
):
|
winner_dir
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
loser_abs = os.path.abspath(loser_dir)
|
||||||
|
if loser_abs in seen_loser_dirs:
|
||||||
|
continue
|
||||||
|
seen_loser_dirs.add(loser_abs)
|
||||||
|
loser_manifest = self._read_manifest(loser_dir)
|
||||||
|
if loser_manifest is None:
|
||||||
|
# Corrupted loser: leave it for the sweep to quarantine.
|
||||||
|
continue
|
||||||
|
for entry in loser_manifest.get("entries") or []:
|
||||||
|
if entry.get("restored"):
|
||||||
continue
|
continue
|
||||||
loser_manifest = self._read_manifest(loser_dir)
|
staged_path = entry.get("staged")
|
||||||
if loser_manifest is None:
|
if not staged_path or not os.path.exists(staged_path):
|
||||||
# Corrupted loser: leave it for the sweep to quarantine.
|
|
||||||
continue
|
continue
|
||||||
for entry in loser_manifest.get("entries") or []:
|
winner_manifest["entries"].append(entry)
|
||||||
if entry.get("restored"):
|
merged_sources.append(loser_abs)
|
||||||
continue
|
|
||||||
staged_path = entry.get("staged")
|
|
||||||
if not staged_path or not os.path.exists(staged_path):
|
|
||||||
continue
|
|
||||||
new_staged = os.path.join(
|
|
||||||
winner_dir, os.path.basename(staged_path)
|
|
||||||
)
|
|
||||||
if os.path.exists(new_staged):
|
|
||||||
# os.rename would silently overwrite the existing
|
|
||||||
# staged file on POSIX - never drop a staged file.
|
|
||||||
# Abort the merge so callers fall back to the
|
|
||||||
# batch_ids array contract.
|
|
||||||
raise OSError(
|
|
||||||
f"Merge collision: {os.path.basename(staged_path)} "
|
|
||||||
"already staged in winner batch"
|
|
||||||
)
|
|
||||||
os.rename(staged_path, new_staged)
|
|
||||||
original_staged = entry["staged"]
|
|
||||||
entry["staged"] = os.path.abspath(new_staged)
|
|
||||||
winner_manifest["entries"].append(entry)
|
|
||||||
moved.append((entry, original_staged, loser_dir))
|
|
||||||
processed_losers.append((loser_id, loser_dir))
|
|
||||||
except OSError as exc:
|
|
||||||
logger.warning(
|
|
||||||
"Merge of %s failed after moving files: %s; rolling back",
|
|
||||||
list(batch_ids),
|
|
||||||
exc,
|
|
||||||
)
|
|
||||||
self._rollback_merge_moves(moved)
|
|
||||||
return None
|
|
||||||
|
|
||||||
# Re-anchor expiry and persist the merged manifest atomically.
|
# Re-anchor expiry and persist the merged manifest atomically - it
|
||||||
|
# becomes the ONLY source of truth for every merged file, wherever
|
||||||
|
# it physically lives.
|
||||||
winner_manifest["expires_at"] = (
|
winner_manifest["expires_at"] = (
|
||||||
int(time.time()) + PENDING_DELETE_TTL_SECONDS
|
int(time.time()) + PENDING_DELETE_TTL_SECONDS
|
||||||
)
|
)
|
||||||
|
if merged_sources:
|
||||||
|
winner_manifest["merged_sources"] = merged_sources
|
||||||
try:
|
try:
|
||||||
self._write_manifest_atomic(winner_dir, winner_manifest)
|
self._write_manifest_atomic(winner_dir, winner_manifest)
|
||||||
except OSError as exc:
|
except OSError as exc:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Failed to write merged manifest for %s: %s; rolling back",
|
"Failed to write merged manifest for %s: %s",
|
||||||
winner_id,
|
winner_id,
|
||||||
exc,
|
exc,
|
||||||
)
|
)
|
||||||
self._rollback_merge_moves(moved)
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# All moves committed: remove loser dirs (must be empty by now)
|
# Stamp each loser manifest so its own purge timer / a later sweep
|
||||||
# and drop them from the registry. Skipped losers (missing /
|
# / a direct undo call no-op: the winner owns those files from
|
||||||
# corrupted / same-dir) stay registered so the sweep still
|
# here on. Best-effort coordination; a failed stamp only risks the
|
||||||
# quarantines them, exactly as before the registry existed.
|
# loser being swept at its own (earlier) expiry after a restart.
|
||||||
for loser_id, loser_dir in processed_losers:
|
for loser_dir in merged_sources:
|
||||||
self._remove_manifest(loser_dir)
|
try:
|
||||||
self._remove_empty_dir(loser_dir)
|
self._mark_merged(loser_dir, winner_id)
|
||||||
await self._forget_batch(loser_id)
|
except OSError as exc: # pragma: no cover - best-effort
|
||||||
|
logger.warning(
|
||||||
|
"Failed to mark merged loser %s: %s", loser_dir, exc
|
||||||
|
)
|
||||||
|
|
||||||
|
# Losers are no longer independently managed.
|
||||||
|
for loser_dir in merged_sources:
|
||||||
|
await self._forget_batch(os.path.basename(loser_dir))
|
||||||
await self._remember_batch(winner_id, winner_dir)
|
await self._remember_batch(winner_id, winner_dir)
|
||||||
|
|
||||||
# Arm a fresh purge timer for the winner with the re-anchored
|
# Arm a fresh purge timer for the winner with the re-anchored
|
||||||
@@ -397,6 +393,16 @@ class PendingDeleteService:
|
|||||||
if manifest is None:
|
if manifest is None:
|
||||||
raise ValueError(f"Manifest missing for batch {batch_id}")
|
raise ValueError(f"Manifest missing for batch {batch_id}")
|
||||||
|
|
||||||
|
merged_into = manifest.get("merged_into")
|
||||||
|
if merged_into:
|
||||||
|
# The batch was merged into another batch: its staged files
|
||||||
|
# are owned by the winner's manifest. Undo via the winner so
|
||||||
|
# the whole merged batch stays consistent.
|
||||||
|
raise ValueError(
|
||||||
|
f"Batch {batch_id} was merged into batch {merged_into}; "
|
||||||
|
"undo that batch instead"
|
||||||
|
)
|
||||||
|
|
||||||
if manifest.get("state") == "restored":
|
if manifest.get("state") == "restored":
|
||||||
return self._undo_result(manifest)
|
return self._undo_result(manifest)
|
||||||
|
|
||||||
@@ -448,6 +454,10 @@ class PendingDeleteService:
|
|||||||
self._remove_manifest(batch_dir)
|
self._remove_manifest(batch_dir)
|
||||||
self._remove_empty_dir(batch_dir)
|
self._remove_empty_dir(batch_dir)
|
||||||
await self._forget_batch(batch_id)
|
await self._forget_batch(batch_id)
|
||||||
|
# Clean up merged loser dirs (their staged files were restored
|
||||||
|
# above) and drop them from the registry too.
|
||||||
|
for loser_id in self._remove_merged_batch_dirs(manifest):
|
||||||
|
await self._forget_batch(loser_id)
|
||||||
|
|
||||||
logger.info("Restored pending-delete batch %s", batch_id)
|
logger.info("Restored pending-delete batch %s", batch_id)
|
||||||
return self._undo_result(manifest)
|
return self._undo_result(manifest)
|
||||||
@@ -500,10 +510,36 @@ class PendingDeleteService:
|
|||||||
QUARANTINE them (preserving the pre-registry sweep semantics). The
|
QUARANTINE them (preserving the pre-registry sweep semantics). The
|
||||||
walk only descends into dirs literally named ``.lm-pending-delete``,
|
walk only descends into dirs literally named ``.lm-pending-delete``,
|
||||||
so false positives are structurally limited.
|
so false positives are structurally limited.
|
||||||
|
|
||||||
|
The filesystem walk itself runs in a worker thread so a large or slow
|
||||||
|
library cannot block the event loop at startup; only the (rare) batch
|
||||||
|
registration awaits run on the loop.
|
||||||
|
"""
|
||||||
|
roots = await self._get_all_model_roots()
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
staging_parents = await loop.run_in_executor(
|
||||||
|
None, # Use default thread pool
|
||||||
|
self._collect_staging_parents, # Run the tree walk off the loop
|
||||||
|
roots,
|
||||||
|
)
|
||||||
|
for staging_parent in staging_parents:
|
||||||
|
await self._register_batch_candidates(staging_parent)
|
||||||
|
|
||||||
|
def _collect_staging_parents(self, roots: Sequence[str]) -> List[str]:
|
||||||
|
"""Walk every model root and return its staging-parent dirs.
|
||||||
|
|
||||||
|
Pure synchronous filesystem discovery with no awaits: walks with
|
||||||
|
``followlinks=True, topdown=True``, prunes symlink cycles via a
|
||||||
|
per-root ``visited`` realpath set (realpath is used ONLY for this
|
||||||
|
dedup set - the returned paths are the unresolved business paths),
|
||||||
|
filters out :func:`_is_excluded_dir` dirs, and collects every dir
|
||||||
|
named ``.lm-pending-delete`` (including the case where a model root
|
||||||
|
itself is one). Results are returned in walk order.
|
||||||
"""
|
"""
|
||||||
from .model_scanner import _is_excluded_dir
|
from .model_scanner import _is_excluded_dir
|
||||||
|
|
||||||
for root in await self._get_all_model_roots():
|
staging_parents: List[str] = []
|
||||||
|
for root in roots:
|
||||||
if not os.path.isdir(root):
|
if not os.path.isdir(root):
|
||||||
continue
|
continue
|
||||||
visited: Set[str] = set()
|
visited: Set[str] = set()
|
||||||
@@ -518,21 +554,20 @@ class PendingDeleteService:
|
|||||||
visited.add(real_dir)
|
visited.add(real_dir)
|
||||||
if os.path.basename(dirpath) == PENDING_DELETE_DIR_NAME:
|
if os.path.basename(dirpath) == PENDING_DELETE_DIR_NAME:
|
||||||
# The current dir IS a staging parent (reachable only when
|
# The current dir IS a staging parent (reachable only when
|
||||||
# a model root itself is one): register its batches.
|
# a model root itself is one): collect its batches.
|
||||||
await self._register_batch_candidates(dirpath)
|
staging_parents.append(dirpath)
|
||||||
dirnames[:] = []
|
dirnames[:] = []
|
||||||
continue
|
continue
|
||||||
next_dirs: List[str] = []
|
next_dirs: List[str] = []
|
||||||
for name in dirnames:
|
for name in dirnames:
|
||||||
if name == PENDING_DELETE_DIR_NAME:
|
if name == PENDING_DELETE_DIR_NAME:
|
||||||
await self._register_batch_candidates(
|
staging_parents.append(os.path.join(dirpath, name))
|
||||||
os.path.join(dirpath, name)
|
|
||||||
)
|
|
||||||
elif _is_excluded_dir(name):
|
elif _is_excluded_dir(name):
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
next_dirs.append(name)
|
next_dirs.append(name)
|
||||||
dirnames[:] = next_dirs
|
dirnames[:] = next_dirs
|
||||||
|
return staging_parents
|
||||||
|
|
||||||
async def _register_batch_candidates(self, staging_parent: str) -> None:
|
async def _register_batch_candidates(self, staging_parent: str) -> None:
|
||||||
"""Register every non-orphaned batch subdir of a staging parent."""
|
"""Register every non-orphaned batch subdir of a staging parent."""
|
||||||
@@ -754,25 +789,35 @@ class PendingDeleteService:
|
|||||||
"Failed to remove staged copy %s: %s", staged_path, exc
|
"Failed to remove staged copy %s: %s", staged_path, exc
|
||||||
)
|
)
|
||||||
|
|
||||||
def _rollback_merge_moves(
|
def _mark_merged(self, loser_dir: str, winner_id: str) -> None:
|
||||||
self, moved: Sequence[Tuple[Dict[str, Any], str, str]]
|
"""Stamp ``merged_into`` on a loser manifest (best-effort).
|
||||||
) -> None:
|
|
||||||
"""Move already-merged files back to their original loser batch dirs."""
|
The stamp makes the loser's own purge timer, post-restart sweeps and
|
||||||
for _entry, original_staged, _loser_dir in reversed(list(moved)):
|
direct undo calls no-op, so the winner's merged batch stays the only
|
||||||
current = _entry.get("staged")
|
owner of the loser's staged files until it is undone or purged.
|
||||||
if not current or not original_staged:
|
"""
|
||||||
|
loser_manifest = self._read_manifest(loser_dir)
|
||||||
|
if loser_manifest is None:
|
||||||
|
return
|
||||||
|
loser_manifest["merged_into"] = winner_id
|
||||||
|
self._write_manifest_atomic(loser_dir, loser_manifest)
|
||||||
|
|
||||||
|
def _remove_merged_batch_dirs(self, manifest: Dict[str, Any]) -> List[str]:
|
||||||
|
"""Remove merged loser batch dirs once their files were handled.
|
||||||
|
|
||||||
|
Called after a merged batch has been fully undone or purged: each
|
||||||
|
loser manifest (stamped ``merged_into``) and its now-empty dir are
|
||||||
|
removed so the sweep never quarantines an orphaned staging dir.
|
||||||
|
Best-effort - returns the removed batch ids for registry cleanup.
|
||||||
|
"""
|
||||||
|
removed: List[str] = []
|
||||||
|
for src in manifest.get("merged_sources") or []:
|
||||||
|
if not isinstance(src, str) or not src:
|
||||||
continue
|
continue
|
||||||
if not os.path.exists(current):
|
self._remove_manifest(src)
|
||||||
continue
|
self._remove_empty_dir(src)
|
||||||
try:
|
removed.append(os.path.basename(src))
|
||||||
os.rename(current, original_staged)
|
return removed
|
||||||
except OSError as exc: # pragma: no cover - best-effort rollback
|
|
||||||
logger.warning(
|
|
||||||
"Failed to roll back merge move %s -> %s: %s",
|
|
||||||
current,
|
|
||||||
original_staged,
|
|
||||||
exc,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _purge_batch_dir(self, batch_dir: str) -> bool:
|
def _purge_batch_dir(self, batch_dir: str) -> bool:
|
||||||
"""Purge one batch dir. Returns True when the batch was purged/removed."""
|
"""Purge one batch dir. Returns True when the batch was purged/removed."""
|
||||||
@@ -786,6 +831,13 @@ class PendingDeleteService:
|
|||||||
self._quarantine_batch_dir(batch_dir)
|
self._quarantine_batch_dir(batch_dir)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
if manifest.get("merged_into"):
|
||||||
|
# Merged into another batch: the winner owns these staged files.
|
||||||
|
# The loser's own purge timer / post-restart sweep must not remove
|
||||||
|
# them early (the winner re-anchored the merged expiry to give the
|
||||||
|
# whole bulk one undo window).
|
||||||
|
return False
|
||||||
|
|
||||||
if manifest.get("state") == "restored":
|
if manifest.get("state") == "restored":
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -817,6 +869,7 @@ class PendingDeleteService:
|
|||||||
|
|
||||||
self._remove_manifest(batch_dir)
|
self._remove_manifest(batch_dir)
|
||||||
self._remove_empty_dir(batch_dir)
|
self._remove_empty_dir(batch_dir)
|
||||||
|
self._remove_merged_batch_dirs(manifest)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def _quarantine_batch_dir(self, batch_dir: str) -> str:
|
def _quarantine_batch_dir(self, batch_dir: str) -> str:
|
||||||
@@ -931,6 +984,7 @@ class PendingDeleteService:
|
|||||||
"get_lora_scanner",
|
"get_lora_scanner",
|
||||||
"get_checkpoint_scanner",
|
"get_checkpoint_scanner",
|
||||||
"get_embedding_scanner",
|
"get_embedding_scanner",
|
||||||
|
"get_other_scanner",
|
||||||
):
|
):
|
||||||
getter = getattr(ServiceRegistry, getter_name, None)
|
getter = getattr(ServiceRegistry, getter_name, None)
|
||||||
if not callable(getter):
|
if not callable(getter):
|
||||||
|
|||||||
@@ -19,6 +19,9 @@ class PersistedCacheData:
|
|||||||
hash_rows: List[Tuple[str, str]]
|
hash_rows: List[Tuple[str, str]]
|
||||||
excluded_models: List[str]
|
excluded_models: List[str]
|
||||||
autov3_hash_rows: List[Tuple[str, str]] = field(default_factory=list)
|
autov3_hash_rows: List[Tuple[str, str]] = field(default_factory=list)
|
||||||
|
# Every directory under the model roots (including empty ones), or None
|
||||||
|
# when the snapshot predates folder recording.
|
||||||
|
all_folders: Optional[List[str]] = None
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_LICENSE_FLAGS = 127 # 127 (0b1111111) encodes default CivitAI permissions with all commercial modes enabled.
|
DEFAULT_LICENSE_FLAGS = 127 # 127 (0b1111111) encodes default CivitAI permissions with all commercial modes enabled.
|
||||||
@@ -128,6 +131,14 @@ class PersistentModelCache:
|
|||||||
"SELECT file_path FROM excluded_models WHERE model_type = ?",
|
"SELECT file_path FROM excluded_models WHERE model_type = ?",
|
||||||
(model_type,),
|
(model_type,),
|
||||||
).fetchall()
|
).fetchall()
|
||||||
|
folder_rows = conn.execute(
|
||||||
|
"SELECT path FROM folders WHERE model_type = ?",
|
||||||
|
(model_type,),
|
||||||
|
).fetchall()
|
||||||
|
folders_recorded = conn.execute(
|
||||||
|
"SELECT value FROM cache_meta WHERE key = ?",
|
||||||
|
(f"folders_recorded:{model_type}",),
|
||||||
|
).fetchone()
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -216,14 +227,20 @@ class PersistentModelCache:
|
|||||||
]
|
]
|
||||||
|
|
||||||
excluded_paths = [row["file_path"] for row in excluded]
|
excluded_paths = [row["file_path"] for row in excluded]
|
||||||
|
all_folders: Optional[List[str]] = None
|
||||||
|
if folders_recorded is not None:
|
||||||
|
all_folders = sorted(
|
||||||
|
(row["path"] for row in folder_rows), key=lambda x: x.lower()
|
||||||
|
)
|
||||||
return PersistedCacheData(
|
return PersistedCacheData(
|
||||||
raw_data=raw_data,
|
raw_data=raw_data,
|
||||||
hash_rows=hash_pairs,
|
hash_rows=hash_pairs,
|
||||||
excluded_models=excluded_paths,
|
excluded_models=excluded_paths,
|
||||||
autov3_hash_rows=autov3_pairs,
|
autov3_hash_rows=autov3_pairs,
|
||||||
|
all_folders=all_folders,
|
||||||
)
|
)
|
||||||
|
|
||||||
def save_cache(self, model_type: str, raw_data: Sequence[Dict[str, Any]], hash_index: Dict[str, List[str]], excluded_models: Sequence[str], autov3_hash_index: Optional[Dict[str, List[str]]] = None) -> None:
|
def save_cache(self, model_type: str, raw_data: Sequence[Dict[str, Any]], hash_index: Dict[str, List[str]], excluded_models: Sequence[str], autov3_hash_index: Optional[Dict[str, List[str]]] = None, all_folders: Optional[Sequence[str]] = None) -> None:
|
||||||
if not self.is_enabled():
|
if not self.is_enabled():
|
||||||
return
|
return
|
||||||
if not self._schema_initialized:
|
if not self._schema_initialized:
|
||||||
@@ -469,6 +486,27 @@ class PersistentModelCache:
|
|||||||
excluded_inserts,
|
excluded_inserts,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if all_folders is not None:
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM folders WHERE model_type = ?",
|
||||||
|
(model_type,),
|
||||||
|
)
|
||||||
|
folder_inserts = [
|
||||||
|
(model_type, path) for path in all_folders if path
|
||||||
|
]
|
||||||
|
if folder_inserts:
|
||||||
|
conn.executemany(
|
||||||
|
"INSERT OR IGNORE INTO folders (model_type, path) VALUES (?, ?)",
|
||||||
|
folder_inserts,
|
||||||
|
)
|
||||||
|
# Mark the snapshot as having folder data even when the
|
||||||
|
# library has no subfolders, so an empty list is not
|
||||||
|
# mistaken for "never recorded" on load.
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR REPLACE INTO cache_meta (key, value) VALUES (?, ?)",
|
||||||
|
(f"folders_recorded:{model_type}", "1"),
|
||||||
|
)
|
||||||
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
@@ -554,6 +592,17 @@ class PersistentModelCache:
|
|||||||
file_path TEXT NOT NULL,
|
file_path TEXT NOT NULL,
|
||||||
PRIMARY KEY (model_type, file_path)
|
PRIMARY KEY (model_type, file_path)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS folders (
|
||||||
|
model_type TEXT NOT NULL,
|
||||||
|
path TEXT NOT NULL,
|
||||||
|
PRIMARY KEY (model_type, path)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS cache_meta (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT
|
||||||
|
);
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
self._ensure_additional_model_columns(conn)
|
self._ensure_additional_model_columns(conn)
|
||||||
|
|||||||
@@ -52,12 +52,13 @@ class PersistentRecipeCache:
|
|||||||
"file_mtime",
|
"file_mtime",
|
||||||
"file_size",
|
"file_size",
|
||||||
"favorite",
|
"favorite",
|
||||||
"repair_version",
|
|
||||||
"preview_nsfw_level",
|
"preview_nsfw_level",
|
||||||
"loras_json",
|
"loras_json",
|
||||||
"checkpoint_json",
|
"checkpoint_json",
|
||||||
"gen_params_json",
|
"gen_params_json",
|
||||||
"tags_json",
|
"tags_json",
|
||||||
|
"has_workflow",
|
||||||
|
"import_info_json",
|
||||||
)
|
)
|
||||||
_instances: Dict[str, "PersistentRecipeCache"] = {}
|
_instances: Dict[str, "PersistentRecipeCache"] = {}
|
||||||
_instance_lock = threading.Lock()
|
_instance_lock = threading.Lock()
|
||||||
@@ -332,6 +333,44 @@ class PersistentRecipeCache:
|
|||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.debug("Failed to persist image_id_map: %s", exc)
|
logger.debug("Failed to persist image_id_map: %s", exc)
|
||||||
|
|
||||||
|
def get_metadata_value(self, key: str) -> Optional[str]:
|
||||||
|
"""Return a value from cache_metadata, or None if missing."""
|
||||||
|
if not self.is_enabled() or not self._schema_initialized:
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self._db_lock:
|
||||||
|
conn = self._connect(readonly=True)
|
||||||
|
try:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT value FROM cache_metadata WHERE key = ?",
|
||||||
|
(key,),
|
||||||
|
).fetchone()
|
||||||
|
return row["value"] if row else None
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def set_metadata_value(self, key: str, value: str) -> None:
|
||||||
|
"""Store a value in cache_metadata without rewriting the full cache."""
|
||||||
|
if not self.is_enabled() or not self._schema_initialized:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self._db_lock:
|
||||||
|
conn = self._connect()
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR REPLACE INTO cache_metadata (key, value) VALUES (?, ?)",
|
||||||
|
(key, value),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("Failed to persist cache metadata %s: %s", key, exc)
|
||||||
|
|
||||||
def get_indexed_recipe_ids(self) -> Set[str]:
|
def get_indexed_recipe_ids(self) -> Set[str]:
|
||||||
"""Return all recipe IDs in the cache.
|
"""Return all recipe IDs in the cache.
|
||||||
|
|
||||||
@@ -402,12 +441,13 @@ class PersistentRecipeCache:
|
|||||||
file_mtime REAL,
|
file_mtime REAL,
|
||||||
file_size INTEGER,
|
file_size INTEGER,
|
||||||
favorite INTEGER DEFAULT 0,
|
favorite INTEGER DEFAULT 0,
|
||||||
repair_version INTEGER DEFAULT 0,
|
|
||||||
preview_nsfw_level INTEGER DEFAULT 0,
|
preview_nsfw_level INTEGER DEFAULT 0,
|
||||||
loras_json TEXT,
|
loras_json TEXT,
|
||||||
checkpoint_json TEXT,
|
checkpoint_json TEXT,
|
||||||
gen_params_json TEXT,
|
gen_params_json TEXT,
|
||||||
tags_json TEXT
|
tags_json TEXT,
|
||||||
|
has_workflow INTEGER DEFAULT 0,
|
||||||
|
import_info_json TEXT
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_recipes_json_path ON recipes(json_path);
|
CREATE INDEX IF NOT EXISTS idx_recipes_json_path ON recipes(json_path);
|
||||||
@@ -426,6 +466,20 @@ class PersistentRecipeCache:
|
|||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass # column already exists
|
pass # column already exists
|
||||||
|
# Migration: add has_workflow column to existing databases
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"ALTER TABLE recipes ADD COLUMN has_workflow INTEGER DEFAULT 0"
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
pass # column already exists
|
||||||
|
# Migration: add import_info_json column to existing databases
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"ALTER TABLE recipes ADD COLUMN import_info_json TEXT"
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
pass # column already exists
|
||||||
conn.commit()
|
conn.commit()
|
||||||
self._schema_initialized = True
|
self._schema_initialized = True
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -457,6 +511,9 @@ class PersistentRecipeCache:
|
|||||||
tags = recipe.get("tags")
|
tags = recipe.get("tags")
|
||||||
tags_json = json.dumps(tags) if tags else None
|
tags_json = json.dumps(tags) if tags else None
|
||||||
|
|
||||||
|
import_info = recipe.get("import_info")
|
||||||
|
import_info_json = json.dumps(import_info) if import_info else None
|
||||||
|
|
||||||
# Get file stats if json_path exists
|
# Get file stats if json_path exists
|
||||||
file_mtime = 0.0
|
file_mtime = 0.0
|
||||||
file_size = 0
|
file_size = 0
|
||||||
@@ -482,12 +539,13 @@ class PersistentRecipeCache:
|
|||||||
file_mtime,
|
file_mtime,
|
||||||
file_size,
|
file_size,
|
||||||
1 if recipe.get("favorite") else 0,
|
1 if recipe.get("favorite") else 0,
|
||||||
int(recipe.get("repair_version") or 0),
|
|
||||||
int(recipe.get("preview_nsfw_level") or 0),
|
int(recipe.get("preview_nsfw_level") or 0),
|
||||||
loras_json,
|
loras_json,
|
||||||
checkpoint_json,
|
checkpoint_json,
|
||||||
gen_params_json,
|
gen_params_json,
|
||||||
tags_json,
|
tags_json,
|
||||||
|
1 if recipe.get("has_workflow") else 0,
|
||||||
|
import_info_json,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _row_to_recipe(self, row: sqlite3.Row) -> Dict[str, Any]:
|
def _row_to_recipe(self, row: sqlite3.Row) -> Dict[str, Any]:
|
||||||
@@ -520,6 +578,13 @@ class PersistentRecipeCache:
|
|||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
import_info = None
|
||||||
|
if row["import_info_json"]:
|
||||||
|
try:
|
||||||
|
import_info = json.loads(row["import_info_json"])
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
pass
|
||||||
|
|
||||||
recipe = {
|
recipe = {
|
||||||
"id": row["recipe_id"],
|
"id": row["recipe_id"],
|
||||||
"file_path": row["file_path"] or "",
|
"file_path": row["file_path"] or "",
|
||||||
@@ -531,8 +596,8 @@ class PersistentRecipeCache:
|
|||||||
"created_date": row["created_date"] or 0.0,
|
"created_date": row["created_date"] or 0.0,
|
||||||
"modified": row["modified"] or 0.0,
|
"modified": row["modified"] or 0.0,
|
||||||
"favorite": bool(row["favorite"]),
|
"favorite": bool(row["favorite"]),
|
||||||
"repair_version": row["repair_version"] or 0,
|
|
||||||
"preview_nsfw_level": row["preview_nsfw_level"] or 0,
|
"preview_nsfw_level": row["preview_nsfw_level"] or 0,
|
||||||
|
"has_workflow": bool(row["has_workflow"]),
|
||||||
"loras": loras,
|
"loras": loras,
|
||||||
"gen_params": gen_params,
|
"gen_params": gen_params,
|
||||||
}
|
}
|
||||||
@@ -543,6 +608,9 @@ class PersistentRecipeCache:
|
|||||||
if checkpoint:
|
if checkpoint:
|
||||||
recipe["checkpoint"] = checkpoint
|
recipe["checkpoint"] = checkpoint
|
||||||
|
|
||||||
|
if import_info:
|
||||||
|
recipe["import_info"] = import_info
|
||||||
|
|
||||||
return recipe
|
return recipe
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,213 @@
|
|||||||
|
"""Process-wide, per-destination rate-limit gate for outbound API traffic.
|
||||||
|
|
||||||
|
Implements the pacing/gating layer designed in
|
||||||
|
``docs/plans/issue-1085-rate-limit-design.md``:
|
||||||
|
|
||||||
|
- **Reactive gate**: a 429 response arms ``next_allowed_send`` from the
|
||||||
|
vendor's ``Retry-After`` (or exponential backoff when the header is
|
||||||
|
missing); subsequent requests to the same destination wait out the window.
|
||||||
|
- **Preemptive pacing**: a minimum inter-request interval per destination
|
||||||
|
spaces consecutive sends so bursts never form in the first place.
|
||||||
|
- **Herd-free**: waiters are serialized through a per-destination lock, so
|
||||||
|
each one claims a distinct send slot instead of thousands of coroutines
|
||||||
|
waking up together.
|
||||||
|
- **Bounded**: waits longer than ``rate_limit_max_wait_seconds`` are refused
|
||||||
|
by raising :class:`RateLimitError`, leaving the final decision to callers.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Dict, Optional
|
||||||
|
|
||||||
|
from .errors import RateLimitError
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DEFAULT_MIN_INTERVAL_SECONDS = 0.75
|
||||||
|
DEFAULT_MAX_WAIT_SECONDS = 300.0
|
||||||
|
BASE_BACKOFF_SECONDS = 30.0
|
||||||
|
MAX_BACKOFF_SECONDS = 1800.0
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _DestinationState:
|
||||||
|
"""Rate-limit bookkeeping for one destination (hostname)."""
|
||||||
|
|
||||||
|
next_allowed_send: float = 0.0 # time.monotonic() timestamp
|
||||||
|
consecutive_429: int = 0
|
||||||
|
last_send_at: float = 0.0 # time.monotonic() timestamp
|
||||||
|
lock: asyncio.Lock = field(default_factory=asyncio.Lock)
|
||||||
|
|
||||||
|
|
||||||
|
class RateLimitCoordinator:
|
||||||
|
"""Coordinates outbound request pacing per destination.
|
||||||
|
|
||||||
|
Singleton mirroring :class:`ConnectivityGuard`'s pattern. All waits are
|
||||||
|
bounded by the ``rate_limit_max_wait_seconds`` setting; when the required
|
||||||
|
wait exceeds the cap, :meth:`wait_for_slot` raises :class:`RateLimitError`
|
||||||
|
instead of parking the caller.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_instance: "RateLimitCoordinator | None" = None
|
||||||
|
_instance_lock = asyncio.Lock()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
async def get_instance(cls) -> "RateLimitCoordinator":
|
||||||
|
async with cls._instance_lock:
|
||||||
|
if cls._instance is None:
|
||||||
|
cls._instance = cls()
|
||||||
|
return cls._instance
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
if hasattr(self, "_initialized"):
|
||||||
|
return
|
||||||
|
self._initialized = True
|
||||||
|
self._states: Dict[str, _DestinationState] = {}
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Settings (read live so settings edits apply without a restart)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _setting(key: str, default):
|
||||||
|
try:
|
||||||
|
from .settings_manager import get_settings_manager
|
||||||
|
|
||||||
|
return get_settings_manager().get(key, default)
|
||||||
|
except Exception: # pragma: no cover - defensive: settings unavailable
|
||||||
|
return default
|
||||||
|
|
||||||
|
@property
|
||||||
|
def enabled(self) -> bool:
|
||||||
|
return bool(self._setting("rate_limit_gate_enabled", True))
|
||||||
|
|
||||||
|
@property
|
||||||
|
def min_interval_seconds(self) -> float:
|
||||||
|
try:
|
||||||
|
return max(0.0, float(self._setting("rate_limit_min_interval_seconds", DEFAULT_MIN_INTERVAL_SECONDS)))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return DEFAULT_MIN_INTERVAL_SECONDS
|
||||||
|
|
||||||
|
@property
|
||||||
|
def max_wait_seconds(self) -> float:
|
||||||
|
try:
|
||||||
|
return max(0.0, float(self._setting("rate_limit_max_wait_seconds", DEFAULT_MAX_WAIT_SECONDS)))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return DEFAULT_MAX_WAIT_SECONDS
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# State helpers
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _normalize(destination: Optional[str]) -> str:
|
||||||
|
if destination is None or not destination.strip():
|
||||||
|
return "__global__"
|
||||||
|
return destination.lower().strip()
|
||||||
|
|
||||||
|
def _state_for(self, destination: Optional[str]) -> _DestinationState:
|
||||||
|
key = self._normalize(destination)
|
||||||
|
if key not in self._states:
|
||||||
|
self._states[key] = _DestinationState()
|
||||||
|
return self._states[key]
|
||||||
|
|
||||||
|
def reset(self) -> None:
|
||||||
|
"""Drop all per-destination state. Test seam."""
|
||||||
|
self._states.clear()
|
||||||
|
|
||||||
|
def in_cooldown(self, destination: Optional[str] = None) -> bool:
|
||||||
|
return self.remaining_seconds(destination) > 0
|
||||||
|
|
||||||
|
def remaining_seconds(self, destination: Optional[str] = None) -> float:
|
||||||
|
state = self._state_for(destination)
|
||||||
|
return max(0.0, state.next_allowed_send - time.monotonic())
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Gate operations
|
||||||
|
|
||||||
|
async def wait_for_slot(self, destination: Optional[str] = None) -> None:
|
||||||
|
"""Block until this caller may send the next request to *destination*.
|
||||||
|
|
||||||
|
Waits for both the rate-limit cooldown (``next_allowed_send``) and the
|
||||||
|
minimum inter-request interval (``last_send_at + min_interval``).
|
||||||
|
Waiters queue on the per-destination lock, so concurrent callers are
|
||||||
|
spaced out instead of stampeding when a cooldown expires.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
RateLimitError: when the required wait exceeds
|
||||||
|
``rate_limit_max_wait_seconds``.
|
||||||
|
"""
|
||||||
|
state = self._state_for(destination)
|
||||||
|
deadline = time.monotonic() + self.max_wait_seconds
|
||||||
|
async with state.lock:
|
||||||
|
now = time.monotonic()
|
||||||
|
wake_at = max(
|
||||||
|
state.next_allowed_send,
|
||||||
|
state.last_send_at + self.min_interval_seconds,
|
||||||
|
)
|
||||||
|
if wake_at > deadline:
|
||||||
|
raise RateLimitError(
|
||||||
|
f"Rate limit wait for '{self._normalize(destination)}' "
|
||||||
|
f"exceeds the {self.max_wait_seconds:.0f}s cap",
|
||||||
|
retry_after=wake_at - now,
|
||||||
|
)
|
||||||
|
delay = wake_at - now
|
||||||
|
if delay > 0:
|
||||||
|
logger.debug(
|
||||||
|
"Rate-limit gate: pacing request to '%s' by %.2fs",
|
||||||
|
self._normalize(destination),
|
||||||
|
delay,
|
||||||
|
)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
state.last_send_at = time.monotonic()
|
||||||
|
|
||||||
|
def register_rate_limit(
|
||||||
|
self,
|
||||||
|
destination: Optional[str],
|
||||||
|
retry_after: Optional[float] = None,
|
||||||
|
) -> float:
|
||||||
|
"""Record a 429 for *destination* and arm the cooldown window.
|
||||||
|
|
||||||
|
Honors the vendor's ``Retry-After`` when present; otherwise grows an
|
||||||
|
exponential backoff (30s base, doubling per consecutive 429, capped at
|
||||||
|
1800s). Returns the cooldown duration in seconds.
|
||||||
|
"""
|
||||||
|
state = self._state_for(destination)
|
||||||
|
state.consecutive_429 += 1
|
||||||
|
if retry_after is not None and retry_after > 0:
|
||||||
|
backoff = min(MAX_BACKOFF_SECONDS, float(retry_after))
|
||||||
|
else:
|
||||||
|
backoff = min(
|
||||||
|
MAX_BACKOFF_SECONDS,
|
||||||
|
BASE_BACKOFF_SECONDS * (2 ** (state.consecutive_429 - 1)),
|
||||||
|
)
|
||||||
|
now = time.monotonic()
|
||||||
|
already_cooling = state.next_allowed_send > now
|
||||||
|
state.next_allowed_send = max(state.next_allowed_send, now + backoff)
|
||||||
|
if already_cooling:
|
||||||
|
logger.debug(
|
||||||
|
"Rate-limit cooldown for '%s' extended by %.0fs (consecutive_429=%d)",
|
||||||
|
self._normalize(destination),
|
||||||
|
backoff,
|
||||||
|
state.consecutive_429,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.info(
|
||||||
|
"Rate limited by '%s'; pausing requests for %.0fs",
|
||||||
|
self._normalize(destination),
|
||||||
|
backoff,
|
||||||
|
)
|
||||||
|
return backoff
|
||||||
|
|
||||||
|
def register_success(self, destination: Optional[str]) -> None:
|
||||||
|
"""Reset rate-limit state after a successful request.
|
||||||
|
|
||||||
|
A 200 proves the vendor is accepting traffic again, so any armed
|
||||||
|
cooldown window is cleared alongside the backoff counter (mirrors
|
||||||
|
``ConnectivityGuard.register_success`` semantics).
|
||||||
|
"""
|
||||||
|
state = self._state_for(destination)
|
||||||
|
state.consecutive_429 = 0
|
||||||
|
state.next_allowed_send = 0.0
|
||||||
@@ -7,13 +7,14 @@ enabling sub-100ms search times even with 20k+ recipes.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import hashlib
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from typing import Any, Dict, List, Optional, Set
|
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||||
|
|
||||||
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
||||||
|
|
||||||
@@ -165,6 +166,7 @@ class RecipeFTSIndex:
|
|||||||
batch_size = 500
|
batch_size = 500
|
||||||
total = len(recipes)
|
total = len(recipes)
|
||||||
inserted = 0
|
inserted = 0
|
||||||
|
indexed_ids: Set[str] = set()
|
||||||
|
|
||||||
for i in range(0, total, batch_size):
|
for i in range(0, total, batch_size):
|
||||||
batch = recipes[i:i + batch_size]
|
batch = recipes[i:i + batch_size]
|
||||||
@@ -179,6 +181,7 @@ class RecipeFTSIndex:
|
|||||||
row = self._prepare_fts_row(recipe)
|
row = self._prepare_fts_row(recipe)
|
||||||
rows.append(row)
|
rows.append(row)
|
||||||
inserted += 1
|
inserted += 1
|
||||||
|
indexed_ids.add(recipe_id)
|
||||||
|
|
||||||
if rows:
|
if rows:
|
||||||
# Insert into FTS table
|
# Insert into FTS table
|
||||||
@@ -213,7 +216,11 @@ class RecipeFTSIndex:
|
|||||||
)
|
)
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)",
|
"INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)",
|
||||||
('recipe_count', str(inserted))
|
(self._COUNT_METADATA_KEY, str(inserted))
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)",
|
||||||
|
(self._FINGERPRINT_METADATA_KEY, self._compute_ids_fingerprint(indexed_ids))
|
||||||
)
|
)
|
||||||
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
@@ -288,6 +295,12 @@ class RecipeFTSIndex:
|
|||||||
with self._lock:
|
with self._lock:
|
||||||
conn = self._connect()
|
conn = self._connect()
|
||||||
try:
|
try:
|
||||||
|
# Check existence via the rowid mapping (fast PK lookup)
|
||||||
|
existed = conn.execute(
|
||||||
|
"SELECT 1 FROM recipe_rowid WHERE recipe_id = ?",
|
||||||
|
(recipe_id,)
|
||||||
|
).fetchone() is not None
|
||||||
|
|
||||||
# Remove existing entry if present
|
# Remove existing entry if present
|
||||||
self._remove_recipe_locked(conn, recipe_id)
|
self._remove_recipe_locked(conn, recipe_id)
|
||||||
|
|
||||||
@@ -312,6 +325,10 @@ class RecipeFTSIndex:
|
|||||||
(recipe_id, result[0])
|
(recipe_id, result[0])
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Keep validation metadata in sync (only a new id changes it)
|
||||||
|
if not existed:
|
||||||
|
self._update_mutation_metadata_locked(conn, recipe_id, delta=1)
|
||||||
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
return True
|
return True
|
||||||
finally:
|
finally:
|
||||||
@@ -339,7 +356,13 @@ class RecipeFTSIndex:
|
|||||||
with self._lock:
|
with self._lock:
|
||||||
conn = self._connect()
|
conn = self._connect()
|
||||||
try:
|
try:
|
||||||
|
existed = conn.execute(
|
||||||
|
"SELECT 1 FROM recipe_rowid WHERE recipe_id = ?",
|
||||||
|
(recipe_id,)
|
||||||
|
).fetchone() is not None
|
||||||
self._remove_recipe_locked(conn, recipe_id)
|
self._remove_recipe_locked(conn, recipe_id)
|
||||||
|
if existed:
|
||||||
|
self._update_mutation_metadata_locked(conn, recipe_id, delta=-1)
|
||||||
conn.commit()
|
conn.commit()
|
||||||
return True
|
return True
|
||||||
finally:
|
finally:
|
||||||
@@ -371,6 +394,15 @@ class RecipeFTSIndex:
|
|||||||
try:
|
try:
|
||||||
conn.execute("DELETE FROM recipe_fts")
|
conn.execute("DELETE FROM recipe_fts")
|
||||||
conn.execute("DELETE FROM recipe_rowid")
|
conn.execute("DELETE FROM recipe_rowid")
|
||||||
|
# Reset validation metadata to the empty index state
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)",
|
||||||
|
(self._COUNT_METADATA_KEY, '0')
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)",
|
||||||
|
(self._FINGERPRINT_METADATA_KEY, self._compute_ids_fingerprint(set()))
|
||||||
|
)
|
||||||
conn.commit()
|
conn.commit()
|
||||||
self._ready.clear()
|
self._ready.clear()
|
||||||
return True
|
return True
|
||||||
@@ -427,10 +459,12 @@ class RecipeFTSIndex:
|
|||||||
"""Check if the FTS index matches the expected recipes.
|
"""Check if the FTS index matches the expected recipes.
|
||||||
|
|
||||||
This method validates whether the existing FTS index can be reused
|
This method validates whether the existing FTS index can be reused
|
||||||
without a full rebuild. It checks:
|
without a full rebuild. It compares the expected count and recipe ID
|
||||||
1. The index has been initialized
|
fingerprint against metadata recorded when the index was (re)built,
|
||||||
2. The count matches
|
so it does not scan the FTS content table. Indexes built by older
|
||||||
3. The recipe IDs match
|
versions lack this metadata; for those the validation falls back to
|
||||||
|
a one-time scan of the content table and records the metadata so
|
||||||
|
subsequent startups are cheap.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
recipe_count: Expected number of recipes.
|
recipe_count: Expected number of recipes.
|
||||||
@@ -446,7 +480,28 @@ class RecipeFTSIndex:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
metadata = self._read_validation_metadata()
|
||||||
|
if metadata is not None:
|
||||||
|
stored_count, stored_fingerprint = metadata
|
||||||
|
if stored_count != recipe_count:
|
||||||
|
logger.debug(
|
||||||
|
"FTS index count mismatch: indexed=%d, expected=%d",
|
||||||
|
stored_count, recipe_count
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
if stored_fingerprint != self._compute_ids_fingerprint(recipe_ids):
|
||||||
|
logger.debug("FTS index recipe ID fingerprint mismatch")
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Legacy fallback: no stored metadata, scan the content table once
|
||||||
|
# and persist the metadata so later validations are cheap.
|
||||||
indexed_count = self.get_indexed_count()
|
indexed_count = self.get_indexed_count()
|
||||||
|
indexed_ids = self.get_indexed_recipe_ids()
|
||||||
|
self._store_validation_metadata(indexed_count, indexed_ids)
|
||||||
|
|
||||||
if indexed_count != recipe_count:
|
if indexed_count != recipe_count:
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"FTS index count mismatch: indexed=%d, expected=%d",
|
"FTS index count mismatch: indexed=%d, expected=%d",
|
||||||
@@ -454,7 +509,6 @@ class RecipeFTSIndex:
|
|||||||
)
|
)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
indexed_ids = self.get_indexed_recipe_ids()
|
|
||||||
if indexed_ids != recipe_ids:
|
if indexed_ids != recipe_ids:
|
||||||
missing = recipe_ids - indexed_ids
|
missing = recipe_ids - indexed_ids
|
||||||
extra = indexed_ids - recipe_ids
|
extra = indexed_ids - recipe_ids
|
||||||
@@ -471,6 +525,112 @@ class RecipeFTSIndex:
|
|||||||
|
|
||||||
# Internal helpers
|
# Internal helpers
|
||||||
|
|
||||||
|
_FINGERPRINT_METADATA_KEY = 'recipe_ids_fingerprint'
|
||||||
|
_COUNT_METADATA_KEY = 'recipe_count'
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _fingerprint_recipe_id(recipe_id: str) -> int:
|
||||||
|
"""Return a stable 64-bit fingerprint contribution for a recipe ID."""
|
||||||
|
digest = hashlib.sha256(recipe_id.encode("utf-8")).digest()
|
||||||
|
return int.from_bytes(digest[:8], "big")
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _compute_ids_fingerprint(cls, recipe_ids: Set[str]) -> str:
|
||||||
|
"""Order-independent fingerprint of a recipe ID set (XOR of per-id hashes)."""
|
||||||
|
fingerprint = 0
|
||||||
|
for recipe_id in recipe_ids:
|
||||||
|
fingerprint ^= cls._fingerprint_recipe_id(str(recipe_id))
|
||||||
|
return f"{fingerprint:016x}"
|
||||||
|
|
||||||
|
def _read_validation_metadata(self) -> Optional[Tuple[int, str]]:
|
||||||
|
"""Return stored (recipe count, ID fingerprint), or None if absent."""
|
||||||
|
try:
|
||||||
|
with self._lock:
|
||||||
|
conn = self._connect(readonly=True)
|
||||||
|
try:
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT key, value FROM fts_metadata WHERE key IN (?, ?)",
|
||||||
|
(self._COUNT_METADATA_KEY, self._FINGERPRINT_METADATA_KEY)
|
||||||
|
).fetchall()
|
||||||
|
values = {row[0]: row[1] for row in rows}
|
||||||
|
fingerprint = values.get(self._FINGERPRINT_METADATA_KEY)
|
||||||
|
if fingerprint is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
count = int(values.get(self._COUNT_METADATA_KEY) or 0)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
return count, fingerprint
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
except FileNotFoundError:
|
||||||
|
return None
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("Failed to read FTS validation metadata: %s", exc)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _store_validation_metadata(self, recipe_count: int, recipe_ids: Set[str]) -> None:
|
||||||
|
"""Persist recipe count and ID fingerprint for cheap future validation."""
|
||||||
|
try:
|
||||||
|
with self._lock:
|
||||||
|
conn = self._connect()
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)",
|
||||||
|
(self._COUNT_METADATA_KEY, str(recipe_count))
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)",
|
||||||
|
(self._FINGERPRINT_METADATA_KEY, self._compute_ids_fingerprint(recipe_ids))
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("Failed to store FTS validation metadata: %s", exc)
|
||||||
|
|
||||||
|
def _update_mutation_metadata_locked(
|
||||||
|
self,
|
||||||
|
conn: sqlite3.Connection,
|
||||||
|
recipe_id: str,
|
||||||
|
delta: int,
|
||||||
|
) -> None:
|
||||||
|
"""Incrementally maintain validation metadata after add/remove.
|
||||||
|
|
||||||
|
Caller must hold the lock. The fingerprint is only updated when it
|
||||||
|
already exists; without it, validation falls back to a one-time scan
|
||||||
|
that records fresh metadata.
|
||||||
|
"""
|
||||||
|
fingerprint_row = conn.execute(
|
||||||
|
"SELECT value FROM fts_metadata WHERE key = ?",
|
||||||
|
(self._FINGERPRINT_METADATA_KEY,)
|
||||||
|
).fetchone()
|
||||||
|
if fingerprint_row and fingerprint_row[0]:
|
||||||
|
try:
|
||||||
|
fingerprint = int(fingerprint_row[0], 16)
|
||||||
|
except ValueError:
|
||||||
|
fingerprint = None
|
||||||
|
if fingerprint is not None:
|
||||||
|
fingerprint ^= self._fingerprint_recipe_id(recipe_id)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)",
|
||||||
|
(self._FINGERPRINT_METADATA_KEY, f"{fingerprint & 0xFFFFFFFFFFFFFFFF:016x}")
|
||||||
|
)
|
||||||
|
|
||||||
|
count_row = conn.execute(
|
||||||
|
"SELECT value FROM fts_metadata WHERE key = ?",
|
||||||
|
(self._COUNT_METADATA_KEY,)
|
||||||
|
).fetchone()
|
||||||
|
if count_row:
|
||||||
|
try:
|
||||||
|
count = max(0, int(count_row[0] or 0) + delta)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR REPLACE INTO fts_metadata (key, value) VALUES (?, ?)",
|
||||||
|
(self._COUNT_METADATA_KEY, str(count))
|
||||||
|
)
|
||||||
|
|
||||||
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
||||||
"""Create a database connection."""
|
"""Create a database connection."""
|
||||||
uri = False
|
uri = False
|
||||||
|
|||||||
+1126
-290
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,7 @@
|
|||||||
"""Recipe service layer implementations."""
|
"""Recipe service layer implementations."""
|
||||||
|
|
||||||
from .analysis_service import RecipeAnalysisService
|
from .analysis_service import RecipeAnalysisService
|
||||||
|
from .import_info import build_import_info, compute_no_loras_reason
|
||||||
from .persistence_service import RecipePersistenceService
|
from .persistence_service import RecipePersistenceService
|
||||||
from .sharing_service import RecipeSharingService
|
from .sharing_service import RecipeSharingService
|
||||||
from .errors import (
|
from .errors import (
|
||||||
@@ -15,6 +16,8 @@ __all__ = [
|
|||||||
"RecipeAnalysisService",
|
"RecipeAnalysisService",
|
||||||
"RecipePersistenceService",
|
"RecipePersistenceService",
|
||||||
"RecipeSharingService",
|
"RecipeSharingService",
|
||||||
|
"build_import_info",
|
||||||
|
"compute_no_loras_reason",
|
||||||
"RecipeServiceError",
|
"RecipeServiceError",
|
||||||
"RecipeValidationError",
|
"RecipeValidationError",
|
||||||
"RecipeNotFoundError",
|
"RecipeNotFoundError",
|
||||||
|
|||||||
@@ -72,15 +72,28 @@ class RecipeAnalysisService:
|
|||||||
metadata = self._exif_utils.extract_image_metadata(temp_path)
|
metadata = self._exif_utils.extract_image_metadata(temp_path)
|
||||||
if not metadata:
|
if not metadata:
|
||||||
return AnalysisResult(
|
return AnalysisResult(
|
||||||
{"error": "No metadata found in this image", "loras": []}
|
{
|
||||||
|
"error": "No metadata found in this image",
|
||||||
|
"loras": [],
|
||||||
|
"diagnostics": {
|
||||||
|
"channel": "upload",
|
||||||
|
"exif_present": False,
|
||||||
|
},
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
return await self._parse_metadata(
|
result = await self._parse_metadata(
|
||||||
metadata,
|
metadata,
|
||||||
recipe_scanner=recipe_scanner,
|
recipe_scanner=recipe_scanner,
|
||||||
image_path=None,
|
image_path=None,
|
||||||
include_image_base64=False,
|
include_image_base64=False,
|
||||||
)
|
)
|
||||||
|
result.payload["diagnostics"] = {
|
||||||
|
"channel": "upload",
|
||||||
|
"exif_present": True,
|
||||||
|
"exif_parser": result.payload.get("parser"),
|
||||||
|
}
|
||||||
|
return result
|
||||||
finally:
|
finally:
|
||||||
self._safe_cleanup(temp_path)
|
self._safe_cleanup(temp_path)
|
||||||
|
|
||||||
@@ -104,9 +117,13 @@ class RecipeAnalysisService:
|
|||||||
image_info: Optional[dict[str, Any]] = None
|
image_info: Optional[dict[str, Any]] = None
|
||||||
is_video = False
|
is_video = False
|
||||||
extension = ".jpg" # Default
|
extension = ".jpg" # Default
|
||||||
|
# Diagnostics collected during analysis; surfaced in the payload so
|
||||||
|
# callers can persist an import_info block explaining empty LoRA lists.
|
||||||
|
diagnostics: dict[str, Any] = {"channel": "url"}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
civitai_image_id = extract_civitai_image_id(url)
|
civitai_image_id = extract_civitai_image_id(url)
|
||||||
|
diagnostics["civitai_image"] = bool(civitai_image_id)
|
||||||
if civitai_image_id:
|
if civitai_image_id:
|
||||||
image_info = await civitai_client.get_image_info(
|
image_info = await civitai_client.get_image_info(
|
||||||
civitai_image_id, source_url=url
|
civitai_image_id, source_url=url
|
||||||
@@ -147,11 +164,23 @@ class RecipeAnalysisService:
|
|||||||
):
|
):
|
||||||
metadata = metadata["meta"]
|
metadata = metadata["meta"]
|
||||||
|
|
||||||
|
# Diagnostics: capture the API meta shape before injecting
|
||||||
|
# modelVersionIds / browsingLevel so the recipe modal can
|
||||||
|
# explain why an import ended up without LoRAs.
|
||||||
|
diagnostics["api_meta_present"] = isinstance(metadata, dict)
|
||||||
|
if isinstance(metadata, dict):
|
||||||
|
diagnostics["api_meta_keys"] = sorted(metadata.keys())
|
||||||
|
|
||||||
# Include modelVersionIds from root level if available.
|
# Include modelVersionIds from root level if available.
|
||||||
# CivitAI API returns modelVersionIds at root level, not in meta.
|
# CivitAI API returns modelVersionIds at root level, not in meta.
|
||||||
# When meta is null (None), create a minimal dict so downstream
|
# When meta is null (None), create a minimal dict so downstream
|
||||||
# parsers can still discover LoRAs and checkpoints.
|
# parsers can still discover LoRAs and checkpoints.
|
||||||
model_version_ids = image_info.get("modelVersionIds")
|
model_version_ids = image_info.get("modelVersionIds")
|
||||||
|
diagnostics["api_model_version_ids"] = (
|
||||||
|
len(model_version_ids)
|
||||||
|
if isinstance(model_version_ids, list)
|
||||||
|
else 0
|
||||||
|
)
|
||||||
if model_version_ids:
|
if model_version_ids:
|
||||||
if isinstance(metadata, dict):
|
if isinstance(metadata, dict):
|
||||||
metadata["modelVersionIds"] = model_version_ids
|
metadata["modelVersionIds"] = model_version_ids
|
||||||
@@ -229,6 +258,8 @@ class RecipeAnalysisService:
|
|||||||
finally:
|
finally:
|
||||||
self._safe_cleanup(orig_temp_path)
|
self._safe_cleanup(orig_temp_path)
|
||||||
|
|
||||||
|
diagnostics["exif_present"] = bool(exif_metadata)
|
||||||
|
|
||||||
# Parse EXIF data (typically a string like parameters/prompt/workflow)
|
# Parse EXIF data (typically a string like parameters/prompt/workflow)
|
||||||
# and API metadata (dict with modelVersionIds, browsingLevel) separately,
|
# and API metadata (dict with modelVersionIds, browsingLevel) separately,
|
||||||
# then merge: API loras/checkpoint override, EXIF gen_params fill in gaps.
|
# then merge: API loras/checkpoint override, EXIF gen_params fill in gaps.
|
||||||
@@ -237,6 +268,7 @@ class RecipeAnalysisService:
|
|||||||
if isinstance(exif_metadata, str):
|
if isinstance(exif_metadata, str):
|
||||||
exif_parser = self._recipe_parser_factory.create_parser(exif_metadata)
|
exif_parser = self._recipe_parser_factory.create_parser(exif_metadata)
|
||||||
if exif_parser:
|
if exif_parser:
|
||||||
|
diagnostics["exif_parser"] = exif_parser.__class__.__name__
|
||||||
exif_data = await exif_parser.parse_metadata(
|
exif_data = await exif_parser.parse_metadata(
|
||||||
exif_metadata, recipe_scanner=recipe_scanner,
|
exif_metadata, recipe_scanner=recipe_scanner,
|
||||||
)
|
)
|
||||||
@@ -270,6 +302,22 @@ class RecipeAnalysisService:
|
|||||||
if merged_gp:
|
if merged_gp:
|
||||||
result.payload["gen_params"] = merged_gp
|
result.payload["gen_params"] = merged_gp
|
||||||
|
|
||||||
|
# The API-only parse (meta=null with only modelVersionIds)
|
||||||
|
# yields a checkpoint but no LoRAs; the image EXIF carries the
|
||||||
|
# full resource list. Fill the gaps the API parse left open.
|
||||||
|
if not result.payload.get("loras"):
|
||||||
|
exif_loras = exif_parsed_result.get("loras") or []
|
||||||
|
if exif_loras:
|
||||||
|
result.payload["loras"] = exif_loras
|
||||||
|
if not result.payload.get("checkpoint") and not result.payload.get("model"):
|
||||||
|
exif_checkpoint = exif_parsed_result.get("model") or exif_parsed_result.get(
|
||||||
|
"checkpoint"
|
||||||
|
)
|
||||||
|
if exif_checkpoint:
|
||||||
|
result.payload["checkpoint"] = exif_checkpoint
|
||||||
|
if not result.payload.get("base_model") and exif_parsed_result.get("base_model"):
|
||||||
|
result.payload["base_model"] = exif_parsed_result["base_model"]
|
||||||
|
|
||||||
if civitai_image_id and image_info and not result.payload.get("error"):
|
if civitai_image_id and image_info and not result.payload.get("error"):
|
||||||
# Use the metadata dict we built (may contain modelVersionIds
|
# Use the metadata dict we built (may contain modelVersionIds
|
||||||
# and browsingLevel from the API root level). Do NOT pass
|
# and browsingLevel from the API root level). Do NOT pass
|
||||||
@@ -308,6 +356,8 @@ class RecipeAnalysisService:
|
|||||||
if isinstance(bl, int) and bl > 0:
|
if isinstance(bl, int) and bl > 0:
|
||||||
result.payload["preview_nsfw_level"] = bl
|
result.payload["preview_nsfw_level"] = bl
|
||||||
|
|
||||||
|
diagnostics["is_video"] = is_video
|
||||||
|
result.payload["diagnostics"] = diagnostics
|
||||||
return result
|
return result
|
||||||
finally:
|
finally:
|
||||||
if temp_path:
|
if temp_path:
|
||||||
@@ -318,6 +368,7 @@ class RecipeAnalysisService:
|
|||||||
*,
|
*,
|
||||||
file_path: str | None,
|
file_path: str | None,
|
||||||
recipe_scanner,
|
recipe_scanner,
|
||||||
|
ignore_recipe_metadata: bool = False,
|
||||||
) -> AnalysisResult:
|
) -> AnalysisResult:
|
||||||
"""Analyze a file already present on disk."""
|
"""Analyze a file already present on disk."""
|
||||||
|
|
||||||
@@ -332,14 +383,41 @@ class RecipeAnalysisService:
|
|||||||
self._exif_utils.extract_image_metadata, normalized_path
|
self._exif_utils.extract_image_metadata, normalized_path
|
||||||
)
|
)
|
||||||
if not metadata:
|
if not metadata:
|
||||||
return self._metadata_not_found_response(normalized_path)
|
result = self._metadata_not_found_response(normalized_path)
|
||||||
|
result.payload["diagnostics"] = {
|
||||||
|
"channel": "local",
|
||||||
|
"exif_present": False,
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
|
||||||
return await self._parse_metadata(
|
if ignore_recipe_metadata:
|
||||||
|
# Re-import: re-parse the original embedded generation metadata
|
||||||
|
# instead of the recipe JSON block LoRA Manager appended on save.
|
||||||
|
from ...recipes.parsers.recipe_format import strip_recipe_metadata
|
||||||
|
|
||||||
|
metadata = strip_recipe_metadata(metadata)
|
||||||
|
if not metadata:
|
||||||
|
result = self._metadata_not_found_response(normalized_path)
|
||||||
|
result.payload["diagnostics"] = {
|
||||||
|
"channel": "local",
|
||||||
|
"exif_present": True,
|
||||||
|
"ignore_recipe_metadata": True,
|
||||||
|
"reason": "only_recipe_metadata",
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
|
||||||
|
result = await self._parse_metadata(
|
||||||
metadata,
|
metadata,
|
||||||
recipe_scanner=recipe_scanner,
|
recipe_scanner=recipe_scanner,
|
||||||
image_path=normalized_path,
|
image_path=normalized_path,
|
||||||
include_image_base64=True,
|
include_image_base64=True,
|
||||||
)
|
)
|
||||||
|
result.payload["diagnostics"] = {
|
||||||
|
"channel": "local",
|
||||||
|
"exif_present": True,
|
||||||
|
"exif_parser": result.payload.get("parser"),
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
|
||||||
async def analyze_widget_metadata(self, *, recipe_scanner) -> AnalysisResult:
|
async def analyze_widget_metadata(self, *, recipe_scanner) -> AnalysisResult:
|
||||||
"""Analyse the most recent generation metadata for widget saves."""
|
"""Analyse the most recent generation metadata for widget saves."""
|
||||||
@@ -436,6 +514,10 @@ class RecipeAnalysisService:
|
|||||||
metadata, recipe_scanner=recipe_scanner
|
metadata, recipe_scanner=recipe_scanner
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Record which parser handled the metadata so import diagnostics
|
||||||
|
# can distinguish e.g. ComfyUI workflow sources.
|
||||||
|
result["parser"] = parser.__class__.__name__
|
||||||
|
|
||||||
if include_image_base64 and image_path:
|
if include_image_base64 and image_path:
|
||||||
result["image_base64"] = self._encode_file(image_path)
|
result["image_base64"] = self._encode_file(image_path)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,129 @@
|
|||||||
|
"""Import provenance helpers for recipes.
|
||||||
|
|
||||||
|
Builds the ``import_info`` block persisted on a recipe: the import channel
|
||||||
|
(batch import / single URL / local file / upload / widget) and, when the
|
||||||
|
recipe ended up with no LoRAs, a machine-readable reason plus the diagnostic
|
||||||
|
details that led to it. The recipe modal renders this block in a collapsed
|
||||||
|
"Why no LoRAs?" panel; legacy recipes without ``import_info`` fall back to a
|
||||||
|
frontend heuristic.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
# Import channels (how the recipe entered the library).
|
||||||
|
CHANNEL_BATCH_IMPORT_URL = "batch_import_url"
|
||||||
|
CHANNEL_BATCH_IMPORT_LOCAL = "batch_import_local"
|
||||||
|
CHANNEL_URL = "url"
|
||||||
|
CHANNEL_LOCAL = "local"
|
||||||
|
CHANNEL_UPLOAD = "upload"
|
||||||
|
CHANNEL_WIDGET = "widget"
|
||||||
|
CHANNEL_REIMPORT_URL = "reimport_url"
|
||||||
|
CHANNEL_REIMPORT_LOCAL = "reimport_local"
|
||||||
|
|
||||||
|
_URL_CHANNELS = frozenset(
|
||||||
|
{CHANNEL_BATCH_IMPORT_URL, CHANNEL_URL, CHANNEL_REIMPORT_URL}
|
||||||
|
)
|
||||||
|
|
||||||
|
# No-LoRA reason codes (persisted, consumed by the recipe modal).
|
||||||
|
REASON_NO_LORAS_USED = "no_loras_used"
|
||||||
|
REASON_API_NO_LORA_RESOURCES = "api_meta_no_lora_resources"
|
||||||
|
REASON_API_META_MISSING = "api_meta_missing"
|
||||||
|
REASON_NO_EMBEDDED_METADATA = "no_embedded_metadata"
|
||||||
|
REASON_WORKFLOW_METADATA_LIMITED = "workflow_metadata_limited"
|
||||||
|
REASON_VIDEO_NO_METADATA = "video_no_metadata"
|
||||||
|
REASON_METADATA_UNSUPPORTED = "metadata_unsupported"
|
||||||
|
REASON_UNKNOWN = "unknown"
|
||||||
|
|
||||||
|
_COMFY_PARSER_NAME = "ComfyMetadataParser"
|
||||||
|
|
||||||
|
# Cap for api_meta_keys kept in details — enough for the UI bullet without
|
||||||
|
# bloating the recipe JSON.
|
||||||
|
_MAX_DETAIL_KEYS = 12
|
||||||
|
|
||||||
|
|
||||||
|
def compute_no_loras_reason(
|
||||||
|
channel: str, diagnostics: Optional[Dict[str, Any]]
|
||||||
|
) -> str:
|
||||||
|
"""Classify why an import produced no LoRA entries.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
channel: One of the CHANNEL_* constants.
|
||||||
|
diagnostics: Signals collected during analysis (see
|
||||||
|
``RecipeAnalysisService``), or None for channels without analysis
|
||||||
|
(e.g. widget saves).
|
||||||
|
"""
|
||||||
|
diag = diagnostics or {}
|
||||||
|
|
||||||
|
if diag.get("is_video"):
|
||||||
|
return REASON_VIDEO_NO_METADATA
|
||||||
|
|
||||||
|
# Embedded metadata that is a ComfyUI workflow: LoRA extraction from
|
||||||
|
# workflows is limited, so report that specifically.
|
||||||
|
parser = diag.get("exif_parser") or diag.get("parser")
|
||||||
|
if parser == _COMFY_PARSER_NAME:
|
||||||
|
return REASON_WORKFLOW_METADATA_LIMITED
|
||||||
|
|
||||||
|
if channel in _URL_CHANNELS:
|
||||||
|
if not diag.get("civitai_image"):
|
||||||
|
# Generic (non-CivitAI) URL: only embedded metadata is available.
|
||||||
|
if not diag.get("exif_present"):
|
||||||
|
return REASON_NO_EMBEDDED_METADATA
|
||||||
|
return (
|
||||||
|
REASON_NO_LORAS_USED if parser else REASON_METADATA_UNSUPPORTED
|
||||||
|
)
|
||||||
|
# NOTE: no "parsed EXIF means no LoRAs were used" shortcut here.
|
||||||
|
# CivitAI's onsite generator writes A1111-style EXIF (prompt, seed,
|
||||||
|
# steps, ...) WITHOUT LoRA references — LoRA usage lives only in
|
||||||
|
# CivitAI-internal data — so cleanly parsed EXIF cannot prove the
|
||||||
|
# generation used no LoRAs. Report the API meta shape instead.
|
||||||
|
api_keys = diag.get("api_meta_keys") or []
|
||||||
|
api_mvids = diag.get("api_model_version_ids") or 0
|
||||||
|
if api_keys or api_mvids:
|
||||||
|
return REASON_API_NO_LORA_RESOURCES
|
||||||
|
return REASON_API_META_MISSING
|
||||||
|
|
||||||
|
if channel == CHANNEL_WIDGET:
|
||||||
|
return REASON_NO_LORAS_USED
|
||||||
|
|
||||||
|
# Local file / upload / local re-import: embedded metadata only.
|
||||||
|
if not diag.get("exif_present"):
|
||||||
|
return REASON_NO_EMBEDDED_METADATA
|
||||||
|
return REASON_NO_LORAS_USED if parser else REASON_METADATA_UNSUPPORTED
|
||||||
|
|
||||||
|
|
||||||
|
def build_import_info(
|
||||||
|
channel: str,
|
||||||
|
diagnostics: Optional[Dict[str, Any]],
|
||||||
|
loras: Optional[List[Dict[str, Any]]],
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Build the ``import_info`` block persisted on a recipe.
|
||||||
|
|
||||||
|
Always records the import channel; adds ``reason`` and ``details`` only
|
||||||
|
when the recipe has no LoRAs.
|
||||||
|
"""
|
||||||
|
info: Dict[str, Any] = {"channel": channel}
|
||||||
|
if loras:
|
||||||
|
return info
|
||||||
|
|
||||||
|
info["reason"] = compute_no_loras_reason(channel, diagnostics)
|
||||||
|
|
||||||
|
diag = diagnostics or {}
|
||||||
|
details: Dict[str, Any] = {}
|
||||||
|
api_keys = diag.get("api_meta_keys")
|
||||||
|
if api_keys:
|
||||||
|
details["api_meta_keys"] = list(api_keys)[:_MAX_DETAIL_KEYS]
|
||||||
|
api_mvids = diag.get("api_model_version_ids")
|
||||||
|
if api_mvids is not None:
|
||||||
|
details["api_model_version_ids"] = api_mvids
|
||||||
|
if "exif_present" in diag:
|
||||||
|
details["exif_present"] = bool(diag.get("exif_present"))
|
||||||
|
if diag.get("exif_parser"):
|
||||||
|
details["exif_parser"] = diag["exif_parser"]
|
||||||
|
if diag.get("is_video"):
|
||||||
|
details["is_video"] = True
|
||||||
|
if details:
|
||||||
|
info["details"] = details
|
||||||
|
|
||||||
|
return info
|
||||||
@@ -13,9 +13,15 @@ from typing import Any, Awaitable, Dict, Iterable, Optional, cast
|
|||||||
|
|
||||||
from ...config import config
|
from ...config import config
|
||||||
from ...recipes.constants import GEN_PARAM_KEYS
|
from ...recipes.constants import GEN_PARAM_KEYS
|
||||||
|
from ...utils.base_model import (
|
||||||
|
RELATION_COMPATIBLE,
|
||||||
|
RELATION_INCOMPATIBLE,
|
||||||
|
base_model_relation,
|
||||||
|
)
|
||||||
from ...utils.utils import calculate_recipe_fingerprint
|
from ...utils.utils import calculate_recipe_fingerprint
|
||||||
from ..pending_delete_service import get_pending_delete_service
|
from ..pending_delete_service import get_pending_delete_service
|
||||||
from .errors import RecipeNotFoundError, RecipeValidationError
|
from .errors import RecipeNotFoundError, RecipeValidationError
|
||||||
|
from .import_info import CHANNEL_UPLOAD, CHANNEL_WIDGET, build_import_info
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -52,6 +58,7 @@ class RecipePersistenceService:
|
|||||||
extension: str | None = None,
|
extension: str | None = None,
|
||||||
recipe_id: str | None = None,
|
recipe_id: str | None = None,
|
||||||
target_dir: str | None = None,
|
target_dir: str | None = None,
|
||||||
|
skip_optimize: bool = False,
|
||||||
) -> PersistenceResult:
|
) -> PersistenceResult:
|
||||||
"""Persist a user uploaded recipe.
|
"""Persist a user uploaded recipe.
|
||||||
|
|
||||||
@@ -61,6 +68,11 @@ class RecipePersistenceService:
|
|||||||
target_dir: If provided, save recipe files to this directory instead
|
target_dir: If provided, save recipe files to this directory instead
|
||||||
of the default recipes_dir. Used by re-import to preserve the
|
of the default recipes_dir. Used by re-import to preserve the
|
||||||
original folder location.
|
original folder location.
|
||||||
|
skip_optimize: If True, store the image bytes verbatim without
|
||||||
|
resizing/re-encoding (recipe metadata is still embedded via a
|
||||||
|
byte-level EXIF update that leaves the pixels untouched). Used
|
||||||
|
by local re-import, where the source is the recipe's own
|
||||||
|
already-optimized preview image.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
missing_fields = []
|
missing_fields = []
|
||||||
@@ -81,9 +93,12 @@ class RecipePersistenceService:
|
|||||||
|
|
||||||
recipe_id = recipe_id or str(uuid.uuid4())
|
recipe_id = recipe_id or str(uuid.uuid4())
|
||||||
|
|
||||||
# Handle video formats by bypassing optimization and metadata embedding
|
# Handle video formats by bypassing optimization and metadata embedding.
|
||||||
|
# Local re-import also bypasses optimization: the source is the
|
||||||
|
# recipe's own already-optimized preview image, so re-compressing it
|
||||||
|
# would only degrade quality.
|
||||||
is_video = extension in [".mp4", ".webm"]
|
is_video = extension in [".mp4", ".webm"]
|
||||||
if is_video:
|
if is_video or skip_optimize:
|
||||||
optimized_image = resolved_image_bytes
|
optimized_image = resolved_image_bytes
|
||||||
# extension is already set
|
# extension is already set
|
||||||
else:
|
else:
|
||||||
@@ -117,6 +132,7 @@ class RecipePersistenceService:
|
|||||||
"loras": loras_data,
|
"loras": loras_data,
|
||||||
"gen_params": gen_params,
|
"gen_params": gen_params,
|
||||||
"fingerprint": fingerprint,
|
"fingerprint": fingerprint,
|
||||||
|
"has_workflow": self._detect_has_workflow(normalized_image_path),
|
||||||
}
|
}
|
||||||
if checkpoint_entry:
|
if checkpoint_entry:
|
||||||
recipe_data["checkpoint"] = checkpoint_entry
|
recipe_data["checkpoint"] = checkpoint_entry
|
||||||
@@ -128,6 +144,22 @@ class RecipePersistenceService:
|
|||||||
if metadata.get("source_path"):
|
if metadata.get("source_path"):
|
||||||
recipe_data["source_path"] = metadata.get("source_path")
|
recipe_data["source_path"] = metadata.get("source_path")
|
||||||
|
|
||||||
|
# Persist import provenance. Batch import / re-import paths pass a
|
||||||
|
# prebuilt import_info; frontend-driven saves (upload, single URL,
|
||||||
|
# local path) carry the analysis payload's diagnostics, from which
|
||||||
|
# import_info is derived here.
|
||||||
|
import_info = metadata.get("import_info")
|
||||||
|
if not isinstance(import_info, dict):
|
||||||
|
diagnostics = metadata.get("diagnostics")
|
||||||
|
if isinstance(diagnostics, dict):
|
||||||
|
import_info = build_import_info(
|
||||||
|
diagnostics.get("channel") or CHANNEL_UPLOAD,
|
||||||
|
diagnostics,
|
||||||
|
loras_data,
|
||||||
|
)
|
||||||
|
if isinstance(import_info, dict) and import_info:
|
||||||
|
recipe_data["import_info"] = import_info
|
||||||
|
|
||||||
nsfw_level = metadata.get("preview_nsfw_level")
|
nsfw_level = metadata.get("preview_nsfw_level")
|
||||||
if nsfw_level is not None and isinstance(nsfw_level, int):
|
if nsfw_level is not None and isinstance(nsfw_level, int):
|
||||||
recipe_data["preview_nsfw_level"] = nsfw_level
|
recipe_data["preview_nsfw_level"] = nsfw_level
|
||||||
@@ -152,7 +184,11 @@ class RecipePersistenceService:
|
|||||||
json.dump(recipe_data, file_obj, indent=4, ensure_ascii=False)
|
json.dump(recipe_data, file_obj, indent=4, ensure_ascii=False)
|
||||||
|
|
||||||
if not is_video:
|
if not is_video:
|
||||||
self._exif_utils.append_recipe_metadata(normalized_image_path, recipe_data)
|
self._exif_utils.append_recipe_metadata(
|
||||||
|
normalized_image_path,
|
||||||
|
recipe_data,
|
||||||
|
pixel_preserving=skip_optimize,
|
||||||
|
)
|
||||||
|
|
||||||
matching_recipes = await self._find_matching_recipes(recipe_scanner, fingerprint, exclude_id=recipe_id)
|
matching_recipes = await self._find_matching_recipes(recipe_scanner, fingerprint, exclude_id=recipe_id)
|
||||||
await recipe_scanner.add_recipe(recipe_data)
|
await recipe_scanner.add_recipe(recipe_data)
|
||||||
@@ -426,10 +462,34 @@ class RecipePersistenceService:
|
|||||||
if not recipe_path or not os.path.exists(recipe_path):
|
if not recipe_path or not os.path.exists(recipe_path):
|
||||||
raise RecipeNotFoundError("Recipe not found")
|
raise RecipeNotFoundError("Recipe not found")
|
||||||
|
|
||||||
target_lora = await recipe_scanner.get_local_lora(target_name)
|
with open(recipe_path, "r", encoding="utf-8") as file_obj:
|
||||||
if not target_lora:
|
recipe_base_model = json.load(file_obj).get("base_model", "")
|
||||||
|
|
||||||
|
matches = await recipe_scanner.find_local_loras_by_name(target_name)
|
||||||
|
if not matches:
|
||||||
raise RecipeNotFoundError(f"Local LoRA not found with name: {target_name}")
|
raise RecipeNotFoundError(f"Local LoRA not found with name: {target_name}")
|
||||||
|
|
||||||
|
# Three-tier base-model guard: exact/unknown labels pass silently;
|
||||||
|
# labels from the same architecture family (e.g. Pony ↔ Illustrious)
|
||||||
|
# pass but are reported so the UI can warn; confident architecture
|
||||||
|
# mismatches stay hard-rejected because they can never load.
|
||||||
|
eligible: list[tuple[dict, str]] = []
|
||||||
|
for match in matches:
|
||||||
|
relation = base_model_relation(recipe_base_model, match.get("base_model"))
|
||||||
|
if relation != RELATION_INCOMPATIBLE:
|
||||||
|
eligible.append((match, relation))
|
||||||
|
|
||||||
|
if not eligible:
|
||||||
|
raise RecipeValidationError(
|
||||||
|
f"Local LoRA '{target_name}' has a different base model than the recipe"
|
||||||
|
)
|
||||||
|
if len(eligible) > 1:
|
||||||
|
raise RecipeValidationError(
|
||||||
|
f"Multiple local LoRAs match '{target_name}'; "
|
||||||
|
"include the folder path to disambiguate"
|
||||||
|
)
|
||||||
|
target_lora, target_relation = eligible[0]
|
||||||
|
|
||||||
recipe_data, updated_lora = await recipe_scanner.update_lora_entry(
|
recipe_data, updated_lora = await recipe_scanner.update_lora_entry(
|
||||||
recipe_id,
|
recipe_id,
|
||||||
lora_index,
|
lora_index,
|
||||||
@@ -437,6 +497,43 @@ class RecipePersistenceService:
|
|||||||
target_lora=target_lora,
|
target_lora=target_lora,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
image_path = recipe_data.get("file_path")
|
||||||
|
if image_path and os.path.exists(image_path):
|
||||||
|
self._exif_utils.append_recipe_metadata(image_path, recipe_data)
|
||||||
|
|
||||||
|
matching_recipes = []
|
||||||
|
if "fingerprint" in recipe_data:
|
||||||
|
matching_recipes = await recipe_scanner.find_recipes_by_fingerprint(recipe_data["fingerprint"])
|
||||||
|
if recipe_id in matching_recipes:
|
||||||
|
matching_recipes.remove(recipe_id)
|
||||||
|
|
||||||
|
payload: dict[str, Any] = {
|
||||||
|
"success": True,
|
||||||
|
"recipe_id": recipe_id,
|
||||||
|
"updated_lora": updated_lora,
|
||||||
|
"matching_recipes": matching_recipes,
|
||||||
|
}
|
||||||
|
if target_relation == RELATION_COMPATIBLE:
|
||||||
|
# Structured data, not prose — the frontend localizes the warning.
|
||||||
|
payload["base_model_mismatch"] = {
|
||||||
|
"recipe_base_model": recipe_base_model,
|
||||||
|
"lora_base_model": target_lora.get("base_model") or "",
|
||||||
|
}
|
||||||
|
return PersistenceResult(payload)
|
||||||
|
|
||||||
|
async def restore_lora(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
recipe_scanner,
|
||||||
|
recipe_id: str,
|
||||||
|
lora_index: int,
|
||||||
|
) -> PersistenceResult:
|
||||||
|
"""Restore a LoRA entry to the state captured before its reconnect."""
|
||||||
|
|
||||||
|
recipe_data, updated_lora = await recipe_scanner.restore_lora_entry(
|
||||||
|
recipe_id, lora_index
|
||||||
|
)
|
||||||
|
|
||||||
image_path = recipe_data.get("file_path")
|
image_path = recipe_data.get("file_path")
|
||||||
if image_path and os.path.exists(image_path):
|
if image_path and os.path.exists(image_path):
|
||||||
self._exif_utils.append_recipe_metadata(image_path, recipe_data)
|
self._exif_utils.append_recipe_metadata(image_path, recipe_data)
|
||||||
@@ -456,6 +553,231 @@ class RecipePersistenceService:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
async def get_reconnect_suggestions(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
recipe_scanner,
|
||||||
|
recipe_id: str,
|
||||||
|
lora_index: int,
|
||||||
|
query: str | None = None,
|
||||||
|
) -> PersistenceResult:
|
||||||
|
"""Return ranked local LoRA candidates for reconnecting a recipe entry."""
|
||||||
|
|
||||||
|
recipe_path = await recipe_scanner.get_recipe_json_path(recipe_id)
|
||||||
|
if not recipe_path or not os.path.exists(recipe_path):
|
||||||
|
raise RecipeNotFoundError("Recipe not found")
|
||||||
|
|
||||||
|
with open(recipe_path, "r", encoding="utf-8") as file_obj:
|
||||||
|
recipe_data = json.load(file_obj)
|
||||||
|
|
||||||
|
loras = recipe_data.get("loras") or []
|
||||||
|
if lora_index < 0 or lora_index >= len(loras):
|
||||||
|
raise RecipeValidationError(f"Invalid lora_index: {lora_index}")
|
||||||
|
|
||||||
|
suggestions = await recipe_scanner.suggest_reconnect_candidates(
|
||||||
|
entry=loras[lora_index],
|
||||||
|
recipe_base_model=recipe_data.get("base_model"),
|
||||||
|
query=query,
|
||||||
|
)
|
||||||
|
|
||||||
|
return PersistenceResult({"success": True, "suggestions": suggestions})
|
||||||
|
|
||||||
|
async def mark_lora_hash_invalid(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
recipe_scanner,
|
||||||
|
recipe_id: str,
|
||||||
|
lora_index: int,
|
||||||
|
hash_invalid: bool = True,
|
||||||
|
) -> PersistenceResult:
|
||||||
|
"""Mark a recipe LoRA entry's hash as unresolvable on CivitAI.
|
||||||
|
|
||||||
|
Called when a download attempt by hash returned "Model not found".
|
||||||
|
The flag makes the entry an unresolved rematch candidate without
|
||||||
|
altering its stored hash/file_name.
|
||||||
|
"""
|
||||||
|
|
||||||
|
recipe_data, updated_lora = await recipe_scanner.set_lora_entry_hash_invalid(
|
||||||
|
recipe_id,
|
||||||
|
lora_index,
|
||||||
|
hash_invalid=hash_invalid,
|
||||||
|
)
|
||||||
|
|
||||||
|
return PersistenceResult(
|
||||||
|
{
|
||||||
|
"success": True,
|
||||||
|
"recipe_id": recipe_id,
|
||||||
|
"hash_invalid": bool(hash_invalid),
|
||||||
|
"updated_lora": updated_lora,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
async def reconnect_checkpoint(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
recipe_scanner,
|
||||||
|
recipe_id: str,
|
||||||
|
target_name: str,
|
||||||
|
) -> PersistenceResult:
|
||||||
|
"""Reconnect the checkpoint entry within an existing recipe."""
|
||||||
|
|
||||||
|
recipe_path = await recipe_scanner.get_recipe_json_path(recipe_id)
|
||||||
|
if not recipe_path or not os.path.exists(recipe_path):
|
||||||
|
raise RecipeNotFoundError("Recipe not found")
|
||||||
|
|
||||||
|
with open(recipe_path, "r", encoding="utf-8") as file_obj:
|
||||||
|
recipe_base_model = json.load(file_obj).get("base_model", "")
|
||||||
|
|
||||||
|
matches = await recipe_scanner.find_local_checkpoints_by_name(target_name)
|
||||||
|
if not matches:
|
||||||
|
raise RecipeNotFoundError(
|
||||||
|
f"Local checkpoint not found with name: {target_name}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Same three-tier base-model guard as reconnect_lora: exact/unknown
|
||||||
|
# labels pass silently; same-architecture-family labels pass but are
|
||||||
|
# reported so the UI can warn; confident mismatches stay hard-rejected.
|
||||||
|
eligible: list[tuple[dict, str]] = []
|
||||||
|
for match in matches:
|
||||||
|
relation = base_model_relation(recipe_base_model, match.get("base_model"))
|
||||||
|
if relation != RELATION_INCOMPATIBLE:
|
||||||
|
eligible.append((match, relation))
|
||||||
|
|
||||||
|
if not eligible:
|
||||||
|
raise RecipeValidationError(
|
||||||
|
f"Local checkpoint '{target_name}' has a different base model "
|
||||||
|
"than the recipe"
|
||||||
|
)
|
||||||
|
if len(eligible) > 1:
|
||||||
|
raise RecipeValidationError(
|
||||||
|
f"Multiple local checkpoints match '{target_name}'; "
|
||||||
|
"include the folder path to disambiguate"
|
||||||
|
)
|
||||||
|
target_checkpoint, target_relation = eligible[0]
|
||||||
|
|
||||||
|
recipe_data, updated_checkpoint = await recipe_scanner.update_checkpoint_entry(
|
||||||
|
recipe_id,
|
||||||
|
target_name=target_name,
|
||||||
|
target_checkpoint=target_checkpoint,
|
||||||
|
)
|
||||||
|
|
||||||
|
image_path = recipe_data.get("file_path")
|
||||||
|
if image_path and os.path.exists(image_path):
|
||||||
|
self._exif_utils.append_recipe_metadata(image_path, recipe_data)
|
||||||
|
|
||||||
|
matching_recipes = []
|
||||||
|
if "fingerprint" in recipe_data:
|
||||||
|
matching_recipes = await recipe_scanner.find_recipes_by_fingerprint(
|
||||||
|
recipe_data["fingerprint"]
|
||||||
|
)
|
||||||
|
if recipe_id in matching_recipes:
|
||||||
|
matching_recipes.remove(recipe_id)
|
||||||
|
|
||||||
|
payload: dict[str, Any] = {
|
||||||
|
"success": True,
|
||||||
|
"recipe_id": recipe_id,
|
||||||
|
"updated_checkpoint": updated_checkpoint,
|
||||||
|
"matching_recipes": matching_recipes,
|
||||||
|
}
|
||||||
|
if target_relation == RELATION_COMPATIBLE:
|
||||||
|
# Structured data, not prose — the frontend localizes the warning.
|
||||||
|
payload["base_model_mismatch"] = {
|
||||||
|
"recipe_base_model": recipe_base_model,
|
||||||
|
"checkpoint_base_model": target_checkpoint.get("base_model") or "",
|
||||||
|
}
|
||||||
|
return PersistenceResult(payload)
|
||||||
|
|
||||||
|
async def restore_checkpoint(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
recipe_scanner,
|
||||||
|
recipe_id: str,
|
||||||
|
) -> PersistenceResult:
|
||||||
|
"""Restore the checkpoint entry to the state captured before its reconnect."""
|
||||||
|
|
||||||
|
recipe_data, updated_checkpoint = await recipe_scanner.restore_checkpoint_entry(
|
||||||
|
recipe_id
|
||||||
|
)
|
||||||
|
|
||||||
|
image_path = recipe_data.get("file_path")
|
||||||
|
if image_path and os.path.exists(image_path):
|
||||||
|
self._exif_utils.append_recipe_metadata(image_path, recipe_data)
|
||||||
|
|
||||||
|
matching_recipes = []
|
||||||
|
if "fingerprint" in recipe_data:
|
||||||
|
matching_recipes = await recipe_scanner.find_recipes_by_fingerprint(
|
||||||
|
recipe_data["fingerprint"]
|
||||||
|
)
|
||||||
|
if recipe_id in matching_recipes:
|
||||||
|
matching_recipes.remove(recipe_id)
|
||||||
|
|
||||||
|
return PersistenceResult(
|
||||||
|
{
|
||||||
|
"success": True,
|
||||||
|
"recipe_id": recipe_id,
|
||||||
|
"updated_checkpoint": updated_checkpoint,
|
||||||
|
"matching_recipes": matching_recipes,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
async def get_checkpoint_reconnect_suggestions(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
recipe_scanner,
|
||||||
|
recipe_id: str,
|
||||||
|
query: str | None = None,
|
||||||
|
) -> PersistenceResult:
|
||||||
|
"""Return ranked local checkpoint candidates for reconnecting a recipe entry."""
|
||||||
|
|
||||||
|
recipe_path = await recipe_scanner.get_recipe_json_path(recipe_id)
|
||||||
|
if not recipe_path or not os.path.exists(recipe_path):
|
||||||
|
raise RecipeNotFoundError("Recipe not found")
|
||||||
|
|
||||||
|
with open(recipe_path, "r", encoding="utf-8") as file_obj:
|
||||||
|
recipe_data = json.load(file_obj)
|
||||||
|
|
||||||
|
checkpoint = recipe_data.get("checkpoint")
|
||||||
|
if not isinstance(checkpoint, dict):
|
||||||
|
raise RecipeValidationError("Recipe has no checkpoint entry")
|
||||||
|
|
||||||
|
suggestions = await recipe_scanner.suggest_checkpoint_reconnect_candidates(
|
||||||
|
entry=checkpoint,
|
||||||
|
recipe_base_model=recipe_data.get("base_model"),
|
||||||
|
query=query,
|
||||||
|
)
|
||||||
|
|
||||||
|
return PersistenceResult({"success": True, "suggestions": suggestions})
|
||||||
|
|
||||||
|
async def mark_checkpoint_hash_invalid(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
recipe_scanner,
|
||||||
|
recipe_id: str,
|
||||||
|
hash_invalid: bool = True,
|
||||||
|
) -> PersistenceResult:
|
||||||
|
"""Mark the recipe checkpoint entry's hash as unresolvable on CivitAI.
|
||||||
|
|
||||||
|
Called when a download attempt by hash returned "Model not found".
|
||||||
|
The flag makes the entry an unresolved rematch candidate without
|
||||||
|
altering its stored hash/file_name.
|
||||||
|
"""
|
||||||
|
|
||||||
|
recipe_data, updated_checkpoint = (
|
||||||
|
await recipe_scanner.set_checkpoint_entry_hash_invalid(
|
||||||
|
recipe_id,
|
||||||
|
hash_invalid=hash_invalid,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return PersistenceResult(
|
||||||
|
{
|
||||||
|
"success": True,
|
||||||
|
"recipe_id": recipe_id,
|
||||||
|
"hash_invalid": bool(hash_invalid),
|
||||||
|
"updated_checkpoint": updated_checkpoint,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
async def bulk_delete(
|
async def bulk_delete(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -519,7 +841,7 @@ class RecipePersistenceService:
|
|||||||
# Merge succeeded: one undo action covers the whole bulk.
|
# Merge succeeded: one undo action covers the whole bulk.
|
||||||
payload["batch_id"] = merged_batch_id
|
payload["batch_id"] = merged_batch_id
|
||||||
else:
|
else:
|
||||||
# Merge failure (e.g. cross-volume move): expose the constituent
|
# Merge unresolvable (defensive): expose the constituent
|
||||||
# batches so the caller can undo them one at a time.
|
# batches so the caller can undo them one at a time.
|
||||||
payload["batch_ids"] = batch_ids
|
payload["batch_ids"] = batch_ids
|
||||||
else:
|
else:
|
||||||
@@ -602,6 +924,12 @@ class RecipePersistenceService:
|
|||||||
if key not in ["checkpoint", "loras"]
|
if key not in ["checkpoint", "loras"]
|
||||||
},
|
},
|
||||||
"loras_stack": lora_stack,
|
"loras_stack": lora_stack,
|
||||||
|
# Widget saves re-encode an in-memory tensor to PNG/WebP with no
|
||||||
|
# embedded metadata chunks, so a workflow can never be present.
|
||||||
|
"has_workflow": False,
|
||||||
|
# Widget saves read LoRAs straight from the current workflow; an
|
||||||
|
# empty list means the workflow used no LoRAs.
|
||||||
|
"import_info": build_import_info(CHANNEL_WIDGET, None, loras_data),
|
||||||
}
|
}
|
||||||
if checkpoint_entry:
|
if checkpoint_entry:
|
||||||
recipe_data["checkpoint"] = checkpoint_entry
|
recipe_data["checkpoint"] = checkpoint_entry
|
||||||
@@ -626,6 +954,20 @@ class RecipePersistenceService:
|
|||||||
|
|
||||||
# Helper methods ---------------------------------------------------
|
# Helper methods ---------------------------------------------------
|
||||||
|
|
||||||
|
def _detect_has_workflow(self, image_path: str) -> bool:
|
||||||
|
"""Detect whether the saved recipe image embeds a ComfyUI workflow.
|
||||||
|
|
||||||
|
Extraction failures (missing file, corrupt image, unsupported format)
|
||||||
|
map to ``False`` and never propagate, mirroring the scanner's behavior.
|
||||||
|
"""
|
||||||
|
if not image_path or not os.path.exists(image_path):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
metadata = self._exif_utils._load_structured_metadata(image_path)
|
||||||
|
return bool(metadata.get("workflow"))
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
async def _build_widget_checkpoint_entry(
|
async def _build_widget_checkpoint_entry(
|
||||||
self,
|
self,
|
||||||
recipe_scanner,
|
recipe_scanner,
|
||||||
@@ -762,6 +1104,7 @@ class RecipePersistenceService:
|
|||||||
"modelName": lora.get("name", ""),
|
"modelName": lora.get("name", ""),
|
||||||
"modelVersionName": lora.get("version", ""),
|
"modelVersionName": lora.get("version", ""),
|
||||||
"isDeleted": lora.get("isDeleted", False),
|
"isDeleted": lora.get("isDeleted", False),
|
||||||
|
"hashInvalid": lora.get("hashInvalid", False),
|
||||||
"exclude": lora.get("exclude", False),
|
"exclude": lora.get("exclude", False),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -297,23 +297,44 @@ class ServiceRegistry:
|
|||||||
async def get_embedding_scanner(cls):
|
async def get_embedding_scanner(cls):
|
||||||
"""Get or create Embedding scanner instance"""
|
"""Get or create Embedding scanner instance"""
|
||||||
service_name = "embedding_scanner"
|
service_name = "embedding_scanner"
|
||||||
|
|
||||||
if service_name in cls._services:
|
if service_name in cls._services:
|
||||||
return cls._services[service_name]
|
return cls._services[service_name]
|
||||||
|
|
||||||
async with cls._get_lock(service_name):
|
async with cls._get_lock(service_name):
|
||||||
# Double-check after acquiring lock
|
# Double-check after acquiring lock
|
||||||
if service_name in cls._services:
|
if service_name in cls._services:
|
||||||
return cls._services[service_name]
|
return cls._services[service_name]
|
||||||
|
|
||||||
# Import here to avoid circular imports
|
# Import here to avoid circular imports
|
||||||
from .embedding_scanner import EmbeddingScanner
|
from .embedding_scanner import EmbeddingScanner
|
||||||
|
|
||||||
scanner = await EmbeddingScanner.get_instance()
|
scanner = await EmbeddingScanner.get_instance()
|
||||||
cls._services[service_name] = scanner
|
cls._services[service_name] = scanner
|
||||||
logger.debug(f"Created and registered {service_name}")
|
logger.debug(f"Created and registered {service_name}")
|
||||||
return scanner
|
return scanner
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
async def get_other_scanner(cls):
|
||||||
|
"""Get or create Other-model scanner instance"""
|
||||||
|
service_name = "other_scanner"
|
||||||
|
|
||||||
|
if service_name in cls._services:
|
||||||
|
return cls._services[service_name]
|
||||||
|
|
||||||
|
async with cls._get_lock(service_name):
|
||||||
|
# Double-check after acquiring lock
|
||||||
|
if service_name in cls._services:
|
||||||
|
return cls._services[service_name]
|
||||||
|
|
||||||
|
# Import here to avoid circular imports
|
||||||
|
from .other_scanner import OtherScanner
|
||||||
|
|
||||||
|
scanner = await OtherScanner.get_instance()
|
||||||
|
cls._services[service_name] = scanner
|
||||||
|
logger.debug(f"Created and registered {service_name}")
|
||||||
|
return scanner
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def clear_services(cls):
|
def clear_services(cls):
|
||||||
"""Clear all registered services - mainly for testing"""
|
"""Clear all registered services - mainly for testing"""
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user