mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-08-14 09:43:22 -03:00
Compare commits
82 Commits
ab4154c57d
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
| f8d98934ad | |||
| 303cca0d85 | |||
| c2f16784b3 | |||
| 5bc6d8286c | |||
| 3f8381ffee | |||
| 1ca99294c9 | |||
| 680f0a57f5 | |||
| 94e3f54571 | |||
| 5c2b2aedcc | |||
| ebc31fb963 | |||
| 9659df6ad9 | |||
| 04d131e9dc | |||
| 78fe6282c7 | |||
| 0c00ee22fc | |||
| 5fd4946b1f | |||
| f1d3ac0cdc | |||
| e2c45905f0 | |||
| b2c68e6a65 | |||
| eb0f6dd3b6 | |||
| 0bf87f9092 | |||
| 1da2433bb2 | |||
| 2d6cf545b9 | |||
| 6a259a14fa | |||
| 41e1fd1e1f | |||
| 95fb3c7fc9 | |||
| 8237e5f9ea | |||
| aa75986178 | |||
| b887922055 | |||
| 68fa0f29c7 | |||
| d9d362c9c9 | |||
| d0bc4be0dc | |||
| 420530f532 | |||
| 3001f0f0ef | |||
| b2a1307d23 | |||
| 64da845a58 | |||
| 27027c4497 | |||
| 86c85c08ec | |||
| 196c8ffc3e | |||
| cfc95ee02a | |||
| 479fa36997 | |||
| 3e1216e9bc | |||
| 007883b7d1 | |||
| dc9200a12c | |||
| d2f955266d | |||
| 8e724538bd | |||
| 6fcdeb799d | |||
| 97b9b1f62b | |||
| 4bf9a4b640 | |||
| c5088772e8 | |||
| 56acefbd6c | |||
| 5ab06c4aae | |||
| c11f4b5c68 | |||
| 86376284f4 | |||
| 2b8a2fc7d8 | |||
| f26e1b41c8 | |||
| c1671af99f | |||
| ac7707d0f6 | |||
| 381cd710a2 | |||
| ad0d18cb79 | |||
| 7980ee77d0 | |||
| 916b8bb327 | |||
| 87e3d4dea9 | |||
| 76a913f5e0 | |||
| d8c192e647 | |||
| c453437620 | |||
| 720fa6d909 | |||
| b4f71089f4 | |||
| 83e6657ead | |||
| 7ea6df4111 | |||
| d9ab92602a | |||
| 5ffadaed31 | |||
| 24f5f7df5d | |||
| daf01fb1d6 | |||
| 0f11b6def9 | |||
| 7df83f44b8 | |||
| 169fa7bed6 | |||
| 027b504fe8 | |||
| 186ef4da78 | |||
| dc674098e7 | |||
| 9087b4b07c | |||
| 8e45c22d7a | |||
| 191c4e03cd |
@@ -1,47 +1,145 @@
|
||||
---
|
||||
name: lora-manager-e2e
|
||||
description: End-to-end testing and validation for LoRa Manager features. Use when performing automated E2E validation of LoRa Manager standalone mode, including starting/restarting the server, using Chrome DevTools MCP to interact with the web UI at http://127.0.0.1:8188/loras, and verifying frontend-to-backend functionality. Covers workflow validation, UI interaction testing, and integration testing between the standalone Python backend and the browser frontend.
|
||||
description: End-to-end testing and validation for LoRa Manager features. Use when performing automated E2E validation of LoRa Manager standalone mode in a SANDBOXED, disposable configuration: check the port, start/restart the standalone server on a free port, use Chrome DevTools MCP to interact with the web UI (http://127.0.0.1:{PORT}/loras), and verify frontend-to-backend functionality. Covers workflow validation, UI interaction testing, and integration testing between the standalone Python backend and the browser frontend. Trigger keywords: E2E, standalone, Chrome DevTools MCP, lora-manager-e2e, sandbox.
|
||||
---
|
||||
|
||||
# LoRa Manager E2E Testing
|
||||
|
||||
This skill provides workflows and utilities for end-to-end testing of LoRa Manager using Chrome DevTools MCP.
|
||||
|
||||
## Conventions Used in This Document
|
||||
|
||||
- **`{PORT}`**: The server port. The default candidate is `8188`, but **`8188` is commonly occupied by a live ComfyUI process** and MUST NOT be assumed to be free. Always check availability first (see [Port Selection](#port-selection)) and use a free port (e.g. `8199`) for the E2E run. Substitute the actual port for every `{PORT}` in the commands below.
|
||||
- **`<repo-root>`**: The repository/worktree root. Always run commands from the repo or worktree root; never assume a specific absolute path (paths such as `/home/<user>/...` differ per machine). The E2E scripts resolve the project root themselves, but fixture/settings paths are relative to `<repo-root>`.
|
||||
|
||||
## SANDBOX (MANDATORY)
|
||||
|
||||
> **Read this section before running anything.** Every E2E run MUST target a throwaway sandbox, never the real user data. A fresh subagent that skips this section WILL permanently mutate real user recipes.
|
||||
|
||||
1. **Portable settings**: create `<repo-root>/settings.json` (gitignored) with `"use_portable_settings": true` plus sandboxed `folder_paths` (lora/checkpoint roots) and `recipes_path`. This keeps the configuration inside the repo instead of the real user config dir (`~/.config/ComfyUI-LoRA-Manager/settings.json`).
|
||||
2. **Sandboxed paths**: point `folder_paths` / `recipes_path` / `example_images_path` at disposable dirs — e.g. under `/tmp/opencode/<plan-name>-e2e/` (or worktree-local dirs). NEVER point the E2E at the real library (`~/models/...`), real recipe dir, or real settings.
|
||||
3. **Never touch the real config**: the real user config at `~/.config/ComfyUI-LoRA-Manager/settings.json` and the real recipe dir must remain byte-identical before and after the run.
|
||||
4. **Record real-data protection proof** before starting and after finishing:
|
||||
```bash
|
||||
# BEFORE: snapshot real config + recipe library state
|
||||
sha256sum ~/.config/ComfyUI-LoRA-Manager/settings.json > /tmp/opencode/<plan>-e2e/settings.before.sha256
|
||||
ls ~/models/recipes/*.recipe.json 2>/dev/null | wc -l > /tmp/opencode/<plan>-e2e/recipes-count.before.txt
|
||||
find ~/models/recipes -name '*.recipe.json' -newermt "$(date -Iseconds)" | head # expect empty after run
|
||||
# AFTER: record again, then diff the two snapshots. Any change = the run leaked into real data.
|
||||
```
|
||||
Also confirm `<repo-root>/git status` stays clean for `settings.json`/`cache/` (both are gitignored).
|
||||
|
||||
### Portable Settings Example
|
||||
|
||||
```json
|
||||
{
|
||||
"use_portable_settings": true,
|
||||
"folder_paths": {
|
||||
"loras": ["/tmp/opencode/<plan>-e2e/models/loras"],
|
||||
"checkpoints": ["/tmp/opencode/<plan>-e2e/models/checkpoints"],
|
||||
"unet": ["/tmp/opencode/<plan>-e2e/models/checkpoints"],
|
||||
"diffusers": []
|
||||
},
|
||||
"recipes_path": "/tmp/opencode/<plan>-e2e/recipes",
|
||||
"example_images_path": "/tmp/opencode/<plan>-e2e/example_images"
|
||||
}
|
||||
```
|
||||
|
||||
The scanner computes and persists model hashes during the library scan, so the sandbox model dirs just need the model files + `.metadata.json` sidecars (see [Fixture + Fresh-State Guidance](#fixture--fresh-state-guidance)).
|
||||
|
||||
## Time Budgets & Abort Guidance
|
||||
|
||||
A fresh subagent should complete a sandboxed standalone E2E **in well under 30 minutes**. Budget each phase:
|
||||
|
||||
| Phase | Expected duration | Abort if |
|
||||
| --- | --- | --- |
|
||||
| Port check + sandbox setup | < 2 min | — |
|
||||
| Server start (detached) + readiness | < 30 s | > 60 s (2x) → stop |
|
||||
| Chrome DevTools MCP connect | < 1 min | > 2 min → stop |
|
||||
| Per entry-point run (after fixtures ready) | < 5 min | > 10 min (2x) → stop |
|
||||
| Fixture reset + cache clear between runs | < 1 min | > 2 min → stop |
|
||||
|
||||
**Abort rule**: if a phase exceeds ~2x its budget, OR any single tool call fails/retries 3+ times in a row, **STOP**. Do not loop or retry blindly. Report `BLOCKED` with: the phase, the last observed state (server PID + `ss -tlnp` output, page snapshot, last API response), and the suspected cause. Record the partial state as evidence; a clean BLOCKED report is more valuable than an hour of retries.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- LoRa Manager project cloned and dependencies installed (`pip install -r requirements.txt`)
|
||||
- LoRa Manager project cloned and dependencies installed (`pip install -r requirements.txt`) — run everything from `<repo-root>`
|
||||
- Chrome browser available for debugging
|
||||
- Chrome DevTools MCP connected
|
||||
- `ss` (or `lsof`/`netstat`) available for port checks: `ss -tlnp`
|
||||
|
||||
## Quick Start Workflow
|
||||
## Port Selection
|
||||
|
||||
### 1. Start LoRa Manager Standalone
|
||||
|
||||
```python
|
||||
# Use the provided script to start the server
|
||||
python .agents/skills/lora-manager-e2e/scripts/start_server.py --port 8188
|
||||
```
|
||||
|
||||
Or manually:
|
||||
```bash
|
||||
cd /home/miao/workspace/ComfyUI/custom_nodes/ComfyUI-Lora-Manager
|
||||
python standalone.py --port 8188
|
||||
```
|
||||
|
||||
Wait for server ready message before proceeding.
|
||||
|
||||
### 2. Open Chrome Debug Mode
|
||||
`8188` is only the *default candidate*. Verify it is actually free before every run:
|
||||
|
||||
```bash
|
||||
# Chrome with remote debugging on port 9222
|
||||
google-chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:8188/loras
|
||||
# Is anything listening on 8188?
|
||||
ss -tlnp | grep ':8188' || echo "8188 is free"
|
||||
```
|
||||
|
||||
### 3. Connect Chrome DevTools MCP
|
||||
- If a process holds `8188` (e.g. a live ComfyUI — pid 6575 on this machine), pick a different free port, e.g. `8199`:
|
||||
```bash
|
||||
ss -tlnp | grep ':8199' || echo "8199 is free"
|
||||
```
|
||||
- **Never** kill a process you did not start for this E2E. The live ComfyUI is off-limits. Pick a free port instead.
|
||||
- Use your chosen port for **all** subsequent commands (server, Chrome launch, browser URLs).
|
||||
|
||||
Ensure the MCP server is connected to Chrome at `http://localhost:9222`.
|
||||
## Quick Start Workflow (sandboxed)
|
||||
|
||||
### 4. Navigate and Interact
|
||||
### 1. Prepare the sandbox
|
||||
|
||||
```bash
|
||||
cd <repo-root> # ALWAYS run from the repo/worktree root
|
||||
mkdir -p /tmp/opencode/<plan>-e2e/models/{loras,checkpoints}
|
||||
mkdir -p /tmp/opencode/<plan>-e2e/{recipes,example_images,recipes-before}
|
||||
# write <repo-root>/settings.json per the portable-settings example above
|
||||
# record real-data protection proof (see SANDBOX section)
|
||||
```
|
||||
|
||||
### 2. Check port availability
|
||||
|
||||
```bash
|
||||
ss -tlnp | grep ':{PORT}' || echo "port {PORT} is free"
|
||||
```
|
||||
|
||||
If `{PORT}` is occupied by an unrelated process, pick a free one and use it everywhere below. When in doubt use `8199`.
|
||||
|
||||
### 3. Start LoRa Manager Standalone (detached)
|
||||
|
||||
The standalone server **dies with the shell unless launched fully detached** — a plain background `&` from the bash tool is killed when the tool call returns. Launch via the helper script:
|
||||
|
||||
```bash
|
||||
python .agents/skills/lora-manager-e2e/scripts/start_server.py --port {PORT} --wait --timeout 30 --detach
|
||||
```
|
||||
|
||||
Or manually (equivalent detached form):
|
||||
|
||||
```bash
|
||||
setsid nohup python standalone.py --port {PORT} --host 127.0.0.1 < /dev/null \
|
||||
>> /tmp/opencode/<plan>-e2e/server.log 2>&1 &
|
||||
echo "started" # record the printed/pidfile PID for cleanup
|
||||
```
|
||||
|
||||
Verify it is listening **before** proceeding (readiness poll is not a substitute for this):
|
||||
|
||||
```bash
|
||||
ss -tlnp | grep ':{PORT}'
|
||||
```
|
||||
|
||||
Record the server PID for cleanup: the helper script writes it to `/tmp/lora-manager-e2e-server-{PORT}.pid`; a manual `setsid` launch has no pidfile, so capture it explicitly (e.g. from `ss -tlnp`).
|
||||
|
||||
### 4. Open Chrome Debug Mode
|
||||
|
||||
```bash
|
||||
# Chrome with remote debugging on port 9222 (note the {PORT} URL)
|
||||
google-chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:{PORT}/loras
|
||||
```
|
||||
|
||||
### 5. Connect Chrome DevTools MCP
|
||||
|
||||
Ensure the MCP server is connected to Chrome at `http://localhost:9222`. Verify with `list_pages` — if it fails with "browser is already running", see [Chrome DevTools MCP Troubleshooting](#chrome-devtools-mcp-troubleshooting).
|
||||
|
||||
### 6. Navigate and Interact
|
||||
|
||||
Use Chrome DevTools MCP tools to:
|
||||
- Take snapshots: `take_snapshot`
|
||||
@@ -56,7 +154,7 @@ Use Chrome DevTools MCP tools to:
|
||||
|
||||
```python
|
||||
# Navigate to LoRA list page
|
||||
navigate_page(type="url", url="http://127.0.0.1:8188/loras")
|
||||
navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
||||
|
||||
# Wait for page to load
|
||||
wait_for(text="LoRAs", timeout=10000)
|
||||
@@ -68,9 +166,10 @@ snapshot = take_snapshot()
|
||||
### Pattern: Restart Server for Configuration Changes
|
||||
|
||||
```python
|
||||
# Stop current server (if running)
|
||||
# Start with new configuration
|
||||
python .agents/skills/lora-manager-e2e/scripts/start_server.py --port 8188 --restart
|
||||
# Stop current server (if running), start with new configuration.
|
||||
# --restart only kills the E2E server this script started before (via its pidfile);
|
||||
# it refuses to blindly kill unrelated processes on the port.
|
||||
python .agents/skills/lora-manager-e2e/scripts/start_server.py --port {PORT} --restart --wait --detach
|
||||
|
||||
# Wait and refresh browser
|
||||
navigate_page(type="reload", ignoreCache=True)
|
||||
@@ -130,24 +229,96 @@ click(uid="modal-submit-button")
|
||||
wait_for(text="Success", timeout=5000)
|
||||
```
|
||||
|
||||
## Fixture + Fresh-State Guidance
|
||||
|
||||
For rematch/repair E2E runs, seed the **sandboxed** `recipes_path` with hand-written fixture recipes. Rules (validated by the task-8 E2E):
|
||||
|
||||
1. **Filename constraint**: each file MUST be named `f"{id}.recipe.json"` **and** the in-JSON `id` field MUST equal the filename. Discovery accepts any `*.recipe.json`, but persistence resolves the path via `get_recipe_json_path` and `_save_recipe_persistently` returns `False` on a mismatch → the fixture would be counted as an error.
|
||||
- `recipe-a.recipe.json` → in-JSON `"id": "recipe-a"`
|
||||
2. **File format**: mirror an existing recipe JSON — top-level `id`, `file_path`, `title`, `loras`, `fingerprint`, `gen_params`; lora entries per the persistence conventions (`hash`, `file_name`, `modelVersionId`, `isDeleted`, ...).
|
||||
3. **Companion image**: each recipe needs an image (e.g. a `.webp` generated with PIL) referenced by `file_path`, used for EXIF verification (`ExifUtils.append_recipe_metadata` writes a `"Recipe metadata: ..."` marker; a freshly generated `.webp` with no marker is the clean "untouched" control).
|
||||
4. **autov3 three-state contract**: for L3 (autov3-only, renamed-file) fixtures the local model's `.metadata.json` sidecar MUST have the `autov3` key **ABSENT** (the "unchecked" state), NOT `""` — `""` is the TERMINAL "checked but unavailable" state that L3 deliberately skips. The scanner computes + persists `autov3` from the file header during the normal library scan (`model_scanner.py` `_process_model_file`), so the live L3 match resolves through the local autov3/hash cache; the computed-autov3 branch for unchecked items is covered by the unit suite.
|
||||
5. **Fixture design for a rematch run** (mirrors the task-8 E2E):
|
||||
- `recipe-a`: lora entry `isDeleted=True`, `hash` = 12-char autov3 computed from the local model (`calculate_autov3`, `py/utils/file_utils.py`), whose local model file was RENAMED after the recipe was written so `file_name` differs (proves L3 match without filename).
|
||||
- `recipe-b`: parser-convention checkpoint entry (uses `id`, no `modelVersionId`) matching a local checkpoint via L2 — the local checkpoint's `.metadata.json` MUST carry civitai version data with that `id` so `version_index` contains it (L2 cannot match otherwise).
|
||||
- `recipe-c`: healthy recipe (no deleted entries) → must remain untouched.
|
||||
|
||||
### Fresh state between entry-point runs
|
||||
|
||||
Each entry point (global / per-recipe / selection-bulk) must start from the same deleted state. Between runs:
|
||||
|
||||
```bash
|
||||
# 1. Reset fixtures to the before-state snapshot (copy back from recipes-before/)
|
||||
cp /tmp/opencode/<plan>-e2e/recipes-before/*.recipe.json /tmp/opencode/<plan>-e2e/recipes/
|
||||
# 2. Clear the recipe/FTS caches so the stale in-memory/library state is gone
|
||||
rm -f <repo-root>/cache/recipe/*.sqlite
|
||||
rm -rf <repo-root>/cache/fts/*
|
||||
# 3. Restart the server (fresh process, fresh scan)
|
||||
python .agents/skills/lora-manager-e2e/scripts/start_server.py --port {PORT} --restart --wait --timeout 30 --detach
|
||||
# 4. Re-verify server listening + reload the browser page
|
||||
```
|
||||
|
||||
## Server Lifecycle
|
||||
|
||||
- **Detached launch is mandatory**: the standalone server dies with the shell unless launched via `setsid` (or the helper script's `--detach`). Use `setsid nohup python standalone.py --port {PORT} --host 127.0.0.1 ... < /dev/null &`.
|
||||
- **Verify with `ss -tlnp`** after every (re)start; do not proceed on a blind "server starting" message.
|
||||
- **Never kill pre-existing processes** — only kill the E2E server PID you started (`start_server.py --restart` kills only PIDs it manages via its pidfile). The live ComfyUI or a stale QA Chrome must never be killed as part of cleanup unless explicitly identified as such (see Chrome troubleshooting).
|
||||
- **Record your PID for cleanup**: note the PID printed/pidfile, and stop exactly that PID at the end (`kill <PID>`, then confirm with `ss -tlnp` that `{PORT}` is released).
|
||||
|
||||
## Chrome DevTools MCP Troubleshooting
|
||||
|
||||
### Stale profile lock ("browser is already running" / `list_pages` fails)
|
||||
|
||||
A Chrome profile can be held by a stale Chrome from a prior MCP session, which makes `list_pages` fail with "browser is already running":
|
||||
|
||||
1. Identify the stale Chrome — it owns the profile dir in `--user-data-dir` (e.g. `~/.config/chrome-dev-profile`). Find its process:
|
||||
```bash
|
||||
ps -ef | grep -i '[c]hrome.*user-data-dir'
|
||||
```
|
||||
2. Confirm it is a QA Chrome from a completed task (its parent is an old MCP/browser process, it is NOT the live ComfyUI server, and it is NOT your current MCP instance).
|
||||
3. Kill ONLY that stale Chrome:
|
||||
```bash
|
||||
kill <stale-chrome-pid>
|
||||
```
|
||||
Never kill the live server or unrelated processes.
|
||||
4. Retry `list_pages`. The current MCP will spawn a fresh browser.
|
||||
|
||||
### Screenshot-write restrictions
|
||||
|
||||
The chrome-devtools MCP may refuse to write into paths outside its configured workspace roots (e.g. the worktree `.omo/evidence/...` canonicalizing to an unmapped path). Workaround:
|
||||
|
||||
```bash
|
||||
# 1. Save the screenshot to /tmp via the MCP
|
||||
# take_screenshot(filePath="/tmp/<plan>-e2e/recipe-b-after.png", format="png")
|
||||
# 2. Copy it into the evidence dir from the shell
|
||||
mkdir -p <repo-root>/.omo/evidence/screenshots
|
||||
cp /tmp/<plan>-e2e/recipe-b-after.png <repo-root>/.omo/evidence/screenshots/
|
||||
```
|
||||
|
||||
## Cancellation Testing (KNOWN GAP)
|
||||
|
||||
Testing the rematch-cancel path E2E requires a run long enough to cancel mid-flight. A tiny 3-recipe fixture set completes in **seconds** — too fast to reliably cancel. The cancel path is currently **unit-covered only** (`rematch_all_recipes` cancellation tests); do not block an E2E run on cancel-path verification. If you must attempt it, you would need an artificially large/deferred fixture set to create a cancellable window — treat this as a research task, not part of the standard E2E.
|
||||
|
||||
## Available Scripts
|
||||
|
||||
### scripts/start_server.py
|
||||
|
||||
Starts or restarts the LoRa Manager standalone server.
|
||||
Starts or restarts the LoRa Manager standalone server for E2E testing.
|
||||
|
||||
```bash
|
||||
python scripts/start_server.py [--port PORT] [--restart] [--wait]
|
||||
python scripts/start_server.py [--port PORT] [--restart] [--wait] [--timeout SECONDS] [--detach]
|
||||
```
|
||||
|
||||
Options:
|
||||
- `--port`: Server port (default: 8188)
|
||||
- `--restart`: Kill existing server before starting
|
||||
- `--wait`: Wait for server to be ready before exiting
|
||||
- `--port`: Server port (default: 8188). The script exits early with a clear message if the port is already in use by an unrelated process.
|
||||
- `--restart`: Kill the E2E server this script previously managed (tracked via `/tmp/lora-manager-e2e-server-{PORT}.pid`) before starting. If unrelated processes still hold the port after that, the script reports them and aborts instead of killing them.
|
||||
- `--wait`: Wait for the server to be ready before exiting.
|
||||
- `--timeout`: Readiness wait timeout in seconds (default: 30).
|
||||
- `--detach`: Launch the server fully detached (`setsid`-style, survives shell death — REQUIRED for E2E). Default off: a normal background process that dies with the shell.
|
||||
|
||||
### scripts/wait_for_server.py
|
||||
|
||||
Polls server until ready or timeout.
|
||||
Polls the server until ready or timeout.
|
||||
|
||||
```bash
|
||||
python scripts/wait_for_server.py [--port PORT] [--timeout SECONDS]
|
||||
@@ -196,6 +367,7 @@ results = performance_stop_trace()
|
||||
## Cleanup
|
||||
|
||||
Always ensure proper cleanup after tests:
|
||||
1. Stop the standalone server
|
||||
2. Close browser pages (keep at least one open)
|
||||
3. Clear temporary data if needed
|
||||
1. Stop the standalone server: `kill <recorded-pid>` (only the PID you started), then confirm `ss -tlnp | grep ':{PORT}'` is empty.
|
||||
2. Close browser pages (keep at least one open).
|
||||
3. Remove the sandbox: `rm -rf /tmp/opencode/<plan>-e2e` and `<repo-root>/settings.json` + `<repo-root>/cache` (both gitignored).
|
||||
4. Re-run the real-data protection check from the SANDBOX section and record the result in your evidence.
|
||||
|
||||
@@ -2,11 +2,13 @@
|
||||
|
||||
Quick reference for common MCP commands used in LoRa Manager E2E testing.
|
||||
|
||||
> **Port convention**: `{PORT}` is the port chosen for the E2E run (default candidate `8188`, but only if actually free — see the SKILL.md Port Selection section; use e.g. `8199` when `8188` is occupied by a live ComfyUI). Always run against the **sandboxed** standalone server, never a live instance.
|
||||
|
||||
## Navigation
|
||||
|
||||
```python
|
||||
# Navigate to LoRA list page
|
||||
navigate_page(type="url", url="http://127.0.0.1:8188/loras")
|
||||
navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
||||
|
||||
# Reload page with cache clear
|
||||
navigate_page(type="reload", ignoreCache=True)
|
||||
@@ -179,7 +181,7 @@ pages = list_pages()
|
||||
select_page(pageId=0, bringToFront=True)
|
||||
|
||||
# Create new page
|
||||
new_page(url="http://127.0.0.1:8188/loras")
|
||||
new_page(url="http://127.0.0.1:{PORT}/loras")
|
||||
|
||||
# Close page (keep at least one open!)
|
||||
close_page(pageId=1)
|
||||
@@ -261,7 +263,7 @@ drag(from_uid="draggable-item", to_uid="drop-zone")
|
||||
### Verify LoRA Cards Loaded
|
||||
|
||||
```python
|
||||
navigate_page(type="url", url="http://127.0.0.1:8188/loras")
|
||||
navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
||||
wait_for(text="LoRAs", timeout=10000)
|
||||
|
||||
# Check if cards loaded
|
||||
@@ -322,3 +324,37 @@ navigate_page(type="reload")
|
||||
errors = list_console_messages(types=["error"])
|
||||
assert len(errors) == 0, f"Console errors: {errors}"
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Stale profile lock ("browser is already running" / `list_pages` fails)
|
||||
|
||||
A Chrome profile held by a stale Chrome from a prior MCP session makes `list_pages`
|
||||
fail with "browser is already running". Fix:
|
||||
|
||||
1. Find the stale Chrome that owns the profile dir (e.g. `~/.config/chrome-dev-profile`):
|
||||
```bash
|
||||
ps -ef | grep -i '[c]hrome.*user-data-dir'
|
||||
```
|
||||
2. Confirm it is a QA Chrome from a completed task (NOT the live ComfyUI server, NOT
|
||||
your current MCP instance).
|
||||
3. Kill ONLY that stale Chrome (`kill <stale-pid>`), then retry `list_pages`.
|
||||
|
||||
### Screenshot-write restrictions
|
||||
|
||||
The MCP may refuse to write into paths outside its configured workspace roots
|
||||
(e.g. `.omo/evidence/screenshots/` under a worktree that canonicalizes to an unmapped
|
||||
path). Save the screenshot to `/tmp` via the MCP, then copy it into the evidence dir:
|
||||
|
||||
```bash
|
||||
# MCP: take_screenshot(filePath="/tmp/<plan>-e2e/recipe-b-after.png", format="png")
|
||||
# Shell:
|
||||
mkdir -p <repo-root>/.omo/evidence/screenshots
|
||||
cp /tmp/<plan>-e2e/recipe-b-after.png <repo-root>/.omo/evidence/screenshots/
|
||||
```
|
||||
|
||||
### Time budgets & abort rule
|
||||
|
||||
See SKILL.md "Time Budgets & Abort Guidance": if a phase exceeds ~2x its budget or a
|
||||
tool call retries 3+ times in a row, STOP and report BLOCKED with the last observed
|
||||
state (server PID + `ss -tlnp`, page snapshot, last API response). Do not loop.
|
||||
|
||||
@@ -2,6 +2,14 @@
|
||||
|
||||
This document provides detailed test scenarios for end-to-end validation of LoRa Manager features.
|
||||
|
||||
> **Run preconditions (from SKILL.md)**: every run uses the **sandboxed** standalone
|
||||
> server on a free port `{PORT}` (default candidate `8188`, only if actually free — pick
|
||||
> e.g. `8199` when `8188` is occupied by a live ComfyUI). Fixtures live in the sandboxed
|
||||
> `recipes_path` as `f"{id}.recipe.json"` files with matching in-JSON `id`; the real user
|
||||
> config and real library are never touched (record protection proof before/after).
|
||||
> Abort if a phase exceeds ~2x its budget or a tool call retries 3+ times (SKILL.md
|
||||
> "Time Budgets & Abort Guidance").
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [LoRA List Page](#lora-list-page)
|
||||
@@ -19,7 +27,7 @@ This document provides detailed test scenarios for end-to-end validation of LoRa
|
||||
**Objective**: Verify the LoRA list page loads correctly and displays models.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to `http://127.0.0.1:8188/loras`
|
||||
1. Navigate to `http://127.0.0.1:{PORT}/loras`
|
||||
2. Wait for page title "LoRAs" to appear
|
||||
3. Take snapshot to verify:
|
||||
- Header with "LoRAs" title is visible
|
||||
@@ -134,7 +142,7 @@ evaluate_script(function="""
|
||||
**Objective**: Verify recipes page loads and displays recipes.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to `http://127.0.0.1:8188/recipes`
|
||||
1. Navigate to `http://127.0.0.1:{PORT}/recipes`
|
||||
2. Wait for "Recipes" title
|
||||
3. Take snapshot
|
||||
|
||||
@@ -176,7 +184,7 @@ evaluate_script(function="""
|
||||
**Objective**: Verify settings page displays correctly.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to `http://127.0.0.1:8188/settings`
|
||||
1. Navigate to `http://127.0.0.1:{PORT}/settings`
|
||||
2. Wait for "Settings" title
|
||||
3. Take snapshot
|
||||
|
||||
@@ -190,7 +198,7 @@ evaluate_script(function="""
|
||||
1. Navigate to settings page
|
||||
2. Change a setting (e.g., default view mode)
|
||||
3. Save settings
|
||||
4. Restart server: `python scripts/start_server.py --restart --wait`
|
||||
4. Restart server: `python scripts/start_server.py --port {PORT} --restart --wait --timeout 30 --detach`
|
||||
5. Refresh browser page
|
||||
6. Navigate to settings
|
||||
|
||||
|
||||
@@ -8,186 +8,208 @@ This script shows how to:
|
||||
3. Verify functionality end-to-end
|
||||
|
||||
Note: This is a template. Actual execution requires Chrome DevTools MCP.
|
||||
|
||||
Port: pick a FREE port for the run — 8188 is commonly occupied by a live
|
||||
ComfyUI (see the skill's Port Selection section). Set PORT below to e.g. 8199
|
||||
when 8188 is taken. Always run against a SANDBOXED standalone server.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
# Choose the E2E port. 8188 is only the default candidate; use 8199 (or any
|
||||
# free port checked with `ss -tlnp`) when 8188 is occupied by a live ComfyUI.
|
||||
PORT = "8188"
|
||||
|
||||
|
||||
def run_test():
|
||||
"""Run example E2E test flow."""
|
||||
|
||||
|
||||
print("=" * 60)
|
||||
print("LoRa Manager E2E Test Example")
|
||||
print("=" * 60)
|
||||
|
||||
# Step 1: Start server
|
||||
|
||||
# Step 1: Start server (detached so it survives the shell)
|
||||
print("\n[1/5] Starting LoRa Manager standalone server...")
|
||||
result = subprocess.run(
|
||||
[sys.executable, "start_server.py", "--port", "8188", "--wait", "--timeout", "30"],
|
||||
[sys.executable, "start_server.py", "--port", PORT, "--wait", "--timeout", "30", "--detach"],
|
||||
capture_output=True,
|
||||
text=True
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
print(f"Failed to start server: {result.stderr}")
|
||||
return 1
|
||||
print("Server ready!")
|
||||
|
||||
|
||||
# Step 2: Open Chrome (manual step - show command)
|
||||
print("\n[2/5] Open Chrome with debug mode:")
|
||||
print("google-chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:8188/loras")
|
||||
print(
|
||||
f"google-chrome --remote-debugging-port=9222 "
|
||||
f"--user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:{PORT}/loras"
|
||||
)
|
||||
print("(In actual test, this would be automated via MCP)")
|
||||
|
||||
|
||||
# Step 3: Navigate and verify page load
|
||||
print("\n[3/5] Page Load Verification:")
|
||||
print("""
|
||||
print(
|
||||
f"""
|
||||
MCP Commands to execute:
|
||||
1. navigate_page(type="url", url="http://127.0.0.1:8188/loras")
|
||||
1. navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
||||
2. wait_for(text="LoRAs", timeout=10000)
|
||||
3. snapshot = take_snapshot()
|
||||
""")
|
||||
|
||||
"""
|
||||
)
|
||||
|
||||
# Step 4: Test search functionality
|
||||
print("\n[4/5] Search Functionality Test:")
|
||||
print("""
|
||||
print(
|
||||
"""
|
||||
MCP Commands to execute:
|
||||
1. fill(uid="search-input", value="test")
|
||||
2. press_key(key="Enter")
|
||||
3. wait_for(text="Results", timeout=5000)
|
||||
4. result = evaluate_script(function="""
|
||||
4. result = evaluate_script(function=`
|
||||
() => {
|
||||
const cards = document.querySelectorAll('.lora-card');
|
||||
return { count: cards.length };
|
||||
}
|
||||
""")
|
||||
""")
|
||||
|
||||
`)
|
||||
"""
|
||||
)
|
||||
|
||||
# Step 5: Verify API
|
||||
print("\n[5/5] API Verification:")
|
||||
print("""
|
||||
print(
|
||||
"""
|
||||
MCP Commands to execute:
|
||||
1. api_result = evaluate_script(function="""
|
||||
1. api_result = evaluate_script(function=`
|
||||
async () => {
|
||||
const response = await fetch('/loras/api/list');
|
||||
const data = await response.json();
|
||||
return { count: data.length, status: response.status };
|
||||
}
|
||||
""")
|
||||
`)
|
||||
2. Verify api_result['status'] == 200
|
||||
""")
|
||||
|
||||
"""
|
||||
)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Test flow completed!")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def example_restart_flow():
|
||||
"""Example: Testing configuration change that requires restart."""
|
||||
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Example: Server Restart Flow")
|
||||
print("=" * 60)
|
||||
|
||||
print("""
|
||||
|
||||
print(
|
||||
f"""
|
||||
Scenario: Change setting and verify after restart
|
||||
|
||||
|
||||
Steps:
|
||||
1. Navigate to settings page
|
||||
- navigate_page(type="url", url="http://127.0.0.1:8188/settings")
|
||||
|
||||
- navigate_page(type="url", url="http://127.0.0.1:{PORT}/settings")
|
||||
|
||||
2. Change a setting (e.g., theme)
|
||||
- fill(uid="theme-select", value="dark")
|
||||
- click(uid="save-settings-button")
|
||||
|
||||
|
||||
3. Restart server
|
||||
- subprocess.run([python, "start_server.py", "--restart", "--wait"])
|
||||
|
||||
- subprocess.run([python, "start_server.py", "--port", "{PORT}", "--restart", "--wait", "--detach"])
|
||||
|
||||
4. Refresh browser
|
||||
- navigate_page(type="reload", ignoreCache=True)
|
||||
- wait_for(text="LoRAs", timeout=15000)
|
||||
|
||||
|
||||
5. Verify setting persisted
|
||||
- navigate_page(type="url", url="http://127.0.0.1:8188/settings")
|
||||
- navigate_page(type="url", url="http://127.0.0.1:{PORT}/settings")
|
||||
- theme = evaluate_script(function="() => document.querySelector('#theme-select').value")
|
||||
- assert theme == "dark"
|
||||
""")
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def example_modal_interaction():
|
||||
"""Example: Testing modal dialog interaction."""
|
||||
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Example: Modal Dialog Interaction")
|
||||
print("=" * 60)
|
||||
|
||||
print("""
|
||||
|
||||
print(
|
||||
"""
|
||||
Scenario: Add new LoRA via modal
|
||||
|
||||
|
||||
Steps:
|
||||
1. Open modal
|
||||
- click(uid="add-lora-button")
|
||||
- wait_for(text="Add LoRA", timeout=3000)
|
||||
|
||||
|
||||
2. Fill form
|
||||
- fill_form(elements=[
|
||||
{"uid": "lora-name", "value": "Test Character"},
|
||||
{"uid": "lora-path", "value": "/models/test.safetensors"},
|
||||
])
|
||||
|
||||
|
||||
3. Submit
|
||||
- click(uid="modal-submit-button")
|
||||
|
||||
|
||||
4. Verify success
|
||||
- wait_for(text="Successfully added", timeout=5000)
|
||||
- snapshot = take_snapshot()
|
||||
""")
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def example_network_monitoring():
|
||||
"""Example: Network request monitoring."""
|
||||
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Example: Network Request Monitoring")
|
||||
print("=" * 60)
|
||||
|
||||
print("""
|
||||
|
||||
print(
|
||||
f"""
|
||||
Scenario: Verify API calls during user interaction
|
||||
|
||||
|
||||
Steps:
|
||||
1. Clear network log (implicit on navigation)
|
||||
- navigate_page(type="url", url="http://127.0.0.1:8188/loras")
|
||||
|
||||
- navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
||||
|
||||
2. Perform action that triggers API call
|
||||
- fill(uid="search-input", value="character")
|
||||
- press_key(key="Enter")
|
||||
|
||||
|
||||
3. List network requests
|
||||
- requests = list_network_requests(resourceTypes=["xhr", "fetch"])
|
||||
|
||||
|
||||
4. Find search API call
|
||||
- search_requests = [r for r in requests if "/api/search" in r.get("url", "")]
|
||||
- assert len(search_requests) > 0, "Search API was not called"
|
||||
|
||||
|
||||
5. Get request details
|
||||
- if search_requests:
|
||||
details = get_network_request(reqid=search_requests[0]["reqid"])
|
||||
- Verify request method, response status, etc.
|
||||
""")
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("LoRa Manager E2E Test Examples\n")
|
||||
print("This script demonstrates E2E testing patterns.\n")
|
||||
print("Note: Actual execution requires Chrome DevTools MCP connection.\n")
|
||||
|
||||
|
||||
run_test()
|
||||
example_restart_flow()
|
||||
example_modal_interaction()
|
||||
example_network_monitoring()
|
||||
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("All examples shown!")
|
||||
print("=" * 60)
|
||||
|
||||
@@ -1,15 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Start or restart LoRa Manager standalone server for E2E testing.
|
||||
|
||||
Backward-compatible CLI: --port, --restart, --wait, --timeout all work as before.
|
||||
New options: --detach (setsid-style fully detached launch, survives shell death).
|
||||
|
||||
Safety rules implemented here:
|
||||
- Never kill processes the script did not start. The script tracks the PIDs it
|
||||
manages in a pidfile (/tmp/lora-manager-e2e-server-{PORT}.pid).
|
||||
- If the port is held by an unrelated process (e.g. a live ComfyUI) the script
|
||||
reports the conflict and exits early instead of killing it.
|
||||
- --restart only kills managed PIDs; if unrelated processes still hold the port
|
||||
afterwards, the script reports them and aborts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import socket
|
||||
import signal
|
||||
import os
|
||||
|
||||
PIDFILE_PREFIX = "/tmp/lora-manager-e2e-server"
|
||||
|
||||
|
||||
def pidfile_path(port: int) -> str:
|
||||
"""Path of the pidfile that records PIDs this script started for a port."""
|
||||
return f"{PIDFILE_PREFIX}-{port}.pid"
|
||||
|
||||
|
||||
def read_managed_pids(port: int) -> list[int]:
|
||||
"""Read PIDs this script previously managed for the port (may be stale)."""
|
||||
path = pidfile_path(port)
|
||||
if not os.path.exists(path):
|
||||
return []
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
return [int(line.strip()) for line in fh if line.strip().isdigit()]
|
||||
except (OSError, ValueError):
|
||||
return []
|
||||
|
||||
|
||||
def write_managed_pids(port: int, pids: list[int]) -> None:
|
||||
"""Record PIDs this script manages for the port."""
|
||||
try:
|
||||
with open(pidfile_path(port), "w", encoding="utf-8") as fh:
|
||||
for pid in pids:
|
||||
fh.write(f"{pid}\n")
|
||||
except OSError as exc:
|
||||
print(f"Warning: could not write pidfile for port {port}: {exc}")
|
||||
|
||||
|
||||
def clear_managed_pids(port: int) -> None:
|
||||
"""Remove the pidfile for the port (no longer managed)."""
|
||||
path = pidfile_path(port)
|
||||
try:
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
except OSError as exc:
|
||||
print(f"Warning: could not remove pidfile {path}: {exc}")
|
||||
|
||||
|
||||
def process_alive(pid: int) -> bool:
|
||||
"""Return True if a process with the given pid exists."""
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
return True
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True # exists but owned by someone else
|
||||
|
||||
|
||||
def find_server_process(port: int) -> list[int]:
|
||||
@@ -19,7 +82,7 @@ def find_server_process(port: int) -> list[int]:
|
||||
["lsof", "-ti", f":{port}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False
|
||||
check=False,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
return [int(pid) for pid in result.stdout.strip().split("\n") if pid]
|
||||
@@ -30,7 +93,7 @@ def find_server_process(port: int) -> list[int]:
|
||||
["netstat", "-tlnp"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False
|
||||
check=False,
|
||||
)
|
||||
pids = []
|
||||
for line in result.stdout.split("\n"):
|
||||
@@ -49,30 +112,48 @@ def find_server_process(port: int) -> list[int]:
|
||||
return []
|
||||
|
||||
|
||||
def kill_server(port: int) -> None:
|
||||
"""Kill processes using the specified port."""
|
||||
pids = find_server_process(port)
|
||||
def describe_processes(pids: list[int]) -> str:
|
||||
"""Human-readable description of a pid list (pid + command line)."""
|
||||
descriptions = []
|
||||
for pid in pids:
|
||||
cmdline = ""
|
||||
try:
|
||||
with open(f"/proc/{pid}/cmdline", "rb") as fh:
|
||||
raw = fh.read().replace(b"\x00", b" ").decode("utf-8", "replace")
|
||||
cmdline = raw.strip()
|
||||
except OSError:
|
||||
pass
|
||||
descriptions.append(f"pid {pid}{' (' + cmdline + ')' if cmdline else ''}")
|
||||
return ", ".join(descriptions) if descriptions else "none"
|
||||
|
||||
|
||||
def kill_pids(pids: list[int], what: str) -> None:
|
||||
"""Send SIGTERM (then SIGKILL) to the given PIDs, only after reporting."""
|
||||
for pid in pids:
|
||||
print(f"Sent SIGTERM to {what} pid {pid}")
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
print(f"Sent SIGTERM to process {pid}")
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
# Wait for processes to terminate
|
||||
time.sleep(1)
|
||||
|
||||
deadline = time.time() + 5
|
||||
while time.time() < deadline:
|
||||
if not any(process_alive(pid) for pid in pids):
|
||||
break
|
||||
time.sleep(0.2)
|
||||
|
||||
# Force kill if still running
|
||||
pids = find_server_process(port)
|
||||
for pid in pids:
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
print(f"Sent SIGKILL to process {pid}")
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
if process_alive(pid):
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
print(f"Sent SIGKILL to {what} pid {pid}")
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
def is_server_ready(port: int, timeout: float = 0.5) -> bool:
|
||||
def is_server_ready(port: int, timeout: float = 2.0) -> bool:
|
||||
"""Check if server is accepting connections."""
|
||||
try:
|
||||
with socket.create_connection(("127.0.0.1", port), timeout=timeout):
|
||||
@@ -84,9 +165,15 @@ def is_server_ready(port: int, timeout: float = 0.5) -> bool:
|
||||
def wait_for_server(port: int, timeout: int = 30) -> bool:
|
||||
"""Wait for server to become ready."""
|
||||
start = time.time()
|
||||
last_report = 0.0
|
||||
while time.time() - start < timeout:
|
||||
if is_server_ready(port):
|
||||
return True
|
||||
# Report progress every ~5s so a slow boot is visible, not silent.
|
||||
elapsed = time.time() - start
|
||||
if elapsed - last_report >= 5:
|
||||
print(f" ...still waiting ({int(elapsed)}s/{timeout}s)")
|
||||
last_report = elapsed
|
||||
time.sleep(0.5)
|
||||
return False
|
||||
|
||||
@@ -99,68 +186,148 @@ def main() -> int:
|
||||
"--port",
|
||||
type=int,
|
||||
default=8188,
|
||||
help="Server port (default: 8188)"
|
||||
help="Server port (default: 8188)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--restart",
|
||||
action="store_true",
|
||||
help="Kill existing server before starting"
|
||||
help="Kill the E2E server previously managed by this script for the port "
|
||||
"(tracked via pidfile) before starting; refuse to kill unrelated processes",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--wait",
|
||||
action="store_true",
|
||||
help="Wait for server to be ready before exiting"
|
||||
help="Wait for server to be ready before exiting",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=int,
|
||||
default=30,
|
||||
help="Timeout for waiting (default: 30)"
|
||||
help="Timeout for waiting (default: 30)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--detach",
|
||||
action="store_true",
|
||||
help="Launch the server fully detached (setsid-style) so it survives shell "
|
||||
"death. REQUIRED for E2E: a plain background process dies with the shell",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
# Get project root (parent of .agents directory)
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
skill_dir = os.path.dirname(script_dir)
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.dirname(skill_dir)))
|
||||
|
||||
# Restart if requested
|
||||
|
||||
managed_pids = read_managed_pids(args.port)
|
||||
|
||||
# Restart if requested: kill ONLY managed PIDs.
|
||||
if args.restart:
|
||||
print(f"Killing existing server on port {args.port}...")
|
||||
kill_server(args.port)
|
||||
alive_managed = [pid for pid in managed_pids if process_alive(pid)]
|
||||
if alive_managed:
|
||||
print(
|
||||
f"Killing E2E server previously started by this script on port "
|
||||
f"{args.port} ({describe_processes(alive_managed)})..."
|
||||
)
|
||||
kill_pids(alive_managed, "managed E2E server")
|
||||
else:
|
||||
print(
|
||||
f"No live managed E2E server for port {args.port} "
|
||||
f"(pidfile: {pidfile_path(args.port)})"
|
||||
)
|
||||
time.sleep(1)
|
||||
|
||||
# Check if already running
|
||||
if is_server_ready(args.port):
|
||||
print(f"Server already running on port {args.port}")
|
||||
return 0
|
||||
|
||||
# Refuse to kill anything the script did not manage.
|
||||
remaining = find_server_process(args.port)
|
||||
if remaining:
|
||||
print(
|
||||
f"ERROR: port {args.port} is still held by process(es) this script "
|
||||
f"did not start: {describe_processes(remaining)}."
|
||||
)
|
||||
print(
|
||||
"These may be unrelated (e.g. a live ComfyUI). The script will NOT "
|
||||
"kill them. Pick a different --port, or stop them manually if you "
|
||||
"are certain they are stale E2E servers."
|
||||
)
|
||||
return 2
|
||||
clear_managed_pids(args.port)
|
||||
|
||||
# Port conflict check before starting: never blind-kill.
|
||||
port_pids = find_server_process(args.port)
|
||||
if port_pids:
|
||||
alive_managed = [pid for pid in port_pids if pid in managed_pids]
|
||||
unmanaged = [pid for pid in port_pids if pid not in managed_pids]
|
||||
if alive_managed and not unmanaged:
|
||||
print(
|
||||
f"Server already running on port {args.port} "
|
||||
f"({describe_processes(alive_managed)}, started by this script). "
|
||||
f"Use --restart to recycle it."
|
||||
)
|
||||
return 0
|
||||
print(
|
||||
f"ERROR: port {args.port} is already in use by process(es): "
|
||||
f"{describe_processes(port_pids)}."
|
||||
)
|
||||
print(
|
||||
"This is likely an unrelated process (e.g. a live ComfyUI holding 8188). "
|
||||
"The script will NOT kill it. Pick a free port with --port, e.g. 8199."
|
||||
)
|
||||
return 2
|
||||
|
||||
# Start server
|
||||
print(f"Starting LoRa Manager standalone server on port {args.port}...")
|
||||
cmd = [sys.executable, "standalone.py", "--port", str(args.port)]
|
||||
|
||||
# Start in background
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
cwd=project_root,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
start_new_session=True
|
||||
)
|
||||
|
||||
print(f"Server process started with PID {process.pid}")
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"standalone.py",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
str(args.port),
|
||||
]
|
||||
|
||||
if args.detach:
|
||||
# Fully detached launch: new session (setsid), no controlling terminal,
|
||||
# stdin from /dev/null, stdout/stderr to a log file. Survives the shell.
|
||||
log_dir = os.path.join(script_dir, "logs")
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
log_path = os.path.join(log_dir, f"server-{args.port}.log")
|
||||
with open(log_path, "ab") as log_fh:
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
cwd=project_root,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=log_fh,
|
||||
stderr=subprocess.STDOUT,
|
||||
start_new_session=True,
|
||||
close_fds=True,
|
||||
)
|
||||
print(f"Detached server process started with PID {process.pid} (setsid)")
|
||||
print(f"Log: {log_path}")
|
||||
else:
|
||||
# Plain background process (legacy behavior): dies with the shell.
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
cwd=project_root,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
start_new_session=True,
|
||||
)
|
||||
print(f"Server process started with PID {process.pid}")
|
||||
print(
|
||||
"NOTE: not detached — this process dies when the launching shell exits. "
|
||||
"For E2E use --detach."
|
||||
)
|
||||
|
||||
write_managed_pids(args.port, [process.pid])
|
||||
|
||||
# Wait for ready if requested
|
||||
if args.wait:
|
||||
print(f"Waiting for server to be ready (timeout: {args.timeout}s)...")
|
||||
if wait_for_server(args.port, args.timeout):
|
||||
print(f"Server ready at http://127.0.0.1:{args.port}/loras")
|
||||
return 0
|
||||
else:
|
||||
print(f"Timeout waiting for server")
|
||||
return 1
|
||||
|
||||
print(f"Timeout waiting for server on port {args.port}")
|
||||
return 1
|
||||
|
||||
print(f"Server starting at http://127.0.0.1:{args.port}/loras")
|
||||
return 0
|
||||
|
||||
|
||||
@@ -1,15 +1,20 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Wait for LoRa Manager server to become ready.
|
||||
|
||||
Timeout is configurable via --timeout (default 30s); the script polls the port
|
||||
until the server accepts connections or the timeout expires.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def is_server_ready(port: int, timeout: float = 0.5) -> bool:
|
||||
def is_server_ready(port: int, timeout: float = 2.0) -> bool:
|
||||
"""Check if server is accepting connections."""
|
||||
try:
|
||||
with socket.create_connection(("127.0.0.1", port), timeout=timeout):
|
||||
@@ -21,9 +26,15 @@ def is_server_ready(port: int, timeout: float = 0.5) -> bool:
|
||||
def wait_for_server(port: int, timeout: int = 30) -> bool:
|
||||
"""Wait for server to become ready."""
|
||||
start = time.time()
|
||||
last_report = 0.0
|
||||
while time.time() - start < timeout:
|
||||
if is_server_ready(port):
|
||||
return True
|
||||
# Report progress every ~5s so a slow boot is visible, not silent.
|
||||
elapsed = time.time() - start
|
||||
if elapsed - last_report >= 5:
|
||||
print(f" ...still waiting ({int(elapsed)}s/{timeout}s)")
|
||||
last_report = elapsed
|
||||
time.sleep(0.5)
|
||||
return False
|
||||
|
||||
@@ -36,25 +47,24 @@ def main() -> int:
|
||||
"--port",
|
||||
type=int,
|
||||
default=8188,
|
||||
help="Server port (default: 8188)"
|
||||
help="Server port (default: 8188)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=int,
|
||||
default=30,
|
||||
help="Timeout in seconds (default: 30)"
|
||||
help="Timeout in seconds (default: 30)",
|
||||
)
|
||||
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
print(f"Waiting for server on port {args.port} (timeout: {args.timeout}s)...")
|
||||
|
||||
|
||||
if wait_for_server(args.port, args.timeout):
|
||||
print(f"Server ready at http://127.0.0.1:{args.port}/loras")
|
||||
return 0
|
||||
else:
|
||||
print(f"Timeout: Server not ready after {args.timeout}s")
|
||||
return 1
|
||||
print(f"Timeout: Server not ready after {args.timeout}s")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -25,6 +25,7 @@ model_cache/
|
||||
reasonix.toml
|
||||
.reasonix/
|
||||
.codegraph/
|
||||
.playwright-mcp/
|
||||
|
||||
# Vue widgets development cache (but keep build output)
|
||||
vue-widgets/node_modules/
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
---
|
||||
slug: undo-delete-staging
|
||||
status: drafting
|
||||
intent: clear
|
||||
review_required: false
|
||||
pending-action: write .omo/plans/undo-delete-staging.md
|
||||
approach: "Option B: delayed physical deletion with Undo. Backend: same-volume rename to per-root staging dir (.lm-pending-delete/) [updated 2026-08: model staging moved to a SIBLING dir inside each deleted model's own folder — see 'Symlink fix (2026-08)' under Decisions] + manifest JSON (batch_id, expires_at, staged->original map) + purge (30s TTL timer + startup sweep + opportunistic) + undo-delete endpoint + settings toggle 'skip undo'. Small files (recipes: JSON+preview) copy to global staging under settings dir instead of rename. Frontend: extend toast system with action button + 30s countdown; delete flows (single model / recipe / bulk / duplicates) consume batch_id from delete response and show Undo toast; expired undo -> 'undo expired' toast. Plus confirm-modal friction (C-friction, NO type-to-confirm): delete button delay-activation 1.5s + modal shows file size 'will free X GB' + Cancel gets initial focus. i18n keys + sync_translation_keys.py."
|
||||
---
|
||||
|
||||
# Draft: undo-delete-staging
|
||||
|
||||
## Components (topology ledger)
|
||||
<!-- Lock the SHAPE before depth. One row per top-level component that can succeed or fail independently. -->
|
||||
<!-- id | outcome (one line) | status: active|deferred | evidence path -->
|
||||
- backend staging module (stage/purge/undo + manifest + per-volume dir resolution) | new module, active | pending exploration: model_lifecycle_service.py delete_model / delete_model_artifacts
|
||||
- delete endpoints return batch_id (model/recipe/bulk/duplicates) | active | pending exploration: handlers + response shapes
|
||||
- undo-delete HTTP endpoint + route registration | active | pending exploration: route registrar pattern
|
||||
- purge scheduling (30s timer + startup sweep + opportunistic) | active | pending exploration: app on_startup hooks
|
||||
- settings toggle "skip undo window" | active | pending exploration: settings service read pattern
|
||||
- frontend toast extension (action button + countdown) | active | pending exploration: showToast impl
|
||||
- frontend delete flows consume batch_id + Undo toast | active | pending exploration: call sites
|
||||
- confirm-modal friction (delay-activate + size display + cancel focus) | active | pending exploration: modal focus behavior
|
||||
- i18n keys + sync_translation_keys.py | active | known
|
||||
|
||||
## Open assumptions (announced defaults)
|
||||
<!-- Record any default you adopt instead of asking, so the user can veto it at the gate. -->
|
||||
<!-- assumption | adopted default | rationale | reversible? -->
|
||||
- Undo window TTL = 30s | 30s balances space-freeing intent vs accident recovery | yes (constant)
|
||||
- Staging dir name: `.lm-pending-delete/` under each model root; recipes: `{settings_dir}/.lm-pending-delete/` | hidden, same-volume [updated 2026-08: same-volume is now guaranteed by sibling staging inside the model's own folder, not by the root location], consistent | yes
|
||||
- Staging failure falls back to existing hard delete | user intent is delete; staging is best-effort; hard delete likely fails identically under same conditions | yes
|
||||
- Purge on startup uses expires_at (not purge-all) so a <30s restart with live tab can still undo | robust, matches client-side timer | yes
|
||||
- Settings toggle label: "Delete permanently immediately (skip undo window)" | power users freeing space | yes
|
||||
- C-friction: delete button enabled after 1.5s + modal shows freed size; NO type-to-confirm (user vetoed) | user explicitly rejected type-to-confirm | n/a
|
||||
- Bulk/duplicates delete: one batch id for whole action, one undo restores all | simplest consistent semantics | yes
|
||||
|
||||
## Findings (cited - path:lines)
|
||||
|
||||
### Backend
|
||||
- `delete_model_artifacts` (py/services/model_lifecycle_service.py:19-48) = physical delete via os.remove; patterns: main file + `{name}.metadata.json` + PREVIEW_EXTENSIONS (py/utils/constants.py:22-37). ALSO called by ModelScanner.bulk_delete_models (py/services/model_scanner.py:2221) - single swap point covers bulk models.
|
||||
- `ModelLifecycleService.delete_model` (model_lifecycle_service.py:101-154): fetches `cached_entry` (111-116) - SNAPSHOT available for cache restore; after delete: cache.raw_data removal + resort + bump_cache_version (136-143), `_hash_index.remove_by_path` (145-146), `_sync_update_for_model` (148; update-service only, no recipe JSON rewrites - recipe refs are hash-based, re-resolve on restore), `_persist_current_cache` (150-152), returns `{"success": True, "deleted_files": [...]}` (154).
|
||||
- Handler `delete_model` (py/routes/handlers/model_handlers.py:478-492): POST /api/lm/{prefix}/delete; response passthrough; `_broadcast_models_changed()` (57-74) after success; 400 `{"success":false,"error"}`; 500 plain text.
|
||||
- Recipe delete: handler (recipe_handlers.py:1422-1438) DELETE /api/lm/recipe/{recipe_id} -> persistence_service.delete_recipe (py/services/recipes/persistence_service.py:193-209): os.remove(recipe_json_path) + os.remove(image_path) (204-206), recipe_scanner.remove_recipe (208), returns `{"success": true, "message": ...}`. PersistenceResult dataclass (20-25).
|
||||
- Bulk models: POST /api/lm/{prefix}/bulk-delete (model_route_registrar.py:39) -> handler (model_handlers.py:974-994) -> lifecycle_service.bulk_delete_models (model_lifecycle_service.py:308-318) -> scanner.bulk_delete_models (model_scanner.py:2181-2269) which calls delete_model_artifacts per file (2221) + `_batch_update_cache_for_deleted_models` (2271-2335); response `{"success","status","total_deleted","total_attempted","cache_updated","results"}` (2254-2269).
|
||||
- Bulk recipes: POST /api/lm/recipes/bulk-delete (recipe_route_registrar.py:50) -> handler (recipe_handlers.py:1554-1573) -> persistence_service.bulk_delete (persistence_service.py:439-482): per-id os.remove x2 (464-466), recipe_scanner.bulk_remove (472); response `{"success","deleted","failed","total_deleted","total_failed"}` (474-482).
|
||||
- Duplicates: NO dedicated delete endpoints (find-only: GET /api/lm/{prefix}/find-duplicates model_route_registrar.py:59, GET /api/lm/recipes/find-duplicates recipe_route_registrar.py:49). Duplicate deletion reuses bulk-delete endpoints.
|
||||
- Startup hooks: lora_manager.py:183-187 `app.on_startup.append(lambda app: cls._initialize_services())` (ComfyUI mode, app = PromptServer.instance.app at :78); standalone.py:370-374 same (StandaloneLoraManager.add_routes). Background tasks: `asyncio.create_task(name=...)` (lora_manager.py:224-239; recipe_handlers.py:793). Singleton+asyncio.Lock pattern: model_scanner.py:40-63.
|
||||
- Settings: DEFAULT_SETTINGS (py/services/settings_manager.py:57-119), `get(key, default)` (1390-1392), get_settings_manager() (2215-2228), reset_settings_manager() (2231). Typed-bool getter example: get_skip_previously_downloaded_model_versions (1253-1262). Handlers: base_model_routes.py:70, base_recipe_routes.py:54.
|
||||
- Model roots: ModelScanner.get_model_roots base NotImplementedError (model_scanner.py:1073-1075); impls lora_scanner.py:31-45, checkpoint_scanner.py:428-441, embedding_scanner.py:24-36. `_find_root_for_file(file_path)` (model_scanner.py:1108-1124) returns containing root - for per-root staging dir computation [updated 2026-08: staging no longer uses the containing root; batches are siblings inside the model's own folder]. Business-path rule (AGENTS.md): use os.path.abspath, never realpath, for staging/undo routing.
|
||||
- Cache restore methods: ModelCache has raw_data + resort (conftest mocks: tests/conftest.py:144-154); ModelHashIndex.add_entry(sha256, file_path, autov3) (py/services/model_hash_index.py:16); RecipeScanner.add_recipe(recipe_data) (recipe_scanner.py:2136) -> recipe_cache.add_recipe (recipe_cache.py:64). No single-file incremental model rescan - use snapshot restore instead of rescan.
|
||||
- Route registrar: model_route_registrar.py:177 add_route(method, path, handler), :180 add_prefixed_route - undo endpoint can be a non-prefixed route via add_route.
|
||||
- Tests: tests/services/test_model_lifecycle_service.py (inline tmp_path files, per-test stub scanners ScannerForDelete/VersionAwareScanner etc); conftest MockScanner/MockCache/MockHashIndex (tests/conftest.py:134-212); integration fixtures tests/integration/conftest.py; lifecycle hook tests tests/routes/test_lora_manager_lifecycle.py:177-178, tests/standalone/test_standalone_server.py:83-84.
|
||||
|
||||
### Frontend
|
||||
- 5 delete call sites:
|
||||
a) Single model: static/js/utils/modalUtils.js confirmDelete (27-42) -> getModelApiClient().deleteModel(path); ignores return.
|
||||
b) Recipe single: static/js/components/RecipeCard.js confirmDeleteRecipe (405-449) - RAW fetch DELETE /api/lm/recipe/{id}, checks only response.ok, showToast toast.recipes.deletedSuccessfully, state.virtualScroller.removeItemByFilePath.
|
||||
c) Bulk: static/js/managers/BulkManager.js confirmBulkDelete (633-672) -> getActiveApiClient() (134-142) -> bulkDeleteModels(filePaths); reads result.cancelled/success/deleted_count/error.
|
||||
d) Recipe duplicates: static/js/components/DuplicatesManager.js confirmDeleteDuplicates (457-494) - RAW fetch POST /api/lm/recipes/bulk-delete, reads data.success/data.total_deleted, exitDuplicateMode().
|
||||
e) Model duplicates: static/js/components/ModelDuplicatesManager.js confirmDeleteDuplicates (710-776) - RAW fetch POST /api/lm/{type}/bulk-delete, reads data.total_deleted, then resetAndReload(true) + find-duplicates re-check.
|
||||
Bonus: static/js/components/shared/ModelVersionsTab.js:1136-1144 client.deleteModel (ignores return).
|
||||
- API clients: BaseModelApiClient.deleteModel (static/js/api/baseModelApi.js:184-216) returns true/false, shows its own toasts, does removeItemByFilePath inside; bulkDeleteModels (1591-1642) returns {success, deleted_count, failed_count, errors} or {success:false, cancelled:true}; RecipeSidebarApiClient.bulkDeleteModels (recipeApi.js:623-664) returns {success, deleted_count: total_deleted, ...}. Endpoint map apiConfig.js:56,64.
|
||||
- Toast: showToast(key, params={}, type='info', fallback=null) (static/js/utils/uiHelpers.js:136-193) - textContent only, NO action/button support; durations 2000/5000ms; CSS static/css/components/toast.css (.toast flex gap:12px - button can be added). Closest action pattern: bannerService.registerBanner actions array + onRegister (static/js/managers/BannerService.js; used uiHelpers.js:18-57).
|
||||
- i18n: locales/en.json delete keys (1303-1314 bulkDelete, 1945-1948 recipes, 1987-1991 models, 2124-2130 duplicates, 2166-2170 toast.api); t()/interpolate (static/js/i18n/index.js:193-248); translate wrapper (utils/i18nHelpers.js:13-23); sync script scripts/sync_translation_keys.py (en reference, [TODO: Translate] placeholders).
|
||||
- Refresh after undo: recipes -> window.recipeManager.loadRecipes(true) (recipes.js:359; used by FilterManager.js:752 etc) or refreshRecipes (recipeApi.js:308); models -> resetAndReload(true) from modelApiFactory (used by ModelDuplicatesManager.js:740).
|
||||
- Size for modal: card.dataset.file_size (ModelCard.js:467), formatFileSize (ModelModal.js:615).
|
||||
- Tests: tests/frontend/utils/uiHelpers.dom.test.js (toast), api/recipeApi.bulk.test.js, components/duplicatesManager.test.js, components/modelDuplicatesManager.test.js, pages/*Page.test.js, i18n tests tests/i18n/test_i18n.py.
|
||||
|
||||
## Decisions (with rationale)
|
||||
|
||||
1. Same-volume rename staging for model files (atomic, no copy cost for multi-GB files); cross-volume rename forbidden. [CORRECTED 2026-08: "same-volume because under the containing root" was only true for plain directories — nested symlinked subdirs could cross volumes. Superseded by sibling staging: `.lm-pending-delete/<batch_id>/` inside the deleted model's own folder makes stage/undo same-device by construction; see "Symlink fix (2026-08)" below.]
|
||||
2. Copy-to-global-staging for recipes (small files; avoids recipe JSON vs preview image cross-volume problem).
|
||||
3. Manifest JSON files are the only state - no DB changes. Manifest includes model cached_entry snapshot for exact cache restore (no rescan needed).
|
||||
4. Undo endpoint returns restored paths; expired batch -> 404-style error -> frontend 'undo expired' toast.
|
||||
5. Skip-undo setting honored server-side (no batch_id in response -> no undo toast client-side).
|
||||
6. Staging failure falls back to existing hard delete (best-effort undo, never blocks delete).
|
||||
7. Undo window TTL = 30s constant (PENDING_DELETE_TTL_SECONDS); startup sweep uses expires_at (survives restart; browser-tab timer survives).
|
||||
8. Purge triple-trigger: per-batch asyncio timer task + on_startup sweep + opportunistic purge at each stage/undo.
|
||||
9. Frontend: new showActionToast (keep showToast signature untouched; extract shared createToastElement/appendToast internals); undo click -> shared handleUndoDelete(batchId, refreshFn); full list refresh after undo (recipes: window.recipeManager.loadRecipes(true); models: resetAndReload(true)).
|
||||
10. C-friction wave (NO type-to-confirm - user vetoed): delete buttons delay-activate 1.5s after modal open, initial focus on Cancel, model delete modal gains "permanently deleted from disk" warning + file size display (card.dataset.file_size + formatFileSize).
|
||||
11. Model cache restore on undo: append snapshot to cache.raw_data (dedupe by file_path) + resort + bump_cache_version + _persist_current_cache + _hash_index.add_entry + _broadcast_models_changed. Recipe restore: copy back files + recipe_scanner.add_recipe(recipe_data loaded from restored JSON).
|
||||
|
||||
### Symlink fix (2026-08)
|
||||
|
||||
Post-execution addendum (plan `.omo/plans/undo-delete-symlink-fix.md`, commits 5fd4946b / 0c00ee22):
|
||||
|
||||
12. Model staging moved from `<model_root>/.lm-pending-delete/<batch_id>/` to `<model_dir>/.lm-pending-delete/<batch_id>/` (sibling of the model artifacts, inside the deleted model's own folder). Stage/undo renames are same-device BY CONSTRUCTION — EXDEV is impossible even when the business path traverses nested symlinks to other volumes (the decision-1 "containing root" guarantee covered only plain directories). EXDEV remains possible only for cross-volume merges, which keep the batch_ids-array fallback. Accepted edge: deleting the model's whole FOLDER during the 30s window destroys that batch (undo returns 404). Batch discovery uses an in-memory registry (`_known_batch_dirs`) with a startup reconciliation scan (`purge_expired(scan_roots=True)`) covering restarts and crash leftovers. Recipe batches unchanged (copy-based settings-dir staging with the `_restore_file` EXDEV fallback).
|
||||
|
||||
## Scope IN
|
||||
|
||||
- Model single delete (model_handlers delete_model / model_lifecycle_service)
|
||||
- Recipe delete (recipe_handlers delete_recipe / persistence_service)
|
||||
- Bulk delete (models scanner + recipes persistence) + duplicates (reuse bulk endpoints)
|
||||
- Undo endpoint POST /api/lm/undo-delete (models + recipes, one batch space)
|
||||
- Purge: timer + startup sweep + opportunistic
|
||||
- Settings toggle delete_undo_enabled + settings page checkbox
|
||||
- Frontend: showActionToast + all 5 delete flows + shared undo handler
|
||||
- C-friction modal changes (delay-activate + cancel focus + warning copy + size display)
|
||||
- i18n keys + sync_translation_keys.py
|
||||
- Backend + frontend tests
|
||||
|
||||
## Scope OUT (Must NOT have)
|
||||
|
||||
- NO type-to-confirm / hold-to-confirm friction (user vetoed)
|
||||
- NO OS trash integration (send2trash) in this iteration
|
||||
- NO persistent recycle-bin UI (no trash browsing page)
|
||||
- NO changes to exclude/unexclude flow
|
||||
- NO DB migrations
|
||||
- NO new dependencies (no send2trash)
|
||||
- NO changes to download flows
|
||||
- NO recipe-JSON rewriting on model undo (hash-based refs re-resolve themselves)
|
||||
|
||||
## Open questions
|
||||
|
||||
None - all implementation details resolved by exploration. Design decisions settled in conversation (B+C, no type-to-confirm).
|
||||
|
||||
## Approval gate
|
||||
status: approved
|
||||
<!-- Approach approved -> rerun scaffold without --draft-only, run Metis gap analysis, APPEND todo batches, fill TL;DR last, run structural self-check, then Phase 4 handoff. -->
|
||||
|
||||
## Review round state (ulw-plan-review-round-state-contract)
|
||||
```json
|
||||
{
|
||||
"transition": "replace",
|
||||
"phase": "review_round_initialized",
|
||||
"applies_when": ["retry_after_plan_change"],
|
||||
"atomic": true,
|
||||
"review_required": true,
|
||||
"plan_path": ".omo/plans/undo-delete-staging.md",
|
||||
"plan_sha256": "8cf7c9be38a76d8ef1fb832aba043d6d7e82b60465b1bf28c6eafa7045117adc",
|
||||
"review_round_id": "rr-undo-del-20260811-006",
|
||||
"round_status": "active",
|
||||
"pending-action": "review .omo/plans/undo-delete-staging.md",
|
||||
"review": {
|
||||
"momus": { "status": "pending", "workspace_root": "/mnt/data/reinstall-backup-2026-04-12/data/workspace/ComfyUI/custom_nodes/ComfyUI-Lora-Manager", "runtime_home": null, "target": ".omo/plans/undo-delete-staging.md", "round_id": "rr-undo-del-20260811-006", "plan_sha256": "8cf7c9be38a76d8ef1fb832aba043d6d7e82b60465b1bf28c6eafa7045117adc", "launch_id": null, "session": null, "result": null },
|
||||
"independent": { "status": "pending", "workspace_root": "/mnt/data/reinstall-backup-2026-04-12/data/workspace/ComfyUI/custom_nodes/ComfyUI-Lora-Manager", "runtime_home": null, "target": ".omo/plans/undo-delete-staging.md", "round_id": "rr-undo-del-20260811-006", "plan_sha256": "8cf7c9be38a76d8ef1fb832aba043d6d7e82b60465b1bf28c6eafa7045117adc", "launch_id": null, "session": null, "result": null }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Review results + fix/retry ledger
|
||||
|
||||
### Round 1 (rr-undo-del-20260811-001, plan sha256 6c52bf99...)
|
||||
- momus: APPROVE (non-blocking notes: todo1+7 duplicate DEFAULT_SETTINGS key -> fixed todo 7 to verify-only; "batch_ids" plural in todos 8/9 acceptance -> fixed; purge OSError note -> folded into todo 1 purge semantics)
|
||||
- independent (oracle): CHANGES_REQUESTED
|
||||
- BLOCKING S1: scanner walks would index .lm-pending-delete staged files as ghost entries -> fixed: todo 1 now mandates scanner walk exclusion at model_scanner.py:706/:867/:1404/_process_model_file + acceptance (o) scanner-visibility test
|
||||
- BLOCKING S2: manifest lacks model_type, undo could restore into wrong cache/hash index -> fixed: manifest now carries model_type + todo 5 resolves per-type scanner via registrar pattern + acceptance (b) checkpoint-batch test
|
||||
- S3 merged-batch expires_at re-anchor -> fixed: merge_batches re-anchors now+TTL in todo 1 + todo 3/4 assertions
|
||||
- S4 manifest-less dir policy -> fixed: quarantine to <batch_id>.orphaned, never delete (todo 1 + acceptance g)
|
||||
- S5 partial-undo retry semantics -> fixed: per-entry restored flag write-through + retry test (acceptance e)
|
||||
- S6 purge locked-file failure semantics -> fixed: skip file, keep batch, never rmtree past errors (todo 1 + acceptance i)
|
||||
- T8 undo-after-restart test -> fixed: todo 5 acceptance (f)
|
||||
- T9 recipe undo -> re-delete test -> fixed: todo 5 acceptance (h)
|
||||
- T7 rescan-stale-entry test -> fixed: todo 5 acceptance (g)
|
||||
- Route registration pinned to shared routes class per mode (NOT per-model-type registrar which registers 3x) -> fixed: todo 5 now creates py/routes/pending_delete_routes.py registered once in lora_manager.py:170-172 + standalone.py:356-358 + duplicate-route test (e)
|
||||
- Version-index staleness on single-delete undo -> fixed: todo 5 follows bulk cache-update pattern incl. rebuild_version_index (model_scanner.py:2324)
|
||||
- Cancelled-bulk batch_id frontend handling -> fixed: todo 9 shows action toast on cancelled+staged-subset
|
||||
- Single-instance assumption -> added to Scope OUT
|
||||
- Occupied-refusal loss UX -> accepted-intent documented in success criteria + modal copy
|
||||
|
||||
### Round 2 (rr-undo-del-20260811-002, plan sha256 f3d52235...)
|
||||
- momus: APPROVE (all 12 round-1 fixes verified present; zero dead references; non-blocking nits only)
|
||||
- independent (oracle): CHANGES_REQUESTED
|
||||
- BLOCK-1: merge_batches file-movement semantics unspecified (silent data-loss vector) -> fixed: todo 1 now specifies move-into-winner-dir + entry re-point + loser-dirs-removed-only-when-empty + abort-on-move-failure (all batches intact) + merge inside service lock + acceptance (k) file-survival assertions + acceptance (l) merge-failure abort test
|
||||
- BLOCK-2: same-file parallel edits within waves (todo 5 vs 6 on lora_manager.py; todo 8 vs 9 on baseModelApi.js) -> fixed: waves/matrix now serialize 5->6 and 8->9 with explicit reasons; matrix updated
|
||||
- Recommended: checkpoint_scanner.py:331 exclusion -> fixed (todo 1 + acceptance p); S5 pre-check skips restored:true entries -> fixed (todo 1); _tags_count restore on undo -> fixed (todo 5 + acceptance j); undo-blind flows documented (ModelVersionsTab + misc_handlers:2456) -> fixed (todo 8 note + Scope OUT); merge-failure no-merge fallback contract (batch_ids array) -> fixed (todos 3/4/9)
|
||||
|
||||
### Round 3 (rr-undo-del-20260811-003, plan sha256 8f2dfd46...)
|
||||
- momus: APPROVE (all round-2 fixes verified present + spot-checked refs; no new contradictions)
|
||||
- independent (oracle): CHANGES_REQUESTED
|
||||
- BLOCKING A: merged batches never timer-purged after re-anchor (winner's original timer no-ops at old expiry; no fresh timer for re-anchored expiry; idle server -> merged batch lingers, violating "30s purge" success criterion; affects EVERY bulk delete) -> fixed: todo 1 merge_batches now ARMS A FRESH PURGE TIMER for the winner with re-anchored expiry + acceptance (q) fresh-timer test + purge_expired must enumerate ALL scanner types' roots (explicit in todo 1)
|
||||
- BLOCKING B: dependency matrix contradicted same-file policy for todos 8/9<->11 (5 shared files) and 12<->11 -> fixed: todo 11 now "Blocked by: 8, 9 (same files...)"; todo 12 blocked by 11 (sync after 11); wave text updated (11, then 12 AFTER 11); "Can parallelize with" columns corrected
|
||||
- BLOCKING C: frontend batch_ids sequential-undo fallback has NO test + merge->undo loser-restore + merge->purge assertions missing -> fixed: todo 9 acceptance now tests the batch_ids fallback path; todo 1 acceptance now has (k2)/(k3)
|
||||
- Notes folded: sub-second toast-tail expiry race accepted; EXDEV fallback = NORMAL path for cross-volume bulks [annotated 2026-08: after the sibling-staging fix, EXDEV can only arise during cross-volume MERGES, never during single stage/undo renames]
|
||||
|
||||
### Round 4 (rr-undo-del-20260811-004, plan sha256 179e7ff7...)
|
||||
- momus: APPROVE (round-3 fixes verified; one non-blocking nit: todo 11 inline "Blocked by: —" stale -> fixed to "8, 9")
|
||||
- independent (oracle): CHANGES_REQUESTED
|
||||
- BLOCKING GAP-1 (NEW, introduced by round-3 fix): todo 8 handleUndoDelete always-refresh/always-toast contract contradicted todo 9's sequential loop "exactly ONE final refresh" -> fixed: handleUndoDelete(batchId, refreshFn, {showToast, refresh}) suppression options; todo 9 loop uses suppressed calls + one final refresh/toast; acceptance extended (loop failure mid-way -> stop + error toast + no final refresh; 404 body discrimination expired vs occupied)
|
||||
- BLOCKING GAP-2: no cross-type purge enumeration test -> fixed: todo 1 acceptance (r) purges expired batches across lora root + checkpoint root + recipe staging dir in one call
|
||||
- Non-blocking folded: GAP-3 404-copy discrimination -> fixed in todo 8 (d); GAP-4 merge partial-failure rollback direction (move back + restore manifests, extended (l) asserts sequential constituent undo still restores everything) -> fixed in todo 1; GAP-5 post-restart timer-loss residual gap documented -> fixed in todo 6; GAP-6 usage_stats.py:424 walk added to exclusion mandate + todo 5 acceptance (k) embeddings undo test
|
||||
|
||||
### Round 5 (rr-undo-del-20260811-005, plan sha256 dfaa39ea...)
|
||||
- momus: APPROVE (all round-4 fixes verified; no new contradictions)
|
||||
- independent (oracle): CHANGES_REQUESTED
|
||||
- BLOCK-1: lock-ordering deadlock ambiguity (asyncio.Lock not re-entrant: opportunistic purge_expired called while stage/undo hold the lock would deadlock on first use) -> fixed: todo 1 now has explicit LOCK HIERARCHY (lock acquired ONLY by stage/merge/undo/purge_batch; purge_expired is lock-free and must be called BEFORE lock acquisition); todo 6 (c) updated with the same rule + acceptance (u) lock-no-deadlock test
|
||||
- BLOCK-2: purge edge semantics unspecified -> fixed: purge_batch treats missing staged files (partially-restored batches) as already-purged (FileNotFoundError silent no-op); sweep skips `.orphaned`-suffixed dirs (quarantine is terminal); acceptance (s) partially-restored purge + (t) quarantine-terminal tests
|
||||
- Non-blocking folded: todo 2/3 test-file collision -> todo 3's bulk tests moved to tests/services/test_model_scanner.py; todo 9 (d) DuplicatesManager refreshFn stated explicitly (recipes loadRecipes / models resetAndReload); modal-copy + bulk-count trade-offs acknowledged in success criteria; acceptance (r) extended with embeddings root
|
||||
|
||||
### Round 6 (rr-undo-del-20260811-006, plan sha256 8cf7c9be...)
|
||||
- momus: APPROVE (all round-5 fixes verified; no new contradictions; references verified)
|
||||
- independent (oracle): APPROVE — no blocking issues; all round-5 items fixed with working, tested solutions; no new race/data-loss/consistency defects
|
||||
- Deferred optional improvements (non-blocking, recorded for executor awareness; plan file left untouched to preserve the approved digest):
|
||||
1. Tag-count asymmetry: single delete_model never decrements _tags_count (lifecycle 101-154), bulk does (scanner 2297-2303); undo re-increment is exact for bulk, over-counts for single until rescan (cosmetic, self-healing). Optional fix riding in todo 2: decrement tags in the single-delete path to mirror bulk.
|
||||
2. Todo 5 factual nit: ModelCache.resort() already rebuilds the version index — explicit rebuild in undo is belt-and-braces, no action needed.
|
||||
3. Todo 8 premise nit: ModelVersionsTab call ignores deleteModel's return entirely — nothing breaks, no adaptation needed.
|
||||
4. Todo 3's pytest command includes test_model_lifecycle_service.py which todo 2 edits in the same wave — run that file's tests after todo 2 lands.
|
||||
5. merge_batches with a missing/quarantined constituent id: any sane fallback (abort -> batch_ids, or skip missing) acceptable — files stay staged either way.
|
||||
|
||||
## Review lifecycle
|
||||
- rounds: 6 (rr-undo-del-20260811-001..006); final round both lanes APPROVE
|
||||
- final live-plan validation: sha256 = 8cf7c9be38a76d8ef1fb832aba043d6d7e82b60465b1bf28c6eafa7045117adc — MATCHES approved round-6 digest
|
||||
- status: APPROVED — ready for execution handoff ($start-work undo-delete-staging)
|
||||
File diff suppressed because one or more lines are too long
@@ -31,7 +31,7 @@ COVERAGE_FILE=coverage/backend/.coverage pytest \
|
||||
--cov-report=xml:coverage/backend/coverage.xml
|
||||
```
|
||||
|
||||
### Frontend Development (Standalone Web UI)
|
||||
### Frontend Development (LoRA Manager Web UI)
|
||||
|
||||
```bash
|
||||
npm install
|
||||
@@ -154,9 +154,9 @@ npm run test:coverage # Generate coverage report
|
||||
|
||||
## Frontend UI Architecture
|
||||
|
||||
### 1. Standalone Web UI
|
||||
### 1. LoRA Manager Web UI
|
||||
- Location: `./static/` and `./templates/`
|
||||
- Tech: Vanilla JS + CSS, served by standalone server
|
||||
- Tech: Vanilla JS + CSS, served by the hosting server (ComfyUI app in plugin mode, `standalone.py` in standalone mode)
|
||||
- Tests via npm in root directory
|
||||
|
||||
### 2. ComfyUI Custom Node Widgets
|
||||
|
||||
@@ -39,6 +39,7 @@ These fields are present in all model metadata files.
|
||||
| `metadata_source` | string\|null | ❌ No | ✅ Yes | Last provider that supplied metadata (see below) |
|
||||
| `last_checked_at` | float | ❌ No (default: `0`) | ✅ Yes | Unix timestamp of last metadata check |
|
||||
| `hash_status` | string | ❌ No (default: `"completed"`) | ✅ Yes | Hash calculation status: `"pending"`, `"calculating"`, `"completed"`, `"failed"` |
|
||||
| `autov3` | string\|null | ❌ No | ✅ Yes | CivitAI AutoV3 hash (first 12 chars, lowercase hex) sourced from the safetensors embedded metadata (`sshs_model_hash` / `modelspec.hash_sha256`). **Absent** = not yet checked (may be backfilled later); **`null`** = checked but unavailable (header has no recognized hash); **12-char hex string** = value |
|
||||
|
||||
---
|
||||
|
||||
@@ -287,6 +288,7 @@ These fields are automatically synchronized with the filesystem:
|
||||
- `preview_url` — Updated if preview file is moved/removed
|
||||
- `sha256` — Updated during hash calculation (when `hash_status="pending"`)
|
||||
- `hash_status` — Updated during hash calculation
|
||||
- `autov3` — Set when metadata is first created (from safetensors header); may be backfilled later for entries where it is absent
|
||||
- `last_checked_at` — Timestamp of scan
|
||||
- `metadata_source` — Set based on metadata provider
|
||||
|
||||
@@ -345,6 +347,7 @@ These fields can be edited by users at any time through the Lora Manager UI or b
|
||||
| `metadata_source` | `null` |
|
||||
| `last_checked_at` | `0` |
|
||||
| `hash_status` | `"completed"` |
|
||||
| `autov3` | absent (not checked) or `null` (checked, no value) |
|
||||
| `usage_tips` | `"{}"` (LoRA only) |
|
||||
| `model_type` | `"checkpoint"` or `"embedding"` (not present in LoRA models) |
|
||||
|
||||
@@ -354,6 +357,7 @@ These fields can be edited by users at any time through the Lora Manager UI or b
|
||||
|
||||
| Version | Date | Changes |
|
||||
|---------|------|---------|
|
||||
| 1.1 | 2026-08 | Added `autov3` field (CivitAI AutoV3 hash with three-state semantics) |
|
||||
| 1.0 | 2026-03 | Initial schema documentation |
|
||||
|
||||
---
|
||||
|
||||
+2290
-2224
File diff suppressed because it is too large
Load Diff
+69
-3
@@ -186,6 +186,16 @@
|
||||
"cancelled": "Repair cancelled. {count} recipes were repaired.",
|
||||
"error": "Recipe repair failed: {message}"
|
||||
},
|
||||
"rematchRecipes": {
|
||||
"label": "Rematch recipes to local models",
|
||||
"loading": "Rematching recipes to local models...",
|
||||
"success": "Matched {entries} entries across {recipes} recipes",
|
||||
"successErrors": "Matched {entries} entries across {recipes} recipes, {failures} failed",
|
||||
"allFailed": "Rematch failed for {failures} of {total} recipes",
|
||||
"noMatch": "No local match found for {entries} entries in {recipes} recipes",
|
||||
"cancelled": "Rematch cancelled. {recipes} recipes updated ({entries} entries).",
|
||||
"error": "Recipe rematch failed: {message}"
|
||||
},
|
||||
"manageExcludedModels": {
|
||||
"label": "Manage Excluded Models"
|
||||
},
|
||||
@@ -449,6 +459,12 @@
|
||||
"compact": "7 (1080p), 8 (2K), 10 (4K)"
|
||||
},
|
||||
"displayDensityWarning": "Warning: Higher densities may cause performance issues on systems with limited resources.",
|
||||
"recipesLayout": "Recipes Layout",
|
||||
"recipesLayoutHelp": "Choose how recipe cards are arranged: a uniform grid or a masonry (Pinterest-style) layout that preserves each image's aspect ratio.",
|
||||
"recipesLayoutOptions": {
|
||||
"grid": "Grid",
|
||||
"masonry": "Masonry"
|
||||
},
|
||||
"showFolderSidebar": "Show Folder Sidebar",
|
||||
"showFolderSidebarHelp": "Toggle the folder navigation sidebar on model pages. When disabled, the sidebar and hover area stay hidden.",
|
||||
"cardInfoDisplay": "Card Info Display",
|
||||
@@ -678,6 +694,7 @@
|
||||
"deepseek": "DeepSeek",
|
||||
"groq": "Groq",
|
||||
"openrouter": "OpenRouter",
|
||||
"google": "Gemini",
|
||||
"opencode-go": "OpenCode Go",
|
||||
"custom": "Custom (OpenAI-compatible)"
|
||||
},
|
||||
@@ -761,6 +778,7 @@
|
||||
"copyAll": "Copy Selected Syntax",
|
||||
"refreshAll": "Refresh Selected Metadata",
|
||||
"repairMetadata": "Repair Metadata for Selected",
|
||||
"rematchMetadata": "Rematch Selected to Local Models",
|
||||
"reimportMetadata": "Re-import from Source",
|
||||
"checkUpdates": "Check Updates for Selected",
|
||||
"moveAll": "Move Selected to Folder",
|
||||
@@ -773,6 +791,8 @@
|
||||
"deleteAll": "Delete Selected",
|
||||
"downloadMissingLoras": "Download Missing LoRAs",
|
||||
"downloadExamples": "Download Example Images",
|
||||
"downloadMissingExamples": "Download Missing",
|
||||
"reprocessExamples": "Re-process All",
|
||||
"clear": "Clear Selection",
|
||||
"skipMetadataRefreshCount": "Skip ({count} models)",
|
||||
"resumeMetadataRefreshCount": "Resume ({count} models)",
|
||||
@@ -808,10 +828,13 @@
|
||||
"sendToWorkflowReplace": "Send to Workflow (Replace)",
|
||||
"openExamples": "Open Examples Folder",
|
||||
"downloadExamples": "Download Example Images",
|
||||
"downloadMissingExamples": "Download Missing",
|
||||
"reprocessExamples": "Re-process All",
|
||||
"replacePreview": "Replace Preview",
|
||||
"setContentRating": "Set Content Rating",
|
||||
"moveToFolder": "Move to Folder",
|
||||
"repairMetadata": "Repair metadata",
|
||||
"rematchMetadata": "Rematch to local models",
|
||||
"reimportMetadata": "Re-import from Source",
|
||||
"excludeModel": "Exclude Model",
|
||||
"restoreModel": "Restore Model",
|
||||
@@ -912,8 +935,16 @@
|
||||
},
|
||||
"duplicates": {
|
||||
"found": "Found {count} duplicate groups",
|
||||
"noGroups": "No duplicate groups found with the current matching basis",
|
||||
"keepLatest": "Keep Latest Versions",
|
||||
"deleteSelected": "Delete Selected"
|
||||
"deleteSelected": "Delete Selected",
|
||||
"includePromptLabel": "Include prompt in matching",
|
||||
"basis": {
|
||||
"loraCombo": "Matched by: LoRA combination",
|
||||
"loraComboAndPrompt": "Matched by: LoRA combination + prompt",
|
||||
"hintLoraCombo": "Recipes with the same LoRAs at identical strengths are grouped.",
|
||||
"hintPromptIncluded": "Recipes are grouped only when they use the same LoRAs at identical strengths AND have the same prompt."
|
||||
}
|
||||
},
|
||||
"contextMenu": {
|
||||
"copyRecipe": {
|
||||
@@ -1246,8 +1277,13 @@
|
||||
}
|
||||
},
|
||||
"deleteModel": {
|
||||
"freesSpace": "Frees {size}",
|
||||
"title": "Delete Model",
|
||||
"message": "Are you sure you want to delete this model and all associated files?"
|
||||
"message": "Are you sure you want to delete this model and all associated files?",
|
||||
"recoverableWarning": "This will permanently delete the file after 20 seconds unless you undo."
|
||||
},
|
||||
"deleteRecipe": {
|
||||
"recoverableWarning": "This action can be undone for 20 seconds."
|
||||
},
|
||||
"excludeModel": {
|
||||
"title": "Exclude Model",
|
||||
@@ -1577,6 +1613,21 @@
|
||||
"downloadCsv": "Download CSV",
|
||||
"columnModelName": "Model Name",
|
||||
"columnError": "Error"
|
||||
},
|
||||
"downloadBatchSummary": {
|
||||
"title": "Batch Download Summary",
|
||||
"statSuccess": "Success",
|
||||
"statFailed": "Failed",
|
||||
"statTotal": "Total",
|
||||
"successMessage": "All {count} models downloaded successfully",
|
||||
"completedWithErrors": "Completed with errors",
|
||||
"failed": "Download failed",
|
||||
"failedItems": "Failed Items ({count})",
|
||||
"columnName": "Model Name",
|
||||
"columnError": "Error",
|
||||
"close": "Close",
|
||||
"copyReport": "Copy Report",
|
||||
"retryFailed": "Retry Failed ({count})"
|
||||
}
|
||||
},
|
||||
"modelTags": {
|
||||
@@ -1675,6 +1726,7 @@
|
||||
"recipeReplaced": "Recipe replaced in workflow",
|
||||
"recipeFailedToSend": "Failed to send recipe to workflow",
|
||||
"noMatchingNodes": "No compatible nodes available in the current workflow",
|
||||
"noPromptTargets": "No compatible prompt targets in the workflow.\nRight-click a node in ComfyUI → Mark as → Send Prompt Target",
|
||||
"noTargetNodeSelected": "No target node selected",
|
||||
"modelUpdated": "Model updated in workflow",
|
||||
"modelFailed": "Failed to update model node",
|
||||
@@ -1925,6 +1977,12 @@
|
||||
"repairBulkComplete": "Repair complete: {repaired} repaired, {skipped} skipped (of {total})",
|
||||
"repairBulkSkipped": "No repair needed for any of the {total} selected recipes",
|
||||
"repairBulkFailed": "Failed to repair selected recipes: {message}",
|
||||
"rematchComplete": "Matched {entries} entries across {recipes} recipes",
|
||||
"rematchCompleteErrors": "Matched {entries} entries across {recipes} recipes, {failures} failed",
|
||||
"rematchAllFailed": "Rematch failed for {failures} of {total} selected recipes",
|
||||
"rematchUnmatched": "No local match found for {entries} entries in {recipes} recipes",
|
||||
"rematchSkipped": "No rematch needed for any of the {total} selected recipes",
|
||||
"rematchFailed": "Failed to rematch selected recipes: {message}",
|
||||
"reimporting": "Re-importing recipe from source...",
|
||||
"reimportSuccess": "Recipe re-imported successfully",
|
||||
"reimportBulkComplete": "Re-import complete: {completed} re-imported, {failed} failed (of {total})",
|
||||
@@ -2033,7 +2091,6 @@
|
||||
"presetNameTooLong": "Preset name must be {max} characters or less",
|
||||
"presetNameInvalidChars": "Preset name contains invalid characters",
|
||||
"presetNameExists": "A preset with this name already exists",
|
||||
"maxPresetsReached": "Maximum {max} presets allowed. Delete one to add more.",
|
||||
"presetNotFound": "Preset not found",
|
||||
"invalidPreset": "Invalid preset data",
|
||||
"deletePresetFailed": "Failed to delete preset",
|
||||
@@ -2062,6 +2119,14 @@
|
||||
"updateFailed": "Failed to update trigger words",
|
||||
"copyFailed": "Copy failed"
|
||||
},
|
||||
"undo": {
|
||||
"action": "Undo",
|
||||
"deleted": "Deleted {name}",
|
||||
"deletedBulk": "Deleted {count} item(s)",
|
||||
"expired": "Undo window expired. The item was permanently deleted.",
|
||||
"failed": "Undo failed: {error}",
|
||||
"restored": "Item restored"
|
||||
},
|
||||
"virtual": {
|
||||
"loadFailed": "Failed to load items",
|
||||
"loadMoreFailed": "Failed to load more items",
|
||||
@@ -2125,6 +2190,7 @@
|
||||
"fileRenameFailed": "Failed to rename file: {error}",
|
||||
"previewUpdated": "Preview updated successfully",
|
||||
"previewUploadFailed": "Failed to upload preview image",
|
||||
"previewDropInvalid": "Unsupported file type: {name}. Drop an image or MP4 video instead.",
|
||||
"refreshComplete": "{action} complete",
|
||||
"refreshFailed": "Failed to {action} {type}s",
|
||||
"metadataRefreshed": "Metadata refreshed successfully",
|
||||
|
||||
+2290
-2224
File diff suppressed because it is too large
Load Diff
+2290
-2224
File diff suppressed because it is too large
Load Diff
+2290
-2224
File diff suppressed because it is too large
Load Diff
+2290
-2224
File diff suppressed because it is too large
Load Diff
+2290
-2224
File diff suppressed because it is too large
Load Diff
+2290
-2224
File diff suppressed because it is too large
Load Diff
+2290
-2224
File diff suppressed because it is too large
Load Diff
+2290
-2224
File diff suppressed because it is too large
Load Diff
+15
-10
@@ -1,9 +1,13 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
import os
|
||||
import platform
|
||||
import posixpath
|
||||
import threading
|
||||
from pathlib import Path
|
||||
import folder_paths # type: ignore
|
||||
import folder_paths # pyright: ignore[reportMissingImports]
|
||||
from typing import Any, Dict, Iterable, List, Mapping, Optional, Set, Tuple
|
||||
import logging
|
||||
import json
|
||||
@@ -90,7 +94,7 @@ def _resolve_valid_default_root(
|
||||
|
||||
|
||||
def _normalize_folder_paths_for_comparison(
|
||||
folder_paths: Mapping[str, Iterable[str]],
|
||||
folder_paths: Mapping[str, Any],
|
||||
) -> Dict[str, Set[str]]:
|
||||
"""Normalize folder paths for comparison across libraries."""
|
||||
|
||||
@@ -482,7 +486,7 @@ class Config:
|
||||
import ctypes
|
||||
|
||||
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
|
||||
attrs = ctypes.windll.kernel32.GetFileAttributesW(str(path)) # type: ignore[attr-defined]
|
||||
attrs = ctypes.windll.kernel32.GetFileAttributesW(str(path)) # pyright: ignore[reportAttributeAccessIssue]
|
||||
return attrs != -1 and (attrs & FILE_ATTRIBUTE_REPARSE_POINT)
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking Windows reparse point: {e}")
|
||||
@@ -491,7 +495,7 @@ class Config:
|
||||
logger.error(f"Error checking link status for {path}: {e}")
|
||||
return False
|
||||
|
||||
def _entry_is_symlink(self, entry: os.DirEntry) -> bool:
|
||||
def _entry_is_symlink(self, entry: os.DirEntry[str]) -> bool:
|
||||
"""Check if a directory entry is a symlink, including Windows junctions."""
|
||||
if entry.is_symlink():
|
||||
return True
|
||||
@@ -500,7 +504,7 @@ class Config:
|
||||
import ctypes
|
||||
|
||||
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
|
||||
attrs = ctypes.windll.kernel32.GetFileAttributesW(entry.path) # type: ignore[attr-defined]
|
||||
attrs = ctypes.windll.kernel32.GetFileAttributesW(entry.path) # pyright: ignore[reportAttributeAccessIssue]
|
||||
return attrs != -1 and (attrs & FILE_ATTRIBUTE_REPARSE_POINT)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -1126,8 +1130,8 @@ class Config:
|
||||
|
||||
def _apply_library_paths(
|
||||
self,
|
||||
folder_paths: Mapping[str, Iterable[str]],
|
||||
extra_folder_paths: Optional[Mapping[str, Iterable[str]]] = None,
|
||||
folder_paths: Mapping[str, Any],
|
||||
extra_folder_paths: Optional[Mapping[str, Any]] = None,
|
||||
recipes_path: str = "",
|
||||
) -> None:
|
||||
self._path_mappings.clear()
|
||||
@@ -1432,12 +1436,13 @@ class Config:
|
||||
# ('_lm_config_cache') that is NEVER removed from sys.modules (its key does
|
||||
# NOT start with 'py.'), so it survives re-imports of py.* modules.
|
||||
_CONFIG_SENTINEL = "_lm_config_cache"
|
||||
config: Config
|
||||
if _CONFIG_SENTINEL in _sys.modules:
|
||||
# Re-import: reuse the existing singleton from the sentinel.
|
||||
config: Config = _sys.modules[_CONFIG_SENTINEL].config # type: ignore[valid-type]
|
||||
config = _sys.modules[_CONFIG_SENTINEL].config
|
||||
else:
|
||||
config: Config = Config()
|
||||
config = Config()
|
||||
# Register the sentinel so re-imports of py.config find us.
|
||||
_sentinel_mod = _types.ModuleType(_CONFIG_SENTINEL)
|
||||
_sentinel_mod.config = config
|
||||
setattr(_sentinel_mod, "config", config)
|
||||
_sys.modules[_CONFIG_SENTINEL] = _sentinel_mod
|
||||
|
||||
+18
-1
@@ -14,7 +14,7 @@ standalone_mode = (
|
||||
if not standalone_mode:
|
||||
setup_logging()
|
||||
|
||||
from server import PromptServer # type: ignore
|
||||
from server import PromptServer # pyright: ignore[reportMissingImports]
|
||||
|
||||
from .config import config
|
||||
from .services.model_service_factory import (
|
||||
@@ -25,10 +25,12 @@ from .routes.recipe_routes import RecipeRoutes
|
||||
from .routes.stats_routes import StatsRoutes
|
||||
from .routes.update_routes import UpdateRoutes
|
||||
from .routes.misc_routes import MiscRoutes
|
||||
from .routes.pending_delete_routes import PendingDeleteRoutes
|
||||
from .routes.preview_routes import PreviewRoutes
|
||||
from .routes.example_images_routes import ExampleImagesRoutes
|
||||
from .services.service_registry import ServiceRegistry
|
||||
from .services.settings_manager import get_settings_manager
|
||||
from .services.pending_delete_service import get_pending_delete_service
|
||||
from .utils.example_images_migration import ExampleImagesMigration
|
||||
from .services.websocket_manager import ws_manager
|
||||
from .services.example_images_cleanup_service import ExampleImagesCleanupService
|
||||
@@ -170,6 +172,7 @@ class LoraManager:
|
||||
RecipeRoutes.setup_routes(app)
|
||||
UpdateRoutes.setup_routes(app)
|
||||
MiscRoutes.setup_routes(app)
|
||||
PendingDeleteRoutes.setup_routes(app)
|
||||
ExampleImagesRoutes.setup_routes(app, ws_manager=ws_manager)
|
||||
PreviewRoutes.setup_routes(app)
|
||||
|
||||
@@ -245,6 +248,20 @@ class LoraManager:
|
||||
cls._run_post_initialization_tasks(init_tasks), name="post_init_tasks"
|
||||
)
|
||||
|
||||
# Startup sweep: purge pending-delete batches that expired during a
|
||||
# previous run. Non-blocking (fire-and-forget); purge_expired only
|
||||
# removes already-expired batches, so a staged undo that survived a
|
||||
# restart stays restorable. scan_roots=True runs the reconciliation
|
||||
# pass first so leftover batches (the in-process registry is empty
|
||||
# after a restart) are re-discovered on disk. Covers both plugin
|
||||
# and standalone modes (StandaloneLoraManager reuses this
|
||||
# classmethod).
|
||||
pending_delete_service = await get_pending_delete_service()
|
||||
asyncio.create_task(
|
||||
pending_delete_service.purge_expired(scan_roots=True),
|
||||
name="pending_delete_startup_sweep",
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"LoRA Manager: All services initialized and background tasks scheduled"
|
||||
)
|
||||
|
||||
@@ -22,7 +22,7 @@ if not standalone_mode:
|
||||
|
||||
logger.info("ComfyUI Metadata Collector initialized")
|
||||
|
||||
def get_metadata(prompt_id=None): # type: ignore[no-redef]
|
||||
def get_metadata(prompt_id=None): # pyright: ignore[reportRedeclaration]
|
||||
"""Helper function to get metadata from the registry"""
|
||||
registry = MetadataRegistry()
|
||||
return registry.get_metadata(prompt_id)
|
||||
@@ -31,6 +31,6 @@ else:
|
||||
def init():
|
||||
logger.info("ComfyUI Metadata Collector disabled in standalone mode")
|
||||
|
||||
def get_metadata(prompt_id=None): # type: ignore[no-redef]
|
||||
def get_metadata(prompt_id=None): # pyright: ignore[reportRedeclaration]
|
||||
"""Dummy implementation for standalone mode"""
|
||||
return {}
|
||||
|
||||
@@ -16,7 +16,7 @@ class MetadataHook:
|
||||
execution = None
|
||||
try:
|
||||
# Try direct import first
|
||||
import execution # type: ignore
|
||||
import execution # pyright: ignore[reportMissingImports]
|
||||
except ImportError:
|
||||
# Try to locate from system modules
|
||||
for module_name in sys.modules:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import time
|
||||
from nodes import NODE_CLASS_MAPPINGS # type: ignore
|
||||
from typing import Any
|
||||
from nodes import NODE_CLASS_MAPPINGS # pyright: ignore[reportMissingImports, reportAttributeAccessIssue]
|
||||
from .node_extractors import NODE_EXTRACTORS, GenericNodeExtractor
|
||||
from .constants import METADATA_CATEGORIES, IMAGES, OVERWRITE
|
||||
|
||||
@@ -9,6 +10,15 @@ class MetadataRegistry:
|
||||
|
||||
_instance = None
|
||||
|
||||
current_prompt_id: Any = None
|
||||
current_prompt: Any = None
|
||||
metadata: dict[str, Any] = {}
|
||||
prompt_metadata: dict[str, Any] = {}
|
||||
executed_nodes: set[str] = set()
|
||||
node_cache: dict[str, Any] = {}
|
||||
max_prompt_history: int = 3
|
||||
metadata_categories: list[str] = METADATA_CATEGORIES
|
||||
|
||||
def __new__(cls):
|
||||
if cls._instance is None:
|
||||
cls._instance = super().__new__(cls)
|
||||
|
||||
@@ -2,7 +2,8 @@ import json
|
||||
import os
|
||||
import re
|
||||
|
||||
from .constants import CLIP_SKIP_SENTINEL, MODELS, PROMPTS, SAMPLING, LORAS, SIZE, IMAGES, IS_SAMPLER, OVERWRITE, METADATA_OVERWRITE_FIELDS
|
||||
from .constants import MODELS, PROMPTS, SAMPLING, LORAS, SIZE, IMAGES, IS_SAMPLER, OVERWRITE
|
||||
from .overwrite_utils import collect_overwrite_params
|
||||
|
||||
|
||||
def _store_checkpoint_metadata(metadata, node_id, model_name):
|
||||
@@ -39,7 +40,7 @@ class GenericNodeExtractor(NodeMetadataExtractor):
|
||||
* ``MODEL`` output: common input fields (ckpt_name, unet_name, etc.)
|
||||
are checked for a model file name and stored as checkpoint metadata.
|
||||
* ``CONDITIONING`` output: common text input fields are checked for
|
||||
prompt text and stored as prompt metadata.
|
||||
prompt text, and conditioning inputs are tracked through transforms.
|
||||
"""
|
||||
|
||||
# Input field names that carry a model path in loader-style nodes.
|
||||
@@ -72,7 +73,7 @@ class GenericNodeExtractor(NodeMetadataExtractor):
|
||||
_store_checkpoint_metadata(metadata, node_id, name)
|
||||
return
|
||||
|
||||
# — CONDITIONING encoder detection (CLIPTextEncode, Flux, custom) —
|
||||
# — CONDITIONING encoder / transform detection —
|
||||
if "CONDITIONING" in return_types or any("CONDITIONING" in str(t) for t in return_types):
|
||||
text = None
|
||||
for field in GenericNodeExtractor._TEXT_FIELDS:
|
||||
@@ -80,12 +81,14 @@ class GenericNodeExtractor(NodeMetadataExtractor):
|
||||
if val and isinstance(val, str) and val.strip():
|
||||
text = val.strip()
|
||||
break
|
||||
if text:
|
||||
prompt_data = metadata.setdefault(PROMPTS, {})
|
||||
prompt_data[node_id] = {
|
||||
"text": text,
|
||||
"node_id": node_id,
|
||||
}
|
||||
|
||||
input_conditionings = _collect_conditioning_inputs(inputs)
|
||||
if text or input_conditionings:
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
if text:
|
||||
prompt_metadata["text"] = text
|
||||
if input_conditionings:
|
||||
prompt_metadata["orig_conditionings"] = input_conditionings
|
||||
|
||||
@staticmethod
|
||||
def update(node_id, outputs, metadata, return_types=None):
|
||||
@@ -97,11 +100,26 @@ class GenericNodeExtractor(NodeMetadataExtractor):
|
||||
return
|
||||
if node_id not in metadata.get(PROMPTS, {}):
|
||||
return
|
||||
if outputs and isinstance(outputs, list) and len(outputs) > 0:
|
||||
if isinstance(outputs[0], tuple) and len(outputs[0]) > 0:
|
||||
cond = outputs[0][0]
|
||||
if cond is not None:
|
||||
metadata[PROMPTS][node_id]["conditioning"] = cond
|
||||
output_tuple = _first_output_tuple(outputs)
|
||||
if not output_tuple or len(output_tuple) < 1:
|
||||
return
|
||||
|
||||
conditioning_index = _first_conditioning_index(return_types)
|
||||
if conditioning_index is None or len(output_tuple) <= conditioning_index:
|
||||
return
|
||||
|
||||
output_conditioning = output_tuple[conditioning_index]
|
||||
if output_conditioning is None:
|
||||
return
|
||||
|
||||
prompt_metadata = metadata[PROMPTS][node_id]
|
||||
prompt_metadata["conditioning"] = output_conditioning
|
||||
_record_conditioning_source(
|
||||
metadata,
|
||||
node_id,
|
||||
output_conditioning,
|
||||
prompt_metadata.get("orig_conditionings", []),
|
||||
)
|
||||
|
||||
class CheckpointLoaderExtractor(NodeMetadataExtractor):
|
||||
@staticmethod
|
||||
@@ -416,6 +434,34 @@ def _first_output_tuple(outputs):
|
||||
return None
|
||||
|
||||
|
||||
def _first_conditioning_index(return_types):
|
||||
"""Return the index of the first CONDITIONING output slot, or None."""
|
||||
if not return_types:
|
||||
return None
|
||||
for index, return_type in enumerate(return_types):
|
||||
if "CONDITIONING" in str(return_type):
|
||||
return index
|
||||
return None
|
||||
|
||||
|
||||
def _collect_conditioning_inputs(inputs):
|
||||
"""Collect conditioning object inputs (``conditioning*`` keys).
|
||||
|
||||
Primitive values (None, str, int, float, bool) are excluded so scalar
|
||||
fields like ``conditioning_strength`` are not mistaken for conditioning
|
||||
objects during provenance tracking.
|
||||
"""
|
||||
if not inputs:
|
||||
return []
|
||||
return [
|
||||
value
|
||||
for input_name, value in inputs.items()
|
||||
if input_name.startswith("conditioning")
|
||||
and value is not None
|
||||
and not isinstance(value, (str, int, float, bool))
|
||||
]
|
||||
|
||||
|
||||
def _record_conditioning_source(
|
||||
metadata, node_id, output_conditioning, input_conditionings
|
||||
):
|
||||
@@ -428,6 +474,14 @@ def _record_conditioning_source(
|
||||
if not sources:
|
||||
return
|
||||
|
||||
# Identity-preserving selectors return one of their inputs unchanged:
|
||||
# only that input contributed to the output, so record it alone instead
|
||||
# of treating every input as a combination source.
|
||||
for conditioning in sources:
|
||||
if id(conditioning) == id(output_conditioning):
|
||||
sources = [conditioning]
|
||||
break
|
||||
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
prompt_metadata.setdefault("conditioning_sources", []).append(
|
||||
{
|
||||
@@ -507,13 +561,7 @@ class ConditioningCombineExtractor(NodeMetadataExtractor):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
input_conditionings = []
|
||||
for input_name in inputs:
|
||||
if (
|
||||
input_name.startswith("conditioning")
|
||||
and inputs[input_name] is not None
|
||||
):
|
||||
input_conditionings.append(inputs[input_name])
|
||||
input_conditionings = _collect_conditioning_inputs(inputs)
|
||||
|
||||
if input_conditionings:
|
||||
prompt_metadata = _ensure_prompt_metadata(metadata, node_id)
|
||||
@@ -1233,14 +1281,7 @@ class MetadataOverwriteExtractor(NodeMetadataExtractor):
|
||||
if not inputs:
|
||||
return
|
||||
|
||||
overwrite_params = {}
|
||||
for key in METADATA_OVERWRITE_FIELDS:
|
||||
value = inputs.get(key)
|
||||
if key == "clip_skip":
|
||||
if value != CLIP_SKIP_SENTINEL:
|
||||
overwrite_params[key] = value
|
||||
elif value: # truthy — only overwrite when user provided a real value
|
||||
overwrite_params[key] = value
|
||||
overwrite_params = collect_overwrite_params(inputs)
|
||||
|
||||
if overwrite_params:
|
||||
metadata.setdefault(OVERWRITE, {})
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Shared helpers for Metadata Overwrite node metadata collection.
|
||||
|
||||
Used by both the MetadataOverwriteLM node (execution time) and the
|
||||
MetadataOverwriteExtractor (hook time) so the conversion/filtering logic
|
||||
cannot drift between the two paths.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
from ..utils.utils import model_patcher_to_name
|
||||
from .constants import CLIP_SKIP_SENTINEL, METADATA_OVERWRITE_FIELDS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def collect_overwrite_params(values: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Convert node input values into non-default overwrite parameters.
|
||||
|
||||
For most fields, a falsy value (empty string, 0) means "not set" and is
|
||||
skipped. clip_skip uses a dedicated sentinel (-25) so that a wired value
|
||||
of 0 is preserved. The ``model`` field accepts either a manual string or
|
||||
a wired MODEL (ModelPatcher) connection; in the latter case the source
|
||||
model name is extracted from the patcher's ``cached_patcher_init`` and
|
||||
stored as a ComfyUI-style relative path.
|
||||
"""
|
||||
result: Dict[str, Any] = {}
|
||||
for key in METADATA_OVERWRITE_FIELDS:
|
||||
value = values.get(key)
|
||||
if key == "model" and not isinstance(value, str):
|
||||
value = model_patcher_to_name(value)
|
||||
if value is None:
|
||||
logger.warning(
|
||||
"Could not extract model name from wired MODEL input "
|
||||
"(no cached_patcher_init); model metadata overwrite skipped"
|
||||
)
|
||||
if key == "clip_skip":
|
||||
if value != CLIP_SKIP_SENTINEL:
|
||||
result[key] = value
|
||||
elif value:
|
||||
result[key] = value
|
||||
return result
|
||||
@@ -43,7 +43,7 @@ SCANNER_GETTER_NAMES = tuple(SCANNER_TYPE_MAP.keys())
|
||||
|
||||
async def _find_model_entry(
|
||||
model_path: str,
|
||||
) -> tuple[object, object, str | None] | tuple[None, None, None]:
|
||||
) -> tuple[Any, object, str | None] | tuple[None, None, None]:
|
||||
"""Iterate all scanners and return the first (scanner, entry, getter_name)
|
||||
that owns *model_path*. Returns ``(None, None, None)`` when no scanner
|
||||
claims it.
|
||||
@@ -73,7 +73,7 @@ async def _find_model_entry(
|
||||
|
||||
async def _find_scanner_for_model(
|
||||
model_path: str,
|
||||
) -> tuple[object, object] | tuple[None, None]:
|
||||
) -> tuple[Any, object] | tuple[None, None]:
|
||||
"""Find the (scanner, cache_entry) responsible for *model_path*."""
|
||||
scanner, entry, _ = await _find_model_entry(model_path)
|
||||
return scanner, entry
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import logging
|
||||
from typing import List, Tuple
|
||||
import comfy.sd # type: ignore
|
||||
import folder_paths # type: ignore
|
||||
import os
|
||||
from typing import Any, List, Tuple
|
||||
import comfy.sd # pyright: ignore[reportMissingImports]
|
||||
import folder_paths # pyright: ignore[reportMissingImports]
|
||||
from ..utils.utils import get_checkpoint_info_absolute, _format_model_name_for_comfyui
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -18,9 +19,9 @@ class CheckpointLoaderLM:
|
||||
CATEGORY = "Lora Manager/loaders"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
def INPUT_TYPES(cls):
|
||||
# Get list of checkpoint names from scanner (includes extra folder paths)
|
||||
checkpoint_names = s._get_checkpoint_names()
|
||||
checkpoint_names = cls._get_checkpoint_names()
|
||||
return {
|
||||
"required": {
|
||||
"ckpt_name": (
|
||||
@@ -58,7 +59,10 @@ class CheckpointLoaderLM:
|
||||
for item in cache.raw_data:
|
||||
if item.get("sub_type") == "checkpoint":
|
||||
file_path = item.get("file_path", "")
|
||||
if file_path:
|
||||
# Only offer models that still exist on disk so ComfyUI
|
||||
# flags missing checkpoints at queue time via
|
||||
# "value not in list" (the scanner cache can be stale).
|
||||
if file_path and os.path.exists(file_path):
|
||||
# Format using relative path with OS-native separator
|
||||
formatted_name = _format_model_name_for_comfyui(
|
||||
file_path, model_roots
|
||||
@@ -89,7 +93,7 @@ class CheckpointLoaderLM:
|
||||
logger.error(f"Error getting checkpoint names: {e}")
|
||||
return []
|
||||
|
||||
def load_checkpoint(self, ckpt_name: str) -> Tuple:
|
||||
def load_checkpoint(self, ckpt_name: str) -> Tuple[Any, Any, Any]:
|
||||
"""Load a checkpoint by name, supporting extra folder paths
|
||||
|
||||
Args:
|
||||
|
||||
@@ -15,6 +15,7 @@ from .utils import (
|
||||
any_type,
|
||||
apply_lora_syntax_format,
|
||||
get_loras_list,
|
||||
validate_lora_entries,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -42,6 +43,11 @@ class CreateHookLoraLM:
|
||||
"optional": FlexibleOptionalInputType(any_type),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, loras=None):
|
||||
"""Queue-time validation: reject missing local LoRAs before execution."""
|
||||
return validate_lora_entries({"loras": loras}) or True
|
||||
|
||||
RETURN_TYPES = ("HOOKS", "STRING", "STRING")
|
||||
RETURN_NAMES = ("HOOKS", "trigger_words", "active_loras")
|
||||
FUNCTION = "create_hook"
|
||||
@@ -57,8 +63,8 @@ class CreateHookLoraLM:
|
||||
del text # used by the frontend widget only
|
||||
|
||||
# Lazy imports: comfy is not available in CI/test environment at module level
|
||||
import comfy.hooks # type: ignore # noqa: C0415
|
||||
import comfy.utils # type: ignore # noqa: C0415
|
||||
import comfy.hooks # pyright: ignore[reportMissingImports] # noqa: C0415
|
||||
import comfy.utils # pyright: ignore[reportMissingImports] # noqa: C0415
|
||||
|
||||
prev_hooks: comfy.hooks.HookGroup | None = kwargs.get("prev_hooks")
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import importlib
|
||||
import logging
|
||||
|
||||
import comfy.sd # type: ignore
|
||||
import comfy.utils # type: ignore
|
||||
import comfy.sd # pyright: ignore[reportMissingImports]
|
||||
import comfy.utils # pyright: ignore[reportMissingImports]
|
||||
|
||||
from ..utils.utils import get_lora_info_absolute
|
||||
from .utils import (
|
||||
@@ -14,6 +14,7 @@ from .utils import (
|
||||
get_loras_list,
|
||||
nunchaku_load_lora,
|
||||
parse_lora_syntax,
|
||||
validate_lora_entries,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -142,6 +143,11 @@ class LoraLoaderLM:
|
||||
"optional": FlexibleOptionalInputType(any_type),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, loras=None):
|
||||
"""Queue-time validation: reject missing local LoRAs before execution."""
|
||||
return validate_lora_entries({"loras": loras}) or True
|
||||
|
||||
RETURN_TYPES = ("MODEL", "CLIP", "STRING", "STRING")
|
||||
RETURN_NAMES = ("MODEL", "CLIP", "trigger_words", "loaded_loras")
|
||||
FUNCTION = "load_loras"
|
||||
|
||||
@@ -9,6 +9,7 @@ and tracks the last used combination for reuse.
|
||||
import logging
|
||||
import os
|
||||
from ..utils.utils import get_lora_info
|
||||
from .utils import validate_lora_entries
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -31,6 +32,11 @@ class LoraRandomizerLM:
|
||||
},
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, loras=None):
|
||||
"""Queue-time validation: reject missing local LoRAs before execution."""
|
||||
return validate_lora_entries({"loras": loras}) or True
|
||||
|
||||
RETURN_TYPES = ("LORA_STACK",)
|
||||
RETURN_NAMES = ("LORA_STACK",)
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@ class LoraStackCombinerLM:
|
||||
|
||||
stack = inspect.stack()
|
||||
if len(stack) > 2 and stack[2].function == "get_input_info":
|
||||
optional_inputs = _LoraStackOptionalInputs(optional_inputs) # type: ignore[assignment]
|
||||
optional_inputs = _LoraStackOptionalInputs(optional_inputs) # pyright: ignore[reportAssignmentType]
|
||||
|
||||
return {
|
||||
"required": {},
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
from ..utils.utils import get_lora_info
|
||||
from .utils import FlexibleOptionalInputType, any_type, apply_lora_syntax_format, extract_lora_name, get_loras_list
|
||||
from .utils import FlexibleOptionalInputType, any_type, apply_lora_syntax_format, extract_lora_name, get_loras_list, validate_lora_entries
|
||||
|
||||
import logging
|
||||
|
||||
@@ -22,6 +22,11 @@ class LoraStackerLM:
|
||||
"optional": FlexibleOptionalInputType(any_type),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, loras=None):
|
||||
"""Queue-time validation: reject missing local LoRAs before execution."""
|
||||
return validate_lora_entries({"loras": loras}) or True
|
||||
|
||||
RETURN_TYPES = ("LORA_STACK", "STRING", "STRING")
|
||||
RETURN_NAMES = ("LORA_STACK", "trigger_words", "active_loras")
|
||||
FUNCTION = "stack_loras"
|
||||
|
||||
@@ -9,10 +9,8 @@ but users may wire 0 to express "no clip skip / default".
|
||||
|
||||
from typing import Any
|
||||
|
||||
from ..metadata_collector.constants import (
|
||||
CLIP_SKIP_SENTINEL as _CLIP_SKIP_SENTINEL,
|
||||
METADATA_OVERWRITE_FIELDS,
|
||||
)
|
||||
from ..metadata_collector.constants import CLIP_SKIP_SENTINEL as _CLIP_SKIP_SENTINEL
|
||||
from ..metadata_collector.overwrite_utils import collect_overwrite_params
|
||||
|
||||
|
||||
class MetadataOverwriteLM:
|
||||
@@ -87,12 +85,16 @@ class MetadataOverwriteLM:
|
||||
},
|
||||
),
|
||||
"model": (
|
||||
"STRING",
|
||||
"STRING,MODEL",
|
||||
{
|
||||
"default": "",
|
||||
"widgetType": "STRING",
|
||||
"tooltip": (
|
||||
"The checkpoint or diffusion model (UNet) used "
|
||||
"for generation. Only overwrites when non-empty."
|
||||
"for generation. Fill in the name manually or "
|
||||
"connect a MODEL output — the model name is then "
|
||||
"extracted automatically. Only overwrites when "
|
||||
"non-empty."
|
||||
),
|
||||
},
|
||||
),
|
||||
@@ -158,13 +160,10 @@ class MetadataOverwriteLM:
|
||||
For most fields, a falsy value (empty string, 0) means "not set"
|
||||
and is skipped. clip_skip uses a dedicated sentinel (-25) so that
|
||||
a wired value of 0 is preserved and reaches the metadata pipeline.
|
||||
|
||||
The ``model`` field accepts either a manual string or a wired MODEL
|
||||
(ModelPatcher) connection; in the latter case the underlying model
|
||||
name is extracted from the patcher's ``cached_patcher_init`` and
|
||||
stored as a ComfyUI-style relative path.
|
||||
"""
|
||||
result: dict[str, Any] = {}
|
||||
for key in METADATA_OVERWRITE_FIELDS:
|
||||
value = kwargs.get(key)
|
||||
if key == "clip_skip":
|
||||
if value != _CLIP_SKIP_SENTINEL:
|
||||
result[key] = value
|
||||
elif value:
|
||||
result[key] = value
|
||||
return (result,)
|
||||
return (collect_overwrite_params(kwargs),)
|
||||
|
||||
+12
-13
@@ -15,15 +15,15 @@ import os
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple, Union
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union, cast
|
||||
|
||||
import comfy.utils # type: ignore
|
||||
import folder_paths # type: ignore
|
||||
import comfy.utils # pyright: ignore[reportMissingImports]
|
||||
import folder_paths # pyright: ignore[reportMissingImports]
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from safetensors import safe_open
|
||||
|
||||
from nunchaku.lora.flux.nunchaku_converter import (
|
||||
from nunchaku.lora.flux.nunchaku_converter import ( # pyright: ignore[reportMissingTypeStubs]
|
||||
pack_lowrank_weight,
|
||||
unpack_lowrank_weight,
|
||||
)
|
||||
@@ -87,10 +87,6 @@ def _rename_layer_underscore_layer_name(old_name: str) -> str:
|
||||
return new_name
|
||||
|
||||
|
||||
def _is_indexable_module(module):
|
||||
return isinstance(module, (nn.ModuleList, nn.Sequential, list, tuple))
|
||||
|
||||
|
||||
def _get_module_by_name(model: nn.Module, name: str) -> Optional[nn.Module]:
|
||||
if not name:
|
||||
return model
|
||||
@@ -100,7 +96,7 @@ def _get_module_by_name(model: nn.Module, name: str) -> Optional[nn.Module]:
|
||||
continue
|
||||
if hasattr(module, part):
|
||||
module = getattr(module, part)
|
||||
elif part.isdigit() and _is_indexable_module(module):
|
||||
elif part.isdigit() and isinstance(module, (nn.ModuleList, nn.Sequential, list, tuple)):
|
||||
try:
|
||||
module = module[int(part)]
|
||||
except (IndexError, TypeError):
|
||||
@@ -267,7 +263,9 @@ def _handle_proj_out_split(lora_dict: Dict[str, Dict[str, torch.Tensor]], base_k
|
||||
return result, consumed
|
||||
|
||||
|
||||
def _apply_lora_to_module(module: nn.Module, a_tensor: torch.Tensor, b_tensor: torch.Tensor, module_name: str, model: nn.Module) -> None:
|
||||
def _apply_lora_to_module(module: Any, a_tensor: torch.Tensor, b_tensor: torch.Tensor, module_name: str, model: Any) -> None:
|
||||
# These modules are dynamic torch containers; monkey-patched attributes
|
||||
# below are set at runtime, so the module/model types are deliberately Any.
|
||||
if not hasattr(module, "in_features") or not hasattr(module, "out_features"):
|
||||
raise ValueError(f"{module_name}: unsupported module without in/out features")
|
||||
if a_tensor.shape[1] != module.in_features or b_tensor.shape[0] != module.out_features:
|
||||
@@ -336,7 +334,7 @@ def _apply_lora_to_module(module: nn.Module, a_tensor: torch.Tensor, b_tensor: t
|
||||
raise ValueError(f"{module_name}: unsupported module type {type(module)}")
|
||||
|
||||
|
||||
def reset_lora_v2(model: nn.Module) -> None:
|
||||
def reset_lora_v2(model: Any) -> None:
|
||||
slots = getattr(model, "_lora_slots", None)
|
||||
if not slots:
|
||||
return
|
||||
@@ -344,6 +342,7 @@ def reset_lora_v2(model: nn.Module) -> None:
|
||||
module = _get_module_by_name(model, name)
|
||||
if module is None:
|
||||
continue
|
||||
module = cast(Any, module)
|
||||
module_type = info.get("type", "nunchaku")
|
||||
if module_type == "nunchaku":
|
||||
base_rank = info["base_rank"]
|
||||
@@ -371,7 +370,7 @@ def reset_lora_v2(model: nn.Module) -> None:
|
||||
def compose_loras_v2(model: nn.Module, lora_configs: List[Tuple[Union[str, Path, Dict[str, torch.Tensor]], float]], apply_awq_mod: bool = True) -> bool:
|
||||
del apply_awq_mod # retained for interface compatibility
|
||||
reset_lora_v2(model)
|
||||
aggregated_weights: Dict[str, List[Dict[str, object]]] = defaultdict(list)
|
||||
aggregated_weights: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
|
||||
saw_supported_format = False
|
||||
unresolved_targets = 0
|
||||
|
||||
@@ -471,7 +470,7 @@ def compose_loras_v2(model: nn.Module, lora_configs: List[Tuple[Union[str, Path,
|
||||
class ComfyQwenImageWrapperLM(nn.Module):
|
||||
def __init__(self, model: nn.Module, config=None, apply_awq_mod: bool = True):
|
||||
super().__init__()
|
||||
self.model = model
|
||||
self.model: Any = model
|
||||
self.config = {} if config is None else config
|
||||
self.dtype = next(model.parameters()).dtype
|
||||
self.loras: List[Tuple[Union[str, Path, Dict[str, torch.Tensor]], float]] = []
|
||||
|
||||
+2
-2
@@ -67,7 +67,7 @@ class PromptLM:
|
||||
|
||||
stack = inspect.stack()
|
||||
if len(stack) > 2 and stack[2].function == "get_input_info":
|
||||
optional_inputs = _PromptOptionalInputs(optional_inputs) # type: ignore[assignment]
|
||||
optional_inputs = _PromptOptionalInputs(optional_inputs) # pyright: ignore[reportAssignmentType]
|
||||
|
||||
return {
|
||||
"required": {
|
||||
@@ -126,7 +126,7 @@ class PromptLM:
|
||||
else:
|
||||
prompt = expanded_text
|
||||
|
||||
from nodes import CLIPTextEncode # type: ignore
|
||||
from nodes import CLIPTextEncode # pyright: ignore[reportMissingImports, reportAttributeAccessIssue]
|
||||
|
||||
conditioning = CLIPTextEncode().encode(clip, prompt)[0]
|
||||
return (conditioning, prompt)
|
||||
|
||||
+22
-9
@@ -5,7 +5,7 @@ import time
|
||||
import uuid
|
||||
from typing import Any, Dict, Optional
|
||||
import numpy as np
|
||||
import folder_paths # type: ignore
|
||||
import folder_paths # pyright: ignore[reportMissingImports]
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
from ..metadata_collector.metadata_processor import MetadataProcessor
|
||||
from ..metadata_collector import get_metadata
|
||||
@@ -13,7 +13,7 @@ from ..utils.constants import CARD_PREVIEW_WIDTH
|
||||
from ..utils.exif_utils import ExifUtils
|
||||
from ..utils.utils import calculate_recipe_fingerprint, sanitize_folder_name
|
||||
from PIL import Image, PngImagePlugin
|
||||
import piexif
|
||||
import piexif # pyright: ignore[reportMissingTypeStubs]
|
||||
import logging
|
||||
|
||||
# Civitai-compatible sampler name mapping: ComfyUI internal → A1111 display name
|
||||
@@ -252,6 +252,13 @@ class SaveImageLM:
|
||||
"tooltip": "When enabled, embeds generation parameters into the saved image metadata. Disable to skip writing generation metadata.",
|
||||
},
|
||||
),
|
||||
"add_loras_to_prompt": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": "When enabled, appends the LoRA syntax line (e.g. <lora:name:strength>) after the positive prompt in the saved metadata.",
|
||||
},
|
||||
),
|
||||
"add_counter_to_filename": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
@@ -348,7 +355,7 @@ class SaveImageLM:
|
||||
type_lower = model_type.lower() if model_type else "other"
|
||||
return f"urn:air:{slug}:{type_lower}:civitai:{model_id}@{version_id}"
|
||||
|
||||
def format_metadata(self, metadata_dict: dict) -> str:
|
||||
def format_metadata(self, metadata_dict: dict[str, Any], add_loras_to_prompt: bool = False) -> str:
|
||||
"""Format metadata as A1111-compatible parameters string with Hashes JSON and Civitai resources."""
|
||||
if not metadata_dict: return ""
|
||||
|
||||
@@ -389,7 +396,7 @@ class SaveImageLM:
|
||||
ckpt_display_name = os.path.splitext(os.path.basename(checkpoint))[0]
|
||||
|
||||
# Resolve LoRA hash and Civitai data from local cache
|
||||
loras_data: list[dict] = []
|
||||
loras_data: list[dict[str, Any]] = []
|
||||
for lora_name, strength in lora_entries:
|
||||
lora_hash, lora_civitai, lora_base_model = self._resolve_model_cache_entry(
|
||||
"lora_scanner", lora_name
|
||||
@@ -411,9 +418,9 @@ class SaveImageLM:
|
||||
hashes[f"LORA:{lora['name']}"] = lora["hash"][:10].upper()
|
||||
|
||||
# Build Civitai resources JSON array
|
||||
civitai_resources: list[dict] = []
|
||||
civitai_resources: list[dict[str, Any]] = []
|
||||
if ckpt_civitai.get("id", 0) > 0:
|
||||
ckpt_resource: dict = {}
|
||||
ckpt_resource: dict[str, Any] = {}
|
||||
ckpt_type = (ckpt_civitai.get("model") or {}).get("type", "Checkpoint")
|
||||
model_id = ckpt_civitai.get("modelId", 0)
|
||||
version_id = ckpt_civitai.get("id", 0)
|
||||
@@ -432,7 +439,7 @@ class SaveImageLM:
|
||||
lora_civitai = lora["civitai"]
|
||||
if not lora_civitai or lora_civitai.get("id", 0) <= 0:
|
||||
continue
|
||||
lora_resource: dict = {"weight": lora["strength"]}
|
||||
lora_resource: dict[str, Any] = {"weight": lora["strength"]}
|
||||
lora_type = (lora_civitai.get("model") or {}).get("type", "LORA")
|
||||
model_id = lora_civitai.get("modelId", 0)
|
||||
version_id = lora_civitai.get("id", 0)
|
||||
@@ -458,7 +465,10 @@ class SaveImageLM:
|
||||
scheduler_name = scheduler_mapping.get(scheduler, scheduler) if scheduler else None
|
||||
|
||||
# Build output lines
|
||||
lines = [prompt] if prompt else [""]
|
||||
prompt_line = prompt if prompt else ""
|
||||
if add_loras_to_prompt and loras_text:
|
||||
prompt_line = f"{prompt_line}\n{loras_text}" if prompt_line else loras_text
|
||||
lines = [prompt_line] if prompt_line else [""]
|
||||
if negative_prompt:
|
||||
lines.append(f"Negative prompt: {negative_prompt}")
|
||||
|
||||
@@ -793,6 +803,7 @@ class SaveImageLM:
|
||||
save_with_metadata=True,
|
||||
add_counter_to_filename=True,
|
||||
save_as_recipe=False,
|
||||
add_loras_to_prompt=False,
|
||||
):
|
||||
"""Save images with metadata"""
|
||||
results = []
|
||||
@@ -801,7 +812,7 @@ class SaveImageLM:
|
||||
raw_metadata = get_metadata()
|
||||
metadata_dict = MetadataProcessor.to_dict(raw_metadata, id)
|
||||
|
||||
metadata = self.format_metadata(metadata_dict)
|
||||
metadata = self.format_metadata(metadata_dict, add_loras_to_prompt)
|
||||
|
||||
# Process filename_prefix with pattern substitution
|
||||
filename_prefix = self.format_filename(filename_prefix, metadata_dict)
|
||||
@@ -943,6 +954,7 @@ class SaveImageLM:
|
||||
save_with_metadata=True,
|
||||
add_counter_to_filename=True,
|
||||
save_as_recipe=False,
|
||||
add_loras_to_prompt=False,
|
||||
):
|
||||
"""Process and save image with metadata"""
|
||||
# Make sure the output directory exists
|
||||
@@ -974,6 +986,7 @@ class SaveImageLM:
|
||||
save_with_metadata,
|
||||
add_counter_to_filename,
|
||||
save_as_recipe,
|
||||
add_loras_to_prompt,
|
||||
)
|
||||
|
||||
return {
|
||||
|
||||
+31
-7
@@ -1,12 +1,27 @@
|
||||
import logging
|
||||
import os
|
||||
from typing import List, Tuple
|
||||
import comfy.sd # type: ignore
|
||||
from typing import Any, List, Tuple
|
||||
import comfy.sd # pyright: ignore[reportMissingImports]
|
||||
from ..utils.utils import get_checkpoint_info_absolute, _format_model_name_for_comfyui
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _reload_gguf_unet(
|
||||
unet_path: str, weight_dtype: str, disable_dynamic: bool = False
|
||||
) -> object:
|
||||
"""Reload a GGUF diffusion model from disk (cached_patcher_init factory).
|
||||
|
||||
Mirrors the GGUF branch of UNETLoaderLM.load_unet so ModelPatcher
|
||||
deepclone/dynamic machinery can rebuild GGUF models with the correct
|
||||
GGMLOps. ``disable_dynamic`` is accepted for signature compatibility
|
||||
with core ComfyUI loaders.
|
||||
"""
|
||||
loader = UNETLoaderLM()
|
||||
model, = loader._load_gguf_unet(unet_path, unet_path, weight_dtype)
|
||||
return model
|
||||
|
||||
|
||||
class UNETLoaderLM:
|
||||
"""UNET Loader with support for extra folder paths
|
||||
|
||||
@@ -19,9 +34,9 @@ class UNETLoaderLM:
|
||||
CATEGORY = "Lora Manager/loaders"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
def INPUT_TYPES(cls):
|
||||
# Get list of unet names from scanner (includes extra folder paths)
|
||||
unet_names = s._get_unet_names()
|
||||
unet_names = cls._get_unet_names()
|
||||
return {
|
||||
"required": {
|
||||
"unet_name": (
|
||||
@@ -59,7 +74,10 @@ class UNETLoaderLM:
|
||||
for item in cache.raw_data:
|
||||
if item.get("sub_type") == "diffusion_model":
|
||||
file_path = item.get("file_path", "")
|
||||
if file_path:
|
||||
# Only offer models that still exist on disk so ComfyUI
|
||||
# flags missing diffusion models at queue time via
|
||||
# "value not in list" (the scanner cache can be stale).
|
||||
if file_path and os.path.exists(file_path):
|
||||
# Format using relative path with OS-native separator
|
||||
formatted_name = _format_model_name_for_comfyui(
|
||||
file_path, model_roots
|
||||
@@ -90,7 +108,7 @@ class UNETLoaderLM:
|
||||
logger.error(f"Error getting unet names: {e}")
|
||||
return []
|
||||
|
||||
def load_unet(self, unet_name: str, weight_dtype: str) -> Tuple:
|
||||
def load_unet(self, unet_name: str, weight_dtype: str) -> Tuple[Any, ...]:
|
||||
"""Load a diffusion model by name, supporting extra folder paths
|
||||
|
||||
Args:
|
||||
@@ -133,7 +151,7 @@ class UNETLoaderLM:
|
||||
|
||||
def _load_gguf_unet(
|
||||
self, unet_path: str, unet_name: str, weight_dtype: str
|
||||
) -> Tuple:
|
||||
) -> Tuple[Any, ...]:
|
||||
"""Load a GGUF format diffusion model
|
||||
|
||||
Args:
|
||||
@@ -196,6 +214,12 @@ class UNETLoaderLM:
|
||||
# Wrap with GGUFModelPatcher
|
||||
model = GGUFModelPatcher.clone(model)
|
||||
|
||||
# Register a reload factory so the MODEL carries its source path
|
||||
# (cached_patcher_init) like core ComfyUI loaders do — required
|
||||
# for model-name extraction downstream and for ModelPatcher
|
||||
# deepclone/dynamic machinery.
|
||||
model.cached_patcher_init = (_reload_gguf_unet, (unet_path, weight_dtype))
|
||||
|
||||
return (model,)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
+159
-3
@@ -1,3 +1,6 @@
|
||||
from typing import Any
|
||||
|
||||
|
||||
class AnyType(str):
|
||||
"""A special class that is always equal in not equal comparisons. Credit to pythongosssss"""
|
||||
|
||||
@@ -6,7 +9,7 @@ class AnyType(str):
|
||||
|
||||
|
||||
# Credit to Regis Gaughan, III (rgthree)
|
||||
class FlexibleOptionalInputType(dict):
|
||||
class FlexibleOptionalInputType(dict[str, Any]):
|
||||
"""A special class to make flexible nodes that pass data to our python handlers.
|
||||
|
||||
Enables both flexible/dynamic input types (like for Any Switch) or a dynamic number of inputs
|
||||
@@ -23,6 +26,7 @@ class FlexibleOptionalInputType(dict):
|
||||
"""
|
||||
|
||||
def __init__(self, type):
|
||||
super().__init__()
|
||||
self.type = type
|
||||
|
||||
def __getitem__(self, key):
|
||||
@@ -40,7 +44,8 @@ import re
|
||||
import logging
|
||||
import copy
|
||||
import sys
|
||||
import folder_paths # type: ignore
|
||||
import asyncio
|
||||
import folder_paths # pyright: ignore[reportMissingImports]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -70,7 +75,7 @@ def extract_lora_name(lora_path):
|
||||
return apply_lora_syntax_format(name_no_ext)
|
||||
|
||||
|
||||
def parse_lora_syntax(text: str) -> list[dict]:
|
||||
def parse_lora_syntax(text: str) -> list[dict[str, Any]]:
|
||||
"""Parse <lora:name:strength> syntax from text input into a list of dicts.
|
||||
|
||||
Each entry contains: name, model_strength, clip_strength.
|
||||
@@ -107,6 +112,157 @@ def get_loras_list(kwargs):
|
||||
return []
|
||||
|
||||
|
||||
_LORA_EXTENSIONS = (".safetensors", ".ckpt", ".pt", ".bin")
|
||||
|
||||
|
||||
def _strip_lora_extension(name: str) -> str:
|
||||
"""Strip a known LoRA model extension from a name (case-insensitive)."""
|
||||
lowered = name.lower()
|
||||
for ext in _LORA_EXTENSIONS:
|
||||
if lowered.endswith(ext):
|
||||
return name[: -len(ext)]
|
||||
return name
|
||||
|
||||
|
||||
def _find_missing_loras(names: list[str]) -> list[str]:
|
||||
"""Return the names that cannot be resolved to an existing local LoRA file.
|
||||
|
||||
Mirrors the matching semantics of ``get_lora_info_absolute``
|
||||
(py/utils/utils.py): after stripping the extension, a name matches a cached
|
||||
LoRA when it equals the cached file name or the ``folder/file`` path. As a
|
||||
fallback, a name containing a folder that only matches by basename resolves
|
||||
to the first basename match (same behavior as the runtime resolver). Raw
|
||||
absolute paths that exist on disk are always considered available.
|
||||
|
||||
The scanner cache is fetched once for all names; the cache may be stale, so
|
||||
resolved paths are additionally verified with ``os.path.isfile``.
|
||||
"""
|
||||
if not names:
|
||||
return []
|
||||
|
||||
async def _check() -> list[str]:
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
|
||||
scanner = await ServiceRegistry.get_lora_scanner()
|
||||
# The scanner cache may not be hydrated yet (startup, library path
|
||||
# change). An empty cache is not authoritative — treat it as "cannot
|
||||
# verify" and skip validation instead of flagging every active LoRA
|
||||
# as missing.
|
||||
if getattr(scanner, "_cache", None) is None or getattr(
|
||||
scanner, "_is_initializing", False
|
||||
):
|
||||
return []
|
||||
cache = await scanner.get_cached_data()
|
||||
|
||||
lookup = {}
|
||||
basename_candidates = {}
|
||||
for item in cache.raw_data:
|
||||
file_path = item.get("file_path")
|
||||
if not file_path:
|
||||
continue
|
||||
file_name = item.get("file_name", "")
|
||||
folder = item.get("folder", "")
|
||||
file_name_no_ext = _strip_lora_extension(file_name)
|
||||
path_name_no_ext = (
|
||||
f"{folder}/{file_name_no_ext}".replace("\\", "/")
|
||||
if folder
|
||||
else file_name_no_ext
|
||||
)
|
||||
lookup.setdefault(file_name_no_ext, file_path)
|
||||
lookup.setdefault(path_name_no_ext, file_path)
|
||||
basename_candidates.setdefault(file_name_no_ext, []).append(
|
||||
(folder, file_path)
|
||||
)
|
||||
|
||||
missing = []
|
||||
for name in names:
|
||||
if not name:
|
||||
continue
|
||||
normalized = name.replace("\\", "/")
|
||||
# Raw absolute paths (outside the library) are usable as-is.
|
||||
if os.path.isfile(normalized):
|
||||
continue
|
||||
no_ext = _strip_lora_extension(normalized)
|
||||
file_path = lookup.get(no_ext)
|
||||
if file_path is None and "/" in no_ext:
|
||||
# A name with a folder that matches only by basename resolves
|
||||
# at runtime like get_lora_info_absolute's fallback does:
|
||||
# prefer a candidate whose folder prefixes the name, else the
|
||||
# first basename match.
|
||||
folder, basename = no_ext.rsplit("/", 1)
|
||||
candidates = basename_candidates.get(basename, [])
|
||||
file_path = next(
|
||||
(
|
||||
fp
|
||||
for fld, fp in candidates
|
||||
if fld and no_ext.startswith(fld + "/")
|
||||
),
|
||||
None,
|
||||
)
|
||||
if file_path is None and candidates:
|
||||
file_path = candidates[0][1]
|
||||
if file_path is None or not os.path.isfile(file_path):
|
||||
missing.append(name)
|
||||
return missing
|
||||
|
||||
try:
|
||||
# Check if we're already in an event loop
|
||||
loop = asyncio.get_running_loop()
|
||||
# If we're in a running loop, run the async check in a separate thread
|
||||
import concurrent.futures
|
||||
|
||||
def run_in_thread():
|
||||
new_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(new_loop)
|
||||
try:
|
||||
return new_loop.run_until_complete(_check())
|
||||
finally:
|
||||
new_loop.close()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
future = executor.submit(run_in_thread)
|
||||
return future.result()
|
||||
except RuntimeError:
|
||||
# No event loop is running, we can use asyncio.run()
|
||||
return asyncio.run(_check())
|
||||
|
||||
|
||||
def validate_lora_entries(kwargs):
|
||||
"""Validate active LoRA widget entries against the local library.
|
||||
|
||||
Used by node ``VALIDATE_INPUTS`` implementations so ComfyUI rejects the
|
||||
prompt at queue time (``custom_validation_failed``) when an active entry
|
||||
references a LoRA that is not available locally — mirroring how built-in
|
||||
loader nodes flag missing models before execution starts.
|
||||
|
||||
Returns:
|
||||
None when every active entry resolves to an existing local file,
|
||||
otherwise a descriptive error string listing the missing LoRAs.
|
||||
Verification failures (e.g. scanner not ready) are treated as valid
|
||||
so queueing is never blocked by validation machinery itself.
|
||||
"""
|
||||
# Missing/empty loras input is always valid; skip get_loras_list so it
|
||||
# does not log a warning for the None case on every queue.
|
||||
if not kwargs.get("loras"):
|
||||
return None
|
||||
loras = get_loras_list(kwargs)
|
||||
active_names = []
|
||||
for lora in loras:
|
||||
if not isinstance(lora, dict):
|
||||
continue
|
||||
if not lora.get("active", False):
|
||||
continue
|
||||
active_names.append(apply_lora_syntax_format(str(lora.get("name") or "")))
|
||||
try:
|
||||
missing = _find_missing_loras(active_names)
|
||||
except Exception:
|
||||
logger.exception("Failed to validate LoRA entries against the local library")
|
||||
return None
|
||||
if not missing:
|
||||
return None
|
||||
return "Missing LoRA(s) in local library: " + ", ".join(missing)
|
||||
|
||||
|
||||
def load_state_dict_in_safetensors(path, device="cpu", filter_prefix=""):
|
||||
"""Simplified version of load_state_dict_in_safetensors that just loads from a local path"""
|
||||
import safetensors.torch
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import os
|
||||
from ..utils.utils import get_lora_info_absolute
|
||||
from ..config import config
|
||||
from .utils import FlexibleOptionalInputType, any_type, get_loras_list
|
||||
from .utils import FlexibleOptionalInputType, any_type, get_loras_list, validate_lora_entries
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -35,6 +35,11 @@ class WanVideoLoraSelectLM:
|
||||
"optional": FlexibleOptionalInputType(any_type),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, loras=None):
|
||||
"""Queue-time validation: reject missing local LoRAs before execution."""
|
||||
return validate_lora_entries({"loras": loras}) or True
|
||||
|
||||
RETURN_TYPES = ("WANVIDLORA", "STRING", "STRING")
|
||||
RETURN_NAMES = ("lora", "trigger_words", "active_loras")
|
||||
FUNCTION = "process_loras"
|
||||
|
||||
+31
-9
@@ -1,3 +1,7 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
"""Base classes for recipe parsers."""
|
||||
|
||||
import json
|
||||
@@ -7,7 +11,7 @@ import re
|
||||
from typing import Dict, List, Any, Optional, Tuple
|
||||
from abc import ABC, abstractmethod
|
||||
from ..config import config
|
||||
from ..utils.constants import VALID_LORA_TYPES, VALID_CHECKPOINT_SUB_TYPES
|
||||
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES, VALID_LORA_TYPES, VALID_CHECKPOINT_SUB_TYPES
|
||||
from ..utils.civitai_utils import rewrite_preview_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -38,7 +42,7 @@ class RecipeMetadataParser(ABC):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
async def populate_lora_from_civitai(lora_entry: Dict[str, Any], civitai_info_tuple: Tuple[Dict[str, Any], Optional[str]],
|
||||
async def populate_lora_from_civitai(lora_entry: Dict[str, Any], civitai_info_tuple: Tuple[Dict[str, Any] | None, str | None] | Dict[str, Any],
|
||||
recipe_scanner=None, base_model_counts=None, hash_value=None) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Populate a lora entry with information from Civitai API response
|
||||
@@ -151,9 +155,9 @@ class RecipeMetadataParser(ABC):
|
||||
|
||||
# Process file information if available
|
||||
if 'files' in civitai_info:
|
||||
# Find the primary model file (type="Model" and primary=true) in the files list
|
||||
# Find the primary model file (weights-type and primary=true) in the files list
|
||||
model_file = next((file for file in civitai_info.get('files', [])
|
||||
if file.get('type') == 'Model' and file.get('primary') == True), None)
|
||||
if file.get('type') in MODEL_WEIGHT_FILE_TYPES and file.get('primary') == True), None)
|
||||
|
||||
if model_file:
|
||||
# Get size
|
||||
@@ -175,10 +179,18 @@ class RecipeMetadataParser(ABC):
|
||||
lora_entry['localPath'] = local_path
|
||||
lora_entry['file_name'] = os.path.splitext(os.path.basename(local_path))[0]
|
||||
|
||||
# Get thumbnail from local preview if available
|
||||
# Get thumbnail from local preview if available.
|
||||
# Match the cache item by local path first (get_path_by_hash
|
||||
# cascade: 10-char autov2 / 12-char autov3), then by hash.
|
||||
lora_cache = await lora_scanner.get_cached_data()
|
||||
lora_item = next((item for item in lora_cache.raw_data
|
||||
if item['sha256'].lower() == lora_entry['hash'].lower()), None)
|
||||
h = (lora_entry.get("hash") or "").lower()
|
||||
lora_item = next((item for item in lora_cache.raw_data
|
||||
if (item.get("file_path") or "") == local_path), None)
|
||||
if lora_item is None:
|
||||
lora_item = next((item for item in lora_cache.raw_data
|
||||
if (item.get("sha256") or "").lower() == h
|
||||
or (item.get("autov3") or "").lower() == h
|
||||
or (item.get("sha256") or "")[:10].lower() == h), None)
|
||||
if lora_item and 'preview_url' in lora_item:
|
||||
lora_entry['thumbnailUrl'] = config.get_preview_static_url(lora_item['preview_url'])
|
||||
except Exception as e:
|
||||
@@ -194,7 +206,7 @@ class RecipeMetadataParser(ABC):
|
||||
return lora_entry
|
||||
|
||||
@staticmethod
|
||||
async def populate_checkpoint_from_civitai(checkpoint: Dict[str, Any], civitai_info: Dict[str, Any]) -> Dict[str, Any]:
|
||||
async def populate_checkpoint_from_civitai(checkpoint: Dict[str, Any], civitai_info: Dict[str, Any] | Tuple[Dict[str, Any] | None, str | None] | None) -> Dict[str, Any]:
|
||||
"""
|
||||
Populate checkpoint information from Civitai API response
|
||||
|
||||
@@ -249,11 +261,21 @@ class RecipeMetadataParser(ABC):
|
||||
checkpoint['id'] = civitai_data.get('id', 0)
|
||||
|
||||
if 'files' in civitai_data:
|
||||
# Prefer the file CivitAI marked primary; fall back to any
|
||||
# weights-type file (providers without primary flags).
|
||||
model_file = next(
|
||||
(
|
||||
file
|
||||
for file in civitai_data.get('files', [])
|
||||
if file.get('type') == 'Model'
|
||||
if file.get('type') in MODEL_WEIGHT_FILE_TYPES
|
||||
and file.get('primary') is True
|
||||
),
|
||||
None,
|
||||
) or next(
|
||||
(
|
||||
file
|
||||
for file in civitai_data.get('files', [])
|
||||
if file.get('type') in MODEL_WEIGHT_FILE_TYPES
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
import logging
|
||||
import json
|
||||
import os
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Factory for creating recipe metadata parsers."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
from .parsers import (
|
||||
RecipeFormatParser,
|
||||
ComfyMetadataParser,
|
||||
@@ -31,7 +32,8 @@ class RecipeParserFactory:
|
||||
# First, try CivitaiApiMetadataParser for dict input
|
||||
if isinstance(metadata, dict):
|
||||
try:
|
||||
if CivitaiApiMetadataParser().is_metadata_matching(metadata):
|
||||
user_comment: Any = metadata
|
||||
if CivitaiApiMetadataParser().is_metadata_matching(user_comment):
|
||||
return CivitaiApiMetadataParser()
|
||||
except Exception as e:
|
||||
logger.debug(f"CivitaiApiMetadataParser check failed: {e}")
|
||||
|
||||
@@ -52,7 +52,7 @@ class AutomaticMetadataParser(RecipeMetadataParser):
|
||||
negative_and_params = ""
|
||||
|
||||
# Initialize metadata
|
||||
metadata = {
|
||||
metadata: Dict[str, Any] = {
|
||||
"prompt": prompt,
|
||||
"loras": []
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ import json
|
||||
import logging
|
||||
from typing import Dict, Any, Union
|
||||
from ..base import RecipeMetadataParser
|
||||
from ..constants import GEN_PARAM_KEYS
|
||||
from ..constants import GEN_PARAM_KEYS, VALID_LORA_TYPES
|
||||
from ...services.metadata_service import get_default_metadata_provider
|
||||
from ...config import config
|
||||
|
||||
@@ -14,15 +14,16 @@ logger = logging.getLogger(__name__)
|
||||
class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
"""Parser for Civitai image metadata format"""
|
||||
|
||||
def is_metadata_matching(self, metadata) -> bool:
|
||||
def is_metadata_matching(self, user_comment) -> bool:
|
||||
"""Check if the metadata matches the Civitai image metadata format
|
||||
|
||||
Args:
|
||||
metadata: The metadata from the image (dict)
|
||||
user_comment: The metadata from the image (dict)
|
||||
|
||||
Returns:
|
||||
bool: True if this parser can handle the metadata
|
||||
"""
|
||||
metadata = user_comment
|
||||
if not metadata or not isinstance(metadata, dict):
|
||||
return False
|
||||
|
||||
@@ -73,7 +74,7 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
|
||||
return False
|
||||
|
||||
async def parse_metadata( # type: ignore[override]
|
||||
async def parse_metadata( # pyright: ignore[reportIncompatibleMethodOverride]
|
||||
self, user_comment, recipe_scanner=None, civitai_client=None,
|
||||
local_cache: dict[str, Any] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
@@ -89,8 +90,7 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
Returns:
|
||||
Dict containing parsed recipe data
|
||||
"""
|
||||
metadata: Dict[str, Any] = user_comment # type: ignore[assignment]
|
||||
metadata = user_comment
|
||||
metadata: Dict[str, Any] = user_comment
|
||||
try:
|
||||
# Get metadata provider instead of using civitai_client directly
|
||||
metadata_provider = await get_default_metadata_provider()
|
||||
@@ -116,7 +116,7 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
metadata = inner_meta
|
||||
|
||||
# Initialize result structure
|
||||
result = {
|
||||
result: Dict[str, Any] = {
|
||||
"base_model": None,
|
||||
"loras": [],
|
||||
"model": None,
|
||||
@@ -125,10 +125,10 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
}
|
||||
|
||||
# Track already added LoRAs to prevent duplicates
|
||||
added_loras = {} # key: model_version_id or hash, value: index in result["loras"]
|
||||
added_loras: Dict[str, Any] = {} # key: model_version_id or hash, value: index in result["loras"]
|
||||
|
||||
# Extract hash information from hashes field for LoRA matching
|
||||
lora_hashes = {}
|
||||
lora_hashes: Dict[str, Any] = {}
|
||||
if "hashes" in metadata and isinstance(metadata["hashes"], dict):
|
||||
for key, hash_value in metadata["hashes"].items():
|
||||
key_str = str(key)
|
||||
@@ -184,7 +184,7 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
if model_info:
|
||||
result["base_model"] = model_info.get("baseModel", "")
|
||||
|
||||
base_model_counts = {}
|
||||
base_model_counts: Dict[str, int] = {}
|
||||
|
||||
# Process standard resources array
|
||||
if "resources" in metadata and isinstance(metadata["resources"], list):
|
||||
@@ -196,7 +196,7 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
# identification because it has an explicit type field and hash,
|
||||
# unlike modelVersionIds which is a flat list with no type info.
|
||||
if resource_type == "model":
|
||||
checkpoint_entry = {
|
||||
checkpoint_entry: Dict[str, Any] = {
|
||||
"id": 0,
|
||||
"modelId": 0,
|
||||
"name": resource.get("name", "Unknown Model"),
|
||||
@@ -216,7 +216,8 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
# Try to look up base model from the checkpoint hash
|
||||
cp_hash = checkpoint_entry.get("hash")
|
||||
if cp_hash and metadata_provider:
|
||||
local_cached = local_cache.get(cp_hash) if local_cache else None
|
||||
# local_cache keys are stored lowercase
|
||||
local_cached = local_cache.get(cp_hash.lower()) if local_cache else None
|
||||
if local_cached:
|
||||
self._populate_entry_from_cache(
|
||||
checkpoint_entry, local_cached
|
||||
@@ -294,8 +295,15 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
|
||||
# Try to get info from Civitai if hash is available
|
||||
if lora_hash and metadata_provider:
|
||||
local_cached = local_cache.get(lora_hash) if local_cache else None
|
||||
# local_cache keys are stored lowercase
|
||||
local_cached = local_cache.get(lora_hash.lower()) if local_cache else None
|
||||
if local_cached:
|
||||
cached_type = self._cache_item_model_type(local_cached)
|
||||
if cached_type and cached_type not in VALID_LORA_TYPES:
|
||||
logger.debug(
|
||||
f"Skipping non-LoRA cache item for hash {lora_hash}"
|
||||
)
|
||||
continue
|
||||
self._populate_entry_from_cache(
|
||||
lora_entry, local_cached
|
||||
)
|
||||
@@ -304,6 +312,12 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
added_loras[str(lora_entry["id"])] = len(
|
||||
result["loras"]
|
||||
)
|
||||
# Mirror base.py:150-151 counts for API-path loras
|
||||
bm = local_cached.get("base_model") or ""
|
||||
if bm:
|
||||
base_model_counts[bm] = base_model_counts.get(
|
||||
bm, 0
|
||||
) + 1
|
||||
else:
|
||||
try:
|
||||
civitai_info = (
|
||||
@@ -649,30 +663,47 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
}
|
||||
|
||||
if metadata_provider:
|
||||
try:
|
||||
civitai_info = await metadata_provider.get_model_by_hash(
|
||||
lora_hash
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
# local_cache keys are stored lowercase
|
||||
local_cached = local_cache.get(lora_hash.lower()) if local_cache else None
|
||||
if local_cached:
|
||||
cached_type = self._cache_item_model_type(local_cached)
|
||||
if cached_type and cached_type not in VALID_LORA_TYPES:
|
||||
logger.debug(
|
||||
f"Skipping non-LoRA cache item for hash {lora_hash}"
|
||||
)
|
||||
continue
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
self._populate_entry_from_cache(lora_entry, local_cached)
|
||||
# Mirror base.py:150-151 counts for API-path loras
|
||||
bm = local_cached.get("base_model") or ""
|
||||
if bm:
|
||||
base_model_counts[bm] = base_model_counts.get(bm, 0) + 1
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_hash}: {e}"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
civitai_info = await metadata_provider.get_model_by_hash(
|
||||
lora_hash
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
continue
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_hash}: {e}"
|
||||
)
|
||||
|
||||
added_loras[lora_hash] = len(result["loras"])
|
||||
result["loras"].append(lora_entry)
|
||||
@@ -711,32 +742,51 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
|
||||
# Try to get info from Civitai if hash is available
|
||||
if lora_entry["hash"] and metadata_provider:
|
||||
try:
|
||||
civitai_info = await metadata_provider.get_model_by_hash(
|
||||
lora_hash
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
# local_cache keys are stored lowercase
|
||||
local_cached = local_cache.get(lora_hash.lower()) if local_cache else None
|
||||
if local_cached:
|
||||
cached_type = self._cache_item_model_type(local_cached)
|
||||
if cached_type and cached_type not in VALID_LORA_TYPES:
|
||||
logger.debug(
|
||||
f"Skipping non-LoRA cache item for hash {lora_hash}"
|
||||
)
|
||||
lora_index += 1
|
||||
continue # Skip invalid LoRA types
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
continue # Skip non-LoRA cache items
|
||||
self._populate_entry_from_cache(lora_entry, local_cached)
|
||||
# Mirror base.py:150-151 counts for API-path loras
|
||||
bm = local_cached.get("base_model") or ""
|
||||
if bm:
|
||||
base_model_counts[bm] = base_model_counts.get(bm, 0) + 1
|
||||
# If we have a version ID from Civitai, track it for deduplication
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_entry['hash']}: {e}"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
civitai_info = await metadata_provider.get_model_by_hash(
|
||||
lora_hash
|
||||
)
|
||||
|
||||
populated_entry = await self.populate_lora_from_civitai(
|
||||
lora_entry,
|
||||
civitai_info,
|
||||
recipe_scanner,
|
||||
base_model_counts,
|
||||
lora_hash,
|
||||
)
|
||||
|
||||
if populated_entry is None:
|
||||
lora_index += 1
|
||||
continue # Skip invalid LoRA types
|
||||
|
||||
lora_entry = populated_entry
|
||||
|
||||
# If we have a version ID from Civitai, track it for deduplication
|
||||
if "id" in lora_entry and lora_entry["id"]:
|
||||
added_loras[str(lora_entry["id"])] = len(result["loras"])
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching Civitai info for LoRA hash {lora_entry['hash']}: {e}"
|
||||
)
|
||||
|
||||
# Track by hash if we have it
|
||||
if lora_hash:
|
||||
@@ -795,3 +845,14 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
base_model = cache_item.get("base_model", "")
|
||||
if base_model:
|
||||
entry["baseModel"] = base_model
|
||||
|
||||
@staticmethod
|
||||
def _cache_item_model_type(cache_item: dict[str, Any]) -> str:
|
||||
"""Lowercased civitai.model.type of a cache item, or '' when unknown."""
|
||||
civ = cache_item.get("civitai")
|
||||
if not isinstance(civ, dict):
|
||||
return ""
|
||||
model_info = civ.get("model")
|
||||
if not isinstance(model_info, dict):
|
||||
return ""
|
||||
return (model_info.get("type") or "").lower()
|
||||
|
||||
@@ -30,7 +30,7 @@ class MetaFormatParser(RecipeMetadataParser):
|
||||
prompt = parts[0].strip()
|
||||
|
||||
# Initialize metadata
|
||||
metadata = {"prompt": prompt, "loras": []}
|
||||
metadata: Dict[str, Any] = {"prompt": prompt, "loras": []}
|
||||
|
||||
# Extract negative prompt and parameters if available
|
||||
if len(parts) > 1:
|
||||
|
||||
@@ -91,7 +91,15 @@ class RecipeFormatParser(RecipeMetadataParser):
|
||||
exists_locally = lora_scanner.has_hash(lora['hash'])
|
||||
if exists_locally:
|
||||
lora_cache = await lora_scanner.get_cached_data()
|
||||
lora_item = next((item for item in lora_cache.raw_data if item['sha256'].lower() == lora['hash'].lower()), None)
|
||||
# Cascade match: full sha256, stored autov3, or autov2 (sha256[:10]).
|
||||
h = (lora.get('hash') or '').lower()
|
||||
lora_item = next(
|
||||
(item for item in lora_cache.raw_data
|
||||
if (item.get("sha256") or "").lower() == h
|
||||
or (item.get("autov3") or "").lower() == h
|
||||
or (item.get("sha256") or "")[:10].lower() == h),
|
||||
None
|
||||
)
|
||||
if lora_item:
|
||||
lora_entry['existsLocally'] = True
|
||||
lora_entry['inLibrary'] = True
|
||||
@@ -148,7 +156,7 @@ class RecipeFormatParser(RecipeMetadataParser):
|
||||
checkpoint_data = recipe_metadata.get('checkpoint') or {}
|
||||
if isinstance(checkpoint_data, dict) and checkpoint_data:
|
||||
version_id = checkpoint_data.get('modelVersionId') or checkpoint_data.get('id')
|
||||
checkpoint_entry = {
|
||||
checkpoint_entry: Dict[str, Any] = {
|
||||
'id': version_id or 0,
|
||||
'modelId': checkpoint_data.get('modelId', 0),
|
||||
'name': checkpoint_data.get('name', 'Unknown Checkpoint'),
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, Callable, Dict, Mapping
|
||||
from typing import TYPE_CHECKING, Awaitable, Callable, Dict, Mapping
|
||||
|
||||
import jinja2
|
||||
from aiohttp import web
|
||||
@@ -30,6 +30,7 @@ from ..services.websocket_progress_callback import (
|
||||
WebSocketProgressCallback,
|
||||
)
|
||||
from ..utils.exif_utils import ExifUtils
|
||||
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES
|
||||
from ..utils.metadata_manager import MetadataManager
|
||||
from .model_route_registrar import COMMON_ROUTE_DEFINITIONS, ModelRouteRegistrar
|
||||
from .handlers.model_handlers import (
|
||||
@@ -84,7 +85,7 @@ class BaseModelRoutes(ABC):
|
||||
self.metadata_progress_callback = WebSocketBroadcastCallback()
|
||||
|
||||
self._handler_set: ModelHandlerSet | None = None
|
||||
self._handler_mapping: Dict[str, Callable[[web.Request], web.StreamResponse]] | None = None
|
||||
self._handler_mapping: Dict[str, Callable[[web.Request], Awaitable[web.Response]]] | None = None
|
||||
|
||||
self._preview_service = PreviewAssetService(
|
||||
metadata_manager=MetadataManager,
|
||||
@@ -131,7 +132,7 @@ class BaseModelRoutes(ABC):
|
||||
self._handler_set = None
|
||||
self._handler_mapping = None
|
||||
|
||||
def _ensure_handler_mapping(self) -> Mapping[str, Callable[[web.Request], web.StreamResponse]]:
|
||||
def _ensure_handler_mapping(self) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||
if self._handler_mapping is None:
|
||||
handler_set = self._create_handler_set()
|
||||
self._handler_set = handler_set
|
||||
@@ -220,7 +221,7 @@ class BaseModelRoutes(ABC):
|
||||
)
|
||||
|
||||
@property
|
||||
def route_handlers(self) -> Mapping[str, Callable[[web.Request], web.StreamResponse]]:
|
||||
def route_handlers(self) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||
return self._ensure_handler_mapping()
|
||||
|
||||
def setup_routes(self, app: web.Application, prefix: str) -> None:
|
||||
@@ -237,7 +238,7 @@ class BaseModelRoutes(ABC):
|
||||
"""Setup model-specific routes."""
|
||||
raise NotImplementedError
|
||||
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict:
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict[str, Any]:
|
||||
"""Parse model-specific parameters - to be overridden by subclasses."""
|
||||
return {}
|
||||
|
||||
@@ -251,9 +252,9 @@ class BaseModelRoutes(ABC):
|
||||
|
||||
def _find_model_file(self, files):
|
||||
"""Find the appropriate model file from the files list - can be overridden by subclasses."""
|
||||
return next((file for file in files if file.get("type") in ("Model", "Diffusion Model") and file.get("primary") is True), None)
|
||||
return next((file for file in files if file.get("type") in MODEL_WEIGHT_FILE_TYPES and file.get("primary") is True), None)
|
||||
|
||||
def get_handler(self, name: str) -> Callable[[web.Request], web.StreamResponse]:
|
||||
def get_handler(self, name: str) -> Callable[[web.Request], Awaitable[web.StreamResponse]]:
|
||||
"""Expose handlers for subclasses or tests."""
|
||||
return self._ensure_handler_mapping()[name]
|
||||
|
||||
@@ -285,7 +286,7 @@ class BaseModelRoutes(ABC):
|
||||
)
|
||||
return self.model_lifecycle_service
|
||||
|
||||
def _make_handler_proxy(self, name: str) -> Callable[[web.Request], web.StreamResponse]:
|
||||
def _make_handler_proxy(self, name: str) -> Callable[[web.Request], Awaitable[web.StreamResponse]]:
|
||||
async def proxy(request: web.Request) -> web.StreamResponse:
|
||||
try:
|
||||
handler = self.get_handler(name)
|
||||
|
||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Callable, Mapping
|
||||
from typing import Awaitable, Callable, Mapping
|
||||
|
||||
import jinja2
|
||||
from aiohttp import web
|
||||
@@ -61,7 +61,9 @@ class BaseRecipeRoutes:
|
||||
self._i18n_registered = False
|
||||
self._startup_hooks_registered = False
|
||||
self._handler_set: RecipeHandlerSet | None = None
|
||||
self._handler_mapping: dict[str, Callable] | None = None
|
||||
self._handler_mapping: Mapping[
|
||||
str, Callable[[web.Request], Awaitable[web.StreamResponse]]
|
||||
] | None = None
|
||||
|
||||
async def attach_dependencies(self, app: web.Application | None = None) -> None:
|
||||
"""Resolve shared services from the registry."""
|
||||
@@ -84,7 +86,9 @@ class BaseRecipeRoutes:
|
||||
app.on_startup.append(self.attach_dependencies)
|
||||
self._startup_hooks_registered = True
|
||||
|
||||
def to_route_mapping(self) -> Mapping[str, Callable]:
|
||||
def to_route_mapping(
|
||||
self,
|
||||
) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||
"""Return a mapping of handler name to coroutine for registrar binding."""
|
||||
|
||||
if self._handler_mapping is None:
|
||||
@@ -124,17 +128,17 @@ class BaseRecipeRoutes:
|
||||
or os.environ.get("HF_HUB_DISABLE_TELEMETRY", "0") == "0"
|
||||
)
|
||||
if not standalone_mode:
|
||||
from ..metadata_collector import get_metadata # type: ignore[import-not-found]
|
||||
from ..metadata_collector.metadata_processor import ( # type: ignore[import-not-found]
|
||||
from ..metadata_collector import get_metadata # pyright: ignore[reportMissingImports]
|
||||
from ..metadata_collector.metadata_processor import ( # pyright: ignore[reportMissingImports]
|
||||
MetadataProcessor,
|
||||
)
|
||||
from ..metadata_collector.metadata_registry import ( # type: ignore[import-not-found]
|
||||
from ..metadata_collector.metadata_registry import ( # pyright: ignore[reportMissingImports]
|
||||
MetadataRegistry,
|
||||
)
|
||||
else: # pragma: no cover - optional dependency path
|
||||
get_metadata = None # type: ignore[assignment]
|
||||
MetadataProcessor = None # type: ignore[assignment]
|
||||
MetadataRegistry = None # type: ignore[assignment]
|
||||
get_metadata = None # pyright: ignore[reportAssignmentType]
|
||||
MetadataProcessor = None # pyright: ignore[reportAssignmentType]
|
||||
MetadataRegistry = None # pyright: ignore[reportAssignmentType]
|
||||
|
||||
analysis_service = RecipeAnalysisService(
|
||||
exif_utils=ExifUtils,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import logging
|
||||
from typing import Dict, List, Set
|
||||
from typing import Any, Dict, List, Set
|
||||
from aiohttp import web
|
||||
|
||||
from .base_model_routes import BaseModelRoutes
|
||||
@@ -28,13 +28,13 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
# Attach service dependencies
|
||||
self.attach_service(self.service)
|
||||
|
||||
def setup_routes(self, app: web.Application):
|
||||
def setup_routes(self, app: web.Application, prefix: str = "checkpoints"):
|
||||
"""Setup Checkpoint routes"""
|
||||
# Schedule service initialization on app startup
|
||||
app.on_startup.append(lambda _: self.initialize_services())
|
||||
|
||||
|
||||
# Setup common routes with 'checkpoints' prefix (includes page route)
|
||||
super().setup_routes(app, 'checkpoints')
|
||||
super().setup_routes(app, prefix)
|
||||
|
||||
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
||||
"""Setup Checkpoint-specific routes"""
|
||||
@@ -53,9 +53,9 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
"""Get expected model types string for error messages"""
|
||||
return "Checkpoint"
|
||||
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict:
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict[str, Any]:
|
||||
"""Parse Checkpoint-specific parameters"""
|
||||
params: Dict = {}
|
||||
params: Dict[str, Any] = {}
|
||||
|
||||
if 'checkpoint_hash' in request.query:
|
||||
params['hash_filters'] = {'single_hash': request.query['checkpoint_hash'].lower()}
|
||||
@@ -70,7 +70,7 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
"""Get detailed information for a specific checkpoint by name"""
|
||||
try:
|
||||
name = request.match_info.get('name', '')
|
||||
checkpoint_info = await self.service.get_model_info_by_name(name)
|
||||
checkpoint_info = await self.service.get_model_info_by_name(name) # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
if checkpoint_info:
|
||||
return web.json_response(checkpoint_info)
|
||||
@@ -89,7 +89,7 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
roots.extend(config.checkpoints_roots or [])
|
||||
roots.extend(config.extra_checkpoints_roots or [])
|
||||
# Remove duplicates while preserving order
|
||||
seen: set = set()
|
||||
seen: set[str] = set()
|
||||
unique_roots: List[str] = []
|
||||
for root in roots:
|
||||
if root and root not in seen:
|
||||
@@ -114,7 +114,7 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
roots.extend(config.unet_roots or [])
|
||||
roots.extend(config.extra_unet_roots or [])
|
||||
# Remove duplicates while preserving order
|
||||
seen: set = set()
|
||||
seen: set[str] = set()
|
||||
unique_roots: List[str] = []
|
||||
for root in roots:
|
||||
if root and root not in seen:
|
||||
|
||||
@@ -26,13 +26,13 @@ class EmbeddingRoutes(BaseModelRoutes):
|
||||
# Attach service dependencies
|
||||
self.attach_service(self.service)
|
||||
|
||||
def setup_routes(self, app: web.Application):
|
||||
def setup_routes(self, app: web.Application, prefix: str = "embeddings"):
|
||||
"""Setup Embedding routes"""
|
||||
# Schedule service initialization on app startup
|
||||
app.on_startup.append(lambda _: self.initialize_services())
|
||||
|
||||
|
||||
# Setup common routes with 'embeddings' prefix (includes page route)
|
||||
super().setup_routes(app, 'embeddings')
|
||||
super().setup_routes(app, prefix)
|
||||
|
||||
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
||||
"""Setup Embedding-specific routes"""
|
||||
@@ -51,7 +51,7 @@ class EmbeddingRoutes(BaseModelRoutes):
|
||||
"""Get detailed information for a specific embedding by name"""
|
||||
try:
|
||||
name = request.match_info.get('name', '')
|
||||
embedding_info = await self.service.get_model_info_by_name(name)
|
||||
embedding_info = await self.service.get_model_info_by_name(name) # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
if embedding_info:
|
||||
return web.json_response(embedding_info)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Callable, Mapping
|
||||
from typing import Any, Awaitable, Callable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -35,7 +35,7 @@ class ExampleImagesRoutes:
|
||||
*,
|
||||
ws_manager,
|
||||
download_manager: DownloadManager | None = None,
|
||||
processor=ExampleImagesProcessor,
|
||||
processor: Any = ExampleImagesProcessor,
|
||||
file_manager=ExampleImagesFileManager,
|
||||
cleanup_service: ExampleImagesCleanupService | None = None,
|
||||
) -> None:
|
||||
@@ -46,7 +46,9 @@ class ExampleImagesRoutes:
|
||||
self._file_manager = file_manager
|
||||
self._cleanup_service = cleanup_service or ExampleImagesCleanupService()
|
||||
self._handler_set: ExampleImagesHandlerSet | None = None
|
||||
self._handler_mapping: Mapping[str, Callable[[web.Request], web.StreamResponse]] | None = None
|
||||
self._handler_mapping: Mapping[
|
||||
str, Callable[[web.Request], Awaitable[web.StreamResponse]]
|
||||
] | None = None
|
||||
|
||||
@classmethod
|
||||
def setup_routes(cls, app: web.Application, *, ws_manager) -> None:
|
||||
@@ -61,7 +63,9 @@ class ExampleImagesRoutes:
|
||||
registrar = ExampleImagesRouteRegistrar(app)
|
||||
registrar.register_routes(self.to_route_mapping())
|
||||
|
||||
def to_route_mapping(self) -> Mapping[str, Callable[[web.Request], web.StreamResponse]]:
|
||||
def to_route_mapping(
|
||||
self,
|
||||
) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||
"""Return the registrar-compatible mapping of handler names to callables."""
|
||||
|
||||
if self._handler_mapping is None:
|
||||
|
||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Mapping
|
||||
from typing import Awaitable, Callable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -170,7 +170,7 @@ class ExampleImagesHandlerSet:
|
||||
management: ExampleImagesManagementHandler
|
||||
files: ExampleImagesFileHandler
|
||||
|
||||
def to_route_mapping(self) -> Mapping[str, Callable[[web.Request], web.StreamResponse]]:
|
||||
def to_route_mapping(self) -> Mapping[str, Callable[[web.Request], Awaitable[web.StreamResponse]]]:
|
||||
"""Flatten handler methods into the registrar mapping."""
|
||||
|
||||
return {
|
||||
|
||||
@@ -276,7 +276,7 @@ def _collect_comfyui_session_logs(
|
||||
) -> dict[str, Any]:
|
||||
if log_entries is None:
|
||||
try:
|
||||
import app.logger as comfy_logger
|
||||
import app.logger as comfy_logger # pyright: ignore[reportMissingImports]
|
||||
|
||||
log_entries = list(comfy_logger.get_logs() or [])
|
||||
except Exception as exc: # pragma: no cover - environment dependent
|
||||
@@ -422,10 +422,10 @@ class PromptServerProtocol(Protocol):
|
||||
"""Subset of PromptServer used by the handlers."""
|
||||
|
||||
instance: "PromptServerProtocol"
|
||||
sockets: dict # maps clientId (sid) → WebSocketResponse
|
||||
sockets: dict[str, Any] # maps clientId (sid) → WebSocketResponse
|
||||
|
||||
def send_sync(
|
||||
self, event: str, payload: dict | None = None, sid: str | None = None
|
||||
self, event: str, payload: dict[str, Any] | None = None, sid: str | None = None
|
||||
) -> None: # pragma: no cover - protocol
|
||||
...
|
||||
|
||||
@@ -443,7 +443,12 @@ class UsageStatsFactory(Protocol):
|
||||
class MetadataProviderProtocol(Protocol):
|
||||
async def get_model_versions(
|
||||
self, model_id: int
|
||||
) -> dict | None: # pragma: no cover - protocol
|
||||
) -> dict[str, Any] | None: # pragma: no cover - protocol
|
||||
...
|
||||
|
||||
async def get_user_models(
|
||||
self, username: str, cursor: str | None = None
|
||||
) -> Any: # pragma: no cover - protocol
|
||||
...
|
||||
|
||||
|
||||
@@ -466,16 +471,16 @@ class MetadataArchiveManagerProtocol(Protocol):
|
||||
class BackupServiceProtocol(Protocol):
|
||||
async def create_snapshot(
|
||||
self, *, snapshot_type: str = "manual", persist: bool = False
|
||||
) -> dict: # pragma: no cover - protocol
|
||||
) -> dict[str, Any]: # pragma: no cover - protocol
|
||||
...
|
||||
|
||||
async def restore_snapshot(self, archive_path: str) -> dict: # pragma: no cover - protocol
|
||||
async def restore_snapshot(self, archive_path: str) -> dict[str, Any]: # pragma: no cover - protocol
|
||||
...
|
||||
|
||||
def get_status(self) -> dict: # pragma: no cover - protocol
|
||||
def get_status(self) -> dict[str, Any]: # pragma: no cover - protocol
|
||||
...
|
||||
|
||||
def get_available_snapshots(self) -> list[dict]: # pragma: no cover - protocol
|
||||
def get_available_snapshots(self) -> list[dict[str, Any]]: # pragma: no cover - protocol
|
||||
...
|
||||
|
||||
|
||||
@@ -491,7 +496,7 @@ class NodeRegistry:
|
||||
def __init__(self) -> None:
|
||||
self._lock = asyncio.Lock()
|
||||
# sid → {unique_id → node_info}
|
||||
self._tab_nodes: Dict[str, Dict[str, dict]] = {}
|
||||
self._tab_nodes: Dict[str, Dict[str, dict[str, Any]]] = {}
|
||||
self._ready = asyncio.Event()
|
||||
self._waiting_clients: set[str] = set()
|
||||
|
||||
@@ -504,7 +509,7 @@ class NodeRegistry:
|
||||
# Helpers to build one node dict (extracted so it's reused for each tab)
|
||||
# ------------------------------------------------------------------
|
||||
@staticmethod
|
||||
def _build_node_dict(node: dict) -> dict:
|
||||
def _build_node_dict(node: dict[str, Any]) -> dict[str, Any]:
|
||||
node_id = node["node_id"]
|
||||
graph_id = str(node["graph_id"])
|
||||
unique_id = f"{graph_id}:{node_id}"
|
||||
@@ -513,11 +518,11 @@ class NodeRegistry:
|
||||
bgcolor = node.get("bgcolor") or DEFAULT_NODE_COLOR
|
||||
|
||||
raw_capabilities = node.get("capabilities")
|
||||
capabilities: dict = {}
|
||||
capabilities: dict[str, Any] = {}
|
||||
if isinstance(raw_capabilities, dict):
|
||||
capabilities = dict(raw_capabilities)
|
||||
|
||||
raw_widget_names: list | None = node.get("widget_names")
|
||||
raw_widget_names: list[Any] | None = node.get("widget_names")
|
||||
if not isinstance(raw_widget_names, list):
|
||||
capability_widget_names = capabilities.get("widget_names")
|
||||
raw_widget_names = (
|
||||
@@ -565,9 +570,9 @@ class NodeRegistry:
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
async def register_nodes(self, sid: str, nodes: list[dict]) -> None:
|
||||
async def register_nodes(self, sid: str, nodes: list[dict[str, Any]]) -> None:
|
||||
"""Register/replace the node list for a single ComfyUI tab (identified by *sid*)."""
|
||||
tab_nodes: dict[str, dict] = {}
|
||||
tab_nodes: dict[str, dict[str, Any]] = {}
|
||||
for node in nodes:
|
||||
nd = self._build_node_dict(node)
|
||||
tab_nodes[nd["unique_id"]] = nd
|
||||
@@ -602,7 +607,7 @@ class NodeRegistry:
|
||||
except asyncio.TimeoutError:
|
||||
return False
|
||||
|
||||
async def get_merged_registry(self, active_sids: set[str] | None = None) -> dict:
|
||||
async def get_merged_registry(self, active_sids: set[str] | None = None) -> dict[str, Any]:
|
||||
"""Return the union of all known tab nodes, pruning any tab that is no
|
||||
longer connected."""
|
||||
async with self._lock:
|
||||
@@ -619,8 +624,8 @@ class NodeRegistry:
|
||||
len(stale_sids), stale_sids,
|
||||
)
|
||||
|
||||
merged: dict[str, dict] = {}
|
||||
tab_info: dict[str, dict] = {}
|
||||
merged: dict[str, dict[str, Any]] = {}
|
||||
tab_info: dict[str, dict[str, Any]] = {}
|
||||
for sid, nodes in self._tab_nodes.items():
|
||||
tab_info[sid] = {
|
||||
"node_count": len(nodes),
|
||||
@@ -653,7 +658,7 @@ class SupportersHandler:
|
||||
def __init__(self, logger: logging.Logger | None = None) -> None:
|
||||
self._logger = logger or logging.getLogger(__name__)
|
||||
|
||||
def _load_supporters(self) -> dict:
|
||||
def _load_supporters(self) -> dict[str, Any]:
|
||||
"""Load supporters data from JSON file."""
|
||||
try:
|
||||
current_file = os.path.abspath(__file__)
|
||||
@@ -1229,10 +1234,8 @@ class DoctorHandler:
|
||||
settings_snapshot = _sanitize_sensitive_data(
|
||||
getattr(self._settings, "settings", {}) or {}
|
||||
)
|
||||
startup_messages_getter = getattr(self._settings, "get_startup_messages", None)
|
||||
startup_messages = (
|
||||
list(startup_messages_getter()) if callable(startup_messages_getter) else []
|
||||
)
|
||||
startup_messages_getter: Any = getattr(self._settings, "get_startup_messages", None)
|
||||
startup_messages = list(startup_messages_getter()) if startup_messages_getter else []
|
||||
|
||||
environment = {
|
||||
"app_version": app_version,
|
||||
@@ -1439,7 +1442,7 @@ class SettingsHandler:
|
||||
*,
|
||||
settings_service=None,
|
||||
metadata_provider_updater: Callable[
|
||||
[], Awaitable[None]
|
||||
[], Awaitable[Any]
|
||||
] = update_metadata_providers,
|
||||
downloader_factory: Callable[
|
||||
[], Awaitable[DownloaderProtocol]
|
||||
@@ -1484,8 +1487,8 @@ class SettingsHandler:
|
||||
settings_file = getattr(self._settings, "settings_file", None)
|
||||
if settings_file:
|
||||
response_data["settings_file"] = settings_file
|
||||
messages_getter = getattr(self._settings, "get_startup_messages", None)
|
||||
messages = list(messages_getter()) if callable(messages_getter) else []
|
||||
messages_getter: Any = getattr(self._settings, "get_startup_messages", None)
|
||||
messages = list(messages_getter()) if messages_getter else []
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
@@ -2005,11 +2008,11 @@ async def _noop_backup_service() -> None:
|
||||
|
||||
@dataclass
|
||||
class ServiceRegistryAdapter:
|
||||
get_lora_scanner: Callable[[], Awaitable]
|
||||
get_checkpoint_scanner: Callable[[], Awaitable]
|
||||
get_embedding_scanner: Callable[[], Awaitable]
|
||||
get_downloaded_version_history_service: Callable[[], Awaitable]
|
||||
get_backup_service: Callable[[], Awaitable] = _noop_backup_service
|
||||
get_lora_scanner: Callable[[], Awaitable[Any]]
|
||||
get_checkpoint_scanner: Callable[[], Awaitable[Any]]
|
||||
get_embedding_scanner: Callable[[], Awaitable[Any]]
|
||||
get_downloaded_version_history_service: Callable[[], Awaitable[Any]]
|
||||
get_backup_service: Callable[[], Awaitable[Any]] = _noop_backup_service
|
||||
|
||||
|
||||
class ModelLibraryHandler:
|
||||
@@ -2050,8 +2053,8 @@ class ModelLibraryHandler:
|
||||
return await self._service_registry.get_downloaded_version_history_service()
|
||||
|
||||
@staticmethod
|
||||
def _with_downloaded_flag(versions: list[dict]) -> list[dict]:
|
||||
enriched: list[dict] = []
|
||||
def _with_downloaded_flag(versions: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
enriched: list[dict[str, Any]] = []
|
||||
for version in versions:
|
||||
entry = dict(version)
|
||||
entry.setdefault("hasBeenDownloaded", True)
|
||||
@@ -2244,7 +2247,7 @@ class ModelLibraryHandler:
|
||||
checkpoint_scanner = await self._service_registry.get_checkpoint_scanner()
|
||||
embedding_scanner = await self._service_registry.get_embedding_scanner()
|
||||
|
||||
results: list[dict] = []
|
||||
results: list[dict[str, Any]] = []
|
||||
for model_id in model_ids:
|
||||
lora_versions = await lora_scanner.get_model_versions_by_id(model_id)
|
||||
if lora_versions:
|
||||
@@ -2353,7 +2356,7 @@ class ModelLibraryHandler:
|
||||
)
|
||||
|
||||
try:
|
||||
model_version_id = int(data.get("modelVersionId"))
|
||||
model_version_id = int(data.get("modelVersionId")) # pyright: ignore[reportArgumentType]
|
||||
except (TypeError, ValueError):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Parameter modelVersionId must be an integer"},
|
||||
@@ -2465,10 +2468,11 @@ class ModelLibraryHandler:
|
||||
"checkpoint": checkpoint_scanner,
|
||||
"embedding": embedding_scanner,
|
||||
}
|
||||
scanner = scanner_map.get(found_type)
|
||||
scanner = scanner_map.get(found_type or "")
|
||||
if scanner:
|
||||
persist = getattr(scanner, "_persist_current_cache", None)
|
||||
if callable(persist):
|
||||
scanner.bump_cache_version()
|
||||
persist: Any = getattr(scanner, "_persist_current_cache", None)
|
||||
if persist:
|
||||
await persist()
|
||||
|
||||
history_service = await self._get_download_history_service()
|
||||
@@ -2649,13 +2653,13 @@ class ModelLibraryHandler:
|
||||
}
|
||||
lora_type_aliases = {model_type.lower() for model_type in VALID_LORA_TYPES}
|
||||
|
||||
type_scanner_map: Dict[str, object | None] = {
|
||||
type_scanner_map: Dict[str, Any] = {
|
||||
**{alias: lora_scanner for alias in lora_type_aliases},
|
||||
"checkpoint": checkpoint_scanner,
|
||||
"textualinversion": embedding_scanner,
|
||||
}
|
||||
|
||||
versions: list[dict] = []
|
||||
versions: list[dict[str, Any]] = []
|
||||
history_service = await self._get_download_history_service()
|
||||
model_ids: list[int] = []
|
||||
model_count = 0
|
||||
@@ -2707,6 +2711,8 @@ class ModelLibraryHandler:
|
||||
tags_value = model.get("tags")
|
||||
tags = tags_value if isinstance(tags_value, list) else []
|
||||
model_id = model.get("id")
|
||||
if model_id is None:
|
||||
continue
|
||||
try:
|
||||
model_id_int = int(model_id)
|
||||
except (TypeError, ValueError):
|
||||
@@ -2722,6 +2728,8 @@ class ModelLibraryHandler:
|
||||
continue
|
||||
|
||||
version_id = version.get("id")
|
||||
if version_id is None:
|
||||
continue
|
||||
try:
|
||||
version_id_int = int(version_id)
|
||||
except (TypeError, ValueError):
|
||||
@@ -2783,7 +2791,7 @@ class MetadataArchiveHandler:
|
||||
] = get_metadata_archive_manager,
|
||||
settings_service=None,
|
||||
metadata_provider_updater: Callable[
|
||||
[], Awaitable[None]
|
||||
[], Awaitable[Any]
|
||||
] = update_metadata_providers,
|
||||
) -> None:
|
||||
self._metadata_archive_manager_factory = metadata_archive_manager_factory
|
||||
@@ -2930,7 +2938,7 @@ class BackupHandler:
|
||||
|
||||
if request.content_type.startswith("multipart/"):
|
||||
reader = await request.multipart()
|
||||
field = await reader.next()
|
||||
field: Any = await reader.next()
|
||||
uploaded = False
|
||||
while field is not None:
|
||||
if getattr(field, "filename", None):
|
||||
@@ -3549,7 +3557,7 @@ class NodeRegistryHandler:
|
||||
except (TypeError, ValueError):
|
||||
parsed_node_id = node_identifier
|
||||
|
||||
payload: dict = {
|
||||
payload: dict[str, Any] = {
|
||||
"id": parsed_node_id,
|
||||
"value": value,
|
||||
"mode": mode,
|
||||
@@ -3673,7 +3681,7 @@ class NodeRegistryHandler:
|
||||
except (TypeError, ValueError):
|
||||
parsed_node_id = node_identifier
|
||||
|
||||
payload: dict = {
|
||||
payload: dict[str, Any] = {
|
||||
"id": parsed_node_id,
|
||||
"value": value,
|
||||
"mode": mode,
|
||||
@@ -3740,8 +3748,8 @@ class MiscHandlerSet:
|
||||
doctor: DoctorHandler,
|
||||
example_workflows: ExampleWorkflowsHandler,
|
||||
base_model: BaseModelHandlerSet,
|
||||
hf_handler: HfHandler | None = None,
|
||||
agent_handler: AgentHandler | None = None,
|
||||
hf_handler: Any = None,
|
||||
agent_handler: Any = None,
|
||||
) -> None:
|
||||
self.health = health
|
||||
self.settings = settings
|
||||
|
||||
@@ -51,6 +51,29 @@ LICENSE_FIELDS = (
|
||||
)
|
||||
|
||||
|
||||
_broadcast_models_changed_tasks: set = set()
|
||||
|
||||
|
||||
def _broadcast_models_changed() -> None:
|
||||
"""Notify connected clients that the local model library changed.
|
||||
|
||||
The ComfyUI graph page listens for this event to invalidate its cached
|
||||
model availability data (loras widget missing-model cues / error flags)
|
||||
without waiting for the cache TTL to expire.
|
||||
"""
|
||||
try:
|
||||
from ...services.websocket_manager import ws_manager
|
||||
|
||||
task = asyncio.create_task(ws_manager.broadcast({"type": "models_changed"}))
|
||||
# Keep a reference so the task is not garbage-collected mid-await.
|
||||
_broadcast_models_changed_tasks.add(task)
|
||||
task.add_done_callback(_broadcast_models_changed_tasks.discard)
|
||||
except Exception:
|
||||
logging.getLogger(__name__).debug(
|
||||
"Failed to broadcast models_changed", exc_info=True
|
||||
)
|
||||
|
||||
|
||||
class ModelPageView:
|
||||
"""Render the HTML view for model listings."""
|
||||
|
||||
@@ -71,7 +94,7 @@ class ModelPageView:
|
||||
self._server_i18n = server_i18n
|
||||
self._logger = logger
|
||||
|
||||
def _load_supporters(self) -> dict:
|
||||
def _load_supporters(self) -> dict[str, Any]:
|
||||
"""Load supporters data from JSON file."""
|
||||
try:
|
||||
current_file = os.path.abspath(__file__)
|
||||
@@ -152,7 +175,7 @@ class ModelPageView:
|
||||
self._template_env.filters["t"] = (
|
||||
self._server_i18n.create_template_filter()
|
||||
)
|
||||
self._template_env._i18n_filter_added = True # type: ignore[attr-defined]
|
||||
self._template_env._i18n_filter_added = True # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
from ...services.llm_service import PROVIDER_PRESETS
|
||||
|
||||
@@ -199,7 +222,7 @@ class ModelListingHandler:
|
||||
self,
|
||||
*,
|
||||
service,
|
||||
parse_specific_params: Callable[[web.Request], Dict],
|
||||
parse_specific_params: Callable[[web.Request], Dict[str, Any]],
|
||||
logger: logging.Logger,
|
||||
) -> None:
|
||||
self._service = service
|
||||
@@ -287,7 +310,7 @@ class ModelListingHandler:
|
||||
)
|
||||
return web.json_response({"error": str(exc)}, status=500)
|
||||
|
||||
def _parse_common_params(self, request: web.Request) -> Dict:
|
||||
def _parse_common_params(self, request: web.Request) -> Dict[str, Any]:
|
||||
page = int(request.query.get("page", "1"))
|
||||
page_size = min(int(request.query.get("page_size", "20")), 100)
|
||||
sort_by = request.query.get("sort_by", "name")
|
||||
@@ -460,6 +483,7 @@ class ModelManagementHandler:
|
||||
return web.Response(text="Model path is required", status=400)
|
||||
|
||||
result = await self._lifecycle_service.delete_model(file_path)
|
||||
_broadcast_models_changed()
|
||||
return web.json_response(result)
|
||||
except ValueError as exc:
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=400)
|
||||
@@ -658,7 +682,7 @@ class ModelManagementHandler:
|
||||
try:
|
||||
reader = await request.multipart()
|
||||
|
||||
field = await reader.next()
|
||||
field: Any = await reader.next()
|
||||
if field is None or field.name != "preview_file":
|
||||
raise ValueError("Expected 'preview_file' field")
|
||||
content_type = field.headers.get("Content-Type", "image/png")
|
||||
@@ -700,7 +724,7 @@ class ModelManagementHandler:
|
||||
{
|
||||
"success": True,
|
||||
"preview_url": config.get_preview_static_url(
|
||||
result["preview_path"]
|
||||
str(result["preview_path"])
|
||||
),
|
||||
"preview_nsfw_level": result["preview_nsfw_level"],
|
||||
}
|
||||
@@ -781,7 +805,7 @@ class ModelManagementHandler:
|
||||
|
||||
result = await self._preview_service.replace_preview(
|
||||
model_path=model_path,
|
||||
preview_data=preview_data,
|
||||
preview_data=preview_bytes,
|
||||
content_type=content_type,
|
||||
original_filename=original_filename,
|
||||
nsfw_level=nsfw_level,
|
||||
@@ -793,7 +817,7 @@ class ModelManagementHandler:
|
||||
{
|
||||
"success": True,
|
||||
"preview_url": config.get_preview_static_url(
|
||||
result["preview_path"]
|
||||
str(result["preview_path"])
|
||||
),
|
||||
"preview_nsfw_level": result["preview_nsfw_level"],
|
||||
}
|
||||
@@ -931,6 +955,8 @@ class ModelManagementHandler:
|
||||
file_path=file_path, new_file_name=new_file_name
|
||||
)
|
||||
|
||||
_broadcast_models_changed()
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
**result,
|
||||
@@ -959,6 +985,7 @@ class ModelManagementHandler:
|
||||
)
|
||||
|
||||
result = await self._lifecycle_service.bulk_delete_models(file_paths)
|
||||
_broadcast_models_changed()
|
||||
return web.json_response(result)
|
||||
except ValueError as exc:
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=400)
|
||||
@@ -1061,6 +1088,7 @@ class ModelQueryHandler:
|
||||
await self._service.scan_models(
|
||||
force_refresh=True, rebuild_cache=full_rebuild
|
||||
)
|
||||
_broadcast_models_changed()
|
||||
if self._service.scanner.is_cancelled():
|
||||
return web.json_response(
|
||||
{
|
||||
@@ -1488,8 +1516,73 @@ class ModelQueryHandler:
|
||||
search = request.query.get("search", "").strip()
|
||||
limit = min(int(request.query.get("limit", "15")), 100)
|
||||
offset = max(0, int(request.query.get("offset", "0")))
|
||||
|
||||
folder = request.query.get("folder")
|
||||
recursive = request.query.get("recursive", "true").lower() == "true"
|
||||
base_models = list(request.query.getall("base_model", []))
|
||||
model_types = list(request.query.getall("model_type", []))
|
||||
|
||||
tag_filters: Dict[str, str] = {}
|
||||
for tag in request.query.getall("tag_include", []):
|
||||
if tag:
|
||||
tag_filters[tag] = "include"
|
||||
for tag in request.query.getall("tag_exclude", []):
|
||||
if tag:
|
||||
tag_filters[tag] = "exclude"
|
||||
|
||||
auto_tag_filters: Dict[str, str] = {}
|
||||
for tag in request.query.getall("auto_tag_include", []):
|
||||
if tag:
|
||||
auto_tag_filters[tag] = "include"
|
||||
for tag in request.query.getall("auto_tag_exclude", []):
|
||||
if tag:
|
||||
auto_tag_filters[tag] = "exclude"
|
||||
|
||||
tag_logic = request.query.get("tag_logic", "any").lower()
|
||||
if tag_logic not in ("any", "all"):
|
||||
tag_logic = "any"
|
||||
|
||||
credit_required = request.query.get("credit_required")
|
||||
if credit_required is not None:
|
||||
credit_required = credit_required.lower() not in ("false", "0", "")
|
||||
|
||||
allow_selling_generated_content = request.query.get(
|
||||
"allow_selling_generated_content"
|
||||
)
|
||||
if allow_selling_generated_content is not None:
|
||||
allow_selling_generated_content = (
|
||||
allow_selling_generated_content.lower() not in ("false", "0", "")
|
||||
)
|
||||
|
||||
# The presence of the recursive param (always sent by the loras
|
||||
# widget when filter mode is on) signals that the filter pipeline
|
||||
# must run even when no concrete filter is set, so global settings
|
||||
# like show_only_sfw stay consistent with the list endpoint.
|
||||
apply_filters = (
|
||||
"recursive" in request.query
|
||||
or folder is not None
|
||||
or bool(base_models)
|
||||
or bool(model_types)
|
||||
or bool(tag_filters)
|
||||
or bool(auto_tag_filters)
|
||||
or credit_required is not None
|
||||
or allow_selling_generated_content is not None
|
||||
)
|
||||
|
||||
matching_paths = await self._service.search_relative_paths(
|
||||
search, limit, offset
|
||||
search,
|
||||
limit,
|
||||
offset,
|
||||
folder=folder,
|
||||
recursive=recursive,
|
||||
base_models=base_models,
|
||||
model_types=model_types,
|
||||
tags=tag_filters,
|
||||
auto_tags=auto_tag_filters,
|
||||
tag_logic=tag_logic,
|
||||
credit_required=credit_required,
|
||||
allow_selling_generated_content=allow_selling_generated_content,
|
||||
apply_filters=apply_filters,
|
||||
)
|
||||
return web.json_response(
|
||||
{"success": True, "relative_paths": matching_paths}
|
||||
@@ -1995,7 +2088,7 @@ class ModelCivitaiHandler:
|
||||
settings_service: SettingsManager,
|
||||
ws_manager: WebSocketManager,
|
||||
logger: logging.Logger,
|
||||
metadata_provider_factory: Callable[[], Awaitable],
|
||||
metadata_provider_factory: Callable[[], Awaitable[Any]],
|
||||
validate_model_type: Callable[[str], bool],
|
||||
expected_model_types: Callable[[], str],
|
||||
find_model_file: Callable[
|
||||
@@ -2060,7 +2153,7 @@ class ModelCivitaiHandler:
|
||||
downloaded_version_ids = set(
|
||||
await history_service.get_downloaded_version_ids(
|
||||
self._service.model_type,
|
||||
model_id,
|
||||
int(model_id),
|
||||
)
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - defensive logging
|
||||
@@ -2170,6 +2263,8 @@ class ModelMoveHandler:
|
||||
result = await self._move_service.move_model(
|
||||
file_path, target_path, use_default_paths=use_default_paths
|
||||
)
|
||||
if result.get("success"):
|
||||
_broadcast_models_changed()
|
||||
status = 200 if result.get("success") else 500
|
||||
return web.json_response(result, status=status)
|
||||
except Exception as exc:
|
||||
@@ -2189,6 +2284,8 @@ class ModelMoveHandler:
|
||||
result = await self._move_service.move_models_bulk(
|
||||
file_paths, target_path, use_default_paths=use_default_paths
|
||||
)
|
||||
if result.get("success"):
|
||||
_broadcast_models_changed()
|
||||
return web.json_response(result)
|
||||
except Exception as exc:
|
||||
self._logger.error("Error moving models in bulk: %s", exc, exc_info=True)
|
||||
@@ -2234,6 +2331,7 @@ class ModelAutoOrganizeHandler:
|
||||
progress_callback=self._progress_callback,
|
||||
exclusion_patterns=exclusion_patterns,
|
||||
)
|
||||
_broadcast_models_changed()
|
||||
return web.json_response(result.to_dict())
|
||||
except AutoOrganizeInProgressError:
|
||||
return web.json_response(
|
||||
@@ -2337,8 +2435,8 @@ class ModelUpdateHandler:
|
||||
self._logger.error("Failed to fetch license info: %s", exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
updated: List[Dict[str, str]] = []
|
||||
errors: List[Dict[str, str]] = []
|
||||
updated: List[Dict[str, Any]] = []
|
||||
errors: List[Dict[str, Any]] = []
|
||||
for model_id in model_ids:
|
||||
license_payload = license_map.get(model_id)
|
||||
if not license_payload:
|
||||
@@ -2351,6 +2449,7 @@ class ModelUpdateHandler:
|
||||
model_section = civitai_section.get("model")
|
||||
if not isinstance(model_section, Mapping):
|
||||
model_section = {}
|
||||
model_section = dict(model_section)
|
||||
model_section.update(resolved_payload)
|
||||
civitai_section["model"] = model_section
|
||||
metadata_payload["civitai"] = civitai_section
|
||||
@@ -2366,7 +2465,7 @@ class ModelUpdateHandler:
|
||||
)
|
||||
errors.append({"filePath": metadata_path, "error": str(exc)})
|
||||
|
||||
response_payload = {"success": True, "updated": updated}
|
||||
response_payload: Dict[str, Any] = {"success": True, "updated": updated}
|
||||
missing_model_ids = [mid for mid in model_ids if mid not in license_map]
|
||||
if missing_model_ids:
|
||||
response_payload["missingModelIds"] = missing_model_ids
|
||||
@@ -2715,6 +2814,7 @@ class ModelUpdateHandler:
|
||||
civitai_payload = metadata_payload.get("civitai")
|
||||
if not isinstance(civitai_payload, Mapping):
|
||||
civitai_payload = {}
|
||||
civitai_payload = dict(civitai_payload)
|
||||
|
||||
model_payload = civitai_payload.get("model")
|
||||
if not isinstance(model_payload, Mapping):
|
||||
@@ -2759,7 +2859,7 @@ class ModelUpdateHandler:
|
||||
|
||||
return aggregated
|
||||
|
||||
def _extract_target_model_ids(self, payload: Dict) -> Optional[List[int]]:
|
||||
def _extract_target_model_ids(self, payload: Dict[str, Any]) -> Optional[List[int]]:
|
||||
if not isinstance(payload, Mapping):
|
||||
return None
|
||||
|
||||
@@ -2787,7 +2887,7 @@ class ModelUpdateHandler:
|
||||
return {}
|
||||
|
||||
to_dict = getattr(metadata, "to_dict", None)
|
||||
if callable(to_dict):
|
||||
if to_dict:
|
||||
try:
|
||||
return to_dict()
|
||||
except Exception:
|
||||
@@ -2798,7 +2898,7 @@ class ModelUpdateHandler:
|
||||
|
||||
return {}
|
||||
|
||||
async def _read_json(self, request: web.Request) -> Dict:
|
||||
async def _read_json(self, request: web.Request) -> Dict[str, Any]:
|
||||
if not request.can_read_body:
|
||||
return {}
|
||||
try:
|
||||
@@ -2830,7 +2930,7 @@ class ModelUpdateHandler:
|
||||
record,
|
||||
*,
|
||||
version_context: Optional[Dict[int, Dict[str, Any]]] = None,
|
||||
) -> Dict:
|
||||
) -> Dict[str, Any]:
|
||||
context = version_context or {}
|
||||
# Check user setting for hiding early access versions
|
||||
hide_early_access = False
|
||||
@@ -2859,7 +2959,7 @@ class ModelUpdateHandler:
|
||||
@staticmethod
|
||||
def _serialize_version(
|
||||
version, context: Optional[Dict[str, Any]]
|
||||
) -> Dict:
|
||||
) -> Dict[str, Any]:
|
||||
context = context or {}
|
||||
preview_override = context.get("preview_override")
|
||||
preview_url = (
|
||||
|
||||
@@ -0,0 +1,323 @@
|
||||
"""Handler for the pending-delete undo endpoint.
|
||||
|
||||
Restores a staged delete batch (models or recipes) via
|
||||
``PendingDeleteService.undo`` and then repairs the affected library caches:
|
||||
the model cache entry is restored from the manifest's ``model_snapshot``
|
||||
(including the version index and hash index), tag counts are re-incremented,
|
||||
and the recipe cache is re-populated via ``RecipeScanner.add_recipe``.
|
||||
|
||||
The per-type scanner is resolved from the manifest's ``model_type`` page value
|
||||
through the SAME ServiceRegistry getters the model route registrars use
|
||||
(lora/checkpoint/embedding) - never a hardcoded lora scanner.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Any, Awaitable, Callable, Dict, List, Optional, Set, cast
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from ...services.pending_delete_service import get_pending_delete_service
|
||||
from .model_handlers import _broadcast_models_changed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Manifest ``model_type`` page values -> ServiceRegistry scanner getter names.
|
||||
# The model route registrars resolve per-type scanners via these getters
|
||||
# (lora_routes / checkpoint_routes / embedding_routes); undo must do the same
|
||||
# so the CORRECT cache is restored for the deleted model's type.
|
||||
_MODEL_TYPE_GETTER_NAMES: Dict[str, str] = {
|
||||
"loras": "get_lora_scanner",
|
||||
"checkpoints": "get_checkpoint_scanner",
|
||||
"embeddings": "get_embedding_scanner",
|
||||
}
|
||||
|
||||
# Staged batch ids are ``uuid.uuid4().hex`` (32 lowercase hex chars). The id is
|
||||
# joined into filesystem paths by ``_find_batch_dir``, so reject anything that
|
||||
# does not match this exact shape (blocks path-traversal via batch_id).
|
||||
_BATCH_ID_RE = re.compile(r"^[0-9a-f]{32}$")
|
||||
|
||||
|
||||
class PendingDeleteHandler:
|
||||
"""Handle undo requests for staged model/recipe deletions."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
service_factory: Callable[[], Awaitable[Any]] = get_pending_delete_service,
|
||||
scanner_getter: Optional[Callable[[str], Awaitable[Any]]] = None,
|
||||
recipe_scanner_getter: Optional[Callable[[], Awaitable[Any]]] = None,
|
||||
) -> None:
|
||||
self._service_factory: Callable[[], Awaitable[Any]] = service_factory
|
||||
self._scanner_getter: Callable[[str], Awaitable[Any]] = (
|
||||
scanner_getter or self._resolve_scanner
|
||||
)
|
||||
self._recipe_scanner_getter: Callable[[], Awaitable[Any]] = (
|
||||
recipe_scanner_getter or self._resolve_recipe_scanner
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _resolve_scanner(model_type: str) -> Any:
|
||||
"""Resolve the per-type scanner for a manifest ``model_type``.
|
||||
|
||||
The getter is looked up on the ServiceRegistry module namespace at call
|
||||
time so tests (and the registry stubs) can patch it.
|
||||
"""
|
||||
from ...services import service_registry
|
||||
|
||||
getter_name = _MODEL_TYPE_GETTER_NAMES.get(model_type)
|
||||
if getter_name is None:
|
||||
raise ValueError(f"Unknown model type: {model_type}")
|
||||
getter = getattr(service_registry.ServiceRegistry, getter_name, None)
|
||||
if not callable(getter):
|
||||
raise ValueError(f"No scanner getter for model type: {model_type}")
|
||||
scanner = await cast(Callable[[], Awaitable[Any]], getter)()
|
||||
if scanner is None:
|
||||
raise ValueError(f"No scanner registered for model type: {model_type}")
|
||||
return scanner
|
||||
|
||||
@staticmethod
|
||||
async def _resolve_recipe_scanner() -> Any:
|
||||
"""Resolve the recipe scanner via the ServiceRegistry module namespace."""
|
||||
from ...services import service_registry
|
||||
|
||||
getter = getattr(service_registry.ServiceRegistry, "get_recipe_scanner", None)
|
||||
if not callable(getter):
|
||||
raise ValueError("Recipe scanner getter unavailable")
|
||||
scanner = await cast(Callable[[], Awaitable[Any]], getter)()
|
||||
if scanner is None:
|
||||
raise ValueError("No recipe scanner registered")
|
||||
return scanner
|
||||
|
||||
async def undo_delete(self, request: web.Request) -> web.Response:
|
||||
"""Restore a staged batch and its library cache entry.
|
||||
|
||||
Body: ``{"batch_id": str}``. On success returns
|
||||
``{"success": True, "restored": [<original paths>], "kind": kind}``.
|
||||
Expired/unknown batches and occupied target paths -> 404.
|
||||
"""
|
||||
try:
|
||||
data = await request.json()
|
||||
except Exception:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Invalid JSON body"}, status=400
|
||||
)
|
||||
if not isinstance(data, dict):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Invalid JSON body"}, status=400
|
||||
)
|
||||
batch_id = data.get("batch_id")
|
||||
if not batch_id or not isinstance(batch_id, str):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "batch_id is required"}, status=400
|
||||
)
|
||||
if not _BATCH_ID_RE.fullmatch(batch_id):
|
||||
# batch_id is joined into a path by _find_batch_dir - restrict to
|
||||
# the exact staged-id shape so traversal payloads get 400.
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Invalid batch_id"}, status=400
|
||||
)
|
||||
|
||||
service = await self._service_factory()
|
||||
try:
|
||||
# Read the manifest BEFORE undo: undo() removes the batch dir.
|
||||
manifest = await self._read_staged_manifest(service, batch_id)
|
||||
result = await service.undo(batch_id)
|
||||
except ValueError as exc:
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=404)
|
||||
except Exception as exc:
|
||||
logger.error("Unexpected error undoing batch %s: %s", batch_id, exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
kind = result.get("kind")
|
||||
try:
|
||||
if kind == "model":
|
||||
if manifest is not None:
|
||||
await self._restore_model_cache(manifest)
|
||||
else:
|
||||
# undo() raises when the manifest is missing, so this only
|
||||
# happens defensively - files are restored regardless.
|
||||
logger.warning(
|
||||
"Manifest missing after undo of %s; skipping cache restore",
|
||||
batch_id,
|
||||
)
|
||||
_broadcast_models_changed()
|
||||
elif kind == "recipe":
|
||||
# Recipe undo is client-refresh only: re-add to the scanner
|
||||
# cache, no models_changed broadcast.
|
||||
if manifest is not None:
|
||||
await self._restore_recipe_cache(result, manifest)
|
||||
else:
|
||||
logger.warning(
|
||||
"Manifest missing after undo of %s; skipping cache restore",
|
||||
batch_id,
|
||||
)
|
||||
except Exception as exc:
|
||||
# Files are already restored; only the cache restoration failed.
|
||||
logger.error(
|
||||
"Cache restoration failed after undo of %s: %s",
|
||||
batch_id,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"restored": result.get("restored", []),
|
||||
"kind": kind,
|
||||
}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _read_staged_manifest(
|
||||
service: Any, batch_id: str
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Locate and read the batch manifest while it still exists on disk."""
|
||||
batch_dir = await service._find_batch_dir(batch_id)
|
||||
if not batch_dir:
|
||||
return None
|
||||
manifest_path = os.path.join(batch_dir, "manifest.json")
|
||||
try:
|
||||
with open(manifest_path, "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
logger.debug("Failed to read manifest for batch %s: %s", batch_id, exc)
|
||||
return None
|
||||
return payload if isinstance(payload, dict) else None
|
||||
|
||||
async def _restore_model_cache(self, manifest: Dict[str, Any]) -> None:
|
||||
"""Re-add every deleted model's cache entry from the manifest.
|
||||
|
||||
Each main-file entry carries the deleted model's ``snapshot`` (added at
|
||||
stage time), so a merged bulk manifest holds ALL snapshots - undo must
|
||||
restore every one, not just the top-level winner's. Old-format
|
||||
manifests without entry snapshots fall back to the top-level
|
||||
``model_snapshot`` (backward compat / single-delete path).
|
||||
"""
|
||||
model_type = manifest.get("model_type")
|
||||
if not model_type or not isinstance(model_type, str):
|
||||
raise ValueError(f"Manifest carries no model_type: {manifest.get('batch_id')}")
|
||||
scanner = await self._scanner_getter(model_type)
|
||||
|
||||
# Collect one snapshot per distinct file_path from the entry snapshots.
|
||||
snapshots: List[Dict[str, Any]] = []
|
||||
seen: Set[str] = set()
|
||||
for entry in manifest.get("entries") or []:
|
||||
snapshot = entry.get("snapshot")
|
||||
if not isinstance(snapshot, dict):
|
||||
continue
|
||||
file_path = snapshot.get("file_path")
|
||||
if not file_path or not isinstance(file_path, str):
|
||||
continue
|
||||
if file_path in seen:
|
||||
continue
|
||||
seen.add(file_path)
|
||||
snapshots.append(snapshot)
|
||||
|
||||
if not snapshots:
|
||||
# Backward compat: pre-F3 manifests carry only the top-level
|
||||
# model_snapshot (single-delete path, unchanged behavior).
|
||||
top = manifest.get("model_snapshot")
|
||||
if isinstance(top, dict) and top.get("file_path"):
|
||||
snapshots = [top]
|
||||
else:
|
||||
logger.warning(
|
||||
"Manifest %s has no restorable model snapshot; skipping cache restore",
|
||||
manifest.get("batch_id"),
|
||||
)
|
||||
return
|
||||
|
||||
cache = await scanner.get_cached_data()
|
||||
if cache is None:
|
||||
logger.warning(
|
||||
"Scanner cache unavailable for %s; skipping cache restore", model_type
|
||||
)
|
||||
return
|
||||
|
||||
for snapshot in snapshots:
|
||||
file_path = str(snapshot["file_path"])
|
||||
# A rescan between delete and undo may have re-added a stale entry
|
||||
# for this path - drop it so exactly one (the snapshot) remains.
|
||||
cache.raw_data = [
|
||||
item for item in cache.raw_data if item.get("file_path") != file_path
|
||||
]
|
||||
|
||||
# Restore tag counts (mirror of the bulk-delete decrement in
|
||||
# _batch_update_cache_for_deleted_models: undo re-increments).
|
||||
tags = snapshot.get("tags")
|
||||
if isinstance(tags, list):
|
||||
for tag in tags:
|
||||
if not isinstance(tag, str) or not tag:
|
||||
continue
|
||||
scanner._tags_count[tag] = scanner._tags_count.get(tag, 0) + 1
|
||||
|
||||
cache.raw_data.append(dict(snapshot))
|
||||
|
||||
# Re-register the path in the hash index (add_entry guards a
|
||||
# missing sha256 internally; still guard defensively here).
|
||||
sha256 = snapshot.get("sha256") or ""
|
||||
autov3 = snapshot.get("autov3")
|
||||
hash_index = getattr(scanner, "_hash_index", None)
|
||||
if hash_index is not None and sha256 and file_path:
|
||||
hash_index.add_entry(sha256, file_path, autov3)
|
||||
|
||||
# Follow the bulk-delete cache-update pattern ONCE after all entries,
|
||||
# including the explicit version-index rebuild so the version index
|
||||
# does not go stale.
|
||||
cache.rebuild_version_index()
|
||||
await cache.resort()
|
||||
|
||||
scanner.bump_cache_version()
|
||||
|
||||
persist = getattr(scanner, "_persist_current_cache", None)
|
||||
if callable(persist):
|
||||
result = persist()
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
|
||||
async def _restore_recipe_cache(
|
||||
self, result: Dict[str, Any], manifest: Dict[str, Any]
|
||||
) -> None:
|
||||
"""Re-add a restored recipe via ``RecipeScanner.add_recipe``.
|
||||
|
||||
The recipe JSON embeds the full recipe_data (incl. id/file_path);
|
||||
``add_recipe`` only READS the ``_json_path_map`` so the forced frontend
|
||||
refresh self-heals any transient path-map gap.
|
||||
"""
|
||||
restored = result.get("restored") or []
|
||||
json_path = next(
|
||||
(p for p in restored if isinstance(p, str) and p.endswith(".json")),
|
||||
None,
|
||||
)
|
||||
if not json_path or not os.path.exists(json_path):
|
||||
# Defensive fallback to the manifest's recipe_snapshot file_path.
|
||||
snapshot = manifest.get("recipe_snapshot") or {}
|
||||
fallback = snapshot.get("file_path")
|
||||
if fallback and os.path.exists(fallback):
|
||||
json_path = fallback
|
||||
else:
|
||||
logger.warning(
|
||||
"Restored recipe JSON not found in %s; skipping cache restore",
|
||||
restored,
|
||||
)
|
||||
return
|
||||
try:
|
||||
with open(json_path, "r", encoding="utf-8") as handle:
|
||||
recipe_data = json.load(handle)
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
logger.warning("Failed to load restored recipe JSON %s: %s", json_path, exc)
|
||||
return
|
||||
if not isinstance(recipe_data, dict):
|
||||
return
|
||||
recipe_scanner = await self._recipe_scanner_getter()
|
||||
await recipe_scanner.add_recipe(recipe_data)
|
||||
|
||||
|
||||
__all__ = ["PendingDeleteHandler"]
|
||||
@@ -10,7 +10,7 @@ import asyncio
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Awaitable, Callable, Dict, List, Mapping, Optional
|
||||
from typing import Any, Awaitable, Callable, Dict, List, Mapping, Optional, Tuple
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -44,6 +44,22 @@ EnsureDependenciesCallable = Callable[[], Awaitable[None]]
|
||||
RecipeScannerGetter = Callable[[], Any]
|
||||
CivitaiClientGetter = Callable[[], Any]
|
||||
|
||||
# Cap concurrent preview-dimension reads across requests. With a cold LRU
|
||||
# cache one page can touch up to page_size image files; 16 balances SSD and
|
||||
# HDD throughput without starving the event loop.
|
||||
_DIMS_READ_SEMAPHORE = asyncio.Semaphore(16)
|
||||
|
||||
|
||||
async def _read_preview_dims(path: str) -> Optional[Tuple[int, int]]:
|
||||
"""Read preview dimensions off the event loop under the concurrency cap.
|
||||
|
||||
PIL I/O runs in a worker thread so it never blocks the event loop, and the
|
||||
semaphore bounds how many files are opened at once even when many list
|
||||
requests land together.
|
||||
"""
|
||||
async with _DIMS_READ_SEMAPHORE:
|
||||
return await asyncio.to_thread(ExifUtils.get_image_dimensions, path)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RecipeHandlerSet:
|
||||
@@ -96,6 +112,11 @@ class RecipeHandlerSet:
|
||||
"repair_recipe": self.management.repair_recipe,
|
||||
"repair_recipes_bulk": self.management.repair_recipes_bulk,
|
||||
"get_repair_progress": self.management.get_repair_progress,
|
||||
"rematch_recipes": self.management.rematch_recipes,
|
||||
"cancel_rematch": self.management.cancel_rematch,
|
||||
"rematch_recipe": self.management.rematch_recipe,
|
||||
"rematch_recipes_bulk": self.management.rematch_recipes_bulk,
|
||||
"get_rematch_progress": self.management.get_rematch_progress,
|
||||
"start_batch_import": self.batch_import.start_batch_import,
|
||||
"get_batch_import_progress": self.batch_import.get_batch_import_progress,
|
||||
"cancel_batch_import": self.batch_import.cancel_batch_import,
|
||||
@@ -246,7 +267,8 @@ class RecipeListingHandler:
|
||||
recursive=recursive,
|
||||
)
|
||||
|
||||
for item in result.get("items", []):
|
||||
items = result.get("items", [])
|
||||
for item in items:
|
||||
file_path = item.get("file_path")
|
||||
if file_path:
|
||||
item["file_url"] = self.format_recipe_file_url(file_path)
|
||||
@@ -255,6 +277,26 @@ class RecipeListingHandler:
|
||||
item.setdefault("loras", [])
|
||||
item.setdefault("base_model", "")
|
||||
|
||||
# Batch preview dimension reads with asyncio.gather. The previous
|
||||
# loop awaited asyncio.to_thread once per item, so a page_size=100
|
||||
# request submitted 100 sequential thread calls (50-300ms cold-page
|
||||
# latency). gather runs them concurrently while the semaphore caps
|
||||
# disk opens; dimensions stay omitted (not null) when a preview has
|
||||
# no readable size (video, missing file).
|
||||
to_read = [
|
||||
(i, item.get("file_path"))
|
||||
for i, item in enumerate(items)
|
||||
if item.get("file_path")
|
||||
]
|
||||
if to_read:
|
||||
dims_list = await asyncio.gather(
|
||||
*(_read_preview_dims(path) for _, path in to_read)
|
||||
)
|
||||
for (idx, _), dims in zip(to_read, dims_list):
|
||||
if dims:
|
||||
item = items[idx]
|
||||
item["width"], item["height"] = dims
|
||||
|
||||
return web.json_response(result)
|
||||
except Exception as exc:
|
||||
self._logger.error("Error retrieving recipes: %s", exc, exc_info=True)
|
||||
@@ -538,7 +580,12 @@ class RecipeQueryHandler:
|
||||
if recipe_scanner is None:
|
||||
raise RuntimeError("Recipe scanner unavailable")
|
||||
|
||||
fingerprint_groups = await recipe_scanner.find_all_duplicate_recipes()
|
||||
include_prompt = (
|
||||
request.query.get("include_prompt", "false").lower() in ("1", "true")
|
||||
)
|
||||
fingerprint_groups = await recipe_scanner.find_all_duplicate_recipes(
|
||||
include_prompt=include_prompt
|
||||
)
|
||||
url_groups = await recipe_scanner.find_duplicate_recipes_by_source()
|
||||
response_data = []
|
||||
|
||||
@@ -571,6 +618,7 @@ class RecipeQueryHandler:
|
||||
response_data.append(
|
||||
{
|
||||
"type": "fingerprint",
|
||||
"key": f"g-{len(response_data) + 1}",
|
||||
"fingerprint": fingerprint,
|
||||
"count": len(recipes),
|
||||
"recipes": recipes,
|
||||
@@ -606,6 +654,7 @@ class RecipeQueryHandler:
|
||||
response_data.append(
|
||||
{
|
||||
"type": "source_path",
|
||||
"key": f"g-{len(response_data) + 1}",
|
||||
"fingerprint": url,
|
||||
"count": len(recipes),
|
||||
"recipes": recipes,
|
||||
@@ -850,6 +899,159 @@ class RecipeManagementHandler:
|
||||
self._logger.error("Error repairing single recipe: %s", exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def rematch_recipes(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
await self._ensure_dependencies_ready()
|
||||
recipe_scanner = self._recipe_scanner_getter()
|
||||
if recipe_scanner is None:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Recipe scanner unavailable"},
|
||||
status=503,
|
||||
)
|
||||
|
||||
# Mutual exclusion: a global rematch cannot start while a rematch
|
||||
# OR a repair is already running — both mutate recipes under the
|
||||
# same mutation lock.
|
||||
if (
|
||||
self._ws_manager.is_recipe_rematch_running()
|
||||
or self._ws_manager.is_recipe_repair_running()
|
||||
):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Recipe rematch already in progress"},
|
||||
status=409,
|
||||
)
|
||||
|
||||
recipe_scanner.reset_cancellation()
|
||||
|
||||
async def progress_callback(data):
|
||||
await self._ws_manager.broadcast_recipe_rematch_progress(data)
|
||||
|
||||
# Run in background to avoid timeout
|
||||
async def run_rematch():
|
||||
try:
|
||||
await recipe_scanner.rematch_all_recipes(
|
||||
progress_callback=progress_callback
|
||||
)
|
||||
except Exception as e:
|
||||
self._logger.error(
|
||||
f"Error in recipe rematch task: {e}", exc_info=True
|
||||
)
|
||||
await self._ws_manager.broadcast_recipe_rematch_progress(
|
||||
{"status": "error", "error": str(e)}
|
||||
)
|
||||
finally:
|
||||
# Keep the final status for a while so the UI can see it
|
||||
await asyncio.sleep(5)
|
||||
self._ws_manager.cleanup_recipe_rematch_progress()
|
||||
|
||||
asyncio.create_task(run_rematch())
|
||||
|
||||
return web.json_response(
|
||||
{"success": True, "message": "Recipe rematch started"}
|
||||
)
|
||||
except Exception as exc:
|
||||
self._logger.error("Error starting recipe rematch: %s", exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def cancel_rematch(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
await self._ensure_dependencies_ready()
|
||||
recipe_scanner = self._recipe_scanner_getter()
|
||||
if recipe_scanner is None:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Recipe scanner unavailable"},
|
||||
status=503,
|
||||
)
|
||||
|
||||
recipe_scanner.cancel_task()
|
||||
return web.json_response(
|
||||
{"success": True, "message": "Cancellation requested"}
|
||||
)
|
||||
except Exception as exc:
|
||||
self._logger.error("Error cancelling recipe rematch: %s", exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def rematch_recipes_bulk(self, request: web.Request) -> web.Response:
|
||||
"""Rematch deleted resources for multiple recipes by their IDs.
|
||||
|
||||
Accepts a JSON body with a "recipe_ids" array. The per-recipe loop is
|
||||
delegated to the scanner's rematch_recipes_bulk; this handler only
|
||||
parses the request and returns the scanner's summary.
|
||||
"""
|
||||
try:
|
||||
await self._ensure_dependencies_ready()
|
||||
recipe_scanner = self._recipe_scanner_getter()
|
||||
if recipe_scanner is None:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Recipe scanner unavailable"},
|
||||
status=503,
|
||||
)
|
||||
|
||||
# A bulk rematch must not queue behind a running global rematch's
|
||||
# mutation lock.
|
||||
if self._ws_manager.is_recipe_rematch_running():
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Recipe rematch already in progress"},
|
||||
status=409,
|
||||
)
|
||||
|
||||
data = await request.json()
|
||||
recipe_ids = data.get("recipe_ids", [])
|
||||
if not recipe_ids:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "recipe_ids are required"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
result = await recipe_scanner.rematch_recipes_bulk(recipe_ids)
|
||||
return web.json_response(result)
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error performing bulk rematch: %s", exc, exc_info=True
|
||||
)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(exc)}, status=500
|
||||
)
|
||||
|
||||
async def rematch_recipe(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
await self._ensure_dependencies_ready()
|
||||
recipe_scanner = self._recipe_scanner_getter()
|
||||
if recipe_scanner is None:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Recipe scanner unavailable"},
|
||||
status=503,
|
||||
)
|
||||
|
||||
# Reject per-recipe rematches while a global run is in progress so
|
||||
# they do not queue behind the mutation lock.
|
||||
if self._ws_manager.is_recipe_rematch_running():
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Recipe rematch already in progress"},
|
||||
status=409,
|
||||
)
|
||||
|
||||
recipe_id = request.match_info["recipe_id"]
|
||||
result = await recipe_scanner.rematch_recipe_by_id(recipe_id)
|
||||
return web.json_response(result)
|
||||
except RecipeNotFoundError as exc:
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=404)
|
||||
except Exception as exc:
|
||||
self._logger.error("Error rematching single recipe: %s", exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def get_rematch_progress(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
progress = self._ws_manager.get_recipe_rematch_progress()
|
||||
if progress:
|
||||
return web.json_response({"success": True, "progress": progress})
|
||||
return web.json_response(
|
||||
{"success": False, "message": "No rematch in progress"}, status=404
|
||||
)
|
||||
except Exception as exc:
|
||||
self._logger.error("Error getting rematch progress: %s", exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def reimport_recipe(self, request: web.Request) -> web.Response:
|
||||
"""Delete a recipe and re-import it from its source URL.
|
||||
|
||||
@@ -1045,10 +1247,10 @@ class RecipeManagementHandler:
|
||||
*,
|
||||
image_url: str,
|
||||
name: str,
|
||||
lora_entries: list,
|
||||
checkpoint_entry: dict,
|
||||
gen_params_request: dict,
|
||||
tags: list,
|
||||
lora_entries: list[Any],
|
||||
checkpoint_entry: Dict[str, Any] | None,
|
||||
gen_params_request: Dict[str, Any] | None,
|
||||
tags: list[Any],
|
||||
base_model: str,
|
||||
source_path: str,
|
||||
) -> web.Response:
|
||||
@@ -1081,6 +1283,12 @@ class RecipeManagementHandler:
|
||||
_original_image_url,
|
||||
) = await self._download_remote_media(image_url)
|
||||
|
||||
# Build a version-cached map of local model hashes to cache items so
|
||||
# CivitaiApiMetadataParser can skip CivitAI API calls for models that
|
||||
# exist on disk. Built once and shared by every parse pass below.
|
||||
local_cache = await recipe_scanner.build_local_hash_cache()
|
||||
from ...recipes.parsers.civitai_image import CivitaiApiMetadataParser
|
||||
|
||||
# Extract embedded EXIF metadata (offloaded to thread pool in this call)
|
||||
embedded_gen_params = {}
|
||||
parsed_embedded = None
|
||||
@@ -1102,9 +1310,16 @@ class RecipeManagementHandler:
|
||||
)
|
||||
)
|
||||
if parser:
|
||||
parsed_embedded = await parser.parse_metadata(
|
||||
raw_embedded, recipe_scanner=recipe_scanner
|
||||
)
|
||||
if isinstance(parser, CivitaiApiMetadataParser):
|
||||
parsed_embedded = await parser.parse_metadata(
|
||||
raw_embedded,
|
||||
recipe_scanner=recipe_scanner,
|
||||
local_cache=local_cache,
|
||||
)
|
||||
else:
|
||||
parsed_embedded = await parser.parse_metadata(
|
||||
raw_embedded, recipe_scanner=recipe_scanner
|
||||
)
|
||||
if parsed_embedded and "gen_params" in parsed_embedded:
|
||||
embedded_gen_params = parsed_embedded["gen_params"]
|
||||
else:
|
||||
@@ -1135,9 +1350,16 @@ class RecipeManagementHandler:
|
||||
civitai_inner_meta
|
||||
)
|
||||
if parser:
|
||||
civitai_parsed = await parser.parse_metadata(
|
||||
civitai_inner_meta, recipe_scanner=recipe_scanner
|
||||
)
|
||||
if isinstance(parser, CivitaiApiMetadataParser):
|
||||
civitai_parsed = await parser.parse_metadata(
|
||||
civitai_inner_meta,
|
||||
recipe_scanner=recipe_scanner,
|
||||
local_cache=local_cache,
|
||||
)
|
||||
else:
|
||||
civitai_parsed = await parser.parse_metadata(
|
||||
civitai_inner_meta, recipe_scanner=recipe_scanner
|
||||
)
|
||||
if civitai_parsed and "gen_params" in civitai_parsed:
|
||||
# Merge: API gen_params override EXIF at field level,
|
||||
# EXIF fills in fields the API doesn't have.
|
||||
@@ -1641,7 +1863,7 @@ class RecipeManagementHandler:
|
||||
if not provider:
|
||||
return ""
|
||||
|
||||
version_info = await provider.get_model_version_info(version_id)
|
||||
version_info = await provider.get_model_version_info(str(version_id))
|
||||
if isinstance(version_info, tuple):
|
||||
version_info = version_info[0]
|
||||
|
||||
@@ -1761,6 +1983,12 @@ class RecipeManagementHandler:
|
||||
await self._download_remote_media(image_url)
|
||||
)
|
||||
|
||||
# Build a version-cached map of local model hashes to cache items so
|
||||
# CivitaiApiMetadataParser can skip CivitAI API calls for models that
|
||||
# exist on disk. Built once and shared by every parse pass below.
|
||||
local_cache = await recipe_scanner.build_local_hash_cache()
|
||||
from ...recipes.parsers.civitai_image import CivitaiApiMetadataParser
|
||||
|
||||
# Extract embedded EXIF metadata
|
||||
embedded_gen_params = {}
|
||||
parsed_embedded = None
|
||||
@@ -1782,9 +2010,16 @@ class RecipeManagementHandler:
|
||||
)
|
||||
)
|
||||
if parser:
|
||||
parsed_embedded = await parser.parse_metadata(
|
||||
raw_embedded, recipe_scanner=recipe_scanner
|
||||
)
|
||||
if isinstance(parser, CivitaiApiMetadataParser):
|
||||
parsed_embedded = await parser.parse_metadata(
|
||||
raw_embedded,
|
||||
recipe_scanner=recipe_scanner,
|
||||
local_cache=local_cache,
|
||||
)
|
||||
else:
|
||||
parsed_embedded = await parser.parse_metadata(
|
||||
raw_embedded, recipe_scanner=recipe_scanner
|
||||
)
|
||||
if parsed_embedded and "gen_params" in parsed_embedded:
|
||||
embedded_gen_params = parsed_embedded["gen_params"]
|
||||
finally:
|
||||
@@ -1822,9 +2057,16 @@ class RecipeManagementHandler:
|
||||
)
|
||||
)
|
||||
if parser:
|
||||
parsed_embedded = await parser.parse_metadata(
|
||||
raw_orig, recipe_scanner=recipe_scanner
|
||||
)
|
||||
if isinstance(parser, CivitaiApiMetadataParser):
|
||||
parsed_embedded = await parser.parse_metadata(
|
||||
raw_orig,
|
||||
recipe_scanner=recipe_scanner,
|
||||
local_cache=local_cache,
|
||||
)
|
||||
else:
|
||||
parsed_embedded = await parser.parse_metadata(
|
||||
raw_orig, recipe_scanner=recipe_scanner
|
||||
)
|
||||
if (
|
||||
parsed_embedded
|
||||
and "gen_params" in parsed_embedded
|
||||
@@ -1858,9 +2100,16 @@ class RecipeManagementHandler:
|
||||
civitai_inner_meta
|
||||
)
|
||||
if parser:
|
||||
civitai_parsed = await parser.parse_metadata(
|
||||
civitai_inner_meta, recipe_scanner=recipe_scanner
|
||||
)
|
||||
if isinstance(parser, CivitaiApiMetadataParser):
|
||||
civitai_parsed = await parser.parse_metadata(
|
||||
civitai_inner_meta,
|
||||
recipe_scanner=recipe_scanner,
|
||||
local_cache=local_cache,
|
||||
)
|
||||
else:
|
||||
civitai_parsed = await parser.parse_metadata(
|
||||
civitai_inner_meta, recipe_scanner=recipe_scanner
|
||||
)
|
||||
if civitai_parsed and "gen_params" in civitai_parsed:
|
||||
# Merge: API gen_params override EXIF at field level,
|
||||
# EXIF fills in fields the API doesn't have.
|
||||
@@ -2072,33 +2321,44 @@ class RecipeManagementHandler:
|
||||
parsed_input = {**image_data, **inner_meta}
|
||||
parsed_input.pop("meta", None)
|
||||
|
||||
# Build a local cache of {hash → cache_item} so the parser can
|
||||
# skip CivitAI API calls for models that exist on disk.
|
||||
local_cache: Dict[str, Dict[str, Any]] = {}
|
||||
lora_scanner = getattr(recipe_scanner, "_lora_scanner", None)
|
||||
if lora_scanner and model_hash:
|
||||
try:
|
||||
parent_cache_data = await lora_scanner.get_cached_data()
|
||||
for item in getattr(parent_cache_data, "raw_data", []):
|
||||
if item.get("sha256", "").lower() == model_hash.lower():
|
||||
local_cache[model_hash.lower()] = item
|
||||
# Compute AutoV3 so the parser can also match on
|
||||
# that hash type (CivitAI metadata resources use
|
||||
# AutoV3).
|
||||
file_path = item.get("file_path")
|
||||
if file_path and os.path.exists(file_path):
|
||||
try:
|
||||
from ...utils.file_utils import (
|
||||
calculate_autov3,
|
||||
)
|
||||
autov3 = calculate_autov3(file_path)
|
||||
if autov3:
|
||||
local_cache[autov3.lower()] = item
|
||||
except Exception:
|
||||
pass
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
# Build the shared local hash cache so the parser can skip CivitAI
|
||||
# API calls for models that exist on disk.
|
||||
local_cache: Dict[str, Dict[str, Any]] = (
|
||||
await recipe_scanner.build_local_hash_cache()
|
||||
)
|
||||
|
||||
# Bounded supplement for un-backfilled parents. The shared builder
|
||||
# never computes autov3; when the parent model exists on disk but
|
||||
# its cached entry has no stored AutoV3, compute it for that single
|
||||
# file and register the AutoV3 key so the parser can also match on
|
||||
# that hash type (CivitAI metadata resources use AutoV3). This runs
|
||||
# whenever the parent is found with an empty autov3, independent of
|
||||
# whether the sha256 key is already present in the shared cache.
|
||||
if model_hash:
|
||||
lora_scanner = getattr(recipe_scanner, "_lora_scanner", None)
|
||||
if lora_scanner:
|
||||
try:
|
||||
parent_cache_data = await lora_scanner.get_cached_data()
|
||||
for item in getattr(parent_cache_data, "raw_data", []):
|
||||
if item.get("sha256", "").lower() == model_hash.lower():
|
||||
autov3 = (item.get("autov3") or "").lower()
|
||||
if not autov3:
|
||||
file_path = item.get("file_path")
|
||||
if file_path and os.path.exists(file_path):
|
||||
try:
|
||||
from ...utils.file_utils import (
|
||||
calculate_autov3,
|
||||
)
|
||||
autov3 = (
|
||||
calculate_autov3(file_path) or ""
|
||||
).lower()
|
||||
except Exception:
|
||||
pass
|
||||
if autov3:
|
||||
local_cache[autov3] = item
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
parser = self._analysis_service._recipe_parser_factory.create_parser(
|
||||
parsed_input
|
||||
@@ -2130,10 +2390,10 @@ class RecipeManagementHandler:
|
||||
parent_model_id: int | None = None
|
||||
parent_version_name: str | None = None
|
||||
parent_model_name: str | None = None
|
||||
# Prefer sha256 key; fall back to any cached entry.
|
||||
# Resolve the parent strictly by its sha256 key. There is no
|
||||
# arbitrary fallback: with a full-library cache, picking any entry
|
||||
# would corrupt the isDeleted reconciliation below.
|
||||
parent_item = local_cache.get(model_hash.lower()) if model_hash else None
|
||||
if parent_item is None and local_cache:
|
||||
parent_item = next(iter(local_cache.values()))
|
||||
if parent_item:
|
||||
civ = parent_item.get("civitai") or {}
|
||||
if isinstance(civ, dict):
|
||||
@@ -2349,7 +2609,7 @@ class RecipeAnalysisHandler:
|
||||
content_type = request.headers.get("Content-Type", "")
|
||||
if "multipart/form-data" in content_type:
|
||||
reader = await request.multipart()
|
||||
field = await reader.next()
|
||||
field: Any = await reader.next()
|
||||
if field is None or field.name != "image":
|
||||
raise RecipeValidationError("No image field found")
|
||||
image_chunks = bytearray()
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from aiohttp import web
|
||||
from typing import Dict
|
||||
from server import PromptServer # type: ignore
|
||||
from typing import Any, Dict
|
||||
from server import PromptServer # pyright: ignore[reportMissingImports]
|
||||
|
||||
from .base_model_routes import BaseModelRoutes
|
||||
from .model_route_registrar import ModelRouteRegistrar
|
||||
@@ -31,13 +31,13 @@ class LoraRoutes(BaseModelRoutes):
|
||||
# Attach service dependencies
|
||||
self.attach_service(self.service)
|
||||
|
||||
def setup_routes(self, app: web.Application):
|
||||
def setup_routes(self, app: web.Application, prefix: str = "loras"):
|
||||
"""Setup LoRA routes"""
|
||||
# Schedule service initialization on app startup
|
||||
app.on_startup.append(lambda _: self.initialize_services())
|
||||
|
||||
# Setup common routes with 'loras' prefix (includes page route)
|
||||
super().setup_routes(app, "loras")
|
||||
super().setup_routes(app, prefix)
|
||||
|
||||
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
||||
"""Setup LoRA-specific routes"""
|
||||
@@ -73,7 +73,7 @@ class LoraRoutes(BaseModelRoutes):
|
||||
"POST", "/api/lm/{prefix}/get_trigger_words", prefix, self.get_trigger_words
|
||||
)
|
||||
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict:
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict[str, Any]:
|
||||
"""Parse LoRA-specific parameters"""
|
||||
params = {}
|
||||
|
||||
@@ -119,25 +119,6 @@ class LoraRoutes(BaseModelRoutes):
|
||||
logger.error(f"Error getting letter counts: {e}")
|
||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||
|
||||
async def get_lora_notes(self, request: web.Request) -> web.Response:
|
||||
"""Get notes for a specific LoRA file"""
|
||||
try:
|
||||
lora_name = request.query.get("name")
|
||||
if not lora_name:
|
||||
return web.Response(text="Lora file name is required", status=400)
|
||||
|
||||
notes = await self.service.get_lora_notes(lora_name)
|
||||
if notes is not None:
|
||||
return web.json_response({"success": True, "notes": notes})
|
||||
else:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "LoRA not found in cache"}, status=404
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting lora notes: {e}", exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||
|
||||
async def get_lora_trigger_words(self, request: web.Request) -> web.Response:
|
||||
"""Get trigger words for a specific LoRA file"""
|
||||
try:
|
||||
@@ -168,52 +149,6 @@ class LoraRoutes(BaseModelRoutes):
|
||||
logger.error(f"Error getting lora usage tips by path: {e}", exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||
|
||||
async def get_lora_preview_url(self, request: web.Request) -> web.Response:
|
||||
"""Get the static preview URL for a LoRA file"""
|
||||
try:
|
||||
lora_name = request.query.get("name")
|
||||
if not lora_name:
|
||||
return web.Response(text="Lora file name is required", status=400)
|
||||
|
||||
preview_url = await self.service.get_lora_preview_url(lora_name)
|
||||
if preview_url:
|
||||
return web.json_response({"success": True, "preview_url": preview_url})
|
||||
else:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "No preview URL found for the specified lora",
|
||||
},
|
||||
status=404,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting lora preview URL: {e}", exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||
|
||||
async def get_lora_civitai_url(self, request: web.Request) -> web.Response:
|
||||
"""Get the Civitai URL for a LoRA file"""
|
||||
try:
|
||||
lora_name = request.query.get("name")
|
||||
if not lora_name:
|
||||
return web.Response(text="Lora file name is required", status=400)
|
||||
|
||||
result = await self.service.get_lora_civitai_url(lora_name)
|
||||
if result["civitai_url"]:
|
||||
return web.json_response({"success": True, **result})
|
||||
else:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "No Civitai data found for the specified lora",
|
||||
},
|
||||
status=404,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting lora Civitai URL: {e}", exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(e)}, status=500)
|
||||
|
||||
async def get_random_loras(self, request: web.Request) -> web.Response:
|
||||
"""Get random LoRAs based on filters and strength ranges"""
|
||||
try:
|
||||
@@ -337,7 +272,7 @@ class LoraRoutes(BaseModelRoutes):
|
||||
graph_identifier = entry.get("graph_id")
|
||||
|
||||
try:
|
||||
parsed_node_id = int(node_identifier)
|
||||
parsed_node_id = int(node_identifier) # pyright: ignore[reportArgumentType]
|
||||
except (TypeError, ValueError):
|
||||
parsed_node_id = node_identifier
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ miscellaneous endpoints share a consistent registration flow.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Iterable, Mapping
|
||||
from typing import Any, Callable, Iterable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -147,7 +147,7 @@ class MiscRouteRegistrar:
|
||||
handler_lookup[definition.handler_name],
|
||||
)
|
||||
|
||||
def _bind(self, method: str, path: str, handler: Callable) -> None:
|
||||
def _bind(self, method: str, path: str, handler: Callable[..., Any]) -> None:
|
||||
add_method_name = self._METHOD_MAP[method.upper()]
|
||||
add_method = getattr(self._app.router, add_method_name)
|
||||
add_method(path, handler)
|
||||
|
||||
@@ -7,7 +7,7 @@ import os
|
||||
from typing import Awaitable, Callable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
from server import PromptServer # type: ignore
|
||||
from server import PromptServer # pyright: ignore[reportMissingImports]
|
||||
|
||||
from ..services.metadata_service import (
|
||||
get_metadata_archive_manager,
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Iterable, Mapping
|
||||
from typing import Any, Callable, Iterable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -174,15 +174,15 @@ class ModelRouteRegistrar:
|
||||
handler_lookup[definition.handler_name],
|
||||
)
|
||||
|
||||
def add_route(self, method: str, path: str, handler: Callable) -> None:
|
||||
def add_route(self, method: str, path: str, handler: Callable[..., Any]) -> None:
|
||||
self._bind_route(method, path, handler)
|
||||
|
||||
def add_prefixed_route(
|
||||
self, method: str, path_template: str, prefix: str, handler: Callable
|
||||
self, method: str, path_template: str, prefix: str, handler: Callable[..., Any]
|
||||
) -> None:
|
||||
self._bind_route(method, path_template.replace("{prefix}", prefix), handler)
|
||||
|
||||
def _bind_route(self, method: str, path: str, handler: Callable) -> None:
|
||||
def _bind_route(self, method: str, path: str, handler: Callable[..., Any]) -> None:
|
||||
add_method_name = self._METHOD_MAP[method.upper()]
|
||||
add_method = getattr(self._app.router, add_method_name)
|
||||
add_method(path, handler)
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Route controller for the pending-delete undo endpoint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from .handlers.pending_delete_handler import PendingDeleteHandler
|
||||
|
||||
|
||||
class PendingDeleteRoutes:
|
||||
"""Shared route controller mirroring MiscRoutes/UpdateRoutes.
|
||||
|
||||
Registered ONCE per mode (py/lora_manager.py, standalone.py); NEVER through
|
||||
the per-model-type ModelRouteRegistrar, which is instantiated per model
|
||||
type and would register this non-prefixed route three times.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def setup_routes(app: web.Application) -> None:
|
||||
"""Register the shared undo-delete endpoint."""
|
||||
handler = PendingDeleteHandler()
|
||||
_ = app.router.add_post("/api/lm/undo-delete", handler.undo_delete)
|
||||
|
||||
|
||||
__all__ = ["PendingDeleteRoutes"]
|
||||
@@ -3,7 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Mapping
|
||||
from typing import Any, Callable, Mapping
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
@@ -61,6 +61,11 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("POST", "/api/lm/recipe/{recipe_id}/repair", "repair_recipe"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/repair-bulk", "repair_recipes_bulk"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/repair-progress", "get_repair_progress"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/rematch", "rematch_recipes"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/rematch-bulk", "rematch_recipes_bulk"),
|
||||
RouteDefinition("POST", "/api/lm/recipe/{recipe_id}/rematch", "rematch_recipe"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/cancel-rematch", "cancel_rematch"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/rematch-progress", "get_rematch_progress"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/batch-import/start", "start_batch_import"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/recipes/batch-import/progress", "get_batch_import_progress"
|
||||
@@ -105,7 +110,7 @@ class RecipeRouteRegistrar:
|
||||
handler = handler_lookup[definition.handler_name]
|
||||
self._bind_route(definition.method, definition.path, handler)
|
||||
|
||||
def _bind_route(self, method: str, path: str, handler: Callable) -> None:
|
||||
def _bind_route(self, method: str, path: str, handler: Callable[..., Any]) -> None:
|
||||
add_method_name = self._METHOD_MAP[method.upper()]
|
||||
add_method = getattr(self._app.router, add_method_name)
|
||||
add_method(path, handler)
|
||||
|
||||
+11
-10
@@ -40,10 +40,11 @@ class StatsRoutes:
|
||||
"""Route handlers for Statistics page and API endpoints"""
|
||||
|
||||
def __init__(self):
|
||||
self.lora_scanner = None
|
||||
self.checkpoint_scanner = None
|
||||
self.embedding_scanner = None
|
||||
self.usage_stats = None
|
||||
self.lora_scanner: Any = None
|
||||
self.checkpoint_scanner: Any = None
|
||||
self.embedding_scanner: Any = None
|
||||
self.usage_stats: Any = None
|
||||
self._i18n_filter_added = False
|
||||
self.template_env = jinja2.Environment(
|
||||
loader=jinja2.FileSystemLoader(config.templates_path),
|
||||
autoescape=True
|
||||
@@ -95,9 +96,9 @@ class StatsRoutes:
|
||||
server_i18n.set_locale(user_language)
|
||||
|
||||
# 为模板环境添加i18n过滤器
|
||||
if not hasattr(self.template_env, '_i18n_filter_added'):
|
||||
if not self._i18n_filter_added:
|
||||
self.template_env.filters['t'] = server_i18n.create_template_filter()
|
||||
self.template_env._i18n_filter_added = True
|
||||
self._i18n_filter_added = True
|
||||
|
||||
template = self.template_env.get_template('statistics.html')
|
||||
rendered = template.render(
|
||||
@@ -549,7 +550,7 @@ class StatsRoutes:
|
||||
'error': str(e)
|
||||
}, status=500)
|
||||
|
||||
def _count_unused_models(self, models: List[Dict], usage_data: Dict) -> int:
|
||||
def _count_unused_models(self, models: List[Dict[str, Any]], usage_data: Dict[str, Any]) -> int:
|
||||
"""Count models that have never been used"""
|
||||
used_hashes = set(usage_data.keys())
|
||||
unused_count = 0
|
||||
@@ -560,7 +561,7 @@ class StatsRoutes:
|
||||
|
||||
return unused_count
|
||||
|
||||
def _get_top_used_models(self, usage_data: Dict, model_map: Dict, limit: int) -> List[Dict]:
|
||||
def _get_top_used_models(self, usage_data: Dict[str, Any], model_map: Dict[str, Any], limit: int) -> List[Dict[str, Any]]:
|
||||
"""Get top used models with their metadata"""
|
||||
sorted_usage = sorted(usage_data.items(), key=lambda x: x[1].get('total', 0), reverse=True)
|
||||
|
||||
@@ -578,7 +579,7 @@ class StatsRoutes:
|
||||
|
||||
return top_models
|
||||
|
||||
def _get_usage_timeline(self, usage_data: Dict, days: int) -> List[Dict]:
|
||||
def _get_usage_timeline(self, usage_data: Dict[str, Any], days: int) -> List[Dict[str, Any]]:
|
||||
"""Get usage timeline for the past N days"""
|
||||
timeline = []
|
||||
today = datetime.now()
|
||||
@@ -614,7 +615,7 @@ class StatsRoutes:
|
||||
|
||||
return list(reversed(timeline)) # Oldest to newest
|
||||
|
||||
def _format_size(self, size_bytes: int) -> str:
|
||||
def _format_size(self, size_bytes: float) -> str:
|
||||
"""Format file size in human readable format"""
|
||||
for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
|
||||
if size_bytes < 1024.0:
|
||||
|
||||
+16
-13
@@ -6,7 +6,7 @@ import shutil
|
||||
import tempfile
|
||||
import asyncio
|
||||
from aiohttp import web, ClientError
|
||||
from typing import Dict, List
|
||||
from typing import Any, Dict, List, cast
|
||||
|
||||
from ..utils.settings_paths import ensure_settings_file
|
||||
from ..services.downloader import get_downloader
|
||||
@@ -467,9 +467,10 @@ class UpdateRoutes:
|
||||
if not success:
|
||||
logger.error(f"Failed to fetch release info: {data}")
|
||||
return False, ""
|
||||
|
||||
zip_url = data.get("zipball_url")
|
||||
version = data.get("tag_name", "unknown")
|
||||
|
||||
release_payload = cast(dict[str, Any], data)
|
||||
zip_url = release_payload.get("zipball_url", "")
|
||||
version = release_payload.get("tag_name", "unknown")
|
||||
|
||||
# Download ZIP to temporary file
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".zip") as tmp_zip:
|
||||
@@ -580,9 +581,10 @@ class UpdateRoutes:
|
||||
logger.warning("Failed to fetch GitHub commit: %s", data)
|
||||
return "main", [], 0, ""
|
||||
|
||||
commit_sha = data.get('sha', '')[:7]
|
||||
commit_message = data.get('commit', {}).get('message', '')
|
||||
commit_date = data.get('commit', {}).get('committer', {}).get('date', '')[:10]
|
||||
commit_payload = cast(dict[str, Any], data)
|
||||
commit_sha = commit_payload.get('sha', '')[:7]
|
||||
commit_message = commit_payload.get('commit', {}).get('message', '')
|
||||
commit_date = commit_payload.get('commit', {}).get('committer', {}).get('date', '')[:10]
|
||||
|
||||
version = f"main-{commit_sha}"
|
||||
changelog = [commit_message] if commit_message else []
|
||||
@@ -598,10 +600,11 @@ class UpdateRoutes:
|
||||
custom_headers={'Accept': 'application/vnd.github+json'}
|
||||
)
|
||||
if c_ok:
|
||||
if c_data.get('status') in ('ahead', 'diverged'):
|
||||
behind_by = c_data.get('ahead_by', 0)
|
||||
compare_payload = cast(dict[str, Any], c_data)
|
||||
if compare_payload.get('status') in ('ahead', 'diverged'):
|
||||
behind_by = compare_payload.get('ahead_by', 0)
|
||||
else:
|
||||
behind_by = c_data.get('behind_by', 0)
|
||||
behind_by = compare_payload.get('behind_by', 0)
|
||||
|
||||
return version, changelog, behind_by, commit_date
|
||||
|
||||
@@ -706,7 +709,7 @@ class UpdateRoutes:
|
||||
logger.info(f"Successfully updated to {new_version}")
|
||||
return True, new_version
|
||||
|
||||
except git.exc.GitError as e:
|
||||
except git.exc.GitError as e: # pyright: ignore[reportAttributeAccessIssue]
|
||||
logger.error(f"Git error during update: {e}")
|
||||
return False, ""
|
||||
except Exception as e:
|
||||
@@ -767,7 +770,7 @@ class UpdateRoutes:
|
||||
return git_info
|
||||
|
||||
@staticmethod
|
||||
async def _get_remote_version() -> tuple[str, List[str], List[Dict]]:
|
||||
async def _get_remote_version() -> tuple[str, List[str], List[Dict[str, Any]]]:
|
||||
"""
|
||||
Fetch remote version from GitHub
|
||||
Returns:
|
||||
@@ -789,7 +792,7 @@ class UpdateRoutes:
|
||||
|
||||
# Parse releases
|
||||
releases = []
|
||||
for i, release in enumerate(data):
|
||||
for i, release in enumerate(cast(list[dict[str, Any]], data)):
|
||||
version = release.get('tag_name', '')
|
||||
if not version.startswith('v'):
|
||||
version = f"v{version}"
|
||||
|
||||
@@ -117,7 +117,7 @@ def _render_prompt(template: str, variables: Dict[str, Any]) -> str:
|
||||
Uses simple regex substitution — no Jinja2 dependency needed.
|
||||
"""
|
||||
|
||||
def replace(match: re.Match) -> str:
|
||||
def replace(match: re.Match[str]) -> str:
|
||||
key = match.group(1).strip()
|
||||
value = variables.get(key, "")
|
||||
if isinstance(value, (dict, list)):
|
||||
|
||||
@@ -295,7 +295,7 @@ class PostProcessor:
|
||||
normalises every tag to lowercase for case-insensitive dedup.
|
||||
"""
|
||||
merged: List[str] = []
|
||||
seen: set = set()
|
||||
seen: set[str] = set()
|
||||
for tag in list(existing) + list(new):
|
||||
t = tag.strip().lower()
|
||||
if t and t not in seen:
|
||||
|
||||
@@ -49,7 +49,7 @@ _FRONTMATTER_RE = re.compile(
|
||||
)
|
||||
|
||||
|
||||
def _parse_skill_file(path: Path) -> tuple[dict, str]:
|
||||
def _parse_skill_file(path: Path) -> tuple[dict[str, Any], str]:
|
||||
"""Read a prompt definition file (``prompt.md`` or legacy ``SKILL.md``) and
|
||||
return (frontmatter_dict, body_text).
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from __future__ import annotations
|
||||
|
||||
import html as html_module
|
||||
import re
|
||||
from typing import List, Tuple
|
||||
from typing import Any, List, Tuple
|
||||
|
||||
|
||||
_REPO_URL_PATTERN = re.compile(r"https?://huggingface\.co/([^/]+/[^/]+)")
|
||||
@@ -18,10 +18,10 @@ _REPO_URL_PATTERN = re.compile(r"https?://huggingface\.co/([^/]+/[^/]+)")
|
||||
def extract_simple_markdown_images(
|
||||
markdown_text: str,
|
||||
repo: str,
|
||||
existing_urls: set | None = None,
|
||||
existing_urls: set[str] | None = None,
|
||||
default_width: int = 512,
|
||||
default_height: int = 512,
|
||||
) -> list[dict]:
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Extract standalone markdown images from the README body.
|
||||
|
||||
Matches ```` on lines that are NOT part of a markdown table
|
||||
@@ -36,8 +36,8 @@ def extract_simple_markdown_images(
|
||||
return []
|
||||
|
||||
base_url = f"https://huggingface.co/{repo}/resolve/main"
|
||||
images: list[dict] = []
|
||||
seen_urls: set = set(existing_urls) if existing_urls else set()
|
||||
images: list[dict[str, Any]] = []
|
||||
seen_urls: set[str] = set(existing_urls) if existing_urls else set()
|
||||
|
||||
# Collect lines that are NOT inside fenced code blocks
|
||||
lines = markdown_text.split("\n")
|
||||
@@ -86,10 +86,10 @@ def extract_simple_markdown_images(
|
||||
def extract_html_img_tags(
|
||||
markdown_text: str,
|
||||
repo: str,
|
||||
existing_urls: set | None = None,
|
||||
existing_urls: set[str] | None = None,
|
||||
default_width: int = 512,
|
||||
default_height: int = 512,
|
||||
) -> list[dict]:
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Extract image URLs from HTML ``<img src=\"...\">`` tags in the README.
|
||||
|
||||
Many HF collection repos (e.g. ``deadman44/Z-Image_LoRA``) use raw HTML
|
||||
@@ -103,8 +103,8 @@ def extract_html_img_tags(
|
||||
return []
|
||||
|
||||
base_url = f"https://huggingface.co/{repo}/resolve/main"
|
||||
images: list[dict] = []
|
||||
seen_urls: set = set(existing_urls) if existing_urls else set()
|
||||
images: list[dict[str, Any]] = []
|
||||
seen_urls: set[str] = set(existing_urls) if existing_urls else set()
|
||||
|
||||
for m in re.finditer(
|
||||
r'<img\s[^>]*src=\"([^\"]+)\"',
|
||||
@@ -175,7 +175,7 @@ def extract_gallery_images(
|
||||
repo: str,
|
||||
default_width: int = 512,
|
||||
default_height: int = 512,
|
||||
) -> List[dict]:
|
||||
) -> List[dict[str, Any]]:
|
||||
"""Extract widget/gallery images from the YAML frontmatter of a HF README.
|
||||
|
||||
Args:
|
||||
@@ -196,7 +196,7 @@ def extract_gallery_images(
|
||||
if not frontmatter:
|
||||
return []
|
||||
|
||||
images: List[dict] = []
|
||||
images: List[dict[str, Any]] = []
|
||||
base_url = f"https://huggingface.co/{repo}/resolve/main"
|
||||
w = default_width or 512
|
||||
h = default_height or 512
|
||||
@@ -258,7 +258,7 @@ def extract_gallery_images(
|
||||
text = raw_text
|
||||
|
||||
if url:
|
||||
image: dict = {
|
||||
image: dict[str, Any] = {
|
||||
"url": url,
|
||||
"type": "image",
|
||||
"nsfwLevel": 0,
|
||||
@@ -276,10 +276,10 @@ def extract_gallery_images(
|
||||
def extract_gallery_table_images(
|
||||
markdown_text: str,
|
||||
repo: str,
|
||||
existing_urls: set | None = None,
|
||||
existing_urls: set[str] | None = None,
|
||||
default_width: int = 512,
|
||||
default_height: int = 512,
|
||||
) -> list[dict]:
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Extract images from ``| Preview | Prompt |`` markdown gallery tables.
|
||||
|
||||
Many HF READMEs include a sample-gallery table in the body (outside
|
||||
@@ -295,8 +295,8 @@ def extract_gallery_table_images(
|
||||
return []
|
||||
|
||||
base_url = f"https://huggingface.co/{repo}/resolve/main"
|
||||
images: list[dict] = []
|
||||
seen_urls: set = set(existing_urls) if existing_urls else set()
|
||||
images: list[dict[str, Any]] = []
|
||||
seen_urls: set[str] = set(existing_urls) if existing_urls else set()
|
||||
lines = markdown_text.split("\n")
|
||||
n = len(lines)
|
||||
i = 0
|
||||
@@ -514,7 +514,7 @@ def _strip_standalone_images(text: str) -> str:
|
||||
URL was stripped entirely, making it impossible for the LLM to return
|
||||
a ``preview_url`` for repos that use HTML ``<img>`` tags exclusively.
|
||||
"""
|
||||
def _img_to_md(match: re.Match) -> str:
|
||||
def _img_to_md(match: re.Match[str]) -> str:
|
||||
"""Convert an ``<img>`` tag to markdown image syntax ````."""
|
||||
tag = match.group(0)
|
||||
src_m = re.search(r'src="([^"]+)"', tag) or re.search(r"src='([^']+)'", tag)
|
||||
@@ -942,7 +942,7 @@ def _strip_badge_images(text: str) -> str:
|
||||
"twitter", "colab", "gradio", "space",
|
||||
)
|
||||
|
||||
def _should_remove(m: re.Match) -> str:
|
||||
def _should_remove(m: re.Match[str]) -> str:
|
||||
alt = (m.group(1) or "").lower()
|
||||
for kw in badge_keywords:
|
||||
if kw in alt:
|
||||
|
||||
+146
-18
@@ -1,3 +1,7 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
@@ -7,6 +11,7 @@ import os
|
||||
import secrets
|
||||
import shutil
|
||||
import socket
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
@@ -20,10 +25,43 @@ from .settings_manager import get_settings_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Maximum times the download poll loop will re-schedule a transfer after it
|
||||
# is lost (daemon restart / RPC outage) before failing the download.
|
||||
MAX_TRANSFER_RECOVERY_ATTEMPTS = 2
|
||||
|
||||
# stderr lines matching these markers indicate a disk write failure inside
|
||||
# aria2 (piece cache flush or raw file write). They are promoted to INFO so
|
||||
# the root cause (disk full, permission denied, file locked by another
|
||||
# process, ...) is visible in the default logs; all other stderr output stays
|
||||
# at DEBUG to avoid noise.
|
||||
_DISK_WRITE_ERROR_MARKERS = (
|
||||
# aria2 wrapper messages (write disk cache flush path)
|
||||
"write disk cache flush failure",
|
||||
"error when trying to flush write cache",
|
||||
"failed to write into the file",
|
||||
"failed to open the file",
|
||||
"failed to seek the file",
|
||||
# underlying root-cause phrases reported via "cause: ..." (POSIX + Windows)
|
||||
"no space left on device",
|
||||
"not enough space on the disk",
|
||||
"input/output error",
|
||||
"permission denied",
|
||||
"access is denied",
|
||||
"disk quota exceeded",
|
||||
"used by another process",
|
||||
"sharing violation",
|
||||
)
|
||||
|
||||
# Minimum interval between INFO-level reports of the same stderr line so a
|
||||
# repeated failure (e.g. aria2 retrying against a full disk) does not spam
|
||||
# the log.
|
||||
STDERR_ERROR_REPORT_INTERVAL = 60.0
|
||||
|
||||
|
||||
def _try_certifi_ca_path() -> str | None:
|
||||
"""Return the certifi CA bundle path if available, else None."""
|
||||
try:
|
||||
import certifi # type: ignore[import-untyped]
|
||||
import certifi # pyright: ignore[reportMissingTypeStubs]
|
||||
|
||||
path = certifi.where()
|
||||
if os.path.isfile(path):
|
||||
@@ -81,10 +119,12 @@ class Aria2Downloader:
|
||||
self._rpc_session: Optional[aiohttp.ClientSession] = None
|
||||
self._rpc_session_lock = asyncio.Lock()
|
||||
self._process_lock = asyncio.Lock()
|
||||
self._register_lock = asyncio.Lock()
|
||||
self._transfers: Dict[str, Aria2Transfer] = {}
|
||||
self._poll_interval = 0.5
|
||||
self._state_store = Aria2TransferStateStore()
|
||||
self._stderr_reader_task: Optional[asyncio.Task] = None
|
||||
self._stderr_reader_task: Optional[asyncio.Task[Any]] = None
|
||||
self._stderr_error_report: Dict[str, float] = {}
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
@@ -99,26 +139,58 @@ class Aria2Downloader:
|
||||
progress_callback=None,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> Tuple[bool, str]:
|
||||
"""Download a file using aria2 RPC and wait for completion."""
|
||||
"""Download a file using aria2 RPC and wait for completion.
|
||||
|
||||
The poll loop is self-healing: when the in-memory transfer entry
|
||||
disappears (e.g. another download restarted the daemon and
|
||||
``close()`` cleared ``_transfers``) or the RPC becomes unreachable,
|
||||
the transfer is re-scheduled with ``continue=true`` so the download
|
||||
resumes from the on-disk ``.aria2`` control file. Recovery is bounded
|
||||
by ``MAX_TRANSFER_RECOVERY_ATTEMPTS``.
|
||||
"""
|
||||
|
||||
await self._ensure_process()
|
||||
save_path = os.path.abspath(save_path)
|
||||
transfer = self._transfers.get(download_id)
|
||||
if transfer is None or os.path.abspath(transfer.save_path) != save_path:
|
||||
gid = await self._schedule_download(
|
||||
url,
|
||||
save_path,
|
||||
download_id=download_id,
|
||||
headers=headers,
|
||||
)
|
||||
transfer = Aria2Transfer(gid=gid, save_path=save_path)
|
||||
self._transfers[download_id] = transfer
|
||||
|
||||
async with self._register_lock:
|
||||
transfer = self._transfers.get(download_id)
|
||||
if transfer is None or os.path.abspath(transfer.save_path) != save_path:
|
||||
transfer = await self._register_transfer(
|
||||
url,
|
||||
save_path,
|
||||
download_id=download_id,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
recovery_attempts = 0
|
||||
try:
|
||||
while True:
|
||||
status = await self._get_status_with_retry(download_id)
|
||||
try:
|
||||
status = await self._get_status_with_retry(download_id)
|
||||
except Aria2Error:
|
||||
status = None
|
||||
|
||||
if status is None:
|
||||
return False, "aria2 download not found"
|
||||
if recovery_attempts >= MAX_TRANSFER_RECOVERY_ATTEMPTS:
|
||||
return False, "aria2 download not found"
|
||||
recovery_attempts += 1
|
||||
logger.warning(
|
||||
"aria2 transfer %s lost; re-scheduling with resume "
|
||||
"(attempt %d/%d)",
|
||||
download_id,
|
||||
recovery_attempts,
|
||||
MAX_TRANSFER_RECOVERY_ATTEMPTS,
|
||||
)
|
||||
await asyncio.sleep(1.0)
|
||||
await self._ensure_process()
|
||||
async with self._register_lock:
|
||||
transfer = await self._register_transfer(
|
||||
url,
|
||||
save_path,
|
||||
download_id=download_id,
|
||||
headers=headers,
|
||||
)
|
||||
continue
|
||||
|
||||
snapshot = self._build_progress_snapshot(status)
|
||||
if progress_callback is not None:
|
||||
@@ -135,7 +207,9 @@ class Aria2Downloader:
|
||||
|
||||
await asyncio.sleep(self._poll_interval)
|
||||
finally:
|
||||
self._transfers.pop(download_id, None)
|
||||
current = self._transfers.get(download_id)
|
||||
if current is not None and current.gid == transfer.gid:
|
||||
self._transfers.pop(download_id, None)
|
||||
|
||||
async def _get_status_with_retry(
|
||||
self, download_id: str, *, max_retries: int = 4, retry_delay: float = 3.0
|
||||
@@ -190,7 +264,7 @@ class Aria2Downloader:
|
||||
download_id,
|
||||
)
|
||||
|
||||
options: Dict[str, str] = {
|
||||
options: Dict[str, Any] = {
|
||||
"dir": save_dir,
|
||||
"out": out_name,
|
||||
"continue": "true",
|
||||
@@ -238,6 +312,25 @@ class Aria2Downloader:
|
||||
)
|
||||
return gid
|
||||
|
||||
async def _register_transfer(
|
||||
self,
|
||||
url: str,
|
||||
save_path: str,
|
||||
*,
|
||||
download_id: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
) -> Aria2Transfer:
|
||||
"""Schedule a download and track it in the in-memory transfer registry."""
|
||||
gid = await self._schedule_download(
|
||||
url,
|
||||
save_path,
|
||||
download_id=download_id,
|
||||
headers=headers,
|
||||
)
|
||||
transfer = Aria2Transfer(gid=gid, save_path=os.path.abspath(save_path))
|
||||
self._transfers[download_id] = transfer
|
||||
return transfer
|
||||
|
||||
async def get_status(self, download_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Return the raw aria2 status payload for a known download."""
|
||||
|
||||
@@ -385,16 +478,51 @@ class Aria2Downloader:
|
||||
blocks, which freezes the entire ``aria2c`` process — including its
|
||||
RPC handler. This background task reads lines from stderr as they
|
||||
arrive and forwards them to Python's logger.
|
||||
|
||||
Lines that indicate a disk write failure (e.g. the "cause: No space
|
||||
left on device" line that follows "Write disk cache flush failure")
|
||||
are promoted to INFO so the root cause is visible without enabling
|
||||
debug logging; every other line stays at DEBUG to avoid noise.
|
||||
"""
|
||||
try:
|
||||
assert self._process is not None and self._process.stderr is not None
|
||||
async for line in self._process.stderr:
|
||||
text = line.decode("utf-8", errors="replace").rstrip()
|
||||
if text:
|
||||
logger.debug("aria2 stderr: %s", text)
|
||||
if self._is_disk_write_error(text):
|
||||
self._report_stderr_error(text)
|
||||
else:
|
||||
logger.debug("aria2 stderr: %s", text)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _is_disk_write_error(text: str) -> bool:
|
||||
lowered = text.lower()
|
||||
return any(marker in lowered for marker in _DISK_WRITE_ERROR_MARKERS)
|
||||
|
||||
def _report_stderr_error(self, text: str) -> None:
|
||||
"""INFO-log a disk write failure line, rate-limited per line text.
|
||||
|
||||
aria2 re-emits the same error chain on every poll/retry while the
|
||||
underlying condition persists; only the first occurrence within
|
||||
``STDERR_ERROR_REPORT_INTERVAL`` seconds is promoted to INFO.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
last = self._stderr_error_report.get(text)
|
||||
if last is not None and now - last < STDERR_ERROR_REPORT_INTERVAL:
|
||||
logger.debug("aria2 stderr (repeated disk write error): %s", text)
|
||||
return
|
||||
# Drop entries older than the window so the map stays bounded even
|
||||
# during a long disk-full episode (piece indexes change per line).
|
||||
self._stderr_error_report = {
|
||||
line: timestamp
|
||||
for line, timestamp in self._stderr_error_report.items()
|
||||
if now - timestamp < STDERR_ERROR_REPORT_INTERVAL
|
||||
}
|
||||
self._stderr_error_report[text] = now
|
||||
logger.info("aria2 disk write failure: %s", text)
|
||||
|
||||
async def _dispatch_progress(self, callback, snapshot: DownloadProgress) -> None:
|
||||
try:
|
||||
result = callback(snapshot, snapshot)
|
||||
|
||||
@@ -8,7 +8,7 @@ from filename, base_model, and CivitAI version name — no manual tagging requir
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Dict, List, Set
|
||||
from typing import Any, Dict, List, Set
|
||||
|
||||
# ── Tag category definitions ──────────────────────────────────────────
|
||||
# Each category maps a display label to a regex pattern.
|
||||
@@ -52,7 +52,7 @@ AUTO_TAG_GROUPS = {
|
||||
DEFAULT_ENABLED_GROUPS = {"mode", "video"}
|
||||
|
||||
|
||||
def _collect_sources(model_data: Dict) -> List[str]:
|
||||
def _collect_sources(model_data: Dict[str, Any]) -> List[str]:
|
||||
"""Collect all text sources from model data for tag matching."""
|
||||
sources: List[str] = []
|
||||
|
||||
@@ -73,7 +73,7 @@ def _collect_sources(model_data: Dict) -> List[str]:
|
||||
return sources
|
||||
|
||||
|
||||
def extract_auto_tags(model_data: Dict) -> List[str]:
|
||||
def extract_auto_tags(model_data: Dict[str, Any]) -> List[str]:
|
||||
"""Extract auto-detected tags from model metadata.
|
||||
|
||||
Uses a two-layer approach:
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
"""Backfill the AutoV3 checked state for models loaded from a persisted snapshot.
|
||||
|
||||
The SQLite persistent cache predates the AutoV3 feature, so entries hydrated
|
||||
from it have a NULL ``autov3`` column (the "not checked yet" state). This
|
||||
service computes the embedded AutoV3 hash for each such model — once per
|
||||
process — and persists it through the scanner's single write path
|
||||
(:meth:`ModelScanner.update_autov3_for_model`), marking every visited row so a
|
||||
subsequent run finds nothing left to do.
|
||||
|
||||
Three-state contract honored here:
|
||||
|
||||
- ``NULL`` (sqlite) / absent (dict) = not checked yet → backfill computes it
|
||||
- ``''`` (sqlite/dict) / JSON null = checked, no value available → never recompute
|
||||
- 12-char lowercase hex = value → never recompute
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover - type-check only; runtime imports are local
|
||||
from .model_scanner import ModelScanner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _resolve_autov3(file_path: str) -> str:
|
||||
"""Resolve the AutoV3 hash for a model file.
|
||||
|
||||
Prefers the Civitai AutoV3 reported for the file whose SHA256 matches
|
||||
(the authoritative value for recipe matching); falls back to the embedded
|
||||
safetensors header hash. Returns ``''`` when neither is available.
|
||||
"""
|
||||
try:
|
||||
metadata_path = f"{os.path.splitext(file_path)[0]}.metadata.json"
|
||||
if os.path.exists(metadata_path):
|
||||
with open(metadata_path, "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
if isinstance(payload, dict):
|
||||
from ..utils.models import autov3_from_civitai_files # local import avoids cycles
|
||||
|
||||
sha256 = (payload.get("sha256") or "").lower()
|
||||
civitai_autov3 = autov3_from_civitai_files(payload.get("civitai"), sha256)
|
||||
if civitai_autov3:
|
||||
return civitai_autov3
|
||||
except Exception:
|
||||
pass
|
||||
from ..utils.file_utils import calculate_autov3 # local import avoids cycles
|
||||
|
||||
return calculate_autov3(file_path) or ""
|
||||
|
||||
|
||||
class Autov3BackfillService:
|
||||
"""Compute and persist AutoV3 hashes for models missing a checked state."""
|
||||
|
||||
_instance: Optional["Autov3BackfillService"] = None
|
||||
_instance_lock = threading.Lock()
|
||||
|
||||
def __init__(self) -> None:
|
||||
# Re-entrancy guard per model type: scanners for different model types
|
||||
# initialize concurrently (lora_manager.py), so a global guard would
|
||||
# silently skip every type but the first to start. Each model type
|
||||
# runs its own backfill; a duplicate trigger for the same type no-ops.
|
||||
self._running_types: set[str] = set()
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls) -> "Autov3BackfillService":
|
||||
"""Return the process-wide singleton instance."""
|
||||
if cls._instance is None:
|
||||
with cls._instance_lock:
|
||||
if cls._instance is None:
|
||||
cls._instance = cls()
|
||||
return cls._instance
|
||||
|
||||
async def backfill(self, scanner: "ModelScanner") -> int:
|
||||
"""Compute AutoV3 for every un-checked model of ``scanner.model_type``.
|
||||
|
||||
Each candidate file is read once via :func:`~py.utils.file_utils.calculate_autov3`
|
||||
(cheap: safetensors header only) and the result is persisted through
|
||||
``scanner.update_autov3_for_model``. Files that no longer exist on
|
||||
disk are skipped — they are intentionally NOT marked, because scanner
|
||||
cleanup removes the stale row later.
|
||||
|
||||
Returns:
|
||||
The number of models successfully updated. Never raises; on any
|
||||
failure a warning is logged and ``0`` is returned. A duplicate
|
||||
trigger for a model type that is already being backfilled returns
|
||||
``0`` immediately; different model types run concurrently.
|
||||
"""
|
||||
model_type = scanner.model_type
|
||||
if model_type in self._running_types:
|
||||
return 0
|
||||
self._running_types.add(model_type)
|
||||
try:
|
||||
# Local imports avoid import cycles at module load time.
|
||||
from .persistent_model_cache import get_persistent_cache
|
||||
from ..utils.file_utils import calculate_autov3
|
||||
|
||||
persistent = getattr(scanner, "_persistent_cache", None) or get_persistent_cache()
|
||||
paths = persistent.get_models_missing_autov3(model_type)
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
count = 0
|
||||
for path in paths:
|
||||
# A file that no longer exists must not be marked; scanner
|
||||
# cleanup removes the stale row later. The existence check and
|
||||
# hash resolution run in the executor so the loop stays
|
||||
# responsive to API requests while the backfill iterates a
|
||||
# large library.
|
||||
if not await loop.run_in_executor(None, os.path.exists, path):
|
||||
continue
|
||||
autov3 = await loop.run_in_executor(None, _resolve_autov3, path)
|
||||
if await scanner.update_autov3_for_model(model_type, path, autov3):
|
||||
count += 1
|
||||
|
||||
if paths:
|
||||
logger.info(
|
||||
"AutoV3 backfill: updated %d/%d models for %s",
|
||||
count,
|
||||
len(paths),
|
||||
model_type,
|
||||
)
|
||||
else:
|
||||
# Steady state after the first run: nothing left to backfill.
|
||||
logger.debug("AutoV3 backfill: nothing to process for %s", model_type)
|
||||
return count
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"AutoV3 backfill failed for %s: %s",
|
||||
getattr(scanner, "model_type", "?"),
|
||||
exc,
|
||||
)
|
||||
return 0
|
||||
finally:
|
||||
self._running_types.discard(model_type)
|
||||
@@ -1,3 +1,7 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
@@ -2,7 +2,7 @@ from abc import ABC, abstractmethod
|
||||
import asyncio
|
||||
import re
|
||||
import random
|
||||
from typing import Any, Dict, List, Optional, Type, Union, TYPE_CHECKING
|
||||
from typing import Any, Awaitable, Dict, List, Optional, Type, Union, TYPE_CHECKING, cast
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
@@ -70,24 +70,24 @@ class BaseModelService(ABC):
|
||||
page: int,
|
||||
page_size: int,
|
||||
sort_by: str = "name",
|
||||
folder: str = None,
|
||||
folder_include: list = None,
|
||||
folder_exclude: list = None,
|
||||
search: str = None,
|
||||
folder: str | None = None,
|
||||
folder_include: list[str] | None = None,
|
||||
folder_exclude: list[str] | None = None,
|
||||
search: str | None = None,
|
||||
fuzzy_search: bool = False,
|
||||
base_models: list = None,
|
||||
model_types: list = None,
|
||||
base_models: list[str] | None = None,
|
||||
model_types: list[str] | None = None,
|
||||
tags: Optional[Dict[str, str]] = None,
|
||||
auto_tags: Optional[Dict[str, str]] = None,
|
||||
search_options: dict = None,
|
||||
hash_filters: dict = None,
|
||||
search_options: dict[str, Any] | None = None,
|
||||
hash_filters: dict[str, Any] | None = None,
|
||||
favorites_only: bool = False,
|
||||
update_available_only: bool = False,
|
||||
credit_required: Optional[bool] = None,
|
||||
allow_selling_generated_content: Optional[bool] = None,
|
||||
tag_logic: str = "any",
|
||||
**kwargs,
|
||||
) -> Dict:
|
||||
) -> Dict[str, Any]:
|
||||
"""Get paginated and filtered model data"""
|
||||
overall_start = time.perf_counter()
|
||||
|
||||
@@ -178,8 +178,8 @@ class BaseModelService(ABC):
|
||||
ufs = self.settings.get("version_grouping", "same_base")
|
||||
group_by_base = ufs == "same_base"
|
||||
|
||||
model_groups: Dict[Any, List[Dict]] = {}
|
||||
ungrouped_standalone: List[Dict] = []
|
||||
model_groups: Dict[Any, List[Dict[str, Any]]] = {}
|
||||
ungrouped_standalone: List[Dict[str, Any]] = []
|
||||
for item in sorted_data:
|
||||
mid = self._extract_group_key(item)
|
||||
if mid is None:
|
||||
@@ -249,7 +249,7 @@ class BaseModelService(ABC):
|
||||
filter_duration = time.perf_counter() - t1
|
||||
post_filter_count = len(filtered_data)
|
||||
|
||||
annotated_for_filter: Optional[List[Dict]] = None
|
||||
annotated_for_filter: Optional[List[Dict[str, Any]]] = None
|
||||
t2 = time.perf_counter()
|
||||
if update_available_only:
|
||||
annotated_for_filter = await self._annotate_update_flags(filtered_data)
|
||||
@@ -296,11 +296,11 @@ class BaseModelService(ABC):
|
||||
page: int,
|
||||
page_size: int,
|
||||
sort_by: str = "name",
|
||||
search: str = None,
|
||||
search: str | None = None,
|
||||
fuzzy_search: bool = False,
|
||||
search_options: dict = None,
|
||||
search_options: dict[str, Any] | None = None,
|
||||
**kwargs,
|
||||
) -> Dict:
|
||||
) -> Dict[str, Any]:
|
||||
"""Get paginated excluded model data."""
|
||||
excluded_paths = list(self.scanner.get_excluded_models())
|
||||
excluded_entries: List[Dict[str, Any]] = []
|
||||
@@ -326,7 +326,7 @@ class BaseModelService(ABC):
|
||||
]
|
||||
persist_current_cache = getattr(self.scanner, "_persist_current_cache", None)
|
||||
if callable(persist_current_cache):
|
||||
await persist_current_cache()
|
||||
await cast(Awaitable[Any], persist_current_cache())
|
||||
|
||||
excluded_entries = self._sort_entries(excluded_entries, sort_by)
|
||||
|
||||
@@ -444,39 +444,50 @@ class BaseModelService(ABC):
|
||||
return entry
|
||||
|
||||
async def _apply_hash_filters(
|
||||
self, data: List[Dict], hash_filters: Dict
|
||||
) -> List[Dict]:
|
||||
"""Apply hash-based filtering"""
|
||||
self, data: List[Dict[str, Any]], hash_filters: Dict[str, Any]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Apply hash-based filtering (SHA256 and AutoV3)."""
|
||||
|
||||
def matches_hash_set(item: Dict[str, Any], hash_set: set[str]) -> bool:
|
||||
"""Check whether an item matches any hash in the set.
|
||||
|
||||
Compares the item's ``sha256`` field and its non-empty ``autov3``
|
||||
field, both case-insensitively.
|
||||
"""
|
||||
if item.get("sha256", "").lower() in hash_set:
|
||||
return True
|
||||
autov3 = item.get("autov3", "")
|
||||
return bool(autov3) and autov3.lower() in hash_set
|
||||
|
||||
single_hash = hash_filters.get("single_hash")
|
||||
multiple_hashes = hash_filters.get("multiple_hashes")
|
||||
|
||||
if single_hash:
|
||||
# Filter by single hash
|
||||
single_hash = single_hash.lower()
|
||||
# Filter by single hash (SHA256 or AutoV3)
|
||||
return [
|
||||
item for item in data if item.get("sha256", "").lower() == single_hash
|
||||
item for item in data if matches_hash_set(item, {single_hash.lower()})
|
||||
]
|
||||
elif multiple_hashes:
|
||||
# Filter by multiple hashes
|
||||
hash_set = set(hash.lower() for hash in multiple_hashes)
|
||||
return [item for item in data if item.get("sha256", "").lower() in hash_set]
|
||||
# Filter by multiple hashes (SHA256 or AutoV3)
|
||||
hash_set = {hash.lower() for hash in multiple_hashes}
|
||||
return [item for item in data if matches_hash_set(item, hash_set)]
|
||||
|
||||
return data
|
||||
|
||||
async def _apply_common_filters(
|
||||
self,
|
||||
data: List[Dict],
|
||||
folder: str = None,
|
||||
folder_include: list = None,
|
||||
folder_exclude: list = None,
|
||||
base_models: list = None,
|
||||
model_types: list = None,
|
||||
data: List[Dict[str, Any]],
|
||||
folder: str | None = None,
|
||||
folder_include: list[str] | None = None,
|
||||
folder_exclude: list[str] | None = None,
|
||||
base_models: list[str] | None = None,
|
||||
model_types: list[str] | None = None,
|
||||
tags: Optional[Dict[str, str]] = None,
|
||||
auto_tags: Optional[Dict[str, str]] = None,
|
||||
favorites_only: bool = False,
|
||||
search_options: dict = None,
|
||||
search_options: dict[str, Any] | None = None,
|
||||
tag_logic: str = "any",
|
||||
) -> List[Dict]:
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Apply common filters that work across all model types"""
|
||||
normalized_options = self.search_strategy.normalize_options(search_options)
|
||||
criteria = FilterCriteria(
|
||||
@@ -495,24 +506,24 @@ class BaseModelService(ABC):
|
||||
|
||||
async def _apply_search_filters(
|
||||
self,
|
||||
data: List[Dict],
|
||||
data: List[Dict[str, Any]],
|
||||
search: str,
|
||||
fuzzy_search: bool,
|
||||
search_options: dict,
|
||||
) -> List[Dict]:
|
||||
search_options: dict[str, Any] | None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Apply search filtering"""
|
||||
normalized_options = self.search_strategy.normalize_options(search_options)
|
||||
return self.search_strategy.apply(
|
||||
data, search, normalized_options, fuzzy_search
|
||||
)
|
||||
|
||||
async def _apply_specific_filters(self, data: List[Dict], **kwargs) -> List[Dict]:
|
||||
async def _apply_specific_filters(self, data: List[Dict[str, Any]], **kwargs) -> List[Dict[str, Any]]:
|
||||
"""Apply model-specific filters - to be overridden by subclasses if needed"""
|
||||
return data
|
||||
|
||||
async def _apply_credit_required_filter(
|
||||
self, data: List[Dict], credit_required: bool
|
||||
) -> List[Dict]:
|
||||
self, data: List[Dict[str, Any]], credit_required: bool
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Apply credit required filtering based on license_flags.
|
||||
|
||||
Args:
|
||||
@@ -542,8 +553,8 @@ class BaseModelService(ABC):
|
||||
return filtered_data
|
||||
|
||||
async def _apply_allow_selling_filter(
|
||||
self, data: List[Dict], allow_selling: bool
|
||||
) -> List[Dict]:
|
||||
self, data: List[Dict[str, Any]], allow_selling: bool
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Apply allow selling generated content filtering based on license_flags.
|
||||
|
||||
Args:
|
||||
@@ -575,8 +586,8 @@ class BaseModelService(ABC):
|
||||
|
||||
async def _annotate_update_flags(
|
||||
self,
|
||||
items: List[Dict],
|
||||
) -> List[Dict]:
|
||||
items: List[Dict[str, Any]],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Attach an update_available flag to each response item.
|
||||
|
||||
Items without a civitai model id default to False.
|
||||
@@ -591,7 +602,7 @@ class BaseModelService(ABC):
|
||||
item["update_available"] = False
|
||||
return annotated
|
||||
|
||||
id_to_items: Dict[int, List[Dict]] = {}
|
||||
id_to_items: Dict[int, List[Dict[str, Any]]] = {}
|
||||
ordered_ids: List[int] = []
|
||||
for item in annotated:
|
||||
model_id = self._extract_model_id(item)
|
||||
@@ -628,7 +639,7 @@ class BaseModelService(ABC):
|
||||
record_method = getattr(self.update_service, "get_records_bulk", None)
|
||||
if callable(record_method):
|
||||
try:
|
||||
records = await record_method(self.model_type, ordered_ids)
|
||||
records = await cast(Awaitable[Any], record_method(self.model_type, ordered_ids))
|
||||
resolved = {
|
||||
model_id: record.has_update(hide_early_access=hide_early_access)
|
||||
for model_id, record in records.items()
|
||||
@@ -648,11 +659,11 @@ class BaseModelService(ABC):
|
||||
bulk_method = getattr(self.update_service, "has_updates_bulk", None)
|
||||
if callable(bulk_method):
|
||||
try:
|
||||
resolved = await bulk_method(
|
||||
resolved = await cast(Awaitable[Any], bulk_method(
|
||||
self.model_type,
|
||||
ordered_ids,
|
||||
hide_early_access=hide_early_access,
|
||||
)
|
||||
))
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"Failed to resolve update status in bulk for %s models (%s): %s",
|
||||
@@ -714,7 +725,7 @@ class BaseModelService(ABC):
|
||||
return annotated
|
||||
|
||||
@staticmethod
|
||||
def _extract_hf_group_key(item: Dict) -> Optional[str]:
|
||||
def _extract_hf_group_key(item: Dict[str, Any]) -> Optional[str]:
|
||||
"""Extract `hf:{owner}/{repo}` from item's ``hf_url``, or None."""
|
||||
hf_url = item.get("hf_url") if isinstance(item, dict) else None
|
||||
if not hf_url or not isinstance(hf_url, str):
|
||||
@@ -727,7 +738,7 @@ class BaseModelService(ABC):
|
||||
return f"hf:{m.group(1)}"
|
||||
|
||||
@staticmethod
|
||||
def _extract_group_key(item: Dict) -> Union[int, str, None]:
|
||||
def _extract_group_key(item: Dict[str, Any]) -> Union[int, str, None]:
|
||||
"""Return the group identity key: CivitAI modelId (int) or HF repo (str).
|
||||
|
||||
Preference order:
|
||||
@@ -741,7 +752,7 @@ class BaseModelService(ABC):
|
||||
return BaseModelService._extract_hf_group_key(item)
|
||||
|
||||
@staticmethod
|
||||
def _extract_model_id(item: Dict) -> Optional[int]:
|
||||
def _extract_model_id(item: Dict[str, Any]) -> Optional[int]:
|
||||
civitai = item.get("civitai") if isinstance(item, dict) else None
|
||||
if not isinstance(civitai, dict):
|
||||
return None
|
||||
@@ -754,7 +765,7 @@ class BaseModelService(ABC):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_version_id(item: Dict) -> Optional[int]:
|
||||
def _extract_version_id(item: Dict[str, Any]) -> Optional[int]:
|
||||
civitai = item.get("civitai") if isinstance(item, dict) else None
|
||||
if not isinstance(civitai, dict):
|
||||
return None
|
||||
@@ -767,7 +778,7 @@ class BaseModelService(ABC):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_base_model(item: Dict) -> Optional[str]:
|
||||
def _extract_base_model(item: Dict[str, Any]) -> Optional[str]:
|
||||
value = item.get("base_model")
|
||||
if value is None:
|
||||
return None
|
||||
@@ -819,7 +830,7 @@ class BaseModelService(ABC):
|
||||
|
||||
return highest_by_base
|
||||
|
||||
def _paginate(self, data: List[Dict], page: int, page_size: int) -> Dict:
|
||||
def _paginate(self, data: List[Dict[str, Any]], page: int, page_size: int) -> Dict[str, Any]:
|
||||
"""Apply pagination to filtered data"""
|
||||
total_items = len(data)
|
||||
start_idx = (page - 1) * page_size
|
||||
@@ -834,7 +845,7 @@ class BaseModelService(ABC):
|
||||
}
|
||||
|
||||
@abstractmethod
|
||||
async def format_response(self, model_data: Dict) -> Optional[Dict]:
|
||||
async def format_response(self, model_data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""Format model data for API response - must be implemented by subclasses.
|
||||
|
||||
Subclasses should return None for corrupted entries so the handler
|
||||
@@ -843,17 +854,17 @@ class BaseModelService(ABC):
|
||||
pass
|
||||
|
||||
# Common service methods that delegate to scanner
|
||||
async def get_top_tags(self, limit: int = 20) -> List[Dict]:
|
||||
async def get_top_tags(self, limit: int = 20) -> List[Dict[str, Any]]:
|
||||
"""Get top tags sorted by frequency"""
|
||||
return await self.scanner.get_top_tags(limit)
|
||||
|
||||
async def search_tags(
|
||||
self, query: str, limit: int = 50
|
||||
) -> List[Dict]:
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Search tags by substring, sorted by frequency"""
|
||||
return await self.scanner.search_tags(query, limit)
|
||||
|
||||
async def get_base_models(self, limit: int = 20) -> List[Dict]:
|
||||
async def get_base_models(self, limit: int = 20) -> List[Dict[str, Any]]:
|
||||
"""Get base models sorted by frequency"""
|
||||
return await self.scanner.get_base_models(limit)
|
||||
|
||||
@@ -920,7 +931,7 @@ class BaseModelService(ABC):
|
||||
"""Get model root directories"""
|
||||
return self.scanner.get_model_roots()
|
||||
|
||||
def filter_civitai_data(self, data: Dict, minimal: bool = False) -> Dict:
|
||||
def filter_civitai_data(self, data: Dict[str, Any], minimal: bool = False) -> Dict[str, Any]:
|
||||
"""Filter relevant fields from CivitAI data"""
|
||||
if not data:
|
||||
return {}
|
||||
@@ -946,7 +957,7 @@ class BaseModelService(ABC):
|
||||
)
|
||||
return {k: data[k] for k in fields if k in data}
|
||||
|
||||
async def get_folder_tree(self, model_root: str) -> Dict:
|
||||
async def get_folder_tree(self, model_root: str) -> Dict[str, Any]:
|
||||
"""Get hierarchical folder tree for a specific model root"""
|
||||
cache = await self.scanner.get_cached_data()
|
||||
|
||||
@@ -975,7 +986,7 @@ class BaseModelService(ABC):
|
||||
|
||||
return tree
|
||||
|
||||
async def get_unified_folder_tree(self) -> Dict:
|
||||
async def get_unified_folder_tree(self) -> Dict[str, Any]:
|
||||
"""Get unified folder tree across all model roots"""
|
||||
cache = await self.scanner.get_cached_data()
|
||||
|
||||
@@ -1004,7 +1015,7 @@ class BaseModelService(ABC):
|
||||
|
||||
return unified_tree
|
||||
|
||||
async def get_model_notes(self, model_name: str) -> Optional[dict]:
|
||||
async def get_model_notes(self, model_name: str) -> Optional[dict[str, Any]]:
|
||||
"""Get notes and file_path for a specific model file.
|
||||
|
||||
Supports both simple names (``OWSMianne_ANIMA_V1``) and full-path
|
||||
@@ -1136,7 +1147,7 @@ class BaseModelService(ABC):
|
||||
|
||||
return {"civitai_url": None, "model_id": None, "version_id": None}
|
||||
|
||||
async def get_model_metadata(self, file_path: str) -> Optional[Dict]:
|
||||
async def get_model_metadata(self, file_path: str) -> Optional[Dict[str, Any]]:
|
||||
"""Load full metadata for a single model.
|
||||
|
||||
Listing/search endpoints return lightweight cache entries; this method performs
|
||||
@@ -1232,7 +1243,7 @@ class BaseModelService(ABC):
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _relative_path_sort_key(relative_path: str, include_terms: List[str]) -> tuple:
|
||||
def _relative_path_sort_key(relative_path: str, include_terms: List[str]) -> tuple[int, int, int, str]:
|
||||
"""Sort paths by how well they satisfy the include tokens.
|
||||
|
||||
Sorts based on path without extension for consistent ordering.
|
||||
@@ -1259,19 +1270,87 @@ class BaseModelService(ABC):
|
||||
)
|
||||
|
||||
async def search_relative_paths(
|
||||
self, search_term: str, limit: int = 15, offset: int = 0
|
||||
self,
|
||||
search_term: str,
|
||||
limit: int = 15,
|
||||
offset: int = 0,
|
||||
*,
|
||||
folder: Optional[str] = None,
|
||||
folder_include: Optional[list[str]] = None,
|
||||
folder_exclude: Optional[list[str]] = None,
|
||||
base_models: Optional[list[str]] = None,
|
||||
model_types: Optional[list[str]] = None,
|
||||
tags: Optional[dict[str, str]] = None,
|
||||
auto_tags: Optional[dict[str, str]] = None,
|
||||
tag_logic: str = "any",
|
||||
credit_required: Optional[bool] = None,
|
||||
allow_selling_generated_content: Optional[bool] = None,
|
||||
recursive: bool = True,
|
||||
apply_filters: bool = False,
|
||||
) -> List[str]:
|
||||
"""Search model relative file paths for autocomplete functionality"""
|
||||
"""Search model relative file paths for autocomplete functionality.
|
||||
|
||||
Optional filter kwargs mirror the filters used by the list endpoint
|
||||
(/api/lm/{prefix}/list). When no filter kwargs are provided the
|
||||
behavior is identical to plain token-based path matching.
|
||||
"""
|
||||
cache = await self.scanner.get_cached_data()
|
||||
include_terms, exclude_terms = self._parse_search_tokens(search_term)
|
||||
|
||||
data = cache.raw_data
|
||||
has_filters = any(
|
||||
[
|
||||
apply_filters,
|
||||
folder is not None,
|
||||
folder_include,
|
||||
folder_exclude,
|
||||
base_models,
|
||||
model_types,
|
||||
tags,
|
||||
auto_tags,
|
||||
credit_required is not None,
|
||||
allow_selling_generated_content is not None,
|
||||
]
|
||||
)
|
||||
if has_filters:
|
||||
# Auto-tags are not stored in the scanner cache — they are computed
|
||||
# on the fly. Pre-compute them only when an auto-tag filter is
|
||||
# active to avoid mutating cache entries unnecessarily.
|
||||
if auto_tags:
|
||||
from .auto_tag_service import extract_auto_tags
|
||||
|
||||
for item in data:
|
||||
if not item.get("auto_tags"):
|
||||
item["auto_tags"] = extract_auto_tags(item)
|
||||
|
||||
criteria = FilterCriteria(
|
||||
folder=folder,
|
||||
folder_include=folder_include,
|
||||
folder_exclude=folder_exclude,
|
||||
base_models=base_models,
|
||||
model_types=model_types,
|
||||
tags=tags,
|
||||
auto_tags=auto_tags,
|
||||
search_options={"recursive": recursive},
|
||||
tag_logic=tag_logic,
|
||||
)
|
||||
data = self.filter_set.apply(data, criteria)
|
||||
if credit_required is not None:
|
||||
data = await self._apply_credit_required_filter(
|
||||
data, credit_required
|
||||
)
|
||||
if allow_selling_generated_content is not None:
|
||||
data = await self._apply_allow_selling_filter(
|
||||
data, allow_selling_generated_content
|
||||
)
|
||||
|
||||
matching_paths = []
|
||||
|
||||
# Get model roots for path calculation
|
||||
model_roots = self.scanner.get_model_roots()
|
||||
|
||||
# Collect all matching paths first (needed for proper sorting and offset)
|
||||
for model in cache.raw_data:
|
||||
for model in data:
|
||||
file_path = model.get("file_path", "")
|
||||
if not file_path:
|
||||
continue
|
||||
|
||||
@@ -59,6 +59,7 @@ class CacheEntryValidator:
|
||||
'notes': ('', False),
|
||||
'usage_tips': ('', False),
|
||||
'hash_status': ('completed', False),
|
||||
'autov3': (None, False),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
@@ -119,8 +120,13 @@ class CacheEntryValidator:
|
||||
if is_required:
|
||||
errors.append(f"Required field '{field_name}' is missing or None")
|
||||
if auto_repair:
|
||||
working_entry[field_name] = cls._get_default_copy(default_value)
|
||||
repaired = True
|
||||
# A missing optional field whose default is None is already
|
||||
# semantically equal to its default (e.g. autov3: absent
|
||||
# means "not checked") — writing None back is a no-op, not
|
||||
# a repair.
|
||||
if default_value is not None:
|
||||
working_entry[field_name] = cls._get_default_copy(default_value)
|
||||
repaired = True
|
||||
continue
|
||||
|
||||
# Validate field type and value
|
||||
@@ -175,6 +181,15 @@ class CacheEntryValidator:
|
||||
# that invalidates the entry, but we also don't mark it repaired.
|
||||
pass
|
||||
|
||||
# Normalize autov3 to lowercase if needed (optional field, never stripped).
|
||||
autov3 = working_entry.get('autov3')
|
||||
if isinstance(autov3, str) and autov3:
|
||||
normalized_autov3 = autov3.lower()
|
||||
if normalized_autov3 != autov3:
|
||||
if auto_repair:
|
||||
working_entry['autov3'] = normalized_autov3
|
||||
repaired = True
|
||||
|
||||
# Determine if entry is valid
|
||||
# Entry is valid if no critical required field errors remain after repair
|
||||
# Critical fields are file_path and sha256
|
||||
@@ -242,6 +257,19 @@ class CacheEntryValidator:
|
||||
"""
|
||||
expected_type = type(default_value)
|
||||
|
||||
# Special case: autov3 is optional with a three-state contract.
|
||||
# None = not checked, "" = checked but unavailable, otherwise a
|
||||
# 12-character hex string (case-insensitive here; normalized to
|
||||
# lowercase separately).
|
||||
if field_name == 'autov3':
|
||||
if value is None or value == "":
|
||||
return None
|
||||
if not isinstance(value, str):
|
||||
return f"Field 'autov3' should be string or None, got {type(value).__name__}"
|
||||
if len(value) != 12 or any(c not in '0123456789abcdefABCDEF' for c in value):
|
||||
return "Field 'autov3' should be a 12-character hex string"
|
||||
return None
|
||||
|
||||
# Special handling for numeric types
|
||||
if expected_type == int:
|
||||
if not isinstance(value, (int, float)):
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
@@ -6,10 +10,10 @@ from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..utils.models import CheckpointMetadata
|
||||
from ..utils.file_utils import find_preview_file, normalize_path
|
||||
from ..utils.file_utils import find_preview_file, normalize_path, calculate_autov3
|
||||
from ..utils.metadata_manager import MetadataManager
|
||||
from ..config import config
|
||||
from .model_scanner import ModelScanner
|
||||
from .model_scanner import ModelScanner, _is_excluded_dir
|
||||
from .model_hash_index import ModelHashIndex
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -62,6 +66,11 @@ class CheckpointScanner(ModelScanner):
|
||||
# Find preview image
|
||||
preview_url = find_preview_file(base_name, dir_path)
|
||||
|
||||
# AutoV3 reads only the safetensors header, so it is cheap even for
|
||||
# large checkpoints; record the checked state at creation time ("" =
|
||||
# checked but unavailable).
|
||||
autov3 = calculate_autov3(real_path)
|
||||
|
||||
# Create metadata WITHOUT calculating hash
|
||||
metadata = CheckpointMetadata(
|
||||
file_name=base_name,
|
||||
@@ -77,6 +86,7 @@ class CheckpointScanner(ModelScanner):
|
||||
sub_type="checkpoint",
|
||||
from_civitai=False, # Mark as local model since no hash yet
|
||||
hash_status="pending", # Mark hash as pending
|
||||
autov3=autov3 or "",
|
||||
)
|
||||
|
||||
# Save the created metadata
|
||||
@@ -120,7 +130,11 @@ class CheckpointScanner(ModelScanner):
|
||||
# that queries get_hash_by_filename first) will miss on every
|
||||
# lookup and keep calling back into this method, creating a
|
||||
# tight loop that never populates the index.
|
||||
self._hash_index.add_entry(metadata.sha256.lower(), file_path)
|
||||
self._hash_index.add_entry(
|
||||
metadata.sha256.lower(),
|
||||
file_path,
|
||||
getattr(metadata, "autov3", None) or None,
|
||||
)
|
||||
return metadata.sha256
|
||||
|
||||
async with self._hash_calculation_lock:
|
||||
@@ -132,7 +146,11 @@ class CheckpointScanner(ModelScanner):
|
||||
and metadata.hash_status == "completed"
|
||||
and metadata.sha256
|
||||
):
|
||||
self._hash_index.add_entry(metadata.sha256.lower(), file_path)
|
||||
self._hash_index.add_entry(
|
||||
metadata.sha256.lower(),
|
||||
file_path,
|
||||
getattr(metadata, "autov3", None) or None,
|
||||
)
|
||||
return metadata.sha256
|
||||
|
||||
task = self._hash_calculation_tasks.get(real_path)
|
||||
@@ -185,7 +203,11 @@ class CheckpointScanner(ModelScanner):
|
||||
if metadata.hash_status == "completed" and metadata.sha256:
|
||||
# Populate the in-memory hash index even for pre-computed
|
||||
# hashes, mirroring the fix in calculate_hash_for_model.
|
||||
self._hash_index.add_entry(metadata.sha256.lower(), file_path)
|
||||
self._hash_index.add_entry(
|
||||
metadata.sha256.lower(),
|
||||
file_path,
|
||||
getattr(metadata, "autov3", None) or None,
|
||||
)
|
||||
return metadata.sha256
|
||||
|
||||
# Update status to calculating
|
||||
@@ -202,7 +224,11 @@ class CheckpointScanner(ModelScanner):
|
||||
await MetadataManager.save_metadata(file_path, metadata)
|
||||
|
||||
# Update hash index
|
||||
self._hash_index.add_entry(sha256.lower(), file_path)
|
||||
self._hash_index.add_entry(
|
||||
sha256.lower(),
|
||||
file_path,
|
||||
getattr(metadata, "autov3", None) or None,
|
||||
)
|
||||
|
||||
# Update the in-memory cache entry so that subsequent
|
||||
# _persist_current_cache / _save_persistent_cache calls
|
||||
@@ -216,6 +242,7 @@ class CheckpointScanner(ModelScanner):
|
||||
if entry.get("file_path") == file_path:
|
||||
entry["sha256"] = sha256.lower()
|
||||
entry["hash_status"] = "completed"
|
||||
self.bump_cache_version()
|
||||
break
|
||||
|
||||
logger.info(f"Hash calculated for checkpoint: {file_path}")
|
||||
@@ -301,7 +328,8 @@ class CheckpointScanner(ModelScanner):
|
||||
if not os.path.exists(root_path):
|
||||
continue
|
||||
|
||||
for dirpath, _dirnames, filenames in os.walk(root_path):
|
||||
for dirpath, dirnames, filenames in os.walk(root_path):
|
||||
dirnames[:] = [d for d in dirnames if not _is_excluded_dir(d)]
|
||||
for filename in filenames:
|
||||
if not filename.endswith(".metadata.json"):
|
||||
continue
|
||||
@@ -405,7 +433,7 @@ class CheckpointScanner(ModelScanner):
|
||||
roots.extend(config.extra_checkpoints_roots or [])
|
||||
roots.extend(config.extra_unet_roots or [])
|
||||
# Remove duplicates while preserving order
|
||||
seen: set = set()
|
||||
seen: set[str] = set()
|
||||
unique_roots: List[str] = []
|
||||
for root in roots:
|
||||
if root not in seen:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import logging
|
||||
from typing import Dict, Optional
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from .base_model_service import BaseModelService
|
||||
from .auto_tag_service import extract_auto_tags
|
||||
@@ -21,58 +21,58 @@ class CheckpointService(BaseModelService):
|
||||
"""
|
||||
super().__init__("checkpoint", scanner, CheckpointMetadata, update_service=update_service)
|
||||
|
||||
async def format_response(self, checkpoint_data: Dict) -> Optional[Dict]:
|
||||
async def format_response(self, model_data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""Format Checkpoint data for API response.
|
||||
|
||||
Returns None when the entry is missing critical fields (corrupted cache
|
||||
row), so the handler layer can filter it out. See issue #730.
|
||||
"""
|
||||
# Guard against corrupted cache entries missing critical fields
|
||||
file_path = checkpoint_data.get("file_path")
|
||||
file_path = model_data.get("file_path")
|
||||
if not file_path or not isinstance(file_path, str):
|
||||
logger.warning(
|
||||
"Skipping corrupted checkpoint entry (missing file_path): %s",
|
||||
checkpoint_data.get("file_name", "<unknown>"),
|
||||
model_data.get("file_name", "<unknown>"),
|
||||
)
|
||||
return None
|
||||
|
||||
# Get sub_type from cache entry (new canonical field)
|
||||
sub_type = checkpoint_data.get("sub_type", "checkpoint")
|
||||
sub_type = model_data.get("sub_type", "checkpoint")
|
||||
|
||||
file_name = checkpoint_data.get("file_name") or ""
|
||||
model_name = checkpoint_data.get("model_name") or file_name
|
||||
folder = checkpoint_data.get("folder") or ""
|
||||
file_name = model_data.get("file_name") or ""
|
||||
model_name = model_data.get("model_name") or file_name
|
||||
folder = model_data.get("folder") or ""
|
||||
|
||||
return {
|
||||
"model_name": model_name,
|
||||
"file_name": file_name,
|
||||
"preview_url": config.get_preview_static_url(checkpoint_data.get("preview_url", "")),
|
||||
"preview_nsfw_level": checkpoint_data.get("preview_nsfw_level", 0),
|
||||
"base_model": checkpoint_data.get("base_model", ""),
|
||||
"preview_url": config.get_preview_static_url(model_data.get("preview_url", "")),
|
||||
"preview_nsfw_level": model_data.get("preview_nsfw_level", 0),
|
||||
"base_model": model_data.get("base_model", ""),
|
||||
"folder": folder,
|
||||
"sha256": checkpoint_data.get("sha256", ""),
|
||||
"sha256": model_data.get("sha256", ""),
|
||||
"file_path": file_path.replace(os.sep, "/"),
|
||||
"file_size": checkpoint_data.get("size", 0),
|
||||
"modified": checkpoint_data.get("modified", ""),
|
||||
"tags": checkpoint_data.get("tags", []),
|
||||
"from_civitai": checkpoint_data.get("from_civitai", True),
|
||||
"usage_count": checkpoint_data.get("usage_count", 0),
|
||||
"notes": checkpoint_data.get("notes", ""),
|
||||
"file_size": model_data.get("size", 0),
|
||||
"modified": model_data.get("modified", ""),
|
||||
"tags": model_data.get("tags", []),
|
||||
"from_civitai": model_data.get("from_civitai", True),
|
||||
"usage_count": model_data.get("usage_count", 0),
|
||||
"notes": model_data.get("notes", ""),
|
||||
"sub_type": sub_type,
|
||||
"favorite": checkpoint_data.get("favorite", False),
|
||||
"exclude": bool(checkpoint_data.get("exclude", False)),
|
||||
"update_available": bool(checkpoint_data.get("update_available", False)),
|
||||
"skip_metadata_refresh": bool(checkpoint_data.get("skip_metadata_refresh", False)),
|
||||
"civitai": self.filter_civitai_data(checkpoint_data.get("civitai", {}), minimal=True),
|
||||
"auto_tags": checkpoint_data.get("auto_tags") or extract_auto_tags(checkpoint_data),
|
||||
"version_count": checkpoint_data.get("version_count"),
|
||||
"hf_url": checkpoint_data.get("hf_url", ""),
|
||||
"favorite": model_data.get("favorite", False),
|
||||
"exclude": bool(model_data.get("exclude", False)),
|
||||
"update_available": bool(model_data.get("update_available", False)),
|
||||
"skip_metadata_refresh": bool(model_data.get("skip_metadata_refresh", False)),
|
||||
"civitai": self.filter_civitai_data(model_data.get("civitai", {}), minimal=True),
|
||||
"auto_tags": model_data.get("auto_tags") or extract_auto_tags(model_data),
|
||||
"version_count": model_data.get("version_count"),
|
||||
"hf_url": model_data.get("hf_url", ""),
|
||||
}
|
||||
|
||||
def find_duplicate_hashes(self) -> Dict:
|
||||
def find_duplicate_hashes(self) -> Dict[str, Any]:
|
||||
"""Find Checkpoints with duplicate SHA256 hashes"""
|
||||
return self.scanner._hash_index.get_duplicate_hashes()
|
||||
|
||||
def find_duplicate_filenames(self) -> Dict:
|
||||
def find_duplicate_filenames(self) -> Dict[str, Any]:
|
||||
"""Find Checkpoints with conflicting filenames"""
|
||||
return self.scanner._hash_index.get_duplicate_filenames()
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
import json
|
||||
import logging
|
||||
import asyncio
|
||||
from copy import deepcopy
|
||||
from typing import Optional, Dict, Tuple, List
|
||||
from typing import Any, Optional, Dict, Tuple, List, cast
|
||||
from .model_metadata_provider import CivArchiveModelMetadataProvider, ModelMetadataProviderManager
|
||||
from .downloader import get_downloader
|
||||
from .errors import RateLimitError
|
||||
@@ -37,8 +41,8 @@ class CivArchiveClient:
|
||||
async def _request_json(
|
||||
self,
|
||||
path: str,
|
||||
params: Optional[Dict[str, str]] = None
|
||||
) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
params: Optional[Dict[str, Any]] = None
|
||||
) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
"""Call CivArchive API and return JSON payload"""
|
||||
success, payload = await self._make_request(path, params=params)
|
||||
if not success:
|
||||
@@ -52,12 +56,12 @@ class CivArchiveClient:
|
||||
self,
|
||||
path: str,
|
||||
*,
|
||||
params: Optional[Dict[str, str]] = None,
|
||||
) -> Tuple[bool, Dict | str]:
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
) -> Tuple[bool, Dict[str, Any] | str]:
|
||||
"""Wrapper around downloader.make_request that surfaces rate limits."""
|
||||
|
||||
downloader = await get_downloader()
|
||||
kwargs: Dict[str, Dict[str, str]] = {}
|
||||
kwargs: Dict[str, Dict[str, Any]] = {}
|
||||
if params:
|
||||
safe_params = {str(key): str(value) for key, value in params.items() if value is not None}
|
||||
if safe_params:
|
||||
@@ -73,10 +77,11 @@ class CivArchiveClient:
|
||||
if payload.provider is None:
|
||||
payload.provider = "civarchive_api"
|
||||
raise payload
|
||||
return success, payload
|
||||
# RateLimitError is always raised above, so the returned payload is a dict or str.
|
||||
return success, cast(Dict[str, Any] | str, payload)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_payload(payload: Dict) -> Dict:
|
||||
def _normalize_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Unwrap CivArchive responses that wrap content under a data key"""
|
||||
if not isinstance(payload, dict):
|
||||
return {}
|
||||
@@ -86,12 +91,12 @@ class CivArchiveClient:
|
||||
return payload
|
||||
|
||||
@staticmethod
|
||||
def _split_context(payload: Dict) -> Tuple[Dict, Dict, List[Dict]]:
|
||||
def _split_context(payload: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any], List[Dict[str, Any]]]:
|
||||
"""Separate version payload from surrounding model context"""
|
||||
data = CivArchiveClient._normalize_payload(payload)
|
||||
context: Dict = {}
|
||||
fallback_files: List[Dict] = []
|
||||
version: Dict = {}
|
||||
context: Dict[str, Any] = {}
|
||||
fallback_files: List[Dict[str, Any]] = []
|
||||
version: Dict[str, Any] = {}
|
||||
|
||||
for key, value in data.items():
|
||||
if key in {"version", "model"}:
|
||||
@@ -115,7 +120,7 @@ class CivArchiveClient:
|
||||
return context, version, fallback_files
|
||||
|
||||
@staticmethod
|
||||
def _ensure_list(value) -> List:
|
||||
def _ensure_list(value: Any) -> List[Any]:
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
if value is None:
|
||||
@@ -123,7 +128,7 @@ class CivArchiveClient:
|
||||
return [value]
|
||||
|
||||
@staticmethod
|
||||
def _build_model_info(context: Dict) -> Dict:
|
||||
def _build_model_info(context: Dict[str, Any]) -> Dict[str, Any]:
|
||||
tags = context.get("tags")
|
||||
if not isinstance(tags, list):
|
||||
tags = list(tags) if isinstance(tags, (set, tuple)) else ([] if tags is None else [tags])
|
||||
@@ -136,7 +141,7 @@ class CivArchiveClient:
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _build_creator_info(context: Dict) -> Dict:
|
||||
def _build_creator_info(context: Dict[str, Any]) -> Dict[str, Any]:
|
||||
username = context.get("creator_username") or context.get("username") or ""
|
||||
image = context.get("creator_image") or context.get("creator_avatar") or ""
|
||||
creator: Dict[str, Optional[str]] = {
|
||||
@@ -150,7 +155,7 @@ class CivArchiveClient:
|
||||
return creator
|
||||
|
||||
@staticmethod
|
||||
def _transform_file_entry(file_data: Dict) -> Dict:
|
||||
def _transform_file_entry(file_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
mirrors = file_data.get("mirrors") or []
|
||||
if not isinstance(mirrors, list):
|
||||
mirrors = [mirrors]
|
||||
@@ -165,7 +170,7 @@ class CivArchiveClient:
|
||||
if not name and available_mirror:
|
||||
name = available_mirror.get("filename")
|
||||
|
||||
transformed: Dict = {
|
||||
transformed: Dict[str, Any] = {
|
||||
"id": file_data.get("id"),
|
||||
"sizeKB": file_data.get("sizeKB"),
|
||||
"name": name,
|
||||
@@ -216,23 +221,23 @@ class CivArchiveClient:
|
||||
|
||||
def _transform_files(
|
||||
self,
|
||||
files: Optional[List[Dict]],
|
||||
fallback_files: Optional[List[Dict]] = None
|
||||
) -> List[Dict]:
|
||||
candidates: List[Dict] = []
|
||||
files: Optional[List[Dict[str, Any]]],
|
||||
fallback_files: Optional[List[Dict[str, Any]]] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
candidates: List[Dict[str, Any]] = []
|
||||
if isinstance(files, list) and files:
|
||||
candidates = files
|
||||
elif isinstance(fallback_files, list):
|
||||
candidates = fallback_files
|
||||
|
||||
transformed_files: List[Dict] = []
|
||||
transformed_files: List[Dict[str, Any]] = []
|
||||
for file_data in candidates:
|
||||
if isinstance(file_data, dict):
|
||||
transformed_files.append(self._transform_file_entry(file_data))
|
||||
|
||||
# Sort: .safetensors first, .ckpt second, others last
|
||||
# so the backend fallback (no file_params) prefers safetensors
|
||||
def _sort_key(f: Dict) -> int:
|
||||
def _sort_key(f: Dict[str, Any]) -> int:
|
||||
fname = f.get("name") or ""
|
||||
if isinstance(fname, str):
|
||||
lower = fname.lower()
|
||||
@@ -247,10 +252,10 @@ class CivArchiveClient:
|
||||
|
||||
def _transform_version(
|
||||
self,
|
||||
context: Dict,
|
||||
version: Dict,
|
||||
fallback_files: Optional[List[Dict]] = None
|
||||
) -> Optional[Dict]:
|
||||
context: Dict[str, Any],
|
||||
version: Dict[str, Any],
|
||||
fallback_files: Optional[List[Dict[str, Any]]] = None
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if not version:
|
||||
return None
|
||||
|
||||
@@ -291,7 +296,7 @@ class CivArchiveClient:
|
||||
|
||||
return version_copy
|
||||
|
||||
async def _resolve_version_from_files(self, payload: Dict) -> Optional[Dict]:
|
||||
async def _resolve_version_from_files(self, payload: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""Fallback to fetch version data when only file metadata is available"""
|
||||
data = self._normalize_payload(payload)
|
||||
files = data.get("files") or payload.get("files") or []
|
||||
@@ -323,7 +328,7 @@ class CivArchiveClient:
|
||||
return resolved
|
||||
return None
|
||||
|
||||
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
"""Find model by SHA256 hash value using CivArchive API"""
|
||||
try:
|
||||
payload, error = await self._request_json(f"/sha256/{model_hash.lower()}")
|
||||
@@ -332,12 +337,12 @@ class CivArchiveClient:
|
||||
return None, "Model not found"
|
||||
return None, error
|
||||
|
||||
context, version_data, fallback_files = self._split_context(payload)
|
||||
context, version_data, fallback_files = self._split_context(cast(Dict[str, Any], payload))
|
||||
transformed = self._transform_version(context, version_data, fallback_files)
|
||||
if transformed:
|
||||
return transformed, None
|
||||
|
||||
resolved = await self._resolve_version_from_files(payload)
|
||||
resolved = await self._resolve_version_from_files(cast(Dict[str, Any], payload))
|
||||
if resolved:
|
||||
return resolved, None
|
||||
|
||||
@@ -350,7 +355,7 @@ class CivArchiveClient:
|
||||
logger.error(f"Error fetching CivArchive model by hash {model_hash[:10]}: {e}")
|
||||
return None, str(e)
|
||||
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict]:
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get all versions of a model using CivArchive API"""
|
||||
try:
|
||||
payload, error = await self._request_json(f"/models/{model_id}")
|
||||
@@ -364,7 +369,7 @@ class CivArchiveClient:
|
||||
context, version_data, fallback_files = self._split_context(payload)
|
||||
|
||||
versions_meta = data.get("versions") or []
|
||||
transformed_versions: List[Dict] = []
|
||||
transformed_versions: List[Dict[str, Any]] = []
|
||||
for meta in versions_meta:
|
||||
if not isinstance(meta, dict):
|
||||
continue
|
||||
@@ -381,7 +386,7 @@ class CivArchiveClient:
|
||||
if primary_version:
|
||||
transformed_versions.insert(0, primary_version)
|
||||
|
||||
ordered_versions: List[Dict] = []
|
||||
ordered_versions: List[Dict[str, Any]] = []
|
||||
seen_ids = set()
|
||||
for version in transformed_versions:
|
||||
version_id = version.get("id")
|
||||
@@ -402,7 +407,7 @@ class CivArchiveClient:
|
||||
logger.error(f"Error fetching CivArchive model versions for {model_id}: {e}")
|
||||
return None
|
||||
|
||||
async def get_model_version(self, model_id: int = None, version_id: int = None) -> Optional[Dict]:
|
||||
async def get_model_version(self, model_id: int | str | None = None, version_id: int | str | None = None) -> Optional[Dict[str, Any]]:
|
||||
"""Get specific model version using CivArchive API
|
||||
|
||||
Args:
|
||||
@@ -459,7 +464,7 @@ class CivArchiveClient:
|
||||
logger.error(f"Error fetching CivArchive model version via API {model_id}/{version_id}: {e}")
|
||||
return None
|
||||
|
||||
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
""" Fetch model version metadata using a known bogus model lookup
|
||||
CivArchive lacks a direct version lookup API, this uses a workaround (which we handle in the main model request now)
|
||||
|
||||
|
||||
@@ -283,7 +283,7 @@ class CivitaiBaseModelService:
|
||||
return None
|
||||
|
||||
if isinstance(result, str):
|
||||
data = json.loads(result)
|
||||
data: Any = json.loads(result)
|
||||
else:
|
||||
data = result
|
||||
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
import asyncio
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from typing import Any, Optional, Dict, Tuple, List, Sequence
|
||||
from typing import Any, Optional, Dict, Tuple, List, Sequence, cast
|
||||
from .connectivity_guard import (
|
||||
OFFLINE_FRIENDLY_MESSAGE,
|
||||
is_expected_offline_error,
|
||||
@@ -17,6 +21,7 @@ from .model_metadata_provider import (
|
||||
from .downloader import get_downloader
|
||||
from .errors import RateLimitError, ResourceNotFoundError
|
||||
from ..utils.civitai_utils import resolve_license_payload
|
||||
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -58,7 +63,7 @@ class CivitaiClient:
|
||||
# Uses OrderedDict with LRU eviction at MAX_CACHE_ENTRIES to prevent
|
||||
# unbounded growth in long-running server processes.
|
||||
self._version_info_cache: OrderedDict[
|
||||
str, Tuple[Optional[Dict], Optional[str]]
|
||||
str, Tuple[Optional[Dict[str, Any]], Optional[str]]
|
||||
] = OrderedDict()
|
||||
self._MAX_CACHE_ENTRIES = 500
|
||||
|
||||
@@ -72,7 +77,7 @@ class CivitaiClient:
|
||||
*,
|
||||
use_auth: bool = False,
|
||||
**kwargs,
|
||||
) -> Tuple[bool, Dict | str]:
|
||||
) -> Tuple[bool, Dict[str, Any] | str]:
|
||||
"""Wrapper around downloader.make_request that surfaces rate limits,
|
||||
with retry for transient server errors (5xx, Cloudflare 524, network flakiness)."""
|
||||
|
||||
@@ -86,7 +91,8 @@ class CivitaiClient:
|
||||
**kwargs,
|
||||
)
|
||||
if success:
|
||||
return True, result
|
||||
# RateLimitError is raised below; a successful result is dict or str.
|
||||
return True, cast(Dict[str, Any] | str, result)
|
||||
|
||||
if isinstance(result, RateLimitError):
|
||||
if result.provider is None:
|
||||
@@ -126,7 +132,7 @@ class CivitaiClient:
|
||||
return False, "Unexpected error in _make_request"
|
||||
|
||||
@staticmethod
|
||||
def _remove_comfy_metadata(model_version: Optional[Dict]) -> None:
|
||||
def _remove_comfy_metadata(model_version: Optional[Dict[str, Any]]) -> None:
|
||||
"""Remove Comfy-specific metadata from model version images."""
|
||||
if not isinstance(model_version, dict):
|
||||
return
|
||||
@@ -173,7 +179,7 @@ class CivitaiClient:
|
||||
|
||||
async def get_model_by_hash(
|
||||
self, model_hash: str
|
||||
) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
try:
|
||||
success, version = await self._make_request(
|
||||
"GET",
|
||||
@@ -220,7 +226,7 @@ class CivitaiClient:
|
||||
# Ensure directory exists
|
||||
os.makedirs(os.path.dirname(save_path), exist_ok=True)
|
||||
with open(save_path, "wb") as f:
|
||||
f.write(content)
|
||||
f.write(content if isinstance(content, bytes) else content.encode("utf-8"))
|
||||
return True
|
||||
return False
|
||||
except Exception as e:
|
||||
@@ -275,7 +281,7 @@ class CivitaiClient:
|
||||
return True
|
||||
return False
|
||||
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict]:
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get all versions of a model with local availability info"""
|
||||
try:
|
||||
success, result = await self._make_request(
|
||||
@@ -283,7 +289,7 @@ class CivitaiClient:
|
||||
f"{self.base_url}/models/{model_id}",
|
||||
use_auth=True,
|
||||
)
|
||||
if success:
|
||||
if success and isinstance(result, dict):
|
||||
# Also return model type along with versions
|
||||
return {
|
||||
"modelVersions": result.get("modelVersions", []),
|
||||
@@ -317,7 +323,7 @@ class CivitaiClient:
|
||||
|
||||
async def get_model_versions_bulk(
|
||||
self, model_ids: Sequence[int]
|
||||
) -> Optional[Dict[int, Dict]]:
|
||||
) -> Optional[Dict[int, Dict[str, Any]]]:
|
||||
"""Fetch model metadata for multiple ids using the batch API."""
|
||||
|
||||
deduped: Dict[int, None] = {}
|
||||
@@ -347,13 +353,13 @@ class CivitaiClient:
|
||||
if not isinstance(items, list):
|
||||
return {}
|
||||
|
||||
payload: Dict[int, Dict] = {}
|
||||
payload: Dict[int, Dict[str, Any]] = {}
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
model_id = item.get("id")
|
||||
try:
|
||||
normalized_id = int(model_id)
|
||||
normalized_id = int(cast(Any, model_id))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
payload[normalized_id] = {
|
||||
@@ -373,8 +379,8 @@ class CivitaiClient:
|
||||
return None
|
||||
|
||||
async def get_model_version(
|
||||
self, model_id: int = None, version_id: int = None
|
||||
) -> Optional[Dict]:
|
||||
self, model_id: int | None = None, version_id: int | None = None
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Get specific model version with additional metadata."""
|
||||
try:
|
||||
if model_id is None and version_id is not None:
|
||||
@@ -392,7 +398,7 @@ class CivitaiClient:
|
||||
logger.error(f"Error fetching model version: {e}")
|
||||
return None
|
||||
|
||||
async def _get_version_by_id_only(self, version_id: int) -> Optional[Dict]:
|
||||
async def _get_version_by_id_only(self, version_id: int) -> Optional[Dict[str, Any]]:
|
||||
version = await self._fetch_version_by_id(version_id)
|
||||
if version is None:
|
||||
return None
|
||||
@@ -411,7 +417,7 @@ class CivitaiClient:
|
||||
|
||||
async def _get_version_with_model_id(
|
||||
self, model_id: int, version_id: Optional[int]
|
||||
) -> Optional[Dict]:
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
model_data = await self._fetch_model_data(model_id)
|
||||
if not model_data:
|
||||
return None
|
||||
@@ -464,20 +470,20 @@ class CivitaiClient:
|
||||
self._remove_comfy_metadata(version)
|
||||
return version
|
||||
|
||||
async def _fetch_model_data(self, model_id: int) -> Optional[Dict]:
|
||||
async def _fetch_model_data(self, model_id: int) -> Optional[Dict[str, Any]]:
|
||||
success, data = await self._make_request(
|
||||
"GET",
|
||||
f"{self.base_url}/models/{model_id}",
|
||||
use_auth=True,
|
||||
)
|
||||
if success:
|
||||
if success and isinstance(data, dict):
|
||||
return data
|
||||
if is_expected_offline_error(data):
|
||||
return None
|
||||
logger.warning(f"Failed to fetch model data for model {model_id}")
|
||||
return None
|
||||
|
||||
async def _fetch_version_by_id(self, version_id: Optional[int]) -> Optional[Dict]:
|
||||
async def _fetch_version_by_id(self, version_id: Optional[int]) -> Optional[Dict[str, Any]]:
|
||||
if version_id is None:
|
||||
return None
|
||||
|
||||
@@ -486,7 +492,7 @@ class CivitaiClient:
|
||||
f"{self.base_url}/model-versions/{version_id}",
|
||||
use_auth=True,
|
||||
)
|
||||
if success:
|
||||
if success and isinstance(version, dict):
|
||||
return version
|
||||
if is_expected_offline_error(version):
|
||||
return None
|
||||
@@ -494,7 +500,7 @@ class CivitaiClient:
|
||||
logger.warning(f"Failed to fetch version by id {version_id}")
|
||||
return None
|
||||
|
||||
async def _fetch_version_by_hash(self, model_hash: Optional[str]) -> Optional[Dict]:
|
||||
async def _fetch_version_by_hash(self, model_hash: Optional[str]) -> Optional[Dict[str, Any]]:
|
||||
if not model_hash:
|
||||
return None
|
||||
|
||||
@@ -503,7 +509,7 @@ class CivitaiClient:
|
||||
f"{self.base_url}/model-versions/by-hash/{model_hash}",
|
||||
use_auth=True,
|
||||
)
|
||||
if success:
|
||||
if success and isinstance(version, dict):
|
||||
return version
|
||||
if is_expected_offline_error(version):
|
||||
return None
|
||||
@@ -512,8 +518,8 @@ class CivitaiClient:
|
||||
return None
|
||||
|
||||
def _select_target_version(
|
||||
self, model_data: Dict, model_id: int, version_id: Optional[int]
|
||||
) -> Optional[Dict]:
|
||||
self, model_data: Dict[str, Any], model_id: int, version_id: Optional[int]
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
model_versions = model_data.get("modelVersions", [])
|
||||
if not model_versions:
|
||||
logger.warning(f"No model versions found for model {model_id}")
|
||||
@@ -532,18 +538,24 @@ class CivitaiClient:
|
||||
|
||||
return model_versions[0]
|
||||
|
||||
def _extract_primary_model_hash(self, version_entry: Dict) -> Optional[str]:
|
||||
def _extract_primary_model_hash(self, version_entry: Dict[str, Any]) -> Optional[str]:
|
||||
# Prefer the generic "Model" file (most reliable version identity);
|
||||
# fall back to any other weights-type primary.
|
||||
for file_info in version_entry.get("files", []):
|
||||
if file_info.get("type") == "Model" and file_info.get("primary"):
|
||||
hashes = file_info.get("hashes", {})
|
||||
model_hash = hashes.get("SHA256")
|
||||
model_hash = (file_info.get("hashes", {}) or {}).get("SHA256")
|
||||
if model_hash:
|
||||
return model_hash
|
||||
for file_info in version_entry.get("files", []):
|
||||
if file_info.get("type") in MODEL_WEIGHT_FILE_TYPES and file_info.get("primary"):
|
||||
model_hash = (file_info.get("hashes", {}) or {}).get("SHA256")
|
||||
if model_hash:
|
||||
return model_hash
|
||||
return None
|
||||
|
||||
def _build_version_from_model_data(
|
||||
self, version_entry: Dict, model_id: int, model_data: Dict
|
||||
) -> Dict:
|
||||
self, version_entry: Dict[str, Any], model_id: int, model_data: Dict[str, Any]
|
||||
) -> Dict[str, Any]:
|
||||
version = copy.deepcopy(version_entry)
|
||||
version.pop("index", None)
|
||||
version["modelId"] = model_id
|
||||
@@ -555,7 +567,7 @@ class CivitaiClient:
|
||||
}
|
||||
return version
|
||||
|
||||
def _enrich_version_with_model_data(self, version: Dict, model_data: Dict) -> None:
|
||||
def _enrich_version_with_model_data(self, version: Dict[str, Any], model_data: Dict[str, Any]) -> None:
|
||||
model_info = version.get("model")
|
||||
if not isinstance(model_info, dict):
|
||||
model_info = {}
|
||||
@@ -571,7 +583,7 @@ class CivitaiClient:
|
||||
|
||||
async def get_model_version_info(
|
||||
self, version_id: str
|
||||
) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
"""Fetch model version metadata from Civitai
|
||||
|
||||
Args:
|
||||
@@ -596,7 +608,7 @@ class CivitaiClient:
|
||||
logger.debug("Resolving Civitai model version info: %s", url)
|
||||
success, result = await self._make_request("GET", url, use_auth=True)
|
||||
|
||||
if success:
|
||||
if success and isinstance(result, dict):
|
||||
logger.debug("Successfully fetched model version info for: %s", version_id)
|
||||
self._remove_comfy_metadata(result)
|
||||
self._version_info_cache[version_id] = (result, None)
|
||||
@@ -626,7 +638,7 @@ class CivitaiClient:
|
||||
|
||||
async def get_image_info(
|
||||
self, image_id: str, source_url: str | None = None
|
||||
) -> Optional[Dict]:
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Fetch image information from Civitai API
|
||||
|
||||
Args:
|
||||
@@ -659,7 +671,7 @@ class CivitaiClient:
|
||||
)
|
||||
return None
|
||||
|
||||
if result and "items" in result and isinstance(result["items"], list):
|
||||
if isinstance(result, dict) and "items" in result and isinstance(result["items"], list):
|
||||
items = result["items"]
|
||||
|
||||
for item in items:
|
||||
@@ -699,7 +711,7 @@ class CivitaiClient:
|
||||
|
||||
async def get_model_versions_by_hashes(
|
||||
self, hashes: List[str]
|
||||
) -> Optional[List[Dict]]:
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
"""Fetch full version details for up to 100 SHA256 hashes via the batch endpoint.
|
||||
|
||||
Uses POST /api/v1/model-versions/by-hash which returns full version
|
||||
@@ -716,7 +728,7 @@ class CivitaiClient:
|
||||
return []
|
||||
|
||||
BATCH_SIZE = 100
|
||||
all_versions: List[Dict] = []
|
||||
all_versions: List[Dict[str, Any]] = []
|
||||
|
||||
for start in range(0, len(hashes), BATCH_SIZE):
|
||||
batch = hashes[start : start + BATCH_SIZE]
|
||||
@@ -736,7 +748,7 @@ class CivitaiClient:
|
||||
continue
|
||||
|
||||
if isinstance(result, list):
|
||||
all_versions.extend(result)
|
||||
all_versions.extend(cast(Any, result))
|
||||
else:
|
||||
logger.debug(
|
||||
"Unexpected by-hash response type: %s", type(result)
|
||||
|
||||
@@ -18,7 +18,7 @@ class DownloadCoordinator:
|
||||
self,
|
||||
*,
|
||||
ws_manager,
|
||||
download_manager_factory: Callable[[], Awaitable],
|
||||
download_manager_factory: Callable[[], Awaitable[Any]],
|
||||
) -> None:
|
||||
self._ws_manager = ws_manager
|
||||
self._download_manager_factory = download_manager_factory
|
||||
@@ -83,6 +83,7 @@ class DownloadCoordinator:
|
||||
save_dir=payload.get("model_root"),
|
||||
relative_path=payload.get("relative_path", ""),
|
||||
use_default_paths=payload.get("use_default_paths", False),
|
||||
use_save_dir_as_root=payload.get("use_save_dir_as_root", False),
|
||||
progress_callback=progress_callback,
|
||||
download_id=download_id,
|
||||
source=payload.get("source"),
|
||||
|
||||
+199
-110
@@ -1,3 +1,7 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
@@ -8,17 +12,18 @@ import zipfile
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from collections import OrderedDict
|
||||
import uuid
|
||||
from typing import Dict, List, Optional, Set, Tuple
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple, cast
|
||||
from urllib.parse import urlparse
|
||||
from ..utils.models import LoraMetadata, CheckpointMetadata, EmbeddingMetadata
|
||||
from ..utils.constants import (
|
||||
CARD_PREVIEW_WIDTH,
|
||||
DIFFUSION_MODEL_BASE_MODELS,
|
||||
MODEL_WEIGHT_FILE_TYPES,
|
||||
SUPPORTED_DOWNLOAD_SKIP_BASE_MODELS,
|
||||
VALID_LORA_TYPES,
|
||||
)
|
||||
from ..utils.civitai_utils import normalize_civitai_download_url, rewrite_preview_url
|
||||
from ..utils.file_utils import calculate_sha256
|
||||
from ..utils.file_utils import calculate_sha256, calculate_autov3
|
||||
from ..utils.preview_selection import resolve_mature_threshold, select_preview_media
|
||||
from ..utils.utils import sanitize_folder_name
|
||||
from ..utils.exif_utils import ExifUtils
|
||||
@@ -42,6 +47,11 @@ CIVITAI_DOWNLOAD_URL_PREFIXES = (
|
||||
)
|
||||
|
||||
|
||||
# File types that are never the intended download target even when CivitAI
|
||||
# marks them primary — configs/archives/workflows are auxiliary artifacts.
|
||||
NON_DOWNLOADABLE_PRIMARY_TYPES = ("Config", "Archive", "Workflow", "Training Data")
|
||||
|
||||
|
||||
class DownloadManager:
|
||||
_instance = None
|
||||
_lock = asyncio.Lock()
|
||||
@@ -121,7 +131,7 @@ class DownloadManager:
|
||||
"delay": 0,
|
||||
}
|
||||
)
|
||||
except DownloadInProgressError:
|
||||
except DownloadInProgressError: # pyright: ignore[reportPossiblyUnboundVariable]
|
||||
logger.info(
|
||||
"Skipping automatic example images download for %s; another example images download is already running",
|
||||
model_hash,
|
||||
@@ -170,7 +180,7 @@ class DownloadManager:
|
||||
logger.error("aria2 download failed for %s: %s", download_url, exc)
|
||||
return False, str(exc)
|
||||
|
||||
download_kwargs = {
|
||||
download_kwargs: Dict[str, Any] = {
|
||||
"progress_callback": progress_callback,
|
||||
"use_auth": use_auth,
|
||||
}
|
||||
@@ -204,16 +214,17 @@ class DownloadManager:
|
||||
|
||||
async def download_from_civitai(
|
||||
self,
|
||||
model_id: int = None,
|
||||
model_version_id: int = None,
|
||||
save_dir: str = None,
|
||||
model_id: int | None = None,
|
||||
model_version_id: int | None = None,
|
||||
save_dir: str | None = None,
|
||||
relative_path: str = "",
|
||||
progress_callback=None,
|
||||
use_default_paths: bool = False,
|
||||
download_id: str = None,
|
||||
source: str = None,
|
||||
file_params: Dict = None,
|
||||
) -> Dict:
|
||||
download_id: str | None = None,
|
||||
source: str | None = None,
|
||||
file_params: Dict[str, Any] | None = None,
|
||||
use_save_dir_as_root: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""Download model from Civitai with task tracking and concurrency control
|
||||
|
||||
Args:
|
||||
@@ -253,6 +264,7 @@ class DownloadManager:
|
||||
"save_dir": save_dir,
|
||||
"relative_path": relative_path,
|
||||
"use_default_paths": bool(use_default_paths),
|
||||
"use_save_dir_as_root": bool(use_save_dir_as_root),
|
||||
"source": source,
|
||||
"file_params": copy.deepcopy(file_params) if file_params is not None else None,
|
||||
"progress": 0,
|
||||
@@ -283,6 +295,7 @@ class DownloadManager:
|
||||
use_default_paths,
|
||||
source,
|
||||
file_params,
|
||||
use_save_dir_as_root,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -309,14 +322,15 @@ class DownloadManager:
|
||||
async def _download_with_semaphore(
|
||||
self,
|
||||
task_id: str,
|
||||
model_id: int,
|
||||
model_version_id: int,
|
||||
save_dir: str,
|
||||
model_id: int | None,
|
||||
model_version_id: int | None,
|
||||
save_dir: str | None,
|
||||
relative_path: str,
|
||||
progress_callback=None,
|
||||
use_default_paths: bool = False,
|
||||
source: str = None,
|
||||
file_params: Dict = None,
|
||||
source: str | None = None,
|
||||
file_params: Dict[str, Any] | None = None,
|
||||
use_save_dir_as_root: bool = False,
|
||||
):
|
||||
"""Execute download with semaphore to limit concurrency"""
|
||||
# Update status to waiting
|
||||
@@ -380,7 +394,8 @@ class DownloadManager:
|
||||
# Use original download implementation
|
||||
try:
|
||||
# Check for cancellation before starting
|
||||
if asyncio.current_task().cancelled():
|
||||
current_task = asyncio.current_task()
|
||||
if current_task is not None and current_task.cancelled():
|
||||
raise asyncio.CancelledError()
|
||||
|
||||
result = await self._execute_original_download(
|
||||
@@ -396,6 +411,7 @@ class DownloadManager:
|
||||
),
|
||||
source,
|
||||
file_params,
|
||||
use_save_dir_as_root=use_save_dir_as_root,
|
||||
)
|
||||
|
||||
# Update status based on result
|
||||
@@ -484,11 +500,11 @@ class DownloadManager:
|
||||
# Schedule cleanup of download record after delay
|
||||
asyncio.create_task(self._cleanup_download_record(task_id))
|
||||
|
||||
def _start_background_download_task(self, download_id: str, coroutine) -> asyncio.Task:
|
||||
def _start_background_download_task(self, download_id: str, coroutine) -> asyncio.Task[Any]:
|
||||
task = asyncio.create_task(coroutine)
|
||||
self._download_tasks[download_id] = task
|
||||
|
||||
def _cleanup_done_task(done_task: asyncio.Task) -> None:
|
||||
def _cleanup_done_task(done_task: asyncio.Task[Any]) -> None:
|
||||
current_task = self._download_tasks.get(download_id)
|
||||
if current_task is done_task:
|
||||
self._download_tasks.pop(download_id, None)
|
||||
@@ -530,7 +546,7 @@ class DownloadManager:
|
||||
async def _cleanup_cancelled_download_files(
|
||||
self,
|
||||
download_id: str,
|
||||
download_info: Optional[Dict],
|
||||
download_info: Optional[Dict[str, Any]],
|
||||
) -> None:
|
||||
target_files = set()
|
||||
persisted = await self._aria2_state_store.get(download_id)
|
||||
@@ -603,19 +619,20 @@ class DownloadManager:
|
||||
self,
|
||||
download_id: str,
|
||||
*,
|
||||
extra: Optional[Dict] = None,
|
||||
extra: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
info = self._active_downloads.get(download_id)
|
||||
if not info:
|
||||
return
|
||||
|
||||
payload = {
|
||||
payload: Dict[str, Any] = {
|
||||
"download_id": download_id,
|
||||
"model_id": info.get("model_id"),
|
||||
"model_version_id": info.get("model_version_id"),
|
||||
"save_dir": info.get("save_dir"),
|
||||
"relative_path": info.get("relative_path", ""),
|
||||
"use_default_paths": bool(info.get("use_default_paths", False)),
|
||||
"use_save_dir_as_root": bool(info.get("use_save_dir_as_root", False)),
|
||||
"source": info.get("source"),
|
||||
"file_params": copy.deepcopy(info.get("file_params")),
|
||||
"transfer_backend": info.get("transfer_backend", "aria2"),
|
||||
@@ -631,13 +648,14 @@ class DownloadManager:
|
||||
|
||||
await self._aria2_state_store.upsert(download_id, payload)
|
||||
|
||||
def _build_restored_download_info(self, record: Dict, save_path: str) -> Dict:
|
||||
def _build_restored_download_info(self, record: Dict[str, Any], save_path: str) -> Dict[str, Any]:
|
||||
return {
|
||||
"model_id": record.get("model_id"),
|
||||
"model_version_id": record.get("model_version_id"),
|
||||
"save_dir": record.get("save_dir"),
|
||||
"relative_path": record.get("relative_path", ""),
|
||||
"use_default_paths": bool(record.get("use_default_paths", False)),
|
||||
"use_save_dir_as_root": bool(record.get("use_save_dir_as_root", False)),
|
||||
"source": record.get("source"),
|
||||
"file_params": copy.deepcopy(record.get("file_params")),
|
||||
"progress": record.get("progress", 0),
|
||||
@@ -653,8 +671,8 @@ class DownloadManager:
|
||||
|
||||
def _is_same_aria2_download_request(
|
||||
self,
|
||||
current_info: Optional[Dict],
|
||||
persisted_record: Dict,
|
||||
current_info: Optional[Dict[str, Any]],
|
||||
persisted_record: Dict[str, Any],
|
||||
) -> bool:
|
||||
if not isinstance(current_info, dict):
|
||||
return False
|
||||
@@ -666,13 +684,15 @@ class DownloadManager:
|
||||
|
||||
return current_version_id == persisted_version_id
|
||||
|
||||
def _build_download_urls_from_file_info(self, file_info: Dict, source: str = None) -> List[str]:
|
||||
def _build_download_urls_from_file_info(self, file_info: Dict[str, Any], source: str | None = None) -> List[str]:
|
||||
mirrors = file_info.get("mirrors") or []
|
||||
download_urls: List[str] = []
|
||||
if mirrors:
|
||||
for mirror in mirrors:
|
||||
if mirror.get("deletedAt") is None and mirror.get("url"):
|
||||
download_urls.append(normalize_civitai_download_url(mirror["url"]))
|
||||
normalized_url = normalize_civitai_download_url(mirror["url"])
|
||||
if normalized_url:
|
||||
download_urls.append(normalized_url)
|
||||
|
||||
if source == "civarchive" and len(download_urls) > 1:
|
||||
civitai_urls = [
|
||||
@@ -688,7 +708,9 @@ class DownloadManager:
|
||||
if not download_urls:
|
||||
download_url = file_info.get("downloadUrl")
|
||||
if download_url:
|
||||
download_urls.append(normalize_civitai_download_url(download_url))
|
||||
normalized_url = normalize_civitai_download_url(download_url)
|
||||
if normalized_url:
|
||||
download_urls.append(normalized_url)
|
||||
|
||||
return download_urls
|
||||
|
||||
@@ -696,8 +718,8 @@ class DownloadManager:
|
||||
self,
|
||||
*,
|
||||
model_type: str,
|
||||
version_info: Dict,
|
||||
file_info: Dict,
|
||||
version_info: Dict[str, Any],
|
||||
file_info: Dict[str, Any],
|
||||
save_path: str,
|
||||
):
|
||||
if model_type == "checkpoint":
|
||||
@@ -706,7 +728,7 @@ class DownloadManager:
|
||||
return EmbeddingMetadata.from_civitai_info(version_info, file_info, save_path)
|
||||
return LoraMetadata.from_civitai_info(version_info, file_info, save_path)
|
||||
|
||||
def _resolve_save_path_from_persisted_record(self, record: Dict) -> Optional[str]:
|
||||
def _resolve_save_path_from_persisted_record(self, record: Dict[str, Any]) -> Optional[str]:
|
||||
save_path = record.get("save_path") or record.get("file_path")
|
||||
if isinstance(save_path, str) and save_path:
|
||||
return os.path.abspath(save_path)
|
||||
@@ -728,7 +750,7 @@ class DownloadManager:
|
||||
|
||||
return os.path.abspath(os.path.join(save_dir, file_name))
|
||||
|
||||
async def _resume_restored_aria2_download(self, download_id: str, record: Dict) -> Dict:
|
||||
async def _resume_restored_aria2_download(self, download_id: str, record: Dict[str, Any]) -> Dict[str, Any]:
|
||||
try:
|
||||
if download_id in self._active_downloads:
|
||||
self._active_downloads[download_id]["status"] = "downloading"
|
||||
@@ -842,7 +864,7 @@ class DownloadManager:
|
||||
self,
|
||||
previous_download_id: str,
|
||||
new_download_id: str,
|
||||
persisted_record: Dict,
|
||||
persisted_record: Dict[str, Any],
|
||||
save_path: str,
|
||||
) -> None:
|
||||
aria2_downloader = await get_aria2_downloader()
|
||||
@@ -938,7 +960,7 @@ class DownloadManager:
|
||||
except Exception:
|
||||
status_payload = None
|
||||
|
||||
if status_payload is not None:
|
||||
if status_payload is not None and isinstance(gid, str):
|
||||
remote_status = status_payload.get("status", "")
|
||||
if remote_status in {"active", "waiting", "paused"}:
|
||||
await aria2_downloader.restore_transfer(download_id, gid, save_path)
|
||||
@@ -992,6 +1014,7 @@ class DownloadManager:
|
||||
bool(restored.get("use_default_paths", False)),
|
||||
restored.get("source"),
|
||||
restored.get("file_params"),
|
||||
bool(restored.get("use_save_dir_as_root", False)),
|
||||
)
|
||||
)
|
||||
continue
|
||||
@@ -1115,17 +1138,18 @@ class DownloadManager:
|
||||
|
||||
async def _execute_original_download(
|
||||
self,
|
||||
model_id,
|
||||
model_version_id,
|
||||
save_dir,
|
||||
relative_path,
|
||||
model_id: int | None,
|
||||
model_version_id: int | None,
|
||||
save_dir: str | None,
|
||||
relative_path: str,
|
||||
progress_callback,
|
||||
use_default_paths,
|
||||
download_id=None,
|
||||
transfer_backend="python",
|
||||
source=None,
|
||||
file_params=None,
|
||||
):
|
||||
use_default_paths: bool,
|
||||
download_id: str | None = None,
|
||||
transfer_backend: str = "python",
|
||||
source: str | None = None,
|
||||
file_params: Dict[str, Any] | None = None,
|
||||
use_save_dir_as_root: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""Wrapper for original download_from_civitai implementation"""
|
||||
try:
|
||||
# Check if model version already exists in library
|
||||
@@ -1172,7 +1196,7 @@ class DownloadManager:
|
||||
|
||||
# Get version info based on the provided identifier
|
||||
version_info = await metadata_provider.get_model_version(
|
||||
model_id, model_version_id
|
||||
cast(int, model_id), cast(int, model_version_id)
|
||||
)
|
||||
|
||||
if not version_info:
|
||||
@@ -1183,7 +1207,7 @@ class DownloadManager:
|
||||
)
|
||||
metadata_provider = await get_default_metadata_provider()
|
||||
version_info = await metadata_provider.get_model_version(
|
||||
model_id, model_version_id
|
||||
cast(int, model_id), cast(int, model_version_id)
|
||||
)
|
||||
|
||||
if not version_info:
|
||||
@@ -1353,41 +1377,48 @@ class DownloadManager:
|
||||
# Handle use_default_paths
|
||||
if use_default_paths:
|
||||
settings_manager = get_settings_manager()
|
||||
# Set save_dir based on model type
|
||||
if model_type == "checkpoint":
|
||||
if is_diffusion_model:
|
||||
default_path = settings_manager.get("default_unet_root")
|
||||
error_msg = "Default unet root path not set in settings"
|
||||
else:
|
||||
default_path = settings_manager.get("default_checkpoint_root")
|
||||
error_msg = "Default checkpoint root path not set in settings"
|
||||
if not default_path:
|
||||
return {
|
||||
"success": False,
|
||||
"error": error_msg,
|
||||
}
|
||||
save_dir = default_path
|
||||
elif model_type == "lora":
|
||||
default_path = settings_manager.get("default_lora_root")
|
||||
if not default_path:
|
||||
return {
|
||||
"success": False,
|
||||
"error": "Default lora root path not set in settings",
|
||||
}
|
||||
save_dir = default_path
|
||||
elif model_type == "embedding":
|
||||
default_path = settings_manager.get("default_embedding_root")
|
||||
if not default_path:
|
||||
return {
|
||||
"success": False,
|
||||
"error": "Default embedding root path not set in settings",
|
||||
}
|
||||
save_dir = default_path
|
||||
# With use_save_dir_as_root, an explicitly provided save_dir is kept
|
||||
# as the base root and the path template is resolved underneath it.
|
||||
# Otherwise fall back to the configured default root, which keeps the
|
||||
# classic "download to default root" behavior for regular downloads.
|
||||
if not save_dir or not use_save_dir_as_root:
|
||||
# Set save_dir based on model type
|
||||
if model_type == "checkpoint":
|
||||
if is_diffusion_model:
|
||||
default_path = settings_manager.get("default_unet_root")
|
||||
error_msg = "Default unet root path not set in settings"
|
||||
else:
|
||||
default_path = settings_manager.get("default_checkpoint_root")
|
||||
error_msg = "Default checkpoint root path not set in settings"
|
||||
if not default_path:
|
||||
return {
|
||||
"success": False,
|
||||
"error": error_msg,
|
||||
}
|
||||
save_dir = default_path
|
||||
elif model_type == "lora":
|
||||
default_path = settings_manager.get("default_lora_root")
|
||||
if not default_path:
|
||||
return {
|
||||
"success": False,
|
||||
"error": "Default lora root path not set in settings",
|
||||
}
|
||||
save_dir = default_path
|
||||
elif model_type == "embedding":
|
||||
default_path = settings_manager.get("default_embedding_root")
|
||||
if not default_path:
|
||||
return {
|
||||
"success": False,
|
||||
"error": "Default embedding root path not set in settings",
|
||||
}
|
||||
save_dir = default_path
|
||||
|
||||
# Calculate relative path using template
|
||||
relative_path = self._calculate_relative_path(version_info, model_type)
|
||||
|
||||
# Update save directory with relative path if provided
|
||||
if not save_dir:
|
||||
return {"success": False, "error": "No save directory specified"}
|
||||
if relative_path:
|
||||
base_save_dir = save_dir
|
||||
save_dir = os.path.join(save_dir, relative_path)
|
||||
@@ -1475,7 +1506,7 @@ class DownloadManager:
|
||||
f
|
||||
for f in files
|
||||
if f.get("primary")
|
||||
and f.get("type") in ("Model", "Negative", "Diffusion Model", "UNet")
|
||||
and f.get("type") in MODEL_WEIGHT_FILE_TYPES
|
||||
),
|
||||
None,
|
||||
)
|
||||
@@ -1515,21 +1546,52 @@ class DownloadManager:
|
||||
# Fallback to primary file if no match found
|
||||
if not file_info:
|
||||
logger.debug("[download] Looking for primary file as fallback")
|
||||
# Prefer a weights-type file CivitAI marked primary; then any
|
||||
# weights-type file (providers without primary flags, e.g.
|
||||
# civarchive); then trust CivitAI's primary flag regardless of
|
||||
# type — newer types like 'Enhancement LoRA' are valid primary
|
||||
# files. Weights files are preferred over non-weights primary
|
||||
# files so a Config/Archive primary never replaces a Model.
|
||||
file_info = next(
|
||||
(
|
||||
f
|
||||
for f in files
|
||||
if f.get("primary") and f.get("type") in ("Model", "Negative", "Diffusion Model", "UNet")
|
||||
if f.get("primary") and f.get("type") in MODEL_WEIGHT_FILE_TYPES
|
||||
),
|
||||
None,
|
||||
)
|
||||
if file_info:
|
||||
logger.debug(
|
||||
"[download] Fallback primary file selected: id=%s, name=%s",
|
||||
"[download] Fallback primary file selected (primary + weights): id=%s, name=%s",
|
||||
file_info.get("id"), file_info.get("name"),
|
||||
)
|
||||
else:
|
||||
logger.debug("[download] No primary file found in fallback lookup")
|
||||
file_info = next(
|
||||
(f for f in files if f.get("type") in MODEL_WEIGHT_FILE_TYPES),
|
||||
None,
|
||||
)
|
||||
if file_info:
|
||||
logger.debug(
|
||||
"[download] Fallback primary file selected (weights type, no primary flag): id=%s, name=%s",
|
||||
file_info.get("id"), file_info.get("name"),
|
||||
)
|
||||
else:
|
||||
file_info = next(
|
||||
(
|
||||
f
|
||||
for f in files
|
||||
if f.get("primary")
|
||||
and f.get("type") not in NON_DOWNLOADABLE_PRIMARY_TYPES
|
||||
),
|
||||
None,
|
||||
)
|
||||
if file_info:
|
||||
logger.debug(
|
||||
"[download] Fallback primary file selected (trusting CivitAI primary flag): id=%s, name=%s, type=%s",
|
||||
file_info.get("id"), file_info.get("name"), file_info.get("type"),
|
||||
)
|
||||
else:
|
||||
logger.debug("[download] No primary file found in fallback lookup")
|
||||
|
||||
if not file_info:
|
||||
return {"success": False, "error": "No suitable file found in metadata"}
|
||||
@@ -1561,6 +1623,11 @@ class DownloadManager:
|
||||
version_info, file_info, save_path
|
||||
)
|
||||
logger.info(f"Creating EmbeddingMetadata for {file_name}")
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f'Unsupported model type "{model_type}"',
|
||||
}
|
||||
|
||||
# 6. Start download process
|
||||
if transfer_backend == "aria2" and download_id:
|
||||
@@ -1580,7 +1647,7 @@ class DownloadManager:
|
||||
},
|
||||
)
|
||||
|
||||
execute_kwargs = {
|
||||
execute_kwargs: Dict[str, Any] = {
|
||||
"download_urls": download_urls,
|
||||
"save_dir": save_dir,
|
||||
"metadata": metadata,
|
||||
@@ -1627,7 +1694,8 @@ class DownloadManager:
|
||||
)
|
||||
|
||||
# If early_access_msg exists and download failed, replace error message
|
||||
if "early_access_msg" in locals() and not result.get("success", False):
|
||||
early_access_msg = locals().get("early_access_msg")
|
||||
if early_access_msg and not result.get("success", False):
|
||||
result["error"] = early_access_msg
|
||||
|
||||
return result
|
||||
@@ -1652,7 +1720,7 @@ class DownloadManager:
|
||||
self,
|
||||
model_type: str,
|
||||
model_id_value,
|
||||
version_info: Dict,
|
||||
version_info: Dict[str, Any],
|
||||
fallback_version_id=None,
|
||||
file_path: str | None = None,
|
||||
) -> None:
|
||||
@@ -1683,8 +1751,8 @@ class DownloadManager:
|
||||
try:
|
||||
await history_service.mark_downloaded(
|
||||
model_type,
|
||||
int(version_id),
|
||||
model_id=int(resolved_model_id) if resolved_model_id is not None else None,
|
||||
int(cast(Any, version_id)),
|
||||
model_id=int(cast(Any, resolved_model_id)) if resolved_model_id is not None else None,
|
||||
source="download",
|
||||
file_path=file_path,
|
||||
)
|
||||
@@ -1701,7 +1769,7 @@ class DownloadManager:
|
||||
self,
|
||||
model_type: str,
|
||||
model_id_value,
|
||||
version_info: Dict,
|
||||
version_info: Dict[str, Any],
|
||||
fallback_version_id=None,
|
||||
) -> None:
|
||||
"""Ensure update tracking reflects a newly downloaded version."""
|
||||
@@ -1725,7 +1793,7 @@ class DownloadManager:
|
||||
if isinstance(model_info, dict):
|
||||
resolved_model_id = model_info.get("id")
|
||||
try:
|
||||
resolved_model_id = int(resolved_model_id)
|
||||
resolved_model_id = int(cast(Any, resolved_model_id))
|
||||
except (TypeError, ValueError):
|
||||
logger.debug(
|
||||
"Skipping update sync; invalid model id: %s", resolved_model_id
|
||||
@@ -1736,7 +1804,7 @@ class DownloadManager:
|
||||
if version_id is None:
|
||||
version_id = fallback_version_id
|
||||
try:
|
||||
version_id = int(version_id)
|
||||
version_id = int(cast(Any, version_id))
|
||||
except (TypeError, ValueError):
|
||||
logger.debug(
|
||||
"Skipping update sync; invalid version id for model %s: %s",
|
||||
@@ -1773,7 +1841,7 @@ class DownloadManager:
|
||||
for entry in local_versions or []:
|
||||
vid = entry.get("versionId")
|
||||
try:
|
||||
version_ids.add(int(vid))
|
||||
version_ids.add(int(cast(Any, vid)))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
|
||||
@@ -1795,7 +1863,7 @@ class DownloadManager:
|
||||
)
|
||||
|
||||
def _calculate_relative_path(
|
||||
self, version_info: Dict, model_type: str = "lora"
|
||||
self, version_info: Dict[str, Any], model_type: str = "lora"
|
||||
) -> str:
|
||||
"""Calculate relative path using template from settings
|
||||
|
||||
@@ -1871,21 +1939,22 @@ class DownloadManager:
|
||||
download_urls: List[str],
|
||||
save_dir: str,
|
||||
metadata,
|
||||
version_info: Dict,
|
||||
version_info: Dict[str, Any],
|
||||
relative_path: str,
|
||||
progress_callback=None,
|
||||
model_type: str = "lora",
|
||||
download_id: str = None,
|
||||
download_id: str | None = None,
|
||||
transfer_backend: Optional[str] = None,
|
||||
) -> Dict:
|
||||
) -> Dict[str, Any]:
|
||||
"""Execute the actual download process including preview images and model files"""
|
||||
metadata_entries: List = []
|
||||
metadata_entries: List[Any] = []
|
||||
metadata_files_for_cleanup: List[str] = []
|
||||
extracted_paths: List[str] = []
|
||||
metadata_path = ""
|
||||
preview_targets: List[str] = []
|
||||
preview_path: str | None = None
|
||||
preview_nsfw_level = 0
|
||||
save_path: str | None = None
|
||||
transfer_backend = (transfer_backend or self._get_model_download_backend()).lower()
|
||||
try:
|
||||
resolved, save_path = await self._resolve_download_target_path(
|
||||
@@ -1933,9 +2002,9 @@ class DownloadManager:
|
||||
mature_threshold=mature_threshold,
|
||||
)
|
||||
|
||||
preview_url = selected_image.get("url") if selected_image else None
|
||||
preview_url = cast(Optional[str], selected_image.get("url")) if selected_image else None
|
||||
media_type = (
|
||||
(selected_image.get("type") or "").lower() if selected_image else ""
|
||||
cast(str, selected_image.get("type") or "").lower() if selected_image else ""
|
||||
)
|
||||
|
||||
def _extension_from_url(url: str, fallback: str) -> str:
|
||||
@@ -1959,9 +2028,10 @@ class DownloadManager:
|
||||
preview_url, media_type="video"
|
||||
)
|
||||
attempt_urls: List[str] = []
|
||||
if rewritten:
|
||||
if rewritten and rewritten_url:
|
||||
attempt_urls.append(rewritten_url)
|
||||
attempt_urls.append(preview_url)
|
||||
if preview_url:
|
||||
attempt_urls.append(preview_url)
|
||||
|
||||
seen_attempts = set()
|
||||
for attempt in attempt_urls:
|
||||
@@ -1978,7 +2048,7 @@ class DownloadManager:
|
||||
rewritten_url, rewritten = rewrite_preview_url(
|
||||
preview_url, media_type="image"
|
||||
)
|
||||
if rewritten:
|
||||
if rewritten and rewritten_url:
|
||||
preview_ext = _extension_from_url(preview_url, ".png")
|
||||
preview_path = os.path.splitext(save_path)[0] + preview_ext
|
||||
success, _ = await downloader.download_file(
|
||||
@@ -2004,7 +2074,9 @@ class DownloadManager:
|
||||
)
|
||||
if success:
|
||||
with open(temp_path, "wb") as temp_file_handle:
|
||||
temp_file_handle.write(content)
|
||||
temp_file_handle.write(
|
||||
content if isinstance(content, bytes) else content.encode("utf-8")
|
||||
)
|
||||
preview_path = (
|
||||
os.path.splitext(save_path)[0] + ".webp"
|
||||
)
|
||||
@@ -2056,6 +2128,8 @@ class DownloadManager:
|
||||
last_error = None
|
||||
for download_url in download_urls:
|
||||
download_url = normalize_civitai_download_url(download_url)
|
||||
if download_url is None:
|
||||
continue
|
||||
use_auth = download_url.startswith(CIVITAI_DOWNLOAD_URL_PREFIXES)
|
||||
if transfer_backend == "aria2" and download_id:
|
||||
await self._persist_aria2_state(
|
||||
@@ -2160,6 +2234,10 @@ class DownloadManager:
|
||||
"error": f"Zip archive does not contain any supported model files ({supported_text})",
|
||||
}
|
||||
actual_file_paths = extracted_paths
|
||||
# The archive entry's AutoV3 (if any) describes the zip itself,
|
||||
# not the extracted models; clear it so per-file header
|
||||
# resolution applies to every extracted model.
|
||||
metadata.autov3 = None
|
||||
try:
|
||||
os.remove(save_path)
|
||||
except OSError as exc:
|
||||
@@ -2235,7 +2313,7 @@ class DownloadManager:
|
||||
entry, normalized_file_path, adjust_root
|
||||
)
|
||||
if adjusted_entry is not None:
|
||||
entry = adjusted_entry
|
||||
entry = cast(Any, adjusted_entry)
|
||||
metadata_entries[index] = entry
|
||||
|
||||
metadata_file_path = (
|
||||
@@ -2355,11 +2433,11 @@ class DownloadManager:
|
||||
|
||||
async def _build_metadata_entries(
|
||||
self, base_metadata, file_paths: List[str]
|
||||
) -> List:
|
||||
) -> List[Any]:
|
||||
if not file_paths:
|
||||
return []
|
||||
|
||||
entries: List = []
|
||||
entries: List[Any] = []
|
||||
for index, file_path in enumerate(file_paths):
|
||||
entry = base_metadata if index == 0 else copy.deepcopy(base_metadata)
|
||||
# Update file paths without modifying size and modified timestamps
|
||||
@@ -2374,6 +2452,16 @@ class DownloadManager:
|
||||
sha256 = await calculate_sha256(file_path)
|
||||
if sha256:
|
||||
entry.sha256 = sha256.lower()
|
||||
# AutoV3: the Civitai-reported value for the downloaded file (set
|
||||
# by from_civitai_info) takes precedence. Only the un-checked
|
||||
# state (None) triggers a header read; '' (checked-unavailable)
|
||||
# is never re-read, honoring the three-state contract so rows
|
||||
# marked at download time stay untouched by later passes.
|
||||
if entry.autov3 is None:
|
||||
autov3 = await asyncio.get_running_loop().run_in_executor(
|
||||
None, calculate_autov3, file_path
|
||||
)
|
||||
entry.autov3 = (autov3 or "").lower()
|
||||
entries.append(entry)
|
||||
|
||||
return entries
|
||||
@@ -2392,7 +2480,7 @@ class DownloadManager:
|
||||
return destination
|
||||
|
||||
def _distribute_preview_to_entries(
|
||||
self, preview_path: str, entries: List
|
||||
self, preview_path: str, entries: List[Any]
|
||||
) -> List[str]:
|
||||
if not preview_path or not entries:
|
||||
return []
|
||||
@@ -2451,7 +2539,7 @@ class DownloadManager:
|
||||
progress_callback, normalized_snapshot, rounded_progress
|
||||
)
|
||||
|
||||
async def cancel_download(self, download_id: str) -> Dict:
|
||||
async def cancel_download(self, download_id: str) -> Dict[str, Any]:
|
||||
"""Cancel an active download by download_id
|
||||
|
||||
Args:
|
||||
@@ -2533,7 +2621,7 @@ class DownloadManager:
|
||||
self._download_tasks.pop(download_id, None)
|
||||
await self._aria2_state_store.remove(download_id)
|
||||
|
||||
async def skip_download(self, download_id: str) -> Dict:
|
||||
async def skip_download(self, download_id: str) -> Dict[str, Any]:
|
||||
"""Skip a download while preserving all partial files on disk.
|
||||
|
||||
Removes all in-memory tracking (asyncio task, semaphore, active/pause
|
||||
@@ -2616,7 +2704,7 @@ class DownloadManager:
|
||||
# Preserve aria2 state store entry so the partial download
|
||||
# info survives restarts and can be resumed later
|
||||
|
||||
async def pause_download(self, download_id: str) -> Dict:
|
||||
async def pause_download(self, download_id: str) -> Dict[str, Any]:
|
||||
"""Pause an active download without losing progress."""
|
||||
|
||||
await self._restore_persisted_downloads()
|
||||
@@ -2663,7 +2751,7 @@ class DownloadManager:
|
||||
|
||||
return {"success": True, "message": "Download paused successfully"}
|
||||
|
||||
async def resume_download(self, download_id: str) -> Dict:
|
||||
async def resume_download(self, download_id: str) -> Dict[str, Any]:
|
||||
"""Resume a previously paused download."""
|
||||
|
||||
await self._restore_persisted_downloads()
|
||||
@@ -2680,7 +2768,7 @@ class DownloadManager:
|
||||
self._pause_events[download_id] = pause_control
|
||||
self._active_downloads[download_id] = self._build_restored_download_info(
|
||||
persisted,
|
||||
os.path.abspath(save_path),
|
||||
os.path.abspath(cast(str, save_path)),
|
||||
)
|
||||
|
||||
if pause_control.is_set():
|
||||
@@ -2724,6 +2812,7 @@ class DownloadManager:
|
||||
bool(persisted.get("use_default_paths", False)),
|
||||
persisted.get("source"),
|
||||
persisted.get("file_params"),
|
||||
bool(persisted.get("use_save_dir_as_root", False)),
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
@@ -2807,7 +2896,7 @@ class DownloadManager:
|
||||
elif asyncio.iscoroutine(result):
|
||||
await result
|
||||
|
||||
async def get_active_downloads(self) -> Dict:
|
||||
async def get_active_downloads(self) -> Dict[str, Any]:
|
||||
"""Get information about all active downloads
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
"""
|
||||
Unified download manager for all HTTP/HTTPS downloads in the application.
|
||||
|
||||
@@ -20,7 +24,7 @@ from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from email.utils import parsedate_to_datetime
|
||||
from urllib.parse import urlparse
|
||||
from typing import Optional, Dict, Tuple, Callable, Union, Awaitable
|
||||
from typing import Optional, Dict, Tuple, Callable, Union, Awaitable, Any, cast
|
||||
from ..services.settings_manager import get_settings_manager
|
||||
from .connectivity_guard import (
|
||||
OFFLINE_COOLDOWN_ERROR,
|
||||
@@ -204,6 +208,7 @@ class Downloader:
|
||||
# Double check after acquiring lock
|
||||
if self._session is None or self._should_refresh_session():
|
||||
await self._create_session()
|
||||
assert self._session is not None
|
||||
return self._session
|
||||
|
||||
@property
|
||||
@@ -231,7 +236,7 @@ class Downloader:
|
||||
)
|
||||
|
||||
try:
|
||||
timeout_value = float(raw_value)
|
||||
timeout_value = float(cast(Any, raw_value))
|
||||
except (TypeError, ValueError):
|
||||
timeout_value = default_timeout
|
||||
|
||||
@@ -243,7 +248,7 @@ class Downloader:
|
||||
raw_value = os.environ.get("COMFYUI_DOWNLOAD_MAX_RETRIES")
|
||||
|
||||
try:
|
||||
retries = int(raw_value)
|
||||
retries = int(cast(Any, raw_value))
|
||||
except (TypeError, ValueError):
|
||||
retries = default_retries
|
||||
|
||||
@@ -320,7 +325,7 @@ class Downloader:
|
||||
# CA coverage across different Python environments (especially
|
||||
# embedded/compatibility Python builds).
|
||||
try:
|
||||
import certifi # type: ignore[import-untyped]
|
||||
import certifi # pyright: ignore[reportMissingTypeStubs]
|
||||
|
||||
ca_path = certifi.where()
|
||||
ssl_context = ssl.create_default_context(cafile=ca_path)
|
||||
@@ -330,7 +335,7 @@ class Downloader:
|
||||
logger.debug("SSL: certifi unavailable; using system default CA bundle")
|
||||
|
||||
# Optimize TCP connection parameters
|
||||
connector_kwargs = dict(
|
||||
connector_kwargs: Dict[str, Any] = dict(
|
||||
ssl=ssl_context,
|
||||
limit=8, # Concurrent connections
|
||||
ttl_dns_cache=300, # DNS cache timeout
|
||||
@@ -890,7 +895,7 @@ class Downloader:
|
||||
use_auth: bool = False,
|
||||
custom_headers: Optional[Dict[str, str]] = None,
|
||||
return_headers: bool = False,
|
||||
) -> Tuple[bool, Union[bytes, str], Optional[Dict]]:
|
||||
) -> Tuple[bool, Union[bytes, str], Optional[Dict[str, Any]]]:
|
||||
"""
|
||||
Download a file to memory (for small files like preview images)
|
||||
|
||||
@@ -976,7 +981,7 @@ class Downloader:
|
||||
url: str,
|
||||
use_auth: bool = False,
|
||||
custom_headers: Optional[Dict[str, str]] = None,
|
||||
) -> Tuple[bool, Union[Dict, str]]:
|
||||
) -> Tuple[bool, Union[Dict[str, Any], str]]:
|
||||
"""
|
||||
Get response headers without downloading the full content
|
||||
|
||||
@@ -1036,7 +1041,7 @@ class Downloader:
|
||||
use_auth: bool = False,
|
||||
custom_headers: Optional[Dict[str, str]] = None,
|
||||
**kwargs,
|
||||
) -> Tuple[bool, Union[Dict, str]]:
|
||||
) -> Tuple[bool, Union[Dict[str, Any], str, RateLimitError]]:
|
||||
"""
|
||||
Make a generic HTTP request and return JSON response
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ class EmbeddingScanner(ModelScanner):
|
||||
roots.extend(config.embeddings_roots or [])
|
||||
roots.extend(config.extra_embeddings_roots or [])
|
||||
# Remove duplicates while preserving order
|
||||
seen: set = set()
|
||||
seen: set[str] = set()
|
||||
unique_roots: List[str] = []
|
||||
for root in roots:
|
||||
if root and root not in seen:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import logging
|
||||
from typing import Dict, Optional
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from .base_model_service import BaseModelService
|
||||
from .auto_tag_service import extract_auto_tags
|
||||
@@ -21,58 +21,58 @@ class EmbeddingService(BaseModelService):
|
||||
"""
|
||||
super().__init__("embedding", scanner, EmbeddingMetadata, update_service=update_service)
|
||||
|
||||
async def format_response(self, embedding_data: Dict) -> Optional[Dict]:
|
||||
async def format_response(self, model_data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""Format Embedding data for API response.
|
||||
|
||||
Returns None when the entry is missing critical fields (corrupted cache
|
||||
row), so the handler layer can filter it out. See issue #730.
|
||||
"""
|
||||
# Guard against corrupted cache entries missing critical fields
|
||||
file_path = embedding_data.get("file_path")
|
||||
file_path = model_data.get("file_path")
|
||||
if not file_path or not isinstance(file_path, str):
|
||||
logger.warning(
|
||||
"Skipping corrupted embedding entry (missing file_path): %s",
|
||||
embedding_data.get("file_name", "<unknown>"),
|
||||
model_data.get("file_name", "<unknown>"),
|
||||
)
|
||||
return None
|
||||
|
||||
# Get sub_type from cache entry (new canonical field)
|
||||
sub_type = embedding_data.get("sub_type", "embedding")
|
||||
sub_type = model_data.get("sub_type", "embedding")
|
||||
|
||||
file_name = embedding_data.get("file_name") or ""
|
||||
model_name = embedding_data.get("model_name") or file_name
|
||||
folder = embedding_data.get("folder") or ""
|
||||
file_name = model_data.get("file_name") or ""
|
||||
model_name = model_data.get("model_name") or file_name
|
||||
folder = model_data.get("folder") or ""
|
||||
|
||||
return {
|
||||
"model_name": model_name,
|
||||
"file_name": file_name,
|
||||
"preview_url": config.get_preview_static_url(embedding_data.get("preview_url", "")),
|
||||
"preview_nsfw_level": embedding_data.get("preview_nsfw_level", 0),
|
||||
"base_model": embedding_data.get("base_model", ""),
|
||||
"preview_url": config.get_preview_static_url(model_data.get("preview_url", "")),
|
||||
"preview_nsfw_level": model_data.get("preview_nsfw_level", 0),
|
||||
"base_model": model_data.get("base_model", ""),
|
||||
"folder": folder,
|
||||
"sha256": embedding_data.get("sha256", ""),
|
||||
"sha256": model_data.get("sha256", ""),
|
||||
"file_path": file_path.replace(os.sep, "/"),
|
||||
"file_size": embedding_data.get("size", 0),
|
||||
"modified": embedding_data.get("modified", ""),
|
||||
"tags": embedding_data.get("tags", []),
|
||||
"from_civitai": embedding_data.get("from_civitai", True),
|
||||
# "usage_count": embedding_data.get("usage_count", 0), # TODO: Enable when embedding usage tracking is implemented
|
||||
"notes": embedding_data.get("notes", ""),
|
||||
"file_size": model_data.get("size", 0),
|
||||
"modified": model_data.get("modified", ""),
|
||||
"tags": model_data.get("tags", []),
|
||||
"from_civitai": model_data.get("from_civitai", True),
|
||||
# "usage_count": model_data.get("usage_count", 0), # TODO: Enable when embedding usage tracking is implemented
|
||||
"notes": model_data.get("notes", ""),
|
||||
"sub_type": sub_type,
|
||||
"favorite": embedding_data.get("favorite", False),
|
||||
"exclude": bool(embedding_data.get("exclude", False)),
|
||||
"update_available": bool(embedding_data.get("update_available", False)),
|
||||
"skip_metadata_refresh": bool(embedding_data.get("skip_metadata_refresh", False)),
|
||||
"civitai": self.filter_civitai_data(embedding_data.get("civitai", {}), minimal=True),
|
||||
"auto_tags": embedding_data.get("auto_tags") or extract_auto_tags(embedding_data),
|
||||
"version_count": embedding_data.get("version_count"),
|
||||
"hf_url": embedding_data.get("hf_url", ""),
|
||||
"favorite": model_data.get("favorite", False),
|
||||
"exclude": bool(model_data.get("exclude", False)),
|
||||
"update_available": bool(model_data.get("update_available", False)),
|
||||
"skip_metadata_refresh": bool(model_data.get("skip_metadata_refresh", False)),
|
||||
"civitai": self.filter_civitai_data(model_data.get("civitai", {}), minimal=True),
|
||||
"auto_tags": model_data.get("auto_tags") or extract_auto_tags(model_data),
|
||||
"version_count": model_data.get("version_count"),
|
||||
"hf_url": model_data.get("hf_url", ""),
|
||||
}
|
||||
|
||||
def find_duplicate_hashes(self) -> Dict:
|
||||
def find_duplicate_hashes(self) -> Dict[str, Any]:
|
||||
"""Find Embeddings with duplicate SHA256 hashes"""
|
||||
return self.scanner._hash_index.get_duplicate_hashes()
|
||||
|
||||
def find_duplicate_filenames(self) -> Dict:
|
||||
def find_duplicate_filenames(self) -> Dict[str, Any]:
|
||||
"""Find Embeddings with conflicting filenames"""
|
||||
return self.scanner._hash_index.get_duplicate_filenames()
|
||||
|
||||
@@ -35,7 +35,7 @@ class CleanupResult:
|
||||
def to_dict(self) -> Dict[str, object]:
|
||||
"""Convert the dataclass to a serialisable dictionary."""
|
||||
|
||||
data = {
|
||||
data: Dict[str, object] = {
|
||||
"success": self.success,
|
||||
"checked_folders": self.checked_folders,
|
||||
"moved_empty_folders": self.moved_empty_folders,
|
||||
|
||||
@@ -201,6 +201,11 @@ PROVIDER_PRESETS: Dict[str, Dict[str, Any]] = {
|
||||
"api_base": "https://openrouter.ai/api/v1",
|
||||
"requires_key": True,
|
||||
},
|
||||
"google": {
|
||||
"name": "Gemini",
|
||||
"api_base": "https://generativelanguage.googleapis.com/v1beta/openai",
|
||||
"requires_key": True,
|
||||
},
|
||||
"opencode-go": {
|
||||
"name": "OpenCode Go",
|
||||
"api_base": "https://opencode.ai/zen/go/v1",
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
from ..utils.models import LoraMetadata
|
||||
from ..config import config
|
||||
from .model_scanner import ModelScanner
|
||||
from .model_hash_index import ModelHashIndex # Changed from LoraHashIndex to ModelHashIndex
|
||||
import sys
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -15,8 +17,10 @@ class LoraScanner(ModelScanner):
|
||||
def __init__(self):
|
||||
# Define supported file extensions
|
||||
file_extensions = {'.safetensors'}
|
||||
|
||||
|
||||
# Initialize parent class with ModelHashIndex
|
||||
from .model_hash_index import ModelHashIndex
|
||||
|
||||
super().__init__(
|
||||
model_type="lora",
|
||||
model_class=LoraMetadata,
|
||||
@@ -26,11 +30,13 @@ class LoraScanner(ModelScanner):
|
||||
|
||||
def get_model_roots(self) -> List[str]:
|
||||
"""Get lora root directories (including extra paths)"""
|
||||
from ..config import config
|
||||
|
||||
roots: List[str] = []
|
||||
roots.extend(config.loras_roots or [])
|
||||
roots.extend(config.extra_loras_roots or [])
|
||||
# Remove duplicates while preserving order
|
||||
seen: set = set()
|
||||
seen: set[str] = set()
|
||||
unique_roots: List[str] = []
|
||||
for root in roots:
|
||||
if root and root not in seen:
|
||||
@@ -68,8 +74,12 @@ class LoraScanner(ModelScanner):
|
||||
test_hash = next(iter(self._hash_index._hash_to_path.keys()))
|
||||
test_path = self._hash_index.get_path(test_hash)
|
||||
logger.debug(f"\nTest lookup by hash: {test_hash[:8]}... -> {test_path}")
|
||||
if test_path is None:
|
||||
return
|
||||
|
||||
# Also test reverse lookup
|
||||
test_hash_result = self._hash_index.get_hash(test_path)
|
||||
if test_hash_result is None:
|
||||
return
|
||||
logger.debug(f"Test reverse lookup: {test_path} -> {test_hash_result[:8]}...\n\n")
|
||||
|
||||
|
||||
+41
-41
@@ -1,7 +1,7 @@
|
||||
import logging
|
||||
import json
|
||||
import os
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base_model_service import BaseModelService
|
||||
from .model_query import resolve_sub_type
|
||||
@@ -24,7 +24,7 @@ class LoraService(BaseModelService):
|
||||
"""
|
||||
super().__init__("lora", scanner, LoraMetadata, update_service=update_service)
|
||||
|
||||
async def format_response(self, lora_data: Dict) -> Optional[Dict]:
|
||||
async def format_response(self, model_data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""Format LoRA data for API response.
|
||||
|
||||
Returns None when the entry is missing critical fields (corrupted cache
|
||||
@@ -32,56 +32,56 @@ class LoraService(BaseModelService):
|
||||
whole listing request. See issue #730.
|
||||
"""
|
||||
# Guard against corrupted cache entries missing critical fields
|
||||
file_path = lora_data.get("file_path")
|
||||
file_path = model_data.get("file_path")
|
||||
if not file_path or not isinstance(file_path, str):
|
||||
logger.warning(
|
||||
"Skipping corrupted LoRA entry (missing file_path): %s",
|
||||
lora_data.get("file_name", "<unknown>"),
|
||||
model_data.get("file_name", "<unknown>"),
|
||||
)
|
||||
return None
|
||||
|
||||
# Resolve sub_type using priority: sub_type > model_type > civitai.model.type > default
|
||||
# Normalize to lowercase for consistent API responses
|
||||
sub_type = resolve_sub_type(lora_data).lower()
|
||||
sub_type = resolve_sub_type(model_data).lower()
|
||||
|
||||
file_name = lora_data.get("file_name") or ""
|
||||
model_name = lora_data.get("model_name") or file_name
|
||||
folder = lora_data.get("folder") or ""
|
||||
file_name = model_data.get("file_name") or ""
|
||||
model_name = model_data.get("model_name") or file_name
|
||||
folder = model_data.get("folder") or ""
|
||||
|
||||
return {
|
||||
"model_name": model_name,
|
||||
"file_name": file_name,
|
||||
"preview_url": config.get_preview_static_url(
|
||||
lora_data.get("preview_url", "")
|
||||
model_data.get("preview_url", "")
|
||||
),
|
||||
"preview_nsfw_level": lora_data.get("preview_nsfw_level", 0),
|
||||
"base_model": lora_data.get("base_model", ""),
|
||||
"preview_nsfw_level": model_data.get("preview_nsfw_level", 0),
|
||||
"base_model": model_data.get("base_model", ""),
|
||||
"folder": folder,
|
||||
"sha256": lora_data.get("sha256", ""),
|
||||
"sha256": model_data.get("sha256", ""),
|
||||
"file_path": file_path.replace(os.sep, "/"),
|
||||
"file_size": lora_data.get("size", 0),
|
||||
"modified": lora_data.get("modified", ""),
|
||||
"tags": lora_data.get("tags", []),
|
||||
"from_civitai": lora_data.get("from_civitai", True),
|
||||
"usage_count": lora_data.get("usage_count", 0),
|
||||
"usage_tips": lora_data.get("usage_tips", ""),
|
||||
"notes": lora_data.get("notes", ""),
|
||||
"favorite": lora_data.get("favorite", False),
|
||||
"exclude": bool(lora_data.get("exclude", False)),
|
||||
"update_available": bool(lora_data.get("update_available", False)),
|
||||
"file_size": model_data.get("size", 0),
|
||||
"modified": model_data.get("modified", ""),
|
||||
"tags": model_data.get("tags", []),
|
||||
"from_civitai": model_data.get("from_civitai", True),
|
||||
"usage_count": model_data.get("usage_count", 0),
|
||||
"usage_tips": model_data.get("usage_tips", ""),
|
||||
"notes": model_data.get("notes", ""),
|
||||
"favorite": model_data.get("favorite", False),
|
||||
"exclude": bool(model_data.get("exclude", False)),
|
||||
"update_available": bool(model_data.get("update_available", False)),
|
||||
"skip_metadata_refresh": bool(
|
||||
lora_data.get("skip_metadata_refresh", False)
|
||||
model_data.get("skip_metadata_refresh", False)
|
||||
),
|
||||
"sub_type": sub_type,
|
||||
"civitai": self.filter_civitai_data(
|
||||
lora_data.get("civitai", {}), minimal=True
|
||||
model_data.get("civitai", {}), minimal=True
|
||||
),
|
||||
"auto_tags": lora_data.get("auto_tags") or extract_auto_tags(lora_data),
|
||||
"version_count": lora_data.get("version_count"),
|
||||
"hf_url": lora_data.get("hf_url", ""),
|
||||
"auto_tags": model_data.get("auto_tags") or extract_auto_tags(model_data),
|
||||
"version_count": model_data.get("version_count"),
|
||||
"hf_url": model_data.get("hf_url", ""),
|
||||
}
|
||||
|
||||
async def _apply_specific_filters(self, data: List[Dict], **kwargs) -> List[Dict]:
|
||||
async def _apply_specific_filters(self, data: List[Dict[str, Any]], **kwargs) -> List[Dict[str, Any]]:
|
||||
"""Apply LoRA-specific filters"""
|
||||
# Handle first_letter filter for LoRAs
|
||||
first_letter = kwargs.get("first_letter")
|
||||
@@ -152,7 +152,7 @@ class LoraService(BaseModelService):
|
||||
|
||||
return data
|
||||
|
||||
def _filter_by_first_letter(self, data: List[Dict], letter: str) -> List[Dict]:
|
||||
def _filter_by_first_letter(self, data: List[Dict[str, Any]], letter: str) -> List[Dict[str, Any]]:
|
||||
"""Filter data by first letter of model name
|
||||
|
||||
Special handling:
|
||||
@@ -307,7 +307,7 @@ class LoraService(BaseModelService):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_recommended_strength_from_lora_data(lora_data: Dict) -> Optional[float]:
|
||||
def get_recommended_strength_from_lora_data(lora_data: Dict[str, Any]) -> Optional[float]:
|
||||
"""Parse usage_tips JSON and extract recommended model strength."""
|
||||
try:
|
||||
usage_tips = lora_data.get("usage_tips", "")
|
||||
@@ -320,7 +320,7 @@ class LoraService(BaseModelService):
|
||||
|
||||
@staticmethod
|
||||
def get_recommended_clip_strength_from_lora_data(
|
||||
lora_data: Dict,
|
||||
lora_data: Dict[str, Any],
|
||||
) -> Optional[float]:
|
||||
"""Parse usage_tips JSON and extract recommended clip strength."""
|
||||
try:
|
||||
@@ -332,7 +332,7 @@ class LoraService(BaseModelService):
|
||||
except (json.JSONDecodeError, TypeError, AttributeError):
|
||||
return None
|
||||
|
||||
async def get_lora_metadata_by_filename(self, filename: str) -> Optional[Dict]:
|
||||
async def get_lora_metadata_by_filename(self, filename: str) -> Optional[Dict[str, Any]]:
|
||||
"""Return cached raw metadata for a LoRA matching the given filename."""
|
||||
cache = await self.scanner.get_cached_data(force_refresh=False)
|
||||
|
||||
@@ -357,11 +357,11 @@ class LoraService(BaseModelService):
|
||||
|
||||
return None
|
||||
|
||||
def find_duplicate_hashes(self) -> Dict:
|
||||
def find_duplicate_hashes(self) -> Dict[str, Any]:
|
||||
"""Find LoRAs with duplicate SHA256 hashes"""
|
||||
return self.scanner._hash_index.get_duplicate_hashes()
|
||||
|
||||
def find_duplicate_filenames(self) -> Dict:
|
||||
def find_duplicate_filenames(self) -> Dict[str, Any]:
|
||||
"""Find LoRAs with conflicting filenames"""
|
||||
return self.scanner._hash_index.get_duplicate_filenames()
|
||||
|
||||
@@ -373,8 +373,8 @@ class LoraService(BaseModelService):
|
||||
use_same_clip_strength: bool = True,
|
||||
clip_strength_min: float = 0.0,
|
||||
clip_strength_max: float = 1.0,
|
||||
locked_loras: Optional[List[Dict]] = None,
|
||||
pool_config: Optional[Dict] = None,
|
||||
locked_loras: Optional[List[Dict[str, Any]]] = None,
|
||||
pool_config: Optional[Dict[str, Any]] = None,
|
||||
count_mode: str = "fixed",
|
||||
count_min: int = 3,
|
||||
count_max: int = 7,
|
||||
@@ -382,7 +382,7 @@ class LoraService(BaseModelService):
|
||||
recommended_strength_scale_min: float = 0.5,
|
||||
recommended_strength_scale_max: float = 1.0,
|
||||
seed: Optional[int] = None,
|
||||
) -> List[Dict]:
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get random LoRAs with specified strength ranges.
|
||||
|
||||
@@ -513,8 +513,8 @@ class LoraService(BaseModelService):
|
||||
return result_loras
|
||||
|
||||
async def _apply_pool_filters(
|
||||
self, available_loras: List[Dict], pool_config: Dict
|
||||
) -> List[Dict]:
|
||||
self, available_loras: List[Dict[str, Any]], pool_config: Dict[str, Any]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Apply pool_config filters to available LoRAs.
|
||||
|
||||
@@ -671,8 +671,8 @@ class LoraService(BaseModelService):
|
||||
return available_loras
|
||||
|
||||
async def get_cycler_list(
|
||||
self, pool_config: Optional[Dict] = None, sort_by: str = "filename"
|
||||
) -> List[Dict]:
|
||||
self, pool_config: Optional[Dict[str, Any]] = None, sort_by: str = "filename"
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get filtered and sorted LoRA list for cycling.
|
||||
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
import os
|
||||
import logging
|
||||
from .model_metadata_provider import (
|
||||
@@ -170,7 +174,7 @@ def _wrap_provider_with_rate_limit(provider_name: str | None, provider: ModelMet
|
||||
return RateLimitRetryingProvider(provider, label=provider_name)
|
||||
|
||||
|
||||
async def get_metadata_provider(provider_name: str = None):
|
||||
async def get_metadata_provider(provider_name: str | None = None):
|
||||
"""Get a specific metadata provider or default provider with rate-limit handling."""
|
||||
|
||||
provider_manager = await ModelMetadataProviderManager.get_instance()
|
||||
|
||||
@@ -6,25 +6,26 @@ import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Any, Awaitable, Callable, Dict, Iterable, Optional
|
||||
from typing import Any, Awaitable, Callable, Dict, Iterable, Optional, Protocol
|
||||
|
||||
from ..services.settings_manager import SettingsManager
|
||||
from ..utils.civitai_utils import resolve_license_payload
|
||||
from ..utils.model_utils import determine_base_model
|
||||
from ..utils.models import autov3_from_civitai_files
|
||||
from .connectivity_guard import OFFLINE_FRIENDLY_MESSAGE, is_expected_offline_error
|
||||
from .errors import RateLimitError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MetadataProviderProtocol:
|
||||
class MetadataProviderProtocol(Protocol):
|
||||
"""Subset of metadata provider interface consumed by the sync service."""
|
||||
|
||||
async def get_model_by_hash(self, sha256: str) -> tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
async def get_model_by_hash(self, model_hash: str) -> tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
...
|
||||
|
||||
async def get_model_version(
|
||||
self, model_id: int, model_version_id: Optional[int]
|
||||
self, model_id: Any = None, version_id: Any = None
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
...
|
||||
|
||||
@@ -38,8 +39,8 @@ class MetadataSyncService:
|
||||
metadata_manager,
|
||||
preview_service,
|
||||
settings: SettingsManager,
|
||||
default_metadata_provider_factory: Callable[[], Awaitable[MetadataProviderProtocol]],
|
||||
metadata_provider_selector: Callable[[str], Awaitable[MetadataProviderProtocol]],
|
||||
default_metadata_provider_factory: Callable[..., Awaitable[MetadataProviderProtocol]],
|
||||
metadata_provider_selector: Callable[..., Awaitable[MetadataProviderProtocol]],
|
||||
) -> None:
|
||||
self._metadata_manager = metadata_manager
|
||||
self._preview_service = preview_service
|
||||
@@ -152,6 +153,18 @@ class MetadataSyncService:
|
||||
civitai_metadata.get("baseModel")
|
||||
)
|
||||
|
||||
# Civitai-first AutoV3 propagation: the freshly fetched version
|
||||
# metadata may report an AutoV3 for the file whose SHA256 matches the
|
||||
# local model. Persist it now so recipe matching sees it immediately —
|
||||
# no full rescan or restart required (the header is never re-read to
|
||||
# upgrade the checked-unavailable '' state).
|
||||
sha256_value = (local_metadata.get("sha256") or "").lower()
|
||||
civitai_autov3 = autov3_from_civitai_files(
|
||||
local_metadata.get("civitai"), sha256_value
|
||||
)
|
||||
if civitai_autov3:
|
||||
local_metadata["autov3"] = civitai_autov3
|
||||
|
||||
await self._preview_service.ensure_preview_for_metadata(
|
||||
metadata_path, local_metadata, civitai_metadata.get("images", [])
|
||||
)
|
||||
@@ -479,7 +492,7 @@ class MetadataSyncService:
|
||||
if not file_paths:
|
||||
raise ValueError("No file paths provided for verification")
|
||||
|
||||
results = {
|
||||
results: Dict[str, Any] = {
|
||||
"verified_as_duplicates": True,
|
||||
"mismatched_files": [],
|
||||
"new_hash_map": {},
|
||||
|
||||
+21
-16
@@ -31,17 +31,22 @@ DISPLAY_NAME_MODES = {"model_name", "file_name"}
|
||||
class ModelCache:
|
||||
"""Cache structure for model data with extensible sorting."""
|
||||
|
||||
raw_data: List[Dict]
|
||||
raw_data: List[Dict[str, Any]]
|
||||
folders: List[str]
|
||||
version_index: Dict[int, Dict] = field(default_factory=dict)
|
||||
version_index: Dict[int, Dict[str, Any]] = field(default_factory=dict)
|
||||
model_id_index: Dict[int, List[Dict[str, Any]]] = field(default_factory=dict)
|
||||
name_display_mode: str = "model_name"
|
||||
_lock: Any = field(init=False, repr=False, default=None)
|
||||
# Cache for last sort: (sort_key, order, seed) -> sorted list
|
||||
_last_sort: Tuple[Optional[str], str, Optional[str]] = field(
|
||||
init=False, repr=False, default=(None, "asc", None)
|
||||
)
|
||||
_last_sorted_data: List[Dict[str, Any]] = field(
|
||||
init=False, repr=False, default_factory=list
|
||||
)
|
||||
|
||||
def __post_init__(self):
|
||||
self._lock = asyncio.Lock()
|
||||
# Cache for last sort: (sort_key, order, seed) -> sorted list
|
||||
self._last_sort: Tuple[Optional[str], str, Optional[str]] = (None, "asc", None)
|
||||
self._last_sorted_data: List[Dict] = []
|
||||
self._normalize_raw_data()
|
||||
self.name_display_mode = self._normalize_display_mode(self.name_display_mode)
|
||||
# Default sort on init
|
||||
@@ -64,7 +69,7 @@ class ModelCache:
|
||||
return ""
|
||||
return str(value)
|
||||
|
||||
def _normalize_item(self, item: Dict) -> None:
|
||||
def _normalize_item(self, item: Dict[str, Any]) -> None:
|
||||
"""Ensure core metadata fields are present and string typed."""
|
||||
|
||||
if not isinstance(item, dict):
|
||||
@@ -80,7 +85,7 @@ class ModelCache:
|
||||
for item in self.raw_data:
|
||||
self._normalize_item(item)
|
||||
|
||||
def _get_display_name(self, item: Dict) -> str:
|
||||
def _get_display_name(self, item: Dict[str, Any]) -> str:
|
||||
"""Return the value used for name-based sorting based on display settings."""
|
||||
|
||||
if self.name_display_mode == "file_name":
|
||||
@@ -114,7 +119,7 @@ class ModelCache:
|
||||
for item in self.raw_data:
|
||||
self.add_to_version_index(item)
|
||||
|
||||
def add_to_version_index(self, item: Dict) -> None:
|
||||
def add_to_version_index(self, item: Dict[str, Any]) -> None:
|
||||
"""Register a cache item in the version/model indexes if possible."""
|
||||
|
||||
civitai_data = item.get('civitai') if isinstance(item, dict) else None
|
||||
@@ -143,7 +148,7 @@ class ModelCache:
|
||||
else:
|
||||
versions.append(descriptor)
|
||||
|
||||
def remove_from_version_index(self, item: Dict) -> None:
|
||||
def remove_from_version_index(self, item: Dict[str, Any]) -> None:
|
||||
"""Remove a cache item from the version/model indexes if present."""
|
||||
|
||||
civitai_data = item.get('civitai') if isinstance(item, dict) else None
|
||||
@@ -177,7 +182,7 @@ class ModelCache:
|
||||
|
||||
def _build_version_descriptor(
|
||||
self,
|
||||
item: Dict,
|
||||
item: Dict[str, Any],
|
||||
civitai_data: Dict[str, Any],
|
||||
version_id: int,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
@@ -204,8 +209,8 @@ class ModelCache:
|
||||
async def resort(self):
|
||||
"""Resort cached data according to last sort mode if set"""
|
||||
async with self._lock:
|
||||
if self._last_sort[0] is not None:
|
||||
sort_key, order, seed = self._last_sort
|
||||
sort_key, order, seed = self._last_sort
|
||||
if sort_key is not None:
|
||||
sorted_data = self._sort_data(self.raw_data, sort_key, order, seed)
|
||||
self._last_sorted_data = sorted_data
|
||||
# Update folder list
|
||||
@@ -219,7 +224,7 @@ class ModelCache:
|
||||
self.folders = sorted(list(all_folders), key=lambda x: x.lower())
|
||||
self.rebuild_version_index()
|
||||
|
||||
def _sort_data(self, data: List[Dict], sort_key: str, order: str, seed: Optional[str] = None) -> List[Dict]:
|
||||
def _sort_data(self, data: List[Dict[str, Any]], sort_key: str, order: str, seed: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""Sort data by sort_key and order"""
|
||||
start_time = time.perf_counter()
|
||||
reverse = (order == 'desc')
|
||||
@@ -293,7 +298,7 @@ class ModelCache:
|
||||
logger.debug("ModelCache._sort_data(%s, %s) for %d items took %.3fs", sort_key, order, len(data), duration)
|
||||
return result
|
||||
|
||||
async def get_sorted_data(self, sort_key: str = 'name', order: str = 'asc', seed: Optional[str] = None) -> List[Dict]:
|
||||
async def get_sorted_data(self, sort_key: str = 'name', order: str = 'asc', seed: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""Get sorted data by sort_key and order, using cache if possible"""
|
||||
async with self._lock:
|
||||
cache_key = (sort_key, order, seed)
|
||||
@@ -321,8 +326,8 @@ class ModelCache:
|
||||
|
||||
self.name_display_mode = normalized
|
||||
|
||||
if self._last_sort[0] == 'name':
|
||||
sort_key, order, seed = self._last_sort
|
||||
sort_key, order, seed = self._last_sort
|
||||
if sort_key == 'name':
|
||||
self._last_sorted_data = self._sort_data(self.raw_data, sort_key, order, seed)
|
||||
|
||||
async def update_preview_url(self, file_path: str, preview_url: str, preview_nsfw_level: int) -> bool:
|
||||
|
||||
@@ -41,7 +41,7 @@ class AutoOrganizeResult:
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary"""
|
||||
result = {
|
||||
result: Dict[str, Any] = {
|
||||
'success': self.status != 'error',
|
||||
'status': self.status,
|
||||
'message': f'Auto-organize {self.operation_type} completed: {self.success_count} moved, {self.skipped_count} skipped, {self.failure_count} failed out of {self.total} total',
|
||||
@@ -418,6 +418,8 @@ class ModelFileService:
|
||||
"""Calculate the target directory for a model"""
|
||||
if is_flat_structure:
|
||||
file_path = model.get('file_path')
|
||||
if not isinstance(file_path, str):
|
||||
return None
|
||||
current_dir = os.path.dirname(file_path)
|
||||
|
||||
# Check if already in root directory
|
||||
|
||||
@@ -8,11 +8,12 @@ class ModelHashIndex:
|
||||
self._hash_to_path: Dict[str, str] = {}
|
||||
self._filename_to_hash: Dict[str, str] = {}
|
||||
self._autov2_to_path: Dict[str, str] = {}
|
||||
self._autov3_to_path: Dict[str, str] = {}
|
||||
# New data structures for tracking duplicates
|
||||
self._duplicate_hashes: Dict[str, List[str]] = {} # sha256 -> list of paths
|
||||
self._duplicate_filenames: Dict[str, List[str]] = {} # filename -> list of paths
|
||||
|
||||
def add_entry(self, sha256: str, file_path: str) -> None:
|
||||
def add_entry(self, sha256: str, file_path: str, autov3: Optional[str] = None) -> None:
|
||||
"""Add or update hash index entry"""
|
||||
if not sha256 or not file_path:
|
||||
return
|
||||
@@ -33,9 +34,14 @@ class ModelHashIndex:
|
||||
self._duplicate_hashes.setdefault(sha256, []).append(file_path)
|
||||
|
||||
# Track duplicates by filename - FIXED LOGIC
|
||||
is_re_registration = False
|
||||
existing_hash: Optional[str] = None
|
||||
if filename in self._filename_to_hash:
|
||||
existing_hash = self._filename_to_hash[filename]
|
||||
existing_path = self._hash_to_path.get(existing_hash)
|
||||
# Same path registered again (e.g. a file replaced in place with
|
||||
# new content) — used below to drop its stale autov3 mapping.
|
||||
is_re_registration = existing_path == file_path
|
||||
|
||||
# If this is a different file with the same filename
|
||||
if existing_path and existing_path != file_path:
|
||||
@@ -67,12 +73,36 @@ class ModelHashIndex:
|
||||
# AutoV2 = first 10 chars of SHA256
|
||||
if len(sha256) >= 10:
|
||||
self._autov2_to_path[sha256[:10]] = file_path
|
||||
# AutoV3 is an independent hash (not derived from SHA256), stored as-is.
|
||||
# Drop stale mappings for a path when it is re-registered with a NEW
|
||||
# sha256 (file replaced in place) or with an explicit new autov3 value
|
||||
# (correction). Re-registering the SAME file with the same sha256 and
|
||||
# no autov3 (e.g. lazy-hash completion) must never clear its existing
|
||||
# mapping. First-time registrations stay O(1).
|
||||
if autov3:
|
||||
autov3 = autov3.lower()
|
||||
if is_re_registration and (existing_hash != sha256 or autov3):
|
||||
stale_autov3_keys = [
|
||||
key for key, mapped_path in self._autov3_to_path.items()
|
||||
if mapped_path == file_path and key != autov3
|
||||
]
|
||||
for key in stale_autov3_keys:
|
||||
del self._autov3_to_path[key]
|
||||
if autov3:
|
||||
self._autov3_to_path[autov3] = file_path
|
||||
|
||||
def add_autov3(self, autov3: str, file_path: str) -> None:
|
||||
"""Add or update an AutoV3-only index entry (used when only AutoV3 is known)"""
|
||||
if not autov3:
|
||||
return
|
||||
autov3 = autov3.lower()
|
||||
self._autov3_to_path[autov3] = file_path
|
||||
|
||||
def _get_filename_from_path(self, file_path: str) -> str:
|
||||
"""Extract filename without extension from path"""
|
||||
return os.path.splitext(os.path.basename(file_path))[0]
|
||||
|
||||
def remove_by_path(self, file_path: str, hash_val: str = None) -> None:
|
||||
def remove_by_path(self, file_path: str, hash_val: Optional[str] = None) -> None:
|
||||
"""Remove entry by file path"""
|
||||
filename = self._get_filename_from_path(file_path)
|
||||
|
||||
@@ -167,6 +197,11 @@ class ModelHashIndex:
|
||||
for k in autov2_keys_to_remove:
|
||||
del self._autov2_to_path[k]
|
||||
|
||||
# Remove from AutoV3 index
|
||||
autov3_keys_to_remove = [k for k, v in self._autov3_to_path.items() if v == file_path]
|
||||
for k in autov3_keys_to_remove:
|
||||
del self._autov3_to_path[k]
|
||||
|
||||
def remove_by_hash(self, sha256: str) -> None:
|
||||
"""Remove entry by hash"""
|
||||
sha256 = sha256.lower()
|
||||
@@ -189,6 +224,11 @@ class ModelHashIndex:
|
||||
autov2_key = sha256[:10]
|
||||
if autov2_key in self._autov2_to_path:
|
||||
del self._autov2_to_path[autov2_key]
|
||||
|
||||
# Remove AutoV3 entries pointing to any removed path
|
||||
autov3_keys_to_remove = [k for k, v in self._autov3_to_path.items() if v in paths_to_remove]
|
||||
for k in autov3_keys_to_remove:
|
||||
del self._autov3_to_path[k]
|
||||
|
||||
# Update filename-to-hash and duplicate filenames for all paths
|
||||
for path_to_remove in paths_to_remove:
|
||||
@@ -209,22 +249,26 @@ class ModelHashIndex:
|
||||
del self._duplicate_filenames[fname]
|
||||
|
||||
def has_hash(self, hash_value: str) -> bool:
|
||||
"""Check if hash exists in index (SHA256 or AutoV2)"""
|
||||
"""Check if hash exists in index (SHA256, AutoV2, or AutoV3)"""
|
||||
normalized = hash_value.lower()
|
||||
if normalized in self._hash_to_path:
|
||||
return True
|
||||
if len(normalized) == 10:
|
||||
return normalized in self._autov2_to_path
|
||||
if len(normalized) == 12:
|
||||
return normalized in self._autov3_to_path
|
||||
return False
|
||||
|
||||
def get_path(self, hash_value: str) -> Optional[str]:
|
||||
"""Get file path for a hash (SHA256 or AutoV2)"""
|
||||
"""Get file path for a hash (SHA256, AutoV2, or AutoV3)"""
|
||||
normalized = hash_value.lower()
|
||||
path = self._hash_to_path.get(normalized)
|
||||
if path is not None:
|
||||
return path
|
||||
if len(normalized) == 10:
|
||||
return self._autov2_to_path.get(normalized)
|
||||
if len(normalized) == 12:
|
||||
return self._autov3_to_path.get(normalized)
|
||||
return None
|
||||
|
||||
def get_hash(self, file_path: str) -> Optional[str]:
|
||||
@@ -243,6 +287,7 @@ class ModelHashIndex:
|
||||
self._hash_to_path.clear()
|
||||
self._filename_to_hash.clear()
|
||||
self._autov2_to_path.clear()
|
||||
self._autov3_to_path.clear()
|
||||
self._duplicate_hashes.clear()
|
||||
self._duplicate_filenames.clear()
|
||||
|
||||
@@ -253,6 +298,10 @@ class ModelHashIndex:
|
||||
def get_all_filenames(self) -> Set[str]:
|
||||
"""Get all filenames in the index"""
|
||||
return set(self._filename_to_hash.keys())
|
||||
|
||||
def get_all_autov3(self) -> Dict[str, str]:
|
||||
"""Get a snapshot of all AutoV3 hashes mapped to their file paths"""
|
||||
return dict(self._autov3_to_path)
|
||||
|
||||
def get_duplicate_hashes(self) -> Dict[str, List[str]]:
|
||||
"""Get dictionary of duplicate hashes and their paths"""
|
||||
|
||||
@@ -4,9 +4,10 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Awaitable, Callable, Dict, Iterable, List, Mapping, Optional, TYPE_CHECKING
|
||||
from typing import Any, Awaitable, Callable, Dict, Iterable, List, Mapping, Optional, TYPE_CHECKING, cast
|
||||
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
from ..services.pending_delete_service import get_pending_delete_service
|
||||
from ..utils.constants import PREVIEW_EXTENSIONS
|
||||
from ..utils.metadata_manager import MetadataManager
|
||||
|
||||
@@ -87,8 +88,8 @@ class ModelLifecycleService:
|
||||
scanner,
|
||||
metadata_manager,
|
||||
metadata_loader: Callable[[str], Awaitable[Dict[str, object]]],
|
||||
recipe_scanner_factory: Callable[[], Awaitable] | None = None,
|
||||
update_service: "ModelUpdateService" | None = None,
|
||||
recipe_scanner_factory: Callable[[], Awaitable[Any]] | None = None,
|
||||
update_service: Optional["ModelUpdateService"] = None,
|
||||
) -> None:
|
||||
self._scanner = scanner
|
||||
self._metadata_manager = metadata_manager
|
||||
@@ -129,15 +130,33 @@ class ModelLifecycleService:
|
||||
target_dir = os.path.dirname(file_path)
|
||||
base_name = os.path.basename(file_path)
|
||||
file_name, main_extension = os.path.splitext(base_name)
|
||||
deleted_files = await delete_model_artifacts(
|
||||
target_dir, file_name, main_extension=main_extension
|
||||
|
||||
# Stage the delete into the pending-delete service when undo is
|
||||
# enabled; a successful stage renames the artifacts away, otherwise
|
||||
# fall back to the direct hard delete.
|
||||
pending_delete_service = await get_pending_delete_service()
|
||||
batch_id = await pending_delete_service.stage_model_delete(
|
||||
scanner=self._scanner,
|
||||
target_dir=target_dir,
|
||||
file_name=file_name,
|
||||
main_extension=main_extension,
|
||||
original_file_path=file_path,
|
||||
cached_entry=cached_entry,
|
||||
)
|
||||
deleted_files: List[str] = []
|
||||
if batch_id is None:
|
||||
deleted_files = await delete_model_artifacts(
|
||||
target_dir, file_name, main_extension=main_extension
|
||||
)
|
||||
|
||||
if cache:
|
||||
cache.raw_data = [
|
||||
item for item in cache.raw_data if item.get("file_path") != file_path
|
||||
]
|
||||
await cache.resort()
|
||||
bump_cache_version = getattr(self._scanner, "bump_cache_version", None)
|
||||
if callable(bump_cache_version):
|
||||
bump_cache_version()
|
||||
|
||||
if hasattr(self._scanner, "_hash_index") and self._scanner._hash_index:
|
||||
self._scanner._hash_index.remove_by_path(file_path)
|
||||
@@ -146,9 +165,13 @@ class ModelLifecycleService:
|
||||
|
||||
persist_current_cache = getattr(self._scanner, "_persist_current_cache", None)
|
||||
if callable(persist_current_cache):
|
||||
await persist_current_cache()
|
||||
await cast(Awaitable[Any], persist_current_cache())
|
||||
|
||||
return {"success": True, "deleted_files": deleted_files}
|
||||
return {
|
||||
"success": True,
|
||||
"deleted_files": deleted_files,
|
||||
"batch_id": batch_id,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _extract_model_id_from_payload(payload: Any) -> Optional[int]:
|
||||
@@ -244,6 +267,9 @@ class ModelLifecycleService:
|
||||
item for item in cache.raw_data if item["file_path"] != file_path
|
||||
]
|
||||
await cache.resort()
|
||||
bump_cache_version = getattr(self._scanner, "bump_cache_version", None)
|
||||
if callable(bump_cache_version):
|
||||
bump_cache_version()
|
||||
|
||||
excluded = getattr(self._scanner, "_excluded_models", None)
|
||||
if isinstance(excluded, list):
|
||||
@@ -252,7 +278,7 @@ class ModelLifecycleService:
|
||||
|
||||
persist_current_cache = getattr(self._scanner, "_persist_current_cache", None)
|
||||
if callable(persist_current_cache):
|
||||
await persist_current_cache()
|
||||
await cast(Awaitable[Any], persist_current_cache())
|
||||
|
||||
message = f"Model {os.path.basename(file_path)} excluded"
|
||||
return {"success": True, "message": message}
|
||||
@@ -357,7 +383,8 @@ class ModelLifecycleService:
|
||||
|
||||
if os.path.exists(metadata_path):
|
||||
metadata = await self._metadata_loader(metadata_path)
|
||||
hash_value = metadata.get("sha256") if isinstance(metadata, dict) else None
|
||||
raw_hash = metadata.get("sha256") if isinstance(metadata, dict) else None
|
||||
hash_value = raw_hash if isinstance(raw_hash, str) else None
|
||||
|
||||
renamed_files: List[str] = []
|
||||
new_metadata_path: Optional[str] = None
|
||||
|
||||
@@ -10,7 +10,7 @@ from .errors import RateLimitError, ResourceNotFoundError
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
except ImportError as exc:
|
||||
BeautifulSoup = None # type: ignore[assignment]
|
||||
BeautifulSoup = None # pyright: ignore[reportAssignmentType]
|
||||
_BS4_IMPORT_ERROR = exc
|
||||
else:
|
||||
_BS4_IMPORT_ERROR = None
|
||||
@@ -18,7 +18,7 @@ else:
|
||||
try:
|
||||
import aiosqlite
|
||||
except ImportError as exc:
|
||||
aiosqlite = None # type: ignore[assignment]
|
||||
aiosqlite = None # pyright: ignore[reportAssignmentType]
|
||||
_AIOSQLITE_IMPORT_ERROR = exc
|
||||
else:
|
||||
_AIOSQLITE_IMPORT_ERROR = None
|
||||
@@ -105,24 +105,24 @@ class ModelMetadataProvider(ABC):
|
||||
"""Base abstract class for all model metadata providers"""
|
||||
|
||||
@abstractmethod
|
||||
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
"""Find model by hash value"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict]:
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get all versions of a model with their details"""
|
||||
pass
|
||||
|
||||
async def get_model_versions_bulk(
|
||||
self, model_ids: Sequence[int]
|
||||
) -> Optional[Dict[int, Dict]]:
|
||||
) -> Optional[Dict[int, Dict[str, Any]]]:
|
||||
"""Fetch model versions for multiple model ids when supported."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_model_versions_by_hashes(
|
||||
self, hashes: List[str]
|
||||
) -> Optional[List[Dict]]:
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
"""Fetch full version details for multiple SHA256 hashes.
|
||||
|
||||
Used specifically to retrieve ``usageControl`` which is only
|
||||
@@ -133,17 +133,17 @@ class ModelMetadataProvider(ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_model_version(self, model_id: int = None, version_id: int = None) -> Optional[Dict]:
|
||||
async def get_model_version(self, model_id: Optional[int] = None, version_id: Optional[int] = None) -> Optional[Dict[str, Any]]:
|
||||
"""Get specific model version with additional metadata"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
"""Fetch model version metadata"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict]:
|
||||
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
||||
"""Fetch one page of models owned by the specified user.
|
||||
|
||||
Returns ``{"items": [...], "nextCursor": <str|None>}`` on success,
|
||||
@@ -161,29 +161,29 @@ class CivitaiModelMetadataProvider(ModelMetadataProvider):
|
||||
def __init__(self, civitai_client):
|
||||
self.client = civitai_client
|
||||
|
||||
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
return await self.client.get_model_by_hash(model_hash)
|
||||
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict]:
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict[str, Any]]:
|
||||
return await self.client.get_model_versions(model_id)
|
||||
|
||||
async def get_model_versions_bulk(
|
||||
self, model_ids: Sequence[int]
|
||||
) -> Optional[Dict[int, Dict]]:
|
||||
) -> Optional[Dict[int, Dict[str, Any]]]:
|
||||
return await self.client.get_model_versions_bulk(model_ids)
|
||||
|
||||
async def get_model_versions_by_hashes(
|
||||
self, hashes: List[str]
|
||||
) -> Optional[List[Dict]]:
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
return await self.client.get_model_versions_by_hashes(hashes)
|
||||
|
||||
async def get_model_version(self, model_id: int = None, version_id: int = None) -> Optional[Dict]:
|
||||
async def get_model_version(self, model_id: Optional[int] = None, version_id: Optional[int] = None) -> Optional[Dict[str, Any]]:
|
||||
return await self.client.get_model_version(model_id, version_id)
|
||||
|
||||
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
return await self.client.get_model_version_info(version_id)
|
||||
|
||||
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict]:
|
||||
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
||||
return await self.client.get_user_models(username, cursor)
|
||||
|
||||
async def get_creator_model_count(self, username: str) -> Optional[int]:
|
||||
@@ -195,19 +195,19 @@ class CivArchiveModelMetadataProvider(ModelMetadataProvider):
|
||||
def __init__(self, civarchive_client):
|
||||
self.client = civarchive_client
|
||||
|
||||
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
return await self.client.get_model_by_hash(model_hash)
|
||||
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict]:
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict[str, Any]]:
|
||||
return await self.client.get_model_versions(model_id)
|
||||
|
||||
async def get_model_version(self, model_id: int = None, version_id: int = None) -> Optional[Dict]:
|
||||
async def get_model_version(self, model_id: Optional[int] = None, version_id: Optional[int] = None) -> Optional[Dict[str, Any]]:
|
||||
return await self.client.get_model_version(model_id, version_id)
|
||||
|
||||
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
return await self.client.get_model_version_info(version_id)
|
||||
|
||||
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict]:
|
||||
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
||||
"""Not supported by CivArchive provider"""
|
||||
return None
|
||||
|
||||
@@ -218,7 +218,7 @@ class SQLiteModelMetadataProvider(ModelMetadataProvider):
|
||||
self.db_path = db_path
|
||||
self._aiosqlite = _require_aiosqlite()
|
||||
|
||||
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
"""Find model by hash value from SQLite database"""
|
||||
async with self._aiosqlite.connect(self.db_path) as db:
|
||||
# Look up in model_files table to get model_id and version_id
|
||||
@@ -243,7 +243,7 @@ class SQLiteModelMetadataProvider(ModelMetadataProvider):
|
||||
result = await self._get_version_with_model_data(db, model_id, version_id)
|
||||
return result, None if result else "Error retrieving model data"
|
||||
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict]:
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get all versions of a model from SQLite database"""
|
||||
async with self._aiosqlite.connect(self.db_path) as db:
|
||||
db.row_factory = self._aiosqlite.Row
|
||||
@@ -299,7 +299,7 @@ class SQLiteModelMetadataProvider(ModelMetadataProvider):
|
||||
'name': model_name
|
||||
}
|
||||
|
||||
async def get_model_version(self, model_id: int = None, version_id: int = None) -> Optional[Dict]:
|
||||
async def get_model_version(self, model_id: Optional[int] = None, version_id: Optional[int] = None) -> Optional[Dict[str, Any]]:
|
||||
"""Get specific model version with additional metadata from SQLite database"""
|
||||
if not model_id and not version_id:
|
||||
return None
|
||||
@@ -339,7 +339,7 @@ class SQLiteModelMetadataProvider(ModelMetadataProvider):
|
||||
# Now we have both model_id and version_id, get the full data
|
||||
return await self._get_version_with_model_data(db, model_id, version_id)
|
||||
|
||||
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
"""Fetch model version metadata from SQLite database"""
|
||||
async with self._aiosqlite.connect(self.db_path) as db:
|
||||
db.row_factory = self._aiosqlite.Row
|
||||
@@ -358,11 +358,11 @@ class SQLiteModelMetadataProvider(ModelMetadataProvider):
|
||||
version_data = await self._get_version_with_model_data(db, model_id, version_id)
|
||||
return version_data, None
|
||||
|
||||
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict]:
|
||||
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
||||
"""Listing models by username is not supported for archive database"""
|
||||
return None
|
||||
|
||||
async def _get_version_with_model_data(self, db, model_id, version_id) -> Optional[Dict]:
|
||||
async def _get_version_with_model_data(self, db, model_id, version_id) -> Optional[Dict[str, Any]]:
|
||||
"""Helper to build version data with model information"""
|
||||
# Get version details
|
||||
version_query = "SELECT name, base_model, data FROM model_versions WHERE id = ? AND model_id = ?"
|
||||
@@ -485,7 +485,7 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
jitter_ratio=self._rate_limit_jitter_ratio,
|
||||
)
|
||||
|
||||
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
for provider, label in self._iter_providers():
|
||||
try:
|
||||
result, error = await self._call_with_rate_limit(
|
||||
@@ -507,7 +507,7 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
continue
|
||||
return None, "Model not found"
|
||||
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict]:
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict[str, Any]]:
|
||||
not_found_confirmed = False
|
||||
for provider, label in self._iter_providers():
|
||||
try:
|
||||
@@ -538,7 +538,7 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
continue
|
||||
return None
|
||||
|
||||
async def get_model_version(self, model_id: int = None, version_id: int = None) -> Optional[Dict]:
|
||||
async def get_model_version(self, model_id: Optional[int] = None, version_id: Optional[int] = None) -> Optional[Dict[str, Any]]:
|
||||
for provider, label in self._iter_providers():
|
||||
try:
|
||||
result = await self._call_with_rate_limit(
|
||||
@@ -561,7 +561,7 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
continue
|
||||
return None
|
||||
|
||||
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
for provider, label in self._iter_providers():
|
||||
try:
|
||||
result, error = await self._call_with_rate_limit(
|
||||
@@ -585,7 +585,7 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
|
||||
async def get_model_versions_by_hashes(
|
||||
self, hashes: List[str]
|
||||
) -> Optional[List[Dict]]:
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
for provider, label in self._iter_providers():
|
||||
try:
|
||||
result = await self._call_with_rate_limit(
|
||||
@@ -613,7 +613,7 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
continue
|
||||
return None
|
||||
|
||||
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict]:
|
||||
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
||||
for provider, label in self._iter_providers():
|
||||
try:
|
||||
result = await self._call_with_rate_limit(
|
||||
@@ -681,14 +681,14 @@ class RateLimitRetryingProvider(ModelMetadataProvider):
|
||||
def __getattr__(self, item):
|
||||
return getattr(self._provider, item)
|
||||
|
||||
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
return await self._rate_limit_helper.run(
|
||||
self._label,
|
||||
self._provider.get_model_by_hash,
|
||||
model_hash,
|
||||
)
|
||||
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict]:
|
||||
async def get_model_versions(self, model_id: str) -> Optional[Dict[str, Any]]:
|
||||
return await self._rate_limit_helper.run(
|
||||
self._label,
|
||||
self._provider.get_model_versions,
|
||||
@@ -698,7 +698,7 @@ class RateLimitRetryingProvider(ModelMetadataProvider):
|
||||
async def get_model_versions_bulk(
|
||||
self,
|
||||
model_ids: Sequence[int],
|
||||
) -> Optional[Dict[int, Dict]]:
|
||||
) -> Optional[Dict[int, Dict[str, Any]]]:
|
||||
return await self._rate_limit_helper.run(
|
||||
self._label,
|
||||
self._provider.get_model_versions_bulk,
|
||||
@@ -707,14 +707,14 @@ class RateLimitRetryingProvider(ModelMetadataProvider):
|
||||
|
||||
async def get_model_versions_by_hashes(
|
||||
self, hashes: List[str]
|
||||
) -> Optional[List[Dict]]:
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
return await self._rate_limit_helper.run(
|
||||
self._label,
|
||||
self._provider.get_model_versions_by_hashes,
|
||||
hashes,
|
||||
)
|
||||
|
||||
async def get_model_version(self, model_id: int = None, version_id: int = None) -> Optional[Dict]:
|
||||
async def get_model_version(self, model_id: Optional[int] = None, version_id: Optional[int] = None) -> Optional[Dict[str, Any]]:
|
||||
return await self._rate_limit_helper.run(
|
||||
self._label,
|
||||
self._provider.get_model_version,
|
||||
@@ -722,14 +722,14 @@ class RateLimitRetryingProvider(ModelMetadataProvider):
|
||||
version_id,
|
||||
)
|
||||
|
||||
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
return await self._rate_limit_helper.run(
|
||||
self._label,
|
||||
self._provider.get_model_version_info,
|
||||
version_id,
|
||||
)
|
||||
|
||||
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict]:
|
||||
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
||||
return await self._rate_limit_helper.run(
|
||||
self._label,
|
||||
self._provider.get_user_models,
|
||||
@@ -762,12 +762,12 @@ class ModelMetadataProviderManager:
|
||||
if is_default or self.default_provider is None:
|
||||
self.default_provider = name
|
||||
|
||||
async def get_model_by_hash(self, model_hash: str, provider_name: str = None) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
async def get_model_by_hash(self, model_hash: str, provider_name: Optional[str] = None) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
"""Find model by hash using specified or default provider"""
|
||||
provider = self._get_provider(provider_name)
|
||||
return await provider.get_model_by_hash(model_hash)
|
||||
|
||||
async def get_model_versions(self, model_id: str, provider_name: str = None) -> Optional[Dict]:
|
||||
async def get_model_versions(self, model_id: str, provider_name: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
||||
"""Get model versions using specified or default provider"""
|
||||
provider = self._get_provider(provider_name)
|
||||
return await provider.get_model_versions(model_id)
|
||||
@@ -775,8 +775,8 @@ class ModelMetadataProviderManager:
|
||||
async def get_model_versions_bulk(
|
||||
self,
|
||||
model_ids: Sequence[int],
|
||||
provider_name: str = None,
|
||||
) -> Optional[Dict[int, Dict]]:
|
||||
provider_name: Optional[str] = None,
|
||||
) -> Optional[Dict[int, Dict[str, Any]]]:
|
||||
"""Fetch model versions for multiple model ids when supported by provider."""
|
||||
provider = self._get_provider(provider_name)
|
||||
try:
|
||||
@@ -784,12 +784,12 @@ class ModelMetadataProviderManager:
|
||||
except NotImplementedError:
|
||||
return None
|
||||
|
||||
async def get_model_version(self, model_id: int = None, version_id: int = None, provider_name: str = None) -> Optional[Dict]:
|
||||
async def get_model_version(self, model_id: Optional[int] = None, version_id: Optional[int] = None, provider_name: Optional[str] = None) -> Optional[Dict[str, Any]]:
|
||||
"""Get specific model version using specified or default provider"""
|
||||
provider = self._get_provider(provider_name)
|
||||
return await provider.get_model_version(model_id, version_id)
|
||||
|
||||
async def get_model_version_info(self, version_id: str, provider_name: str = None) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
async def get_model_version_info(self, version_id: str, provider_name: Optional[str] = None) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
"""Fetch model version info using specified or default provider"""
|
||||
provider = self._get_provider(provider_name)
|
||||
return await provider.get_model_version_info(version_id)
|
||||
@@ -797,8 +797,8 @@ class ModelMetadataProviderManager:
|
||||
async def get_model_versions_by_hashes(
|
||||
self,
|
||||
hashes: List[str],
|
||||
provider_name: str = None,
|
||||
) -> Optional[List[Dict]]:
|
||||
provider_name: Optional[str] = None,
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
provider = self._get_provider(provider_name)
|
||||
try:
|
||||
return await provider.get_model_versions_by_hashes(hashes)
|
||||
@@ -808,19 +808,19 @@ class ModelMetadataProviderManager:
|
||||
async def get_user_models(
|
||||
self,
|
||||
username: str,
|
||||
provider_name: str = None,
|
||||
provider_name: Optional[str] = None,
|
||||
cursor: Optional[str] = None,
|
||||
) -> Optional[Dict]:
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Fetch one page of models owned by the specified user"""
|
||||
provider = self._get_provider(provider_name)
|
||||
return await provider.get_user_models(username, cursor)
|
||||
|
||||
async def get_creator_model_count(self, username: str, provider_name: str = None) -> Optional[int]:
|
||||
async def get_creator_model_count(self, username: str, provider_name: Optional[str] = None) -> Optional[int]:
|
||||
"""Best-effort published model count for the specified user"""
|
||||
provider = self._get_provider(provider_name)
|
||||
return await provider.get_creator_model_count(username)
|
||||
|
||||
def _get_provider(self, provider_name: str = None) -> ModelMetadataProvider:
|
||||
def _get_provider(self, provider_name: Optional[str] = None) -> ModelMetadataProvider:
|
||||
"""Get provider by name or default provider"""
|
||||
if provider_name:
|
||||
if provider_name not in self.providers:
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import (
|
||||
Tuple,
|
||||
Protocol,
|
||||
Callable,
|
||||
cast,
|
||||
)
|
||||
|
||||
from ..utils.constants import NSFW_LEVELS
|
||||
@@ -309,7 +310,7 @@ class ModelFilterSet:
|
||||
else:
|
||||
include_tags.add(normalized)
|
||||
else:
|
||||
include_tags = {tag.strip().lower() for tag in tag_filters if tag}
|
||||
include_tags = {tag.strip().lower() for tag in cast(Iterable[Any], tag_filters) if tag}
|
||||
|
||||
if include_tags:
|
||||
tag_logic = criteria.tag_logic.lower() if criteria.tag_logic else "any"
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user