mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-09-21 03:01:27 -03:00
Compare commits
161 Commits
a7d65fe84a
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 521531111a | |||
| 474da1b264 | |||
| 78d38b449e | |||
| 2bc9860b24 | |||
| 327da0465b | |||
| c8c84bfc54 | |||
| 3b9e8efb3d | |||
| d45a523fb5 | |||
| 6dc9f34f7d | |||
| 5adfa3be36 | |||
| d4b82d98b2 | |||
| 8c1c1691e3 | |||
| e14a084f0d | |||
| c55c6f0a41 | |||
| 7d963b27b5 | |||
| eba03800b9 | |||
| bf497d5144 | |||
| 369613f811 | |||
| 9eeebac40b | |||
| b9a516c9f8 | |||
| ef7fa7d3dd | |||
| 9c67dbbf15 | |||
| 16b0bdf70a | |||
| e09fe5888b | |||
| f67689b0f9 | |||
| 1d6da1787a | |||
| d572292142 | |||
| 1b1a8d63db | |||
| a0a5b13ab0 | |||
| 779bd18e75 | |||
| 01137eed88 | |||
| c6c44b741a | |||
| 5095b23eb2 | |||
| cc25bb3dc2 | |||
| 3b54a13cae | |||
| 9bbe57ee85 | |||
| 4938faa049 | |||
| cc8eedcff7 | |||
| 9734df15b4 | |||
| 2ceb1e2850 | |||
| 942717f0b6 | |||
| 0f160e157f | |||
| e9e9ee20c6 | |||
| f0ee30fc68 | |||
| 51de85a6ca | |||
| 4064ea7d3a | |||
| 35b291ab19 | |||
| e711e643f1 | |||
| db38ad80e6 | |||
| 326df32933 | |||
| 31ef9ffa06 | |||
| 38d4c59b4c | |||
| b9bf006998 | |||
| 5ab0e88abc | |||
| 84146b62fd | |||
| adeb40bfff | |||
| 8a21837ca2 | |||
| 3302147a43 | |||
| 4d87ae7637 | |||
| b1a653f18f | |||
| 6fe0543d2e | |||
| 931dfbe1d3 | |||
| b5c1331911 | |||
| 37f2cba72d | |||
| 0e789cb38c | |||
| f3b3393a16 | |||
| 480a3f4ea5 | |||
| 69a62d739c | |||
| 28fbb86dce | |||
| f88fe2665c | |||
| 3592eab48c | |||
| 1dbdf5b00c | |||
| fc3b2d7c13 | |||
| f2a7297cb9 | |||
| 57729375b6 | |||
| fa7ce725c1 | |||
| 27da7b3ca3 | |||
| 3070838a42 | |||
| fe160134d0 | |||
| 6d3f82976f | |||
| 91b2735dad | |||
| 3112869a21 | |||
| aa630bf85b | |||
| e0052cd237 | |||
| 3cdc5ba7a2 | |||
| 04485e384f | |||
| a03dc4002f | |||
| cc9d3bff42 | |||
| 2672b3331b | |||
| 4963bf2b2e | |||
| 51cad6f852 | |||
| 1b5cbbbaa0 | |||
| e747946f7a | |||
| 53fa22f39c | |||
| 82b34097fb | |||
| a7995db009 | |||
| 5ae4aef30e | |||
| 08023f0cd9 | |||
| 6e2185c182 | |||
| 41302e75ba | |||
| a17399d667 | |||
| e2d85a0a21 | |||
| 303833bbae | |||
| f86b7b55d6 | |||
| 782bb53784 | |||
| 139231e225 | |||
| 121d8d5cea | |||
| ec147bd677 | |||
| 93fc28b499 | |||
| 7afed1a14b | |||
| e6f5142e48 | |||
| 87f05fb66c | |||
| cf64e5baa8 | |||
| 634ea7f299 | |||
| 6ba64ebb3c | |||
| 03569c62df | |||
| a61840b366 | |||
| 726fc178f1 | |||
| 8260bd022d | |||
| b309becdf9 | |||
| 1e375bb8d9 | |||
| 14da8a6f17 | |||
| da71985c3e | |||
| 7c4c8b8f30 | |||
| 77109b3cf8 | |||
| 00095a5398 | |||
| 6b41c3bbb4 | |||
| b37238d790 | |||
| bc33e32c6f | |||
| 49704d801c | |||
| 34ca14d7fc | |||
| f7b247f9e8 | |||
| 3005d2877e | |||
| ed2a17970f | |||
| 9584fa85c9 | |||
| 1fd7cc0123 | |||
| 39e7c1376c | |||
| 2a3c632dc5 | |||
| 8d46d26abe | |||
| d761ac77f7 | |||
| c8b9db5bf4 | |||
| bce7d1d30c | |||
| bccd494a56 | |||
| 3fd29f6943 | |||
| 838a374a56 | |||
| 6e31da7a70 | |||
| fc9088bfd6 | |||
| 675421ea84 | |||
| 2ff98ae089 | |||
| c972c755fc | |||
| ebe3df7d22 | |||
| be44a75b74 | |||
| fd1227d3b8 | |||
| 3a9e02137d | |||
| d8a2be8edc | |||
| 1c46b2e8c3 | |||
| 3c3ac49f2f | |||
| 1a1be95a64 | |||
| 7a36659a20 | |||
| cb18281b14 | |||
| 856c9a87ac |
@@ -1,146 +0,0 @@
|
||||
---
|
||||
name: lora-manager-e2e
|
||||
description: "End-to-end testing and validation for LoRa Manager features. Use ONLY for sandboxed E2E validation of LoRa Manager standalone mode: start the standalone server on a free port with --settings-path, drive the web UI (http://127.0.0.1:{PORT}/loras) via Chrome DevTools MCP, and verify frontend-to-backend integration. NOT for UI behavior checks that unit tests (Vitest/jsdom) can cover. Trigger keywords: E2E, standalone, Chrome DevTools MCP, lora-manager-e2e, sandbox."
|
||||
---
|
||||
|
||||
# LoRa Manager E2E Testing
|
||||
|
||||
End-to-end testing of LoRa Manager standalone mode using Chrome DevTools MCP.
|
||||
|
||||
## When to Use — and When NOT To
|
||||
|
||||
E2E runs are slow and token-heavy. Reach for them only when the question genuinely
|
||||
spans server + browser (routing, scan persistence, websocket updates, EXIF writes).
|
||||
|
||||
- **Default to unit/component tests first**: `npm run test:js` (Vitest/jsdom) covers
|
||||
DOM rendering, modal behavior, event handling and API-client calls deterministically
|
||||
in seconds. Backend logic goes through `pytest`. A UI-behavior question answered by
|
||||
jsdom MUST NOT be escalated to E2E.
|
||||
- **Use E2E only when** the behavior cannot be observed without a live server and a
|
||||
real browser, e.g. template rendering through the aiohttp server, scanner → SQLite
|
||||
persistence → API → DOM round-trips, or real EXIF/image writes.
|
||||
- If you start an E2E and realize a unit test would answer the question, stop and
|
||||
switch.
|
||||
|
||||
**Browser driver is fixed: Chrome DevTools MCP.** Do not substitute kimi-webbridge —
|
||||
it operates on the user's real browser (real tabs, real sessions, synthetic
|
||||
`isTrusted=false` events), which breaks the isolation this skill requires and lacks
|
||||
the console/network inspection E2E debugging relies on. kimi-webbridge is for
|
||||
interactive browsing with the user's real login sessions, not for sandboxed E2E.
|
||||
|
||||
## Conventions
|
||||
|
||||
- **`{PORT}`**: default candidate `8188`, but it is **commonly occupied by a live
|
||||
ComfyUI** — always check first (`ss -tlnp | grep ':{PORT}'`) and use a free port
|
||||
(e.g. `8199`). Substitute the chosen port everywhere below. Never kill a process
|
||||
you did not start for this E2E.
|
||||
- **`<repo-root>`**: the repository/worktree root; run all commands from there.
|
||||
- **`<sandbox>`**: a throwaway dir, e.g. `/tmp/opencode/<plan>-e2e`.
|
||||
|
||||
## SANDBOX (MANDATORY)
|
||||
|
||||
> Every E2E run MUST target a throwaway sandbox, never real user data.
|
||||
|
||||
1. **Explicit settings directory**: always launch with `--settings-path <sandbox>/settings`.
|
||||
This pins ALL runtime data (`settings.json`, `cache/`, `backups/`, `logs/`, `stats/`,
|
||||
`wildcards/`) under the sandbox. **Never** create `<repo-root>/settings.json` — the repo
|
||||
folder is usually the real ComfyUI plugin folder and a portable settings file there is
|
||||
read by the real instance.
|
||||
2. **Sandboxed library paths**: point `folder_paths` / `recipes_path` /
|
||||
`example_images_path` at disposable dirs under `<sandbox>` — never the real library,
|
||||
real recipe dir, or real settings:
|
||||
|
||||
```json
|
||||
{
|
||||
"folder_paths": {
|
||||
"loras": ["<sandbox>/models/loras"],
|
||||
"checkpoints": ["<sandbox>/models/checkpoints"],
|
||||
"unet": ["<sandbox>/models/checkpoints"],
|
||||
"diffusers": []
|
||||
},
|
||||
"recipes_path": "<sandbox>/recipes",
|
||||
"example_images_path": "<sandbox>/example_images"
|
||||
}
|
||||
```
|
||||
|
||||
3. **Real-data protection proof**: before starting and after finishing, snapshot the real
|
||||
config and recipe library and confirm they are byte-identical; also confirm
|
||||
`<repo-root>` gained no `settings.json` or `cache/`:
|
||||
|
||||
```bash
|
||||
sha256sum ~/.config/ComfyUI-LoRA-Manager/settings.json > <sandbox>/settings.before.sha256
|
||||
ls ~/models/recipes/*.recipe.json 2>/dev/null | wc -l > <sandbox>/recipes-count.before.txt
|
||||
# AFTER the run: record again and diff. Any change = the run leaked into real data.
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
cd <repo-root>
|
||||
# 1. Sandbox
|
||||
mkdir -p <sandbox>/settings <sandbox>/models/{loras,checkpoints} <sandbox>/{recipes,example_images}
|
||||
# write <sandbox>/settings/settings.json per the SANDBOX example
|
||||
# 2. Port
|
||||
ss -tlnp | grep ':{PORT}' || echo "port {PORT} is free"
|
||||
# 3. Server — MUST be fully detached (a plain background & dies with the shell);
|
||||
# the helper enforces this and manages its own pidfile
|
||||
python .agents/skills/lora-manager-e2e/scripts/start_server.py \
|
||||
--port {PORT} --settings-path <sandbox>/settings --wait --timeout 30 --detach
|
||||
ss -tlnp | grep ':{PORT}' # verify listening BEFORE proceeding
|
||||
# 4. Chrome with remote debugging, then connect Chrome DevTools MCP (verify via list_pages)
|
||||
google-chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:{PORT}/loras
|
||||
```
|
||||
|
||||
Then drive the UI with the MCP tools (`take_snapshot`, `click`, `fill`, `fill_form`,
|
||||
`evaluate_script`, `wait_for`, `list_network_requests`, `list_console_messages`) —
|
||||
see [references/mcp-cheatsheet.md](references/mcp-cheatsheet.md) for patterns.
|
||||
|
||||
Server restart after config/fixture changes:
|
||||
|
||||
```bash
|
||||
python .agents/skills/lora-manager-e2e/scripts/start_server.py \
|
||||
--port {PORT} --settings-path <sandbox>/settings --restart --wait --detach
|
||||
# then reload the browser page (ignoreCache=True)
|
||||
```
|
||||
|
||||
`--restart` only kills the E2E server the script itself started (via its pidfile) and
|
||||
aborts instead of killing unrelated processes on the port.
|
||||
|
||||
## Abort Rule
|
||||
|
||||
A sandboxed E2E should finish in well under 30 minutes. If any phase exceeds ~2x its
|
||||
expected duration (server readiness > 60 s, MCP connect > 2 min, a single scenario >
|
||||
10 min), or any single tool call fails 3+ times in a row, **STOP** — do not retry
|
||||
blindly. Report `BLOCKED` with the phase, last observed state (server PID,
|
||||
`ss -tlnp` output, page snapshot, last API response) and suspected cause. A clean
|
||||
BLOCKED report beats an hour of retries.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- **"browser is already running" / `list_pages` fails**: a stale Chrome holds the
|
||||
profile dir. Find it (`ps -ef | grep -i '[c]hrome.*user-data-dir'`), confirm it is a
|
||||
leftover QA Chrome (not the live ComfyUI, not your current MCP browser), kill only
|
||||
that PID, then retry `list_pages`.
|
||||
- **MCP refuses to write screenshots into the worktree**: save to `/tmp` via
|
||||
`take_screenshot(filePath="/tmp/...")` and copy into the evidence dir from the shell.
|
||||
|
||||
## Cleanup
|
||||
|
||||
1. Stop the standalone server: `kill <recorded-pid>` (only the PID you started), then
|
||||
confirm `ss -tlnp | grep ':{PORT}'` is empty.
|
||||
2. Close browser pages (keep at least one open).
|
||||
3. `rm -rf <sandbox>`; verify `<repo-root>` gained no `settings.json` or `cache/`.
|
||||
4. Re-run the real-data protection check from the SANDBOX section and record the result.
|
||||
|
||||
## References & Scripts
|
||||
|
||||
- [references/mcp-cheatsheet.md](references/mcp-cheatsheet.md) — Chrome DevTools MCP
|
||||
command patterns (navigation, waiting, snapshots, forms, network, console, performance).
|
||||
- [references/test-scenarios.md](references/test-scenarios.md) — detailed test scenarios
|
||||
(list display, metadata editing, recipes, settings, import/export).
|
||||
- [references/recipe-rematch-fixtures.md](references/recipe-rematch-fixtures.md) —
|
||||
fixture format, fresh-state reset and known gaps for recipe rematch/repair E2E runs.
|
||||
- `scripts/start_server.py` — start/restart the standalone server
|
||||
(`--port --settings-path --restart --wait --timeout --detach`); refuses to touch
|
||||
unrelated processes on the port.
|
||||
- `scripts/wait_for_server.py` — poll readiness (`--port --timeout`).
|
||||
@@ -1,360 +0,0 @@
|
||||
# Chrome DevTools MCP Cheatsheet for LoRa Manager
|
||||
|
||||
Quick reference for common MCP commands used in LoRa Manager E2E testing.
|
||||
|
||||
> **Port convention**: `{PORT}` is the port chosen for the E2E run (default candidate `8188`, but only if actually free — see the SKILL.md Port Selection section; use e.g. `8199` when `8188` is occupied by a live ComfyUI). Always run against the **sandboxed** standalone server, never a live instance.
|
||||
|
||||
## Navigation
|
||||
|
||||
```python
|
||||
# Navigate to LoRA list page
|
||||
navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
||||
|
||||
# Reload page with cache clear
|
||||
navigate_page(type="reload", ignoreCache=True)
|
||||
|
||||
# Go back/forward
|
||||
navigate_page(type="back")
|
||||
navigate_page(type="forward")
|
||||
```
|
||||
|
||||
## Waiting
|
||||
|
||||
```python
|
||||
# Wait for text to appear
|
||||
wait_for(text="LoRAs", timeout=10000)
|
||||
|
||||
# Wait for specific element (via evaluate_script)
|
||||
evaluate_script(function="""
|
||||
() => {
|
||||
return new Promise((resolve) => {
|
||||
const check = () => {
|
||||
if (document.querySelector('.lora-card')) {
|
||||
resolve(true);
|
||||
} else {
|
||||
setTimeout(check, 100);
|
||||
}
|
||||
};
|
||||
check();
|
||||
});
|
||||
}
|
||||
""")
|
||||
```
|
||||
|
||||
## Taking Snapshots
|
||||
|
||||
```python
|
||||
# Full page snapshot
|
||||
snapshot = take_snapshot()
|
||||
|
||||
# Verbose snapshot (more details)
|
||||
snapshot = take_snapshot(verbose=True)
|
||||
|
||||
# Save to file
|
||||
take_snapshot(filePath="test-snapshots/page-load.json")
|
||||
```
|
||||
|
||||
## Element Interaction
|
||||
|
||||
```python
|
||||
# Click element
|
||||
click(uid="element-uid-from-snapshot")
|
||||
|
||||
# Double click
|
||||
click(uid="element-uid", dblClick=True)
|
||||
|
||||
# Fill input
|
||||
fill(uid="search-input", value="test query")
|
||||
|
||||
# Fill multiple inputs
|
||||
fill_form(elements=[
|
||||
{"uid": "input-1", "value": "value 1"},
|
||||
{"uid": "input-2", "value": "value 2"},
|
||||
])
|
||||
|
||||
# Hover
|
||||
hover(uid="lora-card-1")
|
||||
|
||||
# Upload file
|
||||
upload_file(uid="file-input", filePath="/path/to/file.safetensors")
|
||||
```
|
||||
|
||||
## Keyboard Input
|
||||
|
||||
```python
|
||||
# Press key
|
||||
press_key(key="Enter")
|
||||
press_key(key="Escape")
|
||||
press_key(key="Tab")
|
||||
|
||||
# Keyboard shortcuts
|
||||
press_key(key="Control+A") # Select all
|
||||
press_key(key="Control+F") # Find
|
||||
```
|
||||
|
||||
## JavaScript Evaluation
|
||||
|
||||
```python
|
||||
# Simple evaluation
|
||||
result = evaluate_script(function="() => document.title")
|
||||
|
||||
# Async evaluation
|
||||
result = evaluate_script(function="""
|
||||
async () => {
|
||||
const response = await fetch('/loras/api/list');
|
||||
return await response.json();
|
||||
}
|
||||
""")
|
||||
|
||||
# Check element existence
|
||||
exists = evaluate_script(function="""
|
||||
() => document.querySelector('.lora-card') !== null
|
||||
""")
|
||||
|
||||
# Get element count
|
||||
count = evaluate_script(function="""
|
||||
() => document.querySelectorAll('.lora-card').length
|
||||
""")
|
||||
```
|
||||
|
||||
## Network Monitoring
|
||||
|
||||
```python
|
||||
# List all network requests
|
||||
requests = list_network_requests()
|
||||
|
||||
# Filter by resource type
|
||||
xhr_requests = list_network_requests(resourceTypes=["xhr", "fetch"])
|
||||
|
||||
# Get specific request details
|
||||
details = get_network_request(reqid=123)
|
||||
|
||||
# Include preserved requests from previous navigations
|
||||
all_requests = list_network_requests(includePreservedRequests=True)
|
||||
```
|
||||
|
||||
## Console Monitoring
|
||||
|
||||
```python
|
||||
# List all console messages
|
||||
messages = list_console_messages()
|
||||
|
||||
# Filter by type
|
||||
errors = list_console_messages(types=["error", "warn"])
|
||||
|
||||
# Include preserved messages
|
||||
all_messages = list_console_messages(includePreservedMessages=True)
|
||||
|
||||
# Get specific message
|
||||
details = get_console_message(msgid=1)
|
||||
```
|
||||
|
||||
## Performance Testing
|
||||
|
||||
```python
|
||||
# Start trace with page reload
|
||||
performance_start_trace(reload=True, autoStop=False)
|
||||
|
||||
# Start trace without reload
|
||||
performance_start_trace(reload=False, autoStop=True, filePath="trace.json.gz")
|
||||
|
||||
# Stop trace
|
||||
results = performance_stop_trace()
|
||||
|
||||
# Stop and save
|
||||
performance_stop_trace(filePath="trace-results.json.gz")
|
||||
|
||||
# Analyze specific insight
|
||||
insight = performance_analyze_insight(
|
||||
insightSetId="results.insightSets[0].id",
|
||||
insightName="LCPBreakdown"
|
||||
)
|
||||
```
|
||||
|
||||
## Page Management
|
||||
|
||||
```python
|
||||
# List open pages
|
||||
pages = list_pages()
|
||||
|
||||
# Select a page
|
||||
select_page(pageId=0, bringToFront=True)
|
||||
|
||||
# Create new page
|
||||
new_page(url="http://127.0.0.1:{PORT}/loras")
|
||||
|
||||
# Close page (keep at least one open!)
|
||||
close_page(pageId=1)
|
||||
|
||||
# Resize page
|
||||
resize_page(width=1920, height=1080)
|
||||
```
|
||||
|
||||
## Screenshots
|
||||
|
||||
```python
|
||||
# Full page screenshot
|
||||
take_screenshot(fullPage=True)
|
||||
|
||||
# Viewport screenshot
|
||||
take_screenshot()
|
||||
|
||||
# Element screenshot
|
||||
take_screenshot(uid="lora-card-1")
|
||||
|
||||
# Save to file
|
||||
take_screenshot(filePath="screenshots/page.png", format="png")
|
||||
|
||||
# JPEG with quality
|
||||
take_screenshot(filePath="screenshots/page.jpg", format="jpeg", quality=90)
|
||||
```
|
||||
|
||||
## Dialog Handling
|
||||
|
||||
```python
|
||||
# Accept dialog
|
||||
handle_dialog(action="accept")
|
||||
|
||||
# Accept with text input
|
||||
handle_dialog(action="accept", promptText="user input")
|
||||
|
||||
# Dismiss dialog
|
||||
handle_dialog(action="dismiss")
|
||||
```
|
||||
|
||||
## Device Emulation
|
||||
|
||||
```python
|
||||
# Mobile viewport
|
||||
emulate(viewport={"width": 375, "height": 667, "isMobile": True, "hasTouch": True})
|
||||
|
||||
# Tablet viewport
|
||||
emulate(viewport={"width": 768, "height": 1024, "isMobile": True, "hasTouch": True})
|
||||
|
||||
# Desktop viewport
|
||||
emulate(viewport={"width": 1920, "height": 1080})
|
||||
|
||||
# Network throttling
|
||||
emulate(networkConditions="Slow 3G")
|
||||
emulate(networkConditions="Fast 4G")
|
||||
|
||||
# CPU throttling
|
||||
emulate(cpuThrottlingRate=4) # 4x slowdown
|
||||
|
||||
# Geolocation
|
||||
emulate(geolocation={"latitude": 37.7749, "longitude": -122.4194})
|
||||
|
||||
# User agent
|
||||
emulate(userAgent="Mozilla/5.0 (Custom)")
|
||||
|
||||
# Reset emulation
|
||||
emulate(viewport=None, networkConditions="No emulation", userAgent=None)
|
||||
```
|
||||
|
||||
## Drag and Drop
|
||||
|
||||
```python
|
||||
# Drag element to another
|
||||
drag(from_uid="draggable-item", to_uid="drop-zone")
|
||||
```
|
||||
|
||||
## Common LoRa Manager Test Patterns
|
||||
|
||||
### Verify LoRA Cards Loaded
|
||||
|
||||
```python
|
||||
navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
||||
wait_for(text="LoRAs", timeout=10000)
|
||||
|
||||
# Check if cards loaded
|
||||
result = evaluate_script(function="""
|
||||
() => {
|
||||
const cards = document.querySelectorAll('.lora-card');
|
||||
return {
|
||||
count: cards.length,
|
||||
hasData: cards.length > 0
|
||||
};
|
||||
}
|
||||
""")
|
||||
```
|
||||
|
||||
### Search and Verify Results
|
||||
|
||||
```python
|
||||
fill(uid="search-input", value="character")
|
||||
press_key(key="Enter")
|
||||
wait_for(timeout=2000) # Wait for debounce
|
||||
|
||||
# Check results
|
||||
result = evaluate_script(function="""
|
||||
() => {
|
||||
const cards = document.querySelectorAll('.lora-card');
|
||||
const names = Array.from(cards).map(c => c.dataset.name || c.textContent);
|
||||
return { count: cards.length, names };
|
||||
}
|
||||
""")
|
||||
```
|
||||
|
||||
### Check API Response
|
||||
|
||||
```python
|
||||
# Trigger API call
|
||||
evaluate_script(function="""
|
||||
() => window.loraApiCallPromise = fetch('/loras/api/list').then(r => r.json())
|
||||
""")
|
||||
|
||||
# Wait and get result
|
||||
import time
|
||||
time.sleep(1)
|
||||
|
||||
result = evaluate_script(function="""
|
||||
async () => await window.loraApiCallPromise
|
||||
""")
|
||||
```
|
||||
|
||||
### Monitor Console for Errors
|
||||
|
||||
```python
|
||||
# Before test: clear console (navigate reloads)
|
||||
navigate_page(type="reload")
|
||||
|
||||
# ... perform actions ...
|
||||
|
||||
# Check for errors
|
||||
errors = list_console_messages(types=["error"])
|
||||
assert len(errors) == 0, f"Console errors: {errors}"
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Stale profile lock ("browser is already running" / `list_pages` fails)
|
||||
|
||||
A Chrome profile held by a stale Chrome from a prior MCP session makes `list_pages`
|
||||
fail with "browser is already running". Fix:
|
||||
|
||||
1. Find the stale Chrome that owns the profile dir (e.g. `~/.config/chrome-dev-profile`):
|
||||
```bash
|
||||
ps -ef | grep -i '[c]hrome.*user-data-dir'
|
||||
```
|
||||
2. Confirm it is a QA Chrome from a completed task (NOT the live ComfyUI server, NOT
|
||||
your current MCP instance).
|
||||
3. Kill ONLY that stale Chrome (`kill <stale-pid>`), then retry `list_pages`.
|
||||
|
||||
### Screenshot-write restrictions
|
||||
|
||||
The MCP may refuse to write into paths outside its configured workspace roots
|
||||
(e.g. `.omo/evidence/screenshots/` under a worktree that canonicalizes to an unmapped
|
||||
path). Save the screenshot to `/tmp` via the MCP, then copy it into the evidence dir:
|
||||
|
||||
```bash
|
||||
# MCP: take_screenshot(filePath="/tmp/<plan>-e2e/recipe-b-after.png", format="png")
|
||||
# Shell:
|
||||
mkdir -p <repo-root>/.omo/evidence/screenshots
|
||||
cp /tmp/<plan>-e2e/recipe-b-after.png <repo-root>/.omo/evidence/screenshots/
|
||||
```
|
||||
|
||||
### Time budgets & abort rule
|
||||
|
||||
See SKILL.md "Time Budgets & Abort Guidance": if a phase exceeds ~2x its budget or a
|
||||
tool call retries 3+ times in a row, STOP and report BLOCKED with the last observed
|
||||
state (server PID + `ss -tlnp`, page snapshot, last API response). Do not loop.
|
||||
@@ -1,72 +0,0 @@
|
||||
# Recipe Rematch/Repair E2E — Fixtures, Fresh State, Known Gaps
|
||||
|
||||
Specialized guidance for recipe rematch/repair E2E runs, extracted from the SKILL.md
|
||||
main flow. Read the SKILL.md SANDBOX section first — everything here assumes a
|
||||
sandboxed run.
|
||||
|
||||
## Fixture Rules (validated by the task-8 E2E)
|
||||
|
||||
Seed the **sandboxed** `recipes_path` with hand-written fixture recipes:
|
||||
|
||||
1. **Filename constraint**: each file MUST be named `f"{id}.recipe.json"` **and** the
|
||||
in-JSON `id` field MUST equal the filename. Discovery accepts any `*.recipe.json`,
|
||||
but persistence resolves the path via `get_recipe_json_path` and
|
||||
`_save_recipe_persistently` returns `False` on a mismatch → the fixture would be
|
||||
counted as an error.
|
||||
- `recipe-a.recipe.json` → in-JSON `"id": "recipe-a"`
|
||||
2. **File format**: mirror an existing recipe JSON — top-level `id`, `file_path`,
|
||||
`title`, `loras`, `fingerprint`, `gen_params`; lora entries per the persistence
|
||||
conventions (`hash`, `file_name`, `modelVersionId`, `isDeleted`, ...).
|
||||
3. **Companion image**: each recipe needs an image (e.g. a `.webp` generated with PIL)
|
||||
referenced by `file_path`, used for EXIF verification
|
||||
(`ExifUtils.append_recipe_metadata` writes a `"Recipe metadata: ..."` marker; a
|
||||
freshly generated `.webp` with no marker is the clean "untouched" control).
|
||||
4. **autov3 three-state contract**: for L3 (autov3-only, renamed-file) fixtures the
|
||||
local model's `.metadata.json` sidecar MUST have the `autov3` key **ABSENT** (the
|
||||
"unchecked" state), NOT `""` — `""` is the TERMINAL "checked but unavailable" state
|
||||
that L3 deliberately skips. The scanner computes + persists `autov3` from the file
|
||||
header during the normal library scan (`model_scanner.py` `_process_model_file`), so
|
||||
the live L3 match resolves through the local autov3/hash cache; the
|
||||
computed-autov3 branch for unchecked items is covered by the unit suite.
|
||||
5. **Fixture design for a rematch run** (mirrors the task-8 E2E):
|
||||
- `recipe-a`: lora entry `isDeleted=True`, `hash` = 12-char autov3 computed from the
|
||||
local model (`calculate_autov3`, `py/utils/file_utils.py`), whose local model file
|
||||
was RENAMED after the recipe was written so `file_name` differs (proves L3 match
|
||||
without filename).
|
||||
- `recipe-b`: parser-convention checkpoint entry (uses `id`, no `modelVersionId`)
|
||||
matching a local checkpoint via L2 — the local checkpoint's `.metadata.json` MUST
|
||||
carry civitai version data with that `id` so `version_index` contains it (L2
|
||||
cannot match otherwise).
|
||||
- `recipe-c`: healthy recipe (no deleted entries) → must remain untouched.
|
||||
|
||||
The scanner computes and persists model hashes during the library scan, so the sandbox
|
||||
model dirs just need the model files + `.metadata.json` sidecars. With
|
||||
`--settings-path`, all derived data lands under the sandbox settings dir (`cache/`,
|
||||
`backups/`, `logs/`, `stats/`, `wildcards/`), and NO `cache/` appears in the repo root.
|
||||
|
||||
## Fresh State Between Entry-Point Runs
|
||||
|
||||
Each entry point (global / per-recipe / selection-bulk) must start from the same
|
||||
deleted state. Between runs (keep a pristine copy in `<sandbox>/recipes-before/`):
|
||||
|
||||
```bash
|
||||
# 1. Reset fixtures to the before-state snapshot
|
||||
cp <sandbox>/recipes-before/*.recipe.json <sandbox>/recipes/
|
||||
# 2. Clear the recipe/FTS caches (with --settings-path these live under the sandbox
|
||||
# settings dir, NOT <repo-root>/cache)
|
||||
rm -f <sandbox>/settings/cache/recipe/*.sqlite
|
||||
rm -rf <sandbox>/settings/cache/fts/*
|
||||
# 3. Restart the server (fresh process, fresh scan)
|
||||
python .agents/skills/lora-manager-e2e/scripts/start_server.py \
|
||||
--port {PORT} --settings-path <sandbox>/settings --restart --wait --timeout 30 --detach
|
||||
# 4. Re-verify the server is listening + reload the browser page
|
||||
```
|
||||
|
||||
## Cancellation Testing (KNOWN GAP)
|
||||
|
||||
Testing the rematch-cancel path E2E requires a run long enough to cancel mid-flight. A
|
||||
tiny 3-recipe fixture set completes in **seconds** — too fast to reliably cancel. The
|
||||
cancel path is currently **unit-covered only** (`rematch_all_recipes` cancellation
|
||||
tests); do not block an E2E run on cancel-path verification. If you must attempt it,
|
||||
you would need an artificially large/deferred fixture set to create a cancellable
|
||||
window — treat this as a research task, not part of the standard E2E.
|
||||
@@ -1,280 +0,0 @@
|
||||
# LoRa Manager E2E Test Scenarios
|
||||
|
||||
This document provides detailed test scenarios for end-to-end validation of LoRa Manager features.
|
||||
|
||||
> **Run preconditions (from SKILL.md)**: every run uses the **sandboxed** standalone
|
||||
> server on a free port `{PORT}` (default candidate `8188`, only if actually free — pick
|
||||
> e.g. `8199` when `8188` is occupied by a live ComfyUI). Fixtures live in the sandboxed
|
||||
> `recipes_path` as `f"{id}.recipe.json"` files with matching in-JSON `id`; the real user
|
||||
> config and real library are never touched (record protection proof before/after).
|
||||
> Abort if a phase exceeds ~2x its budget or a tool call retries 3+ times (SKILL.md
|
||||
> "Time Budgets & Abort Guidance").
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [LoRA List Page](#lora-list-page)
|
||||
2. [Model Details](#model-details)
|
||||
3. [Recipes](#recipes)
|
||||
4. [Settings](#settings)
|
||||
5. [Import/Export](#importexport)
|
||||
|
||||
---
|
||||
|
||||
## LoRA List Page
|
||||
|
||||
### Scenario: Page Load and Display
|
||||
|
||||
**Objective**: Verify the LoRA list page loads correctly and displays models.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to `http://127.0.0.1:{PORT}/loras`
|
||||
2. Wait for page title "LoRAs" to appear
|
||||
3. Take snapshot to verify:
|
||||
- Header with "LoRAs" title is visible
|
||||
- Search/filter controls are present
|
||||
- Grid/list view toggle exists
|
||||
- LoRA cards are displayed (if models exist)
|
||||
- Pagination controls (if applicable)
|
||||
|
||||
**Expected Result**: Page loads without errors, UI elements are present.
|
||||
|
||||
### Scenario: Search Functionality
|
||||
|
||||
**Objective**: Verify search filters LoRA models correctly.
|
||||
|
||||
**Steps**:
|
||||
1. Ensure at least one LoRA exists with known name (e.g., "test-character")
|
||||
2. Navigate to LoRA list page
|
||||
3. Enter search term in search box: "test"
|
||||
4. Press Enter or click search button
|
||||
5. Wait for results to update
|
||||
|
||||
**Expected Result**: Only LoRAs matching search term are displayed.
|
||||
|
||||
**Verification Script**:
|
||||
```python
|
||||
# After search, verify filtered results
|
||||
evaluate_script(function="""
|
||||
() => {
|
||||
const cards = document.querySelectorAll('.lora-card');
|
||||
const names = Array.from(cards).map(c => c.dataset.name);
|
||||
return { count: cards.length, names };
|
||||
}
|
||||
""")
|
||||
```
|
||||
|
||||
### Scenario: Filter by Tags
|
||||
|
||||
**Objective**: Verify tag filtering works correctly.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to LoRA list page
|
||||
2. Click on a tag (e.g., "character", "style")
|
||||
3. Wait for filtered results
|
||||
|
||||
**Expected Result**: Only LoRAs with selected tag are displayed.
|
||||
|
||||
### Scenario: View Mode Toggle
|
||||
|
||||
**Objective**: Verify grid/list view toggle works.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to LoRA list page
|
||||
2. Click list view button
|
||||
3. Verify list layout
|
||||
4. Click grid view button
|
||||
5. Verify grid layout
|
||||
|
||||
**Expected Result**: View mode changes correctly, layout updates.
|
||||
|
||||
---
|
||||
|
||||
## Model Details
|
||||
|
||||
### Scenario: Open Model Details
|
||||
|
||||
**Objective**: Verify clicking a LoRA opens its details.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to LoRA list page
|
||||
2. Click on a LoRA card
|
||||
3. Wait for details panel/modal to open
|
||||
|
||||
**Expected Result**: Details panel shows:
|
||||
- Model name
|
||||
- Preview image
|
||||
- Metadata (trigger words, tags, etc.)
|
||||
- Action buttons (edit, delete, etc.)
|
||||
|
||||
### Scenario: Edit Model Metadata
|
||||
|
||||
**Objective**: Verify metadata editing works end-to-end.
|
||||
|
||||
**Steps**:
|
||||
1. Open a LoRA's details
|
||||
2. Click "Edit" button
|
||||
3. Modify trigger words field
|
||||
4. Add/remove tags
|
||||
5. Save changes
|
||||
6. Refresh page
|
||||
7. Reopen the same LoRA
|
||||
|
||||
**Expected Result**: Changes persist after refresh.
|
||||
|
||||
### Scenario: Delete Model
|
||||
|
||||
**Objective**: Verify model deletion works.
|
||||
|
||||
**Steps**:
|
||||
1. Open a LoRA's details
|
||||
2. Click "Delete" button
|
||||
3. Confirm deletion in dialog
|
||||
4. Wait for removal
|
||||
|
||||
**Expected Result**: Model removed from list, success message shown.
|
||||
|
||||
---
|
||||
|
||||
## Recipes
|
||||
|
||||
### Scenario: Recipe List Display
|
||||
|
||||
**Objective**: Verify recipes page loads and displays recipes.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to `http://127.0.0.1:{PORT}/recipes`
|
||||
2. Wait for "Recipes" title
|
||||
3. Take snapshot
|
||||
|
||||
**Expected Result**: Recipe list displayed with cards/items.
|
||||
|
||||
### Scenario: Create New Recipe
|
||||
|
||||
**Objective**: Verify recipe creation workflow.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to recipes page
|
||||
2. Click "New Recipe" button
|
||||
3. Fill recipe form:
|
||||
- Name: "Test Recipe"
|
||||
- Description: "E2E test recipe"
|
||||
- Add LoRA models
|
||||
4. Save recipe
|
||||
5. Verify recipe appears in list
|
||||
|
||||
**Expected Result**: New recipe created and displayed.
|
||||
|
||||
### Scenario: Apply Recipe
|
||||
|
||||
**Objective**: Verify applying a recipe to ComfyUI.
|
||||
|
||||
**Steps**:
|
||||
1. Open a recipe
|
||||
2. Click "Apply" or "Load in ComfyUI"
|
||||
3. Verify action completes
|
||||
|
||||
**Expected Result**: Recipe applied successfully.
|
||||
|
||||
---
|
||||
|
||||
## Settings
|
||||
|
||||
### Scenario: Settings Page Load
|
||||
|
||||
**Objective**: Verify settings page displays correctly.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to `http://127.0.0.1:{PORT}/settings`
|
||||
2. Wait for "Settings" title
|
||||
3. Take snapshot
|
||||
|
||||
**Expected Result**: Settings form with various options displayed.
|
||||
|
||||
### Scenario: Change Setting and Restart
|
||||
|
||||
**Objective**: Verify settings persist after restart.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to settings page
|
||||
2. Change a setting (e.g., default view mode)
|
||||
3. Save settings
|
||||
4. Restart server: `python scripts/start_server.py --port {PORT} --restart --wait --timeout 30 --detach`
|
||||
5. Refresh browser page
|
||||
6. Navigate to settings
|
||||
|
||||
**Expected Result**: Changed setting value persists.
|
||||
|
||||
---
|
||||
|
||||
## Import/Export
|
||||
|
||||
### Scenario: Export Models List
|
||||
|
||||
**Objective**: Verify export functionality.
|
||||
|
||||
**Steps**:
|
||||
1. Navigate to LoRA list
|
||||
2. Click "Export" button
|
||||
3. Select format (JSON/CSV)
|
||||
4. Download file
|
||||
|
||||
**Expected Result**: File downloaded with correct data.
|
||||
|
||||
### Scenario: Import Models
|
||||
|
||||
**Objective**: Verify import functionality.
|
||||
|
||||
**Steps**:
|
||||
1. Prepare import file
|
||||
2. Navigate to import page
|
||||
3. Upload file
|
||||
4. Verify import results
|
||||
|
||||
**Expected Result**: Models imported successfully, confirmation shown.
|
||||
|
||||
---
|
||||
|
||||
## API Integration Tests
|
||||
|
||||
### Scenario: Verify API Endpoints
|
||||
|
||||
**Objective**: Verify backend API responds correctly.
|
||||
|
||||
**Test via browser console**:
|
||||
```javascript
|
||||
// List LoRAs
|
||||
fetch('/loras/api/list').then(r => r.json()).then(console.log)
|
||||
|
||||
// Get LoRA details
|
||||
fetch('/loras/api/detail/<id>').then(r => r.json()).then(console.log)
|
||||
|
||||
// Search LoRAs
|
||||
fetch('/loras/api/search?q=test').then(r => r.json()).then(console.log)
|
||||
```
|
||||
|
||||
**Expected Result**: APIs return valid JSON with expected structure.
|
||||
|
||||
---
|
||||
|
||||
## Console Error Monitoring
|
||||
|
||||
During all tests, monitor browser console for errors:
|
||||
|
||||
```python
|
||||
# Check for JavaScript errors
|
||||
messages = list_console_messages(types=["error"])
|
||||
assert len(messages) == 0, f"Console errors found: {messages}"
|
||||
```
|
||||
|
||||
## Network Request Verification
|
||||
|
||||
Verify key API calls are made:
|
||||
|
||||
```python
|
||||
# List XHR requests
|
||||
requests = list_network_requests(resourceTypes=["xhr", "fetch"])
|
||||
|
||||
# Look for specific endpoints
|
||||
lora_list_requests = [r for r in requests if "/api/list" in r.get("url", "")]
|
||||
assert len(lora_list_requests) > 0, "LoRA list API not called"
|
||||
```
|
||||
@@ -1,215 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Example E2E test demonstrating LoRa Manager testing workflow.
|
||||
|
||||
This script shows how to:
|
||||
1. Start the standalone server
|
||||
2. Use Chrome DevTools MCP to interact with the UI
|
||||
3. Verify functionality end-to-end
|
||||
|
||||
Note: This is a template. Actual execution requires Chrome DevTools MCP.
|
||||
|
||||
Port: pick a FREE port for the run — 8188 is commonly occupied by a live
|
||||
ComfyUI (see the skill's Port Selection section). Set PORT below to e.g. 8199
|
||||
when 8188 is taken. Always run against a SANDBOXED standalone server.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
# Choose the E2E port. 8188 is only the default candidate; use 8199 (or any
|
||||
# free port checked with `ss -tlnp`) when 8188 is occupied by a live ComfyUI.
|
||||
PORT = "8188"
|
||||
|
||||
|
||||
def run_test():
|
||||
"""Run example E2E test flow."""
|
||||
|
||||
print("=" * 60)
|
||||
print("LoRa Manager E2E Test Example")
|
||||
print("=" * 60)
|
||||
|
||||
# Step 1: Start server (detached so it survives the shell)
|
||||
print("\n[1/5] Starting LoRa Manager standalone server...")
|
||||
result = subprocess.run(
|
||||
[sys.executable, "start_server.py", "--port", PORT, "--wait", "--timeout", "30", "--detach"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
print(f"Failed to start server: {result.stderr}")
|
||||
return 1
|
||||
print("Server ready!")
|
||||
|
||||
# Step 2: Open Chrome (manual step - show command)
|
||||
print("\n[2/5] Open Chrome with debug mode:")
|
||||
print(
|
||||
f"google-chrome --remote-debugging-port=9222 "
|
||||
f"--user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:{PORT}/loras"
|
||||
)
|
||||
print("(In actual test, this would be automated via MCP)")
|
||||
|
||||
# Step 3: Navigate and verify page load
|
||||
print("\n[3/5] Page Load Verification:")
|
||||
print(
|
||||
f"""
|
||||
MCP Commands to execute:
|
||||
1. navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
||||
2. wait_for(text="LoRAs", timeout=10000)
|
||||
3. snapshot = take_snapshot()
|
||||
"""
|
||||
)
|
||||
|
||||
# Step 4: Test search functionality
|
||||
print("\n[4/5] Search Functionality Test:")
|
||||
print(
|
||||
"""
|
||||
MCP Commands to execute:
|
||||
1. fill(uid="search-input", value="test")
|
||||
2. press_key(key="Enter")
|
||||
3. wait_for(text="Results", timeout=5000)
|
||||
4. result = evaluate_script(function=`
|
||||
() => {
|
||||
const cards = document.querySelectorAll('.lora-card');
|
||||
return { count: cards.length };
|
||||
}
|
||||
`)
|
||||
"""
|
||||
)
|
||||
|
||||
# Step 5: Verify API
|
||||
print("\n[5/5] API Verification:")
|
||||
print(
|
||||
"""
|
||||
MCP Commands to execute:
|
||||
1. api_result = evaluate_script(function=`
|
||||
async () => {
|
||||
const response = await fetch('/loras/api/list');
|
||||
const data = await response.json();
|
||||
return { count: data.length, status: response.status };
|
||||
}
|
||||
`)
|
||||
2. Verify api_result['status'] == 200
|
||||
"""
|
||||
)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Test flow completed!")
|
||||
print("=" * 60)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def example_restart_flow():
|
||||
"""Example: Testing configuration change that requires restart."""
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Example: Server Restart Flow")
|
||||
print("=" * 60)
|
||||
|
||||
print(
|
||||
f"""
|
||||
Scenario: Change setting and verify after restart
|
||||
|
||||
Steps:
|
||||
1. Navigate to settings page
|
||||
- navigate_page(type="url", url="http://127.0.0.1:{PORT}/settings")
|
||||
|
||||
2. Change a setting (e.g., theme)
|
||||
- fill(uid="theme-select", value="dark")
|
||||
- click(uid="save-settings-button")
|
||||
|
||||
3. Restart server
|
||||
- subprocess.run([python, "start_server.py", "--port", "{PORT}", "--restart", "--wait", "--detach"])
|
||||
|
||||
4. Refresh browser
|
||||
- navigate_page(type="reload", ignoreCache=True)
|
||||
- wait_for(text="LoRAs", timeout=15000)
|
||||
|
||||
5. Verify setting persisted
|
||||
- navigate_page(type="url", url="http://127.0.0.1:{PORT}/settings")
|
||||
- theme = evaluate_script(function="() => document.querySelector('#theme-select').value")
|
||||
- assert theme == "dark"
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def example_modal_interaction():
|
||||
"""Example: Testing modal dialog interaction."""
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Example: Modal Dialog Interaction")
|
||||
print("=" * 60)
|
||||
|
||||
print(
|
||||
"""
|
||||
Scenario: Add new LoRA via modal
|
||||
|
||||
Steps:
|
||||
1. Open modal
|
||||
- click(uid="add-lora-button")
|
||||
- wait_for(text="Add LoRA", timeout=3000)
|
||||
|
||||
2. Fill form
|
||||
- fill_form(elements=[
|
||||
{"uid": "lora-name", "value": "Test Character"},
|
||||
{"uid": "lora-path", "value": "/models/test.safetensors"},
|
||||
])
|
||||
|
||||
3. Submit
|
||||
- click(uid="modal-submit-button")
|
||||
|
||||
4. Verify success
|
||||
- wait_for(text="Successfully added", timeout=5000)
|
||||
- snapshot = take_snapshot()
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def example_network_monitoring():
|
||||
"""Example: Network request monitoring."""
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("Example: Network Request Monitoring")
|
||||
print("=" * 60)
|
||||
|
||||
print(
|
||||
f"""
|
||||
Scenario: Verify API calls during user interaction
|
||||
|
||||
Steps:
|
||||
1. Clear network log (implicit on navigation)
|
||||
- navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras")
|
||||
|
||||
2. Perform action that triggers API call
|
||||
- fill(uid="search-input", value="character")
|
||||
- press_key(key="Enter")
|
||||
|
||||
3. List network requests
|
||||
- requests = list_network_requests(resourceTypes=["xhr", "fetch"])
|
||||
|
||||
4. Find search API call
|
||||
- search_requests = [r for r in requests if "/api/search" in r.get("url", "")]
|
||||
- assert len(search_requests) > 0, "Search API was not called"
|
||||
|
||||
5. Get request details
|
||||
- if search_requests:
|
||||
details = get_network_request(reqid=search_requests[0]["reqid"])
|
||||
- Verify request method, response status, etc.
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("LoRa Manager E2E Test Examples\n")
|
||||
print("This script demonstrates E2E testing patterns.\n")
|
||||
print("Note: Actual execution requires Chrome DevTools MCP connection.\n")
|
||||
|
||||
run_test()
|
||||
example_restart_flow()
|
||||
example_modal_interaction()
|
||||
example_network_monitoring()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("All examples shown!")
|
||||
print("=" * 60)
|
||||
@@ -15,6 +15,7 @@ node_modules/
|
||||
coverage/
|
||||
.coverage
|
||||
model_cache/
|
||||
recipe_cache/
|
||||
|
||||
# agent / dev tooling
|
||||
.opencode/
|
||||
|
||||
@@ -72,6 +72,16 @@ python scripts/sync_translation_keys.py
|
||||
|
||||
Locale files are in `locales/` (en, zh-CN, zh-TW, ja, ko, fr, de, es, ru, he).
|
||||
|
||||
After adding keys to `en.json` and syncing, **stop**: the `[TODO: Translate]` placeholders in
|
||||
the other locales are the expected end state during feature development. Do NOT translate
|
||||
proactively — translate only when the feature owner explicitly asks (see
|
||||
`docs/i18n-translation-guidelines.md` §7).
|
||||
|
||||
**Before translating anything, read `docs/i18n-translation-guidelines.md`** — it defines the
|
||||
term conventions (e.g. "Recipe" stays untranslated in French, 配方 in Chinese; model-type and
|
||||
brand names are never translated), per-locale preferred renderings, placeholder rules, and
|
||||
the known confusion hot-spots.
|
||||
|
||||
## Code Style
|
||||
|
||||
### Python
|
||||
@@ -156,10 +166,14 @@ The system runs in two modes:
|
||||
|
||||
### Model Types & Routes
|
||||
|
||||
- API endpoints follow `/loras/*`, `/checkpoints/*`, `/embeddings/*` patterns
|
||||
- API endpoints follow `/loras/*`, `/checkpoints/*`, `/embeddings/*`, `/other/*` patterns
|
||||
- Route registrars organize endpoints by domain: `ModelRouteRegistrar`, `RecipeRouteRegistrar`, etc.
|
||||
- Request handlers in `py/routes/handlers/` implement route logic
|
||||
- All routes use aiohttp, return `web.json_response` or `web.Response`
|
||||
- Endpoints consumed by the companion browser extension (lm-civitai-extension)
|
||||
MUST also accept `GET` with query-string params: the extension is GET-only by
|
||||
convention (see its AGENTS.md), even for state-changing operations such as
|
||||
`GET /api/lm/recipe/{recipe_id}/reimport`
|
||||
|
||||
### Recipe System
|
||||
|
||||
@@ -176,6 +190,17 @@ The system runs in two modes:
|
||||
|
||||
- `py/config.py` manages folder paths for models and handles symlink mappings
|
||||
- Auto-saves paths to `settings.json` in ComfyUI mode
|
||||
- `settings.json.example` is intentionally minimal (see Important Notes); all
|
||||
other defaults live in `DEFAULT_SETTINGS` (`py/services/settings_manager.py`)
|
||||
- **`folder_paths` vs `extra_folder_paths` — different purposes, do not conflate:**
|
||||
- `folder_paths` (primary model roots): in ComfyUI plugin mode these come
|
||||
from the ComfyUI host; in standalone mode they are the ONLY source of
|
||||
model library paths and are currently edited by hand in `settings.json`.
|
||||
- `extra_folder_paths` is a **ComfyUI-plugin-mode feature**: paths visible
|
||||
ONLY to LoRA Manager, not to ComfyUI. Its motivation is that a very large
|
||||
model library slows ComfyUI itself down, while LoRA Manager handles large
|
||||
libraries without performance issues — so users keep ComfyUI's library
|
||||
small and add the bulk via `extra_folder_paths`.
|
||||
|
||||
### Frontend UI Architecture
|
||||
|
||||
@@ -205,6 +230,26 @@ The system runs in two modes:
|
||||
- Vanilla JS tests: `tests/frontend/**/*.test.js` with jsdom; setup in `tests/frontend/setup.js`
|
||||
- Vue widget tests: `vue-widgets/tests/**/*.test.ts` with jsdom + `@vue/test-utils`
|
||||
|
||||
### UI Verification (manual default)
|
||||
|
||||
UI/layout changes are verified by the user by eye — do NOT spin up a sandbox,
|
||||
standalone server, or browser automation to "prove" a visual fix. Ask the user to
|
||||
look instead. The full browser E2E ceremony (server + Chrome DevTools MCP +
|
||||
screenshots) is slow, token-heavy, and fragile; reserve it for genuine
|
||||
server+browser integration bugs, and only when the user explicitly agrees.
|
||||
|
||||
If a cross-layer issue ever needs a live server, the sandboxed helpers live in
|
||||
`scripts/e2e/` (`start_server.py`, `wait_for_server.py`). Non-negotiable rules:
|
||||
|
||||
- Always launch with `--settings-path <sandbox>/settings` and sandboxed
|
||||
`folder_paths` under `/tmp` — the repo folder is the real plugin folder and a
|
||||
`settings.json` there is read by the live instance. Never touch real config or
|
||||
real model libraries.
|
||||
- Never kill a process you did not start; `start_server.py` tracks its own PIDs
|
||||
via pidfile and refuses to touch unrelated processes on the port.
|
||||
- Abort after ~30 minutes or 3 consecutive tool failures; report `BLOCKED` with
|
||||
observed state instead of retrying blindly. Clean up sandbox and server after.
|
||||
|
||||
## Key Integration Points
|
||||
|
||||
- **Settings:** Stored in the user config directory (via `platformdirs`) or portable mode (`"use_portable_settings": true`)
|
||||
@@ -216,6 +261,12 @@ The system runs in two modes:
|
||||
## Important Notes
|
||||
|
||||
- ALWAYS use English for comments (per copilot-instructions.md)
|
||||
- **`settings.json.example` must stay minimal**: only `use_portable_settings`,
|
||||
`civitai_api_key`, and the four core `folder_paths` keys (`loras`,
|
||||
`checkpoints`, `unet`, `embeddings`). Do NOT add optional/default keys
|
||||
(model-category folders, `default_*_root`, `auto_organize_exclusions`, etc.)
|
||||
to this file unless the user explicitly asks for it. Defaults belong in
|
||||
`DEFAULT_SETTINGS` in `py/services/settings_manager.py`.
|
||||
- Run `python scripts/sync_translation_keys.py` after adding UI strings to `locales/en.json`
|
||||
- Symlinks require normalized paths.
|
||||
**Business paths vs real paths**: All stored paths and operation routing use the
|
||||
|
||||
-10
@@ -3,8 +3,6 @@ try: # pragma: no cover - import fallback for pytest collection
|
||||
from .py.nodes.lora_loader import LoraLoaderLM, LoraTextLoaderLM
|
||||
from .py.nodes.checkpoint_loader import CheckpointLoaderLM
|
||||
from .py.nodes.unet_loader import UNETLoaderLM
|
||||
from .py.nodes.random_checkpoint_loader import RandomCheckpointLoaderLM
|
||||
from .py.nodes.random_unet_loader import RandomUNETLoaderLM
|
||||
from .py.nodes.trigger_word_toggle import TriggerWordToggleLM
|
||||
from .py.nodes.prompt import PromptLM
|
||||
from .py.nodes.text import TextLM
|
||||
@@ -42,12 +40,6 @@ except (
|
||||
"py.nodes.checkpoint_loader"
|
||||
).CheckpointLoaderLM
|
||||
UNETLoaderLM = importlib.import_module("py.nodes.unet_loader").UNETLoaderLM
|
||||
RandomCheckpointLoaderLM = importlib.import_module(
|
||||
"py.nodes.random_checkpoint_loader"
|
||||
).RandomCheckpointLoaderLM
|
||||
RandomUNETLoaderLM = importlib.import_module(
|
||||
"py.nodes.random_unet_loader"
|
||||
).RandomUNETLoaderLM
|
||||
TriggerWordToggleLM = importlib.import_module(
|
||||
"py.nodes.trigger_word_toggle"
|
||||
).TriggerWordToggleLM
|
||||
@@ -87,8 +79,6 @@ NODE_CLASS_MAPPINGS = {
|
||||
LoraTextLoaderLM.NAME: LoraTextLoaderLM,
|
||||
CheckpointLoaderLM.NAME: CheckpointLoaderLM,
|
||||
UNETLoaderLM.NAME: UNETLoaderLM,
|
||||
RandomCheckpointLoaderLM.NAME: RandomCheckpointLoaderLM,
|
||||
RandomUNETLoaderLM.NAME: RandomUNETLoaderLM,
|
||||
TriggerWordToggleLM.NAME: TriggerWordToggleLM,
|
||||
LoraStackerLM.NAME: LoraStackerLM,
|
||||
LoraStackCombinerLM.NAME: LoraStackCombinerLM,
|
||||
|
||||
+326
-283
File diff suppressed because it is too large
Load Diff
+124
-8
@@ -62,27 +62,143 @@ Environment variable overrides: `LLM_API_KEY`, `LLM_MODEL`, `LLM_API_BASE`, `LLM
|
||||
|
||||
### enrich_hf_metadata
|
||||
|
||||
Enriches HuggingFace-downloaded models with metadata extracted by an LLM from the HF model card.
|
||||
Enriches models linked to an external model site with metadata extracted by an LLM from the site's model card (README).
|
||||
|
||||
**Entry point**: Right-click context menu → "Enrich Metadata (Agent)"
|
||||
**Entry point**: Right-click context menu → "Enrich Metadata with AI"
|
||||
|
||||
**Supported model sources**:
|
||||
|
||||
| Platform | Link | AI enrichment | Direct download |
|
||||
| --- | --- | --- | --- |
|
||||
| Hugging Face | yes | yes | yes |
|
||||
| ModelScope (`modelscope.cn`) | yes | yes | yes |
|
||||
| ModelScope International (`modelscope.ai`) | yes | yes | yes |
|
||||
| TensorArt | yes | no (see below) | no |
|
||||
|
||||
`modelscope.cn` and `modelscope.ai` are **separate catalogues, not mirrors** — a
|
||||
repository published on one is routinely absent from the other — so each is
|
||||
registered as its own source (`ModelScopeSource` / `ModelScopeIntlSource` in
|
||||
`py/services/model_sources/modelscope.py`). The host therefore decides which
|
||||
API and CDN a model resolves against, and the two deployments get separate
|
||||
version groups (`ms:` / `msai:`) and default download directories. Keep the two
|
||||
tables in `modelSourceHelpers.js` and `registry.py` in step when adding a site.
|
||||
|
||||
TensorArt is link-only: `tensor.art` sits behind a Cloudflare managed challenge and its internal API requires session authorization, so the backend cannot read its model pages. Linking still stores the canonical page URL and the "View on TensorArt" link works.
|
||||
|
||||
**What it does**:
|
||||
1. Reads the model's `.metadata.json` to get the `hf_url`
|
||||
2. Fetches the README.md from the HuggingFace repository
|
||||
3. Sends the README + local metadata to the LLM for structured extraction
|
||||
1. Reads the model's `.metadata.json` to get the source (`source_platform` + `source_url`, or the legacy `hf_url`)
|
||||
2. Fetches the model card through the provider in `py/services/model_sources/` — the README via `fetch_model_card()`, plus any extras the site keeps outside it via `fetch_model_card_context()`
|
||||
3. Sends the README + site-provided extras + local metadata to the LLM for structured extraction
|
||||
4. Writes extracted fields to `.metadata.json`:
|
||||
- `base_model` — only if current value is empty
|
||||
- `trainedWords` — trigger words (LoRA only, if none exist)
|
||||
- `modelDescription` — concise summary (if none exists)
|
||||
- `modelDescription` — the site's author description (if any) followed by the README rendered as HTML
|
||||
- `tags` — merged with existing tags, deduplicated
|
||||
- `civitai.images` — example images
|
||||
- `metadata_source` — audit trail: `agent:enrich_hf_metadata`
|
||||
- `llm_enriched_at` — ISO timestamp
|
||||
5. Downloads and optimizes preview image (if LLM found one in the README)
|
||||
5. Downloads and optimizes a preview image, using the per-file example image the
|
||||
site publishes when the README has none
|
||||
6. Updates the scanner cache
|
||||
7. Broadcasts WebSocket progress events
|
||||
|
||||
#### Site-provided card extras (`fetch_model_card_context`)
|
||||
|
||||
A model card is not always just `README.md`. ModelScope keeps the author's
|
||||
summary (`Description`), the site-curated tags (`OfficialTags`), and — per
|
||||
published version — the model filenames together with that file's example
|
||||
images (`MuseInfo.versions[].coverImages`) and trigger words in its
|
||||
model-detail API. AIGC repositories there often ship an auto-generated
|
||||
boilerplate README and put everything useful in `Description`, so reading only
|
||||
the README yields almost nothing.
|
||||
|
||||
Providers opt in by overriding `ModelSource.fetch_model_card_context()`, which
|
||||
returns a `ModelCardContext`. The wanted file is identified by its sha256 when
|
||||
the caller knows it (the scanner already records one) and by **basename**
|
||||
otherwise, so each checkpoint in a collection repo gets its own images — and
|
||||
keeps getting them after the user renames the weights, which is the only
|
||||
identifier a rename cannot invalidate. Sites with no such extras inherit an
|
||||
empty context, and the pipeline behaves exactly as before.
|
||||
|
||||
The README and the repository metadata describe the whole repository, not one
|
||||
file, so `execute_skill()` creates a `ModelSourceCache` for the duration of a
|
||||
run and passes it down. Enriching the eight checkpoints of one ModelScope
|
||||
repository costs two HTTP requests instead of sixteen; only the per-file
|
||||
selection is redone for each file. Nothing is cached across runs, and download
|
||||
URLs never go through it.
|
||||
|
||||
#### Deterministic data is applied whether or not an LLM is configured
|
||||
|
||||
`AgentService._load_source_card()` runs for every source-backed enrichment, and
|
||||
the post-processor applies what it returns before the LLM output is merged. A
|
||||
user with **no** provider configured therefore still gets the author summary,
|
||||
the example images, the preview, the site-curated tags, the trigger words and
|
||||
the README rendered as the model description.
|
||||
|
||||
The LLM is always consulted when one is configured — invoking **Enrich Metadata
|
||||
with AI** must call the provider every time, and the site data is never treated
|
||||
as a reason to skip it. The deterministic values act as fallbacks that fill
|
||||
gaps the LLM leaves behind:
|
||||
|
||||
| Field | Deterministic source | LLM role |
|
||||
| --- | --- | --- |
|
||||
| `model_name` | site display name (`Name`), written only while the value is still the file stem | — |
|
||||
| `modelDescription` | author summary + README as HTML | — |
|
||||
| `civitai.name` | the matched version's label (`modelVersion.showName`) | — |
|
||||
| `civitai.images` | site example images, then README images | — |
|
||||
| `preview_url` | first available example image | may propose one from the README |
|
||||
| `tags` | site-curated tags, always merged in | proposes additional content tags |
|
||||
| `civitai.description` | author summary | richer 1-2 sentence summary wins |
|
||||
| `base_model` | site hints resolved against the canonical vocabulary (`py/services/agent/base_model_resolver.py`) | mapping it is the LLM's job; the resolver only fills in when the LLM returns nothing |
|
||||
| `trainedWords` | per-file site trigger words, then YAML `instance_prompt` | primary extraction |
|
||||
| `usage_tips` | regex over an explicitly stated strength range | primary extraction |
|
||||
| `notes` | — | LLM-only |
|
||||
|
||||
Models with no source, an unknown source, or a source without model-card access (TensorArt) are skipped with an explicit reason and counted in the run summary.
|
||||
|
||||
**Model types**: LoRA, Checkpoint, Embedding
|
||||
|
||||
### Download-time hydration
|
||||
|
||||
The same deterministic mapping runs automatically when a model is downloaded
|
||||
from a model source, so a ModelScope or Hugging Face download lands with the
|
||||
populated card a CivitAI download produces instead of a bare filename and
|
||||
hash. Nothing needs to be triggered by hand and no provider is called.
|
||||
|
||||
`py/services/model_sources/hydration.py` owns this path:
|
||||
|
||||
* `_save_source_metadata()` in `py/routes/handlers/model_source_handlers.py`
|
||||
creates the sidecar (hash, source link, scanner-cache entry) and then calls
|
||||
`hydrate_from_source()`. It also runs for a file that was already on disk, so
|
||||
models downloaded before this existed get topped up on the next attempt.
|
||||
* Metadata is created through the **owning scanner**
|
||||
(`scanner._create_default_metadata()`) rather than
|
||||
`MetadataManager.create_default_metadata()`, so the per-type lazy-hash rule
|
||||
applies: `CheckpointScanner` and `OtherScanner` store
|
||||
`hash_status="pending"` with an empty `sha256` for their multi-GB files, and
|
||||
the generic helper would read a 10 GB checkpoint end to end inside the
|
||||
download request. Hydration copes with the empty hash — `_matching_versions()`
|
||||
falls back to the repository basename, which the download just wrote.
|
||||
* Hydration reuses `PostProcessor` with an empty `llm_output`, so the two paths
|
||||
cannot drift apart. It reports `metadata_source = "source:<platform>"` rather
|
||||
than the skill's `agent:enrich_hf_metadata`, and — because no provider ran —
|
||||
it does not stamp `llm_enriched_at`.
|
||||
* `model_name` is only written while it still equals the file stem: once a user
|
||||
renames a model, that choice is kept.
|
||||
* Only a model whose stored `source_platform`/`source_url` match the repository
|
||||
being downloaded is updated; a local file that merely shares a name must not
|
||||
receive another model's card.
|
||||
* The README and repository payload describe the *repository*, so a short-lived
|
||||
process-wide `ModelSourceCache` (`shared_source_cache`, 300 s, 32 entries)
|
||||
keeps a batch over one repository to two HTTP requests.
|
||||
* Every failure — unreachable site, changed payload shape, broken post-processor
|
||||
— is logged and swallowed. Metadata hydration can never fail a download.
|
||||
* Neither stage advances the byte counter, so both are announced to the
|
||||
progress UI (`_report_phase()` → `{"status": "metadata", "stage": ...}`) as
|
||||
they start. Without that the bar sits at 100% reporting `0 B/s` for several
|
||||
seconds and the download looks stuck. `stage` and `platform` are
|
||||
machine-readable; the wording is localised in `LoadingManager`.
|
||||
|
||||
## Adding a New Skill
|
||||
|
||||
### 1. Create the skill directory
|
||||
@@ -129,7 +245,7 @@ Use `{{variable}}` placeholders that will be replaced with data from the `prepar
|
||||
```markdown
|
||||
You are an expert assistant...
|
||||
|
||||
Model URL: {{hf_url}}
|
||||
Model URL: {{source_url}}
|
||||
README content:
|
||||
{{readme_content}}
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ The dedicated services encapsulate long-running work so handlers stay thin.
|
||||
| Use case | Entry point | Dependencies | Guarantees |
|
||||
| --- | --- | --- | --- |
|
||||
| `RecipeAnalysisService` | `analyze_uploaded_image`, `analyze_remote_image`, `analyze_local_image`, `analyze_widget_metadata` | `ExifUtils`, `RecipeParserFactory`, downloader factory, optional metadata collector/processor | Normalises missing/invalid payloads into `RecipeValidationError`; generates consistent fingerprint data to keep duplicate detection stable; temporary files are cleaned up after every analysis path. |
|
||||
| `RecipePersistenceService` | `save_recipe`, `delete_recipe`, `update_recipe`, `reconnect_lora`, `bulk_delete`, `save_recipe_from_widget` | `ExifUtils`, recipe scanner, card preview sizing constants | Writes images/JSON metadata atomically; updates scanner caches and hash indices before returning; recalculates fingerprints whenever LoRA assignments change. |
|
||||
| `RecipePersistenceService` | `save_recipe`, `delete_recipe`, `update_recipe`, `reconnect_lora`, `get_reconnect_suggestions`, `bulk_delete`, `save_recipe_from_widget` | `ExifUtils`, recipe scanner, card preview sizing constants | Writes images/JSON metadata atomically; updates scanner caches and hash indices before returning; recalculates fingerprints whenever LoRA assignments change. |
|
||||
| `RecipeSharingService` | `share_recipe`, `prepare_download` | `tempfile`, recipe scanner | Copies originals to TTL-managed temp files; metadata lookups re-use the scanner; expired shares trigger cleanup and `RecipeNotFoundError`. |
|
||||
|
||||
## Maintaining critical invariants
|
||||
|
||||
@@ -0,0 +1,590 @@
|
||||
# i18n Translation Guidelines
|
||||
|
||||
This document is the canonical set of conventions for translating LoRA Manager UI strings.
|
||||
It applies to **human translators and AI agents** alike. Read it before editing anything in
|
||||
`locales/`.
|
||||
|
||||
Source of truth: `locales/en.json` (10 locales, 2025 leaf keys; all locales share the exact
|
||||
same key structure).
|
||||
|
||||
Locales: `en`, `zh-CN`, `zh-TW`, `ja`, `ko`, `fr`, `de`, `es`, `ru`, `he` (RTL).
|
||||
|
||||
> **Status (2026-08 sweep):** a full audit was executed and the terminology, placeholder,
|
||||
> stale-text, and untranslated-block fixes described in §2–§6 were applied across all locales
|
||||
> (commits `3c3ac49f` … `fd1227d3`). The tables below are now the **normative target state**,
|
||||
> not a to-do list — future edits should preserve these renderings and only add what is new.
|
||||
>
|
||||
> **Status (2026-09, Other Models):** the `other` model type (VAE / Upscaler / Text Encoder /
|
||||
> CLIP Vision / ControlNet) and the Other Models opt-in toggles added 36 new keys; all of them
|
||||
> are now translated in all 9 locales (terminology in §2 "Other Models feature"). There are no
|
||||
> remaining `[TODO: Translate]` placeholders in any locale.
|
||||
>
|
||||
> **Status (2026-09, revision):** `other.disabled.description`, `banners.otherModels.content` and
|
||||
> `settings.folderSettings.enableOtherModelsHelp` were refreshed in `en.json` to name all five
|
||||
> sub_types (they had listed four, which read as "these are what enabling manages") and
|
||||
> re-translated in all 9 locales in the same pass. `clip_vision` and `controlnet` are now both
|
||||
> opt-in, so the first two describe **capability** and the third the **master switch**, not the
|
||||
> default set — keep all three enumerating the full five (`VAE / upscaler / text encoder /
|
||||
> CLIP vision / ControlNet` in `en`; locale slash-list casing follows each file's existing
|
||||
> `VAE / Upscaler / Text Encoder / …` style, de compounds as `CLIP-Vision- und ControlNet-Ordner`).
|
||||
>
|
||||
> **Status (2026-09, "no folders found" state):** the Other Models page gained an *enabled but
|
||||
> nothing to scan* empty state with 6 new keys (`other.noPaths.*`); translated in all 9 locales
|
||||
> in the same pass. The `folder_paths` JSON snippet shown in that state lives in
|
||||
> `templates/other.html`, **not** in the locale files, so it is never translated — only the
|
||||
> surrounding prose is. Terminology added in §2.
|
||||
>
|
||||
> **Status (2026-09, model sources):** models can now be linked to ModelScope and TensorArt
|
||||
> alongside Hugging Face, which added 15 keys (`modelCard.actions.viewOnSource`,
|
||||
> `loras.contextMenu.linkModelSource`, `modals.linkModelSource.*`,
|
||||
> `modals.model.versions.sourceGroupInfo`, `toast.contextMenu.enrichNeedsSource`,
|
||||
> `toast.contextMenu.enrichUnsupportedSource`) and refreshed the two `enrichHfAgent` labels,
|
||||
> which had hardcoded "HF" for a button that now also enriches ModelScope models. The
|
||||
> `modals.linkModelSource.urlPlaceholder` value stays byte-identical to `en.json` (it is a URL,
|
||||
> the §6 exception). Terminology in §2, "Model source feature".
|
||||
>
|
||||
> **Status (2026-09, folder sidebar):** the model-root sidebar gained on-disk folder management
|
||||
> (create / rename / delete folders, show empty folders, tree vs list view) plus its `...`
|
||||
> view-options menu, adding 35 `sidebar.*` keys. Those were the only `[TODO: Translate]`
|
||||
> placeholders left behind by the feature series, and all 35 are now translated in all 9
|
||||
> locales, so the "no remaining placeholders" claim above holds again. Terminology in §2,
|
||||
> "Folder sidebar feature".
|
||||
>
|
||||
> **Status (2026-09, chip reordering):** model tags and trigger words now share one drag/`⠿`
|
||||
> grip reorder affordance, which added the single `common.reorder.dragHandle` key (it lives
|
||||
> under `common` because both editors render it). All 9 locales are translated (renderings in
|
||||
> §2, "Chip reordering"). Reordering is pointer-only by design: an `Alt + Arrow` shortcut was
|
||||
> prototyped and removed because it collided with the browser's Alt + Arrow handling and the
|
||||
> modal's arrow-key navigation.
|
||||
|
||||
> **Status (2026-09, standalone no-paths guidance):** the standalone branch of the
|
||||
> `other.noPaths` empty state now shows the real `settings.json` path plus an
|
||||
> `other.noPaths.openSettingsFolder` button (each locale reuses its
|
||||
> `settings.openSettingsFileLocation.label` rendering), and `descriptionStandalone` was
|
||||
> reworded in `en.json` — from "none of the configured folders exist on disk" to "no
|
||||
> other-model folders were found; add the folder keys you need to the `folder_paths`
|
||||
> section" — and re-translated in all 9 locales. The `on disk` phrase now survives only in
|
||||
> the ComfyUI variant (`descriptionComfyUI`).
|
||||
|
||||
> **Status (2026-09, settings Organization tab):** the settings modal split its overloaded
|
||||
> Library tab, adding the single `settings.nav.organization` key (renderings in §2,
|
||||
> "Settings Organization tab"). All 9 locales are translated, so the "no remaining
|
||||
> placeholders" claim holds again.
|
||||
|
||||
> **Status (2026-09, filename templates):** the Filename Templates feature (per-model-type
|
||||
> download filename templates + bulk "Apply to Library Now" rename, with an empty template
|
||||
> restoring recorded original filenames) added 26 keys across `settings.filenameTemplates.*`,
|
||||
> `loras.bulkOperations.filenameTemplateProgress.*`, `modals.filenameTemplateConfirm.*` and
|
||||
> the `toast.loras.filenameTemplate*` / `toast.settings.filenameTemplates*` toasts. All 9
|
||||
> locales are translated (terminology in §2, "Filename Templates feature").
|
||||
|
||||
---
|
||||
|
||||
## 1. Hard rules (do not violate)
|
||||
|
||||
### R1 — Key structure is sacred
|
||||
- Only `locales/en.json` may add/remove/rename keys. All other locales must keep the exact
|
||||
same nested key set. `tests/i18n/test_i18n.py` enforces this.
|
||||
- When a new UI string is added to `en.json`, run
|
||||
`python scripts/sync_translation_keys.py` (adds the missing keys to all locales with
|
||||
`[TODO: Translate]` placeholder copies) — **then stop**. Do NOT translate proactively:
|
||||
placeholders are the expected end state during feature development, and translations are
|
||||
filled in only when the feature owner explicitly asks (workflow details in §7).
|
||||
- Never reorder, re-indent, or reformat a locale file "for tidiness". The sync script
|
||||
preserves formatting; manual reformatting creates noisy diffs.
|
||||
|
||||
### R2 — Placeholders and HTML must be preserved verbatim
|
||||
- `{name}`-style placeholders must appear in the translation exactly as in `en.json`.
|
||||
Do not invent placeholders the source string does not have — the caller may not pass them
|
||||
(example bug: `zh-CN recipes.controls.import.downloadLocationPreview` added `{path}`; the
|
||||
template renders this key with no parameters, so the literal text `{path}` shows in the UI).
|
||||
- `{{...}}` in a locale value is an escaped literal brace — keep it identical.
|
||||
- Keep embedded HTML tags (e.g. `<strong>...</strong>`, `<code>...</code>`) intact.
|
||||
You may move the tag around the sentence if the target language needs different word order.
|
||||
|
||||
### R3 — Never translate or transliterate these
|
||||
- Model types: **LoRA, Checkpoint, Embedding, Diffusion Model**
|
||||
- Products/brands: **LoRA Manager, ComfyUI, CivitAI, CivArchive, HuggingFace, Ko-fi**
|
||||
- Ecosystem names: **LyCORIS, DoRA**, trigger-adjacent jargon **Prompt, Workflow**
|
||||
(these are used as-is in the target-language SD community; see §2 per-language policy)
|
||||
- Theme names: **Nord, Midnight, Monokai, Dracula, Solarized**
|
||||
|
||||
### R4 — The "Recipe" convention (the most important domain term)
|
||||
Product intent: a *Recipe* records a **LoRA combination + generation parameters**
|
||||
(prompt, seed, sampler, …) that reproduces an image style. The metaphor is a **cooking
|
||||
recipe** — "follow it and you get a similar dish". It is **not** a menu, not a dish list,
|
||||
not a prescription.
|
||||
|
||||
Decision per language — translate only into a word whose everyday primary meaning is a
|
||||
cooking recipe; where that word would mislead users, **keep the English "Recipe(s)"**:
|
||||
|
||||
| Locale | Use | Never use |
|
||||
|---|---|---|
|
||||
| fr | **Recipe / Recipes** (keep English) | recette(s) — cooking reading is secondary and it was explicitly judged misleading |
|
||||
| zh-CN / zh-TW | 配方 | 食谱 (reads as "food cookbook") |
|
||||
| ja | レシピ | — (leftover English "Recipe" in `initialization.recipes.title` / `toast.recipes.recipeSaved` → translate) |
|
||||
| ko | 레시피 | — |
|
||||
| de | Rezept / Rezepte | — (cooking meaning dominant; prescription reading acceptable) |
|
||||
| es | receta / recetas | — (cooking meaning dominant) |
|
||||
| ru | рецепт / рецепты | — (leftover English "Recipe" in `initialization.recipes.title` / `toast.recipes.recipeSaved` → translate) |
|
||||
| he | מתכון / מתכונים | — (cooking meaning dominant) |
|
||||
|
||||
Whatever the choice, **one concept = one noun within a locale**. Currently violated in:
|
||||
- `fr` — "Recipe" (~97 keys, incl. nav) mixed with "recette" (~58 keys)
|
||||
- `zh-CN` / `zh-TW` — 配方 (126/122 keys) mixed with 食谱 / 食譜 (14/17 keys, all in the
|
||||
*rematch* flow: `globalContextMenu.rematchRecipes.*`, `toast.recipes.rematch*`)
|
||||
- `de` — "Rezept" (136 keys) mixed with leftover English "Recipe" (5 keys)
|
||||
- `ja` / `ru` — leftover English "Recipe" in `initialization.recipes.title` ("Recipe Manager
|
||||
zu initialisieren" / «Инициализация Recipe Manager») and `toast.recipes.recipeSaved`
|
||||
|
||||
### R5 — One term, one rendering (within each locale)
|
||||
Same source word must not be translated several ways in one file. Known offender areas
|
||||
(see §5 for the full fix list): recipe, Checkpoint, Embedding, prompt, base model, preset,
|
||||
workflow, hash, metadata, tags, bulk. Every locale currently mixes variants of at least one
|
||||
of these — pick the preferred form in the §2 tables and normalize.
|
||||
|
||||
### R6 — Register consistency
|
||||
- `zh-CN` / `zh-TW`: pick 你 or 您 once. Do not mix (zh-CN has 44×你 + 5×您; zh-TW has
|
||||
27×您 + 18×你).
|
||||
- `de`: pick "du" or "Sie" once (currently 143×Sie + ~7×du).
|
||||
- `es`: pick "tú" or "usted" once.
|
||||
|
||||
### R7 — Punctuation per script
|
||||
- Full-width punctuation `:()` is correct **only in CJK locales** (zh-CN, zh-TW, ja, ko).
|
||||
- Latin/Cyrillic/Hebrew locales must use ASCII `: ()` — full-width colons leaked in there
|
||||
are machine-translation artifacts. Known: `fr toast.recipes.createError/createFailed`,
|
||||
`es toast.recipes.createError/createFailed` (e.g. "…de la receta:" should be "…de la receta:").
|
||||
- `fr` apostrophes must be U+2019 `'` / ASCII `'`, never a straight double quote:
|
||||
`fr header.filter.allowSellingGeneratedContentTooltip` currently reads
|
||||
`vendre d"images` → fix to `d'images`. Do not mix `'` and `'` in one file (fr has 299 vs 15).
|
||||
- Ellipsis: use ASCII `...` (project style). Don't introduce `…`.
|
||||
- Keep the sentence-ending period/omission consistent with the source string where the
|
||||
language allows it.
|
||||
- `he` is RTL: mix of Hebrew and Latin scripts is normal; keep Latin term ordering natural.
|
||||
|
||||
### R8 — No untranslated English leftovers
|
||||
Full sentences left byte-identical to `en.json` are bugs (brand names and URL placeholders
|
||||
are the exception). Every locale has them; see §6 for the per-locale checklist.
|
||||
`[TODO: Translate]` placeholders are the sanctioned intermediate state during feature
|
||||
development (see §7) — do not "fix" them unless the feature owner asked for translations.
|
||||
|
||||
### R9 — Mirror the source even when the source is wrong
|
||||
If `en.json` itself contains an inconsistency (e.g. the `Civitai` vs `CivitAI` casing split,
|
||||
or the `CivitArchive` typo in `modals.relinkCivitai.helpText.format4`), translate/transcribe
|
||||
it as-is in your locale and instead **fix the source** in `en.json` (then propagate by
|
||||
re-syncing and re-translating affected keys). Do not silently diverge in one locale only.
|
||||
|
||||
---
|
||||
|
||||
## 2. Per-language term maps
|
||||
|
||||
Preferred rendering per term. "Fix" means the locale currently contains the wrong variant
|
||||
and must be normalized. `en` = keep the English word as-is.
|
||||
|
||||
### fr
|
||||
|
||||
| Term | Use | Fix |
|
||||
|---|---|---|
|
||||
| recipe | Recipe(s) | Replace all "recette(s)" (58 keys, e.g. `recipes.actions.deleteRecipeWithShortcut`, `toast.recipes.rematchComplete`) with "Recipe(s)" |
|
||||
| Checkpoint | Checkpoint | `statistics.modelTypes.checkpoint` = "Point de contrôle" → "Checkpoint" |
|
||||
| trigger words | mot(s)-clé(s) | unify: `modals.model.triggerWords.editWord` uses "mot déclencheur" — pick one |
|
||||
| prompt / negative prompt | Prompt / prompt négatif | — |
|
||||
| base model | modèle(s) de base | — |
|
||||
| preset | préréglage | unify: `modals.model.usageTips.addPresetParameter` "prédéfini", `toast.presets.restored` "par défaut" |
|
||||
| hash | hash | `conflictConfirm.message` "hachage" → "hash" |
|
||||
| tags | tags | `settings.sections.priorityTags` "Étiquettes" → "Tags" |
|
||||
| metadata | métadonnées | `loras.controls.refresh.fullTooltip` keeps English "metadata" |
|
||||
| duplicates | doublon(s) | unify with "dupliqué(e)s" |
|
||||
| bulk | groupé(e) | unify with "par lot / mode lot" variants |
|
||||
|
||||
### de
|
||||
|
||||
| Term | Use | Fix |
|
||||
|---|---|---|
|
||||
| recipe | Rezept/Rezepte | leftover English "Recipe" keys → Rezept (e.g. `toast.recipes.recipeSaved`) |
|
||||
| base model | pick Basis-Modell or Basismodell | currently 27× hyphenated vs 15× closed |
|
||||
| metadata | Metadaten | 4 keys use "Modelldaten" (`onboarding.steps.fetch.title/content`) → Metadaten |
|
||||
| bulk | pick Massen- or Sammelmodus | `loras.controls.bulk.action` = "Massen" reads as "crowds" — use "Massenbearbeitung"/"Mehrfachauswahl" |
|
||||
| register | Sie (formal) | 7 keys use "du/dein" (`settings.backup.managementHelp`, `modals.checkUpdates.message/tip`, `doctor.footer`, …) |
|
||||
|
||||
### es
|
||||
|
||||
| Term | Use | Fix |
|
||||
|---|---|---|
|
||||
| recipe | receta(s) | — |
|
||||
| Checkpoint | Checkpoint | 5 statistics keys "Punto(s) de control" → "Checkpoints" (`statistics.metrics.checkpoints`, `statistics.insights.unusedCheckpoints.*`, `statistics.modelTypes.checkpoint`) |
|
||||
| trigger words | palabra(s) de activación | 2 keys already use it; ~15 keys "palabra(s) clave" (reads as search keyword) → unify |
|
||||
| base model | modelo base | — |
|
||||
| preset | preajuste | 3 keys keep English "preset", 1 "preestablecido" → preajuste |
|
||||
| workflow | pick flujo de trabajo or workflow | currently 21× "flujo de trabajo" vs 10× "workflow" |
|
||||
| bulk | masivo / por lotes | unify; "Batch Import" → traducción |
|
||||
| tags | etiquetas | — |
|
||||
|
||||
### ru
|
||||
|
||||
| Term | Use | Fix |
|
||||
|---|---|---|
|
||||
| recipe | рецепт(ы) | English leftovers: `initialization.recipes.title`, `recipes.batchImport.*`, `toast.recipes.recipeSaved` → translate |
|
||||
| Checkpoint | Checkpoint (recommended) | 3 variants today: "Checkpoint" (17 keys), «Чекпойнт», «Контрольная точка» (statistics, 6 keys) — statistics MUST drop «Контрольная точка» |
|
||||
| Embedding | Embedding | «Эмбеддинг» variant exists in `settings.priorityTags.modelTypes.embedding` — unify |
|
||||
| prompt | промпт | 8 keys use «запрос» (reads as "database/HTTP request") → «промпт» |
|
||||
| base model | базовая модель | — |
|
||||
| preset | пресет | `header.theme.presets` "Предустановки" → пресеты |
|
||||
| workflow | Workflow (recommended) | «рабочий процесс» used in 4 keys — unify |
|
||||
| hash | pick хеш or хэш | both spellings co-occur |
|
||||
| tag(s) | тег(и) | — |
|
||||
| typos | — | `settings.misc.loraSyntaxFormatHelp`: «безпотерьного» → «беспотерьного» |
|
||||
|
||||
### he
|
||||
|
||||
| Term | Use | Fix |
|
||||
|---|---|---|
|
||||
| recipe | מתכון / מתכונים | — |
|
||||
| Checkpoint | Checkpoint | 5 statistics keys «נקודת/נקודות ביקורת» (road/security checkpoint) → "Checkpoint(s)" (`statistics.metrics.checkpoints`, `statistics.modelTypes.checkpoint`, `statistics.insights.unusedCheckpoints.*`) |
|
||||
| Embedding | Embedding | `statistics` keys use הטמעות → Embedding |
|
||||
| prompt | pick הנחיה or פרומפט | 9 keys הנחיה vs 3 פרומפט — unify (recommend פרומפט, SD-community loanword) |
|
||||
| preset | קביעה מראש | `header.filter.presetOverwriteConfirm` uses פריסט → unify |
|
||||
| hash | pick one of האש / גיבוב / hash | 3 variants co-occur — unify (recommend hash or גיבוב) |
|
||||
| metadata | pick מטא-דאטה or מטא-נתונים | 38 vs 17 keys — unify |
|
||||
| model | מודל | 13 keys use דגם/דגמים — unify |
|
||||
| bulk | pick one of 5 variants | 5 different renderings ("כמות גדולה", "המוני", "קבוצתי", "אצווה", …) — unify; `loras.controls.bulk.action` "כמות גדולה" reads as "large quantity" |
|
||||
|
||||
### ja
|
||||
|
||||
| Term | Use | Fix |
|
||||
|---|---|---|
|
||||
| recipe | レシピ | `initialization.recipes.title` keeps English "Recipe Manager" — translate to レシピマネージャー |
|
||||
| Checkpoint | Checkpoint or チェックポイント (pick one) | 3 variants: Checkpoint (~14), checkpoint lowercase (4), チェックポイント (4, e.g. `settings.priorityTags.modelTypes.checkpoint`) |
|
||||
| Embedding | Embedding | 4 keys lowercase "embedding" mid-sentence |
|
||||
| bulk | 一括 | `modals.checkUpdates.tip` "バルクモード" → 一括モード |
|
||||
| recipe counter | 件 or 個 | `globalContextMenu.rematchRecipes.success` uses 件, `.cancelled` uses 個 — unify |
|
||||
|
||||
### ko
|
||||
|
||||
| Term | Use | Fix |
|
||||
|---|---|---|
|
||||
| recipe | 레시피 | — |
|
||||
| Checkpoint | Checkpoint (recommended) | 4 keys transliterate 체크포인트 (`settings.priorityTags.modelTypes.checkpoint`, `toast.recipes.missingCheckpointPath/missingCheckpointInfo/downloadCheckpointFailed`) |
|
||||
| Embedding | Embedding | 3 keys 임베딩 (`settings.priorityTags.modelTypes.embedding`, `uiHelpers.nodeSelector.embedding`) |
|
||||
| base model | 베이스 모델 | 6 keys «기본 모델» read as "default model" → 베이스 모델 (`settings.downloadSkipBaseModels.*`, `toast.loras.downloadSkippedByBaseModel`) |
|
||||
| workflow | pick 워크플로 or 워크플로우 | 26 vs 6 keys — unify |
|
||||
| bulk | 일괄 | `modals.checkUpdates.tip` "벌크 모드" → 일괄 모드 |
|
||||
| tag logic | — | `header.filter.tagLogicAny` = "모든 태그 일치 (OR)" is **inverted** (should be "하나 이상의 태그 일치") and identical to `tagLogicAll` |
|
||||
| particle | — | `modelCard.sendToWorkflow.checkpointNotImplemented`: "Checkpoint을" → "Checkpoint를" |
|
||||
|
||||
### zh-CN / zh-TW
|
||||
|
||||
| Term | zh-CN | zh-TW |
|
||||
|---|---|---|
|
||||
| recipe | 配方 (fix 食谱 → 配方, 14 keys in rematch flow) | 配方 (fix 食譜 → 配方, 17 keys in rematch flow) |
|
||||
| Checkpoint | Checkpoint (fix 检查点 → Checkpoint, 5 keys: `toast.recipes.missingCheckpointPath/missingCheckpointInfo/downloadCheckpointFailed`, `modelCard.actions.checkpointNameCopied`, `modelCard.sendToWorkflow.checkpointNotImplemented`) | Checkpoint (fix 檢查點 → Checkpoint, 4 keys: `modelCard.actions.copyCheckpointName`, `toast.recipes.missing*`×2, `toast.recipes.downloadCheckpointFailed`) |
|
||||
| base model | 基础模型 (fix 基模型 → 基础模型, 3 keys in `modals.model.versions.filters.*`) | 基礎模型 ✓ consistent |
|
||||
| prompt | 提示词 ✓ | 提示詞 ✓ |
|
||||
| preset | 预设 ✓ | 預設 ✓ |
|
||||
| workflow | 工作流 ✓ | 工作流 ✓ |
|
||||
| trigger words | 触发词 ✓ | 觸發詞 ✓ |
|
||||
| hash | 哈希 (哈希值 variant OK) | 雜湊 ✓ |
|
||||
| register | 你 (fix 5×您 → 你) | 您 (fix 18×你 → 您) |
|
||||
|
||||
### Other Models feature (VAE / Upscaler / Text Encoder / CLIP Vision / ControlNet)
|
||||
|
||||
The `other` model type exposes five sub_types. They are **model-type names**, so they follow
|
||||
R3 and stay in Latin in every locale. The `settings.folderSettings.subType*` values are
|
||||
therefore **intentionally byte-identical to `en.json`** (same precedent as
|
||||
`settings.priorityTags.modelTypes` / `checkpoints.modelTypes.checkpoint`) — a §6 sweep must
|
||||
not "fix" them.
|
||||
|
||||
| Term | Rendering | Note |
|
||||
|---|---|---|
|
||||
| VAE | `VAE` everywhere | acronym, always upper-case |
|
||||
| Upscaler | `Upscaler` everywhere | CivitAI `ModelType` name |
|
||||
| Text Encoder | `Text Encoder` everywhere | de compounds as `Text-Encoder-Stammordner` |
|
||||
| CLIP Vision | `CLIP Vision` everywhere | de compounds as `CLIP-Vision-Stammordner` |
|
||||
| ControlNet | `ControlNet` everywhere | brand casing, capital N |
|
||||
|
||||
In prose these names sit next to localized nouns the same way `Diffusion Model` does
|
||||
(zh `VAE 根目录`, ja `VAEルート`, ko `VAE 루트`, ru `Корневая папка VAE`).
|
||||
|
||||
**"Other Models" is the page/feature name, not a model type — translate it:**
|
||||
|
||||
| Locale | `other.title` | `header.navigation.other` |
|
||||
|---|---|---|
|
||||
| fr | Autres modèles | Autres |
|
||||
| zh-CN | 其他模型 | 其他 |
|
||||
| zh-TW | 其他模型 | 其他 |
|
||||
| ja | その他のモデル | その他 |
|
||||
| ko | 기타 모델 | 기타 |
|
||||
| de | Weitere Modelle | Andere |
|
||||
| es | Otros modelos | Otros |
|
||||
| ru | Другие модели | Другое |
|
||||
| he | מודלים אחרים | אחרים |
|
||||
|
||||
`settings.folderSettings.otherSubTypes` ("Managed Types") must name **model** types, matching
|
||||
each locale's `header.filter.modelTypes` rendering (zh `管理的模型类型`, ja `管理するモデルタイプ`,
|
||||
de `Verwaltete Modelltypen`, …).
|
||||
|
||||
The "no folders found" empty state (`other.noPaths.*`) uses two phrases that must stay
|
||||
consistent whenever that copy is edited. `folder key` means the `folder_paths` key name
|
||||
(`vae`, `upscale_models`, … — Latin per the table above); `on disk` means the folder must
|
||||
physically exist:
|
||||
|
||||
| Phrase | Rendering |
|
||||
|---|---|
|
||||
| folder key | zh-CN 文件夹键 · zh-TW 資料夾鍵 · ja フォルダーキー · ko 폴더 키 · fr clé de dossier · de Ordnerschlüssel · es clave de carpeta · ru ключ папки · he מפתח תיקייה |
|
||||
| on disk | zh-CN 在磁盘上 · zh-TW 在磁碟上 · ja ディスク上 · ko 디스크에 · fr sur le disque · de auf dem Datenträger · es en el disco · ru на диске · he בדיסק |
|
||||
|
||||
`settings.json` and `ComfyUI` stay verbatim in every locale; "reload this page" / "restart
|
||||
LoRA Manager" reuse each locale's existing restart wording (`settings.extraFolderPaths.*`).
|
||||
|
||||
### Model source feature (Hugging Face / ModelScope / TensorArt)
|
||||
|
||||
A model file can be linked to the page of an external model site. **Hugging Face**,
|
||||
**ModelScope** and **TensorArt** are brand names and stay Latin in every locale (R3); the
|
||||
generic nouns around them are translated:
|
||||
|
||||
| Term | Rendering |
|
||||
|---|---|
|
||||
| model source | zh-CN 模型来源 · zh-TW 模型來源 · ja モデルソース · ko 모델 소스 · fr source de modèle · de Modellquelle · es fuente de modelo · ru источник модели · he מקור מודל |
|
||||
| model page | zh-CN 模型页面 · zh-TW 模型頁面 · ja モデルページ · ko 모델 페이지 · fr page du modèle · de Modellseite · es página del modelo · ru страница модели · he עמוד המודל |
|
||||
| model card | zh-CN 模型卡 · zh-TW 模型卡 · ja モデルカード · ko 모델 카드 · fr fiche de modèle · de Modellkarte · es ficha de modelo · ru карточка модели · he כרטיס מודל |
|
||||
| AI enrichment (noun) | reuse the existing pair per locale: zh-CN 增强 · zh-TW 增強 · ja 補完 · ko 보강 · fr enrichissement (par IA) · de Anreicherung (KI-) · es enriquecimiento (con IA) · ru обогащение (с помощью ИИ) · he העשרה (AI) |
|
||||
|
||||
`modelCard.actions.viewOnSource` ("View on {source}") follows each locale's existing
|
||||
`viewOnHuggingFace` pattern — de `Auf … ansehen`, ru `Открыть …`, he `צפייה ב-…`,
|
||||
ja `… で見る`, ko `…에서 보기`, zh `在 … 查看`, fr `Voir sur …`, es `Ver en …`. `{source}` is
|
||||
replaced at runtime with the untranslated platform name, so the brand never appears inside the
|
||||
translated text.
|
||||
|
||||
`modals.linkModelSource.enrichNote` states the rule that only sites exposing a readable model
|
||||
card can be enriched and names TensorArt as the current exception. Keep the parenthetical
|
||||
exception in sync if another link-only source is ever added — the sentence is deliberately
|
||||
phrased as a rule, not as an apology for one site.
|
||||
|
||||
The context-menu and bulk-operation enrichment entry points read **"Enrich Metadata with AI"**
|
||||
in `en`, not "Enrich HF Metadata": they cover ModelScope as well, so no locale may reintroduce
|
||||
an `HF` qualifier in `loras.contextMenu.enrichHfAgent` / `loras.bulkOperations.enrichHfAgent`
|
||||
(the key names keep the historical `Hf`; only the values changed).
|
||||
|
||||
### Folder sidebar feature (create / rename / delete folders, empty folders, view options)
|
||||
|
||||
The model-root sidebar manages on-disk folders. "Folder" reuses the noun already fixed in §2
|
||||
(the `folder key` row); the rest is new surface:
|
||||
|
||||
| Term | Rendering |
|
||||
|---|---|
|
||||
| folder | zh-CN 文件夹 · zh-TW 資料夾 · ja フォルダ · ko 폴더 · fr dossier · de Ordner · es carpeta · ru папка · he תיקייה |
|
||||
| model root (as in "no model root is configured") | zh-CN 模型根目录 · zh-TW 模型根目錄 · ja モデルルート · ko 모델 루트 · fr racine de modèle · de Modell-Stammverzeichnis · es raíz de modelo · ru корневая папка моделей · he שורש מודלים — note `sidebar.modelRoot` alone is the shorter 根目录 / 根目錄 / ルート / 루트 / Racine / Stammverzeichnis / Raíz / Корень / שורש |
|
||||
| tree view / list view | zh-CN 树形视图 / 列表视图 · zh-TW 樹狀檢視 / 清單檢視 · ja ツリー表示 / リスト表示 · ko 트리 보기 / 목록 보기 · fr Vue arborescente / Vue liste · de Baumansicht / Listenansicht · es Vista de árbol / Vista de lista · ru Дерево / Список · he תצוגת עץ / תצוגת רשימה |
|
||||
| sidebar | reuse each locale's `sidebar.hideOnThisPage` noun: zh-CN 侧边栏 · zh-TW 側邊欄 · ja サイドバー · ko 사이드바 · fr barre latérale · de Seitenleiste · es barra lateral · ru боковая панель · he סרגל צד |
|
||||
|
||||
Deleting a folder **never cascades over model files** — the backend refuses it and
|
||||
`sidebar.deleteFolderModal.notEmptyMessage` states the rule in every locale, so keep that
|
||||
clause (and its `—`) when the copy is edited. The `{name}` / `{count}` / `{message}` tokens in
|
||||
`sidebar.createFolderResult.*`, `sidebar.deleteFolderResult.*` and `sidebar.renameFolderResult.*`
|
||||
are verbatim §1-R2 placeholders; `successWithFiles` is the only key carrying `{count}`.
|
||||
|
||||
### Settings Organization tab
|
||||
|
||||
The settings modal's fourth nav tab groups everything about how files are arranged on
|
||||
disk: download path templates, priority tags, and auto-organize exclusions. The label is
|
||||
the **noun for arranging files**, matching each locale's existing
|
||||
`settings.sections.autoOrganize` rendering minus the "auto":
|
||||
|
||||
| Locale | `settings.nav.organization` |
|
||||
|---|---|
|
||||
| fr | Organisation |
|
||||
| zh-CN | 整理 |
|
||||
| zh-TW | 整理 |
|
||||
| ja | 整理 |
|
||||
| ko | 정리 |
|
||||
| de | Organisation |
|
||||
| es | Organización |
|
||||
| ru | Организация |
|
||||
| he | ארגון |
|
||||
|
||||
zh-CN/zh-TW use 整理 ("tidying/arranging"), not 组织/組織 (an organization as a group).
|
||||
|
||||
### Filename Templates feature
|
||||
|
||||
Per-model-type templates that name downloaded model files; "Apply to Library Now"
|
||||
bulk-renames existing files, and an **empty template restores the recorded original
|
||||
filenames** (recorded in each model's metadata at its first rename). "Template" follows
|
||||
each locale's existing download-path-template noun (zh-CN 模板 vs zh-TW 範本 — note the
|
||||
split); progress strings mirror `loras.bulkOperations.autoOrganizeProgress` verbatim with
|
||||
the locale's "moved" verb swapped for its "renamed" verb, and the toasts mirror the
|
||||
`autoOrganize*` / `downloadTemplates*` toast shapes.
|
||||
|
||||
| Term | Rendering |
|
||||
|---|---|
|
||||
| filename template(s) | zh-CN 文件名模板 · zh-TW 檔案名稱範本 · ja ファイル名テンプレート · ko 파일명 템플릿 · fr modèle(s) de nom de fichier · de Dateinamen-Vorlage(n) · es plantilla(s) de nombres de archivo · ru шаблон(ы) имён файлов · he תבנית שם קובץ / תבניות שמות קבצים |
|
||||
| Apply to Library Now (button) | zh-CN 立即应用到库 · zh-TW 立即套用至模型庫 · ja ライブラリに今すぐ適用 · ko 지금 라이브러리에 적용 · fr Appliquer à la bibliothèque maintenant · de Jetzt auf Bibliothek anwenden · es Aplicar a la biblioteca ahora · ru Применить к библиотеке сейчас · he החל על הספרייה כעת |
|
||||
| Restore original filenames (modal title / button) | zh-CN 恢复原始文件名?/ 恢复原始文件名 · zh-TW 要還原原始檔案名稱嗎?/ 還原原始檔案名稱 · ja 元のファイル名を復元しますか?/ 元のファイル名を復元 · ko 원본 파일명을 복원하시겠습니까? / 원본 파일명 복원 · fr Restaurer les noms de fichier d'origine ? / Restaurer les noms de fichier d'origine · de Ursprüngliche Dateinamen wiederherstellen? / Ursprüngliche Dateinamen wiederherstellen · es ¿Restaurar los nombres de archivo originales? / Restaurar nombres de archivo originales · ru Восстановить исходные имена файлов? / Восстановить исходные имена файлов · he לשחזר שמות קבצים מקוריים? / שחזר שמות קבצים מקוריים |
|
||||
| "renamed" (progress/toast counter) | zh-CN 已重命名 · zh-TW 已重新命名 · ja リネーム · ko 이름 변경 · fr renommés · de umbenannt · es renombrados · ru переименовано · he שונו שמותם |
|
||||
|
||||
### Chip reordering (model tags / trigger words)
|
||||
|
||||
Model tags and trigger-word chips share a single reorder affordance (drag the chip, or its
|
||||
`⠿` grip where the chip body is click-to-edit), so the copy sits in `common.reorder.dragHandle`
|
||||
instead of a feature namespace. It is used twice per editor: as the grip tooltip and as the
|
||||
hint shown in the edit controls row. There is deliberately **no keyboard shortcut** — an
|
||||
`Alt + Arrow` binding fought the browser's own Alt + Arrow handling and the modal's arrow-key
|
||||
navigation, so reordering is pointer-only and the grip is a decorative, non-focusable
|
||||
affordance. Do not reintroduce a shortcut or a "position X of Y" screen-reader string without
|
||||
re-adding the corresponding keys.
|
||||
|
||||
`dragHandle` is a fragment, not a sentence: it labels both the grip and the hint, so keep it
|
||||
short and imperative and do not append a keyboard hint in any locale.
|
||||
|
||||
| Term | Rendering |
|
||||
|---|---|
|
||||
| drag to reorder | zh-CN 拖拽以调整顺序 · zh-TW 拖曳以調整順序 · ja ドラッグして並べ替え · ko 드래그하여 순서 변경 · fr Glisser pour réordonner · de Zum Neuordnen ziehen · es Arrastra para reordenar · ru Перетащите, чтобы изменить порядок · he גרור כדי לשנות סדר |
|
||||
|
||||
The grip itself is an icon and is never translated.
|
||||
|
||||
---
|
||||
|
||||
## 3. Cross-cutting confusion hot-spots (must-fix list)
|
||||
|
||||
All items below were **resolved** in the 2026-08 sweep — treat them as a regression
|
||||
watch-list: do not reintroduce these renderings.
|
||||
|
||||
1. **Checkpoint rendered as a literal security/road checkpoint** — fr, es, ru, he, zh-CN,
|
||||
zh-TW all had 4–6 keys in the `statistics.*` domain reading as "control point"; reverted
|
||||
to "Checkpoint".
|
||||
2. **"recipe" variants that break the one-noun rule** — fr "recette" → "Recipe", zh
|
||||
食谱/食譜 → 配方, de/ja/ru leftover English "Recipe" translated.
|
||||
3. **ko `header.filter.tagLogicAny`** — was inverted ("모든 태그 일치 (OR)") and identical
|
||||
to `tagLogicAll`; now "어느 하나의 태그와 일치 (OR)".
|
||||
4. **ja `modals.model.versions.actions.viewLocalTooltip`** — was the stale "近日対応予定"
|
||||
("coming soon"); all 9 locales now describe the actual action.
|
||||
5. **Stale help texts** — `settings.downloadSkipBaseModels.help`,
|
||||
`settings.aiProvider.apiBaseHelp`, `settings.hideEarlyAccessUpdates.help` retranslated
|
||||
in all locales to the current `en.json` wording.
|
||||
6. **en.json source bugs** (fixed in source, then mirrored):
|
||||
- "Civitai" → "CivitAI" brand casing (values only; key names `relinkCivitai` etc. keep
|
||||
their lowercase form and must not be renamed)
|
||||
- `modals.relinkCivitai.helpText.format4` "CivitArchive" typo → "CivArchive"
|
||||
- `zh-CN recipes.controls.import.downloadLocationPreview` invented `{path}` removed
|
||||
|
||||
---
|
||||
|
||||
## 4. Placeholder contract deviations (current)
|
||||
|
||||
`{...}` token sets must match `en.json` per key. All deviations found in the 2026-08 sweep
|
||||
were fixed, with one *intentional* exception:
|
||||
|
||||
**`toast.settings.mappingsUpdated`** — the caller passes a hardcoded English inflection
|
||||
(`plural: count !== 1 ? 's' : ''`). Languages that cannot build a plural by appending that
|
||||
`s` (zh-CN/zh-TW, ja, ko, de, ru, he) **drop `{plural}`** and render a count-friendly form
|
||||
(`({count})` or a measure word); fr and es keep it (`mappage{plural}`, `mapeo{plural}`).
|
||||
|
||||
```python
|
||||
# keep a copy of this rule next to the key if it ever moves:
|
||||
# fr/es: "... ({count} mappage{plural})"
|
||||
# de/ru/he: "... ({count})"
|
||||
# zh-CN: "({count} 条映射)" / zh-TW: "({count} 個對應)" / ja: "({count} マッピング)"
|
||||
```
|
||||
|
||||
Do NOT add `{...}` tokens the source lacks (the caller will not supply them, and the literal
|
||||
text renders in the UI), and do NOT rename source tokens (`{typePlural}` stays `{typePlural}`).
|
||||
|
||||
---
|
||||
|
||||
## 5. One term, one rendering — offender matrix
|
||||
|
||||
Cross-locale summary of §2 inconsistencies. "✓" = already consistent. All ✗ cells were
|
||||
resolved in the 2026-08 sweep; the row shows the single rendering now in force per locale.
|
||||
|
||||
| Term | fr | de | es | ru | he | ja | ko | zh-CN | zh-TW |
|
||||
|---|---|---|---|---|---|---|---|---|---|
|
||||
| recipe | Recipe | Rezept | receta | рецепт | מתכון | レシピ | 레시피 | 配方 | 配方 |
|
||||
| Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint | Checkpoint |
|
||||
| Embedding | Embedding | Embedding | Embedding | Embedding | Embedding | Embedding | Embedding | Embedding | Embedding |
|
||||
| prompt | Prompt | Prompt | prompt | промпт | פרומפט | プロンプト | 프롬프트 | 提示词 | 提示詞 |
|
||||
| base model | modèle de base | Basismodell | modelo base | базовая модель | מודל בסיס | ベースモデル | 베이스 모델 | 基础模型 | 基礎模型 |
|
||||
| preset | préréglage | Voreinstellung | preajuste | пресет | קביעה מראש | プリセット | 프리셋 | 预设 | 預設 |
|
||||
| workflow | Workflow | Workflow | workflow | Workflow | workflow | ワークフロー | 워크플로 | 工作流 | 工作流 |
|
||||
| hash | hash | Hash | hash | хеш | hash | ハッシュ | 해시 | 哈希 | 雜湊 |
|
||||
| metadata | métadonnées | Metadaten | metadatos | метаданные | מטא-נתונים | メタデータ | 메타데이터 | 元数据 | 中繼資料 |
|
||||
| tags | Tags | Tags | etiquetas | теги | תגיות | タグ | 태그 | 标签 | 標籤 |
|
||||
| duplicates | en double | Duplikate | duplicados | дубликаты | כפילויות | 重複 | 중복 | 重复项 | 重複項 |
|
||||
| bulk | groupé | Massen- | por lotes | пакетный | בכמות גדולה | 一括 | 일괄 | 批量 | 批量 |
|
||||
|
||||
Watch: ja/ko keep the model-type names **Checkpoint/Embedding** and `Diffusion Model` in
|
||||
Latin (consistent with their model-type sections) — do not transliterate them as
|
||||
チェックポイント/체크포인트.
|
||||
|
||||
---
|
||||
|
||||
## 6. Untranslated English leftovers (status)
|
||||
|
||||
Values byte-identical to `en.json` that are actual UI sentences are bugs (brand names and
|
||||
URL placeholders are the exception). As of the 2026-08 sweep, **all previously untranslated
|
||||
blocks are translated** in every locale: `recipes.batchImport.*` + `toast.recipes.batchImport*`
|
||||
(fr/de/es/ru/he/ja/ko), `banners.communitySupport.*`, `modals.model.license.*`,
|
||||
`globalContextMenu.fetchMissingLicenses.*`, the `doctor.*` issue/action/label subset,
|
||||
`toast.settings.libraryLoadFailed` / `libraryActivateFailed`, `toast.api.moveFailed`,
|
||||
`settings.extraFolderPaths.restartRequired`, `toast.recipes.recipeSaved`,
|
||||
`sidebar.dragDrop.moveUnsupported`, `checkpoints.modelTypes.diffusion_model`
|
||||
(ja/ko keep the English loanword), `initialization.recipes.title`.
|
||||
|
||||
The only values that remain intentionally identical to `en.json` are non-translatable:
|
||||
URL/path placeholders (`https://…`, `C:/…`), numeric presets (`5 (1080p), 6 (2K), 8 (4K)`),
|
||||
example token lists (`character, concept, style(toon|toon_style)`), service/provider names
|
||||
(`CivitAI → CivArchive → Archive DB`), model-type names (`settings.priorityTags.modelTypes.*`,
|
||||
`settings.folderSettings.subTypeVae` … `subTypeControlnet` — see §2), and the external playlist
|
||||
title (`help.updateVlogs.playlistTitle`, de: translated to "LoRA Manager-Update-Playlist").
|
||||
|
||||
Rule for `uiHelpers.workflow.noPromptTargets`: the second line (`Mark as → Send Prompt
|
||||
Target`) quotes literal ComfyUI context-menu items — keep those menu labels in English in
|
||||
every locale because that is what the user actually sees in ComfyUI.
|
||||
|
||||
License labels (`modals.model.license.*`): the restriction labels are now translated in all
|
||||
locales (the sibling `creditRequired` has always been translated).
|
||||
|
||||
---
|
||||
|
||||
## 7. Workflow for agents and translators
|
||||
|
||||
### Adding a new UI string
|
||||
1. Add the key to `locales/en.json` only.
|
||||
2. Run `python scripts/sync_translation_keys.py` — it inserts the key into the other 9
|
||||
locales (as a `[TODO: Translate]` placeholder) preserving formatting.
|
||||
3. **During feature development, stop here.** While the UI copy is still in flux, leave the
|
||||
`[TODO: Translate]` placeholders as-is — translating churning strings into 9 locales is
|
||||
wasted work. Placeholders are a normal intermediate state, not a bug.
|
||||
4. Once the wording is final and the feature owner explicitly asks for translations,
|
||||
translate **all** pending `[TODO: Translate]` keys in every locale (not just the latest
|
||||
feature's), applying §1–§3 (placeholders verbatim, Recipe rule, term maps, register).
|
||||
Find pending keys with: `grep -c "TODO: Translate" locales/*.json`
|
||||
5. If the new string contains new terminology, extend §2 tables.
|
||||
|
||||
### Fixing a translation bug
|
||||
1. Locate the key (dotted path) in the relevant locale file.
|
||||
2. Check the corresponding `en.json` value and the actual caller (grep `static/js` or
|
||||
`web/comfyui` for the key) to learn which placeholders are passed.
|
||||
3. Fix trivially; for normalization sweeps (e.g. "recette" → "Recipe"), do it file-wide for
|
||||
the offending keys only — do not touch unrelated lines.
|
||||
4. If the bug is in `en.json` itself (R9), fix the source first, then re-sync and update all
|
||||
locales.
|
||||
|
||||
### Verification
|
||||
```bash
|
||||
pytest tests/i18n/test_i18n.py # key parity + JSON validity + JS key references
|
||||
python scripts/sync_translation_keys.py --dry-run # shows which keys would change; add --verbose for per-key detail
|
||||
npm test # frontend tests incl. i18n helpers
|
||||
```
|
||||
|
||||
`pytest tests/i18n` only checks structure. Quality conventions in this document are not
|
||||
machine-enforced — a human/agent review pass is required.
|
||||
|
||||
### Anti-patterns checklist
|
||||
- [ ] Placeholders `{x}` / `{{x}}` differ from `en.json`
|
||||
- [ ] Same source term translated 2+ ways in the same file (see §5)
|
||||
- [ ] "Checkpoint" became a literal checkpoint; "recipe" became menu/prescription/food-cookbook
|
||||
- [ ] Brand names translated or transliterated (LoRA, CivitAI, ComfyUI, …)
|
||||
- [ ] Latin locale using full-width `:()`; fr using `"` as apostrophe
|
||||
- [ ] Mixed 你/您, du/Sie, tú/usted
|
||||
- [ ] Full English sentences left behind (see §6)
|
||||
- [ ] Register/typos/mojibake; source string is stale vs `en.json` (compare semantics, not
|
||||
just words)
|
||||
@@ -0,0 +1,107 @@
|
||||
# Plan: Filename Template Follow-ups
|
||||
|
||||
**Issue:** [#1071 — Lora Renaming](https://github.com/willmiao/ComfyUI-Lora-Manager/issues/1071)
|
||||
**Status:** Core feature **implemented** (2026-09-19, commit `2bc9860b`,
|
||||
preceded by the settings-tab split in `327da046`). Follow-ups 1 and 2 were
|
||||
resolved together on 2026-09-19 by redefining the empty template as
|
||||
"revert to recorded original filename" (see below). Follow-up 3 remains open.
|
||||
|
||||
## What shipped in `2bc9860b`
|
||||
|
||||
- Per-model-type `download_filename_templates` setting (empty = keep current
|
||||
filename; opt-in). Placeholders: `{model_name}`, `{version_name}`,
|
||||
`{base_model}`, `{author}`, `{first_tag}`, `{hash_short}`,
|
||||
`{original_name}`.
|
||||
- `calculate_filename_for_model()` in `py/utils/utils.py` renders the
|
||||
template; templates containing path separators are rejected.
|
||||
- Downloads apply the template post-download
|
||||
(`DownloadManager._apply_download_filename_template`); rename conflicts
|
||||
keep the original name and never fail the download.
|
||||
- `ModelLifecycleService.rename_model` records `original_file_name` in the
|
||||
`.metadata.json` sidecar (first rename wins via `setdefault`).
|
||||
- Bulk apply: `GET|POST /api/lm/{prefix}/apply-filename-template`
|
||||
(`FilenameTemplateUseCase`, shares the auto-organize lock, WS progress type
|
||||
`filename_template_progress`).
|
||||
- Settings UI: "Filename Templates" subsection in the new **Organization**
|
||||
settings tab (`templates/components/modals/settings/organization.html`),
|
||||
with validation, live preview, and per-type "Apply to Library Now".
|
||||
|
||||
Sandbox E2E verified: rename incl. companion files (previews, sidecars),
|
||||
metadata pointer updates, `original_file_name` recording, idempotency,
|
||||
conflict handling (failure counted, batch continues), empty-template no-op,
|
||||
GET variant.
|
||||
|
||||
## Follow-ups 1 & 2 — RESOLVED: empty template = revert to recorded original
|
||||
|
||||
Follow-up 1 asked to reword the ambiguous "Valid (keep original filename)"
|
||||
empty-template message; Follow-up 2 asked for a bulk revert to the recorded
|
||||
`original_file_name`. Both were resolved by a single semantic change: **an
|
||||
empty template now means "restore the recorded original filename"** instead of
|
||||
"leave the current filename untouched".
|
||||
|
||||
Rationale: for never-renamed models a revert is a no-op (no recorded
|
||||
original), for renamed models it restores the pre-rename name, and new
|
||||
downloads with an empty template keep the download name as before — so the
|
||||
two contexts (download path and bulk apply) share one coherent meaning, and
|
||||
no separate revert feature or `{recorded_original}` placeholder is needed.
|
||||
|
||||
Implemented changes:
|
||||
|
||||
- `FilenameTemplateUseCase._process_model`: an empty template now resolves
|
||||
the target name from the sidecar's `original_file_name` via the injected
|
||||
`metadata_loader` (default `load_local_metadata`); models without a
|
||||
recorded original or whose original matches the current name are skipped.
|
||||
Cache entries do not project `original_file_name`, so the sidecar is read
|
||||
per model.
|
||||
- `SettingsManager.js`: removed the empty-template early return and the
|
||||
apply-button disable (`updateFilenameTemplateApplyButton` deleted — the
|
||||
button is now always enabled). The browser-native `confirm()` was replaced
|
||||
with `filenameTemplateConfirmModal`
|
||||
(`templates/components/modals/confirm_modals.html`), a **self-managed**
|
||||
modal (like `DirectoryPickerModal`, NOT registered with ModalManager):
|
||||
ModalManager's "close current modal on open" behavior would kill the
|
||||
settings modal underneath. It stacks via `z-index: 10010`
|
||||
(`delete-modal.css`), handles ESC in capture phase with
|
||||
`stopPropagation`, and shows apply vs revert wording
|
||||
(`modals.filenameTemplateConfirm.titleApply` / `titleRevert` /
|
||||
`revertButton`; messages reuse `settings.filenameTemplates.confirmApply` /
|
||||
`confirmRevert`).
|
||||
- `locales/en.json`: reworded `help` / `applyHelp`, replaced
|
||||
`validation.keepOriginal` with `validation.restoreOriginal`
|
||||
("Valid (empty template restores original filenames)"), added
|
||||
`confirmRevert`, removed the now-unused `emptyTemplateInfo`. Other locales
|
||||
re-synced with `[TODO: Translate]` placeholders — retranslation waits for
|
||||
the feature owner's request per `docs/i18n-translation-guidelines.md` §7.
|
||||
- Tests: revert / no-record-skip / same-name-skip cases in
|
||||
`tests/services/test_use_cases.py`; modal confirm-and-revert and
|
||||
cancel paths in
|
||||
`tests/frontend/managers/settingsManager.filenameTemplates.test.js`.
|
||||
|
||||
Sandbox E2E verified (standalone server, sandboxed settings + library under
|
||||
`/tmp`, 2026-09-19): template apply renames and records
|
||||
`original_file_name`; empty-template apply reverts to the recorded name;
|
||||
revert target occupied by a newer file counts as failure and keeps the
|
||||
current name; models without a recorded original are skipped;
|
||||
apply → revert → re-apply cycles repeat cleanly.
|
||||
|
||||
Standing caveats (unchanged):
|
||||
|
||||
- The revert target may collide with an existing file — the existing conflict
|
||||
handling (count as failure, keep current name) covers this.
|
||||
- `original_file_name` only exists for models renamed after `2bc9860b`;
|
||||
older renames have no recorded original and are skipped.
|
||||
- `original_file_name` is kept (not cleared) after a revert, so
|
||||
apply → revert → re-apply stays repeatable.
|
||||
|
||||
## Follow-up 3 — Cross-page refresh after bulk apply
|
||||
|
||||
**Problem:** the settings-modal "Apply to Library Now" button calls
|
||||
`resetAndReload(true)`, which refreshes only the page type currently open.
|
||||
Applying the checkpoint template while on the loras page leaves the loras
|
||||
view refreshed but does not touch the checkpoints page state (same
|
||||
limitation as the existing bulk auto-organize flow in
|
||||
`static/js/managers/SettingsManager.js#applyFilenameTemplate`).
|
||||
|
||||
**Fix options:** broadcast a generic "library changed" event that every
|
||||
page's state listens to, or accept the limitation (the other page reloads
|
||||
its cache on next visit). Low priority.
|
||||
@@ -0,0 +1,363 @@
|
||||
# Plan: "Other Models" Page — Unified Management for VAE / Upscaler / Text Encoder / etc.
|
||||
|
||||
**Status:** v2 — **Phase 1 implemented** (2026-09-12, commits `27da7b3c` backend + `fa7ce725` frontend; verified live against a running ComfyUI instance: scan/hash/sub_type-derivation/fetch/previews all green). **Phase 2 implemented** (2026-09-12, per §9 design; full pytest + vitest green). **Phase 3 implemented** (§11: opt-in management toggles; default off). **i18n done** (2026-09-13): all 36 new keys translated in the 9 non-English locales — the `[TODO: Translate]` placeholders left by the sync script during development are gone (see `docs/i18n-translation-guidelines.md` §2, "Other Models feature"). **Default set revised (pre-release):** only `vae` / `upscaler` / `text_encoder` are managed by default — `clip_vision` and `controlnet` are both opt-in (§2, §11.1.1).
|
||||
**Scope (Phase 1):** scan + manage (list, search, filter, tags, folders, preview, rename, move, delete/exclude, CivitAI metadata fetch) for a new model type `other`, exposed as a new web page. **Phase 2 (§9):** one-click download from CivitAI for these types.
|
||||
|
||||
## 1. Goal
|
||||
|
||||
Today the manager supports three model types:
|
||||
|
||||
| page | model_type | sub_types |
|
||||
|---|---|---|
|
||||
| `/loras` | `lora` | `lora`, `locon`, `dora` |
|
||||
| `/checkpoints` | `checkpoint` | `checkpoint`, `diffusion_model` |
|
||||
| `/embeddings` | `embedding` | `embedding` |
|
||||
|
||||
Add a fourth page that manages "everything else" — VAE, upscalers, text encoders / CLIP, CLIP vision, optionally ControlNet — with a folder→sub_type mapping table so new ComfyUI folder categories can be added later by configuration, not code.
|
||||
|
||||
## 2. Locked Decisions
|
||||
|
||||
1. **Architecture: one scanner + one service + one page, sub_type derived by location.**
|
||||
Replicates the checkpoint pattern (`CheckpointScanner` aggregates `checkpoints` + `unet` roots and derives `checkpoint` vs `diffusion_model` from the root containing the file, `py/services/checkpoint_scanner.py:384-415`). One `OtherScanner` aggregates all enabled folder roots; `resolve_sub_type_for_path()` maps each root to a sub_type. No per-category scanners.
|
||||
|
||||
2. **Naming: internal `model_type = "other"`, route prefix `/other`, page id `other`.**
|
||||
- `misc` is rejected: `py/routes/misc_routes.py` already owns that name for system/settings routes (`/api/lm/settings`, `/api/lm/doctor/*`).
|
||||
- `components` is rejected: `templates/components/` and `static/js/components/` directories would make `components.html` / `components.js` confusing neighbors.
|
||||
- `other` matches CivitAI's `Other` fallback type semantics. The **display name** is an i18n string (`other.title`, e.g. "Other Models") and can be renamed later without touching code.
|
||||
|
||||
3. **sub_type values:** snake_case, aligned with CivitAI `ModelType` semantics:
|
||||
|
||||
| sub_type | ComfyUI `folder_paths` key(s) | CivitAI ModelType | enabled by default |
|
||||
|---|---|---|---|
|
||||
| `vae` | `vae` | `VAE` | yes |
|
||||
| `upscaler` | `upscale_models` | `Upscaler` | yes |
|
||||
| `text_encoder` | `text_encoders`, `clip` (legacy) | `TextEncoder` (CLIP is retired upstream) | yes |
|
||||
| `clip_vision` | `clip_vision` | `CLIPVision` | no (mapping present, opt-in) |
|
||||
| `controlnet` | `controlnet` | `Controlnet` | no (mapping present, opt-in) |
|
||||
|
||||
New folder categories = one line in the mapping table (see §4.1).
|
||||
|
||||
**Why only three are on by default** (revised in Phase 3, before release):
|
||||
VAE, upscalers and text encoders are dependency-style assets every pipeline
|
||||
needs, and "which one am I actually using" is the recurring problem they
|
||||
solve. `clip_vision` and `controlnet` are workflow-driven instead
|
||||
(IPAdapter/SVD image conditioning; per-workflow ControlNet variants), and
|
||||
ControlNet libraries routinely run to dozens of files, so both are treated
|
||||
symmetrically as opt-in. Enumerating all five as "the default set" was not
|
||||
defensible on demand breadth alone.
|
||||
|
||||
4. **Phase 1 = scan/manage only.** Downloads from CivitAI (`download_manager.py` type mapping, default-root settings keys, download routing) are Phase 2 (§9). CivitAI **metadata fetch** for existing files IS in Phase 1 (hash-based lookup is type-agnostic; only the type-validation hook needs new values).
|
||||
|
||||
5. **Out of scope (default off, revisit later):** usage statistics buckets, recipe matching (`recipe_scanner.py` only merges lora+checkpoint scanners), statistics page, embeddings re-classification (stays its own page — merging would be a breaking change).
|
||||
|
||||
## 3. Why This Works With Minimal Churn
|
||||
|
||||
- `ModelScanner` (`py/services/model_scanner.py:93`) is specialized entirely via constructor params (`model_type`, `model_class`, `file_extensions`) + optional hooks (`adjust_metadata`, `adjust_cached_entry`, `resolve_sub_type_for_path`, `model_scanner.py:1429-1443`).
|
||||
- `BaseModelService` subclasses can be one method (`EmbeddingService` implements only `format_response`, `py/services/embedding_service.py:12`).
|
||||
- Routes: `ModelServiceFactory.register_model_type()` (`py/services/model_service_factory.py:120-136`) + `COMMON_ROUTE_DEFINITIONS` (`py/routes/model_route_registrar.py:23-149`) generate the full `/api/lm/{prefix}/*` surface (~50 endpoints) plus the `GET /{prefix}` page route.
|
||||
- `PersistentModelCache` (`py/services/persistent_model_cache.py:526-606`) is a single `models` table keyed `(model_type, file_path)` with `model_type` as free text — **zero schema change**.
|
||||
- Frontend `apiConfig.js` (`static/js/api/apiConfig.js:51`) generates all endpoints from the model-type string; `ModelCard.js:670-675` renders the sub_type badge from data; the checkpoints page already demonstrates the "one page, multiple sub_types" filter (`header.html:298`).
|
||||
|
||||
## 4. Backend Changes
|
||||
|
||||
### 4.1 New constants — `py/utils/constants.py`
|
||||
|
||||
```python
|
||||
# folder_paths key -> sub_type; single source of truth for extensibility
|
||||
OTHER_MODEL_FOLDER_SUBTYPES = {
|
||||
"vae": "vae",
|
||||
"upscale_models": "upscaler",
|
||||
"text_encoders": "text_encoder",
|
||||
"clip": "text_encoder", # legacy ComfyUI key
|
||||
"clip_vision": "clip_vision",
|
||||
"controlnet": "controlnet",
|
||||
}
|
||||
DEFAULT_OTHER_MODEL_FOLDERS = ("vae", "upscale_models", "text_encoders", "clip", "clip_vision")
|
||||
VALID_OTHER_SUB_TYPES = ["vae", "upscaler", "text_encoder", "clip_vision", "controlnet"]
|
||||
# CivitAI model.type values accepted for this page (fetch-metadata validation)
|
||||
VALID_OTHER_CIVITAI_TYPES = {"vae", "upscaler", "textencoder", "clipvision", "controlnet", "other"}
|
||||
```
|
||||
|
||||
Also extend `CIVITAI_USER_MODEL_TYPES` (`constants.py:90`) if user-model queries should include these types.
|
||||
|
||||
### 4.2 New files (mirror the embedding/checkpoint implementations)
|
||||
|
||||
1. **`py/utils/models.py`** — add `OtherModelMetadata(BaseModelMetadata)`: default `sub_type="vae"` placeholder overridden by scanner hook; `from_civitai_info` mapping CivitAI types → our sub_types (`TextEncoder`→`text_encoder`, `CLIPVision`→`clip_vision`, `Upscaler`→`upscaler`, `VAE`→`vae`, `Controlnet`→`controlnet`, else `other`-ish fallback to folder-derived sub_type).
|
||||
2. **`py/services/other_scanner.py`** — `OtherScanner(ModelScanner)`:
|
||||
- `model_type="other"`, extensions: reuse the checkpoint set (`safetensors/pt/pt2/bin/pth/pkl/sft/gguf`).
|
||||
- `get_model_roots()`: iterate `OTHER_MODEL_FOLDER_SUBTYPES` ∩ enabled keys, pull each from `config` (§4.3); dedupe; build `root → sub_type` map (normalized abspaths; multiple keys may share a sub_type).
|
||||
- Implement all three hooks like `CheckpointScanner` (`checkpoint_scanner.py:384-415`): `resolve_sub_type_for_path` by longest-prefix root match, `adjust_metadata`, `adjust_cached_entry` (sub_type is re-derived on cache load, never persisted).
|
||||
- **Lazy hashing, checkpoint-style**: text encoders (T5-XXL ≈ 10 GB) make eager sha256 painful. Copy the `hash_status="pending"` + singleflight `calculate_hash_for_model` pattern from `CheckpointScanner`.
|
||||
3. **`py/services/other_model_service.py`** — `OtherModelService(BaseModelService)`, `format_response` only (no usage_count, like `EmbeddingService`).
|
||||
4. **`py/routes/other_routes.py`** — `OtherRoutes(BaseModelRoutes)`, `template_name="other.html"`, hooks:
|
||||
- `_validate_civitai_model_type` → `VALID_OTHER_CIVITAI_TYPES`
|
||||
- `_get_expected_model_types`, `_parse_specific_params` (no type-specific download params in Phase 1)
|
||||
- `initialize_services()` on `app.on_startup` pulling `ServiceRegistry.get_other_scanner()`.
|
||||
|
||||
### 4.3 `py/config.py`
|
||||
|
||||
- New `other_roots` property: for each enabled key in `OTHER_MODEL_FOLDER_SUBTYPES`, `folder_paths.get_folder_paths(key)` (plugin mode) — standalone mode needs nothing new: `MockFolderPaths` (`standalone.py:66-105`) already serves arbitrary keys from `settings.json.folder_paths`.
|
||||
- Follow the existing per-type recipe: an `_prepare_other_paths()` (dedupe + symlink registration; also **cross-scanner overlap detection** — warn if an `other` root is already covered by checkpoints/unet/embedding roots, mirroring the checkpoint/unet overlap check).
|
||||
- Wire into: `_apply_library_paths`, `_symlink_roots()`, `_rebuild_preview_roots()` (hard requirement — preview images are served per registered root), `save_folder_paths_to_settings()`.
|
||||
|
||||
### 4.4 Existing-file edits (the "type string scatter" — each is a small branch/entry)
|
||||
|
||||
| file | change |
|
||||
|---|---|
|
||||
| `py/services/model_service_factory.py:120` | register `("other", OtherModelService, OtherRoutes)` in `register_default_model_types()` |
|
||||
| `py/services/service_registry.py` | add `get_other_scanner()` (mirror `:297` `get_embedding_scanner`) |
|
||||
| `py/services/model_scanner.py:67` | `PAGE_TYPE_MAP['other'] = 'other'` (WebSocket progress) |
|
||||
| `py/services/base_model_service.py:896-906` | `get_model_types()` branch → `VALID_OTHER_SUB_TYPES` |
|
||||
| `py/lora_manager.py` | `_initialize_services` scanner task list (`:219-242`), `_cleanup` cancel list (`:463`), `_cleanup_backup_files` roots (`:327-330`) |
|
||||
| `py/routes/handlers/misc_handlers.py` | `scanner_getters` (`:657-661`) + `scanner_factories` (`:757-759`) so Doctor / init-status / refresh-all see the new scanner |
|
||||
| `py/services/pending_delete_service.py` | `_PAGE_TYPE` map (`:57-61`) + scanner getter list (`:983-985`) |
|
||||
| `py/metadata_ops/__init__.py:36-38` | `SCANNER_TYPE_MAP['other']` |
|
||||
| `settings.json.example` | document optional `folder_paths` keys: `vae`, `upscale_models`, `text_encoders`, `clip_vision` |
|
||||
|
||||
**Explicitly NOT touched in Phase 1:** `py/services/download_manager.py`, `py/services/download_routing.py`, `py/services/settings_manager.py` default-root keys, `py/routes/stats_routes.py`, `py/utils/usage_stats.py`, `py/services/recipe_scanner.py`, `py/metadata_collector/`, `py/nodes/`.
|
||||
|
||||
**Zero-change confirmations (verified):** `PersistentModelCache`, `ModelUpdateService`, `DownloadedVersionHistoryService`, `MetadataSyncService` + provider chain (type-agnostic hash lookups), `ModelFileService` / `ModelMoveService` / `ModelLifecycleService` (scanner + model_type injected), `ModelCache` / `ModelHashIndex`, `AutoV3BackfillService`.
|
||||
|
||||
## 5. Frontend Changes
|
||||
|
||||
1. **`static/js/api/apiConfig.js`** — `MODEL_TYPES.OTHER = 'other'`; `MODEL_CONFIG.other` entry (displayName, singularName, `supportsMove`, `supportsBulkOperations`; no letter filter); endpoints come free from `getApiEndpoints()` (`:51`).
|
||||
2. **`static/js/api/otherApi.js`** — thin `OtherApiClient extends BaseModelApiClient` (mirror `embeddingApi.js`); register in `modelApiFactory.js`.
|
||||
3. **`static/js/other.js`** — page entry (mirror `embeddings.js`): `appCore.initialize()` + `createPageControls('other')` + `initializePageFeatures()` + `ModelDuplicatesManager` + `initActiveFiltersSync('other')`.
|
||||
4. **Controls & context menu** — `OtherControls extends PageControls` and `OtherContextMenu` (start from the embedding variants — the smallest); add branches in the two factories (`components/controls/index.js:15`, `components/ContextMenu/index.js:15`). Context-menu template block lives in `templates/other.html` (`{% block additional_components %}`, the checkpoints/embeddings pattern — do NOT touch the shared `context_menu.html`).
|
||||
5. **`templates/other.html`** — copy `embeddings.html`: same content blocks (controls + breadcrumb + duplicates banner + folder sidebar + `#modelGrid`), `data-page="other"`, main script `/loras_static/js/other.js`.
|
||||
6. **`templates/components/header.html`** — nav entry (`:23-43`, active when `request.path.startswith('/other')`); enable the `modelTypes` sub_type filter panel for `other` (`:298-305` pattern from checkpoints); check search-options panel conditions (`:199-224`).
|
||||
7. **`static/js/utils/constants.js`** — `MODEL_SUBTYPE_ABBREVIATIONS` (`:115`): `vae→VAE`, `upscaler→UPS`, `text_encoder→TE`, `clip_vision→CV`, `controlnet→CN`; matching `MODEL_SUBTYPE_DISPLAY_NAMES` (`:99`). (Unknown fallback already uppercases 4 chars, but explicit mappings read better.)
|
||||
8. **`static/js/core.js:110` `getPageType()`** — verify `data-page="other"` flows through `state.pages` generically; add only if the page list is enumerated anywhere.
|
||||
9. No change to `web/comfyui/top_menu_extension.js` (it opens `/loras`; page-to-page nav is the header bar).
|
||||
|
||||
## 6. i18n
|
||||
|
||||
- `locales/en.json`: add `other.title` (e.g. "Other Models") + minimal `other.contextMenu.*` / `other.modelTypes.*` keys; reuse `modelCard.*`, `loras.contextMenu.*`, `common.*` wherever possible (the established pattern — checkpoints/embeddings already reuse lora keys).
|
||||
- Run `python scripts/sync_translation_keys.py`; leave `[TODO: Translate]` placeholders in other locales (per `docs/i18n-translation-guidelines.md` §7 — do not translate proactively).
|
||||
|
||||
## 7. Testing
|
||||
|
||||
Follow existing conventions (`pytest.ini`, `tests/frontend/` vitest):
|
||||
|
||||
1. **Backend (pytest, async where needed):**
|
||||
- `OtherScanner` root aggregation + `resolve_sub_type_for_path` (file under `vae/` root → `vae`; `text_encoders` and legacy `clip` both → `text_encoder`; disabled `controlnet` root not scanned).
|
||||
- Cache round-trip: sub_type re-derived via `adjust_cached_entry` (not persisted).
|
||||
- Lazy hash: `hash_status="pending"` default; `calculate_hash_for_model` singleflight.
|
||||
- `OtherRoutes` registration smoke test: `/api/lm/other/...` endpoints exist; `_validate_civitai_model_type` accepts `vae`/`upscaler`/`textencoder`, rejects `lora`.
|
||||
- Config: `other_roots` in both modes (mock `folder_paths`, and standalone `settings.json.folder_paths`).
|
||||
2. **Frontend (vitest + jsdom, `tests/frontend/`):**
|
||||
- `apiConfig`: `getApiEndpoints('other')` URL shapes; `modelApiFactory` returns the Other client.
|
||||
- `ModelCard` badge rendering for new sub_types.
|
||||
- `createPageControls('other')` / `createPageContextMenu('other')` factories.
|
||||
3. **Manual UI verification by the user** (per AGENTS.md — no sandbox/browser automation): page loads, scans a real library, sub_type filter + badges, context menu actions.
|
||||
|
||||
## 8. Execution Order
|
||||
|
||||
1. `constants.py` + `OtherModelMetadata` + `config.py` roots
|
||||
2. `OtherScanner` (+ registry, factory, `PAGE_TYPE_MAP`) → scanner unit tests green
|
||||
3. `OtherModelService` + `OtherRoutes` + handler/registrar wiring + `lora_manager.py` lifecycle → route tests green
|
||||
4. Doctor/pending-delete/metadata-ops scatter entries
|
||||
5. Template + header nav + frontend API/controls/context-menu/card badges → vitest green
|
||||
6. i18n keys + sync script
|
||||
7. `pytest` + `npm test` full runs; hand to user for manual UI check
|
||||
|
||||
## 9. Phase 2 Detailed Design — CivitAI Downloads for `other`
|
||||
|
||||
Designed 2026-09-12 against the Phase-1 code on this branch; decisions marked **[locked]** follow the same recommendations the feature owner approved for Phase 1.
|
||||
|
||||
### 9.1 Download pipeline touch points
|
||||
|
||||
Flow: `POST /api/lm/download-model` (`py/routes/model_route_registrar.py:104`; GET variant `:105` for the browser extension) → `ModelDownloadHandler.download_model` (`model_handlers.py:1740`) → `DownloadModelUseCase.execute` → `DownloadCoordinator.schedule_download` → `DownloadManager.download_from_civitai` (`download_manager.py:386`) → `_execute_original_download` (`:1415`). Inside, seven scatter points need an `other` branch:
|
||||
|
||||
1. **Type map** (`:1496-1507`): accept `model.type.lower() in VALID_OTHER_CIVITAI_TYPES` → `model_type = "other"` (reuses the Phase-1 set, incl. `"other"` itself).
|
||||
2. **Early version-exists gate** (`:1436-1463`): add `other_scanner.check_model_version_exists`.
|
||||
3. **File-level exists gate** (`:1640-1655` → `_find_local_file_entry` `:320-346` → `_get_scanner_for_model_type` `:230-236`): add explicit `other` branch. **Trap**: the function currently falls through to the lora scanner for unknown types — `"other"` would silently dedupe against loras. Also narrow the fall-through to `"lora"` only / raise on unknown.
|
||||
4. **Version-level fallback gate** (`:1656-1688`): add `elif model_type == "other"`.
|
||||
5. **Default-root selection** (`:1690-1727`): for `other`, first resolve sub_type (§9.2), then read `default_other_roots[sub_type]` (§9.3); if sub_type is undecidable or no default root configured → error guiding the user to pick a folder explicitly.
|
||||
6. **Metadata class selection** (`:1909-1928`) + `_build_metadata_for_resume` (`:969-981`): add `OtherModelMetadata.from_civitai_info` branches.
|
||||
7. **Post-download cache write** (`_execute_download_pipeline` `:2622-2679`): add `other` scanner branch; `adjust_metadata` re-derives sub_type from the on-disk root automatically. `_get_supported_extensions_for_type` (`:2720-2744`): `other` reuses the checkpoint extension set.
|
||||
|
||||
Hooks: `_record_downloaded_version_history` (model_type is free text — zero change); `_sync_downloaded_version` (`:1984` → scanner dispatch `:2130-2135`) add `other`; `py/utils/example_images_download_manager.py` scanner dispatch at `:411-421`, `:591-601`, `:1089+` — add `other` at all three (silent no-scanner otherwise).
|
||||
|
||||
Path templates: `get_download_path_template("other")` is unset, so `other` resolves to a **flat** layout (empty template) — downloads land directly under the resolved sub_type root. This is deliberate: other-model roots are already split per sub_type (`default_other_roots`), and `priority_tags` has no `other` entry, so `{first_tag}` would fall back to an arbitrary CivitAI tag and scatter files into unstable folders. Users who want nesting can still set `download_path_templates["other"]` in `settings.json`. See `DEFAULT_DOWNLOAD_PATH_TEMPLATES` (`py/utils/constants.py`) and `DEFAULT_PATH_TEMPLATES` (`static/js/utils/constants.js`).
|
||||
|
||||
### 9.2 File-level routing (model.type / file.type → sub_type) **[locked]**
|
||||
|
||||
Table-driven, mirroring Phase 1. New in `py/utils/constants.py`:
|
||||
|
||||
```python
|
||||
CIVITAI_FILE_TYPE_TO_OTHER_SUB_TYPE = {
|
||||
"VAE": "vae", "Upscaler": "upscaler", "Text Encoder": "text_encoder",
|
||||
"Vision Encoder": "clip_vision", "CLIPVision": "clip_vision",
|
||||
"ControlNet": "controlnet",
|
||||
}
|
||||
```
|
||||
|
||||
`download_routing.py` gains `resolve_other_download_sub_type(civitai_model_type, file_types, selected_file_type=None)` with fixed priority:
|
||||
|
||||
1. **Explicit user file pick** (`file_params` from #1058's `_resolve_target_file`) — if the picked file's type maps, it wins even when model.type is `Checkpoint`.
|
||||
2. **model.type** via the existing `CIVITAI_TYPE_TO_OTHER_SUB_TYPE` (`constants.py:120-127`).
|
||||
3. **file.type fallback** — only when model.type maps to nothing (e.g. model.type `Other` or retired `CLIP`). MUST NOT override a mapped model.type: checkpoint models routinely bundle VAE/Text Encoder component files, and unconditional file-type routing would misroute them.
|
||||
4. Still undecidable → `None`; `use_default_paths` errors and the UI offers all other roots for manual selection.
|
||||
|
||||
HTTP: extend `DownloadRoutingHandler.get_download_routing` (`download_routing_handlers.py:23`) with an `other` branch returning `{root_kind: "other", sub_type: ...}`; add `GET /api/lm/other/roots_by_subtype` in `OtherRoutes.setup_specific_routes` (data from `config._prepare_other_paths`'s per-key roots, aggregating `text_encoders` + legacy `clip` under `text_encoder`).
|
||||
|
||||
### 9.3 Settings: single dict key `default_other_roots` **[locked]**
|
||||
|
||||
Rejected: four flat keys (`default_vae_root`…) — each flat key costs ~13 touch points in `settings_manager.py` (defaults `:82-85`, `_check_and_auto_set` `:890-895`, `set()` `:1621-1628`, `_update_active_library_entry` `:738-805`, upsert/create signatures `:1953-2132`, `_build_library_payload` `:552-612`, `_sync_active_library_to_root` `:519-547`, three library constructors, frontend `DEFAULT_SETTINGS_BASE`), repeated per future sub_type.
|
||||
|
||||
Chosen: one mapping key `default_other_roots: {sub_type: path}`, copying the `extra_folder_paths` precedent (generic Mapping handling at `:533-535`, `:573-578`, `:763-767`). `_check_and_auto_set` generalizes to per-sub_type candidates (union over that sub_type's folder keys — `text_encoder` → `text_encoders` + `clip`). `set()` validates keys against `VALID_OTHER_SUB_TYPES`.
|
||||
|
||||
Also fix the Phase-1 omission: add `"other_scanner"` to `_notify_library_change` (`:2150-2156`) and `_notify_model_name_display_change` (`:1795-1800`) — otherwise switching libraries leaves the other page stale.
|
||||
|
||||
### 9.4 Settings UI
|
||||
|
||||
- `templates/components/modals/settings/library.html:34-40`: sub_type selectors after the existing four `setting_select`s (Jinja loop; controlnet selector only when `enabled_other_folders` includes it). Dict-subkey save helper `saveOtherRootSetting(subType, value)` alongside the flat `saveSelectSetting`.
|
||||
- `static/js/managers/SettingsManager.js:1547-1697`: `loadOtherRoots()` mirroring `loadUnetRoots()`, fed by `/api/lm/other/roots_by_subtype`; current values from `state.global.settings.default_other_roots`. `state/index.js:24` `DEFAULT_SETTINGS_BASE` += `default_other_roots: {}`.
|
||||
- Optional: one `other` row in the download-path-template block (`library.html:153-211`).
|
||||
- i18n: `settings.folderSettings.*` keys into `locales/en.json` + sync script; other locales keep `[TODO: Translate]`.
|
||||
- Settings GET (`misc_handlers.py:1528-1536`) already returns all non-sensitive keys — new key reaches the frontend for free.
|
||||
|
||||
### 9.5 Frontend download entry
|
||||
|
||||
- `templates/components/controls.html:83`: drop the `page_id != 'other'` exclusion on the download button (keyboard shortcut D self-enables via `PageControls.js:196-198`).
|
||||
- `OtherControls.js:22-55`: add `showDownloadModal: () => downloadManager.showDownloadModal()` (mirror `EmbeddingsControls.js:43-45`).
|
||||
- `DownloadManager.js` `proceedToLocationContent` (`:955-1017`): add `_resolveOtherSubType()` (mirror `_resolveIsDiffusionModel` `:1026`): selected file type → `/api/lm/download/routing` → `otherApiClient.fetchModelRoots(subType)` (new); default-root preselect reads `default_other_roots[subType]` instead of `` `default_${singularType}_root` `` (`:974`). Undecidable → list all other roots (`/api/lm/other/roots`) for manual pick; an explicit save_dir skips backend default-root logic, so the two paths cannot disagree.
|
||||
- `ModelVersionsTab` download buttons are modelType-generic and already work via `getModelApiClient('other')`; context menu has no CivitAI download entry — no change.
|
||||
- Version-list type validation (`get_civitai_versions` → `_validate_civitai_model_type`) already accepts `VALID_OTHER_CIVITAI_TYPES` from Phase 1.
|
||||
|
||||
### 9.6 CivitAI type mapping decisions **[locked]**
|
||||
|
||||
- Download accepts exactly `VALID_OTHER_CIVITAI_TYPES` (`VAE, Upscaler, TextEncoder, CLIP, CLIPVision, Controlnet, Other`) — reuse the Phase-1 tables; do NOT create new ones.
|
||||
- Extend `CIVITAI_USER_MODEL_TYPES` (`constants.py:133-137`) with the 7 aliases, and point them at the other scanner / `"other"` history bucket in `misc_handlers.py` (`type_scanner_map` `:2793-2797`, `downloaded_version_map` `:2821-2827`) — otherwise creator pages silently filter these models while downloads claim support.
|
||||
- Fix (small Phase-1 bug): `OtherModelMetadata.from_civitai_info` (`py/utils/models.py:343`) reads `version_info.get("type")`, but the type lives at `version["model"]["type"]` — the mapping never fires and always degrades to the placeholder. Read `version_info.get("model", {}).get("type")` instead. (`CheckpointMetadata:290` has the same shape; leave it alone here.)
|
||||
|
||||
### 9.7 Tests
|
||||
|
||||
Existing base: `tests/services/test_download_manager_basic.py` (incl. `test_download_rejects_unsupported_model_type` `:1336`), `test_download_manager_error.py`, `test_download_manager_concurrent.py`, `tests/integration/test_download_flow.py`, `tests/services/test_settings_manager.py`; frontend `tests/frontend/managers/downloadManager.routing.test.js`, `settingsManager.library.test.js`.
|
||||
|
||||
Add: (1) `resolve_other_download_sub_type` unit tests — every priority tier, bundled-component anti-misrouting, undecidable → None, civarchive-shaped payload; (2) download_manager — six model.types accepted → other scanner (mock), unknown still rejected, no lora-scanner fall-through, per-sub_type default roots + unconfigured error, resume metadata, extension set; (3) settings_manager — `default_other_roots` defaults/auto-set (incl. text_encoder dual-key union)/library sync/upsert passthrough/illegal sub_type rejection; (4) routes — `/api/lm/download/routing` other branch, `roots_by_subtype` shape; (5) example-images dispatch accepts `other` (3 sites); (6) vitest — `_resolveOtherSubType` + root select + default preselect, `loadOtherRoots`; (7) user-models existsLocally for VAE.
|
||||
|
||||
### 9.8 Phase 2 file list
|
||||
|
||||
Backend: `py/utils/constants.py`, `py/services/download_routing.py`, `py/routes/handlers/download_routing_handlers.py`, `py/services/download_manager.py`, `py/utils/example_images_download_manager.py`, `py/services/settings_manager.py`, `py/utils/models.py`, `py/routes/other_routes.py`, `py/routes/handlers/misc_handlers.py`, `settings.json.example`.
|
||||
Frontend/templates: `templates/components/controls.html`, `static/js/components/controls/OtherControls.js`, `static/js/managers/DownloadManager.js`, `static/js/api/otherApi.js`, `templates/components/modals/settings/library.html`, `static/js/managers/SettingsManager.js`, `static/js/state/index.js`, `locales/en.json` + sync.
|
||||
|
||||
## 10. Risks / Open Questions
|
||||
|
||||
- **Root overlap**: a user may point `text_encoders` at a directory already scanned as checkpoints/unet. Realpath dedup inside one scanner won't catch cross-scanner overlap → the `_prepare_other_paths` overlap warning (§4.3) is the mitigation; duplicate cards across pages are cosmetic, not corrupting (cache keyed by `(model_type, file_path)`).
|
||||
- **Huge text encoders + lazy hash**: CivitAI fetch for a pending-hash model must trigger on-demand hash like checkpoints do — verify that flow (`calculate_hash_for_model`) is reachable from the `other` routes' fetch-metadata handler.
|
||||
- **Retired CivitAI types**: `CLIP`/`CLIPVision` are retired upstream (grandfathered for existing models); metadata fetch must tolerate both retired and current types — `VALID_OTHER_CIVITAI_TYPES` includes them deliberately.
|
||||
- **Standalone users** must add the new `folder_paths` keys to `settings.json` themselves; document in `settings.json.example` and the feature doc.
|
||||
- **Page display name** is i18n-only; if "Other Models" tests poorly, rename `other.title` without code changes.
|
||||
|
||||
### Phase 2 risks
|
||||
|
||||
- **Bundled component files**: checkpoint models routinely ship VAE/Text Encoder component files — file.type routing must stay a fallback (or explicit user pick), never an override (§9.2 priority is load-bearing; test it).
|
||||
- **`_get_scanner_for_model_type` lora fall-through** (`download_manager.py:236`): without an explicit `other` branch, dedupe checks run against the lora scanner — the most insidious trap in Phase 2.
|
||||
- **text_encoder dual folder keys** (`text_encoders` + legacy `clip`): default-root candidates, `roots_by_subtype`, and auto-set must all merge both keys; miss one and the default-root dropdown comes up empty.
|
||||
- **Undecidable sub_type** (model.type `Other` + unknown file types): must error and ask, never silently default to the vae folder.
|
||||
- **Lazy hash after download**: downloads carry CivitAI SHA256 (no recompute needed) — ensure the post-download cache write doesn't leave `hash_status="pending"`, or the next metadata fetch re-hashes a 10 GB file.
|
||||
- **CivArchive source**: same `_execute_original_download` path, same payload shape — cover it once in tests.
|
||||
|
||||
## 11. Phase 3 — Opt-in Management Toggles (implemented)
|
||||
|
||||
Designed 2026-09-13 against the Phase-1/2 code. Other Models is **opt-in**: after
|
||||
Phase 3 the feature ships disabled, so no other-model folder is scanned and the
|
||||
page shows an "enable" empty state until the user turns it on.
|
||||
|
||||
### 11.1 Settings (global, not per-library)
|
||||
|
||||
| key | type | default | meaning |
|
||||
|---|---|---|---|
|
||||
| `enable_other_models` | bool | `false` | master switch |
|
||||
| `enabled_other_sub_types` | list[str] | `["vae","upscaler","text_encoder"]` | allow-list; `clip_vision` and `controlnet` are opt-in (see §2) |
|
||||
|
||||
`enabled_other_folders` (the unreleased, additive, no-UI backend key) was removed
|
||||
and replaced by the sub_type-level allow-list; there is no migration because the
|
||||
feature never shipped. `text_encoder` expands to `text_encoders` + legacy `clip`
|
||||
via `OTHER_SUB_TYPE_FOLDER_KEYS`.
|
||||
|
||||
The default allow-list lives on five surfaces that must stay in sync:
|
||||
`DEFAULT_ENABLED_OTHER_SUB_TYPES` (`py/utils/constants.py`), `DEFAULT_SETTINGS`
|
||||
(`py/services/settings_manager.py`), the two `DEFAULT_SETTINGS_BASE` /
|
||||
`createDefaultSettings` lists (`static/js/state/index.js`), the
|
||||
`updateOtherModelsControls()` fallback (`static/js/managers/SettingsManager.js`)
|
||||
and the server-rendered Jinja fallback
|
||||
(`templates/components/modals/settings/library.html`).
|
||||
|
||||
### 11.1.1 Legacy key handling in `Config._init_other_paths`
|
||||
|
||||
ComfyUI's `folder_paths` rewrites legacy names before every access (`clip` →
|
||||
`text_encoders`, `unet` → `diffusion_models`) and registers both legacy
|
||||
directories under the canonical key, so `get_folder_paths("clip")` returns
|
||||
exactly the same list as `get_folder_paths("text_encoders")`. Querying both keys
|
||||
made the overlap guard fire twice with `please fix your path configuration` for a
|
||||
configuration the user cannot fix. `Config._collapse_legacy_folder_keys()` now
|
||||
drops a key when the host exposes `map_legacy` and resolves it to another queried
|
||||
key, and `_prepare_other_paths()` downgrades a same-`sub_type` duplicate to
|
||||
`debug` (a cross-`sub_type` collision still warns). In standalone mode
|
||||
`MockFolderPaths` has no `map_legacy` and its keys are independent
|
||||
`settings.json` entries, so every key is still queried there.
|
||||
|
||||
`settings.json.example` intentionally stays minimal (only `use_portable_settings`,
|
||||
`civitai_api_key`, and the four core `folder_paths` keys: `loras`, `checkpoints`,
|
||||
`unet`, `embeddings`). Optional keys — including the other-model folder paths and
|
||||
`enable_other_models` — are NOT documented there; they live in `DEFAULT_SETTINGS`
|
||||
and reach the user's `settings.json` on demand. This supersedes the Phase-1/Phase-2
|
||||
notes that proposed adding the other-model folder keys to the example.
|
||||
|
||||
### 11.2 Behaviour matrix
|
||||
|
||||
| state | scan | nav / `/other` | other downloads | `default_other_roots` | Doctor / refresh-all |
|
||||
|---|---|---|---|---|---|
|
||||
| master off | nothing (`other_roots == []`) | nav entry hidden (`nav-item--hidden`); `/other` still renders the disabled empty state + Enable button; one-time dismissible announcement banner on first visit | rejected | preserved, never auto-set | scanner skipped |
|
||||
| sub_type off | that sub_type's folder keys excluded | page keeps working, type disappears from data | auto-routing refused (manual folder still allowed) | preserved, not preselected | normal |
|
||||
| all on (after enabling) | Phase-1/2 behaviour | normal | normal | normal | normal |
|
||||
|
||||
### 11.3 Backend touch points
|
||||
|
||||
- `py/utils/constants.py` — `DEFAULT_ENABLED_OTHER_SUB_TYPES`, `OTHER_SUB_TYPE_FOLDER_KEYS`, `normalize_other_sub_types`.
|
||||
- `py/config.py` — `_get_enabled_other_folder_keys()` is the single scan gate (master switch + allow-list); new `refresh_other_roots()` rebuilds roots + preview roots on toggle.
|
||||
- `py/services/settings_manager.py` — new defaults, `set()` normalization, `is_other_models_enabled()` / `get_enabled_other_sub_types()` / `is_other_sub_type_enabled()`, and `_apply_other_model_settings_change()` which reapplies config and calls `other_scanner.on_library_changed(reconcile=True)`.
|
||||
- `py/services/model_scanner.py` — `_should_keep_cached_entry()` hydration hook (default keep) plus `on_library_changed(reconcile=...)` / `initialize_in_background(reconcile=...)`; the hook filters `raw_data` and the hash/autov3 index rows.
|
||||
- `py/services/other_scanner.py` — drops persisted entries whose folder is no longer a managed root (sub_type is location-derived, so config is the source of truth).
|
||||
- `py/routes/other_routes.py` — `_validate_civitai_model_type` rejects everything while off / mapped-but-disabled sub_types; `_get_page_context_provider()` injects `other_disabled` into the template.
|
||||
- `py/routes/handlers/model_handlers.py` + `base_model_routes.py` — optional `page_context_provider` hook on `ModelPageView`.
|
||||
- `py/routes/handlers/download_routing_handlers.py` — returns `{sub_type: None, disabled: true, reason}` instead of guessing.
|
||||
- `py/services/download_manager.py` — rejects other-type downloads while off; disabled sub_type refuses default-path routing with a "pick a folder" error.
|
||||
- `py/routes/handlers/misc_handlers.py` — Doctor / init-status / refresh-all skip the other scanner while off (`_active_scanner_factories` / `_active_scanner_getters`).
|
||||
- `py/services/pending_delete_service.py` — deliberately untouched: the scanner stays registered so staged deletes still merge.
|
||||
|
||||
### 11.4 Frontend
|
||||
|
||||
Discoverability: the nav entry is hidden while the feature is off, and three
|
||||
lightweight surfaces replace it — a one-time announcement banner, the download
|
||||
toast, and the settings toggle itself.
|
||||
|
||||
- `templates/components/header.html` + `static/css/components/header.css` — `nav-item--hidden` class (server-rendered when off, client-toggled after enabling) and the `fa-shapes` icon.
|
||||
- `templates/other.html` — `other_disabled` branch in `content` + `main_script`; page-scoped CSS for the empty state.
|
||||
- `static/js/other_disabled.js` — boots `appCore` (shared header) and delegates to the shared enable helper.
|
||||
- `static/js/utils/otherModels.js` — shared `enableOtherModels()` (POST settings + reload) and `openOtherModelsSettings()` (settings modal on the Library section); used by the disabled page, the banner and the download modal.
|
||||
- `static/js/managers/BannerService.js` — `other-models-announcement` banner (only when off and not dismissed; `priority: 0`, dismissal persisted via `dismissed_banners`) with Enable / Open Settings actions; `removeOtherModelsAnnouncement()` drops it without persisting a dismissal.
|
||||
- `templates/components/modals/settings/library.html` + `SettingsManager.updateOtherModelsControls()` / `saveEnabledOtherSubTypes()` / `updateOtherModelsNavVisibility()` — master toggle + five sub_type checkboxes; unchecked/disabled sub_types have their default-root select disabled.
|
||||
- `static/js/managers/DownloadManager.js` — a disabled routing answer surfaces a `showActionToast` with an "Enable Other Models" action (opening settings) and falls back to manual selection.
|
||||
- i18n: `settings.folderSettings.*`, `other.disabled.*` and `banners.otherModels.*` keys in `locales/en.json` + `scripts/sync_translation_keys.py` (other locales keep `[TODO: Translate]`).
|
||||
|
||||
### 11.5 Cache consistency
|
||||
|
||||
- Disabling purges rows from the in-memory view at hydration time (the
|
||||
`_should_keep_cached_entry` hook) and from SQLite on the reconcile triggered by
|
||||
the toggle; the `.metadata.json` sidecars survive, so re-enabling rescans
|
||||
without recomputing hashes (critical for multi-GB text encoders).
|
||||
- Enabling triggers a reconcile so newly managed roots are scanned immediately.
|
||||
- Editing `settings.json` while the server is stopped is still covered by the
|
||||
hydration hook, so disabled types never appear after a restart.
|
||||
|
||||
### 11.6 Tests
|
||||
|
||||
Backend: opt-in fixtures added to the other-related suites; new coverage for
|
||||
"default off scans nothing", per-sub_type gating, routing/download rejection,
|
||||
`_should_keep_cached_entry`, settings normalization and `other_disabled` page
|
||||
context. Frontend: `updateOtherModelsControls` / `saveEnabledOtherSubTypes` and
|
||||
the disabled-page enable flow.
|
||||
@@ -0,0 +1,58 @@
|
||||
# CivitAI image imports can end up with 0 LoRAs
|
||||
|
||||
## Symptom
|
||||
|
||||
Importing a CivitAI image URL can produce a recipe with **zero LoRA
|
||||
entries**, even though the image page lists LoRAs in its resource panel.
|
||||
|
||||
Reported example: `https://civitai.red/images/140818889` was imported as a
|
||||
local recipe with 0 LoRAs, while the page shows 3 LoRAs. Some images (e.g.
|
||||
NSFW / higher browsing level) additionally require a login to view, so their
|
||||
data is not publicly reachable at all.
|
||||
|
||||
## Root cause
|
||||
|
||||
URL imports use only two data sources:
|
||||
|
||||
1. **CivitAI REST image API** — `GET /api/v1/images?imageId=<id>&nsfw=X&withMeta=true` → `meta`
|
||||
2. **Embedded image metadata** — EXIF/XMP read from the downloaded bytes
|
||||
|
||||
For the same image both sources can be empty, and the one source that does
|
||||
contain the data is never queried. Verified for image 140818889:
|
||||
|
||||
| Source | What it returned |
|
||||
|---|---|
|
||||
| REST image API | `meta` holds only a prompt; `modelVersionIds: []`; no `resources`/`hashes`; `baseModel: null` |
|
||||
| Downloaded image | PNG with **no EXIF/XMP** (the CDN URL ends in `.jpeg`, the body is PNG) |
|
||||
| Image page HTML | `__NEXT_DATA__` embeds the trpc `image.getGenerationData` result → full `resources` list: 3 LoRAs, each with `modelId`, `modelVersionId`, `modelName`, `modelType`, `versionName`, `baseModel` |
|
||||
|
||||
Key points:
|
||||
|
||||
- The page's resource panel is fed by an **internal, non-public trpc
|
||||
endpoint**, not by the public REST image API.
|
||||
- That internal endpoint is **login-gated** for some content — the
|
||||
"requires login" symptom.
|
||||
- Even with the version IDs in hand, `/model-versions/{id}` for these
|
||||
(Krea) versions returns **no `sha256`**, so an exact local-file hash match
|
||||
is impossible; only model/version identity is recoverable.
|
||||
|
||||
## Conclusion / status
|
||||
|
||||
0-LoRA imports are a data-source gap: public REST meta and image EXIF are
|
||||
both empty, while the only complete source (page generation data) is
|
||||
internal, sometimes login-gated, and not used by the importer.
|
||||
|
||||
Such imports **cannot be reliably auto-repaired/completed** by the backend
|
||||
alone. The old "Repair Metadata" feature only re-fetched the same incomplete
|
||||
REST meta and could not fix them; it was deprecated and has been removed.
|
||||
|
||||
**Fixed via the companion browser extension.** When the extension is
|
||||
installed with a valid license, it scrapes the image page's internal trpc
|
||||
generation data with the user's session and calls the payload-capable
|
||||
re-import endpoint (`POST /api/lm/recipe/{recipe_id}/reimport` with
|
||||
`image_url`/`name`/`resources`/`gen_params`/`base_model`/`tags` query
|
||||
params), which rebuilds the recipe from the caller-supplied metadata. The
|
||||
web UI delegates re-import of CivitAI-image-sourced recipes to the extension
|
||||
automatically (probe + `lm:reimport*` DOM events); without the extension,
|
||||
re-import silently falls back to the native path, which remains limited by
|
||||
the data-source gap documented above.
|
||||
@@ -0,0 +1,92 @@
|
||||
# Reconcile 的 Windows 大小写回退分支 - 待验证清单
|
||||
|
||||
> **状态**: 待 Windows 环境验证 | **创建日期**: 2026-09-11
|
||||
> **相关文件**: `py/services/model_scanner.py` (`ModelScanner._reconcile_cache`)
|
||||
> **相关历史**: #871 (`76ee59cd`, 路径重叠去重)、#1108 (按文件夹扫描的需求)
|
||||
|
||||
---
|
||||
|
||||
## 背景
|
||||
|
||||
Refresh 按钮走的是 `_reconcile_cache()`(快速增量对账)。2026-09-11 做了一轮性能优化,把两处"预防性"的
|
||||
realpath 全量遍历改成按需触发(详见下方"已完成")。优化后,一次零变更 Refresh 在 5 万文件库上从
|
||||
~1400 ms 降到 ~120 ms。
|
||||
|
||||
清理过程中发现**唯一一处遗留的可疑点**:Windows 专属的大小写不敏感回退分支。它无法在 Linux 上验证,
|
||||
因此单独记录,留待 Windows 机器上确认。
|
||||
|
||||
---
|
||||
|
||||
## 待验证分支(现状)
|
||||
|
||||
`py/services/model_scanner.py` 中 `_reconcile_cache()` 的 walk 循环内:
|
||||
|
||||
```python
|
||||
# Try case-insensitive match on Windows
|
||||
if os.name == 'nt':
|
||||
lower_path = file_path.lower()
|
||||
matched = False
|
||||
for cached_path in cached_paths: # 每个未命中文件都全量扫一遍缓存
|
||||
if cached_path.lower() == lower_path:
|
||||
found_paths.add(cached_path)
|
||||
matched = True
|
||||
break
|
||||
if matched:
|
||||
continue
|
||||
```
|
||||
|
||||
它排在精确匹配(`file_path in cached_paths`)和 realpath 别名匹配之后,只有**未命中**的文件才会走到。
|
||||
|
||||
### 为什么可疑
|
||||
|
||||
1. **可能不可达**:Windows 上 `os.path.realpath()` 会返回磁盘上的真实大小写,因此"缓存路径大小写与磁盘
|
||||
不一致"的情形,理论上已经被上一步的 realpath 别名匹配覆盖。若如此,这段就是纯冗余代码。
|
||||
2. **一旦可达就是 O(N×M)**:每个未命中文件都要遍历全部 `cached_paths` 做小写比较。若某种路径写法让
|
||||
整个库都变成"未命中"(例如缓存里的盘符/大小写形式与 walk 结果系统性不一致),一次 Refresh 会退化
|
||||
成 文件数 × 缓存条目数 次字符串比较,比真实 IO 还贵。
|
||||
3. **没有测试覆盖**:`tests/services/test_model_scanner.py` 没有任何针对该分支的用例(它在 Linux 上
|
||||
被 `os.name == 'nt'` 短路,无法覆盖)。
|
||||
|
||||
---
|
||||
|
||||
## 待办
|
||||
|
||||
- [ ] **验证可达性**:在 Windows 上构造"缓存路径与磁盘真实大小写不一致"的场景,确认 realpath 别名匹配
|
||||
是否已经命中,即上面的 `if os.name == 'nt'` 分支是否还有进入的必要。
|
||||
- [ ] **若不可达 / 冗余**:删除该分支,并在删除处留注释说明 realpath 已覆盖大小写归一(附验证记录)。
|
||||
- [ ] **若可达**:保留语义但改成 O(1)——预先构建一次 `lower_path -> cached_path` 映射(与
|
||||
`cached_real_paths` 同样按需、懒构建),把内层全量扫描换成一次字典查询。
|
||||
- [ ] **补一个 Windows-only 的回归测试**(`pytest.mark.skipif(os.name != "nt", ...)`),锁定最终结论。
|
||||
- [ ] 把验证结论回填到本文件,并同步更新状态行。
|
||||
|
||||
---
|
||||
|
||||
## 验证方法(Windows)
|
||||
|
||||
1. **构造不一致的大小写**:让缓存里的 `file_path` 与磁盘实际路径大小写不同(例如改过盘符/目录大小写,
|
||||
或从另一台机器迁移了 `settings.json` 与持久化缓存),然后在 UI 点 Refresh。
|
||||
2. **看后端日志判据**:
|
||||
- 若 realpath 已覆盖 → 日志应显示 `Cache reconciliation completed in X seconds. Added 0, removed 0 models.`,
|
||||
且**没有** `Found N new files to process` / `Processing <path>`。
|
||||
- 若回退分支在起作用 → 同样应该是 `Added 0, removed 0`(因为 `found_paths` 被补上),这是"分支可达"
|
||||
的证据;反之若出现大量 `Processing ...` 并重新 hash,说明连回退分支也没命中,问题更严重
|
||||
(缓存路径被当成了新文件 + 旧条目被删)。
|
||||
3. **跑测试**:`python -m pytest tests/services/test_model_scanner.py -k reconcile`(该文件在 Windows 上会
|
||||
真实执行 `os.name == 'nt'` 分支)。
|
||||
4. **量化**:如果需要,可在 `_reconcile_cache` 里临时插桩统计该分支的进入次数与内层迭代次数,确认是否为 0。
|
||||
|
||||
---
|
||||
|
||||
## 已完成(本轮优化,供对照)
|
||||
|
||||
同一次清理里已经落地并验证的部分(Linux,5 万文件库):
|
||||
|
||||
- `cached_real_paths` 别名映射改为**首次未命中时**懒构建(原来每次 Refresh 都对全部缓存条目算一次 realpath)。
|
||||
- 每个文件的 `realpath` 移到精确命中检查**之后**(原来对每个文件都算,命中即丢弃)。
|
||||
- `get_model_roots()` 在新增文件处理阶段只快照一次(原来每个新文件重读一次)。
|
||||
- 全量去重 pass 加了 O(1) 前置判断(`cached_size_before != len(cached_paths) or total_added > 0`),
|
||||
零变更且缓存干净时跳过;快照本身含重复路径时仍会自愈。
|
||||
|
||||
结果:零变更 Refresh 5 万文件 **~1400 ms → ~120 ms**;根目录顺序/符号链接别名翻转场景仍是
|
||||
`re-processed=0`(不重新读 metadata、不重新 hash)。测试:`tests/services/test_model_scanner.py`
|
||||
47 项、全量后端 2567 项全部通过。
|
||||
+608
-247
File diff suppressed because it is too large
Load Diff
+470
-109
File diff suppressed because it is too large
Load Diff
+614
-253
File diff suppressed because it is too large
Load Diff
+626
-265
File diff suppressed because it is too large
Load Diff
+639
-278
File diff suppressed because it is too large
Load Diff
+572
-211
File diff suppressed because it is too large
Load Diff
+576
-215
File diff suppressed because it is too large
Load Diff
+592
-231
File diff suppressed because it is too large
Load Diff
+506
-145
File diff suppressed because it is too large
Load Diff
+523
-162
File diff suppressed because it is too large
Load Diff
+276
-1
@@ -17,6 +17,9 @@ import types as _types
|
||||
import time
|
||||
|
||||
from .utils.cache_paths import CacheType, get_cache_file_path, get_legacy_cache_paths
|
||||
from .utils.constants import (
|
||||
OTHER_MODEL_FOLDER_SUBTYPES,
|
||||
)
|
||||
from .utils.settings_paths import (
|
||||
ensure_settings_file,
|
||||
get_settings_dir,
|
||||
@@ -172,6 +175,13 @@ class Config:
|
||||
self.embeddings_roots = None
|
||||
self.base_models_roots = self._init_checkpoint_paths()
|
||||
self.embeddings_roots = self._init_embedding_paths()
|
||||
# Other-model roots (VAE, upscalers, text encoders, ...): flat deduped
|
||||
# list plus a normalized root -> sub_type map and per-folder_paths-key
|
||||
# roots for settings persistence.
|
||||
self.other_roots: Optional[List[str]] = None
|
||||
self.other_root_subtypes: Dict[str, str] = {}
|
||||
self.other_folder_roots: Dict[str, List[str]] = {}
|
||||
self.other_roots = self._init_other_paths()
|
||||
# Extra paths (only for LoRA Manager, not shared with ComfyUI)
|
||||
self.extra_loras_roots: List[str] = []
|
||||
self.extra_checkpoints_roots: List[str] = []
|
||||
@@ -336,6 +346,10 @@ class Config:
|
||||
"unet": list(self.unet_roots or []),
|
||||
"embeddings": list(self.embeddings_roots or []),
|
||||
}
|
||||
# Persist the other-model roots under their original folder_paths
|
||||
# keys so library switching round-trips them.
|
||||
for key, roots in (self.other_folder_roots or {}).items():
|
||||
target_folder_paths[key] = list(roots)
|
||||
|
||||
normalized_target_paths = _normalize_folder_paths_for_comparison(
|
||||
target_folder_paths
|
||||
@@ -522,6 +536,7 @@ class Config:
|
||||
roots.extend(self.loras_roots or [])
|
||||
roots.extend(self.base_models_roots or [])
|
||||
roots.extend(self.embeddings_roots or [])
|
||||
roots.extend(self.other_roots or [])
|
||||
# Include extra paths for scanning symlinks
|
||||
roots.extend(self.extra_loras_roots or [])
|
||||
roots.extend(self.extra_checkpoints_roots or [])
|
||||
@@ -862,6 +877,8 @@ class Config:
|
||||
preview_roots.update(self._expand_preview_root(root))
|
||||
for root in self.embeddings_roots or []:
|
||||
preview_roots.update(self._expand_preview_root(root))
|
||||
for root in self.other_roots or []:
|
||||
preview_roots.update(self._expand_preview_root(root))
|
||||
# Include extra paths for preview access
|
||||
for root in self.extra_loras_roots or []:
|
||||
preview_roots.update(self._expand_preview_root(root))
|
||||
@@ -882,7 +899,7 @@ class Config:
|
||||
path for path in preview_roots if path.is_absolute()
|
||||
}
|
||||
logger.debug(
|
||||
"Preview roots rebuilt: %d paths from %d lora roots (%d extra), %d checkpoint roots (%d extra), %d embedding roots (%d extra), %d symlink mappings",
|
||||
"Preview roots rebuilt: %d paths from %d lora roots (%d extra), %d checkpoint roots (%d extra), %d embedding roots (%d extra), %d other roots, %d symlink mappings",
|
||||
len(self._preview_root_paths),
|
||||
len(self.loras_roots or []),
|
||||
len(self.extra_loras_roots or []),
|
||||
@@ -890,6 +907,7 @@ class Config:
|
||||
len(self.extra_checkpoints_roots or []),
|
||||
len(self.embeddings_roots or []),
|
||||
len(self.extra_embeddings_roots or []),
|
||||
len(self.other_roots or []),
|
||||
len(self._path_mappings),
|
||||
)
|
||||
|
||||
@@ -1128,6 +1146,155 @@ class Config:
|
||||
|
||||
return unique_paths
|
||||
|
||||
def _get_enabled_other_folder_keys(self) -> List[str]:
|
||||
"""Return the OTHER_MODEL_FOLDER_SUBTYPES keys that are enabled.
|
||||
|
||||
Other Models management is opt-in: while ``enable_other_models`` is
|
||||
off (the default) no other-model folder is scanned at all. When it is
|
||||
on, only the folder keys of the enabled sub_types are scanned
|
||||
(text_encoder merges ``text_encoders`` with the legacy ``clip`` key).
|
||||
"""
|
||||
try:
|
||||
from .services.settings_manager import get_settings_manager
|
||||
|
||||
enabled_sub_types = get_settings_manager().get_enabled_other_sub_types()
|
||||
except Exception:
|
||||
enabled_sub_types = []
|
||||
if not enabled_sub_types:
|
||||
return []
|
||||
allowed = set(enabled_sub_types)
|
||||
return [
|
||||
key
|
||||
for key, sub_type in OTHER_MODEL_FOLDER_SUBTYPES.items()
|
||||
if sub_type in allowed
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _collapse_legacy_folder_keys(keys: List[str]) -> List[str]:
|
||||
"""Drop folder keys the host already normalizes onto another queried key.
|
||||
|
||||
ComfyUI's ``folder_paths`` rewrites legacy names before every access
|
||||
(``clip`` -> ``text_encoders``, ``unet`` -> ``diffusion_models``), and
|
||||
registers both legacy directories under the canonical key, so
|
||||
``get_folder_paths("clip")`` returns exactly the same list as
|
||||
``get_folder_paths("text_encoders")``. Querying both therefore reports
|
||||
every text-encoder folder twice and trips the overlap guard with a
|
||||
conflict the user cannot fix.
|
||||
|
||||
When the host exposes ``map_legacy`` the alias is provably redundant and
|
||||
is skipped (an empty canonical list implies an empty alias list).
|
||||
Without it - the standalone mock, whose keys are independent
|
||||
``settings.json`` entries - every key is kept, because a ``clip``-only
|
||||
configuration is then genuinely distinct.
|
||||
"""
|
||||
map_legacy = getattr(folder_paths, "map_legacy", None)
|
||||
if not callable(map_legacy):
|
||||
return list(keys)
|
||||
|
||||
queried = set(keys)
|
||||
collapsed: List[str] = []
|
||||
for key in keys:
|
||||
try:
|
||||
canonical = map_legacy(key)
|
||||
except Exception:
|
||||
canonical = key
|
||||
if canonical != key and canonical in queried:
|
||||
logger.debug(
|
||||
"Skipping legacy folder key '%s'; the host resolves it to "
|
||||
"'%s', which is queried as well.",
|
||||
key,
|
||||
canonical,
|
||||
)
|
||||
continue
|
||||
collapsed.append(key)
|
||||
return collapsed
|
||||
|
||||
def _prepare_other_paths(
|
||||
self, folder_path_map: Mapping[str, Iterable[str]]
|
||||
) -> Tuple[List[str], Dict[str, str], Dict[str, List[str]]]:
|
||||
"""Prepare other-model paths from a folder_paths-key -> raw paths map.
|
||||
|
||||
Returns:
|
||||
Tuple of (all_unique_roots, business_root -> sub_type map,
|
||||
folder_paths key -> business roots). This method does NOT modify
|
||||
instance variables - callers must set them.
|
||||
"""
|
||||
unique_paths: List[str] = []
|
||||
sub_type_map: Dict[str, str] = {}
|
||||
per_key_roots: Dict[str, List[str]] = {}
|
||||
# real path -> (business path, sub_type) of the category that claimed it
|
||||
seen_real_paths: Dict[str, Tuple[str, str]] = {}
|
||||
|
||||
# Cross-scanner overlap detection: warn when an "other" root is
|
||||
# already covered by the checkpoints/unet or embeddings scanners.
|
||||
# Kept (not dropped) on purpose - duplicate cards across pages are
|
||||
# cosmetic, while dropping would silently unmanage the files.
|
||||
covered_real_paths = {
|
||||
os.path.normpath(os.path.realpath(path)).replace(os.sep, "/"): path
|
||||
for path in [
|
||||
*(self.base_models_roots or []),
|
||||
*(self.embeddings_roots or []),
|
||||
]
|
||||
if isinstance(path, str) and path.strip() and os.path.exists(path)
|
||||
}
|
||||
|
||||
for key, sub_type in OTHER_MODEL_FOLDER_SUBTYPES.items():
|
||||
raw_paths = folder_path_map.get(key)
|
||||
if not raw_paths:
|
||||
continue
|
||||
path_map = self._dedupe_existing_paths(raw_paths)
|
||||
key_roots: List[str] = []
|
||||
for real_path, business_path in sorted(
|
||||
path_map.items(), key=lambda item: item[1].lower()
|
||||
):
|
||||
seen = seen_real_paths.get(real_path)
|
||||
if seen is not None:
|
||||
seen_business_path, seen_sub_type = seen
|
||||
if seen_sub_type == sub_type:
|
||||
# Same category reached through a second folder_paths
|
||||
# key (legacy alias, or a sub_type spanning two keys).
|
||||
# Expected, so never a "fix your configuration" warning.
|
||||
logger.debug(
|
||||
"Ignoring duplicate folder '%s' for category '%s' "
|
||||
"(already covered by '%s').",
|
||||
business_path,
|
||||
sub_type,
|
||||
seen_business_path,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Detected the same folder '%s' under multiple other-model "
|
||||
"categories ('%s' is already mapped as '%s'). Keeping the "
|
||||
"first category; please fix your path configuration.",
|
||||
business_path,
|
||||
seen_business_path,
|
||||
seen_sub_type,
|
||||
)
|
||||
continue
|
||||
seen_real_paths[real_path] = (business_path, sub_type)
|
||||
unique_paths.append(business_path)
|
||||
key_roots.append(business_path)
|
||||
sub_type_map[business_path] = sub_type
|
||||
|
||||
if real_path != business_path:
|
||||
self.add_path_mapping(business_path, real_path)
|
||||
|
||||
covered_by = covered_real_paths.get(real_path)
|
||||
if covered_by:
|
||||
logger.warning(
|
||||
"Detected an other-model root ('%s', category '%s') that "
|
||||
"overlaps an existing checkpoints/embeddings root ('%s'). "
|
||||
"The same files will appear on both pages; please review "
|
||||
"your path configuration.",
|
||||
business_path,
|
||||
key,
|
||||
covered_by,
|
||||
)
|
||||
if key_roots:
|
||||
per_key_roots[key] = key_roots
|
||||
|
||||
return unique_paths, sub_type_map, per_key_roots
|
||||
|
||||
def _apply_library_paths(
|
||||
self,
|
||||
folder_paths: Mapping[str, Any],
|
||||
@@ -1151,6 +1318,16 @@ class Config:
|
||||
) = self._prepare_checkpoint_paths(checkpoint_paths, unet_paths)
|
||||
self.embeddings_roots = self._prepare_embedding_paths(embedding_paths)
|
||||
|
||||
other_path_map = {
|
||||
key: folder_paths.get(key, []) or []
|
||||
for key in self._get_enabled_other_folder_keys()
|
||||
}
|
||||
(
|
||||
self.other_roots,
|
||||
self.other_root_subtypes,
|
||||
self.other_folder_roots,
|
||||
) = self._prepare_other_paths(other_path_map)
|
||||
|
||||
# Process extra paths (only for LoRA Manager, not shared with ComfyUI)
|
||||
extra_paths = extra_folder_paths or {}
|
||||
extra_lora_paths = extra_paths.get("loras", []) or []
|
||||
@@ -1267,6 +1444,104 @@ class Config:
|
||||
logger.warning(f"Error initializing embedding paths: {e}")
|
||||
return []
|
||||
|
||||
def _init_other_paths(self) -> List[str]:
|
||||
"""Initialize and validate other-model paths from ComfyUI settings.
|
||||
|
||||
Iterates the enabled OTHER_MODEL_FOLDER_SUBTYPES keys and pulls each
|
||||
from ``folder_paths.get_folder_paths(key)`` (in standalone mode the
|
||||
mock serves arbitrary keys from ``settings.json.folder_paths``).
|
||||
Legacy aliases the host normalizes onto a canonical key (``clip`` ->
|
||||
``text_encoders``) are collapsed first so the same folders are not
|
||||
reported twice.
|
||||
"""
|
||||
try:
|
||||
folder_path_map: Dict[str, List[str]] = {}
|
||||
for key in self._collapse_legacy_folder_keys(
|
||||
self._get_enabled_other_folder_keys()
|
||||
):
|
||||
try:
|
||||
folder_path_map[key] = folder_paths.get_folder_paths(key)
|
||||
except Exception as exc:
|
||||
logger.debug("Error reading folder paths for '%s': %s", key, exc)
|
||||
|
||||
(
|
||||
unique_paths,
|
||||
self.other_root_subtypes,
|
||||
self.other_folder_roots,
|
||||
) = self._prepare_other_paths(folder_path_map)
|
||||
|
||||
logger.info(
|
||||
"Found other model roots:"
|
||||
+ ("\n - " + "\n - ".join(unique_paths) if unique_paths else "[]")
|
||||
)
|
||||
|
||||
if not unique_paths:
|
||||
logger.info("No valid other-model folders found in configuration")
|
||||
return []
|
||||
|
||||
return unique_paths
|
||||
except Exception as e:
|
||||
logger.warning(f"Error initializing other model paths: {e}")
|
||||
return []
|
||||
|
||||
def refresh_other_roots(self) -> None:
|
||||
"""Rebuild other-model roots after the management toggles changed.
|
||||
|
||||
Called when ``enable_other_models`` / ``enabled_other_sub_types`` are
|
||||
updated so the scanner immediately reflects the new folder set without
|
||||
a full application restart.
|
||||
"""
|
||||
self.other_roots = self._init_other_paths()
|
||||
self._rebuild_preview_roots()
|
||||
|
||||
def get_other_models_availability(self) -> Dict[str, Any]:
|
||||
"""Report the other-model folders the host can actually expose.
|
||||
|
||||
Independent of the opt-in ``enable_other_models`` toggle: this answers
|
||||
"could Other Models management work here at all?". ComfyUI mode almost
|
||||
always has these folder keys registered, while standalone mode only
|
||||
knows the keys present in ``settings.json.folder_paths`` - so the UI
|
||||
uses this to decide whether announcing the feature would be actionable.
|
||||
|
||||
Returns:
|
||||
``{"available": bool, "sub_types": {sub_type: [existing roots]}}``.
|
||||
A folder only counts when it exists on disk; an empty folder still
|
||||
counts because CivitAI downloads can target it.
|
||||
"""
|
||||
sub_types: Dict[str, List[str]] = {}
|
||||
try:
|
||||
keys = self._collapse_legacy_folder_keys(
|
||||
list(OTHER_MODEL_FOLDER_SUBTYPES.keys())
|
||||
)
|
||||
except Exception: # pragma: no cover - defensive
|
||||
keys = list(OTHER_MODEL_FOLDER_SUBTYPES.keys())
|
||||
|
||||
for key in keys:
|
||||
sub_type = OTHER_MODEL_FOLDER_SUBTYPES.get(key)
|
||||
if not sub_type:
|
||||
continue
|
||||
try:
|
||||
raw_paths = folder_paths.get_folder_paths(key)
|
||||
except Exception as exc:
|
||||
logger.debug("Error probing folder paths for '%s': %s", key, exc)
|
||||
continue
|
||||
|
||||
bucket = sub_types.setdefault(sub_type, [])
|
||||
for root in sorted(
|
||||
self._dedupe_existing_paths(raw_paths or []).values(),
|
||||
key=lambda path: path.lower(),
|
||||
):
|
||||
if root not in bucket:
|
||||
bucket.append(root)
|
||||
|
||||
available_sub_types = {
|
||||
sub_type: roots for sub_type, roots in sub_types.items() if roots
|
||||
}
|
||||
return {
|
||||
"available": bool(available_sub_types),
|
||||
"sub_types": available_sub_types,
|
||||
}
|
||||
|
||||
def get_preview_static_url(self, preview_path: str) -> str:
|
||||
if not preview_path:
|
||||
return ""
|
||||
|
||||
+7
-8
@@ -219,6 +219,7 @@ class LoraManager:
|
||||
lora_scanner = await ServiceRegistry.get_lora_scanner()
|
||||
checkpoint_scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||
embedding_scanner = await ServiceRegistry.get_embedding_scanner()
|
||||
other_scanner = await ServiceRegistry.get_other_scanner()
|
||||
|
||||
# Initialize recipe scanner if needed
|
||||
recipe_scanner = await ServiceRegistry.get_recipe_scanner()
|
||||
@@ -236,6 +237,10 @@ class LoraManager:
|
||||
embedding_scanner.initialize_in_background(),
|
||||
name="embedding_cache_init",
|
||||
),
|
||||
asyncio.create_task(
|
||||
other_scanner.initialize_in_background(),
|
||||
name="other_cache_init",
|
||||
),
|
||||
asyncio.create_task(
|
||||
recipe_scanner.initialize_in_background(), name="recipe_cache_init"
|
||||
),
|
||||
@@ -328,6 +333,7 @@ class LoraManager:
|
||||
all_roots.update(config.loras_roots)
|
||||
all_roots.update(config.base_models_roots or [])
|
||||
all_roots.update(config.embeddings_roots or [])
|
||||
all_roots.update(config.other_roots or [])
|
||||
|
||||
total_deleted = 0
|
||||
total_size_freed = 0
|
||||
@@ -460,18 +466,11 @@ class LoraManager:
|
||||
# Cancel any in-flight scanner initialization tasks so thread-pool
|
||||
# workers (e.g. _initialize_cache_sync) can break out of their loops
|
||||
# when the server shuts down (e.g. Ctrl+C on WSL).
|
||||
for name in ("lora_scanner", "checkpoint_scanner", "embedding_scanner"):
|
||||
for name in ("lora_scanner", "checkpoint_scanner", "embedding_scanner", "other_scanner"):
|
||||
scanner = ServiceRegistry.get_service_sync(name)
|
||||
if scanner is not None and hasattr(scanner, "cancel_task"):
|
||||
scanner.cancel_task()
|
||||
logger.debug("LoRA Manager: Cancelled %s", name)
|
||||
|
||||
# Close shared aiohttp sessions to avoid "Unclosed client session" warnings
|
||||
try:
|
||||
from py.routes.handlers.hf_handlers import close_hf_api_session
|
||||
await close_hf_api_session()
|
||||
except Exception as exc:
|
||||
logger.debug("Error closing HF API session: %s", exc)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during cleanup: {e}", exc_info=True)
|
||||
|
||||
@@ -36,6 +36,7 @@ SCANNER_TYPE_MAP: dict[str, str] = {
|
||||
"get_lora_scanner": "lora",
|
||||
"get_checkpoint_scanner": "checkpoint",
|
||||
"get_embedding_scanner": "embedding",
|
||||
"get_other_scanner": "other",
|
||||
}
|
||||
|
||||
SCANNER_GETTER_NAMES = tuple(SCANNER_TYPE_MAP.keys())
|
||||
@@ -80,8 +81,8 @@ async def _find_scanner_for_model(
|
||||
|
||||
|
||||
async def identify_model_type(model_path: str) -> str:
|
||||
"""Determine the model type (``\"lora\"``, ``\"checkpoint\"``, or
|
||||
``\"embedding\"``) for *model_path*.
|
||||
"""Determine the model type (``\"lora\"``, ``\"checkpoint\"``,
|
||||
``\"embedding\"``, or ``\"other\"``) for *model_path*.
|
||||
|
||||
Falls back to ``\"lora\"`` when unknown.
|
||||
"""
|
||||
|
||||
@@ -78,7 +78,7 @@ class CheckpointLoaderLM:
|
||||
|
||||
# Filter only checkpoint type (not diffusion_model) and format names
|
||||
names = []
|
||||
for item in cache.raw_data:
|
||||
for item in list(cache.raw_data):
|
||||
if item.get("sub_type") == "checkpoint":
|
||||
file_path = item.get("file_path", "")
|
||||
# Only offer models that still exist on disk so ComfyUI
|
||||
@@ -126,7 +126,7 @@ class CheckpointLoaderLM:
|
||||
cache = await scanner.get_cached_data()
|
||||
|
||||
base_models = set()
|
||||
for item in cache.raw_data:
|
||||
for item in list(cache.raw_data):
|
||||
if item.get("sub_type") != "checkpoint":
|
||||
continue
|
||||
base_model = item.get("base_model")
|
||||
|
||||
@@ -1,214 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from typing import Any, List, Optional, Tuple
|
||||
import comfy.sd # pyright: ignore[reportMissingImports]
|
||||
import folder_paths # pyright: ignore[reportMissingImports]
|
||||
from ..utils.utils import get_checkpoint_info_absolute, _format_model_name_for_comfyui
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RandomCheckpointLoaderLM:
|
||||
"""Checkpoint Loader that can randomly pick a checkpoint from the pool
|
||||
|
||||
Loads checkpoints from both standard ComfyUI folders and LoRA Manager's
|
||||
extra folder paths. When select_at_random is enabled, ignores ckpt_name
|
||||
and picks a random checkpoint (optionally filtered by base_model) on
|
||||
every run.
|
||||
"""
|
||||
|
||||
NAME = "Random Checkpoint Loader (LoraManager)"
|
||||
CATEGORY = "Lora Manager/loaders"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
# Get list of checkpoint names from scanner (includes extra folder paths)
|
||||
checkpoint_names = cls._get_checkpoint_names()
|
||||
base_models = cls._get_available_base_models()
|
||||
return {
|
||||
"required": {
|
||||
"ckpt_name": (
|
||||
checkpoint_names,
|
||||
{"tooltip": "The name of the checkpoint (model) to load."},
|
||||
),
|
||||
"select_at_random": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": (
|
||||
"Ignore ckpt_name and pick a random checkpoint from the "
|
||||
"pool (optionally filtered by base_model) on every run."
|
||||
),
|
||||
},
|
||||
),
|
||||
"base_model": (
|
||||
base_models,
|
||||
{
|
||||
"default": "Any",
|
||||
"tooltip": "Restrict random selection to this base model. 'Any' uses the full pool.",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("MODEL", "CLIP", "VAE", "STRING")
|
||||
RETURN_NAMES = ("MODEL", "CLIP", "VAE", "model_name")
|
||||
OUTPUT_TOOLTIPS = (
|
||||
"The model used for denoising latents.",
|
||||
"The CLIP model used for encoding text prompts.",
|
||||
"The VAE model used for encoding and decoding images to and from latent space.",
|
||||
"The name of the checkpoint that was loaded (useful when select_at_random is enabled).",
|
||||
)
|
||||
FUNCTION = "load_checkpoint"
|
||||
|
||||
@classmethod
|
||||
def IS_CHANGED(cls, ckpt_name, select_at_random=False, base_model="Any"):
|
||||
# Force re-execution on every run while randomizing, since the widget
|
||||
# values themselves don't change between queue runs.
|
||||
if select_at_random:
|
||||
return float("nan")
|
||||
return ckpt_name
|
||||
|
||||
@staticmethod
|
||||
def _run_async(coro_fn):
|
||||
"""Run an async fetcher, handling the case where an event loop is already running."""
|
||||
import asyncio
|
||||
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
import concurrent.futures
|
||||
|
||||
def run_in_thread():
|
||||
new_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(new_loop)
|
||||
try:
|
||||
return new_loop.run_until_complete(coro_fn())
|
||||
finally:
|
||||
new_loop.close()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
future = executor.submit(run_in_thread)
|
||||
return future.result()
|
||||
except RuntimeError:
|
||||
return asyncio.run(coro_fn())
|
||||
|
||||
@classmethod
|
||||
def _get_checkpoint_names(cls, base_model: Optional[str] = None) -> List[str]:
|
||||
"""Get list of checkpoint names from scanner cache in ComfyUI format (relative path with extension)
|
||||
|
||||
Args:
|
||||
base_model: If given (and not "Any"), only include checkpoints matching this base model.
|
||||
"""
|
||||
try:
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
|
||||
async def _get_names():
|
||||
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||
cache = await scanner.get_cached_data()
|
||||
|
||||
# Get all model roots for calculating relative paths
|
||||
model_roots = scanner.get_model_roots()
|
||||
|
||||
# Filter only checkpoint type (not diffusion_model) and format names
|
||||
names = []
|
||||
for item in cache.raw_data:
|
||||
if item.get("sub_type") != "checkpoint":
|
||||
continue
|
||||
if (
|
||||
base_model
|
||||
and base_model != "Any"
|
||||
and item.get("base_model") != base_model
|
||||
):
|
||||
continue
|
||||
file_path = item.get("file_path", "")
|
||||
# Only offer models that still exist on disk so ComfyUI
|
||||
# flags missing checkpoints at queue time via
|
||||
# "value not in list" (the scanner cache can be stale).
|
||||
if file_path and os.path.exists(file_path):
|
||||
# Format using relative path with OS-native separator
|
||||
formatted_name = _format_model_name_for_comfyui(
|
||||
file_path, model_roots
|
||||
)
|
||||
if formatted_name:
|
||||
names.append(formatted_name)
|
||||
|
||||
return sorted(names)
|
||||
|
||||
return cls._run_async(_get_names)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting checkpoint names: {e}")
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
def _get_available_base_models(cls) -> List[str]:
|
||||
"""Get distinct base_model values present among indexed checkpoints, for the random-selection filter."""
|
||||
try:
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
|
||||
async def _get_base_models():
|
||||
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||
cache = await scanner.get_cached_data()
|
||||
|
||||
base_models = set()
|
||||
for item in cache.raw_data:
|
||||
if item.get("sub_type") != "checkpoint":
|
||||
continue
|
||||
base_model = item.get("base_model")
|
||||
file_path = item.get("file_path", "")
|
||||
if base_model and file_path and os.path.exists(file_path):
|
||||
base_models.add(base_model)
|
||||
|
||||
return sorted(base_models)
|
||||
|
||||
return ["Any"] + cls._run_async(_get_base_models)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting available base models: {e}")
|
||||
return ["Any"]
|
||||
|
||||
def load_checkpoint(
|
||||
self,
|
||||
ckpt_name: str,
|
||||
select_at_random: bool = False,
|
||||
base_model: str = "Any",
|
||||
) -> Tuple[Any, Any, Any, str]:
|
||||
"""Load a checkpoint by name, supporting extra folder paths
|
||||
|
||||
Args:
|
||||
ckpt_name: The name of the checkpoint to load (relative path with extension)
|
||||
select_at_random: If True, ignore ckpt_name and pick randomly from the pool
|
||||
base_model: Restricts random selection to this base model ("Any" = no filter)
|
||||
|
||||
Returns:
|
||||
Tuple of (MODEL, CLIP, VAE, model_name)
|
||||
"""
|
||||
if select_at_random:
|
||||
pool = self._get_checkpoint_names(base_model)
|
||||
if not pool:
|
||||
raise FileNotFoundError(
|
||||
f"No checkpoints found for base model '{base_model}'. "
|
||||
"Pick a different base model or disable 'select_at_random'."
|
||||
)
|
||||
ckpt_name = random.choice(pool)
|
||||
logger.info(
|
||||
f"[RandomCheckpointLoaderLM] Randomly selected checkpoint: {ckpt_name}"
|
||||
)
|
||||
|
||||
# Get absolute path from cache using ComfyUI-style name
|
||||
ckpt_path, metadata = get_checkpoint_info_absolute(ckpt_name)
|
||||
|
||||
if metadata is None:
|
||||
raise FileNotFoundError(
|
||||
f"Checkpoint '{ckpt_name}' not found in LoRA Manager cache. "
|
||||
"Make sure the checkpoint is indexed and try again."
|
||||
)
|
||||
|
||||
# Load regular checkpoint using ComfyUI's API
|
||||
logger.info(f"Loading checkpoint from: {ckpt_path}")
|
||||
out = comfy.sd.load_checkpoint_guess_config(
|
||||
ckpt_path,
|
||||
output_vae=True,
|
||||
output_clip=True,
|
||||
embedding_directory=folder_paths.get_folder_paths("embeddings"),
|
||||
)
|
||||
return out[:3] + (ckpt_name,)
|
||||
@@ -1,326 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from typing import Any, List, Optional, Tuple
|
||||
import comfy.sd # pyright: ignore[reportMissingImports]
|
||||
from ..utils.utils import get_checkpoint_info_absolute, _format_model_name_for_comfyui
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _reload_gguf_unet(
|
||||
unet_path: str, weight_dtype: str, disable_dynamic: bool = False
|
||||
) -> object:
|
||||
"""Reload a GGUF diffusion model from disk (cached_patcher_init factory).
|
||||
|
||||
Mirrors the GGUF branch of RandomUNETLoaderLM.load_unet so ModelPatcher
|
||||
deepclone/dynamic machinery can rebuild GGUF models with the correct
|
||||
GGMLOps. ``disable_dynamic`` is accepted for signature compatibility
|
||||
with core ComfyUI loaders.
|
||||
"""
|
||||
loader = RandomUNETLoaderLM()
|
||||
model, _unet_name = loader._load_gguf_unet(unet_path, unet_path, weight_dtype)
|
||||
return model
|
||||
|
||||
|
||||
class RandomUNETLoaderLM:
|
||||
"""UNET Loader that can randomly pick a diffusion model from the pool
|
||||
|
||||
Loads diffusion models/UNets from both standard ComfyUI folders and LoRA
|
||||
Manager's extra folder paths. Supports both regular diffusion models and
|
||||
GGUF format models. When select_at_random is enabled, ignores unet_name
|
||||
and picks a random diffusion model (optionally filtered by base_model)
|
||||
on every run.
|
||||
"""
|
||||
|
||||
NAME = "Random Unet Loader (LoraManager)"
|
||||
CATEGORY = "Lora Manager/loaders"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
# Get list of unet names from scanner (includes extra folder paths)
|
||||
unet_names = cls._get_unet_names()
|
||||
base_models = cls._get_available_base_models()
|
||||
return {
|
||||
"required": {
|
||||
"unet_name": (
|
||||
unet_names,
|
||||
{"tooltip": "The name of the diffusion model to load."},
|
||||
),
|
||||
"weight_dtype": (
|
||||
["default", "fp8_e4m3fn", "fp8_e4m3fn_fast", "fp8_e5m2"],
|
||||
{"tooltip": "The dtype to use for the model weights."},
|
||||
),
|
||||
"select_at_random": (
|
||||
"BOOLEAN",
|
||||
{
|
||||
"default": False,
|
||||
"tooltip": (
|
||||
"Ignore unet_name and pick a random diffusion model from "
|
||||
"the pool (optionally filtered by base_model) on every run."
|
||||
),
|
||||
},
|
||||
),
|
||||
"base_model": (
|
||||
base_models,
|
||||
{
|
||||
"default": "Any",
|
||||
"tooltip": "Restrict random selection to this base model. 'Any' uses the full pool.",
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("MODEL", "STRING")
|
||||
RETURN_NAMES = ("MODEL", "model_name")
|
||||
OUTPUT_TOOLTIPS = (
|
||||
"The model used for denoising latents.",
|
||||
"The name of the diffusion model that was loaded (useful when select_at_random is enabled).",
|
||||
)
|
||||
FUNCTION = "load_unet"
|
||||
|
||||
@classmethod
|
||||
def IS_CHANGED(
|
||||
cls, unet_name, weight_dtype, select_at_random=False, base_model="Any"
|
||||
):
|
||||
# Force re-execution on every run while randomizing, since the widget
|
||||
# values themselves don't change between queue runs.
|
||||
if select_at_random:
|
||||
return float("nan")
|
||||
return unet_name
|
||||
|
||||
@staticmethod
|
||||
def _run_async(coro_fn):
|
||||
"""Run an async fetcher, handling the case where an event loop is already running."""
|
||||
import asyncio
|
||||
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
import concurrent.futures
|
||||
|
||||
def run_in_thread():
|
||||
new_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(new_loop)
|
||||
try:
|
||||
return new_loop.run_until_complete(coro_fn())
|
||||
finally:
|
||||
new_loop.close()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
future = executor.submit(run_in_thread)
|
||||
return future.result()
|
||||
except RuntimeError:
|
||||
return asyncio.run(coro_fn())
|
||||
|
||||
@classmethod
|
||||
def _get_unet_names(cls, base_model: Optional[str] = None) -> List[str]:
|
||||
"""Get list of diffusion model names from scanner cache in ComfyUI format (relative path with extension)
|
||||
|
||||
Args:
|
||||
base_model: If given (and not "Any"), only include models matching this base model.
|
||||
"""
|
||||
try:
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
|
||||
async def _get_names():
|
||||
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||
cache = await scanner.get_cached_data()
|
||||
|
||||
# Get all model roots for calculating relative paths
|
||||
model_roots = scanner.get_model_roots()
|
||||
|
||||
# Filter only diffusion_model type and format names
|
||||
names = []
|
||||
for item in cache.raw_data:
|
||||
if item.get("sub_type") != "diffusion_model":
|
||||
continue
|
||||
if (
|
||||
base_model
|
||||
and base_model != "Any"
|
||||
and item.get("base_model") != base_model
|
||||
):
|
||||
continue
|
||||
file_path = item.get("file_path", "")
|
||||
# Only offer models that still exist on disk so ComfyUI
|
||||
# flags missing diffusion models at queue time via
|
||||
# "value not in list" (the scanner cache can be stale).
|
||||
if file_path and os.path.exists(file_path):
|
||||
# Format using relative path with OS-native separator
|
||||
formatted_name = _format_model_name_for_comfyui(
|
||||
file_path, model_roots
|
||||
)
|
||||
if formatted_name:
|
||||
names.append(formatted_name)
|
||||
|
||||
return sorted(names)
|
||||
|
||||
return cls._run_async(_get_names)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting unet names: {e}")
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
def _get_available_base_models(cls) -> List[str]:
|
||||
"""Get distinct base_model values present among indexed diffusion models, for the random-selection filter."""
|
||||
try:
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
|
||||
async def _get_base_models():
|
||||
scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||
cache = await scanner.get_cached_data()
|
||||
|
||||
base_models = set()
|
||||
for item in cache.raw_data:
|
||||
if item.get("sub_type") != "diffusion_model":
|
||||
continue
|
||||
base_model = item.get("base_model")
|
||||
file_path = item.get("file_path", "")
|
||||
if base_model and file_path and os.path.exists(file_path):
|
||||
base_models.add(base_model)
|
||||
|
||||
return sorted(base_models)
|
||||
|
||||
return ["Any"] + cls._run_async(_get_base_models)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting available base models: {e}")
|
||||
return ["Any"]
|
||||
|
||||
def load_unet(
|
||||
self,
|
||||
unet_name: str,
|
||||
weight_dtype: str,
|
||||
select_at_random: bool = False,
|
||||
base_model: str = "Any",
|
||||
) -> Tuple[Any, ...]:
|
||||
"""Load a diffusion model by name, supporting extra folder paths
|
||||
|
||||
Args:
|
||||
unet_name: The name of the diffusion model to load (relative path with extension)
|
||||
weight_dtype: The dtype to use for model weights
|
||||
select_at_random: If True, ignore unet_name and pick randomly from the pool
|
||||
base_model: Restricts random selection to this base model ("Any" = no filter)
|
||||
|
||||
Returns:
|
||||
Tuple of (MODEL, model_name)
|
||||
"""
|
||||
import torch
|
||||
|
||||
if select_at_random:
|
||||
pool = self._get_unet_names(base_model)
|
||||
if not pool:
|
||||
raise FileNotFoundError(
|
||||
f"No diffusion models found for base model '{base_model}'. "
|
||||
"Pick a different base model or disable 'select_at_random'."
|
||||
)
|
||||
unet_name = random.choice(pool)
|
||||
logger.info(
|
||||
f"[RandomUNETLoaderLM] Randomly selected diffusion model: {unet_name}"
|
||||
)
|
||||
|
||||
# Get absolute path from cache using ComfyUI-style name
|
||||
unet_path, metadata = get_checkpoint_info_absolute(unet_name)
|
||||
|
||||
if metadata is None:
|
||||
raise FileNotFoundError(
|
||||
f"Diffusion model '{unet_name}' not found in LoRA Manager cache. "
|
||||
"Make sure the model is indexed and try again."
|
||||
)
|
||||
|
||||
# Check if it's a GGUF model
|
||||
if unet_path.endswith(".gguf"):
|
||||
return self._load_gguf_unet(unet_path, unet_name, weight_dtype)
|
||||
|
||||
# Load regular diffusion model using ComfyUI's API
|
||||
logger.info(f"Loading diffusion model from: {unet_path}")
|
||||
|
||||
# Build model options based on weight_dtype
|
||||
model_options = {}
|
||||
if weight_dtype == "fp8_e4m3fn":
|
||||
model_options["dtype"] = torch.float8_e4m3fn
|
||||
elif weight_dtype == "fp8_e4m3fn_fast":
|
||||
model_options["dtype"] = torch.float8_e4m3fn
|
||||
model_options["fp8_optimizations"] = True
|
||||
elif weight_dtype == "fp8_e5m2":
|
||||
model_options["dtype"] = torch.float8_e5m2
|
||||
|
||||
model = comfy.sd.load_diffusion_model(unet_path, model_options=model_options)
|
||||
return (model, unet_name)
|
||||
|
||||
def _load_gguf_unet(
|
||||
self, unet_path: str, unet_name: str, weight_dtype: str
|
||||
) -> Tuple[Any, ...]:
|
||||
"""Load a GGUF format diffusion model
|
||||
|
||||
Args:
|
||||
unet_path: Absolute path to the GGUF file
|
||||
unet_name: Name of the model for error messages
|
||||
weight_dtype: The dtype to use for model weights
|
||||
|
||||
Returns:
|
||||
Tuple of (MODEL, model_name)
|
||||
"""
|
||||
import torch
|
||||
from .gguf_import_helper import get_gguf_modules
|
||||
|
||||
# Get ComfyUI-GGUF modules using helper (handles various import scenarios)
|
||||
try:
|
||||
loader_module, ops_module, nodes_module = get_gguf_modules()
|
||||
gguf_sd_loader = getattr(loader_module, "gguf_sd_loader")
|
||||
GGMLOps = getattr(ops_module, "GGMLOps")
|
||||
GGUFModelPatcher = getattr(nodes_module, "GGUFModelPatcher")
|
||||
except RuntimeError as e:
|
||||
raise RuntimeError(f"Cannot load GGUF model '{unet_name}'. {str(e)}")
|
||||
|
||||
logger.info(f"Loading GGUF diffusion model from: {unet_path}")
|
||||
|
||||
try:
|
||||
# Load GGUF state dict
|
||||
sd, extra = gguf_sd_loader(unet_path)
|
||||
|
||||
# Prepare kwargs for metadata if supported
|
||||
kwargs = {}
|
||||
import inspect
|
||||
|
||||
valid_params = inspect.signature(
|
||||
comfy.sd.load_diffusion_model_state_dict
|
||||
).parameters
|
||||
if "metadata" in valid_params:
|
||||
kwargs["metadata"] = extra.get("metadata", {})
|
||||
|
||||
# Setup custom operations with GGUF support
|
||||
ops = GGMLOps()
|
||||
|
||||
# Handle weight_dtype for GGUF models
|
||||
if weight_dtype in ("default", None):
|
||||
ops.Linear.dequant_dtype = None
|
||||
elif weight_dtype in ["target"]:
|
||||
ops.Linear.dequant_dtype = weight_dtype
|
||||
else:
|
||||
ops.Linear.dequant_dtype = getattr(torch, weight_dtype, None)
|
||||
|
||||
# Load the model
|
||||
model = comfy.sd.load_diffusion_model_state_dict(
|
||||
sd, model_options={"custom_operations": ops}, **kwargs
|
||||
)
|
||||
|
||||
if model is None:
|
||||
raise RuntimeError(
|
||||
f"Could not detect model type for GGUF diffusion model: {unet_path}"
|
||||
)
|
||||
|
||||
# Wrap with GGUFModelPatcher
|
||||
model = GGUFModelPatcher.clone(model)
|
||||
|
||||
# Register a reload factory so the MODEL carries its source path
|
||||
# (cached_patcher_init) like core ComfyUI loaders do — required
|
||||
# for model-name extraction downstream and for ModelPatcher
|
||||
# deepclone/dynamic machinery.
|
||||
model.cached_patcher_init = (_reload_gguf_unet, (unet_path, weight_dtype))
|
||||
|
||||
return (model, unet_name)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error loading GGUF diffusion model '{unet_name}': {e}")
|
||||
raise RuntimeError(
|
||||
f"Failed to load GGUF diffusion model '{unet_name}': {str(e)}"
|
||||
)
|
||||
@@ -601,7 +601,7 @@ class SaveImageLM:
|
||||
os.path.basename(name),
|
||||
os.path.splitext(os.path.basename(name))[0],
|
||||
]
|
||||
for model in getattr(cache, "raw_data", []):
|
||||
for model in list(getattr(cache, "raw_data", [])):
|
||||
file_name = model.get("file_name")
|
||||
if file_name in candidates:
|
||||
return model
|
||||
|
||||
@@ -93,7 +93,7 @@ class UNETLoaderLM:
|
||||
|
||||
# Filter only diffusion_model type and format names
|
||||
names = []
|
||||
for item in cache.raw_data:
|
||||
for item in list(cache.raw_data):
|
||||
if item.get("sub_type") == "diffusion_model":
|
||||
file_path = item.get("file_path", "")
|
||||
# Only offer models that still exist on disk so ComfyUI
|
||||
@@ -141,7 +141,7 @@ class UNETLoaderLM:
|
||||
cache = await scanner.get_cached_data()
|
||||
|
||||
base_models = set()
|
||||
for item in cache.raw_data:
|
||||
for item in list(cache.raw_data):
|
||||
if item.get("sub_type") != "diffusion_model":
|
||||
continue
|
||||
base_model = item.get("base_model")
|
||||
|
||||
+1
-1
@@ -156,7 +156,7 @@ def _find_missing_loras(names: list[str]) -> list[str]:
|
||||
|
||||
lookup = {}
|
||||
basename_candidates = {}
|
||||
for item in cache.raw_data:
|
||||
for item in list(cache.raw_data):
|
||||
file_path = item.get("file_path")
|
||||
if not file_path:
|
||||
continue
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import Dict, Any
|
||||
from ..base import RecipeMetadataParser
|
||||
from ..constants import GEN_PARAM_KEYS
|
||||
from ...services.metadata_service import get_default_metadata_provider
|
||||
from ...utils.constants import is_empty_placeholder_hash
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -146,15 +147,13 @@ class AutomaticMetadataParser(RecipeMetadataParser):
|
||||
# Initialize hashes dict if it doesn't exist
|
||||
if "hashes" not in metadata:
|
||||
metadata["hashes"] = {}
|
||||
# Add as lora type in the same format as
|
||||
# regular hashes. Only override an
|
||||
# existing entry if its value is empty
|
||||
# (Lora hashes is the more reliable
|
||||
# source when Hashes JSON has blanks).
|
||||
# Lora hashes carries the 12-char AutoV3
|
||||
# hash (resolvable on CivitAI and the local
|
||||
# autov3 index); the Hashes JSON value is
|
||||
# only the 10-char AutoV2 prefix, so on
|
||||
# conflict the Lora hashes value wins.
|
||||
key = f"lora:{lora_name}"
|
||||
existing = metadata["hashes"].get(key, "")
|
||||
if not existing:
|
||||
metadata["hashes"][key] = lora_hash
|
||||
metadata["hashes"][key] = lora_hash
|
||||
|
||||
# Remove lora hashes from params section
|
||||
params_section = params_section.replace(lora_hashes_match.group(0), '')
|
||||
@@ -526,6 +525,26 @@ class AutomaticMetadataParser(RecipeMetadataParser):
|
||||
weight = prompt_entries[0][1] if len(prompt_entries) == 1 else 1.0
|
||||
lora_entry = make_lora_entry(lora_type, lora_name, weight, lora_hash)
|
||||
|
||||
if is_empty_placeholder_hash(lora_hash):
|
||||
# The empty-hash placeholder (SHA256 of an empty byte
|
||||
# string) is not a real hash: never look it up in the
|
||||
# local hash index or on CivitAI. Match by filename;
|
||||
# otherwise keep the item as unresolved (no hash, flagged
|
||||
# hashInvalid so the UI shows the unresolvable-hash state
|
||||
# and offers reconnect instead of download) rather than
|
||||
# dropping it.
|
||||
if recipe_scanner and lora_type == 'lora' and basename_key not in queried_local_basenames:
|
||||
local_lora = await recipe_scanner.get_local_lora(lora_name, recipe_base_model)
|
||||
if local_lora:
|
||||
local_entry = self.populate_lora_from_local(lora_entry, local_lora)
|
||||
merge_or_append_local(local_entry)
|
||||
continue
|
||||
lora_entry['hash'] = ''
|
||||
lora_entry['hashInvalid'] = True
|
||||
if not resource_lora_count:
|
||||
loras.append(lora_entry)
|
||||
continue
|
||||
|
||||
if lora_hash and recipe_scanner and lora_type == 'lora':
|
||||
local_lora = await recipe_scanner.get_local_lora_by_hash(lora_hash)
|
||||
if local_lora:
|
||||
|
||||
@@ -115,6 +115,27 @@ class CivitaiApiMetadataParser(RecipeMetadataParser):
|
||||
):
|
||||
metadata = inner_meta
|
||||
|
||||
# Civitai's image API meta parser mangles the A1111 "Lora hashes"
|
||||
# text field into a quote-wrapped dict entry:
|
||||
# '"Daphne Blake Cosplay_v1": "e67ebd5e315f"'
|
||||
# The 12-char AutoV3 it carries is more reliable than the stale
|
||||
# 10-char AutoV2 value in the "hashes" dict, so recover it and
|
||||
# let it override the conflicting entry.
|
||||
if isinstance(metadata, dict):
|
||||
for key, hash_value in list(metadata.items()):
|
||||
if (
|
||||
isinstance(key, str)
|
||||
and key.startswith('"')
|
||||
and isinstance(hash_value, str)
|
||||
and hash_value.endswith('"')
|
||||
):
|
||||
clean_name = key.strip('"').strip()
|
||||
clean_hash = hash_value.strip('"').strip()
|
||||
if clean_name and clean_hash:
|
||||
hashes_dict = metadata.get("hashes")
|
||||
if isinstance(hashes_dict, dict):
|
||||
hashes_dict[f"lora:{clean_name}"] = clean_hash
|
||||
|
||||
# Initialize result structure
|
||||
result: Dict[str, Any] = {
|
||||
"base_model": None,
|
||||
|
||||
@@ -196,7 +196,7 @@ class RecipeFormatParser(RecipeMetadataParser):
|
||||
filtered_gen_params[key] = value
|
||||
|
||||
return {
|
||||
'base_model': checkpoint['baseModel'] if checkpoint and checkpoint.get('baseModel') else recipe_metadata.get('base_model', ''),
|
||||
'base_model': checkpoint['baseModel'] if checkpoint and checkpoint.get('baseModel') else (recipe_metadata.get('base_model') or None),
|
||||
'loras': loras,
|
||||
'gen_params': filtered_gen_params,
|
||||
'tags': recipe_metadata.get('tags', []),
|
||||
@@ -208,3 +208,24 @@ class RecipeFormatParser(RecipeMetadataParser):
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing recipe format metadata: {e}", exc_info=True)
|
||||
return {"error": str(e), "loras": []}
|
||||
|
||||
|
||||
def strip_recipe_metadata(metadata_text: str) -> str:
|
||||
"""Strip the ``Recipe metadata: {...}`` block appended by LoRA Manager.
|
||||
|
||||
The saved recipe image carries the original generation metadata followed
|
||||
by an appended recipe JSON block (see ``ExifUtils.append_recipe_metadata``).
|
||||
Re-import wants to re-parse the original embedded metadata, so this returns
|
||||
only the text before the appended marker. The input is returned unchanged
|
||||
when no marker is present.
|
||||
"""
|
||||
if not metadata_text:
|
||||
return metadata_text
|
||||
match = re.search(
|
||||
RecipeFormatParser.METADATA_MARKER,
|
||||
metadata_text,
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
if not match:
|
||||
return metadata_text
|
||||
return metadata_text[: match.start()].strip()
|
||||
|
||||
@@ -24,9 +24,11 @@ from ..services.use_cases import (
|
||||
AutoOrganizeUseCase,
|
||||
BulkMetadataRefreshUseCase,
|
||||
DownloadModelUseCase,
|
||||
FilenameTemplateUseCase,
|
||||
)
|
||||
from ..services.websocket_progress_callback import (
|
||||
WebSocketBroadcastCallback,
|
||||
WebSocketFilenameTemplateProgressCallback,
|
||||
WebSocketProgressCallback,
|
||||
)
|
||||
from ..utils.exif_utils import ExifUtils
|
||||
@@ -37,6 +39,7 @@ from .handlers.model_handlers import (
|
||||
ModelAutoOrganizeHandler,
|
||||
ModelCivitaiHandler,
|
||||
ModelDownloadHandler,
|
||||
ModelFilenameTemplateHandler,
|
||||
ModelHandlerSet,
|
||||
ModelListingHandler,
|
||||
ModelManagementHandler,
|
||||
@@ -83,6 +86,9 @@ class BaseModelRoutes(ABC):
|
||||
self.model_lifecycle_service: ModelLifecycleService | None = None
|
||||
self.websocket_progress_callback = WebSocketProgressCallback()
|
||||
self.metadata_progress_callback = WebSocketBroadcastCallback()
|
||||
self.filename_template_progress_callback = (
|
||||
WebSocketFilenameTemplateProgressCallback()
|
||||
)
|
||||
|
||||
self._handler_set: ModelHandlerSet | None = None
|
||||
self._handler_mapping: Dict[str, Callable[[web.Request], Awaitable[web.Response]]] | None = None
|
||||
@@ -149,6 +155,7 @@ class BaseModelRoutes(ABC):
|
||||
settings_service=self._settings,
|
||||
server_i18n=self._server_i18n,
|
||||
logger=logger,
|
||||
page_context_provider=self._get_page_context_provider(),
|
||||
)
|
||||
listing = ModelListingHandler(
|
||||
service=service,
|
||||
@@ -201,6 +208,17 @@ class BaseModelRoutes(ABC):
|
||||
ws_manager=self._ws_manager,
|
||||
logger=logger,
|
||||
)
|
||||
filename_template_use_case = FilenameTemplateUseCase(
|
||||
scanner=service.scanner,
|
||||
lifecycle_service=self._ensure_lifecycle_service(),
|
||||
lock_provider=self._ws_manager,
|
||||
model_type=service.model_type,
|
||||
)
|
||||
filename_template = ModelFilenameTemplateHandler(
|
||||
use_case=filename_template_use_case,
|
||||
progress_callback=self.filename_template_progress_callback,
|
||||
logger=logger,
|
||||
)
|
||||
updates = ModelUpdateHandler(
|
||||
service=service,
|
||||
update_service=update_service,
|
||||
@@ -217,6 +235,7 @@ class BaseModelRoutes(ABC):
|
||||
civitai=civitai,
|
||||
move=move,
|
||||
auto_organize=auto_organize,
|
||||
filename_template=filename_template,
|
||||
updates=updates,
|
||||
)
|
||||
|
||||
@@ -250,6 +269,10 @@ class BaseModelRoutes(ABC):
|
||||
"""Get expected model types string for error messages - to be overridden by subclasses."""
|
||||
return "any model type"
|
||||
|
||||
def _get_page_context_provider(self):
|
||||
"""Optional hook returning extra template context for the page view."""
|
||||
return None
|
||||
|
||||
def _find_model_file(self, files):
|
||||
"""Find the appropriate model file from the files list - can be overridden by subclasses."""
|
||||
return next((file for file in files if file.get("type") in MODEL_WEIGHT_FILE_TYPES and file.get("primary") is True), None)
|
||||
|
||||
@@ -47,15 +47,16 @@ class CheckpointRoutes(BaseModelRoutes):
|
||||
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/checkpoints_roots', prefix, self.get_checkpoints_roots)
|
||||
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/unet_roots', prefix, self.get_unet_roots)
|
||||
|
||||
# Name/base_model pool for the Random Checkpoint/Unet Loader nodes
|
||||
# Name/base_model pool for the Checkpoint/Unet Loader nodes' base_model filtering
|
||||
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/loader-pool', prefix, self.get_loader_pool)
|
||||
|
||||
async def get_loader_pool(self, request: web.Request) -> web.Response:
|
||||
"""Return ComfyUI-formatted model names with their base_model.
|
||||
|
||||
Backing data for the Random Checkpoint/Unet Loader nodes: the front-end
|
||||
filters the ckpt_name/unet_name combo options by base_model using this
|
||||
pool, so control_after_generate randomizes within the narrowed set.
|
||||
Backing data for the Checkpoint/Unet Loader nodes'
|
||||
control_after_generate feature: the front-end filters the
|
||||
ckpt_name/unet_name combo options by base_model using this pool, so
|
||||
randomize mode picks within the narrowed set.
|
||||
"""
|
||||
try:
|
||||
sub_type = request.query.get("sub_type", "checkpoint")
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
"""HTTP handler for download target routing decisions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from ...services.download_routing import (
|
||||
is_diffusion_model_download,
|
||||
resolve_other_download_sub_type,
|
||||
)
|
||||
from ...utils.constants import VALID_OTHER_CIVITAI_TYPES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DownloadRoutingHandler:
|
||||
"""Expose the download-time checkpoint/diffusion-model routing decision.
|
||||
|
||||
The web UI calls this when the user reaches the download location step
|
||||
so the root dropdown offers the same root set (checkpoint vs unet) that
|
||||
the download manager would pick for ``use_default_paths``.
|
||||
"""
|
||||
|
||||
async def get_download_routing(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
payload = await request.json()
|
||||
except json.JSONDecodeError:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Invalid JSON payload"}, status=400
|
||||
)
|
||||
|
||||
model_type = payload.get("model_type", "")
|
||||
base_model = payload.get("base_model") or ""
|
||||
file_types = payload.get("file_types") or []
|
||||
selected_file_type = payload.get("selected_file_type")
|
||||
|
||||
if not isinstance(model_type, str) or not model_type:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "model_type is required"}, status=400
|
||||
)
|
||||
if not isinstance(base_model, str) or not isinstance(file_types, list):
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "base_model must be a string and file_types a list",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
if selected_file_type is not None and not isinstance(selected_file_type, str):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "selected_file_type must be a string"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
if model_type.lower() in VALID_OTHER_CIVITAI_TYPES:
|
||||
from ...services.settings_manager import get_settings_manager
|
||||
|
||||
settings = get_settings_manager()
|
||||
if not settings.is_other_models_enabled():
|
||||
# Opt-in feature is off: never auto-route, the UI falls back to
|
||||
# manual folder selection and the download manager rejects it.
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"root_kind": "other",
|
||||
"sub_type": None,
|
||||
"disabled": True,
|
||||
"reason": "other_models_disabled",
|
||||
}
|
||||
)
|
||||
|
||||
sub_type = resolve_other_download_sub_type(
|
||||
model_type,
|
||||
file_types=(str(t) for t in file_types),
|
||||
selected_file_type=selected_file_type,
|
||||
)
|
||||
if sub_type and not settings.is_other_sub_type_enabled(sub_type):
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"root_kind": "other",
|
||||
"sub_type": None,
|
||||
"disabled": True,
|
||||
"reason": "other_sub_type_disabled",
|
||||
"requested_sub_type": sub_type,
|
||||
}
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"root_kind": "other",
|
||||
"sub_type": sub_type,
|
||||
}
|
||||
)
|
||||
|
||||
is_diffusion = is_diffusion_model_download(
|
||||
model_type,
|
||||
file_types=(str(t) for t in file_types),
|
||||
base_model=base_model,
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"is_diffusion_model": is_diffusion,
|
||||
"root_kind": "unet" if is_diffusion else model_type,
|
||||
}
|
||||
)
|
||||
@@ -1,508 +0,0 @@
|
||||
"""Handlers for Hugging Face model listing and download.
|
||||
|
||||
Minimal MVP implementation — uses direct HTTP to the HF API for file
|
||||
listing and the project's existing aiohttp-based Downloader for
|
||||
downloading. No huggingface_hub dependency required.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
from aiohttp import web
|
||||
|
||||
from ...config import config
|
||||
from ...services.downloader import (
|
||||
DownloadProgress,
|
||||
get_downloader,
|
||||
)
|
||||
from ...services.aria2_downloader import Aria2Downloader
|
||||
from ...services.settings_manager import get_settings_manager
|
||||
from ...services.service_registry import ServiceRegistry
|
||||
from ...services.websocket_manager import ws_manager
|
||||
from ...utils.constants import MODEL_FILE_EXTENSIONS
|
||||
from ...utils.metadata_manager import MetadataManager
|
||||
from ...utils.models import LoraMetadata, CheckpointMetadata, EmbeddingMetadata
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_MODEL_CLASS = LoraMetadata
|
||||
_DEFAULT_SCANNER_GETTER = "get_lora_scanner"
|
||||
|
||||
# Shared aiohttp session for HF API calls (created on first use)
|
||||
_hf_api_session: aiohttp.ClientSession | None = None
|
||||
|
||||
|
||||
async def _get_hf_api_session() -> aiohttp.ClientSession:
|
||||
"""Get or create the shared aiohttp session for HF API calls."""
|
||||
global _hf_api_session # needed because we reassign the module-level name
|
||||
if _hf_api_session is None or _hf_api_session.closed:
|
||||
_hf_api_session = aiohttp.ClientSession(
|
||||
headers={"User-Agent": "ComfyUI-LoRA-Manager/1.0"},
|
||||
timeout=aiohttp.ClientTimeout(total=30),
|
||||
)
|
||||
return _hf_api_session
|
||||
|
||||
|
||||
async def close_hf_api_session() -> None:
|
||||
"""Close the shared HF API session, if it was ever created."""
|
||||
global _hf_api_session
|
||||
if _hf_api_session is not None and not _hf_api_session.closed:
|
||||
await _hf_api_session.close()
|
||||
_hf_api_session = None
|
||||
|
||||
|
||||
def _infer_model_type(model_root: str) -> tuple[Any, str]:
|
||||
"""Determine model class and scanner by matching ``model_root`` against the
|
||||
configured root paths for each model type (from ``Config``).
|
||||
|
||||
The ``model_root`` value comes from the frontend's model-root dropdown,
|
||||
which is populated from the current page's scanner roots. By checking
|
||||
which scanner's root list it belongs to, we avoid fragile heuristics
|
||||
like substring-matching path names.
|
||||
"""
|
||||
norm = os.path.normpath(model_root).replace(os.sep, "/")
|
||||
|
||||
# LoRA roots
|
||||
for p in (config.loras_roots or []) + (config.extra_loras_roots or []):
|
||||
if os.path.normpath(p).replace(os.sep, "/") == norm:
|
||||
return LoraMetadata, "get_lora_scanner"
|
||||
|
||||
# Checkpoint / UNet roots
|
||||
for p in (
|
||||
(config.checkpoints_roots or [])
|
||||
+ (config.extra_checkpoints_roots or [])
|
||||
+ (config.unet_roots or [])
|
||||
+ (config.extra_unet_roots or [])
|
||||
):
|
||||
if os.path.normpath(p).replace(os.sep, "/") == norm:
|
||||
return CheckpointMetadata, "get_checkpoint_scanner"
|
||||
|
||||
# Embedding roots
|
||||
for p in (config.embeddings_roots or []) + (config.extra_embeddings_roots or []):
|
||||
if os.path.normpath(p).replace(os.sep, "/") == norm:
|
||||
return EmbeddingMetadata, "get_embedding_scanner"
|
||||
|
||||
# Fallback — should not happen in normal use
|
||||
logger.warning(
|
||||
"Could not determine model type for root '%s'; defaulting to LoRA",
|
||||
model_root,
|
||||
)
|
||||
return _DEFAULT_MODEL_CLASS, _DEFAULT_SCANNER_GETTER
|
||||
|
||||
|
||||
async def _save_hf_metadata(dest_path: str, repo: str, model_root: str) -> None:
|
||||
"""Create a proper .metadata.json and add the model to the scanner cache.
|
||||
|
||||
Uses ``MetadataManager.create_default_metadata()`` which computes the
|
||||
SHA256 hash, extracts safetensors header metadata (base_model), and
|
||||
produces a fully-populated ``LoraMetadata`` (or ``CheckpointMetadata`` /
|
||||
``EmbeddingMetadata``) object. We then overlay HF-specific fields and
|
||||
register the model in the in-memory scanner cache so it appears
|
||||
immediately without a full filesystem walk.
|
||||
"""
|
||||
try:
|
||||
hf_url = f"https://huggingface.co/{repo}"
|
||||
model_class, scanner_getter_name = _infer_model_type(model_root)
|
||||
|
||||
# 1. Create proper metadata (computes SHA256, reads safetensors headers)
|
||||
metadata = await MetadataManager.create_default_metadata(
|
||||
dest_path, model_class=model_class
|
||||
)
|
||||
if metadata is None:
|
||||
logger.warning("create_default_metadata returned None for %s", dest_path)
|
||||
return
|
||||
|
||||
# 2. Overlay HF-specific fields
|
||||
metadata._unknown_fields["hf_url"] = hf_url
|
||||
metadata.from_civitai = False # HF models are not from CivitAI
|
||||
|
||||
metadata_dict = metadata.to_dict()
|
||||
if "trainedWords" in metadata_dict and not metadata_dict["trainedWords"]:
|
||||
del metadata_dict["trainedWords"]
|
||||
|
||||
# 3. Save metadata atomically
|
||||
await MetadataManager.save_metadata(dest_path, metadata_dict)
|
||||
logger.info("Saved HF metadata (with hf_url) for %s", dest_path)
|
||||
|
||||
# 4. Determine relative folder path for cache
|
||||
# model_root is an absolute path; dest_path is under it
|
||||
folder = ""
|
||||
if os.path.isabs(model_root) and dest_path.startswith(model_root):
|
||||
rel = os.path.relpath(os.path.dirname(dest_path), model_root)
|
||||
folder = rel.replace(os.sep, "/") if rel != "." else ""
|
||||
|
||||
# 5. Add to scanner cache (same as CivitAI's _execute_download does)
|
||||
scanner_getter = getattr(ServiceRegistry, scanner_getter_name, None)
|
||||
if scanner_getter is not None:
|
||||
scanner = await scanner_getter()
|
||||
if scanner is not None:
|
||||
metadata_dict = metadata.to_dict()
|
||||
metadata_dict["hf_url"] = hf_url
|
||||
await scanner.add_model_to_cache(metadata_dict, folder)
|
||||
logger.info("Added %s to scanner cache (folder=%s)", dest_path, folder)
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to save HF metadata for %s: %s", dest_path, exc)
|
||||
|
||||
|
||||
def _find_matching_root(dest_dir: str) -> str | None:
|
||||
"""Walk up *dest_dir* to find which configured scanner root it belongs to."""
|
||||
norm = os.path.normpath(dest_dir).replace(os.sep, "/")
|
||||
all_roots = []
|
||||
for root_list in (
|
||||
config.loras_roots or [],
|
||||
config.extra_loras_roots or [],
|
||||
config.checkpoints_roots or [],
|
||||
config.extra_checkpoints_roots or [],
|
||||
config.unet_roots or [],
|
||||
config.extra_unet_roots or [],
|
||||
config.embeddings_roots or [],
|
||||
config.extra_embeddings_roots or [],
|
||||
):
|
||||
all_roots.extend([os.path.normpath(p).replace(os.sep, "/") for p in root_list])
|
||||
# Find the longest matching prefix
|
||||
match: str | None = None
|
||||
for root in all_roots:
|
||||
if norm.startswith(root):
|
||||
if match is None or len(root) > len(match):
|
||||
match = root
|
||||
return match
|
||||
|
||||
|
||||
async def _add_to_scanner_cache(dest_path: str, metadata: dict[str, Any]) -> None:
|
||||
model_dir = os.path.dirname(dest_path)
|
||||
model_root = _find_matching_root(model_dir)
|
||||
if not model_root:
|
||||
raise ValueError(f"File path {dest_path} is not within any configured scanner root")
|
||||
scanner_getter_name = _infer_model_type(model_root)[1]
|
||||
scanner_getter = getattr(ServiceRegistry, scanner_getter_name, None)
|
||||
if scanner_getter is None:
|
||||
raise RuntimeError(f"Scanner getter '{scanner_getter_name}' not found in ServiceRegistry")
|
||||
scanner = await scanner_getter()
|
||||
if scanner is None:
|
||||
raise RuntimeError(f"Scanner '{scanner_getter_name}' returned None")
|
||||
await scanner.update_single_model_cache(dest_path, dest_path, metadata)
|
||||
|
||||
|
||||
class HfHandler:
|
||||
"""Handle Hugging Face model browsing and download."""
|
||||
|
||||
async def set_hf_url(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
payload: dict[str, Any] = await request.json()
|
||||
except json.JSONDecodeError:
|
||||
return web.json_response({"success": False, "error": "Invalid JSON"}, status=400)
|
||||
|
||||
file_path = (payload.get("file_path") or "").strip()
|
||||
hf_url = (payload.get("hf_url") or "").strip()
|
||||
|
||||
if not file_path or not hf_url:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Missing required fields: 'file_path' and 'hf_url'"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
m = re.match(r"^https?://huggingface\.co/([^/]+/[^/]+)/?$", hf_url)
|
||||
if not m:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "Invalid HuggingFace URL. Expected format: https://huggingface.co/user/repo",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
|
||||
if not os.path.isfile(file_path):
|
||||
return web.json_response(
|
||||
{"success": False, "error": f"File not found: {file_path}"},
|
||||
status=404,
|
||||
)
|
||||
|
||||
model_root = _find_matching_root(os.path.dirname(file_path))
|
||||
if not model_root:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "File is not within any configured model directory. Cannot link to HuggingFace.",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
|
||||
try:
|
||||
existing = await MetadataManager.load_metadata_payload(file_path)
|
||||
if existing.get("hf_url") == hf_url:
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": "hf_url already set",
|
||||
"hf_url": hf_url,
|
||||
})
|
||||
|
||||
existing["hf_url"] = hf_url
|
||||
existing["from_civitai"] = False
|
||||
await MetadataManager.save_metadata(file_path, existing)
|
||||
|
||||
await _add_to_scanner_cache(file_path, existing)
|
||||
|
||||
logger.info("Set hf_url=%s for %s", hf_url, file_path)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"hf_url set to {hf_url}",
|
||||
"hf_url": hf_url,
|
||||
})
|
||||
except Exception as exc:
|
||||
logger.error("Failed to set hf_url for %s: %s", file_path, exc)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(exc)},
|
||||
status=500,
|
||||
)
|
||||
|
||||
async def get_hf_repo_files(self, request: web.Request) -> web.Response:
|
||||
"""List model-weight files from a HF repo with real file sizes.
|
||||
|
||||
Uses the HF tree API endpoint which returns accurate file sizes
|
||||
(including LFS-tracked files), unlike the model info endpoint.
|
||||
"""
|
||||
repo = request.query.get("repo", "").strip()
|
||||
if not repo or "/" not in repo:
|
||||
return web.json_response(
|
||||
{"error": "Missing or invalid 'repo' parameter (expected user/repo)"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
url = f"https://huggingface.co/api/models/{repo}/tree/main"
|
||||
|
||||
try:
|
||||
session = await _get_hf_api_session()
|
||||
async with session.get(url) as resp:
|
||||
if resp.status == 404:
|
||||
return web.json_response(
|
||||
{"error": f"Repo '{repo}' not found"}, status=404
|
||||
)
|
||||
if resp.status != 200:
|
||||
text = await resp.text()
|
||||
return web.json_response(
|
||||
{"error": f"HF API error {resp.status}: {text[:200]}"},
|
||||
status=resp.status,
|
||||
)
|
||||
tree: list[dict[str, Any]] = await resp.json()
|
||||
except Exception as exc:
|
||||
logger.error("Failed to fetch HF repo files: %s", exc)
|
||||
return web.json_response({"error": str(exc)}, status=502)
|
||||
|
||||
files: list[dict[str, Any]] = []
|
||||
for entry in tree:
|
||||
path: str = entry.get("path", "")
|
||||
ext = os.path.splitext(path)[1].lower()
|
||||
if ext not in MODEL_FILE_EXTENSIONS:
|
||||
continue
|
||||
size = entry.get("size", 0) or 0
|
||||
if size == 0 and "lfs" in entry:
|
||||
size = entry["lfs"].get("size", 0) or 0
|
||||
files.append({
|
||||
"filename": path,
|
||||
"size": size,
|
||||
})
|
||||
|
||||
files.sort(key=lambda f: f["size"], reverse=True)
|
||||
return web.json_response(files)
|
||||
|
||||
async def download_hf_model(self, request: web.Request) -> web.Response:
|
||||
"""Download a single file from Hugging Face into the model directory.
|
||||
|
||||
POST JSON body::
|
||||
|
||||
{
|
||||
"repo": "dx8152/Flux2-Klein-9B-Consistency",
|
||||
"filename": "Flux2-Klein-9B-consistency-V2.safetensors",
|
||||
"revision": "main",
|
||||
"model_root": "loras",
|
||||
"relative_path": "",
|
||||
"use_default_paths": false,
|
||||
"download_id": "optional-batch-id"
|
||||
}
|
||||
|
||||
If ``download_id`` is provided, real-time progress (bytes, speed,
|
||||
percentage) is broadcast via the WebSocket progress system, matching
|
||||
the CivitAI download experience.
|
||||
|
||||
Respects the ``download_backend`` setting (``aria2`` or ``default``).
|
||||
"""
|
||||
try:
|
||||
payload: dict[str, Any] = await request.json()
|
||||
except json.JSONDecodeError:
|
||||
return web.json_response({"error": "Invalid JSON"}, status=400)
|
||||
|
||||
repo = (payload.get("repo") or "").strip()
|
||||
filename = (payload.get("filename") or "").strip()
|
||||
revision = (payload.get("revision") or "main").strip()
|
||||
model_root = (payload.get("model_root") or "").strip()
|
||||
relative_path = (payload.get("relative_path") or "").strip()
|
||||
use_default_paths = bool(payload.get("use_default_paths", False))
|
||||
download_id: str | None = payload.get("download_id")
|
||||
|
||||
logger.info(
|
||||
"download_hf_model: repo=%s file=%s root=%s download_id=%s",
|
||||
repo, filename, model_root, download_id,
|
||||
)
|
||||
|
||||
if not repo or not filename:
|
||||
return web.json_response(
|
||||
{"error": "Missing required fields: 'repo' and 'filename'"}, status=400
|
||||
)
|
||||
|
||||
# Validate repo format — must be user/repo_name
|
||||
if repo.count("/") != 1 or not re.match(r"^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$", repo):
|
||||
return web.json_response({"error": f"Invalid repo format: {repo}"}, status=400)
|
||||
author, repo_name = repo.split("/", 1)
|
||||
if ".." in (author, repo_name) or "." in (author, repo_name):
|
||||
return web.json_response({"error": f"Invalid repo format: {repo}"}, status=400)
|
||||
|
||||
# Validate filename — must not contain path traversal
|
||||
if ".." in filename:
|
||||
return web.json_response({"error": "Invalid filename"}, status=400)
|
||||
|
||||
# Validate relative_path — must not be absolute or escape base directory
|
||||
if relative_path:
|
||||
if os.path.isabs(relative_path):
|
||||
return web.json_response({"error": "relative_path must not be absolute"}, status=400)
|
||||
if ".." in relative_path.split("/") or "\\" in relative_path:
|
||||
return web.json_response({"error": "Invalid relative_path"}, status=400)
|
||||
|
||||
# Use model_root directly as the base directory — same approach as
|
||||
# CivitAI's download path (download_manager.py). No realpath, no
|
||||
# allowed-roots validation, no path-traversal check; those are
|
||||
# unnecessary when the frontend sends the path from its own dropdown
|
||||
# (populated from scanner roots). Using the "business path" directly
|
||||
# keeps dest_path consistent with scanner roots so that later folder
|
||||
# derivation (in _save_hf_metadata) works correctly.
|
||||
if os.path.isabs(model_root):
|
||||
base_dir = os.path.normpath(model_root)
|
||||
else:
|
||||
base_dir = os.path.normpath(os.path.join(os.getcwd(), "models", model_root))
|
||||
|
||||
if use_default_paths:
|
||||
target_dir = os.path.join(base_dir, "huggingface", author, repo_name)
|
||||
elif relative_path:
|
||||
target_dir = os.path.join(base_dir, relative_path)
|
||||
else:
|
||||
target_dir = base_dir
|
||||
|
||||
# Strip HF repo subdirectory — "diffusion_models/xxx.safetensors"
|
||||
# is an HF repo convention, not meaningful for local storage.
|
||||
file_base = os.path.basename(filename)
|
||||
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
dest_path = os.path.join(target_dir, file_base)
|
||||
|
||||
# Check if already exists (simple skip)
|
||||
if os.path.exists(dest_path) and os.path.getsize(dest_path) > 0:
|
||||
logger.info("download_hf_model: file already exists, skipping — %s", dest_path)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"File already exists: {dest_path}",
|
||||
"path": dest_path,
|
||||
})
|
||||
|
||||
# Build HF resolve URL
|
||||
resolve_url = (
|
||||
f"https://huggingface.co/{repo}/resolve/{revision}/{filename}"
|
||||
)
|
||||
|
||||
# Set up progress callback if download_id is provided
|
||||
progress_callback = None
|
||||
if download_id:
|
||||
|
||||
async def _progress_callback(
|
||||
progress: float | DownloadProgress,
|
||||
snapshot: DownloadProgress | None = None,
|
||||
) -> None:
|
||||
percent = 0.0
|
||||
metrics = snapshot if isinstance(snapshot, DownloadProgress) else None
|
||||
|
||||
if isinstance(progress, DownloadProgress):
|
||||
percent = progress.percent_complete
|
||||
metrics = progress
|
||||
elif isinstance(snapshot, DownloadProgress):
|
||||
percent = snapshot.percent_complete
|
||||
else:
|
||||
percent = float(progress)
|
||||
|
||||
broadcast: dict[str, Any] = {
|
||||
"status": "progress",
|
||||
"progress": round(percent),
|
||||
}
|
||||
if metrics:
|
||||
broadcast["bytes_downloaded"] = metrics.bytes_downloaded
|
||||
broadcast["total_bytes"] = metrics.total_bytes
|
||||
broadcast["bytes_per_second"] = metrics.bytes_per_second
|
||||
|
||||
await ws_manager.broadcast_download_progress(download_id, broadcast)
|
||||
|
||||
progress_callback = _progress_callback
|
||||
|
||||
# Respect download backend setting (aria2 vs default)
|
||||
download_backend = (
|
||||
get_settings_manager().get("download_backend", "default")
|
||||
)
|
||||
|
||||
if download_backend == "aria2":
|
||||
aria2 = await Aria2Downloader.get_instance()
|
||||
aid = download_id or f"hf_{repo}_{filename}"
|
||||
try:
|
||||
hf_success, hf_result = await aria2.download_file(
|
||||
url=resolve_url,
|
||||
save_path=dest_path,
|
||||
download_id=aid,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
if hf_success:
|
||||
await _save_hf_metadata(dest_path, repo, model_root)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"Downloaded to {dest_path}",
|
||||
"path": dest_path,
|
||||
})
|
||||
else:
|
||||
return web.json_response(
|
||||
{"success": False, "error": hf_result or "aria2 download failed"},
|
||||
status=500,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("HF download (aria2) failed: %s", exc)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(exc)}, status=500
|
||||
)
|
||||
|
||||
# Default: use built-in aiohttp Downloader
|
||||
downloader = await get_downloader()
|
||||
try:
|
||||
success, result = await downloader.download_file(
|
||||
url=resolve_url,
|
||||
save_path=dest_path,
|
||||
use_auth=False,
|
||||
allow_resume=True,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
if success:
|
||||
await _save_hf_metadata(dest_path, repo, model_root)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"Downloaded to {result}",
|
||||
"path": result,
|
||||
})
|
||||
else:
|
||||
return web.json_response(
|
||||
{"success": False, "error": result or "Download failed"},
|
||||
status=500,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("HF download failed: %s", exc)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(exc)}, status=500
|
||||
)
|
||||
@@ -53,11 +53,15 @@ from ...utils.constants import (
|
||||
PREVIEW_EXTENSIONS,
|
||||
SUPPORTED_MEDIA_EXTENSIONS,
|
||||
VALID_LORA_TYPES,
|
||||
VALID_OTHER_CIVITAI_TYPES,
|
||||
folder_path_schema,
|
||||
)
|
||||
from .hf_handlers import HfHandler
|
||||
from .model_source_handlers import ModelSourceHandler
|
||||
from .agent_handlers import AgentHandler
|
||||
from .download_routing_handlers import DownloadRoutingHandler
|
||||
from .model_handlers import ModelCivitaiHandler
|
||||
from ...utils.civitai_utils import rewrite_preview_url
|
||||
from ...utils.directory_browser import browse_directory
|
||||
from ...utils.example_images_paths import (
|
||||
find_non_compliant_items_in_example_images_root,
|
||||
is_valid_example_images_root,
|
||||
@@ -419,6 +423,11 @@ def _wsl_to_windows_path(wsl_path: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _has_gui_display() -> bool:
|
||||
"""Check whether a GUI session is reachable for xdg-open."""
|
||||
return bool(os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"))
|
||||
|
||||
|
||||
class PromptServerProtocol(Protocol):
|
||||
"""Subset of PromptServer used by the handlers."""
|
||||
|
||||
@@ -657,9 +666,21 @@ class HealthCheckHandler:
|
||||
"lora": ServiceRegistry.get_lora_scanner,
|
||||
"checkpoint": ServiceRegistry.get_checkpoint_scanner,
|
||||
"embedding": ServiceRegistry.get_embedding_scanner,
|
||||
"other": ServiceRegistry.get_other_scanner,
|
||||
"recipe": ServiceRegistry.get_recipe_scanner,
|
||||
}
|
||||
|
||||
def _active_scanner_getters(
|
||||
self,
|
||||
) -> Mapping[str, Callable[[], Awaitable[Any]]]:
|
||||
"""Drop the opt-in other scanner while Other Models is disabled."""
|
||||
getters = self._scanner_getters
|
||||
if "other" not in getters:
|
||||
return getters
|
||||
if get_settings_manager().is_other_models_enabled():
|
||||
return getters
|
||||
return {name: getter for name, getter in getters.items() if name != "other"}
|
||||
|
||||
async def health_check(self, request: web.Request) -> web.Response:
|
||||
return web.json_response({"status": "ok"})
|
||||
|
||||
@@ -671,7 +692,7 @@ class HealthCheckHandler:
|
||||
page accepts the update and only reloads once all scanners are done.
|
||||
"""
|
||||
pending: list[str] = []
|
||||
for name, getter in self._scanner_getters.items():
|
||||
for name, getter in self._active_scanner_getters().items():
|
||||
try:
|
||||
scanner = await getter()
|
||||
except Exception:
|
||||
@@ -756,10 +777,19 @@ class DoctorHandler:
|
||||
("lora", "LoRAs", ServiceRegistry.get_lora_scanner),
|
||||
("checkpoint", "Checkpoints", ServiceRegistry.get_checkpoint_scanner),
|
||||
("embedding", "Embeddings", ServiceRegistry.get_embedding_scanner),
|
||||
("other", "Other Models", ServiceRegistry.get_other_scanner),
|
||||
)
|
||||
)
|
||||
self._app_version_getter = app_version_getter
|
||||
|
||||
def _active_scanner_factories(
|
||||
self,
|
||||
) -> Sequence[tuple[str, str, Callable[[], Awaitable[Any]]]]:
|
||||
"""Drop the opt-in other scanner while Other Models is disabled."""
|
||||
if self._settings.is_other_models_enabled():
|
||||
return self._scanner_factories
|
||||
return tuple(entry for entry in self._scanner_factories if entry[0] != "other")
|
||||
|
||||
async def get_doctor_diagnostics(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
client_version = (request.query.get("clientVersion") or "").strip()
|
||||
@@ -807,7 +837,7 @@ class DoctorHandler:
|
||||
repaired: list[dict[str, Any]] = []
|
||||
failures: list[dict[str, str]] = []
|
||||
|
||||
for model_type, label, factory in self._scanner_factories:
|
||||
for model_type, label, factory in self._active_scanner_factories():
|
||||
try:
|
||||
scanner = await factory()
|
||||
await scanner.get_cached_data(force_refresh=True, rebuild_cache=True)
|
||||
@@ -839,7 +869,7 @@ class DoctorHandler:
|
||||
renamed: list[dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
for model_type, label, factory in self._scanner_factories:
|
||||
for model_type, label, factory in self._active_scanner_factories():
|
||||
try:
|
||||
scanner = await factory()
|
||||
hash_index = getattr(scanner, "_hash_index", None)
|
||||
@@ -1071,7 +1101,7 @@ class DoctorHandler:
|
||||
overall_status = "ok"
|
||||
summary = "All model caches look healthy."
|
||||
|
||||
for model_type, label, factory in self._scanner_factories:
|
||||
for model_type, label, factory in self._active_scanner_factories():
|
||||
try:
|
||||
scanner = await factory()
|
||||
persisted = None
|
||||
@@ -1156,7 +1186,7 @@ class DoctorHandler:
|
||||
total_conflict_groups = 0
|
||||
total_conflict_files = 0
|
||||
|
||||
for model_type, label, factory in self._scanner_factories:
|
||||
for model_type, label, factory in self._active_scanner_factories():
|
||||
# Duplicate filename detection targets LoRAs which use basename-only
|
||||
# syntax (<lora:name:strength>). Checkpoints/embeddings reference
|
||||
# models via relative paths with extensions, so conflicts there would
|
||||
@@ -1536,6 +1566,46 @@ class SettingsHandler:
|
||||
response_data["civitai_api_key_set"] = bool(raw_key)
|
||||
raw_llm_key = self._settings.get("llm_api_key")
|
||||
response_data["llm_api_key_set"] = bool(raw_llm_key)
|
||||
# Derived capability flag (not persisted): whether the host exposes
|
||||
# any other-model folder at all. Standalone installs only know the
|
||||
# folder_paths keys present in settings.json, so the announcement
|
||||
# banner uses this to avoid promising a page that cannot list
|
||||
# anything.
|
||||
try:
|
||||
availability = config.get_other_models_availability()
|
||||
response_data["other_models_paths_available"] = bool(
|
||||
availability.get("available")
|
||||
)
|
||||
except Exception as availability_error: # pragma: no cover - defensive
|
||||
logger.debug(
|
||||
"Could not resolve Other Models availability: %s",
|
||||
availability_error,
|
||||
)
|
||||
response_data["other_models_paths_available"] = None
|
||||
standalone_mode = os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"
|
||||
response_data["standalone_mode"] = standalone_mode
|
||||
if standalone_mode:
|
||||
# Standalone reads its model roots exclusively from
|
||||
# settings.json, so the Model Paths settings UI needs the
|
||||
# current values plus the editable-key schema. In plugin mode
|
||||
# the paths come from the ComfyUI host and stay hidden.
|
||||
folder_paths = self._settings.get("folder_paths") or {}
|
||||
# A fresh install is seeded from settings.json.example, whose
|
||||
# folder_paths are documentation placeholders — hide them so
|
||||
# the UI starts with empty editors instead of fake paths.
|
||||
get_placeholders = getattr(
|
||||
self._settings, "get_template_folder_path_placeholders", None
|
||||
)
|
||||
placeholders = get_placeholders() if get_placeholders else set()
|
||||
if placeholders:
|
||||
folder_paths = {
|
||||
key: [p for p in paths if p not in placeholders]
|
||||
if isinstance(paths, list)
|
||||
else paths
|
||||
for key, paths in folder_paths.items()
|
||||
}
|
||||
response_data["folder_paths"] = folder_paths
|
||||
response_data["folder_path_schema"] = folder_path_schema()
|
||||
settings_file = getattr(self._settings, "settings_file", None)
|
||||
if settings_file:
|
||||
response_data["settings_file"] = settings_file
|
||||
@@ -2065,6 +2135,7 @@ class ServiceRegistryAdapter:
|
||||
get_embedding_scanner: Callable[[], Awaitable[Any]]
|
||||
get_downloaded_version_history_service: Callable[[], Awaitable[Any]]
|
||||
get_backup_service: Callable[[], Awaitable[Any]] = _noop_backup_service
|
||||
get_other_scanner: Callable[[], Awaitable[Any]] = ServiceRegistry.get_other_scanner
|
||||
|
||||
|
||||
class ModelLibraryHandler:
|
||||
@@ -2089,6 +2160,8 @@ class ModelLibraryHandler:
|
||||
return "checkpoint"
|
||||
if normalized in {"embedding", "textualinversion"}:
|
||||
return "embedding"
|
||||
if normalized in VALID_OTHER_CIVITAI_TYPES:
|
||||
return "other"
|
||||
return None
|
||||
|
||||
async def _get_scanner_for_type(self, model_type: str | None):
|
||||
@@ -2099,6 +2172,13 @@ class ModelLibraryHandler:
|
||||
return normalized_type, await self._service_registry.get_checkpoint_scanner()
|
||||
if normalized_type == "embedding":
|
||||
return normalized_type, await self._service_registry.get_embedding_scanner()
|
||||
if normalized_type == "other":
|
||||
# Opt-in feature: the other scanner only resolves while the master
|
||||
# switch is on, so callers keep returning the legacy "required"
|
||||
# error (400) when it is off.
|
||||
if not get_settings_manager().is_other_models_enabled():
|
||||
return None, None
|
||||
return normalized_type, await self._service_registry.get_other_scanner()
|
||||
return None, None
|
||||
|
||||
async def _get_download_history_service(self):
|
||||
@@ -2190,6 +2270,11 @@ class ModelLibraryHandler:
|
||||
lora_scanner = await self._service_registry.get_lora_scanner()
|
||||
checkpoint_scanner = await self._service_registry.get_checkpoint_scanner()
|
||||
embedding_scanner = await self._service_registry.get_embedding_scanner()
|
||||
# Opt-in: probe the other scanner only while Other Models is enabled,
|
||||
# so the disabled behaviour stays byte-identical to the legacy one.
|
||||
other_scanner = None
|
||||
if get_settings_manager().is_other_models_enabled():
|
||||
other_scanner = await self._service_registry.get_other_scanner()
|
||||
|
||||
if model_version_id_str:
|
||||
try:
|
||||
@@ -2228,6 +2313,13 @@ class ModelLibraryHandler:
|
||||
exists = True
|
||||
model_type = "embedding"
|
||||
matched_scanner = embedding_scanner
|
||||
elif (
|
||||
other_scanner
|
||||
and await other_scanner.check_model_version_exists(model_version_id)
|
||||
):
|
||||
exists = True
|
||||
model_type = "other"
|
||||
matched_scanner = other_scanner
|
||||
|
||||
if exists:
|
||||
return web.json_response(
|
||||
@@ -2245,7 +2337,7 @@ class ModelLibraryHandler:
|
||||
history_service = await self._get_download_history_service()
|
||||
has_been_downloaded = False
|
||||
history_type = None
|
||||
for candidate_type in ("lora", "checkpoint", "embedding"):
|
||||
for candidate_type in ("lora", "checkpoint", "embedding", "other"):
|
||||
if await history_service.has_been_downloaded(
|
||||
candidate_type,
|
||||
model_version_id,
|
||||
@@ -2267,6 +2359,7 @@ class ModelLibraryHandler:
|
||||
lora_versions = await lora_scanner.get_model_versions_by_id(model_id)
|
||||
checkpoint_versions = []
|
||||
embedding_versions = []
|
||||
other_versions = []
|
||||
if not lora_versions and checkpoint_scanner:
|
||||
checkpoint_versions = await checkpoint_scanner.get_model_versions_by_id(
|
||||
model_id
|
||||
@@ -2275,6 +2368,13 @@ class ModelLibraryHandler:
|
||||
embedding_versions = await embedding_scanner.get_model_versions_by_id(
|
||||
model_id
|
||||
)
|
||||
if (
|
||||
not lora_versions
|
||||
and not checkpoint_versions
|
||||
and not embedding_versions
|
||||
and other_scanner
|
||||
):
|
||||
other_versions = await other_scanner.get_model_versions_by_id(model_id)
|
||||
|
||||
model_type = None
|
||||
versions = []
|
||||
@@ -2306,9 +2406,18 @@ class ModelLibraryHandler:
|
||||
"downloadedVersionIds": [],
|
||||
}
|
||||
)
|
||||
if other_versions:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"modelType": "other",
|
||||
"versions": self._with_downloaded_flag(other_versions),
|
||||
"downloadedVersionIds": [],
|
||||
}
|
||||
)
|
||||
|
||||
history_service = await self._get_download_history_service()
|
||||
for candidate_type in ("lora", "checkpoint", "embedding"):
|
||||
for candidate_type in ("lora", "checkpoint", "embedding", "other"):
|
||||
candidate_downloaded_version_ids = (
|
||||
await history_service.get_downloaded_version_ids(
|
||||
candidate_type,
|
||||
@@ -2363,6 +2472,11 @@ class ModelLibraryHandler:
|
||||
lora_scanner = await self._service_registry.get_lora_scanner()
|
||||
checkpoint_scanner = await self._service_registry.get_checkpoint_scanner()
|
||||
embedding_scanner = await self._service_registry.get_embedding_scanner()
|
||||
# Opt-in: keep the other probe last so model cards for lora /
|
||||
# checkpoint / embedding ids are unaffected by the extra scanner.
|
||||
other_scanner = None
|
||||
if get_settings_manager().is_other_models_enabled():
|
||||
other_scanner = await self._service_registry.get_other_scanner()
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
for model_id in model_ids:
|
||||
@@ -2398,6 +2512,17 @@ class ModelLibraryHandler:
|
||||
})
|
||||
continue
|
||||
|
||||
if other_scanner:
|
||||
other_versions = await other_scanner.get_model_versions_by_id(model_id)
|
||||
if other_versions:
|
||||
results.append({
|
||||
"modelId": model_id,
|
||||
"modelType": "other",
|
||||
"versions": self._with_downloaded_flag(other_versions),
|
||||
"downloadedVersionIds": [],
|
||||
})
|
||||
continue
|
||||
|
||||
results.append({
|
||||
"modelId": model_id,
|
||||
"modelType": None,
|
||||
@@ -2665,12 +2790,40 @@ class ModelLibraryHandler:
|
||||
|
||||
normalized_type, scanner = await self._get_scanner_for_type(model_type)
|
||||
if not normalized_type:
|
||||
# The lookup cannot be served as a fully interactive list. Two
|
||||
# cases share this branch: a CivitAI type with no scanner at all
|
||||
# (Wildcards, Workflows, Hypernetwork, Poses, AestheticGradient)
|
||||
# and an Other-model type while the opt-in master switch is off.
|
||||
# Answer 200 with the CivitAI list marked read-only plus a
|
||||
# machine-readable reason, so clients can still show the
|
||||
# versions and explain why the actions are missing. Legacy
|
||||
# clients keep working: they only read `success`/`versions`.
|
||||
reason = (
|
||||
"other_models_disabled"
|
||||
if self._normalize_model_type(model_type) == "other"
|
||||
else "model_type_unsupported"
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": f'Model type "{model_type}" is not supported',
|
||||
},
|
||||
status=400,
|
||||
"success": True,
|
||||
"modelId": model_id,
|
||||
"modelName": model_name,
|
||||
"modelType": model_type,
|
||||
"supported": False,
|
||||
"reason": reason,
|
||||
"versions": [
|
||||
{
|
||||
"id": version.get("id"),
|
||||
"name": version.get("name", ""),
|
||||
"thumbnailUrl": version.get("images")[0]["url"]
|
||||
if version.get("images")
|
||||
else None,
|
||||
"inLibrary": False,
|
||||
"hasBeenDownloaded": False,
|
||||
}
|
||||
for version in versions
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
if not scanner:
|
||||
@@ -2712,6 +2865,7 @@ class ModelLibraryHandler:
|
||||
"modelId": model_id,
|
||||
"modelName": model_name,
|
||||
"modelType": model_type,
|
||||
"supported": True,
|
||||
"versions": enriched_versions,
|
||||
}
|
||||
)
|
||||
@@ -2786,12 +2940,32 @@ class ModelLibraryHandler:
|
||||
model_type.lower() for model_type in CIVITAI_USER_MODEL_TYPES
|
||||
}
|
||||
lora_type_aliases = {model_type.lower() for model_type in VALID_LORA_TYPES}
|
||||
other_type_aliases = {
|
||||
model_type.lower() for model_type in VALID_OTHER_CIVITAI_TYPES
|
||||
}
|
||||
|
||||
# Acquire the other scanner lazily so adapters without it only
|
||||
# fail when the payload actually contains other-type models.
|
||||
# While the opt-in feature is off the scanner still exists (its
|
||||
# cache is empty), so other types simply report inLibrary=False.
|
||||
needs_other_scanner = any(
|
||||
isinstance(model, dict)
|
||||
and str(model.get("type", "")).lower() in other_type_aliases
|
||||
for model in models
|
||||
)
|
||||
other_scanner = None
|
||||
if needs_other_scanner:
|
||||
other_scanner = await self._service_registry.get_other_scanner()
|
||||
|
||||
type_scanner_map: Dict[str, Any] = {
|
||||
**{alias: lora_scanner for alias in lora_type_aliases},
|
||||
"checkpoint": checkpoint_scanner,
|
||||
"textualinversion": embedding_scanner,
|
||||
}
|
||||
if other_scanner is not None:
|
||||
type_scanner_map.update(
|
||||
{alias: other_scanner for alias in other_type_aliases}
|
||||
)
|
||||
|
||||
versions: list[dict[str, Any]] = []
|
||||
history_service = await self._get_download_history_service()
|
||||
@@ -2815,12 +2989,17 @@ class ModelLibraryHandler:
|
||||
"embedding",
|
||||
model_ids,
|
||||
)
|
||||
other_downloaded = await history_service.get_downloaded_version_ids_bulk(
|
||||
"other",
|
||||
model_ids,
|
||||
)
|
||||
downloaded_version_map: Dict[str, Dict[int, set[int]]] = {
|
||||
"lora": lora_downloaded,
|
||||
"locon": lora_downloaded,
|
||||
"dora": lora_downloaded,
|
||||
"checkpoint": checkpoint_downloaded,
|
||||
"textualinversion": embedding_downloaded,
|
||||
**{alias: other_downloaded for alias in VALID_OTHER_CIVITAI_TYPES},
|
||||
}
|
||||
for model in models:
|
||||
if not isinstance(model, dict):
|
||||
@@ -3274,6 +3453,18 @@ class FileSystemHandler:
|
||||
subprocess.Popen(["open", "-R", settings_file])
|
||||
else:
|
||||
folder = os.path.dirname(settings_file)
|
||||
if not _has_gui_display():
|
||||
# Headless/SSH session: xdg-open cannot open a file
|
||||
# manager, so hand the path to the browser for copying
|
||||
# instead of reporting a success that never happened.
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"message": "Headless session: path available for copying",
|
||||
"path": settings_file,
|
||||
"mode": "clipboard",
|
||||
}
|
||||
)
|
||||
subprocess.Popen(["xdg-open", folder])
|
||||
|
||||
return web.json_response(
|
||||
@@ -3307,6 +3498,76 @@ class FileSystemHandler:
|
||||
logger.error("Failed to open wildcards location: %s", exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def browse_directory(self, request: web.Request) -> web.Response:
|
||||
"""Browse a directory for the settings-UI directory picker."""
|
||||
try:
|
||||
data = await request.json()
|
||||
payload, status = browse_directory(data.get("path", ""))
|
||||
return web.json_response(payload, status=status)
|
||||
except json.JSONDecodeError:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Invalid JSON"}, status=400
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - defensive logging
|
||||
logger.error("Failed to browse directory: %s", exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def validate_path(self, request: web.Request) -> web.Response:
|
||||
"""Validate a filesystem path for the settings UI.
|
||||
|
||||
A well-formed request always returns HTTP 200; invalid paths are
|
||||
reported via ``error_code`` in the payload. HTTP 400 is reserved for
|
||||
malformed requests (missing path, invalid JSON).
|
||||
"""
|
||||
try:
|
||||
data = await request.json()
|
||||
raw_path = data.get("path")
|
||||
expect = data.get("expect", "directory")
|
||||
|
||||
if not raw_path or not isinstance(raw_path, str):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Missing path parameter"}, status=400
|
||||
)
|
||||
|
||||
# Business path convention: abspath only, never realpath.
|
||||
path = os.path.abspath(os.path.expanduser(raw_path))
|
||||
|
||||
exists = os.path.exists(path)
|
||||
is_directory = os.path.isdir(path) if exists else False
|
||||
readable = bool(exists and os.access(path, os.R_OK))
|
||||
writable = bool(exists and os.access(path, os.W_OK))
|
||||
|
||||
error_code = None
|
||||
if not exists:
|
||||
error_code = "path_not_found"
|
||||
elif expect == "directory" and not is_directory:
|
||||
error_code = "not_a_directory"
|
||||
elif expect == "file" and not os.path.isfile(path):
|
||||
error_code = "not_a_file"
|
||||
elif not readable:
|
||||
error_code = "not_readable"
|
||||
elif not writable:
|
||||
error_code = "not_writable"
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"path": path,
|
||||
"exists": exists,
|
||||
"is_directory": is_directory,
|
||||
"readable": readable,
|
||||
"writable": writable,
|
||||
"error_code": error_code,
|
||||
}
|
||||
)
|
||||
except json.JSONDecodeError:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Invalid JSON"}, status=400
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - defensive logging
|
||||
logger.error("Failed to validate path: %s", exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
|
||||
class CustomWordsHandler:
|
||||
"""Handler for autocomplete via TagFTSIndex."""
|
||||
@@ -3882,8 +4143,9 @@ class MiscHandlerSet:
|
||||
doctor: DoctorHandler,
|
||||
example_workflows: ExampleWorkflowsHandler,
|
||||
base_model: BaseModelHandlerSet,
|
||||
hf_handler: Any = None,
|
||||
model_source_handler: Any = None,
|
||||
agent_handler: Any = None,
|
||||
download_routing: Any = None,
|
||||
) -> None:
|
||||
self.health = health
|
||||
self.settings = settings
|
||||
@@ -3902,8 +4164,9 @@ class MiscHandlerSet:
|
||||
self.doctor = doctor
|
||||
self.example_workflows = example_workflows
|
||||
self.base_model = base_model
|
||||
self.hf_handler = hf_handler
|
||||
self.model_source_handler = model_source_handler
|
||||
self.agent_handler = agent_handler
|
||||
self.download_routing = download_routing
|
||||
|
||||
def to_route_mapping(
|
||||
self,
|
||||
@@ -3949,19 +4212,27 @@ class MiscHandlerSet:
|
||||
"open_settings_location": self.filesystem.open_settings_location,
|
||||
"open_backup_location": self.filesystem.open_backup_location,
|
||||
"open_wildcards_location": self.filesystem.open_wildcards_location,
|
||||
"browse_directory": self.filesystem.browse_directory,
|
||||
"validate_path": self.filesystem.validate_path,
|
||||
"search_custom_words": self.custom_words.search_custom_words,
|
||||
"search_wildcards": self.wildcards.search_wildcards,
|
||||
"get_supporters": self.supporters.get_supporters,
|
||||
"get_example_workflows": self.example_workflows.get_example_workflows,
|
||||
"get_example_workflow": self.example_workflows.get_example_workflow,
|
||||
# Hugging Face handlers
|
||||
"get_hf_repo_files": self.hf_handler.get_hf_repo_files,
|
||||
"download_hf_model": self.hf_handler.download_hf_model,
|
||||
"set_hf_url": self.hf_handler.set_hf_url,
|
||||
# External model sources (Hugging Face / ModelScope)
|
||||
"list_model_source_files": self.model_source_handler.list_model_source_files,
|
||||
"download_model_source": self.model_source_handler.download_model_source,
|
||||
"get_hf_repo_files": self.model_source_handler.list_model_source_files,
|
||||
"download_hf_model": self.model_source_handler.download_model_source,
|
||||
"set_hf_url": self.model_source_handler.set_hf_url,
|
||||
"get_model_sources": self.model_source_handler.get_model_sources,
|
||||
# Agent skill handlers
|
||||
"get_agent_skills": self.agent_handler.get_agent_skills,
|
||||
"execute_agent_skill": self.agent_handler.execute_agent_skill,
|
||||
"cancel_agent_skill": self.agent_handler.cancel_agent_skill,
|
||||
# Download routing handler
|
||||
"get_download_routing": self.download_routing.get_download_routing,
|
||||
# Base model handlers
|
||||
"get_base_models": self.base_model.get_base_models,
|
||||
"refresh_base_models": self.base_model.refresh_base_models,
|
||||
@@ -3975,6 +4246,7 @@ def build_service_registry_adapter() -> ServiceRegistryAdapter:
|
||||
get_lora_scanner=ServiceRegistry.get_lora_scanner,
|
||||
get_checkpoint_scanner=ServiceRegistry.get_checkpoint_scanner,
|
||||
get_embedding_scanner=ServiceRegistry.get_embedding_scanner,
|
||||
get_other_scanner=ServiceRegistry.get_other_scanner,
|
||||
get_downloaded_version_history_service=ServiceRegistry.get_downloaded_version_history_service,
|
||||
get_backup_service=ServiceRegistry.get_backup_service,
|
||||
)
|
||||
|
||||
@@ -15,6 +15,10 @@ from aiohttp import web
|
||||
import jinja2
|
||||
|
||||
from ...config import config
|
||||
from ...services.active_filters_store import (
|
||||
ActiveFiltersStore,
|
||||
active_filters_to_query_kwargs,
|
||||
)
|
||||
from ...services.download_coordinator import DownloadCoordinator
|
||||
from ...services.connectivity_guard import (
|
||||
OFFLINE_FRIENDLY_MESSAGE,
|
||||
@@ -33,10 +37,14 @@ from ...services.use_cases import (
|
||||
DownloadModelEarlyAccessError,
|
||||
DownloadModelUseCase,
|
||||
DownloadModelValidationError,
|
||||
FilenameTemplateUseCase,
|
||||
MetadataRefreshProgressReporter,
|
||||
)
|
||||
from ...services.websocket_manager import WebSocketManager
|
||||
from ...services.websocket_progress_callback import WebSocketProgressCallback
|
||||
from ...services.websocket_progress_callback import (
|
||||
WebSocketFilenameTemplateProgressCallback,
|
||||
WebSocketProgressCallback,
|
||||
)
|
||||
from ...services.download_queue_service import DownloadQueueService
|
||||
from ...services.errors import RateLimitError, ResourceNotFoundError
|
||||
from ...utils.civitai_utils import resolve_license_payload
|
||||
@@ -86,6 +94,7 @@ class ModelPageView:
|
||||
settings_service: SettingsManager,
|
||||
server_i18n,
|
||||
logger: logging.Logger,
|
||||
page_context_provider: Callable[[web.Request], Dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
self._template_env = template_env
|
||||
self._template_name = template_name
|
||||
@@ -93,6 +102,7 @@ class ModelPageView:
|
||||
self._settings = settings_service
|
||||
self._server_i18n = server_i18n
|
||||
self._logger = logger
|
||||
self._page_context_provider = page_context_provider
|
||||
|
||||
def _load_supporters(self) -> dict[str, Any]:
|
||||
"""Load supporters data from JSON file."""
|
||||
@@ -206,6 +216,16 @@ class ModelPageView:
|
||||
self._logger.error("Error loading cache data: %s", cache_error)
|
||||
template_context["is_initializing"] = True
|
||||
|
||||
if self._page_context_provider is not None:
|
||||
try:
|
||||
extra_context = self._page_context_provider(request)
|
||||
if isinstance(extra_context, dict):
|
||||
template_context.update(extra_context)
|
||||
except Exception as context_error: # pragma: no cover - logging path
|
||||
self._logger.error(
|
||||
"Error building page context: %s", context_error
|
||||
)
|
||||
|
||||
rendered = self._template_env.get_template(self._template_name).render(
|
||||
**template_context
|
||||
)
|
||||
@@ -1595,12 +1615,50 @@ class ModelQueryHandler:
|
||||
allow_selling_generated_content.lower() not in ("false", "0", "")
|
||||
)
|
||||
|
||||
# When requested, merge the manager page's active filters stored
|
||||
# server-side. Explicit query parameters take precedence over the
|
||||
# stored values.
|
||||
use_active_filters = (
|
||||
request.query.get("use_active_filters", "").lower() in ("1", "true")
|
||||
)
|
||||
if use_active_filters:
|
||||
stored = ActiveFiltersStore.get_instance().get_filters(
|
||||
self._service.model_type
|
||||
)
|
||||
injected = active_filters_to_query_kwargs(stored)
|
||||
if folder is None and "folder" in injected:
|
||||
folder = injected["folder"]
|
||||
if "recursive" not in request.query and "recursive" in injected:
|
||||
recursive = injected["recursive"]
|
||||
if not base_models and injected.get("base_models"):
|
||||
base_models = injected["base_models"]
|
||||
if not model_types and injected.get("model_types"):
|
||||
model_types = injected["model_types"]
|
||||
if not tag_filters and injected.get("tags"):
|
||||
tag_filters = injected["tags"]
|
||||
if not auto_tag_filters and injected.get("auto_tags"):
|
||||
auto_tag_filters = injected["auto_tags"]
|
||||
if "tag_logic" not in request.query and injected.get("tag_logic"):
|
||||
injected_logic = str(injected["tag_logic"]).lower()
|
||||
if injected_logic in ("any", "all"):
|
||||
tag_logic = injected_logic
|
||||
if credit_required is None and "credit_required" in injected:
|
||||
credit_required = injected["credit_required"]
|
||||
if (
|
||||
allow_selling_generated_content is None
|
||||
and "allow_selling_generated_content" in injected
|
||||
):
|
||||
allow_selling_generated_content = injected[
|
||||
"allow_selling_generated_content"
|
||||
]
|
||||
|
||||
# The presence of the recursive param (always sent by the loras
|
||||
# widget when filter mode is on) signals that the filter pipeline
|
||||
# must run even when no concrete filter is set, so global settings
|
||||
# like show_only_sfw stay consistent with the list endpoint.
|
||||
apply_filters = (
|
||||
"recursive" in request.query
|
||||
use_active_filters
|
||||
or "recursive" in request.query
|
||||
or folder is not None
|
||||
or bool(base_models)
|
||||
or bool(model_types)
|
||||
@@ -1634,6 +1692,50 @@ class ModelQueryHandler:
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def update_active_filters(self, request: web.Request) -> web.Response:
|
||||
"""Store the manager page's active filters for this model type."""
|
||||
try:
|
||||
payload = await request.json()
|
||||
except Exception:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Invalid JSON body"}, status=400
|
||||
)
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Body must be a JSON object"}, status=400
|
||||
)
|
||||
|
||||
try:
|
||||
ActiveFiltersStore.get_instance().set_filters(
|
||||
self._service.model_type, payload
|
||||
)
|
||||
return web.json_response({"success": True})
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error updating active filters for %s: %s",
|
||||
self._service.model_type,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def get_active_filters(self, request: web.Request) -> web.Response:
|
||||
"""Return the stored active filters for this model type."""
|
||||
try:
|
||||
filters = ActiveFiltersStore.get_instance().get_filters(
|
||||
self._service.model_type
|
||||
)
|
||||
return web.json_response({"success": True, "filters": filters})
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error getting active filters for %s: %s",
|
||||
self._service.model_type,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
|
||||
class ModelDownloadHandler:
|
||||
"""Coordinate downloads and progress reporting."""
|
||||
@@ -1812,6 +1914,11 @@ class ModelDownloadHandler:
|
||||
response_payload["status"] = status
|
||||
if "message" in progress_data:
|
||||
response_payload["message"] = progress_data["message"]
|
||||
# Post-transfer stage (indexing / source metadata); polling
|
||||
# consumers need it to tell "working" from "stuck".
|
||||
for field in ("stage", "platform"):
|
||||
if field in progress_data:
|
||||
response_payload[field] = progress_data[field]
|
||||
elif status is None and "message" in progress_data:
|
||||
response_payload["message"] = progress_data["message"]
|
||||
|
||||
@@ -2381,6 +2488,90 @@ class ModelMoveHandler:
|
||||
self._move_service = move_service
|
||||
self._logger = logger
|
||||
|
||||
async def create_folder(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
data = await request.json()
|
||||
except Exception:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Invalid JSON body"}, status=400
|
||||
)
|
||||
try:
|
||||
folder_path = data.get("folder_path")
|
||||
if not folder_path:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Folder path is required"}, status=400
|
||||
)
|
||||
result = await self._move_service.create_folder(folder_path)
|
||||
status = 200 if result.get("success") else 400
|
||||
return web.json_response(result, status=status)
|
||||
except Exception as exc:
|
||||
self._logger.error("Error creating folder: %s", exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def delete_folder(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
data = await request.json()
|
||||
except Exception:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Invalid JSON body"}, status=400
|
||||
)
|
||||
try:
|
||||
folder_path = data.get("folder_path")
|
||||
if not folder_path:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Folder path is required"}, status=400
|
||||
)
|
||||
dry_run = bool(data.get("dry_run"))
|
||||
result = await self._move_service.delete_folder(
|
||||
folder_path, dry_run=dry_run
|
||||
)
|
||||
if result.get("success"):
|
||||
if not dry_run:
|
||||
_broadcast_models_changed()
|
||||
return web.json_response(result, status=200)
|
||||
|
||||
# "not_empty" / "busy" are conflicts between the tree the client
|
||||
# rendered and the on-disk truth; everything else is a bad request.
|
||||
code = result.get("code")
|
||||
status = 409 if code in ("not_empty", "busy") else 400
|
||||
return web.json_response(result, status=status)
|
||||
except Exception as exc:
|
||||
self._logger.error("Error deleting folder: %s", exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def rename_folder(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
data = await request.json()
|
||||
except Exception:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Invalid JSON body"}, status=400
|
||||
)
|
||||
try:
|
||||
folder_path = data.get("folder_path")
|
||||
new_name = data.get("new_name")
|
||||
if not folder_path:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Folder path is required"}, status=400
|
||||
)
|
||||
if not new_name:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "New folder name is required"}, status=400
|
||||
)
|
||||
result = await self._move_service.rename_folder(folder_path, new_name)
|
||||
if result.get("success"):
|
||||
if result.get("renamed"):
|
||||
_broadcast_models_changed()
|
||||
return web.json_response(result, status=200)
|
||||
|
||||
# A name collision or a staged delete inside the subtree is a
|
||||
# conflict with the state the client rendered, not a bad request.
|
||||
code = result.get("code")
|
||||
status = 409 if code in ("target_exists", "busy") else 400
|
||||
return web.json_response(result, status=status)
|
||||
except Exception as exc:
|
||||
self._logger.error("Error renaming folder: %s", exc, exc_info=True)
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
async def move_model(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
data = await request.json()
|
||||
@@ -2505,6 +2696,71 @@ class ModelAutoOrganizeHandler:
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
|
||||
class ModelFilenameTemplateHandler:
|
||||
"""Apply the configured filename template to existing library models."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
use_case: FilenameTemplateUseCase,
|
||||
progress_callback: WebSocketFilenameTemplateProgressCallback,
|
||||
logger: logging.Logger,
|
||||
) -> None:
|
||||
self._use_case = use_case
|
||||
self._progress_callback = progress_callback
|
||||
self._logger = logger
|
||||
|
||||
async def apply_filename_template(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
file_paths = None
|
||||
if request.method == "POST":
|
||||
try:
|
||||
data = await request.json()
|
||||
file_paths = data.get("file_paths")
|
||||
except Exception: # pragma: no cover - permissive path
|
||||
pass
|
||||
else:
|
||||
# GET variant (browser extension is GET-only): comma-separated
|
||||
# file_paths query parameter.
|
||||
raw_file_paths = request.query.get("file_paths")
|
||||
if raw_file_paths:
|
||||
file_paths = [
|
||||
path.strip()
|
||||
for path in raw_file_paths.split(",")
|
||||
if path.strip()
|
||||
]
|
||||
|
||||
result = await self._use_case.execute(
|
||||
file_paths=file_paths,
|
||||
progress_callback=self._progress_callback,
|
||||
)
|
||||
_broadcast_models_changed()
|
||||
return web.json_response(result.to_dict())
|
||||
except AutoOrganizeInProgressError:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "Another library operation is already running. Please wait for it to complete.",
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
except Exception as exc:
|
||||
self._logger.error(
|
||||
"Error in apply_filename_template: %s", exc, exc_info=True
|
||||
)
|
||||
try:
|
||||
await self._progress_callback.on_progress(
|
||||
{
|
||||
"type": "filename_template_progress",
|
||||
"status": "error",
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
except Exception: # pragma: no cover - defensive reporting
|
||||
pass
|
||||
return web.json_response({"success": False, "error": str(exc)}, status=500)
|
||||
|
||||
|
||||
class ModelUpdateHandler:
|
||||
"""Handle update tracking requests."""
|
||||
|
||||
@@ -3272,6 +3528,7 @@ class ModelHandlerSet:
|
||||
civitai: ModelCivitaiHandler
|
||||
move: ModelMoveHandler
|
||||
auto_organize: ModelAutoOrganizeHandler
|
||||
filename_template: ModelFilenameTemplateHandler
|
||||
updates: ModelUpdateHandler
|
||||
|
||||
def to_route_mapping(
|
||||
@@ -3331,14 +3588,20 @@ class ModelHandlerSet:
|
||||
"get_civitai_model_by_hash": self.civitai.get_civitai_model_by_hash,
|
||||
"move_model": self.move.move_model,
|
||||
"move_models_bulk": self.move.move_models_bulk,
|
||||
"create_folder": self.move.create_folder,
|
||||
"delete_folder": self.move.delete_folder,
|
||||
"rename_folder": self.move.rename_folder,
|
||||
"auto_organize_models": self.auto_organize.auto_organize_models,
|
||||
"get_auto_organize_progress": self.auto_organize.get_auto_organize_progress,
|
||||
"apply_filename_template": self.filename_template.apply_filename_template,
|
||||
"get_model_notes": self.query.get_model_notes,
|
||||
"get_model_preview_url": self.query.get_model_preview_url,
|
||||
"get_model_civitai_url": self.query.get_model_civitai_url,
|
||||
"get_model_metadata": self.query.get_model_metadata,
|
||||
"get_model_description": self.query.get_model_description,
|
||||
"get_relative_paths": self.query.get_relative_paths,
|
||||
"update_active_filters": self.query.update_active_filters,
|
||||
"get_active_filters": self.query.get_active_filters,
|
||||
"refresh_model_updates": self.updates.refresh_model_updates,
|
||||
"fetch_missing_civitai_license_data": self.updates.fetch_missing_civitai_license_data,
|
||||
"set_model_update_ignore": self.updates.set_model_update_ignore,
|
||||
|
||||
@@ -0,0 +1,639 @@
|
||||
"""Handlers for external model sources: linking, file listing and downloads.
|
||||
|
||||
Covers every site registered in :mod:`py.services.model_sources`. The module
|
||||
was Hugging Face only (``hf_handlers.py`` / ``HfHandler``) until ModelScope
|
||||
downloads were added; the per-site differences now live in the providers, so
|
||||
this file has no platform branches beyond the capability lookups.
|
||||
|
||||
The historical route paths (``/api/lm/set-hf-url``, ``/api/lm/hf-repo-files``,
|
||||
``/api/lm/download-hf-model``) are still registered as aliases of the generic
|
||||
handlers, so existing callers keep working.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
from ...config import config
|
||||
from ...services.downloader import (
|
||||
DownloadProgress,
|
||||
get_downloader,
|
||||
)
|
||||
from ...services.aria2_downloader import Aria2Downloader
|
||||
from ...services.model_sources import (
|
||||
ModelSourceError,
|
||||
SourceRef,
|
||||
detect_source,
|
||||
get_download_source,
|
||||
hydrate_from_source,
|
||||
is_valid_source_id,
|
||||
list_sources,
|
||||
normalize_metadata_source,
|
||||
)
|
||||
from ...services.settings_manager import get_settings_manager
|
||||
from ...services.service_registry import ServiceRegistry
|
||||
from ...services.websocket_manager import ws_manager
|
||||
from ...utils.metadata_manager import MetadataManager
|
||||
from ...utils.models import LoraMetadata, CheckpointMetadata, EmbeddingMetadata
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_MODEL_CLASS = LoraMetadata
|
||||
_DEFAULT_SCANNER_GETTER = "get_lora_scanner"
|
||||
|
||||
|
||||
def _infer_model_type(model_root: str) -> tuple[Any, str]:
|
||||
"""Determine model class and scanner by matching ``model_root`` against the
|
||||
configured root paths for each model type (from ``Config``).
|
||||
|
||||
The ``model_root`` value comes from the frontend's model-root dropdown,
|
||||
which is populated from the current page's scanner roots. By checking
|
||||
which scanner's root list it belongs to, we avoid fragile heuristics
|
||||
like substring-matching path names.
|
||||
"""
|
||||
norm = os.path.normpath(model_root).replace(os.sep, "/")
|
||||
|
||||
# LoRA roots
|
||||
for p in (config.loras_roots or []) + (config.extra_loras_roots or []):
|
||||
if os.path.normpath(p).replace(os.sep, "/") == norm:
|
||||
return LoraMetadata, "get_lora_scanner"
|
||||
|
||||
# Checkpoint / UNet roots
|
||||
for p in (
|
||||
(config.checkpoints_roots or [])
|
||||
+ (config.extra_checkpoints_roots or [])
|
||||
+ (config.unet_roots or [])
|
||||
+ (config.extra_unet_roots or [])
|
||||
):
|
||||
if os.path.normpath(p).replace(os.sep, "/") == norm:
|
||||
return CheckpointMetadata, "get_checkpoint_scanner"
|
||||
|
||||
# Embedding roots
|
||||
for p in (config.embeddings_roots or []) + (config.extra_embeddings_roots or []):
|
||||
if os.path.normpath(p).replace(os.sep, "/") == norm:
|
||||
return EmbeddingMetadata, "get_embedding_scanner"
|
||||
|
||||
# Fallback — should not happen in normal use
|
||||
logger.warning(
|
||||
"Could not determine model type for root '%s'; defaulting to LoRA",
|
||||
model_root,
|
||||
)
|
||||
return _DEFAULT_MODEL_CLASS, _DEFAULT_SCANNER_GETTER
|
||||
|
||||
|
||||
async def _report_phase(
|
||||
download_id: str | None, stage: str, platform: str = ""
|
||||
) -> None:
|
||||
"""Tell the progress UI which post-transfer stage is running.
|
||||
|
||||
A download's byte counter stops the moment the last byte lands, but the
|
||||
backend still has to index the file and read the model site's API. Without
|
||||
this the bar sits at 100% reporting "0 B/s" and the download looks stuck for
|
||||
several seconds. *stage* is machine-readable — the UI localises it — and
|
||||
*platform* lets it name the site the metadata comes from.
|
||||
"""
|
||||
|
||||
if not download_id:
|
||||
return
|
||||
try:
|
||||
await ws_manager.broadcast_download_progress(
|
||||
download_id,
|
||||
{
|
||||
"status": "metadata",
|
||||
"stage": stage,
|
||||
"platform": platform,
|
||||
"progress": 100,
|
||||
},
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - progress must never be fatal
|
||||
logger.debug("Failed to report the '%s' phase: %s", stage, exc)
|
||||
|
||||
|
||||
async def _save_source_metadata(
|
||||
dest_path: str, ref: SourceRef, model_root: str, *, download_id: str | None = None
|
||||
) -> None:
|
||||
"""Create a proper .metadata.json and add the model to the scanner cache.
|
||||
|
||||
The metadata is created through the owning scanner rather than
|
||||
``MetadataManager.create_default_metadata()``, because that is the only
|
||||
factory that knows when hashing must be deferred: ``CheckpointScanner`` and
|
||||
``OtherScanner`` deliberately record ``hash_status="pending"`` with an empty
|
||||
``sha256`` for their multi-GB files, and the generic helper would read a
|
||||
10 GB checkpoint end to end *inside the download request*. Scanners for the
|
||||
small types delegate straight back to it, so nothing changes for them.
|
||||
|
||||
The external-source fields are then overlaid and the model is registered in
|
||||
the in-memory scanner cache so it appears immediately without a full
|
||||
filesystem walk.
|
||||
|
||||
Finally the site's own published metadata is applied (see
|
||||
:func:`~py.services.model_sources.hydration.hydrate_from_source`), so a
|
||||
ModelScope or Hugging Face download lands with the same populated model
|
||||
card a CivitAI download produces instead of a bare filename and hash.
|
||||
|
||||
Both post-transfer stages are reported through *download_id* when the UI is
|
||||
watching one, because neither advances the byte counter.
|
||||
"""
|
||||
try:
|
||||
model_class, scanner_getter_name = _infer_model_type(model_root)
|
||||
|
||||
scanner = None
|
||||
scanner_getter = getattr(ServiceRegistry, scanner_getter_name, None)
|
||||
if scanner_getter is not None:
|
||||
scanner = await scanner_getter()
|
||||
|
||||
# 1. Create proper metadata (reads safetensors headers; hashes only for
|
||||
# the model types whose scanner does not defer it)
|
||||
await _report_phase(download_id, "indexing", ref.platform)
|
||||
create_metadata = getattr(scanner, "_create_default_metadata", None)
|
||||
if create_metadata is not None:
|
||||
metadata = await create_metadata(dest_path)
|
||||
else:
|
||||
metadata = await MetadataManager.create_default_metadata(
|
||||
dest_path, model_class=model_class
|
||||
)
|
||||
if metadata is None:
|
||||
logger.warning("create_default_metadata returned None for %s", dest_path)
|
||||
return
|
||||
|
||||
# 2. Overlay the external-source fields (`hf_url` is written by
|
||||
# normalisation for Hugging Face only)
|
||||
fields = metadata._unknown_fields
|
||||
fields["source_url"] = ref.url
|
||||
fields["source_platform"] = ref.platform
|
||||
if ref.platform == "huggingface":
|
||||
fields["hf_url"] = ref.url
|
||||
metadata.from_civitai = False # externally-sourced models are not from CivitAI
|
||||
|
||||
# 3. Save metadata atomically
|
||||
await MetadataManager.save_metadata(dest_path, metadata)
|
||||
logger.info(
|
||||
"Saved %s metadata (source=%s, hash_status=%s) for %s",
|
||||
ref.platform, ref.url, getattr(metadata, "hash_status", "?"), dest_path,
|
||||
)
|
||||
|
||||
# 4. Determine relative folder path for cache
|
||||
# model_root is an absolute path; dest_path is under it
|
||||
folder = ""
|
||||
if os.path.isabs(model_root) and dest_path.startswith(model_root):
|
||||
rel = os.path.relpath(os.path.dirname(dest_path), model_root)
|
||||
folder = rel.replace(os.sep, "/") if rel != "." else ""
|
||||
|
||||
# 5. Add to scanner cache (same as CivitAI's _execute_download does)
|
||||
if scanner is not None:
|
||||
metadata_dict = normalize_metadata_source(metadata.to_dict())
|
||||
await scanner.add_model_to_cache(metadata_dict, folder)
|
||||
logger.info("Added %s to scanner cache (folder=%s)", dest_path, folder)
|
||||
|
||||
# 6. Top up from the site's public API. Runs last so the scanner-cache
|
||||
# refresh it performs lands on the entry created above. It never
|
||||
# raises and never fails the download.
|
||||
await _report_phase(download_id, "source", ref.platform)
|
||||
await hydrate_from_source(dest_path, ref=ref)
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to save source metadata for %s: %s", dest_path, exc)
|
||||
|
||||
|
||||
def _find_matching_root(dest_dir: str) -> str | None:
|
||||
"""Walk up *dest_dir* to find which configured scanner root it belongs to."""
|
||||
norm = os.path.normpath(dest_dir).replace(os.sep, "/")
|
||||
all_roots = []
|
||||
for root_list in (
|
||||
config.loras_roots or [],
|
||||
config.extra_loras_roots or [],
|
||||
config.checkpoints_roots or [],
|
||||
config.extra_checkpoints_roots or [],
|
||||
config.unet_roots or [],
|
||||
config.extra_unet_roots or [],
|
||||
config.embeddings_roots or [],
|
||||
config.extra_embeddings_roots or [],
|
||||
):
|
||||
all_roots.extend([os.path.normpath(p).replace(os.sep, "/") for p in root_list])
|
||||
# Find the longest matching prefix
|
||||
match: str | None = None
|
||||
for root in all_roots:
|
||||
if norm.startswith(root):
|
||||
if match is None or len(root) > len(match):
|
||||
match = root
|
||||
return match
|
||||
|
||||
|
||||
async def _add_to_scanner_cache(dest_path: str, metadata: dict[str, Any]) -> None:
|
||||
model_dir = os.path.dirname(dest_path)
|
||||
model_root = _find_matching_root(model_dir)
|
||||
if not model_root:
|
||||
raise ValueError(f"File path {dest_path} is not within any configured scanner root")
|
||||
scanner_getter_name = _infer_model_type(model_root)[1]
|
||||
scanner_getter = getattr(ServiceRegistry, scanner_getter_name, None)
|
||||
if scanner_getter is None:
|
||||
raise RuntimeError(f"Scanner getter '{scanner_getter_name}' not found in ServiceRegistry")
|
||||
scanner = await scanner_getter()
|
||||
if scanner is None:
|
||||
raise RuntimeError(f"Scanner '{scanner_getter_name}' returned None")
|
||||
await scanner.update_single_model_cache(dest_path, dest_path, metadata)
|
||||
|
||||
|
||||
def _unsupported_platform_error(platform: str) -> web.Response:
|
||||
supported = ", ".join(source.label for source in list_sources() if source.supports_download)
|
||||
return web.json_response(
|
||||
{"error": f"'{platform}' does not support downloads. Supported: {supported}"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
|
||||
class ModelSourceHandler:
|
||||
"""Handle external model browsing, linking and downloads."""
|
||||
|
||||
async def get_model_sources(self, request: web.Request) -> web.Response:
|
||||
"""List the external model sites the UI can link a model to.
|
||||
|
||||
Used by the "Link Model" dialog to validate URLs client-side, to
|
||||
explain which sites support AI metadata enrichment, and to pick the
|
||||
right download endpoint/revision.
|
||||
"""
|
||||
|
||||
return web.json_response([
|
||||
{
|
||||
"platform": source.platform,
|
||||
"label": source.label,
|
||||
"supports_enrichment": source.supports_enrichment,
|
||||
"supports_download": source.supports_download,
|
||||
"default_revision": source.default_revision,
|
||||
"example_url": source.canonical_url(
|
||||
"user/repo" if source.platform != "tensorart" else "827823520299086029"
|
||||
),
|
||||
}
|
||||
for source in list_sources()
|
||||
])
|
||||
|
||||
async def set_hf_url(self, request: web.Request) -> web.Response:
|
||||
"""Link a model file to its page on an external model site.
|
||||
|
||||
Accepts ``source_url`` (preferred) or the legacy ``hf_url`` / ``url``
|
||||
payload key. Every registered site is recognised and the platform is
|
||||
stored alongside the canonical URL. TensorArt models can be linked and
|
||||
browsed, but not AI-enriched.
|
||||
|
||||
The route path keeps its historical ``set-hf-url`` name.
|
||||
"""
|
||||
|
||||
try:
|
||||
payload: dict[str, Any] = await request.json()
|
||||
except json.JSONDecodeError:
|
||||
return web.json_response({"success": False, "error": "Invalid JSON"}, status=400)
|
||||
|
||||
file_path = (payload.get("file_path") or "").strip()
|
||||
raw_url = (
|
||||
payload.get("source_url")
|
||||
or payload.get("hf_url")
|
||||
or payload.get("url")
|
||||
or ""
|
||||
)
|
||||
source_url = raw_url.strip() if isinstance(raw_url, str) else ""
|
||||
|
||||
if not file_path or not source_url:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "Missing required fields: 'file_path' and 'source_url'",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
|
||||
ref = detect_source(source_url, strict=True)
|
||||
if ref is None:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": (
|
||||
"Unsupported model URL. Supported formats: "
|
||||
+ ", ".join(
|
||||
f"{s.label} ({s.canonical_url('user/repo')})"
|
||||
if s.platform != "tensorart"
|
||||
else f"{s.label} (https://tensor.art/models/<id>)"
|
||||
for s in list_sources()
|
||||
)
|
||||
),
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
|
||||
if not os.path.isfile(file_path):
|
||||
return web.json_response(
|
||||
{"success": False, "error": f"File not found: {file_path}"},
|
||||
status=404,
|
||||
)
|
||||
|
||||
model_root = _find_matching_root(os.path.dirname(file_path))
|
||||
if not model_root:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "File is not within any configured model directory. Cannot link to a model source.",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
|
||||
try:
|
||||
existing = await MetadataManager.load_metadata_payload(file_path)
|
||||
|
||||
already_linked = (
|
||||
(existing.get("source_url") or "").strip() == ref.url
|
||||
and (existing.get("source_platform") or "").strip().lower()
|
||||
== ref.platform
|
||||
) or (
|
||||
not existing.get("source_url")
|
||||
and ref.platform == "huggingface"
|
||||
and (existing.get("hf_url") or "").strip() == ref.url
|
||||
)
|
||||
if already_linked:
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": "source_url already set",
|
||||
"source_url": ref.url,
|
||||
"source_platform": ref.platform,
|
||||
"hf_url": ref.url if ref.platform == "huggingface" else "",
|
||||
})
|
||||
|
||||
existing["source_url"] = ref.url
|
||||
existing["source_platform"] = ref.platform
|
||||
if ref.platform == "huggingface":
|
||||
existing["hf_url"] = ref.url
|
||||
else:
|
||||
existing.pop("hf_url", None)
|
||||
normalize_metadata_source(existing)
|
||||
|
||||
# NOTE: deliberately do NOT touch `from_civitai` here. It records
|
||||
# where the metadata came from, and the UI must show the CivitAI
|
||||
# link whenever CivitAI data is present — linking an external
|
||||
# source must not hide it (#1094). Source provenance is tracked
|
||||
# via `source_platform` / `source_url`.
|
||||
await MetadataManager.save_metadata(file_path, existing)
|
||||
|
||||
await _add_to_scanner_cache(file_path, existing)
|
||||
|
||||
logger.info(
|
||||
"Linked %s to %s source (%s)", file_path, ref.platform, ref.url
|
||||
)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"Linked to {ref.url}",
|
||||
"source_url": ref.url,
|
||||
"source_platform": ref.platform,
|
||||
"hf_url": existing.get("hf_url", ""),
|
||||
})
|
||||
except Exception as exc:
|
||||
logger.error("Failed to link %s to a model source: %s", file_path, exc)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(exc)},
|
||||
status=500,
|
||||
)
|
||||
|
||||
async def list_model_source_files(self, request: web.Request) -> web.Response:
|
||||
"""List the downloadable weight files of an external repository.
|
||||
|
||||
Query params: ``platform``, ``repo`` (``owner/name``), ``revision``
|
||||
(optional; each site has its own default branch).
|
||||
|
||||
Returns a JSON array of ``{"filename", "size"}``, largest first —
|
||||
the same shape the Hugging Face endpoint has always returned.
|
||||
"""
|
||||
|
||||
platform = (request.query.get("platform") or "").strip()
|
||||
repo = (request.query.get("repo") or "").strip()
|
||||
revision = (request.query.get("revision") or "").strip()
|
||||
|
||||
source = get_download_source(platform)
|
||||
if source is None:
|
||||
return _unsupported_platform_error(platform)
|
||||
if not is_valid_source_id(repo):
|
||||
return web.json_response(
|
||||
{"error": "Missing or invalid 'repo' parameter (expected owner/name)"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
try:
|
||||
files = await source.list_files(repo, revision)
|
||||
except ModelSourceError as exc:
|
||||
return web.json_response({"error": str(exc)}, status=exc.status)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to list %s files in %s: %s", platform, repo, exc)
|
||||
return web.json_response({"error": str(exc)}, status=502)
|
||||
|
||||
return web.json_response(files)
|
||||
|
||||
async def download_model_source(self, request: web.Request) -> web.Response:
|
||||
"""Download a single file from an external repository.
|
||||
|
||||
POST JSON body::
|
||||
|
||||
{
|
||||
"platform": "modelscope",
|
||||
"repo": "owner/name",
|
||||
"filename": "subdir/model.safetensors",
|
||||
"revision": "master",
|
||||
"model_root": "loras",
|
||||
"relative_path": "",
|
||||
"use_default_paths": false,
|
||||
"download_id": "optional-batch-id"
|
||||
}
|
||||
|
||||
``platform`` defaults to ``huggingface`` when omitted, which keeps the
|
||||
legacy ``/api/lm/download-hf-model`` payload working unchanged.
|
||||
|
||||
If ``download_id`` is provided, real-time progress (bytes, speed,
|
||||
percentage) is broadcast via the WebSocket progress system.
|
||||
|
||||
Respects the ``download_backend`` setting (``aria2`` or ``default``).
|
||||
"""
|
||||
try:
|
||||
payload: dict[str, Any] = await request.json()
|
||||
except json.JSONDecodeError:
|
||||
return web.json_response({"error": "Invalid JSON"}, status=400)
|
||||
|
||||
platform = (payload.get("platform") or "huggingface").strip()
|
||||
repo = (payload.get("repo") or "").strip()
|
||||
filename = (payload.get("filename") or "").strip()
|
||||
revision = (payload.get("revision") or "").strip()
|
||||
model_root = (payload.get("model_root") or "").strip()
|
||||
relative_path = (payload.get("relative_path") or "").strip()
|
||||
use_default_paths = bool(payload.get("use_default_paths", False))
|
||||
download_id: str | None = payload.get("download_id")
|
||||
|
||||
logger.info(
|
||||
"download_model_source: platform=%s repo=%s file=%s root=%s download_id=%s",
|
||||
platform, repo, filename, model_root, download_id,
|
||||
)
|
||||
|
||||
source = get_download_source(platform)
|
||||
if source is None:
|
||||
return _unsupported_platform_error(platform)
|
||||
|
||||
if not repo or not filename:
|
||||
return web.json_response(
|
||||
{"error": "Missing required fields: 'repo' and 'filename'"}, status=400
|
||||
)
|
||||
|
||||
# `owner/name` only; the components become path segments below.
|
||||
if not is_valid_source_id(repo):
|
||||
return web.json_response({"error": f"Invalid repo format: {repo}"}, status=400)
|
||||
owner, repo_name = repo.split("/", 1)
|
||||
|
||||
# Validate filename — must not contain path traversal
|
||||
if ".." in filename:
|
||||
return web.json_response({"error": "Invalid filename"}, status=400)
|
||||
|
||||
# Validate relative_path — must not be absolute or escape base directory
|
||||
if relative_path:
|
||||
if os.path.isabs(relative_path):
|
||||
return web.json_response({"error": "relative_path must not be absolute"}, status=400)
|
||||
if ".." in relative_path.split("/") or "\\" in relative_path:
|
||||
return web.json_response({"error": "Invalid relative_path"}, status=400)
|
||||
|
||||
# Use model_root directly as the base directory — same approach as
|
||||
# CivitAI's download path (download_manager.py). No realpath, no
|
||||
# allowed-roots validation, no path-traversal check; those are
|
||||
# unnecessary when the frontend sends the path from its own dropdown
|
||||
# (populated from scanner roots). Using the "business path" directly
|
||||
# keeps dest_path consistent with scanner roots so that later folder
|
||||
# derivation (in _save_source_metadata) works correctly.
|
||||
if os.path.isabs(model_root):
|
||||
base_dir = os.path.normpath(model_root)
|
||||
else:
|
||||
base_dir = os.path.normpath(os.path.join(os.getcwd(), "models", model_root))
|
||||
|
||||
if use_default_paths:
|
||||
target_dir = os.path.join(base_dir, source.default_subdir, owner, repo_name)
|
||||
elif relative_path:
|
||||
target_dir = os.path.join(base_dir, relative_path)
|
||||
else:
|
||||
target_dir = base_dir
|
||||
|
||||
# Strip the repository sub-directory — "diffusion_models/xxx.safetensors"
|
||||
# is a repository convention, not meaningful for local storage.
|
||||
file_base = os.path.basename(filename)
|
||||
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
dest_path = os.path.join(target_dir, file_base)
|
||||
|
||||
# Built per request: sites that redirect to a CDN hand out a
|
||||
# time-limited token in the redirect, so the URL must never be cached.
|
||||
resolve_url = source.file_download_url(repo, filename, revision)
|
||||
ref = SourceRef(
|
||||
platform=source.platform, source_id=repo, url=source.canonical_url(repo)
|
||||
)
|
||||
|
||||
# Check if already exists (simple skip)
|
||||
if os.path.exists(dest_path) and os.path.getsize(dest_path) > 0:
|
||||
logger.info("download_model_source: file already exists, skipping — %s", dest_path)
|
||||
# The sidecar may predate the source metadata being fetched, or may
|
||||
# have been deleted, so top it up instead of skipping past it.
|
||||
# Hydration no-ops when there is no sidecar to update.
|
||||
await _report_phase(download_id, "source", source.platform)
|
||||
await hydrate_from_source(dest_path, ref=ref)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"File already exists: {dest_path}",
|
||||
"path": dest_path,
|
||||
})
|
||||
|
||||
# Set up progress callback if download_id is provided
|
||||
progress_callback = None
|
||||
if download_id:
|
||||
|
||||
async def _progress_callback(
|
||||
progress: float | DownloadProgress,
|
||||
snapshot: DownloadProgress | None = None,
|
||||
) -> None:
|
||||
percent = 0.0
|
||||
metrics = snapshot if isinstance(snapshot, DownloadProgress) else None
|
||||
|
||||
if isinstance(progress, DownloadProgress):
|
||||
percent = progress.percent_complete
|
||||
metrics = progress
|
||||
elif isinstance(snapshot, DownloadProgress):
|
||||
percent = snapshot.percent_complete
|
||||
else:
|
||||
percent = float(progress)
|
||||
|
||||
broadcast: dict[str, Any] = {
|
||||
"status": "progress",
|
||||
"progress": round(percent),
|
||||
}
|
||||
if metrics:
|
||||
broadcast["bytes_downloaded"] = metrics.bytes_downloaded
|
||||
broadcast["total_bytes"] = metrics.total_bytes
|
||||
broadcast["bytes_per_second"] = metrics.bytes_per_second
|
||||
|
||||
await ws_manager.broadcast_download_progress(download_id, broadcast)
|
||||
|
||||
progress_callback = _progress_callback
|
||||
|
||||
# Respect download backend setting (aria2 vs default)
|
||||
download_backend = (
|
||||
get_settings_manager().get("download_backend", "default")
|
||||
)
|
||||
|
||||
if download_backend == "aria2":
|
||||
aria2 = await Aria2Downloader.get_instance()
|
||||
aid = download_id or f"{source.platform}_{repo}_{filename}"
|
||||
try:
|
||||
ok, result = await aria2.download_file(
|
||||
url=resolve_url,
|
||||
save_path=dest_path,
|
||||
download_id=aid,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
if ok:
|
||||
await _save_source_metadata(
|
||||
dest_path, ref, model_root, download_id=download_id
|
||||
)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"Downloaded to {dest_path}",
|
||||
"path": dest_path,
|
||||
})
|
||||
return web.json_response(
|
||||
{"success": False, "error": result or "aria2 download failed"},
|
||||
status=500,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("%s download (aria2) failed: %s", platform, exc)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(exc)}, status=500
|
||||
)
|
||||
|
||||
# Default: use built-in aiohttp Downloader
|
||||
downloader = await get_downloader()
|
||||
try:
|
||||
success, result = await downloader.download_file(
|
||||
url=resolve_url,
|
||||
save_path=dest_path,
|
||||
use_auth=False,
|
||||
allow_resume=True,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
if success:
|
||||
await _save_source_metadata(
|
||||
dest_path, ref, model_root, download_id=download_id
|
||||
)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"Downloaded to {result}",
|
||||
"path": result,
|
||||
})
|
||||
return web.json_response(
|
||||
{"success": False, "error": result or "Download failed"},
|
||||
status=500,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("%s download failed: %s", platform, exc)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(exc)}, status=500
|
||||
)
|
||||
@@ -35,6 +35,7 @@ _MODEL_TYPE_GETTER_NAMES: Dict[str, str] = {
|
||||
"loras": "get_lora_scanner",
|
||||
"checkpoints": "get_checkpoint_scanner",
|
||||
"embeddings": "get_embedding_scanner",
|
||||
"other": "get_other_scanner",
|
||||
}
|
||||
|
||||
# Staged batch ids are ``uuid.uuid4().hex`` (32 lowercase hex chars). The id is
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -37,6 +37,8 @@ MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("GET", "/api/lm/wildcards/search", "search_wildcards"),
|
||||
RouteDefinition("POST", "/api/lm/wildcards/open-location", "open_wildcards_location"),
|
||||
RouteDefinition("POST", "/api/lm/open-file-location", "open_file_location"),
|
||||
RouteDefinition("POST", "/api/lm/browse-directory", "browse_directory"),
|
||||
RouteDefinition("POST", "/api/lm/validate-path", "validate_path"),
|
||||
RouteDefinition("POST", "/api/lm/update-usage-stats", "update_usage_stats"),
|
||||
RouteDefinition("GET", "/api/lm/get-usage-stats", "get_usage_stats"),
|
||||
RouteDefinition("POST", "/api/lm/update-lora-code", "update_lora_code"),
|
||||
@@ -99,16 +101,31 @@ MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/delete-model-version", "delete_model_version"
|
||||
),
|
||||
# Hugging Face model endpoints
|
||||
# External model source endpoints (Hugging Face / ModelScope).
|
||||
# The hf-* paths are the historical names, kept as aliases.
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/model-source-files", "list_model_source_files"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/hf-repo-files", "get_hf_repo_files"
|
||||
),
|
||||
# Download target routing decision (checkpoint vs diffusion model roots)
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/download/routing", "get_download_routing"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/download-model-source", "download_model_source"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/download-hf-model", "download_hf_model"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/set-hf-url", "set_hf_url"
|
||||
),
|
||||
# Supported external model sites (Hugging Face / ModelScope / TensorArt)
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/model-sources", "get_model_sources"
|
||||
),
|
||||
# Agent skill endpoints
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/agent/skills", "get_agent_skills"
|
||||
|
||||
@@ -39,8 +39,9 @@ from .handlers.misc_handlers import (
|
||||
build_service_registry_adapter,
|
||||
)
|
||||
from .handlers.base_model_handlers import BaseModelHandlerSet
|
||||
from .handlers.hf_handlers import HfHandler
|
||||
from .handlers.model_source_handlers import ModelSourceHandler
|
||||
from .handlers.agent_handlers import AgentHandler
|
||||
from .handlers.download_routing_handlers import DownloadRoutingHandler
|
||||
from .misc_route_registrar import MiscRouteRegistrar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -138,8 +139,9 @@ class MiscRoutes:
|
||||
doctor = DoctorHandler(settings_service=self._settings)
|
||||
example_workflows = ExampleWorkflowsHandler()
|
||||
base_model = BaseModelHandlerSet()
|
||||
hf_handler = HfHandler()
|
||||
model_source_handler = ModelSourceHandler()
|
||||
agent_handler = AgentHandler()
|
||||
download_routing = DownloadRoutingHandler()
|
||||
|
||||
return self._handler_set_factory(
|
||||
health=health,
|
||||
@@ -159,8 +161,9 @@ class MiscRoutes:
|
||||
doctor=doctor,
|
||||
example_workflows=example_workflows,
|
||||
base_model=base_model,
|
||||
hf_handler=hf_handler,
|
||||
model_source_handler=model_source_handler,
|
||||
agent_handler=agent_handler,
|
||||
download_routing=download_routing,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -40,11 +40,20 @@ COMMON_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/verify-duplicates", "verify_duplicates"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/move_model", "move_model"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/move_models_bulk", "move_models_bulk"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/create-folder", "create_folder"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/delete-folder", "delete_folder"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/rename-folder", "rename_folder"),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/auto-organize", "auto_organize_models"),
|
||||
RouteDefinition("POST", "/api/lm/{prefix}/auto-organize", "auto_organize_models"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/{prefix}/auto-organize-progress", "get_auto_organize_progress"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/{prefix}/apply-filename-template", "apply_filename_template"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/{prefix}/apply-filename-template", "apply_filename_template"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/top-tags", "get_top_tags"),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/search-tags", "search_tags"),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/base-models", "get_base_models"),
|
||||
@@ -68,6 +77,8 @@ COMMON_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
"GET", "/api/lm/{prefix}/model-description", "get_model_description"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/relative-paths", "get_relative_paths"),
|
||||
RouteDefinition("PUT", "/api/lm/{prefix}/active-filters", "update_active_filters"),
|
||||
RouteDefinition("GET", "/api/lm/{prefix}/active-filters", "get_active_filters"),
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/{prefix}/civitai/versions/{model_id}", "get_civitai_versions"
|
||||
),
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List
|
||||
from aiohttp import web
|
||||
|
||||
from .base_model_routes import BaseModelRoutes
|
||||
from .model_route_registrar import ModelRouteRegistrar
|
||||
from ..config import config
|
||||
from ..services.other_model_service import OtherModelService
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
from ..utils.constants import (
|
||||
CIVITAI_TYPE_TO_OTHER_SUB_TYPE,
|
||||
OTHER_MODEL_FOLDER_SUBTYPES,
|
||||
VALID_OTHER_CIVITAI_TYPES,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OtherRoutes(BaseModelRoutes):
|
||||
"""Other-model-specific route controller (VAE, upscaler, text encoder, ...)"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Other-model routes with OtherModel service"""
|
||||
super().__init__()
|
||||
self.template_name = "other.html"
|
||||
|
||||
async def initialize_services(self):
|
||||
"""Initialize services from ServiceRegistry"""
|
||||
other_scanner = await ServiceRegistry.get_other_scanner()
|
||||
update_service = await ServiceRegistry.get_model_update_service()
|
||||
self.service = OtherModelService(other_scanner, update_service=update_service)
|
||||
self.set_model_update_service(update_service)
|
||||
|
||||
# Attach service dependencies
|
||||
self.attach_service(self.service)
|
||||
|
||||
def setup_routes(self, app: web.Application, prefix: str = "other"):
|
||||
"""Setup Other-model routes"""
|
||||
# Schedule service initialization on app startup
|
||||
app.on_startup.append(lambda _: self.initialize_services())
|
||||
|
||||
# Setup common routes with 'other' prefix (includes page route)
|
||||
super().setup_routes(app, prefix)
|
||||
|
||||
def setup_specific_routes(self, registrar: ModelRouteRegistrar, prefix: str):
|
||||
"""Setup Other-model-specific routes"""
|
||||
# Other-model info by name
|
||||
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/info/{name}', prefix, self.get_other_model_info)
|
||||
# Other-model roots grouped by sub_type (text_encoders + legacy clip
|
||||
# are aggregated under text_encoder)
|
||||
registrar.add_prefixed_route('GET', '/api/lm/{prefix}/roots_by_subtype', prefix, self.get_roots_by_subtype)
|
||||
|
||||
def _validate_civitai_model_type(self, model_type: str) -> bool:
|
||||
"""Validate CivitAI model type for other models.
|
||||
|
||||
Accepts retired CivitAI types (CLIP, CLIPVision) as well — grandfathered
|
||||
models on CivitAI still carry them. Types whose sub_type is currently
|
||||
disabled (or every type while the opt-in feature is off) are rejected.
|
||||
"""
|
||||
normalized = (model_type or "").strip().lower()
|
||||
if normalized not in VALID_OTHER_CIVITAI_TYPES:
|
||||
return False
|
||||
if not self._settings.is_other_models_enabled():
|
||||
return False
|
||||
|
||||
sub_type = CIVITAI_TYPE_TO_OTHER_SUB_TYPE.get(normalized)
|
||||
if sub_type is None:
|
||||
# CivitAI "Other" has no sub_type of its own; it is only usable
|
||||
# while at least one sub_type is enabled.
|
||||
return bool(self._settings.get_enabled_other_sub_types())
|
||||
return self._settings.is_other_sub_type_enabled(sub_type)
|
||||
|
||||
def _get_page_context_provider(self):
|
||||
"""Expose the opt-in feature state to the Other Models page template."""
|
||||
return self._page_context_for_other
|
||||
|
||||
def _page_context_for_other(self, request: web.Request) -> Dict[str, Any]:
|
||||
if not self._settings.is_other_models_enabled():
|
||||
return {"other_disabled": True, "other_no_paths": False}
|
||||
|
||||
# Enabled but nothing to scan: folder paths for the managed sub_types
|
||||
# resolved to no existing folder. Render an actionable empty state
|
||||
# instead of an apparently broken empty grid.
|
||||
standalone_mode = os.environ.get("LORA_MANAGER_STANDALONE", "0") == "1"
|
||||
context = {
|
||||
"other_disabled": False,
|
||||
"other_no_paths": not bool(config.other_roots),
|
||||
"standalone_mode": standalone_mode,
|
||||
}
|
||||
if standalone_mode:
|
||||
# The empty state points at the Model Paths settings section and
|
||||
# shows the settings.json path as a fallback reference.
|
||||
context["settings_file"] = getattr(self._settings, "settings_file", "") or ""
|
||||
return context
|
||||
|
||||
def _get_expected_model_types(self) -> str:
|
||||
"""Get expected model types string for error messages"""
|
||||
return "VAE, Upscaler, TextEncoder, CLIPVision, Controlnet, or Other"
|
||||
|
||||
def _parse_specific_params(self, request: web.Request) -> Dict[str, Any]:
|
||||
"""Parse other-model-specific parameters (none in Phase 1)."""
|
||||
return {}
|
||||
|
||||
async def get_roots_by_subtype(self, request: web.Request) -> web.Response:
|
||||
"""Return other-model roots grouped by sub_type.
|
||||
|
||||
Aggregates the per-folder_paths-key roots from config
|
||||
(``text_encoders`` and the legacy ``clip`` key both land under
|
||||
``text_encoder``).
|
||||
"""
|
||||
try:
|
||||
roots_by_subtype: Dict[str, List[str]] = {}
|
||||
for key, roots in (config.other_folder_roots or {}).items():
|
||||
sub_type = OTHER_MODEL_FOLDER_SUBTYPES.get(key)
|
||||
if not sub_type:
|
||||
continue
|
||||
bucket = roots_by_subtype.setdefault(sub_type, [])
|
||||
for root in roots:
|
||||
if root and root not in bucket:
|
||||
bucket.append(root)
|
||||
return web.json_response(
|
||||
{"success": True, "roots_by_subtype": roots_by_subtype}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting other roots by sub_type: {e}", exc_info=True)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(e)}, status=500
|
||||
)
|
||||
|
||||
async def get_other_model_info(self, request: web.Request) -> web.Response:
|
||||
"""Get detailed information for a specific other model by name"""
|
||||
try:
|
||||
name = request.match_info.get('name', '')
|
||||
model_info = await self.service.get_model_info_by_name(name) # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
if model_info:
|
||||
return web.json_response(model_info)
|
||||
else:
|
||||
return web.json_response({"error": "Model not found"}, status=404)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in get_other_model_info: {e}", exc_info=True)
|
||||
return web.json_response({"error": str(e)}, status=500)
|
||||
@@ -49,6 +49,31 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("POST", "/api/lm/recipe/move", "move_recipe"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/move-bulk", "move_recipes_bulk"),
|
||||
RouteDefinition("POST", "/api/lm/recipe/lora/reconnect", "reconnect_lora"),
|
||||
RouteDefinition("POST", "/api/lm/recipe/lora/restore", "restore_lora"),
|
||||
RouteDefinition(
|
||||
"GET",
|
||||
"/api/lm/recipe/{recipe_id}/lora/{lora_index}/reconnect-suggestions",
|
||||
"get_reconnect_suggestions",
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/recipe/lora/mark-hash-invalid", "mark_lora_hash_invalid"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/recipe/checkpoint/reconnect", "reconnect_checkpoint"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/recipe/checkpoint/restore", "restore_checkpoint"
|
||||
),
|
||||
RouteDefinition(
|
||||
"GET",
|
||||
"/api/lm/recipe/{recipe_id}/checkpoint/reconnect-suggestions",
|
||||
"get_checkpoint_reconnect_suggestions",
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST",
|
||||
"/api/lm/recipe/checkpoint/mark-hash-invalid",
|
||||
"mark_checkpoint_hash_invalid",
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/recipes/find-duplicates", "find_duplicates"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/bulk-delete", "bulk_delete"),
|
||||
RouteDefinition(
|
||||
@@ -59,11 +84,6 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
"GET", "/api/lm/recipes/for-checkpoint", "get_recipes_for_checkpoint"
|
||||
),
|
||||
RouteDefinition("GET", "/api/lm/recipes/scan", "scan_recipes"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/repair", "repair_recipes"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/cancel-repair", "cancel_repair"),
|
||||
RouteDefinition("POST", "/api/lm/recipe/{recipe_id}/repair", "repair_recipe"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/repair-bulk", "repair_recipes_bulk"),
|
||||
RouteDefinition("GET", "/api/lm/recipes/repair-progress", "get_repair_progress"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/rematch", "rematch_recipes"),
|
||||
RouteDefinition("POST", "/api/lm/recipes/rematch-bulk", "rematch_recipes_bulk"),
|
||||
RouteDefinition("POST", "/api/lm/recipe/{recipe_id}/rematch", "rematch_recipe"),
|
||||
@@ -90,6 +110,11 @@ ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/recipe/{recipe_id}/reimport", "reimport_recipe"
|
||||
),
|
||||
# The companion browser extension only ever issues GET requests, so the
|
||||
# payload-based re-import variant must also be reachable via GET.
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/recipe/{recipe_id}/reimport", "reimport_recipe"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/recipe/{recipe_id}/send-workflow", "send_recipe_workflow"
|
||||
),
|
||||
|
||||
@@ -21,7 +21,20 @@ NETWORK_EXCEPTIONS = (ClientError, OSError, asyncio.TimeoutError)
|
||||
# otherwise delete them because they are untracked and, in released tags,
|
||||
# not listed in ``.gitignore``. ``-e`` excludes a path from cleaning
|
||||
# regardless of whether it is ignored.
|
||||
_PRESERVE_DIRS = ('settings.json', 'civitai', 'wildcards', 'backups', 'stats', 'logs', 'cache', 'model_cache')
|
||||
# ``cache`` covers the resolved cache tree (cache/model, cache/recipe,
|
||||
# cache/fts, ...); the legacy ``recipe_cache`` / ``model_cache`` directories
|
||||
# are listed too because a portable install can predate the cache/ move.
|
||||
_PRESERVE_DIRS = (
|
||||
'settings.json',
|
||||
'civitai',
|
||||
'wildcards',
|
||||
'backups',
|
||||
'stats',
|
||||
'logs',
|
||||
'cache',
|
||||
'model_cache',
|
||||
'recipe_cache',
|
||||
)
|
||||
|
||||
|
||||
def _clean_excludes() -> List[str]:
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
"""In-memory store for the LoRA Manager page's active filters.
|
||||
|
||||
The manager page keeps its filter state in localStorage for its own
|
||||
restoration, but the ComfyUI node autocomplete runs in a potentially
|
||||
different browser/origin (or Electron shell) where that storage is not
|
||||
shared. This store mirrors the active filters server-side so the
|
||||
``/api/lm/{prefix}/relative-paths`` endpoint can inject them into
|
||||
autocomplete searches regardless of which client set them.
|
||||
|
||||
State is process-local and intentionally not persisted; the manager page
|
||||
re-pushes its restored state on load.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Keys copied from the manager page's persisted filter snapshot.
|
||||
_FILTER_KEYS = (
|
||||
"baseModel",
|
||||
"tags",
|
||||
"autoTags",
|
||||
"modelTypes",
|
||||
"tagLogic",
|
||||
"license",
|
||||
)
|
||||
|
||||
|
||||
class ActiveFiltersStore:
|
||||
"""Process-local store of active filters, keyed by model type."""
|
||||
|
||||
_instance: Optional["ActiveFiltersStore"] = None
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._filters: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls) -> "ActiveFiltersStore":
|
||||
if cls._instance is None:
|
||||
cls._instance = cls()
|
||||
return cls._instance
|
||||
|
||||
@classmethod
|
||||
def reset_instance(cls) -> None:
|
||||
"""Drop the singleton (test isolation)."""
|
||||
cls._instance = None
|
||||
|
||||
def set_filters(self, model_type: str, payload: Dict[str, Any]) -> None:
|
||||
"""Replace the stored active filters for a model type.
|
||||
|
||||
Only recognized keys are kept; everything else is discarded.
|
||||
"""
|
||||
filters = payload.get("filters")
|
||||
sanitized: Dict[str, Any] = {
|
||||
"activeFolder": payload.get("activeFolder"),
|
||||
"recursiveSearch": bool(payload.get("recursiveSearch", True)),
|
||||
"filters": (
|
||||
{key: filters[key] for key in _FILTER_KEYS if key in filters}
|
||||
if isinstance(filters, dict)
|
||||
else None
|
||||
),
|
||||
}
|
||||
self._filters[model_type] = sanitized
|
||||
|
||||
def get_filters(self, model_type: str) -> Optional[Dict[str, Any]]:
|
||||
"""Return the stored payload for a model type, or None if unset."""
|
||||
return self._filters.get(model_type)
|
||||
|
||||
def clear(self, model_type: str) -> None:
|
||||
self._filters.pop(model_type, None)
|
||||
|
||||
|
||||
def active_filters_to_query_kwargs(payload: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""Map a stored active-filters payload to ``search_relative_paths`` kwargs.
|
||||
|
||||
Mirrors the query-param mapping that the ComfyUI autocomplete used to
|
||||
build client-side from localStorage (web/comfyui/autocomplete.js).
|
||||
"""
|
||||
kwargs: Dict[str, Any] = {}
|
||||
if not payload:
|
||||
return kwargs
|
||||
|
||||
active_folder = payload.get("activeFolder")
|
||||
recursive = payload.get("recursiveSearch", True)
|
||||
|
||||
if active_folder and active_folder != "null":
|
||||
kwargs["folder"] = active_folder
|
||||
elif not recursive:
|
||||
# Root folder with recursion disabled mirrors the page list,
|
||||
# which matches only root-level files via folder=''.
|
||||
kwargs["folder"] = ""
|
||||
|
||||
filters = payload.get("filters")
|
||||
if isinstance(filters, dict):
|
||||
base_models = filters.get("baseModel")
|
||||
if isinstance(base_models, list):
|
||||
kwargs["base_models"] = [m for m in base_models if m]
|
||||
|
||||
for source_key, target_key in (("tags", "tags"), ("autoTags", "auto_tags")):
|
||||
states = filters.get(source_key)
|
||||
if isinstance(states, dict):
|
||||
mapped = {
|
||||
tag: state
|
||||
for tag, state in states.items()
|
||||
if state in ("include", "exclude")
|
||||
}
|
||||
if mapped:
|
||||
kwargs[target_key] = mapped
|
||||
|
||||
model_types = filters.get("modelTypes")
|
||||
if isinstance(model_types, list):
|
||||
kwargs["model_types"] = [t for t in model_types if t]
|
||||
|
||||
tag_logic = filters.get("tagLogic")
|
||||
if tag_logic:
|
||||
kwargs["tag_logic"] = tag_logic
|
||||
|
||||
license_filter = filters.get("license")
|
||||
if isinstance(license_filter, dict):
|
||||
no_credit = license_filter.get("noCredit")
|
||||
if no_credit == "include":
|
||||
kwargs["credit_required"] = False
|
||||
elif no_credit == "exclude":
|
||||
kwargs["credit_required"] = True
|
||||
allow_selling = license_filter.get("allowSelling")
|
||||
if allow_selling == "include":
|
||||
kwargs["allow_selling_generated_content"] = True
|
||||
elif allow_selling == "exclude":
|
||||
kwargs["allow_selling_generated_content"] = False
|
||||
|
||||
kwargs["recursive"] = recursive
|
||||
return kwargs
|
||||
@@ -19,16 +19,21 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import aiohttp
|
||||
|
||||
import os
|
||||
|
||||
from ...config import config
|
||||
from ..llm_service import LLMService
|
||||
from ..model_sources import (
|
||||
ModelCardContext,
|
||||
ModelSourceCache,
|
||||
get_source,
|
||||
resolve_source_ref,
|
||||
source_label,
|
||||
)
|
||||
from ..model_sources.hydration import load_model_card, resolve_site_base_model
|
||||
from ..websocket_manager import ws_manager
|
||||
from .post_processor import PostProcessor
|
||||
from .skill_registry import SkillRegistry
|
||||
@@ -255,6 +260,11 @@ class AgentService:
|
||||
llm = await self._ensure_llm()
|
||||
llm_configured = llm.is_configured() if skill.llm_required else True
|
||||
|
||||
# A collection repository holds many model files under one source id;
|
||||
# this memo keeps the README and the repository metadata from being
|
||||
# re-fetched once per file. It lives for this run only.
|
||||
source_cache = ModelSourceCache()
|
||||
|
||||
for model_path in model_paths:
|
||||
model_filename = os.path.basename(model_path)
|
||||
logger.info(
|
||||
@@ -267,24 +277,50 @@ class AgentService:
|
||||
from ...metadata_ops import read_metadata
|
||||
metadata = await read_metadata(model_path)
|
||||
|
||||
# Fast-fail: enrich_hf_metadata requires hf_url to have HF README context
|
||||
if skill_name == "enrich_hf_metadata" and not metadata.get("hf_url", ""):
|
||||
logger.info(
|
||||
"[%s] SKIP %s — no hf_url in metadata",
|
||||
skill_name, model_filename,
|
||||
)
|
||||
skipped_count += 1
|
||||
skip_model = True
|
||||
# Fast-fail: enrich_hf_metadata needs an external model source
|
||||
# that exposes an accessible model card.
|
||||
if skill_name == "enrich_hf_metadata":
|
||||
skip_reason = self._enrichment_skip_reason(metadata)
|
||||
if skip_reason:
|
||||
logger.info(
|
||||
"[%s] SKIP %s — %s",
|
||||
skill_name, model_filename, skip_reason,
|
||||
)
|
||||
skipped_count += 1
|
||||
skip_model = True
|
||||
|
||||
if not skip_model:
|
||||
prompt_vars: Dict[str, Any] = {"model_path": model_path}
|
||||
if skill.llm_required and llm_configured:
|
||||
prompt_vars = await self._build_prompt_context(
|
||||
skill_name, model_path, metadata, registry, llm,
|
||||
# The site's own data is deterministic and must land whether
|
||||
# or not an LLM is available: a user without a key still gets
|
||||
# the author summary, the example images and the tags.
|
||||
source_vars, source_context = await self._load_source_card(
|
||||
model_path, metadata, cache=source_cache,
|
||||
)
|
||||
resolved_base_model = ""
|
||||
if skill_name == "enrich_hf_metadata" and not (
|
||||
metadata.get("base_model") or ""
|
||||
).strip():
|
||||
resolved_base_model = await self._resolve_site_base_model(
|
||||
source_context,
|
||||
)
|
||||
|
||||
llm_response: Optional[Dict[str, Any]] = None
|
||||
if skill.llm_required and llm_configured:
|
||||
if skill.llm_required and not llm_configured:
|
||||
# Without a provider the deterministic model-source data
|
||||
# still lands; the LLM-only fields simply stay untouched.
|
||||
logger.info(
|
||||
"[%s] No LLM configured for %s — applying %s data only",
|
||||
skill_name, model_filename,
|
||||
"model-source"
|
||||
if not source_context.is_empty()
|
||||
else "README",
|
||||
)
|
||||
elif skill.llm_required:
|
||||
prompt_vars = await self._build_prompt_context(
|
||||
skill_name, model_path, metadata, registry, llm,
|
||||
source_vars=source_vars,
|
||||
source_context=source_context,
|
||||
)
|
||||
prompt_template = registry.load_prompt(skill_name)
|
||||
rendered = _render_prompt(prompt_template, prompt_vars)
|
||||
llm_response = await llm.chat_completion_json(
|
||||
@@ -307,7 +343,9 @@ class AgentService:
|
||||
model_path=model_path,
|
||||
llm_output=llm_response or {},
|
||||
metadata=metadata,
|
||||
readme_content=prompt_vars.get("readme_content_full", ""),
|
||||
readme_content=source_vars.get("readme_content_full", ""),
|
||||
source_context=source_context,
|
||||
resolved_base_model=resolved_base_model,
|
||||
)
|
||||
|
||||
if model_result.get("success", True):
|
||||
@@ -358,6 +396,28 @@ class AgentService:
|
||||
# Base model grouping (keeps the prompt compact)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _enrichment_skip_reason(metadata: Dict[str, Any]) -> str:
|
||||
"""Return why ``enrich_hf_metadata`` cannot run, or ``""`` if it can.
|
||||
|
||||
Distinguishes the three cases the user can act on: no source linked,
|
||||
a source we don't know, and a known source whose model card is not
|
||||
reachable from the backend (TensorArt).
|
||||
"""
|
||||
|
||||
ref = resolve_source_ref(metadata)
|
||||
if ref is None:
|
||||
return "no model source linked (source_url missing)"
|
||||
source = get_source(ref.platform)
|
||||
if source is None:
|
||||
return f"unsupported model source platform '{ref.platform}'"
|
||||
if not source.supports_enrichment:
|
||||
return (
|
||||
f"{source.label} does not expose a model card to the backend; "
|
||||
"AI metadata enrichment is not available for this source"
|
||||
)
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _format_base_models(models: List[str]) -> str:
|
||||
"""Format the base model list as a flat, one-per-line list.
|
||||
@@ -368,6 +428,82 @@ class AgentService:
|
||||
"""
|
||||
return "\n".join(f"- {m}" for m in models)
|
||||
|
||||
async def _load_source_card(
|
||||
self,
|
||||
model_path: str,
|
||||
metadata: Dict[str, Any],
|
||||
*,
|
||||
cache: Optional[ModelSourceCache] = None,
|
||||
) -> tuple[Dict[str, Any], ModelCardContext]:
|
||||
"""Fetch the model card and site-published extras for one model.
|
||||
|
||||
Runs for every source-backed enrichment regardless of LLM
|
||||
availability, because everything it returns is deterministic data that
|
||||
should be applied even without a configured provider.
|
||||
|
||||
*cache* is the per-run memo created by :meth:`execute_skill`. The
|
||||
README is repository-wide, so it is fetched once per source id; only
|
||||
successful reads are memoised, leaving a transient failure to be
|
||||
retried for the next file.
|
||||
"""
|
||||
|
||||
variables: Dict[str, Any] = {
|
||||
"asset_base_url": "",
|
||||
"source_description": "",
|
||||
"source_base_model": "",
|
||||
"source_official_tags": "",
|
||||
"source_example_images": "",
|
||||
"source_trigger_words": "",
|
||||
"readme_content": "(README not available)",
|
||||
"readme_content_full": "",
|
||||
}
|
||||
|
||||
ref = resolve_source_ref(metadata)
|
||||
source = get_source(ref.platform) if ref is not None else None
|
||||
if ref is None or source is None or not source.supports_enrichment:
|
||||
return variables, ModelCardContext()
|
||||
|
||||
raw_basename = os.path.splitext(os.path.basename(model_path))[0]
|
||||
variables["asset_base_url"] = source.asset_base_url(ref.source_id)
|
||||
|
||||
readme = await load_model_card(source, ref.source_id, cache)
|
||||
|
||||
# Sites such as ModelScope keep part of the model card outside the
|
||||
# README (author summary, curated tags, per-file example images). The
|
||||
# recorded hash identifies the file even after the user renames it.
|
||||
card_context = await source.fetch_model_card_context(
|
||||
ref.source_id,
|
||||
os.path.basename(model_path),
|
||||
sha256=(metadata.get("sha256") or "").strip(),
|
||||
cache=cache,
|
||||
)
|
||||
variables["source_description"] = card_context.description
|
||||
variables["source_base_model"] = card_context.base_model
|
||||
variables["source_official_tags"] = "\n".join(
|
||||
f"- {tag}" for tag in card_context.official_tags
|
||||
)
|
||||
variables["source_example_images"] = "\n".join(
|
||||
f"- {url}" for url in card_context.example_images
|
||||
)
|
||||
variables["source_trigger_words"] = ", ".join(card_context.trigger_words)
|
||||
|
||||
# Trim README to the section relevant to this model file
|
||||
# (collection repos often have multiple models in one README).
|
||||
if readme and raw_basename:
|
||||
trimmed = extract_relevant_section(readme, raw_basename)
|
||||
cleaned = clean_readme_for_llm(trimmed) if trimmed else ""
|
||||
else:
|
||||
cleaned = clean_readme_for_llm(readme) if readme else ""
|
||||
variables["readme_content"] = cleaned if cleaned else "(README not available)"
|
||||
variables["readme_content_full"] = readme or ""
|
||||
|
||||
return variables, card_context
|
||||
|
||||
async def _resolve_site_base_model(self, source_context: ModelCardContext) -> str:
|
||||
"""Resolve the site's base-model hints to a canonical name, or ``""``."""
|
||||
|
||||
return await resolve_site_base_model(source_context)
|
||||
|
||||
async def _build_prompt_context(
|
||||
self,
|
||||
skill_name: str,
|
||||
@@ -375,19 +511,45 @@ class AgentService:
|
||||
metadata: Dict[str, Any],
|
||||
registry: SkillRegistry,
|
||||
llm: Any,
|
||||
*,
|
||||
source_vars: Optional[Dict[str, Any]] = None,
|
||||
source_context: Optional[ModelCardContext] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Gather variables for the skill's prompt template.
|
||||
|
||||
Reads metadata, fetches the HF README (if applicable), lists available
|
||||
Reads metadata, fetches the model card (unless a pre-fetched
|
||||
*source_vars* / *source_context* pair is supplied), lists available
|
||||
base models, loads user priority tags, and returns a dict that maps to
|
||||
``{{variable}}`` placeholders in ``prompt.md``.
|
||||
"""
|
||||
from ...metadata_ops import identify_model_type, list_base_models
|
||||
from ..settings_manager import SettingsManager
|
||||
|
||||
if source_vars is None or source_context is None:
|
||||
source_vars, source_context = await self._load_source_card(
|
||||
model_path, metadata,
|
||||
)
|
||||
|
||||
context: Dict[str, Any] = {
|
||||
"model_path": model_path,
|
||||
"model_basename": "",
|
||||
# Canonical external-source variables
|
||||
"source_url": "",
|
||||
"source_id": "",
|
||||
"source_platform": "",
|
||||
"source_label": "",
|
||||
"asset_base_url": "",
|
||||
# Site-provided card extras (see ModelSource.fetch_model_card_context)
|
||||
"source_description": "",
|
||||
"source_base_model": "",
|
||||
"source_official_tags": "",
|
||||
"source_example_images": "",
|
||||
"source_trigger_words": "",
|
||||
# Carrier for the structured context handed to the post-processor;
|
||||
# never rendered into the prompt.
|
||||
"source_context": ModelCardContext(),
|
||||
# Legacy Hugging Face aliases (kept so older prompt templates and
|
||||
# third-party skills keep rendering)
|
||||
"hf_url": "",
|
||||
"repo": "",
|
||||
"readme_content": "",
|
||||
@@ -407,26 +569,33 @@ class AgentService:
|
||||
"base_model": metadata.get("base_model", ""),
|
||||
"tags": metadata.get("tags", []),
|
||||
"modelDescription": metadata.get("modelDescription", ""),
|
||||
"trainedWords": metadata.get("trainedWords", []),
|
||||
"sha256": (metadata.get("sha256") or "")[:16] + "..." if metadata.get("sha256") else "",
|
||||
"size": metadata.get("size", 0),
|
||||
}
|
||||
|
||||
hf_url = metadata.get("hf_url", "")
|
||||
context["hf_url"] = hf_url
|
||||
repo = self._extract_repo_from_url(hf_url) if hf_url else ""
|
||||
context["repo"] = repo or ""
|
||||
if repo:
|
||||
readme = await self._fetch_readme(repo)
|
||||
# Trim README to the section relevant to this model file
|
||||
# (collection repos often have multiple models in one README).
|
||||
if readme and raw_basename:
|
||||
trimmed = extract_relevant_section(readme, raw_basename)
|
||||
cleaned = clean_readme_for_llm(trimmed) if trimmed else ""
|
||||
else:
|
||||
cleaned = clean_readme_for_llm(readme) if readme else ""
|
||||
context["readme_content"] = cleaned if cleaned else "(README not available)"
|
||||
context["readme_content_full"] = readme or ""
|
||||
ref = resolve_source_ref(metadata)
|
||||
if ref is not None:
|
||||
context["source_url"] = ref.url
|
||||
context["source_id"] = ref.source_id
|
||||
context["source_platform"] = ref.platform
|
||||
context["source_label"] = source_label(ref.platform, ref.platform)
|
||||
if ref.platform == "huggingface":
|
||||
context["hf_url"] = ref.url
|
||||
context["repo"] = ref.source_id
|
||||
|
||||
source = get_source(ref.platform) if ref is not None else None
|
||||
if ref is not None and source is not None and source.supports_enrichment:
|
||||
# Values fetched once by _load_source_card and shared with the
|
||||
# post-processor, so the network is not hit twice per model.
|
||||
context["asset_base_url"] = source_vars["asset_base_url"]
|
||||
context["source_context"] = source_context
|
||||
context["source_description"] = source_vars["source_description"]
|
||||
context["source_base_model"] = source_vars["source_base_model"]
|
||||
context["source_official_tags"] = source_vars["source_official_tags"]
|
||||
context["source_example_images"] = source_vars["source_example_images"]
|
||||
context["source_trigger_words"] = source_vars["source_trigger_words"]
|
||||
context["readme_content"] = source_vars["readme_content"]
|
||||
context["readme_content_full"] = source_vars["readme_content_full"]
|
||||
|
||||
try:
|
||||
raw_models = await list_base_models()
|
||||
@@ -459,20 +628,14 @@ class AgentService:
|
||||
|
||||
@staticmethod
|
||||
async def _fetch_readme(repo: str) -> str:
|
||||
"""Fetch README.md from HuggingFace (tries ``main``, then ``master``)."""
|
||||
async with aiohttp.ClientSession(
|
||||
headers={"User-Agent": "ComfyUI-LoRA-Manager/1.0"},
|
||||
timeout=aiohttp.ClientTimeout(total=30),
|
||||
) as session:
|
||||
for branch in ("main", "master"):
|
||||
url = f"https://huggingface.co/{repo}/raw/{branch}/README.md"
|
||||
try:
|
||||
async with session.get(url) as resp:
|
||||
if resp.status == 200:
|
||||
return await resp.text()
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to fetch README from %s: %s", url, exc)
|
||||
return ""
|
||||
"""Fetch a Hugging Face README (tries ``main``, then ``master``).
|
||||
|
||||
Kept for backward compatibility; new code should go through the
|
||||
model-source registry so every supported site works.
|
||||
"""
|
||||
from ..model_sources import HuggingFaceSource
|
||||
|
||||
return await HuggingFaceSource().fetch_model_card(repo)
|
||||
|
||||
async def _emit_progress(
|
||||
self,
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Map a site-reported base model onto this system's canonical vocabulary.
|
||||
|
||||
Model sites name base models in their own terms: ModelScope publishes
|
||||
``krea/Krea-2-Turbo`` and ``KREA_2_TURBO`` where this system expects the
|
||||
canonical ``Krea 2``. Turning one into the other is normally the LLM's job;
|
||||
this module resolves the cases that can be decided safely so the canonical
|
||||
field is still populated when the LLM returns nothing usable for it.
|
||||
|
||||
The resolver is deliberately strict, because a wrong base model written with
|
||||
apparent authority is worse than no value at all:
|
||||
|
||||
* it only ever returns a name that is already present in *known_names*;
|
||||
* matching is on the normalised form (lowercased, non-alphanumerics removed),
|
||||
so separators and casing are ignored but nothing is inferred;
|
||||
* a bounded set of published variant suffixes may be stripped, and only when
|
||||
the remainder still matches a known name exactly.
|
||||
|
||||
Anything it cannot decide returns ``""``, and the caller falls back to the LLM.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Iterable, Sequence
|
||||
|
||||
#: Variant suffixes sites append to a base-model *family* name. Stripping one
|
||||
#: is only attempted when the remainder matches a known name exactly, so an
|
||||
#: unrecognised suffix can never produce a bogus match.
|
||||
_VARIANT_SUFFIXES: tuple[str, ...] = (
|
||||
"turbo",
|
||||
"schnell",
|
||||
"lightning",
|
||||
"dev",
|
||||
"beta",
|
||||
"alpha",
|
||||
)
|
||||
|
||||
_NON_ALNUM = re.compile(r"[^a-z0-9]+")
|
||||
|
||||
|
||||
def _normalize(value: str) -> str:
|
||||
"""Return the comparison form of *value*.
|
||||
|
||||
Lowercases and drops every non-alphanumeric character, so ``KREA_2``,
|
||||
``Krea 2``, ``krea-2`` and ``krea.2`` all collapse to ``krea2``.
|
||||
"""
|
||||
|
||||
return _NON_ALNUM.sub("", (value or "").lower())
|
||||
|
||||
|
||||
def resolve_base_model(
|
||||
hints: Iterable[str], known_names: Sequence[str]
|
||||
) -> str:
|
||||
"""Return the canonical base model that *hints* refers to, or ``""``.
|
||||
|
||||
Args:
|
||||
hints: Site-reported names, best first (e.g. an architecture enum
|
||||
before a link-style repository id).
|
||||
known_names: The canonical vocabulary; only these are ever returned.
|
||||
|
||||
Returns:
|
||||
One of *known_names*, or ``""`` when nothing matches exactly.
|
||||
"""
|
||||
|
||||
normalized: dict[str, str] = {}
|
||||
for name in known_names:
|
||||
key = _normalize(name)
|
||||
if key and key not in normalized:
|
||||
normalized[key] = name
|
||||
if not normalized:
|
||||
return ""
|
||||
|
||||
ordered = [hint for hint in hints if hint]
|
||||
|
||||
# 1. Exact normalised match — the unambiguous case.
|
||||
for hint in ordered:
|
||||
candidate = _normalize(hint)
|
||||
if candidate in normalized:
|
||||
return normalized[candidate]
|
||||
|
||||
# 2. Drop one published variant suffix and retry exactly.
|
||||
for hint in ordered:
|
||||
candidate = _normalize(hint)
|
||||
for suffix in _VARIANT_SUFFIXES:
|
||||
if not candidate.endswith(suffix) or candidate == suffix:
|
||||
continue
|
||||
stem = candidate[: -len(suffix)]
|
||||
if stem in normalized:
|
||||
return normalized[stem]
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
__all__ = ["resolve_base_model"]
|
||||
@@ -10,12 +10,16 @@ refresh cache). All actual I/O is delegated to :mod:`~py.metadata_ops`.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover - typing only
|
||||
from ..model_sources import ModelCardContext
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -42,6 +46,9 @@ class PostProcessor:
|
||||
llm_output: Dict[str, Any],
|
||||
metadata: Dict[str, Any],
|
||||
readme_content: str = "",
|
||||
source_context: Optional["ModelCardContext"] = None,
|
||||
resolved_base_model: str = "",
|
||||
metadata_source: str = "agent:enrich_hf_metadata",
|
||||
) -> Dict[str, Any]:
|
||||
"""Route *llm_output* to the correct skill post-processor.
|
||||
|
||||
@@ -49,12 +56,26 @@ class PostProcessor:
|
||||
that is converted to HTML and stored as ``modelDescription`` for
|
||||
the description tab.
|
||||
|
||||
*source_context* carries the extras the model site publishes outside
|
||||
the README (author description, per-file example images, trigger
|
||||
words). It is ``None`` for callers that have none.
|
||||
|
||||
*resolved_base_model* is the canonical base-model name the site's own
|
||||
hints resolve to, used when the LLM did not supply one (which is the
|
||||
normal case when the LLM was skipped).
|
||||
|
||||
*metadata_source* records who produced the metadata. The AI skill
|
||||
keeps its historical value; the deterministic download-time hydration
|
||||
passes its own so the two remain distinguishable. ``llm_enriched_at``
|
||||
is only stamped when *llm_output* actually carries a provider answer.
|
||||
|
||||
Returns a dict with keys ``success`` (bool), ``updated_fields`` (list),
|
||||
``preview_downloaded`` (bool), and ``errors`` (list).
|
||||
"""
|
||||
if skill_name == "enrich_hf_metadata":
|
||||
return await self._process_enrich_hf_metadata(
|
||||
model_path, llm_output, metadata, readme_content,
|
||||
model_path, llm_output, metadata, readme_content, source_context,
|
||||
resolved_base_model, metadata_source,
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
@@ -72,12 +93,16 @@ class PostProcessor:
|
||||
llm_output: Dict[str, Any],
|
||||
metadata: Dict[str, Any],
|
||||
readme_content: str = "",
|
||||
source_context: Optional["ModelCardContext"] = None,
|
||||
resolved_base_model: str = "",
|
||||
metadata_source: str = "agent:enrich_hf_metadata",
|
||||
) -> Dict[str, Any]:
|
||||
from ...metadata_ops import (
|
||||
apply_metadata_updates,
|
||||
download_preview,
|
||||
refresh_cache,
|
||||
)
|
||||
from ..model_sources import get_source, has_external_source, resolve_source_ref
|
||||
from .skills.enrich_hf_metadata.readme_processor import (
|
||||
convert_readme_to_html,
|
||||
extract_gallery_images,
|
||||
@@ -85,24 +110,49 @@ class PostProcessor:
|
||||
extract_relevant_section,
|
||||
extract_simple_markdown_images,
|
||||
extract_html_img_tags,
|
||||
extract_repo_from_hf_url,
|
||||
)
|
||||
|
||||
updated_fields: List[str] = []
|
||||
preview_downloaded = False
|
||||
|
||||
# -- Determine whether this is an HF-sourced model -----------------
|
||||
is_hf_model = not metadata.get("from_civitai", True)
|
||||
# -- Determine whether this is an externally-sourced model ---------
|
||||
# Key off the source fields directly: `from_civitai` records provenance
|
||||
# and can be true for a model that is also linked to an external site
|
||||
# (both sources coexist, see #1094), so it must not gate enrichment.
|
||||
is_source_model = has_external_source(metadata)
|
||||
|
||||
source_ref = resolve_source_ref(metadata)
|
||||
source = get_source(source_ref.platform) if source_ref else None
|
||||
source_id = source_ref.source_id if source_ref else ""
|
||||
asset_base_url = (
|
||||
source.asset_base_url(source_id)
|
||||
if source is not None and source_id
|
||||
else None
|
||||
)
|
||||
|
||||
# -- Collect updates -----------------------------------------------
|
||||
updates: Dict[str, Any] = {}
|
||||
|
||||
# base_model
|
||||
# base_model — the LLM's mapping wins; when it returned nothing usable,
|
||||
# fall back to the canonical name the site's own hints resolve to.
|
||||
new_base = (llm_output.get("base_model") or "").strip()
|
||||
if not new_base:
|
||||
new_base = (resolved_base_model or "").strip()
|
||||
current_base = metadata.get("base_model", "") or ""
|
||||
if new_base and self._should_overwrite(current_base, is_hf_model):
|
||||
if new_base and self._should_overwrite(current_base, is_source_model):
|
||||
updates["base_model"] = new_base
|
||||
|
||||
# model_name — the site's own display name, so a source download never
|
||||
# shows up under its local filename. Written only while the name is
|
||||
# still the untouched file stem: once a user renames a model that
|
||||
# choice is theirs to keep.
|
||||
site_name = ((source_context.model_name if source_context else "") or "").strip()
|
||||
if is_source_model and site_name:
|
||||
current_name = (metadata.get("model_name") or "").strip()
|
||||
file_stem = (metadata.get("file_name") or "").strip()
|
||||
if not current_name or current_name == file_stem:
|
||||
updates["model_name"] = site_name
|
||||
|
||||
# trigger words → civitai.trainedWords
|
||||
new_triggers = llm_output.get("trigger_words", [])
|
||||
trigger_words_empty = True
|
||||
@@ -110,45 +160,71 @@ class PostProcessor:
|
||||
cleaned = [t.strip() for t in new_triggers if t.strip()]
|
||||
cleaned = [t for t in cleaned if t.lower() not in ("none", "null", "n/a")]
|
||||
trigger_words_empty = not cleaned
|
||||
current_civitai = metadata.get("civitai") or {}
|
||||
current_triggers = current_civitai.get("trainedWords") or []
|
||||
if self._should_overwrite_list(current_triggers, is_hf_model):
|
||||
trig_civitai = dict(current_civitai)
|
||||
if "civitai" in updates and isinstance(updates["civitai"], dict):
|
||||
trig_civitai.update(updates["civitai"])
|
||||
trig_civitai["trainedWords"] = cleaned
|
||||
updates["civitai"] = trig_civitai
|
||||
current_triggers = (metadata.get("civitai") or {}).get("trainedWords") or []
|
||||
if self._should_overwrite_list(current_triggers, is_source_model):
|
||||
self._merge_civitai(updates, metadata, trainedWords=cleaned)
|
||||
|
||||
# modelDescription — from raw README content (converted to HTML)
|
||||
if readme_content and is_hf_model:
|
||||
converted = convert_readme_to_html(readme_content)
|
||||
if converted:
|
||||
updates["modelDescription"] = converted
|
||||
# modelDescription — the author's own summary (when the site keeps one
|
||||
# outside the README, e.g. ModelScope's ``Description``) followed by the
|
||||
# README converted to HTML.
|
||||
site_description = (
|
||||
(source_context.description if source_context else "") or ""
|
||||
).strip()
|
||||
if is_source_model and (site_description or readme_content):
|
||||
parts: List[str] = []
|
||||
if site_description:
|
||||
parts.append(f"<p>{html.escape(site_description)}</p>")
|
||||
if readme_content:
|
||||
converted = convert_readme_to_html(readme_content)
|
||||
if converted:
|
||||
parts.append(converted)
|
||||
if parts:
|
||||
updates["modelDescription"] = "\n".join(parts)
|
||||
|
||||
# short_description → civitai.description (for "About this version")
|
||||
# short_description → civitai.description (for "About this version").
|
||||
# Falls back to the site's author summary, which for ModelScope AIGC
|
||||
# models is frequently the only human-written text available.
|
||||
short_desc = (llm_output.get("short_description") or "").strip()
|
||||
if short_desc and is_hf_model:
|
||||
current_civitai = metadata.get("civitai") or {}
|
||||
desc_civitai = dict(current_civitai)
|
||||
if "civitai" in updates and isinstance(updates["civitai"], dict):
|
||||
desc_civitai.update(updates["civitai"])
|
||||
desc_civitai["description"] = short_desc
|
||||
updates["civitai"] = desc_civitai
|
||||
if not short_desc:
|
||||
short_desc = site_description
|
||||
if short_desc and is_source_model:
|
||||
self._merge_civitai(updates, metadata, description=short_desc)
|
||||
|
||||
# The version label completes the card the way a CivitAI download does:
|
||||
# the UI renders `civitai.name` as the version chip. It is per file,
|
||||
# so a collection repository shows that checkpoint's own label.
|
||||
site_version = (
|
||||
(source_context.version_name if source_context else "") or ""
|
||||
).strip()
|
||||
if is_source_model and site_version:
|
||||
self._merge_civitai(updates, metadata, name=site_version)
|
||||
|
||||
# gallery images → civitai.images (site example images, YAML frontmatter
|
||||
# widget entries, and Sample Gallery markdown tables in the README body)
|
||||
rec_width = llm_output.get("recommended_width") or 0
|
||||
rec_height = llm_output.get("recommended_height") or 0
|
||||
|
||||
# Example images the site publishes for *this* file. They are matched
|
||||
# by filename, so they are the most precise preview source available
|
||||
# and the only one for repositories whose README carries no images.
|
||||
site_images: List[Dict[str, Any]] = []
|
||||
if is_source_model and source_context is not None:
|
||||
site_images = [
|
||||
_example_image(url, rec_width, rec_height)
|
||||
for url in source_context.example_images
|
||||
if url
|
||||
]
|
||||
|
||||
# gallery images → civitai.images (from YAML frontmatter widget entries
|
||||
# and Sample Gallery markdown tables in the README body)
|
||||
gallery_images: List[Dict[str, Any]] = []
|
||||
if readme_content and is_hf_model:
|
||||
hf_url = metadata.get("hf_url", "") or ""
|
||||
repo = extract_repo_from_hf_url(hf_url)
|
||||
if repo:
|
||||
rec_w = llm_output.get("recommended_width") or 0
|
||||
rec_h = llm_output.get("recommended_height") or 0
|
||||
|
||||
if (readme_content or site_images) and is_source_model:
|
||||
repo = source_id
|
||||
readme_images: List[Dict[str, Any]] = []
|
||||
if readme_content and repo:
|
||||
# 1. Widget images (YAML frontmatter)
|
||||
gallery = extract_gallery_images(
|
||||
readme_content, repo,
|
||||
default_width=rec_w, default_height=rec_h,
|
||||
default_width=rec_width, default_height=rec_height,
|
||||
base_url=asset_base_url,
|
||||
)
|
||||
|
||||
# 2. Sample Gallery table images (markdown body), deduplicated
|
||||
@@ -156,7 +232,8 @@ class PostProcessor:
|
||||
table_images = extract_gallery_table_images(
|
||||
readme_content, repo,
|
||||
existing_urls=existing_urls,
|
||||
default_width=rec_w, default_height=rec_h,
|
||||
default_width=rec_width, default_height=rec_height,
|
||||
base_url=asset_base_url,
|
||||
)
|
||||
existing_urls.update(img["url"] for img in table_images if img.get("url"))
|
||||
|
||||
@@ -164,7 +241,8 @@ class PostProcessor:
|
||||
simple_images = extract_simple_markdown_images(
|
||||
readme_content, repo,
|
||||
existing_urls=existing_urls,
|
||||
default_width=rec_w, default_height=rec_h,
|
||||
default_width=rec_width, default_height=rec_height,
|
||||
base_url=asset_base_url,
|
||||
)
|
||||
existing_urls.update(img["url"] for img in simple_images if img.get("url"))
|
||||
|
||||
@@ -172,54 +250,71 @@ class PostProcessor:
|
||||
html_images = extract_html_img_tags(
|
||||
readme_content, repo,
|
||||
existing_urls=existing_urls,
|
||||
default_width=rec_w, default_height=rec_h,
|
||||
default_width=rec_width, default_height=rec_height,
|
||||
base_url=asset_base_url,
|
||||
)
|
||||
|
||||
all_images = gallery + table_images + simple_images + html_images
|
||||
if all_images:
|
||||
gallery_images = all_images
|
||||
current_civitai = metadata.get("civitai") or {}
|
||||
gallery_civitai = dict(current_civitai)
|
||||
if "civitai" in updates and isinstance(updates["civitai"], dict):
|
||||
gallery_civitai.update(updates["civitai"])
|
||||
gallery_civitai["images"] = all_images
|
||||
updates["civitai"] = gallery_civitai
|
||||
readme_images = gallery + table_images + simple_images + html_images
|
||||
|
||||
# tags
|
||||
# Site images come first so the preview fallback below prefers an
|
||||
# image that is known to belong to this exact file.
|
||||
all_images = _dedupe_images(site_images + readme_images)
|
||||
if all_images:
|
||||
gallery_images = all_images
|
||||
self._merge_civitai(updates, metadata, images=all_images)
|
||||
|
||||
# tags — the site's curated tags are authoritative content vocabulary, so
|
||||
# they are kept alongside whatever the LLM proposed (the LLM is skipped
|
||||
# entirely when the site data is complete, which is why this cannot rely
|
||||
# on ``llm_output`` alone).
|
||||
new_tags = llm_output.get("tags", [])
|
||||
if isinstance(new_tags, list) and new_tags:
|
||||
candidate_tags: List[str] = []
|
||||
if is_source_model and source_context is not None:
|
||||
candidate_tags.extend(source_context.official_tags)
|
||||
if isinstance(new_tags, list):
|
||||
candidate_tags.extend(
|
||||
tag for tag in new_tags if tag not in candidate_tags
|
||||
)
|
||||
if candidate_tags:
|
||||
existing_tags = metadata.get("tags") or []
|
||||
merged = self._merge_tags(existing_tags, new_tags)
|
||||
if len(merged) > len(existing_tags) or is_hf_model:
|
||||
merged = self._merge_tags(existing_tags, candidate_tags)
|
||||
if len(merged) > len(existing_tags) or is_source_model:
|
||||
updates["tags"] = merged
|
||||
|
||||
# metadata_source & llm_enriched_at (always set)
|
||||
updates["metadata_source"] = "agent:enrich_hf_metadata"
|
||||
updates["llm_enriched_at"] = datetime.now(timezone.utc).isoformat()
|
||||
# metadata_source is recorded for provenance; llm_enriched_at only means
|
||||
# something when a provider actually answered, so the deterministic
|
||||
# download-time hydration does not claim an enrichment that never ran.
|
||||
updates["metadata_source"] = metadata_source
|
||||
if llm_output:
|
||||
updates["llm_enriched_at"] = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
# Store LLM confidence in metadata so it's accessible for evaluation
|
||||
# LLM confidence, stored for the enrichment evaluation harness. The key
|
||||
# must NOT start with an underscore: `BaseModelMetadata.from_dict()`
|
||||
# deliberately drops underscore-prefixed keys so they never round-trip,
|
||||
# which silently erased this field on the next metadata write.
|
||||
raw_confidence = (llm_output.get("confidence") or "").strip()
|
||||
if raw_confidence:
|
||||
updates["_llm_confidence"] = raw_confidence
|
||||
updates["llm_confidence"] = raw_confidence
|
||||
|
||||
# Fallback: extract instance_prompt from YAML frontmatter when the LLM
|
||||
# returned empty trigger words but the README has instance_prompt.
|
||||
# Fallback: use the trigger words the site records for this exact file,
|
||||
# then the README's YAML `instance_prompt`, when the LLM returned none.
|
||||
if trigger_words_empty:
|
||||
instance_prompt = _extract_yaml_instance_prompt(readme_content)
|
||||
if instance_prompt:
|
||||
current_civitai = metadata.get("civitai") or {}
|
||||
trig_civitai = dict(current_civitai)
|
||||
if "civitai" in updates and isinstance(updates["civitai"], dict):
|
||||
trig_civitai.update(updates["civitai"])
|
||||
trig_civitai["trainedWords"] = [instance_prompt]
|
||||
updates["civitai"] = trig_civitai
|
||||
site_triggers = (
|
||||
list(source_context.trigger_words) if source_context else []
|
||||
)
|
||||
if not site_triggers:
|
||||
instance_prompt = _extract_yaml_instance_prompt(readme_content)
|
||||
if instance_prompt:
|
||||
site_triggers = [instance_prompt]
|
||||
if site_triggers:
|
||||
self._merge_civitai(updates, metadata, trainedWords=site_triggers)
|
||||
|
||||
preview_remote_url = (llm_output.get("preview_url") or "").strip()
|
||||
# Fallback: if the LLM couldn't find a preview image in the cleaned
|
||||
# README, find the first gallery image from the *model-specific
|
||||
# section* of the README (not the repo-wide first image, which
|
||||
# belongs to a different model in collection repos).
|
||||
if not preview_remote_url and readme_content and is_hf_model:
|
||||
if not preview_remote_url and readme_content and is_source_model:
|
||||
model_basename = os.path.splitext(os.path.basename(model_path))[0]
|
||||
relevant_section = extract_relevant_section(
|
||||
readme_content, model_basename,
|
||||
@@ -245,8 +340,12 @@ class PostProcessor:
|
||||
if new_notes:
|
||||
updates["notes"] = new_notes
|
||||
|
||||
# usage_tips — JSON string (e.g. {"strength_min":0.85,"strength_max":1.4})
|
||||
# usage_tips — JSON string (e.g. {"strength_min":0.85,"strength_max":1.4}).
|
||||
# When the LLM returned nothing, recover an explicitly stated strength
|
||||
# range from the author summary so the value is not lost.
|
||||
raw_tips = (llm_output.get("usage_tips") or "").strip()
|
||||
if not raw_tips or raw_tips == "{}":
|
||||
raw_tips = _extract_usage_tips(site_description)
|
||||
if raw_tips and raw_tips != "{}":
|
||||
try:
|
||||
json.loads(raw_tips)
|
||||
@@ -276,16 +375,35 @@ class PostProcessor:
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _should_overwrite(current_value: str, is_hf_model: bool) -> bool:
|
||||
def _should_overwrite(current_value: str, is_source_model: bool) -> bool:
|
||||
"""Return ``True`` when a scalar field should be overwritten."""
|
||||
return is_hf_model or not current_value or current_value.lower() in (
|
||||
return is_source_model or not current_value or current_value.lower() in (
|
||||
"", "unknown",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _should_overwrite_list(current_list: List[str], is_hf_model: bool) -> bool:
|
||||
def _merge_civitai(
|
||||
updates: Dict[str, Any], metadata: Dict[str, Any], **fields: Any
|
||||
) -> None:
|
||||
"""Layer *fields* onto the ``civitai`` block being assembled.
|
||||
|
||||
Description, version label, trigger words and gallery images all live
|
||||
in the same dict and are contributed by separate branches, so each one
|
||||
starts from what is already on disk and then applies whatever an
|
||||
earlier branch queued in *updates*.
|
||||
"""
|
||||
|
||||
merged = dict(metadata.get("civitai") or {})
|
||||
queued = updates.get("civitai")
|
||||
if isinstance(queued, dict):
|
||||
merged.update(queued)
|
||||
merged.update(fields)
|
||||
updates["civitai"] = merged
|
||||
|
||||
@staticmethod
|
||||
def _should_overwrite_list(current_list: List[str], is_source_model: bool) -> bool:
|
||||
"""Return ``True`` when a list field should be overwritten."""
|
||||
return is_hf_model or not current_list
|
||||
return is_source_model or not current_list
|
||||
|
||||
@staticmethod
|
||||
def _merge_tags(existing: List[str], new: List[str]) -> List[str]:
|
||||
@@ -309,6 +427,129 @@ class PostProcessor:
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
#: Separator between a label and its value. Published model cards routinely
|
||||
#: wrap the numbers in markdown emphasis or quotes (``strength: **0.85 - 1.4**``,
|
||||
#: ``CLIP 强度「0.5」``), so those are absorbed rather than treated as a break.
|
||||
_EMPHASIS = "[\"'\u201c\u201d\u300c\u300d*_`\\s]*"
|
||||
|
||||
#: An explicitly stated strength/weight range, e.g. ``权重0.5-1.2``,
|
||||
#: ``强度 0.8 ~ 1.2``, ``strength: **0.85 - 1.4**``.
|
||||
_RANGE_DASH = "(?:-|\u2010|\u2011|\u2012|\u2013|\u2014|\uff0d|~|\uff5e|\u81f3|\u5230|to)"
|
||||
|
||||
_STRENGTH_RANGE_RE = re.compile(
|
||||
"(?:\u6743\u91cd|\u5f3a\u5ea6|strength|weight)" + _EMPHASIS + "[:\uff1a]?" + _EMPHASIS
|
||||
+ r"(\d+(?:\.\d+)?)" + _EMPHASIS + _RANGE_DASH + _EMPHASIS
|
||||
+ r"(\d+(?:\.\d+)?)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
#: A single strength/weight value, e.g. ``strength: 0.6``, ``权重 0.8``.
|
||||
_STRENGTH_VALUE_RE = re.compile(
|
||||
"(?:\u6743\u91cd|\u5f3a\u5ea6|strength|weight)" + _EMPHASIS + "[:\uff1a]?" + _EMPHASIS
|
||||
+ r"(\d+(?:\.\d+)?)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
#: ``clip strength: 0.5`` / ``CLIP 强度 0.5``.
|
||||
_CLIP_STRENGTH_RE = re.compile(
|
||||
"clip" + _EMPHASIS + "(?:\u5f3a\u5ea6|strength)" + _EMPHASIS + "[:\uff1a]?" + _EMPHASIS
|
||||
+ r"(\d+(?:\.\d+)?)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
#: ``clip skip: 2`` / ``CLIP 跳过 2``.
|
||||
_CLIP_SKIP_RE = re.compile(
|
||||
"clip" + _EMPHASIS + "(?:skip|\u8df3\u8fc7)" + _EMPHASIS + "[:\uff1a]?" + _EMPHASIS
|
||||
+ r"(\d+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _extract_usage_tips(text: str) -> str:
|
||||
"""Extract stated strength/CLIP recommendations from prose.
|
||||
|
||||
This is the deterministic counterpart to the LLM's ``usage_tips`` output,
|
||||
used when the LLM was skipped. It only recognises explicitly written
|
||||
values — it never infers a range — and returns ``""`` when it finds none.
|
||||
|
||||
Returns:
|
||||
A JSON string matching the skill's ``usage_tips`` schema, or ``""``.
|
||||
"""
|
||||
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
tips: Dict[str, Any] = {}
|
||||
|
||||
# CLIP strength is resolved first and then blanked out, so the generic
|
||||
# strength patterns cannot mistake `CLIP 强度 0.5` for the LoRA strength.
|
||||
text_for_strength = text
|
||||
clip_strength = _CLIP_STRENGTH_RE.search(text_for_strength)
|
||||
if clip_strength:
|
||||
tips["clip_strength"] = float(clip_strength.group(1))
|
||||
text_for_strength = (
|
||||
text_for_strength[: clip_strength.start()]
|
||||
+ " "
|
||||
+ text_for_strength[clip_strength.end() :]
|
||||
)
|
||||
|
||||
range_match = _STRENGTH_RANGE_RE.search(text_for_strength)
|
||||
if range_match:
|
||||
low = float(range_match.group(1))
|
||||
high = float(range_match.group(2))
|
||||
if low > high:
|
||||
low, high = high, low
|
||||
tips["strength_min"] = low
|
||||
tips["strength_max"] = high
|
||||
tips["strength_range"] = f"{low:g}-{high:g}"
|
||||
else:
|
||||
value_match = _STRENGTH_VALUE_RE.search(text_for_strength)
|
||||
if value_match:
|
||||
tips["strength"] = float(value_match.group(1))
|
||||
|
||||
clip_skip = _CLIP_SKIP_RE.search(text)
|
||||
if clip_skip:
|
||||
tips["clip_skip"] = int(clip_skip.group(1))
|
||||
|
||||
if not tips:
|
||||
return ""
|
||||
return json.dumps(tips, ensure_ascii=False)
|
||||
|
||||
|
||||
def _example_image(url: str, width: int, height: int) -> Dict[str, Any]:
|
||||
"""Build a ``civitai.images`` entry for a site-provided example image.
|
||||
|
||||
The site publishes no prompt alongside these images, so the entry carries
|
||||
empty prompt metadata and the LLM's recommended dimensions when it found
|
||||
any (falling back to the same 512px placeholder the README extractors use).
|
||||
"""
|
||||
|
||||
return {
|
||||
"url": url,
|
||||
"type": "image",
|
||||
"nsfwLevel": 0,
|
||||
"width": width or 512,
|
||||
"height": height or 512,
|
||||
"meta": {"prompt": "", "negativePrompt": ""},
|
||||
"hasMeta": False,
|
||||
"hasPositivePrompt": False,
|
||||
}
|
||||
|
||||
|
||||
def _dedupe_images(images: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""Drop later entries that repeat an earlier image URL, keeping order."""
|
||||
|
||||
seen: set[str] = set()
|
||||
unique: List[Dict[str, Any]] = []
|
||||
for image in images:
|
||||
url = image.get("url") or ""
|
||||
if not url or url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
unique.append(image)
|
||||
return unique
|
||||
|
||||
|
||||
def _extract_yaml_instance_prompt(readme_content: str) -> str:
|
||||
"""Extract ``instance_prompt`` from the YAML frontmatter of a HF README.
|
||||
|
||||
|
||||
@@ -1,20 +1,23 @@
|
||||
---
|
||||
name: enrich_hf_metadata
|
||||
title: "Enrich Metadata from HuggingFace"
|
||||
title: "Enrich Metadata from Model Card"
|
||||
description: >
|
||||
Parse the HuggingFace model card via LLM to extract description, trigger
|
||||
words, base model, tags, and preview image URL.
|
||||
Parse the model card (README) from HuggingFace, ModelScope, or any other
|
||||
supported model site via LLM to extract description, trigger words, base
|
||||
model, tags, and preview image URL.
|
||||
llm_required: true
|
||||
---
|
||||
|
||||
You are an expert assistant for AI image generation models. Your task is to extract structured metadata from a HuggingFace model card (README.md).
|
||||
You are an expert assistant for AI image generation models. Your task is to extract structured metadata from a model card (README).
|
||||
|
||||
## Model Information
|
||||
|
||||
- **Repository**: {{hf_url}}
|
||||
- **Source site**: {{source_label}} ({{source_platform}})
|
||||
- **Model page**: {{source_url}}
|
||||
- **Model file path**: {{model_path}}
|
||||
- **Model filename**: {{model_basename}}
|
||||
- **Repository ID**: {{repo}}
|
||||
- **Repository ID**: {{source_id}}
|
||||
- **Repository raw-file base URL**: {{asset_base_url}}
|
||||
|
||||
## Current Metadata (may be incomplete)
|
||||
|
||||
@@ -22,6 +25,34 @@ You are an expert assistant for AI image generation models. Your task is to extr
|
||||
{{current_metadata}}
|
||||
```
|
||||
|
||||
## Site-Provided Metadata (any field may be empty)
|
||||
|
||||
The model site publishes the following **alongside** the README. It is
|
||||
first-hand information recorded by the site itself, so it outranks anything
|
||||
you would otherwise guess:
|
||||
|
||||
- **Author description**: {{source_description}}
|
||||
- **Base model reported by the site**: {{source_base_model}}
|
||||
- **Trigger words recorded for this file**: {{source_trigger_words}}
|
||||
- **Site-curated tags**:
|
||||
{{source_official_tags}}
|
||||
- **Example image URLs for this file**:
|
||||
{{source_example_images}}
|
||||
|
||||
Use it as follows:
|
||||
|
||||
- A weight or strength range stated in the **author description** belongs in
|
||||
``usage_tips`` (and in ``notes``); do not leave ``usage_tips`` empty when the
|
||||
description states one.
|
||||
- When the author description exists, base ``short_description`` on it rather
|
||||
than on the README, which on some sites is auto-generated boilerplate.
|
||||
- Treat the **site-curated tags** as strong signals for ``tags``: they are
|
||||
already a curated content vocabulary, so prefer them over invented words.
|
||||
- Treat the **base model reported by the site** as a strong hint for
|
||||
``base_model``, but still map it to the EXACT canonical name from the
|
||||
available base-model list.
|
||||
- Use the **example image URLs** when the README contains no usable image.
|
||||
|
||||
## User Priority Tags Reference
|
||||
|
||||
The user has configured the following list of **meaningful tag categories** for this model type (`{{model_type}}`):
|
||||
@@ -39,7 +70,7 @@ name listed — do not invent aliases or modify variant suffixes.
|
||||
|
||||
{{base_models}}
|
||||
|
||||
## HuggingFace README Content
|
||||
## Model Card Content
|
||||
|
||||
```
|
||||
{{readme_content}}
|
||||
@@ -52,10 +83,11 @@ Extract the following information from the README content above:
|
||||
### base_model
|
||||
The base model this model was trained on. Use EXACTLY one of the names from the **Available Base Models** list above. Do not invent new names or use aliases.
|
||||
|
||||
Check the YAML frontmatter for ``base_model:`` first. If the frontmatter has no ``base_model:``, look at the **model filename** (``{{model_basename}}``), YAML ``tags:``, README title and first paragraph for clues — the base model family is often embedded in the name
|
||||
Check the **base model reported by the site** (above) and the YAML frontmatter ``base_model:`` first. If neither yields a match, look at the **model filename** (``{{model_basename}}``), YAML ``tags:``, README title and first paragraph for clues — the base model family is often embedded in the name
|
||||
|
||||
### trigger_words
|
||||
The trigger words or activation prompts needed to use this LoRA. Look for:
|
||||
- The **trigger words recorded for this file** in the site-provided metadata (most authoritative)
|
||||
- `instance_prompt:` in the YAML frontmatter
|
||||
- Phrases like "trigger word:", "trigger:", "use this prompt:", "activation prompt:"
|
||||
- In collection repos: the trigger section **specific to this model file** (look near matching download links or anchor IDs)
|
||||
@@ -63,12 +95,13 @@ The trigger words or activation prompts needed to use this LoRA. Look for:
|
||||
Return as an array of strings. If none found, return an empty array `[]`. **Never** return `["None"]` or any placeholder value — a truly empty list means no trigger words exist.
|
||||
|
||||
### short_description
|
||||
A concise 1-2 sentence summary of what this model does. Extract from the "Model description" section or the first paragraph. For collection repos, focus on the **specific model version** matching `{{model_basename}}`, not the repo as a whole. Return empty string if the README is too minimal.
|
||||
A concise 1-2 sentence summary of what this model does. For collection repos, focus on the **specific model version** matching `{{model_basename}}`, not the repo as a whole. Prefer the **author description** from the site-provided metadata when it is present; otherwise extract from the "Model description" section or the first paragraph. Return empty string if the available content is too minimal.
|
||||
|
||||
### tags
|
||||
3-8 relevant tags for categorizing this model. **Quality over quantity.**
|
||||
|
||||
Sources to consider:
|
||||
- The **site-curated tags** from the site-provided metadata (these are already filtered content tags — prefer them)
|
||||
- The YAML frontmatter `tags:` list (filter out technical ones — see below)
|
||||
- The subject, style, character, or concept the model represents
|
||||
- The model filename itself may give clues (e.g. "pokemon", "anime", "pixelart")
|
||||
@@ -79,7 +112,9 @@ Sources to consider:
|
||||
|
||||
2. **Cross-reference against the priority_tags reference.** Only include a tag if it meaningfully describes what the model actually creates (subject, style, character type) and is semantically close to one of the priority_tags. If none of the README's tags match meaningful categories, prefer returning a smaller set or an empty array over including low-value tags.
|
||||
|
||||
3. **All lowercase, no spaces, no hyphens** (use single words like `"photorealistic"`, `"anime"`, `"character"`).
|
||||
3. **All lowercase, and keep each tag's own wording.** Prefer the spelling already used by the site, the frontmatter, or the author — including hyphenated and multi-word tags such as `"sci-fi"`, `"semi-realistic"`, `"character-enhancement"` or `"art style"`. Do **not** strip separators or invent a single-word variant of a tag you are already including (e.g. do not emit both `"character-enhancement"` and `"character"`). When a tag is written in another script (e.g. Chinese), likewise keep it verbatim instead of translating it.
|
||||
|
||||
4. **Never invent a tag** that neither the site-provided metadata, the YAML frontmatter, nor the README text supports.
|
||||
|
||||
Return empty array if no meaningful content tags remain after filtering.
|
||||
|
||||
@@ -92,13 +127,13 @@ The URL of the most suitable preview image from the README. Look for:
|
||||
- The YAML frontmatter `widget:` section (which often has `output.url` fields)
|
||||
- In collection repos: the sample images listed **under the section** for this specific model version
|
||||
- Generic `` in the body
|
||||
Choose the first image that appears to be a generation example (not a logo or diagram). Construct the absolute URL as `https://huggingface.co/{{repo}}/resolve/main/{filename}`. If no suitable image is found, return an empty string.
|
||||
Choose the first image that appears to be a generation example (not a logo or diagram). Construct the absolute URL from the repository raw-file base URL (`{{asset_base_url}}`) plus the relative path. If the README has no suitable image, fall back to the site-provided **example image URLs** for this file. If nothing is available, return an empty string.
|
||||
|
||||
### notes
|
||||
A plain-text summary of the model card's key practical usage information. Combine trigger words, style modifiers, recommended parameters (steps, CFG, resolution, sampler), and any setup tips into a readable paragraph. For collection repos, focus on the **specific model version** matching `{{model_basename}}`. Return empty string if the README has no useful usage info.
|
||||
A plain-text summary of the model card's key practical usage information. Combine trigger words, style modifiers, recommended parameters (steps, CFG, resolution, sampler), and any setup tips into a readable paragraph. For collection repos, focus on the **specific model version** matching `{{model_basename}}`. Include the **author description** from the site-provided metadata when it is present. Return empty string if there is no useful usage info.
|
||||
|
||||
### usage_tips
|
||||
A JSON string with structured usage recommendations. Extract from the README any explicit ranges or recommended values (e.g. "Set LoRA strength: **0.85 - 1.4**", "CLIP strength: 0.5"). Possible fields (include only those you can determine):
|
||||
A JSON string with structured usage recommendations. Extract from the **author description** (site-provided metadata) and the README any explicit ranges or recommended values (e.g. "Set LoRA strength: **0.85 - 1.4**", "CLIP strength: 0.5", "权重0.5-1.2"). Possible fields (include only those you can determine):
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -121,7 +156,7 @@ Your confidence level in the extracted data:
|
||||
|
||||
## Important: Handling Collection Repos (multiple model files)
|
||||
|
||||
Many HuggingFace repos contain **multiple model files** in a single repository
|
||||
Many model repositories contain **multiple model files** in a single repository
|
||||
(e.g. a "LoRA collection" with different styles/characters in separate files).
|
||||
|
||||
The model file currently being enriched is: **`{{model_basename}}`**
|
||||
|
||||
@@ -1,8 +1,15 @@
|
||||
"""HF README processing for the ``enrich_hf_metadata`` skill.
|
||||
"""Model card (README) processing for the ``enrich_hf_metadata`` skill.
|
||||
|
||||
Provides README cleaning for LLM injection, gallery/image extraction from
|
||||
multiple formats (YAML widget, markdown, HTML ``<img>``, gallery tables),
|
||||
and section-based README trimming for collection repos.
|
||||
|
||||
The extractors default to Hugging Face asset URLs, but every one of them
|
||||
accepts an explicit ``base_url`` so the same parsing works for any model
|
||||
source (ModelScope, ...). See :mod:`py.services.model_sources`.
|
||||
|
||||
This module deliberately has no package-relative imports: it is also loaded
|
||||
standalone by the README-processing test harness.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -15,12 +22,25 @@ from typing import Any, List, Tuple
|
||||
_REPO_URL_PATTERN = re.compile(r"https?://huggingface\.co/([^/]+/[^/]+)")
|
||||
|
||||
|
||||
def resolve_asset_base_url(repo: str, base_url: str | None = None) -> str:
|
||||
"""Return the base URL used to resolve repository-relative assets.
|
||||
|
||||
Falls back to the historical Hugging Face layout when *base_url* is not
|
||||
supplied, so existing callers keep their behaviour.
|
||||
"""
|
||||
|
||||
if base_url:
|
||||
return base_url.rstrip("/")
|
||||
return f"https://huggingface.co/{repo}/resolve/main"
|
||||
|
||||
|
||||
def extract_simple_markdown_images(
|
||||
markdown_text: str,
|
||||
repo: str,
|
||||
existing_urls: set[str] | None = None,
|
||||
default_width: int = 512,
|
||||
default_height: int = 512,
|
||||
base_url: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Extract standalone markdown images from the README body.
|
||||
|
||||
@@ -32,10 +52,10 @@ def extract_simple_markdown_images(
|
||||
Returns a list of dicts in the same ``civitai.images`` format as
|
||||
:func:`extract_gallery_images`.
|
||||
"""
|
||||
if not markdown_text or not repo:
|
||||
if not markdown_text or not (repo or base_url):
|
||||
return []
|
||||
|
||||
base_url = f"https://huggingface.co/{repo}/resolve/main"
|
||||
base_url = resolve_asset_base_url(repo, base_url)
|
||||
images: list[dict[str, Any]] = []
|
||||
seen_urls: set[str] = set(existing_urls) if existing_urls else set()
|
||||
|
||||
@@ -89,20 +109,21 @@ def extract_html_img_tags(
|
||||
existing_urls: set[str] | None = None,
|
||||
default_width: int = 512,
|
||||
default_height: int = 512,
|
||||
base_url: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Extract image URLs from HTML ``<img src=\"...\">`` tags in the README.
|
||||
|
||||
Many HF collection repos (e.g. ``deadman44/Z-Image_LoRA``) use raw HTML
|
||||
``<img>`` tags exclusively for their sample images, with no markdown
|
||||
``![]()`` equivalents. This function finds those tags and constructs
|
||||
resolvable HF URLs.
|
||||
resolvable URLs.
|
||||
|
||||
Returns a list of dicts in the ``civitai.images`` format.
|
||||
"""
|
||||
if not markdown_text or not repo:
|
||||
if not markdown_text or not (repo or base_url):
|
||||
return []
|
||||
|
||||
base_url = f"https://huggingface.co/{repo}/resolve/main"
|
||||
base_url = resolve_asset_base_url(repo, base_url)
|
||||
images: list[dict[str, Any]] = []
|
||||
seen_urls: set[str] = set(existing_urls) if existing_urls else set()
|
||||
|
||||
@@ -166,7 +187,7 @@ def extract_html_img_tags(
|
||||
|
||||
def extract_repo_from_hf_url(hf_url: str) -> str:
|
||||
"""Extract ``user/repo`` from a HuggingFace URL."""
|
||||
m = _REPO_URL_PATTERN.match(hf_url)
|
||||
m = _REPO_URL_PATTERN.match(hf_url or "")
|
||||
return m.group(1) if m else ""
|
||||
|
||||
|
||||
@@ -175,21 +196,23 @@ def extract_gallery_images(
|
||||
repo: str,
|
||||
default_width: int = 512,
|
||||
default_height: int = 512,
|
||||
base_url: str | None = None,
|
||||
) -> List[dict[str, Any]]:
|
||||
"""Extract widget/gallery images from the YAML frontmatter of a HF README.
|
||||
"""Extract widget/gallery images from the YAML frontmatter of a README.
|
||||
|
||||
Args:
|
||||
markdown_text: Raw README content.
|
||||
repo: HF repo identifier (``user/repo``).
|
||||
repo: Repository identifier (``user/repo``).
|
||||
default_width: Fallback width when the README provides no dimension.
|
||||
default_height: Fallback height when the README provides no dimension.
|
||||
base_url: Overrides the asset base URL (defaults to Hugging Face).
|
||||
|
||||
Returns a list of dicts compatible with the ``civitai.images`` metadata
|
||||
format, each containing ``url`` (absolute HF URL), ``meta.prompt``,
|
||||
format, each containing ``url`` (absolute), ``meta.prompt``,
|
||||
``width``, ``height``, and ``type``. Returns an empty list when no
|
||||
widget entries are found or when *repo* is empty.
|
||||
"""
|
||||
if not markdown_text or not repo:
|
||||
if not markdown_text or not (repo or base_url):
|
||||
return []
|
||||
|
||||
frontmatter = _extract_frontmatter(markdown_text)
|
||||
@@ -197,7 +220,7 @@ def extract_gallery_images(
|
||||
return []
|
||||
|
||||
images: List[dict[str, Any]] = []
|
||||
base_url = f"https://huggingface.co/{repo}/resolve/main"
|
||||
base_url = resolve_asset_base_url(repo, base_url)
|
||||
w = default_width or 512
|
||||
h = default_height or 512
|
||||
|
||||
@@ -279,10 +302,11 @@ def extract_gallery_table_images(
|
||||
existing_urls: set[str] | None = None,
|
||||
default_width: int = 512,
|
||||
default_height: int = 512,
|
||||
base_url: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Extract images from ``| Preview | Prompt |`` markdown gallery tables.
|
||||
|
||||
Many HF READMEs include a sample-gallery table in the body (outside
|
||||
Many READMEs include a sample-gallery table in the body (outside
|
||||
the YAML frontmatter) that shows generation examples with their
|
||||
prompts. This function parses those tables and merges results with
|
||||
the widget-sourced images from :func:`extract_gallery_images`.
|
||||
@@ -291,10 +315,10 @@ def extract_gallery_table_images(
|
||||
:func:`extract_gallery_images`. Already-seen URLs (from *existing_urls*)
|
||||
are skipped.
|
||||
"""
|
||||
if not markdown_text or not repo:
|
||||
if not markdown_text or not (repo or base_url):
|
||||
return []
|
||||
|
||||
base_url = f"https://huggingface.co/{repo}/resolve/main"
|
||||
base_url = resolve_asset_base_url(repo, base_url)
|
||||
images: list[dict[str, Any]] = []
|
||||
seen_urls: set[str] = set(existing_urls) if existing_urls else set()
|
||||
lines = markdown_text.split("\n")
|
||||
@@ -368,12 +392,18 @@ def _extract_frontmatter(text: str) -> str:
|
||||
|
||||
|
||||
def convert_readme_to_html(markdown_text: str | None) -> str:
|
||||
"""Convert HF README markdown to sanitised HTML."""
|
||||
"""Convert HF README markdown to sanitised HTML.
|
||||
|
||||
Site-generated placeholder notices are dropped here too, so a repository
|
||||
whose author wrote nothing does not store the download instructions as its
|
||||
model description; the result is an empty string in that case.
|
||||
"""
|
||||
if not markdown_text:
|
||||
return ""
|
||||
|
||||
text = markdown_text
|
||||
text = _strip_frontmatter(text)
|
||||
text = _strip_generated_card_boilerplate(text)
|
||||
text = _strip_gallery(text)
|
||||
text = _strip_badge_images(text)
|
||||
text = _strip_html_comments(text)
|
||||
@@ -420,6 +450,59 @@ _MASSIVE_LIST_LINE_MIN_LEN = 150
|
||||
#: Minimum consecutive enumeration lines to trigger massive-list stripping.
|
||||
_MASSIVE_LIST_THRESHOLD = 8
|
||||
|
||||
#: Substrings identifying text a *site* generated to fill a model card whose
|
||||
#: author wrote nothing, as opposed to the author's own content. ModelScope
|
||||
#: renders such a card as a placeholder notice, a block of SDK/git download
|
||||
#: instructions, and a closing invitation to improve the card.
|
||||
#:
|
||||
#: Matched as substrings rather than whole headings because the notices are
|
||||
#: prose, and because non-Latin scripts are not space-delimited — the notice
|
||||
#: continues with a full-width period, so the ``title == kw`` style matching
|
||||
#: used for :data:`_BOILERPLATE_HEADERS` would never fire.
|
||||
_GENERATED_CARD_MARKERS: tuple[str, ...] = (
|
||||
"当前模型的贡献者未提供更加详细的模型介绍",
|
||||
"您可以通过如下",
|
||||
"如果您是本模型的贡献者",
|
||||
)
|
||||
|
||||
|
||||
def _strip_generated_card_boilerplate(text: str) -> str:
|
||||
"""Remove the notices a site generates to fill an empty model card.
|
||||
|
||||
A repository whose uploader wrote no README still gets a card: ModelScope
|
||||
answers with "the contributor provided no further description", the SDK
|
||||
and git download commands, and an invitation to complete the card. None
|
||||
of it describes the model, yet it was landing in both the LLM prompt and
|
||||
the stored description.
|
||||
|
||||
A notice that is a heading takes its whole section with it, so the
|
||||
download block goes too; a stand-alone notice line is dropped on its own.
|
||||
Content the author added later — under a heading of equal or higher
|
||||
level — is kept, so an improved card is not thrown away.
|
||||
"""
|
||||
|
||||
lines = text.split("\n")
|
||||
out: list[str] = []
|
||||
skip_until_level: int | None = None
|
||||
|
||||
for line in lines:
|
||||
level = _heading_level(line)
|
||||
|
||||
if any(marker in line for marker in _GENERATED_CARD_MARKERS):
|
||||
if level > 0:
|
||||
skip_until_level = level
|
||||
continue
|
||||
|
||||
if skip_until_level is not None:
|
||||
if level > 0 and level <= skip_until_level:
|
||||
skip_until_level = None
|
||||
else:
|
||||
continue
|
||||
|
||||
out.append(line)
|
||||
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> str:
|
||||
"""Clean a HF README for injection into an LLM metadata-extraction prompt.
|
||||
@@ -429,6 +512,8 @@ def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> s
|
||||
|
||||
* ``widget:`` YAML block (example prompts + output URLs)
|
||||
* ``<Gallery />`` tags and wrappers
|
||||
* Site-generated placeholder notices for a card the author never wrote
|
||||
(see :func:`_strip_generated_card_boilerplate`)
|
||||
* Fenced code blocks (Python / bash / bibtex / yaml)
|
||||
* Standalone ```` image lines and ``<img>`` tags
|
||||
* Training-parameter tables
|
||||
@@ -454,6 +539,7 @@ def clean_readme_for_llm(markdown_text: str | None, max_length: int = 6000) -> s
|
||||
# Order matters — broader strips first, then finer ones.
|
||||
text = _strip_gallery(text)
|
||||
text = _strip_widget_section(text)
|
||||
text = _strip_generated_card_boilerplate(text)
|
||||
text = _strip_fenced_code_blocks(text)
|
||||
text = _strip_standalone_images(text)
|
||||
text = _strip_training_tables(text)
|
||||
|
||||
+127
-14
@@ -82,6 +82,17 @@ CIVITAI_DOWNLOAD_URL_PREFIXES = (
|
||||
)
|
||||
|
||||
|
||||
def _is_no_uri_available_error(message: str) -> bool:
|
||||
"""Return True for aria2's "No URI available" transfer failure.
|
||||
|
||||
aria2 reports this when every URI for the transfer has become unusable.
|
||||
For CivitAI downloads this typically means the temporary signed URL
|
||||
expired mid-download; the transfer can be recovered by resolving a fresh
|
||||
signed URL and re-scheduling with ``continue=true``.
|
||||
"""
|
||||
return "no uri available" in message.lower()
|
||||
|
||||
|
||||
class Aria2Error(RuntimeError):
|
||||
"""Raised when aria2 integration fails."""
|
||||
|
||||
@@ -145,8 +156,16 @@ class Aria2Downloader:
|
||||
disappears (e.g. another download restarted the daemon and
|
||||
``close()`` cleared ``_transfers``) or the RPC becomes unreachable,
|
||||
the transfer is re-scheduled with ``continue=true`` so the download
|
||||
resumes from the on-disk ``.aria2`` control file. Recovery is bounded
|
||||
by ``MAX_TRANSFER_RECOVERY_ATTEMPTS``.
|
||||
resumes from the on-disk ``.aria2`` control file. The same
|
||||
re-scheduling happens when aria2 fails with "No URI available"
|
||||
(typically an expired CivitAI signed URL): a fresh URL is resolved
|
||||
and the partial download continues. Recovery is bounded by
|
||||
``MAX_TRANSFER_RECOVERY_ATTEMPTS``.
|
||||
|
||||
Cancellation never leaks daemon transfers: the gid is tracked in
|
||||
``_transfers`` before any post-``addUri`` await, and a gid accepted
|
||||
by the daemon while the caller is being cancelled is removed again
|
||||
before the ``CancelledError`` propagates.
|
||||
"""
|
||||
|
||||
await self._ensure_process()
|
||||
@@ -201,14 +220,47 @@ class Aria2Downloader:
|
||||
completed_path = self._resolve_completed_path(status, save_path)
|
||||
return True, completed_path
|
||||
if state == "error":
|
||||
return False, status.get("errorMessage") or "aria2 download failed"
|
||||
error_message = status.get("errorMessage") or "aria2 download failed"
|
||||
if (
|
||||
_is_no_uri_available_error(error_message)
|
||||
and recovery_attempts < MAX_TRANSFER_RECOVERY_ATTEMPTS
|
||||
):
|
||||
# The signed URL (e.g. CivitAI's) expired before the
|
||||
# transfer finished. Re-registering resolves a fresh
|
||||
# URL and resumes from the on-disk partial payload and
|
||||
# .aria2 control file via ``continue=true``.
|
||||
recovery_attempts += 1
|
||||
logger.warning(
|
||||
"aria2 transfer %s failed with %r; refreshing the "
|
||||
"URL and resuming the partial download "
|
||||
"(attempt %d/%d)",
|
||||
download_id,
|
||||
error_message,
|
||||
recovery_attempts,
|
||||
MAX_TRANSFER_RECOVERY_ATTEMPTS,
|
||||
)
|
||||
await asyncio.sleep(1.0)
|
||||
await self._ensure_process()
|
||||
async with self._register_lock:
|
||||
transfer = await self._register_transfer(
|
||||
url,
|
||||
save_path,
|
||||
download_id=download_id,
|
||||
headers=headers,
|
||||
)
|
||||
continue
|
||||
return False, error_message
|
||||
if state == "removed":
|
||||
return False, "Download was cancelled"
|
||||
|
||||
await asyncio.sleep(self._poll_interval)
|
||||
finally:
|
||||
current = self._transfers.get(download_id)
|
||||
if current is not None and current.gid == transfer.gid:
|
||||
if (
|
||||
transfer is not None
|
||||
and current is not None
|
||||
and current.gid == transfer.gid
|
||||
):
|
||||
self._transfers.pop(download_id, None)
|
||||
|
||||
async def _get_status_with_retry(
|
||||
@@ -296,21 +348,43 @@ class Aria2Downloader:
|
||||
resolved_url != url,
|
||||
)
|
||||
|
||||
# Shield the addUri RPC from cancellation: the daemon may accept the
|
||||
# download even when the caller is cancelled while the request is in
|
||||
# flight. On cancellation, wait for the RPC result so the freshly
|
||||
# created gid can be removed instead of leaking an untracked
|
||||
# download that keeps running in the daemon.
|
||||
add_task = asyncio.ensure_future(
|
||||
self._rpc_call("aria2.addUri", [[resolved_url], options])
|
||||
)
|
||||
try:
|
||||
gid = await self._rpc_call("aria2.addUri", [[resolved_url], options])
|
||||
gid = await asyncio.shield(add_task)
|
||||
except asyncio.CancelledError:
|
||||
leaked_gid: Any = None
|
||||
try:
|
||||
leaked_gid = await add_task
|
||||
except Exception:
|
||||
leaked_gid = None
|
||||
if isinstance(leaked_gid, str) and leaked_gid:
|
||||
logger.info(
|
||||
"Removing aria2 gid %s accepted while download %s was "
|
||||
"being cancelled",
|
||||
leaked_gid,
|
||||
download_id,
|
||||
)
|
||||
try:
|
||||
await self._rpc_call("aria2.forceRemove", [leaked_gid])
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to remove leaked aria2 gid %s for download %s: %s",
|
||||
leaked_gid,
|
||||
download_id,
|
||||
exc,
|
||||
)
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise Aria2Error(f"Failed to schedule aria2 download: {exc}") from exc
|
||||
|
||||
logger.debug("aria2 accepted download %s with gid %s", download_id, gid)
|
||||
await self._state_store.upsert(
|
||||
download_id,
|
||||
{
|
||||
"gid": gid,
|
||||
"save_path": save_path,
|
||||
"status": "downloading",
|
||||
"url": url,
|
||||
},
|
||||
)
|
||||
return gid
|
||||
|
||||
async def _register_transfer(
|
||||
@@ -329,7 +403,46 @@ class Aria2Downloader:
|
||||
headers=headers,
|
||||
)
|
||||
transfer = Aria2Transfer(gid=gid, save_path=os.path.abspath(save_path))
|
||||
# Register the transfer before any further await: once the daemon
|
||||
# holds the gid, cancel_download() must be able to find it. An await
|
||||
# in between would open a window where a concurrent cancel reports
|
||||
# "Download task not found" and the daemon keeps downloading
|
||||
# untracked.
|
||||
self._transfers[download_id] = transfer
|
||||
try:
|
||||
await self._state_store.upsert(
|
||||
download_id,
|
||||
{
|
||||
"gid": gid,
|
||||
"save_path": transfer.save_path,
|
||||
"status": "downloading",
|
||||
"url": url,
|
||||
},
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
# The task was cancelled while persisting state and the
|
||||
# coordinator's cancel ran before the transfer was registered
|
||||
# above. Remove the daemon transfer unless it was deliberately
|
||||
# paused (skip_download preserves paused transfers for resume).
|
||||
status = None
|
||||
try:
|
||||
status = await self.get_status(download_id)
|
||||
except Exception:
|
||||
status = None
|
||||
if status is not None and status.get("status") != "paused":
|
||||
try:
|
||||
await self._rpc_call("aria2.forceRemove", [gid])
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to remove aria2 gid %s for cancelled download %s: %s",
|
||||
gid,
|
||||
download_id,
|
||||
exc,
|
||||
)
|
||||
current = self._transfers.get(download_id)
|
||||
if current is not None and current.gid == gid:
|
||||
self._transfers.pop(download_id, None)
|
||||
raise
|
||||
return transfer
|
||||
|
||||
async def get_status(self, download_id: str) -> Optional[Dict[str, Any]]:
|
||||
|
||||
@@ -7,7 +7,7 @@ import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
from ..utils.constants import VALID_LORA_SUB_TYPES, VALID_CHECKPOINT_SUB_TYPES
|
||||
from ..utils.constants import VALID_LORA_SUB_TYPES, VALID_CHECKPOINT_SUB_TYPES, VALID_OTHER_SUB_TYPES
|
||||
from ..utils.models import BaseModelMetadata
|
||||
from ..utils.metadata_manager import MetadataManager
|
||||
from ..utils.usage_stats import UsageStats
|
||||
@@ -21,6 +21,7 @@ from .model_query import (
|
||||
resolve_sub_type,
|
||||
)
|
||||
from .settings_manager import get_settings_manager
|
||||
from .model_sources import source_group_key
|
||||
from ..utils.civitai_utils import build_civitai_model_page_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -742,29 +743,32 @@ class BaseModelService(ABC):
|
||||
@staticmethod
|
||||
def _extract_hf_group_key(item: Dict[str, Any]) -> Optional[str]:
|
||||
"""Extract `hf:{owner}/{repo}` from item's ``hf_url``, or None."""
|
||||
hf_url = item.get("hf_url") if isinstance(item, dict) else None
|
||||
if not hf_url or not isinstance(hf_url, str):
|
||||
return None
|
||||
m = re.match(
|
||||
r"https?://huggingface\.co/([^/]+/[^/]+)", hf_url.strip()
|
||||
)
|
||||
if not m:
|
||||
return None
|
||||
return f"hf:{m.group(1)}"
|
||||
key = BaseModelService._extract_source_group_key(item)
|
||||
return key if key and key.startswith("hf:") else None
|
||||
|
||||
@staticmethod
|
||||
def _extract_source_group_key(item: Dict[str, Any]) -> Optional[str]:
|
||||
"""Return the external-source group key for *item*, or None.
|
||||
|
||||
Hugging Face keeps the historical ``hf:{owner}/{repo}`` shape; other
|
||||
platforms use their own short prefix (``ms:`` / ``ta:``).
|
||||
"""
|
||||
return source_group_key(item)
|
||||
|
||||
@staticmethod
|
||||
def _extract_group_key(item: Dict[str, Any]) -> Union[int, str, None]:
|
||||
"""Return the group identity key: CivitAI modelId (int) or HF repo (str).
|
||||
"""Return the group identity key.
|
||||
|
||||
Preference order:
|
||||
1. CivitAI ``modelId`` (int)
|
||||
2. HF repo identity ``hf:{owner}/{repo}`` (str)
|
||||
2. External model source identity, e.g. ``hf:{owner}/{repo}``,
|
||||
``ms:{owner}/{repo}``, ``ta:{model_id}`` (str)
|
||||
3. ``None`` (no known grouping source)
|
||||
"""
|
||||
mid = BaseModelService._extract_model_id(item)
|
||||
if mid is not None:
|
||||
return mid
|
||||
return BaseModelService._extract_hf_group_key(item)
|
||||
return BaseModelService._extract_source_group_key(item)
|
||||
|
||||
@staticmethod
|
||||
def _extract_model_id(item: Dict[str, Any]) -> Optional[int]:
|
||||
@@ -904,6 +908,11 @@ class BaseModelService(ABC):
|
||||
and normalized_type not in VALID_CHECKPOINT_SUB_TYPES
|
||||
):
|
||||
continue
|
||||
if (
|
||||
self.model_type == "other"
|
||||
and normalized_type not in VALID_OTHER_SUB_TYPES
|
||||
):
|
||||
continue
|
||||
|
||||
type_counts[normalized_type] = type_counts.get(normalized_type, 0) + 1
|
||||
|
||||
@@ -1295,6 +1304,27 @@ class BaseModelService(ABC):
|
||||
path_for_sorting,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _relative_path_folder_group_sort_key(
|
||||
relative_path: str, include_terms: List[str]
|
||||
) -> tuple:
|
||||
"""Group paths by folder, then sort by relevance within each group.
|
||||
|
||||
Folders are ordered alphabetically (case-insensitive) by their full
|
||||
folder path, with root-level files (empty folder) first. Within a
|
||||
folder, paths keep the relevance ordering of
|
||||
``_relative_path_sort_key``. This keeps same-folder entries together
|
||||
in the autocomplete dropdown instead of interleaving them by filename.
|
||||
"""
|
||||
path_for_sorting = BaseModelService._remove_model_extension(
|
||||
relative_path.lower()
|
||||
)
|
||||
folder = path_for_sorting.rpartition(os.sep)[0]
|
||||
|
||||
return (folder,) + BaseModelService._relative_path_sort_key(
|
||||
relative_path, include_terms
|
||||
)
|
||||
|
||||
async def search_relative_paths(
|
||||
self,
|
||||
search_term: str,
|
||||
@@ -1404,9 +1434,13 @@ class BaseModelService(ABC):
|
||||
):
|
||||
matching_paths.append(relative_path)
|
||||
|
||||
# Sort by relevance (prefix and earliest hits first, then by length and alphabetically)
|
||||
# Group by folder (root first, then alphabetically) and sort by
|
||||
# relevance (prefix and earliest hits, then length and alphabetically)
|
||||
# within each folder group.
|
||||
matching_paths.sort(
|
||||
key=lambda relative: self._relative_path_sort_key(relative, include_terms)
|
||||
key=lambda relative: self._relative_path_folder_group_sort_key(
|
||||
relative, include_terms
|
||||
)
|
||||
)
|
||||
|
||||
# Apply offset and limit
|
||||
|
||||
@@ -20,6 +20,11 @@ from .recipes import (
|
||||
RecipeDownloadError,
|
||||
RecipeNotFoundError,
|
||||
)
|
||||
from .recipes.import_info import (
|
||||
CHANNEL_BATCH_IMPORT_LOCAL,
|
||||
CHANNEL_BATCH_IMPORT_URL,
|
||||
build_import_info,
|
||||
)
|
||||
|
||||
|
||||
class ImportItemType(Enum):
|
||||
@@ -624,6 +629,17 @@ class BatchImportService:
|
||||
"loras": loras,
|
||||
"gen_params": payload.get("gen_params", {}),
|
||||
"source_path": item.source,
|
||||
# Record why this import ended up with no LoRAs so the
|
||||
# recipe modal can explain it (collapsed by default).
|
||||
"import_info": build_import_info(
|
||||
(
|
||||
CHANNEL_BATCH_IMPORT_URL
|
||||
if item.item_type == ImportItemType.URL
|
||||
else CHANNEL_BATCH_IMPORT_LOCAL
|
||||
),
|
||||
payload.get("diagnostics"),
|
||||
loras,
|
||||
),
|
||||
}
|
||||
|
||||
if payload.get("checkpoint"):
|
||||
|
||||
@@ -410,6 +410,10 @@ class CheckpointScanner(ModelScanner):
|
||||
|
||||
return None
|
||||
|
||||
def resolve_sub_type_for_path(self, file_path: Optional[str]) -> Optional[str]:
|
||||
"""Resolve sub_type from the configured root that contains the file."""
|
||||
return self._resolve_sub_type(self._find_root_for_file(file_path))
|
||||
|
||||
def adjust_metadata(self, metadata, file_path, root_path):
|
||||
"""Adjust metadata during scanning to set sub_type."""
|
||||
sub_type = self._resolve_sub_type(root_path)
|
||||
@@ -419,9 +423,7 @@ class CheckpointScanner(ModelScanner):
|
||||
|
||||
def adjust_cached_entry(self, entry: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Adjust entries loaded from the persisted cache to ensure sub_type is set."""
|
||||
sub_type = self._resolve_sub_type(
|
||||
self._find_root_for_file(entry.get("file_path"))
|
||||
)
|
||||
sub_type = self.resolve_sub_type_for_path(entry.get("file_path"))
|
||||
if sub_type:
|
||||
entry["sub_type"] = sub_type
|
||||
return entry
|
||||
|
||||
@@ -67,6 +67,8 @@ class CheckpointService(BaseModelService):
|
||||
"civitai": self.filter_civitai_data(model_data.get("civitai", {}), minimal=True),
|
||||
"auto_tags": model_data.get("auto_tags") or extract_auto_tags(model_data),
|
||||
"version_count": model_data.get("version_count"),
|
||||
"source_platform": model_data.get("source_platform", ""),
|
||||
"source_url": model_data.get("source_url", ""),
|
||||
"hf_url": model_data.get("hf_url", ""),
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ from .model_metadata_provider import (
|
||||
from .downloader import get_downloader
|
||||
from .errors import RateLimitError, ResourceNotFoundError
|
||||
from ..utils.civitai_utils import resolve_license_payload
|
||||
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES
|
||||
from ..utils.constants import MODEL_WEIGHT_FILE_TYPES, is_empty_placeholder_hash
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -180,6 +180,11 @@ class CivitaiClient:
|
||||
async def get_model_by_hash(
|
||||
self, model_hash: str
|
||||
) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
if is_empty_placeholder_hash(model_hash):
|
||||
# The empty-hash placeholder (SHA256 of an empty byte string)
|
||||
# matches no real file; CivitAI's by-hash index can contain
|
||||
# polluted entries for it, so never resolve it.
|
||||
return None, "Model not found"
|
||||
try:
|
||||
success, version = await self._make_request(
|
||||
"GET",
|
||||
@@ -500,9 +505,55 @@ class CivitaiClient:
|
||||
logger.warning(f"Failed to fetch version by id {version_id}")
|
||||
return None
|
||||
|
||||
async def get_version_file_mini(
|
||||
self, version_id: int, file_id: int
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Fetch raw stored file info via the model-versions/mini endpoint.
|
||||
|
||||
The public REST API rewrites ``files[].name`` to
|
||||
``"{model}_{version}"`` for non-LoRA model types, so every
|
||||
precision variant of a multi-file version shares one name (#1100).
|
||||
The mini endpoint returns the raw ``ModelFile.name`` in
|
||||
``fileName``. ``file_id`` is mandatory: without it mini picks a
|
||||
file via its own primary-file logic, which can disagree with the
|
||||
REST ``primary`` flag.
|
||||
|
||||
Returns the mini payload dict on success, None on any failure.
|
||||
"""
|
||||
try:
|
||||
success, data = await self._make_request(
|
||||
"GET",
|
||||
f"{self.base_url}/model-versions/mini/{version_id}",
|
||||
params={"modelFileId": file_id},
|
||||
use_auth=True,
|
||||
)
|
||||
if success and isinstance(data, dict):
|
||||
return data
|
||||
if is_expected_offline_error(data):
|
||||
return None
|
||||
logger.debug(
|
||||
"Mini endpoint lookup failed for version %s file %s: %s",
|
||||
version_id,
|
||||
file_id,
|
||||
data,
|
||||
)
|
||||
return None
|
||||
except RateLimitError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Error fetching mini info for version %s file %s: %s",
|
||||
version_id,
|
||||
file_id,
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
|
||||
async def _fetch_version_by_hash(self, model_hash: Optional[str]) -> Optional[Dict[str, Any]]:
|
||||
if not model_hash:
|
||||
return None
|
||||
if is_empty_placeholder_hash(model_hash):
|
||||
return None
|
||||
|
||||
success, version = await self._make_request(
|
||||
"GET",
|
||||
|
||||
+327
-41
@@ -17,27 +17,35 @@ from dataclasses import dataclass, field
|
||||
import uuid
|
||||
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, cast
|
||||
from urllib.parse import urlparse
|
||||
from ..utils.models import LoraMetadata, CheckpointMetadata, EmbeddingMetadata
|
||||
from ..utils.models import (
|
||||
LoraMetadata,
|
||||
CheckpointMetadata,
|
||||
EmbeddingMetadata,
|
||||
OtherModelMetadata,
|
||||
)
|
||||
from ..utils.constants import (
|
||||
CARD_PREVIEW_WIDTH,
|
||||
DIFFUSION_MODEL_BASE_MODELS,
|
||||
MODEL_WEIGHT_FILE_TYPES,
|
||||
SUPPORTED_DOWNLOAD_SKIP_BASE_MODELS,
|
||||
VALID_LORA_TYPES,
|
||||
VALID_OTHER_CIVITAI_TYPES,
|
||||
)
|
||||
from ..utils.civitai_utils import normalize_civitai_download_url, rewrite_preview_url
|
||||
from ..utils.file_utils import calculate_sha256, calculate_autov3
|
||||
from ..utils.preview_selection import resolve_mature_threshold, select_preview_media
|
||||
from ..utils.utils import sanitize_folder_name
|
||||
from ..utils.utils import calculate_filename_for_model, sanitize_folder_name
|
||||
from ..utils.exif_utils import ExifUtils
|
||||
from ..utils.metadata_manager import MetadataManager
|
||||
from .service_registry import ServiceRegistry
|
||||
from .download_routing import is_diffusion_model_download, resolve_other_download_sub_type
|
||||
from .settings_manager import get_settings_manager
|
||||
from .metadata_service import get_default_metadata_provider, get_metadata_provider
|
||||
from .downloader import get_downloader, DownloadProgress, DownloadStreamControl
|
||||
from .errors import RateLimitError
|
||||
from .aria2_downloader import Aria2Error, get_aria2_downloader
|
||||
from .aria2_transfer_state import Aria2TransferStateStore
|
||||
from .download_queue_service import DownloadQueueService
|
||||
from .model_lifecycle_service import ModelLifecycleService, load_local_metadata
|
||||
|
||||
# Download to temporary file first
|
||||
import tempfile
|
||||
@@ -227,12 +235,21 @@ class DownloadManager:
|
||||
return False
|
||||
|
||||
async def _get_scanner_for_model_type(self, model_type: str):
|
||||
"""Return the scanner responsible for the given model type."""
|
||||
"""Return the scanner responsible for the given model type.
|
||||
|
||||
Every supported type resolves explicitly — an unknown type must never
|
||||
fall through to the lora scanner (an "other" download would silently
|
||||
dedupe against the lora library).
|
||||
"""
|
||||
if model_type == "checkpoint":
|
||||
return await self._get_checkpoint_scanner()
|
||||
if model_type == "embedding":
|
||||
return await ServiceRegistry.get_embedding_scanner()
|
||||
return await self._get_lora_scanner()
|
||||
if model_type == "other":
|
||||
return await ServiceRegistry.get_other_scanner()
|
||||
if model_type == "lora":
|
||||
return await self._get_lora_scanner()
|
||||
raise ValueError(f'Unknown model type "{model_type}"')
|
||||
|
||||
@staticmethod
|
||||
def _resolve_target_file(
|
||||
@@ -717,6 +734,47 @@ class DownloadManager:
|
||||
await asyncio.sleep(delay)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _reconcile_failed_aria2_partial(save_path: str) -> None:
|
||||
"""Reconcile on-disk partial state after a failed aria2 transfer.
|
||||
|
||||
The payload and its ``.aria2`` control file form a resumable pair and
|
||||
are preserved together so a retry (with a refreshed URL when needed)
|
||||
can resume via aria2's ``continue=true``. A control file without its
|
||||
payload cannot resume anything, so the orphan is reported and removed.
|
||||
"""
|
||||
control_path = f"{save_path}.aria2"
|
||||
payload_exists = os.path.exists(save_path)
|
||||
control_exists = os.path.exists(control_path)
|
||||
|
||||
if payload_exists and not control_exists:
|
||||
# If the .aria2 control file is missing, aria2 considers the
|
||||
# download complete. A transient RPC failure may have made us
|
||||
# think the download failed even though the file is fully on disk.
|
||||
# Keep the file so a retry can find it already complete.
|
||||
logger.warning(
|
||||
"aria2 download reported failure but .aria2 file is absent "
|
||||
"for %s — the file is likely complete. Preserving it for retry.",
|
||||
save_path,
|
||||
)
|
||||
elif payload_exists and control_exists:
|
||||
logger.info(
|
||||
"Preserving aria2 partial download for resume: %s", save_path
|
||||
)
|
||||
elif control_exists:
|
||||
logger.warning(
|
||||
"Orphaned aria2 control file without payload: %s — removing it",
|
||||
control_path,
|
||||
)
|
||||
try:
|
||||
os.remove(control_path)
|
||||
except OSError as exc:
|
||||
logger.warning(
|
||||
"Failed to remove orphaned aria2 control file %s: %s",
|
||||
control_path,
|
||||
exc,
|
||||
)
|
||||
|
||||
async def _cleanup_cancelled_download_files(
|
||||
self,
|
||||
download_id: str,
|
||||
@@ -888,6 +946,42 @@ class DownloadManager:
|
||||
|
||||
return download_urls
|
||||
|
||||
async def _fetch_raw_file_name(
|
||||
self,
|
||||
metadata_provider,
|
||||
version_id: Optional[int],
|
||||
file_id: Any,
|
||||
) -> Optional[str]:
|
||||
"""Best-effort lookup of the raw stored filename via the CivitAI
|
||||
model-versions/mini endpoint (#1100). Returns None on any failure so
|
||||
the caller can fall back to the (possibly rewritten) REST name."""
|
||||
if version_id is None or file_id is None:
|
||||
return None
|
||||
fetch = getattr(metadata_provider, "get_version_file_mini", None)
|
||||
if fetch is None:
|
||||
return None
|
||||
try:
|
||||
mini_info = await fetch(int(version_id), int(file_id))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
except RateLimitError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Mini endpoint lookup failed for version %s file %s: %s",
|
||||
version_id,
|
||||
file_id,
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
if not isinstance(mini_info, dict):
|
||||
return None
|
||||
raw_name = mini_info.get("fileName")
|
||||
if not isinstance(raw_name, str) or not raw_name.strip():
|
||||
return None
|
||||
# Defensive: never let a path component slip into the filename.
|
||||
return os.path.basename(raw_name.strip()) or None
|
||||
|
||||
def _build_metadata_for_resume(
|
||||
self,
|
||||
*,
|
||||
@@ -900,6 +994,8 @@ class DownloadManager:
|
||||
return CheckpointMetadata.from_civitai_info(version_info, file_info, save_path)
|
||||
if model_type == "embedding":
|
||||
return EmbeddingMetadata.from_civitai_info(version_info, file_info, save_path)
|
||||
if model_type == "other":
|
||||
return OtherModelMetadata.from_civitai_info(version_info, file_info, save_path)
|
||||
return LoraMetadata.from_civitai_info(version_info, file_info, save_path)
|
||||
|
||||
def _resolve_save_path_from_persisted_record(self, record: Dict[str, Any]) -> Optional[str]:
|
||||
@@ -1226,6 +1322,24 @@ class DownloadManager:
|
||||
)
|
||||
continue
|
||||
|
||||
if not os.path.exists(save_path) and os.path.exists(control_path):
|
||||
# A control file without its payload cannot resume
|
||||
# anything; report it and clean up the orphan.
|
||||
logger.warning(
|
||||
"Orphaned aria2 control file without payload for %s: "
|
||||
"%s — removing it",
|
||||
download_id,
|
||||
control_path,
|
||||
)
|
||||
try:
|
||||
os.remove(control_path)
|
||||
except OSError as exc:
|
||||
logger.warning(
|
||||
"Failed to remove orphaned aria2 control file %s: %s",
|
||||
control_path,
|
||||
exc,
|
||||
)
|
||||
|
||||
await self._aria2_state_store.remove(download_id)
|
||||
|
||||
self._restored_persisted_downloads = True
|
||||
@@ -1342,6 +1456,7 @@ class DownloadManager:
|
||||
lora_scanner = await self._get_lora_scanner()
|
||||
checkpoint_scanner = await self._get_checkpoint_scanner()
|
||||
embedding_scanner = await ServiceRegistry.get_embedding_scanner()
|
||||
other_scanner = await ServiceRegistry.get_other_scanner()
|
||||
|
||||
# Check lora scanner first
|
||||
if await lora_scanner.check_model_version_exists(model_version_id):
|
||||
@@ -1366,6 +1481,13 @@ class DownloadManager:
|
||||
"error": "Model version already exists in embedding library",
|
||||
}
|
||||
|
||||
# Check other scanner
|
||||
if await other_scanner.check_model_version_exists(model_version_id):
|
||||
return {
|
||||
"success": False,
|
||||
"error": "Model version already exists in other library",
|
||||
}
|
||||
|
||||
# Use CivArchive provider directly when source is 'civarchive'
|
||||
# This prioritizes CivArchive metadata (with mirror availability info) over Civitai
|
||||
if source == "civarchive":
|
||||
@@ -1404,6 +1526,20 @@ class DownloadManager:
|
||||
model_type = "lora"
|
||||
elif model_type_from_info == "textualinversion":
|
||||
model_type = "embedding"
|
||||
elif model_type_from_info in VALID_OTHER_CIVITAI_TYPES:
|
||||
if not get_settings_manager().is_other_models_enabled():
|
||||
return {
|
||||
"success": False,
|
||||
"error": (
|
||||
"Other Models management is disabled. Enable it in "
|
||||
"Settings > Library before downloading VAE, upscaler, "
|
||||
"text encoder or CLIP files."
|
||||
),
|
||||
# Machine-readable failure code consumed by the companion
|
||||
# browser extension (docs/other-models-support.md C4).
|
||||
"reason": "other_models_disabled",
|
||||
}
|
||||
model_type = "other"
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
@@ -1525,27 +1661,13 @@ class DownloadManager:
|
||||
}
|
||||
|
||||
# Check if this checkpoint should be treated as a diffusion model
|
||||
# Priority: (1) any file has type "UNet" or "Diffusion Model",
|
||||
# (2) baseModel is in DIFFUSION_MODEL_BASE_MODELS
|
||||
is_diffusion_model = False
|
||||
if model_type == "checkpoint":
|
||||
# Check file types first (more direct signal from CivitAI)
|
||||
version_files = version_info.get("files", [])
|
||||
for f in version_files:
|
||||
f_type = f.get("type", "")
|
||||
if f_type in ("UNet", "Diffusion Model"):
|
||||
is_diffusion_model = True
|
||||
logger.info(
|
||||
f"File type '{f_type}' detected, routing checkpoint to unet folder"
|
||||
)
|
||||
break
|
||||
|
||||
# Fallback to baseModel name check
|
||||
if not is_diffusion_model and base_model_value in DIFFUSION_MODEL_BASE_MODELS:
|
||||
is_diffusion_model = True
|
||||
logger.info(
|
||||
f"baseModel '{base_model_value}' is a known diffusion model, routing to unet folder"
|
||||
)
|
||||
# (shared with the download routing endpoint so the UI location
|
||||
# step and the actual download agree on the target roots).
|
||||
is_diffusion_model = is_diffusion_model_download(
|
||||
model_type,
|
||||
file_types=(f.get("type", "") for f in version_info.get("files", [])),
|
||||
base_model=base_model_value,
|
||||
)
|
||||
|
||||
# Existence check after the metadata fetch (#1058):
|
||||
# - An explicit file selection only blocks when THIS file is
|
||||
@@ -1604,6 +1726,13 @@ class DownloadManager:
|
||||
"success": False,
|
||||
"error": "Model version already exists in embedding library",
|
||||
}
|
||||
elif model_type == "other":
|
||||
other_scanner = await ServiceRegistry.get_other_scanner()
|
||||
if await other_scanner.check_model_version_exists(version_id):
|
||||
return {
|
||||
"success": False,
|
||||
"error": "Model version already exists in other library",
|
||||
}
|
||||
|
||||
# Handle use_default_paths
|
||||
if use_default_paths:
|
||||
@@ -1643,6 +1772,60 @@ class DownloadManager:
|
||||
"error": "Default embedding root path not set in settings",
|
||||
}
|
||||
save_dir = default_path
|
||||
elif model_type == "other":
|
||||
other_sub_type = resolve_other_download_sub_type(
|
||||
model_type_from_info,
|
||||
file_types=(
|
||||
f.get("type", "")
|
||||
for f in version_info.get("files", [])
|
||||
if isinstance(f, dict)
|
||||
),
|
||||
selected_file_type=(
|
||||
target_file.get("type") if explicit_file else None
|
||||
),
|
||||
)
|
||||
default_other_roots = (
|
||||
settings_manager.get("default_other_roots") or {}
|
||||
)
|
||||
if other_sub_type and not settings_manager.is_other_sub_type_enabled(
|
||||
other_sub_type
|
||||
):
|
||||
return {
|
||||
"success": False,
|
||||
"error": (
|
||||
f"Other-model sub-type '{other_sub_type}' is "
|
||||
f"disabled in settings. Please pick a destination "
|
||||
f"folder explicitly instead of using default paths."
|
||||
),
|
||||
"reason": "other_sub_type_disabled",
|
||||
}
|
||||
default_path = (
|
||||
default_other_roots.get(other_sub_type)
|
||||
if other_sub_type
|
||||
else None
|
||||
)
|
||||
if not isinstance(default_path, str) or not default_path:
|
||||
if other_sub_type:
|
||||
detail = (
|
||||
f"No default root configured for other-model "
|
||||
f"sub-type '{other_sub_type}'"
|
||||
)
|
||||
reason = "other_no_default_root"
|
||||
else:
|
||||
detail = (
|
||||
"Could not determine the other-model sub-type "
|
||||
"from the model metadata"
|
||||
)
|
||||
reason = "other_sub_type_undecidable"
|
||||
return {
|
||||
"success": False,
|
||||
"error": (
|
||||
f"{detail}. Please pick a destination folder "
|
||||
f"explicitly instead of using default paths."
|
||||
),
|
||||
"reason": reason,
|
||||
}
|
||||
save_dir = default_path
|
||||
|
||||
# Calculate relative path using template
|
||||
relative_path = self._calculate_relative_path(version_info, model_type)
|
||||
@@ -1799,6 +1982,24 @@ class DownloadManager:
|
||||
if not download_urls:
|
||||
return {"success": False, "error": "No mirror URL found"}
|
||||
|
||||
# The public REST API rewrites files[].name to
|
||||
# "{model}_{version}" for non-LoRA model types, so every
|
||||
# precision variant of a multi-file version shares one name and
|
||||
# lands on disk with a random short-hash suffix. The mini
|
||||
# endpoint returns the raw stored filename (#1100). CivArchive
|
||||
# already serves raw names.
|
||||
if source != "civarchive":
|
||||
raw_file_name = await self._fetch_raw_file_name(
|
||||
metadata_provider, resolved_version_id, file_info.get("id")
|
||||
)
|
||||
if raw_file_name and raw_file_name != file_info.get("name"):
|
||||
logger.info(
|
||||
"[download] Using raw stored filename '%s' instead of REST name '%s'",
|
||||
raw_file_name,
|
||||
file_info.get("name"),
|
||||
)
|
||||
file_info = {**file_info, "name": raw_file_name}
|
||||
|
||||
# 3. Prepare download
|
||||
file_name = file_info.get("name", "")
|
||||
if not file_name:
|
||||
@@ -1821,6 +2022,11 @@ class DownloadManager:
|
||||
version_info, file_info, save_path
|
||||
)
|
||||
logger.info(f"Creating EmbeddingMetadata for {file_name}")
|
||||
elif model_type == "other":
|
||||
metadata = OtherModelMetadata.from_civitai_info(
|
||||
version_info, file_info, save_path
|
||||
)
|
||||
logger.info(f"Creating OtherModelMetadata for {file_name}")
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
@@ -2033,6 +2239,8 @@ class DownloadManager:
|
||||
scanner = await self._get_checkpoint_scanner()
|
||||
elif model_type == "embedding":
|
||||
scanner = await ServiceRegistry.get_embedding_scanner()
|
||||
elif model_type == "other":
|
||||
scanner = await ServiceRegistry.get_other_scanner()
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to acquire scanner for %s models: %s", model_type, exc)
|
||||
|
||||
@@ -2423,20 +2631,8 @@ class DownloadManager:
|
||||
break
|
||||
|
||||
last_error = result
|
||||
# For aria2: if the .aria2 control file is missing, aria2 considers
|
||||
# the download complete. A transient RPC failure may have made us
|
||||
# think the download failed even though the file is fully on disk.
|
||||
# Keep the file so a retry can find it already complete.
|
||||
if (
|
||||
transfer_backend == "aria2"
|
||||
and os.path.exists(save_path)
|
||||
and not os.path.exists(f"{save_path}.aria2")
|
||||
):
|
||||
logger.warning(
|
||||
"aria2 download reported failure but .aria2 file is absent "
|
||||
"for %s — the file is likely complete. Preserving it for retry.",
|
||||
save_path,
|
||||
)
|
||||
if transfer_backend == "aria2":
|
||||
self._reconcile_failed_aria2_partial(save_path)
|
||||
elif os.path.exists(save_path):
|
||||
try:
|
||||
os.remove(save_path)
|
||||
@@ -2541,6 +2737,9 @@ class DownloadManager:
|
||||
elif model_type == "embedding":
|
||||
scanner = await ServiceRegistry.get_embedding_scanner()
|
||||
logger.info(f"Updating embedding cache for {actual_file_paths[0]}")
|
||||
elif model_type == "other":
|
||||
scanner = await ServiceRegistry.get_other_scanner()
|
||||
logger.info(f"Updating other-model cache for {actual_file_paths[0]}")
|
||||
|
||||
adjust_cached_entry = (
|
||||
getattr(scanner, "adjust_cached_entry", None)
|
||||
@@ -2548,6 +2747,7 @@ class DownloadManager:
|
||||
else None
|
||||
)
|
||||
|
||||
downloaded_metadata: List[Dict[str, Any]] = []
|
||||
for index, entry in enumerate(metadata_entries):
|
||||
file_path_for_adjust = getattr(
|
||||
entry, "file_path", actual_file_paths[index]
|
||||
@@ -2590,6 +2790,15 @@ class DownloadManager:
|
||||
if scanner is not None:
|
||||
await scanner.add_model_to_cache(metadata_dict, relative_path)
|
||||
|
||||
downloaded_metadata.append(metadata_dict)
|
||||
|
||||
await self._apply_download_filename_template(
|
||||
scanner=scanner,
|
||||
model_type=model_type,
|
||||
downloaded_metadata=downloaded_metadata,
|
||||
download_id=download_id,
|
||||
)
|
||||
|
||||
if transfer_backend == "aria2" and download_id:
|
||||
await self._aria2_state_store.remove(download_id)
|
||||
|
||||
@@ -2629,8 +2838,85 @@ class DownloadManager:
|
||||
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
async def _apply_download_filename_template(
|
||||
self,
|
||||
*,
|
||||
scanner,
|
||||
model_type: str,
|
||||
downloaded_metadata: List[Dict[str, Any]],
|
||||
download_id: Optional[str],
|
||||
) -> None:
|
||||
"""Rename freshly downloaded models according to the filename template.
|
||||
|
||||
Best-effort post-download step: any failure (including name conflicts)
|
||||
is logged and skipped so a successful download is never turned into a
|
||||
failure by a rename problem.
|
||||
"""
|
||||
try:
|
||||
if scanner is None or not downloaded_metadata:
|
||||
return
|
||||
|
||||
template = get_settings_manager().get_download_filename_template(
|
||||
model_type
|
||||
)
|
||||
if not template:
|
||||
return
|
||||
|
||||
lifecycle_service = ModelLifecycleService(
|
||||
scanner=scanner,
|
||||
metadata_manager=MetadataManager,
|
||||
metadata_loader=load_local_metadata,
|
||||
recipe_scanner_factory=ServiceRegistry.get_recipe_scanner,
|
||||
)
|
||||
|
||||
for metadata_dict in downloaded_metadata:
|
||||
file_path = metadata_dict.get("file_path")
|
||||
if not isinstance(file_path, str) or not file_path:
|
||||
continue
|
||||
|
||||
new_stem = calculate_filename_for_model(metadata_dict, model_type)
|
||||
if not new_stem:
|
||||
continue
|
||||
|
||||
current_stem = os.path.splitext(os.path.basename(file_path))[0]
|
||||
if new_stem == current_stem or os.path.normcase(
|
||||
new_stem
|
||||
) == os.path.normcase(current_stem):
|
||||
continue
|
||||
|
||||
try:
|
||||
result = await lifecycle_service.rename_model(
|
||||
file_path=file_path, new_file_name=new_stem
|
||||
)
|
||||
except ValueError as exc:
|
||||
logger.warning(
|
||||
"Keeping original filename for %s: %s", file_path, exc
|
||||
)
|
||||
continue
|
||||
|
||||
new_file_path = result.get("new_file_path")
|
||||
if download_id and isinstance(new_file_path, str):
|
||||
info = self._active_downloads.get(download_id)
|
||||
if info is None:
|
||||
continue
|
||||
if info.get("file_path") == file_path:
|
||||
info["file_path"] = new_file_path
|
||||
extracted = info.get("extracted_paths")
|
||||
if isinstance(extracted, list):
|
||||
info["extracted_paths"] = [
|
||||
new_file_path if path == file_path else path
|
||||
for path in extracted
|
||||
]
|
||||
except Exception as exc: # Rename phase must never fail the download
|
||||
logger.warning(
|
||||
"Filename template rename failed for %s download: %s",
|
||||
model_type,
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _get_supported_extensions_for_type(self, model_type: str) -> Set[str]:
|
||||
if model_type == "checkpoint":
|
||||
if model_type in ("checkpoint", "other"):
|
||||
return {
|
||||
".ckpt",
|
||||
".pt",
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Shared download routing logic.
|
||||
|
||||
Decides whether a download initiated from the checkpoint library should be
|
||||
routed to the unet/diffusion-model roots instead of the checkpoint roots.
|
||||
Used by both the download manager (at download time) and the download
|
||||
routing HTTP endpoint (when the user picks a location in the UI), so the
|
||||
two can never disagree.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Iterable, Optional
|
||||
|
||||
from ..utils.constants import (
|
||||
CIVITAI_FILE_TYPE_TO_OTHER_SUB_TYPE,
|
||||
CIVITAI_TYPE_TO_OTHER_SUB_TYPE,
|
||||
DIFFUSION_MODEL_BASE_MODELS,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# File types reported by the CivitAI API that indicate a raw diffusion
|
||||
# model (loaded via UNETLoader in ComfyUI) rather than a full checkpoint.
|
||||
DIFFUSION_FILE_TYPES = frozenset({"UNet", "Diffusion Model"})
|
||||
|
||||
|
||||
def is_diffusion_model_download(
|
||||
model_type: str,
|
||||
file_types: Iterable[str] = (),
|
||||
base_model: str = "",
|
||||
) -> bool:
|
||||
"""Return True when a download should be routed to the unet roots.
|
||||
|
||||
Only applies to downloads initiated from the checkpoint library.
|
||||
Priority: (1) any file has type "UNet" or "Diffusion Model" (the more
|
||||
direct signal from CivitAI), (2) baseModel is a known diffusion model.
|
||||
"""
|
||||
if model_type != "checkpoint":
|
||||
return False
|
||||
|
||||
for file_type in file_types:
|
||||
if file_type in DIFFUSION_FILE_TYPES:
|
||||
logger.info(
|
||||
"File type '%s' detected, routing checkpoint to unet folder",
|
||||
file_type,
|
||||
)
|
||||
return True
|
||||
|
||||
if base_model in DIFFUSION_MODEL_BASE_MODELS:
|
||||
logger.info(
|
||||
"baseModel '%s' is a known diffusion model, routing to unet folder",
|
||||
base_model,
|
||||
)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def resolve_other_download_sub_type(
|
||||
civitai_model_type: str,
|
||||
file_types: Iterable[str] = (),
|
||||
selected_file_type: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
"""Resolve the "other"-page sub_type for a download.
|
||||
|
||||
Fixed priority (locked design, docs/plans/other-models-page.md §9.2):
|
||||
|
||||
1. Explicit user file pick — when the picked file's type maps, it wins
|
||||
even when model.type maps to something else.
|
||||
2. model.type via CIVITAI_TYPE_TO_OTHER_SUB_TYPE.
|
||||
3. file.type fallback — only when model.type maps to nothing. Must NOT
|
||||
override a mapped model.type: checkpoint models routinely bundle
|
||||
VAE/Text Encoder component files.
|
||||
4. Still undecidable -> None (caller must ask the user for a folder).
|
||||
"""
|
||||
if selected_file_type:
|
||||
mapped = CIVITAI_FILE_TYPE_TO_OTHER_SUB_TYPE.get(selected_file_type)
|
||||
if mapped:
|
||||
logger.info(
|
||||
"Explicit file pick type '%s' routes other download to '%s'",
|
||||
selected_file_type,
|
||||
mapped,
|
||||
)
|
||||
return mapped
|
||||
|
||||
normalized_model_type = (civitai_model_type or "").strip().lower()
|
||||
mapped = CIVITAI_TYPE_TO_OTHER_SUB_TYPE.get(normalized_model_type)
|
||||
if mapped:
|
||||
return mapped
|
||||
|
||||
for file_type in file_types:
|
||||
mapped = CIVITAI_FILE_TYPE_TO_OTHER_SUB_TYPE.get(file_type)
|
||||
if mapped:
|
||||
logger.info(
|
||||
"model.type '%s' unmapped; file type '%s' routes other download to '%s'",
|
||||
civitai_model_type,
|
||||
file_type,
|
||||
mapped,
|
||||
)
|
||||
return mapped
|
||||
|
||||
return None
|
||||
@@ -67,6 +67,8 @@ class EmbeddingService(BaseModelService):
|
||||
"civitai": self.filter_civitai_data(model_data.get("civitai", {}), minimal=True),
|
||||
"auto_tags": model_data.get("auto_tags") or extract_auto_tags(model_data),
|
||||
"version_count": model_data.get("version_count"),
|
||||
"source_platform": model_data.get("source_platform", ""),
|
||||
"source_url": model_data.get("source_url", ""),
|
||||
"hf_url": model_data.get("hf_url", ""),
|
||||
}
|
||||
|
||||
|
||||
+150
-83
@@ -11,6 +11,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import aiohttp
|
||||
@@ -32,8 +33,26 @@ _catalog_cache: Optional[Dict[str, List[str]]] = None
|
||||
# ``{provider_id: {model_id: max_output_tokens}}``.
|
||||
_model_output_limits: Dict[str, Dict[str, int]] = {}
|
||||
|
||||
# Monotonic timestamp of the last failed catalog fetch (None = no failure
|
||||
# yet). Failed fetches are negatively cached: further calls return the
|
||||
# empty fallback without hitting the network until the cooldown elapses,
|
||||
# so users on broken networks don't stall on every settings-modal open.
|
||||
_catalog_last_failure: Optional[float] = None
|
||||
_CATALOG_FAILURE_COOLDOWN = 600.0 # seconds
|
||||
|
||||
# Serializes catalog fetches so concurrent callers don't duplicate requests.
|
||||
_catalog_lock = asyncio.Lock()
|
||||
|
||||
_CATALOG_TIMEOUT = aiohttp.ClientTimeout(total=30)
|
||||
|
||||
# Cloudflare serves brotli when the client advertises it, and brotli is a
|
||||
# required dependency here — a corrupted br stream can crash the native
|
||||
# decoder with a Windows access violation (issue #1099). Request gzip
|
||||
# instead; zlib decompression is not affected and corrupt gzip data only
|
||||
# raises ContentEncodingError (an aiohttp.ClientError subclass), which the
|
||||
# exception handlers below already catch.
|
||||
_NO_BROTLI_HEADERS = {"Accept-Encoding": "gzip, deflate"}
|
||||
|
||||
|
||||
async def _load_model_catalog() -> Dict[str, List[str]]:
|
||||
"""Fetch and parse the model catalog.
|
||||
@@ -46,61 +65,85 @@ async def _load_model_catalog() -> Dict[str, List[str]]:
|
||||
value has a ``models`` sub-dict keyed by model ID. The result is cached
|
||||
in memory after the first successful fetch.
|
||||
Subsequent calls return the cached data immediately.
|
||||
|
||||
Failed fetches are negatively cached: further calls return an empty
|
||||
dict without hitting the network until ``_CATALOG_FAILURE_COOLDOWN``
|
||||
has elapsed, so a broken network does not stall every settings-modal
|
||||
open. Concurrent callers are serialized behind :data:`_catalog_lock`
|
||||
so only one request is ever in flight.
|
||||
"""
|
||||
global _catalog_cache, _model_output_limits
|
||||
global _catalog_cache, _model_output_limits, _catalog_last_failure
|
||||
if _catalog_cache is not None:
|
||||
return _catalog_cache
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession(timeout=_CATALOG_TIMEOUT) as session:
|
||||
async with session.get(_MODEL_CATALOG_URL) as resp:
|
||||
if resp.status != 200:
|
||||
logger.warning("Model catalog returned HTTP %s", resp.status)
|
||||
return _catalog_cache or {}
|
||||
data = await resp.json()
|
||||
except (aiohttp.ClientError, asyncio.TimeoutError, json.JSONDecodeError) as exc:
|
||||
logger.warning("Failed to fetch model catalog: %s", exc)
|
||||
return _catalog_cache or {}
|
||||
async with _catalog_lock:
|
||||
# Re-check under the lock: another caller may have fetched (or
|
||||
# failed) while we were waiting.
|
||||
if _catalog_cache is not None:
|
||||
return _catalog_cache
|
||||
if (
|
||||
_catalog_last_failure is not None
|
||||
and time.monotonic() - _catalog_last_failure < _CATALOG_FAILURE_COOLDOWN
|
||||
):
|
||||
logger.debug(
|
||||
"Skipping model catalog fetch: last attempt failed %.0fs ago",
|
||||
time.monotonic() - _catalog_last_failure,
|
||||
)
|
||||
return {}
|
||||
|
||||
if not isinstance(data, dict):
|
||||
logger.warning("Model catalog is not a dict, got %s", type(data).__name__)
|
||||
return _catalog_cache or {}
|
||||
try:
|
||||
async with aiohttp.ClientSession(timeout=_CATALOG_TIMEOUT) as session:
|
||||
async with session.get(_MODEL_CATALOG_URL, headers=_NO_BROTLI_HEADERS) as resp:
|
||||
if resp.status != 200:
|
||||
logger.warning("Model catalog returned HTTP %s", resp.status)
|
||||
_catalog_last_failure = time.monotonic()
|
||||
return {}
|
||||
data = await resp.json()
|
||||
except (aiohttp.ClientError, asyncio.TimeoutError, json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
logger.warning("Failed to fetch model catalog: %s", exc)
|
||||
_catalog_last_failure = time.monotonic()
|
||||
return {}
|
||||
|
||||
result: Dict[str, List[str]] = {}
|
||||
output_limits: Dict[str, Dict[str, int]] = {}
|
||||
for provider_id, provider_info in data.items():
|
||||
if not isinstance(provider_info, dict):
|
||||
continue
|
||||
models_dict = provider_info.get("models")
|
||||
if not isinstance(models_dict, dict):
|
||||
continue
|
||||
model_ids: List[str] = []
|
||||
provider_limits: Dict[str, int] = {}
|
||||
for mid, model_info in models_dict.items():
|
||||
if not isinstance(mid, str):
|
||||
if not isinstance(data, dict):
|
||||
logger.warning("Model catalog is not a dict, got %s", type(data).__name__)
|
||||
_catalog_last_failure = time.monotonic()
|
||||
return {}
|
||||
|
||||
result: Dict[str, List[str]] = {}
|
||||
output_limits: Dict[str, Dict[str, int]] = {}
|
||||
for provider_id, provider_info in data.items():
|
||||
if not isinstance(provider_info, dict):
|
||||
continue
|
||||
model_ids.append(mid)
|
||||
if isinstance(model_info, dict):
|
||||
limit = model_info.get("limit")
|
||||
if isinstance(limit, dict):
|
||||
output = limit.get("output")
|
||||
if isinstance(output, (int, float)) and output > 0:
|
||||
provider_limits[mid] = int(output)
|
||||
if model_ids:
|
||||
result[provider_id] = model_ids
|
||||
if provider_limits:
|
||||
output_limits[provider_id] = provider_limits
|
||||
models_dict = provider_info.get("models")
|
||||
if not isinstance(models_dict, dict):
|
||||
continue
|
||||
model_ids: List[str] = []
|
||||
provider_limits: Dict[str, int] = {}
|
||||
for mid, model_info in models_dict.items():
|
||||
if not isinstance(mid, str):
|
||||
continue
|
||||
model_ids.append(mid)
|
||||
if isinstance(model_info, dict):
|
||||
limit = model_info.get("limit")
|
||||
if isinstance(limit, dict):
|
||||
output = limit.get("output")
|
||||
if isinstance(output, (int, float)) and output > 0:
|
||||
provider_limits[mid] = int(output)
|
||||
if model_ids:
|
||||
result[provider_id] = model_ids
|
||||
if provider_limits:
|
||||
output_limits[provider_id] = provider_limits
|
||||
|
||||
_catalog_cache = result
|
||||
_model_output_limits = output_limits
|
||||
logger.debug(
|
||||
"Loaded model catalog: %d providers, %d total models "
|
||||
"(%d providers have output limits)",
|
||||
len(result),
|
||||
sum(len(m) for m in result.values()),
|
||||
len(output_limits),
|
||||
)
|
||||
return result
|
||||
_catalog_cache = result
|
||||
_model_output_limits = output_limits
|
||||
logger.debug(
|
||||
"Loaded model catalog: %d providers, %d total models "
|
||||
"(%d providers have output limits)",
|
||||
len(result),
|
||||
sum(len(m) for m in result.values()),
|
||||
len(output_limits),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _get_model_max_output(provider: str, model: str) -> Optional[int]:
|
||||
@@ -126,12 +169,12 @@ async def fetch_ollama_models(api_base: str) -> List[str]:
|
||||
url = f"{api_base.rstrip('/')}/models"
|
||||
try:
|
||||
async with aiohttp.ClientSession(timeout=_OLLAMA_API_TIMEOUT) as session:
|
||||
async with session.get(url) as resp:
|
||||
async with session.get(url, headers=_NO_BROTLI_HEADERS) as resp:
|
||||
if resp.status != 200:
|
||||
logger.debug("Ollama API returned HTTP %s from %s", resp.status, api_base)
|
||||
return []
|
||||
data = await resp.json()
|
||||
except (aiohttp.ClientError, asyncio.TimeoutError, json.JSONDecodeError) as exc:
|
||||
except (aiohttp.ClientError, asyncio.TimeoutError, json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
logger.debug("Ollama not reachable at %s: %s", api_base, exc)
|
||||
return []
|
||||
|
||||
@@ -224,6 +267,16 @@ _PROVIDER_DEFAULTS: Dict[str, str] = {
|
||||
# Request timeout for LLM calls (seconds)
|
||||
_LLM_TIMEOUT = aiohttp.ClientTimeout(total=120)
|
||||
|
||||
# Providers that do NOT implement ``response_format: {"type": "json_schema"}``
|
||||
# and reject it with HTTP 400. For these the weaker, widely supported
|
||||
# ``json_object`` mode is used instead (the prompt already specifies the
|
||||
# expected JSON shape, and ``_try_salvage_json`` repairs imperfect output).
|
||||
# DeepSeek answers a json_schema request with
|
||||
# ``{"error":{"message":"This response_format type is unavailable now"}}``.
|
||||
# LM Studio and some other local OpenAI-compatible servers reject
|
||||
# ``json_object`` but accept ``json_schema``, so they are not listed here.
|
||||
_JSON_OBJECT_ONLY_PROVIDERS = frozenset({"deepseek"})
|
||||
|
||||
|
||||
class LLMService:
|
||||
"""Centralized LLM API client.
|
||||
@@ -571,47 +624,61 @@ class LLMService:
|
||||
if effective_max is None:
|
||||
effective_max = 4096
|
||||
|
||||
# Use json_schema (not json_object) for broader provider compatibility:
|
||||
# LM Studio and some other OpenAI-compatible servers reject
|
||||
# json_object but accept json_schema. {"type": "object"} is
|
||||
# functionally equivalent — it accepts any JSON object without
|
||||
# constraining specific fields.
|
||||
response_format = {
|
||||
# Structured-output format. ``json_schema`` is preferred because LM
|
||||
# Studio and other local OpenAI-compatible servers reject
|
||||
# ``json_object`` but accept ``json_schema``; ``{"type": "object"}``
|
||||
# accepts any JSON object without constraining specific fields, so the
|
||||
# two modes are functionally equivalent here. Providers known to
|
||||
# reject json_schema (see _JSON_OBJECT_ONLY_PROVIDERS) get
|
||||
# ``json_object`` instead.
|
||||
schema_format: Dict[str, Any] = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "metadata",
|
||||
"schema": {"type": "object"},
|
||||
},
|
||||
}
|
||||
json_object_format: Dict[str, Any] = {"type": "json_object"}
|
||||
|
||||
try:
|
||||
result = await self.chat_completion(
|
||||
messages=messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
response_format=response_format,
|
||||
max_tokens=effective_max,
|
||||
)
|
||||
except LLMResponseError as e:
|
||||
# Only fall back when the provider rejects the response_format
|
||||
# type value (e.g. "'response_format.type' must be..."). Avoid
|
||||
# catching unrelated 400 errors whose body happens to mention
|
||||
# "response_format" (e.g. "model does not support
|
||||
# response_format restrictions on this endpoint").
|
||||
if "'response_format.type'" not in str(e).lower():
|
||||
raise
|
||||
logger.info(
|
||||
"Provider rejected response_format, retrying without it. "
|
||||
"Falling back to prompt-only JSON mode. Error: %s",
|
||||
e,
|
||||
)
|
||||
result = await self.chat_completion(
|
||||
messages=messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
response_format=None,
|
||||
max_tokens=effective_max,
|
||||
)
|
||||
if self._get_config()["provider"] in _JSON_OBJECT_ONLY_PROVIDERS:
|
||||
format_chain: List[Optional[Dict[str, Any]]] = [
|
||||
json_object_format,
|
||||
None,
|
||||
]
|
||||
else:
|
||||
format_chain = [schema_format, json_object_format, None]
|
||||
|
||||
result: Optional[Dict[str, Any]] = None
|
||||
for index, fmt in enumerate(format_chain):
|
||||
try:
|
||||
result = await self.chat_completion(
|
||||
messages=messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
response_format=fmt,
|
||||
max_tokens=effective_max,
|
||||
)
|
||||
break
|
||||
except LLMResponseError as e:
|
||||
message = str(e).lower()
|
||||
if index + 1 >= len(format_chain):
|
||||
raise
|
||||
# Only downgrade when the failure is about ``response_format``.
|
||||
# Everything else (auth, unknown model, rate limits) must
|
||||
# surface unchanged. Matching on the bare parameter name also
|
||||
# covers variants such as DeepSeek's "This response_format
|
||||
# type is unavailable now" without swallowing unrelated 400s.
|
||||
if "response_format" not in message:
|
||||
raise
|
||||
logger.info(
|
||||
"Provider rejected response_format=%s, retrying with %s. "
|
||||
"Error: %s",
|
||||
(fmt or {}).get("type", "none"),
|
||||
(format_chain[index + 1] or {}).get("type", "none"),
|
||||
e,
|
||||
)
|
||||
|
||||
assert result is not None # non-empty chain always sets or raises
|
||||
|
||||
content = result.get("content", "") or ""
|
||||
if not content:
|
||||
|
||||
@@ -79,6 +79,8 @@ class LoraService(BaseModelService):
|
||||
),
|
||||
"auto_tags": model_data.get("auto_tags") or extract_auto_tags(model_data),
|
||||
"version_count": model_data.get("version_count"),
|
||||
"source_platform": model_data.get("source_platform", ""),
|
||||
"source_url": model_data.get("source_url", ""),
|
||||
"hf_url": model_data.get("hf_url", ""),
|
||||
}
|
||||
|
||||
@@ -712,12 +714,18 @@ class LoraService(BaseModelService):
|
||||
),
|
||||
)
|
||||
|
||||
# Return minimal data needed for cycling
|
||||
return [
|
||||
{
|
||||
# Return minimal data needed for cycling. usage_tips is only included
|
||||
# when non-empty so widget consumers (recommended strength range cues)
|
||||
# can build their lookup without inflating the payload.
|
||||
result = []
|
||||
for lora in available_loras:
|
||||
entry = {
|
||||
"file_name": f"{lora['folder']}/{lora['file_name']}" if lora.get("folder") else lora["file_name"],
|
||||
"model_name": lora.get("model_name", lora["file_name"]),
|
||||
"folder": lora.get("folder", ""),
|
||||
}
|
||||
for lora in available_loras
|
||||
]
|
||||
usage_tips = lora.get("usage_tips")
|
||||
if usage_tips:
|
||||
entry["usage_tips"] = usage_tips
|
||||
result.append(entry)
|
||||
return result
|
||||
|
||||
@@ -14,10 +14,33 @@ from ..utils.model_utils import determine_base_model
|
||||
from ..utils.models import autov3_from_civitai_files
|
||||
from .connectivity_guard import OFFLINE_FRIENDLY_MESSAGE, is_expected_offline_error
|
||||
from .errors import RateLimitError
|
||||
from .model_sources import has_external_source
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _merge_ordered_unique(existing: Iterable[str], new: Iterable[str]) -> list[str]:
|
||||
"""Concatenate two word lists, dropping duplicates without reordering.
|
||||
|
||||
Trigger word order is meaningful: the sequence stored in
|
||||
``civitai.trainedWords`` is the order used when building prompts, and users
|
||||
can reorder it in the UI. A plain ``set`` union used to shuffle that order on
|
||||
every metadata refresh, so existing words are kept first (in their saved
|
||||
order) and newly discovered ones are appended.
|
||||
"""
|
||||
|
||||
merged: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for word in list(existing) + list(new):
|
||||
if word in seen:
|
||||
continue
|
||||
seen.add(word)
|
||||
merged.append(word)
|
||||
|
||||
return merged
|
||||
|
||||
|
||||
class MetadataProviderProtocol(Protocol):
|
||||
"""Subset of metadata provider interface consumed by the sync service."""
|
||||
|
||||
@@ -114,9 +137,10 @@ class MetadataSyncService:
|
||||
)
|
||||
|
||||
if "trainedWords" in existing_civitai:
|
||||
existing_trained = existing_civitai.get("trainedWords", [])
|
||||
new_trained = civitai_metadata.get("trainedWords", [])
|
||||
merged_trained = list(set(existing_trained + new_trained))
|
||||
existing_trained = existing_civitai.get("trainedWords", []) or []
|
||||
new_trained = civitai_metadata.get("trainedWords", []) or []
|
||||
# Order preserving merge: the saved order drives prompt order.
|
||||
merged_trained = _merge_ordered_unique(existing_trained, new_trained)
|
||||
merged_civitai["trainedWords"] = merged_trained
|
||||
|
||||
local_metadata["civitai"] = merged_civitai
|
||||
@@ -222,9 +246,10 @@ class MetadataSyncService:
|
||||
error_msg = "CivitAI model is deleted and no archive provider is available"
|
||||
return False, error_msg
|
||||
else:
|
||||
is_hf_source = bool(model_data.get("hf_url"))
|
||||
is_hf_source = has_external_source(model_data)
|
||||
if is_hf_source:
|
||||
# HF-sourced model: only check CivitAI API directly.
|
||||
# External-source model (Hugging Face / ModelScope /
|
||||
# TensorArt): only check CivitAI API directly.
|
||||
# CivArchive is almost guaranteed to have no record, and
|
||||
# hitting it wastes rate-limit budget.
|
||||
# Use a distinct provider name ("civitai_api" not None) so
|
||||
|
||||
@@ -33,6 +33,11 @@ class ModelCache:
|
||||
|
||||
raw_data: List[Dict[str, Any]]
|
||||
folders: List[str]
|
||||
# Every directory under the model roots (including empty ones), as
|
||||
# recorded by the last scan/hydration. ``None`` means "never recorded"
|
||||
# (e.g. a persisted snapshot predating this field) and triggers a
|
||||
# background filesystem backfill in the scanner.
|
||||
all_folders: Optional[List[str]] = None
|
||||
version_index: Dict[int, Dict[str, Any]] = field(default_factory=dict)
|
||||
model_id_index: Dict[int, List[Dict[str, Any]]] = field(default_factory=dict)
|
||||
# Multi-valued companion to version_index: every local file entry of a
|
||||
|
||||
@@ -2,13 +2,15 @@ import asyncio
|
||||
import fnmatch
|
||||
import os
|
||||
import logging
|
||||
import shutil
|
||||
from typing import Any, Dict, List, Optional, Sequence, Set
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
from ..utils.utils import calculate_relative_path_for_model, remove_empty_dirs
|
||||
from ..utils.constants import AUTO_ORGANIZE_BATCH_SIZE
|
||||
from ..utils.constants import AUTO_ORGANIZE_BATCH_SIZE, MODEL_FILE_EXTENSIONS
|
||||
from ..services.settings_manager import get_settings_manager
|
||||
from ..services.model_lifecycle_service import _require_path_in_library_roots
|
||||
from ..services.pending_delete_service import PENDING_DELETE_DIR_NAME
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -41,10 +43,22 @@ class AutoOrganizeResult:
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert result to dictionary"""
|
||||
if self.operation_type == 'filename_template':
|
||||
message = (
|
||||
f'Filename template applied: {self.success_count} renamed, '
|
||||
f'{self.skipped_count} skipped, {self.failure_count} failed '
|
||||
f'out of {self.total} total'
|
||||
)
|
||||
else:
|
||||
message = (
|
||||
f'Auto-organize {self.operation_type} completed: '
|
||||
f'{self.success_count} moved, {self.skipped_count} skipped, '
|
||||
f'{self.failure_count} failed out of {self.total} total'
|
||||
)
|
||||
result: Dict[str, Any] = {
|
||||
'success': self.status != 'error',
|
||||
'status': self.status,
|
||||
'message': f'Auto-organize {self.operation_type} completed: {self.success_count} moved, {self.skipped_count} skipped, {self.failure_count} failed out of {self.total} total',
|
||||
'message': message,
|
||||
'summary': {
|
||||
'total': self.total,
|
||||
'success': self.success_count,
|
||||
@@ -473,17 +487,368 @@ class ModelFileService:
|
||||
|
||||
class ModelMoveService:
|
||||
"""Service for handling individual model moves"""
|
||||
|
||||
|
||||
def __init__(self, scanner, model_type: str):
|
||||
"""Initialize the service
|
||||
|
||||
|
||||
Args:
|
||||
scanner: Model scanner instance
|
||||
model_type: Type of model (e.g., 'lora', 'checkpoint')
|
||||
"""
|
||||
self.scanner = scanner
|
||||
self.model_type = model_type
|
||||
|
||||
|
||||
async def create_folder(self, folder_path: str) -> Dict[str, Any]:
|
||||
"""Create a directory inside the model library roots.
|
||||
|
||||
Args:
|
||||
folder_path: Absolute path of the directory to create (business
|
||||
path — symlinks are not resolved)
|
||||
|
||||
Returns:
|
||||
Dictionary with success flag, the created path and the
|
||||
library-relative folder name used by folder trees.
|
||||
"""
|
||||
try:
|
||||
if not folder_path or not str(folder_path).strip():
|
||||
return {"success": False, "error": "Folder path is required"}
|
||||
|
||||
_require_path_in_library_roots(folder_path, self.scanner, label="Folder path")
|
||||
|
||||
absolute_path = os.path.abspath(folder_path)
|
||||
already_exists = os.path.isdir(absolute_path)
|
||||
os.makedirs(absolute_path, exist_ok=True)
|
||||
|
||||
relative_folder = self._calculate_relative_folder(absolute_path)
|
||||
if relative_folder:
|
||||
await self.scanner.add_known_folder(relative_folder)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"folder_path": absolute_path.replace(os.sep, "/"),
|
||||
"folder": relative_folder,
|
||||
"created": not already_exists,
|
||||
}
|
||||
except ValueError as exc:
|
||||
return {"success": False, "error": str(exc)}
|
||||
except Exception as exc:
|
||||
logger.error(f"Error creating folder: {exc}", exc_info=True)
|
||||
return {"success": False, "error": str(exc)}
|
||||
|
||||
def _calculate_relative_folder(self, absolute_path: str) -> str:
|
||||
"""Return the library-relative folder for an absolute directory path."""
|
||||
normalized = os.path.abspath(absolute_path)
|
||||
for root in self.scanner.get_model_roots():
|
||||
abs_root = os.path.abspath(root)
|
||||
try:
|
||||
rel = os.path.relpath(normalized, abs_root)
|
||||
except ValueError:
|
||||
continue
|
||||
if rel == ".":
|
||||
return ""
|
||||
if not rel.startswith(".."):
|
||||
return rel.replace(os.sep, "/")
|
||||
return ""
|
||||
|
||||
async def delete_folder(self, folder_path: str, dry_run: bool = False) -> Dict[str, Any]:
|
||||
"""Delete a model-free directory inside the model library roots.
|
||||
|
||||
Only directories whose subtree holds no model weight files can be
|
||||
removed: a folder-level cascade would bypass the per-model lifecycle
|
||||
bookkeeping (metadata sidecars, previews, cache entries, pending-delete
|
||||
staging and recipe references), so it is deliberately refused. Leftover
|
||||
non-model files (stray previews, sidecars, ``.bak`` files) are reported
|
||||
in the manifest before they are removed.
|
||||
|
||||
Args:
|
||||
folder_path: Absolute path of the directory to remove (business
|
||||
path — symlinks are not resolved)
|
||||
dry_run: When true, only report what would be removed
|
||||
|
||||
Returns:
|
||||
Dictionary with the success flag plus a removal manifest
|
||||
(``model_count``/``file_count``/``dir_count``/``symlink_count``/
|
||||
``total_bytes``/``restorable``) on success.
|
||||
"""
|
||||
try:
|
||||
if not folder_path or not str(folder_path).strip():
|
||||
return {"success": False, "error": "Folder path is required"}
|
||||
|
||||
_require_path_in_library_roots(folder_path, self.scanner, label="Folder path")
|
||||
|
||||
absolute_path = os.path.abspath(folder_path)
|
||||
if os.path.islink(absolute_path):
|
||||
# shutil.rmtree refuses symlinked roots, and silently deleting
|
||||
# the link (leaving the real directory behind) is a separate
|
||||
# decision we do not make here.
|
||||
return {
|
||||
"success": False,
|
||||
"error": "Symlinked folders cannot be deleted",
|
||||
}
|
||||
if not os.path.isdir(absolute_path):
|
||||
return {"success": False, "error": "Folder no longer exists"}
|
||||
|
||||
if self._is_model_root(absolute_path):
|
||||
return {
|
||||
"success": False,
|
||||
"error": "The library root itself cannot be deleted",
|
||||
}
|
||||
|
||||
manifest = self._collect_folder_manifest(absolute_path)
|
||||
|
||||
if manifest["pending_delete_job"]:
|
||||
return {
|
||||
"success": False,
|
||||
"code": "busy",
|
||||
"error": (
|
||||
"A staged delete is still pending inside this folder; "
|
||||
"wait for the undo window to expire"
|
||||
),
|
||||
"manifest": manifest,
|
||||
}
|
||||
|
||||
if manifest["model_count"] > 0:
|
||||
return {
|
||||
"success": False,
|
||||
"code": "not_empty",
|
||||
"error": (
|
||||
f"Folder still contains {manifest['model_count']} model "
|
||||
"file(s); delete or move them first"
|
||||
),
|
||||
"manifest": manifest,
|
||||
}
|
||||
|
||||
relative_folder = self._calculate_relative_folder(absolute_path)
|
||||
|
||||
if dry_run:
|
||||
return {
|
||||
"success": True,
|
||||
"dry_run": True,
|
||||
"folder_path": absolute_path.replace(os.sep, "/"),
|
||||
"folder": relative_folder,
|
||||
**manifest,
|
||||
}
|
||||
|
||||
shutil.rmtree(absolute_path)
|
||||
|
||||
await self._forget_folder(relative_folder)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"dry_run": False,
|
||||
"folder_path": absolute_path.replace(os.sep, "/"),
|
||||
"folder": relative_folder,
|
||||
**manifest,
|
||||
}
|
||||
except ValueError as exc:
|
||||
return {"success": False, "error": str(exc)}
|
||||
except Exception as exc:
|
||||
logger.error(f"Error deleting folder: {exc}", exc_info=True)
|
||||
return {"success": False, "error": str(exc)}
|
||||
|
||||
def _is_model_root(self, absolute_path: str) -> bool:
|
||||
"""Return True when the path *is* one of the configured library roots."""
|
||||
normalized = os.path.normpath(absolute_path)
|
||||
for root in self.scanner.get_model_roots():
|
||||
if os.path.normpath(os.path.abspath(root)) == normalized:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _is_model_file(file_name: str) -> bool:
|
||||
"""Return True when the file name carries a model weight extension."""
|
||||
return os.path.splitext(file_name)[1].lower() in MODEL_FILE_EXTENSIONS
|
||||
|
||||
def _collect_folder_manifest(self, absolute_path: str) -> Dict[str, Any]:
|
||||
"""Describe everything a recursive delete of *absolute_path* removes.
|
||||
|
||||
Walking is intentional: the scanner cache can be stale, and a model file
|
||||
that appeared on disk since the last scan must still block the delete.
|
||||
Symbolic links are never followed (``os.walk`` default) and are counted
|
||||
separately — ``shutil.rmtree`` unlinks them without touching their
|
||||
targets.
|
||||
"""
|
||||
model_count = 0
|
||||
file_count = 0
|
||||
dir_count = 0
|
||||
symlink_count = 0
|
||||
total_bytes = 0
|
||||
pending_delete_job = False
|
||||
|
||||
for dirpath, dirnames, filenames in os.walk(absolute_path):
|
||||
if PENDING_DELETE_DIR_NAME in dirnames:
|
||||
pending_delete_job = True
|
||||
|
||||
for name in dirnames:
|
||||
if os.path.islink(os.path.join(dirpath, name)):
|
||||
symlink_count += 1
|
||||
else:
|
||||
dir_count += 1
|
||||
|
||||
for name in filenames:
|
||||
full_path = os.path.join(dirpath, name)
|
||||
if os.path.islink(full_path):
|
||||
symlink_count += 1
|
||||
continue
|
||||
if self._is_model_file(name):
|
||||
model_count += 1
|
||||
else:
|
||||
file_count += 1
|
||||
try:
|
||||
total_bytes += os.path.getsize(full_path)
|
||||
except OSError: # pragma: no cover - defensive
|
||||
pass
|
||||
|
||||
return {
|
||||
"model_count": model_count,
|
||||
"file_count": file_count,
|
||||
"dir_count": dir_count,
|
||||
"symlink_count": symlink_count,
|
||||
"total_bytes": total_bytes,
|
||||
"pending_delete_job": pending_delete_job,
|
||||
# A truly empty directory is the only case an "undo" can restore by
|
||||
# simply recreating it; a folder holding stray files is gone for good.
|
||||
"restorable": (
|
||||
model_count == 0
|
||||
and file_count == 0
|
||||
and dir_count == 0
|
||||
and symlink_count == 0
|
||||
),
|
||||
}
|
||||
|
||||
async def _forget_folder(self, relative_folder: str) -> None:
|
||||
"""Drop a removed directory from the scanner's folder/cache records."""
|
||||
if not relative_folder:
|
||||
return
|
||||
remove_known_folder = getattr(self.scanner, "remove_known_folder", None)
|
||||
if callable(remove_known_folder):
|
||||
await remove_known_folder(relative_folder)
|
||||
|
||||
async def rename_folder(self, folder_path: str, new_name: str) -> Dict[str, Any]:
|
||||
"""Rename a directory inside the model library roots.
|
||||
|
||||
Unlike :meth:`delete_folder` this works on folders that hold models.
|
||||
A rename keeps every file, so no per-model lifecycle step is bypassed:
|
||||
the directory is renamed on disk and the affected folder, cache, hash
|
||||
index and metadata-sidecar records are re-keyed onto the new prefix by
|
||||
the scanner.
|
||||
|
||||
Args:
|
||||
folder_path: Absolute path of the directory to rename (business
|
||||
path — symlinks are not resolved)
|
||||
new_name: New leaf name; a single path segment, not a path
|
||||
|
||||
Returns:
|
||||
Dictionary with the success flag, the previous/next library-relative
|
||||
folder names and whether the directory actually moved.
|
||||
"""
|
||||
try:
|
||||
if not folder_path or not str(folder_path).strip():
|
||||
return {"success": False, "error": "Folder path is required"}
|
||||
|
||||
new_name = str(new_name or "").strip()
|
||||
if not new_name:
|
||||
return {"success": False, "error": "New folder name is required"}
|
||||
if new_name in (".", "..") or any(
|
||||
char in new_name for char in '/\\:*?"<>|'
|
||||
):
|
||||
return {"success": False, "error": "Invalid characters in folder name"}
|
||||
|
||||
_require_path_in_library_roots(folder_path, self.scanner, label="Folder path")
|
||||
|
||||
absolute_path = os.path.abspath(folder_path)
|
||||
if os.path.islink(absolute_path):
|
||||
return {
|
||||
"success": False,
|
||||
"error": "Symlinked folders cannot be renamed",
|
||||
}
|
||||
if not os.path.isdir(absolute_path):
|
||||
return {"success": False, "error": "Folder no longer exists"}
|
||||
|
||||
if self._is_model_root(absolute_path):
|
||||
return {
|
||||
"success": False,
|
||||
"error": "The library root itself cannot be renamed",
|
||||
}
|
||||
|
||||
previous_relative = self._calculate_relative_folder(absolute_path)
|
||||
target = os.path.join(os.path.dirname(absolute_path), new_name)
|
||||
|
||||
if os.path.normpath(target) == os.path.normpath(absolute_path):
|
||||
return {
|
||||
"success": True,
|
||||
"renamed": False,
|
||||
"folder": previous_relative,
|
||||
"previous_folder": previous_relative,
|
||||
"folder_path": absolute_path.replace(os.sep, "/"),
|
||||
}
|
||||
|
||||
if os.path.exists(target):
|
||||
return {
|
||||
"success": False,
|
||||
"code": "target_exists",
|
||||
"error": f"A folder named \"{new_name}\" already exists here",
|
||||
}
|
||||
|
||||
# A staging manifest records absolute original/staged paths, so
|
||||
# moving a folder that holds one would break its undo and purge.
|
||||
if self._has_pending_delete_job(absolute_path):
|
||||
return {
|
||||
"success": False,
|
||||
"code": "busy",
|
||||
"error": (
|
||||
"A staged delete is still pending inside this folder; "
|
||||
"wait for the undo window to expire"
|
||||
),
|
||||
}
|
||||
|
||||
os.rename(absolute_path, target)
|
||||
|
||||
new_relative = self._calculate_relative_folder(target)
|
||||
await self._rename_folder_records(
|
||||
previous_relative, new_relative, absolute_path, target
|
||||
)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"renamed": True,
|
||||
"folder": new_relative,
|
||||
"previous_folder": previous_relative,
|
||||
"folder_path": target.replace(os.sep, "/"),
|
||||
}
|
||||
except ValueError as exc:
|
||||
return {"success": False, "error": str(exc)}
|
||||
except Exception as exc:
|
||||
logger.error(f"Error renaming folder: {exc}", exc_info=True)
|
||||
return {"success": False, "error": str(exc)}
|
||||
|
||||
@staticmethod
|
||||
def _has_pending_delete_job(absolute_path: str) -> bool:
|
||||
"""Return True when a staged-delete batch lives inside the subtree."""
|
||||
for _dirpath, dirnames, _filenames in os.walk(absolute_path):
|
||||
if PENDING_DELETE_DIR_NAME in dirnames:
|
||||
return True
|
||||
return False
|
||||
|
||||
async def _rename_folder_records(
|
||||
self,
|
||||
previous_relative: str,
|
||||
new_relative: str,
|
||||
previous_path: str,
|
||||
new_path: str,
|
||||
) -> None:
|
||||
"""Hand the rename to the scanner so folder/cache records follow it."""
|
||||
if not previous_relative or not new_relative:
|
||||
return
|
||||
rename_known_folder = getattr(self.scanner, "rename_known_folder", None)
|
||||
if callable(rename_known_folder):
|
||||
await rename_known_folder(
|
||||
previous_relative,
|
||||
new_relative,
|
||||
previous_path=previous_path,
|
||||
new_path=new_path,
|
||||
)
|
||||
|
||||
async def move_model(self, file_path: str, target_path: str, use_default_paths: bool = False) -> Dict[str, Any]:
|
||||
"""Move a single model file
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from typing import Dict, Optional, Set, List
|
||||
import os
|
||||
|
||||
from ..utils.constants import is_empty_placeholder_hash
|
||||
|
||||
class ModelHashIndex:
|
||||
"""Index for looking up models by hash or filename"""
|
||||
|
||||
@@ -81,6 +83,8 @@ class ModelHashIndex:
|
||||
# mapping. First-time registrations stay O(1).
|
||||
if autov3:
|
||||
autov3 = autov3.lower()
|
||||
if is_empty_placeholder_hash(autov3):
|
||||
autov3 = None
|
||||
if is_re_registration and (existing_hash != sha256 or autov3):
|
||||
stale_autov3_keys = [
|
||||
key for key, mapped_path in self._autov3_to_path.items()
|
||||
@@ -93,7 +97,7 @@ class ModelHashIndex:
|
||||
|
||||
def add_autov3(self, autov3: str, file_path: str) -> None:
|
||||
"""Add or update an AutoV3-only index entry (used when only AutoV3 is known)"""
|
||||
if not autov3:
|
||||
if not autov3 or is_empty_placeholder_hash(autov3):
|
||||
return
|
||||
autov3 = autov3.lower()
|
||||
self._autov3_to_path[autov3] = file_path
|
||||
@@ -250,6 +254,8 @@ class ModelHashIndex:
|
||||
|
||||
def has_hash(self, hash_value: str) -> bool:
|
||||
"""Check if hash exists in index (SHA256, AutoV2, or AutoV3)"""
|
||||
if is_empty_placeholder_hash(hash_value):
|
||||
return False
|
||||
normalized = hash_value.lower()
|
||||
if normalized in self._hash_to_path:
|
||||
return True
|
||||
@@ -261,6 +267,8 @@ class ModelHashIndex:
|
||||
|
||||
def get_path(self, hash_value: str) -> Optional[str]:
|
||||
"""Get file path for a hash (SHA256, AutoV2, or AutoV3)"""
|
||||
if is_empty_placeholder_hash(hash_value):
|
||||
return None
|
||||
normalized = hash_value.lower()
|
||||
path = self._hash_to_path.get(normalized)
|
||||
if path is not None:
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Awaitable, Callable, Dict, Iterable, List, Mapping, Optional, TYPE_CHECKING, cast
|
||||
@@ -17,6 +18,26 @@ if TYPE_CHECKING:
|
||||
from ..services.model_update_service import ModelUpdateService
|
||||
|
||||
|
||||
async def load_local_metadata(metadata_path: str) -> Dict[str, Any]:
|
||||
"""Load a metadata sidecar JSON, returning an empty dict when missing.
|
||||
|
||||
Thin equivalent of ``MetadataSyncService.load_local_metadata`` for callers
|
||||
(download manager, use cases) that do not hold a sync-service instance.
|
||||
"""
|
||||
|
||||
if not os.path.exists(metadata_path):
|
||||
return {}
|
||||
|
||||
try:
|
||||
with open(metadata_path, "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to load metadata from %s: %s", metadata_path, exc)
|
||||
return {}
|
||||
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
async def delete_model_artifacts(
|
||||
target_dir: str, file_name: str, main_extension: str | None = None
|
||||
) -> List[str]:
|
||||
@@ -404,6 +425,9 @@ class ModelLifecycleService:
|
||||
if metadata and new_metadata_path:
|
||||
metadata["file_name"] = new_file_name
|
||||
metadata["file_path"] = new_file_path
|
||||
# Preserve the pre-rename stem so the original download filename
|
||||
# stays recoverable after template-driven renames.
|
||||
metadata.setdefault("original_file_name", old_file_name)
|
||||
|
||||
if metadata.get("preview_url"):
|
||||
old_preview = str(metadata["preview_url"])
|
||||
|
||||
@@ -169,6 +169,17 @@ class ModelMetadataProvider(ABC):
|
||||
"""Published model count for the user; None when unsupported."""
|
||||
return None
|
||||
|
||||
async def get_version_file_mini(
|
||||
self, version_id: int, file_id: int
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Fetch raw stored file info via CivitAI's model-versions/mini endpoint.
|
||||
|
||||
Only the CivitAI provider implements this (#1100); other providers
|
||||
already serve raw file names (CivArchive) or cannot resolve this
|
||||
lookup (SQLite), so the default is None.
|
||||
"""
|
||||
return None
|
||||
|
||||
class CivitaiModelMetadataProvider(ModelMetadataProvider):
|
||||
"""Provider that uses Civitai API for metadata"""
|
||||
|
||||
@@ -203,6 +214,11 @@ class CivitaiModelMetadataProvider(ModelMetadataProvider):
|
||||
async def get_creator_model_count(self, username: str) -> Optional[int]:
|
||||
return await self.client.get_creator_model_count(username)
|
||||
|
||||
async def get_version_file_mini(
|
||||
self, version_id: int, file_id: int
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
return await self.client.get_version_file_mini(version_id, file_id)
|
||||
|
||||
class CivArchiveModelMetadataProvider(ModelMetadataProvider):
|
||||
"""Provider that uses CivArchive API for metadata"""
|
||||
|
||||
@@ -700,6 +716,37 @@ class FallbackMetadataProvider(ModelMetadataProvider):
|
||||
continue
|
||||
return None
|
||||
|
||||
async def get_version_file_mini(
|
||||
self, version_id: int, file_id: int
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
rate_limited = False
|
||||
for provider, label in self._iter_providers():
|
||||
if rate_limited and label not in _LOCAL_PROVIDER_LABELS:
|
||||
continue
|
||||
try:
|
||||
result = await self._call_with_rate_limit(
|
||||
label,
|
||||
provider.get_version_file_mini,
|
||||
version_id,
|
||||
file_id,
|
||||
)
|
||||
if result:
|
||||
return result
|
||||
except RateLimitError as exc:
|
||||
rate_limited = True
|
||||
logger.warning(
|
||||
"Provider %s is rate-limited (retry_after=%.0fs); not failing over to other network providers",
|
||||
label,
|
||||
exc.retry_after or 0,
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
"Provider %s failed for get_version_file_mini: %s", label, e
|
||||
)
|
||||
continue
|
||||
return None
|
||||
|
||||
def _iter_providers(self):
|
||||
return zip(self.providers, self._provider_labels)
|
||||
|
||||
@@ -791,6 +838,16 @@ class RateLimitRetryingProvider(ModelMetadataProvider):
|
||||
async def get_creator_model_count(self, username: str) -> Optional[int]:
|
||||
return await self._provider.get_creator_model_count(username)
|
||||
|
||||
async def get_version_file_mini(
|
||||
self, version_id: int, file_id: int
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
return await self._rate_limit_helper.run(
|
||||
self._label,
|
||||
self._provider.get_version_file_mini,
|
||||
version_id,
|
||||
file_id,
|
||||
)
|
||||
|
||||
class ModelMetadataProviderManager:
|
||||
"""Manager for selecting and using model metadata providers"""
|
||||
|
||||
|
||||
+734
-98
File diff suppressed because it is too large
Load Diff
@@ -118,19 +118,24 @@ class ModelServiceFactory:
|
||||
|
||||
|
||||
def register_default_model_types():
|
||||
"""Register the default model types (LoRA, Checkpoint, and Embedding)"""
|
||||
"""Register the default model types (LoRA, Checkpoint, Embedding, and Other)"""
|
||||
from ..services.lora_service import LoraService
|
||||
from ..services.checkpoint_service import CheckpointService
|
||||
from ..services.embedding_service import EmbeddingService
|
||||
from ..services.other_model_service import OtherModelService
|
||||
from ..routes.lora_routes import LoraRoutes
|
||||
from ..routes.checkpoint_routes import CheckpointRoutes
|
||||
from ..routes.embedding_routes import EmbeddingRoutes
|
||||
|
||||
from ..routes.other_routes import OtherRoutes
|
||||
|
||||
# Register LoRA model type
|
||||
ModelServiceFactory.register_model_type('lora', LoraService, LoraRoutes)
|
||||
|
||||
|
||||
# Register Checkpoint model type
|
||||
ModelServiceFactory.register_model_type('checkpoint', CheckpointService, CheckpointRoutes)
|
||||
|
||||
|
||||
# Register Embedding model type
|
||||
ModelServiceFactory.register_model_type('embedding', EmbeddingService, EmbeddingRoutes)
|
||||
ModelServiceFactory.register_model_type('embedding', EmbeddingService, EmbeddingRoutes)
|
||||
|
||||
# Register Other model type (VAE, upscaler, text encoder, ...)
|
||||
ModelServiceFactory.register_model_type('other', OtherModelService, OtherRoutes)
|
||||
@@ -0,0 +1,86 @@
|
||||
"""External model-source providers (Hugging Face, ModelScope, TensorArt).
|
||||
|
||||
This package is the single abstraction over "a site that hosts models and
|
||||
a model card". See :mod:`py.services.model_sources.base` for the provider
|
||||
protocol and :mod:`py.services.model_sources.registry` for the lookup and
|
||||
metadata-normalisation helpers used across the codebase.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .base import (
|
||||
GROUP_PREFIXES,
|
||||
HTTP_TIMEOUT,
|
||||
ModelCardContext,
|
||||
ModelSource,
|
||||
ModelSourceCache,
|
||||
ModelSourceError,
|
||||
SourceRef,
|
||||
USER_AGENT,
|
||||
clean_source_url,
|
||||
fetch_json,
|
||||
fetch_text,
|
||||
filter_weight_files,
|
||||
is_valid_source_id,
|
||||
)
|
||||
from .huggingface import HuggingFaceSource
|
||||
from .hydration import (
|
||||
hydrate_from_source,
|
||||
load_model_card,
|
||||
resolve_site_base_model,
|
||||
)
|
||||
from .modelscope import ModelScopeIntlSource, ModelScopeSource
|
||||
from .registry import (
|
||||
LEGACY_HF_URL_FIELD,
|
||||
SOURCE_PLATFORM_FIELD,
|
||||
SOURCE_URL_FIELD,
|
||||
detect_source,
|
||||
downloadable_sources,
|
||||
get_download_source,
|
||||
get_source,
|
||||
get_source_platform,
|
||||
has_external_source,
|
||||
list_sources,
|
||||
normalize_metadata_source,
|
||||
resolve_source_ref,
|
||||
source_group_key,
|
||||
source_label,
|
||||
)
|
||||
from .tensorart import TensorArtSource
|
||||
|
||||
__all__ = [
|
||||
"GROUP_PREFIXES",
|
||||
"HTTP_TIMEOUT",
|
||||
"LEGACY_HF_URL_FIELD",
|
||||
"ModelCardContext",
|
||||
"ModelSource",
|
||||
"ModelSourceCache",
|
||||
"ModelSourceError",
|
||||
"HuggingFaceSource",
|
||||
"ModelScopeIntlSource",
|
||||
"ModelScopeSource",
|
||||
"SOURCE_PLATFORM_FIELD",
|
||||
"SOURCE_URL_FIELD",
|
||||
"SourceRef",
|
||||
"TensorArtSource",
|
||||
"USER_AGENT",
|
||||
"clean_source_url",
|
||||
"detect_source",
|
||||
"downloadable_sources",
|
||||
"fetch_json",
|
||||
"fetch_text",
|
||||
"filter_weight_files",
|
||||
"get_download_source",
|
||||
"get_source",
|
||||
"get_source_platform",
|
||||
"has_external_source",
|
||||
"hydrate_from_source",
|
||||
"is_valid_source_id",
|
||||
"list_sources",
|
||||
"load_model_card",
|
||||
"normalize_metadata_source",
|
||||
"resolve_site_base_model",
|
||||
"resolve_source_ref",
|
||||
"source_group_key",
|
||||
"source_label",
|
||||
]
|
||||
@@ -0,0 +1,446 @@
|
||||
"""Base types for the external model-source provider abstraction.
|
||||
|
||||
A *model source* is a third-party site that hosts model files and a model
|
||||
card (README) describing them — Hugging Face, ModelScope, TensorArt, and
|
||||
whatever gets added later. Everything the rest of the codebase needs to
|
||||
know about such a site is expressed by :class:`ModelSource`:
|
||||
|
||||
* how to recognise one of its URLs (:meth:`ModelSource.parse`)
|
||||
* the canonical page URL for a source id (:meth:`ModelSource.canonical_url`)
|
||||
* how to fetch the model card (:meth:`ModelSource.fetch_model_card`)
|
||||
* how to fetch the extras that live *outside* the README
|
||||
(:meth:`ModelSource.fetch_model_card_context`)
|
||||
* how to turn repository-relative asset paths into absolute URLs
|
||||
(:meth:`ModelSource.asset_base_url`)
|
||||
* which capabilities the site actually supports
|
||||
(``supports_enrichment`` / ``supports_download``)
|
||||
|
||||
Keeping this in one place means the agent pipeline, the scanners, and the
|
||||
HTTP handlers never need site-specific branching.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, Iterable, Optional
|
||||
|
||||
import aiohttp
|
||||
|
||||
from ...utils.constants import MODEL_FILE_EXTENSIONS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: Shared HTTP timeout for model-card fetches.
|
||||
HTTP_TIMEOUT = 30
|
||||
|
||||
#: User agent used for all model-source HTTP requests.
|
||||
USER_AGENT = "ComfyUI-LoRA-Manager/1.0"
|
||||
|
||||
#: Platform → short prefix used when building version-group keys.
|
||||
#: ``huggingface`` keeps the historical ``hf:`` prefix for backward
|
||||
#: compatibility with already-cached group keys.
|
||||
GROUP_PREFIXES: dict[str, str] = {
|
||||
"huggingface": "hf",
|
||||
"modelscope": "ms",
|
||||
"modelscope-ai": "msai",
|
||||
"tensorart": "ta",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceRef:
|
||||
"""A parsed reference to a model hosted on an external site."""
|
||||
|
||||
platform: str
|
||||
"""Canonical platform id, e.g. ``"huggingface"``."""
|
||||
|
||||
source_id: str
|
||||
"""Site-specific identity, e.g. ``"user/repo"`` or ``"827823520299086029"``."""
|
||||
|
||||
url: str
|
||||
"""Canonical URL of the model page."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelCardContext:
|
||||
"""Site-specific extras that accompany a model's README model card.
|
||||
|
||||
A model card is not always just ``README.md``. ModelScope, for example,
|
||||
keeps the author's summary, the site-curated tags, and the per-file
|
||||
example images in its model-detail API rather than in the repository.
|
||||
Sources with no such extras return an empty context (the default), so
|
||||
every field here must be treated as optional by callers.
|
||||
"""
|
||||
|
||||
description: str = ""
|
||||
"""Author-written summary shown on the model page, outside the README."""
|
||||
|
||||
model_name: str = ""
|
||||
"""Site-published display name for the repository.
|
||||
|
||||
Sites publish this next to the repository id (ModelScope's ``Name``).
|
||||
It is what a CivitAI download would store as the model's name, so the
|
||||
card never has to fall back to the local filename.
|
||||
"""
|
||||
|
||||
model_name_localized: str = ""
|
||||
"""Site-published localized name (ModelScope's ``ChineseName``)."""
|
||||
|
||||
version_name: str = ""
|
||||
"""Site-published label for the requested file's version.
|
||||
|
||||
Resolved per file, like :attr:`example_images`: a repository publishes
|
||||
one label per checkpoint (ModelScope's ``modelVersion.showName``).
|
||||
"""
|
||||
|
||||
license: str = ""
|
||||
"""License the site records for the repository."""
|
||||
|
||||
model_type: str = ""
|
||||
"""Site-reported model type, e.g. ModelScope's ``AigcType`` (``LoRA``)."""
|
||||
|
||||
base_model: str = ""
|
||||
"""Base model as reported by the site (possibly a site-local id)."""
|
||||
|
||||
base_model_aliases: list[str] = field(default_factory=list)
|
||||
"""Other names the site uses for the same base model.
|
||||
|
||||
Sites often publish both a link-style id (``krea/Krea-2-Turbo``) and an
|
||||
internal architecture enum (``KREA_2``). The enum usually normalises
|
||||
cleanly onto this system's canonical vocabulary, so it is the better
|
||||
resolution hint for :mod:`py.services.agent.base_model_resolver`.
|
||||
"""
|
||||
|
||||
official_tags: list[str] = field(default_factory=list)
|
||||
"""Content tags curated by the site itself."""
|
||||
|
||||
example_images: list[str] = field(default_factory=list)
|
||||
"""Absolute URLs of example images for the requested model file."""
|
||||
|
||||
trigger_words: list[str] = field(default_factory=list)
|
||||
"""Trigger words the site records for the requested model file."""
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
"""Return ``True`` when the site contributed nothing extra."""
|
||||
|
||||
return not any(
|
||||
(
|
||||
self.description,
|
||||
self.model_name,
|
||||
self.model_name_localized,
|
||||
self.version_name,
|
||||
self.license,
|
||||
self.model_type,
|
||||
self.base_model,
|
||||
self.base_model_aliases,
|
||||
self.official_tags,
|
||||
self.example_images,
|
||||
self.trigger_words,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class ModelSourceError(Exception):
|
||||
"""Raised when a model source cannot satisfy a request.
|
||||
|
||||
Carries the HTTP status the API handler should answer with, so the
|
||||
handlers stay free of per-site error mapping.
|
||||
"""
|
||||
|
||||
def __init__(self, message: str, status: int = 502) -> None:
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
|
||||
|
||||
class ModelSourceCache:
|
||||
"""Per-run memo shared between the agent pipeline and a model source.
|
||||
|
||||
A collection repository publishes many model files under a single source
|
||||
id, so enriching each file re-fetches the same README and the same
|
||||
repository metadata. One cache is created per enrichment run and thrown
|
||||
away afterwards: nothing is retained across runs (a model card can change
|
||||
at any time), and download URLs are never routed through it.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
#: Provider-agnostic: ``"<platform>:<source_id>"`` → raw README text.
|
||||
self.readmes: Dict[str, str] = {}
|
||||
#: Provider-owned scratch space. Keys must be namespaced by the
|
||||
#: provider (``(platform, kind, source_id)``) so two providers can
|
||||
#: never collide. Only successful results should be stored, so a
|
||||
#: transient failure is still retried for the next file.
|
||||
self.provider: Dict[Any, Any] = {}
|
||||
|
||||
|
||||
#: Repository ids are always exactly ``owner/name``. Components may contain
|
||||
#: dots (``black-forest-labs/FLUX.1-dev``) but must not be empty, ``.`` / ``..``,
|
||||
#: or start with a dot - the id is used as a path segment on disk.
|
||||
_SOURCE_ID_COMPONENT = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.\-]*$")
|
||||
|
||||
|
||||
def is_valid_source_id(source_id: str) -> bool:
|
||||
"""Return ``True`` when *source_id* is a safe ``owner/name`` repository id."""
|
||||
|
||||
if not source_id or not isinstance(source_id, str) or source_id.count("/") != 1:
|
||||
return False
|
||||
owner, name = source_id.split("/", 1)
|
||||
return all(
|
||||
part and part not in (".", "..") and _SOURCE_ID_COMPONENT.match(part)
|
||||
for part in (owner, name)
|
||||
)
|
||||
|
||||
|
||||
async def fetch_text(url: str, *, timeout: int = HTTP_TIMEOUT) -> str:
|
||||
"""Fetch *url* and return its body as text, or ``""`` on any failure.
|
||||
|
||||
Network problems are expected (offline installs, rate limits, dead
|
||||
repos) and must never bubble up into the pipeline, so every error is
|
||||
logged at debug level and normalised to an empty string.
|
||||
"""
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession(
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
timeout=aiohttp.ClientTimeout(total=timeout),
|
||||
) as session:
|
||||
async with session.get(url) as resp:
|
||||
if resp.status == 200:
|
||||
return await resp.text()
|
||||
logger.debug("Fetch %s returned HTTP %s", url, resp.status)
|
||||
except Exception as exc: # pragma: no cover - network dependent
|
||||
logger.debug("Failed to fetch %s: %s", url, exc)
|
||||
return ""
|
||||
|
||||
|
||||
async def fetch_json(
|
||||
url: str, *, timeout: int = HTTP_TIMEOUT
|
||||
) -> tuple[int, Any]:
|
||||
"""Fetch *url* and return ``(status, parsed_body)``.
|
||||
|
||||
Unlike :func:`fetch_text` this reports the status, because callers such as
|
||||
the file-listing endpoints need to distinguish "repo not found" (404) from
|
||||
a transport failure. ``parsed_body`` is ``None`` when the response is not
|
||||
JSON or the request failed outright (status ``0``).
|
||||
"""
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession(
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
timeout=aiohttp.ClientTimeout(total=timeout),
|
||||
) as session:
|
||||
async with session.get(url) as resp:
|
||||
if resp.status != 200:
|
||||
return resp.status, None
|
||||
try:
|
||||
return resp.status, await resp.json(content_type=None)
|
||||
except Exception:
|
||||
return resp.status, None
|
||||
except Exception as exc: # pragma: no cover - network dependent
|
||||
logger.debug("Failed to fetch %s: %s", url, exc)
|
||||
return 0, None
|
||||
|
||||
|
||||
class ModelSource:
|
||||
"""Description and I/O for one external model hosting site."""
|
||||
|
||||
#: Canonical platform id stored in metadata.
|
||||
platform: str = ""
|
||||
|
||||
#: Human-readable name used in UI copy and prompts.
|
||||
label: str = ""
|
||||
|
||||
#: Whether the agent skill can fetch a model card and run AI extraction.
|
||||
supports_enrichment: bool = False
|
||||
|
||||
#: Whether models can be downloaded directly from this site.
|
||||
supports_download: bool = False
|
||||
|
||||
#: Branch used when the caller does not pass an explicit revision.
|
||||
default_revision: str = ""
|
||||
|
||||
#: Sub-directory the "use default paths" template places downloads in.
|
||||
default_subdir: str = ""
|
||||
|
||||
#: Lenient pattern used to recognise URLs already stored in metadata.
|
||||
#: Captures the site-specific source id in group ``id``.
|
||||
url_pattern: re.Pattern[str] | None = None
|
||||
|
||||
#: Strict pattern used to validate user input. Must match the whole URL.
|
||||
strict_url_pattern: re.Pattern[str] | None = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Parsing
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def parse(self, url: str, *, strict: bool = False) -> Optional[str]:
|
||||
"""Return the source id contained in *url*, or ``None``.
|
||||
|
||||
With ``strict=True`` the URL must match this site's canonical shape
|
||||
exactly (used when validating what a user pasted); with
|
||||
``strict=False`` sub-paths such as ``/resolve/main/file.bin`` are
|
||||
tolerated (used when normalising already-stored values).
|
||||
"""
|
||||
|
||||
if not url or not isinstance(url, str):
|
||||
return None
|
||||
candidate = url.strip()
|
||||
if not candidate:
|
||||
return None
|
||||
pattern = self.strict_url_pattern if strict else self.url_pattern
|
||||
if pattern is None:
|
||||
return None
|
||||
match = pattern.match(candidate)
|
||||
return match.group("id") if match else None
|
||||
|
||||
def ref(self, url: str, *, strict: bool = False) -> Optional[SourceRef]:
|
||||
"""Return a :class:`SourceRef` for *url*, or ``None`` if not ours."""
|
||||
|
||||
source_id = self.parse(url, strict=strict)
|
||||
if not source_id:
|
||||
return None
|
||||
return SourceRef(
|
||||
platform=self.platform,
|
||||
source_id=source_id,
|
||||
url=self.canonical_url(source_id),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# URLs and content
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def canonical_url(self, source_id: str) -> str:
|
||||
"""Return the canonical model-page URL for *source_id*."""
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
def asset_base_url(self, source_id: str, revision: str = "") -> str:
|
||||
"""Base URL used to resolve repository-relative asset paths."""
|
||||
|
||||
return ""
|
||||
|
||||
def group_key(self, source_id: str) -> str:
|
||||
"""Return the version-group key for *source_id*."""
|
||||
|
||||
prefix = GROUP_PREFIXES.get(self.platform, self.platform)
|
||||
return f"{prefix}:{source_id}"
|
||||
|
||||
async def fetch_model_card(self, source_id: str) -> str:
|
||||
"""Fetch the raw model card (README) markdown for *source_id*."""
|
||||
|
||||
return ""
|
||||
|
||||
async def fetch_model_card_context(
|
||||
self,
|
||||
source_id: str,
|
||||
filename: str = "",
|
||||
*,
|
||||
sha256: str = "",
|
||||
cache: Optional["ModelSourceCache"] = None,
|
||||
) -> ModelCardContext:
|
||||
"""Return the card extras the site keeps outside the README.
|
||||
|
||||
*filename* is the model file's basename (no directory) and *sha256*
|
||||
its content hash; between them they select the right entry when a
|
||||
repository holds several models. A site that records per-file hashes
|
||||
should prefer *sha256*, because it is the only identifier that
|
||||
survives the user renaming the weights.
|
||||
|
||||
*cache* is an optional per-run memo (see :class:`ModelSourceCache`)
|
||||
that lets a provider avoid re-fetching repository-wide data for every
|
||||
file in a collection repository.
|
||||
|
||||
Sites whose model card is fully described by :meth:`fetch_model_card`
|
||||
need no override and inherit this empty context.
|
||||
|
||||
Implementations must never raise: enrichment treats a missing
|
||||
context as "the site had nothing extra to say".
|
||||
"""
|
||||
|
||||
return ModelCardContext()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Download support
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def list_files(
|
||||
self, source_id: str, revision: str = ""
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List downloadable weight files in *source_id*.
|
||||
|
||||
Returns ``[{"filename": <repo-relative path>, "size": <bytes>}]``,
|
||||
largest first, filtered to :data:`MODEL_FILE_EXTENSIONS`. Sites
|
||||
without download support return an empty list.
|
||||
|
||||
Raises :class:`ModelSourceError` when the repository cannot be read,
|
||||
so the handler can surface "not found" separately from a transport
|
||||
failure.
|
||||
"""
|
||||
|
||||
return []
|
||||
|
||||
def file_download_url(
|
||||
self, source_id: str, filename: str, revision: str = ""
|
||||
) -> str:
|
||||
"""Return the direct (redirecting) download URL for one file."""
|
||||
|
||||
raise ModelSourceError(
|
||||
f"{self.label or self.platform} does not support downloads", status=400
|
||||
)
|
||||
|
||||
def resolve_revision(self, revision: str = "") -> str:
|
||||
"""Return *revision*, falling back to this site's default branch."""
|
||||
|
||||
return revision or self.default_revision
|
||||
|
||||
def page_url_for_file(self, source_id: str, filename: str) -> str:
|
||||
"""Return the human-facing page for *filename* inside *source_id*."""
|
||||
|
||||
return self.canonical_url(source_id)
|
||||
|
||||
def __repr__(self) -> str: # pragma: no cover - debugging aid
|
||||
return f"<ModelSource {self.platform}>"
|
||||
|
||||
|
||||
def clean_source_url(url: Any) -> str:
|
||||
"""Normalise a stored source URL value into a stripped string."""
|
||||
|
||||
if not isinstance(url, str):
|
||||
return ""
|
||||
return url.strip()
|
||||
|
||||
|
||||
def filter_weight_files(entries: Iterable[tuple[str, int]]) -> list[dict[str, Any]]:
|
||||
"""Keep model-weight files from ``(path, size)`` pairs, largest first.
|
||||
|
||||
Every site lists a lot more than weights (READMEs, configs, tokenizers,
|
||||
…); the download picker only ever wants the files ComfyUI can load, which
|
||||
is exactly :data:`MODEL_FILE_EXTENSIONS`.
|
||||
"""
|
||||
|
||||
files = [
|
||||
{"filename": path, "size": int(size or 0)}
|
||||
for path, size in entries
|
||||
if path and os.path.splitext(path)[1].lower() in MODEL_FILE_EXTENSIONS
|
||||
]
|
||||
files.sort(key=lambda entry: entry["size"], reverse=True)
|
||||
return files
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GROUP_PREFIXES",
|
||||
"HTTP_TIMEOUT",
|
||||
"ModelCardContext",
|
||||
"ModelSource",
|
||||
"ModelSourceCache",
|
||||
"ModelSourceError",
|
||||
"SourceRef",
|
||||
"USER_AGENT",
|
||||
"clean_source_url",
|
||||
"fetch_json",
|
||||
"fetch_text",
|
||||
"filter_weight_files",
|
||||
"is_valid_source_id",
|
||||
]
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Hugging Face model source."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from .base import (
|
||||
ModelSource,
|
||||
ModelSourceError,
|
||||
fetch_json,
|
||||
fetch_text,
|
||||
filter_weight_files,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: Lenient — used to normalise URLs already stored in metadata; tolerates
|
||||
#: sub-paths such as ``/resolve/main/model.safetensors``.
|
||||
_URL_PATTERN = re.compile(
|
||||
r"https?://(?:www\.)?huggingface\.co/(?P<id>[^/?#\s]+/[^/?#\s]+)"
|
||||
)
|
||||
|
||||
#: Strict — validates what the user pasted into the "link model" dialog.
|
||||
_STRICT_URL_PATTERN = re.compile(
|
||||
r"https?://(?:www\.)?huggingface\.co/(?P<id>[^/?#\s]+/[^/?#\s]+)/?$"
|
||||
)
|
||||
|
||||
|
||||
class HuggingFaceSource(ModelSource):
|
||||
"""Hugging Face Hub (``huggingface.co``)."""
|
||||
|
||||
platform = "huggingface"
|
||||
label = "Hugging Face"
|
||||
supports_enrichment = True
|
||||
supports_download = True
|
||||
default_revision = "main"
|
||||
default_subdir = "huggingface"
|
||||
url_pattern = _URL_PATTERN
|
||||
strict_url_pattern = _STRICT_URL_PATTERN
|
||||
|
||||
def canonical_url(self, source_id: str) -> str:
|
||||
return f"https://huggingface.co/{source_id}"
|
||||
|
||||
def asset_base_url(self, source_id: str, revision: str = "") -> str:
|
||||
return f"https://huggingface.co/{source_id}/resolve/{self.resolve_revision(revision)}"
|
||||
|
||||
async def fetch_model_card(self, source_id: str) -> str:
|
||||
"""Fetch ``README.md`` from Hugging Face (tries ``main``, then ``master``)."""
|
||||
|
||||
for branch in ("main", "master"):
|
||||
text = await fetch_text(
|
||||
f"https://huggingface.co/{source_id}/raw/{branch}/README.md"
|
||||
)
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
async def list_files(
|
||||
self, source_id: str, revision: str = ""
|
||||
) -> list[dict]:
|
||||
"""List weight files via the Hub tree API.
|
||||
|
||||
The tree endpoint (rather than the model-info endpoint) is used
|
||||
because it reports accurate sizes for LFS-tracked files.
|
||||
"""
|
||||
|
||||
revision = self.resolve_revision(revision)
|
||||
status, payload = await fetch_json(
|
||||
f"https://huggingface.co/api/models/{source_id}/tree/{revision}"
|
||||
)
|
||||
|
||||
if status == 404:
|
||||
raise ModelSourceError(f"Repository '{source_id}' not found", status=404)
|
||||
if status != 200 or not isinstance(payload, list):
|
||||
raise ModelSourceError(
|
||||
f"Hugging Face API error while listing '{source_id}' (HTTP {status})"
|
||||
)
|
||||
|
||||
entries = []
|
||||
for entry in payload:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
path = entry.get("path", "")
|
||||
size = entry.get("size", 0) or 0
|
||||
if not size and isinstance(entry.get("lfs"), dict):
|
||||
size = entry["lfs"].get("size", 0) or 0
|
||||
entries.append((path, size))
|
||||
|
||||
return filter_weight_files(entries)
|
||||
|
||||
def file_download_url(
|
||||
self, source_id: str, filename: str, revision: str = ""
|
||||
) -> str:
|
||||
return (
|
||||
f"https://huggingface.co/{source_id}/resolve/"
|
||||
f"{self.resolve_revision(revision)}/{filename}"
|
||||
)
|
||||
|
||||
def page_url_for_file(self, source_id: str, filename: str) -> str:
|
||||
return (
|
||||
f"https://huggingface.co/{source_id}/blob/{self.default_revision}/{filename}"
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["HuggingFaceSource"]
|
||||
@@ -0,0 +1,235 @@
|
||||
"""Deterministic metadata hydration for freshly downloaded source models.
|
||||
|
||||
A CivitAI download writes a fully-populated metadata sidecar as part of the
|
||||
download itself: the name, the description, the tags, the trigger words and
|
||||
the example images all arrive with the file. A download from an external
|
||||
model source (ModelScope, Hugging Face) has the same information behind a
|
||||
public API, but historically landed as a bare filename plus a source URL that
|
||||
the user had to enrich by hand ("Enrich Metadata with AI").
|
||||
|
||||
This module closes that gap without involving an LLM. It fetches the linked
|
||||
site's model card, hands it to the same :class:`~py.services.agent.post_processor.PostProcessor`
|
||||
the AI skill uses, and writes the result. Everything it applies is data the
|
||||
site published, so it is safe to run automatically on every download and to
|
||||
treat as a fallback for the gaps the LLM would otherwise fill.
|
||||
|
||||
Nothing here may break a download: every failure is logged and normalised to
|
||||
"the site had nothing to contribute".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from .base import ModelCardContext, ModelSourceCache
|
||||
from .registry import get_source, resolve_source_ref
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover - typing only
|
||||
from .base import ModelSource, SourceRef
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: How long a fetched repository payload stays usable. A download batch walks
|
||||
#: a repository's files one HTTP request at a time, and the README plus the
|
||||
#: detail payload describe the *repository*, not the file, so re-fetching them
|
||||
#: per file would be pure waste. They expire so an edited model card is still
|
||||
#: picked up by the next batch.
|
||||
SHARED_CACHE_TTL = 300.0
|
||||
|
||||
#: Upper bound on memoised repositories; a long-running server must not grow
|
||||
#: without limit.
|
||||
SHARED_CACHE_MAX_ENTRIES = 32
|
||||
|
||||
#: ``"<platform>:<source_id>"`` → ``(expiry, memo)``.
|
||||
_shared_caches: dict[str, tuple[float, ModelSourceCache]] = {}
|
||||
|
||||
|
||||
def shared_source_cache(platform: str, source_id: str) -> ModelSourceCache:
|
||||
"""Return a short-lived per-repository memo for download-time hydration."""
|
||||
|
||||
now = time.monotonic()
|
||||
key = f"{platform}:{source_id}"
|
||||
entry = _shared_caches.get(key)
|
||||
if entry is not None and entry[0] > now:
|
||||
return entry[1]
|
||||
|
||||
for expired in [k for k, (expiry, _) in _shared_caches.items() if expiry <= now]:
|
||||
_shared_caches.pop(expired, None)
|
||||
if len(_shared_caches) >= SHARED_CACHE_MAX_ENTRIES:
|
||||
oldest = min(_shared_caches, key=lambda k: _shared_caches[k][0])
|
||||
_shared_caches.pop(oldest, None)
|
||||
|
||||
cache = ModelSourceCache()
|
||||
_shared_caches[key] = (now + SHARED_CACHE_TTL, cache)
|
||||
return cache
|
||||
|
||||
|
||||
def reset_shared_caches() -> None:
|
||||
"""Drop every memoised repository — used by tests."""
|
||||
|
||||
_shared_caches.clear()
|
||||
|
||||
|
||||
async def load_model_card(
|
||||
source: "ModelSource",
|
||||
source_id: str,
|
||||
cache: Optional[ModelSourceCache] = None,
|
||||
) -> str:
|
||||
"""Return *source_id*'s README, reusing *cache* when one is supplied.
|
||||
|
||||
Only successful reads are memoised, leaving a transient failure to be
|
||||
retried for the next file of the same repository.
|
||||
"""
|
||||
|
||||
key = f"{source.platform}:{source_id}"
|
||||
if cache is not None:
|
||||
cached = cache.readmes.get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
readme = await source.fetch_model_card(source_id)
|
||||
if cache is not None and readme:
|
||||
cache.readmes[key] = readme
|
||||
return readme or ""
|
||||
|
||||
|
||||
async def resolve_site_base_model(context: ModelCardContext) -> str:
|
||||
"""Resolve the site's base-model hints to a canonical name, or ``""``.
|
||||
|
||||
Sites name base models in their own vocabulary (ModelScope publishes both
|
||||
``krea/Krea-2-Turbo`` and the ``KREA_2_TURBO`` enum). The resolver is
|
||||
strict and only ever returns a name the canonical vocabulary already
|
||||
contains, so an uncertain hint yields ``""`` rather than a plausible-looking
|
||||
wrong value.
|
||||
"""
|
||||
|
||||
hints = [*context.base_model_aliases, context.base_model]
|
||||
if not any(hints):
|
||||
return ""
|
||||
|
||||
# Imported lazily: pulling in the agent package at module scope would make
|
||||
# the model-source package import itself while it is still initialising.
|
||||
try:
|
||||
from ...metadata_ops import list_base_models
|
||||
from ..agent.base_model_resolver import resolve_base_model
|
||||
|
||||
known_names = await list_base_models()
|
||||
except Exception as exc:
|
||||
logger.warning("Could not resolve a site base model: %s", exc)
|
||||
return ""
|
||||
return resolve_base_model(hints, known_names)
|
||||
|
||||
|
||||
async def hydrate_from_source(
|
||||
file_path: str,
|
||||
*,
|
||||
ref: "SourceRef",
|
||||
cache: Optional[ModelSourceCache] = None,
|
||||
) -> list[str]:
|
||||
"""Apply the linked site's published metadata to a downloaded model.
|
||||
|
||||
This is the deterministic counterpart of the ``enrich_hf_metadata`` skill:
|
||||
it produces the same populated model card a CivitAI download produces,
|
||||
without an LLM and without user action.
|
||||
|
||||
Args:
|
||||
file_path: The just-downloaded model file, whose sidecar already
|
||||
carries the SHA256 used to match the right file in a collection
|
||||
repository.
|
||||
ref: The source the file came from.
|
||||
cache: Optional per-call memo; defaults to a short-lived shared one so
|
||||
a batch over one repository fetches its card only once.
|
||||
|
||||
Returns:
|
||||
The names of the metadata fields that changed. Never raises — a site
|
||||
that is down, or an API that changed shape, must not fail a download.
|
||||
"""
|
||||
|
||||
try:
|
||||
source = get_source(ref.platform)
|
||||
if source is None or not source.supports_enrichment:
|
||||
return []
|
||||
|
||||
from ...metadata_ops import read_metadata
|
||||
|
||||
metadata = await read_metadata(file_path)
|
||||
if not metadata:
|
||||
logger.debug("No metadata to hydrate for %s", file_path)
|
||||
return []
|
||||
|
||||
# Only a model that is actually linked to this repository may be
|
||||
# updated. The download path writes those fields just before calling
|
||||
# us; a file that merely shares a name with the requested one must not
|
||||
# be given another model's card.
|
||||
linked = resolve_source_ref(metadata)
|
||||
if linked is None or (linked.platform, linked.source_id) != (
|
||||
ref.platform,
|
||||
ref.source_id,
|
||||
):
|
||||
logger.debug(
|
||||
"Not hydrating %s: linked to %s, not %s",
|
||||
file_path, linked.url if linked else "no model source", ref.url,
|
||||
)
|
||||
return []
|
||||
|
||||
memo = cache if cache is not None else shared_source_cache(
|
||||
ref.platform, ref.source_id
|
||||
)
|
||||
readme = await load_model_card(source, ref.source_id, memo)
|
||||
context = await source.fetch_model_card_context(
|
||||
ref.source_id,
|
||||
os.path.basename(file_path),
|
||||
sha256=(metadata.get("sha256") or "").strip(),
|
||||
cache=memo,
|
||||
)
|
||||
if context.is_empty() and not readme:
|
||||
logger.debug(
|
||||
"No published metadata for %s on %s", ref.source_id, ref.platform
|
||||
)
|
||||
return []
|
||||
|
||||
resolved_base_model = await resolve_site_base_model(context)
|
||||
|
||||
from ..agent.post_processor import PostProcessor
|
||||
|
||||
result = await PostProcessor().process(
|
||||
skill_name="enrich_hf_metadata",
|
||||
model_path=file_path,
|
||||
llm_output={},
|
||||
metadata=metadata,
|
||||
readme_content=readme,
|
||||
source_context=context,
|
||||
resolved_base_model=resolved_base_model,
|
||||
metadata_source=f"source:{ref.platform}",
|
||||
)
|
||||
if not result.get("success", True):
|
||||
logger.debug(
|
||||
"Hydration reported failure for %s: %s",
|
||||
file_path, result.get("errors"),
|
||||
)
|
||||
return []
|
||||
|
||||
updated = list(result.get("updated_fields") or [])
|
||||
logger.info(
|
||||
"Hydrated %s from %s (%s): %s",
|
||||
file_path, source.label or ref.platform, ref.source_id,
|
||||
", ".join(updated) or "nothing to change",
|
||||
)
|
||||
return updated
|
||||
except Exception as exc: # pragma: no cover - defensive by design
|
||||
logger.warning("Source hydration failed for %s: %s", file_path, exc)
|
||||
return []
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SHARED_CACHE_MAX_ENTRIES",
|
||||
"SHARED_CACHE_TTL",
|
||||
"hydrate_from_source",
|
||||
"load_model_card",
|
||||
"reset_shared_caches",
|
||||
"resolve_site_base_model",
|
||||
"shared_source_cache",
|
||||
]
|
||||
@@ -0,0 +1,644 @@
|
||||
"""ModelScope (魔搭社区) model sources.
|
||||
|
||||
ModelScope exposes the same "model card as README.md" convention as
|
||||
Hugging Face, including a YAML frontmatter block that often carries
|
||||
``base_model:`` and ``trigger_words:``. Four public endpoints are used,
|
||||
none of which requires an API key for public models:
|
||||
|
||||
* ``/models/{owner}/{name}/resolve/{revision}/README.md`` — raw model card
|
||||
* ``/api/v1/models/{owner}/{name}/repo?Revision=..&FilePath=README.md`` —
|
||||
the same content through the API, used as a fallback when the resolve
|
||||
URL is unavailable.
|
||||
* ``/api/v1/models/{owner}/{name}`` — the model-detail payload behind the
|
||||
model page. It carries the repository's display name (``Name`` /
|
||||
``ChineseName``), the author's summary (``Description``), the license, the
|
||||
AIGC type, the site tags (``OfficialTags``, falling back to ``Tags``), and,
|
||||
per published version, the model filenames
|
||||
(``MuseInfo.versions[].stats.fileList``) together with that version's label
|
||||
(``modelVersion.showName``), example images (``coverImages``) and trigger
|
||||
words. See :meth:`ModelScopeSource.fetch_model_card_context`.
|
||||
* ``/api/v1/models/{owner}/{name}/repo/files?Revision=..`` — the file
|
||||
listing backing the download picker. It reports real sizes for LFS
|
||||
files (not the pointer size), so no extra HEAD request is needed.
|
||||
|
||||
Downloads go through ``/models/{owner}/{name}/resolve/{revision}/{path}``,
|
||||
which redirects to a CDN URL carrying a time-limited ``auth_key``.
|
||||
Requesting the resolve URL fresh on every attempt (which the shared
|
||||
downloader does, including for resumable Range requests) keeps that key
|
||||
valid; the CDN URL must never be cached.
|
||||
|
||||
The README and the detail payload both describe the whole repository rather
|
||||
than one file, so a per-run ``ModelSourceCache`` keeps them from being read
|
||||
again for every checkpoint of a collection repository.
|
||||
|
||||
Two deployments are served by this module. ``modelscope.cn`` (with
|
||||
``modelscope.com`` as a redirect alias) and ``modelscope.ai`` are *separate
|
||||
catalogues*, not mirrors, so they are registered as distinct sources:
|
||||
:class:`ModelScopeSource` and :class:`ModelScopeIntlSource`. Every URL either
|
||||
class builds is derived from its ``base_url``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import TYPE_CHECKING, Any, Iterable, Optional
|
||||
|
||||
from .base import (
|
||||
ModelCardContext,
|
||||
ModelSource,
|
||||
ModelSourceError,
|
||||
fetch_json,
|
||||
fetch_text,
|
||||
filter_weight_files,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover - typing only
|
||||
from .base import ModelSourceCache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: ModelScope runs two independent catalogues. ``modelscope.com`` is a
|
||||
#: redirect alias of the mainland site, but ``modelscope.ai`` is the
|
||||
#: *international* deployment with its own repository catalogue — a repository
|
||||
#: published on one is routinely absent from the other (``referall13/EM1``
|
||||
#: exists only on ``.ai``, ``jj3550945163/Krea-2-LORA`` only on ``.cn``). The
|
||||
#: host therefore decides which site, API and CDN a model belongs to, and the
|
||||
#: two deployments are registered as separate sources rather than folded into
|
||||
#: one id.
|
||||
_MAINLAND_HOSTS = r"modelscope\.(?:cn|com)"
|
||||
_INTERNATIONAL_HOSTS = r"modelscope\.ai"
|
||||
|
||||
#: Trailing view segments the site appends to a model URL; accepted verbatim
|
||||
#: when the user pastes a browser tab URL.
|
||||
_VIEW_SEGMENTS = r"(?:summary|files|model-file|readme|community|evaluation)?"
|
||||
|
||||
|
||||
def _url_patterns(hosts: str) -> tuple[re.Pattern[str], re.Pattern[str]]:
|
||||
"""Build the lenient and strict model-URL patterns for *hosts*."""
|
||||
|
||||
body = rf"https?://(?:www\.)?(?:{hosts})/models/(?P<id>[^/?#\s]+/[^/?#\s]+)"
|
||||
return re.compile(body), re.compile(rf"{body}/?{_VIEW_SEGMENTS}/?$")
|
||||
|
||||
|
||||
#: ``master`` is ModelScope's default branch; ``main`` is tried as a fallback
|
||||
#: for repos imported from Hugging Face.
|
||||
_REVISIONS = ("master", "main")
|
||||
|
||||
|
||||
class ModelScopeSource(ModelSource):
|
||||
"""ModelScope's mainland site (``modelscope.cn``).
|
||||
|
||||
``modelscope.com`` is accepted as an alias of it. The international
|
||||
deployment is :class:`ModelScopeIntlSource`; everything below is written in
|
||||
terms of ``base_url`` so both share one implementation.
|
||||
"""
|
||||
|
||||
platform = "modelscope"
|
||||
label = "ModelScope"
|
||||
supports_enrichment = True
|
||||
supports_download = True
|
||||
default_revision = "master"
|
||||
default_subdir = "modelscope"
|
||||
|
||||
#: Origin every outgoing URL is built from.
|
||||
base_url = "https://modelscope.cn"
|
||||
|
||||
url_pattern, strict_url_pattern = _url_patterns(_MAINLAND_HOSTS)
|
||||
|
||||
def canonical_url(self, source_id: str) -> str:
|
||||
return f"{self.base_url}/models/{source_id}"
|
||||
|
||||
def asset_base_url(self, source_id: str, revision: str = "") -> str:
|
||||
return (
|
||||
f"{self.base_url}/models/{source_id}/resolve/"
|
||||
f"{self.resolve_revision(revision)}"
|
||||
)
|
||||
|
||||
async def fetch_model_card(self, source_id: str) -> str:
|
||||
"""Fetch the model card, preferring the raw resolve URL."""
|
||||
|
||||
for revision in _REVISIONS:
|
||||
text = await fetch_text(
|
||||
f"{self.base_url}/models/{source_id}/resolve/{revision}/README.md"
|
||||
)
|
||||
if text:
|
||||
return text
|
||||
|
||||
# Fallback: the repo API proxies the same file and is reachable in
|
||||
# environments where the CDN resolve host is blocked.
|
||||
for revision in _REVISIONS:
|
||||
text = await fetch_text(
|
||||
f"{self.base_url}/api/v1/models/"
|
||||
f"{source_id}/repo?Revision={revision}&FilePath=README.md"
|
||||
)
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
async def fetch_model_card_context(
|
||||
self,
|
||||
source_id: str,
|
||||
filename: str = "",
|
||||
*,
|
||||
sha256: str = "",
|
||||
cache: Optional["ModelSourceCache"] = None,
|
||||
) -> ModelCardContext:
|
||||
"""Read the model-detail API that backs the ModelScope model page.
|
||||
|
||||
ModelScope splits a model card in two: ``README.md`` holds the
|
||||
long-form content, while the author's summary, the site-curated tags,
|
||||
and the per-file example images live only here. AIGC repositories
|
||||
frequently ship an auto-generated README ("the contributor provided
|
||||
no further description") and put everything useful in ``Description``,
|
||||
so enrichment that reads only the README comes back nearly empty.
|
||||
|
||||
The wanted file is identified by its sha256 when the caller knows it
|
||||
and by *filename* otherwise; see :func:`_matching_versions`. The
|
||||
images and trigger words returned belong to that exact
|
||||
``.safetensors`` — essential for collection repositories, where every
|
||||
checkpoint has its own sample image.
|
||||
|
||||
The detail payload describes the whole repository and is therefore
|
||||
shared across every file in it, so it is read through *cache* when the
|
||||
caller supplies one; only the per-file selection is redone.
|
||||
"""
|
||||
|
||||
data = await self._fetch_detail(source_id, cache=cache)
|
||||
if data is None:
|
||||
return ModelCardContext()
|
||||
return _build_card_context(data, filename, sha256)
|
||||
|
||||
async def _fetch_detail(
|
||||
self,
|
||||
source_id: str,
|
||||
*,
|
||||
cache: Optional["ModelSourceCache"] = None,
|
||||
) -> Optional[dict[str, Any]]:
|
||||
"""Fetch (or reuse) the model-detail payload for *source_id*."""
|
||||
|
||||
cache_key = (self.platform, "detail", source_id)
|
||||
if cache is not None and cache_key in cache.provider:
|
||||
return cache.provider[cache_key]
|
||||
|
||||
status, payload = await fetch_json(
|
||||
f"{self.base_url}/api/v1/models/{source_id}"
|
||||
)
|
||||
if status != 200 or not isinstance(payload, dict):
|
||||
logger.debug(
|
||||
"ModelScope detail API returned HTTP %s for %s", status, source_id
|
||||
)
|
||||
return None
|
||||
data = payload.get("Data")
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
|
||||
if cache is not None:
|
||||
cache.provider[cache_key] = data
|
||||
return data
|
||||
|
||||
async def list_files(
|
||||
self, source_id: str, revision: str = ""
|
||||
) -> list[dict]:
|
||||
"""List weight files via the repo files API.
|
||||
|
||||
``master`` is the only branch name the API accepts — even repos
|
||||
imported from Hugging Face are addressed as ``master`` (``main``
|
||||
returns 404) — so no fallback probing is done here.
|
||||
"""
|
||||
|
||||
revision = self.resolve_revision(revision)
|
||||
status, payload = await fetch_json(
|
||||
f"{self.base_url}/api/v1/models/"
|
||||
f"{source_id}/repo/files?Revision={revision}"
|
||||
)
|
||||
|
||||
if status == 404:
|
||||
raise ModelSourceError(f"Repository '{source_id}' not found", status=404)
|
||||
if status != 200 or not isinstance(payload, dict):
|
||||
raise ModelSourceError(
|
||||
f"ModelScope API error while listing '{source_id}' (HTTP {status})"
|
||||
)
|
||||
|
||||
entries = []
|
||||
for entry in (payload.get("Data") or {}).get("Files") or []:
|
||||
if not isinstance(entry, dict) or entry.get("Type") != "blob":
|
||||
continue
|
||||
entries.append((entry.get("Path", ""), entry.get("Size", 0) or 0))
|
||||
|
||||
return filter_weight_files(entries)
|
||||
|
||||
def file_download_url(
|
||||
self, source_id: str, filename: str, revision: str = ""
|
||||
) -> str:
|
||||
return (
|
||||
f"{self.base_url}/models/{source_id}/resolve/"
|
||||
f"{self.resolve_revision(revision)}/{filename}"
|
||||
)
|
||||
|
||||
def page_url_for_file(self, source_id: str, filename: str) -> str:
|
||||
return (
|
||||
f"{self.base_url}/models/{source_id}/file/view/"
|
||||
f"{self.default_revision}/{filename}"
|
||||
)
|
||||
|
||||
|
||||
class ModelScopeIntlSource(ModelScopeSource):
|
||||
"""ModelScope's international site (``modelscope.ai``).
|
||||
|
||||
A separate catalogue rather than a mirror, so it is registered under its
|
||||
own platform id: the two deployments must not share a version group, a
|
||||
"use default paths" directory, or a stored ``source_url``. The detail API,
|
||||
the file listing, the resolve URLs and the CDN redirect all behave exactly
|
||||
like the mainland site, which is why every URL here is derived from
|
||||
:attr:`base_url` instead of being duplicated.
|
||||
"""
|
||||
|
||||
platform = "modelscope-ai"
|
||||
label = "ModelScope (International)"
|
||||
default_subdir = "modelscope-ai"
|
||||
base_url = "https://www.modelscope.ai"
|
||||
|
||||
url_pattern, strict_url_pattern = _url_patterns(_INTERNATIONAL_HOSTS)
|
||||
|
||||
|
||||
__all__ = ["ModelScopeIntlSource", "ModelScopeSource"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model-detail API parsing helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Trigger-word values that mean "the author left this blank".
|
||||
_EMPTY_TRIGGER_VALUES = frozenset({"none", "null", "n/a"})
|
||||
|
||||
#: Repository tags that only restate what the model *is* (its library, task or
|
||||
#: framework) rather than what it depicts. ModelScope mixes both into the
|
||||
#: plain ``Tags`` list, and a card tagged "lora" or "text-to-image" is noise.
|
||||
_GENERIC_TAGS = frozenset(
|
||||
{
|
||||
"any-to-any",
|
||||
"checkpoint",
|
||||
"controlnet",
|
||||
"diffusers",
|
||||
"embedding",
|
||||
"image-text-to-text",
|
||||
"image-to-image",
|
||||
"image-to-video",
|
||||
"lora",
|
||||
"lycoris",
|
||||
"onnx",
|
||||
"pytorch",
|
||||
"safetensors",
|
||||
"tensorflow",
|
||||
"text-to-image",
|
||||
"text-to-speech",
|
||||
"text-to-video",
|
||||
"textual-inversion",
|
||||
"vae",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _clean_text(value: Any) -> str:
|
||||
"""Return a stripped string for *value*, or ``""`` for anything else."""
|
||||
|
||||
return value.strip() if isinstance(value, str) else ""
|
||||
|
||||
|
||||
def _first_string(value: Any) -> str:
|
||||
"""Return the first non-empty string in a list, or ``""``."""
|
||||
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
text = _clean_text(item)
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _build_card_context(
|
||||
data: dict[str, Any], filename: str, sha256: str = ""
|
||||
) -> ModelCardContext:
|
||||
"""Turn a model-detail payload into a :class:`ModelCardContext`.
|
||||
|
||||
Separated from the HTTP fetch so the repository-wide payload can be cached
|
||||
across the files of a collection repository while the per-file selection
|
||||
is still redone for each one.
|
||||
"""
|
||||
|
||||
context = ModelCardContext(
|
||||
description=_clean_text(data.get("Description")),
|
||||
model_name=_clean_text(data.get("Name")),
|
||||
model_name_localized=_clean_text(data.get("ChineseName")),
|
||||
license=_clean_text(data.get("License")),
|
||||
model_type=_clean_text(data.get("AigcType")),
|
||||
base_model=_first_string(data.get("BaseModel")),
|
||||
base_model_aliases=_base_model_aliases(data),
|
||||
official_tags=_official_tags(data),
|
||||
)
|
||||
|
||||
versions = _matching_versions(
|
||||
data.get("MuseInfo"),
|
||||
filename,
|
||||
digests=_file_digests(data),
|
||||
sha256=sha256,
|
||||
)
|
||||
if versions:
|
||||
context.version_name = _version_label(versions)
|
||||
context.example_images = _cover_image_urls(versions)
|
||||
context.trigger_words = _version_trigger_words(versions)
|
||||
return context
|
||||
|
||||
|
||||
def _base_model_aliases(data: dict[str, Any]) -> list[str]:
|
||||
"""Return the site's own names for the base model.
|
||||
|
||||
ModelScope publishes a link-style id (``krea/Krea-2-Turbo``) plus its
|
||||
internal architecture enums (``VisionFoundation: KREA_2``,
|
||||
``SubVisionFoundation: KREA_2_TURBO``). The enums are the better
|
||||
resolution hint because they normalise onto this system's canonical
|
||||
vocabulary, so they come first; the owner prefix is also stripped from
|
||||
the link-style ids.
|
||||
"""
|
||||
|
||||
aliases: list[str] = []
|
||||
for key in ("VisionFoundation", "SubVisionFoundation"):
|
||||
value = _clean_text(data.get(key))
|
||||
if value and value not in aliases:
|
||||
aliases.append(value)
|
||||
|
||||
base_models = data.get("BaseModel")
|
||||
if isinstance(base_models, list):
|
||||
for item in base_models:
|
||||
text = _clean_text(item)
|
||||
leaf = text.rsplit("/", 1)[-1] if text else ""
|
||||
if leaf and leaf not in aliases:
|
||||
aliases.append(leaf)
|
||||
return aliases
|
||||
|
||||
|
||||
def _official_tags(data: dict[str, Any]) -> list[str]:
|
||||
"""Return the content tags the site publishes for the repository.
|
||||
|
||||
``OfficialTags`` is ModelScope's curated content vocabulary and is
|
||||
preferred whenever it is populated. Plenty of AIGC repositories leave it
|
||||
empty and carry only the plain ``Tags`` list, which mixes content tags with
|
||||
framework and task categories; those categories are dropped so a card is
|
||||
not handed "lora" and "text-to-image" as if they described the model.
|
||||
"""
|
||||
|
||||
curated = _dedupe(_tag_values(data.get("OfficialTags")))
|
||||
if curated:
|
||||
return curated
|
||||
|
||||
generic = set(_GENERIC_TAGS)
|
||||
for value in (
|
||||
data.get("AigcType"),
|
||||
data.get("Libraries"),
|
||||
data.get("Frameworks"),
|
||||
):
|
||||
for item in value if isinstance(value, list) else [value]:
|
||||
text = _clean_text(item).lower()
|
||||
if text:
|
||||
generic.add(text)
|
||||
|
||||
return _dedupe(
|
||||
tag for tag in _tag_values(data.get("Tags")) if tag.lower() not in generic
|
||||
)
|
||||
|
||||
|
||||
def _tag_values(value: Any) -> list[str]:
|
||||
"""Return the tag strings from either shape ModelScope publishes.
|
||||
|
||||
``OfficialTags`` is a list of ``{"Tag": ..., "ChineseName": ...}`` dicts
|
||||
carrying an English value; the plain ``Tags`` list is already strings.
|
||||
"""
|
||||
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
tags: list[str] = []
|
||||
for entry in value:
|
||||
tag = _clean_text(entry.get("Tag") if isinstance(entry, dict) else entry)
|
||||
if tag:
|
||||
tags.append(tag)
|
||||
return tags
|
||||
|
||||
|
||||
def _dedupe(values: Iterable[str]) -> list[str]:
|
||||
"""Drop empties and repeats, keeping the first spelling seen."""
|
||||
|
||||
unique: list[str] = []
|
||||
for value in values:
|
||||
if value and value not in unique:
|
||||
unique.append(value)
|
||||
return unique
|
||||
|
||||
|
||||
def _version_files(version: dict[str, Any]) -> list[str]:
|
||||
"""Return the model filenames covered by one ``MuseInfo.versions`` entry.
|
||||
|
||||
The listing normally sits in ``stats.fileList``; some payloads only
|
||||
carry the same field as a JSON-encoded string under
|
||||
``modelVersion.stats``, so both shapes are accepted.
|
||||
"""
|
||||
|
||||
stats = version.get("stats")
|
||||
files = stats.get("fileList") if isinstance(stats, dict) else None
|
||||
|
||||
if not isinstance(files, list):
|
||||
model_version = version.get("modelVersion")
|
||||
raw = model_version.get("stats") if isinstance(model_version, dict) else None
|
||||
if isinstance(raw, str) and raw.strip():
|
||||
try:
|
||||
decoded = json.loads(raw)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
decoded = None
|
||||
if isinstance(decoded, dict):
|
||||
files = decoded.get("fileList")
|
||||
|
||||
if not isinstance(files, list):
|
||||
return []
|
||||
return [item for item in files if isinstance(item, str) and item]
|
||||
|
||||
|
||||
def _version_show_name(version: dict[str, Any]) -> str:
|
||||
"""Return the human-facing version label (e.g. ``c1-st1000``)."""
|
||||
|
||||
model_version = version.get("modelVersion")
|
||||
if not isinstance(model_version, dict):
|
||||
return ""
|
||||
return _clean_text(model_version.get("showName")).lower()
|
||||
|
||||
|
||||
def _version_label(versions: list[dict[str, Any]]) -> str:
|
||||
"""Return the first published version label, preserving its spelling.
|
||||
|
||||
Unlike :func:`_version_show_name` this is for display, so the label is
|
||||
not lowercased.
|
||||
"""
|
||||
|
||||
for version in versions:
|
||||
model_version = version.get("modelVersion")
|
||||
if not isinstance(model_version, dict):
|
||||
continue
|
||||
label = _clean_text(model_version.get("showName"))
|
||||
if label:
|
||||
return label
|
||||
return ""
|
||||
|
||||
|
||||
def _file_digests(data: dict[str, Any]) -> dict[str, str]:
|
||||
"""Return ``basename -> sha256`` for every published weight file.
|
||||
|
||||
``ModelInfos`` groups the repository's files by kind (``safetensor``,
|
||||
…) and records a real sha256 for each, which is what makes it possible to
|
||||
recognise a file the user has renamed.
|
||||
"""
|
||||
|
||||
digests: dict[str, str] = {}
|
||||
model_infos = data.get("ModelInfos")
|
||||
if not isinstance(model_infos, dict):
|
||||
return digests
|
||||
for info in model_infos.values():
|
||||
files = info.get("files") if isinstance(info, dict) else None
|
||||
if not isinstance(files, list):
|
||||
continue
|
||||
for entry in files:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
name = _clean_text(entry.get("name"))
|
||||
digest = _clean_text(entry.get("sha256"))
|
||||
if name and digest:
|
||||
digests.setdefault(os.path.basename(name).lower(), digest.lower())
|
||||
return digests
|
||||
|
||||
|
||||
def _matching_versions(
|
||||
muse_info: Any,
|
||||
filename: str,
|
||||
*,
|
||||
digests: dict[str, str] | None = None,
|
||||
sha256: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return the ``versions`` entries that publish the wanted model file.
|
||||
|
||||
Strategies, in order:
|
||||
|
||||
1. **sha256** — the file's content hash, looked up through
|
||||
:func:`_file_digests`. This is the only strategy that survives the
|
||||
user renaming the weights, which is common once a model is filed away.
|
||||
2. **Exact basename** against each version's ``stats.fileList``.
|
||||
3. **``showName`` inside the file stem**, which absorbs the naming drift
|
||||
ModelScope sometimes applies to uploaded weights.
|
||||
|
||||
A known-but-unmatched hash falls through to the filename strategies
|
||||
rather than giving up, in case the local file was re-encoded. All matches
|
||||
are returned so a file re-published across several versions contributes
|
||||
all of its example images. With no *filename* and no *sha256*, only an
|
||||
unambiguous single-version repository is used, because a per-file image
|
||||
must never be attributed to the wrong file.
|
||||
"""
|
||||
|
||||
if not isinstance(muse_info, dict):
|
||||
return []
|
||||
versions = muse_info.get("versions")
|
||||
if not isinstance(versions, list):
|
||||
return []
|
||||
entries = [entry for entry in versions if isinstance(entry, dict)]
|
||||
if not entries:
|
||||
return []
|
||||
|
||||
target_hash = (sha256 or "").strip().lower()
|
||||
if target_hash:
|
||||
known = digests or {}
|
||||
by_hash: list[dict[str, Any]] = []
|
||||
for version in entries:
|
||||
for path in _version_files(version):
|
||||
if known.get(os.path.basename(path).lower()) == target_hash:
|
||||
by_hash.append(version)
|
||||
break
|
||||
if by_hash:
|
||||
return by_hash
|
||||
|
||||
if not filename:
|
||||
return entries if len(entries) == 1 else []
|
||||
|
||||
target = os.path.basename(filename).strip().lower()
|
||||
if not target:
|
||||
return []
|
||||
stem = os.path.splitext(target)[0]
|
||||
|
||||
exact: list[dict[str, Any]] = []
|
||||
fuzzy: list[dict[str, Any]] = []
|
||||
for version in entries:
|
||||
files = {os.path.basename(path).lower() for path in _version_files(version)}
|
||||
if target in files:
|
||||
exact.append(version)
|
||||
continue
|
||||
show_name = _version_show_name(version)
|
||||
if show_name and show_name in stem:
|
||||
fuzzy.append(version)
|
||||
|
||||
return exact or fuzzy
|
||||
|
||||
|
||||
def _cover_image_urls(versions: list[dict[str, Any]]) -> list[str]:
|
||||
"""Collect the example-image URLs published by the given versions."""
|
||||
|
||||
urls: list[str] = []
|
||||
for version in versions:
|
||||
covers = version.get("coverImages")
|
||||
if not isinstance(covers, list):
|
||||
continue
|
||||
for cover in covers:
|
||||
if not isinstance(cover, dict):
|
||||
continue
|
||||
url = _clean_text(cover.get("url"))
|
||||
if url and url not in urls:
|
||||
urls.append(url)
|
||||
return urls
|
||||
|
||||
|
||||
def _version_trigger_words(versions: list[dict[str, Any]]) -> list[str]:
|
||||
"""Return the first non-empty trigger-word list across *versions*."""
|
||||
|
||||
for version in versions:
|
||||
model_version = version.get("modelVersion")
|
||||
raw = (
|
||||
model_version.get("triggerWords")
|
||||
if isinstance(model_version, dict)
|
||||
else None
|
||||
)
|
||||
words = _parse_trigger_words(raw)
|
||||
if words:
|
||||
return words
|
||||
return []
|
||||
|
||||
|
||||
def _parse_trigger_words(raw: Any) -> list[str]:
|
||||
"""Decode ModelScope's JSON-encoded trigger-word string list."""
|
||||
|
||||
if isinstance(raw, list):
|
||||
candidates = raw
|
||||
elif isinstance(raw, str) and raw.strip():
|
||||
try:
|
||||
decoded = json.loads(raw)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return []
|
||||
if not isinstance(decoded, list):
|
||||
return []
|
||||
candidates = decoded
|
||||
else:
|
||||
return []
|
||||
|
||||
words: list[str] = []
|
||||
for item in candidates:
|
||||
word = _clean_text(item)
|
||||
if not word or word.lower() in _EMPTY_TRIGGER_VALUES:
|
||||
continue
|
||||
if word not in words:
|
||||
words.append(word)
|
||||
return words
|
||||
@@ -0,0 +1,228 @@
|
||||
"""Registry and metadata helpers for external model sources.
|
||||
|
||||
The registry is the single place the rest of the codebase asks "which site
|
||||
is this URL from?", "what is this model's source?", and "can we enrich it?".
|
||||
Import from :mod:`py.services.model_sources` rather than this module
|
||||
directly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, Mapping, Optional
|
||||
|
||||
from .base import GROUP_PREFIXES, ModelSource, SourceRef, clean_source_url
|
||||
from .huggingface import HuggingFaceSource
|
||||
from .modelscope import ModelScopeIntlSource, ModelScopeSource
|
||||
from .tensorart import TensorArtSource
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: Order matters only for disambiguation; the URL patterns are disjoint.
|
||||
#: ``modelscope.ai`` is a separate catalogue from ``modelscope.cn`` rather than
|
||||
#: an alias, which is why it gets its own entry (see ``modelscope.py``).
|
||||
_SOURCES: tuple[ModelSource, ...] = (
|
||||
HuggingFaceSource(),
|
||||
ModelScopeSource(),
|
||||
ModelScopeIntlSource(),
|
||||
TensorArtSource(),
|
||||
)
|
||||
|
||||
_BY_PLATFORM: Dict[str, ModelSource] = {s.platform: s for s in _SOURCES}
|
||||
|
||||
#: Metadata keys that carry the canonical external-source identity.
|
||||
SOURCE_PLATFORM_FIELD = "source_platform"
|
||||
SOURCE_URL_FIELD = "source_url"
|
||||
#: Legacy field kept as a read/write alias for Hugging Face models so that
|
||||
#: older sidecars, cached rows, and third-party consumers keep working.
|
||||
LEGACY_HF_URL_FIELD = "hf_url"
|
||||
|
||||
|
||||
def list_sources() -> list[ModelSource]:
|
||||
"""Return every known model source."""
|
||||
|
||||
return list(_SOURCES)
|
||||
|
||||
|
||||
def get_source(platform: Optional[str]) -> Optional[ModelSource]:
|
||||
"""Return the source registered for *platform*, or ``None``."""
|
||||
|
||||
if not platform or not isinstance(platform, str):
|
||||
return None
|
||||
return _BY_PLATFORM.get(platform.strip().lower())
|
||||
|
||||
|
||||
def source_label(platform: Optional[str], default: str = "") -> str:
|
||||
"""Return the human-readable label for *platform*."""
|
||||
|
||||
source = get_source(platform)
|
||||
return source.label if source else default
|
||||
|
||||
|
||||
def downloadable_sources() -> list[ModelSource]:
|
||||
"""Return the sources whose repositories can be downloaded directly."""
|
||||
|
||||
return [source for source in _SOURCES if source.supports_download]
|
||||
|
||||
|
||||
def get_download_source(platform: Optional[str]) -> Optional[ModelSource]:
|
||||
"""Return the source for *platform*, but only when it supports downloads."""
|
||||
|
||||
source = get_source(platform)
|
||||
if source is None or not source.supports_download:
|
||||
return None
|
||||
return source
|
||||
|
||||
|
||||
def detect_source(url: Optional[str], *, strict: bool = False) -> Optional[SourceRef]:
|
||||
"""Return the :class:`SourceRef` for *url*, or ``None`` if unsupported."""
|
||||
|
||||
if not url or not isinstance(url, str):
|
||||
return None
|
||||
for source in _SOURCES:
|
||||
ref = source.ref(url, strict=strict)
|
||||
if ref is not None:
|
||||
return ref
|
||||
return None
|
||||
|
||||
|
||||
def resolve_source_ref(metadata: Mapping[str, Any]) -> Optional[SourceRef]:
|
||||
"""Return the source reference described by a model's metadata.
|
||||
|
||||
Handles all three storage states found in the wild:
|
||||
|
||||
1. ``source_url`` + ``source_platform`` (current format)
|
||||
2. ``hf_url`` only (legacy Hugging Face storage)
|
||||
3. ``hf_url`` plus a newer ``source_url`` (both written by older builds)
|
||||
"""
|
||||
|
||||
if not isinstance(metadata, Mapping):
|
||||
return None
|
||||
|
||||
platform = clean_source_url(metadata.get(SOURCE_PLATFORM_FIELD)).lower()
|
||||
url = clean_source_url(metadata.get(SOURCE_URL_FIELD))
|
||||
legacy = clean_source_url(metadata.get(LEGACY_HF_URL_FIELD))
|
||||
|
||||
source = get_source(platform)
|
||||
if url:
|
||||
if source is not None:
|
||||
ref = source.ref(url)
|
||||
if ref is not None:
|
||||
return ref
|
||||
ref = detect_source(url)
|
||||
if ref is not None:
|
||||
return ref
|
||||
# Unknown platform but a URL is present: keep it addressable.
|
||||
return SourceRef(platform=platform or "unknown", source_id="", url=url)
|
||||
|
||||
if legacy:
|
||||
return detect_source(legacy)
|
||||
return None
|
||||
|
||||
|
||||
def normalize_metadata_source(metadata: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Normalise the external-source fields on *metadata* in place.
|
||||
|
||||
Guarantees that ``source_url``/``source_platform`` are present and
|
||||
consistent, and that ``hf_url`` mirrors ``source_url`` for Hugging Face
|
||||
models (never for other platforms, so a stale alias can't make a
|
||||
ModelScope model look like a Hugging Face one).
|
||||
|
||||
Returns the same dict for convenient chaining.
|
||||
"""
|
||||
|
||||
if not isinstance(metadata, dict):
|
||||
return metadata
|
||||
|
||||
platform = clean_source_url(metadata.get(SOURCE_PLATFORM_FIELD)).lower()
|
||||
url = clean_source_url(metadata.get(SOURCE_URL_FIELD))
|
||||
legacy = clean_source_url(metadata.get(LEGACY_HF_URL_FIELD))
|
||||
|
||||
source = get_source(platform)
|
||||
ref: Optional[SourceRef] = None
|
||||
|
||||
if url:
|
||||
ref = source.ref(url) if source is not None else None
|
||||
if ref is None:
|
||||
ref = detect_source(url)
|
||||
elif legacy:
|
||||
ref = detect_source(legacy)
|
||||
|
||||
if ref is not None and ref.source_id:
|
||||
platform = ref.platform
|
||||
url = ref.url or url
|
||||
|
||||
if platform:
|
||||
metadata[SOURCE_PLATFORM_FIELD] = platform
|
||||
else:
|
||||
metadata.setdefault(SOURCE_PLATFORM_FIELD, "")
|
||||
|
||||
metadata[SOURCE_URL_FIELD] = url
|
||||
|
||||
# Keep the legacy alias in sync, but only for Hugging Face.
|
||||
if url and platform == "huggingface":
|
||||
metadata[LEGACY_HF_URL_FIELD] = url
|
||||
elif LEGACY_HF_URL_FIELD in metadata and platform and platform != "huggingface":
|
||||
metadata[LEGACY_HF_URL_FIELD] = ""
|
||||
elif legacy and not url:
|
||||
metadata[LEGACY_HF_URL_FIELD] = legacy
|
||||
|
||||
return metadata
|
||||
|
||||
|
||||
def has_external_source(item: Mapping[str, Any]) -> bool:
|
||||
"""Return ``True`` when *item* is linked to any external model site."""
|
||||
|
||||
if not isinstance(item, Mapping):
|
||||
return False
|
||||
return bool(
|
||||
clean_source_url(item.get(SOURCE_URL_FIELD))
|
||||
or clean_source_url(item.get(LEGACY_HF_URL_FIELD))
|
||||
)
|
||||
|
||||
|
||||
def get_source_platform(item: Mapping[str, Any]) -> str:
|
||||
"""Return the platform id stored on *item* (may be empty)."""
|
||||
|
||||
if not isinstance(item, Mapping):
|
||||
return ""
|
||||
platform = clean_source_url(item.get(SOURCE_PLATFORM_FIELD)).lower()
|
||||
if platform:
|
||||
return platform
|
||||
ref = resolve_source_ref(item)
|
||||
return ref.platform if ref else ""
|
||||
|
||||
|
||||
def source_group_key(item: Mapping[str, Any]) -> Optional[str]:
|
||||
"""Return the version-group key for *item*, or ``None``.
|
||||
|
||||
Hugging Face keeps the historical ``hf:{owner}/{repo}`` shape; other
|
||||
platforms use their own short prefix (see :data:`GROUP_PREFIXES`).
|
||||
"""
|
||||
|
||||
ref = resolve_source_ref(item)
|
||||
if ref is None or not ref.source_id:
|
||||
return None
|
||||
source = get_source(ref.platform)
|
||||
if source is None:
|
||||
return None
|
||||
return source.group_key(ref.source_id)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GROUP_PREFIXES",
|
||||
"LEGACY_HF_URL_FIELD",
|
||||
"SOURCE_PLATFORM_FIELD",
|
||||
"SOURCE_URL_FIELD",
|
||||
"detect_source",
|
||||
"downloadable_sources",
|
||||
"get_download_source",
|
||||
"get_source",
|
||||
"get_source_platform",
|
||||
"has_external_source",
|
||||
"list_sources",
|
||||
"normalize_metadata_source",
|
||||
"resolve_source_ref",
|
||||
"source_group_key",
|
||||
"source_label",
|
||||
]
|
||||
@@ -0,0 +1,56 @@
|
||||
"""TensorArt model source (link / provenance only).
|
||||
|
||||
TensorArt support is intentionally limited to *linking* a model to its
|
||||
TensorArt page. Automatic metadata extraction is not possible without a
|
||||
user session:
|
||||
|
||||
* ``tensor.art`` sits behind a Cloudflare managed challenge, so plain
|
||||
HTTP clients (aiohttp, requests, curl) receive ``403 "Just a moment..."``.
|
||||
* Its internal API (``ap-east-1.tensorart.cloud`` / ``cn.tensorart.net``)
|
||||
answers every ``/v1/model/*`` route with
|
||||
``{"code":100002,"message":"invalid authorization header"}``.
|
||||
* The official TAMS API requires an AccessKey/SecretKey pair and request
|
||||
signatures, which is a poor fit for a "paste a URL" workflow.
|
||||
|
||||
``supports_enrichment`` is therefore ``False``: the agent pipeline skips
|
||||
these models with an explicit reason instead of failing silently, and the
|
||||
UI keeps showing the "View on TensorArt" link. ``tusi.cn`` is TensorArt's
|
||||
Chinese mirror and is accepted as the same platform.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from .base import ModelSource
|
||||
|
||||
_DOMAINS = r"(?:tensor\.art|tusi\.cn)"
|
||||
|
||||
_URL_PATTERN = re.compile(
|
||||
rf"https?://(?:www\.)?{_DOMAINS}/models/(?P<id>\d+)"
|
||||
)
|
||||
|
||||
_STRICT_URL_PATTERN = re.compile(
|
||||
rf"https?://(?:www\.)?{_DOMAINS}/models/(?P<id>\d+)(?:/[^/?#\s]+)?/?$"
|
||||
)
|
||||
|
||||
|
||||
class TensorArtSource(ModelSource):
|
||||
"""TensorArt (``tensor.art``)."""
|
||||
|
||||
platform = "tensorart"
|
||||
label = "TensorArt"
|
||||
supports_enrichment = False
|
||||
supports_download = False
|
||||
url_pattern = _URL_PATTERN
|
||||
strict_url_pattern = _STRICT_URL_PATTERN
|
||||
|
||||
def canonical_url(self, source_id: str) -> str:
|
||||
return f"https://tensor.art/models/{source_id}"
|
||||
|
||||
def asset_base_url(self, source_id: str, revision: str = "") -> str:
|
||||
# Unreachable today: enrichment is disabled for this platform.
|
||||
return f"https://tensor.art/models/{source_id}"
|
||||
|
||||
|
||||
__all__ = ["TensorArtSource"]
|
||||
@@ -0,0 +1,81 @@
|
||||
import os
|
||||
import logging
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from .base_model_service import BaseModelService
|
||||
from .auto_tag_service import extract_auto_tags
|
||||
from ..utils.models import OtherModelMetadata
|
||||
from ..config import config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OtherModelService(BaseModelService):
|
||||
"""Other-model-specific service implementation (VAE, upscaler, text encoder, ...)"""
|
||||
|
||||
def __init__(self, scanner, update_service=None):
|
||||
"""Initialize Other-model service
|
||||
|
||||
Args:
|
||||
scanner: Other-model scanner instance
|
||||
update_service: Optional service for remote update tracking.
|
||||
"""
|
||||
super().__init__("other", scanner, OtherModelMetadata, update_service=update_service)
|
||||
|
||||
async def format_response(self, model_data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""Format other-model data for API response.
|
||||
|
||||
Returns None when the entry is missing critical fields (corrupted cache
|
||||
row), so the handler layer can filter it out. See issue #730.
|
||||
"""
|
||||
# Guard against corrupted cache entries missing critical fields
|
||||
file_path = model_data.get("file_path")
|
||||
if not file_path or not isinstance(file_path, str):
|
||||
logger.warning(
|
||||
"Skipping corrupted other-model entry (missing file_path): %s",
|
||||
model_data.get("file_name", "<unknown>"),
|
||||
)
|
||||
return None
|
||||
|
||||
# Get sub_type from cache entry (new canonical field)
|
||||
sub_type = model_data.get("sub_type", "vae")
|
||||
|
||||
file_name = model_data.get("file_name") or ""
|
||||
model_name = model_data.get("model_name") or file_name
|
||||
folder = model_data.get("folder") or ""
|
||||
|
||||
return {
|
||||
"model_name": model_name,
|
||||
"file_name": file_name,
|
||||
"preview_url": config.get_preview_static_url(model_data.get("preview_url", "")),
|
||||
"preview_nsfw_level": model_data.get("preview_nsfw_level", 0),
|
||||
"base_model": model_data.get("base_model", ""),
|
||||
"folder": folder,
|
||||
"sha256": model_data.get("sha256", ""),
|
||||
"autov3": model_data.get("autov3"),
|
||||
"file_path": file_path.replace(os.sep, "/"),
|
||||
"file_size": model_data.get("size", 0),
|
||||
"modified": model_data.get("modified", ""),
|
||||
"tags": model_data.get("tags", []),
|
||||
"from_civitai": model_data.get("from_civitai", True),
|
||||
"notes": model_data.get("notes", ""),
|
||||
"sub_type": sub_type,
|
||||
"favorite": model_data.get("favorite", False),
|
||||
"exclude": bool(model_data.get("exclude", False)),
|
||||
"update_available": bool(model_data.get("update_available", False)),
|
||||
"skip_metadata_refresh": bool(model_data.get("skip_metadata_refresh", False)),
|
||||
"civitai": self.filter_civitai_data(model_data.get("civitai", {}), minimal=True),
|
||||
"auto_tags": model_data.get("auto_tags") or extract_auto_tags(model_data),
|
||||
"version_count": model_data.get("version_count"),
|
||||
"source_platform": model_data.get("source_platform", ""),
|
||||
"source_url": model_data.get("source_url", ""),
|
||||
"hf_url": model_data.get("hf_url", ""),
|
||||
}
|
||||
|
||||
def find_duplicate_hashes(self) -> Dict[str, Any]:
|
||||
"""Find other models with duplicate SHA256 hashes"""
|
||||
return self.scanner._hash_index.get_duplicate_hashes()
|
||||
|
||||
def find_duplicate_filenames(self) -> Dict[str, Any]:
|
||||
"""Find other models with conflicting filenames"""
|
||||
return self.scanner._hash_index.get_duplicate_filenames()
|
||||
@@ -0,0 +1,478 @@
|
||||
# pyright: reportImportCycles=false
|
||||
# Lazy (function-local) imports still count as static edges in basedpyright's
|
||||
# reportImportCycles, so the ServiceRegistry singleton pattern necessarily forms
|
||||
# import cycles. Breaking them would require an architectural refactor.
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..utils.models import OtherModelMetadata
|
||||
from ..utils.file_utils import find_preview_file, normalize_path, calculate_autov3
|
||||
from ..utils.metadata_manager import MetadataManager
|
||||
from ..config import config
|
||||
from .model_scanner import ModelScanner, _is_excluded_dir
|
||||
from .model_hash_index import ModelHashIndex
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OtherScanner(ModelScanner):
|
||||
"""Service for scanning and managing "other" model files.
|
||||
|
||||
Aggregates every enabled folder_paths category from
|
||||
OTHER_MODEL_FOLDER_SUBTYPES (VAE, upscalers, text encoders, CLIP vision,
|
||||
opt-in ControlNet) into one scanner; sub_type is derived from the root
|
||||
containing the file (mirrors CheckpointScanner's checkpoints/unet split).
|
||||
|
||||
Hashing is lazy (checkpoint-style): text encoders can be ~10 GB, so the
|
||||
initial scan records hash_status="pending" and the SHA256 is computed
|
||||
on-demand via calculate_hash_for_model (e.g. when fetching CivitAI
|
||||
metadata).
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Same extension set as CheckpointScanner (ComfyUI's
|
||||
# supported_pt_extensions plus ".gguf").
|
||||
file_extensions = {
|
||||
".ckpt",
|
||||
".pt",
|
||||
".pt2",
|
||||
".bin",
|
||||
".pth",
|
||||
".safetensors",
|
||||
".pkl",
|
||||
".sft",
|
||||
".gguf",
|
||||
}
|
||||
super().__init__(
|
||||
model_type="other",
|
||||
model_class=OtherModelMetadata,
|
||||
file_extensions=file_extensions,
|
||||
hash_index=ModelHashIndex(),
|
||||
)
|
||||
if not hasattr(self, "_hash_calculation_lock"):
|
||||
self._hash_calculation_lock = asyncio.Lock()
|
||||
self._hash_calculation_tasks: dict[str, asyncio.Task[Optional[str]]] = {}
|
||||
|
||||
async def _create_default_metadata(
|
||||
self, file_path: str
|
||||
) -> Optional[OtherModelMetadata]:
|
||||
"""Create default metadata without calculating hash (lazy hash).
|
||||
|
||||
Other models include multi-GB text encoders, so hash calculation is
|
||||
deferred until on-demand (e.g. CivitAI metadata fetch).
|
||||
"""
|
||||
try:
|
||||
real_path = os.path.realpath(file_path)
|
||||
if not os.path.exists(real_path):
|
||||
logger.error(f"File not found: {file_path}")
|
||||
return None
|
||||
|
||||
base_name = os.path.splitext(os.path.basename(file_path))[0]
|
||||
dir_path = os.path.dirname(file_path)
|
||||
|
||||
# Find preview image
|
||||
preview_url = find_preview_file(base_name, dir_path)
|
||||
|
||||
# AutoV3 reads only the safetensors header, so it is cheap even for
|
||||
# large files; record the checked state at creation time ("" =
|
||||
# checked but unavailable).
|
||||
autov3 = calculate_autov3(real_path)
|
||||
|
||||
# Create metadata WITHOUT calculating hash
|
||||
metadata = OtherModelMetadata(
|
||||
file_name=base_name,
|
||||
model_name=base_name,
|
||||
file_path=normalize_path(file_path),
|
||||
size=os.path.getsize(real_path),
|
||||
modified=datetime.now().timestamp(),
|
||||
sha256="", # Empty hash - will be calculated on-demand
|
||||
base_model="Unknown",
|
||||
preview_url=normalize_path(preview_url),
|
||||
tags=[],
|
||||
modelDescription="",
|
||||
sub_type=self.resolve_sub_type_for_path(file_path) or "vae",
|
||||
from_civitai=False, # Mark as local model since no hash yet
|
||||
hash_status="pending", # Mark hash as pending
|
||||
autov3=autov3 or "",
|
||||
)
|
||||
|
||||
# Save the created metadata
|
||||
logger.info(f"Creating other-model metadata (hash pending) for {file_path}")
|
||||
await MetadataManager.save_metadata(file_path, metadata)
|
||||
|
||||
return metadata
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error creating default other-model metadata for {file_path}: {e}"
|
||||
)
|
||||
return None
|
||||
|
||||
async def calculate_hash_for_model(self, file_path: str) -> Optional[str]:
|
||||
"""Calculate hash for a model on-demand with per-file singleflight.
|
||||
|
||||
Args:
|
||||
file_path: Path to the model file
|
||||
|
||||
Returns:
|
||||
SHA256 hash string, or None if calculation failed
|
||||
"""
|
||||
try:
|
||||
real_path = os.path.realpath(file_path)
|
||||
if not os.path.exists(real_path):
|
||||
logger.error(f"File not found for hash calculation: {file_path}")
|
||||
return None
|
||||
|
||||
metadata, _ = await MetadataManager.load_metadata(
|
||||
file_path, self.model_class
|
||||
)
|
||||
if (
|
||||
metadata is not None
|
||||
and metadata.hash_status == "completed"
|
||||
and metadata.sha256
|
||||
):
|
||||
# Ensure the in-memory hash index is populated even when
|
||||
# the hash was already computed and persisted to the metadata
|
||||
# file. Without this, usage tracking (and any other caller
|
||||
# that queries get_hash_by_filename first) will miss on every
|
||||
# lookup and keep calling back into this method, creating a
|
||||
# tight loop that never populates the index.
|
||||
self._hash_index.add_entry(
|
||||
metadata.sha256.lower(),
|
||||
file_path,
|
||||
getattr(metadata, "autov3", None) or None,
|
||||
)
|
||||
return metadata.sha256
|
||||
|
||||
async with self._hash_calculation_lock:
|
||||
metadata, _ = await MetadataManager.load_metadata(
|
||||
file_path, self.model_class
|
||||
)
|
||||
if (
|
||||
metadata is not None
|
||||
and metadata.hash_status == "completed"
|
||||
and metadata.sha256
|
||||
):
|
||||
self._hash_index.add_entry(
|
||||
metadata.sha256.lower(),
|
||||
file_path,
|
||||
getattr(metadata, "autov3", None) or None,
|
||||
)
|
||||
return metadata.sha256
|
||||
|
||||
task = self._hash_calculation_tasks.get(real_path)
|
||||
if task is None:
|
||||
task = asyncio.create_task(
|
||||
self._run_hash_calculation_task(file_path, real_path)
|
||||
)
|
||||
self._hash_calculation_tasks[real_path] = task
|
||||
|
||||
return await asyncio.shield(task)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error calculating hash for {file_path}: {e}")
|
||||
return None
|
||||
|
||||
async def _run_hash_calculation_task(
|
||||
self, file_path: str, real_path: str
|
||||
) -> Optional[str]:
|
||||
"""Run a hash calculation task and remove it from the in-flight map."""
|
||||
try:
|
||||
return await self._calculate_hash_for_model_uncached(file_path, real_path)
|
||||
finally:
|
||||
task = asyncio.current_task()
|
||||
async with self._hash_calculation_lock:
|
||||
if self._hash_calculation_tasks.get(real_path) is task:
|
||||
del self._hash_calculation_tasks[real_path]
|
||||
|
||||
async def _calculate_hash_for_model_uncached(
|
||||
self, file_path: str, real_path: str
|
||||
) -> Optional[str]:
|
||||
"""Calculate hash for a model without checking in-flight tasks."""
|
||||
from ..utils.file_utils import calculate_sha256
|
||||
|
||||
try:
|
||||
# Load current metadata
|
||||
metadata, should_skip = await MetadataManager.load_metadata(
|
||||
file_path, self.model_class
|
||||
)
|
||||
if metadata is None:
|
||||
if should_skip:
|
||||
logger.error(f"Invalid metadata found for {file_path}")
|
||||
return None
|
||||
created_metadata = await self._create_default_metadata(file_path)
|
||||
if created_metadata is None:
|
||||
logger.error(f"No metadata found for {file_path}")
|
||||
return None
|
||||
metadata = created_metadata
|
||||
|
||||
# Check if hash is already calculated
|
||||
if metadata.hash_status == "completed" and metadata.sha256:
|
||||
# Populate the in-memory hash index even for pre-computed
|
||||
# hashes, mirroring the fix in calculate_hash_for_model.
|
||||
self._hash_index.add_entry(
|
||||
metadata.sha256.lower(),
|
||||
file_path,
|
||||
getattr(metadata, "autov3", None) or None,
|
||||
)
|
||||
return metadata.sha256
|
||||
|
||||
# Update status to calculating
|
||||
metadata.hash_status = "calculating"
|
||||
await MetadataManager.save_metadata(file_path, metadata)
|
||||
|
||||
# Calculate hash
|
||||
logger.info(f"Calculating hash for other model: {file_path}")
|
||||
sha256 = await calculate_sha256(real_path)
|
||||
|
||||
# Update metadata with hash
|
||||
metadata.sha256 = sha256
|
||||
metadata.hash_status = "completed"
|
||||
await MetadataManager.save_metadata(file_path, metadata)
|
||||
|
||||
# Update hash index
|
||||
self._hash_index.add_entry(
|
||||
sha256.lower(),
|
||||
file_path,
|
||||
getattr(metadata, "autov3", None) or None,
|
||||
)
|
||||
|
||||
# Update the in-memory cache entry so that subsequent
|
||||
# _persist_current_cache / _save_persistent_cache calls
|
||||
# write the hash back to the SQLite models table. Without
|
||||
# this the hash only lives in the metadata file and the
|
||||
# in-memory hash index, both of which are lost across
|
||||
# restarts, causing the same re-computation loop on the
|
||||
# next session.
|
||||
if self._cache is not None and self._cache.raw_data:
|
||||
for entry in self._cache.raw_data:
|
||||
if entry.get("file_path") == file_path:
|
||||
entry["sha256"] = sha256.lower()
|
||||
entry["hash_status"] = "completed"
|
||||
self.bump_cache_version()
|
||||
break
|
||||
|
||||
logger.info(f"Hash calculated for other model: {file_path}")
|
||||
return sha256
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error calculating hash for {file_path}: {e}")
|
||||
# Update status to failed
|
||||
try:
|
||||
metadata, _ = await MetadataManager.load_metadata(
|
||||
file_path, self.model_class
|
||||
)
|
||||
if metadata:
|
||||
metadata.hash_status = "failed"
|
||||
await MetadataManager.save_metadata(file_path, metadata)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
async def calculate_all_pending_hashes(
|
||||
self, progress_callback=None
|
||||
) -> Dict[str, int]:
|
||||
"""Calculate hashes for all other models with pending hash status.
|
||||
|
||||
If cache is not initialized, scans filesystem directly for metadata files
|
||||
with hash_status != 'completed'.
|
||||
|
||||
Args:
|
||||
progress_callback: Optional callback(progress, total, current_file)
|
||||
|
||||
Returns:
|
||||
Dict with 'completed', 'failed', 'total' counts
|
||||
"""
|
||||
# Try to get from cache first
|
||||
cache = await self.get_cached_data()
|
||||
|
||||
if cache and cache.raw_data:
|
||||
# Use cache if available
|
||||
pending_models = [
|
||||
item
|
||||
for item in cache.raw_data
|
||||
if item.get("hash_status") != "completed" or not item.get("sha256")
|
||||
]
|
||||
else:
|
||||
# Cache not initialized, scan filesystem directly
|
||||
pending_models = await self._find_pending_models_from_filesystem()
|
||||
|
||||
if not pending_models:
|
||||
return {"completed": 0, "failed": 0, "total": 0}
|
||||
|
||||
total = len(pending_models)
|
||||
completed = 0
|
||||
failed = 0
|
||||
|
||||
for i, model_data in enumerate(pending_models):
|
||||
file_path = model_data.get("file_path")
|
||||
if not file_path:
|
||||
continue
|
||||
|
||||
try:
|
||||
sha256 = await self.calculate_hash_for_model(file_path)
|
||||
if sha256:
|
||||
completed += 1
|
||||
else:
|
||||
failed += 1
|
||||
except Exception as e:
|
||||
logger.error(f"Error calculating hash for {file_path}: {e}")
|
||||
failed += 1
|
||||
|
||||
if progress_callback:
|
||||
try:
|
||||
await progress_callback(i + 1, total, file_path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {"completed": completed, "failed": failed, "total": total}
|
||||
|
||||
async def _find_pending_models_from_filesystem(self) -> List[Dict[str, Any]]:
|
||||
"""Scan filesystem for other-model metadata files with pending hash status."""
|
||||
pending_models = []
|
||||
|
||||
for root_path in self.get_model_roots():
|
||||
if not os.path.exists(root_path):
|
||||
continue
|
||||
|
||||
for dirpath, dirnames, filenames in os.walk(root_path):
|
||||
dirnames[:] = [d for d in dirnames if not _is_excluded_dir(d)]
|
||||
for filename in filenames:
|
||||
if not filename.endswith(".metadata.json"):
|
||||
continue
|
||||
|
||||
metadata_path = os.path.join(dirpath, filename)
|
||||
try:
|
||||
with open(metadata_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Check if hash is pending
|
||||
hash_status = data.get("hash_status", "completed")
|
||||
sha256 = data.get("sha256", "")
|
||||
|
||||
if hash_status != "completed" or not sha256:
|
||||
# Find corresponding model file
|
||||
model_name = filename.replace(".metadata.json", "")
|
||||
model_path = None
|
||||
|
||||
# Look for model file with matching name
|
||||
for ext in self.file_extensions:
|
||||
potential_path = os.path.join(dirpath, model_name + ext)
|
||||
if os.path.exists(potential_path):
|
||||
model_path = potential_path
|
||||
break
|
||||
|
||||
if model_path:
|
||||
pending_models.append(
|
||||
{
|
||||
"file_path": model_path.replace(os.sep, "/"),
|
||||
"hash_status": hash_status,
|
||||
"sha256": sha256,
|
||||
**{
|
||||
k: v
|
||||
for k, v in data.items()
|
||||
if k
|
||||
not in [
|
||||
"file_path",
|
||||
"hash_status",
|
||||
"sha256",
|
||||
]
|
||||
},
|
||||
}
|
||||
)
|
||||
except (json.JSONDecodeError, Exception) as e:
|
||||
logger.debug(
|
||||
f"Error reading metadata file {metadata_path}: {e}"
|
||||
)
|
||||
continue
|
||||
|
||||
return pending_models
|
||||
|
||||
def _root_sub_type_map(self) -> Dict[str, str]:
|
||||
"""Return the configured business root -> sub_type map."""
|
||||
root_map = getattr(config, "other_root_subtypes", None)
|
||||
return root_map if isinstance(root_map, dict) else {}
|
||||
|
||||
def _resolve_sub_type(self, root_path: Optional[str]) -> Optional[str]:
|
||||
"""Resolve the sub_type for a configured root path."""
|
||||
if not root_path:
|
||||
return None
|
||||
|
||||
normalized_root = self._normalize_path_value(root_path)
|
||||
for root, sub_type in self._root_sub_type_map().items():
|
||||
if self._normalize_path_value(root) == normalized_root:
|
||||
return sub_type
|
||||
|
||||
return None
|
||||
|
||||
def resolve_sub_type_for_path(self, file_path: Optional[str]) -> Optional[str]:
|
||||
"""Resolve sub_type from the configured root that contains the file.
|
||||
|
||||
Uses the longest-prefix match so nested roots (e.g. a controlnet root
|
||||
inside a vae root) resolve to the most specific category.
|
||||
"""
|
||||
normalized_path = self._normalize_path_value(file_path)
|
||||
if not normalized_path:
|
||||
return None
|
||||
|
||||
best_length = 0
|
||||
best_sub_type: Optional[str] = None
|
||||
for root, sub_type in self._root_sub_type_map().items():
|
||||
normalized_root = self._normalize_path_value(root)
|
||||
if not normalized_root:
|
||||
continue
|
||||
if (
|
||||
normalized_path == normalized_root
|
||||
or normalized_path.startswith(f"{normalized_root}/")
|
||||
) and len(normalized_root) > best_length:
|
||||
best_length = len(normalized_root)
|
||||
best_sub_type = sub_type
|
||||
|
||||
return best_sub_type
|
||||
|
||||
def adjust_metadata(self, metadata, file_path, root_path):
|
||||
"""Adjust metadata during scanning to set sub_type."""
|
||||
sub_type = self._resolve_sub_type(root_path) or self.resolve_sub_type_for_path(
|
||||
file_path
|
||||
)
|
||||
if sub_type:
|
||||
metadata.sub_type = sub_type
|
||||
return metadata
|
||||
|
||||
def adjust_cached_entry(self, entry: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Adjust entries loaded from the persisted cache to ensure sub_type is set.
|
||||
|
||||
sub_type is location-derived: it is re-derived on cache load, never
|
||||
trusted from the persisted snapshot.
|
||||
"""
|
||||
sub_type = self.resolve_sub_type_for_path(entry.get("file_path"))
|
||||
if sub_type:
|
||||
entry["sub_type"] = sub_type
|
||||
return entry
|
||||
|
||||
def _should_keep_cached_entry(self, entry: Dict[str, Any]) -> bool:
|
||||
"""Drop persisted entries whose folder is no longer a managed root.
|
||||
|
||||
sub_type is location-derived and config only maps enabled roots, so a
|
||||
file under a disabled sub_type - or under any other root while the
|
||||
feature is off - resolves to None here and is filtered out while the
|
||||
persisted cache is hydrated.
|
||||
"""
|
||||
return self.resolve_sub_type_for_path(entry.get("file_path")) is not None
|
||||
|
||||
def get_model_roots(self) -> List[str]:
|
||||
"""Get other-model root directories"""
|
||||
roots: List[str] = []
|
||||
roots.extend(config.other_roots or [])
|
||||
# Remove duplicates while preserving order
|
||||
seen: set[str] = set()
|
||||
unique_roots: List[str] = []
|
||||
for root in roots:
|
||||
if root and root not in seen:
|
||||
seen.add(root)
|
||||
unique_roots.append(root)
|
||||
return unique_roots
|
||||
@@ -59,6 +59,7 @@ _MODEL_TYPE_PAGE_MAP = {
|
||||
"lora": "loras",
|
||||
"checkpoint": "checkpoints",
|
||||
"embedding": "embeddings",
|
||||
"other": "other",
|
||||
}
|
||||
|
||||
# Module-level alias so tests can spy on timer task creation without patching
|
||||
@@ -983,6 +984,7 @@ class PendingDeleteService:
|
||||
"get_lora_scanner",
|
||||
"get_checkpoint_scanner",
|
||||
"get_embedding_scanner",
|
||||
"get_other_scanner",
|
||||
):
|
||||
getter = getattr(ServiceRegistry, getter_name, None)
|
||||
if not callable(getter):
|
||||
|
||||
@@ -6,7 +6,10 @@ import threading
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple
|
||||
|
||||
from ..utils.cache_db import connect_cache_db
|
||||
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
||||
from ..utils.file_lock import exclusive_lock
|
||||
from .model_sources import normalize_metadata_source
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -19,6 +22,9 @@ class PersistedCacheData:
|
||||
hash_rows: List[Tuple[str, str]]
|
||||
excluded_models: List[str]
|
||||
autov3_hash_rows: List[Tuple[str, str]] = field(default_factory=list)
|
||||
# Every directory under the model roots (including empty ones), or None
|
||||
# when the snapshot predates folder recording.
|
||||
all_folders: Optional[List[str]] = None
|
||||
|
||||
|
||||
DEFAULT_LICENSE_FLAGS = 127 # 127 (0b1111111) encodes default CivitAI permissions with all commercial modes enabled.
|
||||
@@ -59,6 +65,8 @@ class PersistentModelCache:
|
||||
"db_checked",
|
||||
"last_checked_at",
|
||||
"hash_status",
|
||||
"source_platform",
|
||||
"source_url",
|
||||
"hf_url",
|
||||
)
|
||||
_MODEL_UPDATE_COLUMNS: Tuple[str, ...] = _MODEL_COLUMNS[2:]
|
||||
@@ -128,6 +136,14 @@ class PersistentModelCache:
|
||||
"SELECT file_path FROM excluded_models WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
folder_rows = conn.execute(
|
||||
"SELECT path FROM folders WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
folders_recorded = conn.execute(
|
||||
"SELECT value FROM cache_meta WHERE key = ?",
|
||||
(f"folders_recorded:{model_type}",),
|
||||
).fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
@@ -195,8 +211,13 @@ class PersistentModelCache:
|
||||
"skip_metadata_refresh": bool(row["skip_metadata_refresh"]),
|
||||
"license_flags": int(license_value),
|
||||
"hash_status": row["hash_status"] or "completed",
|
||||
"source_platform": row["source_platform"] or "",
|
||||
"source_url": row["source_url"] or "",
|
||||
"hf_url": row["hf_url"] or "",
|
||||
}
|
||||
# Legacy rows only carry `hf_url`; derive the canonical pair so
|
||||
# every consumer sees the same shape.
|
||||
normalize_metadata_source(item)
|
||||
if row["autov3"] is not None:
|
||||
item["autov3"] = (row["autov3"] or "").lower()
|
||||
raw_data.append(item)
|
||||
@@ -216,14 +237,20 @@ class PersistentModelCache:
|
||||
]
|
||||
|
||||
excluded_paths = [row["file_path"] for row in excluded]
|
||||
all_folders: Optional[List[str]] = None
|
||||
if folders_recorded is not None:
|
||||
all_folders = sorted(
|
||||
(row["path"] for row in folder_rows), key=lambda x: x.lower()
|
||||
)
|
||||
return PersistedCacheData(
|
||||
raw_data=raw_data,
|
||||
hash_rows=hash_pairs,
|
||||
excluded_models=excluded_paths,
|
||||
autov3_hash_rows=autov3_pairs,
|
||||
all_folders=all_folders,
|
||||
)
|
||||
|
||||
def save_cache(self, model_type: str, raw_data: Sequence[Dict[str, Any]], hash_index: Dict[str, List[str]], excluded_models: Sequence[str], autov3_hash_index: Optional[Dict[str, List[str]]] = None) -> None:
|
||||
def save_cache(self, model_type: str, raw_data: Sequence[Dict[str, Any]], hash_index: Dict[str, List[str]], excluded_models: Sequence[str], autov3_hash_index: Optional[Dict[str, List[str]]] = None, all_folders: Optional[Sequence[str]] = None) -> None:
|
||||
if not self.is_enabled():
|
||||
return
|
||||
if not self._schema_initialized:
|
||||
@@ -232,246 +259,271 @@ class PersistentModelCache:
|
||||
return
|
||||
try:
|
||||
with self._db_lock:
|
||||
conn = self._connect()
|
||||
try:
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("BEGIN")
|
||||
# Cross-process serialization: another LoRA Manager instance may
|
||||
# share this settings directory, and the read-merge-write below
|
||||
# spans several statements.
|
||||
with exclusive_lock(self._db_path):
|
||||
conn = self._connect()
|
||||
try:
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("BEGIN")
|
||||
|
||||
model_rows = [self._prepare_model_row(model_type, item) for item in raw_data]
|
||||
model_map: Dict[str, Tuple[Any, ...]] = {
|
||||
row[1]: row for row in model_rows if row[1] # row[1] is file_path
|
||||
}
|
||||
model_rows = [self._prepare_model_row(model_type, item) for item in raw_data]
|
||||
model_map: Dict[str, Tuple[Any, ...]] = {
|
||||
row[1]: row for row in model_rows if row[1] # row[1] is file_path
|
||||
}
|
||||
|
||||
existing_models = conn.execute(
|
||||
"SELECT "
|
||||
+ ", ".join(self._MODEL_COLUMNS[1:])
|
||||
+ " FROM models WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_model_map: Dict[str, sqlite3.Row] = {
|
||||
row["file_path"]: row for row in existing_models
|
||||
}
|
||||
|
||||
to_remove_models = [
|
||||
(model_type, path)
|
||||
for path in existing_model_map.keys()
|
||||
if path not in model_map
|
||||
]
|
||||
if to_remove_models:
|
||||
conn.executemany(
|
||||
"DELETE FROM models WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
conn.executemany(
|
||||
"DELETE FROM model_tags WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
conn.executemany(
|
||||
"DELETE FROM hash_index WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
conn.executemany(
|
||||
"DELETE FROM autov3_index WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
conn.executemany(
|
||||
"DELETE FROM excluded_models WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
|
||||
insert_rows: List[Tuple[Any, ...]] = []
|
||||
update_rows: List[Tuple[Any, ...]] = []
|
||||
|
||||
for file_path, row in model_map.items():
|
||||
existing = existing_model_map.get(file_path)
|
||||
if existing is None:
|
||||
insert_rows.append(row)
|
||||
continue
|
||||
|
||||
existing_values = tuple(
|
||||
existing[column] for column in self._MODEL_COLUMNS[1:]
|
||||
)
|
||||
current_values = row[1:]
|
||||
if existing_values != current_values:
|
||||
update_rows.append(row[2:] + (model_type, file_path))
|
||||
|
||||
if insert_rows:
|
||||
conn.executemany(self._insert_model_sql(), insert_rows)
|
||||
|
||||
if update_rows:
|
||||
set_clause = ", ".join(
|
||||
f"{column} = ?"
|
||||
for column in self._MODEL_UPDATE_COLUMNS
|
||||
)
|
||||
update_sql = (
|
||||
f"UPDATE models SET {set_clause} WHERE model_type = ? AND file_path = ?"
|
||||
)
|
||||
conn.executemany(update_sql, update_rows)
|
||||
|
||||
existing_tags_rows = conn.execute(
|
||||
"SELECT file_path, tag FROM model_tags WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_tags: Dict[str, set[str]] = {}
|
||||
for row in existing_tags_rows:
|
||||
existing_tags.setdefault(row["file_path"], set()).add(row["tag"])
|
||||
|
||||
new_tags: Dict[str, set[str]] = {}
|
||||
for item in raw_data:
|
||||
file_path = item.get("file_path")
|
||||
if not file_path:
|
||||
continue
|
||||
tags = set(item.get("tags") or [])
|
||||
if tags:
|
||||
new_tags[file_path] = tags
|
||||
|
||||
tag_inserts: List[Tuple[str, str, str]] = []
|
||||
tag_deletes: List[Tuple[str, str, str]] = []
|
||||
|
||||
all_tag_paths = set(existing_tags.keys()) | set(new_tags.keys())
|
||||
for path in all_tag_paths:
|
||||
existing_set = existing_tags.get(path, set())
|
||||
new_set = new_tags.get(path, set())
|
||||
to_add = new_set - existing_set
|
||||
to_remove = existing_set - new_set
|
||||
|
||||
for tag in to_add:
|
||||
tag_inserts.append((model_type, path, tag))
|
||||
for tag in to_remove:
|
||||
tag_deletes.append((model_type, path, tag))
|
||||
|
||||
if tag_deletes:
|
||||
conn.executemany(
|
||||
"DELETE FROM model_tags WHERE model_type = ? AND file_path = ? AND tag = ?",
|
||||
tag_deletes,
|
||||
)
|
||||
if tag_inserts:
|
||||
conn.executemany(
|
||||
"INSERT INTO model_tags (model_type, file_path, tag) VALUES (?, ?, ?)",
|
||||
tag_inserts,
|
||||
)
|
||||
|
||||
existing_hash_rows = conn.execute(
|
||||
"SELECT sha256, file_path FROM hash_index WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_hash_map: Dict[str, set[str]] = {}
|
||||
for row in existing_hash_rows:
|
||||
sha_value = (row["sha256"] or "").lower()
|
||||
if not sha_value:
|
||||
continue
|
||||
existing_hash_map.setdefault(sha_value, set()).add(row["file_path"])
|
||||
|
||||
new_hash_map: Dict[str, set[str]] = {}
|
||||
for sha_value, paths in hash_index.items():
|
||||
normalized_sha = (sha_value or "").lower()
|
||||
if not normalized_sha:
|
||||
continue
|
||||
bucket = new_hash_map.setdefault(normalized_sha, set())
|
||||
for path in paths:
|
||||
if path:
|
||||
bucket.add(path)
|
||||
|
||||
hash_inserts: List[Tuple[str, str, str]] = []
|
||||
hash_deletes: List[Tuple[str, str, str]] = []
|
||||
|
||||
all_shas = set(existing_hash_map.keys()) | set(new_hash_map.keys())
|
||||
for sha_value in all_shas:
|
||||
existing_paths = existing_hash_map.get(sha_value, set())
|
||||
new_paths = new_hash_map.get(sha_value, set())
|
||||
|
||||
for path in existing_paths - new_paths:
|
||||
hash_deletes.append((model_type, sha_value, path))
|
||||
for path in new_paths - existing_paths:
|
||||
hash_inserts.append((model_type, sha_value, path))
|
||||
|
||||
if hash_deletes:
|
||||
conn.executemany(
|
||||
"DELETE FROM hash_index WHERE model_type = ? AND sha256 = ? AND file_path = ?",
|
||||
hash_deletes,
|
||||
)
|
||||
if hash_inserts:
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO hash_index (model_type, sha256, file_path) VALUES (?, ?, ?)",
|
||||
hash_inserts,
|
||||
)
|
||||
|
||||
if autov3_hash_index is not None:
|
||||
existing_autov3_rows = conn.execute(
|
||||
"SELECT autov3, file_path FROM autov3_index WHERE model_type = ?",
|
||||
existing_models = conn.execute(
|
||||
"SELECT "
|
||||
+ ", ".join(self._MODEL_COLUMNS[1:])
|
||||
+ " FROM models WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_autov3_map: Dict[str, set[str]] = {}
|
||||
for row in existing_autov3_rows:
|
||||
autov3_value = (row["autov3"] or "").lower()
|
||||
if not autov3_value:
|
||||
continue
|
||||
existing_autov3_map.setdefault(autov3_value, set()).add(row["file_path"])
|
||||
existing_model_map: Dict[str, sqlite3.Row] = {
|
||||
row["file_path"]: row for row in existing_models
|
||||
}
|
||||
|
||||
new_autov3_map: Dict[str, set[str]] = {}
|
||||
for autov3_value, paths in autov3_hash_index.items():
|
||||
normalized_autov3 = (autov3_value or "").lower()
|
||||
if not normalized_autov3:
|
||||
to_remove_models = [
|
||||
(model_type, path)
|
||||
for path in existing_model_map.keys()
|
||||
if path not in model_map
|
||||
]
|
||||
if to_remove_models:
|
||||
conn.executemany(
|
||||
"DELETE FROM models WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
conn.executemany(
|
||||
"DELETE FROM model_tags WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
conn.executemany(
|
||||
"DELETE FROM hash_index WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
conn.executemany(
|
||||
"DELETE FROM autov3_index WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
conn.executemany(
|
||||
"DELETE FROM excluded_models WHERE model_type = ? AND file_path = ?",
|
||||
to_remove_models,
|
||||
)
|
||||
|
||||
insert_rows: List[Tuple[Any, ...]] = []
|
||||
update_rows: List[Tuple[Any, ...]] = []
|
||||
|
||||
for file_path, row in model_map.items():
|
||||
existing = existing_model_map.get(file_path)
|
||||
if existing is None:
|
||||
insert_rows.append(row)
|
||||
continue
|
||||
bucket = new_autov3_map.setdefault(normalized_autov3, set())
|
||||
|
||||
existing_values = tuple(
|
||||
existing[column] for column in self._MODEL_COLUMNS[1:]
|
||||
)
|
||||
current_values = row[1:]
|
||||
if existing_values != current_values:
|
||||
update_rows.append(row[2:] + (model_type, file_path))
|
||||
|
||||
if insert_rows:
|
||||
conn.executemany(self._insert_model_sql(), insert_rows)
|
||||
|
||||
if update_rows:
|
||||
set_clause = ", ".join(
|
||||
f"{column} = ?"
|
||||
for column in self._MODEL_UPDATE_COLUMNS
|
||||
)
|
||||
update_sql = (
|
||||
f"UPDATE models SET {set_clause} WHERE model_type = ? AND file_path = ?"
|
||||
)
|
||||
conn.executemany(update_sql, update_rows)
|
||||
|
||||
existing_tags_rows = conn.execute(
|
||||
"SELECT file_path, tag FROM model_tags WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_tags: Dict[str, set[str]] = {}
|
||||
for row in existing_tags_rows:
|
||||
existing_tags.setdefault(row["file_path"], set()).add(row["tag"])
|
||||
|
||||
new_tags: Dict[str, set[str]] = {}
|
||||
for item in raw_data:
|
||||
file_path = item.get("file_path")
|
||||
if not file_path:
|
||||
continue
|
||||
tags = set(item.get("tags") or [])
|
||||
if tags:
|
||||
new_tags[file_path] = tags
|
||||
|
||||
tag_inserts: List[Tuple[str, str, str]] = []
|
||||
tag_deletes: List[Tuple[str, str, str]] = []
|
||||
|
||||
all_tag_paths = set(existing_tags.keys()) | set(new_tags.keys())
|
||||
for path in all_tag_paths:
|
||||
existing_set = existing_tags.get(path, set())
|
||||
new_set = new_tags.get(path, set())
|
||||
to_add = new_set - existing_set
|
||||
to_remove = existing_set - new_set
|
||||
|
||||
for tag in to_add:
|
||||
tag_inserts.append((model_type, path, tag))
|
||||
for tag in to_remove:
|
||||
tag_deletes.append((model_type, path, tag))
|
||||
|
||||
if tag_deletes:
|
||||
conn.executemany(
|
||||
"DELETE FROM model_tags WHERE model_type = ? AND file_path = ? AND tag = ?",
|
||||
tag_deletes,
|
||||
)
|
||||
if tag_inserts:
|
||||
conn.executemany(
|
||||
"INSERT INTO model_tags (model_type, file_path, tag) VALUES (?, ?, ?)",
|
||||
tag_inserts,
|
||||
)
|
||||
|
||||
existing_hash_rows = conn.execute(
|
||||
"SELECT sha256, file_path FROM hash_index WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_hash_map: Dict[str, set[str]] = {}
|
||||
for row in existing_hash_rows:
|
||||
sha_value = (row["sha256"] or "").lower()
|
||||
if not sha_value:
|
||||
continue
|
||||
existing_hash_map.setdefault(sha_value, set()).add(row["file_path"])
|
||||
|
||||
new_hash_map: Dict[str, set[str]] = {}
|
||||
for sha_value, paths in hash_index.items():
|
||||
normalized_sha = (sha_value or "").lower()
|
||||
if not normalized_sha:
|
||||
continue
|
||||
bucket = new_hash_map.setdefault(normalized_sha, set())
|
||||
for path in paths:
|
||||
if path:
|
||||
bucket.add(path)
|
||||
|
||||
autov3_inserts: List[Tuple[str, str, str]] = []
|
||||
autov3_deletes: List[Tuple[str, str, str]] = []
|
||||
hash_inserts: List[Tuple[str, str, str]] = []
|
||||
hash_deletes: List[Tuple[str, str, str]] = []
|
||||
|
||||
all_autov3 = set(existing_autov3_map.keys()) | set(new_autov3_map.keys())
|
||||
for autov3_value in all_autov3:
|
||||
existing_paths = existing_autov3_map.get(autov3_value, set())
|
||||
new_paths = new_autov3_map.get(autov3_value, set())
|
||||
all_shas = set(existing_hash_map.keys()) | set(new_hash_map.keys())
|
||||
for sha_value in all_shas:
|
||||
existing_paths = existing_hash_map.get(sha_value, set())
|
||||
new_paths = new_hash_map.get(sha_value, set())
|
||||
|
||||
for path in existing_paths - new_paths:
|
||||
autov3_deletes.append((model_type, autov3_value, path))
|
||||
hash_deletes.append((model_type, sha_value, path))
|
||||
for path in new_paths - existing_paths:
|
||||
autov3_inserts.append((model_type, autov3_value, path))
|
||||
hash_inserts.append((model_type, sha_value, path))
|
||||
|
||||
if autov3_deletes:
|
||||
if hash_deletes:
|
||||
conn.executemany(
|
||||
"DELETE FROM autov3_index WHERE model_type = ? AND autov3 = ? AND file_path = ?",
|
||||
autov3_deletes,
|
||||
"DELETE FROM hash_index WHERE model_type = ? AND sha256 = ? AND file_path = ?",
|
||||
hash_deletes,
|
||||
)
|
||||
if autov3_inserts:
|
||||
if hash_inserts:
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO autov3_index (model_type, autov3, file_path) VALUES (?, ?, ?)",
|
||||
autov3_inserts,
|
||||
"INSERT OR IGNORE INTO hash_index (model_type, sha256, file_path) VALUES (?, ?, ?)",
|
||||
hash_inserts,
|
||||
)
|
||||
|
||||
existing_excluded_rows = conn.execute(
|
||||
"SELECT file_path FROM excluded_models WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_excluded = {row["file_path"] for row in existing_excluded_rows}
|
||||
new_excluded = {path for path in excluded_models if path}
|
||||
if autov3_hash_index is not None:
|
||||
existing_autov3_rows = conn.execute(
|
||||
"SELECT autov3, file_path FROM autov3_index WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_autov3_map: Dict[str, set[str]] = {}
|
||||
for row in existing_autov3_rows:
|
||||
autov3_value = (row["autov3"] or "").lower()
|
||||
if not autov3_value:
|
||||
continue
|
||||
existing_autov3_map.setdefault(autov3_value, set()).add(row["file_path"])
|
||||
|
||||
excluded_deletes = [
|
||||
(model_type, path)
|
||||
for path in existing_excluded - new_excluded
|
||||
]
|
||||
excluded_inserts = [
|
||||
(model_type, path)
|
||||
for path in new_excluded - existing_excluded
|
||||
]
|
||||
new_autov3_map: Dict[str, set[str]] = {}
|
||||
for autov3_value, paths in autov3_hash_index.items():
|
||||
normalized_autov3 = (autov3_value or "").lower()
|
||||
if not normalized_autov3:
|
||||
continue
|
||||
bucket = new_autov3_map.setdefault(normalized_autov3, set())
|
||||
for path in paths:
|
||||
if path:
|
||||
bucket.add(path)
|
||||
|
||||
if excluded_deletes:
|
||||
conn.executemany(
|
||||
"DELETE FROM excluded_models WHERE model_type = ? AND file_path = ?",
|
||||
excluded_deletes,
|
||||
)
|
||||
if excluded_inserts:
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO excluded_models (model_type, file_path) VALUES (?, ?)",
|
||||
excluded_inserts,
|
||||
)
|
||||
autov3_inserts: List[Tuple[str, str, str]] = []
|
||||
autov3_deletes: List[Tuple[str, str, str]] = []
|
||||
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
all_autov3 = set(existing_autov3_map.keys()) | set(new_autov3_map.keys())
|
||||
for autov3_value in all_autov3:
|
||||
existing_paths = existing_autov3_map.get(autov3_value, set())
|
||||
new_paths = new_autov3_map.get(autov3_value, set())
|
||||
|
||||
for path in existing_paths - new_paths:
|
||||
autov3_deletes.append((model_type, autov3_value, path))
|
||||
for path in new_paths - existing_paths:
|
||||
autov3_inserts.append((model_type, autov3_value, path))
|
||||
|
||||
if autov3_deletes:
|
||||
conn.executemany(
|
||||
"DELETE FROM autov3_index WHERE model_type = ? AND autov3 = ? AND file_path = ?",
|
||||
autov3_deletes,
|
||||
)
|
||||
if autov3_inserts:
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO autov3_index (model_type, autov3, file_path) VALUES (?, ?, ?)",
|
||||
autov3_inserts,
|
||||
)
|
||||
|
||||
existing_excluded_rows = conn.execute(
|
||||
"SELECT file_path FROM excluded_models WHERE model_type = ?",
|
||||
(model_type,),
|
||||
).fetchall()
|
||||
existing_excluded = {row["file_path"] for row in existing_excluded_rows}
|
||||
new_excluded = {path for path in excluded_models if path}
|
||||
|
||||
excluded_deletes = [
|
||||
(model_type, path)
|
||||
for path in existing_excluded - new_excluded
|
||||
]
|
||||
excluded_inserts = [
|
||||
(model_type, path)
|
||||
for path in new_excluded - existing_excluded
|
||||
]
|
||||
|
||||
if excluded_deletes:
|
||||
conn.executemany(
|
||||
"DELETE FROM excluded_models WHERE model_type = ? AND file_path = ?",
|
||||
excluded_deletes,
|
||||
)
|
||||
if excluded_inserts:
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO excluded_models (model_type, file_path) VALUES (?, ?)",
|
||||
excluded_inserts,
|
||||
)
|
||||
|
||||
if all_folders is not None:
|
||||
conn.execute(
|
||||
"DELETE FROM folders WHERE model_type = ?",
|
||||
(model_type,),
|
||||
)
|
||||
folder_inserts = [
|
||||
(model_type, path) for path in all_folders if path
|
||||
]
|
||||
if folder_inserts:
|
||||
conn.executemany(
|
||||
"INSERT OR IGNORE INTO folders (model_type, path) VALUES (?, ?)",
|
||||
folder_inserts,
|
||||
)
|
||||
# Mark the snapshot as having folder data even when the
|
||||
# library has no subfolders, so an empty list is not
|
||||
# mistaken for "never recorded" on load.
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO cache_meta (key, value) VALUES (?, ?)",
|
||||
(f"folders_recorded:{model_type}", "1"),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to persist cache for %s: %s", model_type, exc)
|
||||
|
||||
@@ -524,6 +576,8 @@ class PersistentModelCache:
|
||||
db_checked INTEGER,
|
||||
last_checked_at REAL,
|
||||
hash_status TEXT,
|
||||
source_platform TEXT DEFAULT '',
|
||||
source_url TEXT DEFAULT '',
|
||||
hf_url TEXT DEFAULT '',
|
||||
PRIMARY KEY (model_type, file_path)
|
||||
);
|
||||
@@ -554,6 +608,17 @@ class PersistentModelCache:
|
||||
file_path TEXT NOT NULL,
|
||||
PRIMARY KEY (model_type, file_path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS folders (
|
||||
model_type TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
PRIMARY KEY (model_type, path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cache_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT
|
||||
);
|
||||
"""
|
||||
)
|
||||
self._ensure_additional_model_columns(conn)
|
||||
@@ -580,6 +645,8 @@ class PersistentModelCache:
|
||||
# Persisting without explicit flags should assume CivitAI's documented defaults (0b111001 == 57).
|
||||
"license_flags": f"INTEGER DEFAULT {DEFAULT_LICENSE_FLAGS}",
|
||||
"hash_status": "TEXT DEFAULT 'completed'",
|
||||
"source_platform": "TEXT DEFAULT ''",
|
||||
"source_url": "TEXT DEFAULT ''",
|
||||
"hf_url": "TEXT DEFAULT ''",
|
||||
"autov3": "TEXT",
|
||||
}
|
||||
@@ -589,18 +656,19 @@ class PersistentModelCache:
|
||||
conn.execute(f"ALTER TABLE models ADD COLUMN {column} {definition}")
|
||||
|
||||
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
||||
uri = False
|
||||
path = self._db_path
|
||||
if readonly:
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(path)
|
||||
path = f"file:{path}?mode=ro"
|
||||
uri = True
|
||||
conn = sqlite3.connect(path, check_same_thread=False, uri=uri, detect_types=sqlite3.PARSE_DECLTYPES)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
if readonly and not os.path.exists(self._db_path):
|
||||
raise FileNotFoundError(self._db_path)
|
||||
return connect_cache_db(
|
||||
self._db_path,
|
||||
readonly=readonly,
|
||||
detect_types=sqlite3.PARSE_DECLTYPES,
|
||||
row_factory=sqlite3.Row,
|
||||
)
|
||||
|
||||
def _prepare_model_row(self, model_type: str, item: Dict[str, Any]) -> Tuple[Any, ...]:
|
||||
# Keep `source_*` and the legacy `hf_url` alias consistent no matter
|
||||
# which caller populated the item.
|
||||
normalize_metadata_source(item)
|
||||
civitai = item.get("civitai") or {}
|
||||
trained_words = civitai.get("trainedWords")
|
||||
if isinstance(trained_words, str):
|
||||
@@ -664,6 +732,8 @@ class PersistentModelCache:
|
||||
1 if item.get("db_checked") else 0,
|
||||
float(item.get("last_checked_at") or 0.0),
|
||||
item.get("hash_status", "completed"),
|
||||
item.get("source_platform") or "",
|
||||
item.get("source_url") or "",
|
||||
item.get("hf_url") or "",
|
||||
)
|
||||
|
||||
|
||||
@@ -19,7 +19,9 @@ import threading
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||
|
||||
from ..utils.cache_db import connect_cache_db
|
||||
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
||||
from ..utils.file_lock import exclusive_lock
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -52,13 +54,13 @@ class PersistentRecipeCache:
|
||||
"file_mtime",
|
||||
"file_size",
|
||||
"favorite",
|
||||
"repair_version",
|
||||
"preview_nsfw_level",
|
||||
"loras_json",
|
||||
"checkpoint_json",
|
||||
"gen_params_json",
|
||||
"tags_json",
|
||||
"has_workflow",
|
||||
"import_info_json",
|
||||
)
|
||||
_instances: Dict[str, "PersistentRecipeCache"] = {}
|
||||
_instance_lock = threading.Lock()
|
||||
@@ -170,65 +172,98 @@ class PersistentRecipeCache:
|
||||
recipes: List[Dict[str, Any]],
|
||||
json_paths: Optional[Dict[str, str]] = None,
|
||||
image_id_map: Optional[Dict[str, str]] = None,
|
||||
) -> None:
|
||||
skip_if_empty: bool = False,
|
||||
) -> bool:
|
||||
"""Save all recipes to SQLite cache.
|
||||
|
||||
Args:
|
||||
recipes: List of recipe dictionaries to persist.
|
||||
json_paths: Optional mapping of recipe_id -> json_path for file stats.
|
||||
image_id_map: Optional precomputed civitai image_id → recipe_id mapping.
|
||||
skip_if_empty: When True, refuse to replace a non-empty cache with an
|
||||
empty one. This is the storage-level backstop against a scan that
|
||||
silently loses every recipe (unavailable drive / mis-resolved
|
||||
recipes directory): overwriting both deletes the user's data and
|
||||
destroys their only record of it. Intentional full clears (manual
|
||||
rebuild) must pass ``skip_if_empty=False``.
|
||||
|
||||
Returns:
|
||||
``True`` when the write happened, ``False`` when it was skipped.
|
||||
"""
|
||||
if not self.is_enabled():
|
||||
return
|
||||
return False
|
||||
if not self._schema_initialized:
|
||||
self._initialize_schema()
|
||||
if not self._schema_initialized:
|
||||
return
|
||||
return False
|
||||
|
||||
try:
|
||||
with self._db_lock:
|
||||
conn = self._connect()
|
||||
try:
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("BEGIN")
|
||||
# Cross-process serialization: another LoRA Manager instance may
|
||||
# share this settings directory, and a full-table replace is a
|
||||
# read-modify-write that SQLite alone cannot make atomic.
|
||||
with exclusive_lock(self._db_path):
|
||||
conn = self._connect()
|
||||
try:
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("BEGIN")
|
||||
|
||||
# Clear existing data
|
||||
conn.execute("DELETE FROM recipes")
|
||||
if skip_if_empty and not recipes:
|
||||
existing = conn.execute(
|
||||
"SELECT COUNT(*) FROM recipes"
|
||||
).fetchone()
|
||||
if existing and existing[0]:
|
||||
conn.rollback()
|
||||
logger.warning(
|
||||
"Refusing to persist an empty recipe cache: the "
|
||||
"stored cache still holds %d recipe(s). The scan "
|
||||
"found nothing, which usually means the recipes "
|
||||
"path was unavailable or resolved elsewhere; "
|
||||
"keeping the stored cache so the data stays "
|
||||
"recoverable.",
|
||||
existing[0],
|
||||
)
|
||||
return False
|
||||
|
||||
# Prepare and insert all rows
|
||||
recipe_rows = []
|
||||
for recipe in recipes:
|
||||
recipe_id = str(recipe.get("id", ""))
|
||||
if not recipe_id:
|
||||
continue
|
||||
# Clear existing data
|
||||
conn.execute("DELETE FROM recipes")
|
||||
|
||||
json_path = ""
|
||||
if json_paths:
|
||||
json_path = json_paths.get(recipe_id, "")
|
||||
# Prepare and insert all rows
|
||||
recipe_rows = []
|
||||
for recipe in recipes:
|
||||
recipe_id = str(recipe.get("id", ""))
|
||||
if not recipe_id:
|
||||
continue
|
||||
|
||||
row = self._prepare_recipe_row(recipe, json_path)
|
||||
recipe_rows.append(row)
|
||||
json_path = ""
|
||||
if json_paths:
|
||||
json_path = json_paths.get(recipe_id, "")
|
||||
|
||||
if recipe_rows:
|
||||
placeholders = ", ".join(["?"] * len(self._RECIPE_COLUMNS))
|
||||
columns = ", ".join(self._RECIPE_COLUMNS)
|
||||
conn.executemany(
|
||||
f"INSERT INTO recipes ({columns}) VALUES ({placeholders})",
|
||||
recipe_rows,
|
||||
row = self._prepare_recipe_row(recipe, json_path)
|
||||
recipe_rows.append(row)
|
||||
|
||||
if recipe_rows:
|
||||
placeholders = ", ".join(["?"] * len(self._RECIPE_COLUMNS))
|
||||
columns = ", ".join(self._RECIPE_COLUMNS)
|
||||
conn.executemany(
|
||||
f"INSERT INTO recipes ({columns}) VALUES ({placeholders})",
|
||||
recipe_rows,
|
||||
)
|
||||
|
||||
# Persist image_id_map for O(1) lookups on cache load
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO cache_metadata (key, value) VALUES (?, ?)",
|
||||
("image_id_map", json.dumps(image_id_map or {})),
|
||||
)
|
||||
|
||||
# Persist image_id_map for O(1) lookups on cache load
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO cache_metadata (key, value) VALUES (?, ?)",
|
||||
("image_id_map", json.dumps(image_id_map or {})),
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
logger.debug("Persisted %d recipes to cache", len(recipe_rows))
|
||||
finally:
|
||||
conn.close()
|
||||
conn.commit()
|
||||
logger.debug("Persisted %d recipes to cache", len(recipe_rows))
|
||||
return True
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to persist recipe cache: %s", exc)
|
||||
return False
|
||||
|
||||
def get_file_stats(self) -> Dict[str, Tuple[float, int]]:
|
||||
"""Return stored file stats for all cached recipes.
|
||||
@@ -441,13 +476,13 @@ class PersistentRecipeCache:
|
||||
file_mtime REAL,
|
||||
file_size INTEGER,
|
||||
favorite INTEGER DEFAULT 0,
|
||||
repair_version INTEGER DEFAULT 0,
|
||||
preview_nsfw_level INTEGER DEFAULT 0,
|
||||
loras_json TEXT,
|
||||
checkpoint_json TEXT,
|
||||
gen_params_json TEXT,
|
||||
tags_json TEXT,
|
||||
has_workflow INTEGER DEFAULT 0
|
||||
has_workflow INTEGER DEFAULT 0,
|
||||
import_info_json TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_recipes_json_path ON recipes(json_path);
|
||||
@@ -473,22 +508,27 @@ class PersistentRecipeCache:
|
||||
)
|
||||
except Exception:
|
||||
pass # column already exists
|
||||
# Migration: add import_info_json column to existing databases
|
||||
try:
|
||||
conn.execute(
|
||||
"ALTER TABLE recipes ADD COLUMN import_info_json TEXT"
|
||||
)
|
||||
except Exception:
|
||||
pass # column already exists
|
||||
conn.commit()
|
||||
self._schema_initialized = True
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to initialize persistent recipe cache schema: %s", exc)
|
||||
|
||||
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
||||
uri = False
|
||||
path = self._db_path
|
||||
if readonly:
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(path)
|
||||
path = f"file:{path}?mode=ro"
|
||||
uri = True
|
||||
conn = sqlite3.connect(path, check_same_thread=False, uri=uri, detect_types=sqlite3.PARSE_DECLTYPES)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
if readonly and not os.path.exists(self._db_path):
|
||||
raise FileNotFoundError(self._db_path)
|
||||
return connect_cache_db(
|
||||
self._db_path,
|
||||
readonly=readonly,
|
||||
detect_types=sqlite3.PARSE_DECLTYPES,
|
||||
row_factory=sqlite3.Row,
|
||||
)
|
||||
|
||||
def _prepare_recipe_row(self, recipe: Dict[str, Any], json_path: str) -> Tuple[Any, ...]:
|
||||
"""Convert a recipe dict to a row tuple for SQLite insertion."""
|
||||
@@ -504,6 +544,9 @@ class PersistentRecipeCache:
|
||||
tags = recipe.get("tags")
|
||||
tags_json = json.dumps(tags) if tags else None
|
||||
|
||||
import_info = recipe.get("import_info")
|
||||
import_info_json = json.dumps(import_info) if import_info else None
|
||||
|
||||
# Get file stats if json_path exists
|
||||
file_mtime = 0.0
|
||||
file_size = 0
|
||||
@@ -529,13 +572,13 @@ class PersistentRecipeCache:
|
||||
file_mtime,
|
||||
file_size,
|
||||
1 if recipe.get("favorite") else 0,
|
||||
int(recipe.get("repair_version") or 0),
|
||||
int(recipe.get("preview_nsfw_level") or 0),
|
||||
loras_json,
|
||||
checkpoint_json,
|
||||
gen_params_json,
|
||||
tags_json,
|
||||
1 if recipe.get("has_workflow") else 0,
|
||||
import_info_json,
|
||||
)
|
||||
|
||||
def _row_to_recipe(self, row: sqlite3.Row) -> Dict[str, Any]:
|
||||
@@ -568,6 +611,13 @@ class PersistentRecipeCache:
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
import_info = None
|
||||
if row["import_info_json"]:
|
||||
try:
|
||||
import_info = json.loads(row["import_info_json"])
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
recipe = {
|
||||
"id": row["recipe_id"],
|
||||
"file_path": row["file_path"] or "",
|
||||
@@ -579,7 +629,6 @@ class PersistentRecipeCache:
|
||||
"created_date": row["created_date"] or 0.0,
|
||||
"modified": row["modified"] or 0.0,
|
||||
"favorite": bool(row["favorite"]),
|
||||
"repair_version": row["repair_version"] or 0,
|
||||
"preview_nsfw_level": row["preview_nsfw_level"] or 0,
|
||||
"has_workflow": bool(row["has_workflow"]),
|
||||
"loras": loras,
|
||||
@@ -592,6 +641,9 @@ class PersistentRecipeCache:
|
||||
if checkpoint:
|
||||
recipe["checkpoint"] = checkpoint
|
||||
|
||||
if import_info:
|
||||
recipe["import_info"] = import_info
|
||||
|
||||
return recipe
|
||||
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ import threading
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||
|
||||
from ..utils.cache_db import connect_cache_db
|
||||
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -633,16 +634,13 @@ class RecipeFTSIndex:
|
||||
|
||||
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
||||
"""Create a database connection."""
|
||||
uri = False
|
||||
path = self._db_path
|
||||
if readonly:
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(path)
|
||||
path = f"file:{path}?mode=ro"
|
||||
uri = True
|
||||
conn = sqlite3.connect(path, check_same_thread=False, uri=uri)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
if readonly and not os.path.exists(self._db_path):
|
||||
raise FileNotFoundError(self._db_path)
|
||||
return connect_cache_db(
|
||||
self._db_path,
|
||||
readonly=readonly,
|
||||
row_factory=sqlite3.Row,
|
||||
)
|
||||
|
||||
def _remove_recipe_locked(self, conn: sqlite3.Connection, recipe_id: str) -> None:
|
||||
"""Remove a recipe entry. Caller must hold the lock."""
|
||||
|
||||
+1003
-254
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,7 @@
|
||||
"""Recipe service layer implementations."""
|
||||
|
||||
from .analysis_service import RecipeAnalysisService
|
||||
from .import_info import build_import_info, compute_no_loras_reason
|
||||
from .persistence_service import RecipePersistenceService
|
||||
from .sharing_service import RecipeSharingService
|
||||
from .errors import (
|
||||
@@ -15,6 +16,8 @@ __all__ = [
|
||||
"RecipeAnalysisService",
|
||||
"RecipePersistenceService",
|
||||
"RecipeSharingService",
|
||||
"build_import_info",
|
||||
"compute_no_loras_reason",
|
||||
"RecipeServiceError",
|
||||
"RecipeValidationError",
|
||||
"RecipeNotFoundError",
|
||||
|
||||
@@ -72,15 +72,28 @@ class RecipeAnalysisService:
|
||||
metadata = self._exif_utils.extract_image_metadata(temp_path)
|
||||
if not metadata:
|
||||
return AnalysisResult(
|
||||
{"error": "No metadata found in this image", "loras": []}
|
||||
{
|
||||
"error": "No metadata found in this image",
|
||||
"loras": [],
|
||||
"diagnostics": {
|
||||
"channel": "upload",
|
||||
"exif_present": False,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
return await self._parse_metadata(
|
||||
result = await self._parse_metadata(
|
||||
metadata,
|
||||
recipe_scanner=recipe_scanner,
|
||||
image_path=None,
|
||||
include_image_base64=False,
|
||||
)
|
||||
result.payload["diagnostics"] = {
|
||||
"channel": "upload",
|
||||
"exif_present": True,
|
||||
"exif_parser": result.payload.get("parser"),
|
||||
}
|
||||
return result
|
||||
finally:
|
||||
self._safe_cleanup(temp_path)
|
||||
|
||||
@@ -104,9 +117,13 @@ class RecipeAnalysisService:
|
||||
image_info: Optional[dict[str, Any]] = None
|
||||
is_video = False
|
||||
extension = ".jpg" # Default
|
||||
# Diagnostics collected during analysis; surfaced in the payload so
|
||||
# callers can persist an import_info block explaining empty LoRA lists.
|
||||
diagnostics: dict[str, Any] = {"channel": "url"}
|
||||
|
||||
try:
|
||||
civitai_image_id = extract_civitai_image_id(url)
|
||||
diagnostics["civitai_image"] = bool(civitai_image_id)
|
||||
if civitai_image_id:
|
||||
image_info = await civitai_client.get_image_info(
|
||||
civitai_image_id, source_url=url
|
||||
@@ -147,11 +164,23 @@ class RecipeAnalysisService:
|
||||
):
|
||||
metadata = metadata["meta"]
|
||||
|
||||
# Diagnostics: capture the API meta shape before injecting
|
||||
# modelVersionIds / browsingLevel so the recipe modal can
|
||||
# explain why an import ended up without LoRAs.
|
||||
diagnostics["api_meta_present"] = isinstance(metadata, dict)
|
||||
if isinstance(metadata, dict):
|
||||
diagnostics["api_meta_keys"] = sorted(metadata.keys())
|
||||
|
||||
# Include modelVersionIds from root level if available.
|
||||
# CivitAI API returns modelVersionIds at root level, not in meta.
|
||||
# When meta is null (None), create a minimal dict so downstream
|
||||
# parsers can still discover LoRAs and checkpoints.
|
||||
model_version_ids = image_info.get("modelVersionIds")
|
||||
diagnostics["api_model_version_ids"] = (
|
||||
len(model_version_ids)
|
||||
if isinstance(model_version_ids, list)
|
||||
else 0
|
||||
)
|
||||
if model_version_ids:
|
||||
if isinstance(metadata, dict):
|
||||
metadata["modelVersionIds"] = model_version_ids
|
||||
@@ -229,6 +258,8 @@ class RecipeAnalysisService:
|
||||
finally:
|
||||
self._safe_cleanup(orig_temp_path)
|
||||
|
||||
diagnostics["exif_present"] = bool(exif_metadata)
|
||||
|
||||
# Parse EXIF data (typically a string like parameters/prompt/workflow)
|
||||
# and API metadata (dict with modelVersionIds, browsingLevel) separately,
|
||||
# then merge: API loras/checkpoint override, EXIF gen_params fill in gaps.
|
||||
@@ -237,6 +268,7 @@ class RecipeAnalysisService:
|
||||
if isinstance(exif_metadata, str):
|
||||
exif_parser = self._recipe_parser_factory.create_parser(exif_metadata)
|
||||
if exif_parser:
|
||||
diagnostics["exif_parser"] = exif_parser.__class__.__name__
|
||||
exif_data = await exif_parser.parse_metadata(
|
||||
exif_metadata, recipe_scanner=recipe_scanner,
|
||||
)
|
||||
@@ -270,6 +302,22 @@ class RecipeAnalysisService:
|
||||
if merged_gp:
|
||||
result.payload["gen_params"] = merged_gp
|
||||
|
||||
# The API-only parse (meta=null with only modelVersionIds)
|
||||
# yields a checkpoint but no LoRAs; the image EXIF carries the
|
||||
# full resource list. Fill the gaps the API parse left open.
|
||||
if not result.payload.get("loras"):
|
||||
exif_loras = exif_parsed_result.get("loras") or []
|
||||
if exif_loras:
|
||||
result.payload["loras"] = exif_loras
|
||||
if not result.payload.get("checkpoint") and not result.payload.get("model"):
|
||||
exif_checkpoint = exif_parsed_result.get("model") or exif_parsed_result.get(
|
||||
"checkpoint"
|
||||
)
|
||||
if exif_checkpoint:
|
||||
result.payload["checkpoint"] = exif_checkpoint
|
||||
if not result.payload.get("base_model") and exif_parsed_result.get("base_model"):
|
||||
result.payload["base_model"] = exif_parsed_result["base_model"]
|
||||
|
||||
if civitai_image_id and image_info and not result.payload.get("error"):
|
||||
# Use the metadata dict we built (may contain modelVersionIds
|
||||
# and browsingLevel from the API root level). Do NOT pass
|
||||
@@ -308,6 +356,8 @@ class RecipeAnalysisService:
|
||||
if isinstance(bl, int) and bl > 0:
|
||||
result.payload["preview_nsfw_level"] = bl
|
||||
|
||||
diagnostics["is_video"] = is_video
|
||||
result.payload["diagnostics"] = diagnostics
|
||||
return result
|
||||
finally:
|
||||
if temp_path:
|
||||
@@ -318,6 +368,7 @@ class RecipeAnalysisService:
|
||||
*,
|
||||
file_path: str | None,
|
||||
recipe_scanner,
|
||||
ignore_recipe_metadata: bool = False,
|
||||
) -> AnalysisResult:
|
||||
"""Analyze a file already present on disk."""
|
||||
|
||||
@@ -332,14 +383,41 @@ class RecipeAnalysisService:
|
||||
self._exif_utils.extract_image_metadata, normalized_path
|
||||
)
|
||||
if not metadata:
|
||||
return self._metadata_not_found_response(normalized_path)
|
||||
result = self._metadata_not_found_response(normalized_path)
|
||||
result.payload["diagnostics"] = {
|
||||
"channel": "local",
|
||||
"exif_present": False,
|
||||
}
|
||||
return result
|
||||
|
||||
return await self._parse_metadata(
|
||||
if ignore_recipe_metadata:
|
||||
# Re-import: re-parse the original embedded generation metadata
|
||||
# instead of the recipe JSON block LoRA Manager appended on save.
|
||||
from ...recipes.parsers.recipe_format import strip_recipe_metadata
|
||||
|
||||
metadata = strip_recipe_metadata(metadata)
|
||||
if not metadata:
|
||||
result = self._metadata_not_found_response(normalized_path)
|
||||
result.payload["diagnostics"] = {
|
||||
"channel": "local",
|
||||
"exif_present": True,
|
||||
"ignore_recipe_metadata": True,
|
||||
"reason": "only_recipe_metadata",
|
||||
}
|
||||
return result
|
||||
|
||||
result = await self._parse_metadata(
|
||||
metadata,
|
||||
recipe_scanner=recipe_scanner,
|
||||
image_path=normalized_path,
|
||||
include_image_base64=True,
|
||||
)
|
||||
result.payload["diagnostics"] = {
|
||||
"channel": "local",
|
||||
"exif_present": True,
|
||||
"exif_parser": result.payload.get("parser"),
|
||||
}
|
||||
return result
|
||||
|
||||
async def analyze_widget_metadata(self, *, recipe_scanner) -> AnalysisResult:
|
||||
"""Analyse the most recent generation metadata for widget saves."""
|
||||
@@ -436,6 +514,10 @@ class RecipeAnalysisService:
|
||||
metadata, recipe_scanner=recipe_scanner
|
||||
)
|
||||
|
||||
# Record which parser handled the metadata so import diagnostics
|
||||
# can distinguish e.g. ComfyUI workflow sources.
|
||||
result["parser"] = parser.__class__.__name__
|
||||
|
||||
if include_image_base64 and image_path:
|
||||
result["image_base64"] = self._encode_file(image_path)
|
||||
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Import provenance helpers for recipes.
|
||||
|
||||
Builds the ``import_info`` block persisted on a recipe: the import channel
|
||||
(batch import / single URL / local file / upload / widget) and, when the
|
||||
recipe ended up with no LoRAs, a machine-readable reason plus the diagnostic
|
||||
details that led to it. The recipe modal renders this block in a collapsed
|
||||
"Why no LoRAs?" panel; legacy recipes without ``import_info`` fall back to a
|
||||
frontend heuristic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
# Import channels (how the recipe entered the library).
|
||||
CHANNEL_BATCH_IMPORT_URL = "batch_import_url"
|
||||
CHANNEL_BATCH_IMPORT_LOCAL = "batch_import_local"
|
||||
CHANNEL_URL = "url"
|
||||
CHANNEL_LOCAL = "local"
|
||||
CHANNEL_UPLOAD = "upload"
|
||||
CHANNEL_WIDGET = "widget"
|
||||
CHANNEL_REIMPORT_URL = "reimport_url"
|
||||
CHANNEL_REIMPORT_LOCAL = "reimport_local"
|
||||
|
||||
_URL_CHANNELS = frozenset(
|
||||
{CHANNEL_BATCH_IMPORT_URL, CHANNEL_URL, CHANNEL_REIMPORT_URL}
|
||||
)
|
||||
|
||||
# No-LoRA reason codes (persisted, consumed by the recipe modal).
|
||||
REASON_NO_LORAS_USED = "no_loras_used"
|
||||
REASON_API_NO_LORA_RESOURCES = "api_meta_no_lora_resources"
|
||||
REASON_API_META_MISSING = "api_meta_missing"
|
||||
REASON_NO_EMBEDDED_METADATA = "no_embedded_metadata"
|
||||
REASON_WORKFLOW_METADATA_LIMITED = "workflow_metadata_limited"
|
||||
REASON_VIDEO_NO_METADATA = "video_no_metadata"
|
||||
REASON_METADATA_UNSUPPORTED = "metadata_unsupported"
|
||||
REASON_UNKNOWN = "unknown"
|
||||
|
||||
_COMFY_PARSER_NAME = "ComfyMetadataParser"
|
||||
|
||||
# Cap for api_meta_keys kept in details — enough for the UI bullet without
|
||||
# bloating the recipe JSON.
|
||||
_MAX_DETAIL_KEYS = 12
|
||||
|
||||
|
||||
def compute_no_loras_reason(
|
||||
channel: str, diagnostics: Optional[Dict[str, Any]]
|
||||
) -> str:
|
||||
"""Classify why an import produced no LoRA entries.
|
||||
|
||||
Args:
|
||||
channel: One of the CHANNEL_* constants.
|
||||
diagnostics: Signals collected during analysis (see
|
||||
``RecipeAnalysisService``), or None for channels without analysis
|
||||
(e.g. widget saves).
|
||||
"""
|
||||
diag = diagnostics or {}
|
||||
|
||||
if diag.get("is_video"):
|
||||
return REASON_VIDEO_NO_METADATA
|
||||
|
||||
# Embedded metadata that is a ComfyUI workflow: LoRA extraction from
|
||||
# workflows is limited, so report that specifically.
|
||||
parser = diag.get("exif_parser") or diag.get("parser")
|
||||
if parser == _COMFY_PARSER_NAME:
|
||||
return REASON_WORKFLOW_METADATA_LIMITED
|
||||
|
||||
if channel in _URL_CHANNELS:
|
||||
if not diag.get("civitai_image"):
|
||||
# Generic (non-CivitAI) URL: only embedded metadata is available.
|
||||
if not diag.get("exif_present"):
|
||||
return REASON_NO_EMBEDDED_METADATA
|
||||
return (
|
||||
REASON_NO_LORAS_USED if parser else REASON_METADATA_UNSUPPORTED
|
||||
)
|
||||
# NOTE: no "parsed EXIF means no LoRAs were used" shortcut here.
|
||||
# CivitAI's onsite generator writes A1111-style EXIF (prompt, seed,
|
||||
# steps, ...) WITHOUT LoRA references — LoRA usage lives only in
|
||||
# CivitAI-internal data — so cleanly parsed EXIF cannot prove the
|
||||
# generation used no LoRAs. Report the API meta shape instead.
|
||||
api_keys = diag.get("api_meta_keys") or []
|
||||
api_mvids = diag.get("api_model_version_ids") or 0
|
||||
if api_keys or api_mvids:
|
||||
return REASON_API_NO_LORA_RESOURCES
|
||||
return REASON_API_META_MISSING
|
||||
|
||||
if channel == CHANNEL_WIDGET:
|
||||
return REASON_NO_LORAS_USED
|
||||
|
||||
# Local file / upload / local re-import: embedded metadata only.
|
||||
if not diag.get("exif_present"):
|
||||
return REASON_NO_EMBEDDED_METADATA
|
||||
return REASON_NO_LORAS_USED if parser else REASON_METADATA_UNSUPPORTED
|
||||
|
||||
|
||||
def build_import_info(
|
||||
channel: str,
|
||||
diagnostics: Optional[Dict[str, Any]],
|
||||
loras: Optional[List[Dict[str, Any]]],
|
||||
) -> Dict[str, Any]:
|
||||
"""Build the ``import_info`` block persisted on a recipe.
|
||||
|
||||
Always records the import channel; adds ``reason`` and ``details`` only
|
||||
when the recipe has no LoRAs.
|
||||
"""
|
||||
info: Dict[str, Any] = {"channel": channel}
|
||||
if loras:
|
||||
return info
|
||||
|
||||
info["reason"] = compute_no_loras_reason(channel, diagnostics)
|
||||
|
||||
diag = diagnostics or {}
|
||||
details: Dict[str, Any] = {}
|
||||
api_keys = diag.get("api_meta_keys")
|
||||
if api_keys:
|
||||
details["api_meta_keys"] = list(api_keys)[:_MAX_DETAIL_KEYS]
|
||||
api_mvids = diag.get("api_model_version_ids")
|
||||
if api_mvids is not None:
|
||||
details["api_model_version_ids"] = api_mvids
|
||||
if "exif_present" in diag:
|
||||
details["exif_present"] = bool(diag.get("exif_present"))
|
||||
if diag.get("exif_parser"):
|
||||
details["exif_parser"] = diag["exif_parser"]
|
||||
if diag.get("is_video"):
|
||||
details["is_video"] = True
|
||||
if details:
|
||||
info["details"] = details
|
||||
|
||||
return info
|
||||
@@ -13,9 +13,15 @@ from typing import Any, Awaitable, Dict, Iterable, Optional, cast
|
||||
|
||||
from ...config import config
|
||||
from ...recipes.constants import GEN_PARAM_KEYS
|
||||
from ...utils.base_model import (
|
||||
RELATION_COMPATIBLE,
|
||||
RELATION_INCOMPATIBLE,
|
||||
base_model_relation,
|
||||
)
|
||||
from ...utils.utils import calculate_recipe_fingerprint
|
||||
from ..pending_delete_service import get_pending_delete_service
|
||||
from .errors import RecipeNotFoundError, RecipeValidationError
|
||||
from .import_info import CHANNEL_UPLOAD, CHANNEL_WIDGET, build_import_info
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -52,6 +58,7 @@ class RecipePersistenceService:
|
||||
extension: str | None = None,
|
||||
recipe_id: str | None = None,
|
||||
target_dir: str | None = None,
|
||||
skip_optimize: bool = False,
|
||||
) -> PersistenceResult:
|
||||
"""Persist a user uploaded recipe.
|
||||
|
||||
@@ -61,6 +68,11 @@ class RecipePersistenceService:
|
||||
target_dir: If provided, save recipe files to this directory instead
|
||||
of the default recipes_dir. Used by re-import to preserve the
|
||||
original folder location.
|
||||
skip_optimize: If True, store the image bytes verbatim without
|
||||
resizing/re-encoding (recipe metadata is still embedded via a
|
||||
byte-level EXIF update that leaves the pixels untouched). Used
|
||||
by local re-import, where the source is the recipe's own
|
||||
already-optimized preview image.
|
||||
"""
|
||||
|
||||
missing_fields = []
|
||||
@@ -81,9 +93,12 @@ class RecipePersistenceService:
|
||||
|
||||
recipe_id = recipe_id or str(uuid.uuid4())
|
||||
|
||||
# Handle video formats by bypassing optimization and metadata embedding
|
||||
# Handle video formats by bypassing optimization and metadata embedding.
|
||||
# Local re-import also bypasses optimization: the source is the
|
||||
# recipe's own already-optimized preview image, so re-compressing it
|
||||
# would only degrade quality.
|
||||
is_video = extension in [".mp4", ".webm"]
|
||||
if is_video:
|
||||
if is_video or skip_optimize:
|
||||
optimized_image = resolved_image_bytes
|
||||
# extension is already set
|
||||
else:
|
||||
@@ -129,6 +144,22 @@ class RecipePersistenceService:
|
||||
if metadata.get("source_path"):
|
||||
recipe_data["source_path"] = metadata.get("source_path")
|
||||
|
||||
# Persist import provenance. Batch import / re-import paths pass a
|
||||
# prebuilt import_info; frontend-driven saves (upload, single URL,
|
||||
# local path) carry the analysis payload's diagnostics, from which
|
||||
# import_info is derived here.
|
||||
import_info = metadata.get("import_info")
|
||||
if not isinstance(import_info, dict):
|
||||
diagnostics = metadata.get("diagnostics")
|
||||
if isinstance(diagnostics, dict):
|
||||
import_info = build_import_info(
|
||||
diagnostics.get("channel") or CHANNEL_UPLOAD,
|
||||
diagnostics,
|
||||
loras_data,
|
||||
)
|
||||
if isinstance(import_info, dict) and import_info:
|
||||
recipe_data["import_info"] = import_info
|
||||
|
||||
nsfw_level = metadata.get("preview_nsfw_level")
|
||||
if nsfw_level is not None and isinstance(nsfw_level, int):
|
||||
recipe_data["preview_nsfw_level"] = nsfw_level
|
||||
@@ -153,7 +184,11 @@ class RecipePersistenceService:
|
||||
json.dump(recipe_data, file_obj, indent=4, ensure_ascii=False)
|
||||
|
||||
if not is_video:
|
||||
self._exif_utils.append_recipe_metadata(normalized_image_path, recipe_data)
|
||||
self._exif_utils.append_recipe_metadata(
|
||||
normalized_image_path,
|
||||
recipe_data,
|
||||
pixel_preserving=skip_optimize,
|
||||
)
|
||||
|
||||
matching_recipes = await self._find_matching_recipes(recipe_scanner, fingerprint, exclude_id=recipe_id)
|
||||
await recipe_scanner.add_recipe(recipe_data)
|
||||
@@ -430,20 +465,31 @@ class RecipePersistenceService:
|
||||
with open(recipe_path, "r", encoding="utf-8") as file_obj:
|
||||
recipe_base_model = json.load(file_obj).get("base_model", "")
|
||||
|
||||
target_lora = await recipe_scanner.get_local_lora(target_name, recipe_base_model)
|
||||
if not target_lora:
|
||||
matches = await recipe_scanner.find_local_loras_by_name(target_name)
|
||||
if len(matches) > 1:
|
||||
raise RecipeValidationError(
|
||||
f"Multiple local LoRAs match '{target_name}'; "
|
||||
"include the folder path to disambiguate"
|
||||
)
|
||||
if len(matches) == 1:
|
||||
raise RecipeValidationError(
|
||||
f"Local LoRA '{target_name}' has a different base model than the recipe"
|
||||
)
|
||||
matches = await recipe_scanner.find_local_loras_by_name(target_name)
|
||||
if not matches:
|
||||
raise RecipeNotFoundError(f"Local LoRA not found with name: {target_name}")
|
||||
|
||||
# Three-tier base-model guard: exact/unknown labels pass silently;
|
||||
# labels from the same architecture family (e.g. Pony ↔ Illustrious)
|
||||
# pass but are reported so the UI can warn; confident architecture
|
||||
# mismatches stay hard-rejected because they can never load.
|
||||
eligible: list[tuple[dict, str]] = []
|
||||
for match in matches:
|
||||
relation = base_model_relation(recipe_base_model, match.get("base_model"))
|
||||
if relation != RELATION_INCOMPATIBLE:
|
||||
eligible.append((match, relation))
|
||||
|
||||
if not eligible:
|
||||
raise RecipeValidationError(
|
||||
f"Local LoRA '{target_name}' has a different base model than the recipe"
|
||||
)
|
||||
if len(eligible) > 1:
|
||||
raise RecipeValidationError(
|
||||
f"Multiple local LoRAs match '{target_name}'; "
|
||||
"include the folder path to disambiguate"
|
||||
)
|
||||
target_lora, target_relation = eligible[0]
|
||||
|
||||
recipe_data, updated_lora = await recipe_scanner.update_lora_entry(
|
||||
recipe_id,
|
||||
lora_index,
|
||||
@@ -451,6 +497,43 @@ class RecipePersistenceService:
|
||||
target_lora=target_lora,
|
||||
)
|
||||
|
||||
image_path = recipe_data.get("file_path")
|
||||
if image_path and os.path.exists(image_path):
|
||||
self._exif_utils.append_recipe_metadata(image_path, recipe_data)
|
||||
|
||||
matching_recipes = []
|
||||
if "fingerprint" in recipe_data:
|
||||
matching_recipes = await recipe_scanner.find_recipes_by_fingerprint(recipe_data["fingerprint"])
|
||||
if recipe_id in matching_recipes:
|
||||
matching_recipes.remove(recipe_id)
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"success": True,
|
||||
"recipe_id": recipe_id,
|
||||
"updated_lora": updated_lora,
|
||||
"matching_recipes": matching_recipes,
|
||||
}
|
||||
if target_relation == RELATION_COMPATIBLE:
|
||||
# Structured data, not prose — the frontend localizes the warning.
|
||||
payload["base_model_mismatch"] = {
|
||||
"recipe_base_model": recipe_base_model,
|
||||
"lora_base_model": target_lora.get("base_model") or "",
|
||||
}
|
||||
return PersistenceResult(payload)
|
||||
|
||||
async def restore_lora(
|
||||
self,
|
||||
*,
|
||||
recipe_scanner,
|
||||
recipe_id: str,
|
||||
lora_index: int,
|
||||
) -> PersistenceResult:
|
||||
"""Restore a LoRA entry to the state captured before its reconnect."""
|
||||
|
||||
recipe_data, updated_lora = await recipe_scanner.restore_lora_entry(
|
||||
recipe_id, lora_index
|
||||
)
|
||||
|
||||
image_path = recipe_data.get("file_path")
|
||||
if image_path and os.path.exists(image_path):
|
||||
self._exif_utils.append_recipe_metadata(image_path, recipe_data)
|
||||
@@ -470,6 +553,231 @@ class RecipePersistenceService:
|
||||
}
|
||||
)
|
||||
|
||||
async def get_reconnect_suggestions(
|
||||
self,
|
||||
*,
|
||||
recipe_scanner,
|
||||
recipe_id: str,
|
||||
lora_index: int,
|
||||
query: str | None = None,
|
||||
) -> PersistenceResult:
|
||||
"""Return ranked local LoRA candidates for reconnecting a recipe entry."""
|
||||
|
||||
recipe_path = await recipe_scanner.get_recipe_json_path(recipe_id)
|
||||
if not recipe_path or not os.path.exists(recipe_path):
|
||||
raise RecipeNotFoundError("Recipe not found")
|
||||
|
||||
with open(recipe_path, "r", encoding="utf-8") as file_obj:
|
||||
recipe_data = json.load(file_obj)
|
||||
|
||||
loras = recipe_data.get("loras") or []
|
||||
if lora_index < 0 or lora_index >= len(loras):
|
||||
raise RecipeValidationError(f"Invalid lora_index: {lora_index}")
|
||||
|
||||
suggestions = await recipe_scanner.suggest_reconnect_candidates(
|
||||
entry=loras[lora_index],
|
||||
recipe_base_model=recipe_data.get("base_model"),
|
||||
query=query,
|
||||
)
|
||||
|
||||
return PersistenceResult({"success": True, "suggestions": suggestions})
|
||||
|
||||
async def mark_lora_hash_invalid(
|
||||
self,
|
||||
*,
|
||||
recipe_scanner,
|
||||
recipe_id: str,
|
||||
lora_index: int,
|
||||
hash_invalid: bool = True,
|
||||
) -> PersistenceResult:
|
||||
"""Mark a recipe LoRA entry's hash as unresolvable on CivitAI.
|
||||
|
||||
Called when a download attempt by hash returned "Model not found".
|
||||
The flag makes the entry an unresolved rematch candidate without
|
||||
altering its stored hash/file_name.
|
||||
"""
|
||||
|
||||
recipe_data, updated_lora = await recipe_scanner.set_lora_entry_hash_invalid(
|
||||
recipe_id,
|
||||
lora_index,
|
||||
hash_invalid=hash_invalid,
|
||||
)
|
||||
|
||||
return PersistenceResult(
|
||||
{
|
||||
"success": True,
|
||||
"recipe_id": recipe_id,
|
||||
"hash_invalid": bool(hash_invalid),
|
||||
"updated_lora": updated_lora,
|
||||
}
|
||||
)
|
||||
|
||||
async def reconnect_checkpoint(
|
||||
self,
|
||||
*,
|
||||
recipe_scanner,
|
||||
recipe_id: str,
|
||||
target_name: str,
|
||||
) -> PersistenceResult:
|
||||
"""Reconnect the checkpoint entry within an existing recipe."""
|
||||
|
||||
recipe_path = await recipe_scanner.get_recipe_json_path(recipe_id)
|
||||
if not recipe_path or not os.path.exists(recipe_path):
|
||||
raise RecipeNotFoundError("Recipe not found")
|
||||
|
||||
with open(recipe_path, "r", encoding="utf-8") as file_obj:
|
||||
recipe_base_model = json.load(file_obj).get("base_model", "")
|
||||
|
||||
matches = await recipe_scanner.find_local_checkpoints_by_name(target_name)
|
||||
if not matches:
|
||||
raise RecipeNotFoundError(
|
||||
f"Local checkpoint not found with name: {target_name}"
|
||||
)
|
||||
|
||||
# Same three-tier base-model guard as reconnect_lora: exact/unknown
|
||||
# labels pass silently; same-architecture-family labels pass but are
|
||||
# reported so the UI can warn; confident mismatches stay hard-rejected.
|
||||
eligible: list[tuple[dict, str]] = []
|
||||
for match in matches:
|
||||
relation = base_model_relation(recipe_base_model, match.get("base_model"))
|
||||
if relation != RELATION_INCOMPATIBLE:
|
||||
eligible.append((match, relation))
|
||||
|
||||
if not eligible:
|
||||
raise RecipeValidationError(
|
||||
f"Local checkpoint '{target_name}' has a different base model "
|
||||
"than the recipe"
|
||||
)
|
||||
if len(eligible) > 1:
|
||||
raise RecipeValidationError(
|
||||
f"Multiple local checkpoints match '{target_name}'; "
|
||||
"include the folder path to disambiguate"
|
||||
)
|
||||
target_checkpoint, target_relation = eligible[0]
|
||||
|
||||
recipe_data, updated_checkpoint = await recipe_scanner.update_checkpoint_entry(
|
||||
recipe_id,
|
||||
target_name=target_name,
|
||||
target_checkpoint=target_checkpoint,
|
||||
)
|
||||
|
||||
image_path = recipe_data.get("file_path")
|
||||
if image_path and os.path.exists(image_path):
|
||||
self._exif_utils.append_recipe_metadata(image_path, recipe_data)
|
||||
|
||||
matching_recipes = []
|
||||
if "fingerprint" in recipe_data:
|
||||
matching_recipes = await recipe_scanner.find_recipes_by_fingerprint(
|
||||
recipe_data["fingerprint"]
|
||||
)
|
||||
if recipe_id in matching_recipes:
|
||||
matching_recipes.remove(recipe_id)
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"success": True,
|
||||
"recipe_id": recipe_id,
|
||||
"updated_checkpoint": updated_checkpoint,
|
||||
"matching_recipes": matching_recipes,
|
||||
}
|
||||
if target_relation == RELATION_COMPATIBLE:
|
||||
# Structured data, not prose — the frontend localizes the warning.
|
||||
payload["base_model_mismatch"] = {
|
||||
"recipe_base_model": recipe_base_model,
|
||||
"checkpoint_base_model": target_checkpoint.get("base_model") or "",
|
||||
}
|
||||
return PersistenceResult(payload)
|
||||
|
||||
async def restore_checkpoint(
|
||||
self,
|
||||
*,
|
||||
recipe_scanner,
|
||||
recipe_id: str,
|
||||
) -> PersistenceResult:
|
||||
"""Restore the checkpoint entry to the state captured before its reconnect."""
|
||||
|
||||
recipe_data, updated_checkpoint = await recipe_scanner.restore_checkpoint_entry(
|
||||
recipe_id
|
||||
)
|
||||
|
||||
image_path = recipe_data.get("file_path")
|
||||
if image_path and os.path.exists(image_path):
|
||||
self._exif_utils.append_recipe_metadata(image_path, recipe_data)
|
||||
|
||||
matching_recipes = []
|
||||
if "fingerprint" in recipe_data:
|
||||
matching_recipes = await recipe_scanner.find_recipes_by_fingerprint(
|
||||
recipe_data["fingerprint"]
|
||||
)
|
||||
if recipe_id in matching_recipes:
|
||||
matching_recipes.remove(recipe_id)
|
||||
|
||||
return PersistenceResult(
|
||||
{
|
||||
"success": True,
|
||||
"recipe_id": recipe_id,
|
||||
"updated_checkpoint": updated_checkpoint,
|
||||
"matching_recipes": matching_recipes,
|
||||
}
|
||||
)
|
||||
|
||||
async def get_checkpoint_reconnect_suggestions(
|
||||
self,
|
||||
*,
|
||||
recipe_scanner,
|
||||
recipe_id: str,
|
||||
query: str | None = None,
|
||||
) -> PersistenceResult:
|
||||
"""Return ranked local checkpoint candidates for reconnecting a recipe entry."""
|
||||
|
||||
recipe_path = await recipe_scanner.get_recipe_json_path(recipe_id)
|
||||
if not recipe_path or not os.path.exists(recipe_path):
|
||||
raise RecipeNotFoundError("Recipe not found")
|
||||
|
||||
with open(recipe_path, "r", encoding="utf-8") as file_obj:
|
||||
recipe_data = json.load(file_obj)
|
||||
|
||||
checkpoint = recipe_data.get("checkpoint")
|
||||
if not isinstance(checkpoint, dict):
|
||||
raise RecipeValidationError("Recipe has no checkpoint entry")
|
||||
|
||||
suggestions = await recipe_scanner.suggest_checkpoint_reconnect_candidates(
|
||||
entry=checkpoint,
|
||||
recipe_base_model=recipe_data.get("base_model"),
|
||||
query=query,
|
||||
)
|
||||
|
||||
return PersistenceResult({"success": True, "suggestions": suggestions})
|
||||
|
||||
async def mark_checkpoint_hash_invalid(
|
||||
self,
|
||||
*,
|
||||
recipe_scanner,
|
||||
recipe_id: str,
|
||||
hash_invalid: bool = True,
|
||||
) -> PersistenceResult:
|
||||
"""Mark the recipe checkpoint entry's hash as unresolvable on CivitAI.
|
||||
|
||||
Called when a download attempt by hash returned "Model not found".
|
||||
The flag makes the entry an unresolved rematch candidate without
|
||||
altering its stored hash/file_name.
|
||||
"""
|
||||
|
||||
recipe_data, updated_checkpoint = (
|
||||
await recipe_scanner.set_checkpoint_entry_hash_invalid(
|
||||
recipe_id,
|
||||
hash_invalid=hash_invalid,
|
||||
)
|
||||
)
|
||||
|
||||
return PersistenceResult(
|
||||
{
|
||||
"success": True,
|
||||
"recipe_id": recipe_id,
|
||||
"hash_invalid": bool(hash_invalid),
|
||||
"updated_checkpoint": updated_checkpoint,
|
||||
}
|
||||
)
|
||||
|
||||
async def bulk_delete(
|
||||
self,
|
||||
*,
|
||||
@@ -619,6 +927,9 @@ class RecipePersistenceService:
|
||||
# Widget saves re-encode an in-memory tensor to PNG/WebP with no
|
||||
# embedded metadata chunks, so a workflow can never be present.
|
||||
"has_workflow": False,
|
||||
# Widget saves read LoRAs straight from the current workflow; an
|
||||
# empty list means the workflow used no LoRAs.
|
||||
"import_info": build_import_info(CHANNEL_WIDGET, None, loras_data),
|
||||
}
|
||||
if checkpoint_entry:
|
||||
recipe_data["checkpoint"] = checkpoint_entry
|
||||
@@ -793,6 +1104,7 @@ class RecipePersistenceService:
|
||||
"modelName": lora.get("name", ""),
|
||||
"modelVersionName": lora.get("version", ""),
|
||||
"isDeleted": lora.get("isDeleted", False),
|
||||
"hashInvalid": lora.get("hashInvalid", False),
|
||||
"exclude": lora.get("exclude", False),
|
||||
}
|
||||
|
||||
|
||||
@@ -297,23 +297,44 @@ class ServiceRegistry:
|
||||
async def get_embedding_scanner(cls):
|
||||
"""Get or create Embedding scanner instance"""
|
||||
service_name = "embedding_scanner"
|
||||
|
||||
|
||||
if service_name in cls._services:
|
||||
return cls._services[service_name]
|
||||
|
||||
|
||||
async with cls._get_lock(service_name):
|
||||
# Double-check after acquiring lock
|
||||
if service_name in cls._services:
|
||||
return cls._services[service_name]
|
||||
|
||||
|
||||
# Import here to avoid circular imports
|
||||
from .embedding_scanner import EmbeddingScanner
|
||||
|
||||
|
||||
scanner = await EmbeddingScanner.get_instance()
|
||||
cls._services[service_name] = scanner
|
||||
logger.debug(f"Created and registered {service_name}")
|
||||
return scanner
|
||||
|
||||
|
||||
@classmethod
|
||||
async def get_other_scanner(cls):
|
||||
"""Get or create Other-model scanner instance"""
|
||||
service_name = "other_scanner"
|
||||
|
||||
if service_name in cls._services:
|
||||
return cls._services[service_name]
|
||||
|
||||
async with cls._get_lock(service_name):
|
||||
# Double-check after acquiring lock
|
||||
if service_name in cls._services:
|
||||
return cls._services[service_name]
|
||||
|
||||
# Import here to avoid circular imports
|
||||
from .other_scanner import OtherScanner
|
||||
|
||||
scanner = await OtherScanner.get_instance()
|
||||
cls._services[service_name] = scanner
|
||||
logger.debug(f"Created and registered {service_name}")
|
||||
return scanner
|
||||
|
||||
@classmethod
|
||||
def clear_services(cls):
|
||||
"""Clear all registered services - mainly for testing"""
|
||||
|
||||
+270
-22
@@ -19,19 +19,26 @@ from typing import (
|
||||
Mapping,
|
||||
Optional,
|
||||
Sequence,
|
||||
Set,
|
||||
Tuple,
|
||||
)
|
||||
|
||||
from platformdirs import user_config_dir
|
||||
|
||||
from ..utils.constants import (
|
||||
DEFAULT_DOWNLOAD_PATH_TEMPLATES,
|
||||
DEFAULT_ENABLED_OTHER_SUB_TYPES,
|
||||
DEFAULT_HASH_CHUNK_SIZE_MB,
|
||||
DEFAULT_PRIORITY_TAG_CONFIG,
|
||||
OTHER_SUB_TYPE_FOLDER_KEYS,
|
||||
SUPPORTED_DOWNLOAD_SKIP_BASE_MODELS,
|
||||
VALID_OTHER_SUB_TYPES,
|
||||
normalize_other_sub_types,
|
||||
)
|
||||
from ..utils.preview_selection import VALID_MATURE_BLUR_LEVELS
|
||||
from ..utils.settings_paths import (
|
||||
APP_NAME,
|
||||
_portable_env_override,
|
||||
ensure_settings_file,
|
||||
get_legacy_settings_path,
|
||||
get_settings_dir_override,
|
||||
@@ -83,9 +90,15 @@ DEFAULT_SETTINGS: Dict[str, Any] = {
|
||||
"default_checkpoint_root": "",
|
||||
"default_unet_root": "",
|
||||
"default_embedding_root": "",
|
||||
"default_other_roots": {},
|
||||
# Other Models management is opt-in: nothing is scanned, shown or offered
|
||||
# for download until the user turns the feature on.
|
||||
"enable_other_models": False,
|
||||
"enabled_other_sub_types": list(DEFAULT_ENABLED_OTHER_SUB_TYPES),
|
||||
"recipes_path": "",
|
||||
"base_model_path_mappings": {},
|
||||
"download_path_templates": {},
|
||||
"download_filename_templates": {},
|
||||
"folder_paths": {},
|
||||
"extra_folder_paths": {},
|
||||
"example_images_path": "",
|
||||
@@ -116,6 +129,7 @@ DEFAULT_SETTINGS: Dict[str, Any] = {
|
||||
"backup_retention_count": 5,
|
||||
"use_new_license_icons": True,
|
||||
"group_by_model": False,
|
||||
"sticky_controls": False,
|
||||
# AI / LLM provider configuration (BYOK)
|
||||
"llm_provider": "openai", # "openai" | "ollama" | "custom"
|
||||
"llm_api_key": "",
|
||||
@@ -161,13 +175,23 @@ class SettingsManager:
|
||||
self._check_environment_variables()
|
||||
self._collect_configuration_warnings()
|
||||
|
||||
if (
|
||||
os.environ.get("LORA_MANAGER_PORTABLE", "0") == "1"
|
||||
and not is_settings_dir_pinned()
|
||||
):
|
||||
portable_override = _portable_env_override()
|
||||
if portable_override is True and not is_settings_dir_pinned():
|
||||
if not self.settings.get("use_portable_settings"):
|
||||
self.settings["use_portable_settings"] = True
|
||||
self._save_settings()
|
||||
elif portable_override is False and self.settings.get(
|
||||
"use_portable_settings"
|
||||
):
|
||||
# Explicit opt-out from a persisted portable mode: clear the flag so
|
||||
# later runs go back to the shared settings directory instead of
|
||||
# requiring a manual edit of settings.json.
|
||||
logger.info(
|
||||
"Clearing the persisted portable-mode flag because %s=0",
|
||||
"LORA_MANAGER_PORTABLE",
|
||||
)
|
||||
self.settings["use_portable_settings"] = False
|
||||
self._save_settings()
|
||||
|
||||
if self._needs_initial_save:
|
||||
self._save_settings()
|
||||
@@ -286,6 +310,29 @@ class SettingsManager:
|
||||
|
||||
return payload == template
|
||||
|
||||
def get_template_folder_path_placeholders(self) -> Set[str]:
|
||||
"""Placeholder folder_paths values shipped in settings.json.example.
|
||||
|
||||
A fresh standalone install is seeded from the template, so its
|
||||
documentation-only placeholder paths end up in the live settings
|
||||
file. The Model Paths settings UI hides them; the first real save
|
||||
overwrites them via ``set("folder_paths")``.
|
||||
"""
|
||||
|
||||
template = self._read_template_payload()
|
||||
if not template:
|
||||
return set()
|
||||
|
||||
folder_paths = template.get("folder_paths")
|
||||
if not isinstance(folder_paths, Mapping):
|
||||
return set()
|
||||
|
||||
placeholders: Set[str] = set()
|
||||
for value in folder_paths.values():
|
||||
paths = value if isinstance(value, list) else [value]
|
||||
placeholders.update(p for p in paths if isinstance(p, str) and p)
|
||||
return placeholders
|
||||
|
||||
def _merge_template_with_defaults(
|
||||
self, defaults: Dict[str, Any], template: Mapping[str, Any]
|
||||
) -> Dict[str, Any]:
|
||||
@@ -308,6 +355,7 @@ class SettingsManager:
|
||||
default_checkpoint_root=merged.get("default_checkpoint_root"),
|
||||
default_unet_root=merged.get("default_unet_root"),
|
||||
default_embedding_root=merged.get("default_embedding_root"),
|
||||
default_other_roots=merged.get("default_other_roots"),
|
||||
recipes_path=merged.get("recipes_path"),
|
||||
)
|
||||
}
|
||||
@@ -442,6 +490,7 @@ class SettingsManager:
|
||||
),
|
||||
default_unet_root=self.settings.get("default_unet_root", ""),
|
||||
default_embedding_root=self.settings.get("default_embedding_root", ""),
|
||||
default_other_roots=self.settings.get("default_other_roots"),
|
||||
recipes_path=self.settings.get("recipes_path", ""),
|
||||
)
|
||||
libraries = {library_name: library_payload}
|
||||
@@ -493,6 +542,7 @@ class SettingsManager:
|
||||
default_checkpoint_root=data.get("default_checkpoint_root"),
|
||||
default_unet_root=data.get("default_unet_root"),
|
||||
default_embedding_root=data.get("default_embedding_root"),
|
||||
default_other_roots=data.get("default_other_roots"),
|
||||
recipes_path=data.get("recipes_path"),
|
||||
metadata=data.get("metadata"),
|
||||
base=data,
|
||||
@@ -540,6 +590,9 @@ class SettingsManager:
|
||||
self.settings["default_embedding_root"] = active_library.get(
|
||||
"default_embedding_root", ""
|
||||
)
|
||||
self.settings["default_other_roots"] = self._normalize_default_other_roots(
|
||||
active_library.get("default_other_roots", {})
|
||||
)
|
||||
self.settings["recipes_path"] = active_library.get("recipes_path", "")
|
||||
|
||||
if save:
|
||||
@@ -557,6 +610,7 @@ class SettingsManager:
|
||||
default_checkpoint_root: Optional[str] = None,
|
||||
default_unet_root: Optional[str] = None,
|
||||
default_embedding_root: Optional[str] = None,
|
||||
default_other_roots: Optional[Mapping[str, str]] = None,
|
||||
recipes_path: Optional[str] = None,
|
||||
metadata: Optional[Mapping[str, Any]] = None,
|
||||
base: Optional[Mapping[str, Any]] = None,
|
||||
@@ -596,6 +650,15 @@ class SettingsManager:
|
||||
else:
|
||||
payload.setdefault("default_embedding_root", "")
|
||||
|
||||
if default_other_roots is not None:
|
||||
payload["default_other_roots"] = self._normalize_default_other_roots(
|
||||
default_other_roots
|
||||
)
|
||||
else:
|
||||
payload["default_other_roots"] = self._normalize_default_other_roots(
|
||||
payload.get("default_other_roots", {})
|
||||
)
|
||||
|
||||
if recipes_path is not None:
|
||||
payload["recipes_path"] = recipes_path
|
||||
else:
|
||||
@@ -631,6 +694,71 @@ class SettingsManager:
|
||||
normalized[key] = cleaned
|
||||
return normalized
|
||||
|
||||
def _normalize_default_other_roots(
|
||||
self, value: Any, *, strict: bool = False
|
||||
) -> Dict[str, str]:
|
||||
"""Normalize a ``default_other_roots`` mapping ({sub_type: root path}).
|
||||
|
||||
Unknown sub_type keys and non-string/empty paths are dropped; with
|
||||
``strict=True`` unknown sub_type keys raise instead (used by ``set()``
|
||||
so typos in API payloads surface as errors).
|
||||
"""
|
||||
if not isinstance(value, Mapping):
|
||||
if strict and value is not None:
|
||||
raise ValueError("default_other_roots must be a mapping")
|
||||
return {}
|
||||
normalized: Dict[str, str] = {}
|
||||
for sub_type, path in value.items():
|
||||
if sub_type not in VALID_OTHER_SUB_TYPES:
|
||||
if strict:
|
||||
raise ValueError(
|
||||
f"Unknown other-model sub-type '{sub_type}'; "
|
||||
f"expected one of {sorted(VALID_OTHER_SUB_TYPES)}"
|
||||
)
|
||||
continue
|
||||
if not isinstance(path, str):
|
||||
continue
|
||||
stripped = path.strip()
|
||||
if stripped:
|
||||
normalized[sub_type] = stripped
|
||||
return normalized
|
||||
|
||||
def is_other_models_enabled(self) -> bool:
|
||||
"""Return True when the opt-in Other Models management is enabled."""
|
||||
return bool(self.settings.get("enable_other_models", False))
|
||||
|
||||
def get_enabled_other_sub_types(self) -> List[str]:
|
||||
"""Return the enabled other-model sub_types (empty when the feature is off)."""
|
||||
if not self.is_other_models_enabled():
|
||||
return []
|
||||
return normalize_other_sub_types(self.settings.get("enabled_other_sub_types"))
|
||||
|
||||
def is_other_sub_type_enabled(self, sub_type: Optional[str]) -> bool:
|
||||
"""Return True when ``sub_type`` is currently managed."""
|
||||
if not sub_type:
|
||||
return False
|
||||
return sub_type in self.get_enabled_other_sub_types()
|
||||
|
||||
def _apply_other_model_settings_change(self) -> None:
|
||||
"""Rebuild other-model roots and refresh the other scanner after a toggle."""
|
||||
try:
|
||||
from ..config import config # Local import to avoid circular dependency
|
||||
|
||||
config.refresh_other_roots()
|
||||
except Exception as exc: # pragma: no cover - defensive logging
|
||||
logger.debug("Failed to refresh other-model roots: %s", exc)
|
||||
|
||||
try:
|
||||
from .service_registry import ServiceRegistry # pyright: ignore[reportImportCycles]
|
||||
|
||||
scanner = ServiceRegistry.get_service_sync("other_scanner")
|
||||
if scanner is not None and hasattr(scanner, "on_library_changed"):
|
||||
# reconcile=True lets the scanner pick up newly enabled roots and
|
||||
# purge rows for folders that are no longer managed.
|
||||
scanner.on_library_changed(reconcile=True)
|
||||
except Exception as exc: # pragma: no cover - defensive logging
|
||||
logger.debug("Failed to refresh other scanner after settings change: %s", exc)
|
||||
|
||||
def _has_configured_paths(self, folder_paths: Any) -> bool:
|
||||
if not isinstance(folder_paths, Mapping):
|
||||
return False
|
||||
@@ -743,6 +871,7 @@ class SettingsManager:
|
||||
default_checkpoint_root: Optional[str] = None,
|
||||
default_unet_root: Optional[str] = None,
|
||||
default_embedding_root: Optional[str] = None,
|
||||
default_other_roots: Optional[Mapping[str, str]] = None,
|
||||
recipes_path: Optional[str] = None,
|
||||
) -> bool:
|
||||
libraries = self.settings.get("libraries", {})
|
||||
@@ -793,6 +922,14 @@ class SettingsManager:
|
||||
library["default_embedding_root"] = default_embedding_root
|
||||
changed = True
|
||||
|
||||
if default_other_roots is not None:
|
||||
normalized_other_roots = self._normalize_default_other_roots(
|
||||
default_other_roots
|
||||
)
|
||||
if library.get("default_other_roots") != normalized_other_roots:
|
||||
library["default_other_roots"] = normalized_other_roots
|
||||
changed = True
|
||||
|
||||
if recipes_path is not None and library.get("recipes_path") != recipes_path:
|
||||
library["recipes_path"] = recipes_path
|
||||
changed = True
|
||||
@@ -893,12 +1030,53 @@ class SettingsManager:
|
||||
updated = _check_and_auto_set("unet", "default_unet_root") or updated
|
||||
updated = _check_and_auto_set("embeddings", "default_embedding_root") or updated
|
||||
|
||||
# Other-model default roots: one entry per enabled sub_type; candidates
|
||||
# are the union of that sub_type's folder_paths keys (text_encoder
|
||||
# merges the legacy 'clip' key with 'text_encoders'). When the opt-in
|
||||
# feature is off the existing mapping is left untouched.
|
||||
other_roots = self._normalize_default_other_roots(
|
||||
self.settings.get("default_other_roots")
|
||||
)
|
||||
if self.is_other_models_enabled():
|
||||
for sub_type in self.get_enabled_other_sub_types():
|
||||
candidates: List[str] = []
|
||||
candidate_identities: set[str] = set()
|
||||
for folder_key in OTHER_SUB_TYPE_FOLDER_KEYS.get(sub_type, []):
|
||||
for candidate in self._get_valid_root_candidates(folder_key):
|
||||
identity = _normalize_root_identity(candidate)
|
||||
if identity in candidate_identities:
|
||||
continue
|
||||
candidate_identities.add(identity)
|
||||
candidates.append(candidate)
|
||||
if not candidates:
|
||||
continue
|
||||
current = other_roots.get(sub_type, "")
|
||||
if current and _normalize_root_identity(current) in candidate_identities:
|
||||
continue
|
||||
other_roots[sub_type] = candidates[0]
|
||||
if current:
|
||||
logger.info(
|
||||
"Repaired stale default_other_roots[%s] from '%s' to '%s' because it is not present in primary or extra roots",
|
||||
sub_type,
|
||||
current,
|
||||
candidates[0],
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"Auto-set default_other_roots[%s] to '%s'",
|
||||
sub_type,
|
||||
candidates[0],
|
||||
)
|
||||
updated = True
|
||||
|
||||
if updated:
|
||||
self.settings["default_other_roots"] = other_roots
|
||||
self._update_active_library_entry(
|
||||
default_lora_root=self.settings.get("default_lora_root"),
|
||||
default_checkpoint_root=self.settings.get("default_checkpoint_root"),
|
||||
default_unet_root=self.settings.get("default_unet_root"),
|
||||
default_embedding_root=self.settings.get("default_embedding_root"),
|
||||
default_other_roots=other_roots,
|
||||
)
|
||||
if self._bootstrap_reason == "missing":
|
||||
self._needs_initial_save = True
|
||||
@@ -1066,19 +1244,27 @@ class SettingsManager:
|
||||
if self._bootstrap_reason == "missing":
|
||||
message = (
|
||||
"LoRA Manager created a default settings.json because no configuration was found. "
|
||||
"Edit settings.json to add your model directories so library scanning can run."
|
||||
"Open Settings → Model Paths to add your model directories so library scanning can run."
|
||||
)
|
||||
else:
|
||||
message = (
|
||||
"LoRA Manager could not locate any configured model directories. "
|
||||
"Edit settings.json to add your model folders so library scanning can run."
|
||||
"Open Settings → Model Paths to add your model folders so library scanning can run."
|
||||
)
|
||||
self._add_startup_message(
|
||||
code="missing-model-paths",
|
||||
title="Model folders need setup",
|
||||
message=message,
|
||||
severity="warning",
|
||||
actions=self._default_settings_actions(),
|
||||
actions=[
|
||||
{
|
||||
"action": "open-model-paths-settings",
|
||||
"label": "Configure model folders",
|
||||
"type": "primary",
|
||||
"icon": "fas fa-cog",
|
||||
},
|
||||
*self._default_settings_actions(),
|
||||
],
|
||||
dismissible=False,
|
||||
)
|
||||
|
||||
@@ -1091,6 +1277,7 @@ class SettingsManager:
|
||||
defaults = copy.deepcopy(DEFAULT_SETTINGS)
|
||||
defaults["base_model_path_mappings"] = {}
|
||||
defaults["download_path_templates"] = {}
|
||||
defaults["download_filename_templates"] = {}
|
||||
defaults["priority_tags"] = DEFAULT_PRIORITY_TAG_CONFIG.copy()
|
||||
defaults.setdefault("folder_paths", {})
|
||||
defaults.setdefault("extra_folder_paths", {})
|
||||
@@ -1598,6 +1785,12 @@ class SettingsManager:
|
||||
value = self.normalize_download_skip_base_models(value)
|
||||
elif key == "mature_blur_level":
|
||||
value = self.normalize_mature_blur_level(value)
|
||||
elif key == "default_other_roots":
|
||||
value = self._normalize_default_other_roots(value, strict=True)
|
||||
elif key == "enabled_other_sub_types":
|
||||
value = normalize_other_sub_types(value)
|
||||
elif key == "enable_other_models":
|
||||
value = bool(value)
|
||||
elif key == "recipes_path":
|
||||
current_recipes_dir = self._get_effective_recipes_dir()
|
||||
value = self._normalize_recipes_path_value(value)
|
||||
@@ -1625,6 +1818,8 @@ class SettingsManager:
|
||||
self._update_active_library_entry(default_unet_root=str(value))
|
||||
elif key == "default_embedding_root":
|
||||
self._update_active_library_entry(default_embedding_root=str(value))
|
||||
elif key == "default_other_roots":
|
||||
self._update_active_library_entry(default_other_roots=value)
|
||||
elif key == "recipes_path":
|
||||
self._update_active_library_entry(recipes_path=str(value))
|
||||
elif key == "model_name_display":
|
||||
@@ -1632,6 +1827,8 @@ class SettingsManager:
|
||||
self._save_settings()
|
||||
if key == "recipes_path":
|
||||
self._notify_library_change(self.get_active_library_name())
|
||||
if key in ("enable_other_models", "enabled_other_sub_types"):
|
||||
self._apply_other_model_settings_change()
|
||||
if portable_switch_pending:
|
||||
self._finalize_portable_switch()
|
||||
|
||||
@@ -1795,6 +1992,7 @@ class SettingsManager:
|
||||
"lora_scanner",
|
||||
"checkpoint_scanner",
|
||||
"embedding_scanner",
|
||||
"other_scanner",
|
||||
"recipe_scanner",
|
||||
):
|
||||
service = ServiceRegistry.get_service_sync(service_name)
|
||||
@@ -1959,6 +2157,7 @@ class SettingsManager:
|
||||
default_checkpoint_root: Optional[str] = None,
|
||||
default_unet_root: Optional[str] = None,
|
||||
default_embedding_root: Optional[str] = None,
|
||||
default_other_roots: Optional[Mapping[str, str]] = None,
|
||||
recipes_path: Optional[str] = None,
|
||||
metadata: Optional[Mapping[str, Any]] = None,
|
||||
activate: bool = False,
|
||||
@@ -2003,6 +2202,11 @@ class SettingsManager:
|
||||
if default_embedding_root is not None
|
||||
else existing.get("default_embedding_root")
|
||||
),
|
||||
default_other_roots=(
|
||||
default_other_roots
|
||||
if default_other_roots is not None
|
||||
else existing.get("default_other_roots")
|
||||
),
|
||||
recipes_path=(
|
||||
recipes_path
|
||||
if recipes_path is not None
|
||||
@@ -2035,6 +2239,7 @@ class SettingsManager:
|
||||
default_checkpoint_root: str = "",
|
||||
default_unet_root: str = "",
|
||||
default_embedding_root: str = "",
|
||||
default_other_roots: Optional[Mapping[str, str]] = None,
|
||||
recipes_path: str = "",
|
||||
metadata: Optional[Mapping[str, Any]] = None,
|
||||
activate: bool = False,
|
||||
@@ -2053,6 +2258,7 @@ class SettingsManager:
|
||||
default_checkpoint_root=default_checkpoint_root,
|
||||
default_unet_root=default_unet_root,
|
||||
default_embedding_root=default_embedding_root,
|
||||
default_other_roots=default_other_roots,
|
||||
recipes_path=recipes_path,
|
||||
metadata=metadata,
|
||||
activate=activate,
|
||||
@@ -2113,6 +2319,7 @@ class SettingsManager:
|
||||
default_checkpoint_root: Optional[str] = None,
|
||||
default_unet_root: Optional[str] = None,
|
||||
default_embedding_root: Optional[str] = None,
|
||||
default_other_roots: Optional[Mapping[str, str]] = None,
|
||||
recipes_path: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Update folder paths for the active library."""
|
||||
@@ -2126,6 +2333,7 @@ class SettingsManager:
|
||||
default_checkpoint_root=default_checkpoint_root,
|
||||
default_unet_root=default_unet_root,
|
||||
default_embedding_root=default_embedding_root,
|
||||
default_other_roots=default_other_roots,
|
||||
recipes_path=recipes_path,
|
||||
activate=True,
|
||||
)
|
||||
@@ -2150,6 +2358,7 @@ class SettingsManager:
|
||||
"lora_scanner",
|
||||
"checkpoint_scanner",
|
||||
"embedding_scanner",
|
||||
"other_scanner",
|
||||
"recipe_scanner",
|
||||
"model_update_service",
|
||||
):
|
||||
@@ -2172,10 +2381,14 @@ class SettingsManager:
|
||||
"""Get download path template for specific model type
|
||||
|
||||
Args:
|
||||
model_type: The type of model ('lora', 'checkpoint', 'embedding')
|
||||
model_type: The type of model ('lora', 'checkpoint', 'embedding',
|
||||
'other')
|
||||
|
||||
Returns:
|
||||
Template string for the model type, defaults to '{base_model}/{first_tag}'
|
||||
Template string for the model type. Falls back to the per-type
|
||||
default in ``DEFAULT_DOWNLOAD_PATH_TEMPLATES``; unknown model types
|
||||
resolve to an empty string (flat layout) rather than silently
|
||||
nesting downloads under an unconfigured subfolder.
|
||||
"""
|
||||
templates = self.settings.get("download_path_templates", {})
|
||||
|
||||
@@ -2199,27 +2412,62 @@ class SettingsManager:
|
||||
logger.warning(
|
||||
f"Failed to parse download_path_templates JSON string: {e}. Setting default values."
|
||||
)
|
||||
default_template = "{base_model}/{first_tag}"
|
||||
templates = {
|
||||
"lora": default_template,
|
||||
"checkpoint": default_template,
|
||||
"embedding": default_template,
|
||||
}
|
||||
templates = dict(DEFAULT_DOWNLOAD_PATH_TEMPLATES)
|
||||
self.settings["download_path_templates"] = templates
|
||||
self._save_settings()
|
||||
|
||||
# Ensure templates is a dictionary
|
||||
if not isinstance(templates, dict):
|
||||
default_template = "{base_model}/{first_tag}"
|
||||
templates = {
|
||||
"lora": default_template,
|
||||
"checkpoint": default_template,
|
||||
"embedding": default_template,
|
||||
}
|
||||
templates = dict(DEFAULT_DOWNLOAD_PATH_TEMPLATES)
|
||||
self.settings["download_path_templates"] = templates
|
||||
self._save_settings()
|
||||
|
||||
return templates.get(model_type, "{base_model}/{first_tag}")
|
||||
return templates.get(
|
||||
model_type, DEFAULT_DOWNLOAD_PATH_TEMPLATES.get(model_type, "")
|
||||
)
|
||||
|
||||
def get_download_filename_template(self, model_type: str) -> str:
|
||||
"""Get the download filename template for a specific model type.
|
||||
|
||||
Args:
|
||||
model_type: The type of model ('lora', 'checkpoint', 'embedding',
|
||||
'other')
|
||||
|
||||
Returns:
|
||||
Template string for the model type. Empty string (the default for
|
||||
every model type) means downloaded files keep their original
|
||||
filename.
|
||||
"""
|
||||
templates = self.settings.get("download_filename_templates", {})
|
||||
|
||||
# Handle edge case where templates might be stored as JSON string
|
||||
if isinstance(templates, str):
|
||||
try:
|
||||
parsed_templates = json.loads(templates)
|
||||
if isinstance(parsed_templates, dict):
|
||||
self.settings["download_filename_templates"] = parsed_templates
|
||||
self._save_settings()
|
||||
templates = parsed_templates
|
||||
logger.info(
|
||||
"Successfully parsed download_filename_templates from JSON string"
|
||||
)
|
||||
else:
|
||||
raise ValueError("Parsed JSON is not a dictionary")
|
||||
except (json.JSONDecodeError, ValueError) as e:
|
||||
logger.warning(
|
||||
f"Failed to parse download_filename_templates JSON string: {e}. Resetting to empty templates."
|
||||
)
|
||||
templates = {}
|
||||
self.settings["download_filename_templates"] = templates
|
||||
self._save_settings()
|
||||
|
||||
if not isinstance(templates, dict):
|
||||
templates = {}
|
||||
self.settings["download_filename_templates"] = templates
|
||||
self._save_settings()
|
||||
|
||||
template = templates.get(model_type, "")
|
||||
return template if isinstance(template, str) else ""
|
||||
|
||||
|
||||
_SETTINGS_MANAGER: Optional["SettingsManager"] = None
|
||||
|
||||
@@ -20,6 +20,7 @@ import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
from ..utils.cache_db import connect_cache_db
|
||||
from ..utils.cache_paths import CacheType, resolve_cache_path_with_migration
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -677,16 +678,13 @@ class TagFTSIndex:
|
||||
|
||||
def _connect(self, readonly: bool = False) -> sqlite3.Connection:
|
||||
"""Create a database connection."""
|
||||
uri = False
|
||||
path = self._db_path
|
||||
if readonly:
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(path)
|
||||
path = f"file:{path}?mode=ro"
|
||||
uri = True
|
||||
conn = sqlite3.connect(path, check_same_thread=False, uri=uri)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
if readonly and not os.path.exists(self._db_path):
|
||||
raise FileNotFoundError(self._db_path)
|
||||
return connect_cache_db(
|
||||
self._db_path,
|
||||
readonly=readonly,
|
||||
row_factory=sqlite3.Row,
|
||||
)
|
||||
|
||||
def _build_fts_query(self, query: str) -> str:
|
||||
"""Build an FTS5 query string with prefix matching.
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user