feat(backend): CivitAI download support for other model types with subtype routing

This commit is contained in:
Will Miao
2026-09-12 15:56:47 +08:00
parent 57729375b6
commit f2a7297cb9
18 changed files with 1492 additions and 16 deletions
+88 -5
View File
@@ -17,12 +17,18 @@ from dataclasses import dataclass, field
import uuid
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, cast
from urllib.parse import urlparse
from ..utils.models import LoraMetadata, CheckpointMetadata, EmbeddingMetadata
from ..utils.models import (
LoraMetadata,
CheckpointMetadata,
EmbeddingMetadata,
OtherModelMetadata,
)
from ..utils.constants import (
CARD_PREVIEW_WIDTH,
MODEL_WEIGHT_FILE_TYPES,
SUPPORTED_DOWNLOAD_SKIP_BASE_MODELS,
VALID_LORA_TYPES,
VALID_OTHER_CIVITAI_TYPES,
)
from ..utils.civitai_utils import normalize_civitai_download_url, rewrite_preview_url
from ..utils.file_utils import calculate_sha256, calculate_autov3
@@ -31,7 +37,7 @@ from ..utils.utils import sanitize_folder_name
from ..utils.exif_utils import ExifUtils
from ..utils.metadata_manager import MetadataManager
from .service_registry import ServiceRegistry
from .download_routing import is_diffusion_model_download
from .download_routing import is_diffusion_model_download, resolve_other_download_sub_type
from .settings_manager import get_settings_manager
from .metadata_service import get_default_metadata_provider, get_metadata_provider
from .downloader import get_downloader, DownloadProgress, DownloadStreamControl
@@ -228,12 +234,21 @@ class DownloadManager:
return False
async def _get_scanner_for_model_type(self, model_type: str):
"""Return the scanner responsible for the given model type."""
"""Return the scanner responsible for the given model type.
Every supported type resolves explicitly — an unknown type must never
fall through to the lora scanner (an "other" download would silently
dedupe against the lora library).
"""
if model_type == "checkpoint":
return await self._get_checkpoint_scanner()
if model_type == "embedding":
return await ServiceRegistry.get_embedding_scanner()
return await self._get_lora_scanner()
if model_type == "other":
return await ServiceRegistry.get_other_scanner()
if model_type == "lora":
return await self._get_lora_scanner()
raise ValueError(f'Unknown model type "{model_type}"')
@staticmethod
def _resolve_target_file(
@@ -978,6 +993,8 @@ class DownloadManager:
return CheckpointMetadata.from_civitai_info(version_info, file_info, save_path)
if model_type == "embedding":
return EmbeddingMetadata.from_civitai_info(version_info, file_info, save_path)
if model_type == "other":
return OtherModelMetadata.from_civitai_info(version_info, file_info, save_path)
return LoraMetadata.from_civitai_info(version_info, file_info, save_path)
def _resolve_save_path_from_persisted_record(self, record: Dict[str, Any]) -> Optional[str]:
@@ -1438,6 +1455,7 @@ class DownloadManager:
lora_scanner = await self._get_lora_scanner()
checkpoint_scanner = await self._get_checkpoint_scanner()
embedding_scanner = await ServiceRegistry.get_embedding_scanner()
other_scanner = await ServiceRegistry.get_other_scanner()
# Check lora scanner first
if await lora_scanner.check_model_version_exists(model_version_id):
@@ -1462,6 +1480,13 @@ class DownloadManager:
"error": "Model version already exists in embedding library",
}
# Check other scanner
if await other_scanner.check_model_version_exists(model_version_id):
return {
"success": False,
"error": "Model version already exists in other library",
}
# Use CivArchive provider directly when source is 'civarchive'
# This prioritizes CivArchive metadata (with mirror availability info) over Civitai
if source == "civarchive":
@@ -1500,6 +1525,8 @@ class DownloadManager:
model_type = "lora"
elif model_type_from_info == "textualinversion":
model_type = "embedding"
elif model_type_from_info in VALID_OTHER_CIVITAI_TYPES:
model_type = "other"
else:
return {
"success": False,
@@ -1686,6 +1713,13 @@ class DownloadManager:
"success": False,
"error": "Model version already exists in embedding library",
}
elif model_type == "other":
other_scanner = await ServiceRegistry.get_other_scanner()
if await other_scanner.check_model_version_exists(version_id):
return {
"success": False,
"error": "Model version already exists in other library",
}
# Handle use_default_paths
if use_default_paths:
@@ -1725,6 +1759,45 @@ class DownloadManager:
"error": "Default embedding root path not set in settings",
}
save_dir = default_path
elif model_type == "other":
other_sub_type = resolve_other_download_sub_type(
model_type_from_info,
file_types=(
f.get("type", "")
for f in version_info.get("files", [])
if isinstance(f, dict)
),
selected_file_type=(
target_file.get("type") if explicit_file else None
),
)
default_other_roots = (
settings_manager.get("default_other_roots") or {}
)
default_path = (
default_other_roots.get(other_sub_type)
if other_sub_type
else None
)
if not isinstance(default_path, str) or not default_path:
if other_sub_type:
detail = (
f"No default root configured for other-model "
f"sub-type '{other_sub_type}'"
)
else:
detail = (
"Could not determine the other-model sub-type "
"from the model metadata"
)
return {
"success": False,
"error": (
f"{detail}. Please pick a destination folder "
f"explicitly instead of using default paths."
),
}
save_dir = default_path
# Calculate relative path using template
relative_path = self._calculate_relative_path(version_info, model_type)
@@ -1921,6 +1994,11 @@ class DownloadManager:
version_info, file_info, save_path
)
logger.info(f"Creating EmbeddingMetadata for {file_name}")
elif model_type == "other":
metadata = OtherModelMetadata.from_civitai_info(
version_info, file_info, save_path
)
logger.info(f"Creating OtherModelMetadata for {file_name}")
else:
return {
"success": False,
@@ -2133,6 +2211,8 @@ class DownloadManager:
scanner = await self._get_checkpoint_scanner()
elif model_type == "embedding":
scanner = await ServiceRegistry.get_embedding_scanner()
elif model_type == "other":
scanner = await ServiceRegistry.get_other_scanner()
except Exception as exc:
logger.debug("Failed to acquire scanner for %s models: %s", model_type, exc)
@@ -2629,6 +2709,9 @@ class DownloadManager:
elif model_type == "embedding":
scanner = await ServiceRegistry.get_embedding_scanner()
logger.info(f"Updating embedding cache for {actual_file_paths[0]}")
elif model_type == "other":
scanner = await ServiceRegistry.get_other_scanner()
logger.info(f"Updating other-model cache for {actual_file_paths[0]}")
adjust_cached_entry = (
getattr(scanner, "adjust_cached_entry", None)
@@ -2718,7 +2801,7 @@ class DownloadManager:
return {"success": False, "error": str(e)}
def _get_supported_extensions_for_type(self, model_type: str) -> Set[str]:
if model_type == "checkpoint":
if model_type in ("checkpoint", "other"):
return {
".ckpt",
".pt",
+52 -2
View File
@@ -10,9 +10,13 @@ two can never disagree.
from __future__ import annotations
import logging
from typing import Iterable
from typing import Iterable, Optional
from ..utils.constants import DIFFUSION_MODEL_BASE_MODELS
from ..utils.constants import (
CIVITAI_FILE_TYPE_TO_OTHER_SUB_TYPE,
CIVITAI_TYPE_TO_OTHER_SUB_TYPE,
DIFFUSION_MODEL_BASE_MODELS,
)
logger = logging.getLogger(__name__)
@@ -51,3 +55,49 @@ def is_diffusion_model_download(
return True
return False
def resolve_other_download_sub_type(
civitai_model_type: str,
file_types: Iterable[str] = (),
selected_file_type: Optional[str] = None,
) -> Optional[str]:
"""Resolve the "other"-page sub_type for a download.
Fixed priority (locked design, docs/plans/other-models-page.md §9.2):
1. Explicit user file pick when the picked file's type maps, it wins
even when model.type maps to something else.
2. model.type via CIVITAI_TYPE_TO_OTHER_SUB_TYPE.
3. file.type fallback only when model.type maps to nothing. Must NOT
override a mapped model.type: checkpoint models routinely bundle
VAE/Text Encoder component files.
4. Still undecidable -> None (caller must ask the user for a folder).
"""
if selected_file_type:
mapped = CIVITAI_FILE_TYPE_TO_OTHER_SUB_TYPE.get(selected_file_type)
if mapped:
logger.info(
"Explicit file pick type '%s' routes other download to '%s'",
selected_file_type,
mapped,
)
return mapped
normalized_model_type = (civitai_model_type or "").strip().lower()
mapped = CIVITAI_TYPE_TO_OTHER_SUB_TYPE.get(normalized_model_type)
if mapped:
return mapped
for file_type in file_types:
mapped = CIVITAI_FILE_TYPE_TO_OTHER_SUB_TYPE.get(file_type)
if mapped:
logger.info(
"model.type '%s' unmapped; file type '%s' routes other download to '%s'",
civitai_model_type,
file_type,
mapped,
)
return mapped
return None
+114
View File
@@ -27,7 +27,9 @@ from platformdirs import user_config_dir
from ..utils.constants import (
DEFAULT_HASH_CHUNK_SIZE_MB,
DEFAULT_PRIORITY_TAG_CONFIG,
OTHER_MODEL_FOLDER_SUBTYPES,
SUPPORTED_DOWNLOAD_SKIP_BASE_MODELS,
VALID_OTHER_SUB_TYPES,
)
from ..utils.preview_selection import VALID_MATURE_BLUR_LEVELS
from ..utils.settings_paths import (
@@ -83,6 +85,7 @@ DEFAULT_SETTINGS: Dict[str, Any] = {
"default_checkpoint_root": "",
"default_unet_root": "",
"default_embedding_root": "",
"default_other_roots": {},
"recipes_path": "",
"base_model_path_mappings": {},
"download_path_templates": {},
@@ -309,6 +312,7 @@ class SettingsManager:
default_checkpoint_root=merged.get("default_checkpoint_root"),
default_unet_root=merged.get("default_unet_root"),
default_embedding_root=merged.get("default_embedding_root"),
default_other_roots=merged.get("default_other_roots"),
recipes_path=merged.get("recipes_path"),
)
}
@@ -443,6 +447,7 @@ class SettingsManager:
),
default_unet_root=self.settings.get("default_unet_root", ""),
default_embedding_root=self.settings.get("default_embedding_root", ""),
default_other_roots=self.settings.get("default_other_roots"),
recipes_path=self.settings.get("recipes_path", ""),
)
libraries = {library_name: library_payload}
@@ -494,6 +499,7 @@ class SettingsManager:
default_checkpoint_root=data.get("default_checkpoint_root"),
default_unet_root=data.get("default_unet_root"),
default_embedding_root=data.get("default_embedding_root"),
default_other_roots=data.get("default_other_roots"),
recipes_path=data.get("recipes_path"),
metadata=data.get("metadata"),
base=data,
@@ -541,6 +547,9 @@ class SettingsManager:
self.settings["default_embedding_root"] = active_library.get(
"default_embedding_root", ""
)
self.settings["default_other_roots"] = self._normalize_default_other_roots(
active_library.get("default_other_roots", {})
)
self.settings["recipes_path"] = active_library.get("recipes_path", "")
if save:
@@ -558,6 +567,7 @@ class SettingsManager:
default_checkpoint_root: Optional[str] = None,
default_unet_root: Optional[str] = None,
default_embedding_root: Optional[str] = None,
default_other_roots: Optional[Mapping[str, str]] = None,
recipes_path: Optional[str] = None,
metadata: Optional[Mapping[str, Any]] = None,
base: Optional[Mapping[str, Any]] = None,
@@ -597,6 +607,15 @@ class SettingsManager:
else:
payload.setdefault("default_embedding_root", "")
if default_other_roots is not None:
payload["default_other_roots"] = self._normalize_default_other_roots(
default_other_roots
)
else:
payload["default_other_roots"] = self._normalize_default_other_roots(
payload.get("default_other_roots", {})
)
if recipes_path is not None:
payload["recipes_path"] = recipes_path
else:
@@ -632,6 +651,35 @@ class SettingsManager:
normalized[key] = cleaned
return normalized
def _normalize_default_other_roots(
self, value: Any, *, strict: bool = False
) -> Dict[str, str]:
"""Normalize a ``default_other_roots`` mapping ({sub_type: root path}).
Unknown sub_type keys and non-string/empty paths are dropped; with
``strict=True`` unknown sub_type keys raise instead (used by ``set()``
so typos in API payloads surface as errors).
"""
if not isinstance(value, Mapping):
if strict and value is not None:
raise ValueError("default_other_roots must be a mapping")
return {}
normalized: Dict[str, str] = {}
for sub_type, path in value.items():
if sub_type not in VALID_OTHER_SUB_TYPES:
if strict:
raise ValueError(
f"Unknown other-model sub-type '{sub_type}'; "
f"expected one of {sorted(VALID_OTHER_SUB_TYPES)}"
)
continue
if not isinstance(path, str):
continue
stripped = path.strip()
if stripped:
normalized[sub_type] = stripped
return normalized
def _has_configured_paths(self, folder_paths: Any) -> bool:
if not isinstance(folder_paths, Mapping):
return False
@@ -744,6 +792,7 @@ class SettingsManager:
default_checkpoint_root: Optional[str] = None,
default_unet_root: Optional[str] = None,
default_embedding_root: Optional[str] = None,
default_other_roots: Optional[Mapping[str, str]] = None,
recipes_path: Optional[str] = None,
) -> bool:
libraries = self.settings.get("libraries", {})
@@ -794,6 +843,14 @@ class SettingsManager:
library["default_embedding_root"] = default_embedding_root
changed = True
if default_other_roots is not None:
normalized_other_roots = self._normalize_default_other_roots(
default_other_roots
)
if library.get("default_other_roots") != normalized_other_roots:
library["default_other_roots"] = normalized_other_roots
changed = True
if recipes_path is not None and library.get("recipes_path") != recipes_path:
library["recipes_path"] = recipes_path
changed = True
@@ -894,12 +951,53 @@ class SettingsManager:
updated = _check_and_auto_set("unet", "default_unet_root") or updated
updated = _check_and_auto_set("embeddings", "default_embedding_root") or updated
# Other-model default roots: one entry per sub_type; candidates are the
# union of that sub_type's folder_paths keys (text_encoder merges the
# legacy 'clip' key with 'text_encoders').
sub_type_folder_keys: Dict[str, List[str]] = {}
for folder_key, sub_type in OTHER_MODEL_FOLDER_SUBTYPES.items():
sub_type_folder_keys.setdefault(sub_type, []).append(folder_key)
other_roots = self._normalize_default_other_roots(
self.settings.get("default_other_roots")
)
for sub_type in VALID_OTHER_SUB_TYPES:
candidates: List[str] = []
candidate_identities: set[str] = set()
for folder_key in sub_type_folder_keys.get(sub_type, []):
for candidate in self._get_valid_root_candidates(folder_key):
identity = _normalize_root_identity(candidate)
if identity in candidate_identities:
continue
candidate_identities.add(identity)
candidates.append(candidate)
if not candidates:
continue
current = other_roots.get(sub_type, "")
if current and _normalize_root_identity(current) in candidate_identities:
continue
other_roots[sub_type] = candidates[0]
if current:
logger.info(
"Repaired stale default_other_roots[%s] from '%s' to '%s' because it is not present in primary or extra roots",
sub_type,
current,
candidates[0],
)
else:
logger.info(
"Auto-set default_other_roots[%s] to '%s'", sub_type, candidates[0]
)
updated = True
if updated:
self.settings["default_other_roots"] = other_roots
self._update_active_library_entry(
default_lora_root=self.settings.get("default_lora_root"),
default_checkpoint_root=self.settings.get("default_checkpoint_root"),
default_unet_root=self.settings.get("default_unet_root"),
default_embedding_root=self.settings.get("default_embedding_root"),
default_other_roots=other_roots,
)
if self._bootstrap_reason == "missing":
self._needs_initial_save = True
@@ -1599,6 +1697,8 @@ class SettingsManager:
value = self.normalize_download_skip_base_models(value)
elif key == "mature_blur_level":
value = self.normalize_mature_blur_level(value)
elif key == "default_other_roots":
value = self._normalize_default_other_roots(value, strict=True)
elif key == "recipes_path":
current_recipes_dir = self._get_effective_recipes_dir()
value = self._normalize_recipes_path_value(value)
@@ -1626,6 +1726,8 @@ class SettingsManager:
self._update_active_library_entry(default_unet_root=str(value))
elif key == "default_embedding_root":
self._update_active_library_entry(default_embedding_root=str(value))
elif key == "default_other_roots":
self._update_active_library_entry(default_other_roots=value)
elif key == "recipes_path":
self._update_active_library_entry(recipes_path=str(value))
elif key == "model_name_display":
@@ -1796,6 +1898,7 @@ class SettingsManager:
"lora_scanner",
"checkpoint_scanner",
"embedding_scanner",
"other_scanner",
"recipe_scanner",
):
service = ServiceRegistry.get_service_sync(service_name)
@@ -1960,6 +2063,7 @@ class SettingsManager:
default_checkpoint_root: Optional[str] = None,
default_unet_root: Optional[str] = None,
default_embedding_root: Optional[str] = None,
default_other_roots: Optional[Mapping[str, str]] = None,
recipes_path: Optional[str] = None,
metadata: Optional[Mapping[str, Any]] = None,
activate: bool = False,
@@ -2004,6 +2108,11 @@ class SettingsManager:
if default_embedding_root is not None
else existing.get("default_embedding_root")
),
default_other_roots=(
default_other_roots
if default_other_roots is not None
else existing.get("default_other_roots")
),
recipes_path=(
recipes_path
if recipes_path is not None
@@ -2036,6 +2145,7 @@ class SettingsManager:
default_checkpoint_root: str = "",
default_unet_root: str = "",
default_embedding_root: str = "",
default_other_roots: Optional[Mapping[str, str]] = None,
recipes_path: str = "",
metadata: Optional[Mapping[str, Any]] = None,
activate: bool = False,
@@ -2054,6 +2164,7 @@ class SettingsManager:
default_checkpoint_root=default_checkpoint_root,
default_unet_root=default_unet_root,
default_embedding_root=default_embedding_root,
default_other_roots=default_other_roots,
recipes_path=recipes_path,
metadata=metadata,
activate=activate,
@@ -2114,6 +2225,7 @@ class SettingsManager:
default_checkpoint_root: Optional[str] = None,
default_unet_root: Optional[str] = None,
default_embedding_root: Optional[str] = None,
default_other_roots: Optional[Mapping[str, str]] = None,
recipes_path: Optional[str] = None,
) -> None:
"""Update folder paths for the active library."""
@@ -2127,6 +2239,7 @@ class SettingsManager:
default_checkpoint_root=default_checkpoint_root,
default_unet_root=default_unet_root,
default_embedding_root=default_embedding_root,
default_other_roots=default_other_roots,
recipes_path=recipes_path,
activate=True,
)
@@ -2151,6 +2264,7 @@ class SettingsManager:
"lora_scanner",
"checkpoint_scanner",
"embedding_scanner",
"other_scanner",
"recipe_scanner",
"model_update_service",
):