Compare commits

...

3 Commits

34 changed files with 1260 additions and 98 deletions

View File

@@ -714,7 +714,9 @@
"versionsCount": "Lokale Versionen",
"versionsCountDesc": "Meiste Versionen zuerst",
"versionsCountAsc": "Wenigste Versionen zuerst",
"versionIdDesc": "Neueste Version zuerst"
"versionIdDesc": "Neueste Version zuerst",
"random": "Zufällig",
"randomAction": "Zufällig mischen"
},
"refresh": {
"title": "Modelliste aktualisieren",

View File

@@ -714,7 +714,9 @@
"versionsCount": "Local Versions",
"versionsCountDesc": "Most versions first",
"versionsCountAsc": "Fewest versions first",
"versionIdDesc": "Newest version first"
"versionIdDesc": "Newest version first",
"random": "Random",
"randomAction": "Randomize (shuffle)"
},
"refresh": {
"title": "Refresh model list",

View File

@@ -714,7 +714,9 @@
"versionsCount": "Versiones locales",
"versionsCountDesc": "Más versiones primero",
"versionsCountAsc": "Menos versiones primero",
"versionIdDesc": "Versión más nueva primero"
"versionIdDesc": "Versión más nueva primero",
"random": "Aleatorio",
"randomAction": "Aleatorizar (barajar)"
},
"refresh": {
"title": "Actualizar lista de modelos",

View File

@@ -714,7 +714,9 @@
"versionsCount": "Versions locales",
"versionsCountDesc": "Plus de versions d'abord",
"versionsCountAsc": "Moins de versions d'abord",
"versionIdDesc": "Version la plus récente d'abord"
"versionIdDesc": "Version la plus récente d'abord",
"random": "Aléatoire",
"randomAction": "Aléatoire (mélanger)"
},
"refresh": {
"title": "Actualiser la liste des modèles",

View File

@@ -714,7 +714,9 @@
"versionsCount": "גרסאות מקומיות",
"versionsCountDesc": "הכי הרבה גרסאות ראשונות",
"versionsCountAsc": "הכי מעט גרסאות ראשונות",
"versionIdDesc": "גרסה חדשה ביותר ראשונה"
"versionIdDesc": "גרסה חדשה ביותר ראשונה",
"random": "אקראי",
"randomAction": "ערבוב אקראי"
},
"refresh": {
"title": "רענן רשימת מודלים",

View File

@@ -714,7 +714,9 @@
"versionsCount": "ローカルバージョン数",
"versionsCountDesc": "バージョン数の多い順",
"versionsCountAsc": "バージョン数の少ない順",
"versionIdDesc": "最新バージョン順"
"versionIdDesc": "最新バージョン順",
"random": "ランダム",
"randomAction": "シャッフル(ランダム)"
},
"refresh": {
"title": "モデルリストを更新",

View File

@@ -714,7 +714,9 @@
"versionsCount": "로컬 버전 수",
"versionsCountDesc": "버전 수 많은 순",
"versionsCountAsc": "버전 수 적은 순",
"versionIdDesc": "최신 버전순"
"versionIdDesc": "최신 버전순",
"random": "랜덤",
"randomAction": "셔플 (무작위)"
},
"refresh": {
"title": "모델 목록 새로고침",

View File

@@ -714,7 +714,9 @@
"versionsCount": "Локальные версии",
"versionsCountDesc": "Сначала больше версий",
"versionsCountAsc": "Сначала меньше версий",
"versionIdDesc": "Сначала новые версии"
"versionIdDesc": "Сначала новые версии",
"random": "Случайно",
"randomAction": "Перемешать"
},
"refresh": {
"title": "Обновить список моделей",

View File

@@ -714,7 +714,9 @@
"versionsCount": "本地版本数",
"versionsCountDesc": "版本数从多到少",
"versionsCountAsc": "版本数从少到多",
"versionIdDesc": "最新版本优先"
"versionIdDesc": "最新版本优先",
"random": "随机",
"randomAction": "随机排序(洗牌)"
},
"refresh": {
"title": "刷新模型列表",

View File

@@ -714,7 +714,9 @@
"versionsCount": "本地版本數",
"versionsCountDesc": "版本數從多到少",
"versionsCountAsc": "版本數從少到多",
"versionIdDesc": "最新版本優先"
"versionIdDesc": "最新版本優先",
"random": "隨機",
"randomAction": "隨機排序(洗牌)"
},
"refresh": {
"title": "重新整理模型列表",

View File

@@ -2590,6 +2590,8 @@ class ModelLibraryHandler:
status=400,
)
cursor = request.query.get("cursor")
metadata_provider = await self._metadata_provider_factory()
if not metadata_provider:
return web.json_response(
@@ -2598,7 +2600,7 @@ class ModelLibraryHandler:
)
try:
models = await metadata_provider.get_user_models(username)
result = await metadata_provider.get_user_models(username, cursor)
except NotImplementedError:
return web.json_response(
{
@@ -2608,14 +2610,35 @@ class ModelLibraryHandler:
status=501,
)
if models is None:
if result is None:
return web.json_response(
{"success": False, "error": "Failed to fetch user models"},
status=502,
)
if isinstance(result, dict):
models = result.get("items")
next_cursor = result.get("nextCursor")
else:
# Defensive: tolerate providers that still return a raw list
models = result
next_cursor = None
if not isinstance(models, list):
models = []
if next_cursor is not None and not isinstance(next_cursor, str):
next_cursor = str(next_cursor)
estimated_total = None
if cursor is None:
get_count = getattr(metadata_provider, "get_creator_model_count", None)
if get_count is not None:
try:
estimated_total = await get_count(username)
except Exception: # best-effort only
estimated_total = None
if not isinstance(estimated_total, int):
estimated_total = None
lora_scanner = await self._service_registry.get_lora_scanner()
checkpoint_scanner = await self._service_registry.get_checkpoint_scanner()
@@ -2635,6 +2658,7 @@ class ModelLibraryHandler:
versions: list[dict] = []
history_service = await self._get_download_history_service()
model_ids: list[int] = []
model_count = 0
for model in models:
try:
model_ids.append(int(model.get("id")))
@@ -2668,6 +2692,8 @@ class ModelLibraryHandler:
if model_type not in normalized_allowed_types:
continue
model_count += 1
scanner = type_scanner_map.get(model_type)
if scanner is None:
return web.json_response(
@@ -2733,7 +2759,15 @@ class ModelLibraryHandler:
)
return web.json_response(
{"success": True, "username": username, "versions": versions}
{
"success": True,
"username": username,
"versions": versions,
"modelCount": model_count,
"nextCursor": next_cursor,
"hasMore": next_cursor is not None,
"estimatedTotal": estimated_total,
}
)
except Exception as exc: # pragma: no cover - defensive logging
logger.error("Failed to get Civitai user models: %s", exc, exc_info=True)

View File

@@ -1,6 +1,7 @@
from abc import ABC, abstractmethod
import asyncio
import re
import random
from typing import Any, Dict, List, Optional, Type, Union, TYPE_CHECKING
import logging
import os
@@ -390,6 +391,12 @@ class BaseModelService(ABC):
(item.get("model_name") or item.get("file_name") or "").lower(),
item.get("file_path", "").lower(),
)
elif key_name == "random":
# Seeded random shuffle: same seed -> same order (stable pagination)
rng = random.Random(sort_params.seed or "random")
result = list(data)
rng.shuffle(result)
return result
elif key_name == "size":
key_fn = lambda item: (
int(item.get("size", 0) or 0),

View File

@@ -2,6 +2,7 @@ import asyncio
import copy
import logging
import os
import time
from collections import OrderedDict
from typing import Any, Optional, Dict, Tuple, List, Sequence
from .connectivity_guard import (
@@ -19,6 +20,12 @@ from ..utils.civitai_utils import resolve_license_payload
logger = logging.getLogger(__name__)
# Best-effort cache for creator model counts, keyed by lowercase username.
# Values are (monotonic timestamp, count or None); None results are cached
# too so repeated failures don't hammer the API.
_CREATOR_COUNT_CACHE_TTL_SECONDS = 600
_creator_model_count_cache: Dict[str, Tuple[float, Optional[int]]] = {}
class CivitaiClient:
_instance = None
@@ -743,17 +750,34 @@ class CivitaiClient:
return all_versions if all_versions else None
async def get_user_models(self, username: str) -> Optional[List[Dict]]:
"""Fetch all models for a specific Civitai user."""
async def get_user_models(
self, username: str, cursor: Optional[str] = None
) -> Optional[Dict[str, Any]]:
"""Fetch one page (up to 100 models) for a specific Civitai user.
Returns ``{"items": [...], "nextCursor": <str|None>}`` on success,
or None on failure. Pass ``cursor`` (from a previous response's
``nextCursor``) to fetch subsequent pages.
"""
if not username:
return None
params: Dict[str, Any] = {
"username": username,
"nsfw": "true",
"limit": 100,
"sort": "Newest",
"period": "AllTime",
}
if cursor:
params["cursor"] = cursor
try:
success, result = await self._make_request(
"GET",
f"{self.base_url}/models",
use_auth=True,
params={"username": username, "nsfw": "true"},
params=params,
)
if not success:
@@ -765,7 +789,7 @@ class CivitaiClient:
items = result.get("items") if isinstance(result, dict) else None
if not isinstance(items, list):
return []
items = []
for model in items:
versions = model.get("modelVersions")
@@ -774,9 +798,68 @@ class CivitaiClient:
for version in versions:
self._remove_comfy_metadata(version)
return items
next_cursor: Optional[str] = None
metadata = result.get("metadata") if isinstance(result, dict) else None
if isinstance(metadata, dict):
raw_cursor = metadata.get("nextCursor")
if raw_cursor is not None:
next_cursor = str(raw_cursor)
return {"items": items, "nextCursor": next_cursor}
except RateLimitError:
raise
except Exception as exc: # pragma: no cover - defensive logging
logger.error("Error fetching models for %s: %s", username, exc)
return None
async def get_creator_model_count(self, username: str) -> Optional[int]:
"""Best-effort lookup of a creator's published model count.
Uses the ``/creators`` endpoint (a contains-match query), picking the
entry whose username matches exactly (case-insensitive). Returns None
on any failure; never raises. Results (including None) are cached
for ``_CREATOR_COUNT_CACHE_TTL_SECONDS``.
"""
if not username:
return None
cache_key = username.lower()
cached = _creator_model_count_cache.get(cache_key)
if cached is not None:
cached_at, cached_count = cached
if time.monotonic() - cached_at < _CREATOR_COUNT_CACHE_TTL_SECONDS:
return cached_count
count: Optional[int] = None
try:
success, result = await self._make_request(
"GET",
f"{self.base_url}/creators",
use_auth=True,
params={"query": username, "limit": 10},
)
if success and isinstance(result, dict):
creators = result.get("items")
if isinstance(creators, list):
for creator in creators:
if not isinstance(creator, dict):
continue
creator_name = creator.get("username")
if not isinstance(creator_name, str):
continue
if creator_name.lower() != cache_key:
continue
model_count = creator.get("modelCount")
if isinstance(model_count, (int, float)) and not isinstance(
model_count, bool
):
count = int(model_count)
break
except Exception as exc: # best-effort only, never propagate
logger.debug(
"Failed to fetch creator model count for %s: %s", username, exc
)
_creator_model_count_cache[cache_key] = (time.monotonic(), count)
return count

View File

@@ -1,6 +1,7 @@
import asyncio
import time
import logging
import random
logger = logging.getLogger(__name__)
from typing import Any, Dict, List, Optional, Tuple
@@ -38,8 +39,8 @@ class ModelCache:
def __post_init__(self):
self._lock = asyncio.Lock()
# Cache for last sort: (sort_key, order) -> sorted list
self._last_sort: Tuple[str, str] = (None, None)
# Cache for last sort: (sort_key, order, seed) -> sorted list
self._last_sort: Tuple[Optional[str], str, Optional[str]] = (None, "asc", None)
self._last_sorted_data: List[Dict] = []
self._normalize_raw_data()
self.name_display_mode = self._normalize_display_mode(self.name_display_mode)
@@ -203,9 +204,9 @@ class ModelCache:
async def resort(self):
"""Resort cached data according to last sort mode if set"""
async with self._lock:
if self._last_sort != (None, None):
sort_key, order = self._last_sort
sorted_data = self._sort_data(self.raw_data, sort_key, order)
if self._last_sort[0] is not None:
sort_key, order, seed = self._last_sort
sorted_data = self._sort_data(self.raw_data, sort_key, order, seed)
self._last_sorted_data = sorted_data
# Update folder list
# else: do nothing
@@ -218,7 +219,7 @@ class ModelCache:
self.folders = sorted(list(all_folders), key=lambda x: x.lower())
self.rebuild_version_index()
def _sort_data(self, data: List[Dict], sort_key: str, order: str) -> List[Dict]:
def _sort_data(self, data: List[Dict], sort_key: str, order: str, seed: Optional[str] = None) -> List[Dict]:
"""Sort data by sort_key and order"""
start_time = time.perf_counter()
reverse = (order == 'desc')
@@ -265,6 +266,13 @@ class ModelCache:
),
reverse=reverse
)
elif sort_key == 'random':
# Random shuffle seeded for stable pagination: the same seed
# always yields the same order, so successive page requests
# stay consistent while browsing.
rng = random.Random(seed or 'random')
result = list(data)
rng.shuffle(result)
elif sort_key == 'versions_count':
# Pre-dedup sort: fall back to name sort.
# Actual re-sort by version_count happens in get_paginated_data after dedup.
@@ -285,15 +293,16 @@ class ModelCache:
logger.debug("ModelCache._sort_data(%s, %s) for %d items took %.3fs", sort_key, order, len(data), duration)
return result
async def get_sorted_data(self, sort_key: str = 'name', order: str = 'asc') -> List[Dict]:
async def get_sorted_data(self, sort_key: str = 'name', order: str = 'asc', seed: Optional[str] = None) -> List[Dict]:
"""Get sorted data by sort_key and order, using cache if possible"""
async with self._lock:
if (sort_key, order) == self._last_sort:
cache_key = (sort_key, order, seed)
if cache_key == self._last_sort:
return self._last_sorted_data
start_time = time.perf_counter()
sorted_data = self._sort_data(self.raw_data, sort_key, order)
self._last_sort = (sort_key, order)
sorted_data = self._sort_data(self.raw_data, sort_key, order, seed)
self._last_sort = cache_key
self._last_sorted_data = sorted_data
duration = time.perf_counter() - start_time
@@ -313,8 +322,8 @@ class ModelCache:
self.name_display_mode = normalized
if self._last_sort[0] == 'name':
sort_key, order = self._last_sort
self._last_sorted_data = self._sort_data(self.raw_data, sort_key, order)
sort_key, order, seed = self._last_sort
self._last_sorted_data = self._sort_data(self.raw_data, sort_key, order, seed)
async def update_preview_url(self, file_path: str, preview_url: str, preview_nsfw_level: int) -> bool:
"""Update preview_url for a specific model in all cached data

View File

@@ -143,10 +143,18 @@ class ModelMetadataProvider(ABC):
pass
@abstractmethod
async def get_user_models(self, username: str) -> Optional[List[Dict]]:
"""Fetch models owned by the specified user"""
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict]:
"""Fetch one page of models owned by the specified user.
Returns ``{"items": [...], "nextCursor": <str|None>}`` on success,
or None when unsupported/failed. ``cursor`` continues a previous page.
"""
pass
async def get_creator_model_count(self, username: str) -> Optional[int]:
"""Published model count for the user; None when unsupported."""
return None
class CivitaiModelMetadataProvider(ModelMetadataProvider):
"""Provider that uses Civitai API for metadata"""
@@ -175,8 +183,11 @@ class CivitaiModelMetadataProvider(ModelMetadataProvider):
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict], Optional[str]]:
return await self.client.get_model_version_info(version_id)
async def get_user_models(self, username: str) -> Optional[List[Dict]]:
return await self.client.get_user_models(username)
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict]:
return await self.client.get_user_models(username, cursor)
async def get_creator_model_count(self, username: str) -> Optional[int]:
return await self.client.get_creator_model_count(username)
class CivArchiveModelMetadataProvider(ModelMetadataProvider):
"""Provider that uses CivArchive API for metadata"""
@@ -196,7 +207,7 @@ class CivArchiveModelMetadataProvider(ModelMetadataProvider):
async def get_model_version_info(self, version_id: str) -> Tuple[Optional[Dict], Optional[str]]:
return await self.client.get_model_version_info(version_id)
async def get_user_models(self, username: str) -> Optional[List[Dict]]:
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict]:
"""Not supported by CivArchive provider"""
return None
@@ -347,7 +358,7 @@ class SQLiteModelMetadataProvider(ModelMetadataProvider):
version_data = await self._get_version_with_model_data(db, model_id, version_id)
return version_data, None
async def get_user_models(self, username: str) -> Optional[List[Dict]]:
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict]:
"""Listing models by username is not supported for archive database"""
return None
@@ -602,13 +613,14 @@ class FallbackMetadataProvider(ModelMetadataProvider):
continue
return None
async def get_user_models(self, username: str) -> Optional[List[Dict]]:
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict]:
for provider, label in self._iter_providers():
try:
result = await self._call_with_rate_limit(
label,
provider.get_user_models,
username,
cursor=cursor,
)
if result is not None:
return result
@@ -624,6 +636,19 @@ class FallbackMetadataProvider(ModelMetadataProvider):
continue
return None
async def get_creator_model_count(self, username: str) -> Optional[int]:
for provider, label in self._iter_providers():
try:
result = await provider.get_creator_model_count(username)
if result is not None:
return result
except Exception as e:
logger.debug(
"Provider %s failed for get_creator_model_count: %s", label, e
)
continue
return None
def _iter_providers(self):
return zip(self.providers, self._provider_labels)
@@ -704,13 +729,17 @@ class RateLimitRetryingProvider(ModelMetadataProvider):
version_id,
)
async def get_user_models(self, username: str) -> Optional[List[Dict]]:
async def get_user_models(self, username: str, cursor: Optional[str] = None) -> Optional[Dict]:
return await self._rate_limit_helper.run(
self._label,
self._provider.get_user_models,
username,
cursor=cursor,
)
async def get_creator_model_count(self, username: str) -> Optional[int]:
return await self._provider.get_creator_model_count(username)
class ModelMetadataProviderManager:
"""Manager for selecting and using model metadata providers"""
@@ -776,10 +805,20 @@ class ModelMetadataProviderManager:
except NotImplementedError:
return None
async def get_user_models(self, username: str, provider_name: str = None) -> Optional[List[Dict]]:
"""Fetch models owned by the specified user"""
async def get_user_models(
self,
username: str,
provider_name: str = None,
cursor: Optional[str] = None,
) -> Optional[Dict]:
"""Fetch one page of models owned by the specified user"""
provider = self._get_provider(provider_name)
return await provider.get_user_models(username)
return await provider.get_user_models(username, cursor)
async def get_creator_model_count(self, username: str, provider_name: str = None) -> Optional[int]:
"""Best-effort published model count for the specified user"""
provider = self._get_provider(provider_name)
return await provider.get_creator_model_count(username)
def _get_provider(self, provider_name: str = None) -> ModelMetadataProvider:
"""Get provider by name or default provider"""

View File

@@ -85,6 +85,7 @@ class SortParams:
key: str
order: str
seed: Optional[str] = None
@dataclass(frozen=True)
@@ -116,7 +117,7 @@ class ModelCacheRepository:
async def fetch_sorted(self, params: SortParams) -> List[Dict[str, Any]]:
"""Fetch cached data pre-sorted according to ``params``."""
cache = await self.get_cache()
return await cache.get_sorted_data(params.key, params.order)
return await cache.get_sorted_data(params.key, params.order, params.seed)
@staticmethod
def parse_sort(sort_by: str) -> SortParams:
@@ -132,10 +133,17 @@ class ModelCacheRepository:
sort_key = sort_by.strip().lower() or "name"
order = "asc"
if order not in ("asc", "desc"):
seed = None
if sort_key == "random":
# Random sort: the portion after ':' is the shuffle seed.
# A stable seed keeps paginated requests consistent; order is
# meaningless for a random shuffle.
seed = order if order and order not in ("asc", "desc") else None
order = "asc"
elif order not in ("asc", "desc"):
order = "asc"
return SortParams(key=sort_key, order=order)
return SortParams(key=sort_key, order=order, seed=seed)
class ModelFilterSet:

View File

@@ -1752,7 +1752,7 @@ class ModelScanner:
# ---- Conditional resort (only when sort-key fields changed) ----
need_resort = False
_last = cache._last_sort
sort_key: Optional[str] = _last[0] if _last != (None, None) else None
sort_key: Optional[str] = _last[0] if _last[0] is not None else None
if sort_key == "name":
if (
old_model_name != desired_entry.get("model_name", "")

View File

@@ -14,11 +14,16 @@ from ..services.service_registry import ServiceRegistry
from ..utils.example_images_paths import (
ExampleImagePathResolver,
ensure_library_root_exists,
get_example_images_root,
is_hash_folder,
uses_library_scoped_folders,
)
from ..utils.metadata_manager import MetadataManager
from .example_images_processor import ExampleImagesProcessor
from .example_images_metadata import MetadataUpdater
from .example_images_metadata import (
MetadataUpdater,
update_cache_from_metadata,
)
from ..services.downloader import get_downloader
from ..services.settings_manager import get_settings_manager
@@ -87,6 +92,13 @@ class _DownloadProgress(dict):
return snapshot
# When fewer candidates than this remain in check_pending_models, probe each
# model folder directly (preserving legacy-folder migration semantics). Above
# it, build a folder index with a single directory scan so libraries with
# 100k+ models do not pay one syscall per candidate.
_BULK_LOOKUP_THRESHOLD = 1000
def _model_directory_has_files(path: str) -> bool:
"""Return True when the provided directory exists and contains entries."""
@@ -103,6 +115,36 @@ def _model_directory_has_files(path: str) -> bool:
return False
def _build_example_folder_index(output_dir: str) -> dict[str, bool]:
"""Build a ``{hash: has_files}`` index for a library's example-image folders.
A single directory scan over the library root replaces ``O(candidates)``
per-folder ``os.scandir`` calls, which is required for libraries with
100k+ models. Each hash folder is classified by whether it contains any
entries, matching the semantics of ``_model_directory_has_files``.
"""
index: dict[str, bool] = {}
if not output_dir or not os.path.isdir(output_dir):
return index
try:
with os.scandir(output_dir) as entries:
for entry in entries:
name = entry.name
if not entry.is_dir() or not is_hash_folder(name):
continue
try:
with os.scandir(entry.path) as subentries:
index[name.lower()] = any(subentries)
except OSError:
index[name.lower()] = False
except OSError:
pass
return index
class DownloadManager:
"""Manages downloading example images for models."""
@@ -410,14 +452,49 @@ class DownloadManager:
# Calculate pending count: check which models actually need processing.
# A model is pending if it has a hash, is not already processed or known-failed,
# and its folder doesn't exist or is empty.
pending_hashes = set()
for model_hash, model_name in all_models_with_hash:
if model_hash not in processed_models and model_hash not in failed_models:
candidate_hashes = [
model_hash
for model_hash, _ in all_models_with_hash
if model_hash not in processed_models
and model_hash not in failed_models
]
pending_hashes: set[str] = set()
# For small candidate counts the existing per-folder check is fine
# and handles legacy folder migration.
# For large libraries, scan the library root once and do set lookups.
if len(candidate_hashes) <= _BULK_LOOKUP_THRESHOLD or not output_dir:
for model_hash in candidate_hashes:
model_dir = ExampleImagePathResolver.get_model_folder(
model_hash, active_library
)
if not _model_directory_has_files(model_dir):
pending_hashes.add(model_hash)
else:
folder_index = await asyncio.get_event_loop().run_in_executor(
None, _build_example_folder_index, output_dir
)
# In multi-library mode, folders that have not been consolidated
# into the library root yet (startup migration skipped, failed
# move, or created at the legacy path afterwards) still live at
# the legacy root/<hash> location. Only scan that root when at
# least one candidate is missing from the library-root index, so
# the fully-consolidated case does not pay an extra directory
# pass on every call.
if uses_library_scoped_folders() and any(
not folder_index.get(model_hash, False)
for model_hash in candidate_hashes
):
legacy_root = get_example_images_root()
if legacy_root and legacy_root != output_dir:
legacy_index = await asyncio.get_event_loop().run_in_executor(
None, _build_example_folder_index, legacy_root
)
for hash_key, has_files in legacy_index.items():
folder_index.setdefault(hash_key, has_files)
for model_hash in candidate_hashes:
if not folder_index.get(model_hash, False):
pending_hashes.add(model_hash)
pending_count = len(pending_hashes)
@@ -1343,8 +1420,8 @@ class DownloadManager:
await MetadataManager.save_metadata(file_path, model_copy)
try:
await scanner.update_single_model_cache(
file_path, file_path, model_data
await update_cache_from_metadata(
scanner, file_path, model_copy
)
except AttributeError:
logger.debug(

View File

@@ -1,3 +1,4 @@
import inspect
import logging
import os
import re
@@ -28,6 +29,31 @@ if TYPE_CHECKING: # pragma: no cover - import for type checkers only
from ..services.settings_manager import SettingsManager
async def update_cache_from_metadata(
scanner: Any, file_path: str, metadata: Dict[str, Any]
) -> bool:
"""Update the scanner cache from a metadata dict using the in-place sync path.
``sync_cache_from_metadata`` patches the existing cache entry incrementally
(tag/hash/version indexes, targeted single-row SQL update) and only resorts
when a sort-key field changed. This avoids the ``O(n)`` full-list resort and
full cache rewrite that ``update_single_model_cache`` performs on every call,
which is critical for libraries with 100k+ models.
Falls back to the legacy full update when the scanner does not expose an
async ``sync_cache_from_metadata`` method.
Returns:
``True`` if the cache entry was updated, ``False`` otherwise.
"""
sync_method = getattr(scanner, "sync_cache_from_metadata", None)
if inspect.iscoroutinefunction(sync_method):
return await sync_method(file_path, metadata)
return await scanner.update_single_model_cache(file_path, file_path, metadata)
def _build_metadata_sync_service(settings_manager: "SettingsManager") -> MetadataSyncService:
"""Construct a metadata sync service bound to the provided settings."""
@@ -103,8 +129,8 @@ class MetadataUpdater:
progress['refreshed_models'].add(model_hash)
async def update_cache_func(old_path, new_path, metadata):
return await scanner.update_single_model_cache(old_path, new_path, metadata)
return await update_cache_from_metadata(scanner, new_path, metadata)
await MetadataManager.hydrate_model_data(model_data)
success, error = await _get_metadata_sync_service().fetch_and_update_model(
sha256=model_hash,
@@ -234,6 +260,7 @@ class MetadataUpdater:
# Save metadata to .metadata.json file
file_path = model.get('file_path')
model_copy: Optional[Dict[str, Any]] = None
try:
model_copy = model.copy()
model_copy.pop('folder', None)
@@ -241,14 +268,18 @@ class MetadataUpdater:
logger.info(f"Saved metadata for {model.get('model_name')}")
except Exception as e:
logger.error(f"Failed to save metadata for {model.get('model_name')}: {str(e)}")
# Save updated metadata to scanner cache
success = await scanner.update_single_model_cache(file_path, file_path, model)
if success:
# Save updated metadata to scanner cache. sync_cache_from_metadata
# returns False both for "already in sync" and for actual failures,
# so the cache sync result is deliberately not treated as an error;
# the return value reflects whether the metadata was persisted.
if file_path and model_copy is not None:
await update_cache_from_metadata(scanner, file_path, model_copy)
logger.info(f"Successfully updated metadata for {model.get('model_name')} with {len(images)} local examples")
return True
else:
logger.warning(f"Failed to update metadata for {model.get('model_name')}")
logger.warning(f"Failed to update metadata for {model.get('model_name')}")
return False
return False
except Exception as e:
@@ -336,6 +367,7 @@ class MetadataUpdater:
# Save metadata to .metadata.json file
file_path = model_data.get('file_path')
model_copy: Optional[Dict[str, Any]] = None
if file_path:
try:
model_copy = model_data.copy()
@@ -344,11 +376,11 @@ class MetadataUpdater:
logger.info(f"Saved metadata for {model_data.get('model_name')}")
except Exception as e:
logger.error(f"Failed to save metadata: {str(e)}")
# Save updated metadata to scanner cache
if file_path:
await scanner.update_single_model_cache(file_path, file_path, model_data)
if file_path and model_copy is not None:
await update_cache_from_metadata(scanner, file_path, model_copy)
# Get regular images array (might be None)
regular_images = civitai_data.get('images', [])

View File

@@ -15,6 +15,7 @@ from ..utils.example_images_paths import (
)
from ..utils.metadata_manager import MetadataManager
from ..utils.example_images_processor import ExampleImagesProcessor
from ..utils.example_images_metadata import update_cache_from_metadata
from ..utils.constants import SUPPORTED_MEDIA_EXTENSIONS
logger = logging.getLogger(__name__)
@@ -421,7 +422,7 @@ class ExampleImagesMigration:
await MetadataManager.save_metadata(file_path, model_copy)
# Update scanner cache
await scanner.update_single_model_cache(file_path, file_path, model_metadata)
await update_cache_from_metadata(scanner, file_path, model_copy)
updated_models += 1
except Exception as e:

View File

@@ -9,7 +9,7 @@ from ..utils.constants import SUPPORTED_MEDIA_EXTENSIONS
from ..services.service_registry import ServiceRegistry
from ..services.settings_manager import get_settings_manager
from ..utils.example_images_paths import get_model_folder, get_model_relative_path
from .example_images_metadata import MetadataUpdater
from .example_images_metadata import MetadataUpdater, update_cache_from_metadata
from ..utils.metadata_manager import MetadataManager
logger = logging.getLogger(__name__)
@@ -644,7 +644,7 @@ class ExampleImagesProcessor:
}, status=500)
# Update cache
await scanner.update_single_model_cache(file_path, file_path, model_data)
await update_cache_from_metadata(scanner, file_path, model_data)
# Get regular images array (might be None)
regular_images = civitai_data.get('images', [])
@@ -759,7 +759,7 @@ class ExampleImagesProcessor:
model_copy = model_data.copy()
model_copy.pop('folder', None)
await MetadataManager.save_metadata(file_path, model_copy)
await scanner.update_single_model_cache(file_path, file_path, model_data)
await update_cache_from_metadata(scanner, file_path, model_copy)
return web.json_response({
'success': True,

View File

@@ -108,10 +108,20 @@ export class PageControls {
const sortSelect = document.getElementById('sortSelect');
if (sortSelect) {
initSortDropdown(sortSelect);
sortSelect.value = this.pageState.sortBy;
this.applySortToSelect(this.pageState.sortBy);
sortSelect.addEventListener('change', async (e) => {
this.pageState.sortBy = e.target.value;
this.saveSortPreference(e.target.value);
let value = e.target.value;
if (value.startsWith('random')) {
// Every pick of Random reshuffles the list: generate a
// fresh seed so the backend keeps a stable order across
// paginated requests.
value = this._randomizeSortValue();
}
this.pageState.sortBy = value;
this.saveSortPreference(value);
// Reset the seeded Random option when switching away from
// Random, or re-apply the fresh seed when picking it again.
this.applySortToSelect(value);
await this.resetAndReload();
});
}
@@ -312,6 +322,44 @@ export class PageControls {
}
}
/**
* Apply a sort value to the native sort <select>, keeping the Random
* option's value in sync when the persisted value carries a seed
* (e.g. "random:abc123"). Must be used instead of assigning
* sortSelect.value directly whenever the value may be a seeded random
* sort, otherwise the native select has no matching option.
* @param {string} sortValue - Sort value like "name:asc" or "random:<seed>"
*/
applySortToSelect(sortValue) {
const sortSelect = document.getElementById('sortSelect');
if (!sortSelect) return;
const randomOpt = sortSelect.querySelector('option[value="random"], option[value^="random:"]');
if (randomOpt) {
randomOpt.value = String(sortValue).startsWith('random') ? sortValue : 'random';
}
sortSelect.value = sortValue;
}
/**
* Generate a fresh seeded random sort value ("random:<seed>") and keep
* the native <select> in sync so its value matches the persisted sort
* string and the dropdown shows the selected label.
* @returns {string} The new sort value, e.g. "random:abc123xyz"
*/
_randomizeSortValue() {
const seed = Math.random().toString(36).slice(2, 12);
const value = `random:${seed}`;
const sortSelect = document.getElementById('sortSelect');
if (sortSelect) {
const randomOpt = sortSelect.querySelector('option[value="random"], option[value^="random:"]');
if (randomOpt) {
randomOpt.value = value;
}
sortSelect.value = value;
}
return value;
}
/**
* Load sort preference from storage
*/
@@ -326,10 +374,7 @@ export class PageControls {
// Handle legacy format conversion
const convertedSort = this.convertLegacySortFormat(savedSort);
this.pageState.sortBy = convertedSort;
const sortSelect = document.getElementById('sortSelect');
if (sortSelect) {
sortSelect.value = convertedSort;
}
this.applySortToSelect(convertedSort);
}
}
@@ -523,9 +568,9 @@ export class PageControls {
this.pageState.sortBy = restoredSort;
this.saveSortPreference(restoredSort);
this._removeVlmSortOption();
this.applySortToSelect(restoredSort);
const sortSelect = document.getElementById('sortSelect');
if (sortSelect) {
sortSelect.value = restoredSort;
sortSelect.disabled = false;
}
}
@@ -575,10 +620,7 @@ export class PageControls {
const savedGroupedSort = getStorageItem(groupedKey);
if (savedGroupedSort) {
this.pageState.sortBy = savedGroupedSort;
const sortSelect = document.getElementById('sortSelect');
if (sortSelect) {
sortSelect.value = savedGroupedSort;
}
this.applySortToSelect(savedGroupedSort);
}
} else {
// Leaving group mode: persist current sort for next time, restore non-group sort
@@ -586,10 +628,7 @@ export class PageControls {
const savedNormalSort = getStorageItem(`${this.pageType}_sort`);
if (savedNormalSort) {
this.pageState.sortBy = savedNormalSort;
const sortSelect = document.getElementById('sortSelect');
if (sortSelect) {
sortSelect.value = savedNormalSort;
}
this.applySortToSelect(savedNormalSort);
}
}
}
@@ -874,7 +913,7 @@ export class PageControls {
}
if (sortSelect) {
sortSelect.value = this.pageState.sortBy;
this.applySortToSelect(this.pageState.sortBy);
}
if (searchInput) {
searchInput.value = this.pageState.filters?.search || '';

View File

@@ -96,7 +96,16 @@ export function initSortDropdown(select) {
};
const choose = (value) => {
if (select.value === value) return;
if (select.value === value) {
// Re-picking the already-selected option is normally a no-op,
// matching native <select> behavior. The seeded Random sort is
// the exception: clicking it again should reshuffle, so let the
// change handler (PageControls) generate a fresh seed.
if (String(value).startsWith('random')) {
select.dispatchEvent(new Event('change', { bubbles: true }));
}
return;
}
select.value = value;
select.dispatchEvent(new Event('change', { bubbles: true }));
};
@@ -277,9 +286,10 @@ export function initSortDropdown(select) {
}
// Rebuild the menu when <option>s change (VLM adds/removes a temporary
// option at runtime).
// option at runtime, and the seeded Random sort option gets a new value
// attribute each time it is picked).
const observer = new MutationObserver(() => buildMenu());
observer.observe(select, { childList: true });
observer.observe(select, { childList: true, subtree: true, attributes: true, attributeFilter: ['value'] });
buildMenu();
group.dataset.sortReady = '1';

View File

@@ -48,6 +48,11 @@
<option value="versions_count:asc">{{ t('loras.controls.sort.versionsCountAsc', default='Fewest versions first') }}</option>
</optgroup>
{% endif %}
{% if page_id != 'recipes' %}
<optgroup label="{{ t('loras.controls.sort.random', default='Random') }}">
<option value="random">{{ t('loras.controls.sort.randomAction', default='Randomize (shuffle)') }}</option>
</optgroup>
{% endif %}
{% if page_id == 'recipes' %}
<optgroup label="{{ t('recipes.controls.sort.lorasCount') }}">
<option value="loras_count:desc">{{ t('recipes.controls.sort.lorasCountDesc') }}</option>

View File

@@ -0,0 +1,221 @@
import { describe, it, beforeEach, afterEach, expect, vi } from 'vitest';
const resetAndReloadMock = vi.fn();
const getModelApiClientMock = vi.fn();
vi.mock('../../../static/js/api/modelApiFactory.js', () => ({
getModelApiClient: getModelApiClientMock,
resetAndReload: resetAndReloadMock,
}));
vi.mock('../../../static/js/utils/uiHelpers.js', () => ({
showToast: vi.fn(),
openCivitaiByMetadata: vi.fn(),
updatePanelPositions: vi.fn(),
}));
vi.mock('../../../static/js/managers/DownloadManager.js', () => ({
downloadManager: { showDownloadModal: vi.fn() },
}));
vi.mock('../../../static/js/components/SidebarManager.js', () => ({
sidebarManager: {
setHostPageControls: vi.fn(),
initialize: vi.fn(async () => {}),
refresh: vi.fn(async () => {}),
cleanup: vi.fn(),
isInitialized: false,
},
}));
vi.mock('../../../static/js/components/alphabet/index.js', () => ({
createAlphabetBar: vi.fn(() => ({ destroy: vi.fn() })),
}));
vi.mock('../../../static/js/utils/updateCheckHelpers.js', () => ({
performModelUpdateCheck: vi.fn(async () => ({ status: 'success', displayName: 'LoRA', records: [] })),
}));
beforeEach(() => {
vi.resetModules();
vi.clearAllMocks();
localStorage.clear();
sessionStorage.clear();
resetAndReloadMock.mockResolvedValue(undefined);
getModelApiClientMock.mockReturnValue({});
global.fetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ success: true, base_models: [] }),
});
});
afterEach(() => {
delete window.bulkManager;
delete window.modelDuplicatesManager;
delete global.fetch;
});
function renderControlsDom(pageKey) {
document.body.dataset.page = pageKey;
document.body.innerHTML = `
<div class="controls">
<div id="excludedViewBanner" class="excluded-view-banner hidden">
<button id="excludedViewBackBtn">Back</button>
</div>
<div class="actions">
<div class="action-buttons">
<div class="control-group">
<select id="sortSelect">
<option value="name:asc">Name Asc</option>
<option value="name:desc">Name Desc</option>
<option value="random">Randomize (shuffle)</option>
</select>
</div>
<div class="control-group dropdown-group">
<button data-action="refresh" class="dropdown-main"></button>
<button class="dropdown-toggle"></button>
<div class="dropdown-menu">
<div class="dropdown-item" data-action="full-rebuild"></div>
</div>
</div>
<div class="control-group">
<button data-action="fetch"></button>
</div>
<div class="control-group">
<button data-action="download"></button>
</div>
<div class="control-group">
<button data-action="bulk"></button>
</div>
<div class="control-group">
<button data-action="find-duplicates"></button>
</div>
<div class="control-group">
<button id="favoriteFilterBtn" class="favorite-filter"></button>
</div>
<div class="control-group dropdown-group update-filter-group">
<button id="updateFilterBtn" class="dropdown-main update-filter" aria-busy="false">
<span>Updates</span>
</button>
<button id="updateFilterMenuToggle" class="dropdown-toggle"></button>
<div class="dropdown-menu">
<div id="checkUpdatesMenuItem" class="dropdown-item" data-action="check-updates">
<span>Check updates</span>
</div>
</div>
</div>
</div>
</div>
</div>
<div id="customFilterIndicator" class="control-group hidden">
<div class="filter-active">
<span class="customFilterText" title=""></span>
<i class="fas fa-times-circle clear-filter"></i>
</div>
</div>
<div id="breadcrumbContainer"></div>
<div id="duplicatesBanner" style="display: none;"></div>
<div class="alphabet-bar-container"></div>
`;
}
async function createControls() {
const stateModule = await import('../../../static/js/state/index.js');
stateModule.initPageState('loras');
const { LorasControls } = await import('../../../static/js/components/controls/LorasControls.js');
return { stateModule, controls: new LorasControls() };
}
describe('Random sort option', () => {
it('generates a seeded sort value when Random is picked', async () => {
renderControlsDom('loras');
const { controls } = await createControls();
const sortSelect = document.getElementById('sortSelect');
const randomOpt = sortSelect.querySelector('option[value="random"]');
sortSelect.value = 'random';
sortSelect.dispatchEvent(new Event('change', { bubbles: true }));
await Promise.resolve();
expect(controls.pageState.sortBy).toMatch(/^random:[a-z0-9]+$/);
expect(localStorage.getItem('lora_manager_loras_sort')).toBe(controls.pageState.sortBy);
expect(randomOpt.value).toBe(controls.pageState.sortBy);
expect(sortSelect.value).toBe(controls.pageState.sortBy);
expect(resetAndReloadMock).toHaveBeenCalled();
});
it('reshuffles with a fresh seed every time Random is picked again', async () => {
renderControlsDom('loras');
const { controls } = await createControls();
const sortSelect = document.getElementById('sortSelect');
const randomOpt = sortSelect.querySelector('option[value="random"]');
// First pick
sortSelect.value = 'random';
sortSelect.dispatchEvent(new Event('change', { bubbles: true }));
await Promise.resolve();
const firstSeed = controls.pageState.sortBy;
// Second pick: the option now carries the seeded value, like a menu click
sortSelect.value = randomOpt.value;
sortSelect.dispatchEvent(new Event('change', { bubbles: true }));
await Promise.resolve();
expect(controls.pageState.sortBy).toMatch(/^random:[a-z0-9]+$/);
expect(controls.pageState.sortBy).not.toBe(firstSeed);
});
it('restores a persisted seeded random sort on load', async () => {
renderControlsDom('loras');
const savedSort = 'random:persistedseed';
localStorage.setItem('lora_manager_loras_sort', savedSort);
const { controls } = await createControls();
const sortSelect = document.getElementById('sortSelect');
expect(controls.pageState.sortBy).toBe(savedSort);
expect(sortSelect.value).toBe(savedSort);
expect(sortSelect.querySelector('option[value="random:persistedseed"]')).not.toBeNull();
});
it('applies a non-random sort back to the plain random option', async () => {
renderControlsDom('loras');
const { controls } = await createControls();
const sortSelect = document.getElementById('sortSelect');
const randomOpt = sortSelect.querySelector('option[value="random"]');
// Seed a random sort, then switch to a normal sort
sortSelect.value = 'random';
sortSelect.dispatchEvent(new Event('change', { bubbles: true }));
await Promise.resolve();
controls.applySortToSelect('name:desc');
expect(sortSelect.value).toBe('name:desc');
expect(randomOpt.value).toBe('random');
});
it('resets the seeded option when switching away from Random via the dropdown change handler', async () => {
renderControlsDom('loras');
const { controls } = await createControls();
const sortSelect = document.getElementById('sortSelect');
const randomOpt = sortSelect.querySelector('option[value="random"]');
// Pick Random: the option is now seeded
sortSelect.value = 'random';
sortSelect.dispatchEvent(new Event('change', { bubbles: true }));
await Promise.resolve();
expect(randomOpt.value).toMatch(/^random:[a-z0-9]+$/);
// Switch to a non-random sort through the change handler (as a menu
// click does); the option must go back to the plain "random" value
sortSelect.value = 'name:desc';
sortSelect.dispatchEvent(new Event('change', { bubbles: true }));
await Promise.resolve();
expect(controls.pageState.sortBy).toBe('name:desc');
expect(sortSelect.value).toBe('name:desc');
expect(randomOpt.value).toBe('random');
});
});

View File

@@ -0,0 +1,68 @@
import { describe, it, beforeEach, expect } from 'vitest';
import { initSortDropdown } from '../../../static/js/components/controls/SortDropdown.js';
function renderSortDropdownDom() {
document.body.innerHTML = `
<div class="sort-dropdown-group">
<select id="sortSelect">
<option value="name:asc">Name Asc</option>
<option value="name:desc">Name Desc</option>
<option value="random" selected>Randomize (shuffle)</option>
</select>
<button class="sort-trigger" type="button">
<span class="sort-trigger__label"></span>
</button>
<div class="sort-dropdown-menu"></div>
</div>
`;
return {
select: document.getElementById('sortSelect'),
menu: document.querySelector('.sort-dropdown-menu'),
label: document.querySelector('.sort-trigger__label'),
};
}
describe('SortDropdown menu sync', () => {
let select;
let menu;
let label;
beforeEach(() => {
({ select, menu, label } = renderSortDropdownDom());
initSortDropdown(select);
});
it('rebuilds the menu and highlights the selected item when an option value attribute changes', async () => {
// The seeded Random option gets a new value each time it is picked.
// The select's value getter follows the selected option's new value.
const randomOpt = select.querySelector('option[value="random"]');
randomOpt.value = 'random:abc123';
await Promise.resolve();
const items = [...menu.querySelectorAll('.sort-option')];
expect(items.map((el) => el.dataset.value)).toContain('random:abc123');
const seededItem = items.find((el) => el.dataset.value === 'random:abc123');
expect(seededItem.classList.contains('is-selected')).toBe(true);
expect(label.textContent).toBe('Randomize (shuffle)');
});
it('drops the stale seeded item and re-selects the plain random item when the option is reset', async () => {
const randomOpt = select.querySelector('option[value="random"]');
randomOpt.value = 'random:abc123';
await Promise.resolve();
// The rebuild must have happened: the seeded item is in the menu
const seededItems = [...menu.querySelectorAll('.sort-option')]
.filter((el) => el.dataset.value === 'random:abc123');
expect(seededItems).toHaveLength(1);
// PageControls resets the option to "random" when switching away
randomOpt.value = 'random';
await Promise.resolve();
const items = [...menu.querySelectorAll('.sort-option')];
expect(items.map((el) => el.dataset.value)).not.toContain('random:abc123');
const randomItem = items.find((el) => el.dataset.value === 'random');
expect(randomItem.classList.contains('is-selected')).toBe(true);
});
});

View File

@@ -900,18 +900,28 @@ class FakeMetadataProvider:
async def get_model_versions(self, _model_id):
return {"modelVersions": [], "name": "", "type": "lora"}
async def get_user_models(self, _username):
return []
async def get_user_models(self, _username, cursor=None):
return {"items": [], "nextCursor": None}
async def get_creator_model_count(self, _username):
return None
class FakeUserModelsProvider(FakeMetadataProvider):
def __init__(self, models):
def __init__(self, models, next_cursor=None, estimated_total=None):
self.models = models
self.next_cursor = next_cursor
self.estimated_total = estimated_total
self.received_usernames: list[str] = []
self.received_cursors: list = []
async def get_user_models(self, username):
async def get_user_models(self, username, cursor=None):
self.received_usernames.append(username)
return self.models
self.received_cursors.append(cursor)
return {"items": self.models, "nextCursor": self.next_cursor}
async def get_creator_model_count(self, _username):
return self.estimated_total
async def fake_metadata_provider_factory():
@@ -1286,6 +1296,88 @@ async def test_get_civitai_user_models_requires_username():
assert "username" in payload["error"].lower()
@pytest.mark.asyncio
async def test_get_civitai_user_models_returns_pagination_fields():
models = [
{
"id": 1,
"name": "Model A",
"type": "LORA",
"tags": [],
"modelVersions": [
{"id": 100, "name": "v1", "images": [{"url": "http://example.com/a.jpg"}]},
],
},
{
"id": 2,
"name": "Unsupported",
"type": "Other",
"modelVersions": [{"id": 200, "name": "v1"}],
},
]
provider = FakeUserModelsProvider(models, next_cursor="cursor-token", estimated_total=2140)
async def provider_factory():
return provider
handler = ModelLibraryHandler(
ServiceRegistryAdapter(
get_lora_scanner=fake_scanner_factory,
get_checkpoint_scanner=fake_scanner_factory,
get_embedding_scanner=fake_scanner_factory,
get_downloaded_version_history_service=fake_download_history_service_factory,
),
metadata_provider_factory=provider_factory,
)
response = await handler.get_civitai_user_models(
FakeRequest(query={"username": "pixel"})
)
payload = json.loads(response.text)
assert response.status == 200
assert payload["success"] is True
# modelCount only counts models surviving the type filter
assert payload["modelCount"] == 1
assert payload["nextCursor"] == "cursor-token"
assert payload["hasMore"] is True
# first page includes the estimated total
assert payload["estimatedTotal"] == 2140
assert provider.received_cursors == [None]
@pytest.mark.asyncio
async def test_get_civitai_user_models_passes_cursor_and_omits_estimate():
provider = FakeUserModelsProvider([], next_cursor=None, estimated_total=999)
async def provider_factory():
return provider
handler = ModelLibraryHandler(
ServiceRegistryAdapter(
get_lora_scanner=fake_scanner_factory,
get_checkpoint_scanner=fake_scanner_factory,
get_embedding_scanner=fake_scanner_factory,
get_downloaded_version_history_service=fake_download_history_service_factory,
),
metadata_provider_factory=provider_factory,
)
response = await handler.get_civitai_user_models(
FakeRequest(query={"username": "pixel", "cursor": "opaque-token"})
)
payload = json.loads(response.text)
assert response.status == 200
assert payload["success"] is True
assert payload["nextCursor"] is None
assert payload["hasMore"] is False
# cursor requests must not include the estimated total
assert payload["estimatedTotal"] is None
assert provider.received_cursors == ["opaque-token"]
def test_ensure_handler_mapping_caches_result():
call_records = []

View File

@@ -183,7 +183,7 @@ class FakeCache:
def __init__(self, items):
self.items = list(items)
async def get_sorted_data(self, sort_key, order):
async def get_sorted_data(self, sort_key, order, seed=None):
if sort_key == "name":
data = sorted(self.items, key=lambda x: x["model_name"].lower())
if order == "desc":

View File

@@ -363,6 +363,148 @@ async def test_check_pending_models_handles_corrupted_progress_file(
assert result["pending_count"] == 1
@pytest.mark.asyncio
@pytest.mark.usefixtures("tmp_path")
async def test_check_pending_models_uses_bulk_folder_index_for_large_libraries(
monkeypatch: pytest.MonkeyPatch,
tmp_path,
settings_manager,
):
"""For >1000 candidates the pre-check scans the library root once instead of
probing every folder individually."""
ws_manager = RecordingWebSocketManager()
manager = download_module.DownloadManager(ws_manager=ws_manager)
monkeypatch.setitem(settings_manager.settings, "example_images_path", str(tmp_path))
# 1500 unprocessed models triggers the bulk lookup path
models = [
{"sha256": f"{i:064x}", "model_name": f"Model {i}"}
for i in range(1500)
]
# Create folders with files for the first 500 models
for i in range(500):
model_dir = tmp_path / f"{i:064x}"
model_dir.mkdir()
(model_dir / "image_0.png").write_text("data")
_patch_scanners(monkeypatch, lora_scanner=StubScanner(models))
per_model_checks = 0
def counting_model_directory_has_files(path: str) -> bool:
nonlocal per_model_checks
per_model_checks += 1
return False
monkeypatch.setattr(
download_module,
"_model_directory_has_files",
counting_model_directory_has_files,
)
result = await manager.check_pending_models(["lora"])
assert result["success"] is True
assert result["total_models"] == 1500
assert result["pending_count"] == 1000
assert result["needs_download"] is True
# The per-folder check should not be used once we cross the threshold.
assert per_model_checks == 0
@pytest.mark.asyncio
@pytest.mark.usefixtures("tmp_path")
async def test_check_pending_models_uses_per_folder_check_for_small_candidate_sets(
monkeypatch: pytest.MonkeyPatch,
tmp_path,
settings_manager,
):
"""For <=1000 candidates the pre-check keeps the accurate per-folder path."""
ws_manager = RecordingWebSocketManager()
manager = download_module.DownloadManager(ws_manager=ws_manager)
monkeypatch.setitem(settings_manager.settings, "example_images_path", str(tmp_path))
models = [
{"sha256": f"{i:064x}", "model_name": f"Model {i}"}
for i in range(500)
]
# Create folders with files for the first 200 models
for i in range(200):
model_dir = tmp_path / f"{i:064x}"
model_dir.mkdir()
(model_dir / "image_0.png").write_text("data")
_patch_scanners(monkeypatch, lora_scanner=StubScanner(models))
per_model_checks = 0
original_has_files = download_module._model_directory_has_files
def counting_model_directory_has_files(path: str) -> bool:
nonlocal per_model_checks
per_model_checks += 1
return original_has_files(path)
monkeypatch.setattr(
download_module,
"_model_directory_has_files",
counting_model_directory_has_files,
)
result = await manager.check_pending_models(["lora"])
assert result["success"] is True
assert result["total_models"] == 500
assert result["pending_count"] == 300
assert result["needs_download"] is True
# Per-folder path should run once per candidate.
assert per_model_checks == 500
@pytest.mark.asyncio
@pytest.mark.usefixtures("tmp_path")
async def test_check_pending_models_bulk_index_includes_legacy_folders(
monkeypatch: pytest.MonkeyPatch,
tmp_path,
settings_manager,
):
"""In multi-library mode the bulk index also scans the legacy root so models
whose folders have not been consolidated yet are not reported pending."""
ws_manager = RecordingWebSocketManager()
manager = download_module.DownloadManager(ws_manager=ws_manager)
monkeypatch.setitem(settings_manager.settings, "example_images_path", str(tmp_path))
monkeypatch.setitem(settings_manager.settings, "libraries", {"default": {}, "extra": {}})
monkeypatch.setitem(settings_manager.settings, "active_library", "extra")
# 1500 unprocessed models triggers the bulk lookup path
models = [
{"sha256": f"{i:064x}", "model_name": f"Model {i}"}
for i in range(1500)
]
# Folders live at the LEGACY root/<hash> path (not yet consolidated)
for i in range(500):
model_dir = tmp_path / f"{i:064x}"
model_dir.mkdir()
(model_dir / "image_0.png").write_text("data")
_patch_scanners(monkeypatch, lora_scanner=StubScanner(models))
result = await manager.check_pending_models(["lora"])
assert result["success"] is True
assert result["total_models"] == 1500
assert result["pending_count"] == 1000
assert result["needs_download"] is True
@pytest.fixture
def settings_manager():
return get_settings_manager()

View File

@@ -35,9 +35,11 @@ class DummyDownloader:
def reset_singletons():
CivitaiClient._instance = None
ModelMetadataProviderManager._instance = None
civitai_client_module._creator_model_count_cache.clear()
yield
CivitaiClient._instance = None
ModelMetadataProviderManager._instance = None
civitai_client_module._creator_model_count_cache.clear()
@pytest.fixture
@@ -622,3 +624,162 @@ async def test_get_image_info_handles_invalid_id(monkeypatch, downloader, caplog
assert result is None
assert "Invalid image ID format" in caplog.text
async def test_get_user_models_requests_first_page_with_stable_params(downloader):
request_calls = []
async def fake_make_request(method, url, use_auth=True, **kwargs):
request_calls.append({"method": method, "url": url, "kwargs": kwargs})
return True, {
"items": [
{
"id": 1,
"modelVersions": [
{"id": 100, "images": [{"meta": {"comfy": {"x": 1}}}]}
],
}
],
"metadata": {"nextCursor": "next-token"},
}
downloader.make_request = fake_make_request
client = await CivitaiClient.get_instance()
result = await client.get_user_models("pixel")
assert result is not None
assert result["nextCursor"] == "next-token"
assert len(result["items"]) == 1
# comfy metadata is still stripped
assert "comfy" not in result["items"][0]["modelVersions"][0]["images"][0]["meta"]
call = request_calls[0]
assert call["method"] == "GET"
assert call["url"] == "https://civitai.red/api/v1/models"
params = call["kwargs"]["params"]
assert params["username"] == "pixel"
assert params["nsfw"] == "true"
assert params["limit"] == 100
assert params["sort"] == "Newest"
assert params["period"] == "AllTime"
assert "cursor" not in params
async def test_get_user_models_passes_cursor_and_stringifies_next_cursor(downloader):
request_calls = []
async def fake_make_request(method, url, use_auth=True, **kwargs):
request_calls.append(kwargs)
return True, {"items": [], "metadata": {"nextCursor": 12345}}
downloader.make_request = fake_make_request
client = await CivitaiClient.get_instance()
result = await client.get_user_models("pixel", cursor="opaque-token")
assert request_calls[0]["params"]["cursor"] == "opaque-token"
assert result == {"items": [], "nextCursor": "12345"}
async def test_get_user_models_without_next_cursor_returns_none_cursor(downloader):
async def fake_make_request(method, url, use_auth=True, **kwargs):
return True, {"items": [{"id": 1, "modelVersions": []}], "metadata": {}}
downloader.make_request = fake_make_request
client = await CivitaiClient.get_instance()
result = await client.get_user_models("pixel")
assert result == {"items": [{"id": 1, "modelVersions": []}], "nextCursor": None}
async def test_get_user_models_failure_returns_none(downloader):
async def fake_make_request(method, url, use_auth=True, **kwargs):
return False, "500 server error"
downloader.make_request = fake_make_request
client = await CivitaiClient.get_instance()
result = await client.get_user_models("pixel")
assert result is None
async def test_get_creator_model_count_matches_exact_username(downloader):
request_calls = []
async def fake_make_request(method, url, use_auth=True, **kwargs):
request_calls.append({"url": url, "kwargs": kwargs})
return True, {
"items": [
{"username": "pixelart", "modelCount": 5},
{"username": "Pixel", "modelCount": 2140},
]
}
downloader.make_request = fake_make_request
client = await CivitaiClient.get_instance()
count = await client.get_creator_model_count("pixel")
assert count == 2140
assert request_calls[0]["url"] == "https://civitai.red/api/v1/creators"
assert request_calls[0]["kwargs"]["params"] == {"query": "pixel", "limit": 10}
async def test_get_creator_model_count_without_exact_match_returns_none(downloader):
async def fake_make_request(method, url, use_auth=True, **kwargs):
return True, {"items": [{"username": "pixelart", "modelCount": 5}]}
downloader.make_request = fake_make_request
client = await CivitaiClient.get_instance()
count = await client.get_creator_model_count("pixel")
assert count is None
async def test_get_creator_model_count_caches_results(downloader):
request_count = 0
async def fake_make_request(method, url, use_auth=True, **kwargs):
nonlocal request_count
request_count += 1
return True, {"items": [{"username": "pixel", "modelCount": 42}]}
downloader.make_request = fake_make_request
client = await CivitaiClient.get_instance()
assert await client.get_creator_model_count("pixel") == 42
# case-insensitive cache key, second call served from cache
assert await client.get_creator_model_count("Pixel") == 42
assert request_count == 1
async def test_get_creator_model_count_caches_failures(downloader):
request_count = 0
async def fake_make_request(method, url, use_auth=True, **kwargs):
nonlocal request_count
request_count += 1
return False, "500 server error"
downloader.make_request = fake_make_request
client = await CivitaiClient.get_instance()
assert await client.get_creator_model_count("pixel") is None
assert await client.get_creator_model_count("pixel") is None
assert request_count == 1
async def test_get_creator_model_count_never_raises(downloader):
async def fake_make_request(method, url, use_auth=True, **kwargs):
return True, "unexpected non-dict payload"
downloader.make_request = fake_make_request
client = await CivitaiClient.get_instance()
assert await client.get_creator_model_count("pixel") is None

View File

@@ -26,6 +26,7 @@ class StubScanner:
def __init__(self, models: list[dict]) -> None:
self._cache = SimpleNamespace(raw_data=models)
self.sync_calls: list[tuple[str, dict]] = []
async def get_cached_data(self):
return self._cache
@@ -38,6 +39,14 @@ class StubScanner:
break
return True
async def sync_cache_from_metadata(self, file_path: str, metadata: dict) -> bool:
self.sync_calls.append((file_path, metadata))
for index, model in enumerate(self._cache.raw_data):
if model.get("file_path") == metadata.get("file_path"):
self._cache.raw_data[index] = metadata
break
return True
def _patch_scanner(monkeypatch: pytest.MonkeyPatch, scanner: StubScanner) -> None:
async def _get_lora_scanner(cls):
@@ -588,6 +597,9 @@ async def test_not_found_example_images_are_cleaned(
assert missing_url in downloader.calls
assert manager._progress["failed_models"] == {model_hash}
assert model_hash in manager._progress["processed_models"]
assert scanner.sync_calls
assert len(scanner.sync_calls) == 1
assert scanner.sync_calls[0][0] == str(model_path)
remaining_images = model_metadata["civitai"]["images"]
assert remaining_images == [

View File

@@ -884,7 +884,7 @@ async def test_sync_cache_conditional_resort_skipped(tmp_path: Path, monkeypatch
raw_data=[dict(entry)], folders=[], name_display_mode="model_name"
)
await scanner._cache.resort()
scanner._cache._last_sort = ("name", "asc") # name sort is active
scanner._cache._last_sort = ("name", "asc", None) # name sort is active
scanner._tags_count = {"alpha": 1}
scanner._hash_index.add_entry("abc123", "/m/a.safetensors")
@@ -935,7 +935,7 @@ async def test_sync_cache_conditional_resort_triggered(tmp_path: Path, monkeypat
raw_data=[dict(entry)], folders=[], name_display_mode="model_name"
)
await scanner._cache.resort()
scanner._cache._last_sort = ("name", "asc")
scanner._cache._last_sort = ("name", "asc", None)
scanner._tags_count = {"alpha": 1}
scanner._hash_index.add_entry("abc123", "/m/a.safetensors")

View File

@@ -0,0 +1,97 @@
"""Tests for sort parsing and the seeded random sort mode."""
import asyncio
import pytest
from py.services.model_cache import ModelCache
from py.services.model_query import ModelCacheRepository, SortParams
def _make_cache(items):
return ModelCache(
raw_data=[
{
"file_path": f"/models/{name}.safetensors",
"file_name": f"{name}.safetensors",
"model_name": name,
"folder": "",
"size": 100,
"modified": 0.0,
}
for name in items
],
folders=[],
)
class TestParseSort:
def test_random_with_seed(self):
params = ModelCacheRepository.parse_sort("random:abc123")
assert params == SortParams(key="random", order="asc", seed="abc123")
def test_random_without_seed(self):
params = ModelCacheRepository.parse_sort("random")
assert params == SortParams(key="random", order="asc", seed=None)
def test_random_empty_seed_falls_back_to_none(self):
params = ModelCacheRepository.parse_sort("random:")
assert params.seed is None
def test_regular_sorts_unaffected(self):
params = ModelCacheRepository.parse_sort("name:desc")
assert params == SortParams(key="name", order="desc", seed=None)
class TestRandomShuffle:
@pytest.mark.asyncio
async def test_same_seed_yields_same_order(self):
cache = _make_cache(["a", "b", "c", "d", "e"])
await asyncio.sleep(0) # allow background resort task to run
first = await cache.get_sorted_data("random", "asc", "seed1")
second = await cache.get_sorted_data("random", "asc", "seed1")
assert [item["model_name"] for item in first] == [
item["model_name"] for item in second
]
@pytest.mark.asyncio
async def test_different_seeds_yield_different_orders(self):
cache = _make_cache([f"m{i}" for i in range(20)])
await asyncio.sleep(0)
first = await cache.get_sorted_data("random", "asc", "seed-a")
second = await cache.get_sorted_data("random", "asc", "seed-b")
assert [item["model_name"] for item in first] != [
item["model_name"] for item in second
]
@pytest.mark.asyncio
async def test_shuffle_is_a_permutation(self):
cache = _make_cache(["a", "b", "c", "d", "e"])
await asyncio.sleep(0)
shuffled = await cache.get_sorted_data("random", "asc", "seed")
assert sorted(item["model_name"] for item in shuffled) == [
"a",
"b",
"c",
"d",
"e",
]
assert len({item["file_path"] for item in shuffled}) == 5
@pytest.mark.asyncio
async def test_missing_seed_is_stable(self):
cache = _make_cache(["a", "b", "c", "d", "e"])
await asyncio.sleep(0)
first = await cache.get_sorted_data("random", "asc")
second = await cache.get_sorted_data("random", "asc")
assert [item["model_name"] for item in first] == [
item["model_name"] for item in second
]

View File

@@ -15,6 +15,7 @@ class StubScanner:
def __init__(self, cache_items: List[Dict[str, Any]]) -> None:
self.cache = SimpleNamespace(raw_data=cache_items)
self.updates: List[Tuple[str, str, Dict[str, Any]]] = []
self.sync_updates: List[Tuple[str, Dict[str, Any]]] = []
async def get_cached_data(self):
return self.cache
@@ -23,6 +24,10 @@ class StubScanner:
self.updates.append((old_path, new_path, metadata))
return True
async def sync_cache_from_metadata(self, file_path: str, metadata: Dict[str, Any]) -> bool:
self.sync_updates.append((file_path, metadata))
return True
@pytest.fixture(autouse=True)
def patch_metadata_manager(monkeypatch: pytest.MonkeyPatch):
@@ -83,7 +88,7 @@ async def test_update_metadata_after_import_enriches_entries(monkeypatch: pytest
assert custom[0]["type"] == "image"
assert Path(patch_metadata_manager[0][0]) == model_file
assert scanner.updates
assert scanner.sync_updates
@pytest.mark.asyncio
@@ -151,8 +156,8 @@ async def test_update_metadata_after_import_preserves_existing_metadata(
assert saved_payload["civitai"]["trainedWords"] == ["foo"]
assert {entry["id"] for entry in saved_payload["civitai"]["customImages"]} == {"existing-id", "new-id"}
assert scanner.updates
updated_metadata = scanner.updates[-1][2]
assert scanner.sync_updates
updated_metadata = scanner.sync_updates[-1][1]
assert updated_metadata["civitai"]["images"] == existing_payload["civitai"]["images"]
assert {entry["id"] for entry in updated_metadata["civitai"]["customImages"]} == {"existing-id", "new-id"}