import json
import os
import logging
import asyncio
import time
import shutil
from dataclasses import dataclass
from typing import Any, Awaitable, Callable, Dict, List, Mapping, Optional, Sequence, Set, Tuple, Type, Union, cast

from ..utils.models import BaseModelMetadata, autov3_from_civitai_files
from ..config import config
from ..utils.file_utils import find_preview_file, get_preview_extension, calculate_sha256, calculate_autov3
from ..utils.metadata_manager import MetadataManager
from ..utils.civitai_utils import resolve_license_info
from .model_cache import ModelCache
from .model_hash_index import ModelHashIndex
from .model_lifecycle_service import delete_model_artifacts, _require_path_in_library_roots
from .service_registry import ServiceRegistry
from .websocket_manager import ws_manager
from .persistent_model_cache import get_persistent_cache
from .settings_manager import get_settings_manager
from .pending_delete_service import PENDING_DELETE_DIR_NAME, get_pending_delete_service
from .cache_entry_validator import CacheEntryValidator
from .cache_health_monitor import CacheHealthMonitor, CacheHealthStatus

logger = logging.getLogger(__name__)

# Canonical set of weight-file extensions stripped when normalizing model
# names for matching (ModelScanner.find_matching_models and the recipe rematch
# filename key share this set). It is the union of the LoRA scanner set
# ({".safetensors"}) and the Checkpoint scanner set (ComfyUI's
# supported_pt_extensions plus ".gguf") so type-blind lookups (lora +
# checkpoint merged) cover every format either scanner indexes. ".safebin"
# is deliberately absent — no scanner indexes it, so a recipe entry
# "model.safebin" must not be bound to a local "model.safetensors".
WEIGHT_FILE_EXTENSIONS = frozenset(
    {
        ".safetensors",
        ".ckpt",
        ".pt",
        ".pt2",
        ".bin",
        ".pth",
        ".pkl",
        ".sft",
        ".gguf",
    }
)


def _is_excluded_dir(name: str) -> bool:
    """Return True when a directory entry must be skipped during model walks.

    The pending-delete staging directory is excluded so staged files never
    appear in the library as ghost model entries.
    """
    return name == PENDING_DELETE_DIR_NAME


def _is_hidden_relative_path(rel_path: str) -> bool:
    """Return True when any segment of a relative path is a hidden directory."""
    return any(part.startswith(".") for part in rel_path.replace(os.sep, "/").split("/"))


# TTL (seconds) for the get_all_folders() live-walk cache, so rapid repeated
# requests (modal open + autocomplete) do not re-walk the model roots.
ALL_FOLDERS_CACHE_TTL_SECONDS = 5.0


def _is_pending_delete_path(path: str) -> bool:
    """Return True when any path component is the pending-delete staging dir."""
    normalized = str(path).replace(os.sep, "/")
    return any(part == PENDING_DELETE_DIR_NAME for part in normalized.split("/"))


@dataclass
class CacheBuildResult:
    """Represents the outcome of scanning model files for cache building."""

    raw_data: List[Dict[str, Any]]
    hash_index: ModelHashIndex
    tags_count: Dict[str, int]
    excluded_models: List[str]

class ModelScanner:
    """Base service for scanning and managing model files"""
    
    _instances = {}  # Dictionary to store instances by class
    _locks = {}  # Dictionary to store locks by class
    
    def __new__(cls, *args, **kwargs):
        """Implement singleton pattern for each subclass"""
        if cls not in cls._instances:
            cls._instances[cls] = super().__new__(cls)
        return cls._instances[cls]
    
    @classmethod
    def _get_lock(cls):
        """Get or create a lock for this class"""
        if cls not in cls._locks:
            cls._locks[cls] = asyncio.Lock()
        return cls._locks[cls]
    
    @classmethod
    async def get_instance(cls):
        """Get singleton instance with async support"""
        lock = cls._get_lock()
        async with lock:
            if cls not in cls._instances:
                cls._instances[cls] = cls()  # pyright: ignore[reportCallIssue]
            return cls._instances[cls]
    
    def __init__(self, model_type: str, model_class: Type[BaseModelMetadata], file_extensions: Set[str], hash_index: Optional[ModelHashIndex] = None):
        """Initialize the scanner
        
        Args:
            model_type: Type of model (lora, checkpoint, etc.)
            model_class: Class used to create metadata instances
            file_extensions: Set of supported file extensions including the dot (e.g. {'.safetensors'})
            hash_index: Hash index instance (optional)
        """
        # Ensure initialization happens only once per instance
        if hasattr(self, '_initialized'):
            return
            
        self.model_type = model_type
        self.model_class = model_class
        self.file_extensions = file_extensions
        self._cache: Any = None
        self._cache_version: int = 0
        self._hash_index = hash_index or ModelHashIndex()
        self._tags_count = {}  # Dictionary to store tag counts
        self._is_initializing = False  # Flag to track initialization state
        self._excluded_models = []  # List to track excluded models
        self._persistent_cache = get_persistent_cache()
        self._name_display_mode = self._resolve_name_display_mode()
        self._cancel_requested = False  # Flag for cancellation
        self._autov3_backfill_scheduled = False  # One-time AutoV3 backfill trigger per process
        # Short-lived cache for get_all_folders(): (timestamp, folders) or None
        self._all_folders_ttl_cache: Optional[Tuple[float, List[str]]] = None
        try:
            loop = asyncio.get_running_loop()
        except RuntimeError:
            loop = None
        self._loop = loop
        self.loop = loop
        self._initialized = True

        # Register this service
        asyncio.create_task(self._register_service())

    @property
    def cache_version(self) -> int:
        """Monotonic version counter for the in-memory cache.

        Every write path that mutates scanner cache state calls
        :meth:`bump_cache_version`, so consumers (e.g. RecipeScanner) can
        detect when a cached derivation of the raw data is stale. Reads never
        bump.
        """
        return self._cache_version

    def bump_cache_version(self) -> None:
        """Invalidate derived caches by incrementing the cache version.

        Public because external services (model lifecycle, route handlers)
        rewrite scanner raw_data directly and must be able to invalidate it.
        """
        self._cache_version += 1

    def on_library_changed(self) -> None:
        """Reset caches when the active library changes."""
        self._persistent_cache = get_persistent_cache()
        self._cache = None
        self._hash_index = ModelHashIndex()
        self._tags_count = {}
        self._excluded_models = []
        self._is_initializing = False
        self._name_display_mode = self._resolve_name_display_mode()
        self.invalidate_all_folders_cache()
        self.bump_cache_version()

        try:
            loop = asyncio.get_running_loop()
        except RuntimeError:
            loop = None

        if loop and not loop.is_closed():
            self._loop = loop
            self.loop = loop
            loop.create_task(self.initialize_in_background())

    def _resolve_name_display_mode(self) -> str:
        """Return the configured display mode for name sorting."""

        try:
            manager = get_settings_manager()
        except Exception:  # pragma: no cover - fallback to defaults
            return "model_name"

        value = manager.get("model_name_display", "model_name")
        return ModelCache._normalize_display_mode(value)

    async def on_model_name_display_changed(self, display_mode: str) -> None:
        """Handle updates to the model name display preference."""

        normalized = ModelCache._normalize_display_mode(display_mode)
        self._name_display_mode = normalized

        if self._cache is not None:
            await self._cache.update_name_display_mode(normalized)

    async def _register_service(self):
        """Register this instance with the ServiceRegistry"""
        service_name = f"{self.model_type}_scanner"
        await ServiceRegistry.register_service(service_name, self)

    def _slim_civitai_payload(self, civitai: Optional[Mapping[str, Any]]) -> Optional[Dict[str, Any]]:
        """Return a lightweight civitai payload containing only frequently used keys."""
        if not isinstance(civitai, Mapping) or not civitai:
            return None

        slim: Dict[str, Any] = {}
        for key in ('id', 'modelId', 'name'):
            value = civitai.get(key)
            if value not in (None, '', []):
                slim[key] = value

        creator = civitai.get('creator')
        if isinstance(creator, Mapping):
            username = creator.get('username')
            if username:
                slim['creator'] = {'username': username}

        trained_words = civitai.get('trainedWords')
        if trained_words:
            slim['trainedWords'] = list(trained_words) if isinstance(trained_words, list) else trained_words

        civitai_model = civitai.get('model')
        if isinstance(civitai_model, Mapping):
            model_type_value = civitai_model.get('type')
            if model_type_value not in (None, '', []):
                slim['model'] = {'type': model_type_value}

        return slim or None

    def _build_cache_entry(
        self,
        source: Union[BaseModelMetadata, Mapping[str, Any]],
        *,
        folder: Optional[str] = None,
        file_path_override: Optional[str] = None
    ) -> Dict[str, Any]:
        """Project metadata into the lightweight cache representation."""
        is_mapping = isinstance(source, Mapping)

        def get_value(key: str, default: Any = None) -> Any:
            if isinstance(source, Mapping):
                return source.get(key, default)

            sentinel = object()
            value = getattr(source, key, sentinel)
            if value is not sentinel:
                return value

            unknown = getattr(source, "_unknown_fields", None)
            if isinstance(unknown, dict) and key in unknown:
                return unknown[key]

            return default

        file_path = file_path_override or get_value('file_path', '') or ''
        normalized_path = file_path.replace('\\', '/')

        folder_value = folder if folder is not None else get_value('folder', '') or ''
        normalized_folder = folder_value.replace('\\', '/')

        tags_value = get_value('tags') or []
        if isinstance(tags_value, list):
            tags_list = list(tags_value)
        elif isinstance(tags_value, (set, tuple)):
            tags_list = list(tags_value)
        else:
            tags_list = []

        preview_url = get_value('preview_url', '') or ''
        if isinstance(preview_url, str):
            preview_url = preview_url.replace('\\', '/')
        else:
            preview_url = ''

        civitai_full = get_value('civitai')
        civitai_slim = self._slim_civitai_payload(civitai_full)
        usage_tips = get_value('usage_tips', '') or ''
        if not isinstance(usage_tips, str):
            usage_tips = str(usage_tips)
        notes = get_value('notes', '') or ''
        if not isinstance(notes, str):
            notes = str(notes)

        # AutoV3 three-state contract: absent key / None = "not checked yet",
        # "" = "checked but unavailable" (never re-read the header), else the
        # 12-char lowercase hex value. A metadata object already follows the
        # contract and is passed through unchanged; a payload dict only carries
        # an explicit checked state when the key is present.
        if is_mapping:
            if 'autov3' in source:
                entry_autov3 = source['autov3'] or ''
            else:
                entry_autov3 = None
        else:
            entry_autov3 = get_value('autov3', None)

        entry: Dict[str, Any] = {
            'file_path': normalized_path,
            # file_name is always stored WITHOUT extension (e.g. "OWSMianne_ANIMA_V1",
            # not "OWSMianne_ANIMA_V1.safetensors"). All upstream population points
            # (MetadataManager, from_civitai_info, download manager, etc.) strip the
            # extension via os.path.splitext before writing. Code consuming this field
            # should match against names that are likewise extension-free.
            'file_name': get_value('file_name', '') or '',
            'model_name': get_value('model_name', '') or '',
            'folder': normalized_folder,
            'size': int(get_value('size', 0) or 0),
            'modified': float(get_value('modified', 0.0) or 0.0),
            'sha256': (get_value('sha256', '') or '').lower(),
            'autov3': entry_autov3,
            'base_model': get_value('base_model', '') or '',
            'preview_url': preview_url,
            'preview_nsfw_level': int(get_value('preview_nsfw_level', 0) or 0),
            'from_civitai': bool(get_value('from_civitai', True)),
            'favorite': bool(get_value('favorite', False)),
            'notes': notes,
            'usage_tips': usage_tips,
            'metadata_source': get_value('metadata_source', None),
            'exclude': bool(get_value('exclude', False)),
            'db_checked': bool(get_value('db_checked', False)),
            'last_checked_at': float(get_value('last_checked_at', 0.0) or 0.0),
            'tags': tags_list,
            'civitai': civitai_slim,
            'civitai_deleted': bool(get_value('civitai_deleted', False)),
            'skip_metadata_refresh': bool(get_value('skip_metadata_refresh', False)),
            'hf_url': get_value('hf_url', '') or '',
        }

        license_source: Dict[str, Any] = {}
        if isinstance(civitai_full, Mapping):
            civitai_model = civitai_full.get('model')
            if isinstance(civitai_model, Mapping):
                for key in (
                    'allowNoCredit',
                    'allowCommercialUse',
                    'allowDerivatives',
                    'allowDifferentLicense',
                ):
                    if key in civitai_model:
                        license_source[key] = civitai_model.get(key)

        for key in (
            'allowNoCredit',
            'allowCommercialUse',
            'allowDerivatives',
            'allowDifferentLicense',
        ):
            if key not in license_source:
                value = get_value(key)
                if value is not None:
                    license_source[key] = value

        _, license_flags = resolve_license_info(license_source or {})
        entry['license_flags'] = license_flags

        # Handle sub_type (new canonical field)
        sub_type = get_value('sub_type', None)
        if sub_type:
            entry['sub_type'] = sub_type
        
        # Handle hash_status for lazy hash calculation (checkpoints)
        hash_status = get_value('hash_status', 'completed')
        if hash_status:
            entry['hash_status'] = hash_status

        return entry

    def _ensure_license_flags(self, entry: Dict[str, Any]) -> None:
        """Ensure cached entries include an integer license flag bitset."""

        if not isinstance(entry, dict):
            return

        license_value = entry.get('license_flags')
        if license_value is not None:
            try:
                entry['license_flags'] = int(license_value)
            except (TypeError, ValueError):
                _, fallback_flags = resolve_license_info({})
                entry['license_flags'] = fallback_flags
            return

        license_source = {
            'allowNoCredit': entry.get('allowNoCredit'),
            'allowCommercialUse': entry.get('allowCommercialUse'),
            'allowDerivatives': entry.get('allowDerivatives'),
            'allowDifferentLicense': entry.get('allowDifferentLicense'),
        }
        civitai_full = entry.get('civitai')
        if isinstance(civitai_full, Mapping):
            civitai_model = civitai_full.get('model')
            if isinstance(civitai_model, Mapping):
                for key in (
                    'allowNoCredit',
                    'allowCommercialUse',
                    'allowDerivatives',
                    'allowDifferentLicense',
                ):
                    if key in civitai_model:
                        license_source[key] = civitai_model.get(key)

        _, license_flags = resolve_license_info(license_source)
        entry['license_flags'] = license_flags

    async def initialize_in_background(self) -> None:
        """Initialize cache in background using thread pool"""
        try:
            # Set initial empty cache to avoid None reference errors
            if self._cache is None:
                self._cache = ModelCache(
                    raw_data=[],
                    folders=[],
                    name_display_mode=self._name_display_mode,
                )
            
            # Set initializing flag to true
            self._is_initializing = True
            
            # Determine the page type based on model type
            page_type_map = {
                'lora': 'loras',
                'checkpoint': 'checkpoints',
                'embedding': 'embeddings'
            }
            page_type = page_type_map.get(self.model_type, self.model_type)
            
            # First, try to load from cache
            await ws_manager.broadcast_init_progress({
                'stage': 'loading_cache',
                'progress': 0,
                'details': f"Loading {self.model_type} cache...",
                'scanner_type': self.model_type,
                'pageType': page_type
            })

            cache_loaded = await self._load_persisted_cache(page_type)

            if cache_loaded:
                await asyncio.sleep(0)  # Yield control so the UI can process the cache hydration update
                await ws_manager.broadcast_init_progress({
                    'stage': 'finalizing',
                    'progress': 100,
                    'status': 'complete',
                    'details': f"Loaded {len(self._cache.raw_data)} cached {self.model_type} files from disk.",
                    'scanner_type': self.model_type,
                    'pageType': page_type
                })
                logger.info(
                    f"{self.model_type.capitalize()} cache hydrated from persisted snapshot with {len(self._cache.raw_data)} models"
                )
                return

            # Persistent load failed; fall back to a full scan
            await ws_manager.broadcast_init_progress({
                'stage': 'scan_folders',
                'progress': 0,
                'details': f"Scanning {self.model_type} folders...",
                'scanner_type': self.model_type,
                'pageType': page_type
            })
            
            # Count files in a separate thread to avoid blocking
            loop = asyncio.get_event_loop()
            total_files = await loop.run_in_executor(
                None,  # Use default thread pool
                self._count_model_files  # Run file counting in thread
            )
            
            await ws_manager.broadcast_init_progress({
                'stage': 'count_models',
                'progress': 1, # Changed from 10 to 1
                'details': f"Found {total_files} {self.model_type} files",
                'scanner_type': self.model_type,
                'pageType': page_type
            })
            
            start_time = time.time()
            
            # Use thread pool to execute CPU-intensive operations with progress reporting
            scan_result: Optional[CacheBuildResult] = await loop.run_in_executor(
                None,  # Use default thread pool
                self._initialize_cache_sync,  # Run synchronous version in thread
                total_files,  # Pass the total file count for progress reporting
                page_type  # Pass the page type for progress reporting
            )

            if scan_result:
                await self._apply_scan_result(scan_result)
                await self._save_persistent_cache(scan_result)
                await self._sync_download_history(scan_result.raw_data, source='scan')

            # Send final progress update
            await ws_manager.broadcast_init_progress({
                'stage': 'finalizing',
                'progress': 99, # Changed from 95 to 99
                'details': f"Finalizing {self.model_type} cache...",
                'scanner_type': self.model_type,
                'pageType': page_type
            })
            
            logger.info(f"{self.model_type.capitalize()} cache initialized in {time.time() - start_time:.2f} seconds. Found {len(self._cache.raw_data)} models")
            
            # Send completion message
            await asyncio.sleep(0.5)  # Small delay to ensure final progress message is sent
            await ws_manager.broadcast_init_progress({
                'stage': 'finalizing',
                'progress': 100,
                'status': 'complete',
                'details': f"Completed! Found {len(self._cache.raw_data)} {self.model_type} files.",
                'scanner_type': self.model_type,
                'pageType': page_type
            })
            
        except Exception as e:
            logger.error(f"{self.model_type.capitalize()} Scanner: Error initializing cache in background: {e}")
        finally:
            # Always clear the initializing flag when done
            self._is_initializing = False
    
    async def _load_persisted_cache(self, page_type: str) -> bool:
        """Attempt to hydrate the in-memory cache from the SQLite snapshot.

        The SQLite read and the per-model rebuild (entry adjustment, tag
        counting, validation/repair, hash index reconstruction) run in the
        default executor so the event loop stays responsive; only applying
        the result to shared cache state happens on the loop.
        """
        if not getattr(self, '_persistent_cache', None):
            return False

        loop = asyncio.get_event_loop()
        try:
            rebuilt = await loop.run_in_executor(
                None,
                self._rebuild_persisted_cache
            )
        except FileNotFoundError:
            return False
        except Exception as exc:
            logger.debug("%s Scanner: Could not load persisted cache: %s", self.model_type.capitalize(), exc)
            return False

        if rebuilt is None:
            return False

        scan_result, invalid_entries = rebuilt

        if invalid_entries:
            monitor = CacheHealthMonitor()
            report = monitor.check_health(scan_result.raw_data, auto_repair=True)

            if report.status != CacheHealthStatus.HEALTHY:
                # Broadcast health warning to frontend
                await ws_manager.broadcast_cache_health_warning(report, page_type)
                logger.warning(
                    f"{self.model_type.capitalize()} Scanner: Cache health issue detected - "
                    f"{report.invalid_entries} invalid entries, {report.repaired_entries} repaired"
                )

            # Rebuild tags count from valid entries only
            tags_count = {}
            for item in scan_result.raw_data:
                for tag in item.get('tags') or []:
                    tags_count[tag] = tags_count.get(tag, 0) + 1
            scan_result.tags_count = tags_count

            # Remove invalid entries from hash index
            for invalid_entry in invalid_entries:
                file_path = CacheEntryValidator.get_file_path_safe(invalid_entry)
                sha256 = CacheEntryValidator.get_sha256_safe(invalid_entry)
                if file_path:
                    scan_result.hash_index.remove_by_path(file_path, sha256)

        await self._apply_scan_result(scan_result)
        await self._sync_download_history(scan_result.raw_data, source='scan')

        await ws_manager.broadcast_init_progress({
            'stage': 'loading_cache',
            'progress': 1,
            'details': f"Loaded cached {self.model_type} data from disk",
            'scanner_type': self.model_type,
            'pageType': page_type
        })

        # Schedule the one-time AutoV3 backfill task (at most once per process)
        # so entries loaded from a persisted snapshot that predates autov3 get
        # their checked state computed in the background. The task never blocks
        # or crashes the load path.
        if not self._autov3_backfill_scheduled:
            self._autov3_backfill_scheduled = True
            try:
                loop = asyncio.get_running_loop()
            except RuntimeError:
                loop = None
            if loop is not None:
                loop.create_task(self._run_autov3_backfill())

        return True

    def _rebuild_persisted_cache(self) -> Optional[Tuple[CacheBuildResult, List[Dict[str, Any]]]]:
        """Load the SQLite snapshot and rebuild a ready-to-apply scan result.

        Runs entirely in a worker thread: it must not touch ``self._cache``,
        the websocket manager, or any asyncio primitives. Returns ``None``
        when no usable snapshot exists, otherwise a tuple of the scan result
        (built from validated/repaired entries) and the invalid entries.
        """
        persisted = self._persistent_cache.load_cache(self.model_type)

        if not persisted or not persisted.raw_data:
            return None

        hash_index = ModelHashIndex()
        for sha_value, path in persisted.hash_rows:
            if sha_value and path:
                hash_index.add_entry(sha_value.lower(), path)

        # Rebuild the AutoV3 index from the persisted autov3_index rows. These
        # cover every known autov3 -> path mapping regardless of whether a
        # sha256 row also exists for the same file.
        for autov3_value, path in persisted.autov3_hash_rows:
            if autov3_value and path:
                hash_index.add_autov3(autov3_value.lower(), path)

        tags_count: Dict[str, int] = {}
        adjusted_raw_data: List[Dict[str, Any]] = []
        for item in persisted.raw_data:
            # load_cache builds a fresh dict per row, and validate_batch below
            # works on its own per-entry copy when auto_repair=True, so no
            # additional dict copy is needed here.
            adjusted_item = self.adjust_cached_entry(item)
            adjusted_raw_data.append(adjusted_item)

            for tag in adjusted_item.get('tags') or []:
                tags_count[tag] = tags_count.get(tag, 0) + 1

        # Validate cache entries and check health.
        # Always use the validated/repaired entries — even when there are no
        # invalid entries, auto_repair may have filled in missing optional
        # fields (model_name, file_name, folder) with safe defaults on a copied
        # working_entry.  Without this unconditional replacement the repaired
        # copies are discarded and None values propagate to format_response.
        # See issue #730.
        valid_entries, invalid_entries = CacheEntryValidator.validate_batch(
            adjusted_raw_data, auto_repair=True
        )

        # Always use the validated entries (repaired copies)
        scan_result = CacheBuildResult(
            raw_data=valid_entries,
            hash_index=hash_index,
            tags_count=tags_count,
            excluded_models=list(persisted.excluded_models)
        )
        return scan_result, invalid_entries

    async def _run_autov3_backfill(self) -> None:
        """Backfill autov3 for entries loaded from the persisted cache that lack it."""
        try:
            from ..services.autov3_backfill_service import Autov3BackfillService  # lazy import (module created by another unit)
            await Autov3BackfillService.get_instance().backfill(self)
        except Exception as exc:
            logger.warning("AutoV3 backfill failed: %s", exc)

    async def _save_persistent_cache(self, scan_result: CacheBuildResult) -> None:
        if not scan_result or not getattr(self, '_persistent_cache', None):
            return

        if self.is_cancelled():
            logger.info(
                f"{self.model_type.capitalize()} Scanner: Skipping _save_persistent_cache "
                "after cancellation"
            )
            return

        hash_snapshot = self._build_hash_index_snapshot(scan_result.hash_index)
        autov3_snapshot = self._build_autov3_index_snapshot(scan_result.hash_index)
        loop = asyncio.get_event_loop()
        try:
            await loop.run_in_executor(
                None,
                self._persistent_cache.save_cache,
                self.model_type,
                list(scan_result.raw_data),
                hash_snapshot,
                list(scan_result.excluded_models),
                autov3_snapshot,
            )
        except Exception as exc:
            logger.warning("%s Scanner: Failed to persist cache: %s", self.model_type.capitalize(), exc)

    def _build_hash_index_snapshot(self, hash_index: Optional[ModelHashIndex]) -> Dict[str, List[str]]:
        snapshot: Dict[str, List[str]] = {}
        if not hash_index:
            return snapshot

        for sha_value, path in getattr(hash_index, '_hash_to_path', {}).items():
            if not sha_value or not path:
                continue
            bucket = snapshot.setdefault(sha_value.lower(), [])
            if path not in bucket:
                bucket.append(path)

        for sha_value, paths in getattr(hash_index, '_duplicate_hashes', {}).items():
            if not sha_value:
                continue
            bucket = snapshot.setdefault(sha_value.lower(), [])
            for path in paths:
                if path and path not in bucket:
                    bucket.append(path)
        return snapshot

    def _build_autov3_index_snapshot(self, hash_index: Optional[ModelHashIndex]) -> Dict[str, List[str]]:
        """Build the autov3 -> [paths] snapshot for the persisted cache."""
        snapshot: Dict[str, List[str]] = {}
        if not hash_index:
            return snapshot

        for autov3_value, path in hash_index.get_all_autov3().items():
            if not autov3_value or not path:
                continue
            bucket = snapshot.setdefault(autov3_value.lower(), [])
            if path not in bucket:
                bucket.append(path)
        return snapshot

    async def _persist_current_cache(self) -> None:
        if self._cache is None or not getattr(self, '_persistent_cache', None):
            return

        snapshot = CacheBuildResult(
            raw_data=list(self._cache.raw_data),
            hash_index=self._hash_index,
            tags_count=dict(self._tags_count),
            excluded_models=list(self._excluded_models)
        )
        await self._save_persistent_cache(snapshot)
        await self._sync_download_history(snapshot.raw_data, source='scan')
    def _count_model_files(self) -> int:
        """Count all model files with supported extensions in all roots
        
        Returns:
            int: Total number of model files found
        """
        total_files = 0
        visited_real_paths = set()
        
        for root_path in self.get_model_roots():
            if not os.path.exists(root_path):
                continue
                
            def count_recursive(path):
                nonlocal total_files
                try:
                    real_path = os.path.realpath(path)
                    if real_path in visited_real_paths:
                        return
                    visited_real_paths.add(real_path)
                    
                    with os.scandir(path) as it:
                        for entry in it:
                            try:
                                if entry.is_file(follow_symlinks=True):
                                    ext = os.path.splitext(entry.name)[1].lower()
                                    if ext in self.file_extensions:
                                        total_files += 1
                                elif entry.is_dir(follow_symlinks=True):
                                    if _is_excluded_dir(entry.name):
                                        continue
                                    count_recursive(entry.path)
                            except Exception as e:
                                logger.error(f"Error counting files in entry {entry.path}: {e}")
                except Exception as e:
                    logger.error(f"Error counting files in {path}: {e}")
            
            count_recursive(root_path)
        
        return total_files
    
    def _initialize_cache_sync(self, total_files: int = 0, page_type: str = 'loras') -> Optional[CacheBuildResult]:
        """Synchronous version of cache initialization for thread pool execution"""

        loop = asyncio.new_event_loop()
        try:
            asyncio.set_event_loop(loop)

            last_progress_time = time.time()
            last_progress_percent = 0

            async def progress_callback(processed_files: int, expected_total: int) -> None:
                nonlocal last_progress_time, last_progress_percent

                if expected_total <= 0:
                    return

                current_time = time.time()
                progress_percent = min(99, int(1 + (processed_files / expected_total) * 98))

                if progress_percent <= last_progress_percent:
                    return

                if current_time - last_progress_time <= 0.5 and processed_files != expected_total:
                    return

                last_progress_percent = progress_percent
                last_progress_time = current_time

                await ws_manager.broadcast_init_progress({
                    'stage': 'process_models',
                    'progress': progress_percent,
                    'details': f"Processing {self.model_type} files: {processed_files}/{expected_total}",
                    'scanner_type': self.model_type,
                    'pageType': page_type
                })

            return loop.run_until_complete(
                self._gather_model_data(
                    total_files=total_files,
                    progress_callback=progress_callback
                )
            )
        except Exception as e:
            logger.error(f"Error in thread-based {self.model_type} cache initialization: {e}")
            return None
        finally:
            asyncio.set_event_loop(None)
            loop.close()

    async def get_cached_data(self, force_refresh: bool = False, rebuild_cache: bool = False) -> ModelCache:
        """Get cached model data, refresh if needed
        
        Args:
            force_refresh: Whether to refresh the cache
            rebuild_cache: Whether to completely rebuild the cache
        """
        # If cache is not initialized, return an empty cache
        # Actual initialization should be done via initialize_in_background
        if self._cache is None and not force_refresh:
            return ModelCache(
                raw_data=[],
                folders=[],
                name_display_mode=self._name_display_mode,
            )

        # If force refresh is requested, initialize the cache directly
        if force_refresh:
            if rebuild_cache:
                await self._initialize_cache()
            else:
                await self._reconcile_cache()
        
        return cast(ModelCache, self._cache)

    async def _initialize_cache(self) -> None:
        """Initialize or refresh the cache"""
        self._is_initializing = True  # Set flag
        try:
            start_time = time.time()
            
            # Manually trigger a symlink rescan during a full rebuild.
            # This ensures that any new symlink mappings are correctly picked up.
            config.rebuild_symlink_cache()

            # Determine the page type based on model type
            # Scan for new data
            scan_result = await self._gather_model_data()
            if not self.is_cancelled():
                await self._apply_scan_result(scan_result)
                await self._save_persistent_cache(scan_result)
                await self._sync_download_history(scan_result.raw_data, source='scan')

                logger.info(
                    f"{self.model_type.capitalize()} Scanner: Cache initialization completed in {time.time() - start_time:.2f} seconds, "
                    f"found {len(scan_result.raw_data)} models"
                )
            else:
                logger.info(
                    f"{self.model_type.capitalize()} Scanner: Cache initialization cancelled "
                    f"after {time.time() - start_time:.2f} seconds"
                )
        except Exception as e:
            logger.error(f"{self.model_type.capitalize()} Scanner: Error initializing cache: {e}")
            # Ensure cache is at least an empty structure on error
            if self._cache is None:
                self._cache = ModelCache(
                    raw_data=[],
                    folders=[],
                    name_display_mode=self._name_display_mode,
                )
        finally:
            self._is_initializing = False # Unset flag

    async def _reconcile_cache(self) -> None:
        """Fast cache reconciliation - only process differences between cache and filesystem"""
        self.reset_cancellation()
        self._is_initializing = True # Set flag for reconciliation duration
        try:
            start_time = time.time()
            logger.info(f"{self.model_type.capitalize()} Scanner: Starting fast cache reconciliation...")
            
            # Get current cached file paths
            cached_paths = {item['file_path'] for item in self._cache.raw_data}
            path_to_item = {item['file_path']: item for item in self._cache.raw_data}
            cached_real_paths = {}
            for cached_path in cached_paths:
                try:
                    cached_real_paths.setdefault(os.path.realpath(cached_path), cached_path)
                except Exception:
                    continue
            
            # Track found files and new files
            found_paths = set()
            new_files = []
            visited_real_paths = set()
            discovered_real_files = set()

            # Scan all model roots
            for root_path in self.get_model_roots():
                if not os.path.exists(root_path):
                    continue

                # Recursively scan directory
                for root, dirnames, files in os.walk(root_path, followlinks=True):
                    dirnames[:] = [d for d in dirnames if not _is_excluded_dir(d)]
                    real_root = os.path.realpath(root)
                    if real_root in visited_real_paths:
                        continue
                    visited_real_paths.add(real_root)

                    for file in files:
                        ext = os.path.splitext(file)[1].lower()
                        if ext in self.file_extensions:
                            # Construct paths exactly as they would be in cache
                            file_path = os.path.join(root, file).replace(os.sep, '/')
                            real_file_path = os.path.realpath(os.path.join(root, file))
                            
                            # Check if this file is already in cache
                            if file_path in cached_paths:
                                found_paths.add(file_path)
                                continue

                            cached_real_match = cached_real_paths.get(real_file_path)
                            if cached_real_match:
                                found_paths.add(cached_real_match)
                                continue

                            if file_path in self._excluded_models:
                                continue
                                
                            # Try case-insensitive match on Windows
                            if os.name == 'nt':
                                lower_path = file_path.lower()
                                matched = False
                                for cached_path in cached_paths:
                                    if cached_path.lower() == lower_path:
                                        found_paths.add(cached_path)
                                        matched = True
                                        break
                                if matched:
                                    continue
                                
                            if real_file_path in discovered_real_files:
                                continue

                            discovered_real_files.add(real_file_path)
                            # This is a new file to process
                            new_files.append(file_path)
                    
                    # Yield control periodically
                    await asyncio.sleep(0)
                    if self.is_cancelled():
                        logger.info(f"{self.model_type.capitalize()} Scanner: Reconcile scan cancelled")
                        return

            # Process new files in batches
            total_added = 0
            if new_files:
                logger.info(f"{self.model_type.capitalize()} Scanner: Found {len(new_files)} new files to process")
                batch_size = 50
                for i in range(0, len(new_files), batch_size):
                    batch = new_files[i:i+batch_size]
                    for path in batch:
                        logger.info(f"{self.model_type.capitalize()} Scanner: Processing {path}")
                        try:
                            # Find the appropriate root path for this file
                            root_path = None
                            model_roots = self.get_model_roots()
                            for potential_root in model_roots:
                                # Normalize both paths for comparison
                                normalized_path = os.path.normpath(path)
                                normalized_root = os.path.normpath(potential_root)
                                if normalized_path.startswith(normalized_root):
                                    root_path = potential_root
                                    break
                            
                            if root_path:
                                model_data = await self._process_model_file(path, root_path)
                                if model_data:
                                    model_data = self.adjust_cached_entry(dict(model_data))
                                    if not model_data:
                                        continue

                                    # Validate the new entry before adding
                                    validation_result = CacheEntryValidator.validate(
                                        model_data, auto_repair=True
                                    )
                                    if not validation_result.is_valid:
                                        logger.warning(
                                            f"Skipping invalid entry during reconcile: {path}"
                                        )
                                        continue
                                    model_data = validation_result.entry
                                    if model_data is None:
                                        continue

                                    self._ensure_license_flags(model_data)
                                    # Add to cache
                                    self._cache.raw_data.append(model_data)
                                    self._cache.add_to_version_index(model_data)

                                    # Update hash index if available
                                    if 'sha256' in model_data and 'file_path' in model_data:
                                        self._hash_index.add_entry(
                                            model_data['sha256'].lower(),
                                            model_data['file_path'],
                                            model_data.get('autov3') or None
                                        )
                                    
                                    # Update tags count
                                    if 'tags' in model_data and model_data['tags']:
                                        for tag in model_data['tags']:
                                            self._tags_count[tag] = self._tags_count.get(tag, 0) + 1
                                            
                                    total_added += 1
                            else:
                                logger.error(f"Could not determine root path for {path}")
                        except Exception as e:
                            logger.error(f"Error adding {path} to cache: {e}")
                        
                        if self.is_cancelled():
                            logger.info(f"{self.model_type.capitalize()} Scanner: Reconcile processing cancelled")
                            return
            
            # Find missing files (in cache but not in filesystem)
            missing_files = cached_paths - found_paths
            total_removed = 0
            
            if missing_files:
                logger.info(f"{self.model_type.capitalize()} Scanner: Found {len(missing_files)} files to remove from cache")
                
                # Process files to remove
                for path in missing_files:
                    try:
                        model_to_remove = path_to_item[path]

                        self._cache.remove_from_version_index(model_to_remove)

                        # Update tags count
                        for tag in model_to_remove.get('tags', []):
                            if tag in self._tags_count:
                                self._tags_count[tag] = max(0, self._tags_count[tag] - 1)
                                if self._tags_count[tag] == 0:
                                    del self._tags_count[tag]
                        
                        # Remove from hash index
                        self._hash_index.remove_by_path(path)
                        total_removed += 1
                    except Exception as e:
                        logger.error(f"Error removing {path} from cache: {e}")
                
                # Update cache data
                self._cache.raw_data = [item for item in self._cache.raw_data if item['file_path'] not in missing_files]
            
            dedup_removed = 0
            seen_paths: set[str] = set()
            deduped: list[Dict[str, Any]] = []
            for item in reversed(self._cache.raw_data):
                path = item.get('file_path', '')
                if path not in seen_paths:
                    seen_paths.add(path)
                    deduped.append(item)
                else:
                    for tag in item.get('tags', []):
                        if tag in self._tags_count:
                            self._tags_count[tag] = max(0, self._tags_count[tag] - 1)
                            if self._tags_count[tag] == 0:
                                del self._tags_count[tag]
                    dedup_removed += 1
            if dedup_removed > 0:
                self._cache.raw_data = list(reversed(deduped))
                total_removed += dedup_removed
            
            # Resort cache if changes were made
            if total_added > 0 or total_removed > 0:
                # Update folders list
                all_folders = set(item.get('folder', '') for item in self._cache.raw_data)
                self._cache.folders = sorted(list(all_folders), key=lambda x: x.lower())

                self._cache.rebuild_version_index()

                # Resort cache
                await self._cache.resort()

                await self._persist_current_cache()
                
            logger.info(f"{self.model_type.capitalize()} Scanner: Cache reconciliation completed in {time.time() - start_time:.2f} seconds. Added {total_added}, removed {total_removed} models.")
        except Exception as e:
            logger.error(f"{self.model_type.capitalize()} Scanner: Error reconciling cache: {e}", exc_info=True)
        finally:
            self._is_initializing = False # Unset flag
            self.bump_cache_version()
    
    def is_initializing(self) -> bool:
        """Check if the scanner is currently initializing"""
        return self._is_initializing
    
    def cancel_task(self) -> None:
        """Request cancellation of the current long-running task."""
        self._cancel_requested = True
        logger.info(f"{self.model_type.capitalize()} Scanner: Cancellation requested")

    def reset_cancellation(self) -> None:
        """Reset the cancellation flag."""
        self._cancel_requested = False

    def is_cancelled(self) -> bool:
        """Check if cancellation has been requested."""
        return self._cancel_requested
    
    def get_model_roots(self) -> List[str]:
        """Get model root directories"""
        raise NotImplementedError("Subclasses must implement get_model_roots")

    async def get_all_folders(self) -> List[str]:
        """Enumerate every directory under the model roots, live from disk.

        Unlike the models-only ``cache.folders``, this includes empty
        directories, so it stays accurate even when the in-memory cache was
        hydrated from a persisted snapshot without a filesystem walk. Hidden
        directories (any segment starting with '.') and the pending-delete
        staging dir are excluded. The result is unioned with the model-derived
        folders so it is always a superset of ``cache.folders``, and cached
        for ``ALL_FOLDERS_CACHE_TTL_SECONDS`` to avoid repeated walks.
        """
        now = time.monotonic()
        if self._all_folders_ttl_cache is not None:
            cached_at, cached_folders = self._all_folders_ttl_cache
            if now - cached_at < ALL_FOLDERS_CACHE_TTL_SECONDS:
                return cached_folders

        discovered: Set[str] = set()
        visited_real_paths: Set[str] = set()

        for root_path in self.get_model_roots():
            if not os.path.exists(root_path):
                continue

            for root, dirnames, _files in os.walk(root_path, followlinks=True):
                dirnames[:] = [d for d in dirnames if not _is_excluded_dir(d)]
                # realpath is used only for symlink dedup, never for the
                # recorded path (business paths stay unresolved).
                real_root = os.path.realpath(root)
                if real_root in visited_real_paths:
                    continue
                visited_real_paths.add(real_root)

                rel_dir = os.path.relpath(os.path.abspath(root), os.path.abspath(root_path))
                rel_dir = rel_dir.replace(os.path.sep, "/")
                if rel_dir != "." and not _is_hidden_relative_path(rel_dir):
                    discovered.add(rel_dir)

        folders = set(discovered)
        if self._cache is not None:
            folders |= {item.get('folder', '') for item in self._cache.raw_data}

        result = sorted(folders, key=lambda x: x.lower())
        self._all_folders_ttl_cache = (now, result)
        return result

    def invalidate_all_folders_cache(self) -> None:
        """Drop the cached get_all_folders() result (e.g. after a move)."""
        self._all_folders_ttl_cache = None
    
    async def _create_default_metadata(self, file_path: str) -> Optional[BaseModelMetadata]:
        """Get model file info and metadata (extensible for different model types)"""
        return await MetadataManager.create_default_metadata(file_path, self.model_class)
    
    def _calculate_folder(self, file_path: str) -> str:
        """Calculate the folder path for a model file"""
        for root in self.get_model_roots():
            if file_path.startswith(root):
                rel_path = os.path.relpath(file_path, root)
                return os.path.dirname(rel_path).replace(os.path.sep, '/')
        return ''

    def adjust_metadata(self, metadata, file_path, root_path):
        """Hook for subclasses: adjust metadata during scanning"""
        return metadata

    def adjust_cached_entry(self, entry: Dict[str, Any]) -> Dict[str, Any]:
        """Hook for subclasses: adjust entries loaded from the persisted cache."""
        return entry

    @staticmethod
    def _normalize_path_value(path: Optional[str]) -> str:
        if not path:
            return ''

        normalized = os.path.normpath(path)
        if normalized == '.':
            return ''

        return normalized.replace('\\', '/')

    def _find_root_for_file(self, file_path: Optional[str]) -> Optional[str]:
        """Return the configured root directory that contains ``file_path``."""

        normalized_path = self._normalize_path_value(file_path)
        if not normalized_path:
            return None

        for root in self.get_model_roots() or []:
            normalized_root = self._normalize_path_value(root)
            if not normalized_root:
                continue

            if (
                normalized_path == normalized_root
                or normalized_path.startswith(f"{normalized_root}/")
            ):
                return root

        return None

    async def _process_model_file(
        self,
        file_path: str,
        root_path: str,
        *,
        hash_index: Optional[ModelHashIndex] = None,
        excluded_models: Optional[List[str]] = None
    ) -> Optional[Dict[str, Any]]:
        """Process a single model file and return its metadata"""
        hash_index = hash_index or self._hash_index
        excluded_models = excluded_models if excluded_models is not None else self._excluded_models

        # Belt-and-braces: staged files must never become library entries even
        # if a caller invokes this method directly with a staging path.
        if _is_pending_delete_path(file_path):
            return None

        metadata, should_skip = await MetadataManager.load_metadata(file_path, self.model_class)
    
        if should_skip:
            # Metadata file exists but cannot be parsed - skip this model
            logger.warning(f"Skipping model {file_path} due to corrupted metadata file")
            return None
        
        if metadata is None:
            civitai_info_path = f"{os.path.splitext(file_path)[0]}.civitai.info"
            if os.path.exists(civitai_info_path):
                try:
                    with open(civitai_info_path, 'r', encoding='utf-8') as f:
                        version_info = json.load(f)
                    
                    file_info = next((f for f in version_info.get('files', []) if f.get('primary')), None)
                    if file_info:
                        file_name = os.path.splitext(os.path.basename(file_path))[0]
                        file_info['name'] = file_name
                    
                        metadata = cast(Any, self.model_class).from_civitai_info(version_info, file_info, file_path)
                        metadata.preview_url = find_preview_file(file_name, os.path.dirname(file_path))
                        await MetadataManager.save_metadata(file_path, metadata)
                        logger.info(f"Created metadata from .civitai.info for {file_path} (Reason: .civitai.info was found but .metadata.json was missing)")
                except Exception as e:
                    logger.error(f"Error creating metadata from .civitai.info for {file_path}: {e}")
        else:
            # Check if metadata exists but civitai field is empty - try to restore from civitai.info
            if metadata.civitai is None or metadata.civitai == {}:
                civitai_info_path = f"{os.path.splitext(file_path)[0]}.civitai.info"
                if os.path.exists(civitai_info_path):
                    try:
                        with open(civitai_info_path, 'r', encoding='utf-8') as f:
                            version_info = json.load(f)
                        
                        logger.debug(f"Restoring missing civitai data from .civitai.info for {file_path}")
                        metadata.civitai = version_info
                        
                        # Ensure tags are also updated if they're missing
                        if (not metadata.tags or len(metadata.tags) == 0) and 'model' in version_info:
                            if 'tags' in version_info['model']:
                                metadata.tags = version_info['model']['tags']
                        
                        # Also restore description if missing
                        if (not metadata.modelDescription or metadata.modelDescription == "") and 'model' in version_info:
                            if 'description' in version_info['model']:
                                metadata.modelDescription = version_info['model']['description']
                        
                        # Save the updated metadata
                        await MetadataManager.save_metadata(file_path, metadata)
                        logger.debug(f"Updated metadata with civitai info for {file_path}")
                    except Exception as e:
                        logger.error(f"Error restoring civitai data from .civitai.info for {file_path}: {e}")
            
        if metadata is None:
            metadata = await self._create_default_metadata(file_path)
        
        assert metadata is not None

        # Hook: allow subclasses to adjust metadata
        metadata = self.adjust_metadata(metadata, file_path, root_path)
        
        rel_path = os.path.relpath(file_path, root_path)
        folder = os.path.dirname(rel_path)
        normalized_folder = folder.replace(os.path.sep, '/')

        model_data = self._build_cache_entry(metadata, folder=normalized_folder)

        # Compute SHA256 hash when metadata provided none (e.g., CivitAI API response has empty hashes).
        # Respect hash_status='pending' (set by CheckpointScanner for large models) to defer
        # hash calculation until on-demand — avoids reading entire checkpoint files at startup.
        hash_status = model_data.get('hash_status', '')
        if not model_data.get('sha256') and hash_status != 'pending' and file_path:
            try:
                logger.info(f"Computing SHA256 hash for {file_path} (was empty from metadata)")
                sha256 = await calculate_sha256(file_path)
                if sha256:
                    model_data['sha256'] = sha256.lower()
                    if isinstance(metadata, BaseModelMetadata):
                        metadata.sha256 = sha256.lower()
                    await MetadataManager.save_metadata(file_path, metadata)
            except Exception as e:
                logger.error(f"Failed to compute SHA256 for {file_path}: {e}")

        # AutoV3 resolution: prefer the Civitai AutoV3 reported for the file
        # whose SHA256 matches (authoritative for recipe matching), falling
        # back to the embedded safetensors header hash only for models never
        # checked before (autov3 is None). A checked-unavailable state ('')
        # is only upgraded by Civitai data — the header is never re-read.
        current_autov3 = model_data.get('autov3')
        if current_autov3 in (None, ''):
            try:
                civitai_data = None
                if isinstance(metadata, BaseModelMetadata):
                    civitai_data = metadata.civitai
                elif isinstance(metadata, dict):
                    civitai_data = metadata.get("civitai")
                autov3 = autov3_from_civitai_files(
                    civitai_data, model_data.get("sha256") or ""
                ) or ""
                if not autov3 and current_autov3 is None:
                    autov3 = (calculate_autov3(os.path.realpath(file_path)) or '').lower()
                if autov3 != current_autov3:
                    model_data['autov3'] = autov3
                    if isinstance(metadata, BaseModelMetadata):
                        metadata.autov3 = autov3
                        await MetadataManager.save_metadata(file_path, metadata)
                    elif isinstance(metadata, dict):
                        # Dict payload: JSON null encodes the checked-unavailable state.
                        metadata['autov3'] = autov3 or None
                        await MetadataManager.save_metadata(file_path, metadata)
            except Exception as e:
                logger.error(f"Failed to resolve AutoV3 for {file_path}: {e}")

        # Skip excluded models
        if model_data.get('exclude', False):
            excluded_models.append(model_data['file_path'])
            return None

        return model_data

    async def _apply_scan_result(self, scan_result: CacheBuildResult) -> None:
        """Apply scan results to the cache and associated indexes."""

        if scan_result is None:
            return

        if self.is_cancelled():
            logger.info(
                f"{self.model_type.capitalize()} Scanner: Skipping _apply_scan_result "
                "after cancellation"
            )
            return

        self._hash_index = scan_result.hash_index
        self._tags_count = dict(scan_result.tags_count)
        self._excluded_models = list(scan_result.excluded_models)

        if self._cache is None:
            self._cache = ModelCache(
                raw_data=list(scan_result.raw_data),
                folders=[],
                name_display_mode=self._name_display_mode,
            )
        else:
            self._cache.raw_data = list(scan_result.raw_data)

        # resort() rebuilds folders and the version index on every path, so a
        # separate rebuild_version_index() call here would be redundant.
        await self._cache.resort()

        self._log_duplicate_filename_summary()

        self.bump_cache_version()

    def _log_duplicate_filename_summary(self) -> None:
        """Log a batched summary of duplicate filename conflicts once per scan."""
        # Duplicate filename detection is only relevant for LoRAs, which use
        # basename-only syntax (<lora:name:strength>). Checkpoints and embeddings
        # use full relative paths for resolution, so conflicts are not ambiguous.
        if self._hash_index is None or self.model_type != "lora":
            return

        # When full path syntax is active, duplicate filenames across subfolders
        # are fully qualified, so there is no ambiguity — skip the warning.
        if get_settings_manager().get("lora_syntax_format", "legacy") == "full":
            return

        duplicates = self._hash_index.get_duplicate_filenames()
        if not duplicates:
            return

        total_files = sum(len(paths) for paths in duplicates.values())
        conflict_count = len(duplicates)
        model_type_label = self.model_type or "model"

        logger.warning(
            "Duplicate filename conflict detected: %d %s filename(s) "
            "are shared by %d files total, causing ambiguity in %s resolution. "
            "Open the Doctor panel to resolve one-click.",
            conflict_count,
            model_type_label,
            total_files,
            model_type_label.capitalize(),
        )

    async def _sync_download_history(
        self,
        raw_data: Sequence[Mapping[str, Any]],
        *,
        source: str,
    ) -> None:
        records: List[Dict[str, Any]] = []
        for item in raw_data or []:
            if not isinstance(item, Mapping):
                continue
            civitai = item.get('civitai')
            if not isinstance(civitai, Mapping):
                continue

            version_id = civitai.get('id')
            if version_id in (None, ''):
                continue

            records.append(
                {
                    'version_id': version_id,
                    'model_id': civitai.get('modelId'),
                    'file_path': item.get('file_path'),
                }
            )

        if not records:
            return

        try:
            history_service = await ServiceRegistry.get_downloaded_version_history_service()
            await history_service.mark_downloaded_bulk(
                self.model_type,
                records,
                source=source,
            )
        except Exception as exc:
            logger.debug(
                "%s Scanner: Failed to sync download history: %s",
                self.model_type.capitalize(),
                exc,
            )

    async def _gather_model_data(
        self,
        *,
        total_files: int = 0,
        progress_callback: Optional[Callable[[int, int], Awaitable[None]]] = None
    ) -> CacheBuildResult:
        """Collect metadata for all model files."""

        raw_data: List[Dict[str, Any]] = []
        hash_index = ModelHashIndex()
        tags_count: Dict[str, int] = {}
        excluded_models: List[str] = []
        processed_files = 0
        processed_real_files: Set[str] = set()
        visited_real_dirs: Set[str] = set()

        async def handle_progress() -> None:
            if progress_callback is None:
                return
            try:
                await progress_callback(processed_files, total_files)
            except Exception as exc:  # pragma: no cover - defensive logging
                logger.error(f"Error reporting progress for {self.model_type}: {exc}")

        self.reset_cancellation()

        async def scan_recursive(current_path: str, root_path: str, visited_paths: Set[str]) -> None:
            nonlocal processed_files

            try:
                real_path = os.path.realpath(current_path)
                if real_path in visited_paths or real_path in visited_real_dirs:
                    return
                visited_paths.add(real_path)
                visited_real_dirs.add(real_path)

                with os.scandir(current_path) as iterator:
                    entries = list(iterator)

                for entry in entries:
                    try:
                        if entry.is_file(follow_symlinks=True):
                            ext = os.path.splitext(entry.name)[1].lower()
                            if ext not in self.file_extensions:
                                continue

                            file_path = entry.path.replace(os.sep, "/")
                            real_file_path = os.path.realpath(entry.path)
                            if real_file_path in processed_real_files:
                                continue

                            processed_real_files.add(real_file_path)
                            result = await self._process_model_file(
                                file_path,
                                root_path,
                                hash_index=hash_index,
                                excluded_models=excluded_models
                            )

                            processed_files += 1

                            if result:
                                # Validate the entry before adding
                                validation_result = CacheEntryValidator.validate(
                                    result, auto_repair=True
                                )
                                if not validation_result.is_valid:
                                    logger.warning(
                                        f"Skipping invalid scan result: {file_path}"
                                    )
                                    continue
                                result = validation_result.entry
                                if result is None:
                                    continue

                                self._ensure_license_flags(result)
                                raw_data.append(result)

                                sha_value = result.get('sha256')
                                model_path = result.get('file_path')
                                if sha_value and model_path:
                                    hash_index.add_entry(sha_value.lower(), model_path, result.get('autov3') or None)

                                for tag in result.get('tags') or []:
                                    tags_count[tag] = tags_count.get(tag, 0) + 1

                            await handle_progress()
                            await asyncio.sleep(0)
                            if self.is_cancelled():
                                return
                        elif entry.is_dir(follow_symlinks=True):
                            if _is_excluded_dir(entry.name):
                                continue
                            await scan_recursive(entry.path, root_path, visited_paths)
                    except Exception as entry_error:
                        logger.error(f"Error processing entry {entry.path}: {entry_error}")
            except Exception as scan_error:
                logger.error(f"Error scanning {current_path}: {scan_error}")

            if self.is_cancelled():
                return

        for model_root in self.get_model_roots():
            if not os.path.exists(model_root):
                continue

            await scan_recursive(model_root, model_root, set())

        return CacheBuildResult(
            raw_data=raw_data,
            hash_index=hash_index,
            tags_count=tags_count,
            excluded_models=excluded_models
        )

    async def add_model_to_cache(self, metadata_dict: Dict[str, Any], folder: str = '') -> bool:
        """Add a model to the cache

        Args:
            metadata_dict: The model metadata dictionary
            folder: The relative folder path for the model
            
        Returns:
            bool: True if successful, False otherwise
        """
        try:
            if self._cache is None:
                await self.get_cached_data()
            assert self._cache is not None

            # Update folder in metadata
            metadata_dict['folder'] = folder
            
            file_path = metadata_dict.get('file_path', '')
            if file_path:
                old_entries = [item for item in self._cache.raw_data if item.get('file_path') == file_path]
                for old_entry in old_entries:
                    for tag in old_entry.get('tags', []):
                        if tag in self._tags_count:
                            self._tags_count[tag] = max(0, self._tags_count[tag] - 1)
                            if self._tags_count[tag] == 0:
                                del self._tags_count[tag]
                self._hash_index.remove_by_path(file_path)
                self._cache.raw_data = [item for item in self._cache.raw_data if item.get('file_path') != file_path]

            for tag in metadata_dict.get('tags', []):
                self._tags_count[tag] = self._tags_count.get(tag, 0) + 1

            self._cache.raw_data.append(metadata_dict)

            await self._cache.resort()
            
            # Update the hash index
            self._hash_index.add_entry(
                metadata_dict['sha256'],
                metadata_dict['file_path'],
                metadata_dict.get('autov3') or None,
            )
            await self._persist_current_cache()
            self.bump_cache_version()
            return True
        except Exception as e:
            logger.error(f"Error adding model to cache: {e}")
            return False
    
    async def move_model(self, source_path: str, target_path: str) -> Optional[Dict[str, Any]]:
        """Move a model and its associated files to a new location
        
        Args:
            source_path: Original file path
            target_path: Target directory path
            
        Returns:
            Optional[str]: New file path if successful, None if failed
        """
        try:
            source_path = source_path.replace(os.sep, '/')
            target_path = target_path.replace(os.sep, '/')
            
            file_ext = os.path.splitext(source_path)[1]
            
            if not file_ext or file_ext.lower() not in self.file_extensions:
                logger.error(f"Invalid file extension for model: {file_ext}")
                return None
                
            base_name = os.path.splitext(os.path.basename(source_path))[0]
            source_dir = os.path.dirname(source_path)

            _require_path_in_library_roots(source_path, self, label="Source path")
            _require_path_in_library_roots(target_path, self, label="Target path")
            
            os.makedirs(target_path, exist_ok=True)
            
            def get_source_hash():
                return self.get_hash_by_path(source_path)
            
            # Check for filename conflicts and auto-rename if necessary
            from ..utils.models import BaseModelMetadata
            final_filename = BaseModelMetadata.generate_unique_filename(
                target_path, base_name, file_ext, lambda: get_source_hash() or ""
            )
            
            target_file = os.path.join(target_path, final_filename).replace(os.sep, '/')
            final_base_name = os.path.splitext(final_filename)[0]
            
            # Log if filename was changed due to conflict
            if final_filename != f"{base_name}{file_ext}":
                logger.info(f"Renamed {base_name}{file_ext} to {final_filename} to avoid filename conflict")

            real_source = os.path.realpath(source_path)
            real_target = os.path.realpath(target_file)
            
            shutil.move(real_source, real_target)
            
            # Move all associated files with the same base name
            source_metadata = None
            moved_metadata_path = None
            
            # Find all files with the same base name in the source directory
            files_to_move = []
            try:
                for file in os.listdir(source_dir):
                    if file.startswith(base_name + ".") and file != os.path.basename(source_path):
                        source_file_path = os.path.join(source_dir, file)
                        # Generate new filename with the same base name as the model file
                        file_suffix = file[len(base_name):]  # Get the part after base_name (e.g., ".metadata.json", ".preview.png")
                        new_associated_filename = f"{final_base_name}{file_suffix}"
                        target_associated_path = os.path.join(target_path, new_associated_filename)
                        
                        # Store metadata file path for special handling
                        if file == f"{base_name}.metadata.json":
                            source_metadata = source_file_path
                            moved_metadata_path = target_associated_path
                        else:
                            files_to_move.append((source_file_path, target_associated_path))
            except Exception as e:
                logger.error(f"Error listing files in {source_dir}: {e}")
            
            # Move all associated files
            metadata = None
            for source_file, target_file_path in files_to_move:
                try:
                    shutil.move(source_file, target_file_path)
                except Exception as e:
                    logger.error(f"Error moving associated file {source_file}: {e}")
            
            # Handle metadata file specially to update paths
            if source_metadata and moved_metadata_path and os.path.exists(source_metadata):
                try:
                    shutil.move(source_metadata, moved_metadata_path)
                    metadata = await self._update_metadata_paths(moved_metadata_path, target_file)
                except Exception as e:
                    logger.error(f"Error moving metadata file: {e}")
            
            update_result = await self.update_single_model_cache(source_path, target_file, metadata, recalculate_type=True)
            
            return {
                "new_path": target_file,
                "cache_entry": update_result if isinstance(update_result, dict) else None
            }
            
        except Exception as e:
            logger.error(f"Error moving model: {e}", exc_info=True)
            return None
    
    async def _update_metadata_paths(self, metadata_path: str, model_path: str) -> Optional[Dict[str, Any]]:
        """Update file paths in metadata file"""
        try:
            with open(metadata_path, 'r', encoding='utf-8') as f:
                metadata = json.load(f)
            
            metadata['file_path'] = model_path.replace(os.sep, '/')
            # Update file_name to match the new filename
            metadata['file_name'] = os.path.splitext(os.path.basename(model_path))[0]
            
            if 'preview_url' in metadata and metadata['preview_url']:
                preview_dir = os.path.dirname(model_path)
                # Update preview filename to match the new base name
                new_base_name = os.path.splitext(os.path.basename(model_path))[0]
                preview_ext = get_preview_extension(metadata['preview_url'])
                new_preview_path = os.path.join(preview_dir, f"{new_base_name}{preview_ext}")
                metadata['preview_url'] = new_preview_path.replace(os.sep, '/')
            
            await MetadataManager.save_metadata(metadata_path, metadata)

            return metadata
                
        except Exception as e:
            logger.error(f"Error updating metadata paths: {e}", exc_info=True)
            return None

    async def update_single_model_cache(self, original_path: str, new_path: str, metadata: Optional[Dict[str, Any]], recalculate_type: bool = False) -> Union[bool, Dict[str, Any]]:
        """Update cache after a model has been moved or modified"""
        cache = await self.get_cached_data()

        existing_item = next((item for item in cache.raw_data if item['file_path'] == original_path), None)
        if existing_item:
            cache.remove_from_version_index(existing_item)

        if existing_item and 'tags' in existing_item:
            for tag in existing_item.get('tags', []):
                if tag in self._tags_count:
                    self._tags_count[tag] = max(0, self._tags_count[tag] - 1)
                    if self._tags_count[tag] == 0:
                        del self._tags_count[tag]
        
        self._hash_index.remove_by_path(original_path)
        
        cache.raw_data = [
            item for item in cache.raw_data
            if item['file_path'] != original_path
        ]

        cache_modified = bool(existing_item) or bool(metadata)
        cache_entry: Optional[Dict[str, Any]] = None

        if metadata:
            normalized_new_path = new_path.replace(os.sep, '/')
            if original_path == new_path and existing_item:
                folder_value = existing_item.get('folder', self._calculate_folder(new_path))
            else:
                folder_value = self._calculate_folder(new_path)

            cache_entry = self._build_cache_entry(
                metadata,
                folder=folder_value,
                file_path_override=normalized_new_path,
            )

            # Ensure sha256 is populated even when metadata doesn't have it
            if not cache_entry.get('sha256') and normalized_new_path and os.path.exists(normalized_new_path):
                try:
                    sha256 = await calculate_sha256(normalized_new_path)
                    if sha256:
                        cache_entry['sha256'] = sha256.lower()
                except Exception as e:
                    logger.error(f"Failed to compute SHA256 for {normalized_new_path}: {e}")

            if recalculate_type:
                cache_entry = self.adjust_cached_entry(cache_entry)

            cache.raw_data.append(cache_entry)
            cache.add_to_version_index(cache_entry)

            sha_value = cache_entry.get('sha256')
            if sha_value:
                self._hash_index.add_entry(
                    sha_value.lower(),
                    normalized_new_path,
                    cache_entry.get('autov3') or None,
                )

            all_folders = set(item['folder'] for item in cache.raw_data)
            cache.folders = sorted(list(all_folders), key=lambda x: x.lower())

            for tag in cache_entry.get('tags', []):
                self._tags_count[tag] = self._tags_count.get(tag, 0) + 1

        cache.rebuild_version_index()

        await cache.resort()

        # A move may have created new directories; drop the cached live-walk
        # result so the next include_empty request sees them.
        self.invalidate_all_folders_cache()

        if cache_modified:
            await self._persist_current_cache()
            self.bump_cache_version()

        if metadata and cache_entry is not None:
            return cache_entry
        return True
        
    async def sync_cache_from_metadata(
        self, file_path: str, metadata_dict: Dict[str, Any]
    ) -> bool:
        """Opportunistically sync in-memory and persistent caches from metadata.

        Builds a prospective cache entry from *metadata_dict* (deserialized
        ``.metadata.json`` content) and compares it against the current cache
        entry.  When the two are already identical this method returns
        ``False`` without touching anything — avoiding the overhead of
        ``update_single_model_cache``, which always removes and re-inserts
        the entry, triggers a full resort, and persists via the heavyweight
        ``save_cache()``.

        When differences are detected the update is applied **in-place** with
        targeted operations:

        * The existing ``raw_data`` entry is modified rather than removed and
          re-appended (O(1) instead of O(n)).
        * Tag counts and the hash index are updated incrementally.
        * The version index is rebuilt only for the affected entry.
        * ``resort()`` is called **only** when a sort-relevant field changed
          (``model_name`` / ``file_name`` for name-sort, ``modified`` for
          date-sort, ``size`` for size-sort).
        * The persistent (SQLite) cache receives a targeted single-row update
          via :meth:`PersistentModelCache.update_single_model` rather than a
          full-table ``save_cache()``.

        Returns:
            ``True`` if any cache update was performed, ``False`` if the
            caches were already in sync.

        .. note::

            This is a **best-effort** operation.  Failures are logged but
            never propagated — callers should fire-and-forget via
            :func:`asyncio.create_task`.
        """
        try:
            return await self._sync_cache_from_metadata_impl(
                file_path, metadata_dict
            )
        except Exception:
            logger.warning(
                "sync_cache_from_metadata failed for %s",
                file_path,
                exc_info=True,
            )
            return False

    async def _sync_cache_from_metadata_impl(
        self, file_path: str, metadata_dict: Dict[str, Any]
    ) -> bool:
        cache = await self.get_cached_data()

        # Locate the existing cache entry -----------------------------------
        existing_idx: Optional[int] = None
        existing_entry: Optional[Dict[str, Any]] = None
        for i, item in enumerate(cache.raw_data):
            if item.get("file_path") == file_path:
                existing_entry = item
                existing_idx = i
                break

        # Build the desired entry from metadata ------------------------------
        folder_value = (
            existing_entry.get("folder", "")
            if existing_entry
            else self._calculate_folder(file_path)
        )
        desired_entry = self._build_cache_entry(
            metadata_dict,
            folder=folder_value,
            file_path_override=file_path,
        )

        # Ensure sha256 is populated (defensive — metadata should have it)
        if (
            not desired_entry.get("sha256")
            and file_path
            and os.path.exists(file_path)
        ):
            try:
                sha256 = await calculate_sha256(file_path)
                if sha256:
                    desired_entry["sha256"] = sha256.lower()
            except Exception:
                pass

        # Not in cache at all — delegate to the full update path ------------
        if existing_entry is None:
            result = await self.update_single_model_cache(
                file_path, file_path, metadata_dict
            )
            return bool(result)

        # Compare — skip everything if already in sync -----------------------
        if not self._cache_entries_differ(existing_entry, desired_entry):
            return False

        # Re-validate: the cache may have been replaced concurrently
        # (e.g. by _apply_scan_result).  Use identity check, not equality,
        # so we detect when the raw_data list was swapped out from under us.
        if self._cache is None or not any(
            item is existing_entry for item in self._cache.raw_data
        ):
            return False

        # ---- Differences detected: apply targeted, in-place updates --------

        # Snapshot old values for delta computations
        old_tags = list(existing_entry.get("tags") or [])
        old_sha256: str = existing_entry.get("sha256", "") or ""
        old_model_name: str = existing_entry.get("model_name", "") or ""
        old_file_name: str = existing_entry.get("file_name", "") or ""
        old_modified: float = float(existing_entry.get("modified", 0.0) or 0.0)
        old_size: int = int(existing_entry.get("size", 0) or 0)
        old_civitai = existing_entry.get("civitai")

        # ---- In-place update of the cache entry ----
        existing_entry.clear()
        existing_entry.update(desired_entry)
        self.bump_cache_version()

        # ---- Incremental tag count update ----
        new_tags: set[str] = set(desired_entry.get("tags") or [])
        old_tag_set: set[str] = set(old_tags)
        for tag in old_tag_set - new_tags:
            current = self._tags_count.get(tag, 0)
            if current <= 1:
                self._tags_count.pop(tag, None)
            else:
                self._tags_count[tag] = current - 1
        for tag in new_tags - old_tag_set:
            self._tags_count[tag] = self._tags_count.get(tag, 0) + 1

        # ---- Incremental hash index update ----
        new_sha = (desired_entry.get("sha256", "") or "").lower()
        old_sha = (old_sha256 or "").lower()
        if new_sha != old_sha:
            if old_sha:
                self._hash_index.remove_by_path(file_path)
            if new_sha:
                self._hash_index.add_entry(
                    new_sha,
                    file_path,
                    desired_entry.get('autov3') or None,
                )

        # ---- Incremental version index update ----
        new_civitai = desired_entry.get("civitai")
        if old_civitai != new_civitai:
            temp_old = {
                "file_path": file_path,
                "file_name": old_file_name,
                "civitai": old_civitai,
            }
            cache.remove_from_version_index(temp_old)
            cache.add_to_version_index(existing_entry)

        # ---- Conditional resort (only when sort-key fields changed) ----
        need_resort = False
        _last = cache._last_sort
        sort_key: Optional[str] = _last[0] if _last[0] is not None else None
        if sort_key == "name":
            if (
                old_model_name != desired_entry.get("model_name", "")
                or old_file_name != desired_entry.get("file_name", "")
            ):
                need_resort = True
        elif sort_key == "date":
            if old_modified != float(desired_entry.get("modified", 0.0) or 0.0):
                need_resort = True
        elif sort_key == "size":
            if old_size != int(desired_entry.get("size", 0) or 0):
                need_resort = True

        if need_resort:
            await cache.resort()

        # ---- Targeted SQL update (single row, not full save_cache) ----
        persistent = getattr(self, "_persistent_cache", None)
        if persistent is not None:
            old_item_for_sql: Dict[str, Any] = {
                "file_path": file_path,
                "tags": old_tags,
                "sha256": old_sha256,
            }
            await asyncio.get_event_loop().run_in_executor(
                None,
                persistent.update_single_model,
                self.model_type,
                desired_entry,
                old_item_for_sql,
            )

        return True

    async def update_autov3_for_model(self, model_type: str, file_path: str, autov3: str) -> bool:
        """Persist an AutoV3 hash for a single model (single write path used by the backfill service).

        Locates the in-memory cache entry by ``file_path`` and updates only its
        ``autov3`` field: the in-memory hash index, the SQLite snapshot via
        :meth:`PersistentModelCache.update_single_model`, and the
        ``.metadata.json`` sidecar. sha256, tags, and every other field are
        left untouched, so the persistent delta only ever differs in autov3.

        Returns:
            ``True`` when the entry was found and updated, ``False`` otherwise.
            Never raises — failures are logged and swallowed.
        """
        try:
            if self._cache is None:
                return False

            entry = next(
                (item for item in self._cache.raw_data if item.get('file_path') == file_path),
                None,
            )
            if entry is None:
                return False

            # Normalize once so the memory entry, sidecar, and SQLite row agree.
            autov3 = (autov3 or "").lower()

            # Capture the pre-mutation state so update_single_model only sees
            # an autov3 delta between old and new.
            old_item = dict(entry)

            entry['autov3'] = autov3 or ''

            # Prefer add_entry when a sha256 is known so the sha256 and autov3
            # maps stay in sync; fall back to an autov3-only registration.
            sha_value = entry.get('sha256')
            checked_autov3 = entry.get('autov3') or None
            if sha_value:
                self._hash_index.add_entry(sha_value.lower(), file_path, checked_autov3)
            elif checked_autov3:
                self._hash_index.add_autov3(checked_autov3, file_path)

            persistent = getattr(self, '_persistent_cache', None)
            if persistent is not None:
                await asyncio.get_event_loop().run_in_executor(
                    None,
                    persistent.update_single_model,
                    model_type,
                    entry,
                    old_item,
                )

            # Sidecar write-back: JSON null encodes the checked-unavailable
            # state. Skip silently when the sidecar does not exist.
            metadata_path = f"{os.path.splitext(file_path)[0]}.metadata.json"
            if os.path.exists(metadata_path):
                with open(metadata_path, 'r', encoding='utf-8') as handle:
                    payload = json.load(handle)
                if not isinstance(payload, dict):
                    payload = {}
                payload['autov3'] = entry['autov3'] or None
                await MetadataManager.save_metadata(metadata_path, payload)

            self.bump_cache_version()
            return True
        except Exception as exc:
            logger.warning("Failed to update AutoV3 for %s: %s", file_path, exc)
            return False

    @staticmethod
    def _cache_entries_differ(a: Dict[str, Any], b: Dict[str, Any]) -> bool:
        """Return ``True`` when two cache-entry dicts differ in any field.

        Tag lists are compared order-insensitively; all other keys use
        standard equality.
        """
        a_tags = sorted(a.get("tags") or [])
        b_tags = sorted(b.get("tags") or [])
        if a_tags != b_tags:
            return True

        all_keys = set(a.keys()) | set(b.keys())
        for key in all_keys:
            if key == "tags":
                continue
            if a.get(key) != b.get(key):
                return True
        return False

    def has_hash(self, sha256: str) -> bool:
        """Check if a model with given hash exists"""
        return self._hash_index.has_hash(sha256.lower())
        
    def get_path_by_hash(self, sha256: str) -> Optional[str]:
        """Get file path for a model by its hash"""
        return self._hash_index.get_path(sha256.lower())
        
    def get_hash_by_path(self, file_path: str) -> Optional[str]:
        """Get hash for a model by its file path"""
        if self._cache is None or not self._cache.raw_data:
            return None
            
        # Iterate through cache data to find matching file path
        for model_data in self._cache.raw_data:
            if model_data.get('file_path') == file_path:
                return model_data.get('sha256')
        
        return None
    
    def get_hash_by_filename(self, filename: str) -> Optional[str]:
        """Get hash for a model by its filename without path"""
        return self._hash_index.get_hash_by_filename(filename)

    # TODO: Adjust this method to use metadata instead of finding the file    
    def get_preview_url_by_hash(self, sha256: str) -> Optional[str]:
        """Get preview static URL for a model by its hash"""
        file_path = self._hash_index.get_path(sha256.lower())
        if not file_path:
            return None

        dir_path = os.path.dirname(file_path)
        base_name = os.path.splitext(os.path.basename(file_path))[0]
        preview_path = find_preview_file(base_name, dir_path)
        if preview_path:
            return config.get_preview_static_url(preview_path)

        return None
        
    async def get_top_tags(self, limit: int = 20) -> List[Dict[str, Any]]:
        """Get top tags sorted by count. If limit is 0, return all tags."""
        await self.get_cached_data()
        
        sorted_tags = sorted(
            [{"tag": tag, "count": count} for tag, count in self._tags_count.items()],
            key=lambda x: x['count'],
            reverse=True
        )
        
        if limit == 0:
            return sorted_tags
        return sorted_tags[:limit]

    async def search_tags(
        self, query: str, limit: int = 50
    ) -> List[Dict[str, Any]]:
        """Search tags by case-insensitive substring match, sorted by count.

        If query is empty, behaves like get_top_tags (returns top ``limit``
        tags). If limit is 0, all matching tags are returned.
        """
        await self.get_cached_data()

        normalized_query = (query or "").strip().lower()
        if not normalized_query:
            return await self.get_top_tags(limit if limit > 0 else 20)

        matched = [
            {"tag": tag, "count": count}
            for tag, count in self._tags_count.items()
            if normalized_query in tag.lower()
        ]
        matched.sort(key=lambda x: x["count"], reverse=True)

        if limit == 0:
            return matched
        return matched[:limit]

    async def get_base_models(self, limit: int = 20) -> List[Dict[str, Any]]:
        """Get base models sorted by count. If limit is 0, return all."""
        cache = await self.get_cached_data()
        
        base_model_counts = {}
        for model in cache.raw_data:
            if 'base_model' in model and model['base_model']:
                base_model = model['base_model']
                base_model_counts[base_model] = base_model_counts.get(base_model, 0) + 1
        
        sorted_models = [{'name': model, 'count': count} for model, count in base_model_counts.items()]
        sorted_models.sort(key=lambda x: x['count'], reverse=True)

        if limit == 0:
            return sorted_models
        return sorted_models[:limit]
        
    @staticmethod
    def find_matching_models(
        raw_data: List[Dict[str, Any]],
        name: str,
        *,
        base_model: Optional[str] = None,
        extensions: Optional[Set[str]] = None,
    ) -> List[Dict[str, Any]]:
        """Return all cached models matching ``name`` (case-insensitive).

        A name containing a path separator must equal the model's
        folder-relative path; a bare name matches on basename. When
        ``base_model`` is given, confident mismatches are rejected while
        unknowns on either side stay eligible (lenient guard).
        ``extensions`` should be the scanner's own ``file_extensions`` so
        suffix stripping only covers formats the scanner actually indexes;
        when omitted, the shared :data:`WEIGHT_FILE_EXTENSIONS` set is used.
        """
        # Longest first so overlapping suffixes strip correctly.
        exts = sorted(extensions or WEIGHT_FILE_EXTENSIONS, key=len, reverse=True)

        normalized_name = str(name).replace("\\", "/").casefold()
        for ext in exts:
            if normalized_name.endswith(ext):
                normalized_name = normalized_name[: -len(ext)]
                break
        has_path = "/" in normalized_name
        basename = normalized_name.rsplit("/", 1)[-1]

        matches = []
        for model in raw_data:
            file_name = str(model.get("file_name") or "").replace("\\", "/")
            folder = str(model.get("folder") or "").replace("\\", "/").strip("/")
            model_path = f"{folder}/{file_name}" if folder else file_name
            for ext in exts:
                if model_path.casefold().endswith(ext):
                    model_path = model_path[: -len(ext)]
                    break
            if (has_path and model_path.casefold() == normalized_name) or (
                not has_path and model_path.rsplit("/", 1)[-1].casefold() == basename
            ):
                matches.append(model)

        expected_base = str(base_model or "").strip().casefold()
        if expected_base and expected_base != "unknown":
            matches = [
                model
                for model in matches
                if str(model.get("base_model") or "").strip().casefold()
                in ("", "unknown", expected_base)
            ]
        return matches

    async def find_models_by_name(
        self, name: str, *, base_model: Optional[str] = None
    ) -> List[Dict[str, Any]]:
        """Return every cached model matching ``name`` (see ``find_matching_models``)."""
        try:
            cache = await self.get_cached_data()
            return self.find_matching_models(
                cache.raw_data,
                name,
                base_model=base_model,
                extensions=self.file_extensions,
            )
        except Exception as e:
            logger.error(f"Error finding models by name: {e}", exc_info=True)
            return []

    async def get_model_info_by_name(
        self,
        name: str,
        *,
        require_unique: bool = False,
        base_model: Optional[str] = None,
    ):
        """Get model information by name.

        Default mode keeps the legacy first-match/fallback semantics. With
        ``require_unique`` an ambiguous name is a miss, and ``base_model``
        rejects confident base-model mismatches (unknowns stay eligible).
        """
        if require_unique or base_model:
            try:
                matches = await self.find_models_by_name(name, base_model=base_model)
                if require_unique and len(matches) != 1:
                    return None
                return matches[0] if matches else None
            except Exception as e:
                logger.error(f"Error getting model info by name: {e}", exc_info=True)
                return None

        try:
            cache = await self.get_cached_data()

            name_normalized = name.replace("\\", "/")
            name_no_ext = name_normalized
            for ext in (".safetensors", ".ckpt", ".pt", ".bin"):
                if name_no_ext.lower().endswith(ext):
                    name_no_ext = name_no_ext[: -len(ext)]
                    break

            has_path = "/" in name_no_ext
            basename = os.path.basename(name_no_ext) if has_path else name_no_ext
            best_fallback = None

            for model in cache.raw_data:
                file_name = model.get("file_name", "")
                folder = model.get("folder", "")
                file_name_no_ext = file_name
                for ext in (".safetensors", ".ckpt", ".pt", ".bin"):
                    if file_name_no_ext.lower().endswith(ext):
                        file_name_no_ext = file_name_no_ext[: -len(ext)]
                        break
                path_name = f"{folder}/{file_name_no_ext}".replace("\\", "/") if folder else file_name_no_ext

                if name_no_ext == file_name_no_ext or name_no_ext == path_name:
                    return model

                if has_path and file_name_no_ext == basename:
                    if folder and name_no_ext.startswith(folder.replace("\\", "/") + "/"):
                        best_fallback = model
                    elif best_fallback is None:
                        best_fallback = model

            return best_fallback

        except Exception as e:
            logger.error(f"Error getting model info by name: {e}", exc_info=True)
            return None
        
    def get_excluded_models(self) -> List[str]:
        """Get list of excluded model file paths"""
        return self._excluded_models.copy()

    async def update_preview_in_cache(self, file_path: str, preview_url: str, preview_nsfw_level: int) -> bool:
        """Update preview URL in cache for a specific lora
        
        Args:
            file_path: The file path of the lora to update
            preview_url: The new preview URL
            preview_nsfw_level: The NSFW level of the preview
            
        Returns:
            bool: True if the update was successful, False if cache doesn't exist or lora wasn't found
        """
        if self._cache is None:
            return False

        updated = await self._cache.update_preview_url(file_path, preview_url, preview_nsfw_level)
        if updated:
            await self._persist_current_cache()
        return updated

    async def bulk_delete_models(self, file_paths: List[str]) -> Dict[str, Any]:
        """Delete multiple models and update cache in a batch operation
        
        Args:
            file_paths: List of file paths to delete
            
        Returns:
            Dict containing results of the operation
        """
        try:
            if not file_paths:
                return {
                    'success': False,
                    'error': 'No file paths provided for deletion',
                    'results': []
                }
            
            # Keep track of success and failures
            results = []
            total_deleted = 0
            cache_updated = False
            
            # Get cache data
            cache = await self.get_cached_data()
            
            # Track deleted models to update cache once
            deleted_models = []
            
            # Stage each file into the pending-delete staging area and merge
            # all per-file batches into ONE batch for the whole bulk action.
            pending_delete_service = await get_pending_delete_service()
            batch_ids: List[str] = []
            
            for file_path in file_paths:
                if self.is_cancelled():
                    logger.info(f"{self.model_type.capitalize()} Scanner: Bulk delete cancelled by user")
                    break

                try:
                    _require_path_in_library_roots(file_path, self, label="File path")

                    target_dir = os.path.dirname(file_path)
                    base_name = os.path.basename(file_path)
                    file_name, main_extension = os.path.splitext(base_name)

                    # Snapshot the cache entry BEFORE the cache mutation that
                    # runs after the loop - the manifest needs it for undo.
                    cached_entry = None
                    if cache is not None:
                        cached_entry = next(
                            (item for item in cache.raw_data if item.get('file_path') == file_path),
                            None,
                        )

                    batch_id = await pending_delete_service.stage_model_delete(
                        scanner=self,
                        target_dir=target_dir,
                        file_name=file_name,
                        main_extension=main_extension,
                        original_file_path=file_path,
                        cached_entry=cached_entry,
                    )

                    if batch_id is not None:
                        # Artifacts were renamed into staging: the main file is
                        # gone from its original location.
                        batch_ids.append(batch_id)
                        deleted_files = [file_path]
                    else:
                        deleted_files = await delete_model_artifacts(
                            target_dir,
                            file_name,
                            main_extension=main_extension,
                        )
                    
                    if deleted_files:
                        deleted_models.append(file_path)
                        results.append({
                            'file_path': file_path,
                            'success': True,
                            'deleted_files': deleted_files
                        })
                        total_deleted += 1
                    else:
                        results.append({
                            'file_path': file_path,
                            'success': False,
                            'error': 'No files deleted'
                        })
                except Exception as e:
                    logger.error(f"Error deleting file {file_path}: {e}")
                    results.append({
                        'file_path': file_path,
                        'success': False,
                        'error': str(e)
                    })
            
            # Merge every staged per-file batch into ONE undoable batch. On a
            # merge failure (defensive) the response falls back to the
            # constituent batch_ids array so the frontend can undo them
            # sequentially.
            batch_field: Dict[str, Any] = {}
            if batch_ids:
                merged_id = await pending_delete_service.merge_batches(batch_ids)
                if merged_id is not None:
                    batch_field['batch_id'] = merged_id
                else:
                    batch_field['batch_ids'] = list(batch_ids)
            
            # Batch update cache if any models were deleted
            if deleted_models:
                # Update the cache in a batch operation
                cache_updated = await self._batch_update_cache_for_deleted_models(deleted_models)
                
            return {
                'success': True,
                'status': 'cancelled' if self.is_cancelled() else 'success',
                'total_deleted': total_deleted,
                'total_attempted': len(file_paths),
                'cache_updated': cache_updated,
                'results': results,
                **batch_field
            }
            
        except Exception as e:
            logger.error(f"Error in bulk delete: {e}", exc_info=True)
            return {
                'success': False,
                'error': str(e),
                'results': []
            }
    
    async def _batch_update_cache_for_deleted_models(self, file_paths: List[str]) -> bool:
        """Update cache after multiple models have been deleted
        
        Args:
            file_paths: List of file paths that were deleted
            
        Returns:
            bool: True if cache was updated and saved successfully
        """
        if not file_paths or self._cache is None:
            return False

        if self.is_cancelled():
            logger.info(
                f"{self.model_type.capitalize()} Scanner: Skipping cache update "
                "after cancelled bulk delete"
            )
            return False
            
        try:
            # Get all models that need to be removed from cache
            models_to_remove = [item for item in self._cache.raw_data if item['file_path'] in file_paths]
            
            if not models_to_remove:
                return False
                
            # Update tag counts
            for model in models_to_remove:
                for tag in model.get('tags', []):
                    if tag in self._tags_count:
                        self._tags_count[tag] = max(0, self._tags_count[tag] - 1)
                        if self._tags_count[tag] == 0:
                            del self._tags_count[tag]
            
            # Update hash index
            for model in models_to_remove:
                file_path = model['file_path']
                self._cache.remove_from_version_index(model)
                if hasattr(self, '_hash_index') and self._hash_index:
                    # Get the hash and filename before removal for duplicate checking
                    file_name = os.path.splitext(os.path.basename(file_path))[0]
                    hash_val = model.get('sha256', '').lower()

                    # Remove from hash index
                    self._hash_index.remove_by_path(file_path, hash_val)
                    
                    # Check and clean up duplicates
                    self._cleanup_duplicates_after_removal(hash_val, file_name)
            
            # Update cache data
            self._cache.raw_data = [item for item in self._cache.raw_data if item['file_path'] not in file_paths]

            # Resort cache
            self._cache.rebuild_version_index()
            await self._cache.resort()

            await self._persist_current_cache()

            self.bump_cache_version()

            return True
            
        except Exception as e:
            logger.error(f"Error updating cache after bulk delete: {e}", exc_info=True)
            return False
    
    def _cleanup_duplicates_after_removal(self, hash_val: str, file_name: str) -> None:
        """Clean up duplicate entries in hash index after removing a model
        
        Args:
            hash_val: SHA256 hash of the removed model
            file_name: File name of the removed model without extension
        """
        if not hash_val or not file_name or not hasattr(self, '_hash_index'):
            return
            
        # Clean up hash duplicates if only 0 or 1 entries remain
        if hash_val in self._hash_index._duplicate_hashes:
            if len(self._hash_index._duplicate_hashes[hash_val]) <= 1:
                del self._hash_index._duplicate_hashes[hash_val]
        
        # Clean up filename duplicates if only 0 or 1 entries remain
        if file_name in self._hash_index._duplicate_filenames:
            if len(self._hash_index._duplicate_filenames[file_name]) <= 1:
                del self._hash_index._duplicate_filenames[file_name]

    async def check_model_version_exists(self, model_version_id: int) -> bool:
        """Check if a specific model version exists in the cache

        Args:
            model_version_id: Civitai model version ID

        Returns:
            bool: True if the model version exists, False otherwise
        """
        try:
            normalized_id = int(model_version_id)
        except (TypeError, ValueError):
            return False

        try:
            cache = await self.get_cached_data()
            if not cache:
                return False

            return normalized_id in cache.version_index
        except Exception as e:
            logger.error(f"Error checking model version existence: {e}")
            return False

    async def get_files_for_version(self, model_version_id: int) -> List[Dict[str, Any]]:
        """Get all local file entries for a specific model version (#1058).

        A Civitai model version can have several weight files downloaded;
        unlike the single-valued version_index this returns every entry.

        Args:
            model_version_id: Civitai model version ID

        Returns:
            List[Dict]: Cache entries (may be empty)
        """
        try:
            normalized_id = int(model_version_id)
        except (TypeError, ValueError):
            return []

        try:
            cache = await self.get_cached_data()
            if not cache:
                return []

            getter = getattr(cache, "get_files_by_version_id", None)
            if getter is not None:
                return getter(normalized_id)

            # Fallback for cache implementations without the multi-file index
            entry = cache.version_index.get(normalized_id)
            return [entry] if entry is not None else []
        except Exception as e:
            logger.error(f"Error getting files for model version: {e}")
            return []

    async def get_model_versions_by_id(self, model_id: int) -> List[Dict[str, Any]]:
        """Get all versions of a model by its ID
        
        Args:
            model_id: Civitai model ID
            
        Returns:
            List[Dict]: List of version information dictionaries
        """
        try:
            cache = await self.get_cached_data()
            if not cache:
                return []

            return cache.get_versions_by_model_id(model_id)
        except Exception as e:
            logger.error(f"Error getting model versions: {e}")
            return []
