feat(settings): add configurable hash chunk size

py/services/settings_manager.py

@@ -8,7 +8,7 @@ from datetime import datetime, timezone
 from threading import Lock
 from typing import Any, Awaitable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
 
-from ..utils.constants import DEFAULT_PRIORITY_TAG_CONFIG
+from ..utils.constants import DEFAULT_HASH_CHUNK_SIZE_MB, DEFAULT_PRIORITY_TAG_CONFIG
 from ..utils.settings_paths import ensure_settings_file
 from ..utils.tag_priorities import (
     PriorityTagEntry,
@@ -29,6 +29,7 @@ CORE_USER_SETTING_KEYS: Tuple[str, ...] = (
 DEFAULT_SETTINGS: Dict[str, Any] = {
     "civitai_api_key": "",
     "use_portable_settings": False,
+    "hash_chunk_size_mb": DEFAULT_HASH_CHUNK_SIZE_MB,
     "language": "en",
     "show_only_sfw": False,
     "enable_metadata_archive_db": False,

py/utils/constants.py

@@ -55,6 +55,9 @@ CIVITAI_USER_MODEL_TYPES = [
     'checkpoint',
 ]
 
+# Default chunk size in megabytes used for hashing large files.
+DEFAULT_HASH_CHUNK_SIZE_MB = 4
+
 # Auto-organize settings
 AUTO_ORGANIZE_BATCH_SIZE = 50  # Process models in batches to avoid overwhelming the system
 

py/utils/file_utils.py

@@ -1,17 +1,41 @@
+
+import hashlib
 import logging
 import os
-import hashlib
 
-from .constants import PREVIEW_EXTENSIONS, CARD_PREVIEW_WIDTH
+from .constants import (
+    CARD_PREVIEW_WIDTH,
+    DEFAULT_HASH_CHUNK_SIZE_MB,
+    PREVIEW_EXTENSIONS,
+)
 from .exif_utils import ExifUtils
+from ..services.settings_manager import get_settings_manager
 
 logger = logging.getLogger(__name__)
 
+
+def _get_hash_chunk_size_bytes() -> int:
+    """Return the chunk size used for hashing, in bytes."""
+
+    settings_manager = get_settings_manager()
+    chunk_size_mb = settings_manager.get("hash_chunk_size_mb", DEFAULT_HASH_CHUNK_SIZE_MB)
+    try:
+        chunk_size_value = float(chunk_size_mb)
+    except (TypeError, ValueError):
+        chunk_size_value = float(DEFAULT_HASH_CHUNK_SIZE_MB)
+
+    if chunk_size_value <= 0:
+        chunk_size_value = float(DEFAULT_HASH_CHUNK_SIZE_MB)
+
+    return max(1, int(chunk_size_value * 1024 * 1024))
+
+
 async def calculate_sha256(file_path: str) -> str:
     """Calculate SHA256 hash of a file"""
     sha256_hash = hashlib.sha256()
+    chunk_size = _get_hash_chunk_size_bytes()
     with open(file_path, "rb") as f:
-        for byte_block in iter(lambda: f.read(128 * 1024), b""):
+        for byte_block in iter(lambda: f.read(chunk_size), b""):
             sha256_hash.update(byte_block)
     return sha256_hash.hexdigest()
 
@@ -81,4 +105,4 @@ def get_preview_extension(preview_path: str) -> str:
 
 def normalize_path(path: str) -> str:
     """Normalize file path to use forward slashes"""
-    return path.replace(os.sep, "/") if path else path
+    return path.replace(os.sep, "/") if path else path