diff --git a/py/config.py b/py/config.py index 965e3817..f5bb017e 100644 --- a/py/config.py +++ b/py/config.py @@ -82,8 +82,8 @@ class Config: self._path_mappings: Dict[str, str] = {} # Normalized preview root directories used to validate preview access self._preview_root_paths: Set[Path] = set() - # Optional background rescan thread - self._rescan_thread: Optional[threading.Thread] = None + # Fingerprint of the symlink layout from the last successful scan + self._cached_fingerprint: Optional[Dict[str, object]] = None self.loras_roots = self._init_lora_paths() self.checkpoints_roots = None self.unet_roots = None @@ -231,32 +231,6 @@ class Config: cache_dir.mkdir(parents=True, exist_ok=True) return cache_dir / "symlink_map.json" - def _compute_noise_mtime(self, root: str) -> Optional[int]: - """Return the latest mtime of known noisy paths inside ``root``.""" - - normalized_root = self._normalize_path(root) - noise_paths: List[str] = [] - - # The first LoRA root hosts recipes and stats files which routinely - # update without changing symlink layout. - first_lora_root = self._normalize_path(self.loras_roots[0]) if self.loras_roots else None - if first_lora_root and normalized_root == first_lora_root: - recipes_dir = os.path.join(root, "recipes") - stats_file = os.path.join(root, "lora_manager_stats.json") - noise_paths.extend([recipes_dir, stats_file]) - - mtimes: List[int] = [] - for path in noise_paths: - try: - stat_result = os.stat(path) - mtimes.append(getattr(stat_result, "st_mtime_ns", int(stat_result.st_mtime * 1e9))) - except OSError: - continue - - if not mtimes: - return None - return max(mtimes) - def _symlink_roots(self) -> List[str]: roots: List[str] = [] roots.extend(self.loras_roots or []) @@ -267,26 +241,46 @@ class Config: def _build_symlink_fingerprint(self) -> Dict[str, object]: roots = [self._normalize_path(path) for path in self._symlink_roots() if path] unique_roots = sorted(set(roots)) + # Fingerprint now only contains the root paths to avoid sensitivity to folder content changes. + return {"roots": unique_roots} - stats: Dict[str, Dict[str, int]] = {} - for root in unique_roots: - try: - root_stat = os.stat(root) - noise_mtime = self._compute_noise_mtime(root) - stats[root] = { - "mtime_ns": getattr(root_stat, "st_mtime_ns", int(root_stat.st_mtime * 1e9)), - "inode": getattr(root_stat, "st_ino", 0), - "noise_mtime_ns": noise_mtime, - } - except OSError: - continue + def _initialize_symlink_mappings(self) -> None: + start = time.perf_counter() + cache_loaded = self._load_persisted_cache_into_mappings() - return {"roots": unique_roots, "stats": stats} + if cache_loaded: + logger.info( + "Symlink mappings restored from cache in %.2f ms", + (time.perf_counter() - start) * 1000, + ) + self._rebuild_preview_roots() - def _load_symlink_cache(self) -> bool: + # Only rescan if target roots have changed. + # This is stable across file additions/deletions. + current_fingerprint = self._build_symlink_fingerprint() + cached_fingerprint = self._cached_fingerprint + + if cached_fingerprint and current_fingerprint == cached_fingerprint: + return + + logger.info("Symlink root paths changed; rescanning symbolic links") + + self.rebuild_symlink_cache() + logger.info( + "Symlink mappings rebuilt and cached in %.2f ms", + (time.perf_counter() - start) * 1000, + ) + + def rebuild_symlink_cache(self) -> None: + """Force a fresh scan of all symbolic links and update the persistent cache.""" + self._scan_symbolic_links() + self._save_symlink_cache() + self._rebuild_preview_roots() + + def _load_persisted_cache_into_mappings(self) -> bool: + """Load the symlink cache and store its fingerprint for comparison.""" cache_path = self._get_symlink_cache_path() if not cache_path.exists(): - logger.info("Symlink cache not found at %s", cache_path) return False try: @@ -297,14 +291,15 @@ class Config: return False if not isinstance(payload, dict): - logger.info("Symlink cache payload is not a dict: %s", type(payload)) return False cached_mappings = payload.get("path_mappings") if not isinstance(cached_mappings, Mapping): - logger.info("Symlink cache missing path mappings") return False + # Store the cached fingerprint for comparison during initialization + self._cached_fingerprint = payload.get("fingerprint") + normalized_mappings: Dict[str, str] = {} for target, link in cached_mappings.items(): if not isinstance(target, str) or not isinstance(link, str): @@ -329,30 +324,10 @@ class Config: except Exception as exc: logger.info("Failed to write symlink cache %s: %s", cache_path, exc) - def _initialize_symlink_mappings(self) -> None: - start = time.perf_counter() - cache_loaded = self._load_symlink_cache() - - if cache_loaded: - logger.info( - "Symlink mappings restored from cache in %.2f ms", - (time.perf_counter() - start) * 1000, - ) - self._rebuild_preview_roots() - self._schedule_symlink_rescan() - return - - self._scan_symbolic_links() - self._save_symlink_cache() - self._rebuild_preview_roots() - logger.info( - "Symlink mappings rebuilt and cached in %.2f ms", - (time.perf_counter() - start) * 1000, - ) - def _scan_symbolic_links(self): """Scan all symbolic links in LoRA, Checkpoint, and Embedding root directories""" start = time.perf_counter() + # Reset mappings before rescanning to avoid stale entries self._path_mappings.clear() self._seed_root_symlink_mappings() @@ -365,39 +340,11 @@ class Config: len(self._path_mappings), ) - def _schedule_symlink_rescan(self) -> None: - """Trigger a best-effort background rescan to refresh stale caches.""" - - if self._rescan_thread and self._rescan_thread.is_alive(): - return - - def worker(): - try: - self._scan_symbolic_links() - self._save_symlink_cache() - self._rebuild_preview_roots() - logger.debug("Background symlink rescan completed") - except Exception as exc: # pragma: no cover - defensive logging - logger.info("Background symlink rescan failed: %s", exc) - - thread = threading.Thread( - target=worker, - name="lora-manager-symlink-rescan", - daemon=True, - ) - self._rescan_thread = thread - thread.start() - - def _wait_for_rescan(self, timeout: Optional[float] = None) -> None: - """Block until the background rescan completes (testing convenience).""" - - thread = self._rescan_thread - if thread: - thread.join(timeout=timeout) - def _scan_directory_links(self, root: str, visited_dirs: Set[str]): """Iteratively scan directory symlinks to avoid deep recursion.""" try: + # Note: We only use realpath for the initial root if it's not already resolved + # to ensure we have a valid entry point. root_real = self._normalize_path(os.path.realpath(root)) except OSError: root_real = self._normalize_path(root) @@ -406,45 +353,57 @@ class Config: return visited_dirs.add(root_real) - stack: List[str] = [root] + # Stack entries: (display_path, real_resolved_path) + stack: List[Tuple[str, str]] = [(root, root_real)] while stack: - current = stack.pop() + current_display, current_real = stack.pop() try: - with os.scandir(current) as it: + with os.scandir(current_display) as it: for entry in it: try: - entry_path = entry.path - if self._is_link(entry_path): - target_path = os.path.realpath(entry_path) + # 1. High speed detection using dirent data (is_symlink) + is_link = entry.is_symlink() + + # On Windows, is_symlink handles reparse points + if is_link: + # Only resolve realpath when we actually find a link + target_path = os.path.realpath(entry.path) if not os.path.isdir(target_path): continue normalized_target = self._normalize_path(target_path) - # Always record the mapping even if we already visited - # the real directory via another path. This prevents the - # traversal order from dropping valid link->target pairs. - self.add_path_mapping(entry_path, target_path) + self.add_path_mapping(entry.path, target_path) + if normalized_target in visited_dirs: continue + visited_dirs.add(normalized_target) - stack.append(target_path) + stack.append((target_path, normalized_target)) continue + # 2. Process normal directories if not entry.is_dir(follow_symlinks=False): continue - normalized_real = self._normalize_path(os.path.realpath(entry_path)) - if normalized_real in visited_dirs: + # For normal directories, we avoid realpath() call by + # incrementally building the real path relative to current_real. + # This is safe because 'entry' is NOT a symlink. + entry_real = self._normalize_path(os.path.join(current_real, entry.name)) + + if entry_real in visited_dirs: continue - visited_dirs.add(normalized_real) - stack.append(entry_path) + + visited_dirs.add(entry_real) + stack.append((entry.path, entry_real)) except Exception as inner_exc: logger.debug( "Error processing directory entry %s: %s", entry.path, inner_exc ) except Exception as e: - logger.error(f"Error scanning links in {current}: {e}") + logger.error(f"Error scanning links in {current_display}: {e}") + + def add_path_mapping(self, link_path: str, target_path: str): """Add a symbolic link path mapping @@ -537,7 +496,11 @@ class Config: normalized_path = os.path.normpath(path).replace(os.sep, '/') # Check if the path is contained in any mapped target path for target_path, link_path in self._path_mappings.items(): - if normalized_path.startswith(target_path): + # Match whole path components to avoid prefix collisions (e.g., /a/b vs /a/bc) + if normalized_path == target_path: + return link_path + + if normalized_path.startswith(target_path + '/'): # If the path starts with the target path, replace with link path mapped_path = normalized_path.replace(target_path, link_path, 1) return mapped_path @@ -547,10 +510,14 @@ class Config: """Map a symbolic link path back to the actual path""" normalized_link = os.path.normpath(link_path).replace(os.sep, '/') # Check if the path is contained in any mapped target path - for target_path, link_path in self._path_mappings.items(): - if normalized_link.startswith(link_path): + for target_path, link_path_mapped in self._path_mappings.items(): + # Match whole path components + if normalized_link == link_path_mapped: + return target_path + + if normalized_link.startswith(link_path_mapped + '/'): # If the path starts with the link path, replace with actual path - mapped_path = normalized_link.replace(link_path, target_path, 1) + mapped_path = normalized_link.replace(link_path_mapped, target_path, 1) return mapped_path return link_path diff --git a/py/services/model_scanner.py b/py/services/model_scanner.py index 8fa76e2a..40015516 100644 --- a/py/services/model_scanner.py +++ b/py/services/model_scanner.py @@ -654,6 +654,11 @@ class ModelScanner: self._is_initializing = True # Set flag try: start_time = time.time() + + # Manually trigger a symlink rescan during a full rebuild. + # This ensures that any new symlink mappings are correctly picked up. + config.rebuild_symlink_cache() + # Determine the page type based on model type # Scan for new data scan_result = await self._gather_model_data() diff --git a/tests/config/test_symlink_cache.py b/tests/config/test_symlink_cache.py index 5ba9820b..c32dacb1 100644 --- a/tests/config/test_symlink_cache.py +++ b/tests/config/test_symlink_cache.py @@ -63,7 +63,6 @@ def test_symlink_scan_skips_file_links(monkeypatch: pytest.MonkeyPatch, tmp_path def test_symlink_cache_reuses_previous_scan(monkeypatch: pytest.MonkeyPatch, tmp_path): loras_dir, settings_dir = _setup_paths(monkeypatch, tmp_path) - monkeypatch.setattr(config_module.Config, "_schedule_symlink_rescan", lambda self: None) target_dir = loras_dir / "target" target_dir.mkdir() @@ -87,7 +86,6 @@ def test_symlink_cache_reuses_previous_scan(monkeypatch: pytest.MonkeyPatch, tmp def test_symlink_cache_survives_noise_mtime(monkeypatch: pytest.MonkeyPatch, tmp_path): loras_dir, settings_dir = _setup_paths(monkeypatch, tmp_path) - monkeypatch.setattr(config_module.Config, "_schedule_symlink_rescan", lambda self: None) target_dir = loras_dir / "target" target_dir.mkdir() @@ -114,7 +112,7 @@ def test_symlink_cache_survives_noise_mtime(monkeypatch: pytest.MonkeyPatch, tmp assert second_cfg.map_path_to_link(str(target_dir)) == _normalize(str(dir_link)) -def test_background_rescan_refreshes_cache(monkeypatch: pytest.MonkeyPatch, tmp_path): +def test_manual_rescan_refreshes_cache(monkeypatch: pytest.MonkeyPatch, tmp_path): loras_dir, _ = _setup_paths(monkeypatch, tmp_path) target_dir = loras_dir / "target" @@ -135,12 +133,11 @@ def test_background_rescan_refreshes_cache(monkeypatch: pytest.MonkeyPatch, tmp_ second_cfg = config_module.Config() - # Cache may still point at the old real path immediately after load - initial_mapping = second_cfg.map_path_to_link(str(new_target)) - assert initial_mapping in {str(new_target), _normalize(str(dir_link))} + # Cache still point at the old real path immediately after load + assert second_cfg.map_path_to_link(str(new_target)) == _normalize(str(new_target)) - # Background rescan should refresh the mapping to the new target and update the cache file - second_cfg._wait_for_rescan(timeout=2.0) + # Manual rescan should refresh the mapping to the new target + second_cfg.rebuild_symlink_cache() new_real = _normalize(os.path.realpath(new_target)) assert second_cfg._path_mappings.get(new_real) == _normalize(str(dir_link)) assert second_cfg.map_path_to_link(str(new_target)) == _normalize(str(dir_link)) @@ -170,7 +167,6 @@ def test_symlink_roots_are_preserved(monkeypatch: pytest.MonkeyPatch, tmp_path): monkeypatch.setattr(config_module.folder_paths, "get_folder_paths", fake_get_folder_paths) monkeypatch.setattr(config_module, "standalone_mode", True) monkeypatch.setattr(config_module, "get_settings_dir", lambda create=True: str(settings_dir)) - monkeypatch.setattr(config_module.Config, "_schedule_symlink_rescan", lambda self: None) cfg = config_module.Config() diff --git a/tests/config/test_symlink_fingerprint.py b/tests/config/test_symlink_fingerprint.py new file mode 100644 index 00000000..c69761d3 --- /dev/null +++ b/tests/config/test_symlink_fingerprint.py @@ -0,0 +1,128 @@ +import os +import pytest +from py import config as config_module + +def _setup_paths(monkeypatch: pytest.MonkeyPatch, tmp_path): + settings_dir = tmp_path / "settings" + loras_dir = tmp_path / "loras" + loras_dir.mkdir() + checkpoint_dir = tmp_path / "checkpoints" + checkpoint_dir.mkdir() + embedding_dir = tmp_path / "embeddings" + embedding_dir.mkdir() + + def fake_get_folder_paths(kind: str): + mapping = { + "loras": [str(loras_dir)], + "checkpoints": [str(checkpoint_dir)], + "unet": [], + "embeddings": [str(embedding_dir)], + } + return mapping.get(kind, []) + + monkeypatch.setattr(config_module.folder_paths, "get_folder_paths", fake_get_folder_paths) + monkeypatch.setattr(config_module, "standalone_mode", True) + monkeypatch.setattr(config_module, "get_settings_dir", lambda create=True: str(settings_dir)) + + return loras_dir, settings_dir + +def test_fingerprint_match_skips_rescan(monkeypatch: pytest.MonkeyPatch, tmp_path): + loras_dir, settings_dir = _setup_paths(monkeypatch, tmp_path) + + # Track calls to _scan_symbolic_links + scan_calls = 0 + original_scan = config_module.Config._scan_symbolic_links + + def wrapped_scan(self): + nonlocal scan_calls + scan_calls += 1 + return original_scan(self) + + monkeypatch.setattr(config_module.Config, "_scan_symbolic_links", wrapped_scan) + + # 1. First initialization: should scan and cache + cfg1 = config_module.Config() + assert scan_calls == 1 + + # 2. Second initialization: should load from cache and skip rescan because fingerprint matches + cfg2 = config_module.Config() + # scan_calls should still be 1 because it loaded from cache and skipped the rescan + assert scan_calls == 1 + +def test_mtime_change_does_not_trigger_rescan(monkeypatch: pytest.MonkeyPatch, tmp_path): + loras_dir, settings_dir = _setup_paths(monkeypatch, tmp_path) + + # Track calls to _scan_symbolic_links + scan_calls = 0 + original_scan = config_module.Config._scan_symbolic_links + + def wrapped_scan(self): + nonlocal scan_calls + scan_calls += 1 + return original_scan(self) + + monkeypatch.setattr(config_module.Config, "_scan_symbolic_links", wrapped_scan) + + # 1. First initialization: should scan and cache + cfg1 = config_module.Config() + assert scan_calls == 1 + + # 2. Modify a root directory (change mtime) - this should NOT trigger a rescan anymore + os.utime(loras_dir, (os.path.getatime(loras_dir), os.path.getmtime(loras_dir) + 100)) + + # 3. Third initialization: should load from cache and skip rescan despite mtime change + cfg3 = config_module.Config() + assert scan_calls == 1 + +def test_root_path_change_triggers_rescan(monkeypatch: pytest.MonkeyPatch, tmp_path): + loras_dir, settings_dir = _setup_paths(monkeypatch, tmp_path) + + # Track calls to _scan_symbolic_links + scan_calls = 0 + original_scan = config_module.Config._scan_symbolic_links + + def wrapped_scan(self): + nonlocal scan_calls + scan_calls += 1 + return original_scan(self) + + monkeypatch.setattr(config_module.Config, "_scan_symbolic_links", wrapped_scan) + + # 1. First initialization + config_module.Config() + assert scan_calls == 1 + + # 2. Change root paths + new_lora_dir = tmp_path / "new_loras" + new_lora_dir.mkdir() + + def fake_get_folder_paths_modified(kind: str): + if kind == "loras": + return [str(loras_dir), str(new_lora_dir)] + return [] + + monkeypatch.setattr(config_module.folder_paths, "get_folder_paths", fake_get_folder_paths_modified) + + # 3. Initialization with different roots should trigger rescan + config_module.Config() + assert scan_calls == 2 + +def test_manual_rebuild_symlink_cache(monkeypatch: pytest.MonkeyPatch, tmp_path): + loras_dir, settings_dir = _setup_paths(monkeypatch, tmp_path) + + scan_calls = 0 + original_scan = config_module.Config._scan_symbolic_links + + def wrapped_scan(self): + nonlocal scan_calls + scan_calls += 1 + return original_scan(self) + + monkeypatch.setattr(config_module.Config, "_scan_symbolic_links", wrapped_scan) + + cfg = config_module.Config() + assert scan_calls == 1 + + # Manual trigger + cfg.rebuild_symlink_cache() + assert scan_calls == 2