From 607ab35cce1f43cf4ad5ce84ad35eff7649eb831 Mon Sep 17 00:00:00 2001 From: Will Miao <13051207myq@gmail.com> Date: Thu, 20 Mar 2025 16:55:51 +0800 Subject: [PATCH] Refactor search functionality in Lora and Recipe scanners to utilize fuzzy matching - Introduced a new fuzzy_match utility function for improved search accuracy across Lora and Recipe scanners. - Updated search logic in LoraScanner and RecipeScanner to leverage fuzzy matching for titles, tags, and filenames, enhancing user experience. - Removed deprecated search methods to streamline the codebase and improve maintainability. - Adjusted API routes to ensure compatibility with the new search options, including recursive search handling. --- py/routes/api_routes.py | 6 +- py/routes/recipe_routes.py | 2 - py/services/lora_scanner.py | 137 ++++++++++++---------------------- py/services/recipe_scanner.py | 12 +-- py/utils/utils.py | 37 +++++++++ static/js/api/loraApi.js | 6 +- static/js/recipes.js | 4 +- 7 files changed, 98 insertions(+), 106 deletions(-) diff --git a/py/routes/api_routes.py b/py/routes/api_routes.py index c86d0d6a..3846eabb 100644 --- a/py/routes/api_routes.py +++ b/py/routes/api_routes.py @@ -131,7 +131,6 @@ class ApiRoutes: folder = request.query.get('folder') search = request.query.get('search', '').lower() fuzzy = request.query.get('fuzzy', 'false').lower() == 'true' - recursive = request.query.get('recursive', 'false').lower() == 'true' # Parse base models filter parameter base_models = request.query.get('base_models', '').split(',') @@ -141,6 +140,7 @@ class ApiRoutes: search_filename = request.query.get('search_filename', 'true').lower() == 'true' search_modelname = request.query.get('search_modelname', 'true').lower() == 'true' search_tags = request.query.get('search_tags', 'false').lower() == 'true' + recursive = request.query.get('recursive', 'false').lower() == 'true' # Validate parameters if page < 1 or page_size < 1 or page_size > 100: @@ -165,13 +165,13 @@ class ApiRoutes: folder=folder, search=search, fuzzy=fuzzy, - recursive=recursive, base_models=base_models, # Pass base models filter tags=tags, # Add tags parameter search_options={ 'filename': search_filename, 'modelname': search_modelname, - 'tags': search_tags + 'tags': search_tags, + 'recursive': recursive } ) diff --git a/py/routes/recipe_routes.py b/py/routes/recipe_routes.py index 63be56e5..0761e40b 100644 --- a/py/routes/recipe_routes.py +++ b/py/routes/recipe_routes.py @@ -68,7 +68,6 @@ class RecipeRoutes: async def get_recipes(self, request: web.Request) -> web.Response: """API endpoint for getting paginated recipes""" try: - logger.info(f"get_recipes, Request: {request}") # Get query parameters with defaults page = int(request.query.get('page', '1')) page_size = int(request.query.get('page_size', '20')) @@ -100,7 +99,6 @@ class RecipeRoutes: 'lora_model': search_lora_model } - logger.info(f"get_recipes, Filters: {filters}, Search Options: {search_options}") # Get paginated data result = await self.recipe_scanner.get_paginated_data( page=page, diff --git a/py/services/lora_scanner.py b/py/services/lora_scanner.py index de24f067..8bc7cd1b 100644 --- a/py/services/lora_scanner.py +++ b/py/services/lora_scanner.py @@ -9,10 +9,10 @@ from operator import itemgetter from ..config import config from ..utils.file_utils import load_metadata, get_file_info from .lora_cache import LoraCache -from difflib import SequenceMatcher from .lora_hash_index import LoraHashIndex from .settings_manager import settings from ..utils.constants import NSFW_LEVELS +from ..utils.utils import fuzzy_match import sys logger = logging.getLogger(__name__) @@ -132,45 +132,9 @@ class LoraScanner: folders=[] ) - def fuzzy_match(self, text: str, pattern: str, threshold: float = 0.7) -> bool: - """ - Check if text matches pattern using fuzzy matching. - Returns True if similarity ratio is above threshold. - """ - if not pattern or not text: - return False - - # Convert both to lowercase for case-insensitive matching - text = text.lower() - pattern = pattern.lower() - - # Split pattern into words - search_words = pattern.split() - - # Check each word - for word in search_words: - # First check if word is a substring (faster) - if word in text: - continue - - # If not found as substring, try fuzzy matching - # Check if any part of the text matches this word - found_match = False - for text_part in text.split(): - ratio = SequenceMatcher(None, text_part, word).ratio() - if ratio >= threshold: - found_match = True - break - - if not found_match: - return False - - # All words found either as substrings or fuzzy matches - return True - async def get_paginated_data(self, page: int, page_size: int, sort_by: str = 'name', folder: str = None, search: str = None, fuzzy: bool = False, - recursive: bool = False, base_models: list = None, tags: list = None, + base_models: list = None, tags: list = None, search_options: dict = None) -> Dict: """Get paginated and filtered lora data @@ -181,10 +145,9 @@ class LoraScanner: folder: Filter by folder path search: Search term fuzzy: Use fuzzy matching for search - recursive: Include subfolders when folder filter is applied base_models: List of base models to filter by tags: List of tags to filter by - search_options: Dictionary with search options (filename, modelname, tags) + search_options: Dictionary with search options (filename, modelname, tags, recursive) """ cache = await self.get_cached_data() @@ -193,7 +156,8 @@ class LoraScanner: search_options = { 'filename': True, 'modelname': True, - 'tags': False + 'tags': False, + 'recursive': False } # Get the base data set @@ -208,7 +172,7 @@ class LoraScanner: # Apply folder filtering if folder is not None: - if recursive: + if search_options.get('recursive', False): # Recursive mode: match all paths starting with this folder filtered_data = [ item for item in filtered_data @@ -237,16 +201,47 @@ class LoraScanner: # Apply search filtering if search: - if fuzzy: - filtered_data = [ - item for item in filtered_data - if self._fuzzy_search_match(item, search, search_options) - ] - else: - filtered_data = [ - item for item in filtered_data - if self._exact_search_match(item, search, search_options) - ] + search_results = [] + for item in filtered_data: + # Check filename if enabled + if search_options.get('filename', True): + if fuzzy: + if fuzzy_match(item.get('file_name', ''), search): + search_results.append(item) + continue + else: + if search.lower() in item.get('file_name', '').lower(): + search_results.append(item) + continue + + # Check model name if enabled + if search_options.get('modelname', True): + if fuzzy: + if fuzzy_match(item.get('model_name', ''), search): + search_results.append(item) + continue + else: + if search.lower() in item.get('model_name', '').lower(): + search_results.append(item) + continue + + # Check tags if enabled + if search_options.get('tags', False) and item.get('tags'): + found_tag = False + for tag in item['tags']: + if fuzzy: + if fuzzy_match(tag, search): + found_tag = True + break + else: + if search.lower() in tag.lower(): + found_tag = True + break + if found_tag: + search_results.append(item) + continue + + filtered_data = search_results # Calculate pagination total_items = len(filtered_data) @@ -263,44 +258,6 @@ class LoraScanner: return result - def _fuzzy_search_match(self, item: Dict, search: str, search_options: Dict) -> bool: - """Check if an item matches the search term using fuzzy matching with search options""" - # Check filename if enabled - if search_options.get('filename', True) and self.fuzzy_match(item.get('file_name', ''), search): - return True - - # Check model name if enabled - if search_options.get('modelname', True) and self.fuzzy_match(item.get('model_name', ''), search): - return True - - # Check tags if enabled - if search_options.get('tags', False) and item.get('tags'): - for tag in item['tags']: - if self.fuzzy_match(tag, search): - return True - - return False - - def _exact_search_match(self, item: Dict, search: str, search_options: Dict) -> bool: - """Check if an item matches the search term using exact matching with search options""" - search = search.lower() - - # Check filename if enabled - if search_options.get('filename', True) and search in item.get('file_name', '').lower(): - return True - - # Check model name if enabled - if search_options.get('modelname', True) and search in item.get('model_name', '').lower(): - return True - - # Check tags if enabled - if search_options.get('tags', False) and item.get('tags'): - for tag in item['tags']: - if search in tag.lower(): - return True - - return False - def invalidate_cache(self): """Invalidate the current cache""" self._cache = None diff --git a/py/services/recipe_scanner.py b/py/services/recipe_scanner.py index b99491a1..29b93561 100644 --- a/py/services/recipe_scanner.py +++ b/py/services/recipe_scanner.py @@ -9,6 +9,7 @@ from ..config import config from .recipe_cache import RecipeCache from .lora_scanner import LoraScanner from .civitai_client import CivitaiClient +from ..utils.utils import fuzzy_match import sys logger = logging.getLogger(__name__) @@ -378,25 +379,26 @@ class RecipeScanner: # Build the search predicate based on search options def matches_search(item): # Search in title if enabled - if search_options.get('title', True) and search.lower() in str(item.get('title', '')).lower(): - return True + if search_options.get('title', True): + if fuzzy_match(str(item.get('title', '')), search): + return True # Search in tags if enabled if search_options.get('tags', True) and 'tags' in item: for tag in item['tags']: - if search.lower() in tag.lower(): + if fuzzy_match(tag, search): return True # Search in lora file names if enabled if search_options.get('lora_name', True) and 'loras' in item: for lora in item['loras']: - if search.lower() in str(lora.get('file_name', '')).lower(): + if fuzzy_match(str(lora.get('file_name', '')), search): return True # Search in lora model names if enabled if search_options.get('lora_model', True) and 'loras' in item: for lora in item['loras']: - if search.lower() in str(lora.get('modelName', '')).lower(): + if fuzzy_match(str(lora.get('modelName', '')), search): return True # No match found diff --git a/py/utils/utils.py b/py/utils/utils.py index 8825a5ec..1a3cf326 100644 --- a/py/utils/utils.py +++ b/py/utils/utils.py @@ -1,3 +1,4 @@ +from difflib import SequenceMatcher import requests import tempfile import re @@ -39,3 +40,39 @@ def download_twitter_image(url): except Exception as e: print(f"Error downloading twitter image: {e}") return None + +def fuzzy_match(text: str, pattern: str, threshold: float = 0.7) -> bool: + """ + Check if text matches pattern using fuzzy matching. + Returns True if similarity ratio is above threshold. + """ + if not pattern or not text: + return False + + # Convert both to lowercase for case-insensitive matching + text = text.lower() + pattern = pattern.lower() + + # Split pattern into words + search_words = pattern.split() + + # Check each word + for word in search_words: + # First check if word is a substring (faster) + if word in text: + continue + + # If not found as substring, try fuzzy matching + # Check if any part of the text matches this word + found_match = False + for text_part in text.split(): + ratio = SequenceMatcher(None, text_part, word).ratio() + if ratio >= threshold: + found_match = True + break + + if not found_match: + return False + + # All words found either as substrings or fuzzy matches + return True diff --git a/static/js/api/loraApi.js b/static/js/api/loraApi.js index 3c63c74a..cb56486d 100644 --- a/static/js/api/loraApi.js +++ b/static/js/api/loraApi.js @@ -18,6 +18,7 @@ export async function loadMoreLoras(resetPage = false, updateFolders = false) { // Clear grid if resetting const grid = document.getElementById('loraGrid'); if (grid) grid.innerHTML = ''; + initializeInfiniteScroll(); } const params = new URLSearchParams({ @@ -26,12 +27,8 @@ export async function loadMoreLoras(resetPage = false, updateFolders = false) { sort_by: pageState.sortBy }); - // Use pageState instead of state - const isRecursiveSearch = pageState.searchOptions?.recursive ?? false; - if (pageState.activeFolder !== null) { params.append('folder', pageState.activeFolder); - params.append('recursive', isRecursiveSearch.toString()); } // Add search parameters if there's a search term @@ -44,6 +41,7 @@ export async function loadMoreLoras(resetPage = false, updateFolders = false) { params.append('search_filename', pageState.searchOptions.filename.toString()); params.append('search_modelname', pageState.searchOptions.modelname.toString()); params.append('search_tags', (pageState.searchOptions.tags || false).toString()); + params.append('recursive', (pageState.searchOptions?.recursive ?? false).toString()); } } diff --git a/static/js/recipes.js b/static/js/recipes.js index ebee41f0..8d9ab47d 100644 --- a/static/js/recipes.js +++ b/static/js/recipes.js @@ -131,8 +131,8 @@ class RecipeManager { // Update recipes grid this.updateRecipesGrid(data, resetPage); - // Update pagination state - this.pageState.hasMore = data.has_more || false; + // Update pagination state based on current page and total pages + this.pageState.hasMore = data.page < data.total_pages; } catch (error) { console.error('Error loading recipes:', error);