diff --git a/locales/de.json b/locales/de.json index 2278ad83..d69d4a26 100644 --- a/locales/de.json +++ b/locales/de.json @@ -935,8 +935,16 @@ }, "duplicates": { "found": "{count} Duplikat-Gruppen gefunden", + "noGroups": "Keine Duplikat-Gruppen mit dem aktuellen Abgleichskriterium gefunden", "keepLatest": "Neueste Versionen behalten", - "deleteSelected": "Ausgewählte löschen" + "deleteSelected": "Ausgewählte löschen", + "includePromptLabel": "Prompt beim Abgleich berücksichtigen", + "basis": { + "loraCombo": "Abgeglichen nach: LoRA-Kombination", + "loraComboAndPrompt": "Abgeglichen nach: LoRA-Kombination + Prompt", + "hintLoraCombo": "Rezepte mit denselben LoRAs bei identischen Stärken werden gruppiert.", + "hintPromptIncluded": "Rezepte werden nur gruppiert, wenn sie dieselben LoRAs bei identischen Stärken UND denselben Prompt verwenden." + } }, "contextMenu": { "copyRecipe": { diff --git a/locales/en.json b/locales/en.json index e54eaa12..9f554140 100644 --- a/locales/en.json +++ b/locales/en.json @@ -935,8 +935,16 @@ }, "duplicates": { "found": "Found {count} duplicate groups", + "noGroups": "No duplicate groups found with the current matching basis", "keepLatest": "Keep Latest Versions", - "deleteSelected": "Delete Selected" + "deleteSelected": "Delete Selected", + "includePromptLabel": "Include prompt in matching", + "basis": { + "loraCombo": "Matched by: LoRA combination", + "loraComboAndPrompt": "Matched by: LoRA combination + prompt", + "hintLoraCombo": "Recipes with the same LoRAs at identical strengths are grouped.", + "hintPromptIncluded": "Recipes are grouped only when they use the same LoRAs at identical strengths AND have the same prompt." + } }, "contextMenu": { "copyRecipe": { diff --git a/locales/es.json b/locales/es.json index fe46a19d..39159d84 100644 --- a/locales/es.json +++ b/locales/es.json @@ -935,8 +935,16 @@ }, "duplicates": { "found": "Se encontraron {count} grupos de duplicados", + "noGroups": "No se encontraron grupos de duplicados con el criterio de coincidencia actual", "keepLatest": "Mantener versiones más recientes", - "deleteSelected": "Eliminar seleccionados" + "deleteSelected": "Eliminar seleccionados", + "includePromptLabel": "Incluir prompt en la coincidencia", + "basis": { + "loraCombo": "Coincidencia por: combinación de LoRA", + "loraComboAndPrompt": "Coincidencia por: combinación de LoRA + prompt", + "hintLoraCombo": "Se agrupan las recetas con los mismos LoRAs y las mismas intensidades.", + "hintPromptIncluded": "Las recetas solo se agrupan cuando usan los mismos LoRAs con intensidades idénticas Y tienen el mismo prompt." + } }, "contextMenu": { "copyRecipe": { diff --git a/locales/fr.json b/locales/fr.json index 9fbffbbc..7be374b2 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -935,8 +935,16 @@ }, "duplicates": { "found": "Trouvé {count} groupes de doublons", + "noGroups": "Aucun groupe de doublons trouvé avec le critère de correspondance actuel", "keepLatest": "Garder les dernières versions", - "deleteSelected": "Supprimer la sélection" + "deleteSelected": "Supprimer la sélection", + "includePromptLabel": "Inclure le prompt dans la correspondance", + "basis": { + "loraCombo": "Correspondance : combinaison de LoRA", + "loraComboAndPrompt": "Correspondance : combinaison de LoRA + prompt", + "hintLoraCombo": "Les recettes avec les mêmes LoRAs et des forces identiques sont regroupées.", + "hintPromptIncluded": "Les recettes ne sont regroupées que si elles utilisent les mêmes LoRAs avec des forces identiques ET ont le même prompt." + } }, "contextMenu": { "copyRecipe": { diff --git a/locales/he.json b/locales/he.json index c1e4db01..e292c775 100644 --- a/locales/he.json +++ b/locales/he.json @@ -935,8 +935,16 @@ }, "duplicates": { "found": "נמצאו {count} קבוצות כפולות", + "noGroups": "לא נמצאו קבוצות כפולות לפי קריטריון ההתאמה הנוכחי", "keepLatest": "שמור גרסאות אחרונות", - "deleteSelected": "מחק נבחרים" + "deleteSelected": "מחק נבחרים", + "includePromptLabel": "כלול הנחיה בהתאמה", + "basis": { + "loraCombo": "התאמה לפי: שילוב LoRA", + "loraComboAndPrompt": "התאמה לפי: שילוב LoRA + הנחיה", + "hintLoraCombo": "מתכונים עם אותם LoRAs בעוצמות זהות מקובצים יחד.", + "hintPromptIncluded": "מתכונים מקובצים רק כאשר הם משתמשים באותם LoRAs בעוצמות זהות ויש להם אותה הנחיה." + } }, "contextMenu": { "copyRecipe": { diff --git a/locales/ja.json b/locales/ja.json index 4a1465ac..7fbf6250 100644 --- a/locales/ja.json +++ b/locales/ja.json @@ -935,8 +935,16 @@ }, "duplicates": { "found": "{count} 個の重複グループが見つかりました", + "noGroups": "現在の一致基準では重複グループが見つかりませんでした", "keepLatest": "最新バージョンを保持", - "deleteSelected": "選択したものを削除" + "deleteSelected": "選択したものを削除", + "includePromptLabel": "一致判定にプロンプトを含める", + "basis": { + "loraCombo": "一致基準: LoRA の組み合わせ", + "loraComboAndPrompt": "一致基準: LoRA の組み合わせ + プロンプト", + "hintLoraCombo": "同じ LoRA を同じ強度で使用するレシピがグループ化されます。", + "hintPromptIncluded": "レシピは、同じ LoRA を同じ強度で使用し、かつプロンプトが同じ場合にのみグループ化されます。" + } }, "contextMenu": { "copyRecipe": { diff --git a/locales/ko.json b/locales/ko.json index d6de1d1c..8eaf4173 100644 --- a/locales/ko.json +++ b/locales/ko.json @@ -935,8 +935,16 @@ }, "duplicates": { "found": "{count}개의 중복 그룹 발견", + "noGroups": "현재 일치 기준으로 중복 그룹을 찾을 수 없습니다", "keepLatest": "최신 버전 유지", - "deleteSelected": "선택된 항목 삭제" + "deleteSelected": "선택된 항목 삭제", + "includePromptLabel": "일치 항목에 프롬프트 포함", + "basis": { + "loraCombo": "일치 기준: LoRA 조합", + "loraComboAndPrompt": "일치 기준: LoRA 조합 + 프롬프트", + "hintLoraCombo": "동일한 LoRA를 동일한 강도로 사용하는 레시피가 그룹화됩니다.", + "hintPromptIncluded": "동일한 LoRA를 동일한 강도로 사용하고 프롬프트도 동일한 경우에만 레시피가 그룹화됩니다." + } }, "contextMenu": { "copyRecipe": { diff --git a/locales/ru.json b/locales/ru.json index 01e6898c..4657ae7d 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -935,8 +935,16 @@ }, "duplicates": { "found": "Найдено {count} групп дубликатов", + "noGroups": "Дубликатов с текущим критерием не найдено", "keepLatest": "Оставить последние версии", - "deleteSelected": "Удалить выбранные" + "deleteSelected": "Удалить выбранные", + "includePromptLabel": "Учитывать запрос при поиске дубликатов", + "basis": { + "loraCombo": "Критерий: комбинация LoRA", + "loraComboAndPrompt": "Критерий: комбинация LoRA + запрос", + "hintLoraCombo": "Рецепты с одинаковыми LoRA и одинаковой силой группируются вместе.", + "hintPromptIncluded": "Рецепты группируются только при одинаковых LoRA с одинаковой силой И одинаковом запросе." + } }, "contextMenu": { "copyRecipe": { diff --git a/locales/zh-CN.json b/locales/zh-CN.json index 7085edf3..4fe7fccb 100644 --- a/locales/zh-CN.json +++ b/locales/zh-CN.json @@ -935,8 +935,16 @@ }, "duplicates": { "found": "发现 {count} 个重复组", + "noGroups": "按当前判重依据未找到重复组", "keepLatest": "保留最新版本", - "deleteSelected": "删除已选" + "deleteSelected": "删除已选", + "includePromptLabel": "将提示词纳入判重", + "basis": { + "loraCombo": "判重依据：LoRA 组合", + "loraComboAndPrompt": "判重依据：LoRA 组合 + 提示词", + "hintLoraCombo": "使用相同 LoRA（强度一致）的配方会被分组。", + "hintPromptIncluded": "仅当配方使用相同的 LoRA（强度一致）且提示词相同时才会被分组。" + } }, "contextMenu": { "copyRecipe": { diff --git a/locales/zh-TW.json b/locales/zh-TW.json index 45f504cb..e1d42d59 100644 --- a/locales/zh-TW.json +++ b/locales/zh-TW.json @@ -935,8 +935,16 @@ }, "duplicates": { "found": "發現 {count} 組重複項", + "noGroups": "按目前判重依據未找到重複組", "keepLatest": "保留最新版本", - "deleteSelected": "刪除所選" + "deleteSelected": "刪除所選", + "includePromptLabel": "將提示詞納入判重", + "basis": { + "loraCombo": "判重依據：LoRA 組合", + "loraComboAndPrompt": "判重依據：LoRA 組合 + 提示詞", + "hintLoraCombo": "使用相同 LoRA（強度一致）的配方會被分組。", + "hintPromptIncluded": "僅當配方使用相同的 LoRA（強度一致）且提示詞相同時才會被分組。" + } }, "contextMenu": { "copyRecipe": { diff --git a/py/routes/handlers/recipe_handlers.py b/py/routes/handlers/recipe_handlers.py index 9f74b720..3ef9de9e 100644 --- a/py/routes/handlers/recipe_handlers.py +++ b/py/routes/handlers/recipe_handlers.py @@ -580,7 +580,12 @@ class RecipeQueryHandler: if recipe_scanner is None: raise RuntimeError("Recipe scanner unavailable") - fingerprint_groups = await recipe_scanner.find_all_duplicate_recipes() + include_prompt = ( + request.query.get("include_prompt", "false").lower() in ("1", "true") + ) + fingerprint_groups = await recipe_scanner.find_all_duplicate_recipes( + include_prompt=include_prompt + ) url_groups = await recipe_scanner.find_duplicate_recipes_by_source() response_data = [] @@ -613,6 +618,7 @@ class RecipeQueryHandler: response_data.append( { "type": "fingerprint", + "key": f"g-{len(response_data) + 1}", "fingerprint": fingerprint, "count": len(recipes), "recipes": recipes, @@ -648,6 +654,7 @@ class RecipeQueryHandler: response_data.append( { "type": "source_path", + "key": f"g-{len(response_data) + 1}", "fingerprint": url, "count": len(recipes), "recipes": recipes, diff --git a/py/services/recipe_scanner.py b/py/services/recipe_scanner.py index 4b53a9f4..72e82fc2 100644 --- a/py/services/recipe_scanner.py +++ b/py/services/recipe_scanner.py @@ -3540,32 +3540,70 @@ class RecipeScanner: return matching_recipes - async def find_all_duplicate_recipes(self) -> Dict[str, List[Any]]: + async def find_all_duplicate_recipes( + self, include_prompt: bool = False + ) -> Dict[str, List[Any]]: """Find all recipe duplicates based on fingerprints + When ``include_prompt`` is True, the grouping key additionally + includes the normalized positive prompt, so recipes are only grouped + when they share both the same LoRA combination (with identical + strengths) and the same prompt. Recipes with neither a fingerprint + nor a prompt are skipped. + + Args: + include_prompt: Whether to require an identical prompt as well + Returns: - Dictionary where keys are fingerprints and values are lists of recipe IDs + Dictionary where keys are grouping keys and values are lists of recipe IDs """ # Get all recipes from cache cache = await self.get_cached_data() - # Group recipes by fingerprint + # Group recipes by fingerprint (optionally combined with the prompt) fingerprint_groups = {} for recipe in cache.raw_data: - fingerprint = recipe.get("fingerprint") - if not fingerprint: + grouping_key = self._build_duplicate_grouping_key( + recipe, include_prompt + ) + if not grouping_key: continue - if fingerprint not in fingerprint_groups: - fingerprint_groups[fingerprint] = [] + if grouping_key not in fingerprint_groups: + fingerprint_groups[grouping_key] = [] - fingerprint_groups[fingerprint].append(recipe.get("id")) + fingerprint_groups[grouping_key].append(recipe.get("id")) # Filter to only include groups with more than one recipe duplicate_groups = {k: v for k, v in fingerprint_groups.items() if len(v) > 1} return duplicate_groups + def _build_duplicate_grouping_key( + self, recipe: Dict[str, Any], include_prompt: bool + ) -> str: + """Build the grouping key used for duplicate detection. + + Without ``include_prompt`` this is the stored fingerprint (same LoRA + combination at identical strengths). With it, the normalized positive + prompt is appended (separated by ``\\x1f``), so recipes must share + both factors to be grouped. Recipes with no loras still participate + when they carry a prompt, matching other no-lora recipes with the + same prompt. + """ + fingerprint = recipe.get("fingerprint") or "" + if not include_prompt: + return fingerprint + + from ..utils.utils import normalize_prompt_for_dedup + + prompt = normalize_prompt_for_dedup( + (recipe.get("gen_params") or {}).get("prompt") + ) + if not fingerprint and not prompt: + return "" + return f"{fingerprint}\x1f{prompt}" + async def find_duplicate_recipes_by_source(self) -> Dict[str, List[Any]]: """Find all recipe duplicates based on source_path (Civitai image URLs) diff --git a/py/utils/utils.py b/py/utils/utils.py index cbf67d3d..8670d497 100644 --- a/py/utils/utils.py +++ b/py/utils/utils.py @@ -469,6 +469,24 @@ def calculate_recipe_fingerprint(loras): return fingerprint +def normalize_prompt_for_dedup(prompt) -> str: + """Normalize a positive prompt for duplicate recipe matching. + + Applies casefolding, collapses whitespace runs into single spaces, and + trims leading/trailing whitespace. Missing or non-string prompts + normalize to an empty string. + + Args: + prompt: The positive prompt text (or None) + + Returns: + str: The normalized prompt + """ + if not prompt or not isinstance(prompt, str): + return "" + return re.sub(r"\s+", " ", prompt).strip().casefold() + + def calculate_relative_path_for_model( model_data: Dict[str, Any], model_type: str = "lora" ) -> str: diff --git a/static/css/components/duplicates.css b/static/css/components/duplicates.css index bef12ad9..fdcd772b 100644 --- a/static/css/components/duplicates.css +++ b/static/css/components/duplicates.css @@ -486,6 +486,26 @@ } } +/* Empty-state hint in the duplicates view */ +.duplicates-empty-state { + padding: 48px 16px; + text-align: center; + opacity: 0.7; + font-size: 0.95em; + width: 100%; +} + +/* Matching basis text in the duplicates banner */ +.duplicates-basis { + font-size: 0.85em; + opacity: 0.8; + padding: 3px 10px; + border-radius: var(--border-radius-xs); + background: oklch(var(--color-accent-l) var(--color-accent-c) var(--color-accent-h) / 0.12); + border: 1px solid oklch(var(--color-accent-l) var(--color-accent-c) var(--color-accent-h) / 0.25); + white-space: nowrap; +} + /* Help icon styling */ .help-icon { color: var(--text-color); diff --git a/static/js/components/DuplicatesManager.js b/static/js/components/DuplicatesManager.js index 5c95c7aa..b17e124f 100644 --- a/static/js/components/DuplicatesManager.js +++ b/static/js/components/DuplicatesManager.js @@ -1,5 +1,6 @@ // Duplicates Manager Component import { showToast } from '../utils/uiHelpers.js'; +import { translate } from '../utils/i18nHelpers.js'; import { RecipeCard } from './RecipeCard.js'; import { state, getCurrentPageState } from '../state/index.js'; import { recreateVirtualScroll } from '../utils/infiniteScroll.js'; @@ -10,11 +11,87 @@ export class DuplicatesManager { this.duplicateGroups = []; this.inDuplicateMode = false; this.selectedForDeletion = new Set(); + this._initPromptMatchToggle(); + this._initHelpTooltip(); } - + + _getPromptMatchPreference() { + return localStorage.getItem('recipes_duplicates_include_prompt') === '1'; + } + + _setPromptMatchPreference(enabled) { + localStorage.setItem('recipes_duplicates_include_prompt', enabled ? '1' : '0'); + } + + updateBasisDisplay() { + const basisEl = document.getElementById('duplicatesBasis'); + const helpTextEl = document.getElementById('duplicatesHelpText'); + const checkbox = document.getElementById('promptMatchInput'); + const includePrompt = this._getPromptMatchPreference(); + if (checkbox) { + checkbox.checked = includePrompt; + } + if (basisEl) { + basisEl.textContent = translate( + includePrompt + ? 'recipes.duplicates.basis.loraComboAndPrompt' + : 'recipes.duplicates.basis.loraCombo' + ); + } + if (helpTextEl) { + helpTextEl.textContent = translate( + includePrompt + ? 'recipes.duplicates.basis.hintPromptIncluded' + : 'recipes.duplicates.basis.hintLoraCombo' + ); + } + } + + _initPromptMatchToggle() { + const checkbox = document.getElementById('promptMatchInput'); + if (!checkbox) return; + checkbox.addEventListener('change', async (e) => { + this._setPromptMatchPreference(e.target.checked); + this.updateBasisDisplay(); + checkbox.disabled = true; + try { + await this.findDuplicates(); + } finally { + checkbox.disabled = false; + } + }); + } + + _initHelpTooltip() { + const helpIcon = document.getElementById('duplicatesHelp'); + const helpTooltip = document.getElementById('duplicatesHelpTooltip'); + if (!helpIcon || !helpTooltip) return; + + helpIcon.addEventListener('mouseenter', () => { + const bannerContent = helpIcon.closest('.banner-content'); + if (!bannerContent) return; + const iconRect = helpIcon.getBoundingClientRect(); + const bannerRect = bannerContent.getBoundingClientRect(); + helpTooltip.style.display = 'block'; + helpTooltip.style.top = `${iconRect.bottom - bannerRect.top + 10}px`; + helpTooltip.style.left = `${iconRect.left - bannerRect.left - 10}px`; + const tooltipRect = helpTooltip.getBoundingClientRect(); + if (tooltipRect.right > window.innerWidth - 20) { + helpTooltip.style.left = `${bannerContent.offsetWidth - tooltipRect.width - 20}px`; + } + }); + helpIcon.addEventListener('mouseleave', () => { + helpTooltip.style.display = 'none'; + }); + } + async findDuplicates() { try { - const response = await fetch('/api/lm/recipes/find-duplicates'); + const includePrompt = this._getPromptMatchPreference(); + const endpoint = includePrompt + ? '/api/lm/recipes/find-duplicates?include_prompt=1' + : '/api/lm/recipes/find-duplicates'; + const response = await fetch(endpoint); if (!response.ok) { throw new Error('Failed to find duplicates'); } @@ -28,7 +105,14 @@ export class DuplicatesManager { if (this.duplicateGroups.length === 0) { showToast('toast.duplicates.noDuplicatesFound', { type: 'recipes' }, 'info'); - return false; + // Keep (or enter) the duplicates view when the user is tuning + // the matching basis, so the prompt-matching toggle stays + // reachable; otherwise just toast and stay on the library grid. + if (!this.inDuplicateMode && !includePrompt) { + return false; + } + this.enterDuplicateMode(); + return true; } this.enterDuplicateMode(); @@ -53,9 +137,14 @@ export class DuplicatesManager { const countSpan = document.getElementById('duplicatesCount'); if (banner && countSpan) { - countSpan.textContent = `Found ${this.duplicateGroups.length} duplicate group${this.duplicateGroups.length !== 1 ? 's' : ''}`; + countSpan.textContent = this.duplicateGroups.length === 0 + ? translate('recipes.duplicates.noGroups') + : translate('recipes.duplicates.found', { count: this.duplicateGroups.length }); banner.style.display = 'block'; } + + // Restore the prompt-matching preference and show the matching basis + this.updateBasisDisplay(); // Disable virtual scrolling if active if (state.virtualScroller) { @@ -113,12 +202,23 @@ export class DuplicatesManager { // Clear existing content recipeGrid.innerHTML = ''; + + // Empty-state view: keep the banner (and the matching-basis toggle) + // reachable when no groups match the current basis + if (this.duplicateGroups.length === 0) { + const emptyState = document.createElement('div'); + emptyState.className = 'duplicates-empty-state'; + emptyState.textContent = translate('recipes.duplicates.noGroups'); + recipeGrid.appendChild(emptyState); + return; + } // Render each duplicate group this.duplicateGroups.forEach((group, groupIndex) => { + const groupKey = group.key; const groupDiv = document.createElement('div'); groupDiv.className = 'duplicate-group'; - groupDiv.dataset.fingerprint = group.fingerprint; + groupDiv.dataset.groupKey = groupKey; // Create group header const header = document.createElement('div'); @@ -126,10 +226,10 @@ export class DuplicatesManager { header.innerHTML = ` Duplicate Group #${groupIndex + 1} (${group.recipes.length} recipes) - - @@ -182,7 +282,7 @@ export class DuplicatesManager { checkbox.type = 'checkbox'; checkbox.className = 'selector-checkbox'; checkbox.dataset.recipeId = recipe.id; - checkbox.dataset.groupFingerprint = group.fingerprint; + checkbox.dataset.groupKey = groupKey; // Check if already selected if (this.selectedForDeletion.has(recipe.id)) { @@ -244,8 +344,8 @@ export class DuplicatesManager { } } - toggleSelectAllInGroup(fingerprint) { - const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-fingerprint="${fingerprint}"]`); + toggleSelectAllInGroup(groupKey) { + const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-key="${groupKey}"]`); const allSelected = Array.from(checkboxes).every(checkbox => checkbox.checked); // If all are selected, deselect all; otherwise select all @@ -264,7 +364,7 @@ export class DuplicatesManager { }); // Update the button text - const button = document.querySelector(`.duplicate-group[data-fingerprint="${fingerprint}"] .btn-select-all`); + const button = document.querySelector(`.duplicate-group[data-group-key="${groupKey}"] .btn-select-all`); if (button) { button.textContent = !allSelected ? "Deselect All" : "Select All"; } @@ -272,8 +372,8 @@ export class DuplicatesManager { this.updateSelectedCount(); } - selectAllInGroup(fingerprint) { - const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-fingerprint="${fingerprint}"]`); + selectAllInGroup(groupKey) { + const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-key="${groupKey}"]`); checkboxes.forEach(checkbox => { checkbox.checked = true; this.selectedForDeletion.add(checkbox.dataset.recipeId); @@ -281,7 +381,7 @@ export class DuplicatesManager { }); // Update the button text - const button = document.querySelector(`.duplicate-group[data-fingerprint="${fingerprint}"] .btn-select-all`); + const button = document.querySelector(`.duplicate-group[data-group-key="${groupKey}"] .btn-select-all`); if (button) { button.textContent = "Deselect All"; } @@ -289,12 +389,12 @@ export class DuplicatesManager { this.updateSelectedCount(); } - selectLatestInGroup(fingerprint) { + selectLatestInGroup(groupKey) { // Find all checkboxes in this group - const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-fingerprint="${fingerprint}"]`); + const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-key="${groupKey}"]`); // Get all the recipes in this group - const group = this.duplicateGroups.find(g => g.fingerprint === fingerprint); + const group = this.duplicateGroups.find(g => g.key === groupKey); if (!group) return; // Sort recipes by date (newest first) @@ -328,7 +428,7 @@ export class DuplicatesManager { selectLatestDuplicates() { // For each duplicate group, select all but the latest recipe this.duplicateGroups.forEach(group => { - this.selectLatestInGroup(group.fingerprint); + this.selectLatestInGroup(group.key); }); } diff --git a/templates/recipes.html b/templates/recipes.html index b10eea65..a847f2bb 100644 --- a/templates/recipes.html +++ b/templates/recipes.html @@ -74,7 +74,16 @@ +
+

+
{% include 'components/folder_sidebar.html' %} diff --git a/tests/frontend/components/duplicatesManager.test.js b/tests/frontend/components/duplicatesManager.test.js index b7527691..6ff6251f 100644 --- a/tests/frontend/components/duplicatesManager.test.js +++ b/tests/frontend/components/duplicatesManager.test.js @@ -2,13 +2,22 @@ import { describe, it, beforeEach, afterEach, expect, vi } from 'vitest'; const showToastMock = vi.fn(); const recreateVirtualScrollMock = vi.fn(); +const translateMock = vi.fn((key) => key); vi.mock('../../../static/js/utils/uiHelpers.js', () => ({ showToast: showToastMock, })); +vi.mock('../../../static/js/utils/i18nHelpers.js', () => ({ + translate: translateMock, +})); + vi.mock('../../../static/js/components/RecipeCard.js', () => ({ - RecipeCard: class {}, + RecipeCard: class { + constructor() { + this.element = document.createElement('div'); + } + }, })); vi.mock('../../../static/js/utils/infiniteScroll.js', () => ({ @@ -85,3 +94,120 @@ describe('DuplicatesManager exitDuplicateMode', () => { expect(document.getElementById('duplicatesBanner').style.display).toBe('none'); }); }); + +describe('DuplicatesManager prompt matching toggle', () => { + beforeEach(() => { + vi.clearAllMocks(); + localStorage.clear(); + setCurrentPageType('recipes'); + setupDom(); + state.pendingLayoutRecreate = false; + state.virtualScroller = { enable: vi.fn(), disable: vi.fn() }; + }); + + afterEach(() => { + state.pendingLayoutRecreate = false; + state.virtualScroller = null; + }); + + it('sends include_prompt=1 when the preference is enabled', async () => { + localStorage.setItem('recipes_duplicates_include_prompt', '1'); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ + success: true, + duplicate_groups: [ + { type: 'fingerprint', key: 'g-1', fingerprint: 'abc:0.8', count: 2, recipes: [{ id: 'r1', modified: 1 }, { id: 'r2', modified: 2 }] }, + ], + }), + }); + + const manager = new DuplicatesManager({}); + await manager.findDuplicates(); + + expect(globalThis.fetch).toHaveBeenCalledWith('/api/lm/recipes/find-duplicates?include_prompt=1'); + }); + + it('calls the endpoint without the param when disabled', async () => { + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ success: true, duplicate_groups: [] }), + }); + + const manager = new DuplicatesManager({}); + await manager.findDuplicates(); + + expect(globalThis.fetch).toHaveBeenCalledWith('/api/lm/recipes/find-duplicates'); + }); + + it('stays in duplicate mode with an empty view when a re-run finds no groups', async () => { + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ success: true, duplicate_groups: [] }), + }); + + const manager = new DuplicatesManager({}); + manager.inDuplicateMode = true; + await manager.findDuplicates(); + + // The view stays open (with the empty state) so the matching-basis + // toggle remains reachable — the deadlock fix + expect(manager.inDuplicateMode).toBe(true); + expect(manager.duplicateGroups).toEqual([]); + expect(document.getElementById('duplicatesBanner').style.display).toBe('block'); + expect(document.querySelector('.duplicates-empty-state')).not.toBeNull(); + }); + + it('enters the empty duplicates view when the toggle is on but no groups match', async () => { + localStorage.setItem('recipes_duplicates_include_prompt', '1'); + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ success: true, duplicate_groups: [] }), + }); + + const manager = new DuplicatesManager({}); + await manager.findDuplicates(); + + expect(manager.inDuplicateMode).toBe(true); + expect(document.getElementById('duplicatesBanner').style.display).toBe('block'); + }); + + it('toasts and stays on the library grid when the toggle is off and no groups match', async () => { + globalThis.fetch = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ success: true, duplicate_groups: [] }), + }); + + const manager = new DuplicatesManager({}); + await manager.findDuplicates(); + + expect(manager.inDuplicateMode).toBe(false); + expect(showToastMock).toHaveBeenCalledWith('toast.duplicates.noDuplicatesFound', { type: 'recipes' }, 'info'); + }); + + it('renders the matching basis and checkbox from the stored preference', () => { + document.body.innerHTML = ` + + + + `; + localStorage.setItem('recipes_duplicates_include_prompt', '1'); + + const manager = new DuplicatesManager({}); + manager.updateBasisDisplay(); + + expect(translateMock).toHaveBeenCalledWith('recipes.duplicates.basis.loraComboAndPrompt'); + expect(translateMock).toHaveBeenCalledWith('recipes.duplicates.basis.hintPromptIncluded'); + expect(document.getElementById('promptMatchInput').checked).toBe(true); + }); + + it('shows the lora-combo basis when the preference is disabled', () => { + document.body.innerHTML = ``; + + const manager = new DuplicatesManager({}); + manager.updateBasisDisplay(); + + expect(translateMock).toHaveBeenCalledWith('recipes.duplicates.basis.loraCombo'); + expect(document.getElementById('duplicatesBasis').textContent).toBe('recipes.duplicates.basis.loraCombo'); + }); +}); diff --git a/tests/routes/test_recipe_routes.py b/tests/routes/test_recipe_routes.py index c0085c80..af632cab 100644 --- a/tests/routes/test_recipe_routes.py +++ b/tests/routes/test_recipe_routes.py @@ -97,6 +97,15 @@ class StubRecipeScanner: async def get_recipe_by_id(self, recipe_id: str) -> Optional[Dict[str, Any]]: return self.recipes.get(recipe_id) + async def find_all_duplicate_recipes( + self, include_prompt: bool = False + ) -> Dict[str, List[Any]]: + self.last_duplicate_include_prompt = include_prompt + return dict(getattr(self, "duplicate_groups_override", {})) + + async def find_duplicate_recipes_by_source(self) -> Dict[str, List[Any]]: + return dict(getattr(self, "duplicate_source_groups_override", {})) + async def get_recipes_for_lora(self, lora_hash: str) -> List[Dict[str, Any]]: return list(self.lora_lookup.get(lora_hash.lower(), [])) @@ -1951,3 +1960,56 @@ async def test_get_rematch_progress_returns_stored_progress( assert response.status == 200 assert payload["success"] is True assert payload["progress"]["status"] == "processing" + + +async def test_find_duplicates_defaults_to_fingerprint_only( + monkeypatch, tmp_path: Path +) -> None: + async with recipe_harness(monkeypatch, tmp_path) as harness: + harness.scanner.recipes = { + "r1": {"id": "r1", "title": "One", "modified": 100}, + "r2": {"id": "r2", "title": "Two", "modified": 200}, + } + harness.scanner.duplicate_groups_override = {"abc:0.8": ["r1", "r2"]} + harness.scanner.duplicate_source_groups_override = {} + + response = await harness.client.get("/api/lm/recipes/find-duplicates") + payload = await response.json() + + assert response.status == 200 + assert payload["success"] is True + assert harness.scanner.last_duplicate_include_prompt is False + assert len(payload["duplicate_groups"]) == 1 + group = payload["duplicate_groups"][0] + assert group["type"] == "fingerprint" + assert group["key"] == "g-1" + assert group["fingerprint"] == "abc:0.8" + assert group["count"] == 2 + + +async def test_find_duplicates_forwards_include_prompt_and_assigns_unique_keys( + monkeypatch, tmp_path: Path +) -> None: + async with recipe_harness(monkeypatch, tmp_path) as harness: + harness.scanner.recipes = { + "r1": {"id": "r1", "title": "One", "modified": 100}, + "r2": {"id": "r2", "title": "Two", "modified": 200}, + "r3": {"id": "r3", "title": "Three", "modified": 300}, + "r4": {"id": "r4", "title": "Four", "modified": 400}, + } + harness.scanner.duplicate_groups_override = {"abc:0.8\x1fa girl": ["r1", "r2"]} + harness.scanner.duplicate_source_groups_override = { + "civitai.com/images/9": ["r3", "r4"] + } + + response = await harness.client.get( + "/api/lm/recipes/find-duplicates?include_prompt=1" + ) + payload = await response.json() + + assert response.status == 200 + assert harness.scanner.last_duplicate_include_prompt is True + groups = payload["duplicate_groups"] + assert len(groups) == 2 + assert {g["type"] for g in groups} == {"fingerprint", "source_path"} + assert len({g["key"] for g in groups}) == 2 diff --git a/tests/services/test_recipe_scanner.py b/tests/services/test_recipe_scanner.py index 84b77fda..2263cc60 100644 --- a/tests/services/test_recipe_scanner.py +++ b/tests/services/test_recipe_scanner.py @@ -3209,3 +3209,71 @@ async def test_rematch_all_autov3_cache_reuse_across_calls( # are read once across both calls (Oracle R2-F4). assert len(called) == 1 + + +async def test_find_all_duplicate_recipes_groups_by_fingerprint(recipe_scanner, monkeypatch): + scanner, _ = recipe_scanner + cache = SimpleNamespace( + raw_data=[ + {"id": "r1", "fingerprint": "abc:0.8", "gen_params": {"prompt": "A Girl, blue hair"}}, + {"id": "r2", "fingerprint": "abc:0.8", "gen_params": {"prompt": "a boy"}}, + {"id": "r3", "fingerprint": "abc:0.8", "gen_params": {"prompt": "a boy"}}, + {"id": "r4", "fingerprint": "def:1.0", "gen_params": {"prompt": "A Girl, blue hair"}}, + {"id": "r5", "fingerprint": "", "gen_params": {"prompt": "landscape"}}, + {"id": "r6", "fingerprint": "", "gen_params": {}}, + ] + ) + async def fake_get_cached_data(): + return cache + monkeypatch.setattr(scanner, "get_cached_data", fake_get_cached_data) + + groups = await scanner.find_all_duplicate_recipes() + assert groups == {"abc:0.8": ["r1", "r2", "r3"]} + + +async def test_find_all_duplicate_recipes_include_prompt_composite_key(recipe_scanner, monkeypatch): + scanner, _ = recipe_scanner + cache = SimpleNamespace( + raw_data=[ + {"id": "r1", "fingerprint": "abc:0.8", "gen_params": {"prompt": "A Girl, blue hair"}}, + {"id": "r2", "fingerprint": "abc:0.8", "gen_params": {"prompt": "a girl, blue hair"}}, + {"id": "r3", "fingerprint": "abc:0.8", "gen_params": {"prompt": "a boy"}}, + {"id": "r4", "fingerprint": "", "gen_params": {"prompt": " landscape "}}, + {"id": "r5", "fingerprint": "", "gen_params": {"prompt": "landscape"}}, + {"id": "r6", "fingerprint": "", "gen_params": {}}, + {"id": "r7", "fingerprint": "def:1.0", "gen_params": {"prompt": "a girl, blue hair"}}, + ] + ) + async def fake_get_cached_data(): + return cache + monkeypatch.setattr(scanner, "get_cached_data", fake_get_cached_data) + + groups = await scanner.find_all_duplicate_recipes(include_prompt=True) + assert groups == { + "abc:0.8\x1fa girl, blue hair": ["r1", "r2"], + "\x1flandscape": ["r4", "r5"], + } + # Same-lora recipes with different prompts are no longer duplicates + assert "abc:0.8\x1fa boy" not in groups + # Different-lora recipes with the same prompt are not grouped either + assert "def:1.0\x1fa girl, blue hair" not in groups + # Recipes with neither fingerprint nor prompt are skipped + assert "r6" not in [rid for ids in groups.values() for rid in ids] + + +async def test_find_all_duplicate_recipes_include_prompt_missing_gen_params(recipe_scanner, monkeypatch): + scanner, _ = recipe_scanner + cache = SimpleNamespace( + raw_data=[ + {"id": "r1", "fingerprint": "abc:0.8"}, + {"id": "r2", "fingerprint": "abc:0.8"}, + {"id": "r3", "fingerprint": "abc:0.8", "gen_params": {"prompt": "a boy"}}, + ] + ) + async def fake_get_cached_data(): + return cache + monkeypatch.setattr(scanner, "get_cached_data", fake_get_cached_data) + + groups = await scanner.find_all_duplicate_recipes(include_prompt=True) + # Recipes without gen_params/prompt normalize to empty prompt and match + assert groups == {"abc:0.8\x1f": ["r1", "r2"]} diff --git a/tests/utils/test_fingerprint_fallback.py b/tests/utils/test_fingerprint_fallback.py index cde80915..b8b95f13 100644 --- a/tests/utils/test_fingerprint_fallback.py +++ b/tests/utils/test_fingerprint_fallback.py @@ -1,6 +1,6 @@ """Test for modelVersionId fallback in fingerprint calculation.""" import pytest -from py.utils.utils import calculate_recipe_fingerprint +from py.utils.utils import calculate_recipe_fingerprint, normalize_prompt_for_dedup def test_calculate_fingerprint_with_model_version_id_fallback(): @@ -98,3 +98,15 @@ def test_calculate_fingerprint_without_hash_or_version_id(): ] fingerprint = calculate_recipe_fingerprint(loras) assert fingerprint == "" + + +def test_normalize_prompt_casefolds_and_collapses_whitespace(): + assert normalize_prompt_for_dedup("A Girl, blue hair") == "a girl, blue hair" + assert normalize_prompt_for_dedup(" landscape \n\t with details ") == "landscape with details" + assert normalize_prompt_for_dedup("MASTERPIECE, best quality") == "masterpiece, best quality" + + +def test_normalize_prompt_handles_missing_or_non_string(): + assert normalize_prompt_for_dedup(None) == "" + assert normalize_prompt_for_dedup("") == "" + assert normalize_prompt_for_dedup(12345) == ""