feat(recipes): add prompt-aware duplicate detection toggle

This commit is contained in:
Will Miao
2026-08-10 00:07:14 +08:00
parent 8237e5f9ea
commit 95fb3c7fc9
20 changed files with 582 additions and 39 deletions

View File

@@ -935,8 +935,16 @@
},
"duplicates": {
"found": "{count} Duplikat-Gruppen gefunden",
"noGroups": "Keine Duplikat-Gruppen mit dem aktuellen Abgleichskriterium gefunden",
"keepLatest": "Neueste Versionen behalten",
"deleteSelected": "Ausgewählte löschen"
"deleteSelected": "Ausgewählte löschen",
"includePromptLabel": "Prompt beim Abgleich berücksichtigen",
"basis": {
"loraCombo": "Abgeglichen nach: LoRA-Kombination",
"loraComboAndPrompt": "Abgeglichen nach: LoRA-Kombination + Prompt",
"hintLoraCombo": "Rezepte mit denselben LoRAs bei identischen Stärken werden gruppiert.",
"hintPromptIncluded": "Rezepte werden nur gruppiert, wenn sie dieselben LoRAs bei identischen Stärken UND denselben Prompt verwenden."
}
},
"contextMenu": {
"copyRecipe": {

View File

@@ -935,8 +935,16 @@
},
"duplicates": {
"found": "Found {count} duplicate groups",
"noGroups": "No duplicate groups found with the current matching basis",
"keepLatest": "Keep Latest Versions",
"deleteSelected": "Delete Selected"
"deleteSelected": "Delete Selected",
"includePromptLabel": "Include prompt in matching",
"basis": {
"loraCombo": "Matched by: LoRA combination",
"loraComboAndPrompt": "Matched by: LoRA combination + prompt",
"hintLoraCombo": "Recipes with the same LoRAs at identical strengths are grouped.",
"hintPromptIncluded": "Recipes are grouped only when they use the same LoRAs at identical strengths AND have the same prompt."
}
},
"contextMenu": {
"copyRecipe": {

View File

@@ -935,8 +935,16 @@
},
"duplicates": {
"found": "Se encontraron {count} grupos de duplicados",
"noGroups": "No se encontraron grupos de duplicados con el criterio de coincidencia actual",
"keepLatest": "Mantener versiones más recientes",
"deleteSelected": "Eliminar seleccionados"
"deleteSelected": "Eliminar seleccionados",
"includePromptLabel": "Incluir prompt en la coincidencia",
"basis": {
"loraCombo": "Coincidencia por: combinación de LoRA",
"loraComboAndPrompt": "Coincidencia por: combinación de LoRA + prompt",
"hintLoraCombo": "Se agrupan las recetas con los mismos LoRAs y las mismas intensidades.",
"hintPromptIncluded": "Las recetas solo se agrupan cuando usan los mismos LoRAs con intensidades idénticas Y tienen el mismo prompt."
}
},
"contextMenu": {
"copyRecipe": {

View File

@@ -935,8 +935,16 @@
},
"duplicates": {
"found": "Trouvé {count} groupes de doublons",
"noGroups": "Aucun groupe de doublons trouvé avec le critère de correspondance actuel",
"keepLatest": "Garder les dernières versions",
"deleteSelected": "Supprimer la sélection"
"deleteSelected": "Supprimer la sélection",
"includePromptLabel": "Inclure le prompt dans la correspondance",
"basis": {
"loraCombo": "Correspondance : combinaison de LoRA",
"loraComboAndPrompt": "Correspondance : combinaison de LoRA + prompt",
"hintLoraCombo": "Les recettes avec les mêmes LoRAs et des forces identiques sont regroupées.",
"hintPromptIncluded": "Les recettes ne sont regroupées que si elles utilisent les mêmes LoRAs avec des forces identiques ET ont le même prompt."
}
},
"contextMenu": {
"copyRecipe": {

View File

@@ -935,8 +935,16 @@
},
"duplicates": {
"found": "נמצאו {count} קבוצות כפולות",
"noGroups": "לא נמצאו קבוצות כפולות לפי קריטריון ההתאמה הנוכחי",
"keepLatest": "שמור גרסאות אחרונות",
"deleteSelected": "מחק נבחרים"
"deleteSelected": "מחק נבחרים",
"includePromptLabel": "כלול הנחיה בהתאמה",
"basis": {
"loraCombo": "התאמה לפי: שילוב LoRA",
"loraComboAndPrompt": "התאמה לפי: שילוב LoRA + הנחיה",
"hintLoraCombo": "מתכונים עם אותם LoRAs בעוצמות זהות מקובצים יחד.",
"hintPromptIncluded": "מתכונים מקובצים רק כאשר הם משתמשים באותם LoRAs בעוצמות זהות ויש להם אותה הנחיה."
}
},
"contextMenu": {
"copyRecipe": {

View File

@@ -935,8 +935,16 @@
},
"duplicates": {
"found": "{count} 個の重複グループが見つかりました",
"noGroups": "現在の一致基準では重複グループが見つかりませんでした",
"keepLatest": "最新バージョンを保持",
"deleteSelected": "選択したものを削除"
"deleteSelected": "選択したものを削除",
"includePromptLabel": "一致判定にプロンプトを含める",
"basis": {
"loraCombo": "一致基準: LoRA の組み合わせ",
"loraComboAndPrompt": "一致基準: LoRA の組み合わせ + プロンプト",
"hintLoraCombo": "同じ LoRA を同じ強度で使用するレシピがグループ化されます。",
"hintPromptIncluded": "レシピは、同じ LoRA を同じ強度で使用し、かつプロンプトが同じ場合にのみグループ化されます。"
}
},
"contextMenu": {
"copyRecipe": {

View File

@@ -935,8 +935,16 @@
},
"duplicates": {
"found": "{count}개의 중복 그룹 발견",
"noGroups": "현재 일치 기준으로 중복 그룹을 찾을 수 없습니다",
"keepLatest": "최신 버전 유지",
"deleteSelected": "선택된 항목 삭제"
"deleteSelected": "선택된 항목 삭제",
"includePromptLabel": "일치 항목에 프롬프트 포함",
"basis": {
"loraCombo": "일치 기준: LoRA 조합",
"loraComboAndPrompt": "일치 기준: LoRA 조합 + 프롬프트",
"hintLoraCombo": "동일한 LoRA를 동일한 강도로 사용하는 레시피가 그룹화됩니다.",
"hintPromptIncluded": "동일한 LoRA를 동일한 강도로 사용하고 프롬프트도 동일한 경우에만 레시피가 그룹화됩니다."
}
},
"contextMenu": {
"copyRecipe": {

View File

@@ -935,8 +935,16 @@
},
"duplicates": {
"found": "Найдено {count} групп дубликатов",
"noGroups": "Дубликатов с текущим критерием не найдено",
"keepLatest": "Оставить последние версии",
"deleteSelected": "Удалить выбранные"
"deleteSelected": "Удалить выбранные",
"includePromptLabel": "Учитывать запрос при поиске дубликатов",
"basis": {
"loraCombo": "Критерий: комбинация LoRA",
"loraComboAndPrompt": "Критерий: комбинация LoRA + запрос",
"hintLoraCombo": "Рецепты с одинаковыми LoRA и одинаковой силой группируются вместе.",
"hintPromptIncluded": "Рецепты группируются только при одинаковых LoRA с одинаковой силой И одинаковом запросе."
}
},
"contextMenu": {
"copyRecipe": {

View File

@@ -935,8 +935,16 @@
},
"duplicates": {
"found": "发现 {count} 个重复组",
"noGroups": "按当前判重依据未找到重复组",
"keepLatest": "保留最新版本",
"deleteSelected": "删除已选"
"deleteSelected": "删除已选",
"includePromptLabel": "将提示词纳入判重",
"basis": {
"loraCombo": "判重依据LoRA 组合",
"loraComboAndPrompt": "判重依据LoRA 组合 + 提示词",
"hintLoraCombo": "使用相同 LoRA强度一致的配方会被分组。",
"hintPromptIncluded": "仅当配方使用相同的 LoRA强度一致且提示词相同时才会被分组。"
}
},
"contextMenu": {
"copyRecipe": {

View File

@@ -935,8 +935,16 @@
},
"duplicates": {
"found": "發現 {count} 組重複項",
"noGroups": "按目前判重依據未找到重複組",
"keepLatest": "保留最新版本",
"deleteSelected": "刪除所選"
"deleteSelected": "刪除所選",
"includePromptLabel": "將提示詞納入判重",
"basis": {
"loraCombo": "判重依據LoRA 組合",
"loraComboAndPrompt": "判重依據LoRA 組合 + 提示詞",
"hintLoraCombo": "使用相同 LoRA強度一致的配方會被分組。",
"hintPromptIncluded": "僅當配方使用相同的 LoRA強度一致且提示詞相同時才會被分組。"
}
},
"contextMenu": {
"copyRecipe": {

View File

@@ -580,7 +580,12 @@ class RecipeQueryHandler:
if recipe_scanner is None:
raise RuntimeError("Recipe scanner unavailable")
fingerprint_groups = await recipe_scanner.find_all_duplicate_recipes()
include_prompt = (
request.query.get("include_prompt", "false").lower() in ("1", "true")
)
fingerprint_groups = await recipe_scanner.find_all_duplicate_recipes(
include_prompt=include_prompt
)
url_groups = await recipe_scanner.find_duplicate_recipes_by_source()
response_data = []
@@ -613,6 +618,7 @@ class RecipeQueryHandler:
response_data.append(
{
"type": "fingerprint",
"key": f"g-{len(response_data) + 1}",
"fingerprint": fingerprint,
"count": len(recipes),
"recipes": recipes,
@@ -648,6 +654,7 @@ class RecipeQueryHandler:
response_data.append(
{
"type": "source_path",
"key": f"g-{len(response_data) + 1}",
"fingerprint": url,
"count": len(recipes),
"recipes": recipes,

View File

@@ -3540,32 +3540,70 @@ class RecipeScanner:
return matching_recipes
async def find_all_duplicate_recipes(self) -> Dict[str, List[Any]]:
async def find_all_duplicate_recipes(
self, include_prompt: bool = False
) -> Dict[str, List[Any]]:
"""Find all recipe duplicates based on fingerprints
When ``include_prompt`` is True, the grouping key additionally
includes the normalized positive prompt, so recipes are only grouped
when they share both the same LoRA combination (with identical
strengths) and the same prompt. Recipes with neither a fingerprint
nor a prompt are skipped.
Args:
include_prompt: Whether to require an identical prompt as well
Returns:
Dictionary where keys are fingerprints and values are lists of recipe IDs
Dictionary where keys are grouping keys and values are lists of recipe IDs
"""
# Get all recipes from cache
cache = await self.get_cached_data()
# Group recipes by fingerprint
# Group recipes by fingerprint (optionally combined with the prompt)
fingerprint_groups = {}
for recipe in cache.raw_data:
fingerprint = recipe.get("fingerprint")
if not fingerprint:
grouping_key = self._build_duplicate_grouping_key(
recipe, include_prompt
)
if not grouping_key:
continue
if fingerprint not in fingerprint_groups:
fingerprint_groups[fingerprint] = []
if grouping_key not in fingerprint_groups:
fingerprint_groups[grouping_key] = []
fingerprint_groups[fingerprint].append(recipe.get("id"))
fingerprint_groups[grouping_key].append(recipe.get("id"))
# Filter to only include groups with more than one recipe
duplicate_groups = {k: v for k, v in fingerprint_groups.items() if len(v) > 1}
return duplicate_groups
def _build_duplicate_grouping_key(
self, recipe: Dict[str, Any], include_prompt: bool
) -> str:
"""Build the grouping key used for duplicate detection.
Without ``include_prompt`` this is the stored fingerprint (same LoRA
combination at identical strengths). With it, the normalized positive
prompt is appended (separated by ``\\x1f``), so recipes must share
both factors to be grouped. Recipes with no loras still participate
when they carry a prompt, matching other no-lora recipes with the
same prompt.
"""
fingerprint = recipe.get("fingerprint") or ""
if not include_prompt:
return fingerprint
from ..utils.utils import normalize_prompt_for_dedup
prompt = normalize_prompt_for_dedup(
(recipe.get("gen_params") or {}).get("prompt")
)
if not fingerprint and not prompt:
return ""
return f"{fingerprint}\x1f{prompt}"
async def find_duplicate_recipes_by_source(self) -> Dict[str, List[Any]]:
"""Find all recipe duplicates based on source_path (Civitai image URLs)

View File

@@ -469,6 +469,24 @@ def calculate_recipe_fingerprint(loras):
return fingerprint
def normalize_prompt_for_dedup(prompt) -> str:
"""Normalize a positive prompt for duplicate recipe matching.
Applies casefolding, collapses whitespace runs into single spaces, and
trims leading/trailing whitespace. Missing or non-string prompts
normalize to an empty string.
Args:
prompt: The positive prompt text (or None)
Returns:
str: The normalized prompt
"""
if not prompt or not isinstance(prompt, str):
return ""
return re.sub(r"\s+", " ", prompt).strip().casefold()
def calculate_relative_path_for_model(
model_data: Dict[str, Any], model_type: str = "lora"
) -> str:

View File

@@ -486,6 +486,26 @@
}
}
/* Empty-state hint in the duplicates view */
.duplicates-empty-state {
padding: 48px 16px;
text-align: center;
opacity: 0.7;
font-size: 0.95em;
width: 100%;
}
/* Matching basis text in the duplicates banner */
.duplicates-basis {
font-size: 0.85em;
opacity: 0.8;
padding: 3px 10px;
border-radius: var(--border-radius-xs);
background: oklch(var(--color-accent-l) var(--color-accent-c) var(--color-accent-h) / 0.12);
border: 1px solid oklch(var(--color-accent-l) var(--color-accent-c) var(--color-accent-h) / 0.25);
white-space: nowrap;
}
/* Help icon styling */
.help-icon {
color: var(--text-color);

View File

@@ -1,5 +1,6 @@
// Duplicates Manager Component
import { showToast } from '../utils/uiHelpers.js';
import { translate } from '../utils/i18nHelpers.js';
import { RecipeCard } from './RecipeCard.js';
import { state, getCurrentPageState } from '../state/index.js';
import { recreateVirtualScroll } from '../utils/infiniteScroll.js';
@@ -10,11 +11,87 @@ export class DuplicatesManager {
this.duplicateGroups = [];
this.inDuplicateMode = false;
this.selectedForDeletion = new Set();
this._initPromptMatchToggle();
this._initHelpTooltip();
}
_getPromptMatchPreference() {
return localStorage.getItem('recipes_duplicates_include_prompt') === '1';
}
_setPromptMatchPreference(enabled) {
localStorage.setItem('recipes_duplicates_include_prompt', enabled ? '1' : '0');
}
updateBasisDisplay() {
const basisEl = document.getElementById('duplicatesBasis');
const helpTextEl = document.getElementById('duplicatesHelpText');
const checkbox = document.getElementById('promptMatchInput');
const includePrompt = this._getPromptMatchPreference();
if (checkbox) {
checkbox.checked = includePrompt;
}
if (basisEl) {
basisEl.textContent = translate(
includePrompt
? 'recipes.duplicates.basis.loraComboAndPrompt'
: 'recipes.duplicates.basis.loraCombo'
);
}
if (helpTextEl) {
helpTextEl.textContent = translate(
includePrompt
? 'recipes.duplicates.basis.hintPromptIncluded'
: 'recipes.duplicates.basis.hintLoraCombo'
);
}
}
_initPromptMatchToggle() {
const checkbox = document.getElementById('promptMatchInput');
if (!checkbox) return;
checkbox.addEventListener('change', async (e) => {
this._setPromptMatchPreference(e.target.checked);
this.updateBasisDisplay();
checkbox.disabled = true;
try {
await this.findDuplicates();
} finally {
checkbox.disabled = false;
}
});
}
_initHelpTooltip() {
const helpIcon = document.getElementById('duplicatesHelp');
const helpTooltip = document.getElementById('duplicatesHelpTooltip');
if (!helpIcon || !helpTooltip) return;
helpIcon.addEventListener('mouseenter', () => {
const bannerContent = helpIcon.closest('.banner-content');
if (!bannerContent) return;
const iconRect = helpIcon.getBoundingClientRect();
const bannerRect = bannerContent.getBoundingClientRect();
helpTooltip.style.display = 'block';
helpTooltip.style.top = `${iconRect.bottom - bannerRect.top + 10}px`;
helpTooltip.style.left = `${iconRect.left - bannerRect.left - 10}px`;
const tooltipRect = helpTooltip.getBoundingClientRect();
if (tooltipRect.right > window.innerWidth - 20) {
helpTooltip.style.left = `${bannerContent.offsetWidth - tooltipRect.width - 20}px`;
}
});
helpIcon.addEventListener('mouseleave', () => {
helpTooltip.style.display = 'none';
});
}
async findDuplicates() {
try {
const response = await fetch('/api/lm/recipes/find-duplicates');
const includePrompt = this._getPromptMatchPreference();
const endpoint = includePrompt
? '/api/lm/recipes/find-duplicates?include_prompt=1'
: '/api/lm/recipes/find-duplicates';
const response = await fetch(endpoint);
if (!response.ok) {
throw new Error('Failed to find duplicates');
}
@@ -28,7 +105,14 @@ export class DuplicatesManager {
if (this.duplicateGroups.length === 0) {
showToast('toast.duplicates.noDuplicatesFound', { type: 'recipes' }, 'info');
return false;
// Keep (or enter) the duplicates view when the user is tuning
// the matching basis, so the prompt-matching toggle stays
// reachable; otherwise just toast and stay on the library grid.
if (!this.inDuplicateMode && !includePrompt) {
return false;
}
this.enterDuplicateMode();
return true;
}
this.enterDuplicateMode();
@@ -53,9 +137,14 @@ export class DuplicatesManager {
const countSpan = document.getElementById('duplicatesCount');
if (banner && countSpan) {
countSpan.textContent = `Found ${this.duplicateGroups.length} duplicate group${this.duplicateGroups.length !== 1 ? 's' : ''}`;
countSpan.textContent = this.duplicateGroups.length === 0
? translate('recipes.duplicates.noGroups')
: translate('recipes.duplicates.found', { count: this.duplicateGroups.length });
banner.style.display = 'block';
}
// Restore the prompt-matching preference and show the matching basis
this.updateBasisDisplay();
// Disable virtual scrolling if active
if (state.virtualScroller) {
@@ -113,12 +202,23 @@ export class DuplicatesManager {
// Clear existing content
recipeGrid.innerHTML = '';
// Empty-state view: keep the banner (and the matching-basis toggle)
// reachable when no groups match the current basis
if (this.duplicateGroups.length === 0) {
const emptyState = document.createElement('div');
emptyState.className = 'duplicates-empty-state';
emptyState.textContent = translate('recipes.duplicates.noGroups');
recipeGrid.appendChild(emptyState);
return;
}
// Render each duplicate group
this.duplicateGroups.forEach((group, groupIndex) => {
const groupKey = group.key;
const groupDiv = document.createElement('div');
groupDiv.className = 'duplicate-group';
groupDiv.dataset.fingerprint = group.fingerprint;
groupDiv.dataset.groupKey = groupKey;
// Create group header
const header = document.createElement('div');
@@ -126,10 +226,10 @@ export class DuplicatesManager {
header.innerHTML = `
<span>Duplicate Group #${groupIndex + 1} (${group.recipes.length} recipes)</span>
<span>
<button class="btn-select-all" onclick="recipeManager.duplicatesManager.toggleSelectAllInGroup('${group.fingerprint}')">
<button class="btn-select-all" onclick="recipeManager.duplicatesManager.toggleSelectAllInGroup('${groupKey}')">
Select All
</button>
<button class="btn-select-latest" onclick="recipeManager.duplicatesManager.selectLatestInGroup('${group.fingerprint}')">
<button class="btn-select-latest" onclick="recipeManager.duplicatesManager.selectLatestInGroup('${groupKey}')">
Keep Latest
</button>
</span>
@@ -182,7 +282,7 @@ export class DuplicatesManager {
checkbox.type = 'checkbox';
checkbox.className = 'selector-checkbox';
checkbox.dataset.recipeId = recipe.id;
checkbox.dataset.groupFingerprint = group.fingerprint;
checkbox.dataset.groupKey = groupKey;
// Check if already selected
if (this.selectedForDeletion.has(recipe.id)) {
@@ -244,8 +344,8 @@ export class DuplicatesManager {
}
}
toggleSelectAllInGroup(fingerprint) {
const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-fingerprint="${fingerprint}"]`);
toggleSelectAllInGroup(groupKey) {
const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-key="${groupKey}"]`);
const allSelected = Array.from(checkboxes).every(checkbox => checkbox.checked);
// If all are selected, deselect all; otherwise select all
@@ -264,7 +364,7 @@ export class DuplicatesManager {
});
// Update the button text
const button = document.querySelector(`.duplicate-group[data-fingerprint="${fingerprint}"] .btn-select-all`);
const button = document.querySelector(`.duplicate-group[data-group-key="${groupKey}"] .btn-select-all`);
if (button) {
button.textContent = !allSelected ? "Deselect All" : "Select All";
}
@@ -272,8 +372,8 @@ export class DuplicatesManager {
this.updateSelectedCount();
}
selectAllInGroup(fingerprint) {
const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-fingerprint="${fingerprint}"]`);
selectAllInGroup(groupKey) {
const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-key="${groupKey}"]`);
checkboxes.forEach(checkbox => {
checkbox.checked = true;
this.selectedForDeletion.add(checkbox.dataset.recipeId);
@@ -281,7 +381,7 @@ export class DuplicatesManager {
});
// Update the button text
const button = document.querySelector(`.duplicate-group[data-fingerprint="${fingerprint}"] .btn-select-all`);
const button = document.querySelector(`.duplicate-group[data-group-key="${groupKey}"] .btn-select-all`);
if (button) {
button.textContent = "Deselect All";
}
@@ -289,12 +389,12 @@ export class DuplicatesManager {
this.updateSelectedCount();
}
selectLatestInGroup(fingerprint) {
selectLatestInGroup(groupKey) {
// Find all checkboxes in this group
const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-fingerprint="${fingerprint}"]`);
const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-key="${groupKey}"]`);
// Get all the recipes in this group
const group = this.duplicateGroups.find(g => g.fingerprint === fingerprint);
const group = this.duplicateGroups.find(g => g.key === groupKey);
if (!group) return;
// Sort recipes by date (newest first)
@@ -328,7 +428,7 @@ export class DuplicatesManager {
selectLatestDuplicates() {
// For each duplicate group, select all but the latest recipe
this.duplicateGroups.forEach(group => {
this.selectLatestInGroup(group.fingerprint);
this.selectLatestInGroup(group.key);
});
}

View File

@@ -74,7 +74,16 @@
<div class="banner-content">
<i class="fas fa-exclamation-triangle"></i>
<span id="duplicatesCount">{{ t('recipes.duplicates.found', count=0) }}</span>
<span id="duplicatesBasis" class="duplicates-basis"></span>
<i class="fas fa-question-circle help-icon" id="duplicatesHelp" aria-label="{{ t('common.actions.help') }}"></i>
<div class="banner-actions">
<div class="setting-contro" id="promptMatchControl">
<span>{{ t('recipes.duplicates.includePromptLabel') }}:</span>
<label class="toggle-switch">
<input type="checkbox" id="promptMatchInput">
<span class="toggle-slider"></span>
</label>
</div>
<button class="btn-select-latest" onclick="recipeManager.selectLatestDuplicates()">
{{ t('recipes.duplicates.keepLatest') }}
</button>
@@ -86,6 +95,9 @@
</button>
</div>
</div>
<div class="help-tooltip" id="duplicatesHelpTooltip">
<p id="duplicatesHelpText"></p>
</div>
</div>
{% include 'components/folder_sidebar.html' %}

View File

@@ -2,13 +2,22 @@ import { describe, it, beforeEach, afterEach, expect, vi } from 'vitest';
const showToastMock = vi.fn();
const recreateVirtualScrollMock = vi.fn();
const translateMock = vi.fn((key) => key);
vi.mock('../../../static/js/utils/uiHelpers.js', () => ({
showToast: showToastMock,
}));
vi.mock('../../../static/js/utils/i18nHelpers.js', () => ({
translate: translateMock,
}));
vi.mock('../../../static/js/components/RecipeCard.js', () => ({
RecipeCard: class {},
RecipeCard: class {
constructor() {
this.element = document.createElement('div');
}
},
}));
vi.mock('../../../static/js/utils/infiniteScroll.js', () => ({
@@ -85,3 +94,120 @@ describe('DuplicatesManager exitDuplicateMode', () => {
expect(document.getElementById('duplicatesBanner').style.display).toBe('none');
});
});
describe('DuplicatesManager prompt matching toggle', () => {
beforeEach(() => {
vi.clearAllMocks();
localStorage.clear();
setCurrentPageType('recipes');
setupDom();
state.pendingLayoutRecreate = false;
state.virtualScroller = { enable: vi.fn(), disable: vi.fn() };
});
afterEach(() => {
state.pendingLayoutRecreate = false;
state.virtualScroller = null;
});
it('sends include_prompt=1 when the preference is enabled', async () => {
localStorage.setItem('recipes_duplicates_include_prompt', '1');
globalThis.fetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
success: true,
duplicate_groups: [
{ type: 'fingerprint', key: 'g-1', fingerprint: 'abc:0.8', count: 2, recipes: [{ id: 'r1', modified: 1 }, { id: 'r2', modified: 2 }] },
],
}),
});
const manager = new DuplicatesManager({});
await manager.findDuplicates();
expect(globalThis.fetch).toHaveBeenCalledWith('/api/lm/recipes/find-duplicates?include_prompt=1');
});
it('calls the endpoint without the param when disabled', async () => {
globalThis.fetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ success: true, duplicate_groups: [] }),
});
const manager = new DuplicatesManager({});
await manager.findDuplicates();
expect(globalThis.fetch).toHaveBeenCalledWith('/api/lm/recipes/find-duplicates');
});
it('stays in duplicate mode with an empty view when a re-run finds no groups', async () => {
globalThis.fetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ success: true, duplicate_groups: [] }),
});
const manager = new DuplicatesManager({});
manager.inDuplicateMode = true;
await manager.findDuplicates();
// The view stays open (with the empty state) so the matching-basis
// toggle remains reachable — the deadlock fix
expect(manager.inDuplicateMode).toBe(true);
expect(manager.duplicateGroups).toEqual([]);
expect(document.getElementById('duplicatesBanner').style.display).toBe('block');
expect(document.querySelector('.duplicates-empty-state')).not.toBeNull();
});
it('enters the empty duplicates view when the toggle is on but no groups match', async () => {
localStorage.setItem('recipes_duplicates_include_prompt', '1');
globalThis.fetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ success: true, duplicate_groups: [] }),
});
const manager = new DuplicatesManager({});
await manager.findDuplicates();
expect(manager.inDuplicateMode).toBe(true);
expect(document.getElementById('duplicatesBanner').style.display).toBe('block');
});
it('toasts and stays on the library grid when the toggle is off and no groups match', async () => {
globalThis.fetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ success: true, duplicate_groups: [] }),
});
const manager = new DuplicatesManager({});
await manager.findDuplicates();
expect(manager.inDuplicateMode).toBe(false);
expect(showToastMock).toHaveBeenCalledWith('toast.duplicates.noDuplicatesFound', { type: 'recipes' }, 'info');
});
it('renders the matching basis and checkbox from the stored preference', () => {
document.body.innerHTML = `
<span id="duplicatesBasis"></span>
<span id="duplicatesHelpText"></span>
<input type="checkbox" id="promptMatchInput">
`;
localStorage.setItem('recipes_duplicates_include_prompt', '1');
const manager = new DuplicatesManager({});
manager.updateBasisDisplay();
expect(translateMock).toHaveBeenCalledWith('recipes.duplicates.basis.loraComboAndPrompt');
expect(translateMock).toHaveBeenCalledWith('recipes.duplicates.basis.hintPromptIncluded');
expect(document.getElementById('promptMatchInput').checked).toBe(true);
});
it('shows the lora-combo basis when the preference is disabled', () => {
document.body.innerHTML = `<span id="duplicatesBasis"></span>`;
const manager = new DuplicatesManager({});
manager.updateBasisDisplay();
expect(translateMock).toHaveBeenCalledWith('recipes.duplicates.basis.loraCombo');
expect(document.getElementById('duplicatesBasis').textContent).toBe('recipes.duplicates.basis.loraCombo');
});
});

View File

@@ -97,6 +97,15 @@ class StubRecipeScanner:
async def get_recipe_by_id(self, recipe_id: str) -> Optional[Dict[str, Any]]:
return self.recipes.get(recipe_id)
async def find_all_duplicate_recipes(
self, include_prompt: bool = False
) -> Dict[str, List[Any]]:
self.last_duplicate_include_prompt = include_prompt
return dict(getattr(self, "duplicate_groups_override", {}))
async def find_duplicate_recipes_by_source(self) -> Dict[str, List[Any]]:
return dict(getattr(self, "duplicate_source_groups_override", {}))
async def get_recipes_for_lora(self, lora_hash: str) -> List[Dict[str, Any]]:
return list(self.lora_lookup.get(lora_hash.lower(), []))
@@ -1951,3 +1960,56 @@ async def test_get_rematch_progress_returns_stored_progress(
assert response.status == 200
assert payload["success"] is True
assert payload["progress"]["status"] == "processing"
async def test_find_duplicates_defaults_to_fingerprint_only(
monkeypatch, tmp_path: Path
) -> None:
async with recipe_harness(monkeypatch, tmp_path) as harness:
harness.scanner.recipes = {
"r1": {"id": "r1", "title": "One", "modified": 100},
"r2": {"id": "r2", "title": "Two", "modified": 200},
}
harness.scanner.duplicate_groups_override = {"abc:0.8": ["r1", "r2"]}
harness.scanner.duplicate_source_groups_override = {}
response = await harness.client.get("/api/lm/recipes/find-duplicates")
payload = await response.json()
assert response.status == 200
assert payload["success"] is True
assert harness.scanner.last_duplicate_include_prompt is False
assert len(payload["duplicate_groups"]) == 1
group = payload["duplicate_groups"][0]
assert group["type"] == "fingerprint"
assert group["key"] == "g-1"
assert group["fingerprint"] == "abc:0.8"
assert group["count"] == 2
async def test_find_duplicates_forwards_include_prompt_and_assigns_unique_keys(
monkeypatch, tmp_path: Path
) -> None:
async with recipe_harness(monkeypatch, tmp_path) as harness:
harness.scanner.recipes = {
"r1": {"id": "r1", "title": "One", "modified": 100},
"r2": {"id": "r2", "title": "Two", "modified": 200},
"r3": {"id": "r3", "title": "Three", "modified": 300},
"r4": {"id": "r4", "title": "Four", "modified": 400},
}
harness.scanner.duplicate_groups_override = {"abc:0.8\x1fa girl": ["r1", "r2"]}
harness.scanner.duplicate_source_groups_override = {
"civitai.com/images/9": ["r3", "r4"]
}
response = await harness.client.get(
"/api/lm/recipes/find-duplicates?include_prompt=1"
)
payload = await response.json()
assert response.status == 200
assert harness.scanner.last_duplicate_include_prompt is True
groups = payload["duplicate_groups"]
assert len(groups) == 2
assert {g["type"] for g in groups} == {"fingerprint", "source_path"}
assert len({g["key"] for g in groups}) == 2

View File

@@ -3209,3 +3209,71 @@ async def test_rematch_all_autov3_cache_reuse_across_calls(
# are read once across both calls (Oracle R2-F4).
assert len(called) == 1
async def test_find_all_duplicate_recipes_groups_by_fingerprint(recipe_scanner, monkeypatch):
scanner, _ = recipe_scanner
cache = SimpleNamespace(
raw_data=[
{"id": "r1", "fingerprint": "abc:0.8", "gen_params": {"prompt": "A Girl, blue hair"}},
{"id": "r2", "fingerprint": "abc:0.8", "gen_params": {"prompt": "a boy"}},
{"id": "r3", "fingerprint": "abc:0.8", "gen_params": {"prompt": "a boy"}},
{"id": "r4", "fingerprint": "def:1.0", "gen_params": {"prompt": "A Girl, blue hair"}},
{"id": "r5", "fingerprint": "", "gen_params": {"prompt": "landscape"}},
{"id": "r6", "fingerprint": "", "gen_params": {}},
]
)
async def fake_get_cached_data():
return cache
monkeypatch.setattr(scanner, "get_cached_data", fake_get_cached_data)
groups = await scanner.find_all_duplicate_recipes()
assert groups == {"abc:0.8": ["r1", "r2", "r3"]}
async def test_find_all_duplicate_recipes_include_prompt_composite_key(recipe_scanner, monkeypatch):
scanner, _ = recipe_scanner
cache = SimpleNamespace(
raw_data=[
{"id": "r1", "fingerprint": "abc:0.8", "gen_params": {"prompt": "A Girl, blue hair"}},
{"id": "r2", "fingerprint": "abc:0.8", "gen_params": {"prompt": "a girl, blue hair"}},
{"id": "r3", "fingerprint": "abc:0.8", "gen_params": {"prompt": "a boy"}},
{"id": "r4", "fingerprint": "", "gen_params": {"prompt": " landscape "}},
{"id": "r5", "fingerprint": "", "gen_params": {"prompt": "landscape"}},
{"id": "r6", "fingerprint": "", "gen_params": {}},
{"id": "r7", "fingerprint": "def:1.0", "gen_params": {"prompt": "a girl, blue hair"}},
]
)
async def fake_get_cached_data():
return cache
monkeypatch.setattr(scanner, "get_cached_data", fake_get_cached_data)
groups = await scanner.find_all_duplicate_recipes(include_prompt=True)
assert groups == {
"abc:0.8\x1fa girl, blue hair": ["r1", "r2"],
"\x1flandscape": ["r4", "r5"],
}
# Same-lora recipes with different prompts are no longer duplicates
assert "abc:0.8\x1fa boy" not in groups
# Different-lora recipes with the same prompt are not grouped either
assert "def:1.0\x1fa girl, blue hair" not in groups
# Recipes with neither fingerprint nor prompt are skipped
assert "r6" not in [rid for ids in groups.values() for rid in ids]
async def test_find_all_duplicate_recipes_include_prompt_missing_gen_params(recipe_scanner, monkeypatch):
scanner, _ = recipe_scanner
cache = SimpleNamespace(
raw_data=[
{"id": "r1", "fingerprint": "abc:0.8"},
{"id": "r2", "fingerprint": "abc:0.8"},
{"id": "r3", "fingerprint": "abc:0.8", "gen_params": {"prompt": "a boy"}},
]
)
async def fake_get_cached_data():
return cache
monkeypatch.setattr(scanner, "get_cached_data", fake_get_cached_data)
groups = await scanner.find_all_duplicate_recipes(include_prompt=True)
# Recipes without gen_params/prompt normalize to empty prompt and match
assert groups == {"abc:0.8\x1f": ["r1", "r2"]}

View File

@@ -1,6 +1,6 @@
"""Test for modelVersionId fallback in fingerprint calculation."""
import pytest
from py.utils.utils import calculate_recipe_fingerprint
from py.utils.utils import calculate_recipe_fingerprint, normalize_prompt_for_dedup
def test_calculate_fingerprint_with_model_version_id_fallback():
@@ -98,3 +98,15 @@ def test_calculate_fingerprint_without_hash_or_version_id():
]
fingerprint = calculate_recipe_fingerprint(loras)
assert fingerprint == ""
def test_normalize_prompt_casefolds_and_collapses_whitespace():
assert normalize_prompt_for_dedup("A Girl, blue hair") == "a girl, blue hair"
assert normalize_prompt_for_dedup(" landscape \n\t with details ") == "landscape with details"
assert normalize_prompt_for_dedup("MASTERPIECE, best quality") == "masterpiece, best quality"
def test_normalize_prompt_handles_missing_or_non_string():
assert normalize_prompt_for_dedup(None) == ""
assert normalize_prompt_for_dedup("") == ""
assert normalize_prompt_for_dedup(12345) == ""