mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-08-10 16:00:14 -03:00
feat(recipes): add prompt-aware duplicate detection toggle
This commit is contained in:
@@ -935,8 +935,16 @@
|
||||
},
|
||||
"duplicates": {
|
||||
"found": "{count} Duplikat-Gruppen gefunden",
|
||||
"noGroups": "Keine Duplikat-Gruppen mit dem aktuellen Abgleichskriterium gefunden",
|
||||
"keepLatest": "Neueste Versionen behalten",
|
||||
"deleteSelected": "Ausgewählte löschen"
|
||||
"deleteSelected": "Ausgewählte löschen",
|
||||
"includePromptLabel": "Prompt beim Abgleich berücksichtigen",
|
||||
"basis": {
|
||||
"loraCombo": "Abgeglichen nach: LoRA-Kombination",
|
||||
"loraComboAndPrompt": "Abgeglichen nach: LoRA-Kombination + Prompt",
|
||||
"hintLoraCombo": "Rezepte mit denselben LoRAs bei identischen Stärken werden gruppiert.",
|
||||
"hintPromptIncluded": "Rezepte werden nur gruppiert, wenn sie dieselben LoRAs bei identischen Stärken UND denselben Prompt verwenden."
|
||||
}
|
||||
},
|
||||
"contextMenu": {
|
||||
"copyRecipe": {
|
||||
|
||||
@@ -935,8 +935,16 @@
|
||||
},
|
||||
"duplicates": {
|
||||
"found": "Found {count} duplicate groups",
|
||||
"noGroups": "No duplicate groups found with the current matching basis",
|
||||
"keepLatest": "Keep Latest Versions",
|
||||
"deleteSelected": "Delete Selected"
|
||||
"deleteSelected": "Delete Selected",
|
||||
"includePromptLabel": "Include prompt in matching",
|
||||
"basis": {
|
||||
"loraCombo": "Matched by: LoRA combination",
|
||||
"loraComboAndPrompt": "Matched by: LoRA combination + prompt",
|
||||
"hintLoraCombo": "Recipes with the same LoRAs at identical strengths are grouped.",
|
||||
"hintPromptIncluded": "Recipes are grouped only when they use the same LoRAs at identical strengths AND have the same prompt."
|
||||
}
|
||||
},
|
||||
"contextMenu": {
|
||||
"copyRecipe": {
|
||||
|
||||
@@ -935,8 +935,16 @@
|
||||
},
|
||||
"duplicates": {
|
||||
"found": "Se encontraron {count} grupos de duplicados",
|
||||
"noGroups": "No se encontraron grupos de duplicados con el criterio de coincidencia actual",
|
||||
"keepLatest": "Mantener versiones más recientes",
|
||||
"deleteSelected": "Eliminar seleccionados"
|
||||
"deleteSelected": "Eliminar seleccionados",
|
||||
"includePromptLabel": "Incluir prompt en la coincidencia",
|
||||
"basis": {
|
||||
"loraCombo": "Coincidencia por: combinación de LoRA",
|
||||
"loraComboAndPrompt": "Coincidencia por: combinación de LoRA + prompt",
|
||||
"hintLoraCombo": "Se agrupan las recetas con los mismos LoRAs y las mismas intensidades.",
|
||||
"hintPromptIncluded": "Las recetas solo se agrupan cuando usan los mismos LoRAs con intensidades idénticas Y tienen el mismo prompt."
|
||||
}
|
||||
},
|
||||
"contextMenu": {
|
||||
"copyRecipe": {
|
||||
|
||||
@@ -935,8 +935,16 @@
|
||||
},
|
||||
"duplicates": {
|
||||
"found": "Trouvé {count} groupes de doublons",
|
||||
"noGroups": "Aucun groupe de doublons trouvé avec le critère de correspondance actuel",
|
||||
"keepLatest": "Garder les dernières versions",
|
||||
"deleteSelected": "Supprimer la sélection"
|
||||
"deleteSelected": "Supprimer la sélection",
|
||||
"includePromptLabel": "Inclure le prompt dans la correspondance",
|
||||
"basis": {
|
||||
"loraCombo": "Correspondance : combinaison de LoRA",
|
||||
"loraComboAndPrompt": "Correspondance : combinaison de LoRA + prompt",
|
||||
"hintLoraCombo": "Les recettes avec les mêmes LoRAs et des forces identiques sont regroupées.",
|
||||
"hintPromptIncluded": "Les recettes ne sont regroupées que si elles utilisent les mêmes LoRAs avec des forces identiques ET ont le même prompt."
|
||||
}
|
||||
},
|
||||
"contextMenu": {
|
||||
"copyRecipe": {
|
||||
|
||||
@@ -935,8 +935,16 @@
|
||||
},
|
||||
"duplicates": {
|
||||
"found": "נמצאו {count} קבוצות כפולות",
|
||||
"noGroups": "לא נמצאו קבוצות כפולות לפי קריטריון ההתאמה הנוכחי",
|
||||
"keepLatest": "שמור גרסאות אחרונות",
|
||||
"deleteSelected": "מחק נבחרים"
|
||||
"deleteSelected": "מחק נבחרים",
|
||||
"includePromptLabel": "כלול הנחיה בהתאמה",
|
||||
"basis": {
|
||||
"loraCombo": "התאמה לפי: שילוב LoRA",
|
||||
"loraComboAndPrompt": "התאמה לפי: שילוב LoRA + הנחיה",
|
||||
"hintLoraCombo": "מתכונים עם אותם LoRAs בעוצמות זהות מקובצים יחד.",
|
||||
"hintPromptIncluded": "מתכונים מקובצים רק כאשר הם משתמשים באותם LoRAs בעוצמות זהות ויש להם אותה הנחיה."
|
||||
}
|
||||
},
|
||||
"contextMenu": {
|
||||
"copyRecipe": {
|
||||
|
||||
@@ -935,8 +935,16 @@
|
||||
},
|
||||
"duplicates": {
|
||||
"found": "{count} 個の重複グループが見つかりました",
|
||||
"noGroups": "現在の一致基準では重複グループが見つかりませんでした",
|
||||
"keepLatest": "最新バージョンを保持",
|
||||
"deleteSelected": "選択したものを削除"
|
||||
"deleteSelected": "選択したものを削除",
|
||||
"includePromptLabel": "一致判定にプロンプトを含める",
|
||||
"basis": {
|
||||
"loraCombo": "一致基準: LoRA の組み合わせ",
|
||||
"loraComboAndPrompt": "一致基準: LoRA の組み合わせ + プロンプト",
|
||||
"hintLoraCombo": "同じ LoRA を同じ強度で使用するレシピがグループ化されます。",
|
||||
"hintPromptIncluded": "レシピは、同じ LoRA を同じ強度で使用し、かつプロンプトが同じ場合にのみグループ化されます。"
|
||||
}
|
||||
},
|
||||
"contextMenu": {
|
||||
"copyRecipe": {
|
||||
|
||||
@@ -935,8 +935,16 @@
|
||||
},
|
||||
"duplicates": {
|
||||
"found": "{count}개의 중복 그룹 발견",
|
||||
"noGroups": "현재 일치 기준으로 중복 그룹을 찾을 수 없습니다",
|
||||
"keepLatest": "최신 버전 유지",
|
||||
"deleteSelected": "선택된 항목 삭제"
|
||||
"deleteSelected": "선택된 항목 삭제",
|
||||
"includePromptLabel": "일치 항목에 프롬프트 포함",
|
||||
"basis": {
|
||||
"loraCombo": "일치 기준: LoRA 조합",
|
||||
"loraComboAndPrompt": "일치 기준: LoRA 조합 + 프롬프트",
|
||||
"hintLoraCombo": "동일한 LoRA를 동일한 강도로 사용하는 레시피가 그룹화됩니다.",
|
||||
"hintPromptIncluded": "동일한 LoRA를 동일한 강도로 사용하고 프롬프트도 동일한 경우에만 레시피가 그룹화됩니다."
|
||||
}
|
||||
},
|
||||
"contextMenu": {
|
||||
"copyRecipe": {
|
||||
|
||||
@@ -935,8 +935,16 @@
|
||||
},
|
||||
"duplicates": {
|
||||
"found": "Найдено {count} групп дубликатов",
|
||||
"noGroups": "Дубликатов с текущим критерием не найдено",
|
||||
"keepLatest": "Оставить последние версии",
|
||||
"deleteSelected": "Удалить выбранные"
|
||||
"deleteSelected": "Удалить выбранные",
|
||||
"includePromptLabel": "Учитывать запрос при поиске дубликатов",
|
||||
"basis": {
|
||||
"loraCombo": "Критерий: комбинация LoRA",
|
||||
"loraComboAndPrompt": "Критерий: комбинация LoRA + запрос",
|
||||
"hintLoraCombo": "Рецепты с одинаковыми LoRA и одинаковой силой группируются вместе.",
|
||||
"hintPromptIncluded": "Рецепты группируются только при одинаковых LoRA с одинаковой силой И одинаковом запросе."
|
||||
}
|
||||
},
|
||||
"contextMenu": {
|
||||
"copyRecipe": {
|
||||
|
||||
@@ -935,8 +935,16 @@
|
||||
},
|
||||
"duplicates": {
|
||||
"found": "发现 {count} 个重复组",
|
||||
"noGroups": "按当前判重依据未找到重复组",
|
||||
"keepLatest": "保留最新版本",
|
||||
"deleteSelected": "删除已选"
|
||||
"deleteSelected": "删除已选",
|
||||
"includePromptLabel": "将提示词纳入判重",
|
||||
"basis": {
|
||||
"loraCombo": "判重依据:LoRA 组合",
|
||||
"loraComboAndPrompt": "判重依据:LoRA 组合 + 提示词",
|
||||
"hintLoraCombo": "使用相同 LoRA(强度一致)的配方会被分组。",
|
||||
"hintPromptIncluded": "仅当配方使用相同的 LoRA(强度一致)且提示词相同时才会被分组。"
|
||||
}
|
||||
},
|
||||
"contextMenu": {
|
||||
"copyRecipe": {
|
||||
|
||||
@@ -935,8 +935,16 @@
|
||||
},
|
||||
"duplicates": {
|
||||
"found": "發現 {count} 組重複項",
|
||||
"noGroups": "按目前判重依據未找到重複組",
|
||||
"keepLatest": "保留最新版本",
|
||||
"deleteSelected": "刪除所選"
|
||||
"deleteSelected": "刪除所選",
|
||||
"includePromptLabel": "將提示詞納入判重",
|
||||
"basis": {
|
||||
"loraCombo": "判重依據:LoRA 組合",
|
||||
"loraComboAndPrompt": "判重依據:LoRA 組合 + 提示詞",
|
||||
"hintLoraCombo": "使用相同 LoRA(強度一致)的配方會被分組。",
|
||||
"hintPromptIncluded": "僅當配方使用相同的 LoRA(強度一致)且提示詞相同時才會被分組。"
|
||||
}
|
||||
},
|
||||
"contextMenu": {
|
||||
"copyRecipe": {
|
||||
|
||||
@@ -580,7 +580,12 @@ class RecipeQueryHandler:
|
||||
if recipe_scanner is None:
|
||||
raise RuntimeError("Recipe scanner unavailable")
|
||||
|
||||
fingerprint_groups = await recipe_scanner.find_all_duplicate_recipes()
|
||||
include_prompt = (
|
||||
request.query.get("include_prompt", "false").lower() in ("1", "true")
|
||||
)
|
||||
fingerprint_groups = await recipe_scanner.find_all_duplicate_recipes(
|
||||
include_prompt=include_prompt
|
||||
)
|
||||
url_groups = await recipe_scanner.find_duplicate_recipes_by_source()
|
||||
response_data = []
|
||||
|
||||
@@ -613,6 +618,7 @@ class RecipeQueryHandler:
|
||||
response_data.append(
|
||||
{
|
||||
"type": "fingerprint",
|
||||
"key": f"g-{len(response_data) + 1}",
|
||||
"fingerprint": fingerprint,
|
||||
"count": len(recipes),
|
||||
"recipes": recipes,
|
||||
@@ -648,6 +654,7 @@ class RecipeQueryHandler:
|
||||
response_data.append(
|
||||
{
|
||||
"type": "source_path",
|
||||
"key": f"g-{len(response_data) + 1}",
|
||||
"fingerprint": url,
|
||||
"count": len(recipes),
|
||||
"recipes": recipes,
|
||||
|
||||
@@ -3540,32 +3540,70 @@ class RecipeScanner:
|
||||
|
||||
return matching_recipes
|
||||
|
||||
async def find_all_duplicate_recipes(self) -> Dict[str, List[Any]]:
|
||||
async def find_all_duplicate_recipes(
|
||||
self, include_prompt: bool = False
|
||||
) -> Dict[str, List[Any]]:
|
||||
"""Find all recipe duplicates based on fingerprints
|
||||
|
||||
When ``include_prompt`` is True, the grouping key additionally
|
||||
includes the normalized positive prompt, so recipes are only grouped
|
||||
when they share both the same LoRA combination (with identical
|
||||
strengths) and the same prompt. Recipes with neither a fingerprint
|
||||
nor a prompt are skipped.
|
||||
|
||||
Args:
|
||||
include_prompt: Whether to require an identical prompt as well
|
||||
|
||||
Returns:
|
||||
Dictionary where keys are fingerprints and values are lists of recipe IDs
|
||||
Dictionary where keys are grouping keys and values are lists of recipe IDs
|
||||
"""
|
||||
# Get all recipes from cache
|
||||
cache = await self.get_cached_data()
|
||||
|
||||
# Group recipes by fingerprint
|
||||
# Group recipes by fingerprint (optionally combined with the prompt)
|
||||
fingerprint_groups = {}
|
||||
for recipe in cache.raw_data:
|
||||
fingerprint = recipe.get("fingerprint")
|
||||
if not fingerprint:
|
||||
grouping_key = self._build_duplicate_grouping_key(
|
||||
recipe, include_prompt
|
||||
)
|
||||
if not grouping_key:
|
||||
continue
|
||||
|
||||
if fingerprint not in fingerprint_groups:
|
||||
fingerprint_groups[fingerprint] = []
|
||||
if grouping_key not in fingerprint_groups:
|
||||
fingerprint_groups[grouping_key] = []
|
||||
|
||||
fingerprint_groups[fingerprint].append(recipe.get("id"))
|
||||
fingerprint_groups[grouping_key].append(recipe.get("id"))
|
||||
|
||||
# Filter to only include groups with more than one recipe
|
||||
duplicate_groups = {k: v for k, v in fingerprint_groups.items() if len(v) > 1}
|
||||
|
||||
return duplicate_groups
|
||||
|
||||
def _build_duplicate_grouping_key(
|
||||
self, recipe: Dict[str, Any], include_prompt: bool
|
||||
) -> str:
|
||||
"""Build the grouping key used for duplicate detection.
|
||||
|
||||
Without ``include_prompt`` this is the stored fingerprint (same LoRA
|
||||
combination at identical strengths). With it, the normalized positive
|
||||
prompt is appended (separated by ``\\x1f``), so recipes must share
|
||||
both factors to be grouped. Recipes with no loras still participate
|
||||
when they carry a prompt, matching other no-lora recipes with the
|
||||
same prompt.
|
||||
"""
|
||||
fingerprint = recipe.get("fingerprint") or ""
|
||||
if not include_prompt:
|
||||
return fingerprint
|
||||
|
||||
from ..utils.utils import normalize_prompt_for_dedup
|
||||
|
||||
prompt = normalize_prompt_for_dedup(
|
||||
(recipe.get("gen_params") or {}).get("prompt")
|
||||
)
|
||||
if not fingerprint and not prompt:
|
||||
return ""
|
||||
return f"{fingerprint}\x1f{prompt}"
|
||||
|
||||
async def find_duplicate_recipes_by_source(self) -> Dict[str, List[Any]]:
|
||||
"""Find all recipe duplicates based on source_path (Civitai image URLs)
|
||||
|
||||
|
||||
@@ -469,6 +469,24 @@ def calculate_recipe_fingerprint(loras):
|
||||
return fingerprint
|
||||
|
||||
|
||||
def normalize_prompt_for_dedup(prompt) -> str:
|
||||
"""Normalize a positive prompt for duplicate recipe matching.
|
||||
|
||||
Applies casefolding, collapses whitespace runs into single spaces, and
|
||||
trims leading/trailing whitespace. Missing or non-string prompts
|
||||
normalize to an empty string.
|
||||
|
||||
Args:
|
||||
prompt: The positive prompt text (or None)
|
||||
|
||||
Returns:
|
||||
str: The normalized prompt
|
||||
"""
|
||||
if not prompt or not isinstance(prompt, str):
|
||||
return ""
|
||||
return re.sub(r"\s+", " ", prompt).strip().casefold()
|
||||
|
||||
|
||||
def calculate_relative_path_for_model(
|
||||
model_data: Dict[str, Any], model_type: str = "lora"
|
||||
) -> str:
|
||||
|
||||
@@ -486,6 +486,26 @@
|
||||
}
|
||||
}
|
||||
|
||||
/* Empty-state hint in the duplicates view */
|
||||
.duplicates-empty-state {
|
||||
padding: 48px 16px;
|
||||
text-align: center;
|
||||
opacity: 0.7;
|
||||
font-size: 0.95em;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
/* Matching basis text in the duplicates banner */
|
||||
.duplicates-basis {
|
||||
font-size: 0.85em;
|
||||
opacity: 0.8;
|
||||
padding: 3px 10px;
|
||||
border-radius: var(--border-radius-xs);
|
||||
background: oklch(var(--color-accent-l) var(--color-accent-c) var(--color-accent-h) / 0.12);
|
||||
border: 1px solid oklch(var(--color-accent-l) var(--color-accent-c) var(--color-accent-h) / 0.25);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
/* Help icon styling */
|
||||
.help-icon {
|
||||
color: var(--text-color);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// Duplicates Manager Component
|
||||
import { showToast } from '../utils/uiHelpers.js';
|
||||
import { translate } from '../utils/i18nHelpers.js';
|
||||
import { RecipeCard } from './RecipeCard.js';
|
||||
import { state, getCurrentPageState } from '../state/index.js';
|
||||
import { recreateVirtualScroll } from '../utils/infiniteScroll.js';
|
||||
@@ -10,11 +11,87 @@ export class DuplicatesManager {
|
||||
this.duplicateGroups = [];
|
||||
this.inDuplicateMode = false;
|
||||
this.selectedForDeletion = new Set();
|
||||
this._initPromptMatchToggle();
|
||||
this._initHelpTooltip();
|
||||
}
|
||||
|
||||
|
||||
_getPromptMatchPreference() {
|
||||
return localStorage.getItem('recipes_duplicates_include_prompt') === '1';
|
||||
}
|
||||
|
||||
_setPromptMatchPreference(enabled) {
|
||||
localStorage.setItem('recipes_duplicates_include_prompt', enabled ? '1' : '0');
|
||||
}
|
||||
|
||||
updateBasisDisplay() {
|
||||
const basisEl = document.getElementById('duplicatesBasis');
|
||||
const helpTextEl = document.getElementById('duplicatesHelpText');
|
||||
const checkbox = document.getElementById('promptMatchInput');
|
||||
const includePrompt = this._getPromptMatchPreference();
|
||||
if (checkbox) {
|
||||
checkbox.checked = includePrompt;
|
||||
}
|
||||
if (basisEl) {
|
||||
basisEl.textContent = translate(
|
||||
includePrompt
|
||||
? 'recipes.duplicates.basis.loraComboAndPrompt'
|
||||
: 'recipes.duplicates.basis.loraCombo'
|
||||
);
|
||||
}
|
||||
if (helpTextEl) {
|
||||
helpTextEl.textContent = translate(
|
||||
includePrompt
|
||||
? 'recipes.duplicates.basis.hintPromptIncluded'
|
||||
: 'recipes.duplicates.basis.hintLoraCombo'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
_initPromptMatchToggle() {
|
||||
const checkbox = document.getElementById('promptMatchInput');
|
||||
if (!checkbox) return;
|
||||
checkbox.addEventListener('change', async (e) => {
|
||||
this._setPromptMatchPreference(e.target.checked);
|
||||
this.updateBasisDisplay();
|
||||
checkbox.disabled = true;
|
||||
try {
|
||||
await this.findDuplicates();
|
||||
} finally {
|
||||
checkbox.disabled = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
_initHelpTooltip() {
|
||||
const helpIcon = document.getElementById('duplicatesHelp');
|
||||
const helpTooltip = document.getElementById('duplicatesHelpTooltip');
|
||||
if (!helpIcon || !helpTooltip) return;
|
||||
|
||||
helpIcon.addEventListener('mouseenter', () => {
|
||||
const bannerContent = helpIcon.closest('.banner-content');
|
||||
if (!bannerContent) return;
|
||||
const iconRect = helpIcon.getBoundingClientRect();
|
||||
const bannerRect = bannerContent.getBoundingClientRect();
|
||||
helpTooltip.style.display = 'block';
|
||||
helpTooltip.style.top = `${iconRect.bottom - bannerRect.top + 10}px`;
|
||||
helpTooltip.style.left = `${iconRect.left - bannerRect.left - 10}px`;
|
||||
const tooltipRect = helpTooltip.getBoundingClientRect();
|
||||
if (tooltipRect.right > window.innerWidth - 20) {
|
||||
helpTooltip.style.left = `${bannerContent.offsetWidth - tooltipRect.width - 20}px`;
|
||||
}
|
||||
});
|
||||
helpIcon.addEventListener('mouseleave', () => {
|
||||
helpTooltip.style.display = 'none';
|
||||
});
|
||||
}
|
||||
|
||||
async findDuplicates() {
|
||||
try {
|
||||
const response = await fetch('/api/lm/recipes/find-duplicates');
|
||||
const includePrompt = this._getPromptMatchPreference();
|
||||
const endpoint = includePrompt
|
||||
? '/api/lm/recipes/find-duplicates?include_prompt=1'
|
||||
: '/api/lm/recipes/find-duplicates';
|
||||
const response = await fetch(endpoint);
|
||||
if (!response.ok) {
|
||||
throw new Error('Failed to find duplicates');
|
||||
}
|
||||
@@ -28,7 +105,14 @@ export class DuplicatesManager {
|
||||
|
||||
if (this.duplicateGroups.length === 0) {
|
||||
showToast('toast.duplicates.noDuplicatesFound', { type: 'recipes' }, 'info');
|
||||
return false;
|
||||
// Keep (or enter) the duplicates view when the user is tuning
|
||||
// the matching basis, so the prompt-matching toggle stays
|
||||
// reachable; otherwise just toast and stay on the library grid.
|
||||
if (!this.inDuplicateMode && !includePrompt) {
|
||||
return false;
|
||||
}
|
||||
this.enterDuplicateMode();
|
||||
return true;
|
||||
}
|
||||
|
||||
this.enterDuplicateMode();
|
||||
@@ -53,9 +137,14 @@ export class DuplicatesManager {
|
||||
const countSpan = document.getElementById('duplicatesCount');
|
||||
|
||||
if (banner && countSpan) {
|
||||
countSpan.textContent = `Found ${this.duplicateGroups.length} duplicate group${this.duplicateGroups.length !== 1 ? 's' : ''}`;
|
||||
countSpan.textContent = this.duplicateGroups.length === 0
|
||||
? translate('recipes.duplicates.noGroups')
|
||||
: translate('recipes.duplicates.found', { count: this.duplicateGroups.length });
|
||||
banner.style.display = 'block';
|
||||
}
|
||||
|
||||
// Restore the prompt-matching preference and show the matching basis
|
||||
this.updateBasisDisplay();
|
||||
|
||||
// Disable virtual scrolling if active
|
||||
if (state.virtualScroller) {
|
||||
@@ -113,12 +202,23 @@ export class DuplicatesManager {
|
||||
|
||||
// Clear existing content
|
||||
recipeGrid.innerHTML = '';
|
||||
|
||||
// Empty-state view: keep the banner (and the matching-basis toggle)
|
||||
// reachable when no groups match the current basis
|
||||
if (this.duplicateGroups.length === 0) {
|
||||
const emptyState = document.createElement('div');
|
||||
emptyState.className = 'duplicates-empty-state';
|
||||
emptyState.textContent = translate('recipes.duplicates.noGroups');
|
||||
recipeGrid.appendChild(emptyState);
|
||||
return;
|
||||
}
|
||||
|
||||
// Render each duplicate group
|
||||
this.duplicateGroups.forEach((group, groupIndex) => {
|
||||
const groupKey = group.key;
|
||||
const groupDiv = document.createElement('div');
|
||||
groupDiv.className = 'duplicate-group';
|
||||
groupDiv.dataset.fingerprint = group.fingerprint;
|
||||
groupDiv.dataset.groupKey = groupKey;
|
||||
|
||||
// Create group header
|
||||
const header = document.createElement('div');
|
||||
@@ -126,10 +226,10 @@ export class DuplicatesManager {
|
||||
header.innerHTML = `
|
||||
<span>Duplicate Group #${groupIndex + 1} (${group.recipes.length} recipes)</span>
|
||||
<span>
|
||||
<button class="btn-select-all" onclick="recipeManager.duplicatesManager.toggleSelectAllInGroup('${group.fingerprint}')">
|
||||
<button class="btn-select-all" onclick="recipeManager.duplicatesManager.toggleSelectAllInGroup('${groupKey}')">
|
||||
Select All
|
||||
</button>
|
||||
<button class="btn-select-latest" onclick="recipeManager.duplicatesManager.selectLatestInGroup('${group.fingerprint}')">
|
||||
<button class="btn-select-latest" onclick="recipeManager.duplicatesManager.selectLatestInGroup('${groupKey}')">
|
||||
Keep Latest
|
||||
</button>
|
||||
</span>
|
||||
@@ -182,7 +282,7 @@ export class DuplicatesManager {
|
||||
checkbox.type = 'checkbox';
|
||||
checkbox.className = 'selector-checkbox';
|
||||
checkbox.dataset.recipeId = recipe.id;
|
||||
checkbox.dataset.groupFingerprint = group.fingerprint;
|
||||
checkbox.dataset.groupKey = groupKey;
|
||||
|
||||
// Check if already selected
|
||||
if (this.selectedForDeletion.has(recipe.id)) {
|
||||
@@ -244,8 +344,8 @@ export class DuplicatesManager {
|
||||
}
|
||||
}
|
||||
|
||||
toggleSelectAllInGroup(fingerprint) {
|
||||
const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-fingerprint="${fingerprint}"]`);
|
||||
toggleSelectAllInGroup(groupKey) {
|
||||
const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-key="${groupKey}"]`);
|
||||
const allSelected = Array.from(checkboxes).every(checkbox => checkbox.checked);
|
||||
|
||||
// If all are selected, deselect all; otherwise select all
|
||||
@@ -264,7 +364,7 @@ export class DuplicatesManager {
|
||||
});
|
||||
|
||||
// Update the button text
|
||||
const button = document.querySelector(`.duplicate-group[data-fingerprint="${fingerprint}"] .btn-select-all`);
|
||||
const button = document.querySelector(`.duplicate-group[data-group-key="${groupKey}"] .btn-select-all`);
|
||||
if (button) {
|
||||
button.textContent = !allSelected ? "Deselect All" : "Select All";
|
||||
}
|
||||
@@ -272,8 +372,8 @@ export class DuplicatesManager {
|
||||
this.updateSelectedCount();
|
||||
}
|
||||
|
||||
selectAllInGroup(fingerprint) {
|
||||
const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-fingerprint="${fingerprint}"]`);
|
||||
selectAllInGroup(groupKey) {
|
||||
const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-key="${groupKey}"]`);
|
||||
checkboxes.forEach(checkbox => {
|
||||
checkbox.checked = true;
|
||||
this.selectedForDeletion.add(checkbox.dataset.recipeId);
|
||||
@@ -281,7 +381,7 @@ export class DuplicatesManager {
|
||||
});
|
||||
|
||||
// Update the button text
|
||||
const button = document.querySelector(`.duplicate-group[data-fingerprint="${fingerprint}"] .btn-select-all`);
|
||||
const button = document.querySelector(`.duplicate-group[data-group-key="${groupKey}"] .btn-select-all`);
|
||||
if (button) {
|
||||
button.textContent = "Deselect All";
|
||||
}
|
||||
@@ -289,12 +389,12 @@ export class DuplicatesManager {
|
||||
this.updateSelectedCount();
|
||||
}
|
||||
|
||||
selectLatestInGroup(fingerprint) {
|
||||
selectLatestInGroup(groupKey) {
|
||||
// Find all checkboxes in this group
|
||||
const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-fingerprint="${fingerprint}"]`);
|
||||
const checkboxes = document.querySelectorAll(`.selector-checkbox[data-group-key="${groupKey}"]`);
|
||||
|
||||
// Get all the recipes in this group
|
||||
const group = this.duplicateGroups.find(g => g.fingerprint === fingerprint);
|
||||
const group = this.duplicateGroups.find(g => g.key === groupKey);
|
||||
if (!group) return;
|
||||
|
||||
// Sort recipes by date (newest first)
|
||||
@@ -328,7 +428,7 @@ export class DuplicatesManager {
|
||||
selectLatestDuplicates() {
|
||||
// For each duplicate group, select all but the latest recipe
|
||||
this.duplicateGroups.forEach(group => {
|
||||
this.selectLatestInGroup(group.fingerprint);
|
||||
this.selectLatestInGroup(group.key);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -74,7 +74,16 @@
|
||||
<div class="banner-content">
|
||||
<i class="fas fa-exclamation-triangle"></i>
|
||||
<span id="duplicatesCount">{{ t('recipes.duplicates.found', count=0) }}</span>
|
||||
<span id="duplicatesBasis" class="duplicates-basis"></span>
|
||||
<i class="fas fa-question-circle help-icon" id="duplicatesHelp" aria-label="{{ t('common.actions.help') }}"></i>
|
||||
<div class="banner-actions">
|
||||
<div class="setting-contro" id="promptMatchControl">
|
||||
<span>{{ t('recipes.duplicates.includePromptLabel') }}:</span>
|
||||
<label class="toggle-switch">
|
||||
<input type="checkbox" id="promptMatchInput">
|
||||
<span class="toggle-slider"></span>
|
||||
</label>
|
||||
</div>
|
||||
<button class="btn-select-latest" onclick="recipeManager.selectLatestDuplicates()">
|
||||
{{ t('recipes.duplicates.keepLatest') }}
|
||||
</button>
|
||||
@@ -86,6 +95,9 @@
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="help-tooltip" id="duplicatesHelpTooltip">
|
||||
<p id="duplicatesHelpText"></p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% include 'components/folder_sidebar.html' %}
|
||||
|
||||
@@ -2,13 +2,22 @@ import { describe, it, beforeEach, afterEach, expect, vi } from 'vitest';
|
||||
|
||||
const showToastMock = vi.fn();
|
||||
const recreateVirtualScrollMock = vi.fn();
|
||||
const translateMock = vi.fn((key) => key);
|
||||
|
||||
vi.mock('../../../static/js/utils/uiHelpers.js', () => ({
|
||||
showToast: showToastMock,
|
||||
}));
|
||||
|
||||
vi.mock('../../../static/js/utils/i18nHelpers.js', () => ({
|
||||
translate: translateMock,
|
||||
}));
|
||||
|
||||
vi.mock('../../../static/js/components/RecipeCard.js', () => ({
|
||||
RecipeCard: class {},
|
||||
RecipeCard: class {
|
||||
constructor() {
|
||||
this.element = document.createElement('div');
|
||||
}
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../../../static/js/utils/infiniteScroll.js', () => ({
|
||||
@@ -85,3 +94,120 @@ describe('DuplicatesManager exitDuplicateMode', () => {
|
||||
expect(document.getElementById('duplicatesBanner').style.display).toBe('none');
|
||||
});
|
||||
});
|
||||
|
||||
describe('DuplicatesManager prompt matching toggle', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
localStorage.clear();
|
||||
setCurrentPageType('recipes');
|
||||
setupDom();
|
||||
state.pendingLayoutRecreate = false;
|
||||
state.virtualScroller = { enable: vi.fn(), disable: vi.fn() };
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
state.pendingLayoutRecreate = false;
|
||||
state.virtualScroller = null;
|
||||
});
|
||||
|
||||
it('sends include_prompt=1 when the preference is enabled', async () => {
|
||||
localStorage.setItem('recipes_duplicates_include_prompt', '1');
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
success: true,
|
||||
duplicate_groups: [
|
||||
{ type: 'fingerprint', key: 'g-1', fingerprint: 'abc:0.8', count: 2, recipes: [{ id: 'r1', modified: 1 }, { id: 'r2', modified: 2 }] },
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
const manager = new DuplicatesManager({});
|
||||
await manager.findDuplicates();
|
||||
|
||||
expect(globalThis.fetch).toHaveBeenCalledWith('/api/lm/recipes/find-duplicates?include_prompt=1');
|
||||
});
|
||||
|
||||
it('calls the endpoint without the param when disabled', async () => {
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ success: true, duplicate_groups: [] }),
|
||||
});
|
||||
|
||||
const manager = new DuplicatesManager({});
|
||||
await manager.findDuplicates();
|
||||
|
||||
expect(globalThis.fetch).toHaveBeenCalledWith('/api/lm/recipes/find-duplicates');
|
||||
});
|
||||
|
||||
it('stays in duplicate mode with an empty view when a re-run finds no groups', async () => {
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ success: true, duplicate_groups: [] }),
|
||||
});
|
||||
|
||||
const manager = new DuplicatesManager({});
|
||||
manager.inDuplicateMode = true;
|
||||
await manager.findDuplicates();
|
||||
|
||||
// The view stays open (with the empty state) so the matching-basis
|
||||
// toggle remains reachable — the deadlock fix
|
||||
expect(manager.inDuplicateMode).toBe(true);
|
||||
expect(manager.duplicateGroups).toEqual([]);
|
||||
expect(document.getElementById('duplicatesBanner').style.display).toBe('block');
|
||||
expect(document.querySelector('.duplicates-empty-state')).not.toBeNull();
|
||||
});
|
||||
|
||||
it('enters the empty duplicates view when the toggle is on but no groups match', async () => {
|
||||
localStorage.setItem('recipes_duplicates_include_prompt', '1');
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ success: true, duplicate_groups: [] }),
|
||||
});
|
||||
|
||||
const manager = new DuplicatesManager({});
|
||||
await manager.findDuplicates();
|
||||
|
||||
expect(manager.inDuplicateMode).toBe(true);
|
||||
expect(document.getElementById('duplicatesBanner').style.display).toBe('block');
|
||||
});
|
||||
|
||||
it('toasts and stays on the library grid when the toggle is off and no groups match', async () => {
|
||||
globalThis.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ success: true, duplicate_groups: [] }),
|
||||
});
|
||||
|
||||
const manager = new DuplicatesManager({});
|
||||
await manager.findDuplicates();
|
||||
|
||||
expect(manager.inDuplicateMode).toBe(false);
|
||||
expect(showToastMock).toHaveBeenCalledWith('toast.duplicates.noDuplicatesFound', { type: 'recipes' }, 'info');
|
||||
});
|
||||
|
||||
it('renders the matching basis and checkbox from the stored preference', () => {
|
||||
document.body.innerHTML = `
|
||||
<span id="duplicatesBasis"></span>
|
||||
<span id="duplicatesHelpText"></span>
|
||||
<input type="checkbox" id="promptMatchInput">
|
||||
`;
|
||||
localStorage.setItem('recipes_duplicates_include_prompt', '1');
|
||||
|
||||
const manager = new DuplicatesManager({});
|
||||
manager.updateBasisDisplay();
|
||||
|
||||
expect(translateMock).toHaveBeenCalledWith('recipes.duplicates.basis.loraComboAndPrompt');
|
||||
expect(translateMock).toHaveBeenCalledWith('recipes.duplicates.basis.hintPromptIncluded');
|
||||
expect(document.getElementById('promptMatchInput').checked).toBe(true);
|
||||
});
|
||||
|
||||
it('shows the lora-combo basis when the preference is disabled', () => {
|
||||
document.body.innerHTML = `<span id="duplicatesBasis"></span>`;
|
||||
|
||||
const manager = new DuplicatesManager({});
|
||||
manager.updateBasisDisplay();
|
||||
|
||||
expect(translateMock).toHaveBeenCalledWith('recipes.duplicates.basis.loraCombo');
|
||||
expect(document.getElementById('duplicatesBasis').textContent).toBe('recipes.duplicates.basis.loraCombo');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -97,6 +97,15 @@ class StubRecipeScanner:
|
||||
async def get_recipe_by_id(self, recipe_id: str) -> Optional[Dict[str, Any]]:
|
||||
return self.recipes.get(recipe_id)
|
||||
|
||||
async def find_all_duplicate_recipes(
|
||||
self, include_prompt: bool = False
|
||||
) -> Dict[str, List[Any]]:
|
||||
self.last_duplicate_include_prompt = include_prompt
|
||||
return dict(getattr(self, "duplicate_groups_override", {}))
|
||||
|
||||
async def find_duplicate_recipes_by_source(self) -> Dict[str, List[Any]]:
|
||||
return dict(getattr(self, "duplicate_source_groups_override", {}))
|
||||
|
||||
async def get_recipes_for_lora(self, lora_hash: str) -> List[Dict[str, Any]]:
|
||||
return list(self.lora_lookup.get(lora_hash.lower(), []))
|
||||
|
||||
@@ -1951,3 +1960,56 @@ async def test_get_rematch_progress_returns_stored_progress(
|
||||
assert response.status == 200
|
||||
assert payload["success"] is True
|
||||
assert payload["progress"]["status"] == "processing"
|
||||
|
||||
|
||||
async def test_find_duplicates_defaults_to_fingerprint_only(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
async with recipe_harness(monkeypatch, tmp_path) as harness:
|
||||
harness.scanner.recipes = {
|
||||
"r1": {"id": "r1", "title": "One", "modified": 100},
|
||||
"r2": {"id": "r2", "title": "Two", "modified": 200},
|
||||
}
|
||||
harness.scanner.duplicate_groups_override = {"abc:0.8": ["r1", "r2"]}
|
||||
harness.scanner.duplicate_source_groups_override = {}
|
||||
|
||||
response = await harness.client.get("/api/lm/recipes/find-duplicates")
|
||||
payload = await response.json()
|
||||
|
||||
assert response.status == 200
|
||||
assert payload["success"] is True
|
||||
assert harness.scanner.last_duplicate_include_prompt is False
|
||||
assert len(payload["duplicate_groups"]) == 1
|
||||
group = payload["duplicate_groups"][0]
|
||||
assert group["type"] == "fingerprint"
|
||||
assert group["key"] == "g-1"
|
||||
assert group["fingerprint"] == "abc:0.8"
|
||||
assert group["count"] == 2
|
||||
|
||||
|
||||
async def test_find_duplicates_forwards_include_prompt_and_assigns_unique_keys(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
async with recipe_harness(monkeypatch, tmp_path) as harness:
|
||||
harness.scanner.recipes = {
|
||||
"r1": {"id": "r1", "title": "One", "modified": 100},
|
||||
"r2": {"id": "r2", "title": "Two", "modified": 200},
|
||||
"r3": {"id": "r3", "title": "Three", "modified": 300},
|
||||
"r4": {"id": "r4", "title": "Four", "modified": 400},
|
||||
}
|
||||
harness.scanner.duplicate_groups_override = {"abc:0.8\x1fa girl": ["r1", "r2"]}
|
||||
harness.scanner.duplicate_source_groups_override = {
|
||||
"civitai.com/images/9": ["r3", "r4"]
|
||||
}
|
||||
|
||||
response = await harness.client.get(
|
||||
"/api/lm/recipes/find-duplicates?include_prompt=1"
|
||||
)
|
||||
payload = await response.json()
|
||||
|
||||
assert response.status == 200
|
||||
assert harness.scanner.last_duplicate_include_prompt is True
|
||||
groups = payload["duplicate_groups"]
|
||||
assert len(groups) == 2
|
||||
assert {g["type"] for g in groups} == {"fingerprint", "source_path"}
|
||||
assert len({g["key"] for g in groups}) == 2
|
||||
|
||||
@@ -3209,3 +3209,71 @@ async def test_rematch_all_autov3_cache_reuse_across_calls(
|
||||
# are read once across both calls (Oracle R2-F4).
|
||||
assert len(called) == 1
|
||||
|
||||
|
||||
|
||||
async def test_find_all_duplicate_recipes_groups_by_fingerprint(recipe_scanner, monkeypatch):
|
||||
scanner, _ = recipe_scanner
|
||||
cache = SimpleNamespace(
|
||||
raw_data=[
|
||||
{"id": "r1", "fingerprint": "abc:0.8", "gen_params": {"prompt": "A Girl, blue hair"}},
|
||||
{"id": "r2", "fingerprint": "abc:0.8", "gen_params": {"prompt": "a boy"}},
|
||||
{"id": "r3", "fingerprint": "abc:0.8", "gen_params": {"prompt": "a boy"}},
|
||||
{"id": "r4", "fingerprint": "def:1.0", "gen_params": {"prompt": "A Girl, blue hair"}},
|
||||
{"id": "r5", "fingerprint": "", "gen_params": {"prompt": "landscape"}},
|
||||
{"id": "r6", "fingerprint": "", "gen_params": {}},
|
||||
]
|
||||
)
|
||||
async def fake_get_cached_data():
|
||||
return cache
|
||||
monkeypatch.setattr(scanner, "get_cached_data", fake_get_cached_data)
|
||||
|
||||
groups = await scanner.find_all_duplicate_recipes()
|
||||
assert groups == {"abc:0.8": ["r1", "r2", "r3"]}
|
||||
|
||||
|
||||
async def test_find_all_duplicate_recipes_include_prompt_composite_key(recipe_scanner, monkeypatch):
|
||||
scanner, _ = recipe_scanner
|
||||
cache = SimpleNamespace(
|
||||
raw_data=[
|
||||
{"id": "r1", "fingerprint": "abc:0.8", "gen_params": {"prompt": "A Girl, blue hair"}},
|
||||
{"id": "r2", "fingerprint": "abc:0.8", "gen_params": {"prompt": "a girl, blue hair"}},
|
||||
{"id": "r3", "fingerprint": "abc:0.8", "gen_params": {"prompt": "a boy"}},
|
||||
{"id": "r4", "fingerprint": "", "gen_params": {"prompt": " landscape "}},
|
||||
{"id": "r5", "fingerprint": "", "gen_params": {"prompt": "landscape"}},
|
||||
{"id": "r6", "fingerprint": "", "gen_params": {}},
|
||||
{"id": "r7", "fingerprint": "def:1.0", "gen_params": {"prompt": "a girl, blue hair"}},
|
||||
]
|
||||
)
|
||||
async def fake_get_cached_data():
|
||||
return cache
|
||||
monkeypatch.setattr(scanner, "get_cached_data", fake_get_cached_data)
|
||||
|
||||
groups = await scanner.find_all_duplicate_recipes(include_prompt=True)
|
||||
assert groups == {
|
||||
"abc:0.8\x1fa girl, blue hair": ["r1", "r2"],
|
||||
"\x1flandscape": ["r4", "r5"],
|
||||
}
|
||||
# Same-lora recipes with different prompts are no longer duplicates
|
||||
assert "abc:0.8\x1fa boy" not in groups
|
||||
# Different-lora recipes with the same prompt are not grouped either
|
||||
assert "def:1.0\x1fa girl, blue hair" not in groups
|
||||
# Recipes with neither fingerprint nor prompt are skipped
|
||||
assert "r6" not in [rid for ids in groups.values() for rid in ids]
|
||||
|
||||
|
||||
async def test_find_all_duplicate_recipes_include_prompt_missing_gen_params(recipe_scanner, monkeypatch):
|
||||
scanner, _ = recipe_scanner
|
||||
cache = SimpleNamespace(
|
||||
raw_data=[
|
||||
{"id": "r1", "fingerprint": "abc:0.8"},
|
||||
{"id": "r2", "fingerprint": "abc:0.8"},
|
||||
{"id": "r3", "fingerprint": "abc:0.8", "gen_params": {"prompt": "a boy"}},
|
||||
]
|
||||
)
|
||||
async def fake_get_cached_data():
|
||||
return cache
|
||||
monkeypatch.setattr(scanner, "get_cached_data", fake_get_cached_data)
|
||||
|
||||
groups = await scanner.find_all_duplicate_recipes(include_prompt=True)
|
||||
# Recipes without gen_params/prompt normalize to empty prompt and match
|
||||
assert groups == {"abc:0.8\x1f": ["r1", "r2"]}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Test for modelVersionId fallback in fingerprint calculation."""
|
||||
import pytest
|
||||
from py.utils.utils import calculate_recipe_fingerprint
|
||||
from py.utils.utils import calculate_recipe_fingerprint, normalize_prompt_for_dedup
|
||||
|
||||
|
||||
def test_calculate_fingerprint_with_model_version_id_fallback():
|
||||
@@ -98,3 +98,15 @@ def test_calculate_fingerprint_without_hash_or_version_id():
|
||||
]
|
||||
fingerprint = calculate_recipe_fingerprint(loras)
|
||||
assert fingerprint == ""
|
||||
|
||||
|
||||
def test_normalize_prompt_casefolds_and_collapses_whitespace():
|
||||
assert normalize_prompt_for_dedup("A Girl, blue hair") == "a girl, blue hair"
|
||||
assert normalize_prompt_for_dedup(" landscape \n\t with details ") == "landscape with details"
|
||||
assert normalize_prompt_for_dedup("MASTERPIECE, best quality") == "masterpiece, best quality"
|
||||
|
||||
|
||||
def test_normalize_prompt_handles_missing_or_non_string():
|
||||
assert normalize_prompt_for_dedup(None) == ""
|
||||
assert normalize_prompt_for_dedup("") == ""
|
||||
assert normalize_prompt_for_dedup(12345) == ""
|
||||
|
||||
Reference in New Issue
Block a user