alphard/ComfyUI-Lora-Manager
1
0
Fork 0
mirror of https://github.com/willmiao/ComfyUI-Lora-Manager.git synced 2026-03-21 21:22:11 -03:00
Code Issues Packages Projects Releases Wiki Activity

Add endpoints and functionality for verifying duplicates in Lora and Checkpoints

Browse Source 
- Implemented `/api/loras/verify-duplicates` and `/api/checkpoints/verify-duplicates` endpoints.
- Added `handle_verify_duplicates` method in `ModelRouteUtils` to process duplicate verification requests.
- Enhanced `ModelDuplicatesManager` to manage verification state and display results.
- Updated CSS for verification badges and hash mismatch indicators. Fixes #221
This commit is contained in:
Will Miao
2025-06-12 12:06:01 +08:00
parent 78cac2edc2
commit 92d48335cb
6 changed files with 388 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
py/routes/api_routes.py
View File

@@ -88,6 +88,9 @@ class ApiRoutes:
# Add new endpoint for bulk deleting loras
app.router.add_post('/api/loras/bulk-delete', routes.bulk_delete_loras)
# Add new endpoint for verifying duplicates
app.router.add_post('/api/loras/verify-duplicates', routes.verify_duplicates)
async def delete_model(self, request: web.Request) -> web.Response:
"""Handle model deletion request"""
if self.scanner is None:
@@ -1292,3 +1295,9 @@ class ApiRoutes:
if self.scanner is None:
self.scanner = await ServiceRegistry.get_lora_scanner()
return await ModelRouteUtils.handle_relink_civitai(request, self.scanner)
async def verify_duplicates(self, request: web.Request) -> web.Response:
"""Handle verification of duplicate lora hashes"""
if self.scanner is None:
self.scanner = await ServiceRegistry.get_lora_scanner()
return await ModelRouteUtils.handle_verify_duplicates(request, self.scanner)

7
py/routes/checkpoints_routes.py
View File

@@ -66,6 +66,9 @@ class CheckpointsRoutes:
# Add new endpoint for bulk deleting checkpoints
app.router.add_post('/api/checkpoints/bulk-delete', self.bulk_delete_checkpoints)
# Add new endpoint for verifying duplicates
app.router.add_post('/api/checkpoints/verify-duplicates', self.verify_duplicates)
async def get_checkpoints(self, request):
"""Get paginated checkpoint data"""
try:
@@ -816,3 +819,7 @@ class CheckpointsRoutes:
async def relink_civitai(self, request: web.Request) -> web.Response:
"""Handle CivitAI metadata re-linking request by model version ID for checkpoints"""
return await ModelRouteUtils.handle_relink_civitai(request, self.scanner)
async def verify_duplicates(self, request: web.Request) -> web.Response:
"""Handle verification of duplicate checkpoint hashes"""
return await ModelRouteUtils.handle_verify_duplicates(request, self.scanner)

92
py/utils/routes_common.py
View File

@@ -688,3 +688,95 @@ class ModelRouteUtils:
except Exception as e:
logger.error(f"Error re-linking to CivitAI: {e}", exc_info=True)
return web.json_response({"success": False, "error": str(e)}, status=500)
@staticmethod
async def handle_verify_duplicates(request: web.Request, scanner) -> web.Response:
"""Handle verification of duplicate model hashes
Args:
request: The aiohttp request
scanner: The model scanner instance with cache management methods
Returns:
web.Response: The HTTP response with verification results
"""
try:
data = await request.json()
file_paths = data.get('file_paths', [])
if not file_paths:
return web.json_response({
'success': False,
'error': 'No file paths provided for verification'
}, status=400)
# Results tracking
results = {
'verified_as_duplicates': True, # Start true, set to false if any mismatch
'mismatched_files': [],
'new_hash_map': {}
}
# Get expected hash from the first file's metadata
expected_hash = None
first_metadata_path = os.path.splitext(file_paths[0])[0] + '.metadata.json'
first_metadata = await ModelRouteUtils.load_local_metadata(first_metadata_path)
if first_metadata and 'sha256' in first_metadata:
expected_hash = first_metadata['sha256'].lower()
# Process each file
for file_path in file_paths:
# Skip files that don't exist
if not os.path.exists(file_path):
continue
# Calculate actual hash
try:
from .file_utils import calculate_sha256
actual_hash = await calculate_sha256(file_path)
# Get metadata
metadata_path = os.path.splitext(file_path)[0] + '.metadata.json'
metadata = await ModelRouteUtils.load_local_metadata(metadata_path)
# Compare hashes
stored_hash = metadata.get('sha256', '').lower()
# Set expected hash from first file if not yet set
if not expected_hash:
expected_hash = stored_hash
# Check if hash matches expected hash
if actual_hash != expected_hash:
results['verified_as_duplicates'] = False
results['mismatched_files'].append(file_path)
results['new_hash_map'][file_path] = actual_hash
# Check if stored hash needs updating
if actual_hash != stored_hash:
# Update metadata with actual hash
metadata['sha256'] = actual_hash
# Save updated metadata
with open(metadata_path, 'w', encoding='utf-8') as f:
json.dump(metadata, f, indent=2, ensure_ascii=False)
# Update cache
await scanner.update_single_model_cache(file_path, file_path, metadata)
except Exception as e:
logger.error(f"Error verifying hash for {file_path}: {e}")
results['mismatched_files'].append(file_path)
results['new_hash_map'][file_path] = "error_calculating_hash"
results['verified_as_duplicates'] = False
return web.json_response({
'success': True,
**results
})
except Exception as e:
logger.error(f"Error verifying duplicate models: {e}", exc_info=True)
return web.json_response({
'success': False,
'error': str(e)
}, status=500)

4
static/css/base.css
View File

@@ -39,6 +39,9 @@ html, body {
--lora-warning-l: 75%;
--lora-warning-c: 0.25;
--lora-warning-h: 80;
--lora-success-l: 70%;
--lora-success-c: 0.2;
--lora-success-h: 140;
/* Composed Colors */
--lora-accent: oklch(var(--lora-accent-l) var(--lora-accent-c) var(--lora-accent-h));
@@ -47,6 +50,7 @@ html, body {
--lora-text: oklch(95% 0.02 256);
--lora-error: oklch(75% 0.32 29);
--lora-warning: oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h)); /* Modified to be used with oklch() */
--lora-success: oklch(var(--lora-success-l) var(--lora-success-c) var(--lora-success-h)); /* New green success color */
/* Spacing Scale */
--space-1: calc(8px * 1);

123
static/css/components/duplicates.css
View File

@@ -315,6 +315,7 @@
margin-bottom: 4px;
display: flex;
flex-wrap: wrap;
word-break: break-all; /* Ensure long hashes wrap properly */
}
.model-tooltip .tooltip-info div strong {
@@ -322,6 +323,128 @@
min-width: 70px;
}
/* Latest indicator */
.hash-mismatch-info {
margin-top: 8px;
padding-top: 8px;
border-top: 1px dashed var(--border-color);
color: oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h));
font-weight: bold;
word-break: break-all; /* Ensure long hashes wrap properly */
}
/* Verification Badge Styles */
.verification-badge {
display: inline-flex;
align-items: center;
margin-left: 8px;
padding: 2px 6px;
font-size: 0.8em;
border-radius: var(--border-radius-xs);
font-weight: normal;
}
.verification-badge.metadata {
background-color: var(--bg-color);
border: 1px solid var(--border-color);
color: var(--text-color);
}
.verification-badge.verified {
background-color: oklch(70% 0.2 140); /* Green for verified */
color: white;
}
.verification-badge.mismatch {
background-color: oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h));
color: white;
}
.verification-badge i {
margin-right: 4px;
}
/* Hash Mismatch Styling */
.lora-card.duplicate.hash-mismatch {
border: 2px dashed oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h));
opacity: 0.85;
position: relative;
}
.lora-card.duplicate.hash-mismatch::before {
content: "";
position: absolute;
top: 0;
left: 0;
right: 0;
bottom: 0;
background: repeating-linear-gradient(
45deg,
oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h) / 0.05),
oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h) / 0.05) 10px,
transparent 10px,
transparent 20px
);
z-index: 1;
pointer-events: none;
}
.lora-card.duplicate.hash-mismatch .card-preview {
filter: grayscale(20%);
}
/* Mismatch Badge */
.mismatch-badge {
position: absolute;
top: 10px;
left: 10px; /* Changed from right:10px to left:10px */
background: oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h));
color: white;
font-size: 12px;
padding: 3px 8px;
border-radius: var(--border-radius-xs);
z-index: 5;
}
/* Disabled checkbox style */
.lora-card.duplicate.hash-mismatch .selector-checkbox {
opacity: 0.5;
cursor: not-allowed;
}
/* Hash mismatch info in tooltip */
.hash-mismatch-info {
margin-top: 8px;
padding-top: 8px;
border-top: 1px dashed var(--border-color);
color: oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h));
font-weight: bold;
}
/* Verify hash button styling */
.btn-verify-hashes {
display: flex;
align-items: center;
gap: 6px;
padding: 4px 10px;
background: var(--card-bg);
border: 1px solid var(--border-color);
border-radius: var(--border-radius-xs);
font-size: 0.85em;
cursor: pointer;
transition: all 0.2s ease;
}
.btn-verify-hashes:hover {
background: var(--bg-color);
border-color: oklch(var(--lora-accent-l) var(--lora-accent-c) var(--lora-accent-h));
transform: translateY(-1px);
}
.btn-verify-hashes i {
font-size: 0.9em;
}
/* Badge Styles */
.badge {
display: inline-flex;

155
static/js/components/ModelDuplicatesManager.js
View File

@@ -4,6 +4,7 @@ import { state, getCurrentPageState } from '../state/index.js';
import { formatDate } from '../utils/formatters.js';
import { resetAndReload as resetAndReloadLoras } from '../api/loraApi.js';
import { resetAndReload as resetAndReloadCheckpoints } from '../api/checkpointApi.js';
import { LoadingManager } from '../managers/LoadingManager.js';
export class ModelDuplicatesManager {
constructor(pageManager, modelType = 'loras') {
@@ -13,10 +14,18 @@ export class ModelDuplicatesManager {
this.selectedForDeletion = new Set();
this.modelType = modelType; // Use the provided modelType or default to 'loras'
// Verification tracking
this.verifiedGroups = new Set(); // Track which groups have been verified
this.mismatchedFiles = new Map(); // Map file paths to actual hashes for mismatched files
// Loading manager for verification process
this.loadingManager = new LoadingManager();
// Bind methods
this.renderModelCard = this.renderModelCard.bind(this);
this.renderTooltip = this.renderTooltip.bind(this);
this.checkDuplicatesCount = this.checkDuplicatesCount.bind(this);
this.handleVerifyHashes = this.handleVerifyHashes.bind(this);
// Keep track of which controls need to be re-enabled
this.disabledControls = [];
@@ -247,14 +256,34 @@ export class ModelDuplicatesManager {
// Create group header
const header = document.createElement('div');
header.className = 'duplicate-group-header';
// Create verification status badge
const verificationBadge = document.createElement('span');
verificationBadge.className = 'verification-badge';
if (this.verifiedGroups.has(group.hash)) {
verificationBadge.classList.add('verified');
verificationBadge.innerHTML = '<i class="fas fa-check-circle"></i> Verified';
} else {
verificationBadge.classList.add('metadata');
verificationBadge.innerHTML = '<i class="fas fa-tag"></i> Metadata Hash';
}
header.innerHTML = `
<span>Duplicate Group #${groupIndex + 1} (${group.models.length} models with same hash: ${group.hash})</span>
<span>
<button class="btn-verify-hashes" data-hash="${group.hash}" title="Recalculate SHA256 hashes to verify if these are true duplicates">
<i class="fas fa-fingerprint"></i> Verify Hashes
</button>
<button class="btn-select-all" onclick="modelDuplicatesManager.toggleSelectAllInGroup('${group.hash}')">
Select All
</button>
</span>
`;
// Insert verification badge after the group title
const headerFirstSpan = header.querySelector('span:first-child');
headerFirstSpan.appendChild(verificationBadge);
groupDiv.appendChild(header);
// Create cards container
@@ -287,6 +316,15 @@ export class ModelDuplicatesManager {
groupDiv.appendChild(cardsDiv);
modelGrid.appendChild(groupDiv);
// Add event listener to the verify hashes button
const verifyButton = header.querySelector('.btn-verify-hashes');
if (verifyButton) {
verifyButton.addEventListener('click', (e) => {
e.stopPropagation();
this.handleVerifyHashes(group);
});
}
});
}
@@ -297,6 +335,14 @@ export class ModelDuplicatesManager {
card.dataset.hash = model.sha256;
card.dataset.filePath = model.file_path;
// Check if this model is a mismatched file
const isMismatched = this.mismatchedFiles.has(model.file_path);
// Add mismatched class if needed
if (isMismatched) {
card.classList.add('hash-mismatch');
}
// Create card content using structure similar to createLoraCard in LoraCard.js
const previewContainer = document.createElement('div');
previewContainer.className = 'card-preview';
@@ -338,6 +384,19 @@ export class ModelDuplicatesManager {
previewContainer.appendChild(preview);
// Add hash mismatch badge if needed
if (isMismatched) {
const mismatchBadge = document.createElement('div');
mismatchBadge.className = 'mismatch-badge';
mismatchBadge.innerHTML = '<i class="fas fa-exclamation-triangle"></i> Different Hash';
previewContainer.appendChild(mismatchBadge);
}
// Mark as latest if applicable
if (model.is_latest) {
card.classList.add('latest');
}
// Move tooltip listeners to the preview container for consistent behavior
// regardless of whether the preview is an image or video
previewContainer.addEventListener('mouseover', () => this.renderTooltip(card, model));
@@ -375,6 +434,12 @@ export class ModelDuplicatesManager {
card.classList.add('duplicate-selected');
}
// Disable checkbox for mismatched files
if (isMismatched) {
checkbox.disabled = true;
checkbox.title = "This file has a different actual hash and can't be selected";
}
// Add change event to checkbox
checkbox.addEventListener('change', (e) => {
e.stopPropagation();
@@ -388,6 +453,11 @@ export class ModelDuplicatesManager {
return;
}
// Don't toggle if it's a mismatched file
if (isMismatched) {
return;
}
// Toggle checkbox state
checkbox.checked = !checkbox.checked;
this.toggleCardSelection(model.file_path, card, checkbox);
@@ -406,8 +476,12 @@ export class ModelDuplicatesManager {
const tooltip = document.createElement('div');
tooltip.className = 'model-tooltip';
// Check if this model is a mismatched file and get the actual hash
const isMismatched = this.mismatchedFiles.has(model.file_path);
const actualHash = isMismatched ? this.mismatchedFiles.get(model.file_path) : null;
// Add model information to tooltip
tooltip.innerHTML = `
let tooltipContent = `
<div class="tooltip-header">${model.model_name}</div>
<div class="tooltip-info">
<div><strong>Version:</strong> ${model.civitai?.name || 'Unknown'}</div>
@@ -415,9 +489,17 @@ export class ModelDuplicatesManager {
<div><strong>Path:</strong> ${model.file_path}</div>
<div><strong>Base Model:</strong> ${model.base_model || 'Unknown'}</div>
<div><strong>Modified:</strong> ${formatDate(model.modified)}</div>
</div>
<div><strong>Metadata Hash:</strong> <span class="hash-value">${model.sha256}</span></div>
`;
// Add actual hash information if available
if (isMismatched && actualHash) {
tooltipContent += `<div class="hash-mismatch-info"><strong>Actual Hash:</strong> <span class="hash-value">${actualHash}</span></div>`;
}
tooltipContent += `</div>`;
tooltip.innerHTML = tooltipContent;
// Position tooltip relative to card
const cardRect = card.getBoundingClientRect();
tooltip.style.top = `${cardRect.top + window.scrollY - 10}px`;
@@ -630,4 +712,73 @@ export class ModelDuplicatesManager {
}
});
}
// Handle verify hashes button click
async handleVerifyHashes(group) {
try {
const groupHash = group.hash;
// Check if already verified
if (this.verifiedGroups.has(groupHash)) {
showToast('This group has already been verified', 'info');
return;
}
// Show loading state
this.loadingManager.showSimpleLoading('Verifying hashes...');
// Get file paths for all models in the group
const filePaths = group.models.map(model => model.file_path);
// Make API request to verify hashes
const response = await fetch(`/api/${this.modelType}/verify-duplicates`, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({ file_paths: filePaths })
});
if (!response.ok) {
throw new Error(`Verification failed: ${response.statusText}`);
}
const data = await response.json();
if (!data.success) {
throw new Error(data.error || 'Unknown error during verification');
}
// Process verification results
const verifiedAsDuplicates = data.verified_as_duplicates;
const mismatchedFiles = data.mismatched_files || [];
// Update mismatchedFiles map
if (data.new_hash_map) {
Object.entries(data.new_hash_map).forEach(([path, hash]) => {
this.mismatchedFiles.set(path, hash);
});
}
// Mark this group as verified
this.verifiedGroups.add(groupHash);
// Re-render the duplicate groups to show verification status
this.renderDuplicateGroups();
// Show appropriate toast message
if (mismatchedFiles.length > 0) {
showToast(`Verification complete. ${mismatchedFiles.length} file(s) have different actual hashes.`, 'warning');
} else {
showToast('Verification complete. All files are confirmed duplicates.', 'success');
}
} catch (error) {
console.error('Error verifying hashes:', error);
showToast('Failed to verify hashes: ' + error.message, 'error');
} finally {
// Hide loading state
this.loadingManager.hide();
}
}
}