From 92d48335cb2acb8effa2944ffdd233a6498123b5 Mon Sep 17 00:00:00 2001
From: Will Miao <13051207myq@gmail.com>
Date: Thu, 12 Jun 2025 12:06:01 +0800
Subject: [PATCH] Add endpoints and functionality for verifying duplicates in
Lora and Checkpoints
- Implemented `/api/loras/verify-duplicates` and `/api/checkpoints/verify-duplicates` endpoints.
- Added `handle_verify_duplicates` method in `ModelRouteUtils` to process duplicate verification requests.
- Enhanced `ModelDuplicatesManager` to manage verification state and display results.
- Updated CSS for verification badges and hash mismatch indicators. Fixes #221
---
py/routes/api_routes.py | 9 +
py/routes/checkpoints_routes.py | 7 +
py/utils/routes_common.py | 92 +++++++++++
static/css/base.css | 4 +
static/css/components/duplicates.css | 123 ++++++++++++++
.../js/components/ModelDuplicatesManager.js | 155 +++++++++++++++++-
6 files changed, 388 insertions(+), 2 deletions(-)
diff --git a/py/routes/api_routes.py b/py/routes/api_routes.py
index cb797e7c..a1d7a8a8 100644
--- a/py/routes/api_routes.py
+++ b/py/routes/api_routes.py
@@ -88,6 +88,9 @@ class ApiRoutes:
# Add new endpoint for bulk deleting loras
app.router.add_post('/api/loras/bulk-delete', routes.bulk_delete_loras)
+ # Add new endpoint for verifying duplicates
+ app.router.add_post('/api/loras/verify-duplicates', routes.verify_duplicates)
+
async def delete_model(self, request: web.Request) -> web.Response:
"""Handle model deletion request"""
if self.scanner is None:
@@ -1292,3 +1295,9 @@ class ApiRoutes:
if self.scanner is None:
self.scanner = await ServiceRegistry.get_lora_scanner()
return await ModelRouteUtils.handle_relink_civitai(request, self.scanner)
+
+ async def verify_duplicates(self, request: web.Request) -> web.Response:
+ """Handle verification of duplicate lora hashes"""
+ if self.scanner is None:
+ self.scanner = await ServiceRegistry.get_lora_scanner()
+ return await ModelRouteUtils.handle_verify_duplicates(request, self.scanner)
diff --git a/py/routes/checkpoints_routes.py b/py/routes/checkpoints_routes.py
index 7e605b01..d37aef2a 100644
--- a/py/routes/checkpoints_routes.py
+++ b/py/routes/checkpoints_routes.py
@@ -66,6 +66,9 @@ class CheckpointsRoutes:
# Add new endpoint for bulk deleting checkpoints
app.router.add_post('/api/checkpoints/bulk-delete', self.bulk_delete_checkpoints)
+ # Add new endpoint for verifying duplicates
+ app.router.add_post('/api/checkpoints/verify-duplicates', self.verify_duplicates)
+
async def get_checkpoints(self, request):
"""Get paginated checkpoint data"""
try:
@@ -816,3 +819,7 @@ class CheckpointsRoutes:
async def relink_civitai(self, request: web.Request) -> web.Response:
"""Handle CivitAI metadata re-linking request by model version ID for checkpoints"""
return await ModelRouteUtils.handle_relink_civitai(request, self.scanner)
+
+ async def verify_duplicates(self, request: web.Request) -> web.Response:
+ """Handle verification of duplicate checkpoint hashes"""
+ return await ModelRouteUtils.handle_verify_duplicates(request, self.scanner)
diff --git a/py/utils/routes_common.py b/py/utils/routes_common.py
index 6cdf32e4..3c85877d 100644
--- a/py/utils/routes_common.py
+++ b/py/utils/routes_common.py
@@ -688,3 +688,95 @@ class ModelRouteUtils:
except Exception as e:
logger.error(f"Error re-linking to CivitAI: {e}", exc_info=True)
return web.json_response({"success": False, "error": str(e)}, status=500)
+
+ @staticmethod
+ async def handle_verify_duplicates(request: web.Request, scanner) -> web.Response:
+ """Handle verification of duplicate model hashes
+
+ Args:
+ request: The aiohttp request
+ scanner: The model scanner instance with cache management methods
+
+ Returns:
+ web.Response: The HTTP response with verification results
+ """
+ try:
+ data = await request.json()
+ file_paths = data.get('file_paths', [])
+
+ if not file_paths:
+ return web.json_response({
+ 'success': False,
+ 'error': 'No file paths provided for verification'
+ }, status=400)
+
+ # Results tracking
+ results = {
+ 'verified_as_duplicates': True, # Start true, set to false if any mismatch
+ 'mismatched_files': [],
+ 'new_hash_map': {}
+ }
+
+ # Get expected hash from the first file's metadata
+ expected_hash = None
+ first_metadata_path = os.path.splitext(file_paths[0])[0] + '.metadata.json'
+ first_metadata = await ModelRouteUtils.load_local_metadata(first_metadata_path)
+ if first_metadata and 'sha256' in first_metadata:
+ expected_hash = first_metadata['sha256'].lower()
+
+ # Process each file
+ for file_path in file_paths:
+ # Skip files that don't exist
+ if not os.path.exists(file_path):
+ continue
+
+ # Calculate actual hash
+ try:
+ from .file_utils import calculate_sha256
+ actual_hash = await calculate_sha256(file_path)
+
+ # Get metadata
+ metadata_path = os.path.splitext(file_path)[0] + '.metadata.json'
+ metadata = await ModelRouteUtils.load_local_metadata(metadata_path)
+
+ # Compare hashes
+ stored_hash = metadata.get('sha256', '').lower()
+
+ # Set expected hash from first file if not yet set
+ if not expected_hash:
+ expected_hash = stored_hash
+
+ # Check if hash matches expected hash
+ if actual_hash != expected_hash:
+ results['verified_as_duplicates'] = False
+ results['mismatched_files'].append(file_path)
+ results['new_hash_map'][file_path] = actual_hash
+
+ # Check if stored hash needs updating
+ if actual_hash != stored_hash:
+ # Update metadata with actual hash
+ metadata['sha256'] = actual_hash
+
+ # Save updated metadata
+ with open(metadata_path, 'w', encoding='utf-8') as f:
+ json.dump(metadata, f, indent=2, ensure_ascii=False)
+
+ # Update cache
+ await scanner.update_single_model_cache(file_path, file_path, metadata)
+ except Exception as e:
+ logger.error(f"Error verifying hash for {file_path}: {e}")
+ results['mismatched_files'].append(file_path)
+ results['new_hash_map'][file_path] = "error_calculating_hash"
+ results['verified_as_duplicates'] = False
+
+ return web.json_response({
+ 'success': True,
+ **results
+ })
+
+ except Exception as e:
+ logger.error(f"Error verifying duplicate models: {e}", exc_info=True)
+ return web.json_response({
+ 'success': False,
+ 'error': str(e)
+ }, status=500)
diff --git a/static/css/base.css b/static/css/base.css
index 0f0f1a8c..4ad7452b 100644
--- a/static/css/base.css
+++ b/static/css/base.css
@@ -39,6 +39,9 @@ html, body {
--lora-warning-l: 75%;
--lora-warning-c: 0.25;
--lora-warning-h: 80;
+ --lora-success-l: 70%;
+ --lora-success-c: 0.2;
+ --lora-success-h: 140;
/* Composed Colors */
--lora-accent: oklch(var(--lora-accent-l) var(--lora-accent-c) var(--lora-accent-h));
@@ -47,6 +50,7 @@ html, body {
--lora-text: oklch(95% 0.02 256);
--lora-error: oklch(75% 0.32 29);
--lora-warning: oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h)); /* Modified to be used with oklch() */
+ --lora-success: oklch(var(--lora-success-l) var(--lora-success-c) var(--lora-success-h)); /* New green success color */
/* Spacing Scale */
--space-1: calc(8px * 1);
diff --git a/static/css/components/duplicates.css b/static/css/components/duplicates.css
index 9456e3fb..74f58dcf 100644
--- a/static/css/components/duplicates.css
+++ b/static/css/components/duplicates.css
@@ -315,6 +315,7 @@
margin-bottom: 4px;
display: flex;
flex-wrap: wrap;
+ word-break: break-all; /* Ensure long hashes wrap properly */
}
.model-tooltip .tooltip-info div strong {
@@ -322,6 +323,128 @@
min-width: 70px;
}
+/* Latest indicator */
+.hash-mismatch-info {
+ margin-top: 8px;
+ padding-top: 8px;
+ border-top: 1px dashed var(--border-color);
+ color: oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h));
+ font-weight: bold;
+ word-break: break-all; /* Ensure long hashes wrap properly */
+}
+
+/* Verification Badge Styles */
+.verification-badge {
+ display: inline-flex;
+ align-items: center;
+ margin-left: 8px;
+ padding: 2px 6px;
+ font-size: 0.8em;
+ border-radius: var(--border-radius-xs);
+ font-weight: normal;
+}
+
+.verification-badge.metadata {
+ background-color: var(--bg-color);
+ border: 1px solid var(--border-color);
+ color: var(--text-color);
+}
+
+.verification-badge.verified {
+ background-color: oklch(70% 0.2 140); /* Green for verified */
+ color: white;
+}
+
+.verification-badge.mismatch {
+ background-color: oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h));
+ color: white;
+}
+
+.verification-badge i {
+ margin-right: 4px;
+}
+
+/* Hash Mismatch Styling */
+.lora-card.duplicate.hash-mismatch {
+ border: 2px dashed oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h));
+ opacity: 0.85;
+ position: relative;
+}
+
+.lora-card.duplicate.hash-mismatch::before {
+ content: "";
+ position: absolute;
+ top: 0;
+ left: 0;
+ right: 0;
+ bottom: 0;
+ background: repeating-linear-gradient(
+ 45deg,
+ oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h) / 0.05),
+ oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h) / 0.05) 10px,
+ transparent 10px,
+ transparent 20px
+ );
+ z-index: 1;
+ pointer-events: none;
+}
+
+.lora-card.duplicate.hash-mismatch .card-preview {
+ filter: grayscale(20%);
+}
+
+/* Mismatch Badge */
+.mismatch-badge {
+ position: absolute;
+ top: 10px;
+ left: 10px; /* Changed from right:10px to left:10px */
+ background: oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h));
+ color: white;
+ font-size: 12px;
+ padding: 3px 8px;
+ border-radius: var(--border-radius-xs);
+ z-index: 5;
+}
+
+/* Disabled checkbox style */
+.lora-card.duplicate.hash-mismatch .selector-checkbox {
+ opacity: 0.5;
+ cursor: not-allowed;
+}
+
+/* Hash mismatch info in tooltip */
+.hash-mismatch-info {
+ margin-top: 8px;
+ padding-top: 8px;
+ border-top: 1px dashed var(--border-color);
+ color: oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h));
+ font-weight: bold;
+}
+
+/* Verify hash button styling */
+.btn-verify-hashes {
+ display: flex;
+ align-items: center;
+ gap: 6px;
+ padding: 4px 10px;
+ background: var(--card-bg);
+ border: 1px solid var(--border-color);
+ border-radius: var(--border-radius-xs);
+ font-size: 0.85em;
+ cursor: pointer;
+ transition: all 0.2s ease;
+}
+
+.btn-verify-hashes:hover {
+ background: var(--bg-color);
+ border-color: oklch(var(--lora-accent-l) var(--lora-accent-c) var(--lora-accent-h));
+ transform: translateY(-1px);
+}
+
+.btn-verify-hashes i {
+ font-size: 0.9em;
+}
+
/* Badge Styles */
.badge {
display: inline-flex;
diff --git a/static/js/components/ModelDuplicatesManager.js b/static/js/components/ModelDuplicatesManager.js
index 8317af8d..04cfcc9f 100644
--- a/static/js/components/ModelDuplicatesManager.js
+++ b/static/js/components/ModelDuplicatesManager.js
@@ -4,6 +4,7 @@ import { state, getCurrentPageState } from '../state/index.js';
import { formatDate } from '../utils/formatters.js';
import { resetAndReload as resetAndReloadLoras } from '../api/loraApi.js';
import { resetAndReload as resetAndReloadCheckpoints } from '../api/checkpointApi.js';
+import { LoadingManager } from '../managers/LoadingManager.js';
export class ModelDuplicatesManager {
constructor(pageManager, modelType = 'loras') {
@@ -13,10 +14,18 @@ export class ModelDuplicatesManager {
this.selectedForDeletion = new Set();
this.modelType = modelType; // Use the provided modelType or default to 'loras'
+ // Verification tracking
+ this.verifiedGroups = new Set(); // Track which groups have been verified
+ this.mismatchedFiles = new Map(); // Map file paths to actual hashes for mismatched files
+
+ // Loading manager for verification process
+ this.loadingManager = new LoadingManager();
+
// Bind methods
this.renderModelCard = this.renderModelCard.bind(this);
this.renderTooltip = this.renderTooltip.bind(this);
this.checkDuplicatesCount = this.checkDuplicatesCount.bind(this);
+ this.handleVerifyHashes = this.handleVerifyHashes.bind(this);
// Keep track of which controls need to be re-enabled
this.disabledControls = [];
@@ -247,14 +256,34 @@ export class ModelDuplicatesManager {
// Create group header
const header = document.createElement('div');
header.className = 'duplicate-group-header';
+
+ // Create verification status badge
+ const verificationBadge = document.createElement('span');
+ verificationBadge.className = 'verification-badge';
+ if (this.verifiedGroups.has(group.hash)) {
+ verificationBadge.classList.add('verified');
+ verificationBadge.innerHTML = ' Verified';
+ } else {
+ verificationBadge.classList.add('metadata');
+ verificationBadge.innerHTML = ' Metadata Hash';
+ }
+
header.innerHTML = `
Duplicate Group #${groupIndex + 1} (${group.models.length} models with same hash: ${group.hash})
+
`;
+
+ // Insert verification badge after the group title
+ const headerFirstSpan = header.querySelector('span:first-child');
+ headerFirstSpan.appendChild(verificationBadge);
+
groupDiv.appendChild(header);
// Create cards container
@@ -287,6 +316,15 @@ export class ModelDuplicatesManager {
groupDiv.appendChild(cardsDiv);
modelGrid.appendChild(groupDiv);
+
+ // Add event listener to the verify hashes button
+ const verifyButton = header.querySelector('.btn-verify-hashes');
+ if (verifyButton) {
+ verifyButton.addEventListener('click', (e) => {
+ e.stopPropagation();
+ this.handleVerifyHashes(group);
+ });
+ }
});
}
@@ -297,6 +335,14 @@ export class ModelDuplicatesManager {
card.dataset.hash = model.sha256;
card.dataset.filePath = model.file_path;
+ // Check if this model is a mismatched file
+ const isMismatched = this.mismatchedFiles.has(model.file_path);
+
+ // Add mismatched class if needed
+ if (isMismatched) {
+ card.classList.add('hash-mismatch');
+ }
+
// Create card content using structure similar to createLoraCard in LoraCard.js
const previewContainer = document.createElement('div');
previewContainer.className = 'card-preview';
@@ -338,6 +384,19 @@ export class ModelDuplicatesManager {
previewContainer.appendChild(preview);
+ // Add hash mismatch badge if needed
+ if (isMismatched) {
+ const mismatchBadge = document.createElement('div');
+ mismatchBadge.className = 'mismatch-badge';
+ mismatchBadge.innerHTML = ' Different Hash';
+ previewContainer.appendChild(mismatchBadge);
+ }
+
+ // Mark as latest if applicable
+ if (model.is_latest) {
+ card.classList.add('latest');
+ }
+
// Move tooltip listeners to the preview container for consistent behavior
// regardless of whether the preview is an image or video
previewContainer.addEventListener('mouseover', () => this.renderTooltip(card, model));
@@ -375,6 +434,12 @@ export class ModelDuplicatesManager {
card.classList.add('duplicate-selected');
}
+ // Disable checkbox for mismatched files
+ if (isMismatched) {
+ checkbox.disabled = true;
+ checkbox.title = "This file has a different actual hash and can't be selected";
+ }
+
// Add change event to checkbox
checkbox.addEventListener('change', (e) => {
e.stopPropagation();
@@ -388,6 +453,11 @@ export class ModelDuplicatesManager {
return;
}
+ // Don't toggle if it's a mismatched file
+ if (isMismatched) {
+ return;
+ }
+
// Toggle checkbox state
checkbox.checked = !checkbox.checked;
this.toggleCardSelection(model.file_path, card, checkbox);
@@ -406,8 +476,12 @@ export class ModelDuplicatesManager {
const tooltip = document.createElement('div');
tooltip.className = 'model-tooltip';
+ // Check if this model is a mismatched file and get the actual hash
+ const isMismatched = this.mismatchedFiles.has(model.file_path);
+ const actualHash = isMismatched ? this.mismatchedFiles.get(model.file_path) : null;
+
// Add model information to tooltip
- tooltip.innerHTML = `
+ let tooltipContent = `
+ Metadata Hash: ${model.sha256}
`;
+ // Add actual hash information if available
+ if (isMismatched && actualHash) {
+ tooltipContent += `Actual Hash: ${actualHash}
`;
+ }
+
+ tooltipContent += ``;
+ tooltip.innerHTML = tooltipContent;
+
// Position tooltip relative to card
const cardRect = card.getBoundingClientRect();
tooltip.style.top = `${cardRect.top + window.scrollY - 10}px`;
@@ -630,4 +712,73 @@ export class ModelDuplicatesManager {
}
});
}
+
+ // Handle verify hashes button click
+ async handleVerifyHashes(group) {
+ try {
+ const groupHash = group.hash;
+
+ // Check if already verified
+ if (this.verifiedGroups.has(groupHash)) {
+ showToast('This group has already been verified', 'info');
+ return;
+ }
+
+ // Show loading state
+ this.loadingManager.showSimpleLoading('Verifying hashes...');
+
+ // Get file paths for all models in the group
+ const filePaths = group.models.map(model => model.file_path);
+
+ // Make API request to verify hashes
+ const response = await fetch(`/api/${this.modelType}/verify-duplicates`, {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json'
+ },
+ body: JSON.stringify({ file_paths: filePaths })
+ });
+
+ if (!response.ok) {
+ throw new Error(`Verification failed: ${response.statusText}`);
+ }
+
+ const data = await response.json();
+
+ if (!data.success) {
+ throw new Error(data.error || 'Unknown error during verification');
+ }
+
+ // Process verification results
+ const verifiedAsDuplicates = data.verified_as_duplicates;
+ const mismatchedFiles = data.mismatched_files || [];
+
+ // Update mismatchedFiles map
+ if (data.new_hash_map) {
+ Object.entries(data.new_hash_map).forEach(([path, hash]) => {
+ this.mismatchedFiles.set(path, hash);
+ });
+ }
+
+ // Mark this group as verified
+ this.verifiedGroups.add(groupHash);
+
+ // Re-render the duplicate groups to show verification status
+ this.renderDuplicateGroups();
+
+ // Show appropriate toast message
+ if (mismatchedFiles.length > 0) {
+ showToast(`Verification complete. ${mismatchedFiles.length} file(s) have different actual hashes.`, 'warning');
+ } else {
+ showToast('Verification complete. All files are confirmed duplicates.', 'success');
+ }
+
+ } catch (error) {
+ console.error('Error verifying hashes:', error);
+ showToast('Failed to verify hashes: ' + error.message, 'error');
+ } finally {
+ // Hide loading state
+ this.loadingManager.hide();
+ }
+ }
}