Add endpoints and functionality for verifying duplicates in Lora and Checkpoints

- Implemented `/api/loras/verify-duplicates` and `/api/checkpoints/verify-duplicates` endpoints.
- Added `handle_verify_duplicates` method in `ModelRouteUtils` to process duplicate verification requests.
- Enhanced `ModelDuplicatesManager` to manage verification state and display results.
- Updated CSS for verification badges and hash mismatch indicators. Fixes #221

py/routes/api_routes.py

@@ -88,6 +88,9 @@ class ApiRoutes:
         # Add new endpoint for bulk deleting loras
         app.router.add_post('/api/loras/bulk-delete', routes.bulk_delete_loras)
 
+        # Add new endpoint for verifying duplicates
+        app.router.add_post('/api/loras/verify-duplicates', routes.verify_duplicates)
+
     async def delete_model(self, request: web.Request) -> web.Response:
         """Handle model deletion request"""
         if self.scanner is None:
@@ -1292,3 +1295,9 @@ class ApiRoutes:
         if self.scanner is None:
             self.scanner = await ServiceRegistry.get_lora_scanner()
         return await ModelRouteUtils.handle_relink_civitai(request, self.scanner)
+
+    async def verify_duplicates(self, request: web.Request) -> web.Response:
+        """Handle verification of duplicate lora hashes"""
+        if self.scanner is None:
+            self.scanner = await ServiceRegistry.get_lora_scanner()
+        return await ModelRouteUtils.handle_verify_duplicates(request, self.scanner)

py/routes/checkpoints_routes.py

@@ -66,6 +66,9 @@ class CheckpointsRoutes:
         # Add new endpoint for bulk deleting checkpoints
         app.router.add_post('/api/checkpoints/bulk-delete', self.bulk_delete_checkpoints)
 
+        # Add new endpoint for verifying duplicates
+        app.router.add_post('/api/checkpoints/verify-duplicates', self.verify_duplicates)
+
     async def get_checkpoints(self, request):
         """Get paginated checkpoint data"""
         try:
@@ -816,3 +819,7 @@ class CheckpointsRoutes:
     async def relink_civitai(self, request: web.Request) -> web.Response:
         """Handle CivitAI metadata re-linking request by model version ID for checkpoints"""
         return await ModelRouteUtils.handle_relink_civitai(request, self.scanner)
+
+    async def verify_duplicates(self, request: web.Request) -> web.Response:
+        """Handle verification of duplicate checkpoint hashes"""
+        return await ModelRouteUtils.handle_verify_duplicates(request, self.scanner)

py/utils/routes_common.py

@@ -688,3 +688,95 @@ class ModelRouteUtils:
         except Exception as e:
             logger.error(f"Error re-linking to CivitAI: {e}", exc_info=True)
             return web.json_response({"success": False, "error": str(e)}, status=500)
+
+    @staticmethod
+    async def handle_verify_duplicates(request: web.Request, scanner) -> web.Response:
+        """Handle verification of duplicate model hashes
+        
+        Args:
+            request: The aiohttp request
+            scanner: The model scanner instance with cache management methods
+            
+        Returns:
+            web.Response: The HTTP response with verification results
+        """
+        try:
+            data = await request.json()
+            file_paths = data.get('file_paths', [])
+            
+            if not file_paths:
+                return web.json_response({
+                    'success': False,
+                    'error': 'No file paths provided for verification'
+                }, status=400)
+            
+            # Results tracking
+            results = {
+                'verified_as_duplicates': True,  # Start true, set to false if any mismatch
+                'mismatched_files': [],
+                'new_hash_map': {}
+            }
+            
+            # Get expected hash from the first file's metadata
+            expected_hash = None
+            first_metadata_path = os.path.splitext(file_paths[0])[0] + '.metadata.json'
+            first_metadata = await ModelRouteUtils.load_local_metadata(first_metadata_path)
+            if first_metadata and 'sha256' in first_metadata:
+                expected_hash = first_metadata['sha256'].lower()
+            
+            # Process each file
+            for file_path in file_paths:
+                # Skip files that don't exist
+                if not os.path.exists(file_path):
+                    continue
+                    
+                # Calculate actual hash
+                try:
+                    from .file_utils import calculate_sha256
+                    actual_hash = await calculate_sha256(file_path)
+                    
+                    # Get metadata
+                    metadata_path = os.path.splitext(file_path)[0] + '.metadata.json'
+                    metadata = await ModelRouteUtils.load_local_metadata(metadata_path)
+                    
+                    # Compare hashes
+                    stored_hash = metadata.get('sha256', '').lower()
+                    
+                    # Set expected hash from first file if not yet set
+                    if not expected_hash:
+                        expected_hash = stored_hash
+                    
+                    # Check if hash matches expected hash
+                    if actual_hash != expected_hash:
+                        results['verified_as_duplicates'] = False
+                        results['mismatched_files'].append(file_path)
+                        results['new_hash_map'][file_path] = actual_hash
+                        
+                    # Check if stored hash needs updating
+                    if actual_hash != stored_hash:
+                        # Update metadata with actual hash
+                        metadata['sha256'] = actual_hash
+                        
+                        # Save updated metadata
+                        with open(metadata_path, 'w', encoding='utf-8') as f:
+                            json.dump(metadata, f, indent=2, ensure_ascii=False)
+                        
+                        # Update cache
+                        await scanner.update_single_model_cache(file_path, file_path, metadata)
+                except Exception as e:
+                    logger.error(f"Error verifying hash for {file_path}: {e}")
+                    results['mismatched_files'].append(file_path)
+                    results['new_hash_map'][file_path] = "error_calculating_hash"
+                    results['verified_as_duplicates'] = False
+            
+            return web.json_response({
+                'success': True,
+                **results
+            })
+            
+        except Exception as e:
+            logger.error(f"Error verifying duplicate models: {e}", exc_info=True)
+            return web.json_response({
+                'success': False,
+                'error': str(e)
+            }, status=500)

static/css/base.css

@@ -39,6 +39,9 @@ html, body {
     --lora-warning-l: 75%;
     --lora-warning-c: 0.25;
     --lora-warning-h: 80;
+    --lora-success-l: 70%;
+    --lora-success-c: 0.2;
+    --lora-success-h: 140;
 
     /* Composed Colors */
     --lora-accent: oklch(var(--lora-accent-l) var(--lora-accent-c) var(--lora-accent-h));
@@ -47,6 +50,7 @@ html, body {
     --lora-text: oklch(95% 0.02 256);
     --lora-error: oklch(75% 0.32 29);
     --lora-warning: oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h)); /* Modified to be used with oklch() */
+    --lora-success: oklch(var(--lora-success-l) var(--lora-success-c) var(--lora-success-h)); /* New green success color */
     
     /* Spacing Scale */
     --space-1: calc(8px * 1);

static/css/components/duplicates.css

@@ -315,6 +315,7 @@
     margin-bottom: 4px;
     display: flex;
     flex-wrap: wrap;
+    word-break: break-all; /* Ensure long hashes wrap properly */
 }
 
 .model-tooltip .tooltip-info div strong {
@@ -322,6 +323,128 @@
     min-width: 70px;
 }
 
+/* Latest indicator */
+.hash-mismatch-info {
+    margin-top: 8px;
+    padding-top: 8px;
+    border-top: 1px dashed var(--border-color);
+    color: oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h));
+    font-weight: bold;
+    word-break: break-all; /* Ensure long hashes wrap properly */
+}
+
+/* Verification Badge Styles */
+.verification-badge {
+    display: inline-flex;
+    align-items: center;
+    margin-left: 8px;
+    padding: 2px 6px;
+    font-size: 0.8em;
+    border-radius: var(--border-radius-xs);
+    font-weight: normal;
+}
+
+.verification-badge.metadata {
+    background-color: var(--bg-color);
+    border: 1px solid var(--border-color);
+    color: var(--text-color);
+}
+
+.verification-badge.verified {
+    background-color: oklch(70% 0.2 140); /* Green for verified */
+    color: white;
+}
+
+.verification-badge.mismatch {
+    background-color: oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h));
+    color: white;
+}
+
+.verification-badge i {
+    margin-right: 4px;
+}
+
+/* Hash Mismatch Styling */
+.lora-card.duplicate.hash-mismatch {
+    border: 2px dashed oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h));
+    opacity: 0.85;
+    position: relative;
+}
+
+.lora-card.duplicate.hash-mismatch::before {
+    content: "";
+    position: absolute;
+    top: 0;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    background: repeating-linear-gradient(
+        45deg,
+        oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h) / 0.05),
+        oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h) / 0.05) 10px,
+        transparent 10px,
+        transparent 20px
+    );
+    z-index: 1;
+    pointer-events: none;
+}
+
+.lora-card.duplicate.hash-mismatch .card-preview {
+    filter: grayscale(20%);
+}
+
+/* Mismatch Badge */
+.mismatch-badge {
+    position: absolute;
+    top: 10px;
+    left: 10px; /* Changed from right:10px to left:10px */
+    background: oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h));
+    color: white;
+    font-size: 12px;
+    padding: 3px 8px;
+    border-radius: var(--border-radius-xs);
+    z-index: 5;
+}
+
+/* Disabled checkbox style */
+.lora-card.duplicate.hash-mismatch .selector-checkbox {
+    opacity: 0.5;
+    cursor: not-allowed;
+}
+
+/* Hash mismatch info in tooltip */
+.hash-mismatch-info {
+    margin-top: 8px;
+    padding-top: 8px;
+    border-top: 1px dashed var(--border-color);
+    color: oklch(var(--lora-warning-l) var(--lora-warning-c) var(--lora-warning-h));
+    font-weight: bold;
+}
+
+/* Verify hash button styling */
+.btn-verify-hashes {
+    display: flex;
+    align-items: center;
+    gap: 6px;
+    padding: 4px 10px;
+    background: var(--card-bg);
+    border: 1px solid var(--border-color);
+    border-radius: var(--border-radius-xs);
+    font-size: 0.85em;
+    cursor: pointer;
+    transition: all 0.2s ease;
+}
+
+.btn-verify-hashes:hover {
+    background: var(--bg-color);
+    border-color: oklch(var(--lora-accent-l) var(--lora-accent-c) var(--lora-accent-h));
+    transform: translateY(-1px);
+}
+
+.btn-verify-hashes i {
+    font-size: 0.9em;
+}
+
 /* Badge Styles */
 .badge {
     display: inline-flex;

static/js/components/ModelDuplicatesManager.js

@@ -4,6 +4,7 @@ import { state, getCurrentPageState } from '../state/index.js';
 import { formatDate } from '../utils/formatters.js';
 import { resetAndReload as resetAndReloadLoras } from '../api/loraApi.js';
 import { resetAndReload as resetAndReloadCheckpoints } from '../api/checkpointApi.js';
+import { LoadingManager } from '../managers/LoadingManager.js';
 
 export class ModelDuplicatesManager {
     constructor(pageManager, modelType = 'loras') {
@@ -13,10 +14,18 @@ export class ModelDuplicatesManager {
         this.selectedForDeletion = new Set();
         this.modelType = modelType; // Use the provided modelType or default to 'loras'
         
+        // Verification tracking
+        this.verifiedGroups = new Set(); // Track which groups have been verified
+        this.mismatchedFiles = new Map(); // Map file paths to actual hashes for mismatched files
+        
+        // Loading manager for verification process
+        this.loadingManager = new LoadingManager();
+        
         // Bind methods
         this.renderModelCard = this.renderModelCard.bind(this);
         this.renderTooltip = this.renderTooltip.bind(this);
         this.checkDuplicatesCount = this.checkDuplicatesCount.bind(this);
+        this.handleVerifyHashes = this.handleVerifyHashes.bind(this);
         
         // Keep track of which controls need to be re-enabled
         this.disabledControls = [];
@@ -247,14 +256,34 @@ export class ModelDuplicatesManager {
             // Create group header
             const header = document.createElement('div');
             header.className = 'duplicate-group-header';
+            
+            // Create verification status badge
+            const verificationBadge = document.createElement('span');
+            verificationBadge.className = 'verification-badge';
+            if (this.verifiedGroups.has(group.hash)) {
+                verificationBadge.classList.add('verified');
+                verificationBadge.innerHTML = '<i class="fas fa-check-circle"></i> Verified';
+            } else {
+                verificationBadge.classList.add('metadata');
+                verificationBadge.innerHTML = '<i class="fas fa-tag"></i> Metadata Hash';
+            }
+            
             header.innerHTML = `
                 <span>Duplicate Group #${groupIndex + 1} (${group.models.length} models with same hash: ${group.hash})</span>
                 <span>
+                    <button class="btn-verify-hashes" data-hash="${group.hash}" title="Recalculate SHA256 hashes to verify if these are true duplicates">
+                        <i class="fas fa-fingerprint"></i> Verify Hashes
+                    </button>
                     <button class="btn-select-all" onclick="modelDuplicatesManager.toggleSelectAllInGroup('${group.hash}')">
                         Select All
                     </button>
                 </span>
             `;
+            
+            // Insert verification badge after the group title
+            const headerFirstSpan = header.querySelector('span:first-child');
+            headerFirstSpan.appendChild(verificationBadge);
+            
             groupDiv.appendChild(header);
             
             // Create cards container
@@ -287,6 +316,15 @@ export class ModelDuplicatesManager {
             
             groupDiv.appendChild(cardsDiv);
             modelGrid.appendChild(groupDiv);
+            
+            // Add event listener to the verify hashes button
+            const verifyButton = header.querySelector('.btn-verify-hashes');
+            if (verifyButton) {
+                verifyButton.addEventListener('click', (e) => {
+                    e.stopPropagation();
+                    this.handleVerifyHashes(group);
+                });
+            }
         });
     }
     
@@ -297,6 +335,14 @@ export class ModelDuplicatesManager {
         card.dataset.hash = model.sha256;
         card.dataset.filePath = model.file_path;
         
+        // Check if this model is a mismatched file
+        const isMismatched = this.mismatchedFiles.has(model.file_path);
+        
+        // Add mismatched class if needed
+        if (isMismatched) {
+            card.classList.add('hash-mismatch');
+        }
+        
         // Create card content using structure similar to createLoraCard in LoraCard.js
         const previewContainer = document.createElement('div');
         previewContainer.className = 'card-preview';
@@ -338,6 +384,19 @@ export class ModelDuplicatesManager {
         
         previewContainer.appendChild(preview);
         
+        // Add hash mismatch badge if needed
+        if (isMismatched) {
+            const mismatchBadge = document.createElement('div');
+            mismatchBadge.className = 'mismatch-badge';
+            mismatchBadge.innerHTML = '<i class="fas fa-exclamation-triangle"></i> Different Hash';
+            previewContainer.appendChild(mismatchBadge);
+        }
+        
+        // Mark as latest if applicable
+        if (model.is_latest) {
+            card.classList.add('latest');
+        }
+        
         // Move tooltip listeners to the preview container for consistent behavior
         // regardless of whether the preview is an image or video
         previewContainer.addEventListener('mouseover', () => this.renderTooltip(card, model));
@@ -375,6 +434,12 @@ export class ModelDuplicatesManager {
             card.classList.add('duplicate-selected');
         }
         
+        // Disable checkbox for mismatched files
+        if (isMismatched) {
+            checkbox.disabled = true;
+            checkbox.title = "This file has a different actual hash and can't be selected";
+        }
+        
         // Add change event to checkbox
         checkbox.addEventListener('change', (e) => {
             e.stopPropagation();
@@ -388,6 +453,11 @@ export class ModelDuplicatesManager {
                 return;
             }
             
+            // Don't toggle if it's a mismatched file
+            if (isMismatched) {
+                return;
+            }
+            
             // Toggle checkbox state
             checkbox.checked = !checkbox.checked;
             this.toggleCardSelection(model.file_path, card, checkbox);
@@ -406,8 +476,12 @@ export class ModelDuplicatesManager {
         const tooltip = document.createElement('div');
         tooltip.className = 'model-tooltip';
         
+        // Check if this model is a mismatched file and get the actual hash
+        const isMismatched = this.mismatchedFiles.has(model.file_path);
+        const actualHash = isMismatched ? this.mismatchedFiles.get(model.file_path) : null;
+        
         // Add model information to tooltip
-        tooltip.innerHTML = `
+        let tooltipContent = `
             <div class="tooltip-header">${model.model_name}</div>
             <div class="tooltip-info">
                 <div><strong>Version:</strong> ${model.civitai?.name || 'Unknown'}</div>
@@ -415,9 +489,17 @@ export class ModelDuplicatesManager {
                 <div><strong>Path:</strong> ${model.file_path}</div>
                 <div><strong>Base Model:</strong> ${model.base_model || 'Unknown'}</div>
                 <div><strong>Modified:</strong> ${formatDate(model.modified)}</div>
-            </div>
+                <div><strong>Metadata Hash:</strong> <span class="hash-value">${model.sha256}</span></div>
         `;
         
+        // Add actual hash information if available
+        if (isMismatched && actualHash) {
+            tooltipContent += `<div class="hash-mismatch-info"><strong>Actual Hash:</strong> <span class="hash-value">${actualHash}</span></div>`;
+        }
+        
+        tooltipContent += `</div>`;
+        tooltip.innerHTML = tooltipContent;
+        
         // Position tooltip relative to card
         const cardRect = card.getBoundingClientRect();
         tooltip.style.top = `${cardRect.top + window.scrollY - 10}px`;
@@ -630,4 +712,73 @@ export class ModelDuplicatesManager {
           }
       });
     }
+
+    // Handle verify hashes button click
+    async handleVerifyHashes(group) {
+        try {
+            const groupHash = group.hash;
+            
+            // Check if already verified
+            if (this.verifiedGroups.has(groupHash)) {
+                showToast('This group has already been verified', 'info');
+                return;
+            }
+            
+            // Show loading state
+            this.loadingManager.showSimpleLoading('Verifying hashes...');
+            
+            // Get file paths for all models in the group
+            const filePaths = group.models.map(model => model.file_path);
+            
+            // Make API request to verify hashes
+            const response = await fetch(`/api/${this.modelType}/verify-duplicates`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json'
+                },
+                body: JSON.stringify({ file_paths: filePaths })
+            });
+            
+            if (!response.ok) {
+                throw new Error(`Verification failed: ${response.statusText}`);
+            }
+            
+            const data = await response.json();
+            
+            if (!data.success) {
+                throw new Error(data.error || 'Unknown error during verification');
+            }
+            
+            // Process verification results
+            const verifiedAsDuplicates = data.verified_as_duplicates;
+            const mismatchedFiles = data.mismatched_files || [];
+            
+            // Update mismatchedFiles map
+            if (data.new_hash_map) {
+                Object.entries(data.new_hash_map).forEach(([path, hash]) => {
+                    this.mismatchedFiles.set(path, hash);
+                });
+            }
+            
+            // Mark this group as verified
+            this.verifiedGroups.add(groupHash);
+            
+            // Re-render the duplicate groups to show verification status
+            this.renderDuplicateGroups();
+            
+            // Show appropriate toast message
+            if (mismatchedFiles.length > 0) {
+                showToast(`Verification complete. ${mismatchedFiles.length} file(s) have different actual hashes.`, 'warning');
+            } else {
+                showToast('Verification complete. All files are confirmed duplicates.', 'success');
+            }
+            
+        } catch (error) {
+            console.error('Error verifying hashes:', error);
+            showToast('Failed to verify hashes: ' + error.message, 'error');
+        } finally {
+            // Hide loading state
+            this.loadingManager.hide();
+        }
+    }
 }