Refactor model file processing in ModelScanner to determine root paths and enhance error logging for missing roots.

This commit is contained in:
Will Miao
2025-06-11 15:53:35 +08:00
parent 14238b8d62
commit 06d5bd259c

View File

@@ -669,26 +669,33 @@ class ModelScanner:
batch = new_files[i:i+batch_size] batch = new_files[i:i+batch_size]
for path in batch: for path in batch:
try: try:
model_data = await self.scan_single_model(path) # Find the appropriate root path for this file
if model_data: root_path = None
# Add to cache for potential_root in self.get_model_roots():
self._cache.raw_data.append(model_data) if path.startswith(potential_root):
root_path = potential_root
# Update hash index if available break
if 'sha256' in model_data and 'file_path' in model_data:
self._hash_index.add_entry(model_data['sha256'].lower(), model_data['file_path']) if root_path:
model_data = await self._process_model_file(path, root_path)
# Update tags count if model_data:
if 'tags' in model_data and model_data['tags']: # Add to cache
for tag in model_data['tags']: self._cache.raw_data.append(model_data)
self._tags_count[tag] = self._tags_count.get(tag, 0) + 1
# Update hash index if available
total_added += 1 if 'sha256' in model_data and 'file_path' in model_data:
self._hash_index.add_entry(model_data['sha256'].lower(), model_data['file_path'])
# Update tags count
if 'tags' in model_data and model_data['tags']:
for tag in model_data['tags']:
self._tags_count[tag] = self._tags_count.get(tag, 0) + 1
total_added += 1
else:
logger.error(f"Could not determine root path for {path}")
except Exception as e: except Exception as e:
logger.error(f"Error adding {path} to cache: {e}") logger.error(f"Error adding {path} to cache: {e}")
# Yield control after each batch
await asyncio.sleep(0)
# Find missing files (in cache but not in filesystem) # Find missing files (in cache but not in filesystem)
missing_files = cached_paths - found_paths missing_files = cached_paths - found_paths