diff --git a/py/lora_manager.py b/py/lora_manager.py index 7789d03c..51d1613a 100644 --- a/py/lora_manager.py +++ b/py/lora_manager.py @@ -10,6 +10,7 @@ from .routes.lora_routes import LoraRoutes from .routes.api_routes import ApiRoutes from .routes.recipe_routes import RecipeRoutes from .routes.checkpoints_routes import CheckpointsRoutes +from .routes.stats_routes import StatsRoutes from .routes.update_routes import UpdateRoutes from .routes.misc_routes import MiscRoutes from .routes.example_images_routes import ExampleImagesRoutes @@ -112,10 +113,12 @@ class LoraManager: # Setup feature routes lora_routes = LoraRoutes() checkpoints_routes = CheckpointsRoutes() + stats_routes = StatsRoutes() # Initialize routes lora_routes.setup_routes(app) checkpoints_routes.setup_routes(app) + stats_routes.setup_routes(app) # Add statistics routes ApiRoutes.setup_routes(app) RecipeRoutes.setup_routes(app) UpdateRoutes.setup_routes(app) diff --git a/py/routes/stats_routes.py b/py/routes/stats_routes.py new file mode 100644 index 00000000..58bf0d66 --- /dev/null +++ b/py/routes/stats_routes.py @@ -0,0 +1,438 @@ +import os +import json +import jinja2 +from aiohttp import web +import logging +from datetime import datetime, timedelta +from collections import defaultdict, Counter +from typing import Dict, List, Any + +from ..config import config +from ..services.settings_manager import settings +from ..services.service_registry import ServiceRegistry +from ..utils.usage_stats import UsageStats + +logger = logging.getLogger(__name__) + +class StatsRoutes: + """Route handlers for Statistics page and API endpoints""" + + def __init__(self): + self.lora_scanner = None + self.checkpoint_scanner = None + self.usage_stats = None + self.template_env = jinja2.Environment( + loader=jinja2.FileSystemLoader(config.templates_path), + autoescape=True + ) + + async def init_services(self): + """Initialize services from ServiceRegistry""" + self.lora_scanner = await ServiceRegistry.get_lora_scanner() + self.checkpoint_scanner = await ServiceRegistry.get_checkpoint_scanner() + self.usage_stats = UsageStats() + + async def handle_stats_page(self, request: web.Request) -> web.Response: + """Handle GET /statistics request""" + try: + # Ensure services are initialized + await self.init_services() + + # Check if scanners are initializing + lora_initializing = ( + self.lora_scanner._cache is None or + (hasattr(self.lora_scanner, 'is_initializing') and self.lora_scanner.is_initializing()) + ) + + checkpoint_initializing = ( + self.checkpoint_scanner._cache is None or + (hasattr(self.checkpoint_scanner, '_is_initializing') and self.checkpoint_scanner._is_initializing) + ) + + is_initializing = lora_initializing or checkpoint_initializing + + template = self.template_env.get_template('statistics.html') + rendered = template.render( + is_initializing=is_initializing, + settings=settings, + request=request + ) + + return web.Response( + text=rendered, + content_type='text/html' + ) + + except Exception as e: + logger.error(f"Error handling statistics request: {e}", exc_info=True) + return web.Response( + text="Error loading statistics page", + status=500 + ) + + async def get_collection_overview(self, request: web.Request) -> web.Response: + """Get collection overview statistics""" + try: + await self.init_services() + + # Get LoRA statistics + lora_cache = await self.lora_scanner.get_cached_data() + lora_count = len(lora_cache.raw_data) + lora_size = sum(lora.get('size', 0) for lora in lora_cache.raw_data) + + # Get Checkpoint statistics + checkpoint_cache = await self.checkpoint_scanner.get_cached_data() + checkpoint_count = len(checkpoint_cache.raw_data) + checkpoint_size = sum(cp.get('size', 0) for cp in checkpoint_cache.raw_data) + + # Get usage statistics + usage_data = await self.usage_stats.get_stats() + + return web.json_response({ + 'success': True, + 'data': { + 'total_models': lora_count + checkpoint_count, + 'lora_count': lora_count, + 'checkpoint_count': checkpoint_count, + 'total_size': lora_size + checkpoint_size, + 'lora_size': lora_size, + 'checkpoint_size': checkpoint_size, + 'total_generations': usage_data.get('total_executions', 0), + 'unused_loras': self._count_unused_models(lora_cache.raw_data, usage_data.get('loras', {})), + 'unused_checkpoints': self._count_unused_models(checkpoint_cache.raw_data, usage_data.get('checkpoints', {})) + } + }) + + except Exception as e: + logger.error(f"Error getting collection overview: {e}", exc_info=True) + return web.json_response({ + 'success': False, + 'error': str(e) + }, status=500) + + async def get_usage_analytics(self, request: web.Request) -> web.Response: + """Get usage analytics data""" + try: + await self.init_services() + + # Get usage statistics + usage_data = await self.usage_stats.get_stats() + + # Get model data for enrichment + lora_cache = await self.lora_scanner.get_cached_data() + checkpoint_cache = await self.checkpoint_scanner.get_cached_data() + + # Create hash to model mapping + lora_map = {lora['sha256']: lora for lora in lora_cache.raw_data} + checkpoint_map = {cp['sha256']: cp for cp in checkpoint_cache.raw_data} + + # Prepare top used models + top_loras = self._get_top_used_models(usage_data.get('loras', {}), lora_map, 10) + top_checkpoints = self._get_top_used_models(usage_data.get('checkpoints', {}), checkpoint_map, 10) + + # Prepare usage timeline (last 30 days) + timeline = self._get_usage_timeline(usage_data, 30) + + return web.json_response({ + 'success': True, + 'data': { + 'top_loras': top_loras, + 'top_checkpoints': top_checkpoints, + 'usage_timeline': timeline, + 'total_executions': usage_data.get('total_executions', 0) + } + }) + + except Exception as e: + logger.error(f"Error getting usage analytics: {e}", exc_info=True) + return web.json_response({ + 'success': False, + 'error': str(e) + }, status=500) + + async def get_base_model_distribution(self, request: web.Request) -> web.Response: + """Get base model distribution statistics""" + try: + await self.init_services() + + # Get model data + lora_cache = await self.lora_scanner.get_cached_data() + checkpoint_cache = await self.checkpoint_scanner.get_cached_data() + + # Count by base model + lora_base_models = Counter(lora.get('base_model', 'Unknown') for lora in lora_cache.raw_data) + checkpoint_base_models = Counter(cp.get('base_model', 'Unknown') for cp in checkpoint_cache.raw_data) + + return web.json_response({ + 'success': True, + 'data': { + 'loras': dict(lora_base_models), + 'checkpoints': dict(checkpoint_base_models) + } + }) + + except Exception as e: + logger.error(f"Error getting base model distribution: {e}", exc_info=True) + return web.json_response({ + 'success': False, + 'error': str(e) + }, status=500) + + async def get_tag_analytics(self, request: web.Request) -> web.Response: + """Get tag usage analytics""" + try: + await self.init_services() + + # Get model data + lora_cache = await self.lora_scanner.get_cached_data() + checkpoint_cache = await self.checkpoint_scanner.get_cached_data() + + # Count tag frequencies + all_tags = [] + for lora in lora_cache.raw_data: + all_tags.extend(lora.get('tags', [])) + for cp in checkpoint_cache.raw_data: + all_tags.extend(cp.get('tags', [])) + + tag_counts = Counter(all_tags) + + # Get top 50 tags + top_tags = [{'tag': tag, 'count': count} for tag, count in tag_counts.most_common(50)] + + return web.json_response({ + 'success': True, + 'data': { + 'top_tags': top_tags, + 'total_unique_tags': len(tag_counts) + } + }) + + except Exception as e: + logger.error(f"Error getting tag analytics: {e}", exc_info=True) + return web.json_response({ + 'success': False, + 'error': str(e) + }, status=500) + + async def get_storage_analytics(self, request: web.Request) -> web.Response: + """Get storage usage analytics""" + try: + await self.init_services() + + # Get usage statistics + usage_data = await self.usage_stats.get_stats() + + # Get model data + lora_cache = await self.lora_scanner.get_cached_data() + checkpoint_cache = await self.checkpoint_scanner.get_cached_data() + + # Create models with usage data + lora_storage = [] + for lora in lora_cache.raw_data: + usage_count = 0 + if lora['sha256'] in usage_data.get('loras', {}): + usage_count = usage_data['loras'][lora['sha256']].get('total', 0) + + lora_storage.append({ + 'name': lora['model_name'], + 'size': lora.get('size', 0), + 'usage_count': usage_count, + 'folder': lora.get('folder', ''), + 'base_model': lora.get('base_model', 'Unknown') + }) + + checkpoint_storage = [] + for cp in checkpoint_cache.raw_data: + usage_count = 0 + if cp['sha256'] in usage_data.get('checkpoints', {}): + usage_count = usage_data['checkpoints'][cp['sha256']].get('total', 0) + + checkpoint_storage.append({ + 'name': cp['model_name'], + 'size': cp.get('size', 0), + 'usage_count': usage_count, + 'folder': cp.get('folder', ''), + 'base_model': cp.get('base_model', 'Unknown') + }) + + # Sort by size + lora_storage.sort(key=lambda x: x['size'], reverse=True) + checkpoint_storage.sort(key=lambda x: x['size'], reverse=True) + + return web.json_response({ + 'success': True, + 'data': { + 'loras': lora_storage[:20], # Top 20 by size + 'checkpoints': checkpoint_storage[:20] + } + }) + + except Exception as e: + logger.error(f"Error getting storage analytics: {e}", exc_info=True) + return web.json_response({ + 'success': False, + 'error': str(e) + }, status=500) + + async def get_insights(self, request: web.Request) -> web.Response: + """Get smart insights about the collection""" + try: + await self.init_services() + + # Get usage statistics + usage_data = await self.usage_stats.get_stats() + + # Get model data + lora_cache = await self.lora_scanner.get_cached_data() + checkpoint_cache = await self.checkpoint_scanner.get_cached_data() + + insights = [] + + # Calculate unused models + unused_loras = self._count_unused_models(lora_cache.raw_data, usage_data.get('loras', {})) + unused_checkpoints = self._count_unused_models(checkpoint_cache.raw_data, usage_data.get('checkpoints', {})) + + total_loras = len(lora_cache.raw_data) + total_checkpoints = len(checkpoint_cache.raw_data) + + if total_loras > 0: + unused_lora_percent = (unused_loras / total_loras) * 100 + if unused_lora_percent > 50: + insights.append({ + 'type': 'warning', + 'title': 'High Number of Unused LoRAs', + 'description': f'{unused_lora_percent:.1f}% of your LoRAs ({unused_loras}/{total_loras}) have never been used.', + 'suggestion': 'Consider organizing or archiving unused models to free up storage space.' + }) + + if total_checkpoints > 0: + unused_checkpoint_percent = (unused_checkpoints / total_checkpoints) * 100 + if unused_checkpoint_percent > 30: + insights.append({ + 'type': 'warning', + 'title': 'Unused Checkpoints Detected', + 'description': f'{unused_checkpoint_percent:.1f}% of your checkpoints ({unused_checkpoints}/{total_checkpoints}) have never been used.', + 'suggestion': 'Review and consider removing checkpoints you no longer need.' + }) + + # Storage insights + total_size = sum(lora.get('size', 0) for lora in lora_cache.raw_data) + \ + sum(cp.get('size', 0) for cp in checkpoint_cache.raw_data) + + if total_size > 100 * 1024 * 1024 * 1024: # 100GB + insights.append({ + 'type': 'info', + 'title': 'Large Collection Detected', + 'description': f'Your model collection is using {self._format_size(total_size)} of storage.', + 'suggestion': 'Consider using external storage or cloud solutions for better organization.' + }) + + # Recent activity insight + if usage_data.get('total_executions', 0) > 100: + insights.append({ + 'type': 'success', + 'title': 'Active User', + 'description': f'You\'ve completed {usage_data["total_executions"]} generations so far!', + 'suggestion': 'Keep exploring and creating amazing content with your models.' + }) + + return web.json_response({ + 'success': True, + 'data': { + 'insights': insights + } + }) + + except Exception as e: + logger.error(f"Error getting insights: {e}", exc_info=True) + return web.json_response({ + 'success': False, + 'error': str(e) + }, status=500) + + def _count_unused_models(self, models: List[Dict], usage_data: Dict) -> int: + """Count models that have never been used""" + used_hashes = set(usage_data.keys()) + unused_count = 0 + + for model in models: + if model.get('sha256') not in used_hashes: + unused_count += 1 + + return unused_count + + def _get_top_used_models(self, usage_data: Dict, model_map: Dict, limit: int) -> List[Dict]: + """Get top used models with their metadata""" + sorted_usage = sorted(usage_data.items(), key=lambda x: x[1].get('total', 0), reverse=True) + + top_models = [] + for sha256, usage_info in sorted_usage[:limit]: + if sha256 in model_map: + model = model_map[sha256] + top_models.append({ + 'name': model['model_name'], + 'usage_count': usage_info.get('total', 0), + 'base_model': model.get('base_model', 'Unknown'), + 'preview_url': config.get_preview_static_url(model.get('preview_url', '')), + 'folder': model.get('folder', '') + }) + + return top_models + + def _get_usage_timeline(self, usage_data: Dict, days: int) -> List[Dict]: + """Get usage timeline for the past N days""" + timeline = [] + today = datetime.now() + + for i in range(days): + date = today - timedelta(days=i) + date_str = date.strftime('%Y-%m-%d') + + lora_usage = 0 + checkpoint_usage = 0 + + # Count usage for this date + for model_usage in usage_data.get('loras', {}).values(): + if isinstance(model_usage, dict) and 'history' in model_usage: + lora_usage += model_usage['history'].get(date_str, 0) + + for model_usage in usage_data.get('checkpoints', {}).values(): + if isinstance(model_usage, dict) and 'history' in model_usage: + checkpoint_usage += model_usage['history'].get(date_str, 0) + + timeline.append({ + 'date': date_str, + 'lora_usage': lora_usage, + 'checkpoint_usage': checkpoint_usage, + 'total_usage': lora_usage + checkpoint_usage + }) + + return list(reversed(timeline)) # Oldest to newest + + def _format_size(self, size_bytes: int) -> str: + """Format file size in human readable format""" + for unit in ['B', 'KB', 'MB', 'GB', 'TB']: + if size_bytes < 1024.0: + return f"{size_bytes:.1f} {unit}" + size_bytes /= 1024.0 + return f"{size_bytes:.1f} PB" + + def setup_routes(self, app: web.Application): + """Register routes with the application""" + # Add an app startup handler to initialize services + app.on_startup.append(self._on_startup) + + # Register page route + app.router.add_get('/statistics', self.handle_stats_page) + + # Register API routes + app.router.add_get('/api/stats/collection-overview', self.get_collection_overview) + app.router.add_get('/api/stats/usage-analytics', self.get_usage_analytics) + app.router.add_get('/api/stats/base-model-distribution', self.get_base_model_distribution) + app.router.add_get('/api/stats/tag-analytics', self.get_tag_analytics) + app.router.add_get('/api/stats/storage-analytics', self.get_storage_analytics) + app.router.add_get('/api/stats/insights', self.get_insights) + + async def _on_startup(self, app): + """Initialize services when the app starts""" + await self.init_services() \ No newline at end of file diff --git a/standalone.py b/standalone.py index 79c3bd8d..73d98db2 100644 --- a/standalone.py +++ b/standalone.py @@ -301,13 +301,16 @@ class StandaloneLoraManager(LoraManager): from py.routes.update_routes import UpdateRoutes from py.routes.misc_routes import MiscRoutes from py.routes.example_images_routes import ExampleImagesRoutes + from py.routes.stats_routes import StatsRoutes lora_routes = LoraRoutes() checkpoints_routes = CheckpointsRoutes() + stats_routes = StatsRoutes() # Initialize routes lora_routes.setup_routes(app) checkpoints_routes.setup_routes(app) + stats_routes.setup_routes(app) ApiRoutes.setup_routes(app) RecipeRoutes.setup_routes(app) UpdateRoutes.setup_routes(app) diff --git a/static/css/components/statistics.css b/static/css/components/statistics.css new file mode 100644 index 00000000..c3e5325b --- /dev/null +++ b/static/css/components/statistics.css @@ -0,0 +1,520 @@ +/* Statistics Page Styles */ +.metrics-panel { + margin-bottom: var(--space-3); +} + +.metrics-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: var(--space-2); + margin-bottom: var(--space-3); +} + +.metric-card { + background: var(--card-bg); + border: 1px solid var(--border-color); + border-radius: var(--border-radius-base); + padding: var(--space-2); + text-align: center; + transition: all 0.3s ease; + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); +} + +.metric-card:hover { + transform: translateY(-2px); + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); +} + +.metric-card .metric-icon { + font-size: 2rem; + color: var(--lora-accent); + margin-bottom: var(--space-1); +} + +.metric-card .metric-value { + font-size: 1.8rem; + font-weight: bold; + color: var(--text-color); + margin-bottom: 4px; +} + +.metric-card .metric-label { + font-size: 0.9rem; + color: oklch(var(--text-color) / 0.7); +} + +.metric-card .metric-change { + font-size: 0.8rem; + margin-top: 4px; +} + +.metric-change.positive { + color: var(--lora-success); +} + +.metric-change.negative { + color: var(--lora-error); +} + +/* Dashboard Content */ +.dashboard-content { + background: var(--card-bg); + border: 1px solid var(--border-color); + border-radius: var(--border-radius-base); + overflow: hidden; +} + +.dashboard-tabs { + display: flex; + background: var(--bg-color); + border-bottom: 1px solid var(--border-color); + overflow-x: auto; +} + +.tab-button { + background: none; + border: none; + padding: var(--space-2) var(--space-3); + cursor: pointer; + transition: all 0.3s ease; + color: var(--text-color); + border-bottom: 3px solid transparent; + white-space: nowrap; + font-size: 0.9rem; + display: flex; + align-items: center; + gap: 8px; +} + +.tab-button:hover { + background: oklch(var(--lora-accent) / 0.1); +} + +.tab-button.active { + color: var(--lora-accent); + border-bottom-color: var(--lora-accent); + background: oklch(var(--lora-accent) / 0.05); +} + +.tab-content { + padding: var(--space-3); +} + +.tab-panel { + display: none; +} + +.tab-panel.active { + display: block; +} + +/* Panel Grid Layout */ +.panel-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(350px, 1fr)); + gap: var(--space-3); + align-items: start; +} + +.panel-grid .full-width { + grid-column: 1 / -1; +} + +/* Chart Containers */ +.chart-container { + background: var(--bg-color); + border: 1px solid var(--border-color); + border-radius: var(--border-radius-sm); + padding: var(--space-2); + min-height: 300px; +} + +.chart-container h3 { + margin: 0 0 var(--space-2) 0; + color: var(--text-color); + font-size: 1.1rem; + display: flex; + align-items: center; + gap: 8px; +} + +.chart-container h3 i { + color: var(--lora-accent); +} + +.chart-wrapper { + position: relative; + height: 250px; +} + +.chart-wrapper canvas { + max-height: 100%; +} + +/* List Containers */ +.list-container { + background: var(--bg-color); + border: 1px solid var(--border-color); + border-radius: var(--border-radius-sm); + padding: var(--space-2); + min-height: 300px; +} + +.list-container h3 { + margin: 0 0 var(--space-2) 0; + color: var(--text-color); + font-size: 1.1rem; + display: flex; + align-items: center; + gap: 8px; +} + +.list-container h3 i { + color: var(--lora-accent); +} + +.model-list { + display: flex; + flex-direction: column; + gap: 8px; +} + +.model-item { + display: flex; + align-items: center; + padding: 8px; + background: var(--card-bg); + border: 1px solid var(--border-color); + border-radius: var(--border-radius-xs); + transition: all 0.2s ease; +} + +.model-item:hover { + border-color: var(--lora-accent); + transform: translateX(2px); +} + +.model-item .model-preview { + width: 40px; + height: 40px; + border-radius: var(--border-radius-xs); + margin-right: 12px; + object-fit: cover; + background: var(--border-color); +} + +.model-item .model-info { + flex: 1; + min-width: 0; +} + +.model-item .model-name { + font-weight: 600; + text-shadow: none; + color: var(--text-color); + font-size: 0.9rem; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.model-item .model-meta { + font-size: 0.8rem; + color: oklch(var(--text-color) / 0.7); + margin-top: 2px; +} + +.model-item .model-usage { + text-align: right; + color: var(--lora-accent); + font-weight: 600; + font-size: 0.9rem; +} + +/* Tag Cloud */ +.tag-cloud { + display: flex; + flex-wrap: wrap; + gap: 8px; + padding: var(--space-2) 0; + max-height: 250px; + overflow-y: auto; +} + +.tag-cloud-item { + padding: 4px 8px; + background: oklch(var(--lora-accent) / 0.1); + color: var(--lora-accent); + border-radius: var(--border-radius-xs); + font-size: 0.8rem; + border: 1px solid oklch(var(--lora-accent) / 0.2); + transition: all 0.2s ease; + cursor: pointer; +} + +.tag-cloud-item:hover { + background: oklch(var(--lora-accent) / 0.2); + transform: scale(1.05); +} + +.tag-cloud-item.size-1 { font-size: 0.7rem; } +.tag-cloud-item.size-2 { font-size: 0.8rem; } +.tag-cloud-item.size-3 { font-size: 0.9rem; } +.tag-cloud-item.size-4 { font-size: 1.0rem; } +.tag-cloud-item.size-5 { font-size: 1.1rem; font-weight: 600; } + +/* Analysis Cards */ +.analysis-cards { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: var(--space-2); +} + +.analysis-card { + background: var(--card-bg); + border: 1px solid var(--border-color); + border-radius: var(--border-radius-xs); + padding: var(--space-2); + text-align: center; +} + +.analysis-card .card-icon { + font-size: 1.5rem; + color: var(--lora-accent); + margin-bottom: 8px; +} + +.analysis-card .card-value { + font-size: 1.4rem; + font-weight: bold; + color: var(--text-color); + margin-bottom: 4px; +} + +.analysis-card .card-label { + font-size: 0.85rem; + color: oklch(var(--text-color) / 0.7); +} + +/* Insights */ +.insights-container { + background: var(--bg-color); + border: 1px solid var(--border-color); + border-radius: var(--border-radius-sm); + padding: var(--space-3); +} + +.insights-container h3 { + margin: 0 0 var(--space-2) 0; + color: var(--text-color); + font-size: 1.2rem; + display: flex; + align-items: center; + gap: 8px; +} + +.insights-container h3 i { + color: var(--lora-accent); +} + +.insights-list { + display: flex; + flex-direction: column; + gap: var(--space-2); +} + +.insight-card { + padding: var(--space-2); + border-radius: var(--border-radius-xs); + border: 1px solid var(--border-color); + transition: all 0.3s ease; +} + +.insight-card:hover { + transform: translateY(-1px); + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); +} + +.insight-card.type-success { + border-left: 4px solid var(--lora-success); + background: oklch(var(--lora-success) / 0.05); +} + +.insight-card.type-warning { + border-left: 4px solid var(--lora-warning); + background: oklch(var(--lora-warning) / 0.05); +} + +.insight-card.type-info { + border-left: 4px solid var(--lora-accent); + background: oklch(var(--lora-accent) / 0.05); +} + +.insight-card.type-error { + border-left: 4px solid var(--lora-error); + background: oklch(var(--lora-error) / 0.05); +} + +.insight-title { + font-weight: 600; + color: var(--text-color); + margin-bottom: 8px; + font-size: 1rem; +} + +.insight-description { + color: oklch(var(--text-color) / 0.8); + margin-bottom: 8px; + font-size: 0.9rem; + line-height: 1.4; +} + +.insight-suggestion { + color: oklch(var(--text-color) / 0.7); + font-size: 0.85rem; + font-style: italic; +} + +/* Recommendations Section */ +.recommendations-section { + margin-top: var(--space-3); + padding-top: var(--space-3); + border-top: 1px solid var(--border-color); +} + +.recommendations-section h4 { + margin: 0 0 var(--space-2) 0; + color: var(--text-color); + font-size: 1.1rem; + display: flex; + align-items: center; + gap: 8px; +} + +.recommendations-section h4 i { + color: var(--lora-accent); +} + +.recommendations-list { + display: flex; + flex-direction: column; + gap: 12px; +} + +.recommendation-item { + padding: 12px; + background: var(--card-bg); + border: 1px solid var(--border-color); + border-radius: var(--border-radius-xs); + transition: all 0.2s ease; +} + +.recommendation-item:hover { + border-color: var(--lora-accent); +} + +.recommendation-title { + font-weight: 600; + color: var(--text-color); + margin-bottom: 6px; + font-size: 0.9rem; +} + +.recommendation-description { + color: oklch(var(--text-color) / 0.8); + font-size: 0.85rem; + line-height: 1.4; +} + +/* Loading States */ +.loading-placeholder { + display: flex; + align-items: center; + justify-content: center; + height: 200px; + color: oklch(var(--text-color) / 0.6); + font-size: 0.9rem; +} + +.loading-placeholder i { + margin-right: 8px; + animation: spin 1s linear infinite; +} + +@keyframes spin { + from { transform: rotate(0deg); } + to { transform: rotate(360deg); } +} + +/* Responsive Design */ +@media (max-width: 1200px) { + .panel-grid { + grid-template-columns: 1fr; + } + + .metrics-grid { + grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); + } +} + +@media (max-width: 768px) { + .dashboard-tabs { + flex-wrap: wrap; + } + + .tab-button { + flex: 1; + min-width: 0; + font-size: 0.8rem; + padding: 12px 8px; + } + + .tab-button i { + display: none; + } + + .tab-content { + padding: var(--space-2); + } + + .metrics-grid { + grid-template-columns: repeat(2, 1fr); + gap: var(--space-1); + } + + .metric-card { + padding: var(--space-1); + } + + .metric-card .metric-icon { + font-size: 1.5rem; + } + + .metric-card .metric-value { + font-size: 1.4rem; + } + + .chart-wrapper { + height: 200px; + } + + .model-item .model-preview { + width: 32px; + height: 32px; + } +} + +/* Dark mode adjustments */ +[data-theme="dark"] .chart-container, +[data-theme="dark"] .list-container, +[data-theme="dark"] .insights-container { + border-color: oklch(var(--border-color) / 0.3); +} + +[data-theme="dark"] .metric-card { + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.3); +} + +[data-theme="dark"] .metric-card:hover { + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.4); +} \ No newline at end of file diff --git a/static/css/style.css b/static/css/style.css index 1ee9946d..c2d18a54 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -30,6 +30,7 @@ @import 'components/alphabet-bar.css'; /* Add alphabet bar component */ @import 'components/duplicates.css'; /* Add duplicates component */ @import 'components/keyboard-nav.css'; /* Add keyboard navigation component */ +@import 'components/statistics.css'; /* Add statistics component */ .initialization-notice { display: flex; diff --git a/static/js/components/Header.js b/static/js/components/Header.js index 4d8c7fa7..27f6d7ad 100644 --- a/static/js/components/Header.js +++ b/static/js/components/Header.js @@ -26,6 +26,7 @@ export class HeaderManager { const path = window.location.pathname; if (path.includes('/loras/recipes')) return 'recipes'; if (path.includes('/checkpoints')) return 'checkpoints'; + if (path.includes('/statistics')) return 'statistics'; if (path.includes('/loras')) return 'loras'; return 'unknown'; } @@ -121,14 +122,17 @@ export class HeaderManager { }); } - // Handle help toggle - // const helpToggle = document.querySelector('.help-toggle'); - // if (helpToggle) { - // helpToggle.addEventListener('click', () => { - // if (window.modalManager) { - // window.modalManager.toggleModal('helpModal'); - // } - // }); - // } + // Hide search functionality on Statistics page + this.updateHeaderForPage(); + } + + updateHeaderForPage() { + const headerSearch = document.getElementById('headerSearch'); + + if (this.currentPage === 'statistics' && headerSearch) { + headerSearch.style.display = 'none'; + } else if (headerSearch) { + headerSearch.style.display = 'flex'; + } } } diff --git a/static/js/managers/FilterManager.js b/static/js/managers/FilterManager.js index 1ebdc9e2..86a880b2 100644 --- a/static/js/managers/FilterManager.js +++ b/static/js/managers/FilterManager.js @@ -148,6 +148,8 @@ export class FilterManager { apiEndpoint = '/api/recipes/base-models'; } else if (this.currentPage === 'checkpoints') { apiEndpoint = '/api/checkpoints/base-models'; + } else { + return; } // Fetch base models diff --git a/static/js/statistics.js b/static/js/statistics.js new file mode 100644 index 00000000..8e596729 --- /dev/null +++ b/static/js/statistics.js @@ -0,0 +1,730 @@ +// Statistics page functionality +import { appCore } from './core.js'; +import { showToast } from './utils/uiHelpers.js'; + +// Chart.js import (assuming it's available globally or via CDN) +// If Chart.js isn't available, we'll need to add it to the project + +class StatisticsManager { + constructor() { + this.charts = {}; + this.data = {}; + this.initialized = false; + } + + async initialize() { + if (this.initialized) return; + + console.log('StatisticsManager: Initializing...'); + + // Initialize tab functionality + this.initializeTabs(); + + // Load initial data + await this.loadAllData(); + + // Initialize charts and visualizations + this.initializeVisualizations(); + + this.initialized = true; + } + + initializeTabs() { + const tabButtons = document.querySelectorAll('.tab-button'); + const tabPanels = document.querySelectorAll('.tab-panel'); + + tabButtons.forEach(button => { + button.addEventListener('click', () => { + const tabId = button.dataset.tab; + + // Update active tab button + tabButtons.forEach(btn => btn.classList.remove('active')); + button.classList.add('active'); + + // Update active tab panel + tabPanels.forEach(panel => panel.classList.remove('active')); + const targetPanel = document.getElementById(`${tabId}-panel`); + if (targetPanel) { + targetPanel.classList.add('active'); + + // Refresh charts when tab becomes visible + this.refreshChartsInPanel(tabId); + } + }); + }); + } + + async loadAllData() { + try { + // Load all statistics data in parallel + const [ + collectionOverview, + usageAnalytics, + baseModelDistribution, + tagAnalytics, + storageAnalytics, + insights + ] = await Promise.all([ + this.fetchData('/api/stats/collection-overview'), + this.fetchData('/api/stats/usage-analytics'), + this.fetchData('/api/stats/base-model-distribution'), + this.fetchData('/api/stats/tag-analytics'), + this.fetchData('/api/stats/storage-analytics'), + this.fetchData('/api/stats/insights') + ]); + + this.data = { + collection: collectionOverview.data, + usage: usageAnalytics.data, + baseModels: baseModelDistribution.data, + tags: tagAnalytics.data, + storage: storageAnalytics.data, + insights: insights.data + }; + + console.log('Statistics data loaded:', this.data); + } catch (error) { + console.error('Error loading statistics data:', error); + showToast('Failed to load statistics data', 'error'); + } + } + + async fetchData(endpoint) { + const response = await fetch(endpoint); + if (!response.ok) { + throw new Error(`Failed to fetch ${endpoint}: ${response.statusText}`); + } + return response.json(); + } + + initializeVisualizations() { + // Initialize metrics cards + this.renderMetricsCards(); + + // Initialize charts + this.initializeCharts(); + + // Initialize lists and other components + this.renderTopModelsLists(); + this.renderTagCloud(); + this.renderInsights(); + } + + renderMetricsCards() { + const metricsGrid = document.getElementById('metricsGrid'); + if (!metricsGrid || !this.data.collection) return; + + const metrics = [ + { + icon: 'fas fa-magic', + value: this.data.collection.total_models, + label: 'Total Models', + format: 'number' + }, + { + icon: 'fas fa-database', + value: this.data.collection.total_size, + label: 'Total Storage', + format: 'size' + }, + { + icon: 'fas fa-play-circle', + value: this.data.collection.total_generations, + label: 'Total Generations', + format: 'number' + }, + { + icon: 'fas fa-chart-line', + value: this.calculateUsageRate(), + label: 'Usage Rate', + format: 'percentage' + }, + { + icon: 'fas fa-layer-group', + value: this.data.collection.lora_count, + label: 'LoRAs', + format: 'number' + }, + { + icon: 'fas fa-check-circle', + value: this.data.collection.checkpoint_count, + label: 'Checkpoints', + format: 'number' + } + ]; + + metricsGrid.innerHTML = metrics.map(metric => this.createMetricCard(metric)).join(''); + } + + createMetricCard(metric) { + const formattedValue = this.formatValue(metric.value, metric.format); + + return ` +
+
+ +
+
${formattedValue}
+
${metric.label}
+
+ `; + } + + formatValue(value, format) { + switch (format) { + case 'number': + return new Intl.NumberFormat().format(value); + case 'size': + return this.formatFileSize(value); + case 'percentage': + return `${value.toFixed(1)}%`; + default: + return value; + } + } + + formatFileSize(bytes) { + if (bytes === 0) return '0 Bytes'; + const k = 1024; + const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i]; + } + + calculateUsageRate() { + if (!this.data.collection) return 0; + + const totalModels = this.data.collection.total_models; + const unusedModels = this.data.collection.unused_loras + this.data.collection.unused_checkpoints; + const usedModels = totalModels - unusedModels; + + return totalModels > 0 ? (usedModels / totalModels) * 100 : 0; + } + + initializeCharts() { + // Check if Chart.js is available + if (typeof Chart === 'undefined') { + console.warn('Chart.js is not available. Charts will not be rendered.'); + this.showChartPlaceholders(); + return; + } + + // Collection pie chart + this.createCollectionPieChart(); + + // Base model distribution chart + this.createBaseModelChart(); + + // Usage timeline chart + this.createUsageTimelineChart(); + + // Usage distribution chart + this.createUsageDistributionChart(); + + // Storage chart + this.createStorageChart(); + + // Storage efficiency chart + this.createStorageEfficiencyChart(); + } + + createCollectionPieChart() { + const ctx = document.getElementById('collectionPieChart'); + if (!ctx || !this.data.collection) return; + + const data = { + labels: ['LoRAs', 'Checkpoints'], + datasets: [{ + data: [this.data.collection.lora_count, this.data.collection.checkpoint_count], + backgroundColor: [ + 'oklch(68% 0.28 256)', + 'oklch(68% 0.28 200)' + ], + borderWidth: 2, + borderColor: getComputedStyle(document.documentElement).getPropertyValue('--border-color') + }] + }; + + this.charts.collection = new Chart(ctx, { + type: 'doughnut', + data: data, + options: { + responsive: true, + maintainAspectRatio: false, + plugins: { + legend: { + position: 'bottom' + } + } + } + }); + } + + createBaseModelChart() { + const ctx = document.getElementById('baseModelChart'); + if (!ctx || !this.data.baseModels) return; + + const loraData = this.data.baseModels.loras; + const checkpointData = this.data.baseModels.checkpoints; + + const allModels = new Set([...Object.keys(loraData), ...Object.keys(checkpointData)]); + + const data = { + labels: Array.from(allModels), + datasets: [ + { + label: 'LoRAs', + data: Array.from(allModels).map(model => loraData[model] || 0), + backgroundColor: 'oklch(68% 0.28 256 / 0.7)' + }, + { + label: 'Checkpoints', + data: Array.from(allModels).map(model => checkpointData[model] || 0), + backgroundColor: 'oklch(68% 0.28 200 / 0.7)' + } + ] + }; + + this.charts.baseModels = new Chart(ctx, { + type: 'bar', + data: data, + options: { + responsive: true, + maintainAspectRatio: false, + scales: { + x: { + stacked: true + }, + y: { + stacked: true + } + } + } + }); + } + + createUsageTimelineChart() { + const ctx = document.getElementById('usageTimelineChart'); + if (!ctx || !this.data.usage) return; + + const timeline = this.data.usage.usage_timeline || []; + + const data = { + labels: timeline.map(item => new Date(item.date).toLocaleDateString()), + datasets: [ + { + label: 'LoRA Usage', + data: timeline.map(item => item.lora_usage), + borderColor: 'oklch(68% 0.28 256)', + backgroundColor: 'oklch(68% 0.28 256 / 0.1)', + fill: true + }, + { + label: 'Checkpoint Usage', + data: timeline.map(item => item.checkpoint_usage), + borderColor: 'oklch(68% 0.28 200)', + backgroundColor: 'oklch(68% 0.28 200 / 0.1)', + fill: true + } + ] + }; + + this.charts.timeline = new Chart(ctx, { + type: 'line', + data: data, + options: { + responsive: true, + maintainAspectRatio: false, + interaction: { + mode: 'index', + intersect: false, + }, + scales: { + x: { + display: true, + title: { + display: true, + text: 'Date' + } + }, + y: { + display: true, + title: { + display: true, + text: 'Usage Count' + } + } + } + } + }); + } + + createUsageDistributionChart() { + const ctx = document.getElementById('usageDistributionChart'); + if (!ctx || !this.data.usage) return; + + const topLoras = this.data.usage.top_loras || []; + const topCheckpoints = this.data.usage.top_checkpoints || []; + + // Combine and sort all models by usage + const allModels = [ + ...topLoras.map(m => ({ ...m, type: 'LoRA' })), + ...topCheckpoints.map(m => ({ ...m, type: 'Checkpoint' })) + ].sort((a, b) => b.usage_count - a.usage_count).slice(0, 10); + + const data = { + labels: allModels.map(model => model.name), + datasets: [{ + label: 'Usage Count', + data: allModels.map(model => model.usage_count), + backgroundColor: allModels.map(model => + model.type === 'LoRA' ? 'oklch(68% 0.28 256)' : 'oklch(68% 0.28 200)' + ) + }] + }; + + this.charts.distribution = new Chart(ctx, { + type: 'bar', + data: data, + options: { + responsive: true, + maintainAspectRatio: false, + indexAxis: 'y', + plugins: { + legend: { + display: false + } + } + } + }); + } + + createStorageChart() { + const ctx = document.getElementById('storageChart'); + if (!ctx || !this.data.collection) return; + + const data = { + labels: ['LoRAs', 'Checkpoints'], + datasets: [{ + data: [this.data.collection.lora_size, this.data.collection.checkpoint_size], + backgroundColor: [ + 'oklch(68% 0.28 256)', + 'oklch(68% 0.28 200)' + ] + }] + }; + + this.charts.storage = new Chart(ctx, { + type: 'doughnut', + data: data, + options: { + responsive: true, + maintainAspectRatio: false, + plugins: { + legend: { + position: 'bottom' + }, + tooltip: { + callbacks: { + label: (context) => { + const value = this.formatFileSize(context.raw); + return `${context.label}: ${value}`; + } + } + } + } + } + }); + } + + createStorageEfficiencyChart() { + const ctx = document.getElementById('storageEfficiencyChart'); + if (!ctx || !this.data.storage) return; + + const loraData = this.data.storage.loras || []; + const checkpointData = this.data.storage.checkpoints || []; + + const allData = [ + ...loraData.map(item => ({ ...item, type: 'LoRA' })), + ...checkpointData.map(item => ({ ...item, type: 'Checkpoint' })) + ]; + + const data = { + datasets: [{ + label: 'Models', + data: allData.map(item => ({ + x: item.size, + y: item.usage_count, + name: item.name, + type: item.type + })), + backgroundColor: allData.map(item => + item.type === 'LoRA' ? 'oklch(68% 0.28 256 / 0.6)' : 'oklch(68% 0.28 200 / 0.6)' + ) + }] + }; + + this.charts.efficiency = new Chart(ctx, { + type: 'scatter', + data: data, + options: { + responsive: true, + maintainAspectRatio: false, + scales: { + x: { + title: { + display: true, + text: 'File Size (bytes)' + }, + type: 'logarithmic' + }, + y: { + title: { + display: true, + text: 'Usage Count' + } + } + }, + plugins: { + tooltip: { + callbacks: { + label: (context) => { + const point = context.raw; + return `${point.name}: ${this.formatFileSize(point.x)}, ${point.y} uses`; + } + } + } + } + } + }); + } + + renderTopModelsLists() { + this.renderTopLorasList(); + this.renderTopCheckpointsList(); + this.renderLargestModelsList(); + } + + renderTopLorasList() { + const container = document.getElementById('topLorasList'); + if (!container || !this.data.usage?.top_loras) return; + + const topLoras = this.data.usage.top_loras; + + if (topLoras.length === 0) { + container.innerHTML = '
No usage data available
'; + return; + } + + container.innerHTML = topLoras.map(lora => ` +
+ ${lora.name} +
+
${lora.name}
+
${lora.base_model} • ${lora.folder}
+
+
${lora.usage_count}
+
+ `).join(''); + } + + renderTopCheckpointsList() { + const container = document.getElementById('topCheckpointsList'); + if (!container || !this.data.usage?.top_checkpoints) return; + + const topCheckpoints = this.data.usage.top_checkpoints; + + if (topCheckpoints.length === 0) { + container.innerHTML = '
No usage data available
'; + return; + } + + container.innerHTML = topCheckpoints.map(checkpoint => ` +
+ ${checkpoint.name} +
+
${checkpoint.name}
+
${checkpoint.base_model} • ${checkpoint.folder}
+
+
${checkpoint.usage_count}
+
+ `).join(''); + } + + renderLargestModelsList() { + const container = document.getElementById('largestModelsList'); + if (!container || !this.data.storage) return; + + const loraModels = this.data.storage.loras || []; + const checkpointModels = this.data.storage.checkpoints || []; + + // Combine and sort by size + const allModels = [ + ...loraModels.map(m => ({ ...m, type: 'LoRA' })), + ...checkpointModels.map(m => ({ ...m, type: 'Checkpoint' })) + ].sort((a, b) => b.size - a.size).slice(0, 10); + + if (allModels.length === 0) { + container.innerHTML = '
No storage data available
'; + return; + } + + container.innerHTML = allModels.map(model => ` +
+
+
${model.name}
+
${model.type} • ${model.base_model}
+
+
${this.formatFileSize(model.size)}
+
+ `).join(''); + } + + renderTagCloud() { + const container = document.getElementById('tagCloud'); + if (!container || !this.data.tags?.top_tags) return; + + const topTags = this.data.tags.top_tags.slice(0, 30); // Show top 30 tags + const maxCount = Math.max(...topTags.map(tag => tag.count)); + + container.innerHTML = topTags.map(tagData => { + const size = Math.ceil((tagData.count / maxCount) * 5); + return ` + + ${tagData.tag} + + `; + }).join(''); + } + + renderInsights() { + const container = document.getElementById('insightsList'); + if (!container || !this.data.insights?.insights) return; + + const insights = this.data.insights.insights; + + if (insights.length === 0) { + container.innerHTML = '
No insights available
'; + return; + } + + container.innerHTML = insights.map(insight => ` +
+
${insight.title}
+
${insight.description}
+
${insight.suggestion}
+
+ `).join(''); + + // Render collection analysis cards + this.renderCollectionAnalysis(); + } + + renderCollectionAnalysis() { + const container = document.getElementById('collectionAnalysis'); + if (!container || !this.data.collection) return; + + const analysis = [ + { + icon: 'fas fa-percentage', + value: this.calculateUsageRate(), + label: 'Usage Rate', + format: 'percentage' + }, + { + icon: 'fas fa-tags', + value: this.data.tags?.total_unique_tags || 0, + label: 'Unique Tags', + format: 'number' + }, + { + icon: 'fas fa-clock', + value: this.data.collection.unused_loras + this.data.collection.unused_checkpoints, + label: 'Unused Models', + format: 'number' + }, + { + icon: 'fas fa-chart-line', + value: this.calculateAverageUsage(), + label: 'Avg. Uses/Model', + format: 'decimal' + } + ]; + + container.innerHTML = analysis.map(item => ` +
+
+ +
+
${this.formatValue(item.value, item.format)}
+
${item.label}
+
+ `).join(''); + } + + calculateAverageUsage() { + if (!this.data.usage || !this.data.collection) return 0; + + const totalGenerations = this.data.collection.total_generations; + const totalModels = this.data.collection.total_models; + + return totalModels > 0 ? totalGenerations / totalModels : 0; + } + + showChartPlaceholders() { + const chartCanvases = document.querySelectorAll('canvas'); + chartCanvases.forEach(canvas => { + const container = canvas.parentElement; + container.innerHTML = '
Chart requires Chart.js library
'; + }); + } + + refreshChartsInPanel(panelId) { + // Refresh charts when panels become visible + setTimeout(() => { + Object.values(this.charts).forEach(chart => { + if (chart && typeof chart.resize === 'function') { + chart.resize(); + } + }); + }, 100); + } + + destroy() { + // Clean up charts + Object.values(this.charts).forEach(chart => { + if (chart && typeof chart.destroy === 'function') { + chart.destroy(); + } + }); + this.charts = {}; + this.initialized = false; + } +} + +// Initialize statistics page when DOM is ready +document.addEventListener('DOMContentLoaded', async () => { + // Wait for app core to initialize + await appCore.initialize(); + + // Initialize statistics functionality + const statsManager = new StatisticsManager(); + await statsManager.initialize(); + + // Make statsManager globally available for debugging + window.statsManager = statsManager; + + console.log('Statistics page initialized successfully'); +}); + +// Handle page unload +window.addEventListener('beforeunload', () => { + if (window.statsManager) { + window.statsManager.destroy(); + } +}); \ No newline at end of file diff --git a/static/js/utils/routes.js b/static/js/utils/routes.js index a3b8f859..b9d94f7b 100644 --- a/static/js/utils/routes.js +++ b/static/js/utils/routes.js @@ -40,7 +40,8 @@ export const apiRoutes = { export const pageRoutes = { loras: '/loras', recipes: '/loras/recipes', - checkpoints: '/checkpoints' + checkpoints: '/checkpoints', + statistics: '/statistics' }; // Helper function to get current page type @@ -48,6 +49,7 @@ export function getCurrentPageType() { const path = window.location.pathname; if (path.includes('/loras/recipes')) return 'recipes'; if (path.includes('/checkpoints')) return 'checkpoints'; + if (path.includes('/statistics')) return 'statistics'; if (path.includes('/loras')) return 'loras'; return 'unknown'; } \ No newline at end of file diff --git a/templates/components/header.html b/templates/components/header.html index a16a75e1..00013003 100644 --- a/templates/components/header.html +++ b/templates/components/header.html @@ -16,10 +16,13 @@ Checkpoints + + Statistics + -
+