From 4b247995d11f96402a4b23a8b7d5fef137bde687 Mon Sep 17 00:00:00 2001
From: Will Miao <13051207myq@gmail.com>
Date: Sun, 2 Feb 2025 22:57:22 +0800
Subject: [PATCH] Implement paginated LoRA data fetching with caching and
 infinite scroll

---
 routes/api_routes.py     |  85 +++++++++++
 routes/lora_routes.py    |  21 ++-
 services/lora_scanner.py |  78 +++++++++-
 static/js/script.js      | 320 ++++++++++++++++++++++++++++++---------
 templates/loras.html     |  52 +------
 5 files changed, 429 insertions(+), 127 deletions(-)

diff --git a/routes/api_routes.py b/routes/api_routes.py
index 79b84370..df5dde84 100644
--- a/routes/api_routes.py
+++ b/routes/api_routes.py
@@ -6,12 +6,16 @@ from typing import Dict, List
 from ..services.civitai_client import CivitaiClient
 from ..utils.file_utils import update_civitai_metadata, load_metadata
 from ..config import config
+from ..services.lora_scanner import LoraScanner
 
 logger = logging.getLogger(__name__)
 
 class ApiRoutes:
     """API route handlers for LoRA management"""
 
+    def __init__(self):
+        self.scanner = LoraScanner()
+
     @classmethod
     def setup_routes(cls, app: web.Application):
         """Register API routes"""
@@ -19,6 +23,7 @@ class ApiRoutes:
         app.router.add_post('/api/delete_model', routes.delete_model)
         app.router.add_post('/api/fetch-civitai', routes.fetch_civitai)
         app.router.add_post('/api/replace_preview', routes.replace_preview)
+        app.router.add_get('/api/loras', routes.get_loras)
 
     async def delete_model(self, request: web.Request) -> web.Response:
         """Handle model deletion request"""
@@ -88,6 +93,86 @@ class ApiRoutes:
             logger.error(f"Error replacing preview: {e}", exc_info=True)
             return web.Response(text=str(e), status=500)
 
+    async def get_loras(self, request: web.Request) -> web.Response:
+        """Handle paginated LoRA data request"""
+        try:
+            # Parse query parameters
+            page = int(request.query.get('page', '1'))
+            page_size = int(request.query.get('page_size', '20'))
+            sort_by = request.query.get('sort_by', 'name')
+            folder = request.query.get('folder')
+            
+            # Validate parameters
+            if page < 1 or page_size < 1 or page_size > 100:
+                return web.json_response({
+                    'error': 'Invalid pagination parameters'
+                }, status=400)
+            
+            if sort_by not in ['date', 'name']:
+                return web.json_response({
+                    'error': 'Invalid sort parameter'
+                }, status=400)
+            
+            # Get paginated data
+            result = await self.scanner.get_paginated_data(
+                page=page,
+                page_size=page_size,
+                sort_by=sort_by,
+                folder=folder
+            )
+            
+            # Format the response data
+            formatted_items = [
+                self._format_lora_response(item) 
+                for item in result['items']
+            ]
+            
+            return web.json_response({
+                'items': formatted_items,
+                'total': result['total'],
+                'page': result['page'],
+                'page_size': result['page_size'],
+                'total_pages': result['total_pages']
+            })
+            
+        except ValueError as e:
+            return web.json_response({
+                'error': 'Invalid parameters'
+            }, status=400)
+            
+        except Exception as e:
+            logger.error(f"Error fetching loras: {e}", exc_info=True)
+            return web.json_response({
+                'error': 'Internal server error'
+            }, status=500)
+
+    def _format_lora_response(self, lora: Dict) -> Dict:
+        """Format LoRA data for API response"""
+        return {
+            "model_name": lora["model_name"],
+            "file_name": lora["file_name"],
+            "preview_url": config.get_preview_static_url(lora["preview_url"]),
+            "base_model": lora["base_model"],
+            "folder": lora["folder"],
+            "sha256": lora["sha256"],
+            "file_path": lora["file_path"].replace(os.sep, "/"),
+            "modified": lora["modified"],
+            "from_civitai": lora.get("from_civitai", True),
+            "civitai": self._filter_civitai_data(lora.get("civitai", {}))
+        }
+
+    def _filter_civitai_data(self, data: Dict) -> Dict:
+        """Filter relevant fields from CivitAI data"""
+        if not data:
+            return {}
+            
+        fields = [
+            "id", "modelId", "name", "createdAt", "updatedAt", 
+            "publishedAt", "trainedWords", "baseModel", "description",
+            "model", "images"
+        ]
+        return {k: data[k] for k in fields if k in data}
+
     # Private helper methods
     async def _delete_model_files(self, target_dir: str, file_name: str) -> List[str]:
         """Delete model and associated files"""
diff --git a/routes/lora_routes.py b/routes/lora_routes.py
index cd2158e4..3c61055e 100644
--- a/routes/lora_routes.py
+++ b/routes/lora_routes.py
@@ -48,18 +48,27 @@ class LoraRoutes:
     async def handle_loras_page(self, request: web.Request) -> web.Response:
         """Handle GET /loras request"""
         try:
-            # Scan for loras
-            loras = await self.scanner.scan_all_loras()
+            # Get cached data
+            cache = await self.scanner.get_cached_data()
             
-            # Format data for template
-            formatted_loras = [self.format_lora_data(l) for l in loras]
-            folders = sorted(list(set(l['folder'] for l in loras)))
+            # Format initial data (first page only)
+            initial_data = await self.scanner.get_paginated_data(
+                page=1,
+                page_size=20,
+                sort_by='name'
+            )
+            
+            formatted_loras = [
+                self.format_lora_data(l) 
+                for l in initial_data['items']
+            ]
             
             # Render template
             template = self.template_env.get_template('loras.html')
             rendered = template.render(
                 loras=formatted_loras,
-                folders=folders
+                folders=cache.folders,
+                total_items=initial_data['total']
             )
             
             return web.Response(
diff --git a/services/lora_scanner.py b/services/lora_scanner.py
index a4575743..33fb84a9 100644
--- a/services/lora_scanner.py
+++ b/services/lora_scanner.py
@@ -1,15 +1,91 @@
 import os
 import logging
-from typing import List, Dict
+import time
+from typing import List, Dict, Optional
+from dataclasses import dataclass
+from operator import itemgetter
 from ..config import config
 from ..utils.file_utils import load_metadata, get_file_info, save_metadata
 from ..utils.lora_metadata import extract_lora_metadata
 
 logger = logging.getLogger(__name__)
 
+@dataclass
+class LoraCache:
+    """Cache structure for LoRA data"""
+    raw_data: List[Dict]
+    sorted_by_name: List[Dict]
+    sorted_by_date: List[Dict]
+    folders: List[str]
+    timestamp: float
+
 class LoraScanner:
     """Service for scanning and managing LoRA files"""
 
+    def __init__(self):
+        self._cache: Optional[LoraCache] = None
+        self.cache_ttl = 300  # 5 minutes cache TTL
+
+    async def get_cached_data(self, force_refresh: bool = False) -> LoraCache:
+        """Get cached LoRA data, refresh if needed"""
+        current_time = time.time()
+        
+        if (self._cache is None or 
+            force_refresh or 
+            current_time - self._cache.timestamp > self.cache_ttl):
+            
+            # Scan for new data
+            raw_data = await self.scan_all_loras()
+            
+            # Create sorted views
+            sorted_by_name = sorted(raw_data, key=itemgetter('model_name'))
+            sorted_by_date = sorted(raw_data, key=itemgetter('modified'), reverse=True)
+            folders = sorted(list(set(l['folder'] for l in raw_data)))
+            
+            # Update cache
+            self._cache = LoraCache(
+                raw_data=raw_data,
+                sorted_by_name=sorted_by_name,
+                sorted_by_date=sorted_by_date,
+                folders=folders,
+                timestamp=current_time
+            )
+        
+        return self._cache
+
+    async def get_paginated_data(self, 
+                                page: int, 
+                                page_size: int, 
+                                sort_by: str = 'date',
+                                folder: Optional[str] = None) -> Dict:
+        """Get paginated LoRA data"""
+        cache = await self.get_cached_data()
+        
+        # Select sorted data based on sort_by parameter
+        data = (cache.sorted_by_date if sort_by == 'date' 
+                else cache.sorted_by_name)
+        
+        # Apply folder filter if specified
+        if folder is not None:
+            data = [item for item in data if item['folder'] == folder]
+        
+        # Calculate pagination
+        total_items = len(data)
+        start_idx = (page - 1) * page_size
+        end_idx = min(start_idx + page_size, total_items)
+        
+        return {
+            'items': data[start_idx:end_idx],
+            'total': total_items,
+            'page': page,
+            'page_size': page_size,
+            'total_pages': (total_items + page_size - 1) // page_size
+        }
+
+    def invalidate_cache(self):
+        """Invalidate the current cache"""
+        self._cache = None
+
     async def scan_all_loras(self) -> List[Dict]:
         """Scan all LoRA directories and return metadata"""
         all_loras = []
diff --git a/static/js/script.js b/static/js/script.js
index 5f61d109..6554e3a0 100644
--- a/static/js/script.js
+++ b/static/js/script.js
@@ -207,68 +207,252 @@ class ModalManager {
 
 const modalManager = new ModalManager();
 
-// Data management functions
-async function refreshLoras() {
-    const loraGrid = document.getElementById('loraGrid');
-    const currentSort = document.getElementById('sortSelect').value;
-    const activeFolder = document.querySelector('.tag.active')?.dataset.folder;
+// State management
+let state = {
+    currentPage: 1,
+    isLoading: false,
+    hasMore: true,
+    sortBy: 'name',
+    activeFolder: null,
+    loadingManager: null,
+    observer: null  // 添加 observer 到状态管理中
+};
 
+// Initialize loading manager
+document.addEventListener('DOMContentLoaded', () => {
+    state.loadingManager = new LoadingManager();
+    initializeInfiniteScroll();
+    initializeEventListeners();
+});
+
+// Initialize infinite scroll
+function initializeInfiniteScroll() {
+    // 如果已存在 observer，先断开连接
+    if (state.observer) {
+        state.observer.disconnect();
+    }
+
+    // Create intersection observer for infinite scroll
+    state.observer = new IntersectionObserver(
+        (entries) => {
+            const target = entries[0];
+            if (target.isIntersecting && !state.isLoading && state.hasMore) {
+                loadMoreLoras();
+            }
+        },
+        { threshold: 0.1 }
+    );
+
+    // Add sentinel element for infinite scroll
+    const existingSentinel = document.getElementById('scroll-sentinel');
+    if (existingSentinel) {
+        state.observer.observe(existingSentinel);
+    } else {
+        const sentinel = document.createElement('div');
+        sentinel.id = 'scroll-sentinel';
+        sentinel.style.height = '10px';
+        document.getElementById('loraGrid').appendChild(sentinel);
+        state.observer.observe(sentinel);
+    }
+}
+
+// Initialize event listeners
+function initializeEventListeners() {
+    // Sort select handler
+    const sortSelect = document.getElementById('sortSelect');
+    if (sortSelect) {
+        sortSelect.value = state.sortBy;
+        sortSelect.addEventListener('change', async (e) => {
+            state.sortBy = e.target.value;
+            await resetAndReload();
+        });
+    }
+
+    // Folder filter handler
+    document.querySelectorAll('.folder-tags .tag').forEach(tag => {
+        // 移除原有的 onclick 属性处理方式，改用事件监听器
+        tag.removeAttribute('onclick');
+        tag.addEventListener('click', toggleFolder);
+    });
+}
+
+// Load more loras
+async function loadMoreLoras() {
+    if (state.isLoading || !state.hasMore) return;
+    
+    state.isLoading = true;
     try {
-        loadingManager.showSimpleLoading('Refreshing loras...');
-        const response = await fetch('/loras?refresh=true');
-        if (!response.ok) throw new Error('Refresh failed');
+        // 构建请求参数
+        const params = new URLSearchParams({
+            page: state.currentPage,
+            page_size: 20,
+            sort_by: state.sortBy
+        });
         
-        const doc = new DOMParser().parseFromString(await response.text(), 'text/html');
-        loraGrid.innerHTML = doc.getElementById('loraGrid').innerHTML;
+        // 只在有选中文件夹时添加 folder 参数
+        if (state.activeFolder !== null) {
+            params.append('folder', state.activeFolder);
+        }
+
+        console.log('Loading loras with params:', params.toString()); // 调试日志
+
+        const response = await fetch(`/api/loras?${params}`);
+        if (!response.ok) {
+            throw new Error(`Failed to fetch loras: ${response.statusText}`);
+        }
         
-        initializeLoraCards();
-        sortCards(currentSort);
-        if (activeFolder) filterByFolder(activeFolder);
+        const data = await response.json();
+        console.log('Received data:', data); // 调试日志
         
+        if (data.items.length === 0 && state.currentPage === 1) {
+            // 如果是第一页且没有数据，显示提示
+            const grid = document.getElementById('loraGrid');
+            grid.innerHTML = '<div class="no-results">No loras found in this folder</div>';
+            state.hasMore = false;
+        } else if (data.items.length > 0) {
+            state.hasMore = state.currentPage < data.total_pages;
+            state.currentPage++;
+            appendLoraCards(data.items);
+            
+            // 确保 sentinel 元素被观察
+            const sentinel = document.getElementById('scroll-sentinel');
+            if (sentinel && state.observer) {
+                state.observer.observe(sentinel);
+            }
+        } else {
+            state.hasMore = false;
+        }
+        
+    } catch (error) {
+        console.error('Error loading loras:', error);
+        showToast('Failed to load loras: ' + error.message, 'error');
+    } finally {
+        state.isLoading = false;
+    }
+}
+
+// Reset and reload
+async function resetAndReload() {
+    console.log('Resetting with state:', { ...state }); // 调试日志
+    
+    state.currentPage = 1;
+    state.hasMore = true;
+    state.isLoading = false;
+    
+    const grid = document.getElementById('loraGrid');
+    grid.innerHTML = ''; // 清空网格
+    
+    // 添加 sentinel
+    const sentinel = document.createElement('div');
+    sentinel.id = 'scroll-sentinel';
+    grid.appendChild(sentinel);
+    
+    // 重新初始化无限滚动
+    initializeInfiniteScroll();
+    
+    await loadMoreLoras();
+}
+
+// Append lora cards
+function appendLoraCards(loras) {
+    const grid = document.getElementById('loraGrid');
+    const sentinel = document.getElementById('scroll-sentinel');
+    
+    loras.forEach(lora => {
+        const card = createLoraCard(lora);
+        grid.insertBefore(card, sentinel);
+    });
+}
+
+// Create lora card
+function createLoraCard(lora) {
+    const card = document.createElement('div');
+    card.className = 'lora-card';
+    card.dataset.sha256 = lora.sha256;
+    card.dataset.filepath = lora.file_path;
+    card.dataset.name = lora.model_name;
+    card.dataset.file_name = lora.file_name;
+    card.dataset.folder = lora.folder;
+    card.dataset.modified = lora.modified;
+    card.dataset.from_civitai = lora.from_civitai;
+    card.dataset.meta = JSON.stringify(lora.civitai || {});
+
+    card.innerHTML = `
+        <div class="card-preview">
+            ${lora.preview_url.endsWith('.mp4') ? 
+                `<video controls autoplay muted loop>
+                    <source src="${lora.preview_url}" type="video/mp4">
+                </video>` :
+                `<img src="${lora.preview_url || '/loras_static/images/no-preview.png'}" alt="${lora.model_name}">`
+            }
+            <div class="card-header">
+                <span class="base-model-label" title="${lora.base_model}">
+                    ${lora.base_model}
+                </span>
+                <div class="card-actions">
+                    <i class="fas fa-globe" 
+                       title="${lora.from_civitai ? 'View on Civitai' : 'Not available from Civitai'}"
+                       ${lora.from_civitai ? 
+                           `onclick="event.stopPropagation(); openCivitai('${lora.model_name}')"` : 
+                           'style="opacity: 0.5; cursor: not-allowed"'}>
+                    </i>
+                    <i class="fas fa-copy" 
+                       title="Copy Model Name"
+                       onclick="event.stopPropagation(); navigator.clipboard.writeText('${lora.file_name}')">
+                    </i>
+                    <i class="fas fa-trash" 
+                       title="Delete Model"
+                       onclick="event.stopPropagation(); deleteModel('${lora.file_path}')">
+                    </i>
+                </div>
+            </div>
+            <div class="card-footer">
+                <div class="model-info">
+                    <span class="model-name">${lora.model_name}</span>
+                </div>
+                <div class="card-actions">
+                    <i class="fas fa-image" 
+                       title="Replace Preview Image"
+                       onclick="event.stopPropagation(); replacePreview('${lora.file_path}')">
+                    </i>
+                </div>
+            </div>
+        </div>
+    `;
+
+    // Add click handler for showing modal
+    card.addEventListener('click', () => {
+        const meta = JSON.parse(card.dataset.meta || '{}');
+        if (Object.keys(meta).length) {
+            showLoraModal(meta);
+        } else {
+            showToast(
+                card.dataset.from_civitai === 'true' ?
+                'Click "Fetch" to retrieve metadata' :
+                'No CivitAI information available',
+                'info'
+            );
+        }
+    });
+
+    return card;
+}
+
+// Refresh loras
+async function refreshLoras() {
+    try {
+        state.loadingManager.showSimpleLoading('Refreshing loras...');
+        await resetAndReload();
         showToast('Refresh complete', 'success');
     } catch (error) {
         console.error('Refresh failed:', error);
         showToast('Failed to refresh loras', 'error');
     } finally {
-        loadingManager.hide();
-        loadingManager.restoreProgressBar();
+        state.loadingManager.hide();
+        state.loadingManager.restoreProgressBar();
     }
 }
 
-async function fetchCivitai() {
-    const loraCards = document.querySelectorAll('.lora-card');
-    const totalCards = loraCards.length;
-
-    await loadingManager.showWithProgress(async (loading) => {
-        for (let i = 0; i < totalCards; i++) {
-            const card = loraCards[i];
-            if (card.dataset.meta?.length > 2) continue;
-            
-            const { sha256, filepath: filePath } = card.dataset;
-            if (!sha256 || !filePath) continue;
-
-            loading.setProgress((i / totalCards * 100).toFixed(1));
-            loading.setStatus(`Processing (${i+1}/${totalCards}) ${card.dataset.name}`);
-
-            try {
-                await fetch('/api/fetch-civitai', {
-                    method: 'POST',
-                    headers: { 'Content-Type': 'application/json' },
-                    body: JSON.stringify({ sha256, file_path: filePath })
-                });
-            } catch (error) {
-                console.error(`Failed to fetch ${card.dataset.name}:`, error);
-            }
-        }
-
-        localStorage.setItem('scrollPosition', window.scrollY.toString());
-        window.location.reload();
-    }, {
-        initialMessage: 'Fetching metadata...',
-        completionMessage: 'Metadata update complete'
-    });
-}
-
 // UI interaction functions
 function showLoraModal(lora) {
     const escapedWords = lora.trainedWords?.length ? 
@@ -457,32 +641,28 @@ function toggleTheme() {
 
 let pendingDeletePath = null;
 
-function toggleFolder(element) {
-    // Store the previous state
-    const wasActive = element.classList.contains('active');
+function toggleFolder(tag) {
+    // 确保 tag 是 DOM 元素
+    const tagElement = (tag instanceof HTMLElement) ? tag : this;
+    const folder = tagElement.dataset.folder;
+    const wasActive = tagElement.classList.contains('active');
     
-    // Remove active class from all tags
-    document.querySelectorAll('.tag').forEach(tag => tag.classList.remove('active'));
+    // 清除所有标签的激活状态
+    document.querySelectorAll('.folder-tags .tag').forEach(t => {
+        t.classList.remove('active');
+    });
     
     if (!wasActive) {
-        // Add active class to clicked tag
-        element.classList.add('active');
-        // Store active folder in localStorage
-        localStorage.setItem('activeFolder', element.getAttribute('data-folder'));
-        // Hide all cards first
-        document.querySelectorAll('.lora-card').forEach(card => {
-            if (card.getAttribute('data-folder') === element.getAttribute('data-folder')) {
-                card.style.display = '';
-            } else {
-                card.style.display = 'none';
-            }
-        });
+        // 激活当前标签
+        tagElement.classList.add('active');
+        state.activeFolder = folder;
     } else {
-        // Clear stored folder when deactivating
-        localStorage.removeItem('activeFolder');
-        // Show all cards
-        document.querySelectorAll('.lora-card').forEach(card => card.style.display = '');
+        // 取消激活
+        state.activeFolder = null;
     }
+    
+    // 重置并重新加载数据
+    resetAndReload();
 }
 
 async function confirmDelete() {
diff --git a/templates/loras.html b/templates/loras.html
index 516970a5..d3485a93 100644
--- a/templates/loras.html
+++ b/templates/loras.html
@@ -69,7 +69,7 @@
         <div class="controls">
             <div class="folder-tags">
                 {% for folder in folders %}
-                <div class="tag" data-folder="{{ folder }}" onclick="toggleFolder(this)">{{ folder }}</div>
+                <div class="tag" data-folder="{{ folder }}">{{ folder }}</div>
                 {% endfor %}
             </div>
             
@@ -85,55 +85,7 @@
 
         <!-- Lora卡片容器 -->
         <div class="card-grid" id="loraGrid">
-            {% for lora in loras %}
-            <div class="lora-card" 
-                 data-sha256="{{ lora.sha256 }}"
-                 data-filepath="{{ lora.file_path }}"
-                 data-name="{{ lora.model_name }}"
-                 data-file_name="{{ lora.file_name }}"
-                 data-folder="{{ lora.folder }}"
-                 data-modified="{{ lora.modified }}"
-                 data-from_civitai="{{ lora.from_civitai }}"
-                 data-meta="{{ lora.civitai | default({}) | tojson | forceescape }}">
-                <div class="card-preview">
-                    {% if lora.preview_url.endswith('.mp4') or lora.preview_url.endswith('.webm') %}
-                        <video controls autoplay muted loop>
-                            <source src="{{ lora.preview_url }}" type="video/mp4">
-                            Your browser does not support the video tag.
-                        </video>
-                    {% else %}
-                        <img src="{{ lora.preview_url if lora.preview_url else '/loras_static/images/no-preview.png' }}" alt="{{ lora.name }}">
-                    {% endif %}
-                    <div class="card-header">
-                        <span class="base-model-label" title="{{ lora.base_model }}">
-                            {{ lora.base_model }}
-                        </span>
-                        <div class="card-actions">
-                            <i class="fas fa-globe" 
-                               title="{% if lora.from_civitai %}View on Civitai{% else %}Not available from Civitai{% endif %}"
-                               {% if lora.from_civitai %}onclick="event.stopPropagation(); openCivitai('{{ lora.model_name }}')"{% endif %}
-                               {% if not lora.from_civitai %}style="opacity: 0.5; cursor: not-allowed"{% endif %}></i>
-                            <i class="fas fa-copy" 
-                               title="Copy Model Name"
-                               onclick="event.stopPropagation(); navigator.clipboard.writeText(this.closest('.lora-card').dataset.file_name)"></i>
-                            <i class="fas fa-trash" 
-                               title="Delete Model"
-                               onclick="event.stopPropagation(); deleteModel('{{ lora.file_path }}')"></i>
-                        </div>
-                    </div>
-                    <div class="card-footer">
-                        <div class="model-info">
-                            <span class="model-name">{{ lora.model_name }}</span>
-                        </div>
-                        <div class="card-actions">
-                            <i class="fas fa-image" 
-                               title="Replace Preview Image"
-                               onclick="event.stopPropagation(); replacePreview('{{ lora.file_path }}')"></i>
-                        </div>
-                    </div>
-                </div>
-            </div>
-            {% endfor %}
+            <!-- Cards will be dynamically inserted here -->
         </div>
     </div>