mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-08-14 09:43:22 -03:00
Compare commits
64 Commits
63785f82b5
..
v1.1.7
| Author | SHA1 | Date | |
|---|---|---|---|
| bf6a614e0d | |||
| feab01cd9c | |||
| 966024e534 | |||
| 2018722cc8 | |||
| 9d85c2a44a | |||
| 03dd047e62 | |||
| 86b547c1e0 | |||
| bab9752c8b | |||
| 774cc1be86 | |||
| 234b73c8a2 | |||
| abd06c48f4 | |||
| 6ca411e4e4 | |||
| 6470021e77 | |||
| 71658ab37b | |||
| 4f016a8024 | |||
| f362ed585b | |||
| 196172624f | |||
| 316702b7ab | |||
| a7625b009f | |||
| 5d4a33c90d | |||
| 041a6b8525 | |||
| 2638109ad6 | |||
| b019326747 | |||
| 54b44131b6 | |||
| a1d948025c | |||
| a90b2514ba | |||
| cb4ad27813 | |||
| 637831248b | |||
| 00228deaaa | |||
| 2373edf73c | |||
| e0e1b804a7 | |||
| fecbe8241f | |||
| 5983eaa1ce | |||
| 07fa454f72 | |||
| 4b5aa45379 | |||
| 9a0d866be4 | |||
| 308d8f71b8 | |||
| d0e8938039 | |||
| 13ed898b6b | |||
| e1dfd1c2a6 | |||
| e3e944911b | |||
| 51c0135250 | |||
| 7b19bbb14e | |||
| 5494a70f40 | |||
| 26c9ade1c9 | |||
| 87db23825f | |||
| 8fb00998a7 | |||
| dd3aa97d0a | |||
| 8bee8f4069 | |||
| 817fe21b3e | |||
| 905c37290f | |||
| f7632a47f9 | |||
| 646f1ddfb1 | |||
| 170c8068c5 | |||
| 3494037d20 | |||
| a1fd4e150b | |||
| b22f09bd1d | |||
| 4ed9169646 | |||
| f06c60bd47 | |||
| ee8250c26c | |||
| 88349bf944 | |||
| a8adcaf023 | |||
| 3c83e78d9f | |||
| d7291f73c9 |
@@ -36,3 +36,7 @@ vue-widgets/dist/
|
||||
|
||||
# Working/research notes (not committed)
|
||||
.docs/
|
||||
|
||||
# HF enrichment validation baseline snapshots (contain potentially
|
||||
# NSFW README content fetched from community model repos)
|
||||
tests/enrich_hf_validation/baselines/
|
||||
|
||||
+301
-285
File diff suppressed because it is too large
Load Diff
+34
-8
@@ -505,7 +505,9 @@
|
||||
"saveSuccess": "Zusätzliche Ordnerpfade aktualisiert. Neustart erforderlich, um Änderungen anzuwenden.",
|
||||
"saveError": "Fehler beim Aktualisieren der zusätzlichen Ordnerpfade: {message}",
|
||||
"validation": {
|
||||
"duplicatePath": "Dieser Pfad ist bereits konfiguriert"
|
||||
"duplicatePath": "Dieser Pfad ist bereits konfiguriert",
|
||||
"checkpointUnetOverlap": "Derselbe Pfad kann nicht für Checkpoints und Diffusionsmodelle verwendet werden: {paths}",
|
||||
"checkpointUnetOverlapInline": "Dieser Pfad wird bereits für einen anderen Modelltyp verwendet. Bitte verwenden Sie separate Ordner für Checkpoints und Diffusionsmodelle."
|
||||
}
|
||||
},
|
||||
"priorityTags": {
|
||||
@@ -662,7 +664,15 @@
|
||||
"title": "KI-Anbieter",
|
||||
"provider": "Anbieter",
|
||||
"providerHelp": "Wählen Sie Ihren LLM-Anbieter. OpenAI und Ollama verwenden voreingestellte API-Endpunkte. Mit \"Benutzerdefiniert\" können Sie jeden OpenAI-kompatiblen Endpunkt angeben.",
|
||||
"custom": "Benutzerdefiniert (OpenAI-kompatibel)",
|
||||
"providerOptions": {
|
||||
"openai": "OpenAI",
|
||||
"ollama": "Ollama (lokal)",
|
||||
"deepseek": "DeepSeek",
|
||||
"groq": "Groq",
|
||||
"openrouter": "OpenRouter",
|
||||
"opencode-go": "OpenCode Go",
|
||||
"custom": "Benutzerdefiniert (OpenAI-kompatibel)"
|
||||
},
|
||||
"apiBase": "API-Basis-URL",
|
||||
"apiBaseHelp": "Die Basis-URL für die LLM-API (z.B. https://api.openai.com/v1). Leer lassen, um die Anbietervoreinstellung zu verwenden.",
|
||||
"apiBasePlaceholder": "https://api.openai.com/v1",
|
||||
@@ -673,7 +683,8 @@
|
||||
"apiKeyConfigured": "Konfiguriert",
|
||||
"apiKeySet": "Einrichten",
|
||||
"model": "Modell",
|
||||
"modelHelp": "Der zu verwendende Modellname (z.B. deepseek-v4-flash, gemini-2.5-flash, gemma4:12b). Prüfen Sie Ihren Anbieter auf verfügbare Modelle."
|
||||
"modelHelp": "Der zu verwendende Modellname (z.B. deepseek-v4-flash, gemini-2.5-flash, gemma4:12b). Prüfen Sie Ihren Anbieter auf verfügbare Modelle.",
|
||||
"modelPlaceholder": "Modell auswählen..."
|
||||
}
|
||||
},
|
||||
"loras": {
|
||||
@@ -772,12 +783,14 @@
|
||||
"complete": "Automatische Organisation abgeschlossen",
|
||||
"error": "Fehler: {error}"
|
||||
},
|
||||
"enrichHfAgent": "Metadaten mit KI anreichern"
|
||||
"enrichHfAgent": "HF-Metadaten mit KI anreichern"
|
||||
},
|
||||
"contextMenu": {
|
||||
"refreshMetadata": "Civitai-Daten aktualisieren",
|
||||
"checkUpdates": "Updates prüfen",
|
||||
"relinkCivitai": "Mit Civitai neu verknüpfen",
|
||||
"linkModel": "Modell verknüpfen",
|
||||
"linkCivitai": "Mit Civitai neu verknüpfen",
|
||||
"linkHuggingFace": "Mit HuggingFace verknüpfen",
|
||||
"copySyntax": "LoRA-Syntax kopieren",
|
||||
"copyFilename": "Modell-Dateiname kopieren",
|
||||
"copyRecipeSyntax": "Rezept-Syntax kopieren",
|
||||
@@ -797,7 +810,7 @@
|
||||
"viewAllLoras": "Alle LoRAs anzeigen",
|
||||
"downloadMissingLoras": "Fehlende LoRAs herunterladen",
|
||||
"deleteRecipe": "Rezept löschen",
|
||||
"enrichHfAgent": "Metadaten mit KI anreichern"
|
||||
"enrichHfAgent": "HF-Metadaten mit KI anreichern"
|
||||
}
|
||||
},
|
||||
"recipes": {
|
||||
@@ -1194,7 +1207,9 @@
|
||||
"preparing": "Download wird vorbereitet...",
|
||||
"downloadedPreview": "Vorschaubild heruntergeladen",
|
||||
"downloadingFile": "{type}-Datei wird heruntergeladen",
|
||||
"finalizing": "Download wird abgeschlossen..."
|
||||
"finalizing": "Download wird abgeschlossen...",
|
||||
"cancelling": "Download wird abgebrochen...",
|
||||
"cancelled": "Download abgebrochen"
|
||||
},
|
||||
"progress": {
|
||||
"currentFile": "Aktuelle Datei:",
|
||||
@@ -1310,6 +1325,14 @@
|
||||
"pathPlaceholder": "Ordnerpfad eingeben oder aus Baum unten auswählen...",
|
||||
"root": "Stammverzeichnis"
|
||||
},
|
||||
"linkHuggingFace": {
|
||||
"title": "Mit HuggingFace verknüpfen",
|
||||
"infoText": "Fügen Sie die HuggingFace-Repository-URL ein, um dieses Modell zuzuordnen. Dies ermöglicht die KI-gestützte Metadatenanreicherung.",
|
||||
"urlLabel": "HuggingFace-Repository-URL:",
|
||||
"urlPlaceholder": "https://huggingface.co/user/repo",
|
||||
"helpText": "Geben Sie die vollständige URL des HuggingFace-Repositorys ein.",
|
||||
"confirmAction": "Speichern & Verknüpfen"
|
||||
},
|
||||
"relinkCivitai": {
|
||||
"title": "Mit Civitai neu verknüpfen",
|
||||
"warning": "Warnung:",
|
||||
@@ -1994,7 +2017,8 @@
|
||||
"imagesCompleted": "Beispielbilder {action} abgeschlossen",
|
||||
"imagesFailed": "Beispielbilder {action} fehlgeschlagen",
|
||||
"loadError": "Fehler beim Laden der Downloads: {message}",
|
||||
"downloadError": "Download-Fehler: {message}"
|
||||
"downloadError": "Download-Fehler: {message}",
|
||||
"downloadStopped": "Download abgebrochen"
|
||||
},
|
||||
"import": {
|
||||
"folderTreeFailed": "Fehler beim Laden des Ordnerbaums",
|
||||
@@ -2039,6 +2063,8 @@
|
||||
"contentRatingFailed": "Fehler beim Setzen der Inhaltsbewertung: {message}",
|
||||
"relinkSuccess": "Modell erfolgreich mit Civitai neu verknüpft",
|
||||
"relinkFailed": "Fehler: {message}",
|
||||
"linkHfSuccess": "Modell erfolgreich mit HuggingFace verknüpft",
|
||||
"linkHfFailed": "Fehler: {message}",
|
||||
"fetchMetadataFirst": "Bitte rufen Sie zuerst Metadaten von CivitAI ab",
|
||||
"noCivitaiInfo": "Keine CivitAI-Informationen verfügbar",
|
||||
"missingHash": "Modell-Hash nicht verfügbar"
|
||||
|
||||
+36
-10
@@ -505,7 +505,9 @@
|
||||
"saveSuccess": "Extra folder paths updated. Restart required to apply changes.",
|
||||
"saveError": "Failed to update extra folder paths: {message}",
|
||||
"validation": {
|
||||
"duplicatePath": "This path is already configured"
|
||||
"duplicatePath": "This path is already configured",
|
||||
"checkpointUnetOverlap": "Cannot use the same path for both checkpoints and diffusion models: {paths}",
|
||||
"checkpointUnetOverlapInline": "This path is also used for a different model type. Use separate folders for checkpoints and diffusion models."
|
||||
}
|
||||
},
|
||||
"priorityTags": {
|
||||
@@ -661,10 +663,18 @@
|
||||
"aiProvider": {
|
||||
"title": "AI Provider",
|
||||
"provider": "Provider",
|
||||
"providerHelp": "Choose your LLM provider. OpenAI and Ollama use preset API endpoints. Custom lets you specify any OpenAI-compatible endpoint.",
|
||||
"custom": "Custom (OpenAI-compatible)",
|
||||
"providerHelp": "Choose your LLM provider. Preset providers set the API base URL automatically. Custom lets you specify any OpenAI-compatible endpoint.",
|
||||
"providerOptions": {
|
||||
"openai": "OpenAI",
|
||||
"ollama": "Ollama (local)",
|
||||
"deepseek": "DeepSeek",
|
||||
"groq": "Groq",
|
||||
"openrouter": "OpenRouter",
|
||||
"opencode-go": "OpenCode Go",
|
||||
"custom": "Custom (OpenAI-compatible)"
|
||||
},
|
||||
"apiBase": "API Base URL",
|
||||
"apiBaseHelp": "The base URL for the LLM API (e.g. https://api.openai.com/v1). Leave empty to use the provider default.",
|
||||
"apiBaseHelp": "The base URL for the LLM API. Select a preset or enter a custom URL. The dropdown shows presets for all supported providers.",
|
||||
"apiBasePlaceholder": "https://api.openai.com/v1",
|
||||
"apiKey": "API Key",
|
||||
"apiKeyHelp": "Your LLM provider API key. Stored locally, never sent to any server except your chosen LLM provider.",
|
||||
@@ -673,7 +683,8 @@
|
||||
"apiKeyConfigured": "Configured",
|
||||
"apiKeySet": "Set up",
|
||||
"model": "Model",
|
||||
"modelHelp": "The model name to use (e.g. deepseek-v4-flash, gemini-2.5-flash, gemma4:12b). Check your provider for available models."
|
||||
"modelHelp": "The model to use. Select from the dropdown (fetched from your provider) or type a custom model name.",
|
||||
"modelPlaceholder": "Select a model..."
|
||||
}
|
||||
},
|
||||
"loras": {
|
||||
@@ -772,12 +783,14 @@
|
||||
"complete": "Auto-organize complete",
|
||||
"error": "Error: {error}"
|
||||
},
|
||||
"enrichHfAgent": "Enrich Metadata (Agent)"
|
||||
"enrichHfAgent": "Enrich HF Metadata (AI)"
|
||||
},
|
||||
"contextMenu": {
|
||||
"refreshMetadata": "Refresh Civitai Data",
|
||||
"checkUpdates": "Check Updates",
|
||||
"relinkCivitai": "Re-link to Civitai",
|
||||
"linkModel": "Link Model",
|
||||
"linkCivitai": "Link to Civitai",
|
||||
"linkHuggingFace": "Link to HuggingFace",
|
||||
"copySyntax": "Copy LoRA Syntax",
|
||||
"copyFilename": "Copy Model Filename",
|
||||
"copyRecipeSyntax": "Copy Recipe Syntax",
|
||||
@@ -797,7 +810,7 @@
|
||||
"viewAllLoras": "View All LoRAs",
|
||||
"downloadMissingLoras": "Download Missing LoRAs",
|
||||
"deleteRecipe": "Delete Recipe",
|
||||
"enrichHfAgent": "Enrich Metadata (Agent)"
|
||||
"enrichHfAgent": "Enrich HF Metadata (AI)"
|
||||
}
|
||||
},
|
||||
"recipes": {
|
||||
@@ -1194,7 +1207,9 @@
|
||||
"preparing": "Preparing download...",
|
||||
"downloadedPreview": "Downloaded preview image",
|
||||
"downloadingFile": "Downloading {type} file",
|
||||
"finalizing": "Finalizing download..."
|
||||
"finalizing": "Finalizing download...",
|
||||
"cancelling": "Cancelling download...",
|
||||
"cancelled": "Download cancelled"
|
||||
},
|
||||
"progress": {
|
||||
"currentFile": "Current file:",
|
||||
@@ -1310,6 +1325,14 @@
|
||||
"pathPlaceholder": "Type folder path or select from tree below...",
|
||||
"root": "Root"
|
||||
},
|
||||
"linkHuggingFace": {
|
||||
"title": "Link to HuggingFace",
|
||||
"infoText": "Paste the HuggingFace repository URL to associate this model with its source. This enables AI-powered metadata enrichment.",
|
||||
"urlLabel": "HuggingFace Repository URL:",
|
||||
"urlPlaceholder": "https://huggingface.co/user/repo",
|
||||
"helpText": "Enter the full URL of the HuggingFace repository.",
|
||||
"confirmAction": "Save & Link"
|
||||
},
|
||||
"relinkCivitai": {
|
||||
"title": "Re-link to Civitai",
|
||||
"warning": "Warning:",
|
||||
@@ -1994,7 +2017,8 @@
|
||||
"imagesCompleted": "Example images {action} completed",
|
||||
"imagesFailed": "Example images {action} failed",
|
||||
"loadError": "Error loading downloads: {message}",
|
||||
"downloadError": "Download error: {message}"
|
||||
"downloadError": "Download error: {message}",
|
||||
"downloadStopped": "Download cancelled"
|
||||
},
|
||||
"import": {
|
||||
"folderTreeFailed": "Failed to load folder tree",
|
||||
@@ -2039,6 +2063,8 @@
|
||||
"contentRatingFailed": "Failed to set content rating: {message}",
|
||||
"relinkSuccess": "Model successfully re-linked to Civitai",
|
||||
"relinkFailed": "Error: {message}",
|
||||
"linkHfSuccess": "Model successfully linked to HuggingFace",
|
||||
"linkHfFailed": "Error: {message}",
|
||||
"fetchMetadataFirst": "Please fetch metadata from CivitAI first",
|
||||
"noCivitaiInfo": "No CivitAI information available",
|
||||
"missingHash": "Model hash not available"
|
||||
|
||||
+34
-8
@@ -505,7 +505,9 @@
|
||||
"saveSuccess": "Rutas de carpetas adicionales actualizadas. Se requiere reinicio para aplicar los cambios.",
|
||||
"saveError": "Error al actualizar las rutas de carpetas adicionales: {message}",
|
||||
"validation": {
|
||||
"duplicatePath": "Esta ruta ya está configurada"
|
||||
"duplicatePath": "Esta ruta ya está configurada",
|
||||
"checkpointUnetOverlap": "No se puede usar la misma ruta para checkpoints y modelos de difusión: {paths}",
|
||||
"checkpointUnetOverlapInline": "Esta ruta ya se usa para otro tipo de modelo. Use carpetas separadas para checkpoints y modelos de difusión."
|
||||
}
|
||||
},
|
||||
"priorityTags": {
|
||||
@@ -662,7 +664,15 @@
|
||||
"title": "Proveedor de IA",
|
||||
"provider": "Proveedor",
|
||||
"providerHelp": "Elija su proveedor de LLM. OpenAI y Ollama usan endpoints predefinidos. Personalizado le permite especificar cualquier endpoint compatible con OpenAI.",
|
||||
"custom": "Personalizado (compatible con OpenAI)",
|
||||
"providerOptions": {
|
||||
"openai": "OpenAI",
|
||||
"ollama": "Ollama (local)",
|
||||
"deepseek": "DeepSeek",
|
||||
"groq": "Groq",
|
||||
"openrouter": "OpenRouter",
|
||||
"opencode-go": "OpenCode Go",
|
||||
"custom": "Personalizado (compatible con OpenAI)"
|
||||
},
|
||||
"apiBase": "URL base de la API",
|
||||
"apiBaseHelp": "La URL base para la API LLM (p.ej. https://api.openai.com/v1). Déjelo vacío para usar el valor predeterminado del proveedor.",
|
||||
"apiBasePlaceholder": "https://api.openai.com/v1",
|
||||
@@ -673,7 +683,8 @@
|
||||
"apiKeyConfigured": "Configurada",
|
||||
"apiKeySet": "Configurar",
|
||||
"model": "Modelo",
|
||||
"modelHelp": "El nombre del modelo a usar (p.ej. deepseek-v4-flash, gemini-2.5-flash, gemma4:12b). Consulte a su proveedor para ver los modelos disponibles."
|
||||
"modelHelp": "El nombre del modelo a usar (p.ej. deepseek-v4-flash, gemini-2.5-flash, gemma4:12b). Consulte a su proveedor para ver los modelos disponibles.",
|
||||
"modelPlaceholder": "Seleccionar un modelo..."
|
||||
}
|
||||
},
|
||||
"loras": {
|
||||
@@ -772,12 +783,14 @@
|
||||
"complete": "Auto-organización completada",
|
||||
"error": "Error: {error}"
|
||||
},
|
||||
"enrichHfAgent": "Enriquecer metadatos (IA)"
|
||||
"enrichHfAgent": "Enriquecer metadatos HF (IA)"
|
||||
},
|
||||
"contextMenu": {
|
||||
"refreshMetadata": "Actualizar datos de Civitai",
|
||||
"checkUpdates": "Comprobar actualizaciones",
|
||||
"relinkCivitai": "Re-vincular a Civitai",
|
||||
"linkModel": "Vincular modelo",
|
||||
"linkCivitai": "Re-vincular a Civitai",
|
||||
"linkHuggingFace": "Vincular a HuggingFace",
|
||||
"copySyntax": "Copiar sintaxis de LoRA",
|
||||
"copyFilename": "Copiar nombre de archivo del modelo",
|
||||
"copyRecipeSyntax": "Copiar sintaxis de receta",
|
||||
@@ -797,7 +810,7 @@
|
||||
"viewAllLoras": "Ver todos los LoRAs",
|
||||
"downloadMissingLoras": "Descargar LoRAs faltantes",
|
||||
"deleteRecipe": "Eliminar receta",
|
||||
"enrichHfAgent": "Enriquecer metadatos (IA)"
|
||||
"enrichHfAgent": "Enriquecer metadatos HF (IA)"
|
||||
}
|
||||
},
|
||||
"recipes": {
|
||||
@@ -1194,7 +1207,9 @@
|
||||
"preparing": "Preparando descarga...",
|
||||
"downloadedPreview": "Imagen de vista previa descargada",
|
||||
"downloadingFile": "Descargando archivo de {type}",
|
||||
"finalizing": "Finalizando descarga..."
|
||||
"finalizing": "Finalizando descarga...",
|
||||
"cancelling": "Cancelando descarga...",
|
||||
"cancelled": "Descarga cancelada"
|
||||
},
|
||||
"progress": {
|
||||
"currentFile": "Archivo actual:",
|
||||
@@ -1310,6 +1325,14 @@
|
||||
"pathPlaceholder": "Escribe la ruta de la carpeta o selecciona del árbol de abajo...",
|
||||
"root": "Raíz"
|
||||
},
|
||||
"linkHuggingFace": {
|
||||
"title": "Vincular a HuggingFace",
|
||||
"infoText": "Pegue la URL del repositorio de HuggingFace para asociar este modelo. Esto permite el enriquecimiento de metadatos con IA.",
|
||||
"urlLabel": "URL del repositorio de HuggingFace:",
|
||||
"urlPlaceholder": "https://huggingface.co/user/repo",
|
||||
"helpText": "Ingrese la URL completa del repositorio de HuggingFace.",
|
||||
"confirmAction": "Guardar y vincular"
|
||||
},
|
||||
"relinkCivitai": {
|
||||
"title": "Re-vincular a Civitai",
|
||||
"warning": "Advertencia:",
|
||||
@@ -1994,7 +2017,8 @@
|
||||
"imagesCompleted": "Imágenes de ejemplo {action} completadas",
|
||||
"imagesFailed": "Imágenes de ejemplo {action} fallidas",
|
||||
"loadError": "Error al cargar descargas: {message}",
|
||||
"downloadError": "Error de descarga: {message}"
|
||||
"downloadError": "Error de descarga: {message}",
|
||||
"downloadStopped": "Descarga cancelada"
|
||||
},
|
||||
"import": {
|
||||
"folderTreeFailed": "Error al cargar árbol de carpetas",
|
||||
@@ -2039,6 +2063,8 @@
|
||||
"contentRatingFailed": "Error al establecer clasificación de contenido: {message}",
|
||||
"relinkSuccess": "Modelo re-vinculado exitosamente a Civitai",
|
||||
"relinkFailed": "Error: {message}",
|
||||
"linkHfSuccess": "Modelo vinculado a HuggingFace exitosamente",
|
||||
"linkHfFailed": "Error: {message}",
|
||||
"fetchMetadataFirst": "Por favor obtén metadatos de CivitAI primero",
|
||||
"noCivitaiInfo": "No hay información de CivitAI disponible",
|
||||
"missingHash": "Hash del modelo no disponible"
|
||||
|
||||
+34
-8
@@ -505,7 +505,9 @@
|
||||
"saveSuccess": "Chemins de dossiers supplémentaires mis à jour. Redémarrage requis pour appliquer les changements.",
|
||||
"saveError": "Échec de la mise à jour des chemins de dossiers supplémentaires: {message}",
|
||||
"validation": {
|
||||
"duplicatePath": "Ce chemin est déjà configuré"
|
||||
"duplicatePath": "Ce chemin est déjà configuré",
|
||||
"checkpointUnetOverlap": "Impossible d'utiliser le même chemin pour les checkpoints et les modèles de diffusion : {paths}",
|
||||
"checkpointUnetOverlapInline": "Ce chemin est déjà utilisé pour un autre type de modèle. Utilisez des dossiers séparés pour les checkpoints et les modèles de diffusion."
|
||||
}
|
||||
},
|
||||
"priorityTags": {
|
||||
@@ -662,7 +664,15 @@
|
||||
"title": "Fournisseur d'IA",
|
||||
"provider": "Fournisseur",
|
||||
"providerHelp": "Choisissez votre fournisseur LLM. OpenAI et Ollama utilisent des endpoints prédéfinis. Personnalisé vous permet de spécifier n'importe quel endpoint compatible OpenAI.",
|
||||
"custom": "Personnalisé (compatible OpenAI)",
|
||||
"providerOptions": {
|
||||
"openai": "OpenAI",
|
||||
"ollama": "Ollama (local)",
|
||||
"deepseek": "DeepSeek",
|
||||
"groq": "Groq",
|
||||
"openrouter": "OpenRouter",
|
||||
"opencode-go": "OpenCode Go",
|
||||
"custom": "Personnalisé (compatible OpenAI)"
|
||||
},
|
||||
"apiBase": "URL de base de l'API",
|
||||
"apiBaseHelp": "L'URL de base pour l'API LLM (ex. https://api.openai.com/v1). Laissez vide pour utiliser le fournisseur par défaut.",
|
||||
"apiBasePlaceholder": "https://api.openai.com/v1",
|
||||
@@ -673,7 +683,8 @@
|
||||
"apiKeyConfigured": "Configurée",
|
||||
"apiKeySet": "Configurer",
|
||||
"model": "Modèle",
|
||||
"modelHelp": "Le nom du modèle à utiliser (ex. deepseek-v4-flash, gemini-2.5-flash, gemma4:12b). Consultez votre fournisseur pour les modèles disponibles."
|
||||
"modelHelp": "Le nom du modèle à utiliser (ex. deepseek-v4-flash, gemini-2.5-flash, gemma4:12b). Consultez votre fournisseur pour les modèles disponibles.",
|
||||
"modelPlaceholder": "Sélectionner un modèle..."
|
||||
}
|
||||
},
|
||||
"loras": {
|
||||
@@ -772,12 +783,14 @@
|
||||
"complete": "Auto-organisation terminée",
|
||||
"error": "Erreur : {error}"
|
||||
},
|
||||
"enrichHfAgent": "Enrichir les métadonnées (IA)"
|
||||
"enrichHfAgent": "Enrichir les métadonnées HF (IA)"
|
||||
},
|
||||
"contextMenu": {
|
||||
"refreshMetadata": "Actualiser les données Civitai",
|
||||
"checkUpdates": "Vérifier les mises à jour",
|
||||
"relinkCivitai": "Relier à nouveau à Civitai",
|
||||
"linkModel": "Lier le modèle",
|
||||
"linkCivitai": "Relier à nouveau à Civitai",
|
||||
"linkHuggingFace": "Lier à HuggingFace",
|
||||
"copySyntax": "Copier la syntaxe LoRA",
|
||||
"copyFilename": "Copier le nom de fichier du modèle",
|
||||
"copyRecipeSyntax": "Copier la syntaxe de la recipe",
|
||||
@@ -797,7 +810,7 @@
|
||||
"viewAllLoras": "Voir tous les LoRAs",
|
||||
"downloadMissingLoras": "Télécharger les LoRAs manquants",
|
||||
"deleteRecipe": "Supprimer la recipe",
|
||||
"enrichHfAgent": "Enrichir les métadonnées (IA)"
|
||||
"enrichHfAgent": "Enrichir les métadonnées HF (IA)"
|
||||
}
|
||||
},
|
||||
"recipes": {
|
||||
@@ -1194,7 +1207,9 @@
|
||||
"preparing": "Préparation du téléchargement...",
|
||||
"downloadedPreview": "Image d'aperçu téléchargée",
|
||||
"downloadingFile": "Téléchargement du fichier {type}",
|
||||
"finalizing": "Finalisation du téléchargement..."
|
||||
"finalizing": "Finalisation du téléchargement...",
|
||||
"cancelling": "Annulation du téléchargement...",
|
||||
"cancelled": "Téléchargement annulé"
|
||||
},
|
||||
"progress": {
|
||||
"currentFile": "Fichier actuel :",
|
||||
@@ -1310,6 +1325,14 @@
|
||||
"pathPlaceholder": "Tapez le chemin du dossier ou sélectionnez dans l'arbre ci-dessous...",
|
||||
"root": "Racine"
|
||||
},
|
||||
"linkHuggingFace": {
|
||||
"title": "Lier à HuggingFace",
|
||||
"infoText": "Collez l'URL du dépôt HuggingFace pour associer ce modèle à sa source. Cela permet l'enrichissement des métadonnées par IA.",
|
||||
"urlLabel": "URL du dépôt HuggingFace :",
|
||||
"urlPlaceholder": "https://huggingface.co/user/repo",
|
||||
"helpText": "Entrez l'URL complète du dépôt HuggingFace.",
|
||||
"confirmAction": "Enregistrer & lier"
|
||||
},
|
||||
"relinkCivitai": {
|
||||
"title": "Relier à nouveau à Civitai",
|
||||
"warning": "Attention :",
|
||||
@@ -1994,7 +2017,8 @@
|
||||
"imagesCompleted": "Images d'exemple {action} terminées",
|
||||
"imagesFailed": "Images d'exemple {action} échouées",
|
||||
"loadError": "Erreur lors du chargement des téléchargements : {message}",
|
||||
"downloadError": "Erreur de téléchargement : {message}"
|
||||
"downloadError": "Erreur de téléchargement : {message}",
|
||||
"downloadStopped": "Téléchargement annulé"
|
||||
},
|
||||
"import": {
|
||||
"folderTreeFailed": "Échec du chargement de l'arborescence des dossiers",
|
||||
@@ -2039,6 +2063,8 @@
|
||||
"contentRatingFailed": "Échec de la définition de la classification du contenu : {message}",
|
||||
"relinkSuccess": "Modèle relié à Civitai avec succès",
|
||||
"relinkFailed": "Erreur : {message}",
|
||||
"linkHfSuccess": "Modèle lié à HuggingFace avec succès",
|
||||
"linkHfFailed": "Erreur : {message}",
|
||||
"fetchMetadataFirst": "Veuillez d'abord récupérer les métadonnées depuis CivitAI",
|
||||
"noCivitaiInfo": "Aucune information CivitAI disponible",
|
||||
"missingHash": "Hash du modèle non disponible"
|
||||
|
||||
+34
-8
@@ -505,7 +505,9 @@
|
||||
"saveSuccess": "נתיבי תיקיות נוספים עודכנו. נדרשת הפעלה מחדש כדי להחיל את השינויים.",
|
||||
"saveError": "נכשל בעדכון נתיבי תיקיות נוספים: {message}",
|
||||
"validation": {
|
||||
"duplicatePath": "נתיב זה כבר מוגדר"
|
||||
"duplicatePath": "נתיב זה כבר מוגדר",
|
||||
"checkpointUnetOverlap": "לא ניתן להשתמש באותו נתיב עבור checkpoints ומודלי דיפוזיה: {paths}",
|
||||
"checkpointUnetOverlapInline": "הנתיב הזה כבר נמצא בשימוש עבור סוג מודל אחר. יש להשתמש בתיקיות נפרדות עבור checkpoints ומודלי דיפוזיה."
|
||||
}
|
||||
},
|
||||
"priorityTags": {
|
||||
@@ -662,7 +664,15 @@
|
||||
"title": "ספק AI",
|
||||
"provider": "ספק",
|
||||
"providerHelp": "בחר את ספק ה-LLM שלך. OpenAI ו-Ollama משתמשים בנקודות קצה מוגדרות מראש. מותאם אישית מאפשר לך לציין כל נקודת קצה תואמת OpenAI.",
|
||||
"custom": "מותאם אישית (תואם OpenAI)",
|
||||
"providerOptions": {
|
||||
"openai": "OpenAI",
|
||||
"ollama": "Ollama (מקומי)",
|
||||
"deepseek": "DeepSeek",
|
||||
"groq": "Groq",
|
||||
"openrouter": "OpenRouter",
|
||||
"opencode-go": "OpenCode Go",
|
||||
"custom": "מותאם אישית (תואם OpenAI)"
|
||||
},
|
||||
"apiBase": "כתובת בסיס API",
|
||||
"apiBaseHelp": "כתובת ה-URL הבסיסית ל-API של LLM (לדוגמה https://api.openai.com/v1). השאר ריק לשימוש בברירת המחדל של הספק.",
|
||||
"apiBasePlaceholder": "https://api.openai.com/v1",
|
||||
@@ -673,7 +683,8 @@
|
||||
"apiKeyConfigured": "הוגדר",
|
||||
"apiKeySet": "הגדר",
|
||||
"model": "מודל",
|
||||
"modelHelp": "שם המודל לשימוש (לדוגמה deepseek-v4-flash, gemini-2.5-flash, gemma4:12b). בדוק אצל הספק שלך אילו מודלים זמינים."
|
||||
"modelHelp": "שם המודל לשימוש (לדוגמה deepseek-v4-flash, gemini-2.5-flash, gemma4:12b). בדוק אצל הספק שלך אילו מודלים זמינים.",
|
||||
"modelPlaceholder": "בחר מודל..."
|
||||
}
|
||||
},
|
||||
"loras": {
|
||||
@@ -772,12 +783,14 @@
|
||||
"complete": "ארגון אוטומטי הושלם",
|
||||
"error": "שגיאה: {error}"
|
||||
},
|
||||
"enrichHfAgent": "העשרת מטא-דאטה (AI)"
|
||||
"enrichHfAgent": "העשרת HF מטא-דאטה (AI)"
|
||||
},
|
||||
"contextMenu": {
|
||||
"refreshMetadata": "רענן נתוני Civitai",
|
||||
"checkUpdates": "בדוק עדכונים",
|
||||
"relinkCivitai": "קשר מחדש ל-Civitai",
|
||||
"linkModel": "קישור מודל",
|
||||
"linkCivitai": "קשר מחדש ל-Civitai",
|
||||
"linkHuggingFace": "קישור ל-HuggingFace",
|
||||
"copySyntax": "העתק תחביר LoRA",
|
||||
"copyFilename": "העתק שם קובץ מודל",
|
||||
"copyRecipeSyntax": "העתק תחביר מתכון",
|
||||
@@ -797,7 +810,7 @@
|
||||
"viewAllLoras": "הצג את כל ה-LoRAs",
|
||||
"downloadMissingLoras": "הורד LoRAs חסרים",
|
||||
"deleteRecipe": "מחק מתכון",
|
||||
"enrichHfAgent": "העשרת מטא-דאטה (AI)"
|
||||
"enrichHfAgent": "העשרת HF מטא-דאטה (AI)"
|
||||
}
|
||||
},
|
||||
"recipes": {
|
||||
@@ -1194,7 +1207,9 @@
|
||||
"preparing": "מכין הורדה...",
|
||||
"downloadedPreview": "תמונת תצוגה מקדימה הורדה",
|
||||
"downloadingFile": "מוריד קובץ {type}",
|
||||
"finalizing": "מסיים הורדה..."
|
||||
"finalizing": "מסיים הורדה...",
|
||||
"cancelling": "מבטל הורדה...",
|
||||
"cancelled": "ההורדה בוטלה"
|
||||
},
|
||||
"progress": {
|
||||
"currentFile": "הקובץ הנוכחי:",
|
||||
@@ -1310,6 +1325,14 @@
|
||||
"pathPlaceholder": "הקלד נתיב תיקייה או בחר מהעץ למטה...",
|
||||
"root": "שורש"
|
||||
},
|
||||
"linkHuggingFace": {
|
||||
"title": "קישור ל-HuggingFace",
|
||||
"infoText": "הדבק את כתובת ה-URL של מאגר HuggingFace כדי לשייך מודל זה למקורו. פעולה זו מאפשרת העשרת מטא-דאטה באמצעות AI.",
|
||||
"urlLabel": "כתובת URL של מאגר HuggingFace:",
|
||||
"urlPlaceholder": "https://huggingface.co/user/repo",
|
||||
"helpText": "הזן את כתובת ה-URL המלאה של מאגר HuggingFace.",
|
||||
"confirmAction": "שמור וקשר"
|
||||
},
|
||||
"relinkCivitai": {
|
||||
"title": "קשר מחדש ל-Civitai",
|
||||
"warning": "אזהרה:",
|
||||
@@ -1994,7 +2017,8 @@
|
||||
"imagesCompleted": "{action} תמונות הדוגמה הושלם",
|
||||
"imagesFailed": "{action} תמונות הדוגמה נכשל",
|
||||
"loadError": "שגיאה בטעינת הורדות: {message}",
|
||||
"downloadError": "שגיאת הורדה: {message}"
|
||||
"downloadError": "שגיאת הורדה: {message}",
|
||||
"downloadStopped": "ההורדה בוטלה"
|
||||
},
|
||||
"import": {
|
||||
"folderTreeFailed": "טעינת עץ התיקיות נכשלה",
|
||||
@@ -2039,6 +2063,8 @@
|
||||
"contentRatingFailed": "הגדרת דירוג התוכן נכשלה: {message}",
|
||||
"relinkSuccess": "המודל קושר מחדש ל-Civitai בהצלחה",
|
||||
"relinkFailed": "שגיאה: {message}",
|
||||
"linkHfSuccess": "המודל נקשר בהצלחה ל-HuggingFace",
|
||||
"linkHfFailed": "שגיאה: {message}",
|
||||
"fetchMetadataFirst": "אנא אחזר מטא-דאטה מ-CivitAI תחילה",
|
||||
"noCivitaiInfo": "אין מידע מ-CivitAI זמין",
|
||||
"missingHash": "ה-hash של המודל אינו זמין"
|
||||
|
||||
+34
-8
@@ -505,7 +505,9 @@
|
||||
"saveSuccess": "追加フォルダーパスを更新しました。変更を適用するには再起動が必要です。",
|
||||
"saveError": "追加フォルダーパスの更新に失敗しました: {message}",
|
||||
"validation": {
|
||||
"duplicatePath": "このパスはすでに設定されています"
|
||||
"duplicatePath": "このパスはすでに設定されています",
|
||||
"checkpointUnetOverlap": "checkpoints と diffusion models に同じパスは使用できません:{paths}",
|
||||
"checkpointUnetOverlapInline": "このパスは別のモデルタイプですでに使用されています。checkpoints と diffusion models には別々のフォルダを使用してください。"
|
||||
}
|
||||
},
|
||||
"priorityTags": {
|
||||
@@ -662,7 +664,15 @@
|
||||
"title": "AIプロバイダー",
|
||||
"provider": "プロバイダー",
|
||||
"providerHelp": "LLMプロバイダーを選択してください。OpenAIとOllamaはプリセットのAPIエンドポイントを使用します。カスタムでは任意のOpenAI互換エンドポイントを指定できます。",
|
||||
"custom": "カスタム(OpenAI互換)",
|
||||
"providerOptions": {
|
||||
"openai": "OpenAI",
|
||||
"ollama": "Ollama(ローカル)",
|
||||
"deepseek": "DeepSeek",
|
||||
"groq": "Groq",
|
||||
"openrouter": "OpenRouter",
|
||||
"opencode-go": "OpenCode Go",
|
||||
"custom": "カスタム(OpenAI 互換)"
|
||||
},
|
||||
"apiBase": "APIベースURL",
|
||||
"apiBaseHelp": "LLM APIのベースURL(例:https://api.openai.com/v1)。空の場合はプロバイダーのデフォルトが使用されます。",
|
||||
"apiBasePlaceholder": "https://api.openai.com/v1",
|
||||
@@ -673,7 +683,8 @@
|
||||
"apiKeyConfigured": "設定済み",
|
||||
"apiKeySet": "設定",
|
||||
"model": "モデル",
|
||||
"modelHelp": "使用するモデル名(例:deepseek-v4-flash, gemini-2.5-flash, gemma4:12b)。プロバイダーで利用可能なモデルをご確認ください。"
|
||||
"modelHelp": "使用するモデル名(例:deepseek-v4-flash, gemini-2.5-flash, gemma4:12b)。プロバイダーで利用可能なモデルをご確認ください。",
|
||||
"modelPlaceholder": "モデルを選択..."
|
||||
}
|
||||
},
|
||||
"loras": {
|
||||
@@ -772,12 +783,14 @@
|
||||
"complete": "自動整理が完了しました",
|
||||
"error": "エラー:{error}"
|
||||
},
|
||||
"enrichHfAgent": "メタデータをAIで補完"
|
||||
"enrichHfAgent": "HF メタデータをAIで補完"
|
||||
},
|
||||
"contextMenu": {
|
||||
"refreshMetadata": "Civitaiデータを更新",
|
||||
"checkUpdates": "更新確認",
|
||||
"relinkCivitai": "Civitaiに再リンク",
|
||||
"linkModel": "モデルをリンク",
|
||||
"linkCivitai": "Civitai にリンク",
|
||||
"linkHuggingFace": "HuggingFace にリンク",
|
||||
"copySyntax": "LoRA構文をコピー",
|
||||
"copyFilename": "モデルファイル名をコピー",
|
||||
"copyRecipeSyntax": "レシピ構文をコピー",
|
||||
@@ -797,7 +810,7 @@
|
||||
"viewAllLoras": "すべてのLoRAを表示",
|
||||
"downloadMissingLoras": "不足しているLoRAをダウンロード",
|
||||
"deleteRecipe": "レシピを削除",
|
||||
"enrichHfAgent": "メタデータをAIで補完"
|
||||
"enrichHfAgent": "HF メタデータをAIで補完"
|
||||
}
|
||||
},
|
||||
"recipes": {
|
||||
@@ -1194,7 +1207,9 @@
|
||||
"preparing": "ダウンロードを準備中...",
|
||||
"downloadedPreview": "プレビュー画像をダウンロードしました",
|
||||
"downloadingFile": "{type}ファイルをダウンロード中",
|
||||
"finalizing": "ダウンロードを完了中..."
|
||||
"finalizing": "ダウンロードを完了中...",
|
||||
"cancelling": "ダウンロードをキャンセル中...",
|
||||
"cancelled": "ダウンロードをキャンセルしました"
|
||||
},
|
||||
"progress": {
|
||||
"currentFile": "現在のファイル:",
|
||||
@@ -1310,6 +1325,14 @@
|
||||
"pathPlaceholder": "フォルダパスを入力するか、下のツリーから選択...",
|
||||
"root": "ルート"
|
||||
},
|
||||
"linkHuggingFace": {
|
||||
"title": "HuggingFace にリンク",
|
||||
"infoText": "HuggingFace リポジトリの URL を貼り付けてモデルを関連付けます。AI によるメタデータ補完が有効になります。",
|
||||
"urlLabel": "HuggingFace リポジトリ URL:",
|
||||
"urlPlaceholder": "https://huggingface.co/user/repo",
|
||||
"helpText": "完全な HuggingFace リポジトリ URL を入力してください。",
|
||||
"confirmAction": "保存&リンク"
|
||||
},
|
||||
"relinkCivitai": {
|
||||
"title": "Civitaiに再リンク",
|
||||
"warning": "警告:",
|
||||
@@ -1994,7 +2017,8 @@
|
||||
"imagesCompleted": "例画像 {action} が完了しました",
|
||||
"imagesFailed": "例画像 {action} が失敗しました",
|
||||
"loadError": "ダウンロード読み込みエラー:{message}",
|
||||
"downloadError": "ダウンロードエラー:{message}"
|
||||
"downloadError": "ダウンロードエラー:{message}",
|
||||
"downloadStopped": "ダウンロードをキャンセルしました"
|
||||
},
|
||||
"import": {
|
||||
"folderTreeFailed": "フォルダツリーの読み込みに失敗しました",
|
||||
@@ -2039,6 +2063,8 @@
|
||||
"contentRatingFailed": "コンテンツレーティングの設定に失敗しました:{message}",
|
||||
"relinkSuccess": "モデルがCivitaiに正常に再リンクされました",
|
||||
"relinkFailed": "エラー:{message}",
|
||||
"linkHfSuccess": "モデルを HuggingFace にリンクしました",
|
||||
"linkHfFailed": "エラー:{message}",
|
||||
"fetchMetadataFirst": "最初にCivitAIからメタデータを取得してください",
|
||||
"noCivitaiInfo": "CivitAI情報が利用できません",
|
||||
"missingHash": "モデルハッシュが利用できません"
|
||||
|
||||
+34
-8
@@ -505,7 +505,9 @@
|
||||
"saveSuccess": "추가 폴다 경로가 업데이트되었습니다. 변경 사항을 적용하려면 재시작이 필요합니다.",
|
||||
"saveError": "추가 폴다 경로 업데이트 실패: {message}",
|
||||
"validation": {
|
||||
"duplicatePath": "이 경로는 이미 구성되어 있습니다"
|
||||
"duplicatePath": "이 경로는 이미 구성되어 있습니다",
|
||||
"checkpointUnetOverlap": "checkpoints와 diffusion models에 동일한 경로를 사용할 수 없습니다: {paths}",
|
||||
"checkpointUnetOverlapInline": "이 경로는 다른 모델 유형에 이미 사용 중입니다. checkpoints와 diffusion models에 별도의 폴더를 사용하세요."
|
||||
}
|
||||
},
|
||||
"priorityTags": {
|
||||
@@ -662,7 +664,15 @@
|
||||
"title": "AI 제공자",
|
||||
"provider": "제공자",
|
||||
"providerHelp": "LLM 제공자를 선택하세요. OpenAI와 Ollama는 사전 설정된 API 엔드포인트를 사용합니다. 사용자 정의를 선택하면 모든 OpenAI 호환 엔드포인트를 지정할 수 있습니다.",
|
||||
"custom": "사용자 정의 (OpenAI 호환)",
|
||||
"providerOptions": {
|
||||
"openai": "OpenAI",
|
||||
"ollama": "Ollama (로컬)",
|
||||
"deepseek": "DeepSeek",
|
||||
"groq": "Groq",
|
||||
"openrouter": "OpenRouter",
|
||||
"opencode-go": "OpenCode Go",
|
||||
"custom": "사용자 정의 (OpenAI 호환)"
|
||||
},
|
||||
"apiBase": "API 기본 URL",
|
||||
"apiBaseHelp": "LLM API의 기본 URL입니다 (예: https://api.openai.com/v1). 비워두면 제공자 기본값이 사용됩니다.",
|
||||
"apiBasePlaceholder": "https://api.openai.com/v1",
|
||||
@@ -673,7 +683,8 @@
|
||||
"apiKeyConfigured": "설정됨",
|
||||
"apiKeySet": "설정",
|
||||
"model": "모델",
|
||||
"modelHelp": "사용할 모델 이름 (예: deepseek-v4-flash, gemini-2.5-flash, gemma4:12b). 제공자에서 사용 가능한 모델을 확인하세요."
|
||||
"modelHelp": "사용할 모델 이름 (예: deepseek-v4-flash, gemini-2.5-flash, gemma4:12b). 제공자에서 사용 가능한 모델을 확인하세요.",
|
||||
"modelPlaceholder": "모델 선택..."
|
||||
}
|
||||
},
|
||||
"loras": {
|
||||
@@ -772,12 +783,14 @@
|
||||
"complete": "자동 정리 완료",
|
||||
"error": "오류: {error}"
|
||||
},
|
||||
"enrichHfAgent": "AI로 메타데이터 보강"
|
||||
"enrichHfAgent": "HF AI로 메타데이터 보강"
|
||||
},
|
||||
"contextMenu": {
|
||||
"refreshMetadata": "Civitai 데이터 새로고침",
|
||||
"checkUpdates": "업데이트 확인",
|
||||
"relinkCivitai": "Civitai에 다시 연결",
|
||||
"linkModel": "모델 연결",
|
||||
"linkCivitai": "Civitai에 연결",
|
||||
"linkHuggingFace": "HuggingFace에 연결",
|
||||
"copySyntax": "LoRA 문법 복사",
|
||||
"copyFilename": "모델 파일명 복사",
|
||||
"copyRecipeSyntax": "레시피 문법 복사",
|
||||
@@ -797,7 +810,7 @@
|
||||
"viewAllLoras": "모든 LoRA 보기",
|
||||
"downloadMissingLoras": "누락된 LoRA 다운로드",
|
||||
"deleteRecipe": "레시피 삭제",
|
||||
"enrichHfAgent": "AI로 메타데이터 보강"
|
||||
"enrichHfAgent": "HF AI로 메타데이터 보강"
|
||||
}
|
||||
},
|
||||
"recipes": {
|
||||
@@ -1194,7 +1207,9 @@
|
||||
"preparing": "다운로드 준비 중...",
|
||||
"downloadedPreview": "미리보기 이미지 다운로드됨",
|
||||
"downloadingFile": "{type} 파일 다운로드 중",
|
||||
"finalizing": "다운로드 완료 중..."
|
||||
"finalizing": "다운로드 완료 중...",
|
||||
"cancelling": "다운로드 취소 중...",
|
||||
"cancelled": "다운로드가 취소되었습니다"
|
||||
},
|
||||
"progress": {
|
||||
"currentFile": "현재 파일:",
|
||||
@@ -1310,6 +1325,14 @@
|
||||
"pathPlaceholder": "폴더 경로를 입력하거나 아래 트리에서 선택하세요...",
|
||||
"root": "루트"
|
||||
},
|
||||
"linkHuggingFace": {
|
||||
"title": "HuggingFace에 연결",
|
||||
"infoText": "HuggingFace 저장소 URL을 붙여넣어 모델을 연결합니다. AI 메타데이터 보강 기능을 사용할 수 있습니다.",
|
||||
"urlLabel": "HuggingFace 저장소 URL:",
|
||||
"urlPlaceholder": "https://huggingface.co/user/repo",
|
||||
"helpText": "전체 HuggingFace 저장소 URL을 입력하세요.",
|
||||
"confirmAction": "저장 및 연결"
|
||||
},
|
||||
"relinkCivitai": {
|
||||
"title": "Civitai에 다시 연결",
|
||||
"warning": "경고:",
|
||||
@@ -1994,7 +2017,8 @@
|
||||
"imagesCompleted": "예시 이미지 {action}이(가) 완료되었습니다",
|
||||
"imagesFailed": "예시 이미지 {action}이(가) 실패했습니다",
|
||||
"loadError": "다운로드 로딩 오류: {message}",
|
||||
"downloadError": "다운로드 오류: {message}"
|
||||
"downloadError": "다운로드 오류: {message}",
|
||||
"downloadStopped": "다운로드가 취소되었습니다"
|
||||
},
|
||||
"import": {
|
||||
"folderTreeFailed": "폴더 트리 로딩 실패",
|
||||
@@ -2039,6 +2063,8 @@
|
||||
"contentRatingFailed": "콘텐츠 등급 설정 실패: {message}",
|
||||
"relinkSuccess": "모델이 Civitai에 성공적으로 다시 연결되었습니다",
|
||||
"relinkFailed": "오류: {message}",
|
||||
"linkHfSuccess": "모델이 HuggingFace에 연결되었습니다",
|
||||
"linkHfFailed": "오류: {message}",
|
||||
"fetchMetadataFirst": "먼저 CivitAI에서 메타데이터를 가져와주세요",
|
||||
"noCivitaiInfo": "사용 가능한 CivitAI 정보가 없습니다",
|
||||
"missingHash": "모델 해시를 사용할 수 없습니다"
|
||||
|
||||
+34
-8
@@ -505,7 +505,9 @@
|
||||
"saveSuccess": "Дополнительные пути к папкам обновлены. Требуется перезапуск для применения изменений.",
|
||||
"saveError": "Не удалось обновить дополнительные пути к папкам: {message}",
|
||||
"validation": {
|
||||
"duplicatePath": "Этот путь уже настроен"
|
||||
"duplicatePath": "Этот путь уже настроен",
|
||||
"checkpointUnetOverlap": "Нельзя использовать один и тот же путь для checkpoints и diffusion models: {paths}",
|
||||
"checkpointUnetOverlapInline": "Этот путь уже используется для другого типа модели. Используйте отдельные папки для checkpoints и diffusion models."
|
||||
}
|
||||
},
|
||||
"priorityTags": {
|
||||
@@ -662,7 +664,15 @@
|
||||
"title": "Поставщик ИИ",
|
||||
"provider": "Поставщик",
|
||||
"providerHelp": "Выберите поставщика LLM. OpenAI и Ollama используют предустановленные API-эндпоинты. Пользовательский позволяет указать любой совместимый с OpenAI эндпоинт.",
|
||||
"custom": "Пользовательский (совместимый с OpenAI)",
|
||||
"providerOptions": {
|
||||
"openai": "OpenAI",
|
||||
"ollama": "Ollama (локальный)",
|
||||
"deepseek": "DeepSeek",
|
||||
"groq": "Groq",
|
||||
"openrouter": "OpenRouter",
|
||||
"opencode-go": "OpenCode Go",
|
||||
"custom": "Пользовательский (совместимый с OpenAI)"
|
||||
},
|
||||
"apiBase": "Базовый URL API",
|
||||
"apiBaseHelp": "Базовый URL для LLM API (например, https://api.openai.com/v1). Оставьте пустым, чтобы использовать значение по умолчанию.",
|
||||
"apiBasePlaceholder": "https://api.openai.com/v1",
|
||||
@@ -673,7 +683,8 @@
|
||||
"apiKeyConfigured": "Настроен",
|
||||
"apiKeySet": "Настроить",
|
||||
"model": "Модель",
|
||||
"modelHelp": "Имя модели для использования (например, deepseek-v4-flash, gemini-2.5-flash, gemma4:12b). Проверьте доступные модели у вашего поставщика."
|
||||
"modelHelp": "Имя модели для использования (например, deepseek-v4-flash, gemini-2.5-flash, gemma4:12b). Проверьте доступные модели у вашего поставщика.",
|
||||
"modelPlaceholder": "Выберите модель..."
|
||||
}
|
||||
},
|
||||
"loras": {
|
||||
@@ -772,12 +783,14 @@
|
||||
"complete": "Автоматическая организация завершена",
|
||||
"error": "Ошибка: {error}"
|
||||
},
|
||||
"enrichHfAgent": "Обогатить метаданные (ИИ)"
|
||||
"enrichHfAgent": "Обогатить HF метаданные (ИИ)"
|
||||
},
|
||||
"contextMenu": {
|
||||
"refreshMetadata": "Обновить данные Civitai",
|
||||
"checkUpdates": "Проверить обновления",
|
||||
"relinkCivitai": "Пересвязать с Civitai",
|
||||
"linkModel": "Связать модель",
|
||||
"linkCivitai": "Пересвязать с Civitai",
|
||||
"linkHuggingFace": "Связать с HuggingFace",
|
||||
"copySyntax": "Копировать синтаксис LoRA",
|
||||
"copyFilename": "Копировать имя файла модели",
|
||||
"copyRecipeSyntax": "Копировать синтаксис рецепта",
|
||||
@@ -797,7 +810,7 @@
|
||||
"viewAllLoras": "Посмотреть все LoRAs",
|
||||
"downloadMissingLoras": "Загрузить отсутствующие LoRAs",
|
||||
"deleteRecipe": "Удалить рецепт",
|
||||
"enrichHfAgent": "Обогатить метаданные (ИИ)"
|
||||
"enrichHfAgent": "Обогатить HF метаданные (ИИ)"
|
||||
}
|
||||
},
|
||||
"recipes": {
|
||||
@@ -1194,7 +1207,9 @@
|
||||
"preparing": "Подготовка загрузки...",
|
||||
"downloadedPreview": "Превью изображение загружено",
|
||||
"downloadingFile": "Загрузка файла {type}",
|
||||
"finalizing": "Завершение загрузки..."
|
||||
"finalizing": "Завершение загрузки...",
|
||||
"cancelling": "Отмена загрузки...",
|
||||
"cancelled": "Загрузка отменена"
|
||||
},
|
||||
"progress": {
|
||||
"currentFile": "Текущий файл:",
|
||||
@@ -1310,6 +1325,14 @@
|
||||
"pathPlaceholder": "Введите путь к папке или выберите из дерева ниже...",
|
||||
"root": "Корень"
|
||||
},
|
||||
"linkHuggingFace": {
|
||||
"title": "Связать с HuggingFace",
|
||||
"infoText": "Вставьте URL репозитория HuggingFace, чтобы связать эту модель с её источником. Это позволит обогащать метаданные с помощью ИИ.",
|
||||
"urlLabel": "URL репозитория HuggingFace:",
|
||||
"urlPlaceholder": "https://huggingface.co/user/repo",
|
||||
"helpText": "Введите полный URL репозитория HuggingFace.",
|
||||
"confirmAction": "Сохранить и связать"
|
||||
},
|
||||
"relinkCivitai": {
|
||||
"title": "Пересвязать с Civitai",
|
||||
"warning": "Предупреждение:",
|
||||
@@ -1994,7 +2017,8 @@
|
||||
"imagesCompleted": "Примеры изображений {action} завершены",
|
||||
"imagesFailed": "Примеры изображений {action} не удались",
|
||||
"loadError": "Ошибка загрузки downloads: {message}",
|
||||
"downloadError": "Ошибка загрузки: {message}"
|
||||
"downloadError": "Ошибка загрузки: {message}",
|
||||
"downloadStopped": "Загрузка отменена"
|
||||
},
|
||||
"import": {
|
||||
"folderTreeFailed": "Не удалось загрузить дерево папок",
|
||||
@@ -2039,6 +2063,8 @@
|
||||
"contentRatingFailed": "Не удалось установить рейтинг контента: {message}",
|
||||
"relinkSuccess": "Модель успешно пересвязана с Civitai",
|
||||
"relinkFailed": "Ошибка: {message}",
|
||||
"linkHfSuccess": "Модель успешно связана с HuggingFace",
|
||||
"linkHfFailed": "Ошибка: {message}",
|
||||
"fetchMetadataFirst": "Пожалуйста, сначала получите метаданные с CivitAI",
|
||||
"noCivitaiInfo": "Информация CivitAI недоступна",
|
||||
"missingHash": "Хеш модели недоступен"
|
||||
|
||||
+36
-10
@@ -505,7 +505,9 @@
|
||||
"saveSuccess": "额外文件夹路径已更新,需要重启才能生效。",
|
||||
"saveError": "更新额外文件夹路径失败:{message}",
|
||||
"validation": {
|
||||
"duplicatePath": "此路径已配置"
|
||||
"duplicatePath": "此路径已配置",
|
||||
"checkpointUnetOverlap": "checkpoints 和 diffusion models 不能使用相同的路径:{paths}",
|
||||
"checkpointUnetOverlapInline": "此路径已被用于另一种模型类型。请为 checkpoints 和 diffusion models 使用不同的文件夹。"
|
||||
}
|
||||
},
|
||||
"priorityTags": {
|
||||
@@ -662,18 +664,27 @@
|
||||
"title": "AI 提供商",
|
||||
"provider": "提供商",
|
||||
"providerHelp": "选择您的 LLM 提供商。OpenAI 和 Ollama 使用预设的 API 端点。自定义允许您指定任何兼容 OpenAI 的端点。",
|
||||
"custom": "自定义(兼容 OpenAI)",
|
||||
"providerOptions": {
|
||||
"openai": "OpenAI",
|
||||
"ollama": "Ollama(本地)",
|
||||
"deepseek": "DeepSeek",
|
||||
"groq": "Groq",
|
||||
"openrouter": "OpenRouter",
|
||||
"opencode-go": "OpenCode Go",
|
||||
"custom": "自定义(OpenAI 兼容)"
|
||||
},
|
||||
"apiBase": "API 基础地址",
|
||||
"apiBaseHelp": "LLM API 的基础 URL(例如 https://api.openai.com/v1)。留空则使用提供商默认地址。",
|
||||
"apiBaseHelp": "LLM API 的基础地址。选择预设或输入自定义地址,下拉框显示所有支持的提供商预设。",
|
||||
"apiBasePlaceholder": "https://api.openai.com/v1",
|
||||
"apiKey": "API 密钥",
|
||||
"apiKeyHelp": "您的 LLM 提供商 API 密钥。仅本地存储,不会发送到您选择的 LLM 提供商之外的任何服务器。",
|
||||
"apiKeyHelp": "LLM 提供商的 API 密钥。本地存储,除您选择的 LLM 提供商外不会发送到任何服务器。",
|
||||
"apiKeyPlaceholder": "sk-...",
|
||||
"apiKeyNotSet": "未设置",
|
||||
"apiKeyConfigured": "已配置",
|
||||
"apiKeySet": "设置",
|
||||
"model": "模型",
|
||||
"modelHelp": "要使用的模型名称(例如 deepseek-v4-flash, gemini-2.5-flash, gemma4:12b)。请查看您的提供商支持的可用模型列表。"
|
||||
"modelHelp": "要使用的模型。从下拉框选择(从提供商获取)或输入自定义模型名称。",
|
||||
"modelPlaceholder": "选择一个模型..."
|
||||
}
|
||||
},
|
||||
"loras": {
|
||||
@@ -772,12 +783,14 @@
|
||||
"complete": "自动整理已完成",
|
||||
"error": "错误:{error}"
|
||||
},
|
||||
"enrichHfAgent": "AI 元数据增强"
|
||||
"enrichHfAgent": "AI HF 元数据增强"
|
||||
},
|
||||
"contextMenu": {
|
||||
"refreshMetadata": "刷新 Civitai 数据",
|
||||
"checkUpdates": "检查更新",
|
||||
"relinkCivitai": "重新关联到 Civitai",
|
||||
"linkModel": "链接模型",
|
||||
"linkCivitai": "链接到 Civitai",
|
||||
"linkHuggingFace": "链接到 HuggingFace",
|
||||
"copySyntax": "复制 LoRA 语法",
|
||||
"copyFilename": "复制模型文件名",
|
||||
"copyRecipeSyntax": "复制配方语法",
|
||||
@@ -797,7 +810,7 @@
|
||||
"viewAllLoras": "查看所有 LoRA",
|
||||
"downloadMissingLoras": "下载缺失的 LoRA",
|
||||
"deleteRecipe": "删除配方",
|
||||
"enrichHfAgent": "AI 元数据增强"
|
||||
"enrichHfAgent": "AI HF 元数据增强"
|
||||
}
|
||||
},
|
||||
"recipes": {
|
||||
@@ -1194,7 +1207,9 @@
|
||||
"preparing": "正在准备下载...",
|
||||
"downloadedPreview": "预览图片已下载",
|
||||
"downloadingFile": "正在下载 {type} 文件",
|
||||
"finalizing": "正在完成下载..."
|
||||
"finalizing": "正在完成下载...",
|
||||
"cancelling": "取消下载中...",
|
||||
"cancelled": "下载已取消"
|
||||
},
|
||||
"progress": {
|
||||
"currentFile": "当前文件:",
|
||||
@@ -1310,6 +1325,14 @@
|
||||
"pathPlaceholder": "输入文件夹路径或从下方树中选择...",
|
||||
"root": "根目录"
|
||||
},
|
||||
"linkHuggingFace": {
|
||||
"title": "链接到 HuggingFace",
|
||||
"infoText": "粘贴 HuggingFace 仓库 URL 以关联此模型。关联后可启用 AI 元数据增强功能。",
|
||||
"urlLabel": "HuggingFace 仓库 URL:",
|
||||
"urlPlaceholder": "https://huggingface.co/user/repo",
|
||||
"helpText": "请输入完整的 HuggingFace 仓库 URL。",
|
||||
"confirmAction": "保存并链接"
|
||||
},
|
||||
"relinkCivitai": {
|
||||
"title": "重新关联到 Civitai",
|
||||
"warning": "警告:",
|
||||
@@ -1994,7 +2017,8 @@
|
||||
"imagesCompleted": "示例图片{action}完成",
|
||||
"imagesFailed": "示例图片{action}失败",
|
||||
"loadError": "加载下载项出错:{message}",
|
||||
"downloadError": "下载错误:{message}"
|
||||
"downloadError": "下载错误:{message}",
|
||||
"downloadStopped": "下载已取消"
|
||||
},
|
||||
"import": {
|
||||
"folderTreeFailed": "加载文件夹树失败",
|
||||
@@ -2039,6 +2063,8 @@
|
||||
"contentRatingFailed": "设置内容评级失败:{message}",
|
||||
"relinkSuccess": "模型已成功重新关联到 Civitai",
|
||||
"relinkFailed": "错误:{message}",
|
||||
"linkHfSuccess": "模型已成功链接到 HuggingFace",
|
||||
"linkHfFailed": "错误:{message}",
|
||||
"fetchMetadataFirst": "请先从 CivitAI 获取元数据",
|
||||
"noCivitaiInfo": "无 CivitAI 信息",
|
||||
"missingHash": "模型哈希不可用"
|
||||
|
||||
+38
-12
@@ -505,7 +505,9 @@
|
||||
"saveSuccess": "額外資料夾路徑已更新,需要重啟才能生效。",
|
||||
"saveError": "更新額外資料夾路徑失敗:{message}",
|
||||
"validation": {
|
||||
"duplicatePath": "此路徑已設定"
|
||||
"duplicatePath": "此路徑已設定",
|
||||
"checkpointUnetOverlap": "checkpoints 和 diffusion models 不能使用相同的路徑:{paths}",
|
||||
"checkpointUnetOverlapInline": "此路徑已被用於另一種模型類型。請為 checkpoints 和 diffusion models 使用不同的資料夾。"
|
||||
}
|
||||
},
|
||||
"priorityTags": {
|
||||
@@ -662,18 +664,27 @@
|
||||
"title": "AI 提供者",
|
||||
"provider": "提供者",
|
||||
"providerHelp": "選擇您的 LLM 提供者。OpenAI 和 Ollama 使用預設 API 端點。自訂允許您指定任何相容 OpenAI 的端點。",
|
||||
"custom": "自訂(相容 OpenAI)",
|
||||
"apiBase": "API 基礎位址",
|
||||
"apiBaseHelp": "LLM API 的基礎 URL(例如 https://api.openai.com/v1)。留空則使用提供者預設位址。",
|
||||
"providerOptions": {
|
||||
"openai": "OpenAI",
|
||||
"ollama": "Ollama(本地)",
|
||||
"deepseek": "DeepSeek",
|
||||
"groq": "Groq",
|
||||
"openrouter": "OpenRouter",
|
||||
"opencode-go": "OpenCode Go",
|
||||
"custom": "自訂(OpenAI 相容)"
|
||||
},
|
||||
"apiBase": "API 基礎網址",
|
||||
"apiBaseHelp": "LLM API 的基礎網址。選擇預設或輸入自訂網址,下拉選單顯示所有支援的提供者預設。",
|
||||
"apiBasePlaceholder": "https://api.openai.com/v1",
|
||||
"apiKey": "API 金鑰",
|
||||
"apiKeyHelp": "您的 LLM 提供者 API 金鑰。僅儲存在本地,除了您選擇的 LLM 提供者外,不會發送到任何伺服器。",
|
||||
"apiKeyPlaceholder": "sk-...",
|
||||
"apiKeyHelp": "LLM 提供者的 API 金鑰。儲存在本地,除您選擇的 LLM 提供者外不會傳送到任何伺服器。",
|
||||
"apiKeyPlaceholder": "[TODO: Translate] sk-...",
|
||||
"apiKeyNotSet": "未設定",
|
||||
"apiKeyConfigured": "已設定",
|
||||
"apiKeySet": "設定",
|
||||
"model": "模型",
|
||||
"modelHelp": "要使用的模型名稱(例如 deepseek-v4-flash, gemini-2.5-flash, gemma4:12b)。請查看您的提供者支援的可用模型列表。"
|
||||
"modelHelp": "要使用的模型。從下拉選單選擇(從提供者取得)或輸入自訂模型名稱。",
|
||||
"modelPlaceholder": "選擇一個模型..."
|
||||
}
|
||||
},
|
||||
"loras": {
|
||||
@@ -772,12 +783,14 @@
|
||||
"complete": "自動整理完成",
|
||||
"error": "錯誤:{error}"
|
||||
},
|
||||
"enrichHfAgent": "AI 中繼資料增強"
|
||||
"enrichHfAgent": "AI HF 中繼資料增強"
|
||||
},
|
||||
"contextMenu": {
|
||||
"refreshMetadata": "刷新 Civitai 資料",
|
||||
"checkUpdates": "檢查更新",
|
||||
"relinkCivitai": "重新連結 Civitai",
|
||||
"linkModel": "連結模型",
|
||||
"linkCivitai": "連結到 Civitai",
|
||||
"linkHuggingFace": "連結到 HuggingFace",
|
||||
"copySyntax": "複製 LoRA 語法",
|
||||
"copyFilename": "複製模型檔名",
|
||||
"copyRecipeSyntax": "複製配方語法",
|
||||
@@ -797,7 +810,7 @@
|
||||
"viewAllLoras": "檢視全部 LoRA",
|
||||
"downloadMissingLoras": "下載缺少的 LoRA",
|
||||
"deleteRecipe": "刪除配方",
|
||||
"enrichHfAgent": "AI 中繼資料增強"
|
||||
"enrichHfAgent": "AI HF 中繼資料增強"
|
||||
}
|
||||
},
|
||||
"recipes": {
|
||||
@@ -1194,7 +1207,9 @@
|
||||
"preparing": "準備下載中...",
|
||||
"downloadedPreview": "已下載預覽圖片",
|
||||
"downloadingFile": "正在下載 {type} 檔案",
|
||||
"finalizing": "完成下載中..."
|
||||
"finalizing": "完成下載中...",
|
||||
"cancelling": "取消下載中...",
|
||||
"cancelled": "下載已取消"
|
||||
},
|
||||
"progress": {
|
||||
"currentFile": "目前檔案:",
|
||||
@@ -1310,6 +1325,14 @@
|
||||
"pathPlaceholder": "輸入資料夾路徑或從下方樹狀結構選擇...",
|
||||
"root": "根目錄"
|
||||
},
|
||||
"linkHuggingFace": {
|
||||
"title": "連結到 HuggingFace",
|
||||
"infoText": "貼上 HuggingFace 倉庫 URL 以關聯此模型。關聯後可啟用 AI 中繼資料增強功能。",
|
||||
"urlLabel": "HuggingFace 倉庫 URL:",
|
||||
"urlPlaceholder": "https://huggingface.co/user/repo",
|
||||
"helpText": "請輸入完整的 HuggingFace 倉庫 URL。",
|
||||
"confirmAction": "儲存並連結"
|
||||
},
|
||||
"relinkCivitai": {
|
||||
"title": "重新連結至 Civitai",
|
||||
"warning": "警告:",
|
||||
@@ -1994,7 +2017,8 @@
|
||||
"imagesCompleted": "範例圖片{action}完成",
|
||||
"imagesFailed": "範例圖片{action}失敗",
|
||||
"loadError": "載入下載時發生錯誤:{message}",
|
||||
"downloadError": "下載錯誤:{message}"
|
||||
"downloadError": "下載錯誤:{message}",
|
||||
"downloadStopped": "下載已取消"
|
||||
},
|
||||
"import": {
|
||||
"folderTreeFailed": "載入資料夾樹狀結構失敗",
|
||||
@@ -2039,6 +2063,8 @@
|
||||
"contentRatingFailed": "設定內容分級失敗:{message}",
|
||||
"relinkSuccess": "模型已成功重新連結至 Civitai",
|
||||
"relinkFailed": "錯誤:{message}",
|
||||
"linkHfSuccess": "模型已成功連結到 HuggingFace",
|
||||
"linkHfFailed": "錯誤:{message}",
|
||||
"fetchMetadataFirst": "請先從 CivitAI 取得 metadata",
|
||||
"noCivitaiInfo": "無 CivitAI 資訊",
|
||||
"missingHash": "模型雜湊不可用"
|
||||
|
||||
+2
-3
@@ -177,8 +177,7 @@ class Config:
|
||||
|
||||
# Load extra folder paths from active library settings before symlink scan
|
||||
# so both primary and extra paths are discovered in a single pass.
|
||||
if not standalone_mode:
|
||||
self._load_extra_paths_from_settings()
|
||||
self._load_extra_paths_from_settings()
|
||||
|
||||
# Scan symbolic links during initialization
|
||||
self._initialize_symlink_mappings()
|
||||
@@ -193,7 +192,7 @@ class Config:
|
||||
Called during ``Config.__init__`` before the symlink scan so both primary and
|
||||
extra paths are discovered in a single pass. Mirrors the extra-path
|
||||
portion of ``_apply_library_paths`` without replacing the primary roots
|
||||
that were already resolved from ComfyUI's ``folder_paths``.
|
||||
that were already resolved via ``folder_paths.get_folder_paths``.
|
||||
"""
|
||||
try:
|
||||
from .services.settings_manager import get_settings_manager
|
||||
|
||||
@@ -208,6 +208,10 @@ class LoraManager:
|
||||
# Initialize WebSocket manager
|
||||
await ServiceRegistry.get_websocket_manager()
|
||||
|
||||
# Preload LLM model catalog (background task, non-blocking)
|
||||
from .services.llm_service import LLMService
|
||||
await LLMService.get_instance()
|
||||
|
||||
# Initialize scanners in background
|
||||
lora_scanner = await ServiceRegistry.get_lora_scanner()
|
||||
checkpoint_scanner = await ServiceRegistry.get_checkpoint_scanner()
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Agent CLI — thin in-process wrappers around LoRA Manager internal services.
|
||||
"""Metadata operations — thin in-process wrappers around LoRA Manager internal services.
|
||||
|
||||
All functions are simple Python async functions that delegate to the
|
||||
appropriate internal service. They use **relative imports** within the
|
||||
@@ -7,15 +7,15 @@ risk of double import or circular dependencies.
|
||||
|
||||
Usage (in-process, primary)::
|
||||
|
||||
from py.agent_cli import list_base_models, read_metadata
|
||||
from py.metadata_ops import list_base_models, read_metadata
|
||||
|
||||
models = await list_base_models()
|
||||
meta = await read_metadata("/path/to/model.safetensors")
|
||||
|
||||
Usage (subprocess, debugging / external)::
|
||||
|
||||
python -m py.agent_cli base-models list
|
||||
python -m py.agent_cli metadata read /path/to/model.safetensors
|
||||
python -m py.metadata_ops base-models list
|
||||
python -m py.metadata_ops metadata read /path/to/model.safetensors
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -32,23 +32,26 @@ logger = logging.getLogger(__name__)
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def _find_scanner_for_model(
|
||||
model_path: str,
|
||||
) -> tuple[object, object] | tuple[None, None]:
|
||||
"""Find the (scanner, cache_entry) responsible for *model_path*.
|
||||
SCANNER_TYPE_MAP: dict[str, str] = {
|
||||
"get_lora_scanner": "lora",
|
||||
"get_checkpoint_scanner": "checkpoint",
|
||||
"get_embedding_scanner": "embedding",
|
||||
}
|
||||
|
||||
Iterates all known scanner types and returns the first one whose cache
|
||||
contains the given path. Returns ``(None, None)`` when no scanner
|
||||
claims the model.
|
||||
SCANNER_GETTER_NAMES = tuple(SCANNER_TYPE_MAP.keys())
|
||||
|
||||
|
||||
async def _find_model_entry(
|
||||
model_path: str,
|
||||
) -> tuple[object, object, str | None] | tuple[None, None, None]:
|
||||
"""Iterate all scanners and return the first (scanner, entry, getter_name)
|
||||
that owns *model_path*. Returns ``(None, None, None)`` when no scanner
|
||||
claims it.
|
||||
"""
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
|
||||
normalized = os.path.normpath(model_path)
|
||||
for getter_name in (
|
||||
"get_lora_scanner",
|
||||
"get_checkpoint_scanner",
|
||||
"get_embedding_scanner",
|
||||
):
|
||||
for getter_name in SCANNER_GETTER_NAMES:
|
||||
getter = getattr(ServiceRegistry, getter_name, None)
|
||||
if getter is None:
|
||||
continue
|
||||
@@ -59,15 +62,31 @@ async def _find_scanner_for_model(
|
||||
cache = await scanner.get_cached_data()
|
||||
for entry in cache.raw_data:
|
||||
if os.path.normpath(entry.get("file_path", "")) == normalized:
|
||||
return scanner, entry
|
||||
return scanner, entry, getter_name
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Scanner %s check failed for %s: %s",
|
||||
getter_name,
|
||||
model_path,
|
||||
exc,
|
||||
getter_name, model_path, exc,
|
||||
)
|
||||
return None, None
|
||||
return None, None, None
|
||||
|
||||
|
||||
async def _find_scanner_for_model(
|
||||
model_path: str,
|
||||
) -> tuple[object, object] | tuple[None, None]:
|
||||
"""Find the (scanner, cache_entry) responsible for *model_path*."""
|
||||
scanner, entry, _ = await _find_model_entry(model_path)
|
||||
return scanner, entry
|
||||
|
||||
|
||||
async def identify_model_type(model_path: str) -> str:
|
||||
"""Determine the model type (``\"lora\"``, ``\"checkpoint\"``, or
|
||||
``\"embedding\"``) for *model_path*.
|
||||
|
||||
Falls back to ``\"lora\"`` when unknown.
|
||||
"""
|
||||
_, _, getter_name = await _find_model_entry(model_path)
|
||||
return SCANNER_TYPE_MAP[getter_name] if getter_name else "lora"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -76,38 +95,29 @@ async def _find_scanner_for_model(
|
||||
|
||||
|
||||
async def list_base_models(limit: int = 0) -> List[str]:
|
||||
"""Return deduplicated base model names from all model caches.
|
||||
"""Return all valid CivitAI base model names.
|
||||
|
||||
The result is ordered by frequency (most common first). Pass
|
||||
*limit* = 0 (default) for all models.
|
||||
Uses ``CivitaiBaseModelService.get_base_models()`` which merges a
|
||||
hardcoded list (``SUPPORTED_DOWNLOAD_SKIP_BASE_MODELS``) with remote
|
||||
models fetched from the CivitAI API. Never empty — the hardcoded
|
||||
fallback always provides a complete set.
|
||||
|
||||
The result is sorted alphabetically. Pass *limit* = 0 for all models.
|
||||
"""
|
||||
from ..services.service_registry import ServiceRegistry
|
||||
from ..services.civitai_base_model_service import (
|
||||
CivitaiBaseModelService,
|
||||
)
|
||||
|
||||
counts: Dict[str, int] = {}
|
||||
for getter_name in (
|
||||
"get_lora_scanner",
|
||||
"get_checkpoint_scanner",
|
||||
"get_embedding_scanner",
|
||||
):
|
||||
getter = getattr(ServiceRegistry, getter_name, None)
|
||||
if getter is None:
|
||||
continue
|
||||
try:
|
||||
scanner = await getter()
|
||||
if scanner is None:
|
||||
continue
|
||||
cache = await scanner.get_cached_data()
|
||||
for entry in cache.raw_data:
|
||||
bm = entry.get("base_model")
|
||||
if bm:
|
||||
counts[bm] = counts.get(bm, 0) + 1
|
||||
except Exception as exc:
|
||||
logger.debug("list_base_models scanner %s error: %s", getter_name, exc)
|
||||
|
||||
sorted_names = [name for name, _ in sorted(counts.items(), key=lambda x: -x[1])]
|
||||
try:
|
||||
service = await CivitaiBaseModelService.get_instance()
|
||||
response = await service.get_base_models()
|
||||
names: List[str] = response.get("models", [])
|
||||
except Exception as exc:
|
||||
logger.warning("list_base_models failed: %s", exc)
|
||||
names = []
|
||||
if limit > 0:
|
||||
return sorted_names[:limit]
|
||||
return sorted_names
|
||||
return names[:limit]
|
||||
return names
|
||||
|
||||
|
||||
async def read_metadata(model_path: str) -> Dict[str, Any]:
|
||||
@@ -153,17 +163,17 @@ async def download_preview(
|
||||
*,
|
||||
target_width: int = 480,
|
||||
quality: int = 85,
|
||||
) -> bool:
|
||||
) -> str | None:
|
||||
"""Download a preview image from *url*, optimise to .webp, and save it.
|
||||
|
||||
The output file is placed alongside the model file with a ``.webp``
|
||||
extension. Returns ``True`` on success.
|
||||
extension. Returns the local file path on success, ``None`` on failure.
|
||||
"""
|
||||
from ..services.downloader import get_downloader
|
||||
from ..utils.exif_utils import ExifUtils
|
||||
|
||||
if not url or not url.strip():
|
||||
return False
|
||||
return None
|
||||
|
||||
base_name = os.path.splitext(os.path.basename(model_path))[0]
|
||||
preview_dir = os.path.dirname(model_path)
|
||||
@@ -186,8 +196,7 @@ async def download_preview(
|
||||
)
|
||||
with open(output_path, "wb") as f:
|
||||
f.write(optimized_data)
|
||||
logger.info("Preview downloaded and optimised for %s", model_path)
|
||||
return True
|
||||
return output_path
|
||||
except Exception as exc:
|
||||
logger.warning("Preview optimisation failed, saving raw: %s", exc)
|
||||
# Fall through to raw save
|
||||
@@ -196,12 +205,11 @@ async def download_preview(
|
||||
try:
|
||||
ok, _ = await downloader.download_file(url, output_path, use_auth=False)
|
||||
if ok:
|
||||
logger.info("Preview downloaded (fallback) for %s", model_path)
|
||||
return True
|
||||
return output_path
|
||||
except Exception as exc:
|
||||
logger.warning("Preview fallback download failed for %s: %s", model_path, exc)
|
||||
|
||||
return False
|
||||
return None
|
||||
|
||||
|
||||
async def refresh_cache(model_path: str) -> bool:
|
||||
@@ -1,17 +1,12 @@
|
||||
"""Subprocess entry point for AgentCLI (debugging / external use).
|
||||
"""Subprocess entry point for ``metadata_ops`` (debugging / external use).
|
||||
|
||||
Usage::
|
||||
|
||||
python -m py.agent_cli base-models list [--limit N]
|
||||
python -m py.agent_cli metadata read <path>
|
||||
python -m py.agent_cli metadata update <path> --json '{...}'
|
||||
python -m py.agent_cli preview download <path> --url <url>
|
||||
python -m py.agent_cli cache refresh <path>
|
||||
|
||||
NOTE: This is an **optional** convenience wrapper. The primary consumer of
|
||||
AgentCLI is the :mod:`AgentService` (in-process). This entry point exists
|
||||
for manual debugging and future integration with subprocess-based agent
|
||||
frameworks.
|
||||
python -m py.metadata_ops base-models list [--limit N]
|
||||
python -m py.metadata_ops metadata read <path>
|
||||
python -m py.metadata_ops metadata update <path> --json '{...}'
|
||||
python -m py.metadata_ops preview download <path> --url <url>
|
||||
python -m py.metadata_ops cache refresh <path>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -41,7 +41,12 @@ async def api_json_error(
|
||||
if exc.status < 400:
|
||||
raise
|
||||
|
||||
logger.warning(
|
||||
# Preview 404 is routine (file deleted from disk) — not worth a warning.
|
||||
logger_method = logger.warning
|
||||
if request.path.startswith("/api/lm/previews") and exc.status == 404:
|
||||
logger_method = logger.debug
|
||||
|
||||
logger_method(
|
||||
"API %s %s returned HTTP %d: %s",
|
||||
request.method,
|
||||
request.path,
|
||||
|
||||
@@ -60,21 +60,21 @@ class AgentHandler:
|
||||
skill_name = request.match_info.get("skill_name", "")
|
||||
if not skill_name:
|
||||
return web.json_response(
|
||||
{"error": "Skill name is required"}, status_code=400
|
||||
{"error": "Skill name is required"}, status=400
|
||||
)
|
||||
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception:
|
||||
return web.json_response(
|
||||
{"error": "Invalid JSON body"}, status_code=400
|
||||
{"error": "Invalid JSON body"}, status=400
|
||||
)
|
||||
|
||||
model_paths = body.get("model_paths", [])
|
||||
if not model_paths or not isinstance(model_paths, list):
|
||||
return web.json_response(
|
||||
{"error": "model_paths must be a non-empty array"},
|
||||
status_code=400,
|
||||
status=400,
|
||||
)
|
||||
|
||||
service = await self._ensure_service()
|
||||
@@ -99,12 +99,11 @@ class AgentHandler:
|
||||
# Launch execution in the background
|
||||
progress_reporter = AgentProgressReporter()
|
||||
logger.info(
|
||||
"Agent skill '%s' starting for %d model(s) in background task",
|
||||
"LLM enrichment '%s' starting for %d model(s)",
|
||||
skill_name, len(model_paths),
|
||||
)
|
||||
|
||||
async def _run() -> None:
|
||||
logger.info("_run background task started for skill '%s'", skill_name)
|
||||
try:
|
||||
result = await service.execute_skill(
|
||||
skill_name=skill_name,
|
||||
@@ -112,11 +111,11 @@ class AgentHandler:
|
||||
progress_callback=progress_reporter,
|
||||
)
|
||||
logger.info(
|
||||
"Agent skill '%s' finished: success=%s, summary='%s', errors=%s",
|
||||
"LLM enrichment '%s' finished: success=%s, summary='%s', errors=%s",
|
||||
skill_name, result.success, result.summary, result.errors,
|
||||
)
|
||||
except LLMNotConfiguredError as exc:
|
||||
logger.warning("Agent skill '%s' not configured: %s", skill_name, exc)
|
||||
logger.warning("LLM enrichment '%s' not configured: %s", skill_name, exc)
|
||||
await progress_reporter.on_progress(
|
||||
{
|
||||
"type": "agent_progress",
|
||||
@@ -126,7 +125,7 @@ class AgentHandler:
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("Agent skill '%s' failed: %s", skill_name, exc, exc_info=True)
|
||||
logger.error("LLM enrichment '%s' failed: %s", skill_name, exc, exc_info=True)
|
||||
await progress_reporter.on_progress(
|
||||
{
|
||||
"type": "agent_progress",
|
||||
@@ -137,8 +136,7 @@ class AgentHandler:
|
||||
)
|
||||
|
||||
# Fire and forget — progress comes via WebSocket
|
||||
task = asyncio.create_task(_run())
|
||||
logger.info("Agent skill '%s' background task created (id=%s)", skill_name, task)
|
||||
asyncio.create_task(_run())
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
@@ -163,5 +161,5 @@ class AgentHandler:
|
||||
# TODO: implement cooperative cancellation in AgentService
|
||||
return web.json_response(
|
||||
{"status": "acknowledged", "note": "Cancellation not yet implemented"},
|
||||
status_code=200,
|
||||
status=200,
|
||||
)
|
||||
|
||||
@@ -122,8 +122,12 @@ async def _save_hf_metadata(dest_path: str, repo: str, model_root: str) -> None:
|
||||
metadata._unknown_fields["hf_url"] = hf_url
|
||||
metadata.from_civitai = False # HF models are not from CivitAI
|
||||
|
||||
metadata_dict = metadata.to_dict()
|
||||
if "trainedWords" in metadata_dict and not metadata_dict["trainedWords"]:
|
||||
del metadata_dict["trainedWords"]
|
||||
|
||||
# 3. Save metadata atomically
|
||||
await MetadataManager.save_metadata(dest_path, metadata)
|
||||
await MetadataManager.save_metadata(dest_path, metadata_dict)
|
||||
logger.info("Saved HF metadata (with hf_url) for %s", dest_path)
|
||||
|
||||
# 4. Determine relative folder path for cache
|
||||
@@ -147,9 +151,117 @@ async def _save_hf_metadata(dest_path: str, repo: str, model_root: str) -> None:
|
||||
logger.warning("Failed to save HF metadata for %s: %s", dest_path, exc)
|
||||
|
||||
|
||||
def _find_matching_root(dest_dir: str) -> str | None:
|
||||
"""Walk up *dest_dir* to find which configured scanner root it belongs to."""
|
||||
norm = os.path.normpath(dest_dir).replace(os.sep, "/")
|
||||
all_roots = []
|
||||
for root_list in (
|
||||
config.loras_roots or [],
|
||||
config.extra_loras_roots or [],
|
||||
config.checkpoints_roots or [],
|
||||
config.extra_checkpoints_roots or [],
|
||||
config.unet_roots or [],
|
||||
config.extra_unet_roots or [],
|
||||
config.embeddings_roots or [],
|
||||
config.extra_embeddings_roots or [],
|
||||
):
|
||||
all_roots.extend([os.path.normpath(p).replace(os.sep, "/") for p in root_list])
|
||||
# Find the longest matching prefix
|
||||
match: str | None = None
|
||||
for root in all_roots:
|
||||
if norm.startswith(root):
|
||||
if match is None or len(root) > len(match):
|
||||
match = root
|
||||
return match
|
||||
|
||||
|
||||
async def _add_to_scanner_cache(dest_path: str, metadata: dict[str, Any]) -> None:
|
||||
model_dir = os.path.dirname(dest_path)
|
||||
model_root = _find_matching_root(model_dir)
|
||||
if not model_root:
|
||||
raise ValueError(f"File path {dest_path} is not within any configured scanner root")
|
||||
scanner_getter_name = _infer_model_type(model_root)[1]
|
||||
scanner_getter = getattr(ServiceRegistry, scanner_getter_name, None)
|
||||
if scanner_getter is None:
|
||||
raise RuntimeError(f"Scanner getter '{scanner_getter_name}' not found in ServiceRegistry")
|
||||
scanner = await scanner_getter()
|
||||
if scanner is None:
|
||||
raise RuntimeError(f"Scanner '{scanner_getter_name}' returned None")
|
||||
await scanner.update_single_model_cache(dest_path, dest_path, metadata)
|
||||
|
||||
|
||||
class HfHandler:
|
||||
"""Handle Hugging Face model browsing and download."""
|
||||
|
||||
async def set_hf_url(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
payload: dict[str, Any] = await request.json()
|
||||
except json.JSONDecodeError:
|
||||
return web.json_response({"success": False, "error": "Invalid JSON"}, status=400)
|
||||
|
||||
file_path = (payload.get("file_path") or "").strip()
|
||||
hf_url = (payload.get("hf_url") or "").strip()
|
||||
|
||||
if not file_path or not hf_url:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "Missing required fields: 'file_path' and 'hf_url'"},
|
||||
status=400,
|
||||
)
|
||||
|
||||
m = re.match(r"^https?://huggingface\.co/([^/]+/[^/]+)/?$", hf_url)
|
||||
if not m:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "Invalid HuggingFace URL. Expected format: https://huggingface.co/user/repo",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
|
||||
if not os.path.isfile(file_path):
|
||||
return web.json_response(
|
||||
{"success": False, "error": f"File not found: {file_path}"},
|
||||
status=404,
|
||||
)
|
||||
|
||||
model_root = _find_matching_root(os.path.dirname(file_path))
|
||||
if not model_root:
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "File is not within any configured model directory. Cannot link to HuggingFace.",
|
||||
},
|
||||
status=400,
|
||||
)
|
||||
|
||||
try:
|
||||
existing = await MetadataManager.load_metadata_payload(file_path)
|
||||
if existing.get("hf_url") == hf_url:
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": "hf_url already set",
|
||||
"hf_url": hf_url,
|
||||
})
|
||||
|
||||
existing["hf_url"] = hf_url
|
||||
existing["from_civitai"] = False
|
||||
await MetadataManager.save_metadata(file_path, existing)
|
||||
|
||||
await _add_to_scanner_cache(file_path, existing)
|
||||
|
||||
logger.info("Set hf_url=%s for %s", hf_url, file_path)
|
||||
return web.json_response({
|
||||
"success": True,
|
||||
"message": f"hf_url set to {hf_url}",
|
||||
"hf_url": hf_url,
|
||||
})
|
||||
except Exception as exc:
|
||||
logger.error("Failed to set hf_url for %s: %s", file_path, exc)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(exc)},
|
||||
status=500,
|
||||
)
|
||||
|
||||
async def get_hf_repo_files(self, request: web.Request) -> web.Response:
|
||||
"""List model-weight files from a HF repo with real file sizes.
|
||||
|
||||
@@ -251,8 +363,8 @@ class HfHandler:
|
||||
if ".." in (author, repo_name) or "." in (author, repo_name):
|
||||
return web.json_response({"error": f"Invalid repo format: {repo}"}, status=400)
|
||||
|
||||
# Validate filename — must not contain path separators or ..
|
||||
if "/" in filename or "\\" in filename or ".." in filename:
|
||||
# Validate filename — must not contain path traversal
|
||||
if ".." in filename:
|
||||
return web.json_response({"error": "Invalid filename"}, status=400)
|
||||
|
||||
# Validate relative_path — must not be absolute or escape base directory
|
||||
@@ -262,35 +374,17 @@ class HfHandler:
|
||||
if ".." in relative_path.split("/") or "\\" in relative_path:
|
||||
return web.json_response({"error": "Invalid relative_path"}, status=400)
|
||||
|
||||
# Validate model_root — must not contain path traversal
|
||||
if not os.path.isabs(model_root):
|
||||
# For relative model_root, check it doesn't escape
|
||||
resolved_model_root = os.path.realpath(
|
||||
os.path.join(os.getcwd(), "models", model_root)
|
||||
)
|
||||
# Use model_root directly as the base directory — same approach as
|
||||
# CivitAI's download path (download_manager.py). No realpath, no
|
||||
# allowed-roots validation, no path-traversal check; those are
|
||||
# unnecessary when the frontend sends the path from its own dropdown
|
||||
# (populated from scanner roots). Using the "business path" directly
|
||||
# keeps dest_path consistent with scanner roots so that later folder
|
||||
# derivation (in _save_hf_metadata) works correctly.
|
||||
if os.path.isabs(model_root):
|
||||
base_dir = os.path.normpath(model_root)
|
||||
else:
|
||||
resolved_model_root = os.path.realpath(model_root)
|
||||
|
||||
# Verify model_root is within a configured scanner root
|
||||
allowed_roots = set()
|
||||
for root_list in (
|
||||
config.loras_roots or [],
|
||||
config.extra_loras_roots or [],
|
||||
config.checkpoints_roots or [],
|
||||
config.extra_checkpoints_roots or [],
|
||||
config.unet_roots or [],
|
||||
config.extra_unet_roots or [],
|
||||
config.embeddings_roots or [],
|
||||
config.extra_embeddings_roots or [],
|
||||
):
|
||||
for r in root_list:
|
||||
allowed_roots.add(os.path.realpath(r))
|
||||
|
||||
if not any(resolved_model_root == root or resolved_model_root.startswith(root + os.sep) for root in allowed_roots):
|
||||
logger.warning("Invalid model_root rejected: %s", model_root)
|
||||
return web.json_response({"error": f"Invalid model_root: {model_root}"}, status=400)
|
||||
|
||||
base_dir = resolved_model_root
|
||||
base_dir = os.path.normpath(os.path.join(os.getcwd(), "models", model_root))
|
||||
|
||||
if use_default_paths:
|
||||
target_dir = os.path.join(base_dir, "huggingface", author, repo_name)
|
||||
@@ -299,15 +393,12 @@ class HfHandler:
|
||||
else:
|
||||
target_dir = base_dir
|
||||
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
dest_path = os.path.join(target_dir, filename)
|
||||
# Strip HF repo subdirectory — "diffusion_models/xxx.safetensors"
|
||||
# is an HF repo convention, not meaningful for local storage.
|
||||
file_base = os.path.basename(filename)
|
||||
|
||||
# Resolve symlinks and check for path traversal escape
|
||||
real_dest = os.path.realpath(dest_path)
|
||||
real_base = os.path.realpath(target_dir)
|
||||
if not real_dest.startswith(real_base + os.sep):
|
||||
logger.warning("Path traversal blocked: %s -> %s", dest_path, real_dest)
|
||||
return web.json_response({"error": "Path traversal detected"}, status=400)
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
dest_path = os.path.join(target_dir, file_base)
|
||||
|
||||
# Check if already exists (simple skip)
|
||||
if os.path.exists(dest_path) and os.path.getsize(dest_path) > 0:
|
||||
|
||||
@@ -38,6 +38,12 @@ from ...services.settings_manager import get_settings_manager
|
||||
from ...services.websocket_manager import ws_manager
|
||||
from ...services.downloader import get_downloader
|
||||
from ...services.errors import ResourceNotFoundError
|
||||
from ...services.llm_service import (
|
||||
PROVIDER_PRESETS,
|
||||
fetch_ollama_models,
|
||||
get_all_provider_models,
|
||||
get_provider_model_ids,
|
||||
)
|
||||
from ...services.cache_health_monitor import CacheHealthMonitor, CacheHealthStatus
|
||||
from ...utils.models import BaseModelMetadata
|
||||
from ...utils.constants import (
|
||||
@@ -567,12 +573,18 @@ class NodeRegistry:
|
||||
tab_nodes[nd["unique_id"]] = nd
|
||||
|
||||
async with self._lock:
|
||||
prev_count = len(self._tab_nodes.get(sid, {}))
|
||||
self._tab_nodes[sid] = tab_nodes
|
||||
self._waiting_clients.discard(sid)
|
||||
if not self._waiting_clients:
|
||||
self._ready.set()
|
||||
total_tabs = len(self._tab_nodes)
|
||||
|
||||
logger.debug("Registered %s nodes from client %s", len(nodes), sid)
|
||||
if len(nodes) != prev_count or len(nodes) > 0:
|
||||
logger.debug(
|
||||
"[LM:Registry] stored %s nodes (was %s) for client %s (total tabs: %s)",
|
||||
len(nodes), prev_count, sid, total_tabs,
|
||||
)
|
||||
|
||||
def prepare_for_refresh(self, active_sids: list[str]) -> None:
|
||||
"""Set the list of client IDs we expect to hear from during the next refresh cycle."""
|
||||
@@ -595,10 +607,17 @@ class NodeRegistry:
|
||||
longer connected."""
|
||||
async with self._lock:
|
||||
# Garbage-collect stale entries (disconnected tabs)
|
||||
stale_sids = []
|
||||
if active_sids is not None:
|
||||
for sid in list(self._tab_nodes):
|
||||
if sid not in active_sids:
|
||||
stale_sids.append(sid)
|
||||
del self._tab_nodes[sid]
|
||||
if stale_sids:
|
||||
logger.debug(
|
||||
"[LM:Registry] GC pruned %s disconnected tabs: %s",
|
||||
len(stale_sids), stale_sids,
|
||||
)
|
||||
|
||||
merged: dict[str, dict] = {}
|
||||
tab_info: dict[str, dict] = {}
|
||||
@@ -1400,8 +1419,9 @@ class SettingsHandler:
|
||||
"libraries",
|
||||
"active_library",
|
||||
# Sensitive — never expose the actual value to the frontend;
|
||||
# frontend receives a boolean instead (civitai_api_key_set).
|
||||
# frontend receives a boolean instead (*_set).
|
||||
"civitai_api_key",
|
||||
"llm_api_key",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -1459,6 +1479,8 @@ class SettingsHandler:
|
||||
# Sensitive fields: only expose a boolean indicating whether set
|
||||
raw_key = self._settings.get("civitai_api_key")
|
||||
response_data["civitai_api_key_set"] = bool(raw_key)
|
||||
raw_llm_key = self._settings.get("llm_api_key")
|
||||
response_data["llm_api_key_set"] = bool(raw_llm_key)
|
||||
settings_file = getattr(self._settings, "settings_file", None)
|
||||
if settings_file:
|
||||
response_data["settings_file"] = settings_file
|
||||
@@ -1563,6 +1585,42 @@ class SettingsHandler:
|
||||
logger.error("Error updating settings: %s", exc, exc_info=True)
|
||||
return web.Response(status=500, text=str(exc))
|
||||
|
||||
async def get_llm_models(self, request: web.Request) -> web.Response:
|
||||
"""Return the model list for a provider.
|
||||
|
||||
For ``ollama`` the list is fetched live from the local Ollama API
|
||||
(only models actually pulled locally are shown). For all other
|
||||
providers the opencode model catalog is used.
|
||||
|
||||
Query parameters:
|
||||
provider (required): Internal provider id (``openai``, ``ollama``, etc.).
|
||||
|
||||
Returns:
|
||||
``{"success": true, "models": ["gpt-4o", ...]}``.
|
||||
"""
|
||||
provider_id = request.query.get("provider", "").strip()
|
||||
if not provider_id:
|
||||
return web.json_response(
|
||||
{"success": False, "error": "provider query parameter is required", "models": []},
|
||||
status=400,
|
||||
)
|
||||
|
||||
try:
|
||||
if provider_id == "ollama":
|
||||
api_base = request.query.get("api_base", "").strip() or self._settings.get("llm_api_base", "")
|
||||
if not api_base:
|
||||
api_base = "http://localhost:11434/v1"
|
||||
models = await fetch_ollama_models(api_base)
|
||||
else:
|
||||
models = await get_provider_model_ids(provider_id)
|
||||
return web.json_response({"success": True, "models": models})
|
||||
except Exception as exc:
|
||||
logger.warning("get_llm_models failed for %s: %s", provider_id, exc)
|
||||
return web.json_response(
|
||||
{"success": False, "error": str(exc), "models": []},
|
||||
status=500,
|
||||
)
|
||||
|
||||
def _validate_example_images_path(self, folder_path: str) -> str | None:
|
||||
if not os.path.exists(folder_path):
|
||||
return f"Path does not exist: {folder_path}"
|
||||
@@ -1585,6 +1643,20 @@ class SettingsHandler:
|
||||
def _is_dedicated_example_images_folder(self, folder_path: str) -> bool:
|
||||
return is_valid_example_images_root(folder_path)
|
||||
|
||||
async def get_provider_models(self, request: web.Request) -> web.Response:
|
||||
"""Return the model catalog for all preset providers.
|
||||
|
||||
This endpoint is called asynchronously by the settings UI so that
|
||||
page rendering never blocks on the remote model catalog fetch.
|
||||
"""
|
||||
catalog_provider_ids = [p for p in PROVIDER_PRESETS if p != "custom"]
|
||||
try:
|
||||
provider_models = await get_all_provider_models(catalog_provider_ids)
|
||||
return web.json_response({"success": True, "models": provider_models})
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to fetch provider models: %s", exc)
|
||||
return web.json_response({"success": False, "models": {}, "error": str(exc)})
|
||||
|
||||
|
||||
class UsageStatsHandler:
|
||||
def __init__(self, usage_stats_factory: UsageStatsFactory = UsageStats) -> None:
|
||||
@@ -3057,6 +3129,8 @@ class NodeRegistryHandler:
|
||||
self._node_registry = node_registry
|
||||
self._prompt_server = prompt_server
|
||||
self._standalone_mode = standalone_mode
|
||||
self._refresh_lock = asyncio.Lock()
|
||||
self._last_slow_path_ts: float = 0.0
|
||||
|
||||
async def register_nodes(self, request: web.Request) -> web.Response:
|
||||
try:
|
||||
@@ -3103,7 +3177,12 @@ class NodeRegistryHandler:
|
||||
)
|
||||
graph_name = node.get("graph_name")
|
||||
try:
|
||||
node["node_id"] = int(node_id)
|
||||
# Handle compound node IDs from expanded group subgraphs,
|
||||
# e.g. "252:0" → 0 (parent scope is already in graph_id)
|
||||
if isinstance(node_id, str) and ":" in node_id:
|
||||
node["node_id"] = int(node_id.rsplit(":", 1)[-1])
|
||||
else:
|
||||
node["node_id"] = int(node_id)
|
||||
except (TypeError, ValueError):
|
||||
return web.json_response(
|
||||
{
|
||||
@@ -3144,42 +3223,101 @@ class NodeRegistryHandler:
|
||||
status=503,
|
||||
)
|
||||
|
||||
# Snapshot of currently-connected ComfyUI tabs
|
||||
active_sids = list(self._prompt_server.instance.sockets.keys())
|
||||
self._node_registry.prepare_for_refresh(active_sids)
|
||||
|
||||
try:
|
||||
self._prompt_server.instance.send_sync("lora_registry_refresh", {})
|
||||
logger.debug(
|
||||
"Sent registry refresh request (expecting %s clients)", len(active_sids)
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to send registry refresh message: %s", exc)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "Communication Error",
|
||||
"message": f"Failed to communicate with ComfyUI frontend: {exc}",
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
|
||||
if not await self._node_registry.wait_for_all(timeout=2.0):
|
||||
logger.warning(
|
||||
"Registry refresh timeout after 2s (%s/%s clients responded)",
|
||||
len(active_sids) - self._node_registry.pending_client_count,
|
||||
len(active_sids),
|
||||
)
|
||||
|
||||
# Re-read current sockets after the wait: a tab may have connected
|
||||
# while we were waiting, and we don't want to garbage-collect it.
|
||||
current_sids = set(self._prompt_server.instance.sockets.keys())
|
||||
|
||||
# Fast path: if the frontend has already pushed node data (via
|
||||
# afterConfigureGraph / graphChanged hooks), return it immediately
|
||||
# without triggering a WebSocket round-trip.
|
||||
registry_info = await self._node_registry.get_merged_registry(
|
||||
active_sids=current_sids
|
||||
)
|
||||
if registry_info["tab_count"] > 0:
|
||||
logger.debug(
|
||||
"[LM:Registry] fast path: %s nodes across %s tabs %s",
|
||||
registry_info["node_count"],
|
||||
registry_info["tab_count"],
|
||||
dict(registry_info.get("tabs", {})),
|
||||
)
|
||||
return web.json_response({"success": True, "data": registry_info})
|
||||
|
||||
# Slow path: registry is empty — trigger refresh via WebSocket.
|
||||
# Serialize with an async lock so concurrent callers don't all
|
||||
# trigger separate WS refresh cycles. The second caller will
|
||||
# re-check the fast path and (usually) find populated data.
|
||||
async with self._refresh_lock:
|
||||
# Re-check after acquiring the lock — another concurrent call
|
||||
# may have populated the cache while we were waiting.
|
||||
registry_info = await self._node_registry.get_merged_registry(
|
||||
active_sids=current_sids
|
||||
)
|
||||
if registry_info["tab_count"] > 0:
|
||||
logger.debug(
|
||||
"[LM:Registry] fast path after lock wait: %s nodes across %s tabs",
|
||||
registry_info["node_count"],
|
||||
registry_info["tab_count"],
|
||||
)
|
||||
return web.json_response({"success": True, "data": registry_info})
|
||||
|
||||
# Cooldown: if the slow path ran recently (< 2 s) and
|
||||
# returned empty, skip another WS round-trip.
|
||||
elapsed = time.monotonic() - self._last_slow_path_ts
|
||||
if elapsed < 2.0:
|
||||
logger.debug(
|
||||
"[LM:Registry] slow path cooldown (%.1fs since last refresh), returning empty",
|
||||
elapsed,
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "Empty Registry",
|
||||
"message": "No workflow nodes found — ensure ComfyUI is open and the extension is loaded.",
|
||||
},
|
||||
status=408,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"[LM:Registry] slow path: cache empty, triggering WS refresh (%s connected tabs: %s)",
|
||||
len(current_sids), list(current_sids)[:5],
|
||||
)
|
||||
active_sids = list(current_sids)
|
||||
self._node_registry.prepare_for_refresh(active_sids)
|
||||
|
||||
try:
|
||||
self._prompt_server.instance.send_sync("lora_registry_refresh", {})
|
||||
logger.debug(
|
||||
"Sent registry refresh request (expecting %s clients)", len(active_sids)
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to send registry refresh message: %s", exc)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
"error": "Communication Error",
|
||||
"message": f"Failed to communicate with ComfyUI frontend: {exc}",
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
|
||||
if not await self._node_registry.wait_for_all(timeout=0.5):
|
||||
logger.warning(
|
||||
"Registry refresh timeout after 0.5s (%s/%s clients responded)",
|
||||
len(active_sids) - self._node_registry.pending_client_count,
|
||||
len(active_sids),
|
||||
)
|
||||
|
||||
# Re-read current sockets after the wait: a tab may have connected
|
||||
# while we were waiting, and we don't want to garbage-collect it.
|
||||
current_sids = set(self._prompt_server.instance.sockets.keys())
|
||||
registry_info = await self._node_registry.get_merged_registry(
|
||||
active_sids=current_sids
|
||||
)
|
||||
self._last_slow_path_ts = time.monotonic()
|
||||
|
||||
if registry_info["node_count"] == 0:
|
||||
logger.warning("No nodes registered after refresh")
|
||||
logger.debug(
|
||||
"[LM:Registry] refresh OK — %s connected tab(s) but 0 compatible nodes found",
|
||||
registry_info["tab_count"],
|
||||
)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": False,
|
||||
@@ -3354,6 +3492,8 @@ class MiscHandlerSet:
|
||||
"get_priority_tags": self.settings.get_priority_tags,
|
||||
"get_settings_libraries": self.settings.get_libraries,
|
||||
"activate_library": self.settings.activate_library,
|
||||
"get_llm_models": self.settings.get_llm_models,
|
||||
"get_provider_models": self.settings.get_provider_models,
|
||||
"update_usage_stats": self.usage_stats.update_usage_stats,
|
||||
"get_usage_stats": self.usage_stats.get_usage_stats,
|
||||
"update_lora_code": self.lora_code.update_lora_code,
|
||||
@@ -3387,6 +3527,7 @@ class MiscHandlerSet:
|
||||
# Hugging Face handlers
|
||||
"get_hf_repo_files": self.hf_handler.get_hf_repo_files,
|
||||
"download_hf_model": self.hf_handler.download_hf_model,
|
||||
"set_hf_url": self.hf_handler.set_hf_url,
|
||||
# Agent skill handlers
|
||||
"get_agent_skills": self.agent_handler.get_agent_skills,
|
||||
"execute_agent_skill": self.agent_handler.execute_agent_skill,
|
||||
|
||||
@@ -154,6 +154,14 @@ class ModelPageView:
|
||||
)
|
||||
self._template_env._i18n_filter_added = True # type: ignore[attr-defined]
|
||||
|
||||
from ...services.llm_service import PROVIDER_PRESETS
|
||||
|
||||
# Provider presets are embedded directly (local, no await needed).
|
||||
# Provider model catalogs are fetched asynchronously by the
|
||||
# frontend via GET /api/lm/llm/provider-models so page rendering
|
||||
# never blocks on the remote model catalog (which can take up to
|
||||
# 30s on cold cache).
|
||||
|
||||
template_context = {
|
||||
"is_initializing": is_initializing,
|
||||
"settings": self._settings,
|
||||
@@ -161,6 +169,8 @@ class ModelPageView:
|
||||
"folders": [],
|
||||
"t": self._server_i18n.get_translation,
|
||||
"version": self._get_app_version(),
|
||||
"provider_presets_json": json.dumps(PROVIDER_PRESETS),
|
||||
"provider_models_json": "{}",
|
||||
}
|
||||
|
||||
if not is_initializing:
|
||||
@@ -1303,9 +1313,20 @@ class ModelQueryHandler:
|
||||
}
|
||||
if include_license_flags:
|
||||
model_data = await self._service.get_model_info_by_name(model_name)
|
||||
license_flags = (model_data or {}).get("license_flags")
|
||||
if license_flags is not None:
|
||||
response_payload["license_flags"] = int(license_flags)
|
||||
# Only return license_flags when real CivitAI model license
|
||||
# data exists. This mirrors ModelModal's guard
|
||||
# (modelData?.civitai?.model) so the preview tooltip never
|
||||
# shows misleading license icons for HF or other models
|
||||
# without actual license metadata.
|
||||
civitai_data = (model_data or {}).get("civitai") or {}
|
||||
has_license_data = (
|
||||
isinstance(civitai_data, dict)
|
||||
and isinstance(civitai_data.get("model"), dict)
|
||||
)
|
||||
if has_license_data:
|
||||
license_flags = (model_data or {}).get("license_flags")
|
||||
if license_flags is not None:
|
||||
response_payload["license_flags"] = int(license_flags)
|
||||
# Include the user's license icon style preference so the
|
||||
# ComfyUI tooltip can pick the right set without a separate
|
||||
# API call.
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import mimetypes
|
||||
import urllib.parse
|
||||
@@ -53,6 +54,7 @@ class PreviewHandler:
|
||||
|
||||
if not resolved.is_file():
|
||||
logger.debug("Preview file not found at %s", str(resolved))
|
||||
asyncio.create_task(self._cleanup_stale_preview_url(normalized))
|
||||
raise web.HTTPNotFound(text="Preview file not found")
|
||||
|
||||
# aiohttp's FileResponse handles range requests, content headers, and
|
||||
@@ -69,6 +71,35 @@ class PreviewHandler:
|
||||
resp.headers["Cache-Control"] = "public, max-age=86400"
|
||||
return resp
|
||||
|
||||
async def _cleanup_stale_preview_url(self, normalized_preview_path: str) -> None:
|
||||
"""Fire-and-forget: clear stale preview_url from all model caches.
|
||||
|
||||
When a preview file is no longer on disk, remove its reference from
|
||||
every cached entry so subsequent list API responses return an empty
|
||||
``preview_url``, letting the frontend show the no-preview placeholder.
|
||||
"""
|
||||
try:
|
||||
from ...services.service_registry import ServiceRegistry
|
||||
|
||||
for service_name in ("lora_scanner", "checkpoint_scanner", "embedding_scanner"):
|
||||
scanner = ServiceRegistry.get_service_sync(service_name)
|
||||
if scanner is None or not hasattr(scanner, "_cache"):
|
||||
continue
|
||||
cache = getattr(scanner, "_cache", None)
|
||||
if cache is None or not hasattr(cache, "clear_preview_by_path"):
|
||||
continue
|
||||
cleared = await cache.clear_preview_by_path(normalized_preview_path)
|
||||
if cleared and hasattr(scanner, "_persist_current_cache"):
|
||||
await scanner._persist_current_cache()
|
||||
logger.info(
|
||||
"Cleared stale preview_url for %d %s entries (%s)",
|
||||
cleared,
|
||||
service_name,
|
||||
normalized_preview_path,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to clean up stale preview_url: %s", exc)
|
||||
|
||||
async def _stream_file(
|
||||
self, request: web.Request, path: Path
|
||||
) -> web.StreamResponse:
|
||||
|
||||
@@ -2218,6 +2218,31 @@ class RecipeManagementHandler:
|
||||
"Failed to download image for recipe: %s", exc
|
||||
)
|
||||
|
||||
# Fallback: try to locate a custom image on disk using model_hash + image id
|
||||
if image_bytes is None:
|
||||
image_id = image_data.get("id") or ""
|
||||
if image_id and model_hash:
|
||||
from ...utils.example_images_paths import get_model_folder
|
||||
model_folder = get_model_folder(model_hash)
|
||||
if model_folder and os.path.exists(model_folder):
|
||||
for fname in os.listdir(model_folder):
|
||||
if f"custom_{image_id}" in fname:
|
||||
ext = os.path.splitext(fname)[1].lower()
|
||||
if ext not in (".jpg", ".jpeg", ".png", ".webp", ".gif"):
|
||||
continue
|
||||
fpath = os.path.join(model_folder, fname)
|
||||
if os.path.isfile(fpath):
|
||||
try:
|
||||
with open(fpath, "rb") as f:
|
||||
image_bytes = f.read()
|
||||
extension = ext
|
||||
except Exception as exc:
|
||||
self._logger.warning(
|
||||
"Failed to read custom image file %s: %s",
|
||||
fpath, exc,
|
||||
)
|
||||
break
|
||||
|
||||
prompt = (
|
||||
(parsed.get("gen_params") or {}).get("prompt") or ""
|
||||
)
|
||||
|
||||
@@ -22,6 +22,8 @@ class RouteDefinition:
|
||||
MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition("GET", "/api/lm/settings", "get_settings"),
|
||||
RouteDefinition("POST", "/api/lm/settings", "update_settings"),
|
||||
RouteDefinition("GET", "/api/lm/llm/models", "get_llm_models"),
|
||||
RouteDefinition("GET", "/api/lm/llm/provider-models", "get_provider_models"),
|
||||
RouteDefinition("GET", "/api/lm/doctor/diagnostics", "get_doctor_diagnostics"),
|
||||
RouteDefinition("POST", "/api/lm/doctor/repair-cache", "repair_doctor_cache"),
|
||||
RouteDefinition("POST", "/api/lm/doctor/resolve-filename-conflicts", "resolve_doctor_filename_conflicts"),
|
||||
@@ -101,6 +103,9 @@ MISC_ROUTE_DEFINITIONS: tuple[RouteDefinition, ...] = (
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/download-hf-model", "download_hf_model"
|
||||
),
|
||||
RouteDefinition(
|
||||
"POST", "/api/lm/set-hf-url", "set_hf_url"
|
||||
),
|
||||
# Agent skill endpoints
|
||||
RouteDefinition(
|
||||
"GET", "/api/lm/agent/skills", "get_agent_skills"
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
"""Agent-powered skill system for LoRA Manager.
|
||||
"""LLM-powered metadata enrichment pipeline infrastructure.
|
||||
|
||||
This package provides the orchestration layer for LLM/agent-powered features.
|
||||
This package provides the orchestration layer for LLM-powered features.
|
||||
Skills define *what* to do (prompt template). The :class:`AgentService`
|
||||
handles *how* (LLM calls, context gathering, validation, progress).
|
||||
|
||||
NOTE: The current implementation is a code-driven pipeline, not a true
|
||||
agent loop. Future agent orchestration (LLM-driven tool selection) will
|
||||
live alongside this package with its own namespace.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
"""Agent orchestration service.
|
||||
"""Pipeline orchestration service.
|
||||
|
||||
The :class:`AgentService` coordinates skill execution:
|
||||
The :class:`AgentService` coordinates LLM-powered pipeline execution:
|
||||
|
||||
1. Look up the skill in :class:`SkillRegistry`
|
||||
2. Validate input against the skill's ``input_schema``
|
||||
3. Prepare context via :mod:`~py.agent_cli` (read metadata, list base models, fetch HF README)
|
||||
1. Look up the pipeline definition in :class:`SkillRegistry`
|
||||
2. Validate input against its ``input_schema``
|
||||
3. Prepare context via :mod:`~py.metadata_ops` (read metadata, list base models, fetch HF README)
|
||||
4. If ``llm_required``: call :class:`LLMService` with the rendered prompt
|
||||
5. Post-process via :class:`PostProcessor` (delegates I/O to :mod:`~py.agent_cli`)
|
||||
5. Post-process via :class:`PostProcessor` (delegates I/O to :mod:`~py.metadata_ops`)
|
||||
6. Broadcast progress and completion via :class:`WebSocketManager`
|
||||
|
||||
Skills define *what* to do (prompt template). The AgentService handles *how*
|
||||
(LLM calls, context gathering, validation, progress).
|
||||
Pipeline definitions (*skills*) describe *what* to do (prompt template).
|
||||
The AgentService handles *how* (LLM calls, context gathering, validation,
|
||||
progress).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -24,10 +25,17 @@ from typing import Any, Dict, List, Optional
|
||||
|
||||
import aiohttp
|
||||
|
||||
import os
|
||||
|
||||
from ...config import config
|
||||
from ..llm_service import LLMService
|
||||
from ..websocket_manager import ws_manager
|
||||
from .post_processor import PostProcessor
|
||||
from .skill_registry import SkillRegistry
|
||||
from .skills.enrich_hf_metadata.readme_processor import (
|
||||
clean_readme_for_llm,
|
||||
extract_relevant_section,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -195,11 +203,11 @@ class AgentService:
|
||||
input_data: Dict[str, Any],
|
||||
progress_callback: Optional[AgentProgressReporter] = None,
|
||||
) -> SkillResult:
|
||||
"""Execute an agent skill.
|
||||
"""Execute a pipeline (skill) on the given models.
|
||||
|
||||
Args:
|
||||
skill_name: Name of the skill to execute
|
||||
input_data: Input validated against the skill's ``input_schema``
|
||||
skill_name: Name of the pipeline to execute
|
||||
input_data: Input validated against the pipeline's ``input_schema``
|
||||
progress_callback: Optional WebSocket progress reporter
|
||||
|
||||
Returns:
|
||||
@@ -207,7 +215,6 @@ class AgentService:
|
||||
"""
|
||||
|
||||
registry = await self._ensure_registry()
|
||||
logger.info("execute_skill '%s': looking up skill", skill_name)
|
||||
skill = registry.get_skill(skill_name)
|
||||
if skill is None:
|
||||
return SkillResult(
|
||||
@@ -235,11 +242,11 @@ class AgentService:
|
||||
total = len(model_paths)
|
||||
processed = 0
|
||||
success_count = 0
|
||||
skipped_count = 0
|
||||
updated_models: List[Dict[str, Any]] = []
|
||||
errors: List[str] = []
|
||||
post_processor = PostProcessor()
|
||||
|
||||
logger.info("execute_skill '%s': starting with %d model(s)", skill_name, total)
|
||||
await self._emit_progress(
|
||||
progress_callback, skill_name, status="started",
|
||||
total=total, processed=0, success=0,
|
||||
@@ -249,52 +256,74 @@ class AgentService:
|
||||
llm_configured = llm.is_configured() if skill.llm_required else True
|
||||
|
||||
for model_path in model_paths:
|
||||
model_filename = os.path.basename(model_path)
|
||||
logger.info(
|
||||
"execute_skill '%s': processing model %d/%d: %s",
|
||||
skill_name, processed + 1, total, model_path,
|
||||
"[%s] [%d/%d] %s",
|
||||
skill_name, processed + 1, total, model_filename,
|
||||
)
|
||||
updated_data: Dict[str, Any] = {}
|
||||
skip_model = False
|
||||
try:
|
||||
from ...agent_cli import read_metadata
|
||||
from ...metadata_ops import read_metadata
|
||||
metadata = await read_metadata(model_path)
|
||||
|
||||
prompt_vars: Dict[str, Any] = {"model_path": model_path}
|
||||
if skill.llm_required and llm_configured:
|
||||
prompt_vars = await self._build_prompt_context(
|
||||
skill_name, model_path, metadata, registry, llm,
|
||||
)
|
||||
|
||||
llm_response: Optional[Dict[str, Any]] = None
|
||||
if skill.llm_required and llm_configured:
|
||||
prompt_template = registry.load_prompt(skill_name)
|
||||
rendered = _render_prompt(prompt_template, prompt_vars)
|
||||
# Fast-fail: enrich_hf_metadata requires hf_url to have HF README context
|
||||
if skill_name == "enrich_hf_metadata" and not metadata.get("hf_url", ""):
|
||||
logger.info(
|
||||
"execute_skill '%s': LLM call for %s (prompt=%d chars)",
|
||||
skill_name, model_path, len(rendered),
|
||||
"[%s] SKIP %s — no hf_url in metadata",
|
||||
skill_name, model_filename,
|
||||
)
|
||||
llm_response = await llm.chat_completion_json(
|
||||
system_prompt=prompt_vars.get(
|
||||
"system_prompt",
|
||||
"You are a helpful assistant that extracts structured metadata.",
|
||||
),
|
||||
user_prompt=rendered,
|
||||
skipped_count += 1
|
||||
skip_model = True
|
||||
|
||||
if not skip_model:
|
||||
prompt_vars: Dict[str, Any] = {"model_path": model_path}
|
||||
if skill.llm_required and llm_configured:
|
||||
prompt_vars = await self._build_prompt_context(
|
||||
skill_name, model_path, metadata, registry, llm,
|
||||
)
|
||||
|
||||
llm_response: Optional[Dict[str, Any]] = None
|
||||
if skill.llm_required and llm_configured:
|
||||
prompt_template = registry.load_prompt(skill_name)
|
||||
rendered = _render_prompt(prompt_template, prompt_vars)
|
||||
llm_response = await llm.chat_completion_json(
|
||||
system_prompt=prompt_vars.get(
|
||||
"system_prompt",
|
||||
"You are a helpful assistant that extracts structured metadata.",
|
||||
),
|
||||
user_prompt=rendered,
|
||||
)
|
||||
if llm_response:
|
||||
logger.info(
|
||||
"[%s] [%d/%d] %s → base_model=%s confidence=%s",
|
||||
skill_name, processed + 1, total, model_filename,
|
||||
(llm_response.get("base_model") or "?")[:50],
|
||||
llm_response.get("confidence", "?"),
|
||||
)
|
||||
|
||||
model_result = await post_processor.process(
|
||||
skill_name=skill_name,
|
||||
model_path=model_path,
|
||||
llm_output=llm_response or {},
|
||||
metadata=metadata,
|
||||
readme_content=prompt_vars.get("readme_content_full", ""),
|
||||
)
|
||||
|
||||
model_result = await post_processor.process(
|
||||
skill_name=skill_name,
|
||||
model_path=model_path,
|
||||
llm_output=llm_response or {},
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
if model_result.get("success", True):
|
||||
success_count += 1
|
||||
uf = model_result.get("updated_fields", [])
|
||||
if uf:
|
||||
updated_models.append({"path": model_path, "updated_fields": uf})
|
||||
else:
|
||||
errors.extend(
|
||||
model_result.get("errors", [model_result.get("error", "Unknown error")])
|
||||
)
|
||||
if model_result.get("success", True):
|
||||
success_count += 1
|
||||
uf = model_result.get("updated_fields", [])
|
||||
if uf:
|
||||
updated_models.append({"path": model_path, "updated_fields": uf})
|
||||
updated_data = model_result.get("updates", {})
|
||||
if "preview_url" in updated_data and updated_data["preview_url"]:
|
||||
updated_data["preview_url"] = config.get_preview_static_url(
|
||||
updated_data["preview_url"]
|
||||
)
|
||||
else:
|
||||
errors.extend(
|
||||
model_result.get("errors", [model_result.get("error", "Unknown error")])
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Skill %s failed for %s: %s", skill_name, model_path, exc)
|
||||
@@ -304,25 +333,41 @@ class AgentService:
|
||||
await self._emit_progress(
|
||||
progress_callback, skill_name, status="processing",
|
||||
total=total, processed=processed, success=success_count,
|
||||
skipped=skipped_count,
|
||||
current_path=model_path,
|
||||
updated_data=updated_data,
|
||||
)
|
||||
|
||||
result = SkillResult(
|
||||
success=success_count > 0,
|
||||
updated_models=updated_models,
|
||||
errors=errors,
|
||||
summary=f"Processed {processed}/{total} models, {success_count} succeeded",
|
||||
summary=f"Processed {processed}/{total} models, {success_count} succeeded, {skipped_count} skipped",
|
||||
)
|
||||
|
||||
logger.info("execute_skill '%s': done — %s", skill_name, result.summary)
|
||||
await self._emit_progress(
|
||||
progress_callback, skill_name, status="completed",
|
||||
total=total, processed=processed, success=success_count,
|
||||
skipped=skipped_count,
|
||||
updated_models=updated_models, errors=errors, summary=result.summary,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Base model grouping (keeps the prompt compact)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _format_base_models(models: List[str]) -> str:
|
||||
"""Format the base model list as a flat, one-per-line list.
|
||||
|
||||
Attempts to group by family consistently degraded LLM extraction
|
||||
accuracy — the LLM finds individual model names harder to spot
|
||||
in comma-separated groups than in a simple ``- Name`` list.
|
||||
"""
|
||||
return "\n".join(f"- {m}" for m in models)
|
||||
|
||||
async def _build_prompt_context(
|
||||
self,
|
||||
skill_name: str,
|
||||
@@ -334,20 +379,29 @@ class AgentService:
|
||||
"""Gather variables for the skill's prompt template.
|
||||
|
||||
Reads metadata, fetches the HF README (if applicable), lists available
|
||||
base models, and returns a dict that maps to ``{{variable}}``
|
||||
placeholders in ``prompt.md``.
|
||||
base models, loads user priority tags, and returns a dict that maps to
|
||||
``{{variable}}`` placeholders in ``prompt.md``.
|
||||
"""
|
||||
from ...agent_cli import list_base_models
|
||||
from ...metadata_ops import identify_model_type, list_base_models
|
||||
from ..settings_manager import SettingsManager
|
||||
|
||||
context: Dict[str, Any] = {
|
||||
"model_path": model_path,
|
||||
"model_basename": "",
|
||||
"hf_url": "",
|
||||
"repo": "",
|
||||
"readme_content": "",
|
||||
"readme_content_full": "",
|
||||
"current_metadata": {},
|
||||
"base_models": [],
|
||||
"priority_tags": "",
|
||||
}
|
||||
|
||||
# Extract model basename (filename without extension) for the LLM
|
||||
# to use when locating the matching section in collection repos.
|
||||
raw_basename = os.path.splitext(os.path.basename(model_path))[0]
|
||||
context["model_basename"] = raw_basename or ""
|
||||
|
||||
context["current_metadata"] = {
|
||||
"file_name": metadata.get("file_name", ""),
|
||||
"base_model": metadata.get("base_model", ""),
|
||||
@@ -364,12 +418,34 @@ class AgentService:
|
||||
context["repo"] = repo or ""
|
||||
if repo:
|
||||
readme = await self._fetch_readme(repo)
|
||||
context["readme_content"] = readme[:8000] if readme else "(README not available)"
|
||||
# Trim README to the section relevant to this model file
|
||||
# (collection repos often have multiple models in one README).
|
||||
if readme and raw_basename:
|
||||
trimmed = extract_relevant_section(readme, raw_basename)
|
||||
cleaned = clean_readme_for_llm(trimmed) if trimmed else ""
|
||||
else:
|
||||
cleaned = clean_readme_for_llm(readme) if readme else ""
|
||||
context["readme_content"] = cleaned if cleaned else "(README not available)"
|
||||
context["readme_content_full"] = readme or ""
|
||||
|
||||
try:
|
||||
context["base_models"] = await list_base_models()
|
||||
raw_models = await list_base_models()
|
||||
context["base_models"] = self._format_base_models(raw_models)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to list base models: %s", exc)
|
||||
context["base_models"] = "</not available>"
|
||||
|
||||
# Determine model type and load the corresponding priority_tags
|
||||
try:
|
||||
model_type = await identify_model_type(model_path)
|
||||
context["model_type"] = model_type
|
||||
settings = SettingsManager()
|
||||
priority_config = settings.get_priority_tag_config()
|
||||
context["priority_tags"] = priority_config.get(model_type, "")
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to load priority tags: %s", exc)
|
||||
context["model_type"] = "lora"
|
||||
context["priority_tags"] = ""
|
||||
|
||||
return context
|
||||
|
||||
|
||||
@@ -1,17 +1,19 @@
|
||||
"""Post-processing engine for agent skill outputs.
|
||||
"""Post-processing engine for skill pipeline outputs.
|
||||
|
||||
The :class:`PostProcessor` takes the LLM's structured JSON output and applies
|
||||
it to a model's on-disk metadata via the :mod:`~py.agent_cli` functions.
|
||||
it to a model's on-disk metadata via the :mod:`~py.metadata_ops` functions.
|
||||
|
||||
It handles all the skill-specific business logic — conditions, transformations,
|
||||
and orchestration of multiple side-effects (write metadata, download preview,
|
||||
refresh cache). All actual I/O is delegated to :mod:`~py.agent_cli`.
|
||||
refresh cache). All actual I/O is delegated to :mod:`~py.metadata_ops`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@@ -19,7 +21,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PostProcessor:
|
||||
"""Deterministic post-processor for agent skill outputs.
|
||||
"""Deterministic post-processor for skill pipeline outputs.
|
||||
|
||||
Usage (called by :class:`~py.services.agent.agent_service.AgentService`)::
|
||||
|
||||
@@ -28,7 +30,7 @@ class PostProcessor:
|
||||
skill_name="enrich_hf_metadata",
|
||||
model_path="/path/to/model.safetensors",
|
||||
llm_output={...},
|
||||
metadata={...}, # from agent_cli.read_metadata()
|
||||
metadata={...}, # from metadata_ops.read_metadata()
|
||||
)
|
||||
"""
|
||||
|
||||
@@ -39,15 +41,20 @@ class PostProcessor:
|
||||
model_path: str,
|
||||
llm_output: Dict[str, Any],
|
||||
metadata: Dict[str, Any],
|
||||
readme_content: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
"""Route *llm_output* to the correct skill post-processor.
|
||||
|
||||
*readme_content* is optional raw markdown content (e.g. HF README)
|
||||
that is converted to HTML and stored as ``modelDescription`` for
|
||||
the description tab.
|
||||
|
||||
Returns a dict with keys ``success`` (bool), ``updated_fields`` (list),
|
||||
``preview_downloaded`` (bool), and ``errors`` (list).
|
||||
"""
|
||||
if skill_name == "enrich_hf_metadata":
|
||||
return await self._process_enrich_hf_metadata(
|
||||
model_path, llm_output, metadata,
|
||||
model_path, llm_output, metadata, readme_content,
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
@@ -64,12 +71,22 @@ class PostProcessor:
|
||||
model_path: str,
|
||||
llm_output: Dict[str, Any],
|
||||
metadata: Dict[str, Any],
|
||||
readme_content: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
from ...agent_cli import (
|
||||
from ...metadata_ops import (
|
||||
apply_metadata_updates,
|
||||
download_preview,
|
||||
refresh_cache,
|
||||
)
|
||||
from .skills.enrich_hf_metadata.readme_processor import (
|
||||
convert_readme_to_html,
|
||||
extract_gallery_images,
|
||||
extract_gallery_table_images,
|
||||
extract_relevant_section,
|
||||
extract_simple_markdown_images,
|
||||
extract_html_img_tags,
|
||||
extract_repo_from_hf_url,
|
||||
)
|
||||
|
||||
updated_fields: List[str] = []
|
||||
preview_downloaded = False
|
||||
@@ -86,23 +103,89 @@ class PostProcessor:
|
||||
if new_base and self._should_overwrite(current_base, is_hf_model):
|
||||
updates["base_model"] = new_base
|
||||
|
||||
# trainedWords / trigger words
|
||||
# trigger words → civitai.trainedWords
|
||||
new_triggers = llm_output.get("trigger_words", [])
|
||||
trigger_words_empty = True
|
||||
if isinstance(new_triggers, list):
|
||||
cleaned = [t.strip() for t in new_triggers if t.strip()]
|
||||
if cleaned:
|
||||
current_triggers = metadata.get("trainedWords") or []
|
||||
if self._should_overwrite_list(current_triggers, is_hf_model):
|
||||
updates["trainedWords"] = cleaned
|
||||
cleaned = [t for t in cleaned if t.lower() not in ("none", "null", "n/a")]
|
||||
trigger_words_empty = not cleaned
|
||||
current_civitai = metadata.get("civitai") or {}
|
||||
current_triggers = current_civitai.get("trainedWords") or []
|
||||
if self._should_overwrite_list(current_triggers, is_hf_model):
|
||||
trig_civitai = dict(current_civitai)
|
||||
if "civitai" in updates and isinstance(updates["civitai"], dict):
|
||||
trig_civitai.update(updates["civitai"])
|
||||
trig_civitai["trainedWords"] = cleaned
|
||||
updates["civitai"] = trig_civitai
|
||||
|
||||
# modelDescription
|
||||
new_desc = (llm_output.get("description") or "").strip()
|
||||
if new_desc:
|
||||
current_desc = metadata.get("modelDescription", "") or ""
|
||||
if self._should_overwrite(current_desc, is_hf_model):
|
||||
updates["modelDescription"] = new_desc
|
||||
# modelDescription — from raw README content (converted to HTML)
|
||||
if readme_content and is_hf_model:
|
||||
converted = convert_readme_to_html(readme_content)
|
||||
if converted:
|
||||
updates["modelDescription"] = converted
|
||||
|
||||
# tags — merge with existing, deduplicate (case-insensitive)
|
||||
# short_description → civitai.description (for "About this version")
|
||||
short_desc = (llm_output.get("short_description") or "").strip()
|
||||
if short_desc and is_hf_model:
|
||||
current_civitai = metadata.get("civitai") or {}
|
||||
desc_civitai = dict(current_civitai)
|
||||
if "civitai" in updates and isinstance(updates["civitai"], dict):
|
||||
desc_civitai.update(updates["civitai"])
|
||||
desc_civitai["description"] = short_desc
|
||||
updates["civitai"] = desc_civitai
|
||||
|
||||
# gallery images → civitai.images (from YAML frontmatter widget entries
|
||||
# and Sample Gallery markdown tables in the README body)
|
||||
gallery_images: List[Dict[str, Any]] = []
|
||||
if readme_content and is_hf_model:
|
||||
hf_url = metadata.get("hf_url", "") or ""
|
||||
repo = extract_repo_from_hf_url(hf_url)
|
||||
if repo:
|
||||
rec_w = llm_output.get("recommended_width") or 0
|
||||
rec_h = llm_output.get("recommended_height") or 0
|
||||
|
||||
# 1. Widget images (YAML frontmatter)
|
||||
gallery = extract_gallery_images(
|
||||
readme_content, repo,
|
||||
default_width=rec_w, default_height=rec_h,
|
||||
)
|
||||
|
||||
# 2. Sample Gallery table images (markdown body), deduplicated
|
||||
existing_urls = {img["url"] for img in gallery if img.get("url")}
|
||||
table_images = extract_gallery_table_images(
|
||||
readme_content, repo,
|
||||
existing_urls=existing_urls,
|
||||
default_width=rec_w, default_height=rec_h,
|
||||
)
|
||||
existing_urls.update(img["url"] for img in table_images if img.get("url"))
|
||||
|
||||
# 3. Simple markdown images `` in the body
|
||||
simple_images = extract_simple_markdown_images(
|
||||
readme_content, repo,
|
||||
existing_urls=existing_urls,
|
||||
default_width=rec_w, default_height=rec_h,
|
||||
)
|
||||
existing_urls.update(img["url"] for img in simple_images if img.get("url"))
|
||||
|
||||
# 4. HTML `<img>` tags (used by many collection repos)
|
||||
html_images = extract_html_img_tags(
|
||||
readme_content, repo,
|
||||
existing_urls=existing_urls,
|
||||
default_width=rec_w, default_height=rec_h,
|
||||
)
|
||||
|
||||
all_images = gallery + table_images + simple_images + html_images
|
||||
if all_images:
|
||||
gallery_images = all_images
|
||||
current_civitai = metadata.get("civitai") or {}
|
||||
gallery_civitai = dict(current_civitai)
|
||||
if "civitai" in updates and isinstance(updates["civitai"], dict):
|
||||
gallery_civitai.update(updates["civitai"])
|
||||
gallery_civitai["images"] = all_images
|
||||
updates["civitai"] = gallery_civitai
|
||||
|
||||
# tags
|
||||
new_tags = llm_output.get("tags", [])
|
||||
if isinstance(new_tags, list) and new_tags:
|
||||
existing_tags = metadata.get("tags") or []
|
||||
@@ -114,16 +197,68 @@ class PostProcessor:
|
||||
updates["metadata_source"] = "agent:enrich_hf_metadata"
|
||||
updates["llm_enriched_at"] = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
# -- Persist updates ------------------------------------------------
|
||||
# Store LLM confidence in metadata so it's accessible for evaluation
|
||||
raw_confidence = (llm_output.get("confidence") or "").strip()
|
||||
if raw_confidence:
|
||||
updates["_llm_confidence"] = raw_confidence
|
||||
|
||||
# Fallback: extract instance_prompt from YAML frontmatter when the LLM
|
||||
# returned empty trigger words but the README has instance_prompt.
|
||||
if trigger_words_empty:
|
||||
instance_prompt = _extract_yaml_instance_prompt(readme_content)
|
||||
if instance_prompt:
|
||||
current_civitai = metadata.get("civitai") or {}
|
||||
trig_civitai = dict(current_civitai)
|
||||
if "civitai" in updates and isinstance(updates["civitai"], dict):
|
||||
trig_civitai.update(updates["civitai"])
|
||||
trig_civitai["trainedWords"] = [instance_prompt]
|
||||
updates["civitai"] = trig_civitai
|
||||
|
||||
preview_remote_url = (llm_output.get("preview_url") or "").strip()
|
||||
# Fallback: if the LLM couldn't find a preview image in the cleaned
|
||||
# README, find the first gallery image from the *model-specific
|
||||
# section* of the README (not the repo-wide first image, which
|
||||
# belongs to a different model in collection repos).
|
||||
if not preview_remote_url and readme_content and is_hf_model:
|
||||
model_basename = os.path.splitext(os.path.basename(model_path))[0]
|
||||
relevant_section = extract_relevant_section(
|
||||
readme_content, model_basename,
|
||||
)
|
||||
if relevant_section and relevant_section != readme_content:
|
||||
for img in gallery_images:
|
||||
img_url = img.get("url", "")
|
||||
if img_url and img_url in relevant_section:
|
||||
preview_remote_url = img_url
|
||||
break
|
||||
# Last resort: use the first gallery image from the full README.
|
||||
if not preview_remote_url and gallery_images:
|
||||
preview_remote_url = gallery_images[0].get("url", "")
|
||||
current_preview = metadata.get("preview_url") or ""
|
||||
if preview_remote_url and not (current_preview and os.path.exists(current_preview)):
|
||||
local_path = await download_preview(model_path, preview_remote_url)
|
||||
if local_path:
|
||||
preview_downloaded = True
|
||||
updates["preview_url"] = local_path
|
||||
|
||||
# notes — plain-text summary of usage info from the LLM
|
||||
new_notes = (llm_output.get("notes") or "").strip()
|
||||
if new_notes:
|
||||
updates["notes"] = new_notes
|
||||
|
||||
# usage_tips — JSON string (e.g. {"strength_min":0.85,"strength_max":1.4})
|
||||
raw_tips = (llm_output.get("usage_tips") or "").strip()
|
||||
if raw_tips and raw_tips != "{}":
|
||||
try:
|
||||
json.loads(raw_tips)
|
||||
updates["usage_tips"] = raw_tips
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
logger.warning(
|
||||
"LLM returned invalid usage_tips JSON: %s", raw_tips[:200]
|
||||
)
|
||||
|
||||
if updates:
|
||||
updated_fields = await apply_metadata_updates(model_path, updates)
|
||||
|
||||
# -- Download preview -----------------------------------------------
|
||||
preview_url = (llm_output.get("preview_url") or "").strip()
|
||||
current_preview = metadata.get("preview_url") or ""
|
||||
if preview_url and not (current_preview and os.path.exists(current_preview)):
|
||||
preview_downloaded = await download_preview(model_path, preview_url)
|
||||
|
||||
# -- Refresh scanner cache ------------------------------------------
|
||||
if updated_fields or preview_downloaded:
|
||||
await refresh_cache(model_path)
|
||||
@@ -132,6 +267,7 @@ class PostProcessor:
|
||||
"success": True,
|
||||
"updated_fields": updated_fields,
|
||||
"preview_downloaded": preview_downloaded,
|
||||
"updates": updates,
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
@@ -166,3 +302,35 @@ class PostProcessor:
|
||||
merged.append(t)
|
||||
seen.add(t)
|
||||
return merged
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Module-level helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
|
||||
def _extract_yaml_instance_prompt(readme_content: str) -> str:
|
||||
"""Extract ``instance_prompt`` from the YAML frontmatter of a HF README.
|
||||
|
||||
Returns the prompt text, or empty string if not found. Handles
|
||||
``null`` / ``~`` YAML null values by returning empty string.
|
||||
"""
|
||||
if not readme_content or not readme_content.startswith("---"):
|
||||
return ""
|
||||
|
||||
# Find end of frontmatter
|
||||
end = readme_content.find("---", 3)
|
||||
if end == -1:
|
||||
return ""
|
||||
frontmatter = readme_content[3:end]
|
||||
|
||||
for line in frontmatter.split("\n"):
|
||||
line = line.strip()
|
||||
m = re.match(r"^instance_prompt:\s*(.*)", line)
|
||||
if m:
|
||||
val = m.group(1).strip().strip('"').strip("'")
|
||||
if val.lower() in ("null", "~", "none", ""):
|
||||
return ""
|
||||
return val
|
||||
|
||||
return ""
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Discovery and loading of agent skills.
|
||||
"""Discovery and loading of prompt-based skills.
|
||||
|
||||
Skills live in ``py/services/agent/skills/<name>/`` directories. Each
|
||||
directory must contain a ``SKILL.md`` file with YAML frontmatter::
|
||||
directory must contain a ``prompt.md`` file with YAML frontmatter::
|
||||
|
||||
---
|
||||
name: my_skill
|
||||
@@ -12,6 +12,8 @@ directory must contain a ``SKILL.md`` file with YAML frontmatter::
|
||||
|
||||
Prompt template with ``{{variable}}`` placeholders.
|
||||
|
||||
Legacy ``SKILL.md`` files are also supported for backward compatibility.
|
||||
|
||||
The registry scans the skills directory on first access and caches results.
|
||||
"""
|
||||
|
||||
@@ -32,6 +34,11 @@ logger = logging.getLogger(__name__)
|
||||
# Directory where built-in skills are stored
|
||||
_SKILLS_DIR = Path(__file__).parent / "skills"
|
||||
|
||||
#: Preferred file names for prompt definition files (tried in order).
|
||||
#: ``prompt.md`` is the current convention; ``SKILL.md`` is the legacy name
|
||||
#: kept for backward compatibility.
|
||||
_PROMPT_FILE_NAMES: tuple[str, ...] = ("prompt.md", "SKILL.md")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Frontmatter parser
|
||||
@@ -43,7 +50,8 @@ _FRONTMATTER_RE = re.compile(
|
||||
|
||||
|
||||
def _parse_skill_file(path: Path) -> tuple[dict, str]:
|
||||
"""Read a ``SKILL.md`` file and return (frontmatter_dict, body_text).
|
||||
"""Read a prompt definition file (``prompt.md`` or legacy ``SKILL.md``) and
|
||||
return (frontmatter_dict, body_text).
|
||||
|
||||
Raises ``ValueError`` if the file lacks valid YAML frontmatter.
|
||||
"""
|
||||
@@ -95,6 +103,20 @@ class SkillRegistry:
|
||||
# Discovery
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _find_prompt_file(skill_dir: Path) -> Path | None:
|
||||
"""Return the first prompt definition file that exists in *skill_dir*.
|
||||
|
||||
Tries ``_PROMPT_FILE_NAMES`` in order so that new conventions
|
||||
(``prompt.md``) take precedence while legacy ``SKILL.md`` files
|
||||
still load without changes.
|
||||
"""
|
||||
for name in _PROMPT_FILE_NAMES:
|
||||
candidate = skill_dir / name
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
return None
|
||||
|
||||
def _discover(self) -> None:
|
||||
"""Scan the skills directory and load all valid skill definitions."""
|
||||
|
||||
@@ -107,31 +129,32 @@ class SkillRegistry:
|
||||
for entry in sorted(self._skills_dir.iterdir()):
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
skill_md = entry / "SKILL.md"
|
||||
if not skill_md.exists():
|
||||
prompt_file = self._find_prompt_file(entry)
|
||||
if prompt_file is None:
|
||||
continue
|
||||
try:
|
||||
definition = self._load_skill_definition(skill_md)
|
||||
definition = self._load_skill_definition(prompt_file)
|
||||
if definition is not None:
|
||||
self._skills[definition.name] = definition
|
||||
logger.debug("Loaded skill: %s", definition.name)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to load skill from %s: %s", skill_md, exc)
|
||||
logger.warning("Failed to load skill from %s: %s", prompt_file, exc)
|
||||
|
||||
self._loaded = True
|
||||
logger.info("Discovered %d agent skills", len(self._skills))
|
||||
logger.info("Discovered %d prompt-based skills", len(self._skills))
|
||||
|
||||
def _load_skill_definition(self, path: Path) -> Optional[SkillDefinition]:
|
||||
"""Parse a ``SKILL.md`` frontmatter into a :class:`SkillDefinition`."""
|
||||
"""Parse a prompt definition file's frontmatter into a
|
||||
:class:`SkillDefinition`."""
|
||||
|
||||
try:
|
||||
data, _body = _parse_skill_file(path)
|
||||
except (ValueError, yaml.YAMLError) as exc:
|
||||
logger.warning("Failed to parse SKILL.md %s: %s", path, exc)
|
||||
logger.warning("Failed to parse prompt file %s: %s", path, exc)
|
||||
return None
|
||||
|
||||
if "name" not in data:
|
||||
logger.warning("SKILL.md missing required 'name' field: %s", path)
|
||||
logger.warning("Prompt file %s missing required 'name' field", path)
|
||||
return None
|
||||
|
||||
perm_data = data.get("permissions", {})
|
||||
@@ -171,12 +194,15 @@ class SkillRegistry:
|
||||
return self._skills.get(name)
|
||||
|
||||
def load_prompt(self, name: str) -> str:
|
||||
"""Load and return the prompt template body from a skill's ``SKILL.md``."""
|
||||
"""Load and return the prompt template body for the named skill."""
|
||||
|
||||
skill_dir = self._skills_dir / name
|
||||
skill_path = skill_dir / "SKILL.md"
|
||||
if not skill_path.exists():
|
||||
raise FileNotFoundError(f"SKILL.md not found: {skill_path}")
|
||||
skill_path = self._find_prompt_file(skill_dir)
|
||||
if skill_path is None:
|
||||
raise FileNotFoundError(
|
||||
f"Prompt file not found for skill '{name}' in {skill_dir} "
|
||||
f"(tried {list(_PROMPT_FILE_NAMES)})"
|
||||
)
|
||||
try:
|
||||
_frontmatter, body = _parse_skill_file(skill_path)
|
||||
return body
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
---
|
||||
name: enrich_hf_metadata
|
||||
title: "Enrich Metadata from HuggingFace"
|
||||
description: >
|
||||
Parse the HuggingFace model card via LLM to extract description, trigger
|
||||
words, base model, tags, and preview image URL.
|
||||
llm_required: true
|
||||
---
|
||||
|
||||
You are an expert assistant for AI image generation models. Your task is to extract structured metadata from a HuggingFace model card (README.md).
|
||||
|
||||
## Model Information
|
||||
|
||||
- **Repository**: {{hf_url}}
|
||||
- **Model file path**: {{model_path}}
|
||||
- **Repository ID**: {{repo}}
|
||||
|
||||
## Current Metadata (may be incomplete)
|
||||
|
||||
```json
|
||||
{{current_metadata}}
|
||||
```
|
||||
|
||||
## Available Base Models
|
||||
|
||||
The following base models are currently valid in this system:
|
||||
{{base_models}}
|
||||
|
||||
## HuggingFace README Content
|
||||
|
||||
```
|
||||
{{readme_content}}
|
||||
```
|
||||
|
||||
## Extraction Instructions
|
||||
|
||||
Extract the following information from the README content above:
|
||||
|
||||
### base_model
|
||||
The base model this LoRA/checkpoint was trained on. Use EXACTLY one of the names from the **Available Base Models** list above. Do not invent new names or use aliases.
|
||||
|
||||
Check the YAML frontmatter (between --- markers) for `base_model:` first, then look at the description text and safetensors metadata. If you cannot determine it, return an empty string.
|
||||
|
||||
### trigger_words
|
||||
The trigger words or activation prompts needed to use this LoRA. Look for:
|
||||
- `instance_prompt:` in the YAML frontmatter
|
||||
- Phrases like "trigger word:", "trigger:", "use this prompt:", "activation prompt:"
|
||||
- Example prompts at the start (usually the first word or phrase before any description)
|
||||
Return as an array of strings. If none found, return an empty array.
|
||||
|
||||
### description
|
||||
A concise 1-2 sentence summary of what this model does. Extract from the "Model description" section or the first paragraph. Return empty string if the README is too minimal.
|
||||
|
||||
### tags
|
||||
3-8 relevant tags for categorizing this model. Extract from:
|
||||
- The YAML frontmatter `tags:` list (often contains excellent categorization tags)
|
||||
- The model type (e.g. "lora", "checkpoint", "flux", "sdxl")
|
||||
- The style/subject (e.g. "anime", "photorealistic", "style", "character")
|
||||
All lowercase, no spaces. Return empty array if none found.
|
||||
|
||||
### preview_url
|
||||
The URL of the most suitable preview image from the README. Look for image tags (e.g. ``) and the YAML frontmatter `widget:` section (which often has `output.url` fields). Choose the first image that appears to be a generation example (not a logo or diagram). Construct the absolute URL as `https://huggingface.co/{{repo}}/resolve/main/{filename}`. If no suitable image is found, return an empty string.
|
||||
|
||||
### confidence
|
||||
Your confidence level in the extracted data:
|
||||
- "high" — most fields were explicitly stated in the README
|
||||
- "medium" — some fields were inferred from context
|
||||
- "low" — most fields are guesses based on limited information
|
||||
|
||||
## Output Format
|
||||
|
||||
Return ONLY a JSON object with exactly these fields (no markdown fences, no extra text):
|
||||
|
||||
```json
|
||||
{
|
||||
"model_path": "{{model_path}}",
|
||||
"base_model": "<canonical name or empty string>",
|
||||
"trigger_words": ["<word1>", "<word2>"],
|
||||
"description": "<1-2 sentence summary>",
|
||||
"tags": ["<tag1>", "<tag2>"],
|
||||
"preview_url": "<image URL or empty string>",
|
||||
"confidence": "<high|medium|low>"
|
||||
}
|
||||
```
|
||||
|
||||
Important:
|
||||
- Only include the JSON object, no other text
|
||||
- If a field cannot be determined, use an empty string or empty array
|
||||
- Do not fabricate information not supported by the README
|
||||
@@ -0,0 +1,165 @@
|
||||
---
|
||||
name: enrich_hf_metadata
|
||||
title: "Enrich Metadata from HuggingFace"
|
||||
description: >
|
||||
Parse the HuggingFace model card via LLM to extract description, trigger
|
||||
words, base model, tags, and preview image URL.
|
||||
llm_required: true
|
||||
---
|
||||
|
||||
You are an expert assistant for AI image generation models. Your task is to extract structured metadata from a HuggingFace model card (README.md).
|
||||
|
||||
## Model Information
|
||||
|
||||
- **Repository**: {{hf_url}}
|
||||
- **Model file path**: {{model_path}}
|
||||
- **Model filename**: {{model_basename}}
|
||||
- **Repository ID**: {{repo}}
|
||||
|
||||
## Current Metadata (may be incomplete)
|
||||
|
||||
```json
|
||||
{{current_metadata}}
|
||||
```
|
||||
|
||||
## User Priority Tags Reference
|
||||
|
||||
The user has configured the following list of **meaningful tag categories** for this model type (`{{model_type}}`):
|
||||
|
||||
```
|
||||
{{priority_tags}}
|
||||
```
|
||||
|
||||
These are the subjects, styles, and concepts the user considers useful for categorization. Use this list as a **reference** when evaluating tags (see the **tags** section below).
|
||||
|
||||
## Available Base Models
|
||||
|
||||
The following base models are currently valid in this system. Use the EXACT
|
||||
name listed — do not invent aliases or modify variant suffixes.
|
||||
|
||||
{{base_models}}
|
||||
|
||||
## HuggingFace README Content
|
||||
|
||||
```
|
||||
{{readme_content}}
|
||||
```
|
||||
|
||||
## Extraction Instructions
|
||||
|
||||
Extract the following information from the README content above:
|
||||
|
||||
### base_model
|
||||
The base model this model was trained on. Use EXACTLY one of the names from the **Available Base Models** list above. Do not invent new names or use aliases.
|
||||
|
||||
Check the YAML frontmatter for ``base_model:`` first. If the frontmatter has no ``base_model:``, look at the **model filename** (``{{model_basename}}``), YAML ``tags:``, README title and first paragraph for clues — the base model family is often embedded in the name
|
||||
|
||||
### trigger_words
|
||||
The trigger words or activation prompts needed to use this LoRA. Look for:
|
||||
- `instance_prompt:` in the YAML frontmatter
|
||||
- Phrases like "trigger word:", "trigger:", "use this prompt:", "activation prompt:"
|
||||
- In collection repos: the trigger section **specific to this model file** (look near matching download links or anchor IDs)
|
||||
- Example prompts at the start (usually the first word or phrase before any description)
|
||||
Return as an array of strings. If none found, return an empty array `[]`. **Never** return `["None"]` or any placeholder value — a truly empty list means no trigger words exist.
|
||||
|
||||
### short_description
|
||||
A concise 1-2 sentence summary of what this model does. Extract from the "Model description" section or the first paragraph. For collection repos, focus on the **specific model version** matching `{{model_basename}}`, not the repo as a whole. Return empty string if the README is too minimal.
|
||||
|
||||
### tags
|
||||
3-8 relevant tags for categorizing this model. **Quality over quantity.**
|
||||
|
||||
Sources to consider:
|
||||
- The YAML frontmatter `tags:` list (filter out technical ones — see below)
|
||||
- The subject, style, character, or concept the model represents
|
||||
- The model filename itself may give clues (e.g. "pokemon", "anime", "pixelart")
|
||||
|
||||
**Critical filtering rules — apply them strictly:**
|
||||
|
||||
1. **Exclude technical/generic tags.** Reject any tag that describes the model's **training methodology, framework, architecture, or modality** rather than its content. Examples to exclude: `text-to-image`, `diffusers`, `lora`, `dreambooth`, `diffusers-training`, `flux`, `sdxl`, `checkpoint`, `pytorch`, `safetensors`, `fine-tuning`, `stable-diffusion`, and any variant of these.
|
||||
|
||||
2. **Cross-reference against the priority_tags reference.** Only include a tag if it meaningfully describes what the model actually creates (subject, style, character type) and is semantically close to one of the priority_tags. If none of the README's tags match meaningful categories, prefer returning a smaller set or an empty array over including low-value tags.
|
||||
|
||||
3. **All lowercase, no spaces, no hyphens** (use single words like `"photorealistic"`, `"anime"`, `"character"`).
|
||||
|
||||
Return empty array if no meaningful content tags remain after filtering.
|
||||
|
||||
### recommended_width, recommended_height
|
||||
The recommended image generation resolution for this model, in pixels. Look for sections like "Best Dimensions", "Recommended size", "Suggested resolution", or similar phrasing in the README. Prefer the explicitly marked "Best" or default resolution. If the table/list has multiple entries (e.g. "768 x 1024 (Best)" and "1024 x 1024 (Default)"), use the one marked "Best". Return integers. If no resolution can be determined, return 0 for both.
|
||||
|
||||
### preview_url
|
||||
The URL of the most suitable preview image from the README. Look for:
|
||||
- Image tags near the section matching the model filename (`{{model_basename}}`)
|
||||
- The YAML frontmatter `widget:` section (which often has `output.url` fields)
|
||||
- In collection repos: the sample images listed **under the section** for this specific model version
|
||||
- Generic `` in the body
|
||||
Choose the first image that appears to be a generation example (not a logo or diagram). Construct the absolute URL as `https://huggingface.co/{{repo}}/resolve/main/{filename}`. If no suitable image is found, return an empty string.
|
||||
|
||||
### notes
|
||||
A plain-text summary of the model card's key practical usage information. Combine trigger words, style modifiers, recommended parameters (steps, CFG, resolution, sampler), and any setup tips into a readable paragraph. For collection repos, focus on the **specific model version** matching `{{model_basename}}`. Return empty string if the README has no useful usage info.
|
||||
|
||||
### usage_tips
|
||||
A JSON string with structured usage recommendations. Extract from the README any explicit ranges or recommended values (e.g. "Set LoRA strength: **0.85 - 1.4**", "CLIP strength: 0.5"). Possible fields (include only those you can determine):
|
||||
|
||||
```json
|
||||
{
|
||||
"strength_min": 0.85,
|
||||
"strength_max": 1.4,
|
||||
"strength_range": "0.85-1.4",
|
||||
"strength": 0.6,
|
||||
"clip_strength": 0.5,
|
||||
"clip_skip": 2
|
||||
}
|
||||
```
|
||||
|
||||
Return the JSON string (e.g. `'{"strength_min":0.85,"strength_max":1.4}'`). Return `"{}"` if nothing useful is found.
|
||||
|
||||
### confidence
|
||||
Your confidence level in the extracted data:
|
||||
- "high" — most fields were explicitly stated in the README
|
||||
- "medium" — some fields were inferred from context
|
||||
- "low" — most fields are guesses based on limited information
|
||||
|
||||
## Important: Handling Collection Repos (multiple model files)
|
||||
|
||||
Many HuggingFace repos contain **multiple model files** in a single repository
|
||||
(e.g. a "LoRA collection" with different styles/characters in separate files).
|
||||
|
||||
The model file currently being enriched is: **`{{model_basename}}`**
|
||||
|
||||
To find the correct section in the README:
|
||||
|
||||
1. **Search for download links** containing the filename — the surrounding paragraph is your section.
|
||||
2. **Search for anchor IDs** (`<a id="...">`) or section headings whose text matches words from the filename.
|
||||
3. **Search for HTML headings** (`<h1>`, `<h2>`, `<span>`) containing parts of the filename.
|
||||
4. If no match is found, use the full README as usual — the model may be the only one in the repo.
|
||||
|
||||
When a matching section IS found, prefer metadata from that section.
|
||||
When no section matches (e.g. single-model repos or repos without per-file sections),
|
||||
extract metadata from the full README normally. Do not return empty data just
|
||||
because the filename doesn't appear in the README.
|
||||
|
||||
## Output Format
|
||||
|
||||
Return ONLY a JSON object with exactly these fields (no markdown fences, no extra text):
|
||||
|
||||
```json
|
||||
{
|
||||
"model_path": "{{model_path}}",
|
||||
"base_model": "<canonical name or empty string>",
|
||||
"trigger_words": ["<word1>", "<word2>"],
|
||||
"short_description": "<1-2 sentence summary>",
|
||||
"tags": ["<tag1>", "<tag2>"],
|
||||
"recommended_width": 768,
|
||||
"recommended_height": 1024,
|
||||
"preview_url": "<image URL or empty string>",
|
||||
"notes": "<plain-text usage summary or empty string>",
|
||||
"usage_tips": "<JSON string like '{\"strength_min\":0.85,\"strength_max\":1.4}' or '{}'>",
|
||||
"confidence": "<high|medium|low>"
|
||||
}
|
||||
```
|
||||
|
||||
Important:
|
||||
- Only include the JSON object, no other text
|
||||
- If a field cannot be determined, use an empty string or empty array
|
||||
- Do not fabricate information not supported by the README
|
||||
- Never use placeholder values like `"None"` or `"unknown"` for missing data — use empty string or empty array
|
||||
File diff suppressed because it is too large
Load Diff
@@ -201,6 +201,13 @@ class Aria2Downloader:
|
||||
"auto-file-renaming": "false",
|
||||
"file-allocation": "none",
|
||||
}
|
||||
|
||||
# Pass proxy to aria2 so the actual file transfer goes through the
|
||||
# same proxy used by the aiohttp-based URL resolution step above.
|
||||
downloader = await get_downloader()
|
||||
if downloader.proxy_url:
|
||||
options["all-proxy"] = downloader.proxy_url
|
||||
|
||||
if request_headers:
|
||||
options["header"] = [
|
||||
f"{key}: {value}" for key, value in request_headers.items()
|
||||
|
||||
@@ -304,6 +304,20 @@ class CivArchiveClient:
|
||||
version_id = file_data.get("model_version_id") or file_data.get("modelVersionId")
|
||||
if model_id is None or version_id is None:
|
||||
continue
|
||||
# CivitAI / CivArchive model IDs are small integers (typically ≤ 7
|
||||
# digits). Reject suspiciously large values that indicate the API
|
||||
# returned a malformed payload (e.g. a hash reinterpreted as an ID)
|
||||
# to avoid pointless HTTP 500 errors from CivArchive.
|
||||
_MAX_VALID_CIVITAI_ID = 100_000_000
|
||||
try:
|
||||
if int(model_id) >= _MAX_VALID_CIVITAI_ID or int(version_id) >= _MAX_VALID_CIVITAI_ID:
|
||||
logger.debug(
|
||||
"Skipping implausible CivArchive model_id=%s / version_id=%s",
|
||||
model_id, version_id,
|
||||
)
|
||||
continue
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
resolved = await self.get_model_version(model_id, version_id)
|
||||
if resolved:
|
||||
return resolved
|
||||
|
||||
@@ -213,6 +213,18 @@ class CivitaiBaseModelService:
|
||||
"wan video 2.2 i2v-a14b": "WAN",
|
||||
"wan video 2.5 t2v": "WAN",
|
||||
"wan video 2.5 i2v": "WAN",
|
||||
"wan video 2.7": "WAN",
|
||||
"wan image 2.7": "WI27",
|
||||
"ace audio": "ACE",
|
||||
"boogu": "BOOG",
|
||||
"grok": "GROK",
|
||||
"happyhorse": "HAPP",
|
||||
"hidream-o1": "HIO1",
|
||||
"lens": "LENS",
|
||||
"mai": "MAI",
|
||||
"upscaler": "UPSC",
|
||||
"ideogram 4.0": "ID40",
|
||||
"qwen 2": "QWN2",
|
||||
}
|
||||
|
||||
if lower_name in special_cases:
|
||||
@@ -392,6 +404,7 @@ class CivitaiBaseModelService:
|
||||
"LTXV2",
|
||||
"LTXV 2.3",
|
||||
"CogVideoX",
|
||||
"HappyHorse",
|
||||
"Mochi",
|
||||
"Hunyuan Video",
|
||||
"Wan Video",
|
||||
@@ -404,15 +417,25 @@ class CivitaiBaseModelService:
|
||||
"Wan Video 2.2 I2V-A14B",
|
||||
"Wan Video 2.5 T2V",
|
||||
"Wan Video 2.5 I2V",
|
||||
"Wan Image 2.7",
|
||||
"Wan Video 2.7",
|
||||
],
|
||||
"Other Models": [
|
||||
"ACE Audio",
|
||||
"Illustrious",
|
||||
"Pony",
|
||||
"Pony V7",
|
||||
"Boogu",
|
||||
"HiDream",
|
||||
"HiDream-O1",
|
||||
"Ideogram 4.0",
|
||||
"Qwen",
|
||||
"Qwen 2",
|
||||
"AuraFlow",
|
||||
"Chroma",
|
||||
"Grok",
|
||||
"Lens",
|
||||
"MAI",
|
||||
"ZImageTurbo",
|
||||
"ZImageBase",
|
||||
"PixArt a",
|
||||
@@ -426,6 +449,7 @@ class CivitaiBaseModelService:
|
||||
"Ernie Turbo",
|
||||
"Nucleus",
|
||||
"Krea 2",
|
||||
"Upscaler",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@@ -230,6 +230,12 @@ class DownloadManager:
|
||||
Returns:
|
||||
Dict with download result
|
||||
"""
|
||||
logger.debug(
|
||||
"[download] download_from_civitai called: model_id=%s, model_version_id=%s, "
|
||||
"source=%s, file_params=%s",
|
||||
model_id, model_version_id, source, file_params,
|
||||
)
|
||||
|
||||
# Validate that at least one identifier is provided
|
||||
if not model_id and not model_version_id:
|
||||
return {
|
||||
@@ -250,6 +256,7 @@ class DownloadManager:
|
||||
"source": source,
|
||||
"file_params": copy.deepcopy(file_params) if file_params is not None else None,
|
||||
"progress": 0,
|
||||
|
||||
"status": "queued",
|
||||
"transfer_backend": self._get_model_download_backend(),
|
||||
"bytes_downloaded": 0,
|
||||
@@ -289,8 +296,8 @@ class DownloadManager:
|
||||
return result
|
||||
except asyncio.CancelledError:
|
||||
return {
|
||||
"success": False,
|
||||
"error": "Download was cancelled",
|
||||
"success": True,
|
||||
"cancelled": True,
|
||||
"download_id": task_id,
|
||||
}
|
||||
finally:
|
||||
@@ -1421,14 +1428,35 @@ class DownloadManager:
|
||||
|
||||
# If file_params is provided, try to find matching file
|
||||
if file_params and model_version_id:
|
||||
target_file_id = file_params.get("id")
|
||||
target_type = file_params.get("type", "Model")
|
||||
target_format = file_params.get("format", "SafeTensor")
|
||||
target_size = file_params.get("size", "full")
|
||||
target_format = file_params.get("format")
|
||||
target_size = file_params.get("size")
|
||||
target_fp = file_params.get("fp")
|
||||
is_primary = file_params.get("isPrimary", False)
|
||||
|
||||
if is_primary:
|
||||
# Find primary file
|
||||
logger.debug(
|
||||
"[download] file_params received: id=%s, type=%s, format=%s, size=%s, fp=%s, isPrimary=%s, "
|
||||
"model_version_id=%s, total_files=%d",
|
||||
target_file_id, target_type, target_format, target_size, target_fp, is_primary,
|
||||
model_version_id, len(files),
|
||||
)
|
||||
|
||||
if target_file_id:
|
||||
target_id_str = str(target_file_id)
|
||||
for f in files:
|
||||
f_id = f.get("id")
|
||||
if str(f_id) == target_id_str:
|
||||
file_info = f
|
||||
logger.debug(
|
||||
"[download] MATCH by ID: id=%s name='%s'",
|
||||
f_id, f.get("name"),
|
||||
)
|
||||
break
|
||||
if not file_info:
|
||||
logger.debug("[download] No file found with id=%s", target_file_id)
|
||||
|
||||
elif is_primary:
|
||||
file_info = next(
|
||||
(
|
||||
f
|
||||
@@ -1439,28 +1467,41 @@ class DownloadManager:
|
||||
None,
|
||||
)
|
||||
else:
|
||||
# Match by metadata
|
||||
# Lenient metadata match: only compare fields present on both sides
|
||||
for f in files:
|
||||
f_type = f.get("type", "")
|
||||
f_meta = f.get("metadata", {})
|
||||
|
||||
# Check type match
|
||||
if f_type != target_type:
|
||||
continue
|
||||
|
||||
# Check metadata match
|
||||
if f_meta.get("format") != target_format:
|
||||
f_meta = f.get("metadata", {})
|
||||
f_format = f_meta.get("format") or f.get("format")
|
||||
f_size = f_meta.get("size") or f.get("size")
|
||||
f_fp = f_meta.get("fp") or f.get("fp")
|
||||
|
||||
if target_format and f_format != target_format:
|
||||
continue
|
||||
if f_meta.get("size") != target_size:
|
||||
if target_size and f_size and f_size != target_size:
|
||||
continue
|
||||
if target_fp and f_meta.get("fp") != target_fp:
|
||||
if target_fp and f_fp and f_fp != target_fp:
|
||||
continue
|
||||
|
||||
file_info = f
|
||||
break
|
||||
|
||||
if not file_info:
|
||||
logger.debug(
|
||||
"[download] No match found via file_params — falling back to primary file lookup",
|
||||
)
|
||||
elif not file_params:
|
||||
logger.debug(
|
||||
"[download] No file_params provided (null/None) — will use primary file lookup. "
|
||||
"model_version_id=%s, total_files=%d",
|
||||
model_version_id, len(files),
|
||||
)
|
||||
|
||||
# Fallback to primary file if no match found
|
||||
if not file_info:
|
||||
logger.debug("[download] Looking for primary file as fallback")
|
||||
file_info = next(
|
||||
(
|
||||
f
|
||||
@@ -1469,6 +1510,13 @@ class DownloadManager:
|
||||
),
|
||||
None,
|
||||
)
|
||||
if file_info:
|
||||
logger.debug(
|
||||
"[download] Fallback primary file selected: id=%s, name=%s",
|
||||
file_info.get("id"), file_info.get("name"),
|
||||
)
|
||||
else:
|
||||
logger.debug("[download] No primary file found in fallback lookup")
|
||||
|
||||
if not file_info:
|
||||
return {"success": False, "error": "No suitable file found in metadata"}
|
||||
|
||||
@@ -46,6 +46,30 @@ def is_ssl_cert_verify_error(exc: BaseException) -> bool:
|
||||
return "CERTIFICATE_VERIFY_FAILED" in str(exc)
|
||||
|
||||
|
||||
def _parse_retry_after(value: str) -> int:
|
||||
"""Parse a Retry-After header value into seconds.
|
||||
|
||||
Supports both integer seconds and HTTP-date formats.
|
||||
Returns a default of 60 seconds on invalid/missing input.
|
||||
"""
|
||||
if not value or not value.strip():
|
||||
return 60
|
||||
|
||||
value = value.strip()
|
||||
try:
|
||||
return max(1, int(value))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
try:
|
||||
parsed = parsedate_to_datetime(value)
|
||||
now = datetime.now().astimezone()
|
||||
delta = (parsed - now).total_seconds()
|
||||
return max(1, int(delta))
|
||||
except (ValueError, OverflowError, OSError):
|
||||
return 60
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DownloadProgress:
|
||||
"""Snapshot of a download transfer at a moment in time."""
|
||||
@@ -911,6 +935,19 @@ class Downloader:
|
||||
elif response.status == 404:
|
||||
error_msg = "File not found"
|
||||
return False, error_msg, None
|
||||
elif response.status == 429:
|
||||
raw_retry_after = response.headers.get("Retry-After")
|
||||
retry_after = _parse_retry_after(raw_retry_after or "")
|
||||
if raw_retry_after:
|
||||
logger.warning(
|
||||
"Rate limited (429) for %s, Retry-After: %ss", url, retry_after
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Rate limited (429) for %s, no Retry-After header; defaulting to %ss",
|
||||
url, retry_after,
|
||||
)
|
||||
return False, f"Rate limited (429), retry after {retry_after}s", None
|
||||
else:
|
||||
error_msg = f"Download failed with status {response.status}"
|
||||
return False, error_msg, None
|
||||
|
||||
+442
-68
@@ -19,11 +19,201 @@ from .errors import LLMNotConfiguredError, LLMRateLimitError, LLMResponseError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Default API base URLs per provider
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model catalog sourced from opencode's maintained model registry.
|
||||
# maps provider_id -> list of model IDs.
|
||||
# ---------------------------------------------------------------------------
|
||||
_MODEL_CATALOG_URL = "https://models.dev/api.json"
|
||||
|
||||
# In-memory cache: maps provider slug -> list of model ID strings.
|
||||
_catalog_cache: Optional[Dict[str, List[str]]] = None
|
||||
|
||||
# Per-model max output token limits parsed from the catalog.
|
||||
# ``{provider_id: {model_id: max_output_tokens}}``.
|
||||
_model_output_limits: Dict[str, Dict[str, int]] = {}
|
||||
|
||||
_CATALOG_TIMEOUT = aiohttp.ClientTimeout(total=30)
|
||||
|
||||
|
||||
async def _load_model_catalog() -> Dict[str, List[str]]:
|
||||
"""Fetch and parse the model catalog.
|
||||
|
||||
Returns ``{provider_id: [model_id, ...]}`` and also populates
|
||||
:data:`_model_output_limits` with per-model ``limit.output`` values
|
||||
for use by :func:`_get_model_max_output`.
|
||||
|
||||
The JSON at ``_MODEL_CATALOG_URL`` is a dict keyed by provider slug; each
|
||||
value has a ``models`` sub-dict keyed by model ID. The result is cached
|
||||
in memory after the first successful fetch.
|
||||
Subsequent calls return the cached data immediately.
|
||||
"""
|
||||
global _catalog_cache, _model_output_limits
|
||||
if _catalog_cache is not None:
|
||||
return _catalog_cache
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession(timeout=_CATALOG_TIMEOUT) as session:
|
||||
async with session.get(_MODEL_CATALOG_URL) as resp:
|
||||
if resp.status != 200:
|
||||
logger.warning("Model catalog returned HTTP %s", resp.status)
|
||||
return _catalog_cache or {}
|
||||
data = await resp.json()
|
||||
except (aiohttp.ClientError, asyncio.TimeoutError, json.JSONDecodeError) as exc:
|
||||
logger.warning("Failed to fetch model catalog: %s", exc)
|
||||
return _catalog_cache or {}
|
||||
|
||||
if not isinstance(data, dict):
|
||||
logger.warning("Model catalog is not a dict, got %s", type(data).__name__)
|
||||
return _catalog_cache or {}
|
||||
|
||||
result: Dict[str, List[str]] = {}
|
||||
output_limits: Dict[str, Dict[str, int]] = {}
|
||||
for provider_id, provider_info in data.items():
|
||||
if not isinstance(provider_info, dict):
|
||||
continue
|
||||
models_dict = provider_info.get("models")
|
||||
if not isinstance(models_dict, dict):
|
||||
continue
|
||||
model_ids: List[str] = []
|
||||
provider_limits: Dict[str, int] = {}
|
||||
for mid, model_info in models_dict.items():
|
||||
if not isinstance(mid, str):
|
||||
continue
|
||||
model_ids.append(mid)
|
||||
if isinstance(model_info, dict):
|
||||
limit = model_info.get("limit")
|
||||
if isinstance(limit, dict):
|
||||
output = limit.get("output")
|
||||
if isinstance(output, (int, float)) and output > 0:
|
||||
provider_limits[mid] = int(output)
|
||||
if model_ids:
|
||||
result[provider_id] = model_ids
|
||||
if provider_limits:
|
||||
output_limits[provider_id] = provider_limits
|
||||
|
||||
_catalog_cache = result
|
||||
_model_output_limits = output_limits
|
||||
logger.debug(
|
||||
"Loaded model catalog: %d providers, %d total models "
|
||||
"(%d providers have output limits)",
|
||||
len(result),
|
||||
sum(len(m) for m in result.values()),
|
||||
len(output_limits),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _get_model_max_output(provider: str, model: str) -> Optional[int]:
|
||||
"""Return the model's max output token limit from the catalog, or ``None``.
|
||||
|
||||
Returns ``None`` when the provider or model is not found in the catalog
|
||||
(e.g. local Ollama models, custom models, or user-typed model names).
|
||||
Callers should fall back to a safe default.
|
||||
"""
|
||||
return _model_output_limits.get(provider, {}).get(model)
|
||||
|
||||
|
||||
# Short timeout for Ollama's local API
|
||||
_OLLAMA_API_TIMEOUT = aiohttp.ClientTimeout(total=8)
|
||||
|
||||
|
||||
async def fetch_ollama_models(api_base: str) -> List[str]:
|
||||
"""Fetch locally available models from a running Ollama instance.
|
||||
|
||||
Uses Ollama's OpenAI-compatible ``GET {api_base}/models`` endpoint.
|
||||
Returns an empty list if Ollama is not reachable (not running).
|
||||
"""
|
||||
url = f"{api_base.rstrip('/')}/models"
|
||||
try:
|
||||
async with aiohttp.ClientSession(timeout=_OLLAMA_API_TIMEOUT) as session:
|
||||
async with session.get(url) as resp:
|
||||
if resp.status != 200:
|
||||
logger.debug("Ollama API returned HTTP %s from %s", resp.status, api_base)
|
||||
return []
|
||||
data = await resp.json()
|
||||
except (aiohttp.ClientError, asyncio.TimeoutError, json.JSONDecodeError) as exc:
|
||||
logger.debug("Ollama not reachable at %s: %s", api_base, exc)
|
||||
return []
|
||||
|
||||
raw = data.get("data") if isinstance(data, dict) else None
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
|
||||
return [
|
||||
str(entry["id"]) for entry in raw
|
||||
if isinstance(entry, dict) and isinstance(entry.get("id"), str)
|
||||
]
|
||||
|
||||
|
||||
async def get_provider_model_ids(provider_id: str) -> List[str]:
|
||||
"""Return the list of known model IDs for *provider_id* from the catalog.
|
||||
|
||||
The catalog is loaded on first call and cached thereafter. If the
|
||||
provider is not found an empty list is returned (never raises).
|
||||
"""
|
||||
catalog = await _load_model_catalog()
|
||||
return catalog.get(provider_id, [])
|
||||
|
||||
|
||||
async def get_all_provider_models(
|
||||
provider_ids: List[str],
|
||||
) -> Dict[str, List[str]]:
|
||||
"""Return model lists for a subset of providers in one call.
|
||||
|
||||
Loads the catalog (cached) and returns only the requested providers.
|
||||
Handy for embedding lightweight data into the template context.
|
||||
"""
|
||||
catalog = await _load_model_catalog()
|
||||
return {
|
||||
pid: catalog.get(pid, [])
|
||||
for pid in provider_ids
|
||||
}
|
||||
|
||||
|
||||
# Provider preset definitions.
|
||||
# Each entry contains display metadata and defaults for the UI.
|
||||
# The key is the internal provider id stored in ``llm_provider``.
|
||||
# Models are NOT listed here — they come from the opencode model catalog at
|
||||
# runtime (see :func:`get_provider_model_ids`).
|
||||
PROVIDER_PRESETS: Dict[str, Dict[str, Any]] = {
|
||||
"openai": {
|
||||
"name": "OpenAI",
|
||||
"api_base": "https://api.openai.com/v1",
|
||||
"requires_key": True,
|
||||
},
|
||||
"ollama": {
|
||||
"name": "Ollama (local)",
|
||||
"api_base": "http://localhost:11434/v1",
|
||||
"requires_key": False,
|
||||
},
|
||||
"deepseek": {
|
||||
"name": "DeepSeek",
|
||||
"api_base": "https://api.deepseek.com/v1",
|
||||
"requires_key": True,
|
||||
},
|
||||
"groq": {
|
||||
"name": "Groq",
|
||||
"api_base": "https://api.groq.com/openai/v1",
|
||||
"requires_key": True,
|
||||
},
|
||||
"openrouter": {
|
||||
"name": "OpenRouter",
|
||||
"api_base": "https://openrouter.ai/api/v1",
|
||||
"requires_key": True,
|
||||
},
|
||||
"opencode-go": {
|
||||
"name": "OpenCode Go",
|
||||
"api_base": "https://opencode.ai/zen/go/v1",
|
||||
"requires_key": True,
|
||||
},
|
||||
# "custom" is handled specially (no preset api_base, requires user input)
|
||||
}
|
||||
|
||||
# Legacy lookup derived from PROVIDER_PRESETS for backward compat.
|
||||
_PROVIDER_DEFAULTS: Dict[str, str] = {
|
||||
"openai": "https://api.openai.com/v1",
|
||||
"ollama": "http://localhost:11434/v1",
|
||||
# "custom" requires an explicit llm_api_base from the user
|
||||
pid: info["api_base"]
|
||||
for pid, info in PROVIDER_PRESETS.items()
|
||||
if info.get("api_base")
|
||||
}
|
||||
|
||||
# Request timeout for LLM calls (seconds)
|
||||
@@ -33,8 +223,8 @@ _LLM_TIMEOUT = aiohttp.ClientTimeout(total=120)
|
||||
class LLMService:
|
||||
"""Centralized LLM API client.
|
||||
|
||||
All agent skills call LLMs through this service so that BYOK config,
|
||||
retry logic, and error handling live in one place.
|
||||
All LLM-based enrichment features call through this service so
|
||||
that BYOK config, retry logic, and error handling live in one place.
|
||||
"""
|
||||
|
||||
_instance: Optional["LLMService"] = None
|
||||
@@ -57,6 +247,10 @@ class LLMService:
|
||||
from .settings_manager import get_settings_manager
|
||||
|
||||
cls._instance = cls(get_settings_manager())
|
||||
# Start preloading the model catalog in the background so
|
||||
# the settings UI never blocks on it. The catalog is
|
||||
# cached after the first fetch (see _load_model_catalog).
|
||||
asyncio.create_task(_load_model_catalog())
|
||||
return cls._instance
|
||||
|
||||
@classmethod
|
||||
@@ -79,20 +273,33 @@ class LLMService:
|
||||
"model": self._settings.get("llm_model", ""),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _provider_requires_key(provider: str) -> bool:
|
||||
"""Return ``False`` when the given provider id does not need an API key."""
|
||||
preset = PROVIDER_PRESETS.get(provider, {})
|
||||
return bool(preset.get("requires_key", True))
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
"""Return ``True`` when the LLM provider is minimally configured.
|
||||
|
||||
A provider is considered configured when ``llm_model`` is set and
|
||||
(for non-Ollama) an API key is configured.
|
||||
A provider is considered configured when ``llm_model`` is set,
|
||||
an API key is configured for providers that require one (e.g.
|
||||
Ollama does not), and an API base URL is set for providers that
|
||||
have no preset default (e.g. ``custom``).
|
||||
"""
|
||||
|
||||
cfg = self._get_config()
|
||||
has_model = bool(cfg["model"])
|
||||
has_key = bool(cfg["api_key"]) or cfg["provider"] == "ollama"
|
||||
return has_model and has_key
|
||||
has_key = bool(cfg["api_key"]) or not self._provider_requires_key(cfg["provider"])
|
||||
has_base = bool(cfg["api_base"]) or bool(_PROVIDER_DEFAULTS.get(cfg["provider"]))
|
||||
return has_model and has_key and has_base
|
||||
|
||||
def _resolve_api_base(self, provider: str, api_base: str) -> str:
|
||||
"""Resolve the API base URL for the given provider."""
|
||||
"""Resolve the API base URL for the given provider.
|
||||
|
||||
If ``api_base`` is explicitly set (non-empty), it takes priority.
|
||||
Otherwise the default from :data:`PROVIDER_PRESETS` is used.
|
||||
"""
|
||||
|
||||
if api_base:
|
||||
return api_base.rstrip("/")
|
||||
@@ -109,19 +316,26 @@ class LLMService:
|
||||
def _ensure_configured(self) -> Dict[str, Any]:
|
||||
"""Validate configuration and return it, or raise.
|
||||
|
||||
A provider is considered configured when ``llm_model`` is set and
|
||||
(for non-Ollama) an API key is configured.
|
||||
A provider is considered configured when ``llm_model`` is set,
|
||||
an API key is configured for providers that require one, and
|
||||
an API base URL is set for providers without a preset default.
|
||||
"""
|
||||
|
||||
cfg = self._get_config()
|
||||
has_model = bool(cfg["model"])
|
||||
has_key = bool(cfg["api_key"]) or cfg["provider"] == "ollama"
|
||||
if not (has_model and has_key):
|
||||
needs_key = self._provider_requires_key(cfg["provider"])
|
||||
has_key = bool(cfg["api_key"]) or not needs_key
|
||||
has_base = bool(cfg["api_base"]) or bool(_PROVIDER_DEFAULTS.get(cfg["provider"]))
|
||||
if not (has_model and has_key and has_base):
|
||||
parts = []
|
||||
if not has_model:
|
||||
parts.append("No LLM model specified")
|
||||
if not has_key and cfg["provider"] != "ollama":
|
||||
if not has_key and needs_key:
|
||||
parts.append("No LLM API key configured")
|
||||
if not has_base:
|
||||
parts.append(
|
||||
f"No API base URL for provider '{cfg['provider']}'"
|
||||
)
|
||||
detail = "; ".join(parts) if parts else "LLM provider is not configured"
|
||||
raise LLMNotConfiguredError(
|
||||
f"{detail}. Configure it in Settings → AI Provider."
|
||||
@@ -163,18 +377,68 @@ class LLMService:
|
||||
|
||||
cfg = self._ensure_configured()
|
||||
api_base = self._resolve_api_base(cfg["provider"], cfg["api_base"])
|
||||
url = f"{api_base}/chat/completions"
|
||||
model_name = model or cfg["model"]
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"model": model_name,
|
||||
"messages": messages,
|
||||
"temperature": temperature,
|
||||
}
|
||||
if response_format is not None:
|
||||
payload["response_format"] = response_format
|
||||
if max_tokens is not None:
|
||||
payload["max_tokens"] = max_tokens
|
||||
is_ollama = cfg["provider"] == "ollama"
|
||||
|
||||
if is_ollama:
|
||||
# Use Ollama's native /api/chat endpoint which does NOT expose
|
||||
# a separate reasoning/thinking field (the model's full output
|
||||
# lands directly in message.content). The OpenAI-compatible
|
||||
# endpoint splits thinking into the "reasoning" field, making
|
||||
# content empty when thinking consumes all available tokens.
|
||||
base = api_base.rstrip("/")
|
||||
if base.endswith("/v1"):
|
||||
base = base[:-3]
|
||||
url = f"{base}/api/chat"
|
||||
else:
|
||||
url = f"{api_base}/chat/completions"
|
||||
|
||||
payload: Dict[str, Any]
|
||||
if is_ollama:
|
||||
payload = {
|
||||
"model": model_name,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
# Suppress separate thinking trace — thinking still happens
|
||||
# internally (accuracy preserved) but output goes directly to
|
||||
# message.content instead of being split across content +
|
||||
# thinking. Without this the model can exhaust num_predict
|
||||
# on thinking alone and leave content empty.
|
||||
"think": False,
|
||||
"options": {
|
||||
"temperature": temperature,
|
||||
# 8K context is sufficient for metadata enrichment
|
||||
# (prompt ~2-5K, output ~0.2-1K tokens). The old 32K
|
||||
# value was excessive for this use case and increased
|
||||
# Ollama VRAM usage unnecessarily.
|
||||
"num_ctx": 8192,
|
||||
},
|
||||
}
|
||||
if response_format is not None:
|
||||
payload["format"] = "json"
|
||||
if max_tokens is not None:
|
||||
payload["options"]["num_predict"] = max_tokens
|
||||
else:
|
||||
payload = {
|
||||
"model": model_name,
|
||||
"messages": messages,
|
||||
"temperature": temperature,
|
||||
}
|
||||
if response_format is not None:
|
||||
payload["response_format"] = response_format
|
||||
if max_tokens is not None:
|
||||
payload["max_tokens"] = max_tokens
|
||||
|
||||
if is_ollama:
|
||||
logger.info(
|
||||
"Ollama request: model=%s num_ctx=%s num_predict=%s format=%s think=%s",
|
||||
payload.get("model"),
|
||||
payload.get("options", {}).get("num_ctx"),
|
||||
payload.get("options", {}).get("num_predict"),
|
||||
payload.get("format", "none"),
|
||||
payload.get("think"),
|
||||
)
|
||||
|
||||
headers = self._build_headers(cfg["api_key"])
|
||||
|
||||
@@ -217,8 +481,25 @@ class LLMService:
|
||||
|
||||
# Parse response
|
||||
try:
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
usage = data.get("usage", {})
|
||||
if is_ollama:
|
||||
content = (data.get("message") or {}).get("content") or ""
|
||||
usage = {"completion_tokens": data.get("eval_count", 0)}
|
||||
finish_reason = data.get("done_reason", "")
|
||||
if not content:
|
||||
logger.warning(
|
||||
"LLM returned empty content. Provider=ollama, "
|
||||
"done_reason=%s, eval_count=%s",
|
||||
finish_reason,
|
||||
data.get("eval_count", 0),
|
||||
)
|
||||
else:
|
||||
content = data["choices"][0]["message"].get("content") or ""
|
||||
usage = data.get("usage", {})
|
||||
if not content:
|
||||
logger.warning(
|
||||
"LLM returned empty content. Full response truncated: %s",
|
||||
json.dumps(data, ensure_ascii=False)[:1000],
|
||||
)
|
||||
return {
|
||||
"content": content,
|
||||
"usage": usage,
|
||||
@@ -245,11 +526,15 @@ class LLMService:
|
||||
temperature: float = 0.3,
|
||||
max_tokens: Optional[int] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Call the LLM and return parsed JSON.
|
||||
"""Call the LLM with ``response_format=json_object`` and return parsed JSON.
|
||||
|
||||
Sends ``response_format: {"type": "json_object"}`` when the provider
|
||||
supports it, and parses the response content as JSON. If parsing
|
||||
fails, retries once with a clarifying system message.
|
||||
``max_tokens`` is resolved in this order:
|
||||
1. Explicit caller-supplied ``max_tokens``
|
||||
2. Per-model ``limit.output`` from the model catalog
|
||||
3. A safe default of 4096 (sufficient for metadata enrichment)
|
||||
|
||||
If the response content is empty or not valid JSON, attempts
|
||||
:func:`_try_salvage_json` before raising.
|
||||
|
||||
Args:
|
||||
system_prompt: System-level instructions
|
||||
@@ -264,7 +549,7 @@ class LLMService:
|
||||
Raises:
|
||||
LLMNotConfiguredError: Provider not configured
|
||||
LLMRateLimitError: Rate limited
|
||||
LLMResponseError: JSON parse failure after retry
|
||||
LLMResponseError: Empty response or JSON parse failure
|
||||
"""
|
||||
|
||||
messages = [
|
||||
@@ -272,50 +557,139 @@ class LLMService:
|
||||
{"role": "user", "content": user_prompt},
|
||||
]
|
||||
|
||||
# First attempt with JSON mode
|
||||
# Resolve max_tokens: caller override → catalog lookup → safe default
|
||||
if max_tokens is None:
|
||||
cfg = self._get_config()
|
||||
effective_max = _get_model_max_output(cfg["provider"], cfg["model"])
|
||||
else:
|
||||
effective_max = max_tokens
|
||||
if effective_max is None:
|
||||
effective_max = 4096
|
||||
|
||||
result = await self.chat_completion(
|
||||
messages=messages,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
response_format={"type": "json_object"},
|
||||
max_tokens=max_tokens,
|
||||
max_tokens=effective_max,
|
||||
)
|
||||
|
||||
try:
|
||||
return json.loads(result["content"])
|
||||
except (json.JSONDecodeError, TypeError) as exc:
|
||||
logger.warning(
|
||||
"LLM JSON parse failed on first attempt: %s. Retrying.", exc
|
||||
content = result.get("content", "") or ""
|
||||
if not content:
|
||||
raise LLMResponseError(
|
||||
"LLM returned empty content in json_object mode. "
|
||||
f"Raw response: {json.dumps(result)[:500]}"
|
||||
)
|
||||
|
||||
# Retry with explicit instruction to return valid JSON
|
||||
retry_messages = messages + [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": result["content"],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"The previous response could not be parsed as JSON. "
|
||||
"Please respond with ONLY a valid JSON object, no "
|
||||
"markdown fences or extra text."
|
||||
),
|
||||
},
|
||||
]
|
||||
try:
|
||||
parsed = json.loads(content)
|
||||
logger.debug(
|
||||
"LLM raw content: %s",
|
||||
json.dumps(parsed, ensure_ascii=False)[:2000],
|
||||
)
|
||||
return parsed
|
||||
except (json.JSONDecodeError, TypeError) as exc:
|
||||
logger.info(
|
||||
"LLM raw response (first 800 chars): %s",
|
||||
content[:800],
|
||||
)
|
||||
|
||||
result = await self.chat_completion(
|
||||
messages=retry_messages,
|
||||
model=model,
|
||||
temperature=0.0, # More deterministic for retry
|
||||
response_format={"type": "json_object"},
|
||||
max_tokens=max_tokens,
|
||||
# Last resort: attempt to salvage partial/truncated JSON
|
||||
salvaged = _try_salvage_json(content)
|
||||
if salvaged is not None:
|
||||
logger.warning(
|
||||
"LLM JSON salvaged from partial content (%d chars raw)",
|
||||
len(content),
|
||||
)
|
||||
return salvaged
|
||||
|
||||
raise LLMResponseError(
|
||||
f"LLM response could not be parsed as JSON: {content[:200]}"
|
||||
)
|
||||
|
||||
try:
|
||||
return json.loads(result["content"])
|
||||
except (json.JSONDecodeError, TypeError) as exc:
|
||||
raise LLMResponseError(
|
||||
f"LLM response could not be parsed as JSON after retry: {exc}\n"
|
||||
f"Raw content: {result['content'][:500]}"
|
||||
) from exc
|
||||
|
||||
def _try_salvage_json(raw: str) -> Dict[str, Any] | None:
|
||||
"""Attempt to repair and parse a truncated JSON string.
|
||||
|
||||
Handles common truncation patterns:
|
||||
|
||||
* Incomplete string value at the end (``"foo`` → ``"foo"``)
|
||||
* Missing closing ``}`` or ``]`` (respecting nesting order)
|
||||
* Trailing comma before closing bracket
|
||||
* Extra text after the JSON object (e.g. markdown fences)
|
||||
|
||||
Returns the parsed dict on success, ``None`` if repair is impossible.
|
||||
"""
|
||||
if not raw:
|
||||
return None
|
||||
|
||||
text = raw.strip()
|
||||
|
||||
# Strip markdown fences if the LLM wrapped the JSON
|
||||
if text.startswith("```"):
|
||||
end = text.find("\n")
|
||||
text = text[end + 1:] if end != -1 else text[3:]
|
||||
if text.endswith("```"):
|
||||
text = text[:-3].rstrip()
|
||||
|
||||
# Find the first '{' and strip everything before it
|
||||
start = text.find("{")
|
||||
if start == -1:
|
||||
return None
|
||||
text = text[start:]
|
||||
|
||||
# Try to close an incomplete string at the end (e.g. ``"https://huggingf``)
|
||||
# Pattern: ends mid-string (last quote is open)
|
||||
if text.count('"') % 2 == 1:
|
||||
text += '"'
|
||||
|
||||
# Ensure trailing commas before closing braces work
|
||||
text = _strip_trailing_commas(text)
|
||||
|
||||
# Walk through the text character by character to find unclosed
|
||||
# brackets and close them in the correct (LIFO) order.
|
||||
# We ignore brackets inside quoted strings.
|
||||
stack: list[str] = []
|
||||
in_string = False
|
||||
escape = False
|
||||
for ch in text:
|
||||
if escape:
|
||||
escape = False
|
||||
continue
|
||||
if ch == "\\":
|
||||
escape = True
|
||||
continue
|
||||
if ch == '"':
|
||||
in_string = not in_string
|
||||
continue
|
||||
if in_string:
|
||||
continue
|
||||
if ch in ("{", "["):
|
||||
stack.append(ch)
|
||||
elif ch == "}":
|
||||
if stack and stack[-1] == "{":
|
||||
stack.pop()
|
||||
else:
|
||||
return None # Unmatched closer — unrecoverable
|
||||
elif ch == "]":
|
||||
if stack and stack[-1] == "[":
|
||||
stack.pop()
|
||||
else:
|
||||
return None
|
||||
|
||||
# Close remaining open brackets in reverse order
|
||||
for opener in reversed(stack):
|
||||
text += "}" if opener == "{" else "]"
|
||||
|
||||
try:
|
||||
return json.loads(text)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _strip_trailing_commas(text: str) -> str:
|
||||
"""Remove commas that appear before a closing brace/bracket."""
|
||||
import re as _re
|
||||
text = _re.sub(r",\s*}", "}", text)
|
||||
text = _re.sub(r",\s*]", "]", text)
|
||||
return text
|
||||
|
||||
@@ -209,7 +209,21 @@ class MetadataSyncService:
|
||||
error_msg = "CivitAI model is deleted and no archive provider is available"
|
||||
return False, error_msg
|
||||
else:
|
||||
provider_attempts.append((None, await self._get_default_provider()))
|
||||
is_hf_source = bool(model_data.get("hf_url"))
|
||||
if is_hf_source:
|
||||
# HF-sourced model: only check CivitAI API directly.
|
||||
# CivArchive is almost guaranteed to have no record, and
|
||||
# hitting it wastes rate-limit budget.
|
||||
# Use a distinct provider name ("civitai_api" not None) so
|
||||
# downstream code does NOT interpret a "Model not found"
|
||||
# response as civitai_api_not_found — which would mark the
|
||||
# model civitai_deleted=True when it was never on CivitAI.
|
||||
try:
|
||||
provider_attempts.append(("civitai_api", await self._get_provider("civitai_api")))
|
||||
except Exception as exc: # pragma: no cover - provider resolution fault
|
||||
logger.debug("Unable to resolve civitai_api provider: %s", exc)
|
||||
if not provider_attempts:
|
||||
provider_attempts.append((None, await self._get_default_provider()))
|
||||
|
||||
civitai_metadata: Optional[Dict[str, Any]] = None
|
||||
metadata_provider: Optional[MetadataProviderProtocol] = None
|
||||
|
||||
@@ -337,4 +337,25 @@ class ModelCache:
|
||||
else:
|
||||
return False # Model not found
|
||||
|
||||
return True
|
||||
return True
|
||||
|
||||
async def clear_preview_by_path(self, preview_file_path: str) -> int:
|
||||
"""Clear ``preview_url`` for every cached entry referencing a file path.
|
||||
|
||||
When a preview file has been deleted from disk, this removes its
|
||||
reference from all matching cache entries so the next list-API
|
||||
response returns an empty ``preview_url`` instead of a stale URL
|
||||
that produces 404s.
|
||||
|
||||
Returns the number of entries that were updated.
|
||||
"""
|
||||
normalized = preview_file_path.replace("\\", "/")
|
||||
cleared = 0
|
||||
async with self._lock:
|
||||
for item in self.raw_data:
|
||||
cached_url = item.get("preview_url", "")
|
||||
if cached_url.replace("\\", "/") == normalized:
|
||||
item["preview_url"] = ""
|
||||
item["preview_nsfw_level"] = 0
|
||||
cleared += 1
|
||||
return cleared
|
||||
@@ -152,6 +152,11 @@ class SettingsManager:
|
||||
self._check_environment_variables()
|
||||
self._collect_configuration_warnings()
|
||||
|
||||
if os.environ.get("LORA_MANAGER_PORTABLE", "0") == "1":
|
||||
if not self.settings.get("use_portable_settings"):
|
||||
self.settings["use_portable_settings"] = True
|
||||
self._save_settings()
|
||||
|
||||
if self._needs_initial_save:
|
||||
self._save_settings()
|
||||
self._needs_initial_save = False
|
||||
@@ -625,12 +630,37 @@ class SettingsManager:
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _normalize_path_set(paths: Iterable[str]) -> set[str]:
|
||||
"""Normalize an iterable of paths for set-based overlap comparison.
|
||||
|
||||
Resolves symlinks via ``os.path.realpath`` when the path exists on disk,
|
||||
then applies ``os.path.normcase`` + ``os.path.normpath`` for consistent
|
||||
cross-platform comparison. Non-string / empty entries are skipped.
|
||||
"""
|
||||
result: set[str] = set()
|
||||
for p in paths:
|
||||
if not isinstance(p, str):
|
||||
continue
|
||||
stripped = p.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
if os.path.exists(stripped):
|
||||
stripped = os.path.normpath(os.path.realpath(stripped))
|
||||
result.add(os.path.normcase(stripped))
|
||||
return result
|
||||
|
||||
def _validate_folder_paths(
|
||||
self,
|
||||
library_name: str,
|
||||
folder_paths: Mapping[str, Iterable[str]],
|
||||
) -> None:
|
||||
"""Ensure folder paths do not overlap with other libraries."""
|
||||
"""Ensure folder paths do not overlap with other libraries.
|
||||
|
||||
Also detects checkpoints ↔ unet path overlap within the same library
|
||||
(including via symlink resolution), which is a configuration error since
|
||||
these model types must use separate physical folders.
|
||||
"""
|
||||
libraries = self.settings.get("libraries", {})
|
||||
normalized_new: Dict[str, Dict[str, str]] = {}
|
||||
for key, values in folder_paths.items():
|
||||
@@ -668,6 +698,22 @@ class SettingsManager:
|
||||
f"Folder path(s) {collisions} already assigned to library '{other_name}'"
|
||||
)
|
||||
|
||||
# Checkpoints ↔ unet overlap within the same library
|
||||
ckpt_paths = folder_paths.get("checkpoints", []) or []
|
||||
unet_paths = folder_paths.get("unet", []) or []
|
||||
if ckpt_paths and unet_paths:
|
||||
ckpt_real = self._normalize_path_set(ckpt_paths)
|
||||
unet_real = self._normalize_path_set(unet_paths)
|
||||
overlap = ckpt_real & unet_real
|
||||
if overlap:
|
||||
collisions = ", ".join(sorted(overlap))
|
||||
raise ValueError(
|
||||
f"Path(s) {collisions} are configured for both "
|
||||
f"'checkpoints' and 'unet' (diffusion models). "
|
||||
f"These model types must use separate physical folders. "
|
||||
f"Please remove one of the conflicting entries."
|
||||
)
|
||||
|
||||
def _update_active_library_entry(
|
||||
self,
|
||||
*,
|
||||
@@ -1542,8 +1588,12 @@ class SettingsManager:
|
||||
portable_switch_pending = True
|
||||
self._prepare_portable_switch(value)
|
||||
if key == "folder_paths" and isinstance(value, Mapping):
|
||||
active_name = self.get_active_library_name()
|
||||
self._validate_folder_paths(active_name, value)
|
||||
self._update_active_library_entry(folder_paths=value) # type: ignore[arg-type]
|
||||
elif key == "extra_folder_paths" and isinstance(value, Mapping):
|
||||
active_name = self.get_active_library_name()
|
||||
self._validate_folder_paths(active_name, value)
|
||||
self._update_active_library_entry(extra_folder_paths=value) # type: ignore[arg-type]
|
||||
elif key == "default_lora_root":
|
||||
self._update_active_library_entry(default_lora_root=str(value))
|
||||
@@ -1797,6 +1847,9 @@ class SettingsManager:
|
||||
if key in self.settings:
|
||||
minimal[key] = copy.deepcopy(self.settings[key])
|
||||
|
||||
if self.settings.get("use_portable_settings"):
|
||||
minimal["use_portable_settings"] = True
|
||||
|
||||
if self._seed_template:
|
||||
for key, value in self._seed_template.items():
|
||||
minimal.setdefault(key, copy.deepcopy(value))
|
||||
|
||||
@@ -51,6 +51,10 @@ class BulkMetadataRefreshUseCase:
|
||||
if not model.get("skip_metadata_refresh", False)
|
||||
and not self._is_in_skip_path(model.get("folder", ""), skip_paths)
|
||||
and (not model.get("civitai") or not model["civitai"].get("id"))
|
||||
# Skip models downloaded from Hugging Face — they are not on
|
||||
# CivitAI / CivArchive. Users can still refresh them individually
|
||||
# via the right-click context menu.
|
||||
and not model.get("hf_url", "")
|
||||
and not (
|
||||
# Skip models confirmed not on CivitAI when no need to retry
|
||||
model.get("from_civitai") is False
|
||||
|
||||
+13
-1
@@ -226,9 +226,21 @@ SUPPORTED_DOWNLOAD_SKIP_BASE_MODELS = frozenset(
|
||||
"Wan Video 2.5 I2V",
|
||||
"Hunyuan Video",
|
||||
"Anima",
|
||||
"ACE Audio",
|
||||
"Boogu",
|
||||
"Ernie",
|
||||
"Ernie Turbo",
|
||||
"Nucleus",
|
||||
"Grok",
|
||||
"HappyHorse",
|
||||
"HiDream-O1",
|
||||
"Ideogram 4.0",
|
||||
"Krea 2",
|
||||
"Lens",
|
||||
"MAI",
|
||||
"Nucleus",
|
||||
"Qwen 2",
|
||||
"Upscaler",
|
||||
"Wan Image 2.7",
|
||||
"Wan Video 2.7",
|
||||
]
|
||||
)
|
||||
|
||||
@@ -72,6 +72,7 @@ class _DownloadProgress(dict):
|
||||
refreshed_models=set(),
|
||||
failed_models=set(),
|
||||
reprocessed_models=set(),
|
||||
rate_limited_models=set(),
|
||||
)
|
||||
|
||||
def snapshot(self) -> dict:
|
||||
@@ -82,6 +83,7 @@ class _DownloadProgress(dict):
|
||||
snapshot["refreshed_models"] = list(self["refreshed_models"])
|
||||
snapshot["failed_models"] = list(self["failed_models"])
|
||||
snapshot["reprocessed_models"] = list(self.get("reprocessed_models", set()))
|
||||
snapshot["rate_limited_models"] = list(self.get("rate_limited_models", set()))
|
||||
return snapshot
|
||||
|
||||
|
||||
@@ -153,13 +155,15 @@ class DownloadManager:
|
||||
# Step 3: Load progress file (I/O operation, done outside lock)
|
||||
processed_models = set()
|
||||
failed_models = set()
|
||||
rate_limited_models = set()
|
||||
|
||||
try:
|
||||
progress_file, processed_models, failed_models = await self._load_progress_file(output_dir)
|
||||
progress_file, processed_models, failed_models, rate_limited_models = await self._load_progress_file(output_dir)
|
||||
logger.debug(
|
||||
"Loaded previous progress, %s models already processed, %s models marked as failed",
|
||||
"Loaded previous progress, %s models already processed, %s models marked as failed, %s models rate-limited",
|
||||
len(processed_models),
|
||||
len(failed_models),
|
||||
len(rate_limited_models),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load progress file: {e}")
|
||||
@@ -175,6 +179,7 @@ class DownloadManager:
|
||||
self._progress.reset()
|
||||
self._progress["processed_models"] = processed_models
|
||||
self._progress["failed_models"] = failed_models
|
||||
self._progress["rate_limited_models"] = rate_limited_models
|
||||
self._stop_requested = False
|
||||
self._progress["status"] = "running"
|
||||
self._progress["start_time"] = time.time()
|
||||
@@ -242,8 +247,8 @@ class DownloadManager:
|
||||
"status": self._progress.snapshot(),
|
||||
}
|
||||
|
||||
async def _load_progress_file(self, output_dir: str) -> tuple[str, set, set]:
|
||||
"""Load progress file from disk. Returns (progress_file_path, processed_models, failed_models).
|
||||
async def _load_progress_file(self, output_dir: str) -> tuple[str, set, set, set]:
|
||||
"""Load progress file from disk. Returns (progress_file_path, processed_models, failed_models, rate_limited_models).
|
||||
|
||||
This is a separate async method to allow running in executor to avoid blocking event loop.
|
||||
"""
|
||||
@@ -252,8 +257,12 @@ class DownloadManager:
|
||||
None, self._load_progress_file_sync, output_dir
|
||||
)
|
||||
|
||||
def _load_progress_file_sync(self, output_dir: str) -> tuple[str, set, set]:
|
||||
"""Synchronous implementation of progress file loading."""
|
||||
def _load_progress_file_sync(self, output_dir: str) -> tuple[str, set, set, set]:
|
||||
"""Synchronous implementation of progress file loading.
|
||||
|
||||
Returns:
|
||||
tuple: (progress_file_path, processed_models, failed_models, rate_limited_models)
|
||||
"""
|
||||
progress_file = os.path.join(output_dir, ".download_progress.json")
|
||||
progress_source = progress_file
|
||||
|
||||
@@ -289,6 +298,7 @@ class DownloadManager:
|
||||
|
||||
processed_models = set()
|
||||
failed_models = set()
|
||||
rate_limited_models = set()
|
||||
|
||||
if os.path.exists(progress_source):
|
||||
try:
|
||||
@@ -296,11 +306,11 @@ class DownloadManager:
|
||||
saved_progress = json.load(f)
|
||||
processed_models = set(saved_progress.get("processed_models", []))
|
||||
failed_models = set(saved_progress.get("failed_models", []))
|
||||
rate_limited_models = set(saved_progress.get("rate_limited_models", []))
|
||||
except Exception:
|
||||
# Return empty sets on error
|
||||
pass
|
||||
|
||||
return progress_file, processed_models, failed_models
|
||||
return progress_file, processed_models, failed_models, rate_limited_models
|
||||
|
||||
def _load_progress_sets_sync(self, progress_file: str) -> tuple[set, set]:
|
||||
"""Load only the processed and failed model sets from progress file.
|
||||
@@ -732,11 +742,13 @@ class DownloadManager:
|
||||
success,
|
||||
is_stale,
|
||||
failed_images,
|
||||
rate_limited_images,
|
||||
) = await ExampleImagesProcessor.download_model_images_with_tracking(
|
||||
model_hash, model_name, images, model_dir, optimize, downloader
|
||||
)
|
||||
|
||||
failed_urls: Set[str] = set(failed_images)
|
||||
rate_limited_urls: Set[str] = set(rate_limited_images)
|
||||
|
||||
# If metadata is stale, try to refresh it
|
||||
if is_stale and model_hash not in self._progress["refreshed_models"]:
|
||||
@@ -760,6 +772,7 @@ class DownloadManager:
|
||||
success,
|
||||
_,
|
||||
additional_failed,
|
||||
additional_rate_limited,
|
||||
) = await ExampleImagesProcessor.download_model_images_with_tracking(
|
||||
model_hash,
|
||||
model_name,
|
||||
@@ -770,29 +783,50 @@ class DownloadManager:
|
||||
)
|
||||
|
||||
failed_urls.update(additional_failed)
|
||||
rate_limited_urls.update(additional_rate_limited)
|
||||
|
||||
self._progress["refreshed_models"].add(model_hash)
|
||||
|
||||
if failed_urls:
|
||||
# Separate permanent failures from rate-limited ones
|
||||
permanent_failures = failed_urls - rate_limited_urls
|
||||
|
||||
if permanent_failures:
|
||||
await self._remove_failed_images_from_metadata(
|
||||
model_hash,
|
||||
model_name,
|
||||
model_dir,
|
||||
failed_urls,
|
||||
permanent_failures,
|
||||
scanner,
|
||||
)
|
||||
|
||||
if failed_urls:
|
||||
if rate_limited_urls:
|
||||
self._progress["rate_limited_models"].add(model_hash)
|
||||
logger.warning(
|
||||
"%d example images for %s are rate-limited (429), will retry next time",
|
||||
len(rate_limited_urls),
|
||||
model_name,
|
||||
)
|
||||
# Clear failed_models so non-force runs can retry
|
||||
if force and model_hash in self._progress["failed_models"]:
|
||||
self._progress["failed_models"].discard(model_hash)
|
||||
logger.info(
|
||||
f"Removed {model_name} from failed_models after force retry with rate-limited images"
|
||||
)
|
||||
|
||||
if rate_limited_urls:
|
||||
# Don't mark as failed or fully processed — rate-limited
|
||||
# images will be retried next time.
|
||||
pass
|
||||
elif permanent_failures:
|
||||
self._progress["failed_models"].add(model_hash)
|
||||
self._progress["processed_models"].add(model_hash)
|
||||
logger.info(
|
||||
"Removed %s failed example images for %s",
|
||||
len(failed_urls),
|
||||
len(permanent_failures),
|
||||
model_name,
|
||||
)
|
||||
elif success:
|
||||
self._progress["processed_models"].add(model_hash)
|
||||
# Remove from failed_models if force mode enabled and model was previously failed
|
||||
if force and model_hash in self._progress["failed_models"]:
|
||||
self._progress["failed_models"].discard(model_hash)
|
||||
logger.info(
|
||||
@@ -850,6 +884,7 @@ class DownloadManager:
|
||||
"processed_models": list(self._progress["processed_models"]),
|
||||
"refreshed_models": list(self._progress["refreshed_models"]),
|
||||
"failed_models": list(self._progress["failed_models"]),
|
||||
"rate_limited_models": list(self._progress.get("rate_limited_models", set())),
|
||||
"completed": self._progress["completed"],
|
||||
"total": self._progress["total"],
|
||||
"last_update": time.time(),
|
||||
@@ -1155,11 +1190,13 @@ class DownloadManager:
|
||||
success,
|
||||
is_stale,
|
||||
failed_images,
|
||||
rate_limited_images,
|
||||
) = await ExampleImagesProcessor.download_model_images_with_tracking(
|
||||
model_hash, model_name, images, model_dir, optimize, downloader
|
||||
)
|
||||
|
||||
failed_urls: Set[str] = set(failed_images)
|
||||
rate_limited_urls: Set[str] = set(rate_limited_images)
|
||||
|
||||
# If metadata is stale, try to refresh it
|
||||
if is_stale and model_hash not in self._progress["refreshed_models"]:
|
||||
@@ -1183,6 +1220,7 @@ class DownloadManager:
|
||||
success,
|
||||
_,
|
||||
additional_failed_images,
|
||||
additional_rate_limited,
|
||||
) = await ExampleImagesProcessor.download_model_images_with_tracking(
|
||||
model_hash,
|
||||
model_name,
|
||||
@@ -1192,21 +1230,35 @@ class DownloadManager:
|
||||
downloader,
|
||||
)
|
||||
|
||||
# Combine failed images from both attempts
|
||||
failed_urls.update(additional_failed_images)
|
||||
rate_limited_urls.update(additional_rate_limited)
|
||||
|
||||
self._progress["refreshed_models"].add(model_hash)
|
||||
|
||||
# For forced downloads, remove failed images from metadata
|
||||
if failed_urls:
|
||||
# Separate permanent failures from rate-limited ones
|
||||
permanent_failures = failed_urls - rate_limited_urls
|
||||
|
||||
# Only remove permanently failed images from metadata
|
||||
if permanent_failures:
|
||||
await self._remove_failed_images_from_metadata(
|
||||
model_hash, model_name, model_dir, failed_urls, scanner
|
||||
model_hash, model_name, model_dir, permanent_failures, scanner
|
||||
)
|
||||
|
||||
# Mark as processed
|
||||
if (
|
||||
success or failed_urls
|
||||
): # Mark as processed if we successfully downloaded some images or removed failed ones
|
||||
if rate_limited_urls:
|
||||
self._progress["rate_limited_models"].add(model_hash)
|
||||
logger.warning(
|
||||
"%d example images for %s are rate-limited (429), will retry next time",
|
||||
len(rate_limited_urls),
|
||||
model_name,
|
||||
)
|
||||
|
||||
# Mark as processed only when no rate-limited images remain
|
||||
if rate_limited_urls:
|
||||
pass
|
||||
elif permanent_failures:
|
||||
self._progress["processed_models"].add(model_hash)
|
||||
self._progress["failed_models"].add(model_hash)
|
||||
elif success:
|
||||
self._progress["processed_models"].add(model_hash)
|
||||
|
||||
return True # Return True to indicate a remote download happened
|
||||
@@ -1229,15 +1281,20 @@ class DownloadManager:
|
||||
model_dir: str,
|
||||
failed_images: Iterable[str],
|
||||
scanner,
|
||||
error_type: str = "not_found",
|
||||
) -> None:
|
||||
"""Mark failed images in model metadata so they won't be retried."""
|
||||
"""Mark failed images in model metadata so they won't be retried.
|
||||
|
||||
Args:
|
||||
error_type: Reason string stored in the image's ``downloadError`` field
|
||||
(default ``"not_found"``).
|
||||
"""
|
||||
|
||||
failed_set: Set[str] = {url for url in failed_images if url}
|
||||
if not failed_set:
|
||||
return
|
||||
|
||||
try:
|
||||
# Get current model data
|
||||
model_data = await MetadataUpdater.get_updated_model(model_hash, scanner)
|
||||
if not model_data:
|
||||
logger.warning(
|
||||
@@ -1268,7 +1325,7 @@ class DownloadManager:
|
||||
continue
|
||||
|
||||
image["downloadFailed"] = True
|
||||
image.setdefault("downloadError", "not_found")
|
||||
image.setdefault("downloadError", error_type)
|
||||
logger.debug(
|
||||
"Marked example image %s for %s as failed due to missing remote asset",
|
||||
image_url,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
@@ -194,16 +195,22 @@ class ExampleImagesProcessor:
|
||||
|
||||
return model_success, False # (success, is_metadata_stale)
|
||||
|
||||
@staticmethod
|
||||
def _extract_retry_after(error_message: str) -> int:
|
||||
if not error_message:
|
||||
return 60
|
||||
match = re.search(r"retry after (\d+)s", str(error_message))
|
||||
if match:
|
||||
return max(1, int(match.group(1)))
|
||||
return 60
|
||||
|
||||
@staticmethod
|
||||
async def download_model_images_with_tracking(model_hash, model_name, model_images, model_dir, optimize, downloader):
|
||||
"""Download images for a single model with tracking of failed image URLs
|
||||
|
||||
Returns:
|
||||
tuple: (success, is_stale_metadata, failed_images) - whether download was successful, whether metadata is stale, list of failed image URLs
|
||||
"""
|
||||
model_success = True
|
||||
failed_images = []
|
||||
|
||||
rate_limited_images = []
|
||||
any_successful_download = False
|
||||
|
||||
for i, image in enumerate(model_images):
|
||||
image_url = image.get('url')
|
||||
if not image_url:
|
||||
@@ -221,64 +228,110 @@ class ExampleImagesProcessor:
|
||||
original_url = image_url
|
||||
if optimize and 'civitai.com' in image_url:
|
||||
image_url = ExampleImagesProcessor.get_civitai_optimized_url(image_url)
|
||||
|
||||
# Download the file first to determine the actual file type
|
||||
try:
|
||||
logger.debug(f"Downloading media file {i} for {model_name}")
|
||||
|
||||
# Download using the unified downloader with headers
|
||||
success, content, headers = await downloader.download_to_memory(
|
||||
|
||||
async def _attempt_download() -> tuple:
|
||||
logger.debug("Downloading media file %s for %s", i, model_name)
|
||||
return await downloader.download_to_memory(
|
||||
image_url,
|
||||
use_auth=False, # Example images don't need auth
|
||||
return_headers=True
|
||||
use_auth=False,
|
||||
return_headers=True,
|
||||
)
|
||||
|
||||
|
||||
try:
|
||||
success, content, headers = await _attempt_download()
|
||||
|
||||
if success:
|
||||
# Determine file extension from content or headers
|
||||
media_ext = ExampleImagesProcessor._get_file_extension_from_content_or_headers(
|
||||
content, headers, original_url, image.get("type")
|
||||
)
|
||||
|
||||
# Check if the detected file type is supported
|
||||
is_image = media_ext in SUPPORTED_MEDIA_EXTENSIONS['images']
|
||||
is_video = media_ext in SUPPORTED_MEDIA_EXTENSIONS['videos']
|
||||
|
||||
|
||||
if not (is_image or is_video):
|
||||
logger.debug(f"Skipping unsupported file type: {media_ext}")
|
||||
logger.debug("Skipping unsupported file type: %s", media_ext)
|
||||
continue
|
||||
|
||||
# Use 0-based indexing with the detected extension
|
||||
|
||||
save_filename = f"image_{i}{media_ext}"
|
||||
save_path = os.path.join(model_dir, save_filename)
|
||||
|
||||
# Check if already downloaded
|
||||
|
||||
if os.path.exists(save_path):
|
||||
logger.debug(f"File already exists: {save_path}")
|
||||
logger.debug("File already exists: %s", save_path)
|
||||
continue
|
||||
|
||||
# Save the file
|
||||
|
||||
with open(save_path, 'wb') as f:
|
||||
f.write(content)
|
||||
|
||||
any_successful_download = True
|
||||
|
||||
elif ExampleImagesProcessor._is_not_found_error(content):
|
||||
error_msg = f"Failed to download file: {image_url}, status code: 404 - Model metadata might be stale"
|
||||
logger.warning(error_msg)
|
||||
model_success = False # Mark the model as failed due to 404 error
|
||||
failed_images.append(image_url) # Track failed URL
|
||||
# Return early to trigger metadata refresh attempt
|
||||
return False, True, failed_images # (success, is_metadata_stale, failed_images)
|
||||
model_success = False
|
||||
failed_images.append(image_url)
|
||||
return False, True, failed_images, rate_limited_images
|
||||
|
||||
elif "Rate limited (429)" in str(content):
|
||||
max_attempts = 3
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
wait = ExampleImagesProcessor._extract_retry_after(str(content)) * (2 ** (attempt - 1))
|
||||
logger.warning(
|
||||
"Rate limited (429) for %s, retry %d/%d after %ds",
|
||||
image_url, attempt, max_attempts, wait,
|
||||
)
|
||||
await asyncio.sleep(wait)
|
||||
|
||||
success, content, headers = await _attempt_download()
|
||||
if success:
|
||||
media_ext = ExampleImagesProcessor._get_file_extension_from_content_or_headers(
|
||||
content, headers, original_url, image.get("type")
|
||||
)
|
||||
is_image = media_ext in SUPPORTED_MEDIA_EXTENSIONS['images']
|
||||
is_video = media_ext in SUPPORTED_MEDIA_EXTENSIONS['videos']
|
||||
|
||||
if not (is_image or is_video):
|
||||
logger.debug("Skipping unsupported file type: %s", media_ext)
|
||||
break
|
||||
|
||||
save_filename = f"image_{i}{media_ext}"
|
||||
save_path = os.path.join(model_dir, save_filename)
|
||||
if os.path.exists(save_path):
|
||||
logger.debug("File already exists: %s", save_path)
|
||||
break
|
||||
|
||||
with open(save_path, 'wb') as f:
|
||||
f.write(content)
|
||||
any_successful_download = True
|
||||
break
|
||||
elif "Rate limited (429)" in str(content):
|
||||
continue
|
||||
elif ExampleImagesProcessor._is_not_found_error(content):
|
||||
logger.warning("Failed to download file: %s, status code: 404", image_url)
|
||||
model_success = False
|
||||
failed_images.append(image_url)
|
||||
break
|
||||
else:
|
||||
logger.warning("Failed to download file: %s, error: %s", image_url, content)
|
||||
model_success = False
|
||||
failed_images.append(image_url)
|
||||
break
|
||||
else:
|
||||
logger.warning(
|
||||
"Giving up on %s after %d retries due to rate limiting",
|
||||
image_url, max_attempts,
|
||||
)
|
||||
rate_limited_images.append(image_url)
|
||||
model_success = False
|
||||
else:
|
||||
error_msg = f"Failed to download file: {image_url}, error: {content}"
|
||||
logger.warning(error_msg)
|
||||
model_success = False # Mark the model as failed
|
||||
failed_images.append(image_url) # Track failed URL
|
||||
model_success = False
|
||||
failed_images.append(image_url)
|
||||
except Exception as e:
|
||||
error_msg = f"Error downloading file {image_url}: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
model_success = False # Mark the model as failed
|
||||
failed_images.append(image_url) # Track failed URL
|
||||
|
||||
return model_success, False, failed_images # (success, is_metadata_stale, failed_images)
|
||||
model_success = False
|
||||
failed_images.append(image_url)
|
||||
|
||||
return any_successful_download or model_success, False, failed_images, rate_limited_images
|
||||
|
||||
@staticmethod
|
||||
async def process_local_examples(model_file_path, model_file_name, model_name, model_dir, optimize):
|
||||
|
||||
@@ -12,6 +12,7 @@ from platformdirs import user_config_dir
|
||||
|
||||
|
||||
APP_NAME = "ComfyUI-LoRA-Manager"
|
||||
_LM_PORTABLE_ENV = "LORA_MANAGER_PORTABLE"
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -100,7 +101,11 @@ def ensure_settings_file(logger: Optional[logging.Logger] = None) -> str:
|
||||
|
||||
|
||||
def _should_use_portable_settings(path: str, logger: logging.Logger) -> bool:
|
||||
"""Return ``True`` when the repository settings file enables portable mode."""
|
||||
"""Return ``True`` when the env var forces it or the settings file enables it."""
|
||||
|
||||
if os.environ.get(_LM_PORTABLE_ENV, "0") == "1":
|
||||
logger.debug("Portable mode enabled via %s", _LM_PORTABLE_ENV)
|
||||
return True
|
||||
|
||||
if not os.path.exists(path):
|
||||
return False
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "comfyui-lora-manager"
|
||||
description = "Revolutionize your workflow with the ultimate LoRA companion for ComfyUI!"
|
||||
version = "1.1.6"
|
||||
version = "1.1.7"
|
||||
license = {file = "LICENSE"}
|
||||
dependencies = [
|
||||
"aiohttp",
|
||||
|
||||
@@ -40,6 +40,12 @@
|
||||
margin: 3px 0;
|
||||
}
|
||||
|
||||
.context-menu-item.disabled {
|
||||
opacity: 0.4;
|
||||
cursor: not-allowed;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.context-menu-item.delete-item {
|
||||
color: var(--danger-color);
|
||||
}
|
||||
|
||||
@@ -577,13 +577,14 @@
|
||||
border: 1px solid var(--border-color);
|
||||
border-radius: var(--border-radius-sm);
|
||||
cursor: pointer;
|
||||
transition: var(--transition-base);
|
||||
transition: var(--transition-base), box-shadow var(--transition-fast), transform var(--transition-fast);
|
||||
background: var(--bg-color);
|
||||
}
|
||||
|
||||
.file-option:hover {
|
||||
border-color: var(--lora-accent);
|
||||
box-shadow: var(--shadow-sm);
|
||||
box-shadow: var(--shadow-md);
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.file-option.selected {
|
||||
@@ -698,10 +699,25 @@
|
||||
color: var(--lora-accent);
|
||||
}
|
||||
|
||||
/* Batch Preview List */
|
||||
/* BUG 1 FIX: Single scrollbar — modal-content becomes a flex column so the
|
||||
batch preview step can flex; the list scrolls instead of the modal-content. */
|
||||
#downloadModal .modal-content {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
#batchPreviewStep {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
min-height: 0;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
/* Batch Preview List — no max-height; flexes inside #batchPreviewStep */
|
||||
.batch-preview-list {
|
||||
max-height: 400px;
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
min-height: 0;
|
||||
margin: var(--space-2) 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
@@ -859,6 +875,8 @@
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 1;
|
||||
backdrop-filter: blur(8px);
|
||||
-webkit-backdrop-filter: blur(8px);
|
||||
}
|
||||
|
||||
.batch-preview-select-all input[type="checkbox"] {
|
||||
@@ -884,3 +902,100 @@
|
||||
[data-theme="dark"] .batch-preview-select-all {
|
||||
background: var(--lora-surface);
|
||||
}
|
||||
|
||||
/* FEATURE 2: HF repo grouping — collapsible groups by repo */
|
||||
.batch-preview-group {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
background: var(--surface-base);
|
||||
}
|
||||
|
||||
.batch-preview-group-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 10px 12px;
|
||||
background: var(--color-accent-subtle);
|
||||
border-bottom: 1px solid var(--color-accent-border);
|
||||
cursor: pointer;
|
||||
user-select: none;
|
||||
transition: background var(--transition-fast);
|
||||
}
|
||||
|
||||
.batch-preview-group-header:hover {
|
||||
background: oklch(var(--color-accent-l) var(--color-accent-c) var(--color-accent-h) / 0.18);
|
||||
}
|
||||
|
||||
.batch-preview-group-toggle {
|
||||
width: 14px;
|
||||
font-size: 0.75em;
|
||||
color: var(--text-color);
|
||||
opacity: 0.7;
|
||||
transition: transform var(--transition-fast);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.batch-preview-group-toggle.expanded {
|
||||
transform: rotate(90deg);
|
||||
}
|
||||
|
||||
.batch-preview-group-name {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
font-weight: 600;
|
||||
color: var(--text-color);
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
font-size: 0.95em;
|
||||
}
|
||||
|
||||
.batch-preview-group-count {
|
||||
font-size: 0.8em;
|
||||
color: var(--text-color);
|
||||
opacity: 0.7;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.batch-preview-group-select-all {
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
cursor: pointer;
|
||||
accent-color: var(--lora-accent);
|
||||
flex-shrink: 0;
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.batch-preview-group-body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1px;
|
||||
background: var(--border-color);
|
||||
overflow: hidden;
|
||||
max-height: 0;
|
||||
opacity: 0;
|
||||
transition: max-height 0.35s ease, opacity 0.2s ease;
|
||||
}
|
||||
|
||||
.batch-preview-group-body.expanded {
|
||||
opacity: 1;
|
||||
max-height: 9999px; /* rest state: content visible; JS inline style overrides during transitions */
|
||||
}
|
||||
|
||||
/* Dark theme overrides for group styles */
|
||||
[data-theme="dark"] .batch-preview-group {
|
||||
background: var(--surface-base);
|
||||
}
|
||||
|
||||
[data-theme="dark"] .batch-preview-group-header {
|
||||
background: var(--color-accent-subtle);
|
||||
}
|
||||
|
||||
[data-theme="dark"] .batch-preview-group-header:hover {
|
||||
background: oklch(var(--color-accent-l) var(--color-accent-c) var(--color-accent-h) / 0.22);
|
||||
}
|
||||
|
||||
[data-theme="dark"] .batch-preview-group-body {
|
||||
background: var(--border-color);
|
||||
}
|
||||
|
||||
@@ -21,18 +21,22 @@
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.input-group {
|
||||
#relinkCivitaiModal .input-group,
|
||||
#linkHfModal .input-group {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
margin-bottom: var(--space-2);
|
||||
}
|
||||
|
||||
.input-group label {
|
||||
#relinkCivitaiModal .input-group label,
|
||||
#linkHfModal .input-group label {
|
||||
margin-bottom: var(--space-1);
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.input-group input {
|
||||
#relinkCivitaiModal .input-group input,
|
||||
#linkHfModal .input-group input {
|
||||
width: auto;
|
||||
padding: 8px 12px;
|
||||
border-radius: var(--border-radius-xs);
|
||||
border: 1px solid var(--border-color);
|
||||
|
||||
@@ -1562,6 +1562,29 @@ input:checked + .toggle-slider:before {
|
||||
box-shadow: 0 0 0 2px rgba(var(--lora-accent-rgb, 79, 70, 229), 0.1);
|
||||
}
|
||||
|
||||
.extra-folder-path-row .path-controls .extra-folder-path-input.has-error {
|
||||
border-color: var(--lora-error);
|
||||
background-color: rgba(220, 53, 69, 0.08);
|
||||
background-color: rgba(from var(--lora-error) r g b / 0.08);
|
||||
}
|
||||
|
||||
.extra-folder-path-row .path-controls .extra-folder-path-input.has-error:focus {
|
||||
box-shadow: 0 0 0 2px rgba(220, 53, 69, 0.15);
|
||||
box-shadow: 0 0 0 2px rgba(from var(--lora-error) r g b / 0.15);
|
||||
}
|
||||
|
||||
.extra-folder-path-error {
|
||||
color: var(--lora-error);
|
||||
font-size: 0.8em;
|
||||
margin-top: 4px;
|
||||
line-height: 1.4;
|
||||
display: none;
|
||||
}
|
||||
|
||||
.extra-folder-path-error.visible {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.extra-folder-path-row .path-controls .remove-path-btn {
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
@@ -1592,3 +1615,45 @@ input:checked + .toggle-slider:before {
|
||||
animation: settings-highlight-pulse 1.5s ease-in-out 3;
|
||||
border-radius: var(--border-radius-xs);
|
||||
}
|
||||
|
||||
/* ---- Combobox panel for AI Provider settings ---- */
|
||||
/* The panel is appended to <body> by Combobox.js and positioned relative to
|
||||
the enhanced <input>. Styles reuse settings-modal CSS variables. */
|
||||
|
||||
.lm-combobox-panel {
|
||||
position: absolute;
|
||||
z-index: 10002;
|
||||
max-height: 240px;
|
||||
overflow-y: auto;
|
||||
background: var(--lora-surface, #2a2a2a);
|
||||
border: 1px solid var(--border-color, rgba(255, 255, 255, 0.12));
|
||||
border-radius: var(--border-radius-xs, 6px);
|
||||
box-shadow: var(--shadow-elevated, 0 6px 18px rgba(0, 0, 0, 0.45));
|
||||
font-size: 0.95em;
|
||||
color: var(--text-color, rgba(226, 232, 240, 0.9));
|
||||
padding: 4px 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
.lm-combobox-option {
|
||||
padding: 6px 12px;
|
||||
cursor: pointer;
|
||||
user-select: none;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.lm-combobox-option:hover,
|
||||
.lm-combobox-option.is-active {
|
||||
background: rgba(from var(--lora-accent) r g b / 0.2);
|
||||
color: var(--lora-accent);
|
||||
}
|
||||
|
||||
.lm-combobox-empty {
|
||||
padding: 8px 12px;
|
||||
color: var(--text-color);
|
||||
opacity: 0.45;
|
||||
font-style: italic;
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
@@ -112,6 +112,18 @@ export class BaseModelApiClient {
|
||||
}
|
||||
}
|
||||
|
||||
async cancelDownload(downloadId) {
|
||||
try {
|
||||
const response = await fetch(
|
||||
`${DOWNLOAD_ENDPOINTS.cancelGet}?download_id=${encodeURIComponent(downloadId)}`
|
||||
);
|
||||
return await response.json();
|
||||
} catch (error) {
|
||||
console.error('Error cancelling download:', error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
async loadMoreWithVirtualScroll(resetPage = false, updateFolders = false) {
|
||||
const pageState = this.getPageState();
|
||||
|
||||
|
||||
@@ -0,0 +1,394 @@
|
||||
// Combobox.js — Reusable dropdown-suggestion + free-text input component.
|
||||
//
|
||||
// Enhances an existing <input> element with a dropdown panel that merges static
|
||||
// `presets` with asynchronously fetched options (`fetchOptions`). The input
|
||||
// remains a free-text field — selecting a dropdown option is optional, the
|
||||
// user can always type an arbitrary value.
|
||||
//
|
||||
// Zero dependencies: pure DOM manipulation. Exported on `window.Combobox`
|
||||
// so non-module callers can instantiate it, and as a named ES module export
|
||||
// for callers that import it directly.
|
||||
//
|
||||
// Usage:
|
||||
// const box = new Combobox(inputEl, {
|
||||
// presets: ['masterpiece', 'best quality'],
|
||||
// fetchOptions: async (q) => await fetchSuggestions(q),
|
||||
// placeholder: 'Type a value…',
|
||||
// onSelect: (value) => console.log('chose', value),
|
||||
// });
|
||||
// box.updatePresets(['new', 'presets']);
|
||||
// box.setValue('masterpiece');
|
||||
|
||||
const DEBOUNCE_MS = 300;
|
||||
|
||||
export class Combobox {
|
||||
/**
|
||||
* @param {HTMLInputElement} inputElement Existing <input> to enhance.
|
||||
* @param {Object} options
|
||||
* @param {string[]} [options.presets=[]] Static preset values shown in dropdown.
|
||||
* @param {(inputValue: string) => Promise<string[]>} [options.fetchOptions]
|
||||
* Async function returning dynamic suggestions for the current input.
|
||||
* @param {string} [options.placeholder] Placeholder text for the empty state.
|
||||
* @param {(value: string) => void} [options.onSelect] Callback when an option is chosen.
|
||||
*/
|
||||
constructor(inputElement, options = {}) {
|
||||
if (!inputElement || inputElement.tagName !== 'INPUT') {
|
||||
console.error('Combobox: expected an <input> element');
|
||||
return;
|
||||
}
|
||||
|
||||
this.input = inputElement;
|
||||
this.presets = Array.isArray(options.presets) ? [...options.presets] : [];
|
||||
this.fetchOptions = typeof options.fetchOptions === 'function' ? options.fetchOptions : null;
|
||||
this.placeholder = options.placeholder || '';
|
||||
this.onSelect = typeof options.onSelect === 'function' ? options.onSelect : null;
|
||||
|
||||
// Internal state
|
||||
this._isOpen = false;
|
||||
this._activeIndex = -1;
|
||||
this._renderedOptions = []; // current visible option strings (de-duplicated, merged)
|
||||
this._fetchToken = 0; // guards against out-of-order async fetch results
|
||||
this._fetchTimer = null;
|
||||
this._suppressInputOpen = false; // guards setValue() from reopening the dropdown
|
||||
|
||||
this._buildDropdown();
|
||||
this._bindEvents();
|
||||
}
|
||||
|
||||
// ---- public API ----
|
||||
|
||||
/**
|
||||
* Replace the preset list. Re-renders the dropdown if it is open.
|
||||
* @param {string[]} presets
|
||||
* @returns {void}
|
||||
*/
|
||||
updatePresets(presets) {
|
||||
this.presets = Array.isArray(presets) ? [...presets] : [];
|
||||
if (this._isOpen) {
|
||||
this._refresh();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the input value programmatically without triggering the dropdown
|
||||
* or firing synthetic events.
|
||||
* @param {string} value
|
||||
* @returns {void}
|
||||
*/
|
||||
setValue(value) {
|
||||
const prev = this._suppressInputOpen;
|
||||
this._suppressInputOpen = true;
|
||||
this.input.value = value ?? '';
|
||||
this._suppressInputOpen = prev;
|
||||
if (this._isOpen) {
|
||||
this._refresh();
|
||||
}
|
||||
}
|
||||
|
||||
// ---- build ----
|
||||
|
||||
_buildDropdown() {
|
||||
const panel = document.createElement('div');
|
||||
panel.className = 'lm-combobox-panel';
|
||||
panel.setAttribute('role', 'listbox');
|
||||
panel.style.display = 'none';
|
||||
// Append to <body> so the panel is never clipped by an overflow:hidden
|
||||
// ancestor; positioning is recomputed on each open.
|
||||
document.body.appendChild(panel);
|
||||
this.panel = panel;
|
||||
|
||||
if (this.placeholder) {
|
||||
this.input.setAttribute('placeholder', this.placeholder);
|
||||
}
|
||||
this.input.setAttribute('autocomplete', 'off');
|
||||
this.input.setAttribute('role', 'combobox');
|
||||
this.input.setAttribute('aria-autocomplete', 'list');
|
||||
this.input.setAttribute('aria-expanded', 'false');
|
||||
}
|
||||
|
||||
// ---- event wiring ----
|
||||
|
||||
_bindEvents() {
|
||||
this.input.addEventListener('focus', () => {
|
||||
if (this._suppressInputOpen) return;
|
||||
this._open();
|
||||
});
|
||||
|
||||
this.input.addEventListener('input', () => {
|
||||
if (this._suppressInputOpen) return;
|
||||
this._open(); // no-op if already open
|
||||
this._refresh(); // re-filter by current input value
|
||||
this._scheduleFetch();
|
||||
});
|
||||
|
||||
this.input.addEventListener('keydown', (event) => this._onKeyDown(event));
|
||||
|
||||
// Click an option (delegated)
|
||||
this.panel.addEventListener('click', (event) => {
|
||||
const item = event.target.closest('.lm-combobox-option');
|
||||
if (!item) return;
|
||||
const value = item.dataset.value;
|
||||
if (value !== undefined) {
|
||||
this._choose(value);
|
||||
}
|
||||
});
|
||||
|
||||
// Hover updates the active highlight so keyboard + mouse stay in sync.
|
||||
this.panel.addEventListener('mouseover', (event) => {
|
||||
const item = event.target.closest('.lm-combobox-option');
|
||||
if (!item) return;
|
||||
const idx = Number(item.dataset.index);
|
||||
if (!Number.isNaN(idx)) {
|
||||
this._setActiveIndex(idx);
|
||||
}
|
||||
});
|
||||
|
||||
// Click outside closes the dropdown.
|
||||
this._outsideClickHandler = (event) => {
|
||||
if (this._isOpen && !this.input.contains(event.target) && !this.panel.contains(event.target)) {
|
||||
this._close();
|
||||
}
|
||||
};
|
||||
document.addEventListener('mousedown', this._outsideClickHandler);
|
||||
|
||||
// Reposition on viewport changes while open.
|
||||
this._resizeHandler = () => {
|
||||
if (this._isOpen) this._position();
|
||||
};
|
||||
window.addEventListener('resize', this._resizeHandler);
|
||||
window.addEventListener('scroll', this._resizeHandler, true);
|
||||
}
|
||||
|
||||
// ---- keyboard ----
|
||||
|
||||
_onKeyDown(event) {
|
||||
if (!this._isOpen) {
|
||||
if (event.key === 'ArrowDown') {
|
||||
event.preventDefault();
|
||||
this._open();
|
||||
this._setActiveIndex(0);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
switch (event.key) {
|
||||
case 'ArrowDown':
|
||||
event.preventDefault();
|
||||
this._setActiveIndex(this._activeIndex + 1);
|
||||
break;
|
||||
|
||||
case 'ArrowUp':
|
||||
event.preventDefault();
|
||||
this._setActiveIndex(this._activeIndex - 1);
|
||||
break;
|
||||
|
||||
case 'Enter':
|
||||
// Only intercept Enter to pick an option when one is actively
|
||||
// highlighted; otherwise let the input's default behavior
|
||||
// (form submit / free-text commit) proceed.
|
||||
if (this._activeIndex >= 0 && this._activeIndex < this._renderedOptions.length) {
|
||||
event.preventDefault();
|
||||
this._choose(this._renderedOptions[this._activeIndex]);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'Escape':
|
||||
event.preventDefault();
|
||||
this._close();
|
||||
this.input.focus();
|
||||
break;
|
||||
|
||||
case 'Tab':
|
||||
// Allow normal tab navigation; just close the panel.
|
||||
this._close();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- open / close ----
|
||||
|
||||
_open() {
|
||||
if (this._isOpen) return;
|
||||
this._isOpen = true;
|
||||
this.panel.style.display = 'block';
|
||||
this.input.setAttribute('aria-expanded', 'true');
|
||||
// On open, render ALL presets — do not filter by the current input
|
||||
// value. Filtering on the input event is handled separately.
|
||||
this._render(this.presets);
|
||||
this._position();
|
||||
}
|
||||
|
||||
_close() {
|
||||
if (!this._isOpen) return;
|
||||
this._isOpen = false;
|
||||
this.panel.style.display = 'none';
|
||||
this.input.setAttribute('aria-expanded', 'false');
|
||||
this._activeIndex = -1;
|
||||
this._cancelFetch();
|
||||
}
|
||||
|
||||
_position() {
|
||||
const rect = this.input.getBoundingClientRect();
|
||||
const panelHeight = this.panel.offsetHeight;
|
||||
const viewportHeight = window.innerHeight;
|
||||
const spaceBelow = viewportHeight - rect.bottom;
|
||||
const spaceAbove = rect.top;
|
||||
|
||||
// Flip above the input when there is more room there.
|
||||
const placeAbove = spaceBelow < panelHeight && spaceAbove > spaceBelow;
|
||||
const top = placeAbove
|
||||
? rect.top + window.scrollY - panelHeight
|
||||
: rect.bottom + window.scrollY;
|
||||
|
||||
this.panel.style.top = `${Math.max(0, top)}px`;
|
||||
this.panel.style.left = `${rect.left + window.scrollX}px`;
|
||||
this.panel.style.minWidth = `${rect.width}px`;
|
||||
}
|
||||
|
||||
// ---- rendering ----
|
||||
|
||||
/** Render a list of strings into the panel. */
|
||||
_render(items) {
|
||||
this._renderedOptions = items;
|
||||
this.panel.innerHTML = '';
|
||||
if (items.length === 0) {
|
||||
const empty = document.createElement('div');
|
||||
empty.className = 'lm-combobox-empty';
|
||||
empty.textContent = this.placeholder ? this.placeholder : 'No options';
|
||||
this.panel.appendChild(empty);
|
||||
this._activeIndex = -1;
|
||||
return;
|
||||
}
|
||||
|
||||
const fragment = document.createDocumentFragment();
|
||||
items.forEach((opt, idx) => {
|
||||
const item = document.createElement('div');
|
||||
item.className = 'lm-combobox-option';
|
||||
item.setAttribute('role', 'option');
|
||||
item.dataset.value = opt;
|
||||
item.dataset.index = String(idx);
|
||||
item.textContent = opt;
|
||||
if (idx === this._activeIndex) {
|
||||
item.classList.add('is-active');
|
||||
}
|
||||
fragment.appendChild(item);
|
||||
});
|
||||
this.panel.appendChild(fragment);
|
||||
|
||||
if (this._activeIndex >= items.length) {
|
||||
this._setActiveIndex(items.length - 1);
|
||||
}
|
||||
}
|
||||
|
||||
/** Filter presets by current input value and re-render. */
|
||||
_refresh() {
|
||||
const value = this.input.value;
|
||||
const filtered = this._filterPresets(value);
|
||||
const merged = this._mergeUnique(filtered, this._fetchedOptions || []);
|
||||
this._render(merged);
|
||||
}
|
||||
|
||||
_filterPresets(value) {
|
||||
const v = (value || '').toLowerCase();
|
||||
if (!v) return [...this.presets];
|
||||
return this.presets.filter((p) => String(p).toLowerCase().startsWith(v));
|
||||
}
|
||||
|
||||
_mergeUnique(...lists) {
|
||||
const seen = new Set();
|
||||
const out = [];
|
||||
for (const list of lists) {
|
||||
for (const item of list) {
|
||||
const key = String(item);
|
||||
if (!seen.has(key)) {
|
||||
seen.add(key);
|
||||
out.push(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
_setActiveIndex(idx) {
|
||||
const max = this._renderedOptions.length - 1;
|
||||
const clamped = Math.max(-1, Math.min(max, idx));
|
||||
this._activeIndex = clamped;
|
||||
// Update DOM classes without full re-render.
|
||||
const items = this.panel.querySelectorAll('.lm-combobox-option');
|
||||
items.forEach((el, i) => {
|
||||
el.classList.toggle('is-active', i === clamped);
|
||||
});
|
||||
// Scroll the active item into view inside the panel.
|
||||
if (clamped >= 0 && items[clamped]) {
|
||||
items[clamped].scrollIntoView({ block: 'nearest' });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove the panel from the DOM and detach event listeners.
|
||||
* Call this before discarding the Combobox instance.
|
||||
*/
|
||||
destroy() {
|
||||
this._close();
|
||||
if (this.panel && this.panel.parentNode) {
|
||||
this.panel.parentNode.removeChild(this.panel);
|
||||
}
|
||||
document.removeEventListener('mousedown', this._outsideClickHandler);
|
||||
window.removeEventListener('resize', this._resizeHandler);
|
||||
window.removeEventListener('scroll', this._resizeHandler, true);
|
||||
}
|
||||
|
||||
_choose(value) {
|
||||
this.input.value = value;
|
||||
this._close();
|
||||
if (typeof this.onSelect === 'function') {
|
||||
this.onSelect(value);
|
||||
}
|
||||
// Re-focus without reopening the dropdown.
|
||||
this._suppressInputOpen = true;
|
||||
this.input.focus();
|
||||
this._suppressInputOpen = false;
|
||||
}
|
||||
|
||||
// ---- async fetch (debounced) ----
|
||||
|
||||
_scheduleFetch() {
|
||||
if (!this.fetchOptions) return;
|
||||
this._cancelFetch();
|
||||
this._fetchTimer = setTimeout(() => {
|
||||
this._fetchTimer = null;
|
||||
this._runFetch();
|
||||
}, DEBOUNCE_MS);
|
||||
}
|
||||
|
||||
_cancelFetch() {
|
||||
if (this._fetchTimer) {
|
||||
clearTimeout(this._fetchTimer);
|
||||
this._fetchTimer = null;
|
||||
}
|
||||
this._fetchToken++; // invalidate any in-flight result
|
||||
}
|
||||
|
||||
async _runFetch() {
|
||||
if (!this.fetchOptions) return;
|
||||
const token = this._fetchToken;
|
||||
const value = this.input.value;
|
||||
let results;
|
||||
try {
|
||||
results = await this.fetchOptions(value);
|
||||
} catch (err) {
|
||||
console.error('Combobox fetchOptions error:', err);
|
||||
results = [];
|
||||
}
|
||||
// Stale guard: a newer fetch or close superseded this one.
|
||||
if (token !== this._fetchToken || !this._isOpen) return;
|
||||
this._fetchedOptions = Array.isArray(results) ? results : [];
|
||||
this._refresh();
|
||||
}
|
||||
}
|
||||
|
||||
// Expose for non-module callers (templates load via <script type="module">,
|
||||
// but some widget code reads globals off `window`).
|
||||
if (typeof window !== 'undefined') {
|
||||
window.Combobox = Combobox;
|
||||
}
|
||||
@@ -27,8 +27,9 @@ export class BaseContextMenu {
|
||||
const menuItem = e.target.closest('.context-menu-item');
|
||||
if (!menuItem || !this.currentCard) return;
|
||||
|
||||
// Ignore clicks on submenu trigger (has-submenu parent)
|
||||
// Ignore clicks on submenu trigger (has-submenu parent) or disabled items
|
||||
if (menuItem.classList.contains('has-submenu')) return;
|
||||
if (menuItem.classList.contains('disabled')) return;
|
||||
|
||||
const action = menuItem.dataset.action;
|
||||
if (!action) return;
|
||||
|
||||
@@ -274,7 +274,7 @@ export class BulkContextMenu extends BaseContextMenu {
|
||||
case 'resume-metadata-refresh':
|
||||
bulkManager.setSkipMetadataRefresh(false);
|
||||
break;
|
||||
case 'enrich-hf-agent-bulk':
|
||||
case 'enrich-hf-llm-bulk':
|
||||
this.enrichBulkWithAgent();
|
||||
break;
|
||||
case 'delete-all':
|
||||
@@ -377,7 +377,6 @@ export class BulkContextMenu extends BaseContextMenu {
|
||||
|
||||
const { agentManager } = await import('../../managers/AgentManager.js');
|
||||
|
||||
// Check if LLM is configured
|
||||
const configured = await agentManager.isLlmConfigured();
|
||||
if (!configured) {
|
||||
showToast('toast.agent.llmNotConfigured', {}, 'warning');
|
||||
@@ -386,41 +385,66 @@ export class BulkContextMenu extends BaseContextMenu {
|
||||
|
||||
const modelPaths = [...state.selectedModels];
|
||||
|
||||
// Connect WebSocket for progress
|
||||
agentManager.connect();
|
||||
|
||||
// Set up one-time completion handler
|
||||
const progressUI = state.loadingManager.showEnhancedProgress(
|
||||
`Enriching metadata for ${modelPaths.length} models...`
|
||||
);
|
||||
|
||||
function cleanupCallbacks() {
|
||||
const pIdx = agentManager.progressCallbacks.indexOf(onProgress);
|
||||
if (pIdx >= 0) agentManager.progressCallbacks.splice(pIdx, 1);
|
||||
const cIdx = agentManager.completeCallbacks.indexOf(onComplete);
|
||||
if (cIdx >= 0) agentManager.completeCallbacks.splice(cIdx, 1);
|
||||
const eIdx = agentManager.errorCallbacks.indexOf(onError);
|
||||
if (eIdx >= 0) agentManager.errorCallbacks.splice(eIdx, 1);
|
||||
}
|
||||
|
||||
const onProgress = (data) => {
|
||||
if (data.status === 'processing' && data.current_path && data.updated_data && Object.keys(data.updated_data).length > 0) {
|
||||
if (state.virtualScroller?.updateSingleItem) {
|
||||
state.virtualScroller.updateSingleItem(data.current_path, data.updated_data);
|
||||
}
|
||||
const pct = data.total > 0 ? Math.floor((data.processed / data.total) * 100) : 0;
|
||||
const name = data.current_path.split('/').pop();
|
||||
progressUI.updateProgress(pct, name, `Processing ${data.processed}/${data.total}: ${name}`);
|
||||
}
|
||||
};
|
||||
agentManager.onProgress(onProgress);
|
||||
|
||||
const onComplete = (data) => {
|
||||
const idx = agentManager.completeCallbacks.indexOf(onComplete);
|
||||
if (idx >= 0) agentManager.completeCallbacks.splice(idx, 1);
|
||||
cleanupCallbacks();
|
||||
|
||||
if (data.status === 'completed') {
|
||||
if (state.bulkMode) bulkManager.toggleBulkMode();
|
||||
progressUI.complete(data.summary || 'Enrich complete');
|
||||
showToast(
|
||||
'toast.agent.enrichComplete',
|
||||
{ summary: data.summary || 'Done' },
|
||||
'success'
|
||||
);
|
||||
// Soft reload to reflect updated metadata
|
||||
window.location.reload();
|
||||
} else if (data.status === 'error') {
|
||||
showToast(
|
||||
'toast.agent.enrichFailed',
|
||||
{ error: data.error || 'Unknown error' },
|
||||
'error'
|
||||
);
|
||||
}
|
||||
};
|
||||
agentManager.onComplete(onComplete);
|
||||
|
||||
showToast(
|
||||
'toast.agent.enrichStarted',
|
||||
{ count: modelPaths.length },
|
||||
'info'
|
||||
);
|
||||
const onError = (data) => {
|
||||
cleanupCallbacks();
|
||||
if (state.bulkMode) bulkManager.toggleBulkMode();
|
||||
state.loadingManager.hide();
|
||||
showToast(
|
||||
'toast.agent.enrichFailed',
|
||||
{ error: data.error || 'Unknown error' },
|
||||
'error'
|
||||
);
|
||||
};
|
||||
agentManager.onError(onError);
|
||||
|
||||
try {
|
||||
await agentManager.executeSkill('enrich_hf_metadata', modelPaths);
|
||||
} catch (error) {
|
||||
cleanupCallbacks();
|
||||
if (state.bulkMode) bulkManager.toggleBulkMode();
|
||||
state.loadingManager.hide();
|
||||
showToast(
|
||||
'toast.agent.enrichFailed',
|
||||
{ error: error.message },
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { BaseContextMenu } from './BaseContextMenu.js';
|
||||
import { ModelContextMenuMixin } from './ModelContextMenuMixin.js';
|
||||
import { state } from '../../state/index.js';
|
||||
import { getModelApiClient, resetAndReload } from '../../api/modelApiFactory.js';
|
||||
import { copyLoraSyntax, sendLoraToWorkflow, buildLoraSyntax, showToast } from '../../utils/uiHelpers.js';
|
||||
import { showExcludeModal, showDeleteModal } from '../../utils/modalUtils.js';
|
||||
@@ -23,6 +24,17 @@ export class LoraContextMenu extends BaseContextMenu {
|
||||
showMenu(x, y, card) {
|
||||
super.showMenu(x, y, card);
|
||||
this.updateExcludeMenuItem();
|
||||
this.updateEnrichMenuItem(card);
|
||||
}
|
||||
|
||||
updateEnrichMenuItem(card) {
|
||||
const enrichItem = this.menu?.querySelector('[data-action="enrich-hf-llm"]');
|
||||
if (!enrichItem) return;
|
||||
const hasHfUrl = !!card.dataset.hf_url;
|
||||
enrichItem.classList.toggle('disabled', !hasHfUrl);
|
||||
enrichItem.title = hasHfUrl
|
||||
? ''
|
||||
: 'Link this model to a HuggingFace repo first (Link Model \u2192 Link to HuggingFace)';
|
||||
}
|
||||
|
||||
handleMenuAction(action, menuItem) {
|
||||
@@ -63,7 +75,7 @@ export class LoraContextMenu extends BaseContextMenu {
|
||||
case 'refresh-metadata':
|
||||
getModelApiClient().refreshSingleModelMetadata(this.currentCard.dataset.filepath);
|
||||
break;
|
||||
case 'enrich-hf-agent':
|
||||
case 'enrich-hf-llm':
|
||||
this.enrichWithAgent(this.currentCard.dataset.filepath);
|
||||
break;
|
||||
case 'exclude':
|
||||
@@ -78,39 +90,61 @@ export class LoraContextMenu extends BaseContextMenu {
|
||||
async enrichWithAgent(filePath) {
|
||||
const { agentManager } = await import('../../managers/AgentManager.js');
|
||||
|
||||
// Check if LLM is configured
|
||||
const configured = await agentManager.isLlmConfigured();
|
||||
if (!configured) {
|
||||
showToast('toast.agent.llmNotConfigured', {}, 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
// Connect WebSocket for progress
|
||||
agentManager.connect();
|
||||
|
||||
// Set up one-time completion handler
|
||||
const progressUI = state.loadingManager.showEnhancedProgress(
|
||||
'Enriching metadata with AI...'
|
||||
);
|
||||
|
||||
function cleanupCallbacks() {
|
||||
const pIdx = agentManager.progressCallbacks.indexOf(onProgress);
|
||||
if (pIdx >= 0) agentManager.progressCallbacks.splice(pIdx, 1);
|
||||
const cIdx = agentManager.completeCallbacks.indexOf(onComplete);
|
||||
if (cIdx >= 0) agentManager.completeCallbacks.splice(cIdx, 1);
|
||||
const eIdx = agentManager.errorCallbacks.indexOf(onError);
|
||||
if (eIdx >= 0) agentManager.errorCallbacks.splice(eIdx, 1);
|
||||
}
|
||||
|
||||
const onProgress = (data) => {
|
||||
if (data.status === 'processing' && data.current_path && data.updated_data && Object.keys(data.updated_data).length > 0) {
|
||||
if (state.virtualScroller?.updateSingleItem) {
|
||||
state.virtualScroller.updateSingleItem(data.current_path, data.updated_data);
|
||||
}
|
||||
const pct = data.total > 0 ? Math.floor((data.processed / data.total) * 100) : 0;
|
||||
const name = data.current_path.split('/').pop();
|
||||
progressUI.updateProgress(pct, name, `Processing ${name}`);
|
||||
}
|
||||
};
|
||||
agentManager.onProgress(onProgress);
|
||||
|
||||
const onComplete = (data) => {
|
||||
const idx = agentManager.completeCallbacks.indexOf(onComplete);
|
||||
if (idx >= 0) agentManager.completeCallbacks.splice(idx, 1);
|
||||
cleanupCallbacks();
|
||||
|
||||
if (data.status === 'completed') {
|
||||
progressUI.complete(data.summary || 'Enrich complete');
|
||||
showToast('toast.agent.enrichComplete', { summary: data.summary || 'Done' }, 'success');
|
||||
// Soft reload to reflect updated metadata
|
||||
if (typeof resetAndReload === 'function') {
|
||||
resetAndReload();
|
||||
}
|
||||
} else if (data.status === 'error') {
|
||||
showToast('toast.agent.enrichFailed', { error: data.error || 'Unknown error' }, 'error');
|
||||
}
|
||||
};
|
||||
agentManager.onComplete(onComplete);
|
||||
|
||||
// Show progress toast
|
||||
showToast('toast.agent.enrichStarted', {}, 'info');
|
||||
const onError = (data) => {
|
||||
cleanupCallbacks();
|
||||
state.loadingManager.hide();
|
||||
showToast('toast.agent.enrichFailed', { error: data.error || 'Unknown error' }, 'error');
|
||||
};
|
||||
agentManager.onError(onError);
|
||||
|
||||
try {
|
||||
await agentManager.executeSkill('enrich_hf_metadata', [filePath]);
|
||||
} catch (error) {
|
||||
cleanupCallbacks();
|
||||
state.loadingManager.hide();
|
||||
showToast('toast.agent.enrichFailed', { error: error.message }, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,6 +187,74 @@ export const ModelContextMenuMixin = {
|
||||
setTimeout(() => urlInput.focus(), 50);
|
||||
},
|
||||
|
||||
// HuggingFace linking methods
|
||||
showLinkHfModal() {
|
||||
const filePath = this.currentCard.dataset.filepath;
|
||||
if (!filePath) return;
|
||||
|
||||
const confirmBtn = document.getElementById('confirmLinkHfBtn');
|
||||
const urlInput = document.getElementById('hfModelUrl');
|
||||
const errorDiv = document.getElementById('hfModelUrlError');
|
||||
|
||||
if (this._boundLinkHfHandler) {
|
||||
confirmBtn.removeEventListener('click', this._boundLinkHfHandler);
|
||||
}
|
||||
|
||||
this._boundLinkHfHandler = async () => {
|
||||
const hfUrl = urlInput.value.trim();
|
||||
if (!hfUrl) {
|
||||
errorDiv.textContent = 'Please enter a HuggingFace repository URL.';
|
||||
return;
|
||||
}
|
||||
|
||||
const hfPattern = /^https?:\/\/huggingface\.co\/([^/]+\/[^/]+)\/?$/;
|
||||
if (!hfPattern.test(hfUrl)) {
|
||||
errorDiv.textContent = 'Invalid URL format. Expected: https://huggingface.co/user/repo';
|
||||
return;
|
||||
}
|
||||
|
||||
errorDiv.textContent = '';
|
||||
modalManager.closeModal('linkHfModal');
|
||||
|
||||
try {
|
||||
state.loadingManager.showSimpleLoading('Linking to HuggingFace...');
|
||||
|
||||
const response = await fetch('/api/lm/set-hf-url', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ file_path: filePath, hf_url: hfUrl }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errData = await response.json().catch(() => ({}));
|
||||
throw new Error(errData.error || `Request failed: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
if (data.success) {
|
||||
showToast('toast.contextMenu.linkHfSuccess', {}, 'success');
|
||||
await this.resetAndReload();
|
||||
} else {
|
||||
throw new Error(data.error || 'Failed to link model');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error linking model to HuggingFace:', error);
|
||||
showToast('toast.contextMenu.linkHfFailed', { message: error.message }, 'error');
|
||||
} finally {
|
||||
state.loadingManager.hide();
|
||||
}
|
||||
};
|
||||
|
||||
confirmBtn.addEventListener('click', this._boundLinkHfHandler);
|
||||
|
||||
urlInput.value = '';
|
||||
errorDiv.textContent = '';
|
||||
|
||||
modalManager.showModal('linkHfModal');
|
||||
|
||||
setTimeout(() => urlInput.focus(), 50);
|
||||
},
|
||||
|
||||
extractModelVersionId(url) {
|
||||
return extractCivitaiModelUrlParts(url);
|
||||
},
|
||||
@@ -295,6 +363,9 @@ export const ModelContextMenuMixin = {
|
||||
case 'relink-civitai':
|
||||
this.showRelinkCivitaiModal();
|
||||
return true;
|
||||
case 'link-hf':
|
||||
this.showLinkHfModal();
|
||||
return true;
|
||||
case 'set-nsfw':
|
||||
this.showNSFWLevelSelector(null, null, this.currentCard);
|
||||
return true;
|
||||
|
||||
@@ -358,7 +358,7 @@ class RecipeCard {
|
||||
<div class="delete-preview">
|
||||
${isVideo ?
|
||||
`<video src="${previewUrl}" controls muted loop playsinline style="max-width: 100%;"></video>` :
|
||||
`<img src="${previewUrl}" alt="${this.recipe.title}">`
|
||||
`<img src="${previewUrl}" alt="${this.recipe.title}" onerror="this.onerror=null; this.src='/loras_static/images/no-preview.png'">`
|
||||
}
|
||||
</div>
|
||||
<div class="delete-info">
|
||||
|
||||
@@ -757,7 +757,7 @@ class RecipeModal {
|
||||
`<video class="thumbnail-video" autoplay loop muted playsinline>
|
||||
<source src="${lora.preview_url}" type="video/mp4">
|
||||
</video>` :
|
||||
`<img src="${lora.preview_url || '/loras_static/images/no-preview.png'}" alt="LoRA preview">`;
|
||||
`<img src="${lora.preview_url || '/loras_static/images/no-preview.png'}" alt="LoRA preview" onerror="this.onerror=null; this.src='/loras_static/images/no-preview.png'">`;
|
||||
|
||||
let loraItemClass = 'recipe-lora-item';
|
||||
if (existsLocally) {
|
||||
@@ -1606,7 +1606,7 @@ class RecipeModal {
|
||||
<video class="thumbnail-video" autoplay loop muted playsinline>
|
||||
<source src="${previewUrl}" type="video/mp4">
|
||||
</video>
|
||||
` : `<img src="${previewUrl}" alt="Checkpoint preview">`;
|
||||
` : `<img src="${previewUrl}" alt="Checkpoint preview" onerror="this.onerror=null; this.src='/loras_static/images/no-preview.png'">`;
|
||||
|
||||
const badge = existsLocally ? `
|
||||
<div class="local-badge">
|
||||
|
||||
@@ -643,7 +643,7 @@ export function createModelCard(model, modelType) {
|
||||
<div class="card-preview ${shouldBlur ? 'blurred' : ''}">
|
||||
${isVideo ?
|
||||
`<video ${videoAttrs.join(' ')} style="pointer-events: none;"></video>` :
|
||||
`<img src="${versionedPreviewUrl}" alt="${model.model_name}">`
|
||||
`<img src="${versionedPreviewUrl}" alt="${model.model_name}" onerror="this.onerror=null; this.src='/loras_static/images/no-preview.png'">`
|
||||
}
|
||||
<div class="card-header">
|
||||
${shouldBlur ?
|
||||
|
||||
@@ -432,7 +432,7 @@ function renderMediaMarkup(version) {
|
||||
|
||||
return `
|
||||
<div class="version-media">
|
||||
<img src="${escapeHtml(version.previewUrl)}" alt="${escapeHtml(version.name || 'preview')}">
|
||||
<img src="${escapeHtml(version.previewUrl)}" alt="${escapeHtml(version.name || 'preview')}" onerror="this.onerror=null; this.src='/loras_static/images/no-preview.png'">
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
@@ -586,6 +586,7 @@ export function initMediaControlHandlers(container) {
|
||||
const imageMetaRaw = this.dataset.imageMeta;
|
||||
const imageUrl = this.dataset.imageUrl;
|
||||
const imageNsfw = this.dataset.imageNsfw;
|
||||
const imgId = this.dataset.imgId || '';
|
||||
const localPath = this.dataset.localPath || '';
|
||||
const showcaseSection = this.closest('.showcase-section');
|
||||
const modelHash = showcaseSection ? showcaseSection.dataset.modelHash : '';
|
||||
@@ -613,6 +614,7 @@ export function initMediaControlHandlers(container) {
|
||||
meta: imageMeta,
|
||||
url: imageUrl,
|
||||
nsfwLevel: imageNsfw ? parseInt(imageNsfw, 10) : undefined,
|
||||
id: imgId || undefined,
|
||||
},
|
||||
model_hash: modelHash,
|
||||
model_name: modelName || modelHash,
|
||||
|
||||
@@ -174,7 +174,10 @@ function renderMediaItem(img, index, exampleFiles) {
|
||||
const localUrl = localFile ? localFile.path : '';
|
||||
|
||||
// Calculate appropriate aspect ratio
|
||||
const aspectRatio = (img.height / img.width) * 100;
|
||||
// Defensive fallback: 0 width/height → 4:3 default (prevents NaN layout)
|
||||
const safeW = img.width || 4;
|
||||
const safeH = img.height || 3;
|
||||
const aspectRatio = (safeH / safeW) * 100;
|
||||
const containerWidth = 800; // modal content maximum width
|
||||
const minHeightPercent = 40;
|
||||
const maxHeightPercent = (window.innerHeight * 0.6 / containerWidth) * 100;
|
||||
@@ -210,8 +213,8 @@ function renderMediaItem(img, index, exampleFiles) {
|
||||
const model = meta.Model || '';
|
||||
const steps = meta.steps || '';
|
||||
const sampler = meta.sampler || '';
|
||||
const cfgScale = meta.cfgScale || '';
|
||||
const clipSkip = meta.clipSkip || '';
|
||||
const cfgScale = meta.cfg_scale || meta.cfgScale || '';
|
||||
const clipSkip = meta.clip_skip || meta.clipSkip || '';
|
||||
|
||||
// Check if we have any meaningful generation parameters
|
||||
const hasParams = seed || model || steps || sampler || cfgScale || clipSkip;
|
||||
@@ -242,6 +245,7 @@ function renderMediaItem(img, index, exampleFiles) {
|
||||
data-image-url="${img.url || ''}"
|
||||
data-image-nsfw="${img.nsfwLevel ?? ''}"
|
||||
data-image-id="${cdnImageId}"
|
||||
data-img-id="${img.id || ''}"
|
||||
data-local-path="${localFile ? localFile.path : ''}">
|
||||
<i class="fas fa-book-open"></i>
|
||||
</button>
|
||||
|
||||
@@ -15,6 +15,7 @@ import { initTheme, initBackToTop } from './utils/uiHelpers.js';
|
||||
import { initializeInfiniteScroll } from './utils/infiniteScroll.js';
|
||||
import { i18n } from './i18n/index.js';
|
||||
import { onboardingManager } from './managers/OnboardingManager.js';
|
||||
import './components/Combobox.js';
|
||||
import { BulkContextMenu } from './components/ContextMenu/BulkContextMenu.js';
|
||||
import { createPageContextMenu, createGlobalContextMenu } from './components/ContextMenu/index.js';
|
||||
import { initializeEventManagement } from './utils/eventManagementInit.js';
|
||||
|
||||
@@ -165,6 +165,18 @@ class AgentManager {
|
||||
*
|
||||
* @returns {Promise<boolean>}
|
||||
*/
|
||||
_readProviderRequiresKey(providerId) {
|
||||
const script = document.getElementById('llmProviderPresets');
|
||||
if (!script) return true; // safe default
|
||||
try {
|
||||
const presets = JSON.parse(script.textContent);
|
||||
const preset = presets[providerId];
|
||||
return preset ? preset.requires_key !== false : true;
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
async isLlmConfigured() {
|
||||
try {
|
||||
const response = await fetch('/api/lm/settings');
|
||||
@@ -172,8 +184,9 @@ class AgentManager {
|
||||
const data = await response.json();
|
||||
const provider = data.settings?.llm_provider;
|
||||
const hasModel = !!data.settings?.llm_model;
|
||||
const hasKey = !!data.settings?.llm_api_key;
|
||||
return hasModel && (hasKey || provider === 'ollama');
|
||||
const hasKey = !!(data.settings?.llm_api_key_set || data.settings?.llm_api_key);
|
||||
const needsKey = this._readProviderRequiresKey(provider);
|
||||
return hasModel && (hasKey || !needsKey);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { modalManager } from './ModalManager.js';
|
||||
import { showToast } from '../utils/uiHelpers.js';
|
||||
import { showToast, setupAutoNewlineOnPaste } from '../utils/uiHelpers.js';
|
||||
import { translate } from '../utils/i18nHelpers.js';
|
||||
import { WS_ENDPOINTS } from '../api/apiConfig.js';
|
||||
import { getStorageItem, setStorageItem } from '../utils/storageHelpers.js';
|
||||
@@ -43,6 +43,9 @@ export class BatchImportManager {
|
||||
setStorageItem('batch_import_skip_no_metadata', e.target.checked);
|
||||
});
|
||||
}
|
||||
|
||||
// Auto-append newline after pasting a URL in the batch URL input
|
||||
setupAutoNewlineOnPaste('batchUrlInput');
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -633,7 +633,7 @@ export class BulkManager {
|
||||
filePaths.forEach(path => {
|
||||
state.virtualScroller.removeItemByFilePath(path);
|
||||
});
|
||||
this.clearSelection();
|
||||
if (state.bulkMode) this.toggleBulkMode();
|
||||
|
||||
if (window.modelDuplicatesManager) {
|
||||
window.modelDuplicatesManager.updateDuplicatesBadgeAfterRefresh();
|
||||
@@ -763,8 +763,9 @@ export class BulkManager {
|
||||
`Re-import complete: ${completed} re-imported, ${failed} failed`
|
||||
);
|
||||
const { resetAndReload: recipeResetAndReload } = await import('../api/recipeApi.js');
|
||||
recipeResetAndReload(false, { preserveScroll: false });
|
||||
this.clearSelection();
|
||||
if (state.bulkMode) this.toggleBulkMode();
|
||||
recipeResetAndReload(false, { preserveScroll: false });
|
||||
} else {
|
||||
state.loadingManager.hide();
|
||||
showToast('toast.recipes.reimportBulkFailed', {}, 'error');
|
||||
@@ -829,7 +830,7 @@ export class BulkManager {
|
||||
);
|
||||
}
|
||||
|
||||
this.clearSelection();
|
||||
if (state.bulkMode) this.toggleBulkMode();
|
||||
} else {
|
||||
throw new Error(result.error || 'Bulk repair failed');
|
||||
}
|
||||
@@ -874,6 +875,8 @@ export class BulkManager {
|
||||
if (this.isStripVisible) {
|
||||
this.updateThumbnailStrip();
|
||||
}
|
||||
|
||||
if (state.bulkMode) this.toggleBulkMode();
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
@@ -927,6 +930,7 @@ export class BulkManager {
|
||||
showToast('toast.models.bulkUpdatesNone', { type: typeLabel }, 'info');
|
||||
}
|
||||
|
||||
if (state.bulkMode) this.toggleBulkMode();
|
||||
await resetAndReload(false);
|
||||
} catch (error) {
|
||||
console.error('Error checking updates for selected models:', error);
|
||||
@@ -1273,6 +1277,8 @@ export class BulkManager {
|
||||
showToast(toastKey, { count: failCount }, 'warning');
|
||||
}
|
||||
|
||||
if (state.bulkMode) this.toggleBulkMode();
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error during bulk tag operation:', error);
|
||||
const toastKey = mode === 'replace' ? 'toast.models.bulkTagsReplaceFailed' : 'toast.models.bulkTagsAddFailed';
|
||||
@@ -1398,6 +1404,8 @@ export class BulkManager {
|
||||
} else {
|
||||
showToast('toast.models.bulkFavoriteFailed', {}, 'error');
|
||||
}
|
||||
|
||||
if (state.bulkMode) this.toggleBulkMode();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1526,6 +1534,8 @@ export class BulkManager {
|
||||
showToast('toast.models.bulkContentRatingFailed', {}, 'error');
|
||||
}
|
||||
|
||||
if (state.bulkMode) this.toggleBulkMode();
|
||||
|
||||
return successCount > 0;
|
||||
}
|
||||
|
||||
@@ -1580,6 +1590,8 @@ export class BulkManager {
|
||||
} else {
|
||||
showToast('toast.models.skipMetadataRefreshFailed', {}, 'error');
|
||||
}
|
||||
|
||||
if (state.bulkMode) this.toggleBulkMode();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1674,6 +1686,8 @@ export class BulkManager {
|
||||
showToast('toast.models.bulkBaseModelUpdateFailed', {}, 'error');
|
||||
}
|
||||
|
||||
if (state.bulkMode) this.toggleBulkMode();
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error during bulk base model operation:', error);
|
||||
showToast('toast.models.bulkBaseModelUpdateFailed', {}, 'error');
|
||||
@@ -1711,6 +1725,7 @@ export class BulkManager {
|
||||
// Call the auto-organize method with selected file paths
|
||||
await apiClient.autoOrganizeModels(filePaths);
|
||||
|
||||
if (state.bulkMode) this.toggleBulkMode();
|
||||
resetAndReload(true);
|
||||
} catch (error) {
|
||||
console.error('Error during bulk auto-organize:', error);
|
||||
|
||||
@@ -196,6 +196,17 @@ export class BulkMissingLoraDownloadManager {
|
||||
let completedDownloads = 0;
|
||||
let failedDownloads = 0;
|
||||
let currentLoraProgress = 0;
|
||||
let cancelled = false;
|
||||
|
||||
loadingManager.showCancelButton(async () => {
|
||||
if (cancelled) return;
|
||||
cancelled = true;
|
||||
try {
|
||||
await this.loraApiClient.cancelDownload(batchDownloadId);
|
||||
} catch (e) {
|
||||
console.error('Cancel request failed:', e);
|
||||
}
|
||||
});
|
||||
|
||||
// Set up WebSocket message handler
|
||||
ws.onmessage = (event) => {
|
||||
@@ -207,6 +218,11 @@ export class BulkMissingLoraDownloadManager {
|
||||
return;
|
||||
}
|
||||
|
||||
if (data.status === 'cancelled') {
|
||||
cancelled = true;
|
||||
return;
|
||||
}
|
||||
|
||||
// Process progress updates
|
||||
if (data.status === 'progress' && data.download_id && data.download_id.startsWith(batchDownloadId)) {
|
||||
currentLoraProgress = data.progress;
|
||||
@@ -249,6 +265,8 @@ export class BulkMissingLoraDownloadManager {
|
||||
|
||||
// Download each LoRA sequentially
|
||||
for (let i = 0; i < lorasToDownload.length; i++) {
|
||||
if (cancelled) break;
|
||||
|
||||
const lora = lorasToDownload[i];
|
||||
|
||||
currentLoraProgress = 0;
|
||||
@@ -275,11 +293,13 @@ export class BulkMissingLoraDownloadManager {
|
||||
modelId,
|
||||
versionId,
|
||||
loraRoot,
|
||||
'', // Empty relative path, use default paths
|
||||
'',
|
||||
useDefaultPaths,
|
||||
batchDownloadId
|
||||
);
|
||||
|
||||
if (cancelled) break;
|
||||
|
||||
if (!response.success) {
|
||||
console.error(`Failed to download LoRA ${lora.name || lora.file_name}: ${response.error}`);
|
||||
failedDownloads++;
|
||||
@@ -288,8 +308,10 @@ export class BulkMissingLoraDownloadManager {
|
||||
updateProgress(100, completedDownloads, '');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error downloading LoRA ${lora.name || lora.file_name}:`, error);
|
||||
failedDownloads++;
|
||||
if (!cancelled) {
|
||||
console.error(`Error downloading LoRA ${lora.name || lora.file_name}:`, error);
|
||||
failedDownloads++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -300,7 +322,10 @@ export class BulkMissingLoraDownloadManager {
|
||||
loadingManager.hide();
|
||||
|
||||
// Show completion message
|
||||
if (failedDownloads === 0) {
|
||||
if (cancelled) {
|
||||
showToast('toast.downloads.downloadStopped', {}, 'info',
|
||||
`Download cancelled. ${completedDownloads} item(s) completed.`);
|
||||
} else if (failedDownloads === 0) {
|
||||
showToast('toast.loras.allDownloadSuccessful', { count: completedDownloads }, 'success');
|
||||
} else {
|
||||
showToast('toast.loras.downloadPartialSuccess', {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { modalManager } from './ModalManager.js';
|
||||
import { showToast } from '../utils/uiHelpers.js';
|
||||
import { showToast, setupAutoNewlineOnPaste } from '../utils/uiHelpers.js';
|
||||
import { state } from '../state/index.js';
|
||||
import { LoadingManager } from './LoadingManager.js';
|
||||
import { getModelApiClient, resetAndReload } from '../api/modelApiFactory.js';
|
||||
@@ -31,6 +31,7 @@ export class DownloadManager {
|
||||
// HF download state
|
||||
this.hfRepoId = null;
|
||||
this.hfSelectedFiles = [];
|
||||
this.hfRepoCollapsed = {};
|
||||
|
||||
this.loadingManager = new LoadingManager();
|
||||
this.folderTreeManager = new FolderTreeManager();
|
||||
@@ -107,7 +108,8 @@ export class DownloadManager {
|
||||
// Default path toggle handler
|
||||
document.getElementById('useDefaultPath').addEventListener('change', this.handleToggleDefaultPath);
|
||||
|
||||
|
||||
// Auto-append newline after pasting a URL so users can paste multiple URLs in succession
|
||||
setupAutoNewlineOnPaste('modelUrl');
|
||||
}
|
||||
|
||||
updateModalLabels() {
|
||||
@@ -173,6 +175,7 @@ export class DownloadManager {
|
||||
// Reset HF state
|
||||
this.hfRepoId = null;
|
||||
this.hfSelectedFiles = [];
|
||||
this.hfRepoCollapsed = {};
|
||||
}
|
||||
|
||||
async retrieveVersionsForModel(modelId, source = null) {
|
||||
@@ -463,8 +466,8 @@ export class DownloadManager {
|
||||
const trimmed = url.trim();
|
||||
if (!trimmed) return null;
|
||||
|
||||
// CivitAI
|
||||
if (/civitai\.com\/models\//i.test(trimmed) || /civitaiarchive|civarchive/i.test(trimmed)) {
|
||||
// CivitAI — matches civitai.com, civitai.red, civitai.green, etc.
|
||||
if (/civitai\.(?:com|red|green)\/models\//i.test(trimmed) || /civitaiarchive|civarchive/i.test(trimmed)) {
|
||||
// Will be parsed by existing CivitAI logic
|
||||
return { type: 'civitai' };
|
||||
}
|
||||
@@ -725,14 +728,23 @@ export class DownloadManager {
|
||||
|
||||
confirmFileSelection() {
|
||||
const selectedRadio = document.querySelector('#fileSelectionList input[type="radio"]:checked');
|
||||
if (!selectedRadio) return;
|
||||
if (!selectedRadio) {
|
||||
console.warn('[download] confirmFileSelection: no radio button checked');
|
||||
return;
|
||||
}
|
||||
|
||||
const version = this.currentVersion;
|
||||
if (!version) return;
|
||||
if (!version) {
|
||||
console.warn('[download] confirmFileSelection: no currentVersion set');
|
||||
return;
|
||||
}
|
||||
|
||||
const modelFiles = (version.files || []).filter(f => f.type === 'Model' || f.type === 'UNet' || f.type === 'Diffusion Model');
|
||||
this.selectedFile = modelFiles.find(f => f.id.toString() === selectedRadio.value);
|
||||
|
||||
console.log('[download] confirmFileSelection: selected file id=%s, name="%s", type="%s", metadata=%o',
|
||||
this.selectedFile?.id, this.selectedFile?.name, this.selectedFile?.type, this.selectedFile?.metadata);
|
||||
|
||||
document.getElementById('fileSelectionStep').style.display = 'none';
|
||||
document.getElementById('locationStep').style.display = 'block';
|
||||
this.proceedToLocationContent();
|
||||
@@ -869,16 +881,26 @@ export class DownloadManager {
|
||||
const displayName = versionName || `#${versionId}`;
|
||||
let ws = null;
|
||||
let updateProgress = () => { };
|
||||
let cancelled = false;
|
||||
const downloadId = Date.now().toString();
|
||||
|
||||
try {
|
||||
this.loadingManager.restoreProgressBar();
|
||||
updateProgress = this.loadingManager.showDownloadProgress(1);
|
||||
updateProgress(0, 0, displayName);
|
||||
|
||||
const downloadId = Date.now().toString();
|
||||
const wsProtocol = window.location.protocol === 'https:' ? 'wss://' : 'ws://';
|
||||
ws = new WebSocket(`${wsProtocol}${window.location.host}/ws/download-progress?id=${downloadId}`);
|
||||
|
||||
this.loadingManager.showCancelButton(async () => {
|
||||
if (cancelled) return;
|
||||
cancelled = true;
|
||||
try {
|
||||
await this.apiClient.cancelDownload(downloadId);
|
||||
} catch (e) {
|
||||
console.error('Cancel request failed:', e);
|
||||
}
|
||||
});
|
||||
|
||||
ws.onmessage = event => {
|
||||
const data = JSON.parse(event.data);
|
||||
|
||||
@@ -887,6 +909,12 @@ export class DownloadManager {
|
||||
return;
|
||||
}
|
||||
|
||||
if (data.status === 'cancelled') {
|
||||
cancelled = true;
|
||||
this.loadingManager.setStatus(translate('modals.download.status.cancelled', {}, 'Download cancelled'));
|
||||
return;
|
||||
}
|
||||
|
||||
if (data.status === 'progress' && data.download_id === downloadId) {
|
||||
const metrics = {
|
||||
bytesDownloaded: data.bytes_downloaded,
|
||||
@@ -925,6 +953,10 @@ export class DownloadManager {
|
||||
fileParams
|
||||
);
|
||||
|
||||
if (cancelled) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (response?.skipped) {
|
||||
this.loadingManager.setStatus(translate('modals.download.status.finalizing'));
|
||||
updateProgress(100, 0, displayName);
|
||||
@@ -965,8 +997,12 @@ export class DownloadManager {
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('Failed to download model version:', error);
|
||||
showToast('toast.downloads.downloadError', { message: error?.message }, 'error');
|
||||
if (cancelled) {
|
||||
console.log('Download cancelled by user:', downloadId);
|
||||
} else {
|
||||
console.error('Failed to download model version:', error);
|
||||
showToast('toast.downloads.downloadError', { message: error?.message }, 'error');
|
||||
}
|
||||
return false;
|
||||
} finally {
|
||||
try {
|
||||
@@ -986,16 +1022,33 @@ export class DownloadManager {
|
||||
const totalFiles = this.hfSelectedFiles.length;
|
||||
const updateProgress = this.loadingManager.showDownloadProgress(totalFiles);
|
||||
|
||||
let cancelled = false;
|
||||
let currentDownloadId = null;
|
||||
|
||||
this.loadingManager.showCancelButton(async () => {
|
||||
if (cancelled) return;
|
||||
cancelled = true;
|
||||
if (currentDownloadId) {
|
||||
try {
|
||||
await this.apiClient.cancelDownload(currentDownloadId);
|
||||
} catch (e) {
|
||||
console.error('Cancel request failed:', e);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
let completedDownloads = 0;
|
||||
for (let i = 0; i < totalFiles; i++) {
|
||||
if (cancelled) break;
|
||||
|
||||
const filename = this.hfSelectedFiles[i];
|
||||
updateProgress(0, completedDownloads, filename);
|
||||
this.loadingManager.setStatus(`Downloading ${filename}...`);
|
||||
|
||||
const downloadId = Date.now().toString() + '_' + i;
|
||||
currentDownloadId = Date.now().toString() + '_' + i;
|
||||
const wsProtocol = window.location.protocol === 'https:' ? 'wss://' : 'ws://';
|
||||
const ws = new WebSocket(`${wsProtocol}${window.location.host}/ws/download-progress?id=${downloadId}`);
|
||||
const ws = new WebSocket(`${wsProtocol}${window.location.host}/ws/download-progress?id=${currentDownloadId}`);
|
||||
|
||||
try {
|
||||
await new Promise((resolve, reject) => {
|
||||
@@ -1003,12 +1056,13 @@ export class DownloadManager {
|
||||
ws.onerror = reject;
|
||||
});
|
||||
|
||||
// Capture completed count at WS creation time so progress
|
||||
// updates arriving after completedDownloads increments still
|
||||
// show the correct "N / total" position.
|
||||
const snapshotCompleted = completedDownloads;
|
||||
ws.onmessage = (event) => {
|
||||
const data = JSON.parse(event.data);
|
||||
if (data.status === 'cancelled') {
|
||||
cancelled = true;
|
||||
return;
|
||||
}
|
||||
if (data.status === 'progress') {
|
||||
const metrics = {
|
||||
bytesDownloaded: data.bytes_downloaded,
|
||||
@@ -1026,9 +1080,11 @@ export class DownloadManager {
|
||||
modelRoot,
|
||||
relativePath: targetFolder,
|
||||
useDefaultPaths,
|
||||
download_id: downloadId,
|
||||
download_id: currentDownloadId,
|
||||
});
|
||||
|
||||
if (cancelled) break;
|
||||
|
||||
if (response?.success) {
|
||||
completedDownloads++;
|
||||
updateProgress(100, completedDownloads, filename);
|
||||
@@ -1038,13 +1094,19 @@ export class DownloadManager {
|
||||
}
|
||||
}
|
||||
|
||||
showToast('toast.loras.downloadCompleted', {}, 'success');
|
||||
// Reload page data — model is already in scanner cache via backend
|
||||
if (cancelled) {
|
||||
showToast('toast.downloads.downloadStopped', {}, 'info',
|
||||
`Download cancelled. ${completedDownloads} item(s) completed.`);
|
||||
} else {
|
||||
showToast('toast.loras.downloadCompleted', {}, 'success');
|
||||
}
|
||||
await resetAndReload(true);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('Failed to download HF model:', error);
|
||||
showToast('toast.downloads.downloadError', { message: error?.message }, 'error');
|
||||
if (!cancelled) {
|
||||
console.error('Failed to download HF model:', error);
|
||||
showToast('toast.downloads.downloadError', { message: error?.message }, 'error');
|
||||
}
|
||||
return false;
|
||||
} finally {
|
||||
this.loadingManager.hide();
|
||||
@@ -1077,7 +1139,7 @@ export class DownloadManager {
|
||||
|
||||
showBatchPreviewStep() {
|
||||
document.querySelectorAll('.download-step').forEach(step => step.style.display = 'none');
|
||||
document.getElementById('batchPreviewStep').style.display = 'block';
|
||||
document.getElementById('batchPreviewStep').style.display = 'flex';
|
||||
|
||||
const validCount = this.batchModels.filter(m => {
|
||||
if (m.error) return false;
|
||||
@@ -1091,56 +1153,36 @@ export class DownloadManager {
|
||||
const list = document.getElementById('batchPreviewList');
|
||||
const hasHfItems = this.batchModels.some(m => m.source === 'huggingface' && !m.error);
|
||||
|
||||
let itemsHtml = this.batchModels.map((item, index) => {
|
||||
if (item.error) {
|
||||
return `
|
||||
<div class="batch-preview-item batch-preview-error" data-index="${index}">
|
||||
<div class="batch-preview-icon">
|
||||
<i class="fas fa-exclamation-triangle"></i>
|
||||
</div>
|
||||
<div class="batch-preview-info">
|
||||
<div class="batch-preview-name">${item.url}</div>
|
||||
<div class="batch-preview-meta batch-preview-error-text">${item.error}</div>
|
||||
</div>
|
||||
<button class="batch-preview-remove" data-index="${index}" title="${translate('common.actions.remove', {}, 'Remove')}">
|
||||
<i class="fas fa-times"></i>
|
||||
</button>
|
||||
// Error items render flat, outside any group
|
||||
const errorItemsHtml = this.batchModels.map((item, index) => {
|
||||
if (!item.error) return null;
|
||||
return `
|
||||
<div class="batch-preview-item batch-preview-error" data-index="${index}">
|
||||
<div class="batch-preview-icon">
|
||||
<i class="fas fa-exclamation-triangle"></i>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
<div class="batch-preview-info">
|
||||
<div class="batch-preview-name">${item.url}</div>
|
||||
<div class="batch-preview-meta batch-preview-error-text">${item.error}</div>
|
||||
</div>
|
||||
<button class="batch-preview-remove" data-index="${index}" title="${translate('common.actions.remove', {}, 'Remove')}">
|
||||
<i class="fas fa-times"></i>
|
||||
</button>
|
||||
</div>
|
||||
`;
|
||||
}).filter(Boolean).join('');
|
||||
|
||||
// CivitAI items render flat, outside any group (unchanged)
|
||||
const civitaiItemsHtml = this.batchModels.map((item, index) => {
|
||||
if (item.error) return null;
|
||||
if (item.source === 'huggingface') return null;
|
||||
const ver = item.selectedVersion;
|
||||
|
||||
// HF batch item rendering with checkbox
|
||||
if (item.source === 'huggingface') {
|
||||
const hfSize = item.fileSizeBytes
|
||||
? formatFileSize(item.fileSizeBytes)
|
||||
: '?';
|
||||
return `
|
||||
<div class="batch-preview-item" data-index="${index}">
|
||||
<input type="checkbox" class="batch-preview-checkbox"
|
||||
data-index="${index}" ${item.checked !== false ? 'checked' : ''} />
|
||||
<div class="batch-preview-info">
|
||||
<div class="batch-preview-name">${item.displayName || item.filename || `HF #${index}`} <span class="hf-badge">HF</span></div>
|
||||
<div class="batch-preview-meta">
|
||||
<span>${hfSize}</span>
|
||||
<span>${item.repo || ''}</span>
|
||||
</div>
|
||||
</div>
|
||||
<button class="batch-preview-remove" data-index="${index}" title="${translate('common.actions.remove', {}, 'Remove')}">
|
||||
<i class="fas fa-times"></i>
|
||||
</button>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
const firstImage = ver?.images?.find(img => !img.url.endsWith('.mp4'));
|
||||
const thumbnailUrl = firstImage ? firstImage.url : '/loras_static/images/no-preview.png';
|
||||
const fileSize = ver?.modelSizeKB
|
||||
? (ver.modelSizeKB / 1024).toFixed(1)
|
||||
: (ver?.files?.[0]?.sizeKB ? (ver.files[0].sizeKB / 1024).toFixed(1) : '?');
|
||||
const existsLocally = ver?.existsLocally;
|
||||
|
||||
return `
|
||||
<div class="batch-preview-item ${existsLocally ? 'batch-preview-local' : ''}" data-index="${index}">
|
||||
<div class="batch-preview-thumbnail">
|
||||
@@ -1161,8 +1203,59 @@ export class DownloadManager {
|
||||
` : ''}
|
||||
</div>
|
||||
`;
|
||||
}).filter(Boolean).join('');
|
||||
|
||||
// Group HF items by repo (data model stays flat — only rendering groups)
|
||||
const hfGroups = {};
|
||||
this.batchModels.forEach((item, index) => {
|
||||
if (item.error || item.source !== 'huggingface') return;
|
||||
const repo = item.repo || 'unknown';
|
||||
if (!hfGroups[repo]) hfGroups[repo] = [];
|
||||
hfGroups[repo].push({ item, index });
|
||||
});
|
||||
|
||||
const renderHfItem = ({ item, index }) => {
|
||||
const hfSize = item.fileSizeBytes ? formatFileSize(item.fileSizeBytes) : '?';
|
||||
return `
|
||||
<div class="batch-preview-item" data-index="${index}">
|
||||
<input type="checkbox" class="batch-preview-checkbox"
|
||||
data-index="${index}" ${item.checked !== false ? 'checked' : ''} />
|
||||
<div class="batch-preview-info">
|
||||
<div class="batch-preview-name">${item.displayName || item.filename || `HF #${index}`} <span class="hf-badge">HF</span></div>
|
||||
<div class="batch-preview-meta">
|
||||
<span>${hfSize}</span>
|
||||
<span>${item.repo || ''}</span>
|
||||
</div>
|
||||
</div>
|
||||
<button class="batch-preview-remove" data-index="${index}" title="${translate('common.actions.remove', {}, 'Remove')}">
|
||||
<i class="fas fa-times"></i>
|
||||
</button>
|
||||
</div>
|
||||
`;
|
||||
};
|
||||
|
||||
const hfGroupsHtml = Object.keys(hfGroups).map(repo => {
|
||||
const items = hfGroups[repo];
|
||||
const isCollapsed = this.hfRepoCollapsed[repo] === true;
|
||||
const allChecked = items.every(({ item }) => item.checked !== false);
|
||||
const fileCount = items.length;
|
||||
return `
|
||||
<div class="batch-preview-group" data-repo="${repo}">
|
||||
<div class="batch-preview-group-header">
|
||||
<i class="fas fa-chevron-right batch-preview-group-toggle ${isCollapsed ? '' : 'expanded'}"></i>
|
||||
<span class="batch-preview-group-name">${repo}</span>
|
||||
<span class="batch-preview-group-count">${fileCount} ${translate('modals.download.fileSelection.files', {}, 'files')}</span>
|
||||
<input type="checkbox" class="batch-preview-group-select-all" data-repo="${repo}" ${allChecked ? 'checked' : ''} />
|
||||
</div>
|
||||
<div class="batch-preview-group-body ${isCollapsed ? '' : 'expanded'}">
|
||||
${items.map(renderHfItem).join('')}
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
|
||||
let itemsHtml = errorItemsHtml + civitaiItemsHtml + hfGroupsHtml;
|
||||
|
||||
// Prepend select-all toolbar if there are HF items with checkboxes
|
||||
if (hasHfItems) {
|
||||
const allChecked = this.batchModels
|
||||
@@ -1178,7 +1271,90 @@ export class DownloadManager {
|
||||
|
||||
list.innerHTML = itemsHtml;
|
||||
|
||||
const updateCountAndSelectAll = () => {
|
||||
const checkedCount = this.batchModels.filter(
|
||||
m => !m.error && m.checked !== false
|
||||
).length;
|
||||
document.getElementById('downloadModalTitle').textContent =
|
||||
translate('modals.download.titleWithType', { type: this.apiClient.apiConfig.config.displayName }) +
|
||||
` (${checkedCount})`;
|
||||
const nextBtn = document.getElementById('nextFromBatchBtn');
|
||||
nextBtn.disabled = checkedCount === 0;
|
||||
nextBtn.classList.toggle('disabled', checkedCount === 0);
|
||||
// Global select-all
|
||||
const selectAll = document.getElementById('batchSelectAll');
|
||||
if (selectAll) {
|
||||
const hfItems = this.batchModels.filter(m => m.source === 'huggingface' && !m.error);
|
||||
selectAll.checked = hfItems.length > 0 && hfItems.every(m => m.checked !== false);
|
||||
}
|
||||
// Per-group select-all
|
||||
list.querySelectorAll('.batch-preview-group-select-all').forEach(gsa => {
|
||||
const repo = gsa.dataset.repo;
|
||||
const repoItems = this.batchModels.filter(m => m.source === 'huggingface' && !m.error && m.repo === repo);
|
||||
gsa.checked = repoItems.length > 0 && repoItems.every(m => m.checked !== false);
|
||||
});
|
||||
};
|
||||
|
||||
list.onclick = (e) => {
|
||||
// Per-group select-all checkbox
|
||||
const groupSelectAll = e.target.closest('.batch-preview-group-select-all');
|
||||
if (groupSelectAll) {
|
||||
const repo = groupSelectAll.dataset.repo;
|
||||
const checked = groupSelectAll.checked;
|
||||
this.batchModels.forEach((m, idx) => {
|
||||
if (m.source === 'huggingface' && !m.error && m.repo === repo) {
|
||||
m.checked = checked;
|
||||
const cb = list.querySelector(`.batch-preview-checkbox[data-index="${idx}"]`);
|
||||
if (cb) cb.checked = checked;
|
||||
}
|
||||
});
|
||||
updateCountAndSelectAll();
|
||||
return;
|
||||
}
|
||||
|
||||
const header = e.target.closest('.batch-preview-group-header');
|
||||
if (header) {
|
||||
const group = header.closest('.batch-preview-group');
|
||||
const repo = group.dataset.repo;
|
||||
const body = group.querySelector('.batch-preview-group-body');
|
||||
const toggle = group.querySelector('.batch-preview-group-toggle');
|
||||
const isCollapsed = this.hfRepoCollapsed[repo];
|
||||
if (isCollapsed) {
|
||||
this.hfRepoCollapsed[repo] = false;
|
||||
body.style.transition = ''; // restore in case collapse was interrupted
|
||||
body.classList.add('expanded');
|
||||
toggle.classList.add('expanded');
|
||||
// force reflow so expanded class is registered before setting height
|
||||
void body.offsetHeight;
|
||||
body.style.maxHeight = body.scrollHeight + 'px';
|
||||
const onEnd = (e) => {
|
||||
if (e.propertyName !== 'max-height') return;
|
||||
if (this.hfRepoCollapsed[repo] !== false) return;
|
||||
body.style.maxHeight = ''; // fall back to .expanded's 9999px
|
||||
body.removeEventListener('transitionend', onEnd);
|
||||
};
|
||||
body.addEventListener('transitionend', onEnd);
|
||||
} else {
|
||||
this.hfRepoCollapsed[repo] = true;
|
||||
body.style.maxHeight = body.scrollHeight + 'px';
|
||||
requestAnimationFrame(() => {
|
||||
// animate only max-height; keep expanded so opacity stays 1
|
||||
body.style.transition = 'max-height 0.35s ease';
|
||||
body.style.maxHeight = '0';
|
||||
toggle.classList.remove('expanded');
|
||||
const onEnd = (e) => {
|
||||
if (e.propertyName !== 'max-height') return;
|
||||
if (this.hfRepoCollapsed[repo] !== true) return; // state changed since
|
||||
body.classList.remove('expanded');
|
||||
body.style.transition = '';
|
||||
body.removeEventListener('transitionend', onEnd);
|
||||
};
|
||||
body.addEventListener('transitionend', onEnd);
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const removeBtn = e.target.closest('.batch-preview-remove');
|
||||
if (removeBtn) {
|
||||
const idx = parseInt(removeBtn.dataset.index);
|
||||
@@ -1193,7 +1369,7 @@ export class DownloadManager {
|
||||
}
|
||||
};
|
||||
|
||||
// Checkbox handler for HF batch items
|
||||
// Individual HF checkbox handler
|
||||
const checkboxes = list.querySelectorAll('.batch-preview-checkbox');
|
||||
checkboxes.forEach(cb => {
|
||||
cb.addEventListener('change', (e) => {
|
||||
@@ -1201,26 +1377,11 @@ export class DownloadManager {
|
||||
if (this.batchModels[idx]) {
|
||||
this.batchModels[idx].checked = e.target.checked;
|
||||
}
|
||||
// Update valid count in title and Next button
|
||||
const checkedCount = this.batchModels.filter(
|
||||
m => !m.error && m.checked !== false
|
||||
).length;
|
||||
document.getElementById('downloadModalTitle').textContent =
|
||||
translate('modals.download.titleWithType', { type: this.apiClient.apiConfig.config.displayName }) +
|
||||
` (${checkedCount})`;
|
||||
const nextBtn = document.getElementById('nextFromBatchBtn');
|
||||
nextBtn.disabled = checkedCount === 0;
|
||||
nextBtn.classList.toggle('disabled', checkedCount === 0);
|
||||
// Update select-all checkbox state
|
||||
const selectAll = document.getElementById('batchSelectAll');
|
||||
if (selectAll) {
|
||||
const hfItems = this.batchModels.filter(m => m.source === 'huggingface' && !m.error);
|
||||
selectAll.checked = hfItems.length > 0 && hfItems.every(m => m.checked !== false);
|
||||
}
|
||||
updateCountAndSelectAll();
|
||||
});
|
||||
});
|
||||
|
||||
// Select-all handler
|
||||
// Global select-all handler
|
||||
const selectAll = document.getElementById('batchSelectAll');
|
||||
if (selectAll) {
|
||||
selectAll.addEventListener('change', (e) => {
|
||||
@@ -1233,16 +1394,7 @@ export class DownloadManager {
|
||||
this.batchModels[idx].checked = checked;
|
||||
}
|
||||
});
|
||||
// Update valid count in title and Next button
|
||||
const checkedCount = this.batchModels.filter(
|
||||
m => !m.error && m.checked !== false
|
||||
).length;
|
||||
document.getElementById('downloadModalTitle').textContent =
|
||||
translate('modals.download.titleWithType', { type: this.apiClient.apiConfig.config.displayName }) +
|
||||
` (${checkedCount})`;
|
||||
const nextBtn = document.getElementById('nextFromBatchBtn');
|
||||
nextBtn.disabled = checkedCount === 0;
|
||||
nextBtn.classList.toggle('disabled', checkedCount === 0);
|
||||
updateCountAndSelectAll();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1333,12 +1485,23 @@ export class DownloadManager {
|
||||
}
|
||||
|
||||
const fileParams = this.selectedFile ? {
|
||||
id: this.selectedFile.id,
|
||||
type: this.selectedFile.type || 'Model',
|
||||
format: this.selectedFile.metadata?.format || 'SafeTensor',
|
||||
size: this.selectedFile.metadata?.size || 'full',
|
||||
fp: this.selectedFile.metadata?.fp,
|
||||
format: this.selectedFile.metadata?.format || null,
|
||||
size: this.selectedFile.metadata?.size || null,
|
||||
fp: this.selectedFile.metadata?.fp || null,
|
||||
} : null;
|
||||
|
||||
if (fileParams) {
|
||||
console.log('[download] startDownload (single): fileParams built from selectedFile — id=%s, type=%s, format=%s, size=%s, fp=%s',
|
||||
fileParams.id, fileParams.type, fileParams.format, fileParams.size, fileParams.fp);
|
||||
} else {
|
||||
console.log('[download] startDownload (single): this.selectedFile is null — no file selection, will download primary/default file. version=%s has %d files',
|
||||
this.currentVersion?.id, (this.currentVersion?.files || []).length);
|
||||
}
|
||||
|
||||
modalManager.closeModal('downloadModal');
|
||||
|
||||
return this.executeDownloadWithProgress({
|
||||
modelId: this.modelId,
|
||||
versionId: this.currentVersion.id,
|
||||
@@ -1377,11 +1540,27 @@ export class DownloadManager {
|
||||
|
||||
let completedDownloads = 0;
|
||||
let failedDownloads = 0;
|
||||
let cancelled = false;
|
||||
|
||||
loadingManager.showCancelButton(async () => {
|
||||
if (cancelled) return;
|
||||
cancelled = true;
|
||||
try {
|
||||
await this.apiClient.cancelDownload(batchDownloadId);
|
||||
} catch (e) {
|
||||
console.error('Cancel request failed:', e);
|
||||
}
|
||||
});
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
const data = JSON.parse(event.data);
|
||||
if (data.type === 'download_id') return;
|
||||
|
||||
if (data.status === 'cancelled') {
|
||||
cancelled = true;
|
||||
return;
|
||||
}
|
||||
|
||||
if (data.status === 'progress' && data.download_id?.startsWith(batchDownloadId)) {
|
||||
const current = downloadItems[completedDownloads + failedDownloads];
|
||||
const name = current?.selectedVersion?.name || current?.displayName || current?.filename || `#${completedDownloads + failedDownloads + 1}`;
|
||||
@@ -1400,6 +1579,8 @@ export class DownloadManager {
|
||||
});
|
||||
|
||||
for (let i = 0; i < downloadItems.length; i++) {
|
||||
if (cancelled) break;
|
||||
|
||||
const item = downloadItems[i];
|
||||
const name = item.displayName || item.filename || (item.selectedVersion?.name || `Model #${item.modelId}`);
|
||||
const isHf = item.source === 'huggingface';
|
||||
@@ -1410,7 +1591,6 @@ export class DownloadManager {
|
||||
try {
|
||||
let response;
|
||||
if (isHf) {
|
||||
// Per-file WebSocket for real-time progress
|
||||
const downloadId = Date.now().toString() + '_hf_' + i;
|
||||
const wsHf = new WebSocket(`${wsProtocol}${window.location.host}/ws/download-progress?id=${downloadId}`);
|
||||
try {
|
||||
@@ -1444,6 +1624,8 @@ export class DownloadManager {
|
||||
wsHf.close();
|
||||
}
|
||||
} else {
|
||||
console.log('[download] batch download: fileParams NOT passed for modelId=%s, versionId=%s — backend will use primary file',
|
||||
item.modelId, item.selectedVersion?.id);
|
||||
response = await this.apiClient.downloadModel(
|
||||
item.modelId,
|
||||
item.selectedVersion.id,
|
||||
@@ -1455,6 +1637,8 @@ export class DownloadManager {
|
||||
);
|
||||
}
|
||||
|
||||
if (cancelled) break;
|
||||
|
||||
if (!response.success) {
|
||||
failedDownloads++;
|
||||
} else {
|
||||
@@ -1462,15 +1646,20 @@ export class DownloadManager {
|
||||
updateProgress(100, completedDownloads, '');
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Failed to download ${name}:`, err);
|
||||
failedDownloads++;
|
||||
if (!cancelled) {
|
||||
console.error(`Failed to download ${name}:`, err);
|
||||
failedDownloads++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ws.close();
|
||||
loadingManager.hide();
|
||||
|
||||
if (failedDownloads === 0) {
|
||||
if (cancelled) {
|
||||
showToast('toast.downloads.downloadStopped', {}, 'info',
|
||||
`Download cancelled. ${completedDownloads} item(s) completed.`);
|
||||
} else if (failedDownloads === 0) {
|
||||
showToast('toast.loras.allDownloadSuccessful', { count: completedDownloads }, 'success');
|
||||
} else {
|
||||
showToast('toast.loras.downloadPartialSuccess', {
|
||||
@@ -1488,6 +1677,10 @@ export class DownloadManager {
|
||||
modelRoot = '',
|
||||
targetFolder = ''
|
||||
} = {}) {
|
||||
console.warn('[download] downloadVersionWithDefaults: NO fileParams will be sent — backend will always use primary file. '
|
||||
+ 'modelType=%s, modelId=%s, versionId=%s, versionName="%s"',
|
||||
modelType, modelId, versionId, versionName);
|
||||
|
||||
try {
|
||||
this.apiClient = getModelApiClient(modelType);
|
||||
} catch (error) {
|
||||
|
||||
@@ -281,6 +281,10 @@ export class LoadingManager {
|
||||
// Initialize transfer stats with empty data
|
||||
updateTransferStats();
|
||||
|
||||
if (this.cancelButton) {
|
||||
this.loadingContent.appendChild(this.cancelButton);
|
||||
}
|
||||
|
||||
// Return update function
|
||||
return (currentProgress, currentIndex = 0, currentName = '', metrics = {}) => {
|
||||
// Update current item progress
|
||||
|
||||
@@ -264,6 +264,19 @@ export class ModalManager {
|
||||
});
|
||||
}
|
||||
|
||||
// Add linkHfModal registration
|
||||
const linkHfModal = document.getElementById('linkHfModal');
|
||||
if (linkHfModal) {
|
||||
this.registerModal('linkHfModal', {
|
||||
element: linkHfModal,
|
||||
onClose: () => {
|
||||
this.getModal('linkHfModal').element.style.display = 'none';
|
||||
document.body.classList.remove('modal-open');
|
||||
},
|
||||
closeOnOutsideClick: true
|
||||
});
|
||||
}
|
||||
|
||||
// Add exampleAccessModal registration
|
||||
const exampleAccessModal = document.getElementById('exampleAccessModal');
|
||||
if (exampleAccessModal) {
|
||||
|
||||
@@ -330,8 +330,9 @@ class MoveManager {
|
||||
.filter(r => r.success)
|
||||
.map(r => ({ original_file_path: r.original_file_path, new_file_path: r.new_file_path }));
|
||||
|
||||
// Deselect moving items
|
||||
// Deselect moving items and exit bulk mode
|
||||
this.bulkFilePaths.forEach(path => bulkManager.deselectItem(path));
|
||||
if (state.bulkMode) bulkManager.toggleBulkMode();
|
||||
} else {
|
||||
// Single move mode
|
||||
const result = await apiClient.moveSingleModel(this.currentFilePath, targetPath, this.useDefaultPath);
|
||||
|
||||
@@ -789,6 +789,27 @@ export class SettingsManager {
|
||||
}
|
||||
}
|
||||
|
||||
async _fetchProviderModelsAsync() {
|
||||
try {
|
||||
const resp = await fetch('/api/lm/llm/provider-models');
|
||||
if (!resp.ok) return;
|
||||
const data = await resp.json();
|
||||
if (data.success && data.models) {
|
||||
this._providerModels = data.models;
|
||||
// Refresh model combobox if the settings modal is still open.
|
||||
// Skip when provider is Ollama — it fetches its own live list
|
||||
// from the local Ollama API and we must not overwrite it.
|
||||
const llmProviderSelect = document.getElementById('llmProvider');
|
||||
const provider = llmProviderSelect ? llmProviderSelect.value : 'openai';
|
||||
if (this._llmModelCombobox && provider !== 'ollama') {
|
||||
this._llmModelCombobox.updatePresets(this._providerModels[provider] || []);
|
||||
}
|
||||
}
|
||||
} catch (_) {
|
||||
// Silently ignore — models stay empty until next modal open
|
||||
}
|
||||
}
|
||||
|
||||
async loadSettingsToUI() {
|
||||
// Set frontend settings from state
|
||||
const blurMatureContentCheckbox = document.getElementById('blurMatureContent');
|
||||
@@ -829,20 +850,112 @@ export class SettingsManager {
|
||||
this.updateApiKeyStatus();
|
||||
this.updateLlmApiKeyStatus();
|
||||
|
||||
// AI Provider settings
|
||||
// ── AI Provider settings ──────────────────────────────────────
|
||||
// Load provider presets from the JSON script tag embedded in the template
|
||||
this._providerPresets = {};
|
||||
this._providerModels = {};
|
||||
const presetsScript = document.getElementById('llmProviderPresets');
|
||||
if (presetsScript) {
|
||||
try {
|
||||
this._providerPresets = JSON.parse(presetsScript.textContent);
|
||||
} catch (_) {
|
||||
this._providerPresets = {};
|
||||
}
|
||||
}
|
||||
const modelsScript = document.getElementById('llmProviderModels');
|
||||
if (modelsScript) {
|
||||
try {
|
||||
this._providerModels = JSON.parse(modelsScript.textContent);
|
||||
} catch (_) {
|
||||
this._providerModels = {};
|
||||
}
|
||||
}
|
||||
|
||||
// If the embedded provider models is empty (server did not block on
|
||||
// the remote catalog during page render), fetch asynchronously.
|
||||
if (!this._providerModels || Object.keys(this._providerModels).length === 0) {
|
||||
this._fetchProviderModelsAsync();
|
||||
}
|
||||
|
||||
const llmProviderSelect = document.getElementById('llmProvider');
|
||||
if (llmProviderSelect) {
|
||||
llmProviderSelect.value = state.global.settings.llm_provider || 'openai';
|
||||
}
|
||||
|
||||
// Destroy previous combobox instances before creating new ones,
|
||||
// since loadSettingsToUI() runs on every modal open.
|
||||
if (this._llmApiBaseCombobox) { this._llmApiBaseCombobox.destroy(); }
|
||||
if (this._llmModelCombobox) { this._llmModelCombobox.destroy(); }
|
||||
|
||||
const llmApiBaseInput = document.getElementById('llmApiBase');
|
||||
if (llmApiBaseInput) {
|
||||
llmApiBaseInput.value = state.global.settings.llm_api_base || '';
|
||||
const presetUrls = Object.values(this._providerPresets)
|
||||
.map(p => p.api_base)
|
||||
.filter(Boolean);
|
||||
if (typeof Combobox !== 'undefined') {
|
||||
this._llmApiBaseCombobox = new Combobox(llmApiBaseInput, {
|
||||
presets: presetUrls,
|
||||
placeholder: 'https://api.openai.com/v1',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Helper to update model Combobox presets from catalog / Ollama API
|
||||
const llmModelInput = document.getElementById('llmModel');
|
||||
if (llmModelInput) {
|
||||
llmModelInput.value = state.global.settings.llm_model || '';
|
||||
this._llmModelCombobox = null;
|
||||
if (llmModelInput && typeof Combobox !== 'undefined') {
|
||||
const currentProvider = llmProviderSelect ? llmProviderSelect.value : 'openai';
|
||||
const fallbackModels = currentProvider === 'ollama' ? [] : (this._providerModels[currentProvider] || []);
|
||||
this._llmModelCombobox = new Combobox(llmModelInput, {
|
||||
presets: fallbackModels,
|
||||
placeholder: translate('settings.aiProvider.modelPlaceholder', {}, 'Select a model...'),
|
||||
onSelect: (value) => {
|
||||
state.global.settings.llm_model = value;
|
||||
this.saveSetting('llm_model', value)
|
||||
.then(() => showToast('toast.settings.settingsUpdated', { setting: 'model' }, 'success'))
|
||||
.catch(() => {});
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const _loadModelPresets = async (provider) => {
|
||||
if (!this._llmModelCombobox) return;
|
||||
if (provider === 'ollama') {
|
||||
try {
|
||||
const apiBase = document.getElementById('llmApiBase')?.value?.trim() || 'http://localhost:11434/v1';
|
||||
const resp = await fetch(`/api/lm/llm/models?provider=ollama&api_base=${encodeURIComponent(apiBase)}`);
|
||||
if (resp.ok) {
|
||||
const data = await resp.json();
|
||||
if (data.success && Array.isArray(data.models)) {
|
||||
this._llmModelCombobox.updatePresets(data.models);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch (_) {}
|
||||
this._llmModelCombobox.updatePresets([]);
|
||||
} else {
|
||||
this._llmModelCombobox.updatePresets(this._providerModels[provider] || []);
|
||||
}
|
||||
};
|
||||
_loadModelPresets(llmProviderSelect ? llmProviderSelect.value : 'openai');
|
||||
|
||||
// Provider change → auto-fill API Base URL + update model presets
|
||||
if (llmProviderSelect) {
|
||||
llmProviderSelect.addEventListener('change', () => {
|
||||
const provider = llmProviderSelect.value;
|
||||
const preset = this._providerPresets[provider];
|
||||
if (preset) {
|
||||
if (llmApiBaseInput && preset.api_base) {
|
||||
llmApiBaseInput.value = preset.api_base;
|
||||
if (this._llmApiBaseCombobox) {
|
||||
this._llmApiBaseCombobox.setValue(preset.api_base);
|
||||
}
|
||||
llmApiBaseInput.dispatchEvent(new Event('blur'));
|
||||
}
|
||||
}
|
||||
_loadModelPresets(provider);
|
||||
});
|
||||
}
|
||||
|
||||
const civitaiHostSelect = document.getElementById('civitaiHost');
|
||||
@@ -1580,13 +1693,15 @@ export class SettingsManager {
|
||||
<input type="text" class="extra-folder-path-input"
|
||||
placeholder="${translate('settings.extraFolderPaths.pathPlaceholder', {}, '/path/to/models')}" value="${path}"
|
||||
onblur="settingsManager.updateExtraFolderPaths('${modelType}')"
|
||||
onfocus="settingsManager.clearExtraFolderPathError(this)"
|
||||
onkeydown="if(event.key === 'Enter') { this.blur(); }" />
|
||||
<button type="button" class="remove-path-btn"
|
||||
onclick="this.parentElement.parentElement.remove(); settingsManager.updateExtraFolderPaths('${modelType}')"
|
||||
onclick="settingsManager.removeExtraFolderPathRow(this, '${modelType}')"
|
||||
title="${translate('common.actions.delete', {}, 'Delete')}">
|
||||
<i class="fas fa-times"></i>
|
||||
</button>
|
||||
</div>
|
||||
<div class="extra-folder-path-error"></div>
|
||||
`;
|
||||
|
||||
container.appendChild(row);
|
||||
@@ -1600,7 +1715,63 @@ export class SettingsManager {
|
||||
}
|
||||
}
|
||||
|
||||
clearExtraFolderPathError(input) {
|
||||
input.classList.remove('has-error');
|
||||
const row = input.closest('.extra-folder-path-row');
|
||||
if (row) {
|
||||
const errEl = row.querySelector('.extra-folder-path-error');
|
||||
if (errEl) {
|
||||
errEl.classList.remove('visible');
|
||||
errEl.textContent = '';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_clearAllExtraFolderPathErrors() {
|
||||
document.querySelectorAll('.extra-folder-path-input.has-error').forEach((input) => {
|
||||
input.classList.remove('has-error');
|
||||
});
|
||||
document.querySelectorAll('.extra-folder-path-error.visible').forEach((el) => {
|
||||
el.classList.remove('visible');
|
||||
el.textContent = '';
|
||||
});
|
||||
}
|
||||
|
||||
_markExtraFolderPathsError(modelType, overlappingPaths, showMessage = false) {
|
||||
const container = document.getElementById(`extraFolderPaths-${modelType}`);
|
||||
if (!container) return;
|
||||
|
||||
const inputs = container.querySelectorAll('.extra-folder-path-input');
|
||||
inputs.forEach((input) => {
|
||||
const val = input.value.trim();
|
||||
if (val && overlappingPaths.includes(val)) {
|
||||
input.classList.add('has-error');
|
||||
if (showMessage) {
|
||||
const row = input.closest('.extra-folder-path-row');
|
||||
if (row) {
|
||||
const errEl = row.querySelector('.extra-folder-path-error');
|
||||
if (errEl) {
|
||||
errEl.textContent = translate('settings.extraFolderPaths.validation.checkpointUnetOverlapInline', {}, 'This path is also used for a different model type. Use separate folders for checkpoints and diffusion models.');
|
||||
errEl.classList.add('visible');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
removeExtraFolderPathRow(btn, modelType) {
|
||||
const row = btn.closest('.extra-folder-path-row');
|
||||
if (row) {
|
||||
row.remove();
|
||||
this.updateExtraFolderPaths(modelType);
|
||||
}
|
||||
}
|
||||
|
||||
async updateExtraFolderPaths(changedModelType) {
|
||||
// Clear previous errors
|
||||
this._clearAllExtraFolderPathErrors();
|
||||
|
||||
const extraFolderPaths = {};
|
||||
|
||||
// Collect paths for all model types
|
||||
@@ -1621,6 +1792,32 @@ export class SettingsManager {
|
||||
extraFolderPaths[modelType] = paths;
|
||||
});
|
||||
|
||||
// Client-side pre-check: checkpoints and unet must not share the same path.
|
||||
// Normalise paths to reduce false negatives vs the backend's realpath + normcase.
|
||||
const normalise = (p) => p.replace(/[/\\]+$/, '').toLowerCase();
|
||||
const ckptSet = new Set((extraFolderPaths.checkpoints || []).map(normalise));
|
||||
const unetSet = new Set((extraFolderPaths.unet || []).map(normalise));
|
||||
const ckptOverlap = (extraFolderPaths.checkpoints || []).filter(p => p && unetSet.has(normalise(p)));
|
||||
const unetOverlap = (extraFolderPaths.unet || []).filter(p => p && ckptSet.has(normalise(p)));
|
||||
const hasOverlap = ckptOverlap.length > 0 || unetOverlap.length > 0;
|
||||
|
||||
if (hasOverlap) {
|
||||
// Error message only on the side the user just edited.
|
||||
// The other side gets red border only (passive conflict indicator).
|
||||
if (changedModelType === 'checkpoints') {
|
||||
this._markExtraFolderPathsError('checkpoints', ckptOverlap, true);
|
||||
this._markExtraFolderPathsError('unet', unetOverlap, false);
|
||||
} else if (changedModelType === 'unet') {
|
||||
this._markExtraFolderPathsError('unet', unetOverlap, true);
|
||||
this._markExtraFolderPathsError('checkpoints', ckptOverlap, false);
|
||||
} else {
|
||||
// Pre-existing conflict from direct config edit — mark both without messages
|
||||
this._markExtraFolderPathsError('checkpoints', ckptOverlap, false);
|
||||
this._markExtraFolderPathsError('unet', unetOverlap, false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if paths have actually changed
|
||||
const currentPaths = state.global.settings.extra_folder_paths || {};
|
||||
const pathsChanged = JSON.stringify(currentPaths) !== JSON.stringify(extraFolderPaths);
|
||||
@@ -2949,7 +3146,7 @@ export class SettingsManager {
|
||||
}
|
||||
|
||||
updateLlmApiKeyStatus() {
|
||||
const hasKey = !!state.global.settings.llm_api_key;
|
||||
const hasKey = !!(state.global.settings.llm_api_key_set || state.global.settings.llm_api_key);
|
||||
const statusText = document.getElementById('llmApiKeyStatusText');
|
||||
const actionBtn = document.getElementById('llmApiKeyActionBtn');
|
||||
if (!statusText || !actionBtn) return;
|
||||
|
||||
@@ -168,6 +168,18 @@ export class DownloadManager {
|
||||
let failedDownloads = 0;
|
||||
let accessFailures = 0;
|
||||
let currentLoraProgress = 0;
|
||||
let cancelled = false;
|
||||
|
||||
this.importManager.loadingManager.showCancelButton(async () => {
|
||||
if (cancelled) return;
|
||||
cancelled = true;
|
||||
try {
|
||||
const loraClient = getModelApiClient(MODEL_TYPES.LORA);
|
||||
await loraClient.cancelDownload(batchDownloadId);
|
||||
} catch (e) {
|
||||
console.error('Cancel request failed:', e);
|
||||
}
|
||||
});
|
||||
|
||||
// Set up progress tracking for current download
|
||||
ws.onmessage = (event) => {
|
||||
@@ -179,6 +191,11 @@ export class DownloadManager {
|
||||
return;
|
||||
}
|
||||
|
||||
if (data.status === 'cancelled') {
|
||||
cancelled = true;
|
||||
return;
|
||||
}
|
||||
|
||||
// Process progress updates for our current active download
|
||||
if (data.status === 'progress' && data.download_id && data.download_id.startsWith(batchDownloadId)) {
|
||||
// Update current LoRA progress
|
||||
@@ -221,6 +238,8 @@ export class DownloadManager {
|
||||
const useDefaultPaths = getStorageItem('use_default_path_loras', false);
|
||||
|
||||
for (let i = 0; i < this.importManager.downloadableLoRAs.length; i++) {
|
||||
if (cancelled) break;
|
||||
|
||||
const lora = this.importManager.downloadableLoRAs[i];
|
||||
|
||||
// Reset current LoRA progress for new download
|
||||
@@ -241,15 +260,13 @@ export class DownloadManager {
|
||||
batchDownloadId
|
||||
);
|
||||
|
||||
if (cancelled) break;
|
||||
|
||||
if (!response.success) {
|
||||
console.error(`Failed to download LoRA ${lora.name}: ${response.error}`);
|
||||
|
||||
failedDownloads++;
|
||||
// Continue with next download
|
||||
} else {
|
||||
completedDownloads++;
|
||||
|
||||
// Update progress to show completion of current LoRA
|
||||
updateProgress(100, completedDownloads, '');
|
||||
|
||||
if (completedDownloads + failedDownloads < this.importManager.downloadableLoRAs.length) {
|
||||
@@ -259,9 +276,10 @@ export class DownloadManager {
|
||||
}
|
||||
}
|
||||
} catch (downloadError) {
|
||||
console.error(`Error downloading LoRA ${lora.name}:`, downloadError);
|
||||
failedDownloads++;
|
||||
// Continue with next download
|
||||
if (!cancelled) {
|
||||
console.error(`Error downloading LoRA ${lora.name}:`, downloadError);
|
||||
failedDownloads++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -269,7 +287,10 @@ export class DownloadManager {
|
||||
ws.close();
|
||||
|
||||
// Show appropriate completion message based on results
|
||||
if (failedDownloads === 0) {
|
||||
if (cancelled) {
|
||||
showToast('toast.downloads.downloadStopped', {}, 'info',
|
||||
`Download cancelled. ${completedDownloads} item(s) completed.`);
|
||||
} else if (failedDownloads === 0) {
|
||||
showToast('toast.loras.allDownloadSuccessful', { count: completedDownloads }, 'success');
|
||||
} else {
|
||||
if (accessFailures > 0) {
|
||||
|
||||
@@ -66,11 +66,23 @@ export const BASE_MODELS = {
|
||||
HUNYUAN_VIDEO: "Hunyuan Video",
|
||||
// Other models
|
||||
ANIMA: "Anima",
|
||||
ACE_AUDIO: "ACE Audio",
|
||||
BOOGU: "Boogu",
|
||||
ERNIE: "Ernie",
|
||||
ERNIE_TURBO: "Ernie Turbo",
|
||||
NUCLEUS: "Nucleus",
|
||||
PONY_V7: "Pony V7",
|
||||
GROK: "Grok",
|
||||
HAPPY_HORSE: "HappyHorse",
|
||||
HIDREAM_O1: "HiDream-O1",
|
||||
IDEOGRAM_4_0: "Ideogram 4.0",
|
||||
KREA_2: "Krea 2",
|
||||
LENS: "Lens",
|
||||
PONY_V7: "Pony V7",
|
||||
MAI: "MAI",
|
||||
NUCLEUS: "Nucleus",
|
||||
QWEN_2: "Qwen 2",
|
||||
UPSCALER: "Upscaler",
|
||||
WAN_IMAGE_2_7: "Wan Image 2.7",
|
||||
WAN_VIDEO_2_7: "Wan Video 2.7",
|
||||
// Default
|
||||
UNKNOWN: "Other"
|
||||
};
|
||||
@@ -143,22 +155,6 @@ export const BASE_MODEL_ABBREVIATIONS = {
|
||||
[BASE_MODELS.FLUX_2_KLEIN_4B]: 'FK4',
|
||||
[BASE_MODELS.FLUX_2_KLEIN_4B_BASE]: 'FK4B',
|
||||
|
||||
// Other diffusion models
|
||||
[BASE_MODELS.AURAFLOW]: 'AF',
|
||||
[BASE_MODELS.CHROMA]: 'CHR',
|
||||
[BASE_MODELS.PIXART_A]: 'PXA',
|
||||
[BASE_MODELS.PIXART_E]: 'PXE',
|
||||
[BASE_MODELS.HUNYUAN_1]: 'HY',
|
||||
[BASE_MODELS.LUMINA]: 'L',
|
||||
[BASE_MODELS.KOLORS]: 'KLR',
|
||||
[BASE_MODELS.NOOBAI]: 'NAI',
|
||||
[BASE_MODELS.ILLUSTRIOUS]: 'IL',
|
||||
[BASE_MODELS.PONY]: 'PONY',
|
||||
[BASE_MODELS.HIDREAM]: 'HID',
|
||||
[BASE_MODELS.QWEN]: 'QWEN',
|
||||
[BASE_MODELS.ZIMAGE_TURBO]: 'ZIT',
|
||||
[BASE_MODELS.ZIMAGE_BASE]: 'ZIB',
|
||||
|
||||
// Video models
|
||||
[BASE_MODELS.SVD]: 'SVD',
|
||||
[BASE_MODELS.LTXV]: 'LTXV',
|
||||
@@ -195,10 +191,22 @@ export const BASE_MODEL_ABBREVIATIONS = {
|
||||
[BASE_MODELS.ZIMAGE_TURBO]: 'ZIT',
|
||||
[BASE_MODELS.ZIMAGE_BASE]: 'ZIB',
|
||||
[BASE_MODELS.ANIMA]: 'ANI',
|
||||
[BASE_MODELS.ACE_AUDIO]: 'ACE',
|
||||
[BASE_MODELS.BOOGU]: 'BOOG',
|
||||
[BASE_MODELS.ERNIE]: 'ERNI',
|
||||
[BASE_MODELS.ERNIE_TURBO]: 'ETRB',
|
||||
[BASE_MODELS.NUCLEUS]: 'NUCL',
|
||||
[BASE_MODELS.GROK]: 'GROK',
|
||||
[BASE_MODELS.HAPPY_HORSE]: 'HAPP',
|
||||
[BASE_MODELS.HIDREAM_O1]: 'HIO1',
|
||||
[BASE_MODELS.IDEOGRAM_4_0]: 'ID40',
|
||||
[BASE_MODELS.KREA_2]: 'KR2',
|
||||
[BASE_MODELS.LENS]: 'LENS',
|
||||
[BASE_MODELS.MAI]: 'MAI',
|
||||
[BASE_MODELS.NUCLEUS]: 'NUCL',
|
||||
[BASE_MODELS.QWEN_2]: 'QWN2',
|
||||
[BASE_MODELS.UPSCALER]: 'UPSC',
|
||||
[BASE_MODELS.WAN_IMAGE_2_7]: 'WI27',
|
||||
[BASE_MODELS.WAN_VIDEO_2_7]: 'WAN',
|
||||
|
||||
// Default
|
||||
[BASE_MODELS.UNKNOWN]: 'OTH'
|
||||
@@ -394,7 +402,9 @@ export const BASE_MODEL_CATEGORIES = {
|
||||
BASE_MODELS.WAN_VIDEO_14B_I2V_480P, BASE_MODELS.WAN_VIDEO_14B_I2V_720P,
|
||||
BASE_MODELS.WAN_VIDEO_2_2_TI2V_5B, BASE_MODELS.WAN_VIDEO_2_2_T2V_A14B,
|
||||
BASE_MODELS.WAN_VIDEO_2_2_I2V_A14B, BASE_MODELS.WAN_VIDEO_2_5_T2V,
|
||||
BASE_MODELS.WAN_VIDEO_2_5_I2V
|
||||
BASE_MODELS.WAN_VIDEO_2_5_I2V,
|
||||
BASE_MODELS.HAPPY_HORSE,
|
||||
BASE_MODELS.WAN_IMAGE_2_7, BASE_MODELS.WAN_VIDEO_2_7
|
||||
],
|
||||
'Flux Models': [BASE_MODELS.FLUX_1_D, BASE_MODELS.FLUX_1_S, BASE_MODELS.FLUX_1_KONTEXT, BASE_MODELS.FLUX_1_KREA, BASE_MODELS.FLUX_2_D, BASE_MODELS.FLUX_2_KLEIN_9B, BASE_MODELS.FLUX_2_KLEIN_9B_BASE, BASE_MODELS.FLUX_2_KLEIN_4B, BASE_MODELS.FLUX_2_KLEIN_4B_BASE],
|
||||
'Other Models': [
|
||||
@@ -402,8 +412,10 @@ export const BASE_MODEL_CATEGORIES = {
|
||||
BASE_MODELS.QWEN, BASE_MODELS.AURAFLOW, BASE_MODELS.CHROMA, BASE_MODELS.ZIMAGE_TURBO, BASE_MODELS.ZIMAGE_BASE,
|
||||
BASE_MODELS.PIXART_A, BASE_MODELS.PIXART_E, BASE_MODELS.HUNYUAN_1,
|
||||
BASE_MODELS.LUMINA, BASE_MODELS.KOLORS, BASE_MODELS.NOOBAI, BASE_MODELS.ANIMA,
|
||||
BASE_MODELS.ERNIE, BASE_MODELS.ERNIE_TURBO, BASE_MODELS.NUCLEUS,
|
||||
BASE_MODELS.KREA_2,
|
||||
BASE_MODELS.ACE_AUDIO, BASE_MODELS.BOOGU, BASE_MODELS.ERNIE, BASE_MODELS.ERNIE_TURBO,
|
||||
BASE_MODELS.GROK, BASE_MODELS.HIDREAM_O1, BASE_MODELS.IDEOGRAM_4_0,
|
||||
BASE_MODELS.LENS, BASE_MODELS.MAI, BASE_MODELS.NUCLEUS,
|
||||
BASE_MODELS.QWEN_2, BASE_MODELS.KREA_2, BASE_MODELS.UPSCALER,
|
||||
BASE_MODELS.UNKNOWN
|
||||
]
|
||||
};
|
||||
|
||||
@@ -552,6 +552,8 @@ async function fetchWorkflowRegistry() {
|
||||
if (!registryData.success) {
|
||||
if (registryData.error === 'Standalone Mode Active') {
|
||||
showToast('toast.general.cannotInteractStandalone', {}, 'warning');
|
||||
} else if (registryData.error === 'Empty Registry') {
|
||||
showToast('uiHelpers.workflow.noSupportedNodes', {}, 'warning');
|
||||
} else {
|
||||
showToast('toast.general.failedWorkflowInfo', {}, 'error');
|
||||
}
|
||||
@@ -1482,3 +1484,40 @@ export async function openExampleImagesFolder(modelHash) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up a paste handler on a textarea that automatically appends a newline
|
||||
* after pasted content that looks like a URL (http/https). This lets users
|
||||
* paste multiple URLs one after another without manually pressing Enter.
|
||||
* @param {string} textareaId - The id of the textarea element
|
||||
*/
|
||||
export function setupAutoNewlineOnPaste(textareaId) {
|
||||
const el = document.getElementById(textareaId);
|
||||
if (!el || el.tagName !== 'TEXTAREA') return;
|
||||
|
||||
el.addEventListener('paste', (e) => {
|
||||
const pastedText = (e.clipboardData || window.clipboardData).getData('text');
|
||||
// Only apply to text that starts with http:// or https://
|
||||
if (/^https?:\/\//.test(pastedText) && !pastedText.endsWith('\n')) {
|
||||
e.preventDefault();
|
||||
|
||||
const start = el.selectionStart;
|
||||
const end = el.selectionEnd;
|
||||
const text = el.value;
|
||||
const before = text.substring(0, start);
|
||||
const after = text.substring(end);
|
||||
|
||||
// Append newline after the pasted URL
|
||||
const modifiedText = pastedText + '\n';
|
||||
el.value = before + modifiedText + after;
|
||||
|
||||
// Move cursor to just after the inserted text
|
||||
const newCursorPos = start + modifiedText.length;
|
||||
el.selectionStart = el.selectionEnd = newCursorPos;
|
||||
|
||||
// Trigger input event so any listeners stay in sync
|
||||
el.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}
|
||||
// Non-URL text or text already ending with \n — let default paste happen
|
||||
});
|
||||
}
|
||||
|
||||
@@ -12,7 +12,19 @@
|
||||
<div id="checkpointContextMenu" class="context-menu" style="display: none;">
|
||||
<!-- Metadata -->
|
||||
<div class="context-menu-item" data-action="refresh-metadata"><i class="fas fa-sync"></i> {{ t('loras.contextMenu.refreshMetadata') }}</div>
|
||||
<div class="context-menu-item" data-action="relink-civitai"><i class="fas fa-link"></i> {{ t('loras.contextMenu.relinkCivitai') }}</div>
|
||||
<div class="context-menu-item has-submenu" data-has-submenu="link-model">
|
||||
<i class="fas fa-link"></i>
|
||||
<span>{{ t('loras.contextMenu.linkModel') }}</span>
|
||||
<i class="fas fa-chevron-right submenu-arrow"></i>
|
||||
<div class="context-submenu">
|
||||
<div class="context-menu-item" data-action="relink-civitai">
|
||||
<i class="fas fa-external-link-alt"></i> <span>{{ t('loras.contextMenu.linkCivitai') }}</span>
|
||||
</div>
|
||||
<div class="context-menu-item" data-action="link-hf">
|
||||
<i class="fas fa-robot"></i> <span>{{ t('loras.contextMenu.linkHuggingFace') }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="context-menu-separator menu-section-break"></div>
|
||||
<!-- Workflow -->
|
||||
<div class="context-menu-item" data-action="copyname"><i class="fas fa-copy"></i> {{ t('loras.contextMenu.copyFilename') }}</div>
|
||||
|
||||
@@ -12,11 +12,21 @@
|
||||
<div class="context-menu-item" data-action="check-updates">
|
||||
<i class="fas fa-bell"></i> <span>{{ t('loras.contextMenu.checkUpdates') }}</span>
|
||||
</div>
|
||||
<div class="context-menu-item" data-action="enrich-hf-agent">
|
||||
<i class="fas fa-wand-magic-sparkles"></i> <span>{{ t('loras.contextMenu.enrichHfAgent') }}</span>
|
||||
<div class="context-menu-item has-submenu" data-has-submenu="link-model">
|
||||
<i class="fas fa-link"></i>
|
||||
<span>{{ t('loras.contextMenu.linkModel') }}</span>
|
||||
<i class="fas fa-chevron-right submenu-arrow"></i>
|
||||
<div class="context-submenu">
|
||||
<div class="context-menu-item" data-action="relink-civitai">
|
||||
<i class="fas fa-external-link-alt"></i> <span>{{ t('loras.contextMenu.linkCivitai') }}</span>
|
||||
</div>
|
||||
<div class="context-menu-item" data-action="link-hf">
|
||||
<i class="fas fa-robot"></i> <span>{{ t('loras.contextMenu.linkHuggingFace') }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="context-menu-item" data-action="relink-civitai">
|
||||
<i class="fas fa-link"></i> <span>{{ t('loras.contextMenu.relinkCivitai') }}</span>
|
||||
<div class="context-menu-item" data-action="enrich-hf-llm">
|
||||
<i class="fas fa-wand-magic-sparkles"></i> <span>{{ t('loras.contextMenu.enrichHfAgent') }}</span>
|
||||
</div>
|
||||
<div class="context-menu-separator menu-section-break"></div>
|
||||
<!-- Workflow -->
|
||||
@@ -86,7 +96,7 @@
|
||||
<div class="context-menu-item" data-action="resume-metadata-refresh">
|
||||
<i class="fas fa-redo"></i> <span>{{ t('loras.bulkOperations.resumeMetadataRefresh') }}</span>
|
||||
</div>
|
||||
<div class="context-menu-item" data-action="enrich-hf-agent-bulk">
|
||||
<div class="context-menu-item" data-action="enrich-hf-llm-bulk">
|
||||
<i class="fas fa-wand-magic-sparkles"></i> <span>{{ t('loras.bulkOperations.enrichHfAgent') }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
{% include 'components/modals/update_modal.html' %}
|
||||
{% include 'components/modals/help_modal.html' %}
|
||||
{% include 'components/modals/relink_civitai_modal.html' %}
|
||||
{% include 'components/modals/link_hf_modal.html' %}
|
||||
{% include 'components/modals/example_access_modal.html' %}
|
||||
{% include 'components/modals/download_modal.html' %}
|
||||
{% include 'components/modals/move_modal.html' %}
|
||||
|
||||
@@ -112,6 +112,10 @@
|
||||
<a href="https://github.com/willmiao/ComfyUI-Lora-Manager/wiki/Priority-Tags-Configuration-Guide" target="_blank">
|
||||
Priority Tags Configuration Guide
|
||||
<span class="new-content-badge inline">{{ t('help.documentation.newBadge') }}</span>
|
||||
<li>
|
||||
<a href="https://github.com/willmiao/ComfyUI-Lora-Manager/wiki/AI-Provider-Setup" target="_blank">
|
||||
AI Provider Setup
|
||||
<span class="new-content-badge inline">{{ t('help.documentation.newBadge') }}</span>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
<!-- Link to HuggingFace Modal -->
|
||||
<div id="linkHfModal" class="modal">
|
||||
<div class="modal-content">
|
||||
<button class="close" onclick="modalManager.closeModal('linkHfModal')">×</button>
|
||||
<h2>{{ t('modals.linkHuggingFace.title') }}</h2>
|
||||
<div class="warning-box">
|
||||
<i class="fas fa-info-circle"></i>
|
||||
<p>{{ t('modals.linkHuggingFace.infoText') }}</p>
|
||||
</div>
|
||||
<div class="input-group">
|
||||
<label for="hfModelUrl">{{ t('modals.linkHuggingFace.urlLabel') }}</label>
|
||||
<input type="text" id="hfModelUrl" placeholder="{{ t('modals.linkHuggingFace.urlPlaceholder') }}" />
|
||||
<div class="input-error" id="hfModelUrlError"></div>
|
||||
<div class="input-help">
|
||||
{{ t('modals.linkHuggingFace.helpText') }}<br>
|
||||
<strong>https://huggingface.co/user/repo</strong>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-actions">
|
||||
<button class="cancel-btn" onclick="modalManager.closeModal('linkHfModal')">{{ t('common.actions.cancel') }}</button>
|
||||
<button class="confirm-btn" id="confirmLinkHfBtn">{{ t('modals.linkHuggingFace.confirmAction') }}</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -157,9 +157,13 @@
|
||||
</div>
|
||||
<div class="setting-control select-control">
|
||||
<select id="llmProvider" onchange="settingsManager.saveSelectSetting('llmProvider', 'llm_provider')">
|
||||
<option value="openai">OpenAI</option>
|
||||
<option value="ollama">Ollama (local)</option>
|
||||
<option value="custom">{{ t('settings.aiProvider.custom') }}</option>
|
||||
<option value="openai">{{ t('settings.aiProvider.providerOptions.openai') }}</option>
|
||||
<option value="ollama">{{ t('settings.aiProvider.providerOptions.ollama') }}</option>
|
||||
<option value="deepseek">{{ t('settings.aiProvider.providerOptions.deepseek') }}</option>
|
||||
<option value="groq">{{ t('settings.aiProvider.providerOptions.groq') }}</option>
|
||||
<option value="openrouter">{{ t('settings.aiProvider.providerOptions.openrouter') }}</option>
|
||||
<option value="opencode-go">{{ t('settings.aiProvider.providerOptions.opencode-go') }}</option>
|
||||
<option value="custom">{{ t('settings.aiProvider.providerOptions.custom') }}</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
@@ -171,10 +175,12 @@
|
||||
<i class="fas fa-info-circle info-icon" data-tooltip="{{ t('settings.aiProvider.apiBaseHelp') }}"></i>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<div class="text-input-wrapper">
|
||||
<div class="text-input-wrapper lm-combobox-container">
|
||||
<input type="text" id="llmApiBase"
|
||||
class="lm-combobox-input"
|
||||
value="{{ settings.get('llm_api_base', '') }}"
|
||||
placeholder="{{ t('settings.aiProvider.apiBasePlaceholder') }}"
|
||||
autocomplete="off"
|
||||
onblur="settingsManager.saveInputSetting('llmApiBase', 'llm_api_base')"
|
||||
onkeydown="if(event.key === 'Enter') { this.blur(); }" />
|
||||
</div>
|
||||
@@ -222,10 +228,12 @@
|
||||
<i class="fas fa-info-circle info-icon" data-tooltip="{{ t('settings.aiProvider.modelHelp') }}"></i>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<div class="text-input-wrapper">
|
||||
<div class="text-input-wrapper lm-combobox-container">
|
||||
<input type="text" id="llmModel"
|
||||
class="lm-combobox-input"
|
||||
value="{{ settings.get('llm_model', '') }}"
|
||||
placeholder="e.g. gpt-4o-mini"
|
||||
placeholder="{{ t('settings.aiProvider.modelPlaceholder') }}"
|
||||
autocomplete="off"
|
||||
onblur="settingsManager.saveInputSetting('llmModel', 'llm_model')"
|
||||
onkeydown="if(event.key === 'Enter') { this.blur(); }" />
|
||||
</div>
|
||||
@@ -234,6 +242,14 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Provider presets + model lists for frontend -->
|
||||
<script id="llmProviderPresets" type="application/json">
|
||||
{{ provider_presets_json | safe }}
|
||||
</script>
|
||||
<script id="llmProviderModels" type="application/json">
|
||||
{{ provider_models_json | safe }}
|
||||
</script>
|
||||
|
||||
<div class="settings-subsection">
|
||||
<div class="settings-subsection-header">
|
||||
<h4>{{ t('settings.sections.downloads') }}</h4>
|
||||
|
||||
@@ -12,7 +12,19 @@
|
||||
<div id="embeddingContextMenu" class="context-menu" style="display: none;">
|
||||
<!-- Metadata -->
|
||||
<div class="context-menu-item" data-action="refresh-metadata"><i class="fas fa-sync"></i> {{ t('loras.contextMenu.refreshMetadata') }}</div>
|
||||
<div class="context-menu-item" data-action="relink-civitai"><i class="fas fa-link"></i> {{ t('loras.contextMenu.relinkCivitai') }}</div>
|
||||
<div class="context-menu-item has-submenu" data-has-submenu="link-model">
|
||||
<i class="fas fa-link"></i>
|
||||
<span>{{ t('loras.contextMenu.linkModel') }}</span>
|
||||
<i class="fas fa-chevron-right submenu-arrow"></i>
|
||||
<div class="context-submenu">
|
||||
<div class="context-menu-item" data-action="relink-civitai">
|
||||
<i class="fas fa-external-link-alt"></i> <span>{{ t('loras.contextMenu.linkCivitai') }}</span>
|
||||
</div>
|
||||
<div class="context-menu-item" data-action="link-hf">
|
||||
<i class="fas fa-robot"></i> <span>{{ t('loras.contextMenu.linkHuggingFace') }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="context-menu-separator menu-section-break"></div>
|
||||
<!-- Workflow -->
|
||||
<div class="context-menu-item" data-action="copyname"><i class="fas fa-copy"></i> {{ t('loras.contextMenu.copyFilename') }}</div>
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# Test suite package.
|
||||
@@ -1,317 +0,0 @@
|
||||
"""Tests for the AgentCLI module (py/agent_cli/).
|
||||
|
||||
All tests mock the underlying services (scanner, MetadataManager, downloader)
|
||||
since the AgentCLI is a thin delegation layer.
|
||||
|
||||
Mock targets must match where imports are resolved inside each function
|
||||
(lazy imports via ``from X import Y`` inside function body).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from py.agent_cli import (
|
||||
list_base_models,
|
||||
read_metadata,
|
||||
apply_metadata_updates,
|
||||
download_preview,
|
||||
refresh_cache,
|
||||
)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Helpers
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class MockCache:
|
||||
def __init__(self, raw_data: list[dict] | None = None):
|
||||
self.raw_data = raw_data or []
|
||||
|
||||
|
||||
class MockScanner:
|
||||
"""Simulates a ModelScanner for testing."""
|
||||
|
||||
def __init__(self, raw_data: list[dict] | None = None):
|
||||
self._raw_data = raw_data or []
|
||||
self.update_single_model_cache = mock.AsyncMock(return_value=True)
|
||||
|
||||
async def get_cached_data(self):
|
||||
return MockCache(self._raw_data)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# list_base_models -- imports ServiceRegistry internally
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestListBaseModels:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_cache(self):
|
||||
scanner = MockScanner([])
|
||||
with mock.patch(
|
||||
"py.services.service_registry.ServiceRegistry",
|
||||
get_lora_scanner=mock.AsyncMock(return_value=scanner),
|
||||
get_checkpoint_scanner=mock.AsyncMock(return_value=None),
|
||||
get_embedding_scanner=mock.AsyncMock(return_value=None),
|
||||
):
|
||||
result = await list_base_models()
|
||||
assert result == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_merges_all_scanners(self):
|
||||
lora_scanner = MockScanner([
|
||||
{"base_model": "SDXL 1.0"},
|
||||
{"base_model": "Flux.1 D"},
|
||||
{"base_model": "SDXL 1.0"},
|
||||
])
|
||||
ckpt_scanner = MockScanner([
|
||||
{"base_model": "SDXL 1.0"},
|
||||
{"base_model": "SD 1.5"},
|
||||
])
|
||||
with mock.patch(
|
||||
"py.services.service_registry.ServiceRegistry",
|
||||
get_lora_scanner=mock.AsyncMock(return_value=lora_scanner),
|
||||
get_checkpoint_scanner=mock.AsyncMock(return_value=ckpt_scanner),
|
||||
get_embedding_scanner=mock.AsyncMock(return_value=None),
|
||||
):
|
||||
result = await list_base_models()
|
||||
assert result == ["SDXL 1.0", "Flux.1 D", "SD 1.5"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_limit(self):
|
||||
scanner = MockScanner([
|
||||
{"base_model": "A"}, {"base_model": "B"}, {"base_model": "C"},
|
||||
])
|
||||
with mock.patch(
|
||||
"py.services.service_registry.ServiceRegistry",
|
||||
get_lora_scanner=mock.AsyncMock(return_value=scanner),
|
||||
get_checkpoint_scanner=mock.AsyncMock(return_value=None),
|
||||
get_embedding_scanner=mock.AsyncMock(return_value=None),
|
||||
):
|
||||
result = await list_base_models(limit=2)
|
||||
assert result == ["A", "B"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_all_scanners_return_none(self):
|
||||
with mock.patch(
|
||||
"py.services.service_registry.ServiceRegistry",
|
||||
get_lora_scanner=mock.AsyncMock(return_value=None),
|
||||
get_checkpoint_scanner=mock.AsyncMock(return_value=None),
|
||||
get_embedding_scanner=mock.AsyncMock(return_value=None),
|
||||
):
|
||||
result = await list_base_models()
|
||||
assert result == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_skips_empty_or_missing_base_model(self):
|
||||
scanner = MockScanner([
|
||||
{"base_model": "SDXL 1.0"},
|
||||
{"file_name": "foo.safetensors"}, # no base_model key
|
||||
{"base_model": ""}, # empty
|
||||
])
|
||||
with mock.patch(
|
||||
"py.services.service_registry.ServiceRegistry",
|
||||
get_lora_scanner=mock.AsyncMock(return_value=scanner),
|
||||
get_checkpoint_scanner=mock.AsyncMock(return_value=None),
|
||||
get_embedding_scanner=mock.AsyncMock(return_value=None),
|
||||
):
|
||||
result = await list_base_models()
|
||||
assert result == ["SDXL 1.0"]
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# read_metadata -- imports MetadataManager from py.utils.metadata_manager
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestReadMetadata:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delegates_to_metadata_manager(self):
|
||||
fake = {"file_name": "test", "base_model": "SDXL 1.0"}
|
||||
with mock.patch("py.utils.metadata_manager.MetadataManager") as mm:
|
||||
mm.load_metadata_payload = mock.AsyncMock(return_value=fake)
|
||||
result = await read_metadata("/p.safetensors")
|
||||
assert result == fake
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exception_returns_empty_dict(self):
|
||||
with mock.patch("py.utils.metadata_manager.MetadataManager") as mm:
|
||||
mm.load_metadata_payload = mock.AsyncMock(side_effect=ValueError("x"))
|
||||
result = await read_metadata("/p.safetensors")
|
||||
assert result == {}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_none_coerces_to_empty_dict(self):
|
||||
with mock.patch("py.utils.metadata_manager.MetadataManager") as mm:
|
||||
mm.load_metadata_payload = mock.AsyncMock(return_value=None)
|
||||
result = await read_metadata("/p.safetensors")
|
||||
assert result == {}
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# apply_metadata_updates -- uses read_metadata + MetadataManager.save_metadata
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestApplyMetadataUpdates:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_updates_field(self):
|
||||
with (
|
||||
mock.patch("py.agent_cli.read_metadata") as mock_read,
|
||||
mock.patch("py.utils.metadata_manager.MetadataManager") as mm,
|
||||
):
|
||||
mock_read.return_value = {"base_model": "", "tags": []}
|
||||
mm.save_metadata = mock.AsyncMock(return_value=True)
|
||||
updated = await apply_metadata_updates(
|
||||
"/p.safetensors", {"base_model": "Flux.1 D"}
|
||||
)
|
||||
assert updated == ["base_model"]
|
||||
mm.save_metadata.assert_awaited_once_with(
|
||||
"/p.safetensors", {"base_model": "Flux.1 D", "tags": []},
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_noop_when_value_unchanged(self):
|
||||
with (
|
||||
mock.patch("py.agent_cli.read_metadata") as mock_read,
|
||||
mock.patch("py.utils.metadata_manager.MetadataManager") as mm,
|
||||
):
|
||||
mock_read.return_value = {"base_model": "Flux.1 D"}
|
||||
updated = await apply_metadata_updates(
|
||||
"/p.safetensors", {"base_model": "Flux.1 D"}
|
||||
)
|
||||
assert updated == []
|
||||
mm.save_metadata.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_fields(self):
|
||||
with (
|
||||
mock.patch("py.agent_cli.read_metadata") as mock_read,
|
||||
mock.patch("py.utils.metadata_manager.MetadataManager") as mm,
|
||||
):
|
||||
mm.save_metadata = mock.AsyncMock(return_value=True)
|
||||
mock_read.return_value = {
|
||||
"base_model": "", "modelDescription": "", "tags": [],
|
||||
}
|
||||
updated = await apply_metadata_updates(
|
||||
"/p.safetensors",
|
||||
{"base_model": "SDXL 1.0", "modelDescription": "A", "tags": ["flux"]},
|
||||
)
|
||||
assert sorted(updated) == sorted(["base_model", "modelDescription", "tags"])
|
||||
saved = mm.save_metadata.call_args[0][1]
|
||||
assert saved["base_model"] == "SDXL 1.0"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_updates_noop(self):
|
||||
with (
|
||||
mock.patch("py.agent_cli.read_metadata"),
|
||||
mock.patch("py.utils.metadata_manager.MetadataManager") as mm,
|
||||
):
|
||||
updated = await apply_metadata_updates("/p.safetensors", {})
|
||||
assert updated == []
|
||||
mm.save_metadata.assert_not_called()
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# download_preview -- imports get_downloader + ExifUtils
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestDownloadPreview:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_url_returns_false(self, tmp_path):
|
||||
mp = tmp_path / "m.safetensors"
|
||||
mp.write_bytes(b"fake")
|
||||
assert await download_preview(str(mp), "") is False
|
||||
assert await download_preview(str(mp), " ") is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_successful_download_and_optimise(self, tmp_path):
|
||||
mp = tmp_path / "t.safetensors"
|
||||
mp.write_bytes(b"fake")
|
||||
with (
|
||||
mock.patch("py.services.downloader.get_downloader") as get_dl,
|
||||
mock.patch("py.utils.exif_utils.ExifUtils") as exif,
|
||||
):
|
||||
dl = mock.AsyncMock()
|
||||
dl.download_to_memory = mock.AsyncMock(return_value=(True, b"raw", {}))
|
||||
get_dl.return_value = dl
|
||||
exif.optimize_image.return_value = (b"optimized_webp", {})
|
||||
result = await download_preview(str(mp), "https://ex.com/i.png")
|
||||
assert result is True
|
||||
assert (tmp_path / "t.webp").exists()
|
||||
assert (tmp_path / "t.webp").read_bytes() == b"optimized_webp"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_failure_returns_false(self, tmp_path):
|
||||
mp = tmp_path / "t.safetensors"
|
||||
mp.write_bytes(b"fake")
|
||||
with mock.patch("py.services.downloader.get_downloader") as get_dl:
|
||||
dl = mock.AsyncMock()
|
||||
dl.download_to_memory = mock.AsyncMock(return_value=(False, None, {}))
|
||||
dl.download_file = mock.AsyncMock(return_value=(False, None))
|
||||
get_dl.return_value = dl
|
||||
result = await download_preview(str(mp), "https://ex.com/i.png")
|
||||
assert result is False
|
||||
assert not (tmp_path / "t.webp").exists()
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# refresh_cache -- uses _find_scanner_for_model (ServiceRegistry)
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestRefreshCache:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_found_and_refreshed(self):
|
||||
scanner = MockScanner([{"file_path": "/some/path.safetensors"}])
|
||||
with (
|
||||
mock.patch(
|
||||
"py.services.service_registry.ServiceRegistry",
|
||||
get_lora_scanner=mock.AsyncMock(return_value=scanner),
|
||||
get_checkpoint_scanner=mock.AsyncMock(return_value=None),
|
||||
get_embedding_scanner=mock.AsyncMock(return_value=None),
|
||||
),
|
||||
mock.patch("py.agent_cli.read_metadata") as mock_read,
|
||||
):
|
||||
mock_read.return_value = {"base_model": "SDXL 1.0"}
|
||||
result = await refresh_cache("/some/path.safetensors")
|
||||
assert result is True
|
||||
scanner.update_single_model_cache.assert_awaited_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_not_found_in_any_scanner(self):
|
||||
scanner = MockScanner([])
|
||||
with mock.patch(
|
||||
"py.services.service_registry.ServiceRegistry",
|
||||
get_lora_scanner=mock.AsyncMock(return_value=scanner),
|
||||
get_checkpoint_scanner=mock.AsyncMock(return_value=None),
|
||||
get_embedding_scanner=mock.AsyncMock(return_value=None),
|
||||
):
|
||||
result = await refresh_cache("/nonexistent/path.safetensors")
|
||||
assert result is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_metadata_returns_false(self):
|
||||
scanner = MockScanner([{"file_path": "/some/path.safetensors"}])
|
||||
with (
|
||||
mock.patch(
|
||||
"py.services.service_registry.ServiceRegistry",
|
||||
get_lora_scanner=mock.AsyncMock(return_value=scanner),
|
||||
get_checkpoint_scanner=mock.AsyncMock(return_value=None),
|
||||
get_embedding_scanner=mock.AsyncMock(return_value=None),
|
||||
),
|
||||
mock.patch("py.agent_cli.read_metadata") as mock_read,
|
||||
):
|
||||
mock_read.return_value = {}
|
||||
result = await refresh_cache("/some/path.safetensors")
|
||||
assert result is False
|
||||
@@ -0,0 +1 @@
|
||||
# HF Metadata Enrichment validation suite.
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Configuration for the HF metadata enrichment validation suite.
|
||||
|
||||
Loads user settings, defines paths, and pulls constants from the main
|
||||
codebase (``py.utils.constants``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Default paths
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DEFAULT_MODELS_FILE = os.path.join(
|
||||
os.path.dirname(__file__), "test_data", "hf_lora_models_with_safetensors.txt"
|
||||
)
|
||||
_DEFAULT_SETTINGS_PATH = os.path.expanduser(
|
||||
"~/.config/ComfyUI-LoRA-Manager/settings.json"
|
||||
)
|
||||
_DEFAULT_OUTPUT_DIR = "/tmp/hf_enrich_validation"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants from the main codebase (copied at import time)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Priority tags used in the LLM prompt for tag selection guidance.
|
||||
CIVITAI_MODEL_TAGS: List[str] = [
|
||||
"character", "concept", "clothing", "realistic", "anime", "toon",
|
||||
"furry", "style", "poses", "background", "tool", "vehicle",
|
||||
"buildings", "objects", "assets", "animal", "action",
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Base model resolution — dynamically fetched from production code
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Module-level cache — populated by init_supported_base_models().
|
||||
# Falls back to a comprehensive hardcoded list when the live fetch fails.
|
||||
SUPPORTED_BASE_MODELS: List[str] = []
|
||||
|
||||
# Fallback base models when the production list_base_models() is unavailable.
|
||||
_FALLBACK_BASE_MODELS: List[str] = [
|
||||
"SD 1.4", "SD 1.5", "SD 1.5 LCM", "SD 1.5 Hyper",
|
||||
"SD 2.0", "SD 2.1",
|
||||
"SD 3", "SD 3.5", "SD 3.5 Medium", "SD 3.5 Large", "SD 3.5 Large Turbo",
|
||||
"SDXL 1.0", "SDXL Lightning", "SDXL Hyper",
|
||||
"Flux.1 D", "Flux.1 S", "Flux.1 Krea", "Flux.1 Kontext",
|
||||
"Flux.2 D", "Flux.2 Klein 9B", "Flux.2 Klein 9B-base",
|
||||
"Flux.2 Klein 4B", "Flux.2 Klein 4B-base",
|
||||
"AuraFlow", "Chroma", "PixArt a", "PixArt E",
|
||||
"Hunyuan 1", "Lumina", "Kolors",
|
||||
"NoobAI", "Illustrious", "Pony", "Pony V7",
|
||||
"HiDream", "Qwen", "ZImageTurbo", "ZImageBase",
|
||||
"SVD", "LTXV", "LTXV2", "LTXV 2.3",
|
||||
"CogVideoX", "Mochi",
|
||||
"Wan Video", "Wan Video 1.3B t2v", "Wan Video 14B t2v",
|
||||
"Wan Video 14B i2v 480p", "Wan Video 14B i2v 720p",
|
||||
"Wan Video 2.2 TI2V-5B", "Wan Video 2.2 T2V-A14B",
|
||||
"Wan Video 2.2 I2V-A14B",
|
||||
"Wan Video 2.5 T2V", "Wan Video 2.5 I2V",
|
||||
"Hunyuan Video", "Anima", "Ernie", "Ernie Turbo",
|
||||
"Nucleus", "Krea 2",
|
||||
]
|
||||
|
||||
|
||||
async def init_supported_base_models() -> None:
|
||||
"""Populate ``SUPPORTED_BASE_MODELS`` from the production codebase.
|
||||
|
||||
Calls ``py.metadata_ops.list_base_models()`` which merges a hardcoded
|
||||
fallback with models fetched from the CivitAI API. When the call
|
||||
fails (e.g. offline, API error), falls back to ``_FALLBACK_BASE_MODELS``.
|
||||
|
||||
Must be called from within an async event loop (i.e. during
|
||||
``run_validation.main()``, not at module level).
|
||||
"""
|
||||
try:
|
||||
from py.metadata_ops import list_base_models
|
||||
|
||||
models = await list_base_models()
|
||||
if models:
|
||||
SUPPORTED_BASE_MODELS[:] = models
|
||||
logger.info("Loaded %d base models from production code", len(models))
|
||||
return
|
||||
logger.warning("list_base_models returned empty list, using fallback")
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to load base models from production: %s", exc)
|
||||
|
||||
SUPPORTED_BASE_MODELS[:] = _FALLBACK_BASE_MODELS
|
||||
logger.info("Using fallback base model list (%d entries)", len(SUPPORTED_BASE_MODELS))
|
||||
|
||||
|
||||
# Placeholder values the LLM sometimes emits that should count as "empty".
|
||||
PLACEHOLDER_VALUES = frozenset({
|
||||
"none", "null", "n/a", "unknown", "not available",
|
||||
"not specified", "no trigger words", "no trigger word",
|
||||
})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# User settings loader
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def load_settings(settings_path: str) -> Dict[str, Any]:
|
||||
"""Load LoRA Manager settings from *settings_path*.
|
||||
|
||||
Returns a flat dict with the LLM configuration fields that the
|
||||
enrichment pipeline depends on.
|
||||
"""
|
||||
path = os.path.expanduser(settings_path)
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(
|
||||
f"Settings file not found: {path}\n"
|
||||
"Please provide a valid --settings path."
|
||||
)
|
||||
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
raw: Dict[str, Any] = json.load(fh)
|
||||
|
||||
# Extract LLM-relevant config
|
||||
return {
|
||||
"llm_provider": raw.get("llm_provider", "ollama"),
|
||||
"llm_model": raw.get("llm_model", "qwen3.5:9b"),
|
||||
"llm_api_base": raw.get("llm_api_base", "http://localhost:11434/v1"),
|
||||
"llm_api_key": raw.get("llm_api_key", ""),
|
||||
"settings_path": path,
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Execute the ``enrich_hf_metadata`` skill serially over a list of models.
|
||||
|
||||
Design decisions (local Ollama, no rate limits):
|
||||
|
||||
- Sequential execution: one model at a time. 100 models at ~30-90 s/call
|
||||
→ roughly 1-2 h total.
|
||||
- Progress persisted to a JSON checkpoint file so the run can be resumed
|
||||
with ``--resume``.
|
||||
- Per-model timeout guards against a stuck Ollama inference.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SKILL_NAME = "enrich_hf_metadata"
|
||||
|
||||
# How long to wait for a single LLM call before marking it timed-out.
|
||||
_PER_MODEL_TIMEOUT = 240 # seconds
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Progress checkpoint helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PROGRESS_FILE = "progress.json"
|
||||
|
||||
|
||||
def _load_progress(output_dir: str) -> Dict[str, Any]:
|
||||
path = os.path.join(output_dir, _PROGRESS_FILE)
|
||||
if os.path.exists(path):
|
||||
with open(path, "r") as fh:
|
||||
return json.load(fh)
|
||||
return {"completed": [], "failed": [], "timed_out": []}
|
||||
|
||||
|
||||
def _save_progress(output_dir: str, progress: Dict[str, Any]) -> None:
|
||||
path = os.path.join(output_dir, _PROGRESS_FILE)
|
||||
with open(path, "w") as fh:
|
||||
json.dump(progress, fh, indent=2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core runner
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class EnrichmentRunner:
|
||||
"""Serial enrichment runner with checkpoint resume."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
output_dir: str,
|
||||
*,
|
||||
per_model_timeout: int = _PER_MODEL_TIMEOUT,
|
||||
) -> None:
|
||||
self._output_dir = output_dir
|
||||
self._per_model_timeout = per_model_timeout
|
||||
self._agent_service: Optional[Any] = None
|
||||
|
||||
async def _ensure_agent_service(self) -> Any:
|
||||
"""Lazy-init AgentService (expensive — needs LLMService init)."""
|
||||
if self._agent_service is not None:
|
||||
return self._agent_service
|
||||
from py.services.agent.agent_service import AgentService
|
||||
|
||||
self._agent_service = await AgentService.get_instance()
|
||||
return self._agent_service
|
||||
|
||||
async def run(
|
||||
self,
|
||||
model_paths: List[str],
|
||||
repos: List[str],
|
||||
) -> Dict[str, Any]:
|
||||
"""Run enrichment over *model_paths* (one-by-one).
|
||||
|
||||
Args:
|
||||
model_paths: model paths in the same order as *repos*.
|
||||
repos: HF repo IDs (for display / checkpoint labelling).
|
||||
|
||||
Returns:
|
||||
A dict with keys ``results``, ``progress``, ``durations``.
|
||||
"""
|
||||
assert len(model_paths) == len(repos)
|
||||
|
||||
progress = _load_progress(self._output_dir)
|
||||
completed_set = set(progress["completed"])
|
||||
failed_set = set(progress["failed"])
|
||||
timed_out_set = set(progress.get("timed_out", []))
|
||||
|
||||
agent = await self._ensure_agent_service()
|
||||
results: List[Dict[str, Any]] = []
|
||||
durations: Dict[str, float] = {}
|
||||
|
||||
total = len(model_paths)
|
||||
processed_before = len(completed_set | failed_set | timed_out_set)
|
||||
|
||||
logger.info(
|
||||
"Enrichment runner: %d models total, %d already processed",
|
||||
total,
|
||||
processed_before,
|
||||
)
|
||||
|
||||
for idx, (model_path, repo_id) in enumerate(zip(model_paths, repos)):
|
||||
if repo_id in completed_set:
|
||||
logger.info("[%d/%d] SKIP (already done): %s", idx + 1, total, repo_id)
|
||||
continue
|
||||
if repo_id in failed_set or repo_id in timed_out_set:
|
||||
logger.info(
|
||||
"[%d/%d] SKIP (previously failed/timeout): %s",
|
||||
idx + 1, total, repo_id,
|
||||
)
|
||||
continue
|
||||
|
||||
logger.info(
|
||||
"[%d/%d] Enriching %s ...", idx + 1, total, repo_id,
|
||||
)
|
||||
t0 = time.perf_counter()
|
||||
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
agent.execute_skill(
|
||||
skill_name=_SKILL_NAME,
|
||||
input_data={"model_paths": [model_path]},
|
||||
progress_callback=None,
|
||||
),
|
||||
timeout=self._per_model_timeout,
|
||||
)
|
||||
|
||||
elapsed = time.perf_counter() - t0
|
||||
durations[repo_id] = round(elapsed, 2)
|
||||
|
||||
if result.success:
|
||||
completed_set.add(repo_id)
|
||||
progress["completed"].append(repo_id)
|
||||
logger.info(
|
||||
" ✓ %s (%.1f s) — %s",
|
||||
repo_id, elapsed, result.summary,
|
||||
)
|
||||
else:
|
||||
failed_set.add(repo_id)
|
||||
progress["failed"].append(repo_id)
|
||||
logger.warning(
|
||||
" ✗ %s (%.1f s) — %s",
|
||||
repo_id, elapsed,
|
||||
"; ".join(result.errors) if result.errors else result.summary,
|
||||
)
|
||||
|
||||
results.append({
|
||||
"repo_id": repo_id,
|
||||
"model_path": model_path,
|
||||
"success": result.success,
|
||||
"updated_fields": result.updated_models,
|
||||
"errors": result.errors,
|
||||
"summary": result.summary,
|
||||
"duration_s": round(elapsed, 2),
|
||||
})
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
elapsed = time.perf_counter() - t0
|
||||
durations[repo_id] = round(elapsed, 2)
|
||||
timed_out_set.add(repo_id)
|
||||
progress.setdefault("timed_out", []).append(repo_id)
|
||||
logger.warning(
|
||||
" ⏱ TIMEOUT %s (%.1f s, limit=%ds)",
|
||||
repo_id, elapsed, self._per_model_timeout,
|
||||
)
|
||||
results.append({
|
||||
"repo_id": repo_id,
|
||||
"model_path": model_path,
|
||||
"success": False,
|
||||
"errors": [f"Timeout after {self._per_model_timeout}s"],
|
||||
"summary": "LLM call timed out",
|
||||
"duration_s": round(elapsed, 2),
|
||||
})
|
||||
|
||||
except Exception as exc:
|
||||
elapsed = time.perf_counter() - t0
|
||||
durations[repo_id] = round(elapsed, 2)
|
||||
failed_set.add(repo_id)
|
||||
progress["failed"].append(repo_id)
|
||||
logger.error(
|
||||
" ✗ %s (%.1f s) — %s",
|
||||
repo_id, elapsed, exc,
|
||||
)
|
||||
results.append({
|
||||
"repo_id": repo_id,
|
||||
"model_path": model_path,
|
||||
"success": False,
|
||||
"errors": [str(exc)],
|
||||
"summary": f"Exception: {exc}",
|
||||
"duration_s": round(elapsed, 2),
|
||||
})
|
||||
|
||||
# Checkpoint after each model
|
||||
_save_progress(self._output_dir, progress)
|
||||
|
||||
return {
|
||||
"results": results,
|
||||
"progress": progress,
|
||||
"durations": durations,
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
"""Evaluate enriched ``.metadata.json`` quality across multiple dimensions.
|
||||
|
||||
Scoring rubric (per field):
|
||||
|
||||
- **Completeness**: Is the field populated with meaningful content?
|
||||
- **Validity**: Does the value conform to expected constraints (controlled
|
||||
vocab, non-placeholder, parsable JSON)?
|
||||
- **Accuracy**: (sub-sample only — requires manual verification against
|
||||
the HF README).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
from .config import (
|
||||
CIVITAI_MODEL_TAGS,
|
||||
PLACEHOLDER_VALUES,
|
||||
SUPPORTED_BASE_MODELS,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scoring helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MIN_TAGS = 1
|
||||
_MAX_TAGS = 8
|
||||
_MIN_DESC_LENGTH = 20
|
||||
_MIN_NOTES_LENGTH = 30
|
||||
|
||||
# Tags that the LLM sometimes emits but which are not meaningful content tags.
|
||||
_TECH_TAGS = frozenset({
|
||||
"lora", "dreambooth", "text-to-image", "diffusers", "flux",
|
||||
"sdxl", "checkpoint", "pytorch", "safetensors", "fine-tuning",
|
||||
"stable-diffusion", "training", "stablediffusion",
|
||||
})
|
||||
|
||||
|
||||
def _is_placeholder(val: str) -> bool:
|
||||
return val.strip().lower() in PLACEHOLDER_VALUES
|
||||
|
||||
|
||||
def _is_valid_trigger_words(words: List[str]) -> bool:
|
||||
"""Return True if *words* is a non-empty list of real trigger words."""
|
||||
if not words:
|
||||
return False
|
||||
cleaned = [w.strip() for w in words if w.strip()]
|
||||
if not cleaned:
|
||||
return False
|
||||
# Reject if ALL entries are placeholders
|
||||
non_placeholder = [w for w in cleaned if not _is_placeholder(w)]
|
||||
return len(non_placeholder) > 0
|
||||
|
||||
|
||||
def _is_valid_tags(tags: List[str]) -> bool:
|
||||
"""Return True if *tags* is a reasonable list of content tags."""
|
||||
if not tags:
|
||||
return False
|
||||
cleaned = [t.strip().lower() for t in tags if t.strip()]
|
||||
if not cleaned:
|
||||
return False
|
||||
# At least one tag that isn't a technical keyword
|
||||
meaningful = [t for t in cleaned if t not in _TECH_TAGS]
|
||||
return len(meaningful) >= _MIN_TAGS
|
||||
|
||||
|
||||
def _tag_priority_coverage(tags: List[str]) -> float:
|
||||
"""Fraction of tags that align with the user's priority tag vocabulary."""
|
||||
if not tags:
|
||||
return 0.0
|
||||
priority_lower = {t.lower() for t in CIVITAI_MODEL_TAGS}
|
||||
matched = sum(1 for t in tags if t.strip().lower() in priority_lower)
|
||||
return matched / len(tags)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-model evaluation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Type alias for a score record
|
||||
ScoreRecord = Dict[str, Any]
|
||||
|
||||
|
||||
def evaluate_model(
|
||||
metadata: Dict[str, Any],
|
||||
model_path: str,
|
||||
repo_id: str,
|
||||
*,
|
||||
enrichment_success: bool,
|
||||
enrichment_errors: List[str],
|
||||
) -> ScoreRecord:
|
||||
"""Score a single enriched model's metadata.
|
||||
|
||||
Returns a dict with per-field scores, a total score, and a list of
|
||||
flagged issues.
|
||||
"""
|
||||
civitai = metadata.get("civitai") or {}
|
||||
trained_words: List[str] = civitai.get("trainedWords") or metadata.get("trainedWords") or []
|
||||
short_desc: str = civitai.get("description") or ""
|
||||
tags: List[str] = metadata.get("tags") or []
|
||||
notes: str = metadata.get("notes") or ""
|
||||
usage_tips_raw: str = metadata.get("usage_tips") or "{}"
|
||||
model_description: str = metadata.get("modelDescription") or ""
|
||||
base_model: str = metadata.get("base_model") or ""
|
||||
preview_url: str = metadata.get("preview_url") or ""
|
||||
confidence: str = metadata.get("_llm_confidence") or ""
|
||||
|
||||
# --- base_model ---
|
||||
base_model_valid = base_model in SUPPORTED_BASE_MODELS
|
||||
base_model_filled = bool(base_model) and base_model != "Unknown"
|
||||
|
||||
# --- trigger_words (trainedWords) ---
|
||||
triggers_valid = _is_valid_trigger_words(trained_words)
|
||||
|
||||
# --- short_description (civitai.description) ---
|
||||
desc_filled = len(short_desc.strip()) >= _MIN_DESC_LENGTH
|
||||
|
||||
# --- tags ---
|
||||
tags_valid = _is_valid_tags(tags)
|
||||
tags_priority_coverage = _tag_priority_coverage(tags)
|
||||
tags_no_technical = (
|
||||
sum(1 for t in tags if t.strip().lower() not in _TECH_TAGS) >= _MIN_TAGS
|
||||
if tags else False
|
||||
)
|
||||
|
||||
# --- notes ---
|
||||
notes_filled = len(notes.strip()) >= _MIN_NOTES_LENGTH
|
||||
|
||||
# --- usage_tips ---
|
||||
usage_tips_valid = False
|
||||
if usage_tips_raw.strip() and usage_tips_raw.strip() != "{}":
|
||||
try:
|
||||
parsed = json.loads(usage_tips_raw)
|
||||
if isinstance(parsed, dict) and len(parsed) > 0:
|
||||
usage_tips_valid = True
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
|
||||
# --- modelDescription (README → HTML) ---
|
||||
desc_html_filled = len(model_description.strip()) > 100
|
||||
|
||||
# --- preview_url ---
|
||||
preview_filled = bool(preview_url) and os.path.exists(preview_url)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Composite score (0-100)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
field_scores = {
|
||||
"base_model": _score_bool(base_model_filled and base_model_valid, weight=15),
|
||||
"trigger_words": _score_bool(triggers_valid, weight=15),
|
||||
"short_description": _score_bool(desc_filled, weight=10),
|
||||
"tags": _score_bool(tags_valid, weight=15),
|
||||
"tags_priority_coverage": _score_continuous(tags_priority_coverage, weight=5),
|
||||
"notes": _score_bool(notes_filled, weight=5),
|
||||
"usage_tips": _score_bool(usage_tips_valid, weight=5),
|
||||
"modelDescription_html": _score_bool(desc_html_filled, weight=10),
|
||||
"preview_downloaded": _score_bool(preview_filled, weight=10),
|
||||
}
|
||||
|
||||
# Deduct points for enrichment-level failures
|
||||
penalty = 0
|
||||
if enrichment_errors:
|
||||
penalty += 10
|
||||
if not enrichment_success:
|
||||
penalty += 20
|
||||
|
||||
total_raw = sum(field_scores.values())
|
||||
total = max(0, min(100, total_raw - penalty))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Flagged issues
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
issues: List[str] = []
|
||||
if not base_model_filled:
|
||||
issues.append("base_model is empty or 'Unknown'")
|
||||
elif not base_model_valid:
|
||||
issues.append(f"base_model '{base_model}' not in SUPPORTED_BASE_MODELS")
|
||||
if not triggers_valid:
|
||||
issues.append("trigger_words are missing or contain only placeholders")
|
||||
if not desc_filled:
|
||||
issues.append("short_description is too short or empty")
|
||||
if not tags_valid:
|
||||
issues.append("tags are missing, too few, or purely technical")
|
||||
if tags_valid and tags_priority_coverage < 0.5:
|
||||
issues.append("tags have low overlap with priority_tags (< 50%)")
|
||||
if not notes_filled:
|
||||
issues.append("notes are too short or empty")
|
||||
if not usage_tips_valid:
|
||||
issues.append("usage_tips is empty or invalid JSON")
|
||||
if not desc_html_filled:
|
||||
issues.append("modelDescription is too short (README may not have been converted)")
|
||||
if not preview_filled:
|
||||
issues.append("preview image not downloaded (URL missing or download failed)")
|
||||
|
||||
return {
|
||||
"repo_id": repo_id,
|
||||
"model_path": model_path,
|
||||
"enrichment_success": enrichment_success,
|
||||
"total_score": total,
|
||||
"field_scores": field_scores,
|
||||
"issues": issues,
|
||||
"confidence_from_llm": confidence,
|
||||
"raw_values": {
|
||||
"base_model": base_model,
|
||||
"trigger_words": trained_words,
|
||||
"short_description": short_desc,
|
||||
"tags": tags,
|
||||
"notes": notes,
|
||||
"usage_tips": usage_tips_raw,
|
||||
"preview_url": preview_url,
|
||||
"has_modelDescription": len(model_description) > 0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _score_bool(condition: bool, weight: int = 10) -> int:
|
||||
return weight if condition else 0
|
||||
|
||||
|
||||
def _score_continuous(value: float, weight: int = 10) -> int:
|
||||
"""Linear interpolation: value 0.0 → 0, value 1.0 → *weight*."""
|
||||
return int(round(value * weight))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Batch evaluation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def evaluate_batch(
|
||||
enriched: List[Dict[str, Any]],
|
||||
) -> List[ScoreRecord]:
|
||||
"""Evaluate a list of enrichment results.
|
||||
|
||||
Each entry in *enriched* should have keys:
|
||||
``repo_id``, ``model_path``, ``metadata`` (the enriched dict),
|
||||
``success``, ``errors``.
|
||||
"""
|
||||
scores: List[ScoreRecord] = []
|
||||
for entry in enriched:
|
||||
record = evaluate_model(
|
||||
metadata=entry.get("metadata", {}),
|
||||
model_path=entry.get("model_path", ""),
|
||||
repo_id=entry.get("repo_id", ""),
|
||||
enrichment_success=entry.get("success", False),
|
||||
enrichment_errors=entry.get("errors", []),
|
||||
)
|
||||
scores.append(record)
|
||||
return scores
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Aggregate statistics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def aggregate_scores(scores: List[ScoreRecord]) -> Dict[str, Any]:
|
||||
"""Compute aggregate stats across all scored models."""
|
||||
n = len(scores)
|
||||
if n == 0:
|
||||
return {"error": "no scores to aggregate"}
|
||||
|
||||
field_names = [
|
||||
"base_model", "trigger_words", "short_description", "tags",
|
||||
"tags_priority_coverage", "notes", "usage_tips",
|
||||
"modelDescription_html", "preview_downloaded",
|
||||
]
|
||||
possible = {f: 15 if f == "base_model" or f == "trigger_words" or f == "tags" else
|
||||
10 if f == "short_description" or f == "modelDescription_html" or f == "preview_downloaded" else
|
||||
5
|
||||
for f in field_names}
|
||||
|
||||
# Per-field aggregate
|
||||
field_agg: Dict[str, Any] = {}
|
||||
for fn in field_names:
|
||||
vals = [s["field_scores"].get(fn, 0) for s in scores]
|
||||
max_per_field = possible[fn]
|
||||
field_agg[fn] = {
|
||||
"mean": round(sum(vals) / n, 1) if n else 0,
|
||||
"fill_rate_pct": round(
|
||||
sum(1 for v in vals if v >= max_per_field) / n * 100, 1
|
||||
) if n else 0.0,
|
||||
"partial_rate_pct": round(
|
||||
sum(1 for v in vals if 0 < v < max_per_field) / n * 100, 1
|
||||
) if n else 0.0,
|
||||
"empty_rate_pct": round(
|
||||
sum(1 for v in vals if v == 0) / n * 100, 1
|
||||
) if n else 0.0,
|
||||
}
|
||||
|
||||
# Total score distribution
|
||||
total_scores = [s["total_score"] for s in scores]
|
||||
total_agg = {
|
||||
"mean": round(sum(total_scores) / n, 1) if n else 0,
|
||||
"median": _median(total_scores),
|
||||
"min": min(total_scores) if total_scores else 0,
|
||||
"max": max(total_scores) if total_scores else 0,
|
||||
"bins": {
|
||||
"excellent_80+": sum(1 for s in total_scores if s >= 80),
|
||||
"good_60_79": sum(1 for s in total_scores if 60 <= s < 80),
|
||||
"fair_40_59": sum(1 for s in total_scores if 40 <= s < 60),
|
||||
"poor_20_39": sum(1 for s in total_scores if 20 <= s < 40),
|
||||
"bad_0_19": sum(1 for s in total_scores if s < 20),
|
||||
},
|
||||
}
|
||||
|
||||
# Issue frequency
|
||||
issue_counter: Dict[str, int] = {}
|
||||
for s in scores:
|
||||
for issue in s["issues"]:
|
||||
issue_counter[issue] = issue_counter.get(issue, 0) + 1
|
||||
top_issues = sorted(issue_counter.items(), key=lambda x: -x[1])
|
||||
|
||||
# Confidence distribution
|
||||
conf_counter: Dict[str, int] = {"high": 0, "medium": 0, "low": 0, "": 0}
|
||||
for s in scores:
|
||||
c = (s.get("confidence_from_llm") or "").strip().lower()
|
||||
if c in conf_counter:
|
||||
conf_counter[c] += 1
|
||||
else:
|
||||
conf_counter[""] += 1
|
||||
|
||||
# Success / timeout / failure stats
|
||||
success_count = sum(1 for s in scores if s["enrichment_success"])
|
||||
fail_count = n - success_count
|
||||
|
||||
return {
|
||||
"model_count": n,
|
||||
"success_count": success_count,
|
||||
"fail_count": fail_count,
|
||||
"total_score": total_agg,
|
||||
"field_aggregates": field_agg,
|
||||
"top_issues": top_issues[:15],
|
||||
"confidence_distribution": conf_counter,
|
||||
}
|
||||
|
||||
|
||||
def _median(values: List[float]) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
sorted_v = sorted(values)
|
||||
m = len(sorted_v) // 2
|
||||
if len(sorted_v) % 2 == 0:
|
||||
return round((sorted_v[m - 1] + sorted_v[m]) / 2, 1)
|
||||
return round(sorted_v[m], 1)
|
||||
@@ -0,0 +1,202 @@
|
||||
"""Construct initial ``.metadata.json`` sidecars for HF model repos.
|
||||
|
||||
Each HF repo + safetensors pair gets a minimal metadata file — no real model
|
||||
file is needed. The enrichment pipeline reads only the sidecar.
|
||||
|
||||
Data format (one line per entry)::
|
||||
|
||||
repo_id, model_name.safetensors
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from .config import CIVITAI_MODEL_TAGS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data types
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# A validated entry parsed from the models file:
|
||||
# (repo_id, safetensors_name)
|
||||
RepoEntry = Tuple[str, str]
|
||||
|
||||
|
||||
def load_repo_ids(path: str, max_models: int | None = None) -> List[RepoEntry]:
|
||||
"""Read ``repo_id, safetensors_name`` pairs from *path*.
|
||||
|
||||
Format (one per line, blanks and ``#`` comments ignored)::
|
||||
|
||||
user/repo-name, lora_zimage_turbo_myjs_alpha01.safetensors
|
||||
|
||||
Returns a list of ``(repo_id, safetensors_name)`` tuples.
|
||||
"""
|
||||
path = os.path.expanduser(path)
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(f"Models file not found: {path}")
|
||||
|
||||
entries: List[RepoEntry] = []
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
for raw_line in fh:
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
|
||||
# Split on the first comma
|
||||
if "," not in line:
|
||||
logger.warning("Skipping malformed line (no comma): %s", raw_line.rstrip())
|
||||
continue
|
||||
|
||||
repo_id, safetensors_name = [part.strip() for part in line.split(",", 1)]
|
||||
if not repo_id or not safetensors_name:
|
||||
logger.warning("Skipping malformed line (empty fields): %s", raw_line.rstrip())
|
||||
continue
|
||||
if not safetensors_name.lower().endswith(".safetensors"):
|
||||
logger.warning(
|
||||
"Skipping line — safetensors_name doesn't end with .safetensors: %s",
|
||||
raw_line.rstrip(),
|
||||
)
|
||||
continue
|
||||
|
||||
entries.append((repo_id, safetensors_name))
|
||||
|
||||
if max_models is not None and max_models > 0:
|
||||
entries = entries[:max_models]
|
||||
|
||||
logger.info("Loaded %d HF repo entries from %s", len(entries), path)
|
||||
return entries
|
||||
|
||||
|
||||
def sanitize_repo_id(repo_id: str) -> str:
|
||||
"""Turn ``user/repo-name`` into a safe directory name."""
|
||||
return repo_id.replace("/", "__").replace(".", "_")
|
||||
|
||||
|
||||
def build_model_dir(output_dir: str, repo_id: str) -> str:
|
||||
"""Return the per-model working directory."""
|
||||
return os.path.join(output_dir, "models", sanitize_repo_id(repo_id))
|
||||
|
||||
|
||||
def build_model_path(model_dir: str, safetensors_name: str) -> str:
|
||||
"""Return the model file path using the real safetensors filename."""
|
||||
return os.path.join(model_dir, safetensors_name)
|
||||
|
||||
|
||||
def build_metadata_path(model_path: str) -> str:
|
||||
"""Return the sidecar path for a model file.
|
||||
|
||||
This MUST match the convention used by ``MetadataManager`` /
|
||||
``apply_metadata_updates``, which derives the sidecar path via
|
||||
``os.path.splitext(model_path)[0] + '.metadata.json'``.
|
||||
For a model file ``lora_x.safetensors`` the sidecar is
|
||||
``lora_x.metadata.json`` — *not* ``lora_x.safetensors.metadata.json``.
|
||||
"""
|
||||
return f"{os.path.splitext(model_path)[0]}.metadata.json"
|
||||
|
||||
|
||||
def create_initial_metadata(
|
||||
output_dir: str,
|
||||
repo_id: str,
|
||||
safetensors_name: str,
|
||||
) -> str:
|
||||
"""Write a minimal ``.metadata.json`` for *repo_id* + *safetensors_name*.
|
||||
|
||||
Args:
|
||||
output_dir: Root output directory.
|
||||
repo_id: HuggingFace repo identifier (``user/repo``).
|
||||
safetensors_name: The specific model file name (e.g.
|
||||
``lora_zimage_turbo_myjs_alpha01.safetensors``).
|
||||
|
||||
Returns the **model path** (the ``.safetensors`` path whose sidecar was
|
||||
written). The caller passes this path to ``AgentService.execute_skill``.
|
||||
The basename (filename without extension) will match the real model file,
|
||||
so ``extract_relevant_section`` can reliably match against the README.
|
||||
"""
|
||||
model_dir = build_model_dir(output_dir, repo_id)
|
||||
os.makedirs(model_dir, exist_ok=True)
|
||||
model_path = build_model_path(model_dir, safetensors_name)
|
||||
metadata_path = build_metadata_path(model_path)
|
||||
|
||||
hf_url = f"https://huggingface.co/{repo_id}"
|
||||
file_name = safetensors_name
|
||||
|
||||
metadata: Dict[str, Any] = {
|
||||
"file_name": file_name,
|
||||
"model_name": safetensors_name,
|
||||
"file_path": model_path.replace(os.sep, "/"),
|
||||
"size": 0,
|
||||
"modified": 0,
|
||||
"sha256": "",
|
||||
"base_model": "Unknown",
|
||||
"preview_url": "",
|
||||
"preview_nsfw_level": 0,
|
||||
"notes": "",
|
||||
"from_civitai": False,
|
||||
"civitai": {},
|
||||
"tags": [],
|
||||
"modelDescription": "",
|
||||
"civitai_deleted": False,
|
||||
"favorite": False,
|
||||
"exclude": False,
|
||||
"db_checked": False,
|
||||
"skip_metadata_refresh": False,
|
||||
"metadata_source": "",
|
||||
"last_checked_at": 0,
|
||||
"hash_status": "completed",
|
||||
"trainedWords": [],
|
||||
"hf_url": hf_url,
|
||||
"usage_tips": "{}",
|
||||
}
|
||||
|
||||
with open(metadata_path, "w", encoding="utf-8") as fh:
|
||||
json.dump(metadata, fh, indent=2, ensure_ascii=False)
|
||||
|
||||
logger.debug("Created initial metadata for %s -> %s", repo_id, metadata_path)
|
||||
return model_path
|
||||
|
||||
|
||||
def create_all_initial_metadata(
|
||||
entries: List[RepoEntry],
|
||||
output_dir: str,
|
||||
*,
|
||||
skip_existing: bool = True,
|
||||
) -> Tuple[List[str], List[str]]:
|
||||
"""Create initial metadata for every repo entry.
|
||||
|
||||
Args:
|
||||
entries: List of ``(repo_id, safetensors_name)`` tuples.
|
||||
output_dir: Root output directory.
|
||||
skip_existing: If True, skip repos whose metadata already exists.
|
||||
|
||||
Returns:
|
||||
A tuple ``(model_paths, repo_ids)`` — two parallel lists in the same
|
||||
order as *entries*. This keeps downstream code (enrichment runner,
|
||||
evaluation engine) unchanged.
|
||||
"""
|
||||
model_paths: List[str] = []
|
||||
repo_ids: List[str] = []
|
||||
for repo_id, safetensors_name in entries:
|
||||
model_dir = build_model_dir(output_dir, repo_id)
|
||||
model_path = build_model_path(model_dir, safetensors_name)
|
||||
metadata_path = build_metadata_path(model_path)
|
||||
|
||||
if skip_existing and os.path.exists(metadata_path):
|
||||
model_paths.append(model_path)
|
||||
repo_ids.append(repo_id)
|
||||
continue
|
||||
|
||||
model_paths.append(create_initial_metadata(output_dir, repo_id, safetensors_name))
|
||||
repo_ids.append(repo_id)
|
||||
|
||||
logger.info(
|
||||
"Constructed initial metadata for %d/%d repos",
|
||||
len(model_paths),
|
||||
len(entries),
|
||||
)
|
||||
return model_paths, repo_ids
|
||||
@@ -0,0 +1,467 @@
|
||||
"""Preprocessing audit for the HF metadata enrichment validation pipeline.
|
||||
|
||||
Phase 1.5 — runs between Phase 1 (metadata creation) and Phase 2 (enrichment).
|
||||
|
||||
Audits the README preprocessing pipeline (section extraction + cleaning)
|
||||
for each repo in the dataset, capturing intermediate outputs so we can
|
||||
distinguish between:
|
||||
|
||||
(A) Preprocessing failed → LLM never saw the right content
|
||||
(B) Preprocessing succeeded → LLM/prompt needs improvement
|
||||
|
||||
This prevents wasted effort optimizing prompts when the actual problem is
|
||||
that ``extract_relevant_section`` or ``clean_readme_for_llm`` removed or
|
||||
misaligned the content the LLM needed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
import aiohttp
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Audit record
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class AuditRecord:
|
||||
"""Preprocessing audit for a single repo entry."""
|
||||
|
||||
# Identity
|
||||
repo_id: str
|
||||
safetensors_name: str
|
||||
basename: str # filename without .safetensors
|
||||
|
||||
# Raw README stats
|
||||
raw_readme_length: int
|
||||
raw_readme_line_count: int
|
||||
has_yaml_frontmatter: bool
|
||||
yaml_has_base_model: bool
|
||||
yaml_has_tags: bool
|
||||
|
||||
# Section extraction
|
||||
section_extraction_activated: bool # output < 95% of input length
|
||||
section_length: int
|
||||
section_line_count: int
|
||||
basename_in_section: bool # basename appears in extracted section text
|
||||
|
||||
# Cleaning
|
||||
cleaned_length: int
|
||||
cleaned_line_count: int
|
||||
compression_pct: float # (1 - cleaned/raw) * 100
|
||||
|
||||
# Widget section (stripped by _strip_widget_section)
|
||||
widget_section_found: bool
|
||||
widget_section_length: int
|
||||
|
||||
# Flags (list of anomaly descriptions)
|
||||
flags: List[str] = field(default_factory=list)
|
||||
|
||||
# Local file path to the saved raw README (for cross-reference)
|
||||
readme_file: str = ""
|
||||
|
||||
# Staged intermediate output for report detail
|
||||
raw_readme_preview: str = "" # first 200 chars
|
||||
section_preview: str = "" # first 300 chars
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_HF_RAW_URL = "https://huggingface.co/{repo_id}/raw/main/README.md"
|
||||
|
||||
# Thresholds for flagging
|
||||
_SECTION_ACTIVATION_RATIO = 0.95
|
||||
_MIN_CLEANED_LENGTH = 100
|
||||
_MAX_COMPRESSION_PCT = 99.0
|
||||
_MIN_SECTION_LINES = 3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module loader — bypasses parent-package __init__ that imports ComfyUI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_readme_processor_module = None
|
||||
|
||||
|
||||
def _load_readme_processor():
|
||||
"""Import ``readme_processor`` without triggering ``folder_paths`` import.
|
||||
|
||||
The normal import path (``py.services.agent.skills.enrich_hf_metadata.
|
||||
readme_processor``) triggers ``py.services.agent.__init__`` which
|
||||
imports ``agent_service.py`` → ``py/config.py`` → ComfyUI's
|
||||
``folder_paths``, which is not available in standalone mode.
|
||||
"""
|
||||
global _readme_processor_module
|
||||
if _readme_processor_module is not None:
|
||||
return _readme_processor_module
|
||||
|
||||
import importlib.util
|
||||
|
||||
_RP_PATH = os.path.join(
|
||||
os.path.dirname(__file__), # tests/enrich_hf_validation/
|
||||
"..", "..",
|
||||
"py", "services", "agent", "skills", "enrich_hf_metadata",
|
||||
"readme_processor.py",
|
||||
)
|
||||
rp_path = os.path.normpath(_RP_PATH)
|
||||
if not os.path.exists(rp_path):
|
||||
logger.error("readme_processor.py not found at %s", rp_path)
|
||||
return None
|
||||
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"readme_processor", rp_path,
|
||||
)
|
||||
if spec is None or spec.loader is None:
|
||||
logger.error("Could not create spec for readme_processor.py")
|
||||
return None
|
||||
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
try:
|
||||
spec.loader.exec_module(mod)
|
||||
except Exception as exc:
|
||||
logger.error("Failed to load readme_processor.py: %s", exc)
|
||||
return None
|
||||
|
||||
_readme_processor_module = mod
|
||||
return mod
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HF README fetcher
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _fetch_readme(repo_id: str, session: aiohttp.ClientSession) -> str:
|
||||
"""Fetch the raw README.md from HuggingFace."""
|
||||
url = _HF_RAW_URL.format(repo_id=repo_id)
|
||||
try:
|
||||
async with session.get(url, timeout=aiohttp.ClientTimeout(total=30)) as resp:
|
||||
if resp.status == 200:
|
||||
return await resp.text()
|
||||
logger.warning("Failed to fetch README for %s: HTTP %d", repo_id, resp.status)
|
||||
return ""
|
||||
except (asyncio.TimeoutError, aiohttp.ClientError) as exc:
|
||||
logger.warning("Failed to fetch README for %s: %s", repo_id, exc)
|
||||
return ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Analysis helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _has_yaml_frontmatter(text: str) -> bool:
|
||||
return bool(text.strip().startswith("---"))
|
||||
|
||||
|
||||
def _extract_yaml_field(text: str, field: str) -> bool:
|
||||
"""Check if the given YAML field exists in the frontmatter."""
|
||||
lines = text.split("\n")
|
||||
if not lines or not lines[0].strip().startswith("---"):
|
||||
return False
|
||||
end = 1
|
||||
while end < len(lines):
|
||||
if lines[end].strip().startswith("---"):
|
||||
break
|
||||
end += 1
|
||||
if end >= len(lines):
|
||||
return False
|
||||
frontmatter = "\n".join(lines[1:end])
|
||||
pattern = rf"^{field}:"
|
||||
return bool(re.search(pattern, frontmatter, re.MULTILINE))
|
||||
|
||||
|
||||
def _find_widget_section_length(text: str) -> int:
|
||||
"""Find the ``widget:`` YAML section and return its length (0 if none)."""
|
||||
if not _has_yaml_frontmatter(text):
|
||||
return 0
|
||||
frontmatter_end = text.find("---", 3)
|
||||
if frontmatter_end == -1:
|
||||
return 0
|
||||
frontmatter = text[3:frontmatter_end]
|
||||
|
||||
# Match widget: through to the next top-level key or frontmatter end
|
||||
m = re.search(r"\nwidget:", frontmatter)
|
||||
if not m:
|
||||
return 0
|
||||
# Length from widget: to end of frontmatter (the next \n\w+: or \n---)
|
||||
return len(frontmatter[m.start():])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core auditor
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def run_audit(
|
||||
entries: List[Tuple[str, str]],
|
||||
*,
|
||||
concurrency: int = 10,
|
||||
readmes_dir: str | None = None,
|
||||
) -> Tuple[List[AuditRecord], Dict[str, Any]]:
|
||||
"""Run the preprocessing audit over all repo entries.
|
||||
|
||||
Args:
|
||||
entries: List of ``(repo_id, safetensors_name)``.
|
||||
concurrency: Max parallel fetches to HuggingFace.
|
||||
readmes_dir: If set, saves each fetched README as
|
||||
``{sanitized_repo_id}.md`` in this directory for offline
|
||||
cross-reference against audit results.
|
||||
|
||||
Returns:
|
||||
Tuple of ``(records, summary)`` where *summary* is a dict with
|
||||
aggregate statistics.
|
||||
"""
|
||||
semaphore = asyncio.Semaphore(concurrency)
|
||||
records: List[AuditRecord] = []
|
||||
flag_counter: Dict[str, int] = {}
|
||||
|
||||
if readmes_dir:
|
||||
os.makedirs(readmes_dir, exist_ok=True)
|
||||
|
||||
connector = aiohttp.TCPConnector(limit=concurrency)
|
||||
async with aiohttp.ClientSession(connector=connector) as session:
|
||||
tasks = [_audit_one(entry, session, semaphore, readmes_dir=readmes_dir) for entry in entries]
|
||||
gathered = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
for entry, result in zip(entries, gathered):
|
||||
if isinstance(result, Exception):
|
||||
logger.error("Audit failed for %s: %s", entry[0], result)
|
||||
records.append(
|
||||
AuditRecord(
|
||||
repo_id=entry[0],
|
||||
safetensors_name=entry[1],
|
||||
basename=os.path.splitext(entry[1])[0],
|
||||
raw_readme_length=0,
|
||||
raw_readme_line_count=0,
|
||||
has_yaml_frontmatter=False,
|
||||
yaml_has_base_model=False,
|
||||
yaml_has_tags=False,
|
||||
section_extraction_activated=False,
|
||||
section_length=0,
|
||||
section_line_count=0,
|
||||
basename_in_section=False,
|
||||
cleaned_length=0,
|
||||
cleaned_line_count=0,
|
||||
compression_pct=0.0,
|
||||
widget_section_found=False,
|
||||
widget_section_length=0,
|
||||
readme_file="",
|
||||
flags=[f"Audit exception: {result}"],
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
# The continue above ensures result is AuditRecord here
|
||||
assert isinstance(result, AuditRecord)
|
||||
records.append(result)
|
||||
for flag in result.flags:
|
||||
flag_counter[flag] = flag_counter.get(flag, 0) + 1
|
||||
|
||||
summary = _build_summary(records, flag_counter)
|
||||
return records, summary
|
||||
|
||||
|
||||
def _sanitize_repo_id(repo_id: str) -> str:
|
||||
"""Turn ``user/repo-name`` into a safe filename."""
|
||||
return repo_id.replace("/", "__").replace(".", "_")
|
||||
|
||||
|
||||
async def _audit_one(
|
||||
entry: Tuple[str, str],
|
||||
session: aiohttp.ClientSession,
|
||||
semaphore: asyncio.Semaphore,
|
||||
*,
|
||||
readmes_dir: str | None = None,
|
||||
) -> AuditRecord:
|
||||
"""Audit a single repo entry."""
|
||||
repo_id, safetensors_name = entry
|
||||
basename = os.path.splitext(safetensors_name)[0]
|
||||
|
||||
async with semaphore:
|
||||
# Import production preprocessing functions.
|
||||
# Use importlib to bypass py.services.agent.__init__ which triggers
|
||||
# ComfyUI's folder_paths module (not available in standalone mode).
|
||||
_rp = _load_readme_processor()
|
||||
if _rp is None:
|
||||
return AuditRecord(
|
||||
repo_id=repo_id,
|
||||
safetensors_name=safetensors_name,
|
||||
basename=basename,
|
||||
raw_readme_length=0, raw_readme_line_count=0,
|
||||
has_yaml_frontmatter=False, yaml_has_base_model=False, yaml_has_tags=False,
|
||||
readme_file="",
|
||||
section_extraction_activated=False, section_length=0, section_line_count=0,
|
||||
basename_in_section=False, cleaned_length=0, cleaned_line_count=0,
|
||||
compression_pct=0.0, widget_section_found=False, widget_section_length=0,
|
||||
flags=["IMPORT_FAILED"],
|
||||
)
|
||||
clean_readme_for_llm = _rp.clean_readme_for_llm
|
||||
extract_relevant_section = _rp.extract_relevant_section
|
||||
|
||||
# Step 1: Fetch the raw README
|
||||
raw_text = await _fetch_readme(repo_id, session)
|
||||
if not raw_text:
|
||||
return AuditRecord(
|
||||
repo_id=repo_id,
|
||||
safetensors_name=safetensors_name,
|
||||
basename=basename,
|
||||
raw_readme_length=0,
|
||||
raw_readme_line_count=0,
|
||||
has_yaml_frontmatter=False,
|
||||
yaml_has_base_model=False,
|
||||
yaml_has_tags=False,
|
||||
section_extraction_activated=False,
|
||||
section_length=0,
|
||||
section_line_count=0,
|
||||
basename_in_section=False,
|
||||
readme_file="",
|
||||
cleaned_length=0,
|
||||
cleaned_line_count=0,
|
||||
compression_pct=0.0,
|
||||
widget_section_found=False,
|
||||
widget_section_length=0,
|
||||
flags=["README_FETCH_FAILED"],
|
||||
)
|
||||
|
||||
# Save the raw README to disk for offline cross-reference
|
||||
readme_path = ""
|
||||
if readmes_dir:
|
||||
safe_name = _sanitize_repo_id(repo_id)
|
||||
readme_path = os.path.join(readmes_dir, f"{safe_name}.md")
|
||||
try:
|
||||
with open(readme_path, "w", encoding="utf-8") as fh:
|
||||
fh.write(raw_text)
|
||||
except OSError as exc:
|
||||
logger.warning("Failed to save README for %s: %s", repo_id, exc)
|
||||
readme_path = ""
|
||||
|
||||
raw_lines = raw_text.split("\n")
|
||||
raw_len = len(raw_text)
|
||||
raw_line_count = len(raw_lines)
|
||||
|
||||
# Step 2: Analyze raw README
|
||||
yaml_fm = _has_yaml_frontmatter(raw_text)
|
||||
yaml_has_bm = _extract_yaml_field(raw_text, "base_model") if yaml_fm else False
|
||||
yaml_has_tg = _extract_yaml_field(raw_text, "tags") if yaml_fm else False
|
||||
widget_len = _find_widget_section_length(raw_text)
|
||||
|
||||
# Step 3: Section extraction
|
||||
section = extract_relevant_section(raw_text, basename)
|
||||
section_len = len(section)
|
||||
section_line_count = len(section.split("\n"))
|
||||
section_activated = section_len < raw_len * _SECTION_ACTIVATION_RATIO
|
||||
basename_in_sec = basename.lower() in section.lower()
|
||||
|
||||
# Step 4: Cleaning for LLM
|
||||
cleaned = clean_readme_for_llm(section)
|
||||
cleaned_len = len(cleaned)
|
||||
cleaned_line_count = len(cleaned.split("\n"))
|
||||
compression_pct = round((1 - cleaned_len / raw_len) * 100, 1) if raw_len else 0.0
|
||||
|
||||
# Step 5: Flag anomalies
|
||||
flags: List[str] = []
|
||||
if not raw_text.strip():
|
||||
flags.append("README_EMPTY")
|
||||
if not yaml_fm:
|
||||
flags.append("NO_YAML_FRONTMATTER")
|
||||
if not section_activated:
|
||||
# Check if basename is extremely short/generic (likely synthetic)
|
||||
if len(basename) <= 5:
|
||||
flags.append("BASENAME_TOO_SHORT_SECTION_NOT_EXPECTED")
|
||||
else:
|
||||
flags.append("SECTION_EXTRACTION_NOT_ACTIVATED")
|
||||
elif not basename_in_sec:
|
||||
flags.append("BASENAME_NOT_IN_EXTRACTED_SECTION")
|
||||
if widget_len == 0:
|
||||
# Not necessarily a problem — many repos lack a widget section
|
||||
pass
|
||||
if cleaned_len < _MIN_CLEANED_LENGTH:
|
||||
flags.append("CLEANED_README_TOO_SHORT")
|
||||
if compression_pct > _MAX_COMPRESSION_PCT:
|
||||
flags.append("EXTREME_COMPRESSION")
|
||||
if section_activated and section_line_count < _MIN_SECTION_LINES:
|
||||
flags.append("SECTION_TOO_SMALL")
|
||||
|
||||
return AuditRecord(
|
||||
repo_id=repo_id,
|
||||
safetensors_name=safetensors_name,
|
||||
basename=basename,
|
||||
raw_readme_length=raw_len,
|
||||
raw_readme_line_count=raw_line_count,
|
||||
has_yaml_frontmatter=yaml_fm,
|
||||
yaml_has_base_model=yaml_has_bm,
|
||||
yaml_has_tags=yaml_has_tg,
|
||||
section_extraction_activated=section_activated,
|
||||
section_length=section_len,
|
||||
section_line_count=section_line_count,
|
||||
basename_in_section=basename_in_sec,
|
||||
cleaned_length=cleaned_len,
|
||||
cleaned_line_count=cleaned_line_count,
|
||||
compression_pct=compression_pct,
|
||||
widget_section_found=widget_len > 0,
|
||||
widget_section_length=widget_len,
|
||||
readme_file=readme_path,
|
||||
flags=flags,
|
||||
raw_readme_preview=raw_text[:200],
|
||||
section_preview=section[:300],
|
||||
)
|
||||
|
||||
|
||||
def _build_summary(
|
||||
records: List[AuditRecord],
|
||||
flag_counter: Dict[str, int],
|
||||
) -> Dict[str, Any]:
|
||||
"""Aggregate audit statistics."""
|
||||
n = len(records)
|
||||
if n == 0:
|
||||
return {"error": "no records", "model_count": 0}
|
||||
|
||||
activated = sum(1 for r in records if r.section_extraction_activated)
|
||||
basename_hit = sum(1 for r in records if r.basename_in_section)
|
||||
with_yaml = sum(1 for r in records if r.has_yaml_frontmatter)
|
||||
with_widget = sum(1 for r in records if r.widget_section_found)
|
||||
fetch_failed = sum(1 for r in records if "README_FETCH_FAILED" in r.flags)
|
||||
|
||||
avg_compression = round(
|
||||
sum(r.compression_pct for r in records if r.raw_readme_length > 0) / max(n - fetch_failed, 1),
|
||||
1,
|
||||
)
|
||||
avg_cleaned = round(
|
||||
sum(r.cleaned_length for r in records if r.raw_readme_length > 0) / max(n - fetch_failed, 1),
|
||||
)
|
||||
|
||||
top_flags = sorted(flag_counter.items(), key=lambda x: -x[1])[:10]
|
||||
|
||||
return {
|
||||
"model_count": n,
|
||||
"fetch_failed_count": fetch_failed,
|
||||
"section_extraction_activated": activated,
|
||||
"section_extraction_pct": round(activated / max(n - fetch_failed, 1) * 100, 1),
|
||||
"basename_in_section": basename_hit,
|
||||
"basename_in_section_pct": round(basename_hit / max(n - fetch_failed, 1) * 100, 1),
|
||||
"with_yaml_frontmatter": with_yaml,
|
||||
"with_yaml_frontmatter_pct": round(with_yaml / max(n - fetch_failed, 1) * 100, 1),
|
||||
"with_widget_section": with_widget,
|
||||
"avg_compression_pct": avg_compression,
|
||||
"avg_cleaned_length": avg_cleaned,
|
||||
"top_flags": top_flags,
|
||||
}
|
||||
|
||||
|
||||
def audit_records_to_serializable(records: List[AuditRecord]) -> List[Dict[str, Any]]:
|
||||
"""Convert AuditRecord dataclasses to plain dicts for JSON serialization."""
|
||||
return [asdict(r) for r in records]
|
||||
@@ -0,0 +1,391 @@
|
||||
"""Generate structured reports from evaluation results.
|
||||
|
||||
Produces:
|
||||
|
||||
1. A JSON data dump (``report.json``) with all scores and aggregations.
|
||||
2. A human-readable Markdown report (``report.md``) with summary stats,
|
||||
issue patterns, and actionable optimisation suggestions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from .config import SUPPORTED_BASE_MODELS
|
||||
from .evaluation_engine import ScoreRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Markdown report
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _fmt_pct(value: float) -> str:
|
||||
return f"{value:.1f}%"
|
||||
|
||||
|
||||
def _bar(value: float, width: int = 20) -> str:
|
||||
filled = int(round(value / 100 * width))
|
||||
return "█" * filled + "░" * (width - filled)
|
||||
|
||||
|
||||
def generate_optimisation_suggestions(
|
||||
agg: Dict[str, Any],
|
||||
scores: List[ScoreRecord],
|
||||
) -> List[str]:
|
||||
"""Analyse evaluation results and produce concrete suggestions."""
|
||||
suggestions: List[str] = []
|
||||
fa = agg.get("field_aggregates", {})
|
||||
|
||||
# --- base_model ---
|
||||
bm = fa.get("base_model", {})
|
||||
if bm and bm.get("empty_rate_pct", 0) > 30:
|
||||
suggestions.append(
|
||||
"- **base_model 空置率高 ({:.0f}%)**: 多数 HF 模型卡片未在 YAML frontmatter 中声明 "
|
||||
"`base_model:` 字段,LLM 无法推断。可考虑在 prompt 中增加 \"look at the model file name "
|
||||
"for clues\" 的引导,或在后处理中增加基于文件名规则的 fallback 猜测。".format(
|
||||
bm.get("empty_rate_pct", 0)
|
||||
)
|
||||
)
|
||||
bm_invalid = sum(
|
||||
1
|
||||
for s in scores
|
||||
if s["raw_values"]["base_model"]
|
||||
and s["raw_values"]["base_model"] != "Unknown"
|
||||
and s["raw_values"]["base_model"] not in set(SUPPORTED_BASE_MODELS)
|
||||
)
|
||||
if bm_invalid > 5:
|
||||
suggestions.append(
|
||||
"- **base_model 含非标准值 ({} 个)**: LLM 输出了未在当前生产系统的 base model 列表 "
|
||||
"中的名称。建议在 prompt 中强调 \"Use EXACTLY one name from the list\" 并在 "
|
||||
"`PostProcessor` 中加一层验证过滤,非标准值直接丢弃。".format(bm_invalid)
|
||||
)
|
||||
|
||||
# --- trigger_words ---
|
||||
tw = fa.get("trigger_words", {})
|
||||
if tw and tw.get("empty_rate_pct", 0) > 40:
|
||||
suggestions.append(
|
||||
"- **trigger_words 空置率高 ({:.0f}%)**: 大量 HF 模型卡没有明确的 "
|
||||
"`instance_prompt:` 或 trigger word 说明。当前 prompt 已覆盖常见模式。若确认这些模型确实"
|
||||
"没有 trigger words(例如 style lora),空数组是正确结果,不需优化。".format(
|
||||
tw.get("empty_rate_pct", 0)
|
||||
)
|
||||
)
|
||||
|
||||
# --- tags ---
|
||||
tag = fa.get("tags", {})
|
||||
if tag and tag.get("empty_rate_pct", 0) > 30:
|
||||
suggestions.append(
|
||||
"- **tags 空置率高 ({:.0f}%)**: 当前 prompt 要求 tags 必须与 "
|
||||
"`priority_tags`(CIVITAI_MODEL_TAGS)对齐。HF 模型的标签体系与 Civitai 不同,"
|
||||
"很多 model card 使用细粒度标签(如 `pokemon`、`watercolor`)而不在 priority list 中。"
|
||||
"建议: 扩大 priority_tags 范围,或允许 LLM 自由生成 tags 后只做去重不做严格过滤。".format(
|
||||
tag.get("empty_rate_pct", 0)
|
||||
)
|
||||
)
|
||||
|
||||
# --- tags priority coverage ---
|
||||
low_coverage = sum(
|
||||
1
|
||||
for s in scores
|
||||
if s["field_scores"].get("tags_priority_coverage", 5) < 3 # < 60% of max
|
||||
and s["field_scores"].get("tags", 0) > 0
|
||||
)
|
||||
if low_coverage > 10:
|
||||
suggestions.append(
|
||||
"- **{} 个模型的 tags 与 priority_tags 匹配度低于 60%**: "
|
||||
"LLM 生成了有意义但不属于 CIVITAI_MODEL_TAGS 的标签。这说明 priority_tags "
|
||||
"的覆盖范围对 HF 模型不足,建议按 HF 模型的实际分布补充新类别。".format(low_coverage)
|
||||
)
|
||||
|
||||
# --- preview ---
|
||||
prev = fa.get("preview_downloaded", {})
|
||||
if prev and prev.get("empty_rate_pct", 0) > 50:
|
||||
suggestions.append(
|
||||
"- **预览图下载成功率低 ({:.0f}%)**: 很多 HF 模型卡没有 embed 图片(仅使用 YAML widget "
|
||||
"或 external link)。当前 `readme_processor.py` 的 `extract_gallery_images` 和 "
|
||||
"`extract_gallery_table_images` 已覆盖了多数场景。若预览图不重要,可降低此字段权重。".format(
|
||||
prev.get("empty_rate_pct", 0)
|
||||
)
|
||||
)
|
||||
|
||||
# --- usage_tips ---
|
||||
ut = fa.get("usage_tips", {})
|
||||
if ut and ut.get("empty_rate_pct", 0) > 70:
|
||||
suggestions.append(
|
||||
"- **usage_tips 空置率极高 ({:.0f}%)**: 这是预期行为。HF 模型卡通常不包含 LoRA "
|
||||
"强度/CLIP skip 等结构化参数。当前提取策略已合理。若需要可用数据,"
|
||||
"可以考虑使用模型类型的通用默认值。".format(
|
||||
ut.get("empty_rate_pct", 0)
|
||||
)
|
||||
)
|
||||
|
||||
# --- short_description ---
|
||||
sd = fa.get("short_description", {})
|
||||
if sd and sd.get("empty_rate_pct", 0) > 40:
|
||||
suggestions.append(
|
||||
"- **short_description 空置率 ({:.0f}%)**: 部分 HF 模型卡 README 内容极少(仅含标签和训练参数)。".format(
|
||||
sd.get("empty_rate_pct", 0)
|
||||
)
|
||||
)
|
||||
|
||||
if not suggestions:
|
||||
suggestions.append("- 未发现明显问题模式,各字段填充率均在可接受范围。")
|
||||
|
||||
return suggestions
|
||||
|
||||
|
||||
def generate_markdown_report(
|
||||
agg: Dict[str, Any],
|
||||
scores: List[ScoreRecord],
|
||||
output_dir: str,
|
||||
duration_summary: Dict[str, Any] | None = None,
|
||||
*,
|
||||
audit_summary: Dict[str, Any] | None = None,
|
||||
config_warnings: List[str] | None = None,
|
||||
) -> str:
|
||||
"""Write ``report.md`` and return its content.
|
||||
|
||||
Args:
|
||||
agg: Aggregate evaluation scores.
|
||||
scores: Per-model evaluation records.
|
||||
output_dir: Output directory for the report file.
|
||||
duration_summary: Optional timing statistics.
|
||||
audit_summary: Optional preprocessing audit summary (Phase 1.5).
|
||||
config_warnings: Optional LLM config consistency warnings.
|
||||
"""
|
||||
lines: List[str] = []
|
||||
def wl(text: str = "") -> None:
|
||||
lines.append(text)
|
||||
|
||||
wl("# HF Metadata Enrichment Validation Report")
|
||||
wl()
|
||||
wl(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
wl(f"Models evaluated: **{agg.get('model_count', 0)}**")
|
||||
wl(f"Successful enrichments: **{agg.get('success_count', 0)}**")
|
||||
wl(f"Failures: **{agg.get('fail_count', 0)}**")
|
||||
wl()
|
||||
|
||||
# ---- Preprocessing Audit Section ----
|
||||
if audit_summary and audit_summary.get("model_count", 0) > 0:
|
||||
wl("## Preprocessing Audit")
|
||||
wl()
|
||||
wl(f"| Metric | Value |")
|
||||
wl(f"|--------|-------|")
|
||||
wl(f"| Models audited | {audit_summary.get('model_count', 0)} |")
|
||||
wl(f"| README fetch failed | {audit_summary.get('fetch_failed_count', 0)} |")
|
||||
wl(f"| Section extraction activated | {_fmt_pct(audit_summary.get('section_extraction_pct', 0))} |")
|
||||
wl(f"| Basename found in section | {_fmt_pct(audit_summary.get('basename_in_section_pct', 0))} |")
|
||||
wl(f"| Has YAML frontmatter | {_fmt_pct(audit_summary.get('with_yaml_frontmatter_pct', 0))} |")
|
||||
wl(f"| Has YAML widget section | {_fmt_pct(audit_summary.get('with_widget_section', 0))} |")
|
||||
wl(f"| Avg README compression | {audit_summary.get('avg_compression_pct', 0)}% |")
|
||||
wl(f"| Avg cleaned length | {audit_summary.get('avg_cleaned_length', 0)} chars |")
|
||||
wl()
|
||||
|
||||
if audit_summary.get("top_flags"):
|
||||
wl("### Audit Flags (most frequent)")
|
||||
wl()
|
||||
for flag, count in audit_summary["top_flags"]:
|
||||
wl(f"- **{flag}**: {count}x")
|
||||
wl()
|
||||
|
||||
wl("**Interpretation:**")
|
||||
wl()
|
||||
act_pct = audit_summary.get("section_extraction_pct", 0)
|
||||
if act_pct < 50:
|
||||
wl(
|
||||
"- ⚠️ Section extraction activated for fewer than 50% of repos. "
|
||||
"This may indicate the basename doesn't match README content, or the "
|
||||
"repos are mostly single-model (where full README is expected)."
|
||||
)
|
||||
else:
|
||||
wl(
|
||||
"- ✅ Section extraction is working for most repos — the LLM is "
|
||||
"receiving focused README sections."
|
||||
)
|
||||
|
||||
if audit_summary.get("basename_in_section_pct", 100) < 80:
|
||||
wl(
|
||||
"- ⚠️ The safetensors basename was NOT found in the extracted section "
|
||||
"for many repos. This could mean the section extraction matched the wrong "
|
||||
"section, or the README doesn't explicitly reference the filename."
|
||||
)
|
||||
wl()
|
||||
|
||||
# ---- Config warnings ----
|
||||
if config_warnings:
|
||||
wl("## ⚠️ Configuration Warnings")
|
||||
wl()
|
||||
for w in config_warnings:
|
||||
wl(f"- {w}")
|
||||
wl()
|
||||
|
||||
# ---- Duration ----
|
||||
if duration_summary:
|
||||
wl("## Timing")
|
||||
wl()
|
||||
wl(f"- Total wall time: **{duration_summary.get('total_wall_s', 0):.0f} s** ")
|
||||
wl(f" ({duration_summary.get('total_wall_s', 0) / 60:.1f} min)")
|
||||
wl(f"- Mean per model: **{duration_summary.get('mean_s', 0):.1f} s**")
|
||||
wl(f"- Median per model: **{duration_summary.get('median_s', 0):.1f} s**")
|
||||
wl(f"- Fastest: **{duration_summary.get('min_s', 0):.1f} s**")
|
||||
wl(f"- Slowest: **{duration_summary.get('max_s', 0):.1f} s**")
|
||||
wl()
|
||||
|
||||
# ---- Overall score ----
|
||||
ts = agg.get("total_score", {})
|
||||
wl("## Overall Score Distribution (0–100)")
|
||||
wl()
|
||||
wl(f"| Metric | Value |")
|
||||
wl(f"|--------|-------|")
|
||||
wl(f"| Mean | {ts.get('mean', 'N/A')} |")
|
||||
wl(f"| Median | {ts.get('median', 'N/A')} |")
|
||||
wl(f"| Min | {ts.get('min', 'N/A')} |")
|
||||
wl(f"| Max | {ts.get('max', 'N/A')} |")
|
||||
wl()
|
||||
for label, key in [
|
||||
("Excellent (≥80)", "excellent_80+"),
|
||||
("Good (60–79)", "good_60_79"),
|
||||
("Fair (40–59)", "fair_40_59"),
|
||||
("Poor (20–39)", "poor_20_39"),
|
||||
("Bad (<20)", "bad_0_19"),
|
||||
]:
|
||||
count = ts.get("bins", {}).get(key, 0)
|
||||
pct = count / agg["model_count"] * 100 if agg["model_count"] else 0
|
||||
wl(f"- **{label}**: {count} models ({_fmt_pct(pct)})")
|
||||
wl()
|
||||
|
||||
# ---- Per-field aggregates ----
|
||||
wl("## Per-Field Completeness")
|
||||
wl()
|
||||
wl("| Field | Mean Score | Fill Rate | Empty Rate |")
|
||||
wl("|-------|-----------:|----------:|-----------:|")
|
||||
fa = agg.get("field_aggregates", {})
|
||||
for fn in [
|
||||
"base_model", "trigger_words", "short_description", "tags",
|
||||
"tags_priority_coverage", "notes", "usage_tips",
|
||||
"modelDescription_html", "preview_downloaded",
|
||||
]:
|
||||
f = fa.get(fn, {})
|
||||
if not f:
|
||||
continue
|
||||
wl(
|
||||
f"| {fn} "
|
||||
f"| {f.get('mean', 'N/A')} "
|
||||
f"| {_fmt_pct(f.get('fill_rate_pct', 0))} "
|
||||
f"| {_fmt_pct(f.get('empty_rate_pct', 0))} |"
|
||||
)
|
||||
wl()
|
||||
|
||||
# ---- Confidence distribution ----
|
||||
wl("## LLM Confidence Distribution")
|
||||
wl()
|
||||
cd = agg.get("confidence_distribution", {})
|
||||
total_conf = sum(cd.values()) or 1
|
||||
for level in ["high", "medium", "low", ""]:
|
||||
count = cd.get(level, 0)
|
||||
label = level if level else "(not reported)"
|
||||
pct = count / total_conf * 100
|
||||
bar = _bar(pct)
|
||||
wl(f"- **{label}**: {count} {bar} {_fmt_pct(pct)}")
|
||||
wl()
|
||||
|
||||
# ---- Top issues ----
|
||||
wl("## Most Frequent Issues")
|
||||
wl()
|
||||
for issue, count in agg.get("top_issues", []):
|
||||
pct = count / agg["model_count"] * 100 if agg["model_count"] else 0
|
||||
wl(f"- **{issue}** — {count}/{agg['model_count']} ({_fmt_pct(pct)})")
|
||||
wl()
|
||||
|
||||
# ---- Optimisation suggestions ----
|
||||
wl("## Optimisation Suggestions")
|
||||
wl()
|
||||
suggestions = generate_optimisation_suggestions(agg, scores)
|
||||
for s in suggestions:
|
||||
wl(s)
|
||||
wl()
|
||||
|
||||
# ---- Per-model detail ----
|
||||
wl("## Per-Model Detail")
|
||||
wl()
|
||||
wl("<details>")
|
||||
wl("<summary>Click to expand</summary>")
|
||||
wl()
|
||||
wl("| # | Repo ID | Score | Issues | Confidence |")
|
||||
wl("|---|---------|------:|--------|------------|")
|
||||
for i, s in enumerate(scores, 1):
|
||||
issue_count = len(s["issues"])
|
||||
issue_str = (
|
||||
f"{issue_count} issue(s)" if issue_count else "✓ ok"
|
||||
)
|
||||
wl(
|
||||
f"| {i} "
|
||||
f"| {s['repo_id']} "
|
||||
f"| {s['total_score']} "
|
||||
f"| {issue_str} "
|
||||
f"| {s.get('confidence_from_llm', '') or '-'} |"
|
||||
)
|
||||
wl()
|
||||
wl("</details>")
|
||||
wl()
|
||||
|
||||
content = "\n".join(lines)
|
||||
report_path = os.path.join(output_dir, "report.md")
|
||||
with open(report_path, "w", encoding="utf-8") as fh:
|
||||
fh.write(content)
|
||||
logger.info("Markdown report written to %s", report_path)
|
||||
return content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON dump
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def save_json_report(
|
||||
agg: Dict[str, Any],
|
||||
scores: List[ScoreRecord],
|
||||
enrichment_results: List[Dict[str, Any]],
|
||||
output_dir: str,
|
||||
duration_summary: Dict[str, Any] | None = None,
|
||||
*,
|
||||
audit_summary: Dict[str, Any] | None = None,
|
||||
config_warnings: List[str] | None = None,
|
||||
) -> str:
|
||||
"""Write ``report.json`` and return the path.
|
||||
|
||||
Args:
|
||||
agg: Aggregate evaluation scores.
|
||||
scores: Per-model evaluation records.
|
||||
enrichment_results: Raw enrichment phase results.
|
||||
output_dir: Output directory.
|
||||
duration_summary: Optional timing statistics.
|
||||
audit_summary: Optional preprocessing audit summary.
|
||||
config_warnings: Optional LLM config consistency warnings.
|
||||
"""
|
||||
report: Dict[str, Any] = {
|
||||
"metadata": {
|
||||
"generated_at": datetime.now().isoformat(),
|
||||
"model_count": agg.get("model_count", 0),
|
||||
},
|
||||
"aggregate": agg,
|
||||
"timing": duration_summary or {},
|
||||
"per_model_scores": scores,
|
||||
"enrichment_results": enrichment_results,
|
||||
}
|
||||
if audit_summary:
|
||||
report["preprocessing_audit"] = audit_summary
|
||||
if config_warnings:
|
||||
report["config_warnings"] = config_warnings
|
||||
|
||||
path = os.path.join(output_dir, "report.json")
|
||||
with open(path, "w", encoding="utf-8") as fh:
|
||||
json.dump(report, fh, indent=2, ensure_ascii=False)
|
||||
logger.info("JSON report written to %s", path)
|
||||
return path
|
||||
@@ -0,0 +1,451 @@
|
||||
#!/usr/bin/env python3
|
||||
"""CLI entry point for the HF metadata enrichment validation suite.
|
||||
|
||||
Usage::
|
||||
|
||||
# Full run (44 models, serial, ~1-2 h)
|
||||
python -m tests.enrich_hf_validation.run_validation \\
|
||||
--output /tmp/hf_enrich_validation
|
||||
|
||||
# Quick smoke test with 2 models
|
||||
python -m tests.enrich_hf_validation.run_validation --sample 2
|
||||
|
||||
# Resume from a previous partial run
|
||||
python -m tests.enrich_hf_validation.run_validation --resume
|
||||
|
||||
# Audit preprocessing only (no LLM calls, fast)
|
||||
python -m tests.enrich_hf_validation.run_validation --audit-only
|
||||
|
||||
# Custom settings file
|
||||
python -m tests.enrich_hf_validation.run_validation \\
|
||||
--settings /custom/path/settings.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
# Ensure the project root is on sys.path so that ``from py import ...`` works.
|
||||
_PROJECT_ROOT = os.path.normpath(
|
||||
os.path.join(os.path.dirname(__file__), "..", "..")
|
||||
)
|
||||
if _PROJECT_ROOT not in sys.path:
|
||||
sys.path.insert(0, _PROJECT_ROOT)
|
||||
|
||||
# Add ComfyUI root to sys.path so ``folder_paths`` can be imported.
|
||||
# Project layout: ComfyUI/custom_nodes/ComfyUI-Lora-Manager/
|
||||
_COMFYUI_ROOT = os.path.normpath(os.path.join(_PROJECT_ROOT, "..", ".."))
|
||||
if _COMFYUI_ROOT not in sys.path:
|
||||
sys.path.insert(0, _COMFYUI_ROOT)
|
||||
|
||||
from tests.enrich_hf_validation.config import (
|
||||
init_supported_base_models,
|
||||
load_settings,
|
||||
)
|
||||
from tests.enrich_hf_validation.metadata_constructor import (
|
||||
RepoEntry,
|
||||
create_all_initial_metadata,
|
||||
load_repo_ids,
|
||||
)
|
||||
from tests.enrich_hf_validation.enrichment_runner import EnrichmentRunner
|
||||
from tests.enrich_hf_validation.evaluation_engine import (
|
||||
aggregate_scores,
|
||||
evaluate_batch,
|
||||
)
|
||||
from tests.enrich_hf_validation.preprocessing_auditor import (
|
||||
audit_records_to_serializable,
|
||||
run_audit,
|
||||
)
|
||||
from tests.enrich_hf_validation.report_generator import (
|
||||
generate_markdown_report,
|
||||
save_json_report,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _setup_logging(verbose: bool) -> None:
|
||||
level = logging.DEBUG if verbose else logging.INFO
|
||||
fmt = "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
|
||||
logging.basicConfig(level=level, format=fmt, stream=sys.stderr)
|
||||
|
||||
# Quiet noisy third-party loggers
|
||||
for name in ("aiohttp", "asyncio", "urllib3"):
|
||||
logging.getLogger(name).setLevel(logging.WARNING)
|
||||
|
||||
|
||||
def _parse_args(argv: List[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Validate and optimise HF metadata enrichment via LLM.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--models",
|
||||
default=os.path.join(os.path.dirname(__file__), "test_data", "hf_lora_models_with_safetensors.txt"),
|
||||
help="Path to the HF repo entries file (format: repo_id, model_name.safetensors per line)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--settings",
|
||||
default="~/.config/ComfyUI-LoRA-Manager/settings.json",
|
||||
help="Path to LoRA Manager settings.json",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default="/tmp/hf_enrich_validation",
|
||||
help="Output directory for reports and intermediate data",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sample",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Process only the first N models (for quick smoke tests)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--resume",
|
||||
action="store_true",
|
||||
help="Resume from previous partial run (uses progress.json)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-enrich",
|
||||
action="store_true",
|
||||
help="Skip enrichment phase (evaluate existing metadata only)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--audit-only",
|
||||
action="store_true",
|
||||
help="Run preprocessing audit only (no enrichment, no evaluation)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=int,
|
||||
default=240,
|
||||
help="Per-model LLM timeout in seconds (default: 240)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v", "--verbose",
|
||||
action="store_true",
|
||||
help="Enable debug logging",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _phase_header(label: str) -> None:
|
||||
sep = "=" * 60
|
||||
print(f"\n{sep}", file=sys.stderr)
|
||||
print(f" PHASE: {label}", file=sys.stderr)
|
||||
print(sep, file=sys.stderr)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Read back LLM config after enrichment (for consistency reporting)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_actual_llm_config() -> Dict[str, str]:
|
||||
"""Read what LLMService is actually using, if initialized.
|
||||
|
||||
Only meaningful when called AFTER enrichment has started (i.e. after
|
||||
``AgentService.get_instance()`` has been called).
|
||||
"""
|
||||
try:
|
||||
from py.services.llm_service import LLMService
|
||||
|
||||
instance = LLMService._instance
|
||||
if instance is None:
|
||||
return {"status": "not initialized"}
|
||||
cfg = instance._get_config()
|
||||
return {
|
||||
"provider": cfg.get("provider", ""),
|
||||
"model": cfg.get("model", ""),
|
||||
"api_base": cfg.get("api_base", ""),
|
||||
}
|
||||
except Exception as exc:
|
||||
return {"status": f"error: {exc}"}
|
||||
|
||||
|
||||
def _compare_llm_config(
|
||||
pipeline_cfg: Dict[str, Any],
|
||||
actual_cfg: Dict[str, str],
|
||||
) -> List[str]:
|
||||
"""Compare pipeline-loaded vs LLMService-used config.
|
||||
|
||||
Returns warning messages if they differ.
|
||||
"""
|
||||
warnings: List[str] = []
|
||||
if not actual_cfg or actual_cfg.get("status", "") == "not initialized":
|
||||
warnings.append(
|
||||
"LLMService was not initialized during this run — cannot verify "
|
||||
"config consistency."
|
||||
)
|
||||
return warnings
|
||||
|
||||
field_map = [
|
||||
("llm_provider", "provider"),
|
||||
("llm_model", "model"),
|
||||
("llm_api_base", "api_base"),
|
||||
]
|
||||
for pipeline_key, llm_key in field_map:
|
||||
pv = (pipeline_cfg.get(pipeline_key) or "").strip()
|
||||
lv = (actual_cfg.get(llm_key) or "").strip()
|
||||
if pv and lv and pv != lv:
|
||||
warnings.append(
|
||||
f"LLM config mismatch: --settings has '{pv}' for {pipeline_key}, "
|
||||
f"but LLMService uses '{lv}'. "
|
||||
f"The pipeline's --settings path ({pipeline_cfg.get('settings_path', '?')}) "
|
||||
"may differ from where SettingsManager reads."
|
||||
)
|
||||
if not warnings and actual_cfg:
|
||||
warnings.append(
|
||||
"✅ LLM config matches between pipeline --settings and LLMService."
|
||||
)
|
||||
return warnings
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 1.5: preprocessing audit
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _run_preprocessing_audit(
|
||||
entries: List[RepoEntry],
|
||||
output_dir: str,
|
||||
) -> Dict[str, Any]:
|
||||
"""Execute the preprocessing audit and save results."""
|
||||
_phase_header("Preprocessing audit")
|
||||
print(f" Auditing {len(entries)} repos ...", file=sys.stderr)
|
||||
|
||||
readmes_dir = os.path.join(output_dir, "readmes")
|
||||
t0 = time.perf_counter()
|
||||
records, summary = await run_audit(entries, readmes_dir=readmes_dir)
|
||||
elapsed = time.perf_counter() - t0
|
||||
|
||||
# Save audit data
|
||||
audit_path = os.path.join(output_dir, "preprocessing_audit.json")
|
||||
with open(audit_path, "w", encoding="utf-8") as fh:
|
||||
json.dump(
|
||||
{
|
||||
"summary": summary,
|
||||
"records": audit_records_to_serializable(records),
|
||||
},
|
||||
fh,
|
||||
indent=2,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
print(f" Audit complete: {len(records)} repos in {elapsed:.0f}s", file=sys.stderr)
|
||||
print(f" Section extraction activated: {summary.get('section_extraction_pct', 0)}%", file=sys.stderr)
|
||||
print(f" Basename in extracted section: {summary.get('basename_in_section_pct', 0)}%", file=sys.stderr)
|
||||
print(f" Avg compression: {summary.get('avg_compression_pct', 0)}%", file=sys.stderr)
|
||||
print(f" Avg cleaned length: {summary.get('avg_cleaned_length', 0)} chars", file=sys.stderr)
|
||||
print(f" Audit data: {audit_path}", file=sys.stderr)
|
||||
|
||||
if summary.get("top_flags"):
|
||||
print(" Top flags:", file=sys.stderr)
|
||||
for flag, count in summary["top_flags"][:5]:
|
||||
print(f" - {flag}: {count}x", file=sys.stderr)
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
async def _run_enrichment(
|
||||
model_paths: List[str],
|
||||
repos: List[str],
|
||||
output_dir: str,
|
||||
timeout: int,
|
||||
verbose: bool,
|
||||
) -> Dict[str, Any]:
|
||||
"""Execute the enrichment phase."""
|
||||
runner = EnrichmentRunner(
|
||||
output_dir=output_dir,
|
||||
per_model_timeout=timeout,
|
||||
)
|
||||
result = await runner.run(model_paths, repos)
|
||||
|
||||
# Print quick summary
|
||||
progress = result["progress"]
|
||||
total_done = (
|
||||
len(progress.get("completed", []))
|
||||
+ len(progress.get("failed", []))
|
||||
+ len(progress.get("timed_out", []))
|
||||
)
|
||||
print(
|
||||
f"\n Enrichment complete: {total_done} processed "
|
||||
f"({len(progress.get('completed', []))} ok, "
|
||||
f"{len(progress.get('failed', []))} failed, "
|
||||
f"{len(progress.get('timed_out', []))} timed out)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _collect_enriched_metadata(
|
||||
model_paths: List[str],
|
||||
repos: List[str],
|
||||
results: List[Dict[str, Any]],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Read enriched .metadata.json for each model.
|
||||
|
||||
Uses the same path convention as the rest of the codebase:
|
||||
``os.path.splitext(model_path)[0] + '.metadata.json'``.
|
||||
|
||||
Returns a list of dicts with keys: repo_id, model_path, success,
|
||||
errors, metadata.
|
||||
"""
|
||||
enriched: List[Dict[str, Any]] = []
|
||||
# Build a lookup from repo_id to enrichment result
|
||||
result_lookup: Dict[str, Dict[str, Any]] = {}
|
||||
for r in results:
|
||||
result_lookup[r["repo_id"]] = r
|
||||
|
||||
for model_path, repo_id in zip(model_paths, repos):
|
||||
res = result_lookup.get(repo_id, {})
|
||||
metadata_path = f"{os.path.splitext(model_path)[0]}.metadata.json"
|
||||
metadata: Dict[str, Any] = {}
|
||||
if os.path.exists(metadata_path):
|
||||
try:
|
||||
with open(metadata_path, "r", encoding="utf-8") as fh:
|
||||
metadata = json.load(fh)
|
||||
except (json.JSONDecodeError, OSError) as exc:
|
||||
logger.warning("Failed to read %s: %s", metadata_path, exc)
|
||||
else:
|
||||
logger.warning(
|
||||
"Metadata file not found for %s (expected: %s)",
|
||||
repo_id, metadata_path,
|
||||
)
|
||||
|
||||
enriched.append({
|
||||
"repo_id": repo_id,
|
||||
"model_path": model_path,
|
||||
"success": res.get("success", False),
|
||||
"errors": res.get("errors", []),
|
||||
"metadata": metadata,
|
||||
})
|
||||
|
||||
return enriched
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def main(argv: List[str]) -> int:
|
||||
args = _parse_args(argv)
|
||||
_setup_logging(args.verbose)
|
||||
|
||||
output_dir = os.path.abspath(os.path.expanduser(args.output))
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
# ---- Phase 0: Initialise shared state ----
|
||||
_phase_header("Initialise")
|
||||
settings = load_settings(args.settings)
|
||||
logger.info(
|
||||
"LLM config from --settings: provider=%s model=%s api_base=%s",
|
||||
settings["llm_provider"],
|
||||
settings["llm_model"],
|
||||
settings["llm_api_base"],
|
||||
)
|
||||
# Load the production base model list (replaces the old hardcoded list)
|
||||
await init_supported_base_models()
|
||||
|
||||
# ---- Load entries ----
|
||||
_phase_header("Load repo entries & construct initial metadata")
|
||||
entries = load_repo_ids(args.models, max_models=args.sample if args.sample > 0 else None)
|
||||
model_paths, repo_ids = create_all_initial_metadata(
|
||||
entries, output_dir, skip_existing=True,
|
||||
)
|
||||
print(f" {len(model_paths)} repos ready", file=sys.stderr)
|
||||
|
||||
# ---- Phase 1.5: Preprocessing audit ----
|
||||
audit_summary: Dict[str, Any] = {}
|
||||
t_start = time.perf_counter()
|
||||
audit_summary = await _run_preprocessing_audit(entries, output_dir)
|
||||
|
||||
if args.audit_only:
|
||||
total_wall = time.perf_counter() - t_start
|
||||
print(f"\n Audit-only done in {total_wall:.0f}s", file=sys.stderr)
|
||||
print(f" Audit data: {output_dir}/preprocessing_audit.json", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
# ---- Phase 2: Enrichment ----
|
||||
enrichment_results: List[Dict[str, Any]] = []
|
||||
if not args.no_enrich:
|
||||
_phase_header("Enrich metadata via LLM")
|
||||
enrichment_out = await _run_enrichment(
|
||||
model_paths, repo_ids, output_dir, args.timeout, args.verbose,
|
||||
)
|
||||
enrichment_results = enrichment_out["results"]
|
||||
else:
|
||||
print(" Enrichment skipped (--no-enrich)", file=sys.stderr)
|
||||
|
||||
t_enrich = time.perf_counter()
|
||||
|
||||
# ---- Phase 3: Evaluation ----
|
||||
_phase_header("Evaluate enriched metadata")
|
||||
enriched = _collect_enriched_metadata(model_paths, repo_ids, enrichment_results)
|
||||
scores = evaluate_batch(enriched)
|
||||
agg = aggregate_scores(scores)
|
||||
print(
|
||||
f" Mean total score: {agg.get('total_score', {}).get('mean', 'N/A')} / 100",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(
|
||||
f" Models scored: {agg.get('model_count', 0)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
# ---- Phase 4: Report generation ----
|
||||
_phase_header("Generate reports")
|
||||
duration_summary: Dict[str, Any] | None = None
|
||||
if enrichment_results:
|
||||
durations = [r.get("duration_s", 0) for r in enrichment_results if r.get("duration_s")]
|
||||
if durations:
|
||||
sorted_d = sorted(durations)
|
||||
m = len(sorted_d) // 2
|
||||
duration_summary = {
|
||||
"total_wall_s": round(t_enrich - t_start, 1),
|
||||
"mean_s": round(sum(durations) / len(durations), 1),
|
||||
"median_s": round(sorted_d[m] if len(sorted_d) % 2 else (sorted_d[m - 1] + sorted_d[m]) / 2, 1),
|
||||
"min_s": round(min(durations), 1),
|
||||
"max_s": round(max(durations), 1),
|
||||
}
|
||||
|
||||
# Check LLM config consistency after enrichment (LLMService is now initialized)
|
||||
actual_llm_cfg = _get_actual_llm_config()
|
||||
config_warnings = _compare_llm_config(settings, actual_llm_cfg)
|
||||
|
||||
save_json_report(
|
||||
agg, scores, enrichment_results, output_dir, duration_summary,
|
||||
audit_summary=audit_summary, config_warnings=config_warnings,
|
||||
)
|
||||
generate_markdown_report(
|
||||
agg, scores, output_dir, duration_summary,
|
||||
audit_summary=audit_summary, config_warnings=config_warnings,
|
||||
)
|
||||
|
||||
# ---- Final summary ----
|
||||
total_wall = time.perf_counter() - t_start
|
||||
print(f"\n Done in {total_wall:.0f}s ({total_wall / 60:.1f} min)", file=sys.stderr)
|
||||
print(f" Reports: {output_dir}/report.md, {output_dir}/report.json", file=sys.stderr)
|
||||
print(file=sys.stderr)
|
||||
|
||||
return 0 if agg.get("success_count", 0) > 0 else 1
|
||||
|
||||
|
||||
def entry_point() -> int:
|
||||
return asyncio.run(main(sys.argv[1:]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(entry_point())
|
||||
@@ -0,0 +1,376 @@
|
||||
{
|
||||
"description": "Ground truth base_model mapping for HF LoRA enrichment test data",
|
||||
"generated_at": "2026-07-05T18:20:00+08:00",
|
||||
"inference_method": "Manual analysis of YAML base_model field + README content + filename clues",
|
||||
"canonical_list_source": "Fallback list in config.py + CivitAI production API (73 models total)",
|
||||
"entries": [
|
||||
{
|
||||
"repo_id": "k2styles/krea-2-cobalt-sky-anime-lora",
|
||||
"safetensors_name": "cobalt-sky-anime.safetensors",
|
||||
"yaml_base_model_raw": "krea/Krea-2-Turbo",
|
||||
"correct_base_model": "Krea 2",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "k2styles/krea-2-azure-gouache-daylight-lora",
|
||||
"safetensors_name": "azure-gouache-daylight.safetensors",
|
||||
"yaml_base_model_raw": "krea/Krea-2-Turbo",
|
||||
"correct_base_model": "Krea 2",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "TheDivergentAI/krea2-turbo-distill-lora",
|
||||
"safetensors_name": "krea2_turbo_distill_r128.safetensors",
|
||||
"yaml_base_model_raw": "krea/Krea-2-Raw",
|
||||
"correct_base_model": "Krea 2",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "DeverStyle/Krea2-Loras",
|
||||
"safetensors_name": "n0t_f4l_000001000.safetensors",
|
||||
"yaml_base_model_raw": "krea/Krea-2-Turbo",
|
||||
"correct_base_model": "Krea 2",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "Komorebi1995/krea2-raw-jpaf-celpaint-lora",
|
||||
"safetensors_name": "krea2_raw_jpaf_celpaint_full_v1.safetensors",
|
||||
"yaml_base_model_raw": null,
|
||||
"correct_base_model": "Krea 2",
|
||||
"confidence": "high",
|
||||
"evidence": "Filename contains 'krea2'"
|
||||
},
|
||||
{
|
||||
"repo_id": "artificialguybr/pixelartredmond-1-5v-pixel-art-loras-for-sd-1-5",
|
||||
"safetensors_name": "PixelArtRedmond15V-PixelArt-PIXARFK.safetensors",
|
||||
"yaml_base_model_raw": "runwayml/stable-diffusion-v1-5",
|
||||
"correct_base_model": "SD 1.5",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "Shakker-Labs/FLUX.1-dev-LoRA-Logo-Design",
|
||||
"safetensors_name": "FLUX-dev-lora-Logo-Design.safetensors",
|
||||
"yaml_base_model_raw": "black-forest-labs/FLUX.1-dev",
|
||||
"correct_base_model": "Flux.1 D",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML base_model: FLUX.1-dev → dev → D"
|
||||
},
|
||||
{
|
||||
"repo_id": "glif-loradex-trainer/bingbangboom_flux_surf",
|
||||
"safetensors_name": "flux_surf_000001500.safetensors",
|
||||
"yaml_base_model_raw": "black-forest-labs/FLUX.1-dev",
|
||||
"correct_base_model": "Flux.1 D",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "prithivMLmods/Ton618-Epic-Realism-Flux-LoRA",
|
||||
"safetensors_name": "Epic-Realism-Unpruned.safetensors",
|
||||
"yaml_base_model_raw": "black-forest-labs/FLUX.1-dev",
|
||||
"correct_base_model": "Flux.1 D",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "prithivMLmods/Fashion-Hut-Modeling-LoRA",
|
||||
"safetensors_name": "Fashion-Modeling.safetensors",
|
||||
"yaml_base_model_raw": "black-forest-labs/FLUX.1-dev",
|
||||
"correct_base_model": "Flux.1 D",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "prithivMLmods/Retro-Pixel-Flux-LoRA",
|
||||
"safetensors_name": "Retro-Pixel.safetensors",
|
||||
"yaml_base_model_raw": "black-forest-labs/FLUX.1-dev",
|
||||
"correct_base_model": "Flux.1 D",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "D1-3105/HiDream-E1-Full_lora",
|
||||
"safetensors_name": "HiDream-E1-Full.safetensors",
|
||||
"yaml_base_model_raw": "HiDream-ai/HiDream-E1-Full",
|
||||
"correct_base_model": "HiDream",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field; filename contains 'HiDream'"
|
||||
},
|
||||
{
|
||||
"repo_id": "renderartist/Classic-Painting-Z-Image-Turbo-LoRA",
|
||||
"safetensors_name": "Classic_Painting_Z_Image_Turbo_v1_renderartist_1750.safetensors",
|
||||
"yaml_base_model_raw": "Tongyi-MAI/Z-Image-Turbo",
|
||||
"correct_base_model": "ZImageTurbo",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field; filename contains 'Z-Image-Turbo'"
|
||||
},
|
||||
{
|
||||
"repo_id": "DeverStyle/Z-Image-loras",
|
||||
"safetensors_name": "z_image_archer_style.safetensors",
|
||||
"yaml_base_model_raw": "Tongyi-MAI/Z-Image-Turbo",
|
||||
"correct_base_model": "ZImageTurbo",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "deadman44/Z-Image_LoRA",
|
||||
"safetensors_name": "lora_zimage_turbo_myjs_alpha01.safetensors",
|
||||
"yaml_base_model_raw": null,
|
||||
"correct_base_model": "ZImageTurbo",
|
||||
"confidence": "high",
|
||||
"evidence": "Filename contains 'zimage_turbo'"
|
||||
},
|
||||
{
|
||||
"repo_id": "zyuzuguldu/vton-lora-linen",
|
||||
"safetensors_name": "pytorch_lora_weights.safetensors",
|
||||
"yaml_base_model_raw": "stabilityai/stable-diffusion-xl-base-1.0",
|
||||
"correct_base_model": "SDXL 1.0",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "svntax-dev/pixel_spritesheet_4walk_small_lora_v1",
|
||||
"safetensors_name": "pixel_4walk_small_flux2_klein_base_4b_v1_000002750.safetensors",
|
||||
"yaml_base_model_raw": "black-forest-labs/FLUX.2-klein-base-4B",
|
||||
"correct_base_model": "Flux.2 Klein 4B-base",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML base_model: FLUX.2-klein-base-4B"
|
||||
},
|
||||
{
|
||||
"repo_id": "Haruka041/z-image-anime-lora",
|
||||
"safetensors_name": "sk_anime_style_v1.0.safetensors",
|
||||
"yaml_base_model_raw": "Tongyi-MAI/Z-Image-Turbo",
|
||||
"correct_base_model": "ZImageTurbo",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "systms/SYSTMS-INFL8-LoRA-Wan22",
|
||||
"safetensors_name": "SYSTMS_INFL8_LORA_WAN22_low_noise.safetensors",
|
||||
"yaml_base_model_raw": "Wan-AI/Wan2.2-I2V-A14B",
|
||||
"correct_base_model": "Wan Video 2.2 I2V-A14B",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML base_model: Wan2.2-I2V-A14B"
|
||||
},
|
||||
{
|
||||
"repo_id": "crafiq/flux-2-klein-9b-360-panorama-lora",
|
||||
"safetensors_name": "flux-2-klein-9b-360-panorama-lora.safetensors",
|
||||
"yaml_base_model_raw": "black-forest-labs/FLUX.2-klein-base-9B",
|
||||
"correct_base_model": "Flux.2 Klein 9B-base",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML base_model: FLUX.2-klein-base-9B; filename contains 'flux-2-klein-9b'"
|
||||
},
|
||||
{
|
||||
"repo_id": "Leon1000/pixel_spritesheet_4walk_small_lora_v1",
|
||||
"safetensors_name": "pixel_4walk_small_flux2_klein_base_4b_v1_000002750.safetensors",
|
||||
"yaml_base_model_raw": "black-forest-labs/FLUX.2-klein-base-4B",
|
||||
"correct_base_model": "Flux.2 Klein 4B-base",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML base_model: FLUX.2-klein-base-4B; filename contains 'flux2_klein_base_4b'"
|
||||
},
|
||||
{
|
||||
"repo_id": "Muapi/pov-missionary-legs-together-lora",
|
||||
"safetensors_name": "pov-missionary-legs-together-lora.safetensors",
|
||||
"yaml_base_model_raw": "OnomaAIResearch/Illustrious-xl-early-release-v0",
|
||||
"correct_base_model": "Illustrious",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML base_model: OnomaAIResearch/Illustrious-*"
|
||||
},
|
||||
{
|
||||
"repo_id": "ostris/ideogram_4_unconditional_lora",
|
||||
"safetensors_name": "ideogram_4_unconditional_lora_r16.safetensors",
|
||||
"yaml_base_model_raw": "ideogram-ai/ideogram-4-fp8",
|
||||
"correct_base_model": "Ideogram 4.0",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML base_model: ideogram-ai/ideogram-4 → Ideogram 4.0; filename contains 'ideogram_4'"
|
||||
},
|
||||
{
|
||||
"repo_id": "ilkerzgi/krea-2-bleached-surreal-uncanny-lora",
|
||||
"safetensors_name": "bleached-surreal-uncanny-comfy.safetensors",
|
||||
"yaml_base_model_raw": "krea/Krea-2-Turbo",
|
||||
"correct_base_model": "Krea 2",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "ilkerzgi/krea-2-azure-surreal-collage-lora",
|
||||
"safetensors_name": "azure-surreal-collage-comfy.safetensors",
|
||||
"yaml_base_model_raw": "krea/Krea-2-Turbo",
|
||||
"correct_base_model": "Krea 2",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "ilkerzgi/krea-2-airy-gouache-minimalist-lora",
|
||||
"safetensors_name": "airy-gouache-minimalist-comfy.safetensors",
|
||||
"yaml_base_model_raw": "krea/Krea-2-Turbo",
|
||||
"correct_base_model": "Krea 2",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "k2styles/krea-2-airy-watercolor-chibi-lora",
|
||||
"safetensors_name": "airy-watercolor-chibi.safetensors",
|
||||
"yaml_base_model_raw": "krea/Krea-2-Turbo",
|
||||
"correct_base_model": "Krea 2",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "TakeAswing/sdxl-lora-lofi",
|
||||
"safetensors_name": "pytorch_lora_weights.safetensors",
|
||||
"yaml_base_model_raw": "stabilityai/stable-diffusion-xl-base-1.0",
|
||||
"correct_base_model": "SDXL 1.0",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field; repo name contains 'sdxl'"
|
||||
},
|
||||
{
|
||||
"repo_id": "heville/anna-lora-krea2",
|
||||
"safetensors_name": "pytorch_lora_weights.safetensors",
|
||||
"yaml_base_model_raw": "krea/Krea-2-Raw",
|
||||
"correct_base_model": "Krea 2",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field; repo name contains 'krea2'"
|
||||
},
|
||||
{
|
||||
"repo_id": "Brioch/krea2_loras",
|
||||
"safetensors_name": "mashap_ohwx_woman_krea2.safetensors",
|
||||
"yaml_base_model_raw": "krea/Krea-2-Raw",
|
||||
"correct_base_model": "Krea 2",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "hr16/Miwano-Rag-LoRA",
|
||||
"safetensors_name": "Miwano-Rag-epoch10.lora.safetensors",
|
||||
"yaml_base_model_raw": null,
|
||||
"correct_base_model": "SD 1.5",
|
||||
"confidence": "high",
|
||||
"evidence": "README: base model is Kanianime (SD 1.5 fine-tune)"
|
||||
},
|
||||
{
|
||||
"repo_id": "ikuseiso/Personal_Lora_collections",
|
||||
"safetensors_name": "vergil_devil_may_cry.safetensors",
|
||||
"yaml_base_model_raw": null,
|
||||
"correct_base_model": "SD 1.5",
|
||||
"confidence": "high",
|
||||
"evidence": "Sample prompt shows Model: AbyssOrangeMix (SD 1.5), 512x768"
|
||||
},
|
||||
{
|
||||
"repo_id": "Tanger/LoraByTanger",
|
||||
"safetensors_name": "(v4)layila-000005.safetensors",
|
||||
"yaml_base_model_raw": null,
|
||||
"correct_base_model": "SD 1.5",
|
||||
"confidence": "high",
|
||||
"evidence": "README: trained on anything4.5 (SD 1.5) and nai (SD 1.5); test images on AbyssOrangeMix2_hard"
|
||||
},
|
||||
{
|
||||
"repo_id": "DS-Archive/ds-LoRA",
|
||||
"safetensors_name": "dsharu-v2_lc.safetensors",
|
||||
"yaml_base_model_raw": null,
|
||||
"correct_base_model": "SD 1.5",
|
||||
"confidence": "high",
|
||||
"evidence": "README explicitly states 'Stable Diffusion 1.5'"
|
||||
},
|
||||
{
|
||||
"repo_id": "soknife/loras",
|
||||
"safetensors_name": "irys-regular-subject-more.safetensors",
|
||||
"yaml_base_model_raw": null,
|
||||
"correct_base_model": "SD 1.5",
|
||||
"confidence": "high",
|
||||
"evidence": "README mentions SD 1.5 fine-tune models (PastelMix, AbyssOrangeMix, Anything)"
|
||||
},
|
||||
{
|
||||
"repo_id": "prompthero/openjourney-lora",
|
||||
"safetensors_name": "openjourneyLora.safetensors",
|
||||
"yaml_base_model_raw": "stabilityai/stable-diffusion-2-1-base",
|
||||
"correct_base_model": "SD 2.1",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "Banano/banchan-lora",
|
||||
"safetensors_name": "Bananochan-PonySDXL-v2.safetensors",
|
||||
"yaml_base_model_raw": null,
|
||||
"correct_base_model": "Pony",
|
||||
"confidence": "medium",
|
||||
"evidence": "Filename contains 'PonySDXL-v2' → Pony base model"
|
||||
},
|
||||
{
|
||||
"repo_id": "Maisman/No-Game-NoLife-LoRAs",
|
||||
"safetensors_name": "ShiroNGNL2_Lora.safetensors",
|
||||
"yaml_base_model_raw": null,
|
||||
"correct_base_model": "SD 1.5",
|
||||
"confidence": "high",
|
||||
"evidence": "Sample prompts show Model: abyssorangemix2_Hardcore (SD 1.5), 512x768"
|
||||
},
|
||||
{
|
||||
"repo_id": "EarthnDusk/Gambit_Xmen_Anime_Lora_V1.1",
|
||||
"safetensors_name": "RemyLebeau.safetensors",
|
||||
"yaml_base_model_raw": null,
|
||||
"correct_base_model": "SD 1.5",
|
||||
"confidence": "high",
|
||||
"evidence": "Trained Feb 2023 via Kohya LoRA (pre-SDXL era), SD 1.5 lineage"
|
||||
},
|
||||
{
|
||||
"repo_id": "EarthnDusk/DuskfallArt_LoRa",
|
||||
"safetensors_name": "DuskfallArt.safetensors",
|
||||
"yaml_base_model_raw": "stable-diffusion-v1-5/stable-diffusion-v1-5",
|
||||
"correct_base_model": "SD 1.5",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "gaoxiao/pokemon-lora",
|
||||
"safetensors_name": "pytorch_lora_weights.safetensors",
|
||||
"yaml_base_model_raw": "runwayml/stable-diffusion-v1-5",
|
||||
"correct_base_model": "SD 1.5",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "wtcherr/sd-unsplash_10k_canny-model-control-lora",
|
||||
"safetensors_name": "diffusion_pytorch_model.safetensors",
|
||||
"yaml_base_model_raw": "runwayml/stable-diffusion-v1-5",
|
||||
"correct_base_model": "SD 1.5",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "wtcherr/sd-unsplash_10k_blur_rand_KS-model-control-lora",
|
||||
"safetensors_name": "diffusion_pytorch_model.safetensors",
|
||||
"yaml_base_model_raw": "runwayml/stable-diffusion-v1-5",
|
||||
"correct_base_model": "SD 1.5",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML frontmatter base_model field"
|
||||
},
|
||||
{
|
||||
"repo_id": "samurai-architects/lora-starbucks",
|
||||
"safetensors_name": "starbucks_interior.safetensors",
|
||||
"yaml_base_model_raw": null,
|
||||
"correct_base_model": null,
|
||||
"confidence": "none",
|
||||
"evidence": "README too minimal, no base_model in YAML, cannot determine"
|
||||
},
|
||||
{
|
||||
"repo_id": "prithivMLmods/Flux-Long-Toon-LoRA",
|
||||
"safetensors_name": "Long-Toon.safetensors",
|
||||
"yaml_base_model_raw": "black-forest-labs/FLUX.1-dev",
|
||||
"correct_base_model": "Flux.1 D",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML base_model: FLUX.1-dev (dev → D)"
|
||||
},
|
||||
{
|
||||
"repo_id": "Limbicnation/pixel-art-lora",
|
||||
"safetensors_name": "pytorch_lora_weights.comfyui.safetensors",
|
||||
"yaml_base_model_raw": "black-forest-labs/FLUX.2-klein-4B",
|
||||
"correct_base_model": "Flux.2 Klein 4B",
|
||||
"confidence": "high",
|
||||
"evidence": "YAML base_model: FLUX.2-klein-4B; README explicitly states base model"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
k2styles/krea-2-cobalt-sky-anime-lora, cobalt-sky-anime.safetensors
|
||||
k2styles/krea-2-azure-gouache-daylight-lora, azure-gouache-daylight.safetensors
|
||||
TheDivergentAI/krea2-turbo-distill-lora, krea2_turbo_distill_r128.safetensors
|
||||
DeverStyle/Krea2-Loras, n0t_f4l_000001000.safetensors
|
||||
Komorebi1995/krea2-raw-jpaf-celpaint-lora, krea2_raw_jpaf_celpaint_full_v1.safetensors
|
||||
artificialguybr/pixelartredmond-1-5v-pixel-art-loras-for-sd-1-5, PixelArtRedmond15V-PixelArt-PIXARFK.safetensors
|
||||
Shakker-Labs/FLUX.1-dev-LoRA-Logo-Design, FLUX-dev-lora-Logo-Design.safetensors
|
||||
glif-loradex-trainer/bingbangboom_flux_surf, flux_surf_000001500.safetensors
|
||||
prithivMLmods/Ton618-Epic-Realism-Flux-LoRA, Epic-Realism-Unpruned.safetensors
|
||||
prithivMLmods/Fashion-Hut-Modeling-LoRA, Fashion-Modeling.safetensors
|
||||
prithivMLmods/Retro-Pixel-Flux-LoRA, Retro-Pixel.safetensors
|
||||
D1-3105/HiDream-E1-Full_lora, HiDream-E1-Full.safetensors
|
||||
renderartist/Classic-Painting-Z-Image-Turbo-LoRA, Classic_Painting_Z_Image_Turbo_v1_renderartist_1750.safetensors
|
||||
DeverStyle/Z-Image-loras, z_image_archer_style.safetensors
|
||||
deadman44/Z-Image_LoRA, lora_zimage_turbo_myjs_alpha01.safetensors
|
||||
zyuzuguldu/vton-lora-linen, pytorch_lora_weights.safetensors
|
||||
svntax-dev/pixel_spritesheet_4walk_small_lora_v1, pixel_4walk_small_flux2_klein_base_4b_v1_000002750.safetensors
|
||||
Haruka041/z-image-anime-lora, sk_anime_style_v1.0.safetensors
|
||||
systms/SYSTMS-INFL8-LoRA-Wan22, SYSTMS_INFL8_LORA_WAN22_low_noise.safetensors
|
||||
crafiq/flux-2-klein-9b-360-panorama-lora, flux-2-klein-9b-360-panorama-lora.safetensors
|
||||
Leon1000/pixel_spritesheet_4walk_small_lora_v1, pixel_4walk_small_flux2_klein_base_4b_v1_000002750.safetensors
|
||||
Muapi/pov-missionary-legs-together-lora, pov-missionary-legs-together-lora.safetensors
|
||||
ostris/ideogram_4_unconditional_lora, ideogram_4_unconditional_lora_r16.safetensors
|
||||
ilkerzgi/krea-2-bleached-surreal-uncanny-lora, bleached-surreal-uncanny-comfy.safetensors
|
||||
ilkerzgi/krea-2-azure-surreal-collage-lora, azure-surreal-collage-comfy.safetensors
|
||||
ilkerzgi/krea-2-airy-gouache-minimalist-lora, airy-gouache-minimalist-comfy.safetensors
|
||||
k2styles/krea-2-airy-watercolor-chibi-lora, airy-watercolor-chibi.safetensors
|
||||
TakeAswing/sdxl-lora-lofi, pytorch_lora_weights.safetensors
|
||||
heville/anna-lora-krea2, pytorch_lora_weights.safetensors
|
||||
Brioch/krea2_loras, mashap_ohwx_woman_krea2.safetensors
|
||||
hr16/Miwano-Rag-LoRA, Miwano-Rag-epoch10.lora.safetensors
|
||||
ikuseiso/Personal_Lora_collections, vergil_devil_may_cry.safetensors
|
||||
Tanger/LoraByTanger, (v4)layila-000005.safetensors
|
||||
DS-Archive/ds-LoRA, dsharu-v2_lc.safetensors
|
||||
soknife/loras, irys-regular-subject-more.safetensors
|
||||
prompthero/openjourney-lora, openjourneyLora.safetensors
|
||||
Banano/banchan-lora, Bananochan-PonySDXL-v2.safetensors
|
||||
Maisman/No-Game-NoLife-LoRAs, ShiroNGNL2_Lora.safetensors
|
||||
EarthnDusk/Gambit_Xmen_Anime_Lora_V1.1, RemyLebeau.safetensors
|
||||
EarthnDusk/DuskfallArt_LoRa, DuskfallArt.safetensors
|
||||
gaoxiao/pokemon-lora, pytorch_lora_weights.safetensors
|
||||
wtcherr/sd-unsplash_10k_canny-model-control-lora, diffusion_pytorch_model.safetensors
|
||||
wtcherr/sd-unsplash_10k_blur_rand_KS-model-control-lora, diffusion_pytorch_model.safetensors
|
||||
samurai-architects/lora-starbucks, starbucks_interior.safetensors
|
||||
prithivMLmods/Flux-Long-Toon-LoRA, Long-Toon.safetensors
|
||||
Limbicnation/pixel-art-lora, pytorch_lora_weights.comfyui.safetensors
|
||||
@@ -62,6 +62,20 @@ describe('DownloadManager.detectUrlType — HF URL detection', () => {
|
||||
expect(result).toEqual({ type: 'civitai' });
|
||||
});
|
||||
|
||||
it('detects CivitAI URL on civitai.red domain', () => {
|
||||
const result = DownloadManager.detectUrlType(
|
||||
'https://civitai.red/models/12345/my-model'
|
||||
);
|
||||
expect(result).toEqual({ type: 'civitai' });
|
||||
});
|
||||
|
||||
it('detects CivitAI URL on civitai.green domain', () => {
|
||||
const result = DownloadManager.detectUrlType(
|
||||
'https://civitai.green/models/67890/another-model'
|
||||
);
|
||||
expect(result).toEqual({ type: 'civitai' });
|
||||
});
|
||||
|
||||
it('detects CivArchive URL', () => {
|
||||
const result = DownloadManager.detectUrlType(
|
||||
'https://civarchive.com/models/456'
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,490 @@
|
||||
"""Tests for ``readme_processor.py`` — HF README processing for enrich_hf_metadata.
|
||||
|
||||
Import via ``importlib`` to avoid the ``folder_paths`` dependency in
|
||||
``py.services.agent.__init__``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_MODULE_PATH = Path(__file__).parents[2] / "py" / "services" / "agent" / "skills" / "enrich_hf_metadata" / "readme_processor.py"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def R():
|
||||
"""Load the ``readme_processor`` module once per session."""
|
||||
spec = importlib.util.spec_from_file_location("readme_processor", str(_MODULE_PATH))
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# extract_gallery_images
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestExtractGalleryImages:
|
||||
def test_empty(self, R):
|
||||
assert R.extract_gallery_images("", "repo") == []
|
||||
assert R.extract_gallery_images("no frontmatter", "repo") == []
|
||||
|
||||
def test_no_widget(self, R):
|
||||
readme = "---\ntags: [test]\n---\nbody"
|
||||
assert R.extract_gallery_images(readme, "repo") == []
|
||||
|
||||
def test_widget_simple_text(self, R):
|
||||
"""YAML ``text: 'plain'`` → extracted as-is."""
|
||||
readme = """---
|
||||
widget:
|
||||
- text: 'a cute cat'
|
||||
output:
|
||||
url: images/cat.png
|
||||
---"""
|
||||
imgs = R.extract_gallery_images(readme, "user/repo")
|
||||
assert len(imgs) == 1
|
||||
assert imgs[0]["meta"]["prompt"] == "a cute cat"
|
||||
assert "images/cat.png" in imgs[0]["url"]
|
||||
|
||||
def test_widget_unquoted_text(self, R):
|
||||
"""YAML ``text: plain value`` without quotes."""
|
||||
readme = """---
|
||||
widget:
|
||||
- text: simple text
|
||||
output:
|
||||
url: img.png
|
||||
---"""
|
||||
imgs = R.extract_gallery_images(readme, "user/repo")
|
||||
assert len(imgs) == 1
|
||||
assert imgs[0]["meta"]["prompt"] == "simple text"
|
||||
|
||||
def test_widget_block_scalar(self, R):
|
||||
"""YAML ``text: >-`` folded block scalar — extract actual content."""
|
||||
readme = """---
|
||||
widget:
|
||||
- text: >-
|
||||
Long toons, a close-up of a cartoon characters face is featured in a
|
||||
vibrant red backdrop.
|
||||
output:
|
||||
url: images/LT4.png
|
||||
---"""
|
||||
imgs = R.extract_gallery_images(readme, "user/repo")
|
||||
assert len(imgs) == 1
|
||||
prompt = imgs[0]["meta"]["prompt"]
|
||||
assert "Long toons" in prompt
|
||||
assert "vibrant red backdrop" in prompt
|
||||
assert prompt != ">-"
|
||||
|
||||
def test_widget_dash_prefix_output(self, R):
|
||||
"""YAML ``- output:`` (dash prefix) — regression for widget parsing."""
|
||||
readme = """---
|
||||
widget:
|
||||
- output:
|
||||
url: images/test.png
|
||||
text: dash test
|
||||
---"""
|
||||
imgs = R.extract_gallery_images(readme, "user/repo")
|
||||
assert len(imgs) == 1
|
||||
assert imgs[0]["meta"]["prompt"] == "dash test"
|
||||
assert "images/test.png" in imgs[0]["url"]
|
||||
|
||||
def test_widget_mixed_entries(self, R):
|
||||
"""Multiple widget entries with different text styles."""
|
||||
readme = """---
|
||||
widget:
|
||||
- text: >-
|
||||
First entry description.
|
||||
output:
|
||||
url: img1.png
|
||||
- text: second entry
|
||||
output:
|
||||
url: img2.png
|
||||
- text: 'third entry'
|
||||
output:
|
||||
url: img3.png
|
||||
---"""
|
||||
imgs = R.extract_gallery_images(readme, "user/repo")
|
||||
assert len(imgs) == 3
|
||||
assert imgs[0]["meta"]["prompt"] == "First entry description."
|
||||
assert imgs[1]["meta"]["prompt"] == "second entry"
|
||||
assert imgs[2]["meta"]["prompt"] == "third entry"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# extract_simple_markdown_images
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestExtractSimpleMarkdownImages:
|
||||
def test_empty(self, R):
|
||||
assert R.extract_simple_markdown_images("", "repo") == []
|
||||
|
||||
def test_basic_markdown_image(self, R):
|
||||
"""```` → absolute URL."""
|
||||
imgs = R.extract_simple_markdown_images("", "u/r")
|
||||
assert len(imgs) == 1
|
||||
assert "image_0.png" in imgs[0]["url"]
|
||||
assert imgs[0]["meta"]["prompt"] == "test"
|
||||
|
||||
def test_absolute_url(self, R):
|
||||
"""```` → keep as-is."""
|
||||
imgs = R.extract_simple_markdown_images(
|
||||
"", "u/r"
|
||||
)
|
||||
assert len(imgs) == 1
|
||||
assert imgs[0]["url"] == "https://example.com/img.png"
|
||||
|
||||
def test_skips_code_fences(self, R):
|
||||
"""Inside ``` blocks should be ignored."""
|
||||
text = """outside
|
||||
```
|
||||

|
||||
```
|
||||
outside again
|
||||
"""
|
||||
imgs = R.extract_simple_markdown_images(text, "u/r")
|
||||
assert len(imgs) == 1
|
||||
assert "valid.png" in imgs[0]["url"]
|
||||
|
||||
def test_deduplicates(self, R):
|
||||
text = "\n"
|
||||
imgs = R.extract_simple_markdown_images(text, "u/r")
|
||||
assert len(imgs) == 1 # deduplicated
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# extract_html_img_tags
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestExtractHtmlImgTags:
|
||||
def test_double_quoted_src(self, R):
|
||||
imgs = R.extract_html_img_tags('<img src="./img.png">', "u/r")
|
||||
assert len(imgs) == 1
|
||||
assert "img.png" in imgs[0]["url"]
|
||||
|
||||
def test_single_quoted_src(self, R):
|
||||
imgs = R.extract_html_img_tags("<img src='./img.png'>", "u/r")
|
||||
assert len(imgs) == 1
|
||||
assert "img.png" in imgs[0]["url"]
|
||||
|
||||
def test_absolute_url(self, R):
|
||||
imgs = R.extract_html_img_tags(
|
||||
'<img src="https://cdn.example.com/img.png">', "u/r"
|
||||
)
|
||||
assert len(imgs) == 1
|
||||
assert imgs[0]["url"] == "https://cdn.example.com/img.png"
|
||||
|
||||
def test_deduplicates_across_formats(self, R):
|
||||
text = '<img src="./img.png">\n<img src=\'./img.png\'>'
|
||||
imgs = R.extract_html_img_tags(text, "u/r")
|
||||
assert len(imgs) == 1
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# extract_gallery_table_images
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestExtractGalleryTableImages:
|
||||
def test_gallery_table(self, R):
|
||||
text = """| Preview | Prompt |
|
||||
|--------|--------|
|
||||
|  | a cat |
|
||||
|  | a dog |"""
|
||||
imgs = R.extract_gallery_table_images(text, "u/r")
|
||||
assert len(imgs) == 2
|
||||
assert imgs[0]["meta"]["prompt"] == "a cat"
|
||||
assert "a.png" in imgs[0]["url"]
|
||||
assert imgs[1]["meta"]["prompt"] == "a dog"
|
||||
|
||||
def test_skips_non_gallery_table(self, R):
|
||||
text = """| Parameter | Value |
|
||||
|----------|-------|
|
||||
| Steps | 4 |"""
|
||||
imgs = R.extract_gallery_table_images(text, "u/r")
|
||||
assert len(imgs) == 0
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# clean_readme_for_llm + strip helpers
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCleanReadmeForLlm:
|
||||
def test_preserves_plain_code_block(self, R):
|
||||
"""`` ``` `` without language tag → preserved (trigger words)."""
|
||||
text = """Before
|
||||
```
|
||||
pixel art sprite, game asset
|
||||
```
|
||||
After"""
|
||||
cleaned = R.clean_readme_for_llm(text)
|
||||
assert "pixel art sprite" in cleaned
|
||||
assert "game asset" in cleaned
|
||||
|
||||
def test_strips_fenced_code_with_lang(self, R):
|
||||
"""`` ```python `` → stripped."""
|
||||
text = "before\n```python\nimport torch\n```\nafter"
|
||||
cleaned = R.clean_readme_for_llm(text)
|
||||
assert "import torch" not in cleaned
|
||||
assert "before" in cleaned
|
||||
assert "after" in cleaned
|
||||
|
||||
def test_preserves_markdown_image_url(self, R):
|
||||
"""```` → URL kept for LLM preview extraction."""
|
||||
text = ""
|
||||
cleaned = R.clean_readme_for_llm(text)
|
||||
assert "./preview.png" in cleaned
|
||||
|
||||
def test_converts_html_img_tag_to_markdown_image(self, R):
|
||||
"""``<img src="...">`` → ```` preserving URL for LLM."""
|
||||
text = 'before\n<img src="logo.png">\nafter'
|
||||
cleaned = R.clean_readme_for_llm(text)
|
||||
assert "" in cleaned
|
||||
assert "logo.png" in cleaned # URL preserved for LLM extraction
|
||||
|
||||
def test_widget_stripped_frontmatter_preserved(self, R):
|
||||
"""Widget YAML stripped but ``base_model:`` kept."""
|
||||
text = """---
|
||||
tags: [test]
|
||||
widget:
|
||||
- text: >-
|
||||
long description here
|
||||
output:
|
||||
url: img.png
|
||||
base_model: black-forest-labs/FLUX.1-dev
|
||||
instance_prompt: test
|
||||
---"""
|
||||
cleaned = R.clean_readme_for_llm(text)
|
||||
assert "widget:" not in cleaned
|
||||
assert "black-forest-labs/FLUX.1-dev" in cleaned
|
||||
assert "instance_prompt: test" in cleaned
|
||||
|
||||
def test_training_table_stripped(self, R):
|
||||
"""Training-parameter table → stripped."""
|
||||
text = """before
|
||||
| LR Scheduler | constant |
|
||||
|--------------|---------|
|
||||
| Optimizer | AdamW |
|
||||
after"""
|
||||
cleaned = R.clean_readme_for_llm(text)
|
||||
assert "LR Scheduler" not in cleaned
|
||||
assert "Optimizer" not in cleaned
|
||||
assert "before" in cleaned
|
||||
assert "after" in cleaned
|
||||
|
||||
def test_best_dimensions_table_kept(self, R):
|
||||
"""Non-training table (Best Dimensions) → kept."""
|
||||
text = """## Best Dimensions
|
||||
- 768 x 1024 (Best)
|
||||
- 1024 x 1024 (Default)"""
|
||||
cleaned = R.clean_readme_for_llm(text)
|
||||
assert "768 x 1024" in cleaned
|
||||
|
||||
def test_boilerplate_section_stripped(self, R):
|
||||
text = """stuff
|
||||
## Download model
|
||||
[link](url)
|
||||
## Next section
|
||||
content"""
|
||||
cleaned = R.clean_readme_for_llm(text)
|
||||
assert "Download model" not in cleaned
|
||||
assert "Next section" in cleaned
|
||||
assert "content" in cleaned
|
||||
|
||||
def test_returns_empty_for_none(self, R):
|
||||
assert R.clean_readme_for_llm(None) == ""
|
||||
|
||||
def test_returns_empty_for_empty(self, R):
|
||||
assert R.clean_readme_for_llm("") == ""
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _is_heading / _heading_level
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestHeadingDetection:
|
||||
@pytest.mark.parametrize(
|
||||
"line,expected",
|
||||
[
|
||||
("# Title", 1),
|
||||
("## Sub", 2),
|
||||
("### Subsub", 3),
|
||||
("#### Subsubsub", 4),
|
||||
("<h1>Title</h1>", 1),
|
||||
("<h2>Sub</h2>", 2),
|
||||
("<h3 class='x'>Sub</h3>", 3),
|
||||
("<h4 id='y'>Sub</h4>", 4),
|
||||
("not a heading", 0),
|
||||
("###", 0), # no text after ###
|
||||
("</h2>", 0), # closing tag, not a heading
|
||||
("", 0),
|
||||
],
|
||||
)
|
||||
def test_heading_level(self, R, line, expected):
|
||||
assert R._heading_level(line) == expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"line,expected",
|
||||
[
|
||||
("# Title", True),
|
||||
("<h2>Sub</h2>", True),
|
||||
("</h2>", False), # closing tag
|
||||
("not heading", False),
|
||||
],
|
||||
)
|
||||
def test_is_heading(self, R, line, expected):
|
||||
assert R._is_heading(line) == expected
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# extract_relevant_section
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestExtractRelevantSection:
|
||||
def test_fallback_full_readme(self, R):
|
||||
"""No match → full README returned."""
|
||||
readme = "# Title\n\nsome content"
|
||||
assert R.extract_relevant_section(readme, "nonexistent") == readme
|
||||
|
||||
def test_empty_basename_returns_full(self, R):
|
||||
readme = "# Title"
|
||||
assert R.extract_relevant_section(readme, "") == readme
|
||||
|
||||
def test_match_heading_includes_yaml(self, R):
|
||||
"""Matching heading should still include YAML frontmatter."""
|
||||
readme = """---
|
||||
base_model: foo
|
||||
---
|
||||
# My-Model-Title
|
||||
|
||||
content
|
||||
## Subsection
|
||||
more"""
|
||||
section = R.extract_relevant_section(readme, "My-Model")
|
||||
assert "base_model: foo" in section
|
||||
assert "content" in section
|
||||
assert "Subsection" in section
|
||||
|
||||
def test_match_heading_includes_subheadings(self, R):
|
||||
"""``# Title`` match includes all ``##`` children."""
|
||||
readme = """# Main Title
|
||||
|
||||
## Child A
|
||||
content A
|
||||
## Child B
|
||||
content B
|
||||
## Child C
|
||||
content C"""
|
||||
section = R.extract_relevant_section(readme, "Main Title")
|
||||
assert "Child A" in section
|
||||
assert "Child B" in section
|
||||
assert "Child C" in section
|
||||
|
||||
def test_match_download_link(self, R):
|
||||
"""Download link containing basename → section extracted."""
|
||||
readme = """# Collection
|
||||
## Model A
|
||||
[Download](./model_a.safetensors)
|
||||
## MyModel
|
||||
[Download](./mymodel.safetensors)
|
||||
content here
|
||||
## Model B
|
||||
other"""
|
||||
section = R.extract_relevant_section(readme, "mymodel")
|
||||
assert "content here" in section
|
||||
assert "Model A" not in section # should not include sibling
|
||||
|
||||
def test_heading_closing_tag_not_boundary(self, R):
|
||||
"""``</h2>`` should NOT be treated as a section boundary."""
|
||||
readme = """# Title
|
||||
<p>some text</p>
|
||||
</h2>
|
||||
## Real Section
|
||||
content"""
|
||||
section = R.extract_relevant_section(readme, "Title")
|
||||
assert "Real Section" in section # forward walk should not stop at </h2>
|
||||
assert "content" in section
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _extract_frontmatter
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestExtractFrontmatter:
|
||||
def test_basic(self, R):
|
||||
assert R._extract_frontmatter("---\ntags: [a]\n---\nbody") == "\ntags: [a]\n"
|
||||
|
||||
def test_no_frontmatter(self, R):
|
||||
assert R._extract_frontmatter("no dashes") == ""
|
||||
|
||||
def test_empty_string(self, R):
|
||||
assert R._extract_frontmatter("") == ""
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _strip_widget_section
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestStripWidgetSection:
|
||||
def test_strip_widget_keep_base_model(self, R):
|
||||
"""Widget stripped but ``base_model:`` preserved."""
|
||||
text = """---
|
||||
tags: [test]
|
||||
widget:
|
||||
- text: >-
|
||||
long text
|
||||
output:
|
||||
url: img.png
|
||||
base_model: black-forest-labs/FLUX.1-dev
|
||||
---"""
|
||||
result = R._strip_widget_section(text)
|
||||
assert "widget:" not in result
|
||||
assert "black-forest-labs/FLUX.1-dev" in result
|
||||
|
||||
def test_no_widget_no_change(self, R):
|
||||
text = "---\ntags: [a]\n---"
|
||||
assert R._strip_widget_section(text) == text
|
||||
|
||||
def test_widget_at_end_of_frontmatter(self, R):
|
||||
"""Widget is the last YAML key before closing ---."""
|
||||
text = """---
|
||||
base_model: a
|
||||
widget:
|
||||
- text: x
|
||||
output:
|
||||
url: y.png
|
||||
---"""
|
||||
result = R._strip_widget_section(text)
|
||||
assert "widget:" not in result
|
||||
assert "base_model: a" in result
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _strip_fenced_code_blocks
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestStripFencedCodeBlocks:
|
||||
def test_strips_with_language(self, R):
|
||||
text = "a\n```python\ncode\n```\nb"
|
||||
assert R._strip_fenced_code_blocks(text) == "a\nb"
|
||||
|
||||
def test_keeps_plain_fence(self, R):
|
||||
"""`` ``` `` without language → preserved."""
|
||||
text = "a\n```\ntrigger words\n```\nb"
|
||||
assert "trigger words" in R._strip_fenced_code_blocks(text)
|
||||
|
||||
def test_pattern(self, R):
|
||||
text = "x\n```yaml\nkey: val\n```\ny"
|
||||
assert "key: val" not in R._strip_fenced_code_blocks(text)
|
||||
@@ -28,6 +28,7 @@
|
||||
'settings': dict({
|
||||
'civitai_api_key_set': True,
|
||||
'language': 'en',
|
||||
'llm_api_key_set': False,
|
||||
'theme': 'dark',
|
||||
}),
|
||||
'success': True,
|
||||
|
||||
@@ -728,6 +728,54 @@ async def test_register_nodes_includes_capabilities():
|
||||
assert stored_node["widget_names"] == ["ckpt_name"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_register_nodes_accepts_compound_node_ids():
|
||||
"""Subgraph nodes from expanded group nodes have compound IDs like '252:0'."""
|
||||
node_registry = NodeRegistry()
|
||||
handler = NodeRegistryHandler(
|
||||
node_registry=node_registry,
|
||||
prompt_server=FakePromptServer,
|
||||
standalone_mode=False,
|
||||
)
|
||||
|
||||
request = FakeRequest(
|
||||
json_data={
|
||||
"nodes": [
|
||||
{
|
||||
"node_id": "252:0",
|
||||
"graph_id": "252",
|
||||
"type": "CheckpointLoaderSimple",
|
||||
"title": "Checkpoint Loader (subgraph)",
|
||||
},
|
||||
{
|
||||
"node_id": "252:1",
|
||||
"graph_id": "252",
|
||||
"type": "CLIPLoader",
|
||||
"title": "CLIP Loader (subgraph)",
|
||||
},
|
||||
],
|
||||
"client_id": "test-client-1",
|
||||
}
|
||||
)
|
||||
|
||||
response = await handler.register_nodes(request)
|
||||
payload = json.loads(response.text)
|
||||
|
||||
assert response.status == 200
|
||||
assert payload["success"] is True
|
||||
assert "2 nodes registered" in payload["message"]
|
||||
|
||||
registry = await node_registry.get_merged_registry()
|
||||
assert registry["node_count"] == 2
|
||||
|
||||
nodes_map = registry["nodes"]
|
||||
assert "252:0" in nodes_map
|
||||
assert "252:1" in nodes_map
|
||||
assert nodes_map["252:0"]["id"] == 0
|
||||
assert nodes_map["252:0"]["graph_id"] == "252"
|
||||
assert nodes_map["252:1"]["id"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_node_widget_sends_payload():
|
||||
send_calls: list[tuple[str, dict]] = []
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user