mirror of
https://github.com/willmiao/ComfyUI-Lora-Manager.git
synced 2026-09-21 03:01:27 -03:00
Will Miao
5ab0e88abc
A model file could only ever be linked to huggingface.co: `set_hf_url` validated the URL with a huggingface-only regex, the agent fetched the card from a hardcoded HF URL, and the readme processor built every relative image path off `https://huggingface.co/{repo}/resolve/main`. ModelScope publishes the same model-card convention (README.md + YAML frontmatter, often carrying `base_model:` and `trigger_words:`) behind a public, key-less API, so the enrichment pipeline could already serve it - it was the plumbing that was HF-shaped, not the idea. Make the external source a first-class, provider-driven concept: - New `py/services/model_sources/` registry. A `ModelSource` owns URL recognition (lenient for stored values, strict for user input), the canonical page URL, model-card fetching, the asset base URL and the capability flags. `HuggingFaceSource` is the previous logic relocated; `ModelScopeSource` reads `/models/{o}/{n}/resolve/{master|main}/README.md` and falls back to `/api/v1/models/{o}/{n}/repo`. `TensorArtSource` is link-only on purpose: tensor.art answers plain HTTP clients with a Cloudflare challenge and its internal API (ap-east-1.tensorart.cloud / cn.tensorart.net) rejects every /v1/model/* route with "invalid authorization header", so it declares supports_enrichment=False rather than failing silently later. - Metadata gains `source_platform` + `source_url`; `hf_url` stays as a read/write alias, written only for Hugging Face, so existing sidecars, cached rows and third-party consumers keep working. Normalisation runs at the scanner, the persistent cache (both directions, plus two new columns behind an ALTER migration) and the linking handler - which is what stops a user who switches sources from leaving a stale `hf_url` on a ModelScope model. - The agent pipeline keys off the provider instead of `hf_url`: the fast-fail gate now explains *why* a model is skipped (no source / unknown source / source without a reachable card), the prompt context exposes source_url/source_id/source_label/asset_base_url while still filling the legacy hf_url/repo aliases, and the four README image extractors take a base_url (defaulting to HF) so relative paths resolve against the right site. Version grouping generalises to hf: / ms: / ta: keys. - `POST /api/lm/set-hf-url` keeps its path and its legacy payload keys but accepts `source_url`, validates against every provider and returns the platform. `GET /api/lm/model-sources` lets the UI render the supported-site list from the server. - Frontend: a `modelSourceHelpers` mirror of the registry drives the link dialog, the card/modal globe (branded "View on ModelScope/TensorArt"), the version-group key and the enrichment gate; the versions tab no longer sends ms:/ta: keys to the CivitAI API. TensorArt stays in the list because provenance is worth keeping even when the card is unreadable - the dialog says so plainly ("Sites that don't expose one (currently TensorArt) can only be linked") and the context menu disables enrichment with a matching tooltip, instead of the user getting "Unsupported URL". Verified against the real ModelScope API: jj3550945163/Krea-2-LORA returns a 1882-byte card whose frontmatter carries base_model/tags/trigger_words, and relative images resolve to .../resolve/master/.... Tests: backend 2815 passed; frontend 1130 JS + 91 Vue passed; pytest tests/i18n and a Jinja compile pass over templates/. The nine locales carry [TODO: Translate] for the new strings, completed in the next commit.
82 lines
3.6 KiB
Python
82 lines
3.6 KiB
Python
import os
|
|
import logging
|
|
from typing import Any, Dict, Optional
|
|
|
|
from .base_model_service import BaseModelService
|
|
from .auto_tag_service import extract_auto_tags
|
|
from ..utils.models import EmbeddingMetadata
|
|
from ..config import config
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class EmbeddingService(BaseModelService):
|
|
"""Embedding-specific service implementation"""
|
|
|
|
def __init__(self, scanner, update_service=None):
|
|
"""Initialize Embedding service
|
|
|
|
Args:
|
|
scanner: Embedding scanner instance
|
|
update_service: Optional service for remote update tracking.
|
|
"""
|
|
super().__init__("embedding", scanner, EmbeddingMetadata, update_service=update_service)
|
|
|
|
async def format_response(self, model_data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|
"""Format Embedding data for API response.
|
|
|
|
Returns None when the entry is missing critical fields (corrupted cache
|
|
row), so the handler layer can filter it out. See issue #730.
|
|
"""
|
|
# Guard against corrupted cache entries missing critical fields
|
|
file_path = model_data.get("file_path")
|
|
if not file_path or not isinstance(file_path, str):
|
|
logger.warning(
|
|
"Skipping corrupted embedding entry (missing file_path): %s",
|
|
model_data.get("file_name", "<unknown>"),
|
|
)
|
|
return None
|
|
|
|
# Get sub_type from cache entry (new canonical field)
|
|
sub_type = model_data.get("sub_type", "embedding")
|
|
|
|
file_name = model_data.get("file_name") or ""
|
|
model_name = model_data.get("model_name") or file_name
|
|
folder = model_data.get("folder") or ""
|
|
|
|
return {
|
|
"model_name": model_name,
|
|
"file_name": file_name,
|
|
"preview_url": config.get_preview_static_url(model_data.get("preview_url", "")),
|
|
"preview_nsfw_level": model_data.get("preview_nsfw_level", 0),
|
|
"base_model": model_data.get("base_model", ""),
|
|
"folder": folder,
|
|
"sha256": model_data.get("sha256", ""),
|
|
"autov3": model_data.get("autov3"),
|
|
"file_path": file_path.replace(os.sep, "/"),
|
|
"file_size": model_data.get("size", 0),
|
|
"modified": model_data.get("modified", ""),
|
|
"tags": model_data.get("tags", []),
|
|
"from_civitai": model_data.get("from_civitai", True),
|
|
# "usage_count": model_data.get("usage_count", 0), # TODO: Enable when embedding usage tracking is implemented
|
|
"notes": model_data.get("notes", ""),
|
|
"sub_type": sub_type,
|
|
"favorite": model_data.get("favorite", False),
|
|
"exclude": bool(model_data.get("exclude", False)),
|
|
"update_available": bool(model_data.get("update_available", False)),
|
|
"skip_metadata_refresh": bool(model_data.get("skip_metadata_refresh", False)),
|
|
"civitai": self.filter_civitai_data(model_data.get("civitai", {}), minimal=True),
|
|
"auto_tags": model_data.get("auto_tags") or extract_auto_tags(model_data),
|
|
"version_count": model_data.get("version_count"),
|
|
"source_platform": model_data.get("source_platform", ""),
|
|
"source_url": model_data.get("source_url", ""),
|
|
"hf_url": model_data.get("hf_url", ""),
|
|
}
|
|
|
|
def find_duplicate_hashes(self) -> Dict[str, Any]:
|
|
"""Find Embeddings with duplicate SHA256 hashes"""
|
|
return self.scanner._hash_index.get_duplicate_hashes()
|
|
|
|
def find_duplicate_filenames(self) -> Dict[str, Any]:
|
|
"""Find Embeddings with conflicting filenames"""
|
|
return self.scanner._hash_index.get_duplicate_filenames()
|