feat(metadata): add CivArchive API toggle and provider fallback order settings

- Add enable_civarchive_api toggle (default on) to allow disabling
  CivArchive to avoid its rate-limit windows entirely
- Add metadata_provider_order dropdown with two presets:
  CivitAI → CivArchive → Archive DB (default) and
  CivitAI → Archive DB → CivArchive
- Wire both settings through backend (metadata_service, settings_manager,
  misc_handlers) and frontend (SettingsManager, state, settings modal)
- Reorder Metadata section in settings modal: toggles → status/management
  → fallback order, for natural top-down workflow
- Make update_metadata_providers() log the effective provider chain
  using actually-registered providers rather than settings assumptions
- Add 5 test cases covering all provider-combination paths
- Complete i18n translations for 6 new keys across all 9 non-English locales
This commit is contained in:
Will Miao
2026-07-19 17:51:50 +08:00
parent c27e4d1bfc
commit a8d742b031
17 changed files with 20055 additions and 19776 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -640,7 +640,13 @@
"preparing": "Preparing download...",
"connecting": "Connecting to download server...",
"completed": "Completed",
"downloadComplete": "Download completed successfully"
"downloadComplete": "Download completed successfully",
"enableCivarchiveApi": "Enable CivArchive API as metadata provider",
"enableCivarchiveApiHelp": "When on, CivArchive API is used as a fallback source for model metadata (e.g. for models deleted from CivitAI). Turn off to avoid CivArchive rate limits entirely.",
"providerOrder": "Metadata provider fallback order",
"providerOrderHelp": "CivitAI API is always tried first. Choose the order of the remaining providers when looking up metadata.",
"providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB",
"providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive"
},
"proxySettings": {
"enableProxy": "Enable App-level Proxy",

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1570,7 +1570,11 @@ class SettingsHandler:
else:
self._settings.set(key, value)
if key == "enable_metadata_archive_db":
if key in (
"enable_metadata_archive_db",
"enable_civarchive_api",
"metadata_provider_order",
):
await self._metadata_provider_updater()
if key in self._PROXY_KEYS:

View File

@@ -15,6 +15,17 @@ from .service_registry import ServiceRegistry
logger = logging.getLogger(__name__)
_PROVIDER_DISPLAY_NAMES = {
"civitai_api": "CivitAI",
"civarchive_api": "CivArchive",
"sqlite": "Archive DB",
}
_PRESET_PROVIDER_ORDERS = {
"civitai_archive_sqlite": ["civitai_api", "civarchive_api", "sqlite"],
"civitai_sqlite_archive": ["civitai_api", "sqlite", "civarchive_api"],
}
async def initialize_metadata_providers():
"""Initialize and configure all metadata providers based on settings"""
provider_manager = await ModelMetadataProviderManager.get_instance()
@@ -26,7 +37,9 @@ async def initialize_metadata_providers():
# Get settings
settings_manager = get_settings_manager()
enable_archive_db = settings_manager.get('enable_metadata_archive_db', False)
enable_civarchive_api = settings_manager.get('enable_civarchive_api', True)
provider_order = settings_manager.get('metadata_provider_order', 'civitai_archive_sqlite')
providers = []
# Initialize archive database provider if enabled
@@ -59,27 +72,48 @@ async def initialize_metadata_providers():
except Exception as e:
logger.error(f"Failed to initialize Civitai API metadata provider: {e}")
# Register CivArchive provider, and all add to fallback providers
try:
civarchive_client = await ServiceRegistry.get_civarchive_client()
civarchive_provider = CivArchiveModelMetadataProvider(civarchive_client)
provider_manager.register_provider('civarchive_api', civarchive_provider)
providers.append(('civarchive_api', civarchive_provider))
logger.debug("CivArchive metadata provider registered (also included in fallback)")
except Exception as e:
logger.error(f"Failed to initialize CivArchive metadata provider: {e}")
# Register CivArchive provider when enabled. Civitai API is always
# preferred (better metadata); CivArchive mainly recovers metadata for
# models deleted from Civitai, so it can be turned off to avoid its long
# rate-limit windows entirely.
if enable_civarchive_api:
try:
civarchive_client = await ServiceRegistry.get_civarchive_client()
civarchive_provider = CivArchiveModelMetadataProvider(civarchive_client)
provider_manager.register_provider('civarchive_api', civarchive_provider)
providers.append(('civarchive_api', civarchive_provider))
logger.debug("CivArchive metadata provider registered (also included in fallback)")
except Exception as e:
logger.error(f"Failed to initialize CivArchive metadata provider: {e}")
else:
logger.debug("CivArchive metadata provider disabled by setting 'enable_civarchive_api'")
# Preset fallback orderings (see module-level _PRESET_PROVIDER_ORDERS).
# civitai_api is always first (better metadata); the remaining providers
# are arranged by the configured preset. Providers that are not
# registered (disabled/unavailable) are simply skipped, so each preset
# degrades gracefully.
desired_order = _PRESET_PROVIDER_ORDERS.get(
provider_order, _PRESET_PROVIDER_ORDERS["civitai_archive_sqlite"]
)
# Set up fallback provider based on available providers
if len(providers) > 1:
# Always use Civitai API (it has better metadata), then CivArchive API, then Archive DB
ordered_providers: list[tuple[str, ModelMetadataProvider]] = []
ordered_providers.extend([p for p in providers if p[0] == 'civitai_api'])
ordered_providers.extend([p for p in providers if p[0] == 'civarchive_api'])
ordered_providers.extend([p for p in providers if p[0] == 'sqlite'])
for name in desired_order:
ordered_providers.extend([p for p in providers if p[0] == name])
# Include any provider not covered by the preset (defensive) at the end
for p in providers:
if p not in ordered_providers:
ordered_providers.append(p)
if ordered_providers:
fallback_provider = FallbackMetadataProvider(ordered_providers)
provider_manager.register_provider('fallback', fallback_provider, is_default=True)
logger.debug(
"Metadata fallback provider order: %s",
", ".join(name for name, _ in ordered_providers),
)
elif len(providers) == 1:
# Only one provider available, set it as default
provider_name, provider = providers[0]
@@ -96,11 +130,30 @@ async def update_metadata_providers():
# Get current settings
settings_manager = get_settings_manager()
enable_archive_db = settings_manager.get('enable_metadata_archive_db', False)
enable_civarchive_api = settings_manager.get('enable_civarchive_api', True)
provider_order = settings_manager.get('metadata_provider_order', 'civitai_archive_sqlite')
# Reinitialize all providers with new settings
provider_manager = await initialize_metadata_providers()
logger.info(f"Updated metadata providers, archive_db enabled: {enable_archive_db}")
# Build effective provider chain for logging (use actually-registered
# providers, not just settings, so a failed init is reflected correctly)
registered = set(provider_manager.providers.keys())
desired = _PRESET_PROVIDER_ORDERS.get(
provider_order, _PRESET_PROVIDER_ORDERS["civitai_archive_sqlite"]
)
chain = "".join(
_PROVIDER_DISPLAY_NAMES[p]
for p in desired
if p in registered and p in _PROVIDER_DISPLAY_NAMES
)
logger.info(
"Updated metadata providers: archive_db=%s, civarchive_api=%s, chain=%s",
enable_archive_db,
enable_civarchive_api,
chain,
)
return provider_manager
except Exception as e:
logger.error(f"Failed to update metadata providers: {e}")

View File

@@ -65,6 +65,8 @@ DEFAULT_SETTINGS: Dict[str, Any] = {
"onboarding_completed": False,
"dismissed_banners": [],
"enable_metadata_archive_db": False,
"enable_civarchive_api": True,
"metadata_provider_order": "civitai_archive_sqlite",
"proxy_enabled": False,
"proxy_host": "",
"proxy_port": "",

View File

@@ -2346,6 +2346,16 @@ export class SettingsManager {
enableMetadataArchiveCheckbox.checked = state.global.settings.enable_metadata_archive_db || false;
}
const enableCivarchiveApiCheckbox = document.getElementById('enableCivarchiveApi');
if (enableCivarchiveApiCheckbox) {
enableCivarchiveApiCheckbox.checked = state.global.settings.enable_civarchive_api ?? true;
}
const metadataProviderOrderSelect = document.getElementById('metadataProviderOrder');
if (metadataProviderOrderSelect) {
metadataProviderOrderSelect.value = state.global.settings.metadata_provider_order || 'civitai_archive_sqlite';
}
// Load status
await this.updateMetadataArchiveStatus();
} catch (error) {

View File

@@ -13,6 +13,8 @@ const DEFAULT_SETTINGS_BASE = Object.freeze({
language: 'en',
show_only_sfw: false,
enable_metadata_archive_db: false,
enable_civarchive_api: true,
metadata_provider_order: 'civitai_archive_sqlite',
proxy_enabled: false,
proxy_type: 'http',
proxy_host: '',

View File

@@ -1401,7 +1401,26 @@
<div class="settings-input-error-message" id="metadataRefreshSkipPathsError"></div>
</div>
<!-- Metadata Archive -->
<!-- CivArchive API provider toggle -->
<div class="setting-item">
<div class="setting-row">
<div class="setting-info">
<label for="enableCivarchiveApi">
{{ t('settings.metadataArchive.enableCivarchiveApi') }}
<i class="fas fa-info-circle info-icon" data-tooltip="{{ t('settings.metadataArchive.enableCivarchiveApiHelp') }}"></i>
</label>
</div>
<div class="setting-control">
<label class="toggle-switch">
<input type="checkbox" id="enableCivarchiveApi"
onchange="settingsManager.saveToggleSetting('enableCivarchiveApi', 'enable_civarchive_api')">
<span class="toggle-slider"></span>
</label>
</div>
</div>
</div>
<!-- Metadata Archive DB -->
<div class="setting-item">
<div class="setting-row">
<div class="setting-info">
@@ -1419,13 +1438,13 @@
</div>
</div>
</div>
<div class="setting-item">
<div class="metadata-archive-status" id="metadataArchiveStatus">
<!-- Status will be populated by JavaScript -->
</div>
</div>
<div class="setting-item">
<div class="setting-row">
<div class="setting-info">
@@ -1444,6 +1463,24 @@
</div>
</div>
</div>
<!-- Metadata provider fallback order -->
<div class="setting-item">
<div class="setting-row">
<div class="setting-info">
<label for="metadataProviderOrder">
{{ t('settings.metadataArchive.providerOrder') }}
<i class="fas fa-info-circle info-icon" data-tooltip="{{ t('settings.metadataArchive.providerOrderHelp') }}"></i>
</label>
</div>
<div class="setting-control select-control">
<select id="metadataProviderOrder" onchange="settingsManager.saveSelectSetting('metadataProviderOrder', 'metadata_provider_order')">
<option value="civitai_archive_sqlite">{{ t('settings.metadataArchive.providerOrderCivitaiArchiveSqlite') }}</option>
<option value="civitai_sqlite_archive">{{ t('settings.metadataArchive.providerOrderCivitaiSqliteArchive') }}</option>
</select>
</div>
</div>
</div>
</div>
</div>
</div>

View File

@@ -60,3 +60,114 @@ async def test_get_metadata_provider_returns_fallback_as_is(monkeypatch):
provider = await metadata_service.get_metadata_provider()
assert provider is fallback
# ---------------------------------------------------------------------------
# initialize_metadata_providers — provider gating + fallback ordering
# ---------------------------------------------------------------------------
def _stub_settings(**overrides):
"""Minimal settings stub returning configured values."""
base = {
"enable_metadata_archive_db": False,
"enable_civarchive_api": True,
"metadata_provider_order": "civitai_archive_sqlite",
}
base.update(overrides)
return SimpleNamespace(get=lambda key, default=None: base.get(key, default))
async def _run_initialize(monkeypatch, settings):
# Fresh provider manager for each test
monkeypatch.setattr(
metadata_service.ModelMetadataProviderManager,
"get_instance",
AsyncMock(return_value=metadata_service.ModelMetadataProviderManager()),
)
monkeypatch.setattr(
metadata_service, "get_settings_manager", lambda: settings
)
monkeypatch.setattr(
metadata_service.ServiceRegistry,
"get_civitai_client",
AsyncMock(return_value=object()),
)
monkeypatch.setattr(
metadata_service.ServiceRegistry,
"get_civarchive_client",
AsyncMock(return_value=object()),
)
# Make MetadataArchiveManager report a usable db path when enabled
fake_archive = SimpleNamespace(get_database_path=lambda: "/tmp/fake.db")
monkeypatch.setattr(
metadata_service, "MetadataArchiveManager", lambda _base: fake_archive
)
# Pretend the db file exists
monkeypatch.setattr(metadata_service.os.path, "exists", lambda _p: True)
manager = await metadata_service.initialize_metadata_providers()
return manager
def _fallback_provider_order(manager):
"""Return the ordered list of provider labels inside the fallback provider."""
fallback = manager.providers.get("fallback")
assert isinstance(fallback, FallbackMetadataProvider), "expected a fallback provider"
return list(fallback._provider_labels)
@pytest.mark.asyncio
async def test_initialize_providers_default_order(monkeypatch):
settings = _stub_settings(enable_metadata_archive_db=True)
manager = await _run_initialize(monkeypatch, settings)
assert _fallback_provider_order(manager) == ["civitai_api", "civarchive_api", "sqlite"]
@pytest.mark.asyncio
async def test_initialize_providers_prefer_sqlite_order(monkeypatch):
settings = _stub_settings(
enable_metadata_archive_db=True,
metadata_provider_order="civitai_sqlite_archive",
)
manager = await _run_initialize(monkeypatch, settings)
assert _fallback_provider_order(manager) == ["civitai_api", "sqlite", "civarchive_api"]
@pytest.mark.asyncio
async def test_initialize_providers_disables_civarchive(monkeypatch):
settings = _stub_settings(
enable_metadata_archive_db=True,
enable_civarchive_api=False,
)
manager = await _run_initialize(monkeypatch, settings)
# civarchive_api must not be registered at all
assert "civarchive_api" not in manager.providers
assert _fallback_provider_order(manager) == ["civitai_api", "sqlite"]
@pytest.mark.asyncio
async def test_initialize_providers_skips_unavailable_sqlite_in_preset(monkeypatch):
# Preset wants sqlite before civarchive, but archive db is disabled ->
# sqlite is unavailable and must be skipped, civarchive stays.
settings = _stub_settings(
enable_metadata_archive_db=False,
metadata_provider_order="civitai_sqlite_archive",
)
manager = await _run_initialize(monkeypatch, settings)
assert _fallback_provider_order(manager) == ["civitai_api", "civarchive_api"]
@pytest.mark.asyncio
async def test_initialize_providers_single_provider_when_only_civitai(monkeypatch):
# Both archive db and civarchive disabled -> only civitai_api remains,
# which takes the single-provider path (registered as default, no fallback).
settings = _stub_settings(
enable_metadata_archive_db=False,
enable_civarchive_api=False,
)
manager = await _run_initialize(monkeypatch, settings)
assert "fallback" not in manager.providers
assert manager.default_provider == "civitai_api"