feat(metadata): add CivArchive API toggle and provider fallback order settings

- Add enable_civarchive_api toggle (default on) to allow disabling
  CivArchive to avoid its rate-limit windows entirely
- Add metadata_provider_order dropdown with two presets:
  CivitAI → CivArchive → Archive DB (default) and
  CivitAI → Archive DB → CivArchive
- Wire both settings through backend (metadata_service, settings_manager,
  misc_handlers) and frontend (SettingsManager, state, settings modal)
- Reorder Metadata section in settings modal: toggles → status/management
  → fallback order, for natural top-down workflow
- Make update_metadata_providers() log the effective provider chain
  using actually-registered providers rather than settings assumptions
- Add 5 test cases covering all provider-combination paths
- Complete i18n translations for 6 new keys across all 9 non-English locales
This commit is contained in:
Will Miao
2026-07-19 17:51:50 +08:00
parent c27e4d1bfc
commit a8d742b031
17 changed files with 20055 additions and 19776 deletions
+2201 -2195
View File
File diff suppressed because it is too large Load Diff
+7 -1
View File
@@ -640,7 +640,13 @@
"preparing": "Preparing download...",
"connecting": "Connecting to download server...",
"completed": "Completed",
"downloadComplete": "Download completed successfully"
"downloadComplete": "Download completed successfully",
"enableCivarchiveApi": "Enable CivArchive API as metadata provider",
"enableCivarchiveApiHelp": "When on, CivArchive API is used as a fallback source for model metadata (e.g. for models deleted from CivitAI). Turn off to avoid CivArchive rate limits entirely.",
"providerOrder": "Metadata provider fallback order",
"providerOrderHelp": "CivitAI API is always tried first. Choose the order of the remaining providers when looking up metadata.",
"providerOrderCivitaiArchiveSqlite": "CivitAI → CivArchive → Archive DB",
"providerOrderCivitaiSqliteArchive": "CivitAI → Archive DB → CivArchive"
},
"proxySettings": {
"enableProxy": "Enable App-level Proxy",
+2201 -2195
View File
File diff suppressed because it is too large Load Diff
+2201 -2195
View File
File diff suppressed because it is too large Load Diff
+2201 -2195
View File
File diff suppressed because it is too large Load Diff
+2201 -2195
View File
File diff suppressed because it is too large Load Diff
+2201 -2195
View File
File diff suppressed because it is too large Load Diff
+2201 -2195
View File
File diff suppressed because it is too large Load Diff
+2201 -2195
View File
File diff suppressed because it is too large Load Diff
+2201 -2195
View File
File diff suppressed because it is too large Load Diff
+5 -1
View File
@@ -1570,7 +1570,11 @@ class SettingsHandler:
else:
self._settings.set(key, value)
if key == "enable_metadata_archive_db":
if key in (
"enable_metadata_archive_db",
"enable_civarchive_api",
"metadata_provider_order",
):
await self._metadata_provider_updater()
if key in self._PROXY_KEYS:
+69 -16
View File
@@ -15,6 +15,17 @@ from .service_registry import ServiceRegistry
logger = logging.getLogger(__name__)
_PROVIDER_DISPLAY_NAMES = {
"civitai_api": "CivitAI",
"civarchive_api": "CivArchive",
"sqlite": "Archive DB",
}
_PRESET_PROVIDER_ORDERS = {
"civitai_archive_sqlite": ["civitai_api", "civarchive_api", "sqlite"],
"civitai_sqlite_archive": ["civitai_api", "sqlite", "civarchive_api"],
}
async def initialize_metadata_providers():
"""Initialize and configure all metadata providers based on settings"""
provider_manager = await ModelMetadataProviderManager.get_instance()
@@ -26,7 +37,9 @@ async def initialize_metadata_providers():
# Get settings
settings_manager = get_settings_manager()
enable_archive_db = settings_manager.get('enable_metadata_archive_db', False)
enable_civarchive_api = settings_manager.get('enable_civarchive_api', True)
provider_order = settings_manager.get('metadata_provider_order', 'civitai_archive_sqlite')
providers = []
# Initialize archive database provider if enabled
@@ -59,27 +72,48 @@ async def initialize_metadata_providers():
except Exception as e:
logger.error(f"Failed to initialize Civitai API metadata provider: {e}")
# Register CivArchive provider, and all add to fallback providers
try:
civarchive_client = await ServiceRegistry.get_civarchive_client()
civarchive_provider = CivArchiveModelMetadataProvider(civarchive_client)
provider_manager.register_provider('civarchive_api', civarchive_provider)
providers.append(('civarchive_api', civarchive_provider))
logger.debug("CivArchive metadata provider registered (also included in fallback)")
except Exception as e:
logger.error(f"Failed to initialize CivArchive metadata provider: {e}")
# Register CivArchive provider when enabled. Civitai API is always
# preferred (better metadata); CivArchive mainly recovers metadata for
# models deleted from Civitai, so it can be turned off to avoid its long
# rate-limit windows entirely.
if enable_civarchive_api:
try:
civarchive_client = await ServiceRegistry.get_civarchive_client()
civarchive_provider = CivArchiveModelMetadataProvider(civarchive_client)
provider_manager.register_provider('civarchive_api', civarchive_provider)
providers.append(('civarchive_api', civarchive_provider))
logger.debug("CivArchive metadata provider registered (also included in fallback)")
except Exception as e:
logger.error(f"Failed to initialize CivArchive metadata provider: {e}")
else:
logger.debug("CivArchive metadata provider disabled by setting 'enable_civarchive_api'")
# Preset fallback orderings (see module-level _PRESET_PROVIDER_ORDERS).
# civitai_api is always first (better metadata); the remaining providers
# are arranged by the configured preset. Providers that are not
# registered (disabled/unavailable) are simply skipped, so each preset
# degrades gracefully.
desired_order = _PRESET_PROVIDER_ORDERS.get(
provider_order, _PRESET_PROVIDER_ORDERS["civitai_archive_sqlite"]
)
# Set up fallback provider based on available providers
if len(providers) > 1:
# Always use Civitai API (it has better metadata), then CivArchive API, then Archive DB
ordered_providers: list[tuple[str, ModelMetadataProvider]] = []
ordered_providers.extend([p for p in providers if p[0] == 'civitai_api'])
ordered_providers.extend([p for p in providers if p[0] == 'civarchive_api'])
ordered_providers.extend([p for p in providers if p[0] == 'sqlite'])
for name in desired_order:
ordered_providers.extend([p for p in providers if p[0] == name])
# Include any provider not covered by the preset (defensive) at the end
for p in providers:
if p not in ordered_providers:
ordered_providers.append(p)
if ordered_providers:
fallback_provider = FallbackMetadataProvider(ordered_providers)
provider_manager.register_provider('fallback', fallback_provider, is_default=True)
logger.debug(
"Metadata fallback provider order: %s",
", ".join(name for name, _ in ordered_providers),
)
elif len(providers) == 1:
# Only one provider available, set it as default
provider_name, provider = providers[0]
@@ -96,11 +130,30 @@ async def update_metadata_providers():
# Get current settings
settings_manager = get_settings_manager()
enable_archive_db = settings_manager.get('enable_metadata_archive_db', False)
enable_civarchive_api = settings_manager.get('enable_civarchive_api', True)
provider_order = settings_manager.get('metadata_provider_order', 'civitai_archive_sqlite')
# Reinitialize all providers with new settings
provider_manager = await initialize_metadata_providers()
logger.info(f"Updated metadata providers, archive_db enabled: {enable_archive_db}")
# Build effective provider chain for logging (use actually-registered
# providers, not just settings, so a failed init is reflected correctly)
registered = set(provider_manager.providers.keys())
desired = _PRESET_PROVIDER_ORDERS.get(
provider_order, _PRESET_PROVIDER_ORDERS["civitai_archive_sqlite"]
)
chain = "".join(
_PROVIDER_DISPLAY_NAMES[p]
for p in desired
if p in registered and p in _PROVIDER_DISPLAY_NAMES
)
logger.info(
"Updated metadata providers: archive_db=%s, civarchive_api=%s, chain=%s",
enable_archive_db,
enable_civarchive_api,
chain,
)
return provider_manager
except Exception as e:
logger.error(f"Failed to update metadata providers: {e}")
+2
View File
@@ -65,6 +65,8 @@ DEFAULT_SETTINGS: Dict[str, Any] = {
"onboarding_completed": False,
"dismissed_banners": [],
"enable_metadata_archive_db": False,
"enable_civarchive_api": True,
"metadata_provider_order": "civitai_archive_sqlite",
"proxy_enabled": False,
"proxy_host": "",
"proxy_port": "",
+10
View File
@@ -2346,6 +2346,16 @@ export class SettingsManager {
enableMetadataArchiveCheckbox.checked = state.global.settings.enable_metadata_archive_db || false;
}
const enableCivarchiveApiCheckbox = document.getElementById('enableCivarchiveApi');
if (enableCivarchiveApiCheckbox) {
enableCivarchiveApiCheckbox.checked = state.global.settings.enable_civarchive_api ?? true;
}
const metadataProviderOrderSelect = document.getElementById('metadataProviderOrder');
if (metadataProviderOrderSelect) {
metadataProviderOrderSelect.value = state.global.settings.metadata_provider_order || 'civitai_archive_sqlite';
}
// Load status
await this.updateMetadataArchiveStatus();
} catch (error) {
+2
View File
@@ -13,6 +13,8 @@ const DEFAULT_SETTINGS_BASE = Object.freeze({
language: 'en',
show_only_sfw: false,
enable_metadata_archive_db: false,
enable_civarchive_api: true,
metadata_provider_order: 'civitai_archive_sqlite',
proxy_enabled: false,
proxy_type: 'http',
proxy_host: '',
@@ -1401,7 +1401,26 @@
<div class="settings-input-error-message" id="metadataRefreshSkipPathsError"></div>
</div>
<!-- Metadata Archive -->
<!-- CivArchive API provider toggle -->
<div class="setting-item">
<div class="setting-row">
<div class="setting-info">
<label for="enableCivarchiveApi">
{{ t('settings.metadataArchive.enableCivarchiveApi') }}
<i class="fas fa-info-circle info-icon" data-tooltip="{{ t('settings.metadataArchive.enableCivarchiveApiHelp') }}"></i>
</label>
</div>
<div class="setting-control">
<label class="toggle-switch">
<input type="checkbox" id="enableCivarchiveApi"
onchange="settingsManager.saveToggleSetting('enableCivarchiveApi', 'enable_civarchive_api')">
<span class="toggle-slider"></span>
</label>
</div>
</div>
</div>
<!-- Metadata Archive DB -->
<div class="setting-item">
<div class="setting-row">
<div class="setting-info">
@@ -1419,13 +1438,13 @@
</div>
</div>
</div>
<div class="setting-item">
<div class="metadata-archive-status" id="metadataArchiveStatus">
<!-- Status will be populated by JavaScript -->
</div>
</div>
<div class="setting-item">
<div class="setting-row">
<div class="setting-info">
@@ -1444,6 +1463,24 @@
</div>
</div>
</div>
<!-- Metadata provider fallback order -->
<div class="setting-item">
<div class="setting-row">
<div class="setting-info">
<label for="metadataProviderOrder">
{{ t('settings.metadataArchive.providerOrder') }}
<i class="fas fa-info-circle info-icon" data-tooltip="{{ t('settings.metadataArchive.providerOrderHelp') }}"></i>
</label>
</div>
<div class="setting-control select-control">
<select id="metadataProviderOrder" onchange="settingsManager.saveSelectSetting('metadataProviderOrder', 'metadata_provider_order')">
<option value="civitai_archive_sqlite">{{ t('settings.metadataArchive.providerOrderCivitaiArchiveSqlite') }}</option>
<option value="civitai_sqlite_archive">{{ t('settings.metadataArchive.providerOrderCivitaiSqliteArchive') }}</option>
</select>
</div>
</div>
</div>
</div>
</div>
</div>
+111
View File
@@ -60,3 +60,114 @@ async def test_get_metadata_provider_returns_fallback_as_is(monkeypatch):
provider = await metadata_service.get_metadata_provider()
assert provider is fallback
# ---------------------------------------------------------------------------
# initialize_metadata_providers — provider gating + fallback ordering
# ---------------------------------------------------------------------------
def _stub_settings(**overrides):
"""Minimal settings stub returning configured values."""
base = {
"enable_metadata_archive_db": False,
"enable_civarchive_api": True,
"metadata_provider_order": "civitai_archive_sqlite",
}
base.update(overrides)
return SimpleNamespace(get=lambda key, default=None: base.get(key, default))
async def _run_initialize(monkeypatch, settings):
# Fresh provider manager for each test
monkeypatch.setattr(
metadata_service.ModelMetadataProviderManager,
"get_instance",
AsyncMock(return_value=metadata_service.ModelMetadataProviderManager()),
)
monkeypatch.setattr(
metadata_service, "get_settings_manager", lambda: settings
)
monkeypatch.setattr(
metadata_service.ServiceRegistry,
"get_civitai_client",
AsyncMock(return_value=object()),
)
monkeypatch.setattr(
metadata_service.ServiceRegistry,
"get_civarchive_client",
AsyncMock(return_value=object()),
)
# Make MetadataArchiveManager report a usable db path when enabled
fake_archive = SimpleNamespace(get_database_path=lambda: "/tmp/fake.db")
monkeypatch.setattr(
metadata_service, "MetadataArchiveManager", lambda _base: fake_archive
)
# Pretend the db file exists
monkeypatch.setattr(metadata_service.os.path, "exists", lambda _p: True)
manager = await metadata_service.initialize_metadata_providers()
return manager
def _fallback_provider_order(manager):
"""Return the ordered list of provider labels inside the fallback provider."""
fallback = manager.providers.get("fallback")
assert isinstance(fallback, FallbackMetadataProvider), "expected a fallback provider"
return list(fallback._provider_labels)
@pytest.mark.asyncio
async def test_initialize_providers_default_order(monkeypatch):
settings = _stub_settings(enable_metadata_archive_db=True)
manager = await _run_initialize(monkeypatch, settings)
assert _fallback_provider_order(manager) == ["civitai_api", "civarchive_api", "sqlite"]
@pytest.mark.asyncio
async def test_initialize_providers_prefer_sqlite_order(monkeypatch):
settings = _stub_settings(
enable_metadata_archive_db=True,
metadata_provider_order="civitai_sqlite_archive",
)
manager = await _run_initialize(monkeypatch, settings)
assert _fallback_provider_order(manager) == ["civitai_api", "sqlite", "civarchive_api"]
@pytest.mark.asyncio
async def test_initialize_providers_disables_civarchive(monkeypatch):
settings = _stub_settings(
enable_metadata_archive_db=True,
enable_civarchive_api=False,
)
manager = await _run_initialize(monkeypatch, settings)
# civarchive_api must not be registered at all
assert "civarchive_api" not in manager.providers
assert _fallback_provider_order(manager) == ["civitai_api", "sqlite"]
@pytest.mark.asyncio
async def test_initialize_providers_skips_unavailable_sqlite_in_preset(monkeypatch):
# Preset wants sqlite before civarchive, but archive db is disabled ->
# sqlite is unavailable and must be skipped, civarchive stays.
settings = _stub_settings(
enable_metadata_archive_db=False,
metadata_provider_order="civitai_sqlite_archive",
)
manager = await _run_initialize(monkeypatch, settings)
assert _fallback_provider_order(manager) == ["civitai_api", "civarchive_api"]
@pytest.mark.asyncio
async def test_initialize_providers_single_provider_when_only_civitai(monkeypatch):
# Both archive db and civarchive disabled -> only civitai_api remains,
# which takes the single-provider path (registered as default, no fallback).
settings = _stub_settings(
enable_metadata_archive_db=False,
enable_civarchive_api=False,
)
manager = await _run_initialize(monkeypatch, settings)
assert "fallback" not in manager.providers
assert manager.default_provider == "civitai_api"