feat: add license information handling for Civitai models

Add license resolution utilities and integrate license information into model metadata processing. The changes include:

- Add `resolve_license_payload` function to extract license data from Civitai model responses
- Integrate license information into model metadata in CivitaiClient and MetadataSyncService
- Add license flags support in model scanning and caching
- Implement CommercialUseLevel enum for standardized license classification
- Update model scanner to handle unknown fields when extracting metadata values

This ensures proper license attribution and compliance when working with Civitai models.
This commit is contained in:
Will Miao
2025-11-06 17:05:31 +08:00
parent 4301b3455f
commit ddf9e33961
10 changed files with 364 additions and 10 deletions

View File

@@ -75,6 +75,10 @@ async def test_update_model_metadata_merges_and_persists():
"description": "desc",
"tags": ["style"],
"creator": {"id": 2},
"allowNoCredit": False,
"allowCommercialUse": ["Image"],
"allowDerivatives": False,
"allowDifferentLicense": True,
},
"baseModel": "sdxl",
"images": ["img"],
@@ -92,6 +96,13 @@ async def test_update_model_metadata_merges_and_persists():
assert result["modelDescription"] == "desc"
assert result["tags"] == ["style"]
assert result["base_model"] == "SDXL 1.0"
civitai_model = result["civitai"]["model"]
assert civitai_model["allowNoCredit"] is False
assert civitai_model["allowCommercialUse"] == ["Image"]
assert civitai_model["allowDerivatives"] is False
assert civitai_model["allowDifferentLicense"] is True
for key in ("allowNoCredit", "allowCommercialUse", "allowDerivatives", "allowDifferentLicense"):
assert key not in result
helpers.preview_service.ensure_preview_for_metadata.assert_awaited_once()
helpers.metadata_manager.save_metadata.assert_awaited_once_with(
@@ -142,6 +153,13 @@ async def test_fetch_and_update_model_success_updates_cache(tmp_path):
assert model_data["civitai_deleted"] is False
assert "civitai" in model_data
assert model_data["metadata_source"] == "civitai_api"
civitai_model = model_data["civitai"]["model"]
assert civitai_model["allowNoCredit"] is True
assert civitai_model["allowDerivatives"] is True
assert civitai_model["allowDifferentLicense"] is True
assert civitai_model["allowCommercialUse"] == ["Sell"]
for key in ("allowNoCredit", "allowCommercialUse", "allowDerivatives", "allowDifferentLicense"):
assert key not in model_data
helpers.metadata_manager.hydrate_model_data.assert_not_awaited()
assert model_data["hydrated"] is True
@@ -151,7 +169,13 @@ async def test_fetch_and_update_model_success_updates_cache(tmp_path):
assert await_args, "expected metadata to be persisted"
last_call = await_args[-1]
assert last_call.args[0] == metadata_path
assert last_call.args[1]["hydrated"] is True
persisted_payload = last_call.args[1]
assert persisted_payload["hydrated"] is True
civitai_model = persisted_payload["civitai"]["model"]
assert civitai_model["allowNoCredit"] is True
assert civitai_model["allowCommercialUse"] == ["Sell"]
for key in ("allowNoCredit", "allowCommercialUse", "allowDerivatives", "allowDifferentLicense"):
assert key not in persisted_payload
update_cache.assert_awaited_once()
@@ -422,4 +446,3 @@ async def test_relink_metadata_raises_when_version_missing():
model_id=9,
model_version_id=None,
)

View File

@@ -11,7 +11,8 @@ from py.services import model_scanner
from py.services.model_cache import ModelCache
from py.services.model_hash_index import ModelHashIndex
from py.services.model_scanner import CacheBuildResult, ModelScanner
from py.services.persistent_model_cache import PersistentModelCache
from py.services.persistent_model_cache import PersistentModelCache, DEFAULT_LICENSE_FLAGS
from py.utils.civitai_utils import build_license_flags
from py.utils.models import BaseModelMetadata
@@ -122,6 +123,7 @@ async def test_initialize_cache_populates_cache(tmp_path: Path):
_normalize_path(tmp_path / "one.txt"),
_normalize_path(tmp_path / "nested" / "two.txt"),
}
assert {item["license_flags"] for item in cache.raw_data} == {DEFAULT_LICENSE_FLAGS}
assert scanner._hash_index.get_path("hash-one") == _normalize_path(tmp_path / "one.txt")
assert scanner._hash_index.get_path("hash-two") == _normalize_path(tmp_path / "nested" / "two.txt")
@@ -179,12 +181,49 @@ async def test_initialize_in_background_applies_scan_result(tmp_path: Path, monk
_normalize_path(tmp_path / "one.txt"),
_normalize_path(tmp_path / "nested" / "two.txt"),
}
assert {item["license_flags"] for item in cache.raw_data} == {DEFAULT_LICENSE_FLAGS}
assert scanner._hash_index.get_path("hash-two") == _normalize_path(tmp_path / "nested" / "two.txt")
assert scanner._tags_count == {"alpha": 1, "beta": 1}
assert scanner._excluded_models == [_normalize_path(tmp_path / "skip-file.txt")]
assert ws_stub.payloads[-1]["progress"] == 100
@pytest.mark.asyncio
async def test_build_cache_entry_encodes_license_flags(tmp_path: Path):
scanner = DummyScanner(tmp_path)
metadata = {
"file_path": _normalize_path(tmp_path / "sample.txt"),
"file_name": "sample",
"model_name": "Sample",
"folder": "",
"size": 1,
"modified": 1.0,
"sha256": "hash",
"tags": [],
"civitai": {
"model": {
"allowNoCredit": False,
"allowCommercialUse": ["Image", "Rent"],
"allowDerivatives": True,
"allowDifferentLicense": False,
}
},
}
expected_flags = build_license_flags(
{
"allowNoCredit": False,
"allowCommercialUse": ["Image", "Rent"],
"allowDerivatives": True,
"allowDifferentLicense": False,
}
)
entry = scanner._build_cache_entry(metadata)
assert entry["license_flags"] == expected_flags
@pytest.mark.asyncio
async def test_initialize_in_background_uses_persisted_cache_without_full_scan(tmp_path: Path, monkeypatch):
monkeypatch.setenv('LORA_MANAGER_DISABLE_PERSISTENT_CACHE', '0')

View File

@@ -2,7 +2,7 @@ from pathlib import Path
import pytest
from py.services.persistent_model_cache import PersistentModelCache
from py.services.persistent_model_cache import PersistentModelCache, DEFAULT_LICENSE_FLAGS
def test_persistent_cache_roundtrip(tmp_path: Path, monkeypatch) -> None:
@@ -43,6 +43,7 @@ def test_persistent_cache_roundtrip(tmp_path: Path, monkeypatch) -> None:
'trainedWords': ['word1'],
'creator': {'username': 'artist42'},
},
'license_flags': 13,
},
{
'file_path': file_b,
@@ -91,12 +92,14 @@ def test_persistent_cache_roundtrip(tmp_path: Path, monkeypatch) -> None:
assert first['metadata_source'] == 'civitai_api'
assert first['civitai']['creator']['username'] == 'artist42'
assert first['civitai_deleted'] is False
assert first['license_flags'] == 13
second = items[file_b]
assert second['exclude'] is True
assert second['civitai'] is None
assert second['metadata_source'] is None
assert second['civitai_deleted'] is True
assert second['license_flags'] == DEFAULT_LICENSE_FLAGS
expected_hash_pairs = {
('hash-a', file_a),

View File

@@ -0,0 +1,35 @@
from py.utils.civitai_utils import (
CommercialUseLevel,
build_license_flags,
resolve_license_info,
resolve_license_payload,
)
def test_resolve_license_payload_defaults():
payload, flags = resolve_license_info({})
assert payload["allowNoCredit"] is True
assert payload["allowDerivatives"] is True
assert payload["allowDifferentLicense"] is True
assert payload["allowCommercialUse"] == ["Sell"]
assert flags == 57
def test_build_license_flags_custom_values():
source = {
"allowNoCredit": False,
"allowCommercialUse": {"Image", "Sell"},
"allowDerivatives": False,
"allowDifferentLicense": False,
}
payload = resolve_license_payload(source)
assert payload["allowNoCredit"] is False
assert set(payload["allowCommercialUse"]) == {"Image", "Sell"}
assert payload["allowDerivatives"] is False
assert payload["allowDifferentLicense"] is False
flags = build_license_flags(source)
# Highest commercial level is SELL -> level 4 -> shifted by 1 == 8.
assert flags == (CommercialUseLevel.SELL << 1)