fix(network): add offline cooldown guard for remote metadata requests

tests/services/test_civitai_client.py

@@ -5,6 +5,7 @@ import pytest
 
 from py.services import civitai_client as civitai_client_module
 from py.services.civitai_client import CivitaiClient
+from py.services.connectivity_guard import OFFLINE_COOLDOWN_ERROR, OFFLINE_FRIENDLY_MESSAGE
 from py.services.errors import RateLimitError, ResourceNotFoundError
 from py.services.model_metadata_provider import ModelMetadataProviderManager
 
@@ -115,6 +116,20 @@ async def test_get_model_by_hash_handles_not_found(monkeypatch, downloader):
     assert error == "Model not found"
 
 
+async def test_get_model_by_hash_handles_offline_cooldown(downloader):
+    async def fake_make_request(method, url, use_auth=True, **kwargs):
+        return False, OFFLINE_COOLDOWN_ERROR
+
+    downloader.make_request = fake_make_request
+
+    client = await CivitaiClient.get_instance()
+
+    result, error = await client.get_model_by_hash("missing")
+
+    assert result is None
+    assert error == OFFLINE_FRIENDLY_MESSAGE
+
+
 async def test_get_model_by_hash_propagates_rate_limit(monkeypatch, downloader):
     async def fake_make_request(method, url, use_auth=True, **kwargs):
         return False, RateLimitError("limited", retry_after=4)

tests/services/test_connectivity_guard.py

@@ -0,0 +1,74 @@
+import asyncio
+import errno
+from datetime import datetime, timedelta
+
+import pytest
+
+from py.services.connectivity_guard import (
+    OFFLINE_COOLDOWN_ERROR,
+    ConnectivityGuard,
+)
+from py.services.downloader import Downloader
+
+
+@pytest.fixture(autouse=True)
+def reset_connectivity_guard_singleton():
+    ConnectivityGuard._instance = None
+    yield
+    ConnectivityGuard._instance = None
+
+
+async def test_connectivity_guard_enters_cooldown_after_threshold():
+    guard = await ConnectivityGuard.get_instance()
+
+    assert guard.online is True
+    assert guard.should_block_request() is False
+
+    guard.register_network_failure(OSError(errno.ENETUNREACH, "unreachable"))
+    guard.register_network_failure(asyncio.TimeoutError("timeout"))
+
+    assert guard.should_block_request() is False
+    assert guard.failure_count == 2
+
+    guard.register_network_failure(ConnectionRefusedError("refused"))
+
+    assert guard.online is False
+    assert guard.failure_count == 3
+    assert guard.should_block_request() is True
+    assert guard.cooldown_remaining_seconds() > 0
+
+
+async def test_connectivity_guard_recovers_after_success():
+    guard = await ConnectivityGuard.get_instance()
+    guard.online = False
+    guard.failure_count = 5
+    guard.cooldown_until = datetime.now() + timedelta(seconds=90)
+
+    guard.register_success()
+
+    assert guard.online is True
+    assert guard.failure_count == 0
+    assert guard.cooldown_until is None
+    assert guard.should_block_request() is False
+
+
+async def test_downloader_short_circuits_all_request_helpers_during_cooldown():
+    guard = await ConnectivityGuard.get_instance()
+    guard.cooldown_until = datetime.now() + timedelta(seconds=30)
+    guard.online = False
+    guard.failure_count = 3
+
+    downloader = Downloader()
+
+    ok, payload = await downloader.make_request("GET", "https://example.invalid")
+    assert ok is False
+    assert payload == OFFLINE_COOLDOWN_ERROR
+
+    ok, payload, headers = await downloader.download_to_memory("https://example.invalid")
+    assert ok is False
+    assert payload == OFFLINE_COOLDOWN_ERROR
+    assert headers is None
+
+    ok, payload = await downloader.get_response_headers("https://example.invalid")
+    assert ok is False
+    assert payload == OFFLINE_COOLDOWN_ERROR

tests/services/test_metadata_sync_service.py

@@ -4,6 +4,7 @@ from unittest.mock import AsyncMock
 
 import pytest
 
+from py.services.connectivity_guard import OFFLINE_COOLDOWN_ERROR, OFFLINE_FRIENDLY_MESSAGE
 from py.services.errors import RateLimitError
 from py.services.metadata_sync_service import MetadataSyncService
 
@@ -243,17 +244,32 @@ async def test_fetch_and_update_model_handles_missing_remote_metadata(tmp_path):
 
     assert not ok
     assert "Model not found" in error
-    assert model_data["from_civitai"] is False
-    assert model_data["civitai_deleted"] is True
 
-    helpers.metadata_manager.hydrate_model_data.assert_not_awaited()
-    assert model_data["hydrated"] is True
 
-    helpers.metadata_manager.save_metadata.assert_awaited_once()
-    call_args = helpers.metadata_manager.save_metadata.await_args
-    assert call_args.args[0].endswith("model.safetensors")
-    assert "folder" not in call_args.args[1]
-    assert call_args.args[1]["hydrated"] is True
+@pytest.mark.asyncio
+async def test_fetch_and_update_model_returns_friendly_offline_message(tmp_path):
+    helpers = build_service()
+    helpers.default_provider.get_model_by_hash.return_value = (None, OFFLINE_COOLDOWN_ERROR)
+
+    model_path = tmp_path / "model.safetensors"
+    model_data = {
+        "model_name": "Local",
+        "folder": "root",
+        "file_path": str(model_path),
+    }
+    update_cache = AsyncMock(return_value=True)
+
+    ok, error = await helpers.service.fetch_and_update_model(
+        sha256="abc",
+        file_path=str(model_path),
+        model_data=model_data,
+        update_cache_func=update_cache,
+    )
+
+    assert ok is False
+    assert error is not None
+    assert OFFLINE_FRIENDLY_MESSAGE in error
+    update_cache.assert_not_awaited()
 
 
 @pytest.mark.asyncio