feat(agent): add LLM-powered metadata enrichment system with AgentCLI and PostProcessor

Introduce an agent skill framework for LLM-driven metadata enrichment:

- AgentCLI (py/agent_cli/): in-process wrappers around internal services
  using standard relative imports, eliminating the need for sys.path hacks
- LLMService: centralized BYOK (bring-your-own-key) LLM client supporting
  OpenAI, Ollama, and custom OpenAI-compatible endpoints
- PostProcessor: deterministic engine that applies LLM output via AgentCLI
  (replaces old handler.py + _BASE_MODEL_ALIASES approach)
- SkillRegistry: filesystem-based skill discovery (skill.yaml + prompt.md)
- AgentService: orchestrates skill execution with WebSocket progress
- Frontend AgentManager: WebSocket listeners, skill execution, config UI
- Context menu entries (single + bulk) for "Enrich Metadata (Agent)"
- Settings UI for AI Provider configuration (BYOK)
- Full i18n support across 9 locales

Bug fixes found during review:
- aiohttp.web.json_response: status_code= -> status=
- settings_modal cancelEditApiKey: wrong argument position
- AgentManager.isLlmConfigured: allow Ollama without API key
- PostProcessor._merge_tags: lowercase all tags to match TagUpdateService
This commit is contained in:
Will Miao
2026-07-02 20:51:11 +08:00
parent fe90f7f9b1
commit cf898da193
44 changed files with 5937 additions and 2180 deletions

View File

@@ -274,6 +274,9 @@ export class BulkContextMenu extends BaseContextMenu {
case 'resume-metadata-refresh':
bulkManager.setSkipMetadataRefresh(false);
break;
case 'enrich-hf-agent-bulk':
this.enrichBulkWithAgent();
break;
case 'delete-all':
bulkManager.showBulkDeleteModal();
break;
@@ -363,4 +366,66 @@ export class BulkContextMenu extends BaseContextMenu {
console.error('Bulk download example images failed:', error);
}
}
/**
* Enrich metadata for selected models via LLM agent skill.
*/
async enrichBulkWithAgent() {
if (state.selectedModels.size === 0) {
return;
}
const { agentManager } = await import('../../managers/AgentManager.js');
// Check if LLM is configured
const configured = await agentManager.isLlmConfigured();
if (!configured) {
showToast('toast.agent.llmNotConfigured', {}, 'warning');
return;
}
const modelPaths = [...state.selectedModels];
// Connect WebSocket for progress
agentManager.connect();
// Set up one-time completion handler
const onComplete = (data) => {
const idx = agentManager.completeCallbacks.indexOf(onComplete);
if (idx >= 0) agentManager.completeCallbacks.splice(idx, 1);
if (data.status === 'completed') {
showToast(
'toast.agent.enrichComplete',
{ summary: data.summary || 'Done' },
'success'
);
// Soft reload to reflect updated metadata
window.location.reload();
} else if (data.status === 'error') {
showToast(
'toast.agent.enrichFailed',
{ error: data.error || 'Unknown error' },
'error'
);
}
};
agentManager.onComplete(onComplete);
showToast(
'toast.agent.enrichStarted',
{ count: modelPaths.length },
'info'
);
try {
await agentManager.executeSkill('enrich_hf_metadata', modelPaths);
} catch (error) {
showToast(
'toast.agent.enrichFailed',
{ error: error.message },
'error'
);
}
}
}

View File

@@ -1,7 +1,7 @@
import { BaseContextMenu } from './BaseContextMenu.js';
import { ModelContextMenuMixin } from './ModelContextMenuMixin.js';
import { getModelApiClient, resetAndReload } from '../../api/modelApiFactory.js';
import { copyLoraSyntax, sendLoraToWorkflow, buildLoraSyntax } from '../../utils/uiHelpers.js';
import { copyLoraSyntax, sendLoraToWorkflow, buildLoraSyntax, showToast } from '../../utils/uiHelpers.js';
import { showExcludeModal, showDeleteModal } from '../../utils/modalUtils.js';
import { moveManager } from '../../managers/MoveManager.js';
@@ -63,6 +63,9 @@ export class LoraContextMenu extends BaseContextMenu {
case 'refresh-metadata':
getModelApiClient().refreshSingleModelMetadata(this.currentCard.dataset.filepath);
break;
case 'enrich-hf-agent':
this.enrichWithAgent(this.currentCard.dataset.filepath);
break;
case 'exclude':
showExcludeModal(this.currentCard.dataset.filepath);
break;
@@ -72,6 +75,46 @@ export class LoraContextMenu extends BaseContextMenu {
}
}
async enrichWithAgent(filePath) {
const { agentManager } = await import('../../managers/AgentManager.js');
// Check if LLM is configured
const configured = await agentManager.isLlmConfigured();
if (!configured) {
showToast('toast.agent.llmNotConfigured', {}, 'warning');
return;
}
// Connect WebSocket for progress
agentManager.connect();
// Set up one-time completion handler
const onComplete = (data) => {
const idx = agentManager.completeCallbacks.indexOf(onComplete);
if (idx >= 0) agentManager.completeCallbacks.splice(idx, 1);
if (data.status === 'completed') {
showToast('toast.agent.enrichComplete', { summary: data.summary || 'Done' }, 'success');
// Soft reload to reflect updated metadata
if (typeof resetAndReload === 'function') {
resetAndReload();
}
} else if (data.status === 'error') {
showToast('toast.agent.enrichFailed', { error: data.error || 'Unknown error' }, 'error');
}
};
agentManager.onComplete(onComplete);
// Show progress toast
showToast('toast.agent.enrichStarted', {}, 'info');
try {
await agentManager.executeSkill('enrich_hf_metadata', [filePath]);
} catch (error) {
showToast('toast.agent.enrichFailed', { error: error.message }, 'error');
}
}
sendLoraToWorkflow(replaceMode) {
const card = this.currentCard;
const usageTips = JSON.parse(card.dataset.usage_tips || '{}');

View File

@@ -0,0 +1,196 @@
/**
* AgentManager — WebSocket listener for agent skill progress events.
*
* Connects to the generic WebSocket endpoint and filters for
* `type: "agent_progress"` messages. Dispatches progress and completion
* events to registered callbacks.
*/
class AgentManager {
constructor() {
this.websocket = null;
this.progressCallbacks = [];
this.completeCallbacks = [];
this.errorCallbacks = [];
this.connected = false;
}
/**
* Connect to the WebSocket endpoint for agent progress events.
* Safe to call multiple times — won't reconnect if already connected.
*/
connect() {
if (this.connected && this.websocket?.readyState === WebSocket.OPEN) {
return;
}
const wsProtocol = window.location.protocol === 'https:' ? 'wss://' : 'ws://';
try {
this.websocket = new WebSocket(
`${wsProtocol}${window.location.host}/ws/fetch-progress`
);
} catch (e) {
console.error('AgentManager: Failed to create WebSocket:', e);
return;
}
this.websocket.onopen = () => {
this.connected = true;
console.debug('AgentManager: WebSocket connected');
};
this.websocket.onmessage = (event) => {
try {
const data = JSON.parse(event.data);
if (data.type !== 'agent_progress') return;
this._dispatch(data);
} catch (e) {
// Not JSON or wrong format — ignore
}
};
this.websocket.onerror = (error) => {
console.error('AgentManager: WebSocket error:', error);
this.connected = false;
};
this.websocket.onclose = () => {
this.connected = false;
console.debug('AgentManager: WebSocket closed');
};
}
/**
* Dispatch a parsed agent event to the appropriate callbacks.
* @param {Object} data - The parsed WebSocket message
*/
_dispatch(data) {
const { status, skill } = data;
if (status === 'error') {
this.errorCallbacks.forEach((cb) => {
try {
cb(data);
} catch (e) {
console.error('AgentManager error callback failed:', e);
}
});
return;
}
if (status === 'completed') {
this.completeCallbacks.forEach((cb) => {
try {
cb(data);
} catch (e) {
console.error('AgentManager complete callback failed:', e);
}
});
return;
}
// started, processing — general progress
this.progressCallbacks.forEach((cb) => {
try {
cb(data);
} catch (e) {
console.error('AgentManager progress callback failed:', e);
}
});
}
/**
* Register a callback for progress events (started, processing).
* @param {Function} callback - Receives the event data
*/
onProgress(callback) {
this.progressCallbacks.push(callback);
}
/**
* Register a callback for completion events.
* @param {Function} callback - Receives the event data
*/
onComplete(callback) {
this.completeCallbacks.push(callback);
}
/**
* Register a callback for error events.
* @param {Function} callback - Receives the event data
*/
onError(callback) {
this.errorCallbacks.push(callback);
}
/**
* Clear all registered callbacks.
*/
clearCallbacks() {
this.progressCallbacks = [];
this.completeCallbacks = [];
this.errorCallbacks = [];
}
/**
* Execute an agent skill on the provided model paths.
*
* @param {string} skillName - The skill to execute
* @param {string[]} modelPaths - Model file paths to process
* @returns {Promise<Object>} The response JSON
*/
async executeSkill(skillName, modelPaths) {
const response = await fetch(
`/api/lm/agent/execute/${encodeURIComponent(skillName)}`,
{
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ model_paths: modelPaths }),
}
);
if (!response.ok) {
const errorData = await response.json().catch(() => ({}));
throw new Error(
errorData.error || `HTTP ${response.status}: ${response.statusText}`
);
}
return response.json();
}
/**
* Check if the LLM provider is configured.
*
* Returns true when both an API key and a model name are set.
*
* @returns {Promise<boolean>}
*/
async isLlmConfigured() {
try {
const response = await fetch('/api/lm/settings');
if (!response.ok) return false;
const data = await response.json();
const provider = data.settings?.llm_provider;
const hasModel = !!data.settings?.llm_model;
const hasKey = !!data.settings?.llm_api_key;
return hasModel && (hasKey || provider === 'ollama');
} catch {
return false;
}
}
/**
* Get the list of available agent skills.
*
* @returns {Promise<Array>}
*/
async listSkills() {
const response = await fetch('/api/lm/agent/skills');
if (!response.ok) return [];
const data = await response.json();
return data.skills || [];
}
}
// Export as singleton
export const agentManager = new AgentManager();

View File

@@ -827,6 +827,23 @@ export class SettingsManager {
// Update API key status display (do NOT pre-fill the input)
this.updateApiKeyStatus();
this.updateLlmApiKeyStatus();
// AI Provider settings
const llmProviderSelect = document.getElementById('llmProvider');
if (llmProviderSelect) {
llmProviderSelect.value = state.global.settings.llm_provider || 'openai';
}
const llmApiBaseInput = document.getElementById('llmApiBase');
if (llmApiBaseInput) {
llmApiBaseInput.value = state.global.settings.llm_api_base || '';
}
const llmModelInput = document.getElementById('llmModel');
if (llmModelInput) {
llmModelInput.value = state.global.settings.llm_model || '';
}
const civitaiHostSelect = document.getElementById('civitaiHost');
if (civitaiHostSelect) {
@@ -2931,42 +2948,70 @@ export class SettingsManager {
}
}
editApiKey() {
const statusEl = document.getElementById('civitaiApiKeyStatus');
updateLlmApiKeyStatus() {
const hasKey = !!state.global.settings.llm_api_key;
const statusText = document.getElementById('llmApiKeyStatusText');
const actionBtn = document.getElementById('llmApiKeyActionBtn');
if (!statusText || !actionBtn) return;
if (hasKey) {
statusText.classList.remove('api-key-status--unconfigured');
statusText.classList.add('api-key-status--configured');
statusText.innerHTML = '<i class="fas fa-check-circle text-success"></i> '
+ translate('settings.aiProvider.apiKeyConfigured', {}, 'Configured');
actionBtn.textContent = translate('common.actions.change', {}, 'Change');
} else {
statusText.classList.remove('api-key-status--configured');
statusText.classList.add('api-key-status--unconfigured');
statusText.innerHTML = '<i class="fas fa-times-circle text-error"></i> '
+ translate('settings.aiProvider.apiKeyNotSet', {}, 'Not set');
actionBtn.textContent = translate('settings.aiProvider.apiKeySet', {}, 'Set up');
}
}
editApiKey(settingsKey = 'civitai_api_key', inputId = 'civitaiApiKey') {
const statusId = inputId + 'Status';
const editId = inputId + 'Edit';
const statusEl = document.getElementById(statusId);
if (statusEl) statusEl.classList.add('is-hidden');
const editContainer = document.getElementById('civitaiApiKeyEdit');
const editContainer = document.getElementById(editId);
if (editContainer) editContainer.classList.remove('is-hidden');
// Focus the input
const input = document.getElementById('civitaiApiKey');
const input = document.getElementById(inputId);
if (input) {
input.value = ''; // Never pre-fill the secret
setTimeout(() => input.focus(), 50);
}
}
cancelEditApiKey(silent) {
const editContainer = document.getElementById('civitaiApiKeyEdit');
cancelEditApiKey(silent, inputId = 'civitaiApiKey') {
const editId = inputId + 'Edit';
const statusId = inputId + 'Status';
const editContainer = document.getElementById(editId);
if (editContainer) editContainer.classList.add('is-hidden');
const statusContainer = document.getElementById('civitaiApiKeyStatus');
const statusContainer = document.getElementById(statusId);
if (statusContainer) statusContainer.classList.remove('is-hidden');
// Clear any typed value
const input = document.getElementById('civitaiApiKey');
const input = document.getElementById(inputId);
if (input) input.value = '';
if (!silent) {
this.updateApiKeyStatus();
if (inputId === 'civitaiApiKey') {
this.updateApiKeyStatus();
}
}
}
async saveApiKey() {
const input = document.getElementById('civitaiApiKey');
async saveApiKey(settingsKey = 'civitai_api_key', inputId = 'civitaiApiKey') {
const input = document.getElementById(inputId);
if (!input) return;
const value = input.value.trim();
try {
await this.saveSetting('civitai_api_key', value);
await this.saveSetting(settingsKey, value);
const labelName = settingsKey === 'civitai_api_key' ? 'CivitAI API Key' : 'LLM API Key';
showToast('toast.settings.settingsUpdated',
{ setting: 'CivitAI API Key' }, 'success');
{ setting: labelName }, 'success');
} catch (error) {
showToast('toast.settings.settingSaveFailed',
{ message: error.message }, 'error');
@@ -2974,9 +3019,13 @@ export class SettingsManager {
}
// Update the in-memory flag so the UI reflects the change
state.global.settings.civitai_api_key_set = !!value;
this.cancelEditApiKey(true);
this.updateApiKeyStatus();
if (settingsKey === 'civitai_api_key') {
state.global.settings.civitai_api_key_set = !!value;
}
this.cancelEditApiKey(true, inputId);
if (inputId === 'civitaiApiKey') {
this.updateApiKeyStatus();
}
}
toggleInputVisibility(button) {

View File

@@ -55,6 +55,10 @@ const DEFAULT_SETTINGS_BASE = Object.freeze({
strip_lora_on_copy: false,
use_new_license_icons: true,
group_by_model: false,
llm_provider: 'openai',
llm_api_key: '',
llm_api_base: '',
llm_model: '',
});
export function createDefaultSettings() {