From d0bc4be0dcf2486722e44319d83ec54c3f4d6d87 Mon Sep 17 00:00:00 2001 From: Will Miao Date: Sun, 9 Aug 2026 11:31:04 +0800 Subject: [PATCH] chore(skill): harden lora-manager-e2e for sandboxed E2E --- .agents/skills/lora-manager-e2e/SKILL.md | 246 +++++++++++++--- .../references/mcp-cheatsheet.md | 42 ++- .../references/test-scenarios.md | 16 +- .../scripts/example_e2e_test.py | 134 +++++---- .../lora-manager-e2e/scripts/start_server.py | 271 ++++++++++++++---- .../scripts/wait_for_server.py | 28 +- 6 files changed, 576 insertions(+), 161 deletions(-) diff --git a/.agents/skills/lora-manager-e2e/SKILL.md b/.agents/skills/lora-manager-e2e/SKILL.md index 316212d6..c6d422de 100644 --- a/.agents/skills/lora-manager-e2e/SKILL.md +++ b/.agents/skills/lora-manager-e2e/SKILL.md @@ -1,47 +1,145 @@ --- name: lora-manager-e2e -description: End-to-end testing and validation for LoRa Manager features. Use when performing automated E2E validation of LoRa Manager standalone mode, including starting/restarting the server, using Chrome DevTools MCP to interact with the web UI at http://127.0.0.1:8188/loras, and verifying frontend-to-backend functionality. Covers workflow validation, UI interaction testing, and integration testing between the standalone Python backend and the browser frontend. +description: End-to-end testing and validation for LoRa Manager features. Use when performing automated E2E validation of LoRa Manager standalone mode in a SANDBOXED, disposable configuration: check the port, start/restart the standalone server on a free port, use Chrome DevTools MCP to interact with the web UI (http://127.0.0.1:{PORT}/loras), and verify frontend-to-backend functionality. Covers workflow validation, UI interaction testing, and integration testing between the standalone Python backend and the browser frontend. Trigger keywords: E2E, standalone, Chrome DevTools MCP, lora-manager-e2e, sandbox. --- # LoRa Manager E2E Testing This skill provides workflows and utilities for end-to-end testing of LoRa Manager using Chrome DevTools MCP. +## Conventions Used in This Document + +- **`{PORT}`**: The server port. The default candidate is `8188`, but **`8188` is commonly occupied by a live ComfyUI process** and MUST NOT be assumed to be free. Always check availability first (see [Port Selection](#port-selection)) and use a free port (e.g. `8199`) for the E2E run. Substitute the actual port for every `{PORT}` in the commands below. +- **``**: The repository/worktree root. Always run commands from the repo or worktree root; never assume a specific absolute path (paths such as `/home//...` differ per machine). The E2E scripts resolve the project root themselves, but fixture/settings paths are relative to ``. + +## SANDBOX (MANDATORY) + +> **Read this section before running anything.** Every E2E run MUST target a throwaway sandbox, never the real user data. A fresh subagent that skips this section WILL permanently mutate real user recipes. + +1. **Portable settings**: create `/settings.json` (gitignored) with `"use_portable_settings": true` plus sandboxed `folder_paths` (lora/checkpoint roots) and `recipes_path`. This keeps the configuration inside the repo instead of the real user config dir (`~/.config/ComfyUI-LoRA-Manager/settings.json`). +2. **Sandboxed paths**: point `folder_paths` / `recipes_path` / `example_images_path` at disposable dirs — e.g. under `/tmp/opencode/-e2e/` (or worktree-local dirs). NEVER point the E2E at the real library (`~/models/...`), real recipe dir, or real settings. +3. **Never touch the real config**: the real user config at `~/.config/ComfyUI-LoRA-Manager/settings.json` and the real recipe dir must remain byte-identical before and after the run. +4. **Record real-data protection proof** before starting and after finishing: + ```bash + # BEFORE: snapshot real config + recipe library state + sha256sum ~/.config/ComfyUI-LoRA-Manager/settings.json > /tmp/opencode/-e2e/settings.before.sha256 + ls ~/models/recipes/*.recipe.json 2>/dev/null | wc -l > /tmp/opencode/-e2e/recipes-count.before.txt + find ~/models/recipes -name '*.recipe.json' -newermt "$(date -Iseconds)" | head # expect empty after run + # AFTER: record again, then diff the two snapshots. Any change = the run leaked into real data. + ``` + Also confirm `/git status` stays clean for `settings.json`/`cache/` (both are gitignored). + +### Portable Settings Example + +```json +{ + "use_portable_settings": true, + "folder_paths": { + "loras": ["/tmp/opencode/-e2e/models/loras"], + "checkpoints": ["/tmp/opencode/-e2e/models/checkpoints"], + "unet": ["/tmp/opencode/-e2e/models/checkpoints"], + "diffusers": [] + }, + "recipes_path": "/tmp/opencode/-e2e/recipes", + "example_images_path": "/tmp/opencode/-e2e/example_images" +} +``` + +The scanner computes and persists model hashes during the library scan, so the sandbox model dirs just need the model files + `.metadata.json` sidecars (see [Fixture + Fresh-State Guidance](#fixture--fresh-state-guidance)). + +## Time Budgets & Abort Guidance + +A fresh subagent should complete a sandboxed standalone E2E **in well under 30 minutes**. Budget each phase: + +| Phase | Expected duration | Abort if | +| --- | --- | --- | +| Port check + sandbox setup | < 2 min | — | +| Server start (detached) + readiness | < 30 s | > 60 s (2x) → stop | +| Chrome DevTools MCP connect | < 1 min | > 2 min → stop | +| Per entry-point run (after fixtures ready) | < 5 min | > 10 min (2x) → stop | +| Fixture reset + cache clear between runs | < 1 min | > 2 min → stop | + +**Abort rule**: if a phase exceeds ~2x its budget, OR any single tool call fails/retries 3+ times in a row, **STOP**. Do not loop or retry blindly. Report `BLOCKED` with: the phase, the last observed state (server PID + `ss -tlnp` output, page snapshot, last API response), and the suspected cause. Record the partial state as evidence; a clean BLOCKED report is more valuable than an hour of retries. + ## Prerequisites -- LoRa Manager project cloned and dependencies installed (`pip install -r requirements.txt`) +- LoRa Manager project cloned and dependencies installed (`pip install -r requirements.txt`) — run everything from `` - Chrome browser available for debugging - Chrome DevTools MCP connected +- `ss` (or `lsof`/`netstat`) available for port checks: `ss -tlnp` -## Quick Start Workflow +## Port Selection -### 1. Start LoRa Manager Standalone - -```python -# Use the provided script to start the server -python .agents/skills/lora-manager-e2e/scripts/start_server.py --port 8188 -``` - -Or manually: -```bash -cd /home/miao/workspace/ComfyUI/custom_nodes/ComfyUI-Lora-Manager -python standalone.py --port 8188 -``` - -Wait for server ready message before proceeding. - -### 2. Open Chrome Debug Mode +`8188` is only the *default candidate*. Verify it is actually free before every run: ```bash -# Chrome with remote debugging on port 9222 -google-chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:8188/loras +# Is anything listening on 8188? +ss -tlnp | grep ':8188' || echo "8188 is free" ``` -### 3. Connect Chrome DevTools MCP +- If a process holds `8188` (e.g. a live ComfyUI — pid 6575 on this machine), pick a different free port, e.g. `8199`: + ```bash + ss -tlnp | grep ':8199' || echo "8199 is free" + ``` +- **Never** kill a process you did not start for this E2E. The live ComfyUI is off-limits. Pick a free port instead. +- Use your chosen port for **all** subsequent commands (server, Chrome launch, browser URLs). -Ensure the MCP server is connected to Chrome at `http://localhost:9222`. +## Quick Start Workflow (sandboxed) -### 4. Navigate and Interact +### 1. Prepare the sandbox + +```bash +cd # ALWAYS run from the repo/worktree root +mkdir -p /tmp/opencode/-e2e/models/{loras,checkpoints} +mkdir -p /tmp/opencode/-e2e/{recipes,example_images,recipes-before} +# write /settings.json per the portable-settings example above +# record real-data protection proof (see SANDBOX section) +``` + +### 2. Check port availability + +```bash +ss -tlnp | grep ':{PORT}' || echo "port {PORT} is free" +``` + +If `{PORT}` is occupied by an unrelated process, pick a free one and use it everywhere below. When in doubt use `8199`. + +### 3. Start LoRa Manager Standalone (detached) + +The standalone server **dies with the shell unless launched fully detached** — a plain background `&` from the bash tool is killed when the tool call returns. Launch via the helper script: + +```bash +python .agents/skills/lora-manager-e2e/scripts/start_server.py --port {PORT} --wait --timeout 30 --detach +``` + +Or manually (equivalent detached form): + +```bash +setsid nohup python standalone.py --port {PORT} --host 127.0.0.1 < /dev/null \ + >> /tmp/opencode/-e2e/server.log 2>&1 & +echo "started" # record the printed/pidfile PID for cleanup +``` + +Verify it is listening **before** proceeding (readiness poll is not a substitute for this): + +```bash +ss -tlnp | grep ':{PORT}' +``` + +Record the server PID for cleanup: the helper script writes it to `/tmp/lora-manager-e2e-server-{PORT}.pid`; a manual `setsid` launch has no pidfile, so capture it explicitly (e.g. from `ss -tlnp`). + +### 4. Open Chrome Debug Mode + +```bash +# Chrome with remote debugging on port 9222 (note the {PORT} URL) +google-chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:{PORT}/loras +``` + +### 5. Connect Chrome DevTools MCP + +Ensure the MCP server is connected to Chrome at `http://localhost:9222`. Verify with `list_pages` — if it fails with "browser is already running", see [Chrome DevTools MCP Troubleshooting](#chrome-devtools-mcp-troubleshooting). + +### 6. Navigate and Interact Use Chrome DevTools MCP tools to: - Take snapshots: `take_snapshot` @@ -56,7 +154,7 @@ Use Chrome DevTools MCP tools to: ```python # Navigate to LoRA list page -navigate_page(type="url", url="http://127.0.0.1:8188/loras") +navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras") # Wait for page to load wait_for(text="LoRAs", timeout=10000) @@ -68,9 +166,10 @@ snapshot = take_snapshot() ### Pattern: Restart Server for Configuration Changes ```python -# Stop current server (if running) -# Start with new configuration -python .agents/skills/lora-manager-e2e/scripts/start_server.py --port 8188 --restart +# Stop current server (if running), start with new configuration. +# --restart only kills the E2E server this script started before (via its pidfile); +# it refuses to blindly kill unrelated processes on the port. +python .agents/skills/lora-manager-e2e/scripts/start_server.py --port {PORT} --restart --wait --detach # Wait and refresh browser navigate_page(type="reload", ignoreCache=True) @@ -130,24 +229,96 @@ click(uid="modal-submit-button") wait_for(text="Success", timeout=5000) ``` +## Fixture + Fresh-State Guidance + +For rematch/repair E2E runs, seed the **sandboxed** `recipes_path` with hand-written fixture recipes. Rules (validated by the task-8 E2E): + +1. **Filename constraint**: each file MUST be named `f"{id}.recipe.json"` **and** the in-JSON `id` field MUST equal the filename. Discovery accepts any `*.recipe.json`, but persistence resolves the path via `get_recipe_json_path` and `_save_recipe_persistently` returns `False` on a mismatch → the fixture would be counted as an error. + - `recipe-a.recipe.json` → in-JSON `"id": "recipe-a"` +2. **File format**: mirror an existing recipe JSON — top-level `id`, `file_path`, `title`, `loras`, `fingerprint`, `gen_params`; lora entries per the persistence conventions (`hash`, `file_name`, `modelVersionId`, `isDeleted`, ...). +3. **Companion image**: each recipe needs an image (e.g. a `.webp` generated with PIL) referenced by `file_path`, used for EXIF verification (`ExifUtils.append_recipe_metadata` writes a `"Recipe metadata: ..."` marker; a freshly generated `.webp` with no marker is the clean "untouched" control). +4. **autov3 three-state contract**: for L3 (autov3-only, renamed-file) fixtures the local model's `.metadata.json` sidecar MUST have the `autov3` key **ABSENT** (the "unchecked" state), NOT `""` — `""` is the TERMINAL "checked but unavailable" state that L3 deliberately skips. The scanner computes + persists `autov3` from the file header during the normal library scan (`model_scanner.py` `_process_model_file`), so the live L3 match resolves through the local autov3/hash cache; the computed-autov3 branch for unchecked items is covered by the unit suite. +5. **Fixture design for a rematch run** (mirrors the task-8 E2E): + - `recipe-a`: lora entry `isDeleted=True`, `hash` = 12-char autov3 computed from the local model (`calculate_autov3`, `py/utils/file_utils.py`), whose local model file was RENAMED after the recipe was written so `file_name` differs (proves L3 match without filename). + - `recipe-b`: parser-convention checkpoint entry (uses `id`, no `modelVersionId`) matching a local checkpoint via L2 — the local checkpoint's `.metadata.json` MUST carry civitai version data with that `id` so `version_index` contains it (L2 cannot match otherwise). + - `recipe-c`: healthy recipe (no deleted entries) → must remain untouched. + +### Fresh state between entry-point runs + +Each entry point (global / per-recipe / selection-bulk) must start from the same deleted state. Between runs: + +```bash +# 1. Reset fixtures to the before-state snapshot (copy back from recipes-before/) +cp /tmp/opencode/-e2e/recipes-before/*.recipe.json /tmp/opencode/-e2e/recipes/ +# 2. Clear the recipe/FTS caches so the stale in-memory/library state is gone +rm -f /cache/recipe/*.sqlite +rm -rf /cache/fts/* +# 3. Restart the server (fresh process, fresh scan) +python .agents/skills/lora-manager-e2e/scripts/start_server.py --port {PORT} --restart --wait --timeout 30 --detach +# 4. Re-verify server listening + reload the browser page +``` + +## Server Lifecycle + +- **Detached launch is mandatory**: the standalone server dies with the shell unless launched via `setsid` (or the helper script's `--detach`). Use `setsid nohup python standalone.py --port {PORT} --host 127.0.0.1 ... < /dev/null &`. +- **Verify with `ss -tlnp`** after every (re)start; do not proceed on a blind "server starting" message. +- **Never kill pre-existing processes** — only kill the E2E server PID you started (`start_server.py --restart` kills only PIDs it manages via its pidfile). The live ComfyUI or a stale QA Chrome must never be killed as part of cleanup unless explicitly identified as such (see Chrome troubleshooting). +- **Record your PID for cleanup**: note the PID printed/pidfile, and stop exactly that PID at the end (`kill `, then confirm with `ss -tlnp` that `{PORT}` is released). + +## Chrome DevTools MCP Troubleshooting + +### Stale profile lock ("browser is already running" / `list_pages` fails) + +A Chrome profile can be held by a stale Chrome from a prior MCP session, which makes `list_pages` fail with "browser is already running": + +1. Identify the stale Chrome — it owns the profile dir in `--user-data-dir` (e.g. `~/.config/chrome-dev-profile`). Find its process: + ```bash + ps -ef | grep -i '[c]hrome.*user-data-dir' + ``` +2. Confirm it is a QA Chrome from a completed task (its parent is an old MCP/browser process, it is NOT the live ComfyUI server, and it is NOT your current MCP instance). +3. Kill ONLY that stale Chrome: + ```bash + kill + ``` + Never kill the live server or unrelated processes. +4. Retry `list_pages`. The current MCP will spawn a fresh browser. + +### Screenshot-write restrictions + +The chrome-devtools MCP may refuse to write into paths outside its configured workspace roots (e.g. the worktree `.omo/evidence/...` canonicalizing to an unmapped path). Workaround: + +```bash +# 1. Save the screenshot to /tmp via the MCP +# take_screenshot(filePath="/tmp/-e2e/recipe-b-after.png", format="png") +# 2. Copy it into the evidence dir from the shell +mkdir -p /.omo/evidence/screenshots +cp /tmp/-e2e/recipe-b-after.png /.omo/evidence/screenshots/ +``` + +## Cancellation Testing (KNOWN GAP) + +Testing the rematch-cancel path E2E requires a run long enough to cancel mid-flight. A tiny 3-recipe fixture set completes in **seconds** — too fast to reliably cancel. The cancel path is currently **unit-covered only** (`rematch_all_recipes` cancellation tests); do not block an E2E run on cancel-path verification. If you must attempt it, you would need an artificially large/deferred fixture set to create a cancellable window — treat this as a research task, not part of the standard E2E. + ## Available Scripts ### scripts/start_server.py -Starts or restarts the LoRa Manager standalone server. +Starts or restarts the LoRa Manager standalone server for E2E testing. ```bash -python scripts/start_server.py [--port PORT] [--restart] [--wait] +python scripts/start_server.py [--port PORT] [--restart] [--wait] [--timeout SECONDS] [--detach] ``` Options: -- `--port`: Server port (default: 8188) -- `--restart`: Kill existing server before starting -- `--wait`: Wait for server to be ready before exiting +- `--port`: Server port (default: 8188). The script exits early with a clear message if the port is already in use by an unrelated process. +- `--restart`: Kill the E2E server this script previously managed (tracked via `/tmp/lora-manager-e2e-server-{PORT}.pid`) before starting. If unrelated processes still hold the port after that, the script reports them and aborts instead of killing them. +- `--wait`: Wait for the server to be ready before exiting. +- `--timeout`: Readiness wait timeout in seconds (default: 30). +- `--detach`: Launch the server fully detached (`setsid`-style, survives shell death — REQUIRED for E2E). Default off: a normal background process that dies with the shell. ### scripts/wait_for_server.py -Polls server until ready or timeout. +Polls the server until ready or timeout. ```bash python scripts/wait_for_server.py [--port PORT] [--timeout SECONDS] @@ -196,6 +367,7 @@ results = performance_stop_trace() ## Cleanup Always ensure proper cleanup after tests: -1. Stop the standalone server -2. Close browser pages (keep at least one open) -3. Clear temporary data if needed +1. Stop the standalone server: `kill ` (only the PID you started), then confirm `ss -tlnp | grep ':{PORT}'` is empty. +2. Close browser pages (keep at least one open). +3. Remove the sandbox: `rm -rf /tmp/opencode/-e2e` and `/settings.json` + `/cache` (both gitignored). +4. Re-run the real-data protection check from the SANDBOX section and record the result in your evidence. diff --git a/.agents/skills/lora-manager-e2e/references/mcp-cheatsheet.md b/.agents/skills/lora-manager-e2e/references/mcp-cheatsheet.md index ff534ea1..2f92035d 100644 --- a/.agents/skills/lora-manager-e2e/references/mcp-cheatsheet.md +++ b/.agents/skills/lora-manager-e2e/references/mcp-cheatsheet.md @@ -2,11 +2,13 @@ Quick reference for common MCP commands used in LoRa Manager E2E testing. +> **Port convention**: `{PORT}` is the port chosen for the E2E run (default candidate `8188`, but only if actually free — see the SKILL.md Port Selection section; use e.g. `8199` when `8188` is occupied by a live ComfyUI). Always run against the **sandboxed** standalone server, never a live instance. + ## Navigation ```python # Navigate to LoRA list page -navigate_page(type="url", url="http://127.0.0.1:8188/loras") +navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras") # Reload page with cache clear navigate_page(type="reload", ignoreCache=True) @@ -179,7 +181,7 @@ pages = list_pages() select_page(pageId=0, bringToFront=True) # Create new page -new_page(url="http://127.0.0.1:8188/loras") +new_page(url="http://127.0.0.1:{PORT}/loras") # Close page (keep at least one open!) close_page(pageId=1) @@ -261,7 +263,7 @@ drag(from_uid="draggable-item", to_uid="drop-zone") ### Verify LoRA Cards Loaded ```python -navigate_page(type="url", url="http://127.0.0.1:8188/loras") +navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras") wait_for(text="LoRAs", timeout=10000) # Check if cards loaded @@ -322,3 +324,37 @@ navigate_page(type="reload") errors = list_console_messages(types=["error"]) assert len(errors) == 0, f"Console errors: {errors}" ``` + +## Troubleshooting + +### Stale profile lock ("browser is already running" / `list_pages` fails) + +A Chrome profile held by a stale Chrome from a prior MCP session makes `list_pages` +fail with "browser is already running". Fix: + +1. Find the stale Chrome that owns the profile dir (e.g. `~/.config/chrome-dev-profile`): + ```bash + ps -ef | grep -i '[c]hrome.*user-data-dir' + ``` +2. Confirm it is a QA Chrome from a completed task (NOT the live ComfyUI server, NOT + your current MCP instance). +3. Kill ONLY that stale Chrome (`kill `), then retry `list_pages`. + +### Screenshot-write restrictions + +The MCP may refuse to write into paths outside its configured workspace roots +(e.g. `.omo/evidence/screenshots/` under a worktree that canonicalizes to an unmapped +path). Save the screenshot to `/tmp` via the MCP, then copy it into the evidence dir: + +```bash +# MCP: take_screenshot(filePath="/tmp/-e2e/recipe-b-after.png", format="png") +# Shell: +mkdir -p /.omo/evidence/screenshots +cp /tmp/-e2e/recipe-b-after.png /.omo/evidence/screenshots/ +``` + +### Time budgets & abort rule + +See SKILL.md "Time Budgets & Abort Guidance": if a phase exceeds ~2x its budget or a +tool call retries 3+ times in a row, STOP and report BLOCKED with the last observed +state (server PID + `ss -tlnp`, page snapshot, last API response). Do not loop. diff --git a/.agents/skills/lora-manager-e2e/references/test-scenarios.md b/.agents/skills/lora-manager-e2e/references/test-scenarios.md index 54dbb497..1ddcc6f0 100644 --- a/.agents/skills/lora-manager-e2e/references/test-scenarios.md +++ b/.agents/skills/lora-manager-e2e/references/test-scenarios.md @@ -2,6 +2,14 @@ This document provides detailed test scenarios for end-to-end validation of LoRa Manager features. +> **Run preconditions (from SKILL.md)**: every run uses the **sandboxed** standalone +> server on a free port `{PORT}` (default candidate `8188`, only if actually free — pick +> e.g. `8199` when `8188` is occupied by a live ComfyUI). Fixtures live in the sandboxed +> `recipes_path` as `f"{id}.recipe.json"` files with matching in-JSON `id`; the real user +> config and real library are never touched (record protection proof before/after). +> Abort if a phase exceeds ~2x its budget or a tool call retries 3+ times (SKILL.md +> "Time Budgets & Abort Guidance"). + ## Table of Contents 1. [LoRA List Page](#lora-list-page) @@ -19,7 +27,7 @@ This document provides detailed test scenarios for end-to-end validation of LoRa **Objective**: Verify the LoRA list page loads correctly and displays models. **Steps**: -1. Navigate to `http://127.0.0.1:8188/loras` +1. Navigate to `http://127.0.0.1:{PORT}/loras` 2. Wait for page title "LoRAs" to appear 3. Take snapshot to verify: - Header with "LoRAs" title is visible @@ -134,7 +142,7 @@ evaluate_script(function=""" **Objective**: Verify recipes page loads and displays recipes. **Steps**: -1. Navigate to `http://127.0.0.1:8188/recipes` +1. Navigate to `http://127.0.0.1:{PORT}/recipes` 2. Wait for "Recipes" title 3. Take snapshot @@ -176,7 +184,7 @@ evaluate_script(function=""" **Objective**: Verify settings page displays correctly. **Steps**: -1. Navigate to `http://127.0.0.1:8188/settings` +1. Navigate to `http://127.0.0.1:{PORT}/settings` 2. Wait for "Settings" title 3. Take snapshot @@ -190,7 +198,7 @@ evaluate_script(function=""" 1. Navigate to settings page 2. Change a setting (e.g., default view mode) 3. Save settings -4. Restart server: `python scripts/start_server.py --restart --wait` +4. Restart server: `python scripts/start_server.py --port {PORT} --restart --wait --timeout 30 --detach` 5. Refresh browser page 6. Navigate to settings diff --git a/.agents/skills/lora-manager-e2e/scripts/example_e2e_test.py b/.agents/skills/lora-manager-e2e/scripts/example_e2e_test.py index 6b2c15bd..d3519bbe 100755 --- a/.agents/skills/lora-manager-e2e/scripts/example_e2e_test.py +++ b/.agents/skills/lora-manager-e2e/scripts/example_e2e_test.py @@ -8,186 +8,208 @@ This script shows how to: 3. Verify functionality end-to-end Note: This is a template. Actual execution requires Chrome DevTools MCP. + +Port: pick a FREE port for the run — 8188 is commonly occupied by a live +ComfyUI (see the skill's Port Selection section). Set PORT below to e.g. 8199 +when 8188 is taken. Always run against a SANDBOXED standalone server. """ import subprocess import sys -import time + +# Choose the E2E port. 8188 is only the default candidate; use 8199 (or any +# free port checked with `ss -tlnp`) when 8188 is occupied by a live ComfyUI. +PORT = "8188" def run_test(): """Run example E2E test flow.""" - + print("=" * 60) print("LoRa Manager E2E Test Example") print("=" * 60) - - # Step 1: Start server + + # Step 1: Start server (detached so it survives the shell) print("\n[1/5] Starting LoRa Manager standalone server...") result = subprocess.run( - [sys.executable, "start_server.py", "--port", "8188", "--wait", "--timeout", "30"], + [sys.executable, "start_server.py", "--port", PORT, "--wait", "--timeout", "30", "--detach"], capture_output=True, - text=True + text=True, ) if result.returncode != 0: print(f"Failed to start server: {result.stderr}") return 1 print("Server ready!") - + # Step 2: Open Chrome (manual step - show command) print("\n[2/5] Open Chrome with debug mode:") - print("google-chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:8188/loras") + print( + f"google-chrome --remote-debugging-port=9222 " + f"--user-data-dir=/tmp/chrome-lora-manager http://127.0.0.1:{PORT}/loras" + ) print("(In actual test, this would be automated via MCP)") - + # Step 3: Navigate and verify page load print("\n[3/5] Page Load Verification:") - print(""" + print( + f""" MCP Commands to execute: - 1. navigate_page(type="url", url="http://127.0.0.1:8188/loras") + 1. navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras") 2. wait_for(text="LoRAs", timeout=10000) 3. snapshot = take_snapshot() - """) - + """ + ) + # Step 4: Test search functionality print("\n[4/5] Search Functionality Test:") - print(""" + print( + """ MCP Commands to execute: 1. fill(uid="search-input", value="test") 2. press_key(key="Enter") 3. wait_for(text="Results", timeout=5000) - 4. result = evaluate_script(function=""" + 4. result = evaluate_script(function=` () => { const cards = document.querySelectorAll('.lora-card'); return { count: cards.length }; } - """) - """) - + `) + """ + ) + # Step 5: Verify API print("\n[5/5] API Verification:") - print(""" + print( + """ MCP Commands to execute: - 1. api_result = evaluate_script(function=""" + 1. api_result = evaluate_script(function=` async () => { const response = await fetch('/loras/api/list'); const data = await response.json(); return { count: data.length, status: response.status }; } - """) + `) 2. Verify api_result['status'] == 200 - """) - + """ + ) + print("\n" + "=" * 60) print("Test flow completed!") print("=" * 60) - + return 0 def example_restart_flow(): """Example: Testing configuration change that requires restart.""" - + print("\n" + "=" * 60) print("Example: Server Restart Flow") print("=" * 60) - - print(""" + + print( + f""" Scenario: Change setting and verify after restart - + Steps: 1. Navigate to settings page - - navigate_page(type="url", url="http://127.0.0.1:8188/settings") - + - navigate_page(type="url", url="http://127.0.0.1:{PORT}/settings") + 2. Change a setting (e.g., theme) - fill(uid="theme-select", value="dark") - click(uid="save-settings-button") - + 3. Restart server - - subprocess.run([python, "start_server.py", "--restart", "--wait"]) - + - subprocess.run([python, "start_server.py", "--port", "{PORT}", "--restart", "--wait", "--detach"]) + 4. Refresh browser - navigate_page(type="reload", ignoreCache=True) - wait_for(text="LoRAs", timeout=15000) - + 5. Verify setting persisted - - navigate_page(type="url", url="http://127.0.0.1:8188/settings") + - navigate_page(type="url", url="http://127.0.0.1:{PORT}/settings") - theme = evaluate_script(function="() => document.querySelector('#theme-select').value") - assert theme == "dark" - """) + """ + ) def example_modal_interaction(): """Example: Testing modal dialog interaction.""" - + print("\n" + "=" * 60) print("Example: Modal Dialog Interaction") print("=" * 60) - - print(""" + + print( + """ Scenario: Add new LoRA via modal - + Steps: 1. Open modal - click(uid="add-lora-button") - wait_for(text="Add LoRA", timeout=3000) - + 2. Fill form - fill_form(elements=[ {"uid": "lora-name", "value": "Test Character"}, {"uid": "lora-path", "value": "/models/test.safetensors"}, ]) - + 3. Submit - click(uid="modal-submit-button") - + 4. Verify success - wait_for(text="Successfully added", timeout=5000) - snapshot = take_snapshot() - """) + """ + ) def example_network_monitoring(): """Example: Network request monitoring.""" - + print("\n" + "=" * 60) print("Example: Network Request Monitoring") print("=" * 60) - - print(""" + + print( + f""" Scenario: Verify API calls during user interaction - + Steps: 1. Clear network log (implicit on navigation) - - navigate_page(type="url", url="http://127.0.0.1:8188/loras") - + - navigate_page(type="url", url="http://127.0.0.1:{PORT}/loras") + 2. Perform action that triggers API call - fill(uid="search-input", value="character") - press_key(key="Enter") - + 3. List network requests - requests = list_network_requests(resourceTypes=["xhr", "fetch"]) - + 4. Find search API call - search_requests = [r for r in requests if "/api/search" in r.get("url", "")] - assert len(search_requests) > 0, "Search API was not called" - + 5. Get request details - if search_requests: details = get_network_request(reqid=search_requests[0]["reqid"]) - Verify request method, response status, etc. - """) + """ + ) if __name__ == "__main__": print("LoRa Manager E2E Test Examples\n") print("This script demonstrates E2E testing patterns.\n") print("Note: Actual execution requires Chrome DevTools MCP connection.\n") - + run_test() example_restart_flow() example_modal_interaction() example_network_monitoring() - + print("\n" + "=" * 60) print("All examples shown!") print("=" * 60) diff --git a/.agents/skills/lora-manager-e2e/scripts/start_server.py b/.agents/skills/lora-manager-e2e/scripts/start_server.py index bb1716f7..8c1a17bf 100755 --- a/.agents/skills/lora-manager-e2e/scripts/start_server.py +++ b/.agents/skills/lora-manager-e2e/scripts/start_server.py @@ -1,15 +1,78 @@ #!/usr/bin/env python3 """ Start or restart LoRa Manager standalone server for E2E testing. + +Backward-compatible CLI: --port, --restart, --wait, --timeout all work as before. +New options: --detach (setsid-style fully detached launch, survives shell death). + +Safety rules implemented here: +- Never kill processes the script did not start. The script tracks the PIDs it + manages in a pidfile (/tmp/lora-manager-e2e-server-{PORT}.pid). +- If the port is held by an unrelated process (e.g. a live ComfyUI) the script + reports the conflict and exits early instead of killing it. +- --restart only kills managed PIDs; if unrelated processes still hold the port + afterwards, the script reports them and aborts. """ +from __future__ import annotations + import argparse +import os +import signal +import socket import subprocess import sys import time -import socket -import signal -import os + +PIDFILE_PREFIX = "/tmp/lora-manager-e2e-server" + + +def pidfile_path(port: int) -> str: + """Path of the pidfile that records PIDs this script started for a port.""" + return f"{PIDFILE_PREFIX}-{port}.pid" + + +def read_managed_pids(port: int) -> list[int]: + """Read PIDs this script previously managed for the port (may be stale).""" + path = pidfile_path(port) + if not os.path.exists(path): + return [] + try: + with open(path, "r", encoding="utf-8") as fh: + return [int(line.strip()) for line in fh if line.strip().isdigit()] + except (OSError, ValueError): + return [] + + +def write_managed_pids(port: int, pids: list[int]) -> None: + """Record PIDs this script manages for the port.""" + try: + with open(pidfile_path(port), "w", encoding="utf-8") as fh: + for pid in pids: + fh.write(f"{pid}\n") + except OSError as exc: + print(f"Warning: could not write pidfile for port {port}: {exc}") + + +def clear_managed_pids(port: int) -> None: + """Remove the pidfile for the port (no longer managed).""" + path = pidfile_path(port) + try: + if os.path.exists(path): + os.remove(path) + except OSError as exc: + print(f"Warning: could not remove pidfile {path}: {exc}") + + +def process_alive(pid: int) -> bool: + """Return True if a process with the given pid exists.""" + try: + os.kill(pid, 0) + return True + except ProcessLookupError: + return False + except PermissionError: + return True # exists but owned by someone else def find_server_process(port: int) -> list[int]: @@ -19,7 +82,7 @@ def find_server_process(port: int) -> list[int]: ["lsof", "-ti", f":{port}"], capture_output=True, text=True, - check=False + check=False, ) if result.returncode == 0 and result.stdout.strip(): return [int(pid) for pid in result.stdout.strip().split("\n") if pid] @@ -30,7 +93,7 @@ def find_server_process(port: int) -> list[int]: ["netstat", "-tlnp"], capture_output=True, text=True, - check=False + check=False, ) pids = [] for line in result.stdout.split("\n"): @@ -49,30 +112,48 @@ def find_server_process(port: int) -> list[int]: return [] -def kill_server(port: int) -> None: - """Kill processes using the specified port.""" - pids = find_server_process(port) +def describe_processes(pids: list[int]) -> str: + """Human-readable description of a pid list (pid + command line).""" + descriptions = [] for pid in pids: + cmdline = "" + try: + with open(f"/proc/{pid}/cmdline", "rb") as fh: + raw = fh.read().replace(b"\x00", b" ").decode("utf-8", "replace") + cmdline = raw.strip() + except OSError: + pass + descriptions.append(f"pid {pid}{' (' + cmdline + ')' if cmdline else ''}") + return ", ".join(descriptions) if descriptions else "none" + + +def kill_pids(pids: list[int], what: str) -> None: + """Send SIGTERM (then SIGKILL) to the given PIDs, only after reporting.""" + for pid in pids: + print(f"Sent SIGTERM to {what} pid {pid}") try: os.kill(pid, signal.SIGTERM) - print(f"Sent SIGTERM to process {pid}") except ProcessLookupError: pass - + # Wait for processes to terminate - time.sleep(1) - + deadline = time.time() + 5 + while time.time() < deadline: + if not any(process_alive(pid) for pid in pids): + break + time.sleep(0.2) + # Force kill if still running - pids = find_server_process(port) for pid in pids: - try: - os.kill(pid, signal.SIGKILL) - print(f"Sent SIGKILL to process {pid}") - except ProcessLookupError: - pass + if process_alive(pid): + try: + os.kill(pid, signal.SIGKILL) + print(f"Sent SIGKILL to {what} pid {pid}") + except ProcessLookupError: + pass -def is_server_ready(port: int, timeout: float = 0.5) -> bool: +def is_server_ready(port: int, timeout: float = 2.0) -> bool: """Check if server is accepting connections.""" try: with socket.create_connection(("127.0.0.1", port), timeout=timeout): @@ -84,9 +165,15 @@ def is_server_ready(port: int, timeout: float = 0.5) -> bool: def wait_for_server(port: int, timeout: int = 30) -> bool: """Wait for server to become ready.""" start = time.time() + last_report = 0.0 while time.time() - start < timeout: if is_server_ready(port): return True + # Report progress every ~5s so a slow boot is visible, not silent. + elapsed = time.time() - start + if elapsed - last_report >= 5: + print(f" ...still waiting ({int(elapsed)}s/{timeout}s)") + last_report = elapsed time.sleep(0.5) return False @@ -99,68 +186,148 @@ def main() -> int: "--port", type=int, default=8188, - help="Server port (default: 8188)" + help="Server port (default: 8188)", ) parser.add_argument( "--restart", action="store_true", - help="Kill existing server before starting" + help="Kill the E2E server previously managed by this script for the port " + "(tracked via pidfile) before starting; refuse to kill unrelated processes", ) parser.add_argument( "--wait", action="store_true", - help="Wait for server to be ready before exiting" + help="Wait for server to be ready before exiting", ) parser.add_argument( "--timeout", type=int, default=30, - help="Timeout for waiting (default: 30)" + help="Timeout for waiting (default: 30)", ) - + parser.add_argument( + "--detach", + action="store_true", + help="Launch the server fully detached (setsid-style) so it survives shell " + "death. REQUIRED for E2E: a plain background process dies with the shell", + ) + args = parser.parse_args() - + # Get project root (parent of .agents directory) script_dir = os.path.dirname(os.path.abspath(__file__)) skill_dir = os.path.dirname(script_dir) project_root = os.path.dirname(os.path.dirname(os.path.dirname(skill_dir))) - - # Restart if requested + + managed_pids = read_managed_pids(args.port) + + # Restart if requested: kill ONLY managed PIDs. if args.restart: - print(f"Killing existing server on port {args.port}...") - kill_server(args.port) + alive_managed = [pid for pid in managed_pids if process_alive(pid)] + if alive_managed: + print( + f"Killing E2E server previously started by this script on port " + f"{args.port} ({describe_processes(alive_managed)})..." + ) + kill_pids(alive_managed, "managed E2E server") + else: + print( + f"No live managed E2E server for port {args.port} " + f"(pidfile: {pidfile_path(args.port)})" + ) time.sleep(1) - - # Check if already running - if is_server_ready(args.port): - print(f"Server already running on port {args.port}") - return 0 - + # Refuse to kill anything the script did not manage. + remaining = find_server_process(args.port) + if remaining: + print( + f"ERROR: port {args.port} is still held by process(es) this script " + f"did not start: {describe_processes(remaining)}." + ) + print( + "These may be unrelated (e.g. a live ComfyUI). The script will NOT " + "kill them. Pick a different --port, or stop them manually if you " + "are certain they are stale E2E servers." + ) + return 2 + clear_managed_pids(args.port) + + # Port conflict check before starting: never blind-kill. + port_pids = find_server_process(args.port) + if port_pids: + alive_managed = [pid for pid in port_pids if pid in managed_pids] + unmanaged = [pid for pid in port_pids if pid not in managed_pids] + if alive_managed and not unmanaged: + print( + f"Server already running on port {args.port} " + f"({describe_processes(alive_managed)}, started by this script). " + f"Use --restart to recycle it." + ) + return 0 + print( + f"ERROR: port {args.port} is already in use by process(es): " + f"{describe_processes(port_pids)}." + ) + print( + "This is likely an unrelated process (e.g. a live ComfyUI holding 8188). " + "The script will NOT kill it. Pick a free port with --port, e.g. 8199." + ) + return 2 + # Start server print(f"Starting LoRa Manager standalone server on port {args.port}...") - cmd = [sys.executable, "standalone.py", "--port", str(args.port)] - - # Start in background - process = subprocess.Popen( - cmd, - cwd=project_root, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - start_new_session=True - ) - - print(f"Server process started with PID {process.pid}") - + cmd = [ + sys.executable, + "standalone.py", + "--host", + "127.0.0.1", + "--port", + str(args.port), + ] + + if args.detach: + # Fully detached launch: new session (setsid), no controlling terminal, + # stdin from /dev/null, stdout/stderr to a log file. Survives the shell. + log_dir = os.path.join(script_dir, "logs") + os.makedirs(log_dir, exist_ok=True) + log_path = os.path.join(log_dir, f"server-{args.port}.log") + with open(log_path, "ab") as log_fh: + process = subprocess.Popen( + cmd, + cwd=project_root, + stdin=subprocess.DEVNULL, + stdout=log_fh, + stderr=subprocess.STDOUT, + start_new_session=True, + close_fds=True, + ) + print(f"Detached server process started with PID {process.pid} (setsid)") + print(f"Log: {log_path}") + else: + # Plain background process (legacy behavior): dies with the shell. + process = subprocess.Popen( + cmd, + cwd=project_root, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + ) + print(f"Server process started with PID {process.pid}") + print( + "NOTE: not detached — this process dies when the launching shell exits. " + "For E2E use --detach." + ) + + write_managed_pids(args.port, [process.pid]) + # Wait for ready if requested if args.wait: print(f"Waiting for server to be ready (timeout: {args.timeout}s)...") if wait_for_server(args.port, args.timeout): print(f"Server ready at http://127.0.0.1:{args.port}/loras") return 0 - else: - print(f"Timeout waiting for server") - return 1 - + print(f"Timeout waiting for server on port {args.port}") + return 1 + print(f"Server starting at http://127.0.0.1:{args.port}/loras") return 0 diff --git a/.agents/skills/lora-manager-e2e/scripts/wait_for_server.py b/.agents/skills/lora-manager-e2e/scripts/wait_for_server.py index 170ff684..af988227 100755 --- a/.agents/skills/lora-manager-e2e/scripts/wait_for_server.py +++ b/.agents/skills/lora-manager-e2e/scripts/wait_for_server.py @@ -1,15 +1,20 @@ #!/usr/bin/env python3 """ Wait for LoRa Manager server to become ready. + +Timeout is configurable via --timeout (default 30s); the script polls the port +until the server accepts connections or the timeout expires. """ +from __future__ import annotations + import argparse import socket import sys import time -def is_server_ready(port: int, timeout: float = 0.5) -> bool: +def is_server_ready(port: int, timeout: float = 2.0) -> bool: """Check if server is accepting connections.""" try: with socket.create_connection(("127.0.0.1", port), timeout=timeout): @@ -21,9 +26,15 @@ def is_server_ready(port: int, timeout: float = 0.5) -> bool: def wait_for_server(port: int, timeout: int = 30) -> bool: """Wait for server to become ready.""" start = time.time() + last_report = 0.0 while time.time() - start < timeout: if is_server_ready(port): return True + # Report progress every ~5s so a slow boot is visible, not silent. + elapsed = time.time() - start + if elapsed - last_report >= 5: + print(f" ...still waiting ({int(elapsed)}s/{timeout}s)") + last_report = elapsed time.sleep(0.5) return False @@ -36,25 +47,24 @@ def main() -> int: "--port", type=int, default=8188, - help="Server port (default: 8188)" + help="Server port (default: 8188)", ) parser.add_argument( "--timeout", type=int, default=30, - help="Timeout in seconds (default: 30)" + help="Timeout in seconds (default: 30)", ) - + args = parser.parse_args() - + print(f"Waiting for server on port {args.port} (timeout: {args.timeout}s)...") - + if wait_for_server(args.port, args.timeout): print(f"Server ready at http://127.0.0.1:{args.port}/loras") return 0 - else: - print(f"Timeout: Server not ready after {args.timeout}s") - return 1 + print(f"Timeout: Server not ready after {args.timeout}s") + return 1 if __name__ == "__main__":