From 38b2deba6b814a4ee4fe4586cca123138f61bdc3 Mon Sep 17 00:00:00 2001 From: Glauber Ferreira Date: Fri, 17 Jul 2026 01:44:47 -0300 Subject: [PATCH] primeiro commit --- .gitignore | 4 + Makefile | 39 + README.md | 98 ++ cmd/filedb/main.go | 2134 ++++++++++++++++++++++++++++++++++++++++++++ go.mod | 23 + go.sum | 58 ++ 6 files changed, 2356 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.md create mode 100644 cmd/filedb/main.go create mode 100644 go.mod create mode 100644 go.sum diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..44693ea --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +bin/ +.agents +.codex +/filedb diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bc0b042 --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ +APP := filedb +CMD := ./cmd/filedb +BUILD_DIR := bin +BIN := $(BUILD_DIR)/$(APP) + +.PHONY: help build run install fmt tidy test clean + +help: + @echo "Targets:" + @echo " make build Build binary to $(BIN)" + @echo " make run ARGS='...' Run CLI with optional ARGS" + @echo " make install Install CLI to GOPATH/bin" + @echo " make fmt Format Go code" + @echo " make tidy Tidy go modules" + @echo " make test Run tests" + @echo " make clean Remove build artifacts" + +build: + @mkdir -p $(BUILD_DIR) + go build -o $(BIN) $(CMD) + +run: + go run $(CMD) $(ARGS) + +install: + go install $(CMD) + +fmt: + gofmt -w $$(find . -type f -name '*.go' -not -path './vendor/*') + +tidy: + go mod tidy + +test: + go test ./... + +clean: + rm -rf $(BUILD_DIR) + diff --git a/README.md b/README.md new file mode 100644 index 0000000..afe3d77 --- /dev/null +++ b/README.md @@ -0,0 +1,98 @@ +# filedb + +`filedb` is a Go CLI utility for indexing file metadata across multiple libraries and collections. + +- Library: logical grouping of collections, persisted as its own DuckDB database file. +- Collection: a backup-like dataset backed by either: + - `fs`: local filesystem path + - `rclone`: `rclone lsjson` listing for a remote/path + +Checks are informational only. The tool does not modify or delete files. + +## Storage + +- Default application directory: `~/.local/filedb` +- Library databases: `~/.local/filedb/libraries/*.duckdb` (one database per library) +- Override app directory with: `--app-dir ` + +## Features + +- Create/list libraries +- Create/list collections inside libraries +- Delete libraries or collections +- Move a collection between libraries +- Update one collection or a full library with one operation (file set + metadata only; no MD5 processing) +- Update MD5 hashes with dedicated `update-hashes` command +- Store metadata: size, mod time, create time (if available), MD5 (if possible) +- Compare indexed metadata with current live files (`compare-live`) +- Offline duplicate checking inside indexed collection (`compare-offline`) +- Store reports inside each library DB by default +- Optional report export to JSON or HTML and optional no-store mode + +## Build + +```bash +go build -o filedb ./cmd/filedb +``` + +## Usage + +```bash +filedb library-create --name archive +filedb library-delete --name old-archive +filedb collection-create --library archive --name tape-001 --type fs --source /data/tape-001 +filedb collection-create --library archive --name remote-photos --type rclone --source myremote:photos +filedb collection-create --library archive --name remote-root --type rclone --source myremote: +filedb collection-delete --library archive --name tape-001 +filedb collection-move --library archive --name remote-photos --target-library cold-storage + +filedb update --library archive --collection tape-001 +filedb update --library archive + +filedb update-hashes --library archive --collection tape-001 +filedb update-hashes --library archive --force-all +filedb update-hashes --library archive --modified-only +filedb update-hashes --library archive --collection tape-001 --file path/in/collection/file.iso +filedb update-hashes --library archive --collection tape-001 --file-list /tmp/files.txt + +filedb compare-live --library archive --metadata size,md5 --collection tape-001 +filedb compare-live --library archive --metadata size,md5 +filedb compare-offline --library archive --metadata filename,size,md5,mtime --collection tape-001 --collection tape-002 + +filedb compare-offline --library archive --metadata filename,size,md5 --no-store-report --export html --output report.html + +filedb report-list +filedb report-list --library archive +filedb report-show --id --library archive +filedb report-export --id --library archive --format json --output report.json +``` + +With custom app directory: + +```bash +filedb --app-dir /tmp/filedb-data library-list +``` + +## Metadata Fields + +For comparison keys (`--metadata`): + +- `filename` (offline only; not allowed in `compare-live`) +- `size` +- `mtime` +- `ctime` +- `md5` + +Example: + +```bash +filedb compare-offline --library archive --metadata filename,size,mtime +``` + +## Notes + +- `update` does not compute hashes. Use `update-hashes` for MD5 processing. +- `ctime` availability is OS/filesystem dependent. +- Rclone collections require `rclone` in `PATH` (validated at collection creation and indexing). +- `collection-create --source` is validated at create time: filesystem sources must be existing directories, and rclone sources must use `remote:` or `remote:path` syntax. +- `update` skips disconnected/unreachable filesystem sources and rclone indexing failures, and continues with other collections. diff --git a/cmd/filedb/main.go b/cmd/filedb/main.go new file mode 100644 index 0000000..4e848b4 --- /dev/null +++ b/cmd/filedb/main.go @@ -0,0 +1,2134 @@ +package main + +import ( + "crypto/md5" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "html/template" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "syscall" + "time" + + _ "github.com/marcboeker/go-duckdb" +) + +type Collection struct { + Name string + Type string + Source string + IndexedAt *time.Time + LastIndexError string +} + +type FileEntry struct { + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + ModTime *time.Time `json:"mod_time,omitempty"` + CTime *time.Time `json:"create_time,omitempty"` + MD5 string `json:"md5,omitempty"` + HashSize *int64 `json:"hash_size,omitempty"` + HashModTime *time.Time `json:"hash_mod_time,omitempty"` + HashUpdatedAt *time.Time `json:"hash_updated_at,omitempty"` +} + +type Report struct { + ID string `json:"id"` + Kind string `json:"kind"` + CreatedAt time.Time `json:"created_at"` + Library string `json:"library"` + Collection string `json:"collection"` + Fields []string `json:"fields,omitempty"` + Summary map[string]int `json:"summary"` + Rows []map[string]any `json:"rows"` + Meta map[string]string `json:"meta,omitempty"` +} + +type compareCfg struct { + Library string + Collections []string + Metadata []string + NoStore bool + ExportFormat string + Output string +} + +type hashUpdateCfg struct { + Library string + Collections []string + ForceAll bool + ModifiedOnly bool + File string + FileList string +} + +type stringListFlag []string + +func (s *stringListFlag) String() string { + return strings.Join(*s, ",") +} + +func (s *stringListFlag) Set(value string) error { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + for _, p := range strings.Split(value, ",") { + p = strings.TrimSpace(p) + if p != "" { + *s = append(*s, p) + } + } + return nil +} + +var showFileRead bool + +func infof(format string, args ...any) { + fmt.Printf("[info] "+format+"\n", args...) +} + +func main() { + root := flag.NewFlagSet("filedb", flag.ContinueOnError) + root.Usage = usage + appDirFlag := root.String("app-dir", "", "Application data directory (default: ~/.local/filedb)") + showFileReadFlag := root.Bool("show-file-read", false, "Show each file path when a local file is read") + helpFlag := root.Bool("help", false, "Show help") + root.BoolVar(helpFlag, "h", false, "Show help") + if err := root.Parse(os.Args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + usage() + return + } + os.Exit(2) + } + args := root.Args() + if *helpFlag { + usage() + return + } + if len(args) == 0 { + usage() + os.Exit(1) + } + + appDir, err := resolveAppDir(*appDirFlag) + if err != nil { + fmt.Fprintf(os.Stderr, "resolve app dir: %v\n", err) + os.Exit(1) + } + if err := ensureAppLayout(appDir); err != nil { + fmt.Fprintf(os.Stderr, "init app dir: %v\n", err) + os.Exit(1) + } + showFileRead = *showFileReadFlag + infof("app-dir=%s", appDir) + infof("show-file-read=%t", showFileRead) + + cmd := args[0] + rest := args[1:] + infof("command=%s args=%v", cmd, rest) + + switch cmd { + case "library-create": + err = cmdLibraryCreate(appDir, rest) + case "library-delete": + err = cmdLibraryDelete(appDir, rest) + case "library-list": + err = cmdLibraryList(appDir) + case "collection-create": + err = cmdCollectionCreate(appDir, rest) + case "collection-delete": + err = cmdCollectionDelete(appDir, rest) + case "collection-list": + err = cmdCollectionList(appDir, rest) + case "collection-move": + err = cmdCollectionMove(appDir, rest) + case "update", "index", "update-library": + err = cmdUpdate(appDir, rest) + case "compare-live": + err = cmdCompareLive(appDir, rest) + case "compare-offline": + err = cmdCompareOffline(appDir, rest) + case "update-hashes": + err = cmdUpdateHashes(appDir, rest) + case "report-list": + err = cmdReportList(appDir, rest) + case "report-show": + err = cmdReportShow(appDir, rest) + case "report-export": + err = cmdReportExport(appDir, rest) + case "help": + usage() + return + default: + err = fmt.Errorf("unknown command: %s", cmd) + } + + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +func usage() { + fmt.Print(`filedb - file metadata database utility + +Commands: + library-create --name + library-delete --name + library-list + collection-create --library --name --type --source + collection-delete --library --name + collection-list --library + collection-move --library --name --target-library + update --library [--collection ] + update-hashes --library [--collection ] [--force-all] [--modified-only] [--file ] [--file-list ] + compare-live --library --metadata [--collection ] [--no-store-report] [--export json|html] [--output file] + compare-offline --library --metadata [--collection ] [--no-store-report] [--export json|html] [--output file] + report-list [--library ] + report-show --id [--library ] + report-export --id --format json|html --output [--library ] + +Global flags: + -app-dir, --app-dir Application data directory (default: ~/.local/filedb) + -show-file-read Show each file path when a local file is read + -h, --help Show this help text +`) +} + +func resolveAppDir(override string) (string, error) { + if strings.TrimSpace(override) != "" { + return override, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, ".local", "filedb"), nil +} + +func ensureAppLayout(appDir string) error { + return os.MkdirAll(filepath.Join(appDir, "libraries"), 0o755) +} + +func libraryPath(appDir, name string) string { + h := md5.Sum([]byte(name)) + hash := hex.EncodeToString(h[:])[:12] + base := strings.ToLower(strings.TrimSpace(name)) + base = strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + return r + } + if r == '-' || r == '_' { + return r + } + return '_' + }, base) + if base == "" { + base = "library" + } + if len(base) > 40 { + base = base[:40] + } + return filepath.Join(appDir, "libraries", fmt.Sprintf("%s-%s.duckdb", base, hash)) +} + +func openLibraryDB(appDir, name string) (*sql.DB, error) { + path := libraryPath(appDir, name) + if _, err := os.Stat(path); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("library not found: %s", name) + } + return nil, err + } + db, err := sql.Open("duckdb", path) + if err != nil { + return nil, err + } + if err := ensureSchema(db); err != nil { + _ = db.Close() + return nil, err + } + return db, nil +} + +func createLibraryDB(appDir, name string) error { + path := libraryPath(appDir, name) + if _, err := os.Stat(path); err == nil { + return fmt.Errorf("library already exists: %s", name) + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + db, err := sql.Open("duckdb", path) + if err != nil { + return err + } + defer db.Close() + if err := ensureSchema(db); err != nil { + return err + } + now := time.Now().UTC().Format(time.RFC3339Nano) + _, err = db.Exec(`INSERT INTO library_info(name, created_at, updated_at) VALUES (?, ?, ?)`, name, now, now) + return err +} + +func ensureSchema(db *sql.DB) error { + stmts := []string{ + `CREATE TABLE IF NOT EXISTS library_info(name TEXT PRIMARY KEY, created_at TEXT, updated_at TEXT)`, + `CREATE TABLE IF NOT EXISTS collections(name TEXT PRIMARY KEY, type TEXT, source TEXT, indexed_at TEXT, last_index_error TEXT)`, + `CREATE TABLE IF NOT EXISTS files(collection_name TEXT, path TEXT, name TEXT, size BIGINT, mod_time TEXT, ctime TEXT, md5 TEXT, hash_size BIGINT, hash_mod_time TEXT, hash_updated_at TEXT, PRIMARY KEY(collection_name, path))`, + `CREATE TABLE IF NOT EXISTS reports(id TEXT PRIMARY KEY, kind TEXT, created_at TEXT, library_name TEXT, collection_name TEXT, fields_json TEXT, summary_json TEXT, rows_json TEXT, meta_json TEXT)`, + `ALTER TABLE files ADD COLUMN IF NOT EXISTS hash_size BIGINT`, + `ALTER TABLE files ADD COLUMN IF NOT EXISTS hash_mod_time TEXT`, + `ALTER TABLE files ADD COLUMN IF NOT EXISTS hash_updated_at TEXT`, + } + for _, s := range stmts { + if _, err := db.Exec(s); err != nil { + return err + } + } + return nil +} + +func readLibraryInfo(db *sql.DB) (name string, collections int, updated time.Time, err error) { + var updatedStr string + err = db.QueryRow(`SELECT name, updated_at FROM library_info LIMIT 1`).Scan(&name, &updatedStr) + if err != nil { + return "", 0, time.Time{}, err + } + if updatedStr != "" { + updated, _ = time.Parse(time.RFC3339Nano, updatedStr) + } + err = db.QueryRow(`SELECT COUNT(*) FROM collections`).Scan(&collections) + if err != nil { + return "", 0, time.Time{}, err + } + return name, collections, updated, nil +} + +func touchLibrary(db *sql.DB) error { + now := time.Now().UTC().Format(time.RFC3339Nano) + _, err := db.Exec(`UPDATE library_info SET updated_at = ?`, now) + return err +} + +func listLibraryFiles(appDir string) ([]string, error) { + entries, err := os.ReadDir(filepath.Join(appDir, "libraries")) + if err != nil { + return nil, err + } + var out []string + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".duckdb") { + continue + } + out = append(out, filepath.Join(appDir, "libraries", e.Name())) + } + sort.Strings(out) + return out, nil +} + +func cmdLibraryCreate(appDir string, args []string) error { + fs := flag.NewFlagSet("library-create", flag.ContinueOnError) + name := fs.String("name", "", "Library name") + if err := fs.Parse(args); err != nil { + return err + } + if strings.TrimSpace(*name) == "" { + return errors.New("--name is required") + } + infof("creating library name=%q db=%s", *name, libraryPath(appDir, *name)) + if err := createLibraryDB(appDir, *name); err != nil { + return err + } + fmt.Printf("created library %q\n", *name) + infof("library created successfully") + return nil +} + +func cmdLibraryDelete(appDir string, args []string) error { + fs := flag.NewFlagSet("library-delete", flag.ContinueOnError) + name := fs.String("name", "", "Library name") + if err := fs.Parse(args); err != nil { + return err + } + if strings.TrimSpace(*name) == "" { + return errors.New("--name is required") + } + path := libraryPath(appDir, *name) + if _, err := os.Stat(path); err != nil { + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("library not found: %s", *name) + } + return err + } + infof("deleting library name=%q db=%s", *name, path) + if err := os.Remove(path); err != nil { + return err + } + if err := os.Remove(path + ".wal"); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + fmt.Printf("deleted library %q\n", *name) + infof("library deleted successfully") + return nil +} + +func cmdLibraryList(appDir string) error { + infof("listing libraries in %s", filepath.Join(appDir, "libraries")) + paths, err := listLibraryFiles(appDir) + if err != nil { + return err + } + if len(paths) == 0 { + fmt.Println("no libraries") + return nil + } + type row struct { + name string + collections int + updated time.Time + } + var rows []row + for _, path := range paths { + infof("reading library db=%s", path) + db, err := sql.Open("duckdb", path) + if err != nil { + continue + } + _ = ensureSchema(db) + name, count, updated, err := readLibraryInfo(db) + _ = db.Close() + if err != nil { + continue + } + rows = append(rows, row{name: name, collections: count, updated: updated}) + } + sort.Slice(rows, func(i, j int) bool { return rows[i].name < rows[j].name }) + for _, r := range rows { + fmt.Printf("%s collections=%d updated=%s\n", r.name, r.collections, r.updated.Format(time.RFC3339)) + } + if len(rows) == 0 { + fmt.Println("no libraries") + } + infof("libraries listed=%d", len(rows)) + return nil +} + +func cmdCollectionCreate(appDir string, args []string) error { + fs := flag.NewFlagSet("collection-create", flag.ContinueOnError) + library := fs.String("library", "", "Library name") + name := fs.String("name", "", "Collection name") + ctype := fs.String("type", "", "Collection type: fs or rclone") + source := fs.String("source", "", "Filesystem path or rclone remote:path") + remote := fs.String("remote", "", "Deprecated: use --source ") + remotePath := fs.String("remote-path", "", "Deprecated: use --source ") + if err := fs.Parse(args); err != nil { + return err + } + if *library == "" || *name == "" || *ctype == "" { + return errors.New("--library --name and --type are required") + } + if *ctype != "fs" && *ctype != "rclone" { + return errors.New("--type must be fs or rclone") + } + finalSource, err := resolveCollectionSource(*ctype, *source, *remote, *remotePath) + if err != nil { + return err + } + infof("creating collection library=%q name=%q type=%q source=%q", *library, *name, *ctype, finalSource) + db, err := openLibraryDB(appDir, *library) + if err != nil { + return err + } + defer db.Close() + _, err = db.Exec(`INSERT INTO collections(name, type, source, indexed_at, last_index_error) VALUES (?, ?, ?, '', '')`, *name, *ctype, finalSource) + if err != nil { + return err + } + if err := touchLibrary(db); err != nil { + return err + } + fmt.Printf("created collection %q in library %q\n", *name, *library) + infof("collection created successfully") + return nil +} + +func cmdCollectionList(appDir string, args []string) error { + fs := flag.NewFlagSet("collection-list", flag.ContinueOnError) + library := fs.String("library", "", "Library name") + if err := fs.Parse(args); err != nil { + return err + } + if *library == "" { + return errors.New("--library is required") + } + infof("listing collections for library=%q", *library) + db, err := openLibraryDB(appDir, *library) + if err != nil { + return err + } + defer db.Close() + rows, err := db.Query(`SELECT name, type, source, indexed_at, last_index_error FROM collections ORDER BY name`) + if err != nil { + return err + } + defer rows.Close() + count := 0 + for rows.Next() { + count++ + var name, ctype, source, indexed, idxErr string + if err := rows.Scan(&name, &ctype, &source, &indexed, &idxErr); err != nil { + return err + } + if indexed == "" { + indexed = "never" + } + fileCount, _ := countCollectionFiles(db, name) + fmt.Printf("%s type=%s source=%q files=%d indexed=%s\n", name, ctype, source, fileCount, indexed) + } + if err := rows.Err(); err != nil { + return err + } + if count == 0 { + fmt.Println("no collections") + } + infof("collections listed=%d", count) + return nil +} + +func cmdCollectionDelete(appDir string, args []string) error { + fs := flag.NewFlagSet("collection-delete", flag.ContinueOnError) + library := fs.String("library", "", "Library name") + name := fs.String("name", "", "Collection name") + if err := fs.Parse(args); err != nil { + return err + } + if strings.TrimSpace(*library) == "" || strings.TrimSpace(*name) == "" { + return errors.New("--library and --name are required") + } + infof("deleting collection library=%q name=%q", *library, *name) + db, err := openLibraryDB(appDir, *library) + if err != nil { + return err + } + defer db.Close() + if _, err := getCollection(db, *name); err != nil { + return err + } + fileCount, _ := countCollectionFiles(db, *name) + if err := deleteCollection(db, *name); err != nil { + return err + } + if err := touchLibrary(db); err != nil { + return err + } + fmt.Printf("deleted collection %q from library %q (removed %d indexed files)\n", *name, *library, fileCount) + infof("collection deleted successfully files=%d", fileCount) + return nil +} + +func cmdCollectionMove(appDir string, args []string) error { + fs := flag.NewFlagSet("collection-move", flag.ContinueOnError) + library := fs.String("library", "", "Source library name") + name := fs.String("name", "", "Collection name") + targetLibrary := fs.String("target-library", "", "Target library name") + fs.StringVar(targetLibrary, "to-library", "", "Target library name") + if err := fs.Parse(args); err != nil { + return err + } + if strings.TrimSpace(*library) == "" || strings.TrimSpace(*name) == "" || strings.TrimSpace(*targetLibrary) == "" { + return errors.New("--library --name and --target-library are required") + } + if *library == *targetLibrary { + return errors.New("--target-library must be different from --library") + } + infof("moving collection name=%q from library=%q to library=%q", *name, *library, *targetLibrary) + srcDB, err := openLibraryDB(appDir, *library) + if err != nil { + return err + } + defer srcDB.Close() + dstDB, err := openLibraryDB(appDir, *targetLibrary) + if err != nil { + return err + } + defer dstDB.Close() + + col, err := getCollection(srcDB, *name) + if err != nil { + return err + } + if _, err := getCollection(dstDB, *name); err == nil { + return fmt.Errorf("collection already exists in target library %q: %s", *targetLibrary, *name) + } else if !strings.Contains(err.Error(), "collection not found:") { + return err + } + files, err := loadCollectionFiles(srcDB, *name) + if err != nil { + return err + } + if err := insertCollectionWithFiles(dstDB, col, files); err != nil { + return err + } + if err := touchLibrary(dstDB); err != nil { + return err + } + if err := deleteCollection(srcDB, *name); err != nil { + return err + } + if err := touchLibrary(srcDB); err != nil { + return err + } + fmt.Printf("moved collection %q from library %q to %q (%d indexed files)\n", *name, *library, *targetLibrary, len(files)) + infof("collection moved successfully files=%d", len(files)) + return nil +} + +func cmdUpdate(appDir string, args []string) error { + fs := flag.NewFlagSet("update", flag.ContinueOnError) + library := fs.String("library", "", "Library name") + collection := fs.String("collection", "", "Collection name (optional)") + if err := fs.Parse(args); err != nil { + return err + } + if *library == "" { + return errors.New("--library is required") + } + infof("update request library=%q collection=%q", *library, strings.TrimSpace(*collection)) + db, err := openLibraryDB(appDir, *library) + if err != nil { + return err + } + defer db.Close() + + if strings.TrimSpace(*collection) != "" { + col, err := getCollection(db, *collection) + if err != nil { + return err + } + infof("updating single collection name=%q type=%q source=%q", col.Name, col.Type, col.Source) + status, count, detail, err := updateCollection(db, col) + if err != nil { + return err + } + if err := touchLibrary(db); err != nil { + return err + } + if status == "updated" { + fmt.Printf("updated %s/%s (%d files)\n", *library, col.Name, count) + infof("single collection update completed status=updated files=%d", count) + } else { + fmt.Printf("skipped %s/%s: %s\n", *library, col.Name, detail) + infof("single collection update completed status=skipped reason=%q", detail) + } + return nil + } + + rows, err := db.Query(`SELECT name, type, source, indexed_at, last_index_error FROM collections ORDER BY name`) + if err != nil { + return err + } + defer rows.Close() + + var failed []string + updated := 0 + skipped := 0 + for rows.Next() { + col, err := scanCollection(rows) + if err != nil { + return err + } + infof("processing collection name=%q type=%q source=%q", col.Name, col.Type, col.Source) + status, count, detail, err := updateCollection(db, col) + if err != nil { + failed = append(failed, fmt.Sprintf("%s: %v", col.Name, err)) + continue + } + if status == "updated" { + updated++ + fmt.Printf("updated %s (%d files)\n", col.Name, count) + } else { + skipped++ + fmt.Printf("skipped %s: %s\n", col.Name, detail) + } + } + if err := rows.Err(); err != nil { + return err + } + if err := touchLibrary(db); err != nil { + return err + } + fmt.Printf("update summary: updated=%d skipped=%d failed=%d\n", updated, skipped, len(failed)) + infof("update completed library=%q updated=%d skipped=%d failed=%d", *library, updated, skipped, len(failed)) + if len(failed) > 0 { + return fmt.Errorf("update completed with errors: %s", strings.Join(failed, "; ")) + } + return nil +} + +func cmdUpdateHashes(appDir string, args []string) error { + cfg, err := parseHashUpdateFlags(args) + if err != nil { + return err + } + infof("update-hashes library=%q collections=%v force-all=%t modified-only=%t file=%q file-list=%q", cfg.Library, cfg.Collections, cfg.ForceAll, cfg.ModifiedOnly, cfg.File, cfg.FileList) + db, err := openLibraryDB(appDir, cfg.Library) + if err != nil { + return err + } + defer db.Close() + + cols, err := resolveTargetCollections(db, cfg.Collections) + if err != nil { + return err + } + if len(cols) == 0 { + return errors.New("no collections selected") + } + + targetSet, err := loadTargetFileSet(cfg.File, cfg.FileList) + if err != nil { + return err + } + if len(targetSet) > 0 && len(cols) != 1 { + return errors.New("--file/--file-list requires exactly one --collection") + } + + totalHashed := 0 + totalFailed := 0 + totalSkipped := 0 + for _, col := range cols { + hashed, failed, skipped, err := updateCollectionHashes(db, col, cfg, targetSet) + if err != nil { + return err + } + totalHashed += hashed + totalFailed += failed + totalSkipped += skipped + fmt.Printf("hash summary %s: hashed=%d failed=%d skipped=%d\n", col.Name, hashed, failed, skipped) + } + if err := touchLibrary(db); err != nil { + return err + } + fmt.Printf("hash update summary: hashed=%d failed=%d skipped=%d\n", totalHashed, totalFailed, totalSkipped) + return nil +} + +func parseHashUpdateFlags(args []string) (hashUpdateCfg, error) { + fs := flag.NewFlagSet("update-hashes", flag.ContinueOnError) + library := fs.String("library", "", "Library name") + var collections stringListFlag + fs.Var(&collections, "collection", "Collection name (can be repeated or comma-separated). If omitted, all collections are used") + forceAll := fs.Bool("force-all", false, "Force update all hashes") + modifiedOnly := fs.Bool("modified-only", false, "Update hashes only for files modified since hash was last stored (size or mtime changed)") + filePath := fs.String("file", "", "Force update hash for a single file path in the selected collection") + fileList := fs.String("file-list", "", "Force update hashes for file paths listed in a text file (one path per line)") + if err := fs.Parse(args); err != nil { + return hashUpdateCfg{}, err + } + if *library == "" { + return hashUpdateCfg{}, errors.New("--library is required") + } + if *filePath != "" && *fileList != "" { + return hashUpdateCfg{}, errors.New("--file and --file-list are mutually exclusive") + } + modeCount := 0 + if *forceAll { + modeCount++ + } + if *modifiedOnly { + modeCount++ + } + if *filePath != "" || *fileList != "" { + modeCount++ + } + if modeCount > 1 { + return hashUpdateCfg{}, errors.New("choose only one mode: default(missing hashes), --force-all, --modified-only, or --file/--file-list") + } + return hashUpdateCfg{ + Library: *library, + Collections: collections, + ForceAll: *forceAll, + ModifiedOnly: *modifiedOnly, + File: strings.TrimSpace(*filePath), + FileList: strings.TrimSpace(*fileList), + }, nil +} + +func loadTargetFileSet(single, listPath string) (map[string]bool, error) { + out := map[string]bool{} + if strings.TrimSpace(single) != "" { + out[normalizeStoredPath(single)] = true + return out, nil + } + if strings.TrimSpace(listPath) == "" { + return out, nil + } + data, err := os.ReadFile(listPath) + if err != nil { + return nil, err + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + out[normalizeStoredPath(line)] = true + } + return out, nil +} + +func normalizeStoredPath(p string) string { + return filepath.ToSlash(strings.TrimSpace(p)) +} + +func updateCollectionHashes(db *sql.DB, col Collection, cfg hashUpdateCfg, targetSet map[string]bool) (hashed, failed, skipped int, err error) { + files, err := loadCollectionFiles(db, col.Name) + if err != nil { + return 0, 0, 0, err + } + if len(files) == 0 { + return 0, 0, 0, nil + } + infof("updating hashes collection=%q type=%q files=%d", col.Name, col.Type, len(files)) + + targets := selectHashTargets(files, cfg, targetSet) + if len(targets) == 0 { + return 0, 0, len(files), nil + } + + rcloneHashes := map[string]string{} + if col.Type == "rclone" { + if err := requireRclone(); err != nil { + return 0, 0, 0, err + } + rcloneHashes, err = fetchRcloneMD5Map(col.Source) + if err != nil { + return 0, 0, 0, err + } + } + + now := time.Now().UTC() + for i := range files { + f := &files[i] + if !targets[f.Path] { + skipped++ + continue + } + var hash string + switch col.Type { + case "fs": + abs := filepath.Join(col.Source, filepath.FromSlash(f.Path)) + hash, err = md5File(abs) + if err != nil { + failed++ + infof("hash failed collection=%q file=%q err=%v", col.Name, f.Path, err) + continue + } + case "rclone": + hash = strings.ToLower(strings.TrimSpace(rcloneHashes[f.Path])) + if hash == "" { + failed++ + infof("hash missing from rclone output collection=%q file=%q", col.Name, f.Path) + continue + } + default: + return 0, 0, 0, fmt.Errorf("unsupported collection type: %s", col.Type) + } + f.MD5 = hash + size := f.Size + f.HashSize = &size + if f.ModTime != nil { + mt := f.ModTime.UTC() + f.HashModTime = &mt + } else { + f.HashModTime = nil + } + ts := now + f.HashUpdatedAt = &ts + hashed++ + } + + if err := replaceCollectionFiles(db, col.Name, files); err != nil { + return 0, 0, 0, err + } + return hashed, failed, skipped, nil +} + +func selectHashTargets(files []FileEntry, cfg hashUpdateCfg, targetSet map[string]bool) map[string]bool { + targets := make(map[string]bool, len(files)) + if len(targetSet) > 0 { + for _, f := range files { + if targetSet[f.Path] { + targets[f.Path] = true + } + } + return targets + } + for _, f := range files { + switch { + case cfg.ForceAll: + targets[f.Path] = true + case cfg.ModifiedOnly: + if fileModifiedSinceLastHash(f) { + targets[f.Path] = true + } + default: + if strings.TrimSpace(f.MD5) == "" { + targets[f.Path] = true + } + } + } + return targets +} + +func fileModifiedSinceLastHash(f FileEntry) bool { + if f.HashSize == nil || f.HashModTime == nil { + return true + } + if *f.HashSize != f.Size { + return true + } + return !timesEqual(f.HashModTime, f.ModTime) +} + +func fetchRcloneMD5Map(source string) (map[string]string, error) { + cmd := exec.Command("rclone", "lsjson", "--recursive", "--hash", source) + out, err := cmd.CombinedOutput() + if err != nil { + msg := strings.TrimSpace(string(out)) + if msg == "" { + return nil, fmt.Errorf("rclone lsjson failed for %s: %w", source, err) + } + return nil, fmt.Errorf("rclone lsjson failed for %s: %s", source, msg) + } + type ritem struct { + Path string `json:"Path"` + IsDir bool `json:"IsDir"` + Hashes map[string]string `json:"Hashes"` + } + var items []ritem + if err := json.Unmarshal(out, &items); err != nil { + return nil, err + } + m := make(map[string]string, len(items)) + for _, it := range items { + if it.IsDir { + continue + } + if v, ok := it.Hashes["MD5"]; ok { + m[filepath.ToSlash(it.Path)] = strings.ToLower(v) + } + } + return m, nil +} + +func updateCollection(db *sql.DB, col Collection) (status string, fileCount int, detail string, err error) { + existing, err := loadCollectionFiles(db, col.Name) + if err != nil { + return "", 0, "", err + } + files, idxErr := indexCollection(col) + now := time.Now().UTC().Format(time.RFC3339Nano) + if idxErr != nil { + _, _ = db.Exec(`UPDATE collections SET indexed_at = ?, last_index_error = ? WHERE name = ?`, now, idxErr.Error(), col.Name) + if shouldSkipCollectionError(col, idxErr) { + return "skipped", 0, idxErr.Error(), nil + } + return "", 0, "", idxErr + } + merged := mergeScannedFilesWithExisting(files, existing) + if err := replaceCollectionFiles(db, col.Name, merged); err != nil { + return "", 0, "", err + } + if _, err := db.Exec(`UPDATE collections SET indexed_at = ?, last_index_error = '' WHERE name = ?`, now, col.Name); err != nil { + return "", 0, "", err + } + return "updated", len(merged), "", nil +} + +func mergeScannedFilesWithExisting(scanned, existing []FileEntry) []FileEntry { + oldByPath := make(map[string]FileEntry, len(existing)) + for _, f := range existing { + oldByPath[f.Path] = f + } + out := make([]FileEntry, 0, len(scanned)) + for _, cur := range scanned { + old, ok := oldByPath[cur.Path] + if !ok { + out = append(out, cur) + continue + } + if old.Name == cur.Name && old.Size == cur.Size && timesEqual(old.ModTime, cur.ModTime) && timesEqual(old.CTime, cur.CTime) { + cur.MD5 = old.MD5 + cur.HashSize = old.HashSize + cur.HashModTime = old.HashModTime + cur.HashUpdatedAt = old.HashUpdatedAt + } + out = append(out, cur) + } + return out +} + +func timesEqual(a, b *time.Time) bool { + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + return a.UTC().Equal(b.UTC()) +} + +func shouldSkipCollectionError(col Collection, err error) bool { + if col.Type == "rclone" { + return true + } + if col.Type == "fs" { + return isFilesystemUnavailableError(err) + } + return false +} + +func isFilesystemUnavailableError(err error) bool { + if errors.Is(err, os.ErrNotExist) || + errors.Is(err, syscall.ENOTCONN) || + errors.Is(err, syscall.EHOSTDOWN) || + errors.Is(err, syscall.ENETDOWN) || + errors.Is(err, syscall.ENETUNREACH) || + errors.Is(err, syscall.ETIMEDOUT) || + errors.Is(err, syscall.ECONNREFUSED) || + errors.Is(err, syscall.ESTALE) { + return true + } + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "transport endpoint is not connected") || + strings.Contains(msg, "network is unreachable") || + strings.Contains(msg, "host is down") || + strings.Contains(msg, "connection timed out") || + strings.Contains(msg, "stale file handle") || + strings.Contains(msg, "no such file or directory") { + return true + } + return false +} + +func cmdCompareLive(appDir string, args []string) error { + cfg, err := parseCompareFlags("compare-live", args) + if err != nil { + return err + } + infof("compare-live library=%q collections=%v metadata=%v", cfg.Library, cfg.Collections, cfg.Metadata) + db, err := openLibraryDB(appDir, cfg.Library) + if err != nil { + return err + } + defer db.Close() + cols, err := resolveTargetCollections(db, cfg.Collections) + if err != nil { + return err + } + if len(cols) == 0 { + return errors.New("no collections selected") + } + report := &Report{ + ID: reportID("compare-live"), + Kind: "compare-live", + CreatedAt: time.Now().UTC(), + Library: cfg.Library, + Collection: joinCollectionNames(cols), + Fields: cfg.Metadata, + Summary: map[string]int{ + "collections_total": len(cols), + "keys_total": 0, + "match": 0, + "missing_live": 0, + "new_live": 0, + "count_mismatch": 0, + }, + Rows: []map[string]any{}, + } + for _, col := range cols { + stored, err := loadCollectionFiles(db, col.Name) + if err != nil { + return err + } + if len(stored) == 0 { + infof("collection=%q skipped in compare-live: no indexed files", col.Name) + continue + } + live, err := indexCollectionForMetadata(col, cfg.Metadata) + if err != nil { + return err + } + sub := compareLive(cfg.Library, col.Name, stored, live, cfg.Metadata) + report.Summary["keys_total"] += sub.Summary["keys_total"] + report.Summary["match"] += sub.Summary["match"] + report.Summary["missing_live"] += sub.Summary["missing_live"] + report.Summary["new_live"] += sub.Summary["new_live"] + report.Summary["count_mismatch"] += sub.Summary["count_mismatch"] + for _, row := range sub.Rows { + row["collection"] = col.Name + report.Rows = append(report.Rows, row) + } + } + if err := finalizeReport(db, report, cfg.NoStore, cfg.ExportFormat, cfg.Output); err != nil { + return err + } + printReportSummary(report) + return nil +} + +func cmdCompareOffline(appDir string, args []string) error { + cfg, err := parseCompareFlags("compare-offline", args) + if err != nil { + return err + } + infof("compare-offline library=%q collections=%v metadata=%v", cfg.Library, cfg.Collections, cfg.Metadata) + db, err := openLibraryDB(appDir, cfg.Library) + if err != nil { + return err + } + defer db.Close() + cols, err := resolveTargetCollections(db, cfg.Collections) + if err != nil { + return err + } + if len(cols) == 0 { + return errors.New("no collections selected") + } + combined := make([]FileEntry, 0) + for _, col := range cols { + stored, err := loadCollectionFiles(db, col.Name) + if err != nil { + return err + } + for _, f := range stored { + f.Path = col.Name + ":" + f.Path + combined = append(combined, f) + } + } + report := compareOffline(cfg.Library, joinCollectionNames(cols), combined, cfg.Metadata) + report.Summary["collections_total"] = len(cols) + if err := finalizeReport(db, report, cfg.NoStore, cfg.ExportFormat, cfg.Output); err != nil { + return err + } + printReportSummary(report) + return nil +} + +func cmdReportList(appDir string, args []string) error { + fs := flag.NewFlagSet("report-list", flag.ContinueOnError) + library := fs.String("library", "", "Library name (optional)") + if err := fs.Parse(args); err != nil { + return err + } + if strings.TrimSpace(*library) != "" { + infof("listing reports for library=%q", *library) + db, err := openLibraryDB(appDir, *library) + if err != nil { + return err + } + defer db.Close() + return listReportsInDB(db) + } + infof("listing reports across all libraries") + libs, err := listLibraries(appDir) + if err != nil { + return err + } + printed := 0 + for _, lib := range libs { + db, err := openLibraryDB(appDir, lib) + if err != nil { + continue + } + before := printed + rows, err := db.Query(`SELECT id, kind, library_name, collection_name, created_at FROM reports ORDER BY created_at`) + if err == nil { + for rows.Next() { + var id, kind, libraryName, collection, created string + if scanErr := rows.Scan(&id, &kind, &libraryName, &collection, &created); scanErr == nil { + fmt.Printf("%s kind=%s lib=%s collection=%s at=%s\n", id, kind, libraryName, collection, created) + printed++ + } + } + _ = rows.Close() + } + _ = db.Close() + if before != printed { + continue + } + } + if printed == 0 { + fmt.Println("no reports") + } + infof("reports listed=%d", printed) + return nil +} + +func cmdReportShow(appDir string, args []string) error { + fs := flag.NewFlagSet("report-show", flag.ContinueOnError) + id := fs.String("id", "", "Report ID") + library := fs.String("library", "", "Library name (optional)") + if err := fs.Parse(args); err != nil { + return err + } + if *id == "" { + return errors.New("--id is required") + } + infof("showing report id=%q library=%q", *id, *library) + report, err := findReport(appDir, *id, *library) + if err != nil { + return err + } + data, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + fmt.Println(string(data)) + infof("report show completed") + return nil +} + +func cmdReportExport(appDir string, args []string) error { + fs := flag.NewFlagSet("report-export", flag.ContinueOnError) + id := fs.String("id", "", "Report ID") + format := fs.String("format", "", "json or html") + output := fs.String("output", "", "Output file") + library := fs.String("library", "", "Library name (optional)") + if err := fs.Parse(args); err != nil { + return err + } + if *id == "" || *format == "" || *output == "" { + return errors.New("--id --format --output are required") + } + infof("exporting report id=%q library=%q format=%q output=%q", *id, *library, *format, *output) + report, err := findReport(appDir, *id, *library) + if err != nil { + return err + } + if err := exportReport(report, *format, *output); err != nil { + return err + } + infof("report export completed") + return nil +} + +func parseCompareFlags(name string, args []string) (compareCfg, error) { + fs := flag.NewFlagSet(name, flag.ContinueOnError) + library := fs.String("library", "", "Library name") + var collections stringListFlag + fs.Var(&collections, "collection", "Collection name (can be repeated or comma-separated). If omitted, all collections are used") + metadata := fs.String("metadata", "", "Required comma-separated metadata fields") + noStore := fs.Bool("no-store-report", false, "Do not store report in DB") + export := fs.String("export", "", "Optional export format: json or html") + output := fs.String("output", "", "Output file for export") + if err := fs.Parse(args); err != nil { + return compareCfg{}, err + } + if *library == "" { + return compareCfg{}, errors.New("--library is required") + } + parsedMetadata, err := normalizeMetadata(*metadata) + if err != nil { + return compareCfg{}, err + } + if name == "compare-live" && containsString(parsedMetadata, "filename") { + return compareCfg{}, errors.New("compare-live does not support metadata field 'filename'") + } + if *export != "" && *output == "" { + return compareCfg{}, errors.New("--output is required when using --export") + } + if *export != "" && *export != "json" && *export != "html" { + return compareCfg{}, errors.New("--export must be json or html") + } + return compareCfg{ + Library: *library, + Collections: collections, + Metadata: parsedMetadata, + NoStore: *noStore, + ExportFormat: *export, + Output: *output, + }, nil +} + +func getCollection(db *sql.DB, name string) (Collection, error) { + row := db.QueryRow(`SELECT name, type, source, indexed_at, last_index_error FROM collections WHERE name = ?`, name) + var c Collection + var indexed string + if err := row.Scan(&c.Name, &c.Type, &c.Source, &indexed, &c.LastIndexError); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return Collection{}, fmt.Errorf("collection not found: %s", name) + } + return Collection{}, err + } + if indexed != "" { + if t, err := time.Parse(time.RFC3339Nano, indexed); err == nil { + c.IndexedAt = &t + } + } + return c, nil +} + +func scanCollection(rows *sql.Rows) (Collection, error) { + var c Collection + var indexed string + if err := rows.Scan(&c.Name, &c.Type, &c.Source, &indexed, &c.LastIndexError); err != nil { + return Collection{}, err + } + if indexed != "" { + if t, err := time.Parse(time.RFC3339Nano, indexed); err == nil { + c.IndexedAt = &t + } + } + return c, nil +} + +func listCollections(db *sql.DB) ([]Collection, error) { + rows, err := db.Query(`SELECT name, type, source, indexed_at, last_index_error FROM collections ORDER BY name`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Collection + for rows.Next() { + c, err := scanCollection(rows) + if err != nil { + return nil, err + } + out = append(out, c) + } + if err := rows.Err(); err != nil { + return nil, err + } + return out, nil +} + +func resolveTargetCollections(db *sql.DB, selected []string) ([]Collection, error) { + if len(selected) == 0 { + return listCollections(db) + } + out := make([]Collection, 0, len(selected)) + seen := map[string]bool{} + for _, name := range selected { + if seen[name] { + continue + } + seen[name] = true + c, err := getCollection(db, name) + if err != nil { + return nil, err + } + out = append(out, c) + } + return out, nil +} + +func joinCollectionNames(cols []Collection) string { + if len(cols) == 0 { + return "" + } + names := make([]string, 0, len(cols)) + for _, c := range cols { + names = append(names, c.Name) + } + sort.Strings(names) + return strings.Join(names, ",") +} + +func countCollectionFiles(db *sql.DB, collection string) (int, error) { + var c int + err := db.QueryRow(`SELECT COUNT(*) FROM files WHERE collection_name = ?`, collection).Scan(&c) + return c, err +} + +func replaceCollectionFiles(db *sql.DB, collection string, files []FileEntry) error { + if _, err := db.Exec(`DELETE FROM files WHERE collection_name = ?`, collection); err != nil { + return err + } + stmt, err := db.Prepare(`INSERT INTO files(collection_name, path, name, size, mod_time, ctime, md5, hash_size, hash_mod_time, hash_updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) + if err != nil { + return err + } + defer stmt.Close() + for _, f := range files { + mod := "" + ctime := "" + hashMod := "" + hashUpdated := "" + var hashSize any = nil + if f.ModTime != nil { + mod = f.ModTime.UTC().Format(time.RFC3339Nano) + } + if f.CTime != nil { + ctime = f.CTime.UTC().Format(time.RFC3339Nano) + } + if f.HashModTime != nil { + hashMod = f.HashModTime.UTC().Format(time.RFC3339Nano) + } + if f.HashUpdatedAt != nil { + hashUpdated = f.HashUpdatedAt.UTC().Format(time.RFC3339Nano) + } + if f.HashSize != nil { + hashSize = *f.HashSize + } + if _, err := stmt.Exec(collection, f.Path, f.Name, f.Size, mod, ctime, strings.ToLower(f.MD5), hashSize, hashMod, hashUpdated); err != nil { + return err + } + } + return nil +} + +func deleteCollection(db *sql.DB, collection string) error { + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.Exec(`DELETE FROM files WHERE collection_name = ?`, collection); err != nil { + return err + } + if _, err := tx.Exec(`DELETE FROM collections WHERE name = ?`, collection); err != nil { + return err + } + return tx.Commit() +} + +func insertCollectionWithFiles(db *sql.DB, col Collection, files []FileEntry) error { + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + indexed := "" + if col.IndexedAt != nil { + indexed = col.IndexedAt.UTC().Format(time.RFC3339Nano) + } + if _, err := tx.Exec( + `INSERT INTO collections(name, type, source, indexed_at, last_index_error) VALUES (?, ?, ?, ?, ?)`, + col.Name, + col.Type, + col.Source, + indexed, + col.LastIndexError, + ); err != nil { + return err + } + stmt, err := tx.Prepare(`INSERT INTO files(collection_name, path, name, size, mod_time, ctime, md5, hash_size, hash_mod_time, hash_updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) + if err != nil { + return err + } + defer stmt.Close() + for _, f := range files { + mod, ctime, hashMod, hashUpdated, hashSize := fileDBValues(f) + if _, err := stmt.Exec(col.Name, f.Path, f.Name, f.Size, mod, ctime, strings.ToLower(f.MD5), hashSize, hashMod, hashUpdated); err != nil { + return err + } + } + return tx.Commit() +} + +func fileDBValues(f FileEntry) (mod, ctime, hashMod, hashUpdated string, hashSize any) { + if f.ModTime != nil { + mod = f.ModTime.UTC().Format(time.RFC3339Nano) + } + if f.CTime != nil { + ctime = f.CTime.UTC().Format(time.RFC3339Nano) + } + if f.HashModTime != nil { + hashMod = f.HashModTime.UTC().Format(time.RFC3339Nano) + } + if f.HashUpdatedAt != nil { + hashUpdated = f.HashUpdatedAt.UTC().Format(time.RFC3339Nano) + } + if f.HashSize != nil { + hashSize = *f.HashSize + } + return mod, ctime, hashMod, hashUpdated, hashSize +} + +func loadCollectionFiles(db *sql.DB, collection string) ([]FileEntry, error) { + rows, err := db.Query(`SELECT path, name, size, mod_time, ctime, md5, hash_size, hash_mod_time, hash_updated_at FROM files WHERE collection_name = ? ORDER BY path`, collection) + if err != nil { + return nil, err + } + defer rows.Close() + out := []FileEntry{} + for rows.Next() { + var f FileEntry + var mod, ctime, md5v, hashMod, hashUpdated sql.NullString + var hashSize sql.NullInt64 + if err := rows.Scan(&f.Path, &f.Name, &f.Size, &mod, &ctime, &md5v, &hashSize, &hashMod, &hashUpdated); err != nil { + return nil, err + } + if mod.Valid && mod.String != "" { + if t, err := time.Parse(time.RFC3339Nano, mod.String); err == nil { + t = t.UTC() + f.ModTime = &t + } + } + if ctime.Valid && ctime.String != "" { + if t, err := time.Parse(time.RFC3339Nano, ctime.String); err == nil { + t = t.UTC() + f.CTime = &t + } + } + if md5v.Valid { + f.MD5 = md5v.String + } + if hashSize.Valid { + v := hashSize.Int64 + f.HashSize = &v + } + if hashMod.Valid && hashMod.String != "" { + if t, err := time.Parse(time.RFC3339Nano, hashMod.String); err == nil { + t = t.UTC() + f.HashModTime = &t + } + } + if hashUpdated.Valid && hashUpdated.String != "" { + if t, err := time.Parse(time.RFC3339Nano, hashUpdated.String); err == nil { + t = t.UTC() + f.HashUpdatedAt = &t + } + } + out = append(out, f) + } + if err := rows.Err(); err != nil { + return nil, err + } + return out, nil +} + +func requireRclone() error { + if _, err := exec.LookPath("rclone"); err != nil { + return errors.New("rclone binary not found in PATH") + } + return nil +} + +func resolveCollectionSource(ctype, source, remote, remotePath string) (string, error) { + source = strings.TrimSpace(source) + remote = strings.TrimSpace(remote) + remotePath = strings.TrimSpace(remotePath) + + switch ctype { + case "fs": + if remote != "" || remotePath != "" { + return "", errors.New("--remote and --remote-path are only valid for --type rclone; use --source for filesystem paths") + } + if source == "" { + return "", errors.New("--source is required for --type fs") + } + info, err := os.Stat(source) + if err != nil { + return "", fmt.Errorf("filesystem source unavailable (%s): %w", source, err) + } + if !info.IsDir() { + return "", fmt.Errorf("filesystem source is not a directory: %s", source) + } + return source, nil + case "rclone": + if err := requireRclone(); err != nil { + return "", err + } + finalSource, err := resolveRcloneSource(source, remote, remotePath) + if err != nil { + return "", err + } + if err := validateRcloneSource(finalSource); err != nil { + return "", err + } + return finalSource, nil + default: + return "", fmt.Errorf("unsupported collection type: %s", ctype) + } +} + +func resolveRcloneSource(source, remote, remotePath string) (string, error) { + source = strings.TrimSpace(source) + remote = strings.TrimSpace(remote) + remotePath = strings.TrimSpace(remotePath) + + if source != "" && remote != "" { + return "", errors.New("use either --source or --remote/--remote-path for rclone, not both") + } + if source != "" { + return source, nil + } + if remote == "" { + return "", errors.New("for --type rclone use --source or --remote (with optional --remote-path)") + } + + remote = strings.TrimSuffix(remote, ":") + if remote == "" { + return "", errors.New("--remote must not be empty") + } + remotePath = strings.TrimPrefix(remotePath, "/") + if remotePath == "" { + return remote + ":", nil + } + return remote + ":" + remotePath, nil +} + +func validateRcloneSource(source string) error { + if source == "" { + return errors.New("--source is required for --type rclone") + } + colon := strings.Index(source, ":") + if colon <= 0 { + return fmt.Errorf("--source must be an rclone path in the form remote: or remote:path: %s", source) + } + remote := strings.TrimSpace(source[:colon]) + if remote == "" || strings.ContainsAny(remote, `/\`) { + return fmt.Errorf("invalid rclone remote in --source: %s", source) + } + return nil +} + +func indexCollection(c Collection) ([]FileEntry, error) { + return indexCollectionWithHashOption(c, false) +} + +func indexCollectionForMetadata(c Collection, metadata []string) ([]FileEntry, error) { + return indexCollectionWithHashOption(c, containsString(metadata, "md5")) +} + +func containsString(items []string, value string) bool { + for _, item := range items { + if strings.EqualFold(strings.TrimSpace(item), value) { + return true + } + } + return false +} + +func indexCollectionWithHashOption(c Collection, includeMD5 bool) ([]FileEntry, error) { + infof("indexing collection name=%q type=%q source=%q", c.Name, c.Type, c.Source) + switch c.Type { + case "fs": + return indexFilesystem(c.Source, includeMD5) + case "rclone": + if err := requireRclone(); err != nil { + return nil, err + } + return indexRclone(c.Source, includeMD5) + default: + return nil, fmt.Errorf("unsupported collection type: %s", c.Type) + } +} + +func indexFilesystem(root string, includeMD5 bool) ([]FileEntry, error) { + infof("indexing filesystem source=%q", root) + info, err := os.Stat(root) + if err != nil { + return nil, fmt.Errorf("filesystem source unavailable (%s): %w", root, err) + } + if !info.IsDir() { + return nil, fmt.Errorf("filesystem source is not a directory: %s", root) + } + + var out []FileEntry + err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + rel, err := filepath.Rel(root, path) + if err != nil { + rel = path + } + mod := info.ModTime().UTC() + entry := FileEntry{Path: filepath.ToSlash(rel), Name: info.Name(), Size: info.Size(), ModTime: &mod} + if ctime, ok := extractCTime(info); ok { + entry.CTime = &ctime + } + if includeMD5 { + if sum, err := md5File(path); err == nil { + entry.MD5 = sum + } + } + out = append(out, entry) + return nil + }) + if err != nil { + return nil, err + } + infof("filesystem index completed source=%q files=%d", root, len(out)) + return out, nil +} + +func indexRclone(remote string, includeMD5 bool) ([]FileEntry, error) { + infof("indexing rclone source=%q", remote) + cmd := exec.Command("rclone", "lsjson", "--recursive", "--hash", remote) + stdout, err := cmd.CombinedOutput() + if err != nil { + msg := strings.TrimSpace(string(stdout)) + if msg == "" { + return nil, fmt.Errorf("rclone lsjson failed for %s: %w", remote, err) + } + return nil, fmt.Errorf("rclone lsjson failed for %s: %s", remote, msg) + } + type ritem struct { + Path string `json:"Path"` + Name string `json:"Name"` + Size int64 `json:"Size"` + Mod string `json:"ModTime"` + IsDir bool `json:"IsDir"` + Hashes map[string]string `json:"Hashes"` + } + var items []ritem + if err := json.Unmarshal(stdout, &items); err != nil { + return nil, err + } + out := make([]FileEntry, 0, len(items)) + for _, item := range items { + if item.IsDir { + continue + } + entry := FileEntry{Path: filepath.ToSlash(item.Path), Name: item.Name, Size: item.Size} + if item.Mod != "" { + if t, err := time.Parse(time.RFC3339Nano, item.Mod); err == nil { + t = t.UTC() + entry.ModTime = &t + } + } + if includeMD5 { + if v, ok := item.Hashes["MD5"]; ok { + entry.MD5 = strings.ToLower(v) + } + } + out = append(out, entry) + } + infof("rclone index completed source=%q files=%d", remote, len(out)) + return out, nil +} + +func extractCTime(info os.FileInfo) (time.Time, bool) { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return time.Time{}, false + } + sec := st.Ctim.Sec + nsec := st.Ctim.Nsec + if sec == 0 && nsec == 0 { + return time.Time{}, false + } + return time.Unix(sec, nsec).UTC(), true +} + +func md5File(path string) (string, error) { + if showFileRead { + fmt.Printf("[read] %s\n", path) + } + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := md5.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func normalizeMetadata(raw string) ([]string, error) { + if strings.TrimSpace(raw) == "" { + return nil, errors.New("--metadata is required with at least one value") + } + allowed := map[string]struct{}{"filename": {}, "size": {}, "mtime": {}, "ctime": {}, "md5": {}} + seen := map[string]bool{} + var fields []string + for _, part := range strings.Split(raw, ",") { + f := strings.ToLower(strings.TrimSpace(part)) + if f == "" || seen[f] { + continue + } + if _, ok := allowed[f]; !ok { + return nil, fmt.Errorf("unsupported field: %s", f) + } + seen[f] = true + fields = append(fields, f) + } + if len(fields) == 0 { + return nil, errors.New("no valid fields selected") + } + return fields, nil +} + +func metadataKey(entry FileEntry, fields []string) string { + parts := make([]string, 0, len(fields)) + for _, f := range fields { + switch f { + case "filename": + parts = append(parts, "filename="+entry.Name) + case "size": + parts = append(parts, fmt.Sprintf("size=%d", entry.Size)) + case "mtime": + if entry.ModTime != nil { + parts = append(parts, "mtime="+entry.ModTime.UTC().Format(time.RFC3339Nano)) + } else { + parts = append(parts, "mtime=") + } + case "ctime": + if entry.CTime != nil { + parts = append(parts, "ctime="+entry.CTime.UTC().Format(time.RFC3339Nano)) + } else { + parts = append(parts, "ctime=") + } + case "md5": + parts = append(parts, "md5="+strings.ToLower(entry.MD5)) + } + } + return strings.Join(parts, "|") +} + +func compareLive(library, collection string, stored, live []FileEntry, fields []string) *Report { + dbMap := make(map[string][]string) + liveMap := make(map[string][]string) + for _, f := range stored { + key := metadataKey(f, fields) + dbMap[key] = append(dbMap[key], f.Path) + } + for _, f := range live { + key := metadataKey(f, fields) + liveMap[key] = append(liveMap[key], f.Path) + } + all := map[string]struct{}{} + for k := range dbMap { + all[k] = struct{}{} + } + for k := range liveMap { + all[k] = struct{}{} + } + keys := make([]string, 0, len(all)) + for k := range all { + keys = append(keys, k) + } + sort.Strings(keys) + + report := &Report{ + ID: reportID("compare-live"), + Kind: "compare-live", + CreatedAt: time.Now().UTC(), + Library: library, + Collection: collection, + Fields: fields, + Summary: map[string]int{"keys_total": len(keys), "match": 0, "missing_live": 0, "new_live": 0, "count_mismatch": 0}, + Rows: []map[string]any{}, + } + for _, key := range keys { + dc := len(dbMap[key]) + lc := len(liveMap[key]) + status := "match" + switch { + case dc == 0: + status = "new_live" + case lc == 0: + status = "missing_live" + case dc != lc: + status = "count_mismatch" + } + report.Summary[status]++ + report.Rows = append(report.Rows, map[string]any{ + "key": key, + "status": status, + "db_count": dc, + "live_count": lc, + "db_paths": dbMap[key], + "live_paths": liveMap[key], + }) + } + return report +} + +func compareOffline(library, collection string, stored []FileEntry, fields []string) *Report { + groups := map[string][]string{} + for _, f := range stored { + key := metadataKey(f, fields) + groups[key] = append(groups[key], f.Path) + } + keys := make([]string, 0, len(groups)) + for k := range groups { + if len(groups[k]) > 1 { + keys = append(keys, k) + } + } + sort.Slice(keys, func(i, j int) bool { + li := len(groups[keys[i]]) + lj := len(groups[keys[j]]) + if li == lj { + return keys[i] < keys[j] + } + return li > lj + }) + report := &Report{ + ID: reportID("compare-offline"), + Kind: "compare-offline", + CreatedAt: time.Now().UTC(), + Library: library, + Collection: collection, + Fields: fields, + Summary: map[string]int{ + "files_total": len(stored), + "duplicate_groups": len(keys), + }, + Rows: []map[string]any{}, + } + for _, key := range keys { + report.Rows = append(report.Rows, map[string]any{ + "key": key, + "count": len(groups[key]), + "paths": groups[key], + }) + } + return report +} + +func finalizeReport(db *sql.DB, report *Report, noStore bool, exportFormat, output string) error { + infof("finalizing report id=%q kind=%q rows=%d store=%t export=%q", report.ID, report.Kind, len(report.Rows), !noStore, exportFormat) + if !noStore { + fieldsJSON, _ := json.Marshal(report.Fields) + summaryJSON, _ := json.Marshal(report.Summary) + rowsJSON, _ := json.Marshal(report.Rows) + metaJSON, _ := json.Marshal(report.Meta) + _, err := db.Exec(`INSERT INTO reports(id, kind, created_at, library_name, collection_name, fields_json, summary_json, rows_json, meta_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + report.ID, + report.Kind, + report.CreatedAt.UTC().Format(time.RFC3339Nano), + report.Library, + report.Collection, + string(fieldsJSON), + string(summaryJSON), + string(rowsJSON), + string(metaJSON), + ) + if err != nil { + return err + } + fmt.Printf("stored report %s\n", report.ID) + } else { + fmt.Println("report storage skipped (--no-store-report)") + } + if exportFormat != "" { + if err := exportReport(report, exportFormat, output); err != nil { + return err + } + fmt.Printf("exported report to %s\n", output) + } + return nil +} + +func listReportsInDB(db *sql.DB) error { + infof("querying reports in current library") + rows, err := db.Query(`SELECT id, kind, library_name, collection_name, created_at FROM reports ORDER BY created_at`) + if err != nil { + return err + } + defer rows.Close() + count := 0 + for rows.Next() { + count++ + var id, kind, library, collection, created string + if err := rows.Scan(&id, &kind, &library, &collection, &created); err != nil { + return err + } + fmt.Printf("%s kind=%s lib=%s collection=%s at=%s\n", id, kind, library, collection, created) + } + if err := rows.Err(); err != nil { + return err + } + if count == 0 { + fmt.Println("no reports") + } + infof("reports listed=%d", count) + return nil +} + +func findReport(appDir, id, library string) (*Report, error) { + if strings.TrimSpace(library) != "" { + db, err := openLibraryDB(appDir, library) + if err != nil { + return nil, err + } + defer db.Close() + r, err := loadReport(db, id) + if err != nil { + return nil, err + } + return r, nil + } + libs, err := listLibraries(appDir) + if err != nil { + return nil, err + } + var found *Report + for _, lib := range libs { + db, err := openLibraryDB(appDir, lib) + if err != nil { + continue + } + r, err := loadReport(db, id) + _ = db.Close() + if err == nil && r != nil { + if found != nil { + return nil, errors.New("report id found in multiple libraries; use --library") + } + found = r + } + } + if found == nil { + return nil, fmt.Errorf("report not found: %s", id) + } + return found, nil +} + +func listLibraries(appDir string) ([]string, error) { + paths, err := listLibraryFiles(appDir) + if err != nil { + return nil, err + } + libs := []string{} + for _, path := range paths { + db, err := sql.Open("duckdb", path) + if err != nil { + continue + } + _ = ensureSchema(db) + var name string + if err := db.QueryRow(`SELECT name FROM library_info LIMIT 1`).Scan(&name); err == nil { + libs = append(libs, name) + } + _ = db.Close() + } + sort.Strings(libs) + return libs, nil +} + +func loadReport(db *sql.DB, id string) (*Report, error) { + row := db.QueryRow(`SELECT id, kind, created_at, library_name, collection_name, fields_json, summary_json, rows_json, meta_json FROM reports WHERE id = ?`, id) + var report Report + var created, fieldsJSON, summaryJSON, rowsJSON, metaJSON string + if err := row.Scan(&report.ID, &report.Kind, &created, &report.Library, &report.Collection, &fieldsJSON, &summaryJSON, &rowsJSON, &metaJSON); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("report not found: %s", id) + } + return nil, err + } + if created != "" { + if t, err := time.Parse(time.RFC3339Nano, created); err == nil { + report.CreatedAt = t.UTC() + } + } + _ = json.Unmarshal([]byte(fieldsJSON), &report.Fields) + _ = json.Unmarshal([]byte(summaryJSON), &report.Summary) + _ = json.Unmarshal([]byte(rowsJSON), &report.Rows) + _ = json.Unmarshal([]byte(metaJSON), &report.Meta) + if report.Summary == nil { + report.Summary = map[string]int{} + } + if report.Rows == nil { + report.Rows = []map[string]any{} + } + if report.Meta == nil { + report.Meta = map[string]string{} + } + return &report, nil +} + +func exportReport(report *Report, format, output string) error { + switch format { + case "json": + data, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + return os.WriteFile(output, data, 0o644) + case "html": + return exportHTML(report, output) + default: + return fmt.Errorf("unsupported format: %s", format) + } +} + +func exportHTML(report *Report, output string) error { + const page = ` + +filedb report + + + +

Report {{.ID}}

+

Kind: {{.Kind}}
+Created: {{.CreatedAt}}
+Library: {{.Library}}
+Collection: {{.Collection}}
+Fields: {{range $i,$f := .Fields}}{{if $i}}, {{end}}{{$f}}{{end}}

+

Summary

+
    {{range $k,$v := .Summary}}
  • {{$k}}: {{$v}}
    • {{end}}
+

Rows

+ + +{{range .Rows}}{{end}} +
Data
{{json .}}
+` + funcs := template.FuncMap{ + "json": func(v any) string { + b, _ := json.MarshalIndent(v, "", " ") + return string(b) + }, + } + tpl, err := template.New("report").Funcs(funcs).Parse(page) + if err != nil { + return err + } + f, err := os.Create(output) + if err != nil { + return err + } + defer f.Close() + return tpl.Execute(f, report) +} + +func printReportSummary(report *Report) { + infof("report summary id=%q", report.ID) + fmt.Printf("report %s (%s) rows=%d\n", report.ID, report.Kind, len(report.Rows)) + keys := make([]string, 0, len(report.Summary)) + for k := range report.Summary { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + fmt.Printf(" %s: %d\n", k, report.Summary[k]) + } +} + +func reportID(prefix string) string { + return fmt.Sprintf("%s-%d", prefix, time.Now().UTC().UnixNano()) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..febaf0a --- /dev/null +++ b/go.mod @@ -0,0 +1,23 @@ +module filedb + +go 1.24 + +require github.com/marcboeker/go-duckdb v1.8.5 + +require ( + github.com/apache/arrow-go/v18 v18.1.0 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/flatbuffers v25.1.24+incompatible // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect + golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c // indirect + golang.org/x/mod v0.22.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/tools v0.29.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..1144140 --- /dev/null +++ b/go.sum @@ -0,0 +1,58 @@ +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/apache/arrow-go/v18 v18.1.0 h1:agLwJUiVuwXZdwPYVrlITfx7bndULJ/dggbnLFgDp/Y= +github.com/apache/arrow-go/v18 v18.1.0/go.mod h1:tigU/sIgKNXaesf5d7Y95jBBKS5KsxTqYBKXFsvKzo0= +github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= +github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v25.1.24+incompatible h1:4wPqL3K7GzBd1CwyhSd3usxLKOaJN/AC6puCca6Jm7o= +github.com/google/flatbuffers v25.1.24+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/marcboeker/go-duckdb v1.8.5 h1:tkYp+TANippy0DaIOP5OEfBEwbUINqiFqgwMQ44jME0= +github.com/marcboeker/go-duckdb v1.8.5/go.mod h1:6mK7+WQE4P4u5AFLvVBmhFxY5fvhymFptghgJX6B+/8= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c h1:KL/ZBHXgKGVmuZBZ01Lt57yE5ws8ZPSkkihmEyq7FXc= +golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= +golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= +gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=