primeiro commit

2026-02-12 15:28:20 -03:00
commit b264fb5c95
5 changed files with 216 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -0,0 +1,2 @@
+bin/
+filedb
--- a/39
+++ b/39
@@ -0,0 +1,39 @@
+APP := filedb
+CMD := ./cmd/filedb
+BUILD_DIR := bin
+BIN := $(BUILD_DIR)/$(APP)
+
+.PHONY: help build run install fmt tidy test clean
+
+help:
+	@echo "Targets:"
+	@echo "  make build             Build binary to $(BIN)"
+	@echo "  make run ARGS='...'    Run CLI with optional ARGS"
+	@echo "  make install           Install CLI to GOPATH/bin"
+	@echo "  make fmt               Format Go code"
+	@echo "  make tidy              Tidy go modules"
+	@echo "  make test              Run tests"
+	@echo "  make clean             Remove build artifacts"
+
+build:
+	@mkdir -p $(BUILD_DIR)
+	go build -o $(BIN) $(CMD)
+
+run:
+	go run $(CMD) $(ARGS)
+
+install:
+	go install $(CMD)
+
+fmt:
+	gofmt -w $$(find . -type f -name '*.go' -not -path './vendor/*')
+
+tidy:
+	go mod tidy
+
+test:
+	go test ./...
+
+clean:
+	rm -rf $(BUILD_DIR)
+
--- a/README.md
+++ b/README.md
@@ -0,0 +1,94 @@
+# filedb
+
+`filedb` is a Go CLI utility for indexing file metadata across multiple libraries and collections.
+
+- Library: logical grouping of collections, persisted as its own DuckDB database file.
+- Collection: a backup-like dataset backed by either:
+  - `fs`: local filesystem path
+  - `rclone`: `rclone lsjson` listing for a remote/path
+
+Checks are informational only. The tool does not modify or delete files.
+
+## Storage
+
+- Default application directory: `~/.local/filedb`
+- Library databases: `~/.local/filedb/libraries/*.duckdb` (one database per library)
+- Override app directory with: `--app-dir <path>`
+
+## Features
+
+- Create/list libraries
+- Create/list collections inside libraries
+- Update one collection or a full library with one operation (file set + metadata only; no MD5 processing)
+- Update MD5 hashes with dedicated `update-hashes` command
+- Store metadata: size, mod time, create time (if available), MD5 (if possible)
+- Compare indexed metadata with current live files (`compare-live`)
+- Offline duplicate checking inside indexed collection (`compare-offline`)
+- Store reports inside each library DB by default
+- Optional report export to JSON or HTML and optional no-store mode
+
+## Build
+
+```bash
+go build -o filedb ./cmd/filedb
+```
+
+## Usage
+
+```bash
+filedb library-create --name archive
+filedb collection-create --library archive --name tape-001 --type fs --source /data/tape-001
+filedb collection-create --library archive --name remote-photos --type rclone --source myremote:photos
+filedb collection-create --library archive --name remote-root --type rclone --remote myremote
+filedb collection-create --library archive --name remote-subdir --type rclone --remote myremote --remote-path photos/2024
+
+filedb update --library archive --collection tape-001
+filedb update --library archive
+
+filedb update-hashes --library archive --collection tape-001
+filedb update-hashes --library archive --force-all
+filedb update-hashes --library archive --modified-only
+filedb update-hashes --library archive --collection tape-001 --file path/in/collection/file.iso
+filedb update-hashes --library archive --collection tape-001 --file-list /tmp/files.txt
+
+filedb compare-live --library archive --metadata size,md5 --collection tape-001
+filedb compare-live --library archive --metadata size,md5
+filedb compare-offline --library archive --metadata filename,size,md5,mtime --collection tape-001 --collection tape-002
+
+filedb compare-offline --library archive --metadata filename,size,md5 --no-store-report --export html --output report.html
+
+filedb report-list
+filedb report-list --library archive
+filedb report-show --id <report-id> --library archive
+filedb report-export --id <report-id> --library archive --format json --output report.json
+```
+
+With custom app directory:
+
+```bash
+filedb --app-dir /tmp/filedb-data library-list
+```
+
+## Metadata Fields
+
+For comparison keys (`--metadata`):
+
+- `filename` (offline only; not allowed in `compare-live`)
+- `size`
+- `mtime`
+- `ctime`
+- `md5`
+
+Example:
+
+```bash
+filedb compare-offline --library archive --metadata filename,size,mtime
+```
+
+## Notes
+
+- `update` does not compute hashes. Use `update-hashes` for MD5 processing.
+- `ctime` availability is OS/filesystem dependent.
+- Rclone collections require `rclone` in `PATH` (validated at collection creation and indexing).
+- For rclone collections use either `--source <remote-or-remote:path>` or `--remote <remote> [--remote-path <path>]`.
+- `update` skips disconnected/unreachable filesystem sources and rclone indexing failures, and continues with other collections.
--- a/go.mod
+++ b/go.mod
@@ -0,0 +1,23 @@
+module filedb
+
+go 1.24
+
+require github.com/marcboeker/go-duckdb v1.8.5
+
+require (
+	github.com/apache/arrow-go/v18 v18.1.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
+	github.com/goccy/go-json v0.10.5 // indirect
+	github.com/google/flatbuffers v25.1.24+incompatible // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
+	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
+	golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c // indirect
+	golang.org/x/mod v0.22.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
+	golang.org/x/tools v0.29.0 // indirect
+	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
+)
--- a/go.sum
+++ b/go.sum
@@ -0,0 +1,58 @@
+github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
+github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
+github.com/apache/arrow-go/v18 v18.1.0 h1:agLwJUiVuwXZdwPYVrlITfx7bndULJ/dggbnLFgDp/Y=
+github.com/apache/arrow-go/v18 v18.1.0/go.mod h1:tigU/sIgKNXaesf5d7Y95jBBKS5KsxTqYBKXFsvKzo0=
+github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE=
+github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
+github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
+github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/flatbuffers v25.1.24+incompatible h1:4wPqL3K7GzBd1CwyhSd3usxLKOaJN/AC6puCca6Jm7o=
+github.com/google/flatbuffers v25.1.24+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
+github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
+github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
+github.com/marcboeker/go-duckdb v1.8.5 h1:tkYp+TANippy0DaIOP5OEfBEwbUINqiFqgwMQ44jME0=
+github.com/marcboeker/go-duckdb v1.8.5/go.mod h1:6mK7+WQE4P4u5AFLvVBmhFxY5fvhymFptghgJX6B+/8=
+github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
+github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
+github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
+github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
+github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
+github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
+github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
+github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
+golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c h1:KL/ZBHXgKGVmuZBZ01Lt57yE5ws8ZPSkkihmEyq7FXc=
+golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU=
+golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
+golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE=
+golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588=
+golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
+golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
+gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0=
+gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=