commit b264fb5c954aa31faf8aed10f3850b8f2640ff0f Author: Glauber Ferreira Date: Thu Feb 12 15:28:20 2026 -0300 primeiro commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fc746ba --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +bin/ +filedb diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bc0b042 --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ +APP := filedb +CMD := ./cmd/filedb +BUILD_DIR := bin +BIN := $(BUILD_DIR)/$(APP) + +.PHONY: help build run install fmt tidy test clean + +help: + @echo "Targets:" + @echo " make build Build binary to $(BIN)" + @echo " make run ARGS='...' Run CLI with optional ARGS" + @echo " make install Install CLI to GOPATH/bin" + @echo " make fmt Format Go code" + @echo " make tidy Tidy go modules" + @echo " make test Run tests" + @echo " make clean Remove build artifacts" + +build: + @mkdir -p $(BUILD_DIR) + go build -o $(BIN) $(CMD) + +run: + go run $(CMD) $(ARGS) + +install: + go install $(CMD) + +fmt: + gofmt -w $$(find . -type f -name '*.go' -not -path './vendor/*') + +tidy: + go mod tidy + +test: + go test ./... + +clean: + rm -rf $(BUILD_DIR) + diff --git a/README.md b/README.md new file mode 100644 index 0000000..80e4e01 --- /dev/null +++ b/README.md @@ -0,0 +1,94 @@ +# filedb + +`filedb` is a Go CLI utility for indexing file metadata across multiple libraries and collections. + +- Library: logical grouping of collections, persisted as its own DuckDB database file. +- Collection: a backup-like dataset backed by either: + - `fs`: local filesystem path + - `rclone`: `rclone lsjson` listing for a remote/path + +Checks are informational only. The tool does not modify or delete files. + +## Storage + +- Default application directory: `~/.local/filedb` +- Library databases: `~/.local/filedb/libraries/*.duckdb` (one database per library) +- Override app directory with: `--app-dir ` + +## Features + +- Create/list libraries +- Create/list collections inside libraries +- Update one collection or a full library with one operation (file set + metadata only; no MD5 processing) +- Update MD5 hashes with dedicated `update-hashes` command +- Store metadata: size, mod time, create time (if available), MD5 (if possible) +- Compare indexed metadata with current live files (`compare-live`) +- Offline duplicate checking inside indexed collection (`compare-offline`) +- Store reports inside each library DB by default +- Optional report export to JSON or HTML and optional no-store mode + +## Build + +```bash +go build -o filedb ./cmd/filedb +``` + +## Usage + +```bash +filedb library-create --name archive +filedb collection-create --library archive --name tape-001 --type fs --source /data/tape-001 +filedb collection-create --library archive --name remote-photos --type rclone --source myremote:photos +filedb collection-create --library archive --name remote-root --type rclone --remote myremote +filedb collection-create --library archive --name remote-subdir --type rclone --remote myremote --remote-path photos/2024 + +filedb update --library archive --collection tape-001 +filedb update --library archive + +filedb update-hashes --library archive --collection tape-001 +filedb update-hashes --library archive --force-all +filedb update-hashes --library archive --modified-only +filedb update-hashes --library archive --collection tape-001 --file path/in/collection/file.iso +filedb update-hashes --library archive --collection tape-001 --file-list /tmp/files.txt + +filedb compare-live --library archive --metadata size,md5 --collection tape-001 +filedb compare-live --library archive --metadata size,md5 +filedb compare-offline --library archive --metadata filename,size,md5,mtime --collection tape-001 --collection tape-002 + +filedb compare-offline --library archive --metadata filename,size,md5 --no-store-report --export html --output report.html + +filedb report-list +filedb report-list --library archive +filedb report-show --id --library archive +filedb report-export --id --library archive --format json --output report.json +``` + +With custom app directory: + +```bash +filedb --app-dir /tmp/filedb-data library-list +``` + +## Metadata Fields + +For comparison keys (`--metadata`): + +- `filename` (offline only; not allowed in `compare-live`) +- `size` +- `mtime` +- `ctime` +- `md5` + +Example: + +```bash +filedb compare-offline --library archive --metadata filename,size,mtime +``` + +## Notes + +- `update` does not compute hashes. Use `update-hashes` for MD5 processing. +- `ctime` availability is OS/filesystem dependent. +- Rclone collections require `rclone` in `PATH` (validated at collection creation and indexing). +- For rclone collections use either `--source ` or `--remote [--remote-path ]`. +- `update` skips disconnected/unreachable filesystem sources and rclone indexing failures, and continues with other collections. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..febaf0a --- /dev/null +++ b/go.mod @@ -0,0 +1,23 @@ +module filedb + +go 1.24 + +require github.com/marcboeker/go-duckdb v1.8.5 + +require ( + github.com/apache/arrow-go/v18 v18.1.0 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/flatbuffers v25.1.24+incompatible // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect + golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c // indirect + golang.org/x/mod v0.22.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/tools v0.29.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..1144140 --- /dev/null +++ b/go.sum @@ -0,0 +1,58 @@ +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/apache/arrow-go/v18 v18.1.0 h1:agLwJUiVuwXZdwPYVrlITfx7bndULJ/dggbnLFgDp/Y= +github.com/apache/arrow-go/v18 v18.1.0/go.mod h1:tigU/sIgKNXaesf5d7Y95jBBKS5KsxTqYBKXFsvKzo0= +github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= +github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v25.1.24+incompatible h1:4wPqL3K7GzBd1CwyhSd3usxLKOaJN/AC6puCca6Jm7o= +github.com/google/flatbuffers v25.1.24+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/marcboeker/go-duckdb v1.8.5 h1:tkYp+TANippy0DaIOP5OEfBEwbUINqiFqgwMQ44jME0= +github.com/marcboeker/go-duckdb v1.8.5/go.mod h1:6mK7+WQE4P4u5AFLvVBmhFxY5fvhymFptghgJX6B+/8= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c h1:KL/ZBHXgKGVmuZBZ01Lt57yE5ws8ZPSkkihmEyq7FXc= +golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= +golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= +gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=