diff --git a/AGENTS.md b/AGENTS.md index 8dc58ae..e1601f5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,6 +36,7 @@ cd mobile && npm run test:db - `service/` — business logic (permissions, upload, create folder, move) - `handlers/` — HTTP handlers (bind the routes; currently 501 not-implemented stubs) - `repository/` — Postgres persistence (`golang-migrate` + `lib/pq`); IDs are TEXT 32-hex (never UUID conversion, cf. `docs/api-v1.md`) +- `db/` — package migrations (`golang-migrate/v4`, embarquées via `embed` dans `db/migrations/*.sql`) : `db.MigrateDatabase(url)` au boot du serveur ; test harness `db/migrations_test.go` (up → assertions schéma → down, `TEST_DATABASE_URL`, skip si PG indisponible) - `ocr/` — OCR engine behind an interface (Tesseract system call, `OCR_LANG` défaut `fra+eng`) - Response helpers: `pkg/api/response.go` - File uploads stored in `backend/uploads/` diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index 63f99ee..e120e53 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -6,6 +6,7 @@ import ( "github.com/gin-gonic/gin" "github.com/vaultdrop/backend/config" + "github.com/vaultdrop/backend/db" "github.com/vaultdrop/backend/handlers" "github.com/vaultdrop/backend/pkg/auth" ) @@ -55,6 +56,10 @@ func main() { } handlers.Auth = authManager + if err := db.MigrateDatabase(cfg.DatabaseURL); err != nil { + log.Fatalf("migrations postgres: %v", err) + } + if err := newRouter().Run(fmt.Sprintf(":%d", cfg.Port)); err != nil { log.Fatalln(err) } diff --git a/backend/config/config.go b/backend/config/config.go index f55da92..6974bab 100644 --- a/backend/config/config.go +++ b/backend/config/config.go @@ -33,7 +33,7 @@ func LoadApplicationConfig() (error, *ApplicationConfig) { return nil, &ApplicationConfig{ Port: port, - DatabaseURL: get("DATABASE_URL", "postgres://vaultdrop:vaultdrop@localhost:5432/vaultdrop?sslmode=disable"), + DatabaseURL: get("DATABASE_URL", "postgres://vaultdrop:vaultdrop@localhost:5432/vaultdrop_dev?sslmode=disable"), UploadDir: get("UPLOAD_DIR", "./uploads"), MaxFileSizeMB: int64(maxSize), OcrLang: get("OCR_LANG", "fra+eng"), diff --git a/backend/db/db.go b/backend/db/db.go new file mode 100644 index 0000000..8b81e1c --- /dev/null +++ b/backend/db/db.go @@ -0,0 +1,80 @@ +package db + +import ( + "database/sql" + "embed" + "errors" + "fmt" + + "github.com/golang-migrate/migrate/v4" + migratepostgres "github.com/golang-migrate/migrate/v4/database/postgres" + "github.com/golang-migrate/migrate/v4/source/iofs" + _ "github.com/lib/pq" +) + +//go:embed migrations/*.sql +var migrationsFS embed.FS + +// Open opens a raw *sql.DB (Postgres). Callers must Close it. +func Open(databaseURL string) (*sql.DB, error) { + conn, err := sql.Open("postgres", databaseURL) + if err != nil { + return nil, fmt.Errorf("open database: %w", err) + } + if err := conn.Ping(); err != nil { + conn.Close() + return nil, fmt.Errorf("ping database: %w", err) + } + return conn, nil +} + +func newMigrator(conn *sql.DB) (*migrate.Migrate, error) { + source, err := iofs.New(migrationsFS, "migrations") + if err != nil { + return nil, fmt.Errorf("load embedded migrations: %w", err) + } + driver, err := migratepostgres.WithInstance(conn, &migratepostgres.Config{}) + if err != nil { + return nil, fmt.Errorf("init postgres migrate driver: %w", err) + } + m, err := migrate.NewWithInstance("iofs", source, "postgres", driver) + if err != nil { + return nil, fmt.Errorf("new migrate: %w", err) + } + return m, nil +} + +// MigrateDatabase applies all pending migrations up to the latest version. +// migrate.ErrNoChange (already up-to-date) is not an error. +func MigrateDatabase(databaseURL string) error { + conn, err := Open(databaseURL) + if err != nil { + return err + } + defer conn.Close() + m, err := newMigrator(conn) + if err != nil { + return err + } + if err := m.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) { + return err + } + return nil +} + +// MigrateDownDatabase rolls back every migration (used by tests). +func MigrateDownDatabase(databaseURL string) error { + conn, err := Open(databaseURL) + if err != nil { + return err + } + defer conn.Close() + m, err := newMigrator(conn) + if err != nil { + return err + } + if err := m.Down(); err != nil && !errors.Is(err, migrate.ErrNoChange) { + return err + } + return nil +} diff --git a/backend/db/migrations/000001_create_users.down.sql b/backend/db/migrations/000001_create_users.down.sql new file mode 100644 index 0000000..2e8d4c1 --- /dev/null +++ b/backend/db/migrations/000001_create_users.down.sql @@ -0,0 +1,3 @@ +DROP INDEX IF EXISTS idx_users_parent; +DROP INDEX IF EXISTS idx_users_email; +DROP TABLE IF EXISTS users; \ No newline at end of file diff --git a/backend/db/migrations/000001_create_users.up.sql b/backend/db/migrations/000001_create_users.up.sql new file mode 100644 index 0000000..43d33f2 --- /dev/null +++ b/backend/db/migrations/000001_create_users.up.sql @@ -0,0 +1,13 @@ +CREATE TABLE users ( + id TEXT PRIMARY KEY CHECK (id ~ '^[0-9a-f]{32}$'), + email VARCHAR(255), + display_name VARCHAR(255), + parent_user_id TEXT REFERENCES users(id) ON DELETE SET NULL, + storage_quota_bytes BIGINT DEFAULT 10737418240, + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW(), + deleted_at TIMESTAMPTZ +); + +CREATE UNIQUE INDEX idx_users_email ON users(email) WHERE email IS NOT NULL AND deleted_at IS NULL; +CREATE INDEX idx_users_parent ON users(parent_user_id) WHERE deleted_at IS NULL; \ No newline at end of file diff --git a/backend/db/migrations/000002_create_devices.down.sql b/backend/db/migrations/000002_create_devices.down.sql new file mode 100644 index 0000000..be1897c --- /dev/null +++ b/backend/db/migrations/000002_create_devices.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_devices_user; +DROP TABLE IF EXISTS devices; \ No newline at end of file diff --git a/backend/db/migrations/000002_create_devices.up.sql b/backend/db/migrations/000002_create_devices.up.sql new file mode 100644 index 0000000..7389991 --- /dev/null +++ b/backend/db/migrations/000002_create_devices.up.sql @@ -0,0 +1,9 @@ +CREATE TABLE devices ( + device_id TEXT PRIMARY KEY CHECK (device_id ~ '^[0-9a-f]{32}$'), + user_id TEXT REFERENCES users(id) ON DELETE SET NULL, + created_at TIMESTAMPTZ DEFAULT NOW(), + registered_at TIMESTAMPTZ DEFAULT NOW(), + last_seen_at TIMESTAMPTZ +); + +CREATE INDEX idx_devices_user ON devices(user_id) WHERE user_id IS NOT NULL; \ No newline at end of file diff --git a/backend/db/migrations/000003_create_resources.down.sql b/backend/db/migrations/000003_create_resources.down.sql new file mode 100644 index 0000000..8007c24 --- /dev/null +++ b/backend/db/migrations/000003_create_resources.down.sql @@ -0,0 +1,4 @@ +DROP INDEX IF EXISTS idx_resources_category; +DROP INDEX IF EXISTS idx_resources_parent; +DROP INDEX IF EXISTS idx_resources_owner; +DROP TABLE IF EXISTS resources; \ No newline at end of file diff --git a/backend/db/migrations/000003_create_resources.up.sql b/backend/db/migrations/000003_create_resources.up.sql new file mode 100644 index 0000000..198d2f1 --- /dev/null +++ b/backend/db/migrations/000003_create_resources.up.sql @@ -0,0 +1,21 @@ +CREATE TABLE resources ( + resource_id TEXT PRIMARY KEY CHECK (resource_id ~ '^[0-9a-f]{32}$'), + type TEXT NOT NULL CHECK (type IN ('file', 'folder')), + name TEXT NOT NULL, + parent_id TEXT REFERENCES resources(resource_id) ON DELETE CASCADE, + owner_id TEXT NOT NULL REFERENCES devices(device_id) ON DELETE CASCADE, + content_hash VARCHAR(128), + size_bytes BIGINT DEFAULT 0, + mime_type VARCHAR(255), + extension VARCHAR(64), + category VARCHAR(50) CHECK (category IN ('photo', 'video', 'document', 'audio', 'other')), + taken_at TIMESTAMPTZ, + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW(), + deleted_at TIMESTAMPTZ, + CONSTRAINT resources_unique_name_per_parent UNIQUE (parent_id, name) +); + +CREATE INDEX idx_resources_owner ON resources(owner_id) WHERE deleted_at IS NULL; +CREATE INDEX idx_resources_parent ON resources(parent_id); +CREATE INDEX idx_resources_category ON resources(category) WHERE deleted_at IS NULL; \ No newline at end of file diff --git a/backend/db/migrations/000004_create_operations.down.sql b/backend/db/migrations/000004_create_operations.down.sql new file mode 100644 index 0000000..3131c28 --- /dev/null +++ b/backend/db/migrations/000004_create_operations.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_operations_device_created; +DROP TABLE IF EXISTS operations; \ No newline at end of file diff --git a/backend/db/migrations/000004_create_operations.up.sql b/backend/db/migrations/000004_create_operations.up.sql new file mode 100644 index 0000000..656424b --- /dev/null +++ b/backend/db/migrations/000004_create_operations.up.sql @@ -0,0 +1,14 @@ +CREATE TABLE operations ( + id BIGSERIAL PRIMARY KEY, + device_id TEXT NOT NULL REFERENCES devices(device_id) ON DELETE CASCADE, + operation_id TEXT NOT NULL CHECK (operation_id ~ '^[0-9a-f]{32}$'), + op_type TEXT NOT NULL, + payload JSONB NOT NULL, + status TEXT NOT NULL DEFAULT 'applied' CHECK (status IN ('applied', 'failed')), + error_code TEXT, + applied_at TIMESTAMPTZ DEFAULT NOW(), + created_at TIMESTAMPTZ DEFAULT NOW(), + CONSTRAINT operations_unique_per_device UNIQUE (device_id, operation_id) +); + +CREATE INDEX idx_operations_device_created ON operations(device_id, created_at); \ No newline at end of file diff --git a/backend/db/migrations/000005_create_ocr_jobs.down.sql b/backend/db/migrations/000005_create_ocr_jobs.down.sql new file mode 100644 index 0000000..4932afa --- /dev/null +++ b/backend/db/migrations/000005_create_ocr_jobs.down.sql @@ -0,0 +1,3 @@ +DROP INDEX IF EXISTS idx_ocr_jobs_file; +DROP INDEX IF EXISTS idx_ocr_jobs_status; +DROP TABLE IF EXISTS ocr_jobs; \ No newline at end of file diff --git a/backend/db/migrations/000005_create_ocr_jobs.up.sql b/backend/db/migrations/000005_create_ocr_jobs.up.sql new file mode 100644 index 0000000..9f21c11 --- /dev/null +++ b/backend/db/migrations/000005_create_ocr_jobs.up.sql @@ -0,0 +1,15 @@ +CREATE TABLE ocr_jobs ( + job_id TEXT PRIMARY KEY CHECK (job_id ~ '^[0-9a-f]{32}$'), + file_id TEXT NOT NULL REFERENCES resources(resource_id) ON DELETE CASCADE, + device_id TEXT NOT NULL REFERENCES devices(device_id) ON DELETE CASCADE, + status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'processing', 'done', 'failed')), + language VARCHAR(64) DEFAULT 'fra+eng', + text TEXT, + error TEXT, + created_at TIMESTAMPTZ DEFAULT NOW(), + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ +); + +CREATE INDEX idx_ocr_jobs_status ON ocr_jobs(status); +CREATE INDEX idx_ocr_jobs_file ON ocr_jobs(file_id); \ No newline at end of file diff --git a/backend/db/migrations_test.go b/backend/db/migrations_test.go new file mode 100644 index 0000000..57f3643 --- /dev/null +++ b/backend/db/migrations_test.go @@ -0,0 +1,153 @@ +package db + +import ( + "database/sql" + "os" + "strings" + "testing" + + _ "github.com/lib/pq" +) + +const defaultTestDatabaseURL = "postgres://vaultdrop:vaultdrop@localhost:5432/vaultdrop_migrations_test?sslmode=disable" + +func testDatabaseURL(t *testing.T) string { + t.Helper() + url := os.Getenv("TEST_DATABASE_URL") + if url == "" { + url = defaultTestDatabaseURL + } + conn, err := sql.Open("postgres", url) + if err != nil { + t.Fatalf("open: %v", err) + } + defer conn.Close() + if err := conn.Ping(); err != nil { + t.Skipf("postgres indisponible (%v) — lancez `docker compose up postgres -d`", err) + } + return url +} + +func resetSchema(t *testing.T, url string) { + t.Helper() + conn, err := sql.Open("postgres", url) + if err != nil { + t.Fatalf("open: %v", err) + } + defer conn.Close() + if _, err := conn.Exec(`DROP SCHEMA public CASCADE; CREATE SCHEMA public;`); err != nil { + t.Fatalf("reset schema: %v", err) + } +} + +func tableNames(t *testing.T, conn *sql.DB) map[string]bool { + t.Helper() + rows, err := conn.Query(` + SELECT table_name FROM information_schema.tables + WHERE table_schema = 'public' AND table_type = 'BASE TABLE'`) + if err != nil { + t.Fatalf("list tables: %v", err) + } + defer rows.Close() + names := map[string]bool{} + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("scan: %v", err) + } + names[name] = true + } + return names +} + +func assertTables(t *testing.T, conn *sql.DB, expected ...string) { + t.Helper() + got := tableNames(t, conn) + for _, table := range expected { + if !got[table] { + t.Errorf("table manquante après migrate up : %s", table) + } + } +} + +func assertHexCheck(t *testing.T, conn *sql.DB, table, column string) { + t.Helper() + var def string + err := conn.QueryRow(` + SELECT pg_get_constraintdef(c.oid) + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + WHERE t.relname = $1 AND c.contype = 'c' + AND pg_get_constraintdef(c.oid) LIKE '%' || $2 || '%'`, + table, column).Scan(&def) + if err != nil { + t.Fatalf("contrainte CHECK(hex) manquante sur %s.%s: %v", table, column, err) + } + if !strings.Contains(def, "^[0-9a-f]{32}$") { + t.Errorf("CHECK %s.%s attendu avec pattern 32-hex, obtenu : %s", table, column, def) + } +} + +func TestMigrationsUpDown(t *testing.T) { + url := testDatabaseURL(t) + resetSchema(t, url) + + if err := MigrateDatabase(url); err != nil { + t.Fatalf("migrate up: %v", err) + } + + conn, err := sql.Open("postgres", url) + if err != nil { + t.Fatalf("open: %v", err) + } + defer conn.Close() + if err := conn.Ping(); err != nil { + t.Fatalf("ping: %v", err) + } + + assertTables(t, conn, "users", "devices", "resources", "operations", "ocr_jobs", "schema_migrations") + + assertHexCheck(t, conn, "devices", "device_id") + assertHexCheck(t, conn, "resources", "resource_id") + assertHexCheck(t, conn, "operations", "operation_id") + assertHexCheck(t, conn, "ocr_jobs", "job_id") + + var resourceTypeCheck int + err = conn.QueryRow(` + SELECT COUNT(*) FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + WHERE t.relname = 'resources' + AND pg_get_constraintdef(c.oid) LIKE '%''file''%' + AND pg_get_constraintdef(c.oid) LIKE '%''folder''%'`).Scan(&resourceTypeCheck) + if err != nil { + t.Fatalf("check type resources: %v", err) + } + if resourceTypeCheck == 0 { + t.Error("CHECK (type IN ('file','folder')) manquant sur resources") + } + + var opUnique int + err = conn.QueryRow(` + SELECT COUNT(*) FROM pg_index i + JOIN pg_class t ON t.oid = i.indrelid + WHERE t.relname = 'operations' AND i.indisunique + AND ARRAY(SELECT a.attname FROM unnest(i.indkey) WITH ORDINALITY k(attnum, ord) + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum + ORDER BY k.ord)::text[] = ARRAY['device_id','operation_id']`).Scan(&opUnique) + if err != nil { + t.Fatalf("unique operations: %v", err) + } + if opUnique == 0 { + t.Error("contrainte UNIQUE(device_id, operation_id) manquante sur operations") + } + + if err := MigrateDownDatabase(url); err != nil { + t.Fatalf("migrate down: %v", err) + } + + remaining := tableNames(t, conn) + delete(remaining, "schema_migrations") + if len(remaining) > 0 { + t.Errorf("tables restantes après migrate down : %v", remaining) + } +} diff --git a/backend/go.mod b/backend/go.mod index f7276f4..c7fcc94 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -6,11 +6,8 @@ require ( aidanwoods.dev/go-paseto v1.6.0 github.com/gin-gonic/gin v1.12.0 github.com/golang-migrate/migrate/v4 v4.19.1 - github.com/google/uuid v1.6.0 github.com/joho/godotenv v1.5.1 - github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 github.com/lib/pq v1.12.3 - golang.org/x/crypto v0.52.0 ) require ( @@ -39,6 +36,7 @@ require ( github.com/ugorji/go/codec v1.3.1 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect golang.org/x/arch v0.22.0 // indirect + golang.org/x/crypto v0.52.0 // indirect golang.org/x/net v0.54.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect diff --git a/backend/go.sum b/backend/go.sum index e28ed09..63ec109 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -63,16 +63,12 @@ github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjY github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 h1:QwWKgMY28TAXaDl+ExRDqGQltzXqN/xypdKP86niVn8= -github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728/go.mod h1:1fEHWurg7pvf5SG6XNE5Q8UZmOwex51Mkx3SLhrW5B4= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= diff --git a/docker-compose.yml b/docker-compose.yml index 82ec649..e77f6c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: ports: - "5432:5432" environment: - POSTGRES_DB: vaultdrop + POSTGRES_DB: vaultdrop_dev POSTGRES_USER: vaultdrop POSTGRES_PASSWORD: vaultdrop volumes: @@ -35,7 +35,7 @@ services: # - "8080:8080" # environment: # - PORT=8080 - # - DATABASE_URL=postgres://vaultdrop:vaultdrop@postgres:5432/vaultdrop?sslmode=disable + # - DATABASE_URL=postgres://vaultdrop:vaultdrop@postgres:5432/vaultdrop_dev?sslmode=disable # volumes: # - ./backend/uploads:/app/uploads # depends_on: