From dd053a50c8f99d7a3ae926003152401aadf1cfb2 Mon Sep 17 00:00:00 2001 From: venture Date: Fri, 25 Sep 2026 18:49:57 +0200 Subject: [PATCH] V2 : flag serveur has_content (migration 000014, lazy healing, FileDto) --- AGENTS.md | 4 +- CHANGELOG.md | 1 + .../000014_resources_has_content.down.sql | 1 + .../000014_resources_has_content.up.sql | 6 ++ backend/handlers/files_test.go | 70 +++++++++++++++++++ backend/repository/resources.go | 46 +++++++++--- backend/service/resources.go | 49 +++++++++---- docs/api-v1.md | 1 + .../com/kazier/mobile/data/remote/dto/Dtos.kt | 6 ++ 9 files changed, 156 insertions(+), 28 deletions(-) create mode 100644 backend/db/migrations/000014_resources_has_content.down.sql create mode 100644 backend/db/migrations/000014_resources_has_content.up.sql diff --git a/AGENTS.md b/AGENTS.md index 555dd1d..9914314 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,8 +39,8 @@ Tests mobiles : JVM unit tests dans `app/src/test` (JUnit + Robolectric + Room i - `models/` — domain entities (users, devices, documents/resources, clients) - `service/` — business logic (permissions, upload, create folder, move, **sync outbox + snapshot**, bootstrap admin) - `handlers/` — HTTP handlers (health, devices register, **auth/login**, **users resolve + change password**, files CRUD/upload/search, folders, **sync/ops + sync/permissions, ocr/jobs, files/:id/content, files/:id/shares, files/:id/links** — réels) ; middleware `RequireAuth` -- `repository/` — Postgres persistence (`repository.Resources` : insert/list/get/soft-delete scoping `owner_id`, search, move, rename, root-name unique index, `ListOwned` pour le snapshot, `OpenContent` (bytes streaming), `ListShares`/`ListLinks` (owner-only) ; `repository.Devices` : Upsert, Exists, MarkUser ; `repository.Operations` : trace outbox idempotente `(device_id, operation_id)` ; `repository.OcrJobs` : jobs queued→processing→done/failed ; `repository.Users` : GetByUsernameNormalized, ResolveExact, UpdatePassword ; `repository.Shares` : Upsert/Revoke, CreateLink/RevokeLink, ListActiveForResource, ListActiveLinksForResource, snapshot `ListEffectivePermissions`) ; IDs sont TEXT 32-hex, `NewID()` = `crypto/rand` 16 octets hex (jamais UUID conversion, cf. `docs/api-v1.md`) -- `db/` — package migrations (`golang-migrate/v4`, embarquées via `embed` dans `db/migrations/*.sql`, 000001→000013) : `db.MigrateDatabase(url)` au boot du serveur ; test harness `db/migrations_test.go` (up → assertions schéma → down, `TEST_DATABASE_URL`, skip si PG indisponible) ; `dbtest/` — helper cross-package pour les tests repo/handlers (crée la DB test si absente, reset schema, migrate ; skip si PG down) +- `repository/` — Postgres persistence (`repository.Resources` : insert/list/get/soft-delete scoping `owner_id`, search, move, rename, root-name unique index, `ListOwned` pour le snapshot, `OpenContent` (bytes streaming + lazy healing `has_content`), `ListShares`/`ListLinks` (owner-only), `SetHasContent` ; `repository.Devices` : Upsert, Exists, MarkUser ; `repository.Operations` : trace outbox idempotente `(device_id, operation_id)` ; `repository.OcrJobs` : jobs queued→processing→done/failed ; `repository.Users` : GetByUsernameNormalized, ResolveExact, UpdatePassword ; `repository.Shares` : Upsert/Revoke, CreateLink/RevokeLink, ListActiveForResource, ListActiveLinksForResource, snapshot `ListEffectivePermissions`) ; IDs sont TEXT 32-hex, `NewID()` = `crypto/rand` 16 octets hex (jamais UUID conversion, cf. `docs/api-v1.md`) +- `db/` — package migrations (`golang-migrate/v4`, embarquées via `embed` dans `db/migrations/*.sql`, 000001→000014) : `db.MigrateDatabase(url)` au boot du serveur ; test harness `db/migrations_test.go` (up → assertions schéma → down, `TEST_DATABASE_URL`, skip si PG indisponible) ; `dbtest/` — helper cross-package pour les tests repo/handlers (crée la DB test si absente, reset schema, migrate ; skip si PG down) - `ocr/` — OCR engine behind an interface (Tesseract system call, `OCR_LANG` défaut `fra+eng`) - `pkg/api/` — response helpers (`response.go`) ; `pkg/auth/` — tokens **paseto v4-local** (subject = `user_id`, claim = `device_id`, TTL 7j, voir `docs/api-v1.md`) ; `pkg/passwd/` — hashing/vérification argon2id (comparaison `subtle`, délai égalisé) - File uploads stored in `backend/uploads/` diff --git a/CHANGELOG.md b/CHANGELOG.md index 201ce6f..1dbca2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Le format suit [Keep a Changelog](https://keepachangelog.com/fr/1.1.0/) et le ve ### Ajouté - Upload V2 des octets : worker `BytesUploadWorker` (fichier par fichier, overwrite ciblée idempotente, cap 3 échecs, backoff), marqueurs `bytes_uploaded`/`bytes_upload_attempts` (Room v17, migration 16→17), multipart SAF mutualisé (`SafMultipart.kt`). +- Flag serveur `has_content` (migration `000014`, lazy healing : TRUE à l'upload / au `OpenContent` réussi, FALSE si octets disparus), exposé en `FileDto.hasContent` (persistance Room prévue en V2-2). ## [1.0.0] - 2026-09-25 diff --git a/backend/db/migrations/000014_resources_has_content.down.sql b/backend/db/migrations/000014_resources_has_content.down.sql new file mode 100644 index 0000000..2c3eb46 --- /dev/null +++ b/backend/db/migrations/000014_resources_has_content.down.sql @@ -0,0 +1 @@ +ALTER TABLE resources DROP COLUMN IF EXISTS has_content; diff --git a/backend/db/migrations/000014_resources_has_content.up.sql b/backend/db/migrations/000014_resources_has_content.up.sql new file mode 100644 index 0000000..e3aad33 --- /dev/null +++ b/backend/db/migrations/000014_resources_has_content.up.sql @@ -0,0 +1,6 @@ +-- V2 placements : `has_content` = « octets vus présents » (lazy healing). +-- TRUE à chaque upload (nouveau ou ciblé), healen FALSE→TRUE au premier +-- `OpenContent` réussi, TRUE→FALSE quand les octets ont disparu sous +-- UPLOAD_DIR. Les lignes existantes partent à FALSE (inconnu, pas absent) : +-- le premier accès les heal, aucun scan au boot. +ALTER TABLE resources ADD COLUMN has_content BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/backend/handlers/files_test.go b/backend/handlers/files_test.go index d16021a..0e0c64b 100644 --- a/backend/handlers/files_test.go +++ b/backend/handlers/files_test.go @@ -7,6 +7,8 @@ import ( "mime/multipart" "net/http" "net/http/httptest" + "os" + "path/filepath" "testing" "github.com/gin-gonic/gin" @@ -748,6 +750,74 @@ func TestFilesContentStreaming(t *testing.T) { } } +func TestFilesHasContentLazy(t *testing.T) { + r, store, repo := setup(t) + ownerDevice := repository.NewID() + ownerToken, owner := registerAndLogin(t, r, repo, testUserUsername(ownerDevice, "hc"), "hascontent-password", ownerDevice) + + getHasContent := func(fileID, label string) bool { + t.Helper() + rec, _ := doRequest(t, r, http.MethodGet, "/api/v1/files/"+fileID, ownerToken, nil, "") + env := expectOK(t, rec, label) + var dto struct { + HasContent bool `json:"hasContent"` + } + if err := json.Unmarshal(env.Data, &dto); err != nil { + t.Fatalf("%s: unmarshal: %v", label, err) + } + return dto.HasContent + } + + // Upload avec octets → TRUE ; metadata-only → FALSE. + fileID := repository.NewID() + if err := repo.Resources.InsertFile(owner, fileID, "note.txt", "", 0, nil, nil); err != nil { + t.Fatalf("insert file: %v", err) + } + if getHasContent(fileID, "hascontent-bare") { + t.Errorf("metadata-only : hasContent attendu false") + } + rec := uploadMultipartToResource(t, r, ownerToken, fileID, "note.txt", []byte("hello")) + if rec.Code != http.StatusOK { + t.Fatalf("upload ciblé: attendu 200, got %d body=%s", rec.Code, rec.Body.String()) + } + if !getHasContent(fileID, "hascontent-uploaded") { + t.Errorf("après upload : hasContent attendu true") + } + + // Suppression manuelle sous UPLOAD_DIR puis GET content → 404 + heal FALSE. + matches, _ := filepath.Glob(filepath.Join(store.UploadDir, owner, fileID+".*")) + if len(matches) == 0 { + t.Fatalf("fichier physique introuvable sous %s", store.UploadDir) + } + if err := os.Remove(matches[0]); err != nil { + t.Fatalf("remove physique: %v", err) + } + rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/files/"+fileID+"/content", ownerToken, nil, "") + expectError(t, rec, http.StatusNotFound, "NOT_FOUND", "hascontent-missing-bytes") + if getHasContent(fileID, "hascontent-healed-down") { + t.Errorf("octets disparus : hasContent attendu false après heal") + } + + // Octets replacés à la main sur une ligne FALSE → GET content 200 + heal TRUE. + bareID := repository.NewID() + if err := repo.Resources.InsertFile(owner, bareID, "manuel.txt", "", 0, nil, nil); err != nil { + t.Fatalf("insert bare: %v", err) + } + if err := os.MkdirAll(filepath.Join(store.UploadDir, owner), 0o700); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(store.UploadDir, owner, bareID+".txt"), []byte("hi"), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/files/"+bareID+"/content", ownerToken, nil, "") + if rec.Code != http.StatusOK || rec.Body.String() != "hi" { + t.Fatalf("content-heal-up: attendu 200 hi, got %d %q", rec.Code, rec.Body.String()) + } + if !getHasContent(bareID, "hascontent-healed-up") { + t.Errorf("octets constatés : hasContent attendu true après heal") + } +} + func TestFilesSharesLinksListing(t *testing.T) { r, _, repo := setup(t) ownerDevice := repository.NewID() diff --git a/backend/repository/resources.go b/backend/repository/resources.go index ed87a5d..3bd6788 100644 --- a/backend/repository/resources.go +++ b/backend/repository/resources.go @@ -52,14 +52,15 @@ func isForeignKeyViolation(err error) bool { } type FileRow struct { - ID string - Name string - Size int64 - MimeType string - FolderID string - CreatedAt time.Time - UpdatedAt time.Time - OcrText string + ID string + Name string + Size int64 + MimeType string + FolderID string + CreatedAt time.Time + UpdatedAt time.Time + OcrText string + HasContent bool } type FolderRow struct { @@ -172,11 +173,11 @@ func (r *Resources) InsertFolder(ownerID, id, name, parentResourceID string) err return r.insert(ownerID, id, name, parentResourceID, "folder", 0, nil, nil) } -const fileColumns = `resource_id, name, size_bytes, COALESCE(mime_type, ''), COALESCE(parent_id, ''), created_at, updated_at, COALESCE(ocr_text, '')` +const fileColumns = `resource_id, name, size_bytes, COALESCE(mime_type, ''), COALESCE(parent_id, ''), created_at, updated_at, COALESCE(ocr_text, ''), COALESCE(has_content, FALSE)` func (r *Resources) scanFile(scan func(...any) error) (FileRow, error) { var row FileRow - err := scan(&row.ID, &row.Name, &row.Size, &row.MimeType, &row.FolderID, &row.CreatedAt, &row.UpdatedAt, &row.OcrText) + err := scan(&row.ID, &row.Name, &row.Size, &row.MimeType, &row.FolderID, &row.CreatedAt, &row.UpdatedAt, &row.OcrText, &row.HasContent) return row, err } @@ -500,9 +501,10 @@ func (r *Resources) UpdateNameByID(resourceID, name string) error { // (size_bytes / mime_type / extension) after a targeted re-upload — used so a // resource created metadata-only (outbox create_resource) receives its bytes // for server-side OCR. Absent / not owned → ErrNotFound. Never renames or moves. +// Des octets viennent d'arriver : `has_content` passe à TRUE (V2 lazy). func (r *Resources) UpdatePhysical(ownerID, resourceID string, size int64, mimeType, extension *string) error { result, err := r.DB.Exec( - `UPDATE resources SET size_bytes = $3, mime_type = $4, extension = $5, updated_at = NOW() + `UPDATE resources SET size_bytes = $3, mime_type = $4, extension = $5, has_content = TRUE, updated_at = NOW() WHERE resource_id = $1 AND user_id = $2 AND type = 'file' AND deleted_at IS NULL`, resourceID, ownerID, size, mimeType, extension, ) @@ -519,6 +521,28 @@ func (r *Resources) UpdatePhysical(ownerID, resourceID string, size int64, mimeT return nil } +// SetHasContent flips the V2 `has_content` flag (lazy healing) : TRUE quand +// des octets sont constatés (upload, OpenContent réussi), FALSE quand ils ont +// disparu sous UPLOAD_DIR. Best-effort côté service (jamais d'erreur vers le +// client). Absent / not owned → ErrNotFound (ignoré par l'appelant). +func (r *Resources) SetHasContent(ownerID, resourceID string, has bool) error { + result, err := r.DB.Exec( + `UPDATE resources SET has_content = $3, updated_at = NOW() + WHERE resource_id = $1 AND user_id = $2 AND type = 'file' AND deleted_at IS NULL`, + resourceID, ownerID, has, + ) + if err != nil { + return err + } + affected, err := result.RowsAffected() + if err != nil { + return err + } + if affected == 0 { + return ErrNotFound + } + return nil +} // UpdateOcrText writes the server-computed OCR extract onto the resource // (owner-scoped, bumping updated_at so the change flows through the sync // snapshot). The text is then readable by every device of the user AND by diff --git a/backend/service/resources.go b/backend/service/resources.go index 4c19597..8b230f2 100644 --- a/backend/service/resources.go +++ b/backend/service/resources.go @@ -24,14 +24,15 @@ var ErrInvalidUpload = errors.New("invalid upload name") // FileDTO serializes exactly as mobile/api/types.ts FileDto. type FileDTO struct { - ID string `json:"id"` - Name string `json:"name"` - Size int64 `json:"size"` - MimeType string `json:"mimeType,omitempty"` - FolderID string `json:"folderId,omitempty"` - Tags []string `json:"tags,omitempty"` - CreatedAt string `json:"createdAt,omitempty"` - UpdatedAt string `json:"updatedAt,omitempty"` + ID string `json:"id"` + Name string `json:"name"` + Size int64 `json:"size"` + MimeType string `json:"mimeType,omitempty"` + FolderID string `json:"folderId,omitempty"` + Tags []string `json:"tags,omitempty"` + HasContent bool `json:"hasContent"` + CreatedAt string `json:"createdAt,omitempty"` + UpdatedAt string `json:"updatedAt,omitempty"` } // FolderDTO serializes exactly as mobile/api/types.ts FolderDto. @@ -201,8 +202,21 @@ func (s *Resources) OpenContent(userID, resourceID string) (ContentInfo, error) if mime == "" { mime = "application/octet-stream" } + // Octets constatés : heal `has_content` → TRUE (V2 lazy, best-effort). + if !row.HasContent { + if herr := s.Repo.SetHasContent(ownerID, resourceID, true); herr != nil { + log.Printf("content: heal has_content(%s): %v", resourceID, herr) + } + } return ContentInfo{Path: resolved, Name: row.Name, MimeType: mime, Size: fi.Size()}, nil } + // Aucun octet mais flag TRUE (suppression manuelle sous UPLOAD_DIR) : + // heal → FALSE (V2 lazy, best-effort). + if row.HasContent { + if herr := s.Repo.SetHasContent(ownerID, resourceID, false); herr != nil { + log.Printf("content: heal has_content(%s): %v", resourceID, herr) + } + } return ContentInfo{}, repository.ErrNotFound } @@ -327,6 +341,10 @@ func (s *Resources) Upload(ownerID string, file *multipart.FileHeader, folderID, err = s.Repo.UpdatePhysical(ownerID, resourceID, written, &mimeType, extPtr) } else { err = s.Repo.InsertFile(ownerID, resourceID, name, folderID, written, &mimeType, extPtr) + if err == nil { + // Upload avec octets : `has_content` TRUE (V2 lazy). + err = s.Repo.SetHasContent(ownerID, resourceID, true) + } } if err != nil { // Best-effort : supprime le nouveau fichier sauf s'il préexistait à l'identique. @@ -422,12 +440,13 @@ func copyMultipartAtomic(file *multipart.FileHeader, destDir, destPath string, m func toFileDTO(row repository.FileRow) FileDTO { return FileDTO{ - ID: row.ID, - Name: row.Name, - Size: row.Size, - MimeType: row.MimeType, - FolderID: row.FolderID, - CreatedAt: row.CreatedAt.UTC().Format(time.RFC3339), - UpdatedAt: row.UpdatedAt.UTC().Format(time.RFC3339), + ID: row.ID, + Name: row.Name, + Size: row.Size, + MimeType: row.MimeType, + FolderID: row.FolderID, + HasContent: row.HasContent, + CreatedAt: row.CreatedAt.UTC().Format(time.RFC3339), + UpdatedAt: row.UpdatedAt.UTC().Format(time.RFC3339), } } diff --git a/docs/api-v1.md b/docs/api-v1.md index a040e11..972d5dd 100644 --- a/docs/api-v1.md +++ b/docs/api-v1.md @@ -62,6 +62,7 @@ type FileDto = { mimeType?: string | null; folderId?: string | null; // resource_id 32-hex tags?: string[]; + hasContent: boolean; // V2 : octets vus présents (lazy healing, FALSE = jamais constatés) createdAt?: string; updatedAt?: string; }; diff --git a/mobile-kotlin/app/src/main/java/com/kazier/mobile/data/remote/dto/Dtos.kt b/mobile-kotlin/app/src/main/java/com/kazier/mobile/data/remote/dto/Dtos.kt index e435a1e..6f6d6f5 100644 --- a/mobile-kotlin/app/src/main/java/com/kazier/mobile/data/remote/dto/Dtos.kt +++ b/mobile-kotlin/app/src/main/java/com/kazier/mobile/data/remote/dto/Dtos.kt @@ -45,6 +45,12 @@ data class FileDto( @Json(name = "mimeType") val mimeType: String? = null, @Json(name = "folderId") val folderId: String? = null, @Json(name = "tags") val tags: List? = null, + /** + * V2 : octets vus présents côté serveur (lazy healing — `FALSE` = + * jamais constatés, pas « certifié absent »). Persistance Room prévue + * en V2-2 (avec `wanted_offline`). + */ + @Json(name = "hasContent") val hasContent: Boolean = false, @Json(name = "createdAt") val createdAt: String? = null, @Json(name = "updatedAt") val updatedAt: String? = null, )