V2 : flag serveur has_content (migration 000014, lazy healing, FileDto)

This commit is contained in:
venture
2026-09-25 18:49:57 +02:00
parent c77af98a68
commit dd053a50c8
9 changed files with 156 additions and 28 deletions
+2 -2
View File
@@ -39,8 +39,8 @@ Tests mobiles : JVM unit tests dans `app/src/test` (JUnit + Robolectric + Room i
- `models/` — domain entities (users, devices, documents/resources, clients)
- `service/` — business logic (permissions, upload, create folder, move, **sync outbox + snapshot**, bootstrap admin)
- `handlers/` — HTTP handlers (health, devices register, **auth/login**, **users resolve + change password**, files CRUD/upload/search, folders, **sync/ops + sync/permissions, ocr/jobs, files/:id/content, files/:id/shares, files/:id/links** — réels) ; middleware `RequireAuth`
- `repository/` — Postgres persistence (`repository.Resources` : insert/list/get/soft-delete scoping `owner_id`, search, move, rename, root-name unique index, `ListOwned` pour le snapshot, `OpenContent` (bytes streaming), `ListShares`/`ListLinks` (owner-only) ; `repository.Devices` : Upsert, Exists, MarkUser ; `repository.Operations` : trace outbox idempotente `(device_id, operation_id)` ; `repository.OcrJobs` : jobs queued→processing→done/failed ; `repository.Users` : GetByUsernameNormalized, ResolveExact, UpdatePassword ; `repository.Shares` : Upsert/Revoke, CreateLink/RevokeLink, ListActiveForResource, ListActiveLinksForResource, snapshot `ListEffectivePermissions`) ; IDs sont TEXT 32-hex, `NewID()` = `crypto/rand` 16 octets hex (jamais UUID conversion, cf. `docs/api-v1.md`)
- `db/` — package migrations (`golang-migrate/v4`, embarquées via `embed` dans `db/migrations/*.sql`, 000001→000013) : `db.MigrateDatabase(url)` au boot du serveur ; test harness `db/migrations_test.go` (up → assertions schéma → down, `TEST_DATABASE_URL`, skip si PG indisponible) ; `dbtest/` — helper cross-package pour les tests repo/handlers (crée la DB test si absente, reset schema, migrate ; skip si PG down)
- `repository/` — Postgres persistence (`repository.Resources` : insert/list/get/soft-delete scoping `owner_id`, search, move, rename, root-name unique index, `ListOwned` pour le snapshot, `OpenContent` (bytes streaming + lazy healing `has_content`), `ListShares`/`ListLinks` (owner-only), `SetHasContent` ; `repository.Devices` : Upsert, Exists, MarkUser ; `repository.Operations` : trace outbox idempotente `(device_id, operation_id)` ; `repository.OcrJobs` : jobs queued→processing→done/failed ; `repository.Users` : GetByUsernameNormalized, ResolveExact, UpdatePassword ; `repository.Shares` : Upsert/Revoke, CreateLink/RevokeLink, ListActiveForResource, ListActiveLinksForResource, snapshot `ListEffectivePermissions`) ; IDs sont TEXT 32-hex, `NewID()` = `crypto/rand` 16 octets hex (jamais UUID conversion, cf. `docs/api-v1.md`)
- `db/` — package migrations (`golang-migrate/v4`, embarquées via `embed` dans `db/migrations/*.sql`, 000001→000014) : `db.MigrateDatabase(url)` au boot du serveur ; test harness `db/migrations_test.go` (up → assertions schéma → down, `TEST_DATABASE_URL`, skip si PG indisponible) ; `dbtest/` — helper cross-package pour les tests repo/handlers (crée la DB test si absente, reset schema, migrate ; skip si PG down)
- `ocr/` — OCR engine behind an interface (Tesseract system call, `OCR_LANG` défaut `fra+eng`)
- `pkg/api/` — response helpers (`response.go`) ; `pkg/auth/` — tokens **paseto v4-local** (subject = `user_id`, claim = `device_id`, TTL 7j, voir `docs/api-v1.md`) ; `pkg/passwd/` — hashing/vérification argon2id (comparaison `subtle`, délai égalisé)
- File uploads stored in `backend/uploads/`
+1
View File
@@ -7,6 +7,7 @@ Le format suit [Keep a Changelog](https://keepachangelog.com/fr/1.1.0/) et le ve
### Ajouté
- Upload V2 des octets : worker `BytesUploadWorker` (fichier par fichier, overwrite ciblée idempotente, cap 3 échecs, backoff), marqueurs `bytes_uploaded`/`bytes_upload_attempts` (Room v17, migration 16→17), multipart SAF mutualisé (`SafMultipart.kt`).
- Flag serveur `has_content` (migration `000014`, lazy healing : TRUE à l'upload / au `OpenContent` réussi, FALSE si octets disparus), exposé en `FileDto.hasContent` (persistance Room prévue en V2-2).
## [1.0.0] - 2026-09-25
@@ -0,0 +1 @@
ALTER TABLE resources DROP COLUMN IF EXISTS has_content;
@@ -0,0 +1,6 @@
-- V2 placements : `has_content` = « octets vus présents » (lazy healing).
-- TRUE à chaque upload (nouveau ou ciblé), healen FALSE→TRUE au premier
-- `OpenContent` réussi, TRUE→FALSE quand les octets ont disparu sous
-- UPLOAD_DIR. Les lignes existantes partent à FALSE (inconnu, pas absent) :
-- le premier accès les heal, aucun scan au boot.
ALTER TABLE resources ADD COLUMN has_content BOOLEAN NOT NULL DEFAULT FALSE;
+70
View File
@@ -7,6 +7,8 @@ import (
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"github.com/gin-gonic/gin"
@@ -748,6 +750,74 @@ func TestFilesContentStreaming(t *testing.T) {
}
}
func TestFilesHasContentLazy(t *testing.T) {
r, store, repo := setup(t)
ownerDevice := repository.NewID()
ownerToken, owner := registerAndLogin(t, r, repo, testUserUsername(ownerDevice, "hc"), "hascontent-password", ownerDevice)
getHasContent := func(fileID, label string) bool {
t.Helper()
rec, _ := doRequest(t, r, http.MethodGet, "/api/v1/files/"+fileID, ownerToken, nil, "")
env := expectOK(t, rec, label)
var dto struct {
HasContent bool `json:"hasContent"`
}
if err := json.Unmarshal(env.Data, &dto); err != nil {
t.Fatalf("%s: unmarshal: %v", label, err)
}
return dto.HasContent
}
// Upload avec octets → TRUE ; metadata-only → FALSE.
fileID := repository.NewID()
if err := repo.Resources.InsertFile(owner, fileID, "note.txt", "", 0, nil, nil); err != nil {
t.Fatalf("insert file: %v", err)
}
if getHasContent(fileID, "hascontent-bare") {
t.Errorf("metadata-only : hasContent attendu false")
}
rec := uploadMultipartToResource(t, r, ownerToken, fileID, "note.txt", []byte("hello"))
if rec.Code != http.StatusOK {
t.Fatalf("upload ciblé: attendu 200, got %d body=%s", rec.Code, rec.Body.String())
}
if !getHasContent(fileID, "hascontent-uploaded") {
t.Errorf("après upload : hasContent attendu true")
}
// Suppression manuelle sous UPLOAD_DIR puis GET content → 404 + heal FALSE.
matches, _ := filepath.Glob(filepath.Join(store.UploadDir, owner, fileID+".*"))
if len(matches) == 0 {
t.Fatalf("fichier physique introuvable sous %s", store.UploadDir)
}
if err := os.Remove(matches[0]); err != nil {
t.Fatalf("remove physique: %v", err)
}
rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/files/"+fileID+"/content", ownerToken, nil, "")
expectError(t, rec, http.StatusNotFound, "NOT_FOUND", "hascontent-missing-bytes")
if getHasContent(fileID, "hascontent-healed-down") {
t.Errorf("octets disparus : hasContent attendu false après heal")
}
// Octets replacés à la main sur une ligne FALSE → GET content 200 + heal TRUE.
bareID := repository.NewID()
if err := repo.Resources.InsertFile(owner, bareID, "manuel.txt", "", 0, nil, nil); err != nil {
t.Fatalf("insert bare: %v", err)
}
if err := os.MkdirAll(filepath.Join(store.UploadDir, owner), 0o700); err != nil {
t.Fatalf("mkdir: %v", err)
}
if err := os.WriteFile(filepath.Join(store.UploadDir, owner, bareID+".txt"), []byte("hi"), 0o600); err != nil {
t.Fatalf("write: %v", err)
}
rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/files/"+bareID+"/content", ownerToken, nil, "")
if rec.Code != http.StatusOK || rec.Body.String() != "hi" {
t.Fatalf("content-heal-up: attendu 200 hi, got %d %q", rec.Code, rec.Body.String())
}
if !getHasContent(bareID, "hascontent-healed-up") {
t.Errorf("octets constatés : hasContent attendu true après heal")
}
}
func TestFilesSharesLinksListing(t *testing.T) {
r, _, repo := setup(t)
ownerDevice := repository.NewID()
+35 -11
View File
@@ -52,14 +52,15 @@ func isForeignKeyViolation(err error) bool {
}
type FileRow struct {
ID string
Name string
Size int64
MimeType string
FolderID string
CreatedAt time.Time
UpdatedAt time.Time
OcrText string
ID string
Name string
Size int64
MimeType string
FolderID string
CreatedAt time.Time
UpdatedAt time.Time
OcrText string
HasContent bool
}
type FolderRow struct {
@@ -172,11 +173,11 @@ func (r *Resources) InsertFolder(ownerID, id, name, parentResourceID string) err
return r.insert(ownerID, id, name, parentResourceID, "folder", 0, nil, nil)
}
const fileColumns = `resource_id, name, size_bytes, COALESCE(mime_type, ''), COALESCE(parent_id, ''), created_at, updated_at, COALESCE(ocr_text, '')`
const fileColumns = `resource_id, name, size_bytes, COALESCE(mime_type, ''), COALESCE(parent_id, ''), created_at, updated_at, COALESCE(ocr_text, ''), COALESCE(has_content, FALSE)`
func (r *Resources) scanFile(scan func(...any) error) (FileRow, error) {
var row FileRow
err := scan(&row.ID, &row.Name, &row.Size, &row.MimeType, &row.FolderID, &row.CreatedAt, &row.UpdatedAt, &row.OcrText)
err := scan(&row.ID, &row.Name, &row.Size, &row.MimeType, &row.FolderID, &row.CreatedAt, &row.UpdatedAt, &row.OcrText, &row.HasContent)
return row, err
}
@@ -500,9 +501,10 @@ func (r *Resources) UpdateNameByID(resourceID, name string) error {
// (size_bytes / mime_type / extension) after a targeted re-upload — used so a
// resource created metadata-only (outbox create_resource) receives its bytes
// for server-side OCR. Absent / not owned → ErrNotFound. Never renames or moves.
// Des octets viennent d'arriver : `has_content` passe à TRUE (V2 lazy).
func (r *Resources) UpdatePhysical(ownerID, resourceID string, size int64, mimeType, extension *string) error {
result, err := r.DB.Exec(
`UPDATE resources SET size_bytes = $3, mime_type = $4, extension = $5, updated_at = NOW()
`UPDATE resources SET size_bytes = $3, mime_type = $4, extension = $5, has_content = TRUE, updated_at = NOW()
WHERE resource_id = $1 AND user_id = $2 AND type = 'file' AND deleted_at IS NULL`,
resourceID, ownerID, size, mimeType, extension,
)
@@ -519,6 +521,28 @@ func (r *Resources) UpdatePhysical(ownerID, resourceID string, size int64, mimeT
return nil
}
// SetHasContent flips the V2 `has_content` flag (lazy healing) : TRUE quand
// des octets sont constatés (upload, OpenContent réussi), FALSE quand ils ont
// disparu sous UPLOAD_DIR. Best-effort côté service (jamais d'erreur vers le
// client). Absent / not owned → ErrNotFound (ignoré par l'appelant).
func (r *Resources) SetHasContent(ownerID, resourceID string, has bool) error {
result, err := r.DB.Exec(
`UPDATE resources SET has_content = $3, updated_at = NOW()
WHERE resource_id = $1 AND user_id = $2 AND type = 'file' AND deleted_at IS NULL`,
resourceID, ownerID, has,
)
if err != nil {
return err
}
affected, err := result.RowsAffected()
if err != nil {
return err
}
if affected == 0 {
return ErrNotFound
}
return nil
}
// UpdateOcrText writes the server-computed OCR extract onto the resource
// (owner-scoped, bumping updated_at so the change flows through the sync
// snapshot). The text is then readable by every device of the user AND by
+34 -15
View File
@@ -24,14 +24,15 @@ var ErrInvalidUpload = errors.New("invalid upload name")
// FileDTO serializes exactly as mobile/api/types.ts FileDto.
type FileDTO struct {
ID string `json:"id"`
Name string `json:"name"`
Size int64 `json:"size"`
MimeType string `json:"mimeType,omitempty"`
FolderID string `json:"folderId,omitempty"`
Tags []string `json:"tags,omitempty"`
CreatedAt string `json:"createdAt,omitempty"`
UpdatedAt string `json:"updatedAt,omitempty"`
ID string `json:"id"`
Name string `json:"name"`
Size int64 `json:"size"`
MimeType string `json:"mimeType,omitempty"`
FolderID string `json:"folderId,omitempty"`
Tags []string `json:"tags,omitempty"`
HasContent bool `json:"hasContent"`
CreatedAt string `json:"createdAt,omitempty"`
UpdatedAt string `json:"updatedAt,omitempty"`
}
// FolderDTO serializes exactly as mobile/api/types.ts FolderDto.
@@ -201,8 +202,21 @@ func (s *Resources) OpenContent(userID, resourceID string) (ContentInfo, error)
if mime == "" {
mime = "application/octet-stream"
}
// Octets constatés : heal `has_content` → TRUE (V2 lazy, best-effort).
if !row.HasContent {
if herr := s.Repo.SetHasContent(ownerID, resourceID, true); herr != nil {
log.Printf("content: heal has_content(%s): %v", resourceID, herr)
}
}
return ContentInfo{Path: resolved, Name: row.Name, MimeType: mime, Size: fi.Size()}, nil
}
// Aucun octet mais flag TRUE (suppression manuelle sous UPLOAD_DIR) :
// heal → FALSE (V2 lazy, best-effort).
if row.HasContent {
if herr := s.Repo.SetHasContent(ownerID, resourceID, false); herr != nil {
log.Printf("content: heal has_content(%s): %v", resourceID, herr)
}
}
return ContentInfo{}, repository.ErrNotFound
}
@@ -327,6 +341,10 @@ func (s *Resources) Upload(ownerID string, file *multipart.FileHeader, folderID,
err = s.Repo.UpdatePhysical(ownerID, resourceID, written, &mimeType, extPtr)
} else {
err = s.Repo.InsertFile(ownerID, resourceID, name, folderID, written, &mimeType, extPtr)
if err == nil {
// Upload avec octets : `has_content` TRUE (V2 lazy).
err = s.Repo.SetHasContent(ownerID, resourceID, true)
}
}
if err != nil {
// Best-effort : supprime le nouveau fichier sauf s'il préexistait à l'identique.
@@ -422,12 +440,13 @@ func copyMultipartAtomic(file *multipart.FileHeader, destDir, destPath string, m
func toFileDTO(row repository.FileRow) FileDTO {
return FileDTO{
ID: row.ID,
Name: row.Name,
Size: row.Size,
MimeType: row.MimeType,
FolderID: row.FolderID,
CreatedAt: row.CreatedAt.UTC().Format(time.RFC3339),
UpdatedAt: row.UpdatedAt.UTC().Format(time.RFC3339),
ID: row.ID,
Name: row.Name,
Size: row.Size,
MimeType: row.MimeType,
FolderID: row.FolderID,
HasContent: row.HasContent,
CreatedAt: row.CreatedAt.UTC().Format(time.RFC3339),
UpdatedAt: row.UpdatedAt.UTC().Format(time.RFC3339),
}
}
+1
View File
@@ -62,6 +62,7 @@ type FileDto = {
mimeType?: string | null;
folderId?: string | null; // resource_id 32-hex
tags?: string[];
hasContent: boolean; // V2 : octets vus présents (lazy healing, FALSE = jamais constatés)
createdAt?: string;
updatedAt?: string;
};
@@ -45,6 +45,12 @@ data class FileDto(
@Json(name = "mimeType") val mimeType: String? = null,
@Json(name = "folderId") val folderId: String? = null,
@Json(name = "tags") val tags: List<String>? = null,
/**
* V2 : octets vus présents côté serveur (lazy healing — `FALSE` =
* jamais constatés, pas « certifié absent »). Persistance Room prévue
* en V2-2 (avec `wanted_offline`).
*/
@Json(name = "hasContent") val hasContent: Boolean = false,
@Json(name = "createdAt") val createdAt: String? = null,
@Json(name = "updatedAt") val updatedAt: String? = null,
)