ocr is working

This commit is contained in:
m
2026-09-16 12:44:48 +02:00
parent 050434325a
commit 906d48702c
21 changed files with 595 additions and 33 deletions
+31 -3
View File
@@ -1,9 +1,15 @@
package main package main
import ( import (
"context"
"errors"
"fmt" "fmt"
"log" "log"
"net/http"
"os" "os"
"os/signal"
"syscall"
"time"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/vaultdrop/backend/config" "github.com/vaultdrop/backend/config"
@@ -64,8 +70,30 @@ func main() {
) )
handlers.Ocr = service.NewOcr(repo, cfg.UploadDir, cfg.OcrLang, ocr.NewTesseract()) handlers.Ocr = service.NewOcr(repo, cfg.UploadDir, cfg.OcrLang, ocr.NewTesseract())
if err := newRouter().Run(fmt.Sprintf(":%d", cfg.Port)); err != nil { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
log.Fatalln(err) defer stop()
}
// Worker OCR borné (OCR_WORKERS) : file `ocr_jobs`, hors request handlers.
go handlers.Ocr.Run(ctx, cfg.OcrWorkers)
log.Printf("ocr worker démarré (%d workers)", cfg.OcrWorkers)
srv := &http.Server{
Addr: fmt.Sprintf(":%d", cfg.Port),
Handler: newRouter(),
}
go func() {
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Fatalf("server: %v", err)
}
}()
<-ctx.Done()
stop()
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := srv.Shutdown(shutdownCtx); err != nil {
log.Printf("shutdown: %v", err)
}
log.Println("arrêt propre")
} }
+7
View File
@@ -13,6 +13,7 @@ type ApplicationConfig struct {
UploadDir string UploadDir string
MaxFileSizeMB int64 MaxFileSizeMB int64
OcrLang string OcrLang string
OcrWorkers int
AuthSecret string AuthSecret string
AdminUsername string AdminUsername string
AdminPassword string AdminPassword string
@@ -33,12 +34,18 @@ func LoadApplicationConfig() (*ApplicationConfig, error) {
return nil, err return nil, err
} }
ocrWorkers, err := getInt("OCR_WORKERS", 2)
if err != nil {
return nil, err
}
return &ApplicationConfig{ return &ApplicationConfig{
Port: port, Port: port,
DatabaseURL: get("DATABASE_URL", "postgres://vaultdrop:vaultdrop@localhost:5432/vaultdrop_dev?sslmode=disable"), DatabaseURL: get("DATABASE_URL", "postgres://vaultdrop:vaultdrop@localhost:5432/vaultdrop_dev?sslmode=disable"),
UploadDir: get("UPLOAD_DIR", "./uploads"), UploadDir: get("UPLOAD_DIR", "./uploads"),
MaxFileSizeMB: int64(maxSize), MaxFileSizeMB: int64(maxSize),
OcrLang: get("OCR_LANG", "fra+eng"), OcrLang: get("OCR_LANG", "fra+eng"),
OcrWorkers: ocrWorkers,
AuthSecret: get("AUTH_SECRET", "dev-secret-change-me"), AuthSecret: get("AUTH_SECRET", "dev-secret-change-me"),
AdminUsername: get("ADMIN_USERNAME", ""), AdminUsername: get("ADMIN_USERNAME", ""),
AdminPassword: get("ADMIN_PASSWORD", ""), AdminPassword: get("ADMIN_PASSWORD", ""),
+6 -2
View File
@@ -9,7 +9,7 @@ func clearEnv(t *testing.T) {
t.Helper() t.Helper()
for _, name := range []string{ for _, name := range []string{
"PORT", "DATABASE_URL", "UPLOAD_DIR", "MAX_FILE_SIZE_MB", "PORT", "DATABASE_URL", "UPLOAD_DIR", "MAX_FILE_SIZE_MB",
"OCR_LANG", "AUTH_SECRET", "ADMIN_USERNAME", "ADMIN_PASSWORD", "OCR_LANG", "OCR_WORKERS", "AUTH_SECRET", "ADMIN_USERNAME", "ADMIN_PASSWORD",
} { } {
t.Setenv(name, "") t.Setenv(name, "")
} }
@@ -37,6 +37,9 @@ func TestLoadApplicationDefaults(t *testing.T) {
if cfg.OcrLang != "fra+eng" { if cfg.OcrLang != "fra+eng" {
t.Errorf("OcrLang défaut inattendu: %q", cfg.OcrLang) t.Errorf("OcrLang défaut inattendu: %q", cfg.OcrLang)
} }
if cfg.OcrWorkers != 2 {
t.Errorf("OcrWorkers = %d, attendu 2", cfg.OcrWorkers)
}
if cfg.AuthSecret != "dev-secret-change-me" { if cfg.AuthSecret != "dev-secret-change-me" {
t.Errorf("AuthSecret défaut inattendu: %q", cfg.AuthSecret) t.Errorf("AuthSecret défaut inattendu: %q", cfg.AuthSecret)
} }
@@ -52,6 +55,7 @@ func TestLoadApplicationEnvOverrides(t *testing.T) {
t.Setenv("DATABASE_URL", "postgres://u:p@host:5433/db?sslmode=disable") t.Setenv("DATABASE_URL", "postgres://u:p@host:5433/db?sslmode=disable")
t.Setenv("UPLOAD_DIR", "/tmp/up") t.Setenv("UPLOAD_DIR", "/tmp/up")
t.Setenv("OCR_LANG", "eng") t.Setenv("OCR_LANG", "eng")
t.Setenv("OCR_WORKERS", "4")
t.Setenv("AUTH_SECRET", "super-secret") t.Setenv("AUTH_SECRET", "super-secret")
t.Setenv("ADMIN_USERNAME", "root") t.Setenv("ADMIN_USERNAME", "root")
t.Setenv("ADMIN_PASSWORD", "toor") t.Setenv("ADMIN_PASSWORD", "toor")
@@ -61,7 +65,7 @@ func TestLoadApplicationEnvOverrides(t *testing.T) {
t.Fatalf("LoadApplicationConfig: %v", err) t.Fatalf("LoadApplicationConfig: %v", err)
} }
if cfg.Port != 9090 || cfg.MaxFileSizeMB != 120 || cfg.DatabaseURL != "postgres://u:p@host:5433/db?sslmode=disable" || if cfg.Port != 9090 || cfg.MaxFileSizeMB != 120 || cfg.DatabaseURL != "postgres://u:p@host:5433/db?sslmode=disable" ||
cfg.UploadDir != "/tmp/up" || cfg.OcrLang != "eng" || cfg.AuthSecret != "super-secret" || cfg.UploadDir != "/tmp/up" || cfg.OcrLang != "eng" || cfg.OcrWorkers != 4 || cfg.AuthSecret != "super-secret" ||
cfg.AdminUsername != "root" || cfg.AdminPassword != "toor" { cfg.AdminUsername != "root" || cfg.AdminPassword != "toor" {
t.Errorf("overrides non appliqués: %+v", cfg) t.Errorf("overrides non appliqués: %+v", cfg)
} }
+6
View File
@@ -42,6 +42,12 @@ func setupOcr(t *testing.T) (*gin.Engine, string, *repository.Repository) {
handlers.Store = store handlers.Store = store
handlers.Ocr = service.NewOcr(repo, uploadDir, "fra+eng", stubEngine{text: "HELLO OCR"}) handlers.Ocr = service.NewOcr(repo, uploadDir, "fra+eng", stubEngine{text: "HELLO OCR"})
// Worker OCR borné — le handler n'enfile plus que le job, le traitement
// tourne sur un worker de contexte.
ctx, cancel := context.WithCancel(context.Background())
go handlers.Ocr.Run(ctx, 1)
t.Cleanup(cancel)
gin.SetMode(gin.TestMode) gin.SetMode(gin.TestMode)
r := gin.New() r := gin.New()
handlers.RegisterRoutes(r) handlers.RegisterRoutes(r)
+47
View File
@@ -3,6 +3,7 @@ package repository
import ( import (
"database/sql" "database/sql"
"errors" "errors"
"time"
) )
// OcrJobRow maps ocr_jobs. // OcrJobRow maps ocr_jobs.
@@ -18,6 +19,9 @@ type OcrJobRow struct {
// ErrJobNotFound marks an OCR job absent or owned by another device. // ErrJobNotFound marks an OCR job absent or owned by another device.
var ErrJobNotFound = errors.New("ocr job not found") var ErrJobNotFound = errors.New("ocr job not found")
// ErrNoQueuedJobs marks an OCR queue drained (no job to claim).
var ErrNoQueuedJobs = errors.New("no queued ocr jobs")
type OcrJobs struct{ DB *sql.DB } type OcrJobs struct{ DB *sql.DB }
func (o *OcrJobs) Create(jobID, deviceID, fileID string) error { func (o *OcrJobs) Create(jobID, deviceID, fileID string) error {
@@ -62,6 +66,49 @@ func (o *OcrJobs) TouchProcessing(deviceID, jobID string) error {
return err return err
} }
// ClaimNext atomically picks the oldest queued job (FIFO) and moves it to
// `processing`. Safe for concurrent workers: `FOR UPDATE SKIP LOCKED` blocks
// the row as part of the same statement. Empty queue → ErrNoQueuedJobs.
func (o *OcrJobs) ClaimNext() (OcrJobRow, error) {
var row OcrJobRow
err := o.DB.QueryRow(
`WITH next AS (
SELECT job_id FROM ocr_jobs
WHERE status = 'queued'
ORDER BY created_at, job_id
LIMIT 1
FOR UPDATE SKIP LOCKED
)
UPDATE ocr_jobs
SET status = 'processing', started_at = NOW()
FROM next
WHERE ocr_jobs.job_id = next.job_id
RETURNING ocr_jobs.job_id, ocr_jobs.file_id, ocr_jobs.device_id, ocr_jobs.status`,
).Scan(&row.ID, &row.FileID, &row.DeviceID, &row.Status)
if err == sql.ErrNoRows {
return OcrJobRow{}, ErrNoQueuedJobs
}
if err != nil {
return OcrJobRow{}, err
}
return row, nil
}
// ResetStaleProcessing requeues jobs stuck in `processing` (worker crash /
// processus redémarré) et plus vieux que `olderThan`. Retourne le nb de jobs
// requeued.
func (o *OcrJobs) ResetStaleProcessing(olderThan time.Duration) (int64, error) {
res, err := o.DB.Exec(
`UPDATE ocr_jobs SET status = 'queued', started_at = NULL
WHERE status = 'processing' AND started_at < $1`,
time.Now().Add(-olderThan),
)
if err != nil {
return 0, err
}
return res.RowsAffected()
}
func (o *OcrJobs) Complete(deviceID, jobID, text string) error { func (o *OcrJobs) Complete(deviceID, jobID, text string) error {
_, err := o.DB.Exec( _, err := o.DB.Exec(
`UPDATE ocr_jobs SET status = 'done', text = NULLIF($3, ''), started_at = COALESCE(started_at, NOW()), completed_at = NOW() `UPDATE ocr_jobs SET status = 'done', text = NULLIF($3, ''), started_at = COALESCE(started_at, NOW()), completed_at = NOW()
+89
View File
@@ -3,6 +3,7 @@ package repository
import ( import (
"errors" "errors"
"testing" "testing"
"time"
"github.com/vaultdrop/backend/dbtest" "github.com/vaultdrop/backend/dbtest"
) )
@@ -90,3 +91,91 @@ func TestOcrJobsGetScopedByDevice(t *testing.T) {
t.Errorf("get par un autre device : attendu ErrJobNotFound, got %v", err) t.Errorf("get par un autre device : attendu ErrJobNotFound, got %v", err)
} }
} }
func TestOcrJobsClaimNextFifo(t *testing.T) {
repo, _, deviceID, fileID := seedOcrJobFixture(t)
var jobIDs []string
for i := 0; i < 3; i++ {
jobID := NewID()
if err := repo.OcrJobs.Create(jobID, deviceID, fileID); err != nil {
t.Fatalf("create %d: %v", i, err)
}
jobIDs = append(jobIDs, jobID)
}
for i, want := range jobIDs {
row, err := repo.OcrJobs.ClaimNext()
if err != nil {
t.Fatalf("claim %d: %v", i, err)
}
if row.ID != want {
t.Errorf("claim %d = %q, attendu %q (FIFO)", i, row.ID, want)
}
if row.Status != "processing" {
t.Errorf("claim %d status = %q, attendu processing", i, row.Status)
}
}
if _, err := repo.OcrJobs.ClaimNext(); !errors.Is(err, ErrNoQueuedJobs) {
t.Errorf("queue vide : attendu ErrNoQueuedJobs, got %v", err)
}
}
func TestOcrJobsClaimNextSkipsProcessingAndDone(t *testing.T) {
repo, _, deviceID, fileID := seedOcrJobFixture(t)
job1 := NewID()
job2 := NewID()
if err := repo.OcrJobs.Create(job1, deviceID, fileID); err != nil {
t.Fatalf("create job1: %v", err)
}
if err := repo.OcrJobs.Create(job2, deviceID, fileID); err != nil {
t.Fatalf("create job2: %v", err)
}
if err := repo.OcrJobs.TouchProcessing(deviceID, job2); err != nil {
t.Fatalf("touch job2 processing: %v", err)
}
row, err := repo.OcrJobs.ClaimNext()
if err != nil {
t.Fatalf("claim: %v", err)
}
if row.ID != job1 {
t.Errorf("claim = %q, attendu job1 %q", row.ID, job1)
}
}
func TestOcrJobsResetStaleProcessing(t *testing.T) {
repo, _, deviceID, fileID := seedOcrJobFixture(t)
jobID := NewID()
if err := repo.OcrJobs.Create(jobID, deviceID, fileID); err != nil {
t.Fatalf("create: %v", err)
}
if err := repo.OcrJobs.TouchProcessing(deviceID, jobID); err != nil {
t.Fatalf("touch processing: %v", err)
}
// Started_at = NOW : pas encore stale → aucun requeue.
if n, err := repo.OcrJobs.ResetStaleProcessing(time.Minute); err != nil {
t.Fatalf("reset: %v", err)
} else if n != 0 {
t.Errorf("reset récent : attendu 0, got %d", n)
}
// Vieillit le started_at puis re-reset → 1 requeue.
if _, err := repo.OcrJobs.DB.Exec(`UPDATE ocr_jobs SET started_at = NOW() - interval '2 minutes' WHERE job_id = $1`, jobID); err != nil {
t.Fatalf("vieillir started_at: %v", err)
}
if n, err := repo.OcrJobs.ResetStaleProcessing(time.Minute); err != nil {
t.Fatalf("reset stale: %v", err)
} else if n != 1 {
t.Errorf("reset stale : attendu 1, got %d", n)
}
row, err := repo.OcrJobs.Get(deviceID, jobID)
if err != nil {
t.Fatalf("get: %v", err)
}
if row.Status != "queued" {
t.Errorf("status après reset = %q, attendu queued", row.Status)
}
}
+15
View File
@@ -254,6 +254,21 @@ func (r *Resources) GetFileVisible(userID, resourceID string) (FileRow, error) {
return file, err return file, err
} }
// GetResourceOwner returns the owning user_id of any resource (file or folder).
// No-rows → ErrNotFound.
func (r *Resources) GetResourceOwner(resourceID string) (string, error) {
var ownerID string
err := r.DB.QueryRow(
`SELECT user_id FROM resources
WHERE resource_id = $1 AND deleted_at IS NULL`,
resourceID,
).Scan(&ownerID)
if errors.Is(err, sql.ErrNoRows) {
return "", ErrNotFound
}
return ownerID, err
}
// DeleteFile soft-deletes the file (deleted_at), returning its id. // DeleteFile soft-deletes the file (deleted_at), returning its id.
func (r *Resources) DeleteFile(ownerID, resourceID string) (string, error) { func (r *Resources) DeleteFile(ownerID, resourceID string) (string, error) {
result, err := r.DB.Exec( result, err := r.DB.Exec(
+72 -14
View File
@@ -2,13 +2,25 @@ package service
import ( import (
"context" "context"
"errors"
"log"
"path/filepath" "path/filepath"
"strings" "strings"
"sync"
"time"
"github.com/vaultdrop/backend/ocr" "github.com/vaultdrop/backend/ocr"
"github.com/vaultdrop/backend/repository" "github.com/vaultdrop/backend/repository"
) )
const (
// StaleClaimTimeout : un job `processing` plus vieux que ça est ré-enfilé
// au boot (worker mort / process redémarré).
StaleClaimTimeout = 15 * time.Minute
// QueuePollInterval : cadence de scrutation quand la file est vide.
QueuePollInterval = time.Second
)
// OcrJobDTO serializes exactly as mobile/api/types.ts OcrJob. // OcrJobDTO serializes exactly as mobile/api/types.ts OcrJob.
type OcrJobDTO struct { type OcrJobDTO struct {
ID string `json:"id"` ID string `json:"id"`
@@ -17,22 +29,24 @@ type OcrJobDTO struct {
Error string `json:"error,omitempty"` Error string `json:"error,omitempty"`
} }
// Ocr queues OCR jobs and processes them asynchronously (V1 : goroutine par // Ocr queues OCR jobs in `ocr_jobs` (status `queued`) and processes them
// job ; le client poll GET /ocr/jobs/:id toutes les 3s). // asynchronously through a bounded worker ([Run]) — Tesseract ne tourne jamais
// dans un request handler. Le client poll GET /ocr/jobs/:id toutes les 3s.
type Ocr struct { type Ocr struct {
Repository *repository.Repository Repository *repository.Repository
UploadDir string UploadDir string
Lang string Lang string
Engine ocr.Engine Engine ocr.Engine
StaleAfter time.Duration
} }
func NewOcr(repo *repository.Repository, uploadDir, lang string, engine ocr.Engine) *Ocr { func NewOcr(repo *repository.Repository, uploadDir, lang string, engine ocr.Engine) *Ocr {
return &Ocr{Repository: repo, UploadDir: uploadDir, Lang: lang, Engine: engine} return &Ocr{Repository: repo, UploadDir: uploadDir, Lang: lang, Engine: engine, StaleAfter: StaleClaimTimeout}
} }
// Create valide la ressource (ownership par user), met le job en file et // Create valide la ressource (ownership par user) et l'enfile uniquement
// lance le traitement. Le job reste scopé par device (le service génère les // (status `queued`) — le traitement est délégué au worker [Run] pour ne pas
// jobs du device courant ; l'outbox OCR n'est pas synchronisée entre devices). // stresser le serveur. Le job reste scopé par device.
func (o *Ocr) Create(userID, deviceID, fileID string) (OcrJobDTO, error) { func (o *Ocr) Create(userID, deviceID, fileID string) (OcrJobDTO, error) {
if _, err := o.Repository.Resources.GetFile(userID, fileID); err != nil { if _, err := o.Repository.Resources.GetFile(userID, fileID); err != nil {
return OcrJobDTO{}, err return OcrJobDTO{}, err
@@ -41,7 +55,6 @@ func (o *Ocr) Create(userID, deviceID, fileID string) (OcrJobDTO, error) {
if err := o.Repository.OcrJobs.Create(jobID, deviceID, fileID); err != nil { if err := o.Repository.OcrJobs.Create(jobID, deviceID, fileID); err != nil {
return OcrJobDTO{}, err return OcrJobDTO{}, err
} }
go o.process(userID, deviceID, jobID, fileID)
return OcrJobDTO{ID: jobID, Status: "queued"}, nil return OcrJobDTO{ID: jobID, Status: "queued"}, nil
} }
@@ -53,22 +66,67 @@ func (o *Ocr) Get(deviceID, jobID string) (OcrJobDTO, error) {
return toOcrJobDTO(row), nil return toOcrJobDTO(row), nil
} }
func (o *Ocr) process(userID, deviceID, jobID, fileID string) { // Run fait tourner le worker OCR : au démarrage il requeue les jobs laissés
ctx := context.Background() // en `processing` (crash), puis `workers` boucles consomment la file FIFO via
if err := o.Repository.OcrJobs.TouchProcessing(deviceID, jobID); err != nil { // ClaimNext (atomique, `FOR UPDATE SKIP LOCKED`). `workers` borne la
// concurrence Tesseract. Retourne quand ctx est annulé.
func (o *Ocr) Run(ctx context.Context, workers int) {
if workers < 1 {
workers = 1
}
if _, err := o.Repository.OcrJobs.ResetStaleProcessing(o.StaleAfter); err != nil {
log.Printf("ocr: requeue jobs stale: %v", err)
}
var wg sync.WaitGroup
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
o.workerLoop(ctx)
}()
}
wg.Wait()
}
// workerLoop consomme la file : un job `queued` → processing → done/failed.
// File vide → scrutation à QueuePollInterval jusqu'à annulation du ctx.
func (o *Ocr) workerLoop(ctx context.Context) {
for {
row, err := o.Repository.OcrJobs.ClaimNext()
if err != nil {
if !errors.Is(err, repository.ErrNoQueuedJobs) {
log.Printf("ocr: claim: %v", err)
}
select {
case <-ctx.Done():
return
case <-time.After(QueuePollInterval):
}
continue
}
o.process(ctx, row)
}
}
// process exécute l'extraction pour un job déjà réclamé (`processing`).
func (o *Ocr) process(ctx context.Context, row repository.OcrJobRow) {
userID, err := o.Repository.Resources.GetResourceOwner(row.FileID)
if err != nil {
_ = o.Repository.OcrJobs.Fail(row.DeviceID, row.ID, "file not readable")
return return
} }
path, err := o.physicalPath(userID, fileID) path, err := o.physicalPath(userID, row.FileID)
if err != nil { if err != nil {
_ = o.Repository.OcrJobs.Fail(deviceID, jobID, "file not readable") _ = o.Repository.OcrJobs.Fail(row.DeviceID, row.ID, "file not readable")
return return
} }
text, err := o.Engine.ExtractText(ctx, path, o.Lang) text, err := o.Engine.ExtractText(ctx, path, o.Lang)
if err != nil { if err != nil {
_ = o.Repository.OcrJobs.Fail(deviceID, jobID, err.Error()) _ = o.Repository.OcrJobs.Fail(row.DeviceID, row.ID, err.Error())
return return
} }
_ = o.Repository.OcrJobs.Complete(deviceID, jobID, text) _ = o.Repository.OcrJobs.Complete(row.DeviceID, row.ID, text)
} }
// physicalPath résout UPLOAD_DIR/<user_id>/<resource_id>.<ext> — l'ext est // physicalPath résout UPLOAD_DIR/<user_id>/<resource_id>.<ext> — l'ext est
+7 -2
View File
@@ -26,13 +26,18 @@ func (s stubEngine) ExtractText(_ context.Context, _ string, _ string) (string,
return s.text, nil return s.text, nil
} }
// newTestOcr builds an Ocr over a fresh DB and returns it with the ids. // newTestOcr builds an Ocr over a fresh DB, starts the bounded worker (1
// worker) and returns it with the ids.
func newTestOcr(t *testing.T, uploadDir string, engine ocr.Engine) (*Ocr, string, string, string) { func newTestOcr(t *testing.T, uploadDir string, engine ocr.Engine) (*Ocr, string, string, string) {
t.Helper() t.Helper()
s := newServiceStore(t) s := newServiceStore(t)
userID := mustCreateUser(t, s.Repository, "ocr-failed") userID := mustCreateUser(t, s.Repository, "ocr-failed")
deviceID := mustRegisterDevice(t, s.Repository, repository.NewID()) deviceID := mustRegisterDevice(t, s.Repository, repository.NewID())
return NewOcr(s.Repository, uploadDir, "fra+eng", engine), userID, deviceID, uploadDir o := NewOcr(s.Repository, uploadDir, "fra+eng", engine)
ctx, cancel := context.WithCancel(context.Background())
go o.Run(ctx, 1)
t.Cleanup(cancel)
return o, userID, deviceID, uploadDir
} }
// waitTillTerminal poll jusqu'à un statut terminal (done/failed). // waitTillTerminal poll jusqu'à un statut terminal (done/failed).
+1 -1
View File
@@ -75,7 +75,7 @@ type OcrJob = { id: string; status: OcrJobStatus; text?: string | null; error?:
## 5. OCR ## 5. OCR
- `POST /ocr/jobs { fileId }``OcrJob` immédiat (`status: queued`), traitement **asynchrone** (goroutine par job côté serveur, V1). - `POST /ocr/jobs { fileId }``OcrJob` immédiat (`status: queued`), traitement **asynchrone** par une **file serveur bornée** (`ocr_jobs` + worker `Ocr.Run`, `OCR_WORKERS` défaut 2) — le handler n'enfile **jamais** de traitement, Tesseract tourne hors request handler (FIFO, récupération des `processing` orphelins au boot).
- `GET /ocr/jobs/:id` → statut. Le mobile **poll toutes les 3s** jusqu'à `done`/`failed` (`hooks/useUpload.ts`). Cycle : `queued → processing → done | failed` ; `done` renvoie `text`, `failed` renvoie `error`. - `GET /ocr/jobs/:id` → statut. Le mobile **poll toutes les 3s** jusqu'à `done`/`failed` (`hooks/useUpload.ts`). Cycle : `queued → processing → done | failed` ; `done` renvoie `text`, `failed` renvoie `error`.
- Moteur : **Tesseract en appel système** (`ocr/tesseract.go`), langue `OCR_LANG` (défaut `fra+eng`). Les images sont passées directement à `tesseract` ; les **PDF** subissent une extraction du calque texte (`ledongthuc/pdf`, déjà en go.mod) — un PDF scanné produit un texte vide plutôt qu'un rendu/OCR (hors scope V1). - Moteur : **Tesseract en appel système** (`ocr/tesseract.go`), langue `OCR_LANG` (défaut `fra+eng`). Les images sont passées directement à `tesseract` ; les **PDF** subissent une extraction du calque texte (`ledongthuc/pdf`, déjà en go.mod) — un PDF scanné produit un texte vide plutôt qu'un rendu/OCR (hors scope V1).
- `fileId` inconnu/pas du user → `NOT_FOUND`. Fichier physique introuvable (ex. suppression manuelle sous `UPLOAD_DIR`) → job `failed` `"file not readable"`. Le job est créé par le device courant (`ocr_jobs.device_id`) mais la validation de la ressource est scopée par le **user**. - `fileId` inconnu/pas du user → `NOT_FOUND`. Fichier physique introuvable (ex. suppression manuelle sous `UPLOAD_DIR`) → job `failed` `"file not readable"`. Le job est créé par le device courant (`ocr_jobs.device_id`) mais la validation de la ressource est scopée par le **user**.
@@ -20,14 +20,16 @@ import com.vaultdrop.mobile.data.local.entity.UserPreferenceEntity
* v5: created_in_app sur folders ; v6: processed sur files (mode review) ; * v5: created_in_app sur folders ; v6: processed sur files (mode review) ;
* v7: pending_operations (outbox) ; v8: scan_sessions + scan_pages (scanner) ; * v7: pending_operations (outbox) ; v8: scan_sessions + scan_pages (scanner) ;
* v9: content sur files (corps des notes créées dans l'app) ; * v9: content sur files (corps des notes créées dans l'app) ;
* v10: ocr_text sur files (extrait OCR serveur persisté localement). * v10: ocr_text sur files (extrait OCR serveur persisté localement) ;
* v11: ocr_attempts + ocr_queued_at sur files (driver OCR auto côté client —
* compteur d'échecs terminaux ≤ 3, marqueur « en cours » de l'upload).
*/ */
@Database( @Database(
entities = [ entities = [
FolderEntity::class, UserPreferenceEntity::class, FileEntity::class, FolderEntity::class, UserPreferenceEntity::class, FileEntity::class,
PendingOperationEntity::class, ScanSessionEntity::class, ScanPageEntity::class, PendingOperationEntity::class, ScanSessionEntity::class, ScanPageEntity::class,
], ],
version = 10, version = 11,
exportSchema = false, exportSchema = false,
) )
abstract class AppDatabase : RoomDatabase() { abstract class AppDatabase : RoomDatabase() {
@@ -126,4 +126,33 @@ interface FileDao {
/** Persiste l'extrait OCR d'un fichier (résultat local du job serveur). */ /** Persiste l'extrait OCR d'un fichier (résultat local du job serveur). */
@Query("UPDATE files SET ocr_text = :text, updated_at = :updatedAt WHERE resource_id = :resourceId") @Query("UPDATE files SET ocr_text = :text, updated_at = :updatedAt WHERE resource_id = :resourceId")
suspend fun updateOcrText(resourceId: String, text: String?, updatedAt: Long) suspend fun updateOcrText(resourceId: String, text: String?, updatedAt: Long)
/** Candidate OCR auto : fichier `processed`, `local-cloud`, avec copie
* locale, format potentiellement extractible, jamais extrait, pas en cours
* et pas épuisé (< 3 échecs terminaux). Trié par dernier upload. */
@Query("""
SELECT * FROM files
WHERE "exists" = 1
AND processed = 1
AND sync_status = 'local-cloud'
AND uri IS NOT NULL
AND ocr_text IS NULL
AND ocr_attempts < 3
AND ocr_queued_at IS NULL
AND category IN ('PDF', 'IMAGE', 'TEXT')
ORDER BY COALESCE(last_modified, added_at) DESC, name ASC
""")
suspend fun getOcrAutoCandidates(): List<FileEntity>
/** Pose le marqueur « en cours » du driver OCR auto (upload en vol). */
@Query("UPDATE files SET ocr_queued_at = :queuedAt, updated_at = :now WHERE resource_id = :resourceId")
suspend fun markOcrQueued(resourceId: String, queuedAt: Long, now: Long)
/** Levé du marqueur + reset du compteur d'échecs (succès / erreur transitoire). */
@Query("UPDATE files SET ocr_queued_at = NULL, ocr_attempts = 0, updated_at = :now WHERE resource_id = :resourceId")
suspend fun clearOcrQueued(resourceId: String, now: Long)
/** Échec terminal : +1 tentative, marqueur levé (ne rejoue pas à l'infini). */
@Query("UPDATE files SET ocr_attempts = ocr_attempts + 1, ocr_queued_at = NULL, updated_at = :now WHERE resource_id = :resourceId")
suspend fun markOcrFailure(resourceId: String, now: Long)
} }
@@ -57,6 +57,10 @@ data class FileEntity(
val processed: Boolean = false, val processed: Boolean = false,
@ColumnInfo(name = "ocr_text") @ColumnInfo(name = "ocr_text")
val ocrText: String? = null, val ocrText: String? = null,
@ColumnInfo(name = "ocr_attempts")
val ocrAttempts: Int = 0,
@ColumnInfo(name = "ocr_queued_at")
val ocrQueuedAt: Long? = null,
@ColumnInfo(name = "added_at") @ColumnInfo(name = "added_at")
val addedAt: Long, val addedAt: Long,
@ColumnInfo(name = "updated_at") @ColumnInfo(name = "updated_at")
@@ -174,7 +174,14 @@ object Migrations {
} }
} }
private val MIGRATION_10_11 = object : Migration(10, 11) {
override fun migrate(db: SupportSQLiteDatabase) {
db.execSQL("ALTER TABLE `files` ADD COLUMN `ocr_attempts` INTEGER NOT NULL DEFAULT 0")
db.execSQL("ALTER TABLE `files` ADD COLUMN `ocr_queued_at` INTEGER")
}
}
val ALL: Array<Migration> = arrayOf( val ALL: Array<Migration> = arrayOf(
MIGRATION_1_2, MIGRATION_2_3, MIGRATION_3_4, MIGRATION_4_5, MIGRATION_5_6, MIGRATION_6_7, MIGRATION_7_8, MIGRATION_8_9, MIGRATION_9_10, MIGRATION_1_2, MIGRATION_2_3, MIGRATION_3_4, MIGRATION_4_5, MIGRATION_5_6, MIGRATION_6_7, MIGRATION_7_8, MIGRATION_8_9, MIGRATION_9_10, MIGRATION_10_11,
) )
} }
@@ -7,6 +7,7 @@ import androidx.core.net.toUri
import com.vaultdrop.mobile.data.local.dao.FileDao import com.vaultdrop.mobile.data.local.dao.FileDao
import com.vaultdrop.mobile.data.local.entity.FileEntity import com.vaultdrop.mobile.data.local.entity.FileEntity
import com.vaultdrop.mobile.data.remote.ApiClient import com.vaultdrop.mobile.data.remote.ApiClient
import com.vaultdrop.mobile.data.remote.ApiException
import com.vaultdrop.mobile.data.remote.dto.OcrJobDto import com.vaultdrop.mobile.data.remote.dto.OcrJobDto
import com.vaultdrop.mobile.data.remote.dto.OcrJobStatus import com.vaultdrop.mobile.data.remote.dto.OcrJobStatus
import dagger.hilt.android.qualifiers.ApplicationContext import dagger.hilt.android.qualifiers.ApplicationContext
@@ -74,11 +75,71 @@ class OcrRepository @Inject constructor(
fileDao.updateOcrText(resourceId, text?.takeIf { it.isNotBlank() }, System.currentTimeMillis()) fileDao.updateOcrText(resourceId, text?.takeIf { it.isNotBlank() }, System.currentTimeMillis())
} }
/**
* OCR auto d'un fichier (un seul à la fois) : marqueur « en cours », upload
* multipart ciblé, soumission du job serveur, poll jusqu'au terminal puis
* persistance. Le worker [OcrAutoWorker] appelle cette méthode fichier par
* fichier pour rester séquentiel — le traitement lourd reste la file
* bornée côté serveur.
*/
suspend fun processAuto(file: FileEntity): OcrAutoOutcome {
val now = System.currentTimeMillis()
fileDao.markOcrQueued(file.resourceId, now, now)
return try {
ensurePhysical(file)
val jobId = submit(file.resourceId)
val final = poll(jobId)
when (final.status) {
OcrJobStatus.DONE -> {
saveResult(file.resourceId, final.text)
fileDao.clearOcrQueued(file.resourceId, System.currentTimeMillis())
OcrAutoOutcome.Done
}
else -> {
fileDao.markOcrFailure(file.resourceId, System.currentTimeMillis())
OcrAutoOutcome.Failed
}
}
} catch (e: ApiException) {
when {
e.code == "NETWORK_ERROR" || e.httpCode >= 500 -> {
// Transitoire : levé le marqueur, WorkManager retente avec backoff.
fileDao.clearOcrQueued(file.resourceId, System.currentTimeMillis())
OcrAutoOutcome.Retryable
}
e.httpCode == 401 -> OcrAutoOutcome.Unauthorized
else -> {
// Erreur permanente : +1 tentative (≤ 3), relance manuelle possible.
fileDao.markOcrFailure(file.resourceId, System.currentTimeMillis())
OcrAutoOutcome.Failed
}
}
} catch (e: Exception) {
fileDao.markOcrFailure(file.resourceId, System.currentTimeMillis())
OcrAutoOutcome.Failed
}
}
companion object { companion object {
const val POLL_INTERVAL_MS = 3_000L const val POLL_INTERVAL_MS = 3_000L
} }
} }
/** Résultat d'une passe d'OCR auto (un fichier max). */
sealed interface OcrAutoOutcome {
/** Traité avec succès : `ocr_text` persisté. */
data object Done : OcrAutoOutcome
/** Job terminal `failed` / erreur permanente : `ocr_attempts` incrémenté. */
data object Failed : OcrAutoOutcome
/** Erreur transitoire : à retenter via backoff WorkManager. */
data object Retryable : OcrAutoOutcome
/** Token expiré/révoqué : le re-login passera par l'UI. */
data object Unauthorized : OcrAutoOutcome
}
/** /**
* `RequestBody` paresseux : le flux SAF est ouvert à l'écriture (multipart), * `RequestBody` paresseux : le flux SAF est ouvert à l'écriture (multipart),
* jamais chargé en mémoire — les fichiers jusqu'à `MAX_FILE_SIZE_MB` restent * jamais chargé en mémoire — les fichiers jusqu'à `MAX_FILE_SIZE_MB` restent
@@ -0,0 +1,100 @@
package com.vaultdrop.mobile.features.ocr
import android.content.Context
import androidx.hilt.work.HiltWorker
import androidx.work.BackoffPolicy
import androidx.work.Constraints
import androidx.work.CoroutineWorker
import androidx.work.ExistingWorkPolicy
import androidx.work.NetworkType
import androidx.work.OneTimeWorkRequestBuilder
import androidx.work.WorkManager
import androidx.work.WorkerParameters
import com.vaultdrop.mobile.auth.TokenProvider
import com.vaultdrop.mobile.data.local.dao.FileDao
import com.vaultdrop.mobile.data.remote.ApiException
import com.vaultdrop.mobile.data.remote.dto.OcrJobStatus
import com.vaultdrop.mobile.data.repository.OcrAutoOutcome
import com.vaultdrop.mobile.data.repository.OcrRepository
import dagger.assisted.Assisted
import dagger.assisted.AssistedInject
import java.util.concurrent.TimeUnit
import timber.log.Timber
/**
* OCR auto fichier par fichier : à chaque exécution, un seul candidat
* (`processed`, `local-cloud`, `ocr_text` nul, URI SAF présente,
* `ocr_attempts < 3`) est uploadé puis enfilé (`POST /ocr/jobs`) dans la
* **file bornée serveur** (`Ocr.Run`). Un seul run à la fois via
* `enqueueUniqueWork(KEEP)`. Le worker se re-enchaine tant qu'il reste des
* candidats *distincts* du dernier fichier traité (pas de relance immédiate
* d'un fichier en échec, pour respecter le cap de 3 tentatives).
*
* Erreurs :
* - **réseau / 5xx** → `Result.retry()` (backoff 30 s) sans incrémenter
* `ocr_attempts` (le marqueur in-flight est levé) ;
* - **4xx non-idempotente** → `ocr_attempts` incrémenté, pas de re-enchaine ;
* - **token expiré (401)** → `Result.success()` (re-login passera par l'UI).
*/
@HiltWorker
class OcrAutoWorker @AssistedInject constructor(
@Assisted appContext: Context,
@Assisted workerParams: WorkerParameters,
private val ocrRepository: OcrRepository,
private val fileDao: FileDao,
private val tokenProvider: TokenProvider,
) : CoroutineWorker(appContext, workerParams) {
override suspend fun doWork(): Result {
if (tokenProvider.current == null) return Result.success()
val file = fileDao.getOcrAutoCandidates().firstOrNull()
?: return Result.success()
Timber.d("ocr-auto: candidate %s (%s)", file.name, file.resourceId)
val outcome = try {
ocrRepository.processAuto(file)
} catch (e: ApiException) {
when {
e.code == "NETWORK_ERROR" || e.httpCode >= 500 -> OcrAutoOutcome.Retryable
e.httpCode == 401 -> OcrAutoOutcome.Unauthorized
else -> OcrAutoOutcome.Failed
}
} catch (e: Exception) {
Timber.w(e, "ocr-auto: unexpected failure for %s", file.resourceId)
OcrAutoOutcome.Failed
}
when (outcome) {
is OcrAutoOutcome.Done ->
Timber.d("ocr-auto: done %s", file.resourceId)
is OcrAutoOutcome.Failed ->
Timber.w("ocr-auto: failed %s", file.resourceId)
is OcrAutoOutcome.Retryable -> return Result.retry()
is OcrAutoOutcome.Unauthorized -> return Result.success()
}
// Self-chain : un autre candidat *distinct* du dernier attend ?
val more = fileDao.getOcrAutoCandidates().any { it.resourceId != file.resourceId }
if (more) {
enqueue(applicationContext)
}
return Result.success()
}
companion object {
const val NAME = "ocr_auto"
fun enqueue(context: Context) {
val constraints = Constraints.Builder()
.setRequiredNetworkType(NetworkType.CONNECTED)
.build()
val request = OneTimeWorkRequestBuilder<OcrAutoWorker>()
.setConstraints(constraints)
.setBackoffCriteria(BackoffPolicy.EXPONENTIAL, 30, TimeUnit.SECONDS)
.build()
WorkManager.getInstance(context)
.enqueueUniqueWork(NAME, ExistingWorkPolicy.KEEP, request)
}
}
}
@@ -22,6 +22,7 @@ import com.vaultdrop.mobile.data.local.entity.PendingOperationType
import com.vaultdrop.mobile.data.remote.ApiClient import com.vaultdrop.mobile.data.remote.ApiClient
import com.vaultdrop.mobile.data.remote.ApiException import com.vaultdrop.mobile.data.remote.ApiException
import com.vaultdrop.mobile.data.remote.dto.SyncOpDto import com.vaultdrop.mobile.data.remote.dto.SyncOpDto
import com.vaultdrop.mobile.features.ocr.OcrAutoWorker
import dagger.assisted.Assisted import dagger.assisted.Assisted
import dagger.assisted.AssistedInject import dagger.assisted.AssistedInject
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
@@ -62,6 +63,9 @@ class OutboxSyncWorker @AssistedInject constructor(
Types.newParameterizedType(Map::class.java, String::class.java, Any::class.java), Types.newParameterizedType(Map::class.java, String::class.java, Any::class.java),
) )
/** Set à true si un fichier a été promu pendant ce run → enchaîne OcrAutoWorker. */
private var filePromoted = false
override suspend fun doWork(): Result { override suspend fun doWork(): Result {
// Mode local : sans compte connecté, rien à pousser. // Mode local : sans compte connecté, rien à pousser.
if (tokenProvider.current == null) return Result.success() if (tokenProvider.current == null) return Result.success()
@@ -71,10 +75,13 @@ class OutboxSyncWorker @AssistedInject constructor(
fileDao.backfillSyncedStatus(System.currentTimeMillis()) fileDao.backfillSyncedStatus(System.currentTimeMillis())
folderDao.backfillSyncedStatus(System.currentTimeMillis()) folderDao.backfillSyncedStatus(System.currentTimeMillis())
filePromoted = false
while (true) { while (true) {
val pending = pendingOperationDao.selectPending(BATCH_SIZE) val pending = pendingOperationDao.selectPending(BATCH_SIZE)
if (pending.isEmpty()) { if (pending.isEmpty()) {
pendingOperationDao.purgeSynced(System.currentTimeMillis() - PURGE_AGE_MS) pendingOperationDao.purgeSynced(System.currentTimeMillis() - PURGE_AGE_MS)
if (filePromoted) OcrAutoWorker.enqueue(applicationContext)
return Result.success() return Result.success()
} }
@@ -139,7 +146,10 @@ class OutboxSyncWorker @AssistedInject constructor(
PendingOperationType.CREATE_RESOURCE, PendingOperationType.CREATE_RESOURCE,
PendingOperationType.MOVE_RESOURCE -> { PendingOperationType.MOVE_RESOURCE -> {
when (op.resourceType) { when (op.resourceType) {
"file" -> fileDao.promoteSyncStatus(resourceId, System.currentTimeMillis()) "file" -> {
fileDao.promoteSyncStatus(resourceId, System.currentTimeMillis())
filePromoted = true
}
"folder" -> folderDao.promoteSyncStatus(resourceId, System.currentTimeMillis()) "folder" -> folderDao.promoteSyncStatus(resourceId, System.currentTimeMillis())
} }
} }
@@ -11,6 +11,7 @@ import com.vaultdrop.mobile.data.local.entity.PendingOperationEntity
import com.vaultdrop.mobile.data.repository.FolderRepository import com.vaultdrop.mobile.data.repository.FolderRepository
import com.vaultdrop.mobile.data.repository.SaveFolderInput import com.vaultdrop.mobile.data.repository.SaveFolderInput
import com.vaultdrop.mobile.data.repository.ShareRepository import com.vaultdrop.mobile.data.repository.ShareRepository
import com.vaultdrop.mobile.features.ocr.OcrAutoWorker
import dagger.hilt.android.lifecycle.HiltViewModel import dagger.hilt.android.lifecycle.HiltViewModel
import dagger.hilt.android.qualifiers.ApplicationContext import dagger.hilt.android.qualifiers.ApplicationContext
import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CancellationException
@@ -107,6 +108,8 @@ class SyncViewModel @Inject constructor(
// Les nouvelles ressources découvertes sont dans l'outbox // Les nouvelles ressources découvertes sont dans l'outbox
// → drainer vers POST /sync/ops (single-flight via KEEP). // → drainer vers POST /sync/ops (single-flight via KEEP).
OutboxSyncWorker.enqueue(appContext) OutboxSyncWorker.enqueue(appContext)
// OCR auto : uploader les octets des fichiers récemment promus (local-cloud).
OcrAutoWorker.enqueue(appContext)
} }
.onFailure { e -> Timber.w(e, "syncAll failed, retrying later") } .onFailure { e -> Timber.w(e, "syncAll failed, retrying later") }
// Hydrate les ressources partagées depuis le snapshot serveur. // Hydrate les ressources partagées depuis le snapshot serveur.
@@ -9,6 +9,7 @@ import com.vaultdrop.mobile.data.remote.ApiException
import com.vaultdrop.mobile.data.repository.AuthRepository import com.vaultdrop.mobile.data.repository.AuthRepository
import com.vaultdrop.mobile.data.repository.ShareRepository import com.vaultdrop.mobile.data.repository.ShareRepository
import com.vaultdrop.mobile.features.sync.OutboxSyncWorker import com.vaultdrop.mobile.features.sync.OutboxSyncWorker
import com.vaultdrop.mobile.features.ocr.OcrAutoWorker
import dagger.hilt.android.lifecycle.HiltViewModel import dagger.hilt.android.lifecycle.HiltViewModel
import dagger.hilt.android.qualifiers.ApplicationContext import dagger.hilt.android.qualifiers.ApplicationContext
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
@@ -51,6 +52,7 @@ class AuthViewModel @Inject constructor(
authRepository.registerDevice() authRepository.registerDevice()
// Session restaurée → drainer l'outbox laissée en attente. // Session restaurée → drainer l'outbox laissée en attente.
OutboxSyncWorker.enqueue(appContext) OutboxSyncWorker.enqueue(appContext)
OcrAutoWorker.enqueue(appContext)
if (session != null) { if (session != null) {
// Snapshot complet des permissions partagées (convergence). // Snapshot complet des permissions partagées (convergence).
runCatching { shareRepository.syncSnapshot() } runCatching { shareRepository.syncSnapshot() }
@@ -73,6 +75,7 @@ class AuthViewModel @Inject constructor(
_authState.value = AuthState.SignedIn(response.user) _authState.value = AuthState.SignedIn(response.user)
// Connexion réussie → pousser les mutations locales en attente. // Connexion réussie → pousser les mutations locales en attente.
OutboxSyncWorker.enqueue(appContext) OutboxSyncWorker.enqueue(appContext)
OcrAutoWorker.enqueue(appContext)
// Snapshot complet des permissions partagées (convergence). // Snapshot complet des permissions partagées (convergence).
runCatching { shareRepository.syncSnapshot() } runCatching { shareRepository.syncSnapshot() }
.onFailure { e -> Timber.d("syncSnapshot on login failed: %s", e.message) } .onFailure { e -> Timber.d("syncSnapshot on login failed: %s", e.message) }
@@ -7,6 +7,7 @@ import com.vaultdrop.mobile.data.remote.ApiClient
import com.vaultdrop.mobile.data.remote.ApiException import com.vaultdrop.mobile.data.remote.ApiException
import com.vaultdrop.mobile.data.repository.OutboxRepository import com.vaultdrop.mobile.data.repository.OutboxRepository
import com.vaultdrop.mobile.features.sync.OutboxSyncWorker import com.vaultdrop.mobile.features.sync.OutboxSyncWorker
import com.vaultdrop.mobile.features.ocr.OcrAutoWorker
import dagger.hilt.android.lifecycle.HiltViewModel import dagger.hilt.android.lifecycle.HiltViewModel
import dagger.hilt.android.qualifiers.ApplicationContext import dagger.hilt.android.qualifiers.ApplicationContext
import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.MutableStateFlow
@@ -68,6 +69,7 @@ class ShareViewModel @Inject constructor(
) )
}.onSuccess { }.onSuccess {
OutboxSyncWorker.enqueue(appContext) OutboxSyncWorker.enqueue(appContext)
OcrAutoWorker.enqueue(appContext)
Timber.d("share: op enqueued for %s", resourceId) Timber.d("share: op enqueued for %s", resourceId)
_uiState.update { it.copy(sharing = false, enqueued = true) } _uiState.update { it.copy(sharing = false, enqueued = true) }
}.onFailure { e -> }.onFailure { e ->
@@ -30,6 +30,7 @@ import okhttp3.MultipartBody
import okhttp3.RequestBody import okhttp3.RequestBody
import org.junit.After import org.junit.After
import org.junit.Assert.assertEquals import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue import org.junit.Assert.assertTrue
import org.junit.Before import org.junit.Before
import org.junit.Test import org.junit.Test
@@ -117,18 +118,24 @@ class OcrRepositoryTest {
assertEquals("boom", final.error) assertEquals("boom", final.error)
} }
private fun insertFile(uri: String): FileEntity { private fun insertFile(
uri: String?,
ocrText: String? = null,
category: String? = "TEXT",
): FileEntity {
val file = FileEntity( val file = FileEntity(
resourceId = "aabbccddeeff11223344556677889900", resourceId = generateResourceId(),
uri = uri, uri = uri,
name = "sample.txt", name = uri?.substringAfterLast('/') ?: "cloud.txt",
folderResourceId = "f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0", folderResourceId = "f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0",
extension = "txt", extension = uri?.substringAfterLast('.')?.take(8),
size = 13L, size = 10L,
mimeType = "text/plain", mimeType = "text/plain",
exists = 1, exists = 1,
category = category,
syncStatus = FileStatus.LOCAL_CLOUD, syncStatus = FileStatus.LOCAL_CLOUD,
processed = true, processed = true,
ocrText = ocrText,
addedAt = 1_700_000_000_000L, addedAt = 1_700_000_000_000L,
updatedAt = 1_700_000_000_000L, updatedAt = 1_700_000_000_000L,
) )
@@ -136,8 +143,83 @@ class OcrRepositoryTest {
return file return file
} }
private var idSeq = 0L
private fun generateResourceId(): String {
val hex = (++idSeq).toString(16).padStart(32, '0')
return hex
}
private fun moshi(): Moshi = private fun moshi(): Moshi =
Moshi.Builder().add(KotlinJsonAdapterFactory()).build() Moshi.Builder().add(KotlinJsonAdapterFactory()).build()
@Test
fun `getOcrAutoCandidates_exclut_ocr_text_present`() = runTest {
val withText = insertFile("content://ocr/a.txt", ocrText = "extrait")
val withoutText = insertFile("content://ocr/b.txt", ocrText = null)
val candidates = fileDao.getOcrAutoCandidates()
assertEquals(1, candidates.size)
assertEquals(withoutText.resourceId, candidates[0].resourceId)
}
@Test
fun `getOcrAutoCandidates_exclut_uri_absente_et_cloud_only`() = runTest {
insertFile("content://ocr/a.txt", ocrText = null)
insertFile(null, ocrText = null)
val candidates = fileDao.getOcrAutoCandidates()
assertEquals(1, candidates.size)
}
@Test
fun `getOcrAutoCandidates_exclut_exhausted_attempts_3`() = runTest {
val exhausted = insertFile("content://ocr/a.txt", ocrText = null)
fileDao.markOcrFailure(exhausted.resourceId, System.currentTimeMillis())
fileDao.markOcrFailure(exhausted.resourceId, System.currentTimeMillis())
fileDao.markOcrFailure(exhausted.resourceId, System.currentTimeMillis())
assertTrue(fileDao.getOcrAutoCandidates().isEmpty())
}
@Test
fun `processAuto_done_persiste_texte_et_clear_queued`() = runTest {
val uri = "content://ocr/auto-done.txt"
shadowOf(context.contentResolver).registerInputStream(uri.toUri(), ByteArrayInputStream("auto".toByteArray()))
val file = insertFile(uri)
apiService.jobSequence = listOf(
OcrJobDto(id = "auto-1", status = OcrJobStatus.QUEUED),
OcrJobDto(id = "auto-1", status = OcrJobStatus.DONE, text = "autoOCR"),
)
val outcome = repository.processAuto(file)
assertTrue(outcome is OcrAutoOutcome.Done)
val stored = fileDao.getByResourceId(file.resourceId)
assertEquals("autoOCR", stored?.ocrText)
assertEquals(0, stored?.ocrAttempts)
assertNull(stored?.ocrQueuedAt)
}
@Test
fun `processAuto_failed_incremente_attempts_et_clear_queued`() = runTest {
val uri = "content://ocr/auto-fail.txt"
shadowOf(context.contentResolver).registerInputStream(uri.toUri(), ByteArrayInputStream(byteArrayOf()))
val file = insertFile(uri)
apiService.jobSequence = listOf(
OcrJobDto(id = "auto-f", status = OcrJobStatus.FAILED, error = "tess err"),
)
val outcome = repository.processAuto(file)
assertTrue(outcome is OcrAutoOutcome.Failed)
val stored = fileDao.getByResourceId(file.resourceId)
assertEquals(1, stored?.ocrAttempts)
assertNull(stored?.ocrQueuedAt)
assertNull(stored?.ocrText)
}
} }
/** /**