Thirdshop/Kazier
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ocr is working

Browse Source
This commit is contained in:
m
2026-09-16 12:44:48 +02:00
parent 050434325a
commit 906d48702c
21 changed files with 595 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/cmd/server/main.go
+31 -3
View File
@@ -1,9 +1,15 @@
package main
import (
"context"
"errors"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"github.com/gin-gonic/gin"
"github.com/vaultdrop/backend/config"
@@ -64,8 +70,30 @@ func main() {
)
handlers.Ocr = service.NewOcr(repo, cfg.UploadDir, cfg.OcrLang, ocr.NewTesseract())
if err := newRouter().Run(fmt.Sprintf(":%d", cfg.Port)); err != nil {
log.Fatalln(err)
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
// Worker OCR borné (OCR_WORKERS) : file `ocr_jobs`, hors request handlers.
go handlers.Ocr.Run(ctx, cfg.OcrWorkers)
log.Printf("ocr worker démarré (%d workers)", cfg.OcrWorkers)
srv := &http.Server{
Addr: fmt.Sprintf(":%d", cfg.Port),
Handler: newRouter(),
}
go func() {
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Fatalf("server: %v", err)
}
}()
<-ctx.Done()
stop()
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := srv.Shutdown(shutdownCtx); err != nil {
log.Printf("shutdown: %v", err)
}
log.Println("arrêt propre")
}
backend/config/config.go
+7
View File
@@ -13,6 +13,7 @@ type ApplicationConfig struct {
UploadDir string
MaxFileSizeMB int64
OcrLang string
OcrWorkers int
AuthSecret string
AdminUsername string
AdminPassword string
@@ -33,12 +34,18 @@ func LoadApplicationConfig() (*ApplicationConfig, error) {
return nil, err
}
ocrWorkers, err := getInt("OCR_WORKERS", 2)
if err != nil {
return nil, err
}
return &ApplicationConfig{
Port: port,
DatabaseURL: get("DATABASE_URL", "postgres://vaultdrop:vaultdrop@localhost:5432/vaultdrop_dev?sslmode=disable"),
UploadDir: get("UPLOAD_DIR", "./uploads"),
MaxFileSizeMB: int64(maxSize),
OcrLang: get("OCR_LANG", "fra+eng"),
OcrWorkers: ocrWorkers,
AuthSecret: get("AUTH_SECRET", "dev-secret-change-me"),
AdminUsername: get("ADMIN_USERNAME", ""),
AdminPassword: get("ADMIN_PASSWORD", ""),
backend/config/config_test.go
+6 -2
View File
@@ -9,7 +9,7 @@ func clearEnv(t *testing.T) {
t.Helper()
for _, name := range []string{
"PORT", "DATABASE_URL", "UPLOAD_DIR", "MAX_FILE_SIZE_MB",
"OCR_LANG", "AUTH_SECRET", "ADMIN_USERNAME", "ADMIN_PASSWORD",
"OCR_LANG", "OCR_WORKERS", "AUTH_SECRET", "ADMIN_USERNAME", "ADMIN_PASSWORD",
} {
t.Setenv(name, "")
}
@@ -37,6 +37,9 @@ func TestLoadApplicationDefaults(t *testing.T) {
if cfg.OcrLang != "fra+eng" {
t.Errorf("OcrLang défaut inattendu: %q", cfg.OcrLang)
}
if cfg.OcrWorkers != 2 {
t.Errorf("OcrWorkers = %d, attendu 2", cfg.OcrWorkers)
}
if cfg.AuthSecret != "dev-secret-change-me" {
t.Errorf("AuthSecret défaut inattendu: %q", cfg.AuthSecret)
}
@@ -52,6 +55,7 @@ func TestLoadApplicationEnvOverrides(t *testing.T) {
t.Setenv("DATABASE_URL", "postgres://u:p@host:5433/db?sslmode=disable")
t.Setenv("UPLOAD_DIR", "/tmp/up")
t.Setenv("OCR_LANG", "eng")
t.Setenv("OCR_WORKERS", "4")
t.Setenv("AUTH_SECRET", "super-secret")
t.Setenv("ADMIN_USERNAME", "root")
t.Setenv("ADMIN_PASSWORD", "toor")
@@ -61,7 +65,7 @@ func TestLoadApplicationEnvOverrides(t *testing.T) {
t.Fatalf("LoadApplicationConfig: %v", err)
}
if cfg.Port != 9090 || cfg.MaxFileSizeMB != 120 || cfg.DatabaseURL != "postgres://u:p@host:5433/db?sslmode=disable" ||
cfg.UploadDir != "/tmp/up" || cfg.OcrLang != "eng" || cfg.AuthSecret != "super-secret" ||
cfg.UploadDir != "/tmp/up" || cfg.OcrLang != "eng" || cfg.OcrWorkers != 4 || cfg.AuthSecret != "super-secret" ||
cfg.AdminUsername != "root" || cfg.AdminPassword != "toor" {
t.Errorf("overrides non appliqués: %+v", cfg)
}
backend/handlers/ocr_test.go
+6
View File
@@ -42,6 +42,12 @@ func setupOcr(t *testing.T) (*gin.Engine, string, *repository.Repository) {
handlers.Store = store
handlers.Ocr = service.NewOcr(repo, uploadDir, "fra+eng", stubEngine{text: "HELLO OCR"})
// Worker OCR borné — le handler n'enfile plus que le job, le traitement
// tourne sur un worker de contexte.
ctx, cancel := context.WithCancel(context.Background())
go handlers.Ocr.Run(ctx, 1)
t.Cleanup(cancel)
gin.SetMode(gin.TestMode)
r := gin.New()
handlers.RegisterRoutes(r)
backend/repository/ocr_jobs.go
+47
View File
@@ -3,6 +3,7 @@ package repository
import (
"database/sql"
"errors"
"time"
)
// OcrJobRow maps ocr_jobs.
@@ -18,6 +19,9 @@ type OcrJobRow struct {
// ErrJobNotFound marks an OCR job absent or owned by another device.
var ErrJobNotFound = errors.New("ocr job not found")
// ErrNoQueuedJobs marks an OCR queue drained (no job to claim).
var ErrNoQueuedJobs = errors.New("no queued ocr jobs")
type OcrJobs struct{ DB *sql.DB }
func (o *OcrJobs) Create(jobID, deviceID, fileID string) error {
@@ -62,6 +66,49 @@ func (o *OcrJobs) TouchProcessing(deviceID, jobID string) error {
return err
}
// ClaimNext atomically picks the oldest queued job (FIFO) and moves it to
// `processing`. Safe for concurrent workers: `FOR UPDATE SKIP LOCKED` blocks
// the row as part of the same statement. Empty queue → ErrNoQueuedJobs.
func (o *OcrJobs) ClaimNext() (OcrJobRow, error) {
var row OcrJobRow
err := o.DB.QueryRow(
`WITH next AS (
SELECT job_id FROM ocr_jobs
WHERE status = 'queued'
ORDER BY created_at, job_id
LIMIT 1
FOR UPDATE SKIP LOCKED
)
UPDATE ocr_jobs
SET status = 'processing', started_at = NOW()
FROM next
WHERE ocr_jobs.job_id = next.job_id
RETURNING ocr_jobs.job_id, ocr_jobs.file_id, ocr_jobs.device_id, ocr_jobs.status`,
).Scan(&row.ID, &row.FileID, &row.DeviceID, &row.Status)
if err == sql.ErrNoRows {
return OcrJobRow{}, ErrNoQueuedJobs
}
if err != nil {
return OcrJobRow{}, err
}
return row, nil
}
// ResetStaleProcessing requeues jobs stuck in `processing` (worker crash /
// processus redémarré) et plus vieux que `olderThan`. Retourne le nb de jobs
// requeued.
func (o *OcrJobs) ResetStaleProcessing(olderThan time.Duration) (int64, error) {
res, err := o.DB.Exec(
`UPDATE ocr_jobs SET status = 'queued', started_at = NULL
WHERE status = 'processing' AND started_at < $1`,
time.Now().Add(-olderThan),
)
if err != nil {
return 0, err
}
return res.RowsAffected()
}
func (o *OcrJobs) Complete(deviceID, jobID, text string) error {
_, err := o.DB.Exec(
`UPDATE ocr_jobs SET status = 'done', text = NULLIF($3, ''), started_at = COALESCE(started_at, NOW()), completed_at = NOW()
backend/repository/ocr_jobs_test.go
+89
View File
@@ -3,6 +3,7 @@ package repository
import (
"errors"
"testing"
"time"
"github.com/vaultdrop/backend/dbtest"
)
@@ -90,3 +91,91 @@ func TestOcrJobsGetScopedByDevice(t *testing.T) {
t.Errorf("get par un autre device : attendu ErrJobNotFound, got %v", err)
}
}
func TestOcrJobsClaimNextFifo(t *testing.T) {
repo, _, deviceID, fileID := seedOcrJobFixture(t)
var jobIDs []string
for i := 0; i < 3; i++ {
jobID := NewID()
if err := repo.OcrJobs.Create(jobID, deviceID, fileID); err != nil {
t.Fatalf("create %d: %v", i, err)
}
jobIDs = append(jobIDs, jobID)
}
for i, want := range jobIDs {
row, err := repo.OcrJobs.ClaimNext()
if err != nil {
t.Fatalf("claim %d: %v", i, err)
}
if row.ID != want {
t.Errorf("claim %d = %q, attendu %q (FIFO)", i, row.ID, want)
}
if row.Status != "processing" {
t.Errorf("claim %d status = %q, attendu processing", i, row.Status)
}
}
if _, err := repo.OcrJobs.ClaimNext(); !errors.Is(err, ErrNoQueuedJobs) {
t.Errorf("queue vide : attendu ErrNoQueuedJobs, got %v", err)
}
}
func TestOcrJobsClaimNextSkipsProcessingAndDone(t *testing.T) {
repo, _, deviceID, fileID := seedOcrJobFixture(t)
job1 := NewID()
job2 := NewID()
if err := repo.OcrJobs.Create(job1, deviceID, fileID); err != nil {
t.Fatalf("create job1: %v", err)
}
if err := repo.OcrJobs.Create(job2, deviceID, fileID); err != nil {
t.Fatalf("create job2: %v", err)
}
if err := repo.OcrJobs.TouchProcessing(deviceID, job2); err != nil {
t.Fatalf("touch job2 processing: %v", err)
}
row, err := repo.OcrJobs.ClaimNext()
if err != nil {
t.Fatalf("claim: %v", err)
}
if row.ID != job1 {
t.Errorf("claim = %q, attendu job1 %q", row.ID, job1)
}
}
func TestOcrJobsResetStaleProcessing(t *testing.T) {
repo, _, deviceID, fileID := seedOcrJobFixture(t)
jobID := NewID()
if err := repo.OcrJobs.Create(jobID, deviceID, fileID); err != nil {
t.Fatalf("create: %v", err)
}
if err := repo.OcrJobs.TouchProcessing(deviceID, jobID); err != nil {
t.Fatalf("touch processing: %v", err)
}
// Started_at = NOW : pas encore stale → aucun requeue.
if n, err := repo.OcrJobs.ResetStaleProcessing(time.Minute); err != nil {
t.Fatalf("reset: %v", err)
} else if n != 0 {
t.Errorf("reset récent : attendu 0, got %d", n)
}
// Vieillit le started_at puis re-reset → 1 requeue.
if _, err := repo.OcrJobs.DB.Exec(`UPDATE ocr_jobs SET started_at = NOW() - interval '2 minutes' WHERE job_id = $1`, jobID); err != nil {
t.Fatalf("vieillir started_at: %v", err)
}
if n, err := repo.OcrJobs.ResetStaleProcessing(time.Minute); err != nil {
t.Fatalf("reset stale: %v", err)
} else if n != 1 {
t.Errorf("reset stale : attendu 1, got %d", n)
}
row, err := repo.OcrJobs.Get(deviceID, jobID)
if err != nil {
t.Fatalf("get: %v", err)
}
if row.Status != "queued" {
t.Errorf("status après reset = %q, attendu queued", row.Status)
}
}
backend/repository/resources.go
+15
View File
@@ -254,6 +254,21 @@ func (r *Resources) GetFileVisible(userID, resourceID string) (FileRow, error) {
return file, err
}
// GetResourceOwner returns the owning user_id of any resource (file or folder).
// No-rows → ErrNotFound.
func (r *Resources) GetResourceOwner(resourceID string) (string, error) {
var ownerID string
err := r.DB.QueryRow(
`SELECT user_id FROM resources
WHERE resource_id = $1 AND deleted_at IS NULL`,
resourceID,
).Scan(&ownerID)
if errors.Is(err, sql.ErrNoRows) {
return "", ErrNotFound
}
return ownerID, err
}
// DeleteFile soft-deletes the file (deleted_at), returning its id.
func (r *Resources) DeleteFile(ownerID, resourceID string) (string, error) {
result, err := r.DB.Exec(
backend/service/ocr.go
+72 -14
View File
@@ -2,13 +2,25 @@ package service
import (
"context"
"errors"
"log"
"path/filepath"
"strings"
"sync"
"time"
"github.com/vaultdrop/backend/ocr"
"github.com/vaultdrop/backend/repository"
)
const (
// StaleClaimTimeout : un job `processing` plus vieux que ça est ré-enfilé
// au boot (worker mort / process redémarré).
StaleClaimTimeout = 15 * time.Minute
// QueuePollInterval : cadence de scrutation quand la file est vide.
QueuePollInterval = time.Second
)
// OcrJobDTO serializes exactly as mobile/api/types.ts OcrJob.
type OcrJobDTO struct {
ID string `json:"id"`
@@ -17,22 +29,24 @@ type OcrJobDTO struct {
Error string `json:"error,omitempty"`
}
// Ocr queues OCR jobs and processes them asynchronously (V1 : goroutine par
// job ; le client poll GET /ocr/jobs/:id toutes les 3s).
// Ocr queues OCR jobs in `ocr_jobs` (status `queued`) and processes them
// asynchronously through a bounded worker ([Run]) — Tesseract ne tourne jamais
// dans un request handler. Le client poll GET /ocr/jobs/:id toutes les 3s.
type Ocr struct {
Repository *repository.Repository
UploadDir string
Lang string
Engine ocr.Engine
StaleAfter time.Duration
}
func NewOcr(repo *repository.Repository, uploadDir, lang string, engine ocr.Engine) *Ocr {
return &Ocr{Repository: repo, UploadDir: uploadDir, Lang: lang, Engine: engine}
return &Ocr{Repository: repo, UploadDir: uploadDir, Lang: lang, Engine: engine, StaleAfter: StaleClaimTimeout}
}
// Create valide la ressource (ownership par user), met le job en file et
// lance le traitement. Le job reste scopé par device (le service génère les
// jobs du device courant ; l'outbox OCR n'est pas synchronisée entre devices).
// Create valide la ressource (ownership par user) et l'enfile uniquement
// (status `queued`) — le traitement est délégué au worker [Run] pour ne pas
// stresser le serveur. Le job reste scopé par device.
func (o *Ocr) Create(userID, deviceID, fileID string) (OcrJobDTO, error) {
if _, err := o.Repository.Resources.GetFile(userID, fileID); err != nil {
return OcrJobDTO{}, err
@@ -41,7 +55,6 @@ func (o *Ocr) Create(userID, deviceID, fileID string) (OcrJobDTO, error) {
if err := o.Repository.OcrJobs.Create(jobID, deviceID, fileID); err != nil {
return OcrJobDTO{}, err
}
go o.process(userID, deviceID, jobID, fileID)
return OcrJobDTO{ID: jobID, Status: "queued"}, nil
}
@@ -53,22 +66,67 @@ func (o *Ocr) Get(deviceID, jobID string) (OcrJobDTO, error) {
return toOcrJobDTO(row), nil
}
func (o *Ocr) process(userID, deviceID, jobID, fileID string) {
ctx := context.Background()
if err := o.Repository.OcrJobs.TouchProcessing(deviceID, jobID); err != nil {
// Run fait tourner le worker OCR : au démarrage il requeue les jobs laissés
// en `processing` (crash), puis `workers` boucles consomment la file FIFO via
// ClaimNext (atomique, `FOR UPDATE SKIP LOCKED`). `workers` borne la
// concurrence Tesseract. Retourne quand ctx est annulé.
func (o *Ocr) Run(ctx context.Context, workers int) {
if workers < 1 {
workers = 1
}
if _, err := o.Repository.OcrJobs.ResetStaleProcessing(o.StaleAfter); err != nil {
log.Printf("ocr: requeue jobs stale: %v", err)
}
var wg sync.WaitGroup
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
o.workerLoop(ctx)
}()
}
wg.Wait()
}
// workerLoop consomme la file : un job `queued` → processing → done/failed.
// File vide → scrutation à QueuePollInterval jusqu'à annulation du ctx.
func (o *Ocr) workerLoop(ctx context.Context) {
for {
row, err := o.Repository.OcrJobs.ClaimNext()
if err != nil {
if !errors.Is(err, repository.ErrNoQueuedJobs) {
log.Printf("ocr: claim: %v", err)
}
select {
case <-ctx.Done():
return
case <-time.After(QueuePollInterval):
}
continue
}
o.process(ctx, row)
}
}
// process exécute l'extraction pour un job déjà réclamé (`processing`).
func (o *Ocr) process(ctx context.Context, row repository.OcrJobRow) {
userID, err := o.Repository.Resources.GetResourceOwner(row.FileID)
if err != nil {
_ = o.Repository.OcrJobs.Fail(row.DeviceID, row.ID, "file not readable")
return
}
path, err := o.physicalPath(userID, fileID)
path, err := o.physicalPath(userID, row.FileID)
if err != nil {
_ = o.Repository.OcrJobs.Fail(deviceID, jobID, "file not readable")
_ = o.Repository.OcrJobs.Fail(row.DeviceID, row.ID, "file not readable")
return
}
text, err := o.Engine.ExtractText(ctx, path, o.Lang)
if err != nil {
_ = o.Repository.OcrJobs.Fail(deviceID, jobID, err.Error())
_ = o.Repository.OcrJobs.Fail(row.DeviceID, row.ID, err.Error())
return
}
_ = o.Repository.OcrJobs.Complete(deviceID, jobID, text)
_ = o.Repository.OcrJobs.Complete(row.DeviceID, row.ID, text)
}
// physicalPath résout UPLOAD_DIR/<user_id>/<resource_id>.<ext> — l'ext est
backend/service/ocr_test.go
+7 -2
View File
@@ -26,13 +26,18 @@ func (s stubEngine) ExtractText(_ context.Context, _ string, _ string) (string,
return s.text, nil
}
// newTestOcr builds an Ocr over a fresh DB and returns it with the ids.
// newTestOcr builds an Ocr over a fresh DB, starts the bounded worker (1
// worker) and returns it with the ids.
func newTestOcr(t *testing.T, uploadDir string, engine ocr.Engine) (*Ocr, string, string, string) {
t.Helper()
s := newServiceStore(t)
userID := mustCreateUser(t, s.Repository, "ocr-failed")
deviceID := mustRegisterDevice(t, s.Repository, repository.NewID())
return NewOcr(s.Repository, uploadDir, "fra+eng", engine), userID, deviceID, uploadDir
o := NewOcr(s.Repository, uploadDir, "fra+eng", engine)
ctx, cancel := context.WithCancel(context.Background())
go o.Run(ctx, 1)
t.Cleanup(cancel)
return o, userID, deviceID, uploadDir
}
// waitTillTerminal poll jusqu'à un statut terminal (done/failed).
docs/api-v1.md
+1 -1
View File
@@ -75,7 +75,7 @@ type OcrJob = { id: string; status: OcrJobStatus; text?: string | null; error?:
## 5. OCR
- `POST /ocr/jobs { fileId }` → `OcrJob` immédiat (`status: queued`), traitement **asynchrone** (goroutine par job côté serveur, V1).
- `POST /ocr/jobs { fileId }` → `OcrJob` immédiat (`status: queued`), traitement **asynchrone** par une **file serveur bornée** (`ocr_jobs` + worker `Ocr.Run`, `OCR_WORKERS` défaut 2) — le handler n'enfile **jamais** de traitement, Tesseract tourne hors request handler (FIFO, récupération des `processing` orphelins au boot).
- `GET /ocr/jobs/:id` → statut. Le mobile **poll toutes les 3s** jusqu'à `done`/`failed` (`hooks/useUpload.ts`). Cycle : `queued → processing → done | failed` ; `done` renvoie `text`, `failed` renvoie `error`.
- Moteur : **Tesseract en appel système** (`ocr/tesseract.go`), langue `OCR_LANG` (défaut `fra+eng`). Les images sont passées directement à `tesseract` ; les **PDF** subissent une extraction du calque texte (`ledongthuc/pdf`, déjà en go.mod) — un PDF scanné produit un texte vide plutôt qu'un rendu/OCR (hors scope V1).
- `fileId` inconnu/pas du user → `NOT_FOUND`. Fichier physique introuvable (ex. suppression manuelle sous `UPLOAD_DIR`) → job `failed` `"file not readable"`. Le job est créé par le device courant (`ocr_jobs.device_id`) mais la validation de la ressource est scopée par le **user**.
mobile-kotlin/app/src/main/java/com/vaultdrop/mobile/data/local/AppDatabase.kt
+4 -2
View File
@@ -20,14 +20,16 @@ import com.vaultdrop.mobile.data.local.entity.UserPreferenceEntity
* v5: created_in_app sur folders ; v6: processed sur files (mode review) ;
* v7: pending_operations (outbox) ; v8: scan_sessions + scan_pages (scanner) ;
* v9: content sur files (corps des notes créées dans l'app) ;
* v10: ocr_text sur files (extrait OCR serveur persisté localement).
* v10: ocr_text sur files (extrait OCR serveur persisté localement) ;
* v11: ocr_attempts + ocr_queued_at sur files (driver OCR auto côté client —
* compteur d'échecs terminaux ≤ 3, marqueur « en cours » de l'upload).
*/
@Database(
entities = [
FolderEntity::class, UserPreferenceEntity::class, FileEntity::class,
PendingOperationEntity::class, ScanSessionEntity::class, ScanPageEntity::class,
],
version = 10,
version = 11,
exportSchema = false,
)
abstract class AppDatabase : RoomDatabase() {
mobile-kotlin/app/src/main/java/com/vaultdrop/mobile/data/local/dao/FileDao.kt
+29
View File
@@ -126,4 +126,33 @@ interface FileDao {
/** Persiste l'extrait OCR d'un fichier (résultat local du job serveur). */
@Query("UPDATE files SET ocr_text = :text, updated_at = :updatedAt WHERE resource_id = :resourceId")
suspend fun updateOcrText(resourceId: String, text: String?, updatedAt: Long)
/** Candidate OCR auto : fichier `processed`, `local-cloud`, avec copie
* locale, format potentiellement extractible, jamais extrait, pas en cours
* et pas épuisé (< 3 échecs terminaux). Trié par dernier upload. */
@Query("""
SELECT * FROM files
WHERE "exists" = 1
AND processed = 1
AND sync_status = 'local-cloud'
AND uri IS NOT NULL
AND ocr_text IS NULL
AND ocr_attempts < 3
AND ocr_queued_at IS NULL
AND category IN ('PDF', 'IMAGE', 'TEXT')
ORDER BY COALESCE(last_modified, added_at) DESC, name ASC
""")
suspend fun getOcrAutoCandidates(): List<FileEntity>
/** Pose le marqueur « en cours » du driver OCR auto (upload en vol). */
@Query("UPDATE files SET ocr_queued_at = :queuedAt, updated_at = :now WHERE resource_id = :resourceId")
suspend fun markOcrQueued(resourceId: String, queuedAt: Long, now: Long)
/** Levé du marqueur + reset du compteur d'échecs (succès / erreur transitoire). */
@Query("UPDATE files SET ocr_queued_at = NULL, ocr_attempts = 0, updated_at = :now WHERE resource_id = :resourceId")
suspend fun clearOcrQueued(resourceId: String, now: Long)
/** Échec terminal : +1 tentative, marqueur levé (ne rejoue pas à l'infini). */
@Query("UPDATE files SET ocr_attempts = ocr_attempts + 1, ocr_queued_at = NULL, updated_at = :now WHERE resource_id = :resourceId")
suspend fun markOcrFailure(resourceId: String, now: Long)
}
mobile-kotlin/app/src/main/java/com/vaultdrop/mobile/data/local/entity/FileEntity.kt
+4
View File
@@ -57,6 +57,10 @@ data class FileEntity(
val processed: Boolean = false,
@ColumnInfo(name = "ocr_text")
val ocrText: String? = null,
@ColumnInfo(name = "ocr_attempts")
val ocrAttempts: Int = 0,
@ColumnInfo(name = "ocr_queued_at")
val ocrQueuedAt: Long? = null,
@ColumnInfo(name = "added_at")
val addedAt: Long,
@ColumnInfo(name = "updated_at")
mobile-kotlin/app/src/main/java/com/vaultdrop/mobile/data/local/migration/Migrations.kt
+8 -1
View File
@@ -174,7 +174,14 @@ object Migrations {
}
}
private val MIGRATION_10_11 = object : Migration(10, 11) {
override fun migrate(db: SupportSQLiteDatabase) {
db.execSQL("ALTER TABLE `files` ADD COLUMN `ocr_attempts` INTEGER NOT NULL DEFAULT 0")
db.execSQL("ALTER TABLE `files` ADD COLUMN `ocr_queued_at` INTEGER")
}
}
val ALL: Array<Migration> = arrayOf(
MIGRATION_1_2, MIGRATION_2_3, MIGRATION_3_4, MIGRATION_4_5, MIGRATION_5_6, MIGRATION_6_7, MIGRATION_7_8, MIGRATION_8_9, MIGRATION_9_10,
MIGRATION_1_2, MIGRATION_2_3, MIGRATION_3_4, MIGRATION_4_5, MIGRATION_5_6, MIGRATION_6_7, MIGRATION_7_8, MIGRATION_8_9, MIGRATION_9_10, MIGRATION_10_11,
)
}
mobile-kotlin/app/src/main/java/com/vaultdrop/mobile/data/repository/OcrRepository.kt
+61
View File
@@ -7,6 +7,7 @@ import androidx.core.net.toUri
import com.vaultdrop.mobile.data.local.dao.FileDao
import com.vaultdrop.mobile.data.local.entity.FileEntity
import com.vaultdrop.mobile.data.remote.ApiClient
import com.vaultdrop.mobile.data.remote.ApiException
import com.vaultdrop.mobile.data.remote.dto.OcrJobDto
import com.vaultdrop.mobile.data.remote.dto.OcrJobStatus
import dagger.hilt.android.qualifiers.ApplicationContext
@@ -74,11 +75,71 @@ class OcrRepository @Inject constructor(
fileDao.updateOcrText(resourceId, text?.takeIf { it.isNotBlank() }, System.currentTimeMillis())
}
/**
* OCR auto d'un fichier (un seul à la fois) : marqueur « en cours », upload
* multipart ciblé, soumission du job serveur, poll jusqu'au terminal puis
* persistance. Le worker [OcrAutoWorker] appelle cette méthode fichier par
* fichier pour rester séquentiel — le traitement lourd reste la file
* bornée côté serveur.
*/
suspend fun processAuto(file: FileEntity): OcrAutoOutcome {
val now = System.currentTimeMillis()
fileDao.markOcrQueued(file.resourceId, now, now)
return try {
ensurePhysical(file)
val jobId = submit(file.resourceId)
val final = poll(jobId)
when (final.status) {
OcrJobStatus.DONE -> {
saveResult(file.resourceId, final.text)
fileDao.clearOcrQueued(file.resourceId, System.currentTimeMillis())
OcrAutoOutcome.Done
}
else -> {
fileDao.markOcrFailure(file.resourceId, System.currentTimeMillis())
OcrAutoOutcome.Failed
}
}
} catch (e: ApiException) {
when {
e.code == "NETWORK_ERROR" || e.httpCode >= 500 -> {
// Transitoire : levé le marqueur, WorkManager retente avec backoff.
fileDao.clearOcrQueued(file.resourceId, System.currentTimeMillis())
OcrAutoOutcome.Retryable
}
e.httpCode == 401 -> OcrAutoOutcome.Unauthorized
else -> {
// Erreur permanente : +1 tentative (≤ 3), relance manuelle possible.
fileDao.markOcrFailure(file.resourceId, System.currentTimeMillis())
OcrAutoOutcome.Failed
}
}
} catch (e: Exception) {
fileDao.markOcrFailure(file.resourceId, System.currentTimeMillis())
OcrAutoOutcome.Failed
}
}
companion object {
const val POLL_INTERVAL_MS = 3_000L
}
}
/** Résultat d'une passe d'OCR auto (un fichier max). */
sealed interface OcrAutoOutcome {
/** Traité avec succès : `ocr_text` persisté. */
data object Done : OcrAutoOutcome
/** Job terminal `failed` / erreur permanente : `ocr_attempts` incrémenté. */
data object Failed : OcrAutoOutcome
/** Erreur transitoire : à retenter via backoff WorkManager. */
data object Retryable : OcrAutoOutcome
/** Token expiré/révoqué : le re-login passera par l'UI. */
data object Unauthorized : OcrAutoOutcome
}
/**
* `RequestBody` paresseux : le flux SAF est ouvert à l'écriture (multipart),
* jamais chargé en mémoire — les fichiers jusqu'à `MAX_FILE_SIZE_MB` restent
mobile-kotlin/app/src/main/java/com/vaultdrop/mobile/features/ocr/OcrAutoWorker.kt
+100
View File
@@ -0,0 +1,100 @@
package com.vaultdrop.mobile.features.ocr
import android.content.Context
import androidx.hilt.work.HiltWorker
import androidx.work.BackoffPolicy
import androidx.work.Constraints
import androidx.work.CoroutineWorker
import androidx.work.ExistingWorkPolicy
import androidx.work.NetworkType
import androidx.work.OneTimeWorkRequestBuilder
import androidx.work.WorkManager
import androidx.work.WorkerParameters
import com.vaultdrop.mobile.auth.TokenProvider
import com.vaultdrop.mobile.data.local.dao.FileDao
import com.vaultdrop.mobile.data.remote.ApiException
import com.vaultdrop.mobile.data.remote.dto.OcrJobStatus
import com.vaultdrop.mobile.data.repository.OcrAutoOutcome
import com.vaultdrop.mobile.data.repository.OcrRepository
import dagger.assisted.Assisted
import dagger.assisted.AssistedInject
import java.util.concurrent.TimeUnit
import timber.log.Timber
/**
* OCR auto fichier par fichier : à chaque exécution, un seul candidat
* (`processed`, `local-cloud`, `ocr_text` nul, URI SAF présente,
* `ocr_attempts < 3`) est uploadé puis enfilé (`POST /ocr/jobs`) dans la
* **file bornée serveur** (`Ocr.Run`). Un seul run à la fois via
* `enqueueUniqueWork(KEEP)`. Le worker se re-enchaine tant qu'il reste des
* candidats *distincts* du dernier fichier traité (pas de relance immédiate
* d'un fichier en échec, pour respecter le cap de 3 tentatives).
*
* Erreurs :
* - **réseau / 5xx** → `Result.retry()` (backoff 30 s) sans incrémenter
* `ocr_attempts` (le marqueur in-flight est levé) ;
* - **4xx non-idempotente** → `ocr_attempts` incrémenté, pas de re-enchaine ;
* - **token expiré (401)** → `Result.success()` (re-login passera par l'UI).
*/
@HiltWorker
class OcrAutoWorker @AssistedInject constructor(
@Assisted appContext: Context,
@Assisted workerParams: WorkerParameters,
private val ocrRepository: OcrRepository,
private val fileDao: FileDao,
private val tokenProvider: TokenProvider,
) : CoroutineWorker(appContext, workerParams) {
override suspend fun doWork(): Result {
if (tokenProvider.current == null) return Result.success()
val file = fileDao.getOcrAutoCandidates().firstOrNull()
?: return Result.success()
Timber.d("ocr-auto: candidate %s (%s)", file.name, file.resourceId)
val outcome = try {
ocrRepository.processAuto(file)
} catch (e: ApiException) {
when {
e.code == "NETWORK_ERROR" || e.httpCode >= 500 -> OcrAutoOutcome.Retryable
e.httpCode == 401 -> OcrAutoOutcome.Unauthorized
else -> OcrAutoOutcome.Failed
}
} catch (e: Exception) {
Timber.w(e, "ocr-auto: unexpected failure for %s", file.resourceId)
OcrAutoOutcome.Failed
}
when (outcome) {
is OcrAutoOutcome.Done ->
Timber.d("ocr-auto: done %s", file.resourceId)
is OcrAutoOutcome.Failed ->
Timber.w("ocr-auto: failed %s", file.resourceId)
is OcrAutoOutcome.Retryable -> return Result.retry()
is OcrAutoOutcome.Unauthorized -> return Result.success()
}
// Self-chain : un autre candidat *distinct* du dernier attend ?
val more = fileDao.getOcrAutoCandidates().any { it.resourceId != file.resourceId }
if (more) {
enqueue(applicationContext)
}
return Result.success()
}
companion object {
const val NAME = "ocr_auto"
fun enqueue(context: Context) {
val constraints = Constraints.Builder()
.setRequiredNetworkType(NetworkType.CONNECTED)
.build()
val request = OneTimeWorkRequestBuilder<OcrAutoWorker>()
.setConstraints(constraints)
.setBackoffCriteria(BackoffPolicy.EXPONENTIAL, 30, TimeUnit.SECONDS)
.build()
WorkManager.getInstance(context)
.enqueueUniqueWork(NAME, ExistingWorkPolicy.KEEP, request)
}
}
}
mobile-kotlin/app/src/main/java/com/vaultdrop/mobile/features/sync/OutboxSyncWorker.kt
+11 -1
View File
@@ -22,6 +22,7 @@ import com.vaultdrop.mobile.data.local.entity.PendingOperationType
import com.vaultdrop.mobile.data.remote.ApiClient
import com.vaultdrop.mobile.data.remote.ApiException
import com.vaultdrop.mobile.data.remote.dto.SyncOpDto
import com.vaultdrop.mobile.features.ocr.OcrAutoWorker
import dagger.assisted.Assisted
import dagger.assisted.AssistedInject
import java.util.concurrent.TimeUnit
@@ -62,6 +63,9 @@ class OutboxSyncWorker @AssistedInject constructor(
Types.newParameterizedType(Map::class.java, String::class.java, Any::class.java),
)
/** Set à true si un fichier a été promu pendant ce run → enchaîne OcrAutoWorker. */
private var filePromoted = false
override suspend fun doWork(): Result {
// Mode local : sans compte connecté, rien à pousser.
if (tokenProvider.current == null) return Result.success()
@@ -71,10 +75,13 @@ class OutboxSyncWorker @AssistedInject constructor(
fileDao.backfillSyncedStatus(System.currentTimeMillis())
folderDao.backfillSyncedStatus(System.currentTimeMillis())
filePromoted = false
while (true) {
val pending = pendingOperationDao.selectPending(BATCH_SIZE)
if (pending.isEmpty()) {
pendingOperationDao.purgeSynced(System.currentTimeMillis() - PURGE_AGE_MS)
if (filePromoted) OcrAutoWorker.enqueue(applicationContext)
return Result.success()
}
@@ -139,7 +146,10 @@ class OutboxSyncWorker @AssistedInject constructor(
PendingOperationType.CREATE_RESOURCE,
PendingOperationType.MOVE_RESOURCE -> {
when (op.resourceType) {
"file" -> fileDao.promoteSyncStatus(resourceId, System.currentTimeMillis())
"file" -> {
fileDao.promoteSyncStatus(resourceId, System.currentTimeMillis())
filePromoted = true
}
"folder" -> folderDao.promoteSyncStatus(resourceId, System.currentTimeMillis())
}
}
mobile-kotlin/app/src/main/java/com/vaultdrop/mobile/features/sync/SyncViewModel.kt
+3
View File
@@ -11,6 +11,7 @@ import com.vaultdrop.mobile.data.local.entity.PendingOperationEntity
import com.vaultdrop.mobile.data.repository.FolderRepository
import com.vaultdrop.mobile.data.repository.SaveFolderInput
import com.vaultdrop.mobile.data.repository.ShareRepository
import com.vaultdrop.mobile.features.ocr.OcrAutoWorker
import dagger.hilt.android.lifecycle.HiltViewModel
import dagger.hilt.android.qualifiers.ApplicationContext
import kotlinx.coroutines.CancellationException
@@ -107,6 +108,8 @@ class SyncViewModel @Inject constructor(
// Les nouvelles ressources découvertes sont dans l'outbox
// → drainer vers POST /sync/ops (single-flight via KEEP).
OutboxSyncWorker.enqueue(appContext)
// OCR auto : uploader les octets des fichiers récemment promus (local-cloud).
OcrAutoWorker.enqueue(appContext)
}
.onFailure { e -> Timber.w(e, "syncAll failed, retrying later") }
// Hydrate les ressources partagées depuis le snapshot serveur.
mobile-kotlin/app/src/main/java/com/vaultdrop/mobile/ui/auth/AuthViewModel.kt
+3
View File
@@ -9,6 +9,7 @@ import com.vaultdrop.mobile.data.remote.ApiException
import com.vaultdrop.mobile.data.repository.AuthRepository
import com.vaultdrop.mobile.data.repository.ShareRepository
import com.vaultdrop.mobile.features.sync.OutboxSyncWorker
import com.vaultdrop.mobile.features.ocr.OcrAutoWorker
import dagger.hilt.android.lifecycle.HiltViewModel
import dagger.hilt.android.qualifiers.ApplicationContext
import kotlinx.coroutines.Dispatchers
@@ -51,6 +52,7 @@ class AuthViewModel @Inject constructor(
authRepository.registerDevice()
// Session restaurée → drainer l'outbox laissée en attente.
OutboxSyncWorker.enqueue(appContext)
OcrAutoWorker.enqueue(appContext)
if (session != null) {
// Snapshot complet des permissions partagées (convergence).
runCatching { shareRepository.syncSnapshot() }
@@ -73,6 +75,7 @@ class AuthViewModel @Inject constructor(
_authState.value = AuthState.SignedIn(response.user)
// Connexion réussie → pousser les mutations locales en attente.
OutboxSyncWorker.enqueue(appContext)
OcrAutoWorker.enqueue(appContext)
// Snapshot complet des permissions partagées (convergence).
runCatching { shareRepository.syncSnapshot() }
.onFailure { e -> Timber.d("syncSnapshot on login failed: %s", e.message) }
mobile-kotlin/app/src/main/java/com/vaultdrop/mobile/ui/share/ShareViewModel.kt
+2
View File
@@ -7,6 +7,7 @@ import com.vaultdrop.mobile.data.remote.ApiClient
import com.vaultdrop.mobile.data.remote.ApiException
import com.vaultdrop.mobile.data.repository.OutboxRepository
import com.vaultdrop.mobile.features.sync.OutboxSyncWorker
import com.vaultdrop.mobile.features.ocr.OcrAutoWorker
import dagger.hilt.android.lifecycle.HiltViewModel
import dagger.hilt.android.qualifiers.ApplicationContext
import kotlinx.coroutines.flow.MutableStateFlow
@@ -68,6 +69,7 @@ class ShareViewModel @Inject constructor(
)
}.onSuccess {
OutboxSyncWorker.enqueue(appContext)
OcrAutoWorker.enqueue(appContext)
Timber.d("share: op enqueued for %s", resourceId)
_uiState.update { it.copy(sharing = false, enqueued = true) }
}.onFailure { e ->
mobile-kotlin/app/src/test/java/com/vaultdrop/mobile/data/repository/OcrRepositoryTest.kt
+87 -5
View File
@@ -30,6 +30,7 @@ import okhttp3.MultipartBody
import okhttp3.RequestBody
import org.junit.After
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Test
@@ -117,18 +118,24 @@ class OcrRepositoryTest {
assertEquals("boom", final.error)
}
private fun insertFile(uri: String): FileEntity {
private fun insertFile(
uri: String?,
ocrText: String? = null,
category: String? = "TEXT",
): FileEntity {
val file = FileEntity(
resourceId = "aabbccddeeff11223344556677889900",
resourceId = generateResourceId(),
uri = uri,
name = "sample.txt",
name = uri?.substringAfterLast('/') ?: "cloud.txt",
folderResourceId = "f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0",
extension = "txt",
size = 13L,
extension = uri?.substringAfterLast('.')?.take(8),
size = 10L,
mimeType = "text/plain",
exists = 1,
category = category,
syncStatus = FileStatus.LOCAL_CLOUD,
processed = true,
ocrText = ocrText,
addedAt = 1_700_000_000_000L,
updatedAt = 1_700_000_000_000L,
)
@@ -136,8 +143,83 @@ class OcrRepositoryTest {
return file
}
private var idSeq = 0L
private fun generateResourceId(): String {
val hex = (++idSeq).toString(16).padStart(32, '0')
return hex
}
private fun moshi(): Moshi =
Moshi.Builder().add(KotlinJsonAdapterFactory()).build()
@Test
fun `getOcrAutoCandidates_exclut_ocr_text_present`() = runTest {
val withText = insertFile("content://ocr/a.txt", ocrText = "extrait")
val withoutText = insertFile("content://ocr/b.txt", ocrText = null)
val candidates = fileDao.getOcrAutoCandidates()
assertEquals(1, candidates.size)
assertEquals(withoutText.resourceId, candidates[0].resourceId)
}
@Test
fun `getOcrAutoCandidates_exclut_uri_absente_et_cloud_only`() = runTest {
insertFile("content://ocr/a.txt", ocrText = null)
insertFile(null, ocrText = null)
val candidates = fileDao.getOcrAutoCandidates()
assertEquals(1, candidates.size)
}
@Test
fun `getOcrAutoCandidates_exclut_exhausted_attempts_3`() = runTest {
val exhausted = insertFile("content://ocr/a.txt", ocrText = null)
fileDao.markOcrFailure(exhausted.resourceId, System.currentTimeMillis())
fileDao.markOcrFailure(exhausted.resourceId, System.currentTimeMillis())
fileDao.markOcrFailure(exhausted.resourceId, System.currentTimeMillis())
assertTrue(fileDao.getOcrAutoCandidates().isEmpty())
}
@Test
fun `processAuto_done_persiste_texte_et_clear_queued`() = runTest {
val uri = "content://ocr/auto-done.txt"
shadowOf(context.contentResolver).registerInputStream(uri.toUri(), ByteArrayInputStream("auto".toByteArray()))
val file = insertFile(uri)
apiService.jobSequence = listOf(
OcrJobDto(id = "auto-1", status = OcrJobStatus.QUEUED),
OcrJobDto(id = "auto-1", status = OcrJobStatus.DONE, text = "autoOCR"),
)
val outcome = repository.processAuto(file)
assertTrue(outcome is OcrAutoOutcome.Done)
val stored = fileDao.getByResourceId(file.resourceId)
assertEquals("autoOCR", stored?.ocrText)
assertEquals(0, stored?.ocrAttempts)
assertNull(stored?.ocrQueuedAt)
}
@Test
fun `processAuto_failed_incremente_attempts_et_clear_queued`() = runTest {
val uri = "content://ocr/auto-fail.txt"
shadowOf(context.contentResolver).registerInputStream(uri.toUri(), ByteArrayInputStream(byteArrayOf()))
val file = insertFile(uri)
apiService.jobSequence = listOf(
OcrJobDto(id = "auto-f", status = OcrJobStatus.FAILED, error = "tess err"),
)
val outcome = repository.processAuto(file)
assertTrue(outcome is OcrAutoOutcome.Failed)
val stored = fileDao.getByResourceId(file.resourceId)
assertEquals(1, stored?.ocrAttempts)
assertNull(stored?.ocrQueuedAt)
assertNull(stored?.ocrText)
}
}
/**