Thirdshop/Kazier
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ocr is working

Browse Source
This commit is contained in:
m
2026-09-16 12:44:48 +02:00
parent 050434325a
commit 906d48702c
21 changed files with 595 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/service/ocr.go
+73 -15
View File
@@ -2,13 +2,25 @@ package service
import (
"context"
"errors"
"log"
"path/filepath"
"strings"
"sync"
"time"
"github.com/vaultdrop/backend/ocr"
"github.com/vaultdrop/backend/repository"
)
const (
// StaleClaimTimeout : un job `processing` plus vieux que ça est ré-enfilé
// au boot (worker mort / process redémarré).
StaleClaimTimeout = 15 * time.Minute
// QueuePollInterval : cadence de scrutation quand la file est vide.
QueuePollInterval = time.Second
)
// OcrJobDTO serializes exactly as mobile/api/types.ts OcrJob.
type OcrJobDTO struct {
ID string `json:"id"`
@@ -17,22 +29,24 @@ type OcrJobDTO struct {
Error string `json:"error,omitempty"`
}
// Ocr queues OCR jobs and processes them asynchronously (V1 : goroutine par
// job ; le client poll GET /ocr/jobs/:id toutes les 3s).
// Ocr queues OCR jobs in `ocr_jobs` (status `queued`) and processes them
// asynchronously through a bounded worker ([Run]) — Tesseract ne tourne jamais
// dans un request handler. Le client poll GET /ocr/jobs/:id toutes les 3s.
type Ocr struct {
Repository *repository.Repository
UploadDir string
Lang string
Engine ocr.Engine
StaleAfter time.Duration
}
func NewOcr(repo *repository.Repository, uploadDir, lang string, engine ocr.Engine) *Ocr {
return &Ocr{Repository: repo, UploadDir: uploadDir, Lang: lang, Engine: engine}
return &Ocr{Repository: repo, UploadDir: uploadDir, Lang: lang, Engine: engine, StaleAfter: StaleClaimTimeout}
}
// Create valide la ressource (ownership par user), met le job en file et
// lance le traitement. Le job reste scopé par device (le service génère les
// jobs du device courant ; l'outbox OCR n'est pas synchronisée entre devices).
// Create valide la ressource (ownership par user) et l'enfile uniquement
// (status `queued`) — le traitement est délégué au worker [Run] pour ne pas
// stresser le serveur. Le job reste scopé par device.
func (o *Ocr) Create(userID, deviceID, fileID string) (OcrJobDTO, error) {
if _, err := o.Repository.Resources.GetFile(userID, fileID); err != nil {
return OcrJobDTO{}, err
@@ -41,7 +55,6 @@ func (o *Ocr) Create(userID, deviceID, fileID string) (OcrJobDTO, error) {
if err := o.Repository.OcrJobs.Create(jobID, deviceID, fileID); err != nil {
return OcrJobDTO{}, err
}
go o.process(userID, deviceID, jobID, fileID)
return OcrJobDTO{ID: jobID, Status: "queued"}, nil
}
@@ -53,22 +66,67 @@ func (o *Ocr) Get(deviceID, jobID string) (OcrJobDTO, error) {
return toOcrJobDTO(row), nil
}
func (o *Ocr) process(userID, deviceID, jobID, fileID string) {
ctx := context.Background()
if err := o.Repository.OcrJobs.TouchProcessing(deviceID, jobID); err != nil {
// Run fait tourner le worker OCR : au démarrage il requeue les jobs laissés
// en `processing` (crash), puis `workers` boucles consomment la file FIFO via
// ClaimNext (atomique, `FOR UPDATE SKIP LOCKED`). `workers` borne la
// concurrence Tesseract. Retourne quand ctx est annulé.
func (o *Ocr) Run(ctx context.Context, workers int) {
if workers < 1 {
workers = 1
}
if _, err := o.Repository.OcrJobs.ResetStaleProcessing(o.StaleAfter); err != nil {
log.Printf("ocr: requeue jobs stale: %v", err)
}
var wg sync.WaitGroup
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
o.workerLoop(ctx)
}()
}
wg.Wait()
}
// workerLoop consomme la file : un job `queued` → processing → done/failed.
// File vide → scrutation à QueuePollInterval jusqu'à annulation du ctx.
func (o *Ocr) workerLoop(ctx context.Context) {
for {
row, err := o.Repository.OcrJobs.ClaimNext()
if err != nil {
if !errors.Is(err, repository.ErrNoQueuedJobs) {
log.Printf("ocr: claim: %v", err)
}
select {
case <-ctx.Done():
return
case <-time.After(QueuePollInterval):
}
continue
}
o.process(ctx, row)
}
}
// process exécute l'extraction pour un job déjà réclamé (`processing`).
func (o *Ocr) process(ctx context.Context, row repository.OcrJobRow) {
userID, err := o.Repository.Resources.GetResourceOwner(row.FileID)
if err != nil {
_ = o.Repository.OcrJobs.Fail(row.DeviceID, row.ID, "file not readable")
return
}
path, err := o.physicalPath(userID, fileID)
path, err := o.physicalPath(userID, row.FileID)
if err != nil {
_ = o.Repository.OcrJobs.Fail(deviceID, jobID, "file not readable")
_ = o.Repository.OcrJobs.Fail(row.DeviceID, row.ID, "file not readable")
return
}
text, err := o.Engine.ExtractText(ctx, path, o.Lang)
if err != nil {
_ = o.Repository.OcrJobs.Fail(deviceID, jobID, err.Error())
_ = o.Repository.OcrJobs.Fail(row.DeviceID, row.ID, err.Error())
return
}
_ = o.Repository.OcrJobs.Complete(deviceID, jobID, text)
_ = o.Repository.OcrJobs.Complete(row.DeviceID, row.ID, text)
}
// physicalPath résout UPLOAD_DIR/<user_id>/<resource_id>.<ext> — l'ext est
@@ -95,4 +153,4 @@ func toOcrJobDTO(row repository.OcrJobRow) OcrJobDTO {
dto.Error = *row.Error
}
return dto
}
}
backend/service/ocr_test.go
+7 -2
View File
@@ -26,13 +26,18 @@ func (s stubEngine) ExtractText(_ context.Context, _ string, _ string) (string,
return s.text, nil
}
// newTestOcr builds an Ocr over a fresh DB and returns it with the ids.
// newTestOcr builds an Ocr over a fresh DB, starts the bounded worker (1
// worker) and returns it with the ids.
func newTestOcr(t *testing.T, uploadDir string, engine ocr.Engine) (*Ocr, string, string, string) {
t.Helper()
s := newServiceStore(t)
userID := mustCreateUser(t, s.Repository, "ocr-failed")
deviceID := mustRegisterDevice(t, s.Repository, repository.NewID())
return NewOcr(s.Repository, uploadDir, "fra+eng", engine), userID, deviceID, uploadDir
o := NewOcr(s.Repository, uploadDir, "fra+eng", engine)
ctx, cancel := context.WithCancel(context.Background())
go o.Run(ctx, 1)
t.Cleanup(cancel)
return o, userID, deviceID, uploadDir
}
// waitTillTerminal poll jusqu'à un statut terminal (done/failed).