Thirdshop/Kazier
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(api): OCR réel — POST /ocr/jobs + GET /ocr/jobs/:id (tesseract syscall, jobs asynchrones)

Browse Source 
- ocr/tesseract.go : Engine → Tesseract subprocess (OCR_LANG fra+eng) ; PDF → calque texte via ledongthuc/pdf (go.mod : nouveau dep)
- repository/ocr_jobs.go : queued→processing→done/failed, scoping device, text/error NULLIFés
- service/ocr.go : Create valide le fichier (GetFile), queue + goroutine de traitement ; physique résolu par glob UPLOAD_DIR/<device>/<id>.* ; fail propre (fichier illisible, erreur moteur)
- handlers/ocr.go : réels (validate fileId, NOT_FOUND si job d'un autre device), fin 501 OCR
- tests end-to-end : cycle queued→done (stub moteur), NOT_FOUND fichier inconnu, scoping device
- smoke réel : PNG 'VAULTDROP' → job done text='VAULTDROP' (tesseract installé)
- docs/AGENTS : §5 + état des routes (tout V1 réel)
This commit is contained in:
m
2026-09-10 19:54:16 +02:00
parent 5d96814853
commit d5886c678b
12 changed files with 452 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/cmd/server/main.go
+4 -1
View File
@@ -8,6 +8,7 @@ import (
"github.com/vaultdrop/backend/config"
"github.com/vaultdrop/backend/db"
"github.com/vaultdrop/backend/handlers"
"github.com/vaultdrop/backend/ocr"
"github.com/vaultdrop/backend/pkg/auth"
"github.com/vaultdrop/backend/repository"
"github.com/vaultdrop/backend/service"
@@ -42,11 +43,13 @@ func main() {
}
handlers.Auth = authManager
repo := repository.NewRepository(conn)
handlers.Store = service.NewResources(
repository.NewRepository(conn),
repo,
cfg.UploadDir,
cfg.MaxFileSizeMB*1024*1024,
)
handlers.Ocr = service.NewOcr(repo, cfg.UploadDir, cfg.OcrLang, ocr.NewTesseract())
if err := newRouter().Run(fmt.Sprintf(":%d", cfg.Port)); err != nil {
log.Fatalln(err)
backend/go.mod
+1
View File
@@ -25,6 +25,7 @@ require (
github.com/goccy/go-yaml v1.19.2 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/ledongthuc/pdf v0.0.0-20260907135840-6c8c28e0e8a0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
backend/go.sum
+2
View File
@@ -69,6 +69,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/ledongthuc/pdf v0.0.0-20260907135840-6c8c28e0e8a0 h1:7Q+xNAZFmnfYOMweHN3c/PDFUKKfY1pVJ26K++QvVfU=
github.com/ledongthuc/pdf v0.0.0-20260907135840-6c8c28e0e8a0/go.mod h1:1fEHWurg7pvf5SG6XNE5Q8UZmOwex51Mkx3SLhrW5B4=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ=
backend/handlers/ocr.go
+46 -2
View File
@@ -1,9 +1,53 @@
package handlers
import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/vaultdrop/backend/pkg/api"
"github.com/vaultdrop/backend/repository"
)
func OcrJobsCreate(c *gin.Context) { api.NotImplemented(c) }
func OcrJobsGet(c *gin.Context) { api.NotImplemented(c) }
type ocrJobRequest struct {
FileID string `json:"fileId"`
}
func OcrJobsCreate(c *gin.Context) {
if Store == nil || Ocr == nil {
api.Error(c, http.StatusServiceUnavailable, "SERVICE_UNAVAILABLE", "backend not initialized")
return
}
deviceID := c.GetString(DeviceIDKey)
var req ocrJobRequest
if err := c.ShouldBindJSON(&req); err != nil {
api.Error(c, http.StatusBadRequest, "INVALID_REQUEST", "invalid request")
return
}
job, err := Ocr.Create(deviceID, req.FileID)
if err != nil {
writeError(c, err)
return
}
api.OK(c, job)
}
func OcrJobsGet(c *gin.Context) {
if Ocr == nil {
api.Error(c, http.StatusServiceUnavailable, "SERVICE_UNAVAILABLE", "backend not initialized")
return
}
deviceID := c.GetString(DeviceIDKey)
job, err := Ocr.Get(deviceID, c.Param("id"))
if err != nil {
if err == repository.ErrJobNotFound {
api.Error(c, http.StatusNotFound, "NOT_FOUND", "ocr job not found")
return
}
writeError(c, err)
return
}
api.OK(c, job)
}
backend/handlers/ocr_test.go
+143
View File
@@ -0,0 +1,143 @@
package handlers_test
import (
"context"
"encoding/json"
"net/http"
"os"
"path/filepath"
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/vaultdrop/backend/dbtest"
"github.com/vaultdrop/backend/handlers"
"github.com/vaultdrop/backend/pkg/auth"
"github.com/vaultdrop/backend/repository"
"github.com/vaultdrop/backend/service"
)
type stubEngine struct {
text string
err error
}
func (s stubEngine) ExtractText(_ context.Context, _ string, _ string) (string, error) {
if s.err != nil {
return "", s.err
}
return s.text, nil
}
func setupOcr(t *testing.T) (*gin.Engine, string) {
t.Helper()
conn := dbtest.OpenTestDatabase(t, handlersTestURL)
repo := repository.NewRepository(conn)
uploadDir := t.TempDir()
store := service.NewResources(repo, uploadDir, 100*1024)
manager, _ := auth.NewManager("test-secret")
handlers.Auth = manager
handlers.Store = store
handlers.Ocr = service.NewOcr(repo, uploadDir, "fra+eng", stubEngine{text: "HELLO OCR"})
gin.SetMode(gin.TestMode)
r := gin.New()
handlers.RegisterRoutes(r)
return r, uploadDir
}
type ocrJobDTO struct {
ID string `json:"id"`
Status string `json:"status"`
Text string `json:"text"`
Error string `json:"error"`
}
func waitTerminal(t *testing.T, r *gin.Engine, token, jobID string) ocrJobDTO {
t.Helper()
deadline := time.Now().Add(3 * time.Second)
for time.Now().Before(deadline) {
rec, _ := doRequest(t, r, http.MethodGet, "/api/v1/ocr/jobs/"+jobID, token, nil, "")
env := expectOK(t, rec, "ocr-get")
var job ocrJobDTO
if err := json.Unmarshal(env.Data, &job); err != nil {
t.Fatalf("ocr-get: unmarshal: %v", err)
}
if job.Status == "done" || job.Status == "failed" {
return job
}
time.Sleep(50 * time.Millisecond)
}
t.Fatal("job OCR jamais terminal")
return ocrJobDTO{}
}
func TestOcrJobsLifecycle(t *testing.T) {
r, uploadDir := setupOcr(t)
device := repository.NewID()
token := registerDevice(t, r, device)
// Fichier + fichier physique (simule UPLOAD_DIR/<device>/<id>.txt)
fileID := repository.NewID()
if err := os.MkdirAll(filepath.Join(uploadDir, device), 0o755); err != nil {
t.Fatalf("mkdir: %v", err)
}
if err := os.WriteFile(filepath.Join(uploadDir, device, fileID+".txt"), []byte("ignored by stub"), 0o644); err != nil {
t.Fatalf("write: %v", err)
}
repo := handlers.Store.Repository
if err := repo.Resources.InsertFile(device, fileID, "scan.png", "", 128, nil, nil); err != nil {
t.Fatalf("insert: %v", err)
}
// Création du job
body, _ := json.Marshal(map[string]string{"fileId": fileID})
rec, _ := doRequest(t, r, http.MethodPost, "/api/v1/ocr/jobs", token, body, "application/json")
env := expectOK(t, rec, "ocr-create")
var created ocrJobDTO
if err := json.Unmarshal(env.Data, &created); err != nil {
t.Fatalf("ocr-create: unmarshal: %v", err)
}
if created.Status != "queued" || created.ID == "" {
t.Errorf("job attendu queued: %+v", created)
}
// Poll jusqu'au terminal
job := waitTerminal(t, r, token, created.ID)
if job.Status != "done" || job.Text != "HELLO OCR" {
t.Errorf("job terminal: %+v", job)
}
// Fichier inconnu → NOT_FOUND
rec, _ = doRequest(t, r, http.MethodPost, "/api/v1/ocr/jobs", token, bodyFor(repository.NewID()), "application/json")
expectError(t, rec, http.StatusNotFound, "NOT_FOUND", "ocr-missing-file")
}
func TestOcrJobsScopedByDevice(t *testing.T) {
r, _ := setupOcr(t)
deviceA := repository.NewID()
deviceB := repository.NewID()
tokenA := registerDevice(t, r, deviceA)
tokenB := registerDevice(t, r, deviceB)
fileID := repository.NewID()
if err := handlers.Store.Repository.Resources.InsertFile(deviceA, fileID, "scan.png", "", 128, nil, nil); err != nil {
t.Fatalf("insert: %v", err)
}
body, _ := json.Marshal(map[string]string{"fileId": fileID})
rec, _ := doRequest(t, r, http.MethodPost, "/api/v1/ocr/jobs", tokenA, body, "application/json")
env := expectOK(t, rec, "ocr-create")
var created ocrJobDTO
_ = json.Unmarshal(env.Data, &created)
// Un autre device ne voit pas le job
rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/ocr/jobs/"+created.ID, tokenB, nil, "")
expectError(t, rec, http.StatusNotFound, "NOT_FOUND", "ocr-other-device")
}
func bodyFor(id string) []byte {
b, _ := json.Marshal(map[string]string{"fileId": id})
return b
}
backend/handlers/store.go
+3
View File
@@ -13,6 +13,9 @@ import (
// (cmd/server). Nil until then.
var Store *service.Resources
// Ocr queues/reads OCR jobs; set once at startup alongside Store.
var Ocr *service.Ocr
// writeError maps repository/service sentinels to contract error codes.
func writeError(c *gin.Context, err error) {
switch {
backend/ocr/tesseract.go
+67
View File
@@ -0,0 +1,67 @@
package ocr
import (
"bytes"
"context"
"errors"
"os/exec"
"path/filepath"
"strings"
"github.com/ledongthuc/pdf"
)
// TesseractEngine runs the `tesseract` binary in a subprocess.
// Images are OCR'd directly; PDFs have their text layer extracted first
// (scanned PDFs → empty text, no rendering pipeline in V1).
type TesseractEngine struct{}
func NewTesseract() *TesseractEngine { return &TesseractEngine{} }
// ExtractText implements Engine.
func (t *TesseractEngine) ExtractText(ctx context.Context, filePath, lang string) (string, error) {
if strings.ToLower(filepath.Ext(filePath)) == ".pdf" {
return extractPDFText(filePath)
}
return runTesseract(ctx, filePath, lang)
}
func runTesseract(ctx context.Context, filePath, lang string) (string, error) {
cmd := exec.CommandContext(ctx, "tesseract", filePath, "stdout", "-l", lang)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
if errors.Is(ctx.Err(), context.Canceled) {
return "", ctx.Err()
}
msg := strings.TrimSpace(stderr.String())
if msg == "" {
msg = err.Error()
}
return "", errors.New("tesseract: " + msg)
}
return strings.TrimSpace(stdout.String()), nil
}
func extractPDFText(filePath string) (string, error) {
f, r, err := pdf.Open(filePath)
if err != nil {
return "", errors.New("pdf: " + err.Error())
}
defer f.Close()
var builder strings.Builder
for i := 1; i <= r.NumPage(); i++ {
p := r.Page(i)
if p.V.IsNull() {
continue
}
plain, err := p.GetPlainText(nil)
if err != nil {
continue
}
builder.WriteString(plain)
builder.WriteString("\n")
}
return strings.TrimSpace(builder.String()), nil
}
backend/repository/ocr_jobs.go
+81
View File
@@ -0,0 +1,81 @@
package repository
import (
"database/sql"
"errors"
)
// OcrJobRow maps ocr_jobs.
type OcrJobRow struct {
ID string
FileID string
DeviceID string
Status string
Text *string
Error *string
}
// ErrJobNotFound marks an OCR job absent or owned by another device.
var ErrJobNotFound = errors.New("ocr job not found")
type OcrJobs struct{ DB *sql.DB }
func (o *OcrJobs) Create(jobID, deviceID, fileID string) error {
_, err := o.DB.Exec(
`INSERT INTO ocr_jobs (job_id, device_id, file_id) VALUES ($1, $2, $3)`,
jobID, deviceID, fileID,
)
return err
}
// Get returns a job scoped by device (no-rows → ErrJobNotFound).
func (o *OcrJobs) Get(deviceID, jobID string) (OcrJobRow, error) {
var row OcrJobRow
var text, errMsg sql.NullString
err := o.DB.QueryRow(
`SELECT job_id, file_id, device_id, status,
NULLIF(text, ''), NULLIF(error, '')
FROM ocr_jobs WHERE job_id = $1 AND device_id = $2`,
jobID, deviceID,
).Scan(&row.ID, &row.FileID, &row.DeviceID, &row.Status, &text, &errMsg)
if err == sql.ErrNoRows {
return OcrJobRow{}, ErrJobNotFound
}
if err != nil {
return OcrJobRow{}, err
}
if text.Valid {
row.Text = &text.String
}
if errMsg.Valid {
row.Error = &errMsg.String
}
return row, nil
}
func (o *OcrJobs) TouchProcessing(deviceID, jobID string) error {
_, err := o.DB.Exec(
`UPDATE ocr_jobs SET status = 'processing', started_at = NOW()
WHERE job_id = $1 AND device_id = $2 AND status = 'queued'`,
jobID, deviceID,
)
return err
}
func (o *OcrJobs) Complete(deviceID, jobID, text string) error {
_, err := o.DB.Exec(
`UPDATE ocr_jobs SET status = 'done', text = NULLIF($3, ''), started_at = COALESCE(started_at, NOW()), completed_at = NOW()
WHERE job_id = $1 AND device_id = $2`,
jobID, deviceID, text,
)
return err
}
func (o *OcrJobs) Fail(deviceID, jobID, message string) error {
_, err := o.DB.Exec(
`UPDATE ocr_jobs SET status = 'failed', error = NULLIF($3, ''), started_at = COALESCE(started_at, NOW()), completed_at = NOW()
WHERE job_id = $1 AND device_id = $2`,
jobID, deviceID, message,
)
return err
}
backend/repository/repository.go
+2
View File
@@ -9,6 +9,7 @@ type Repository struct {
Resources *Resources
Devices *Devices
Operations *Operations
OcrJobs *OcrJobs
}
func NewRepository(conn *sql.DB) *Repository {
@@ -16,5 +17,6 @@ func NewRepository(conn *sql.DB) *Repository {
Resources: &Resources{DB: conn},
Devices: &Devices{DB: conn},
Operations: &Operations{DB: conn},
OcrJobs: &OcrJobs{DB: conn},
}
}
backend/service/ocr.go
+96
View File
@@ -0,0 +1,96 @@
package service
import (
"context"
"path/filepath"
"strings"
"github.com/vaultdrop/backend/ocr"
"github.com/vaultdrop/backend/repository"
)
// OcrJobDTO serializes exactly as mobile/api/types.ts OcrJob.
type OcrJobDTO struct {
ID string `json:"id"`
Status string `json:"status"`
Text string `json:"text,omitempty"`
Error string `json:"error,omitempty"`
}
// Ocr queues OCR jobs and processes them asynchronously (V1 : goroutine par
// job ; le client poll GET /ocr/jobs/:id toutes les 3s).
type Ocr struct {
Repository *repository.Repository
UploadDir string
Lang string
Engine ocr.Engine
}
func NewOcr(repo *repository.Repository, uploadDir, lang string, engine ocr.Engine) *Ocr {
return &Ocr{Repository: repo, UploadDir: uploadDir, Lang: lang, Engine: engine}
}
// Create validates the file, queued the job, and starts processing.
func (o *Ocr) Create(deviceID, fileID string) (OcrJobDTO, error) {
if _, err := o.Repository.Resources.GetFile(deviceID, fileID); err != nil {
return OcrJobDTO{}, err
}
jobID := repository.NewID()
if err := o.Repository.OcrJobs.Create(jobID, deviceID, fileID); err != nil {
return OcrJobDTO{}, err
}
go o.process(deviceID, jobID, fileID)
return OcrJobDTO{ID: jobID, Status: "queued"}, nil
}
func (o *Ocr) Get(deviceID, jobID string) (OcrJobDTO, error) {
row, err := o.Repository.OcrJobs.Get(deviceID, jobID)
if err != nil {
return OcrJobDTO{}, err
}
return toOcrJobDTO(row), nil
}
func (o *Ocr) process(deviceID, jobID, fileID string) {
ctx := context.Background()
if err := o.Repository.OcrJobs.TouchProcessing(deviceID, jobID); err != nil {
return
}
path, err := o.physicalPath(deviceID, fileID)
if err != nil {
_ = o.Repository.OcrJobs.Fail(deviceID, jobID, "file not readable")
return
}
text, err := o.Engine.ExtractText(ctx, path, o.Lang)
if err != nil {
_ = o.Repository.OcrJobs.Fail(deviceID, jobID, err.Error())
return
}
_ = o.Repository.OcrJobs.Complete(deviceID, jobID, text)
}
// physicalPath resolves UPLOAD_DIR/<device_id>/<resource_id>.<ext> — the ext
// is chosen at upload time, so the actual file is matched by prefix.
func (o *Ocr) physicalPath(deviceID, fileID string) (string, error) {
matches, err := filepath.Glob(filepath.Join(o.UploadDir, deviceID, fileID+".*"))
if err != nil {
return "", err
}
for _, m := range matches {
if strings.HasPrefix(filepath.Base(m), fileID+".") {
return m, nil
}
}
return "", repository.ErrNotFound
}
func toOcrJobDTO(row repository.OcrJobRow) OcrJobDTO {
dto := OcrJobDTO{ID: row.ID, Status: row.Status}
if row.Text != nil {
dto.Text = *row.Text
}
if row.Error != nil {
dto.Error = *row.Error
}
return dto
}