V3 S1-S2 : corbeille + versions + extract générique

S1 filet (backend + mobile) : corbeille 30j (deleted_at réutilisé,
trash/restore récursifs, purge auto serveur), versions N=3 à chaque
overwrite, ops outbox trash/restore/purge (delete legacy mappé), Room
v19 (trashed_at), Deleters -> trash, écran Corbeille, walk guards.

S2 extract (backend) : package extract (règles FR stdlib, corpus
testé), resource_facts 000019 (origin auto/user), hook worker OCR +
backfill boot, GET /files/:id/facts (viewer+).

Contrat docs/api-v1.md étendu, V3.md (vision + plan).
This commit is contained in:
venture
2026-09-26 10:14:07 +02:00
parent a34220bf2d
commit 95b4cf79cf
75 changed files with 5220 additions and 119 deletions
+51
View File
@@ -87,6 +87,16 @@ func main() {
)
handlers.Ocr = service.NewOcr(repo, cfg.UploadDir, cfg.OcrLang, ocr.NewTesseract())
// V3 S2 : faits génériques — chaque texte OCR extrait alimente
// `resource_facts` (origin='auto'). Enrichissement best-effort : un
// échec d'extract ne fait jamais échouer le job OCR.
extractSvc := service.NewExtract(repo)
handlers.Ocr.OnText = func(userID, resourceID, text string) {
if err := extractSvc.ProcessText(resourceID, text); err != nil {
log.Printf("extract: %s: %v", resourceID, err)
}
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
@@ -102,6 +112,47 @@ func main() {
}()
log.Printf("ocr worker démarré (%d workers)", cfg.OcrWorkers)
// V3 S2 : backfill extract (une passe bornée au boot — les ressources
// avec ocr_text mais sans faits auto, ex. OCR d'avant la V3).
go func() {
n, err := extractSvc.Backfill(200)
if err != nil {
log.Printf("extract backfill: %v", err)
return
}
if n > 0 {
log.Printf("extract backfill: %d ressources enrichies", n)
}
}()
// V3 corbeille : passe de purge (trash > TrashRetention) au boot puis
// toutes les heures. Best-effort : une passe ratée sera rejouée à la
// suivante, jamais de crash serveur sur une erreur de purge.
go func() {
purge := func() {
n, err := handlers.Store.PurgeExpiredTrash(service.TrashRetention)
if err != nil {
log.Printf("purge corbeille: %v", err)
return
}
if n > 0 {
log.Printf("purge corbeille: %d ressources détruites", n)
}
}
purge()
ticker := time.NewTicker(time.Hour)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
purge()
}
}
}()
log.Printf("purge corbeille démarrée (rétention %s)", service.TrashRetention)
srv := &http.Server{
Addr: fmt.Sprintf(":%d", cfg.Port),
Handler: newRouter(),
+5
View File
@@ -17,10 +17,15 @@ var expectedRoutes = []string{
"GET /api/v1/files/folders",
"GET /api/v1/files/:id",
"GET /api/v1/files/:id/ocr",
"GET /api/v1/files/:id/facts",
"GET /api/v1/files/:id/content",
"GET /api/v1/files/:id/shares",
"GET /api/v1/files/:id/links",
"GET /api/v1/files/:id/placements",
"GET /api/v1/files/:id/versions",
"POST /api/v1/files/:id/restore",
"POST /api/v1/files/:id/versions/:n/restore",
"GET /api/v1/trash",
"DELETE /api/v1/files/:id",
"POST /api/v1/files/upload",
@@ -0,0 +1 @@
DROP INDEX IF EXISTS idx_resources_trashed_active;
@@ -0,0 +1,7 @@
-- V3 corbeille : `deleted_at` est le timestamp de trash (restauration =
-- `deleted_at = NULL`, purge = hard delete après 30j via PurgeExpired).
-- Toutes les lectures filtrent déjà `deleted_at IS NULL` : les trashés sont
-- invisibles partout sauf via les requêtes dédiées (ListTrashed). Cet index
-- partiel accélère la corbeille (`GET /trash`) et la passe de purge.
CREATE INDEX IF NOT EXISTS idx_resources_trashed_active
ON resources(user_id, deleted_at) WHERE deleted_at IS NOT NULL;
@@ -0,0 +1 @@
DROP TABLE IF EXISTS resource_versions;
@@ -0,0 +1,19 @@
-- V3 versions : historique léger des octets écrasés (N=3, cf. V3.md P1).
-- À chaque overwrite (upload ciblé), l'ancienne copie physique est déplacée
-- sous `stored_path` (relatif à UPLOAD_DIR) et sa métadonnée figée ici.
-- Restaurer = recopier vers l'emplacement live (l'actuel est à son tour
-- versionné). `ON DELETE CASCADE` : purger la ressource purge ses versions.
CREATE TABLE resource_versions (
id TEXT PRIMARY KEY CHECK (id ~ '^[0-9a-f]{32}$'),
resource_id TEXT NOT NULL REFERENCES resources(resource_id) ON DELETE CASCADE,
version_no INT NOT NULL CHECK (version_no >= 1),
size_bytes BIGINT NOT NULL DEFAULT 0,
mime_type VARCHAR(255),
extension VARCHAR(64),
content_hash VARCHAR(128),
stored_path TEXT NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE (resource_id, version_no)
);
CREATE INDEX idx_versions_resource ON resource_versions(resource_id);
@@ -0,0 +1 @@
DROP TABLE IF EXISTS resource_facts;
+23
View File
@@ -0,0 +1,23 @@
-- V3 extract générique (S2) : faits universels extraits du texte OCR
-- (dates, montants, émetteurs, références, type suggéré) — cf. V3.md §2.
-- `doc_type` n'est qu'un fait comme un autre (suggestion, jamais gate) ;
-- `origin` distingue l'auto (remplaçable à chaque passe) du `user` (S3,
-- correction en un tap — l'auto ne l'écrase jamais). Cascade : purger la
-- ressource purge ses faits.
CREATE TABLE resource_facts (
id TEXT PRIMARY KEY CHECK (id ~ '^[0-9a-f]{32}$'),
resource_id TEXT NOT NULL REFERENCES resources(resource_id) ON DELETE CASCADE,
fact_type TEXT NOT NULL CHECK (fact_type IN (
'date', 'due_date', 'period', 'amount', 'issuer', 'reference', 'doc_type'
)),
value_text TEXT,
value_num NUMERIC,
value_date DATE,
confidence REAL NOT NULL CHECK (confidence >= 0 AND confidence <= 1),
origin TEXT NOT NULL DEFAULT 'auto' CHECK (origin IN ('auto', 'user')),
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
CREATE INDEX idx_facts_resource ON resource_facts(resource_id);
CREATE INDEX idx_facts_type_value ON resource_facts(fact_type, value_text);
+53
View File
@@ -369,6 +369,59 @@ func TestMigrationsUpDown(t *testing.T) {
// --- 000016 : groupes + cible groupe sur shares -------------------------
assertTables(t, conn, "users", "devices", "resources", "operations", "ocr_jobs", "shares", "share_links", "storage_locations", "resource_placements", "groups", "group_members", "schema_migrations")
// --- 000017 : corbeille (index partiel deleted_at) ----------------------
var trashedIdx int
err = conn.QueryRow(`
SELECT COUNT(*) FROM pg_index i
JOIN pg_class t ON t.oid = i.indrelid
WHERE t.relname = 'resources' AND i.indisvalid
AND pg_get_indexdef(i.indexrelid) LIKE '%idx_resources_trashed_active%'`).Scan(&trashedIdx)
if err != nil {
t.Fatalf("000017 index: %v", err)
}
if trashedIdx == 0 {
t.Error("index idx_resources_trashed_active manquant après 000017")
}
// --- 000018 : versions --------------------------------------------------
assertTables(t, conn, "users", "devices", "resources", "operations", "ocr_jobs", "shares", "share_links", "storage_locations", "resource_placements", "groups", "group_members", "resource_versions", "schema_migrations")
assertHexCheck(t, conn, "resource_versions", "id")
var versionUniq int
err = conn.QueryRow(`
SELECT COUNT(*) FROM pg_index i
JOIN pg_class t ON t.oid = i.indrelid
WHERE t.relname = 'resource_versions' AND i.indisunique
AND ARRAY(SELECT a.attname FROM unnest(i.indkey) WITH ORDINALITY k(attnum, ord)
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum
ORDER BY k.ord)::text[] = ARRAY['resource_id','version_no']`).Scan(&versionUniq)
if err != nil {
t.Fatalf("versions unique: %v", err)
}
if versionUniq == 0 {
t.Error("contrainte UNIQUE(resource_id, version_no) manquante sur resource_versions après 000018")
}
// --- 000019 : faits génériques -----------------------------------------
assertTables(t, conn, "users", "devices", "resources", "operations", "ocr_jobs", "shares", "share_links", "storage_locations", "resource_placements", "groups", "group_members", "resource_versions", "resource_facts", "schema_migrations")
assertHexCheck(t, conn, "resource_facts", "id")
var factTypeCheck int
err = conn.QueryRow(`
SELECT COUNT(*) FROM pg_constraint c
JOIN pg_class t ON t.oid = c.conrelid
WHERE t.relname = 'resource_facts'
AND pg_get_constraintdef(c.oid) LIKE '%doc_type%'
AND pg_get_constraintdef(c.oid) LIKE '%due_date%'`).Scan(&factTypeCheck)
if err != nil {
t.Fatalf("check resource_facts.fact_type: %v", err)
}
if factTypeCheck == 0 {
t.Error("CHECK fact_type manquant sur resource_facts après 000019")
}
if err := MigrateDownDatabase(url); err != nil {
t.Fatalf("migrate down: %v", err)
}
+408
View File
@@ -0,0 +1,408 @@
// Package extract derives generic, verifiable facts from OCR text (V3 S2).
//
// Rules-based French heuristics, zero ML : every fact carries a confidence,
// low-confidence facts are suggested (greyed in UI) and user-correctable (S3),
// and search demotes but never excludes on them. No LLM, no network, pure
// stdlib — it runs inside the OCR worker right after Tesseract.
package extract
import (
"fmt"
"regexp"
"strconv"
"strings"
"unicode"
)
// Fact types — mirror of the resource_facts CHECK (migration 000019).
type FactType string
const (
FactDate FactType = "date"
FactDueDate FactType = "due_date"
FactPeriod FactType = "period"
FactAmount FactType = "amount"
FactIssuer FactType = "issuer"
FactReference FactType = "reference"
FactDocType FactType = "doc_type"
)
// Fact is one extracted statement. ValueText holds issuer names, reference
// codes ("facture:FA-2026-0031"), doc types and periods ("2026-03") ;
// ValueNum holds amounts (HasNum) ; ValueDate holds ISO dates ("2026-03-12").
type Fact struct {
Type FactType
ValueText string
ValueNum float64
HasNum bool
ValueDate string
Confidence float64
}
// MaxFacts caps the output per resource (OCR garbage must stay bounded).
const MaxFacts = 50
// FromText extracts generic facts from raw OCR text. Deterministic output
// order (document order within each type, types in fixed order), duplicates
// removed.
func FromText(text string) []Fact {
e := &extractor{seen: map[string]bool{}}
e.extract(collectLines(text))
out := e.facts
if len(out) > MaxFacts {
out = out[:MaxFacts]
}
return out
}
type extractor struct {
facts []Fact
seen map[string]bool
}
func (e *extractor) add(f Fact) {
if f.Confidence < 0 {
f.Confidence = 0
}
if f.Confidence > 1 {
f.Confidence = 1
}
key := string(f.Type) + "\x00" + f.ValueText + "\x00" + f.ValueDate + "\x00" + fmt.Sprintf("%.2f", f.ValueNum)
if e.seen[key] {
return
}
e.seen[key] = true
e.facts = append(e.facts, f)
}
type line struct {
raw string
folded string
}
func collectLines(text string) []line {
raw := strings.Split(text, "\n")
out := make([]line, 0, len(raw))
for _, r := range raw {
r = strings.TrimSpace(strings.ReplaceAll(r, "\t", " "))
if r == "" {
continue
}
out = append(out, line{raw: r, folded: fold(r)})
}
return out
}
func (e *extractor) extract(lines []line) {
for _, l := range lines {
e.extractAmounts(l)
e.extractDates(l)
e.extractReferences(l)
}
e.extractIssuers(lines)
e.extractDocType(lines)
}
// --- amounts ---------------------------------------------------------------
// Note : pas de `\b` après `€` — c'est un symbole (non-mot), donc il n'y a
// jamais de frontière avec un espace ou une fin de ligne. `eur` garde son
// `\b` (évite "europe"), `euros?` n'en a pas besoin.
var amountPattern = regexp.MustCompile(`(\d[\d\s\x{a0}\x{202f}]*[,\.]\d{2}|\d[\d\s\x{a0}\x{202f}]*)\s*(€|euros?|eur\b)`)
func (e *extractor) extractAmounts(l line) {
// Percentages are not amounts.
if strings.Contains(l.raw, "%") && !strings.ContainsAny(l.raw, "€") {
return
}
for _, m := range amountPattern.FindAllStringSubmatch(l.raw, -1) {
num, ok := parseFrenchNumber(m[1])
if !ok || num <= 0 || num >= 1e13 {
continue
}
conf := 0.7
if amountKeywordPattern.MatchString(l.folded) {
conf = 0.9
}
e.add(Fact{Type: FactAmount, ValueNum: num, HasNum: true, ValueText: formatAmount(num), Confidence: conf})
}
}
var amountKeywordPattern = regexp.MustCompile(`\b(total|montant|a payer|net|ttc|solde|du)\b`)
func formatAmount(num float64) string {
return strconv.FormatFloat(num, 'f', 2, 64) + " €"
}
// parseFrenchNumber reads "1 234,56" / "1234.56" / "84 €"-style magnitudes.
// Comma = decimal separator (French), dots/spaces (incl. NBSP U+00A0 and
// narrow NBSP U+202F, common in "1 234 €") = thousands.
func parseFrenchNumber(s string) (float64, bool) {
s = strings.ReplaceAll(s, "\u00a0", "")
s = strings.ReplaceAll(s, "\u202f", "")
s = strings.ReplaceAll(s, " ", "")
if strings.Contains(s, ",") {
s = strings.ReplaceAll(s, ".", "")
s = strings.ReplaceAll(s, ",", ".")
}
num, err := strconv.ParseFloat(s, 64)
return num, err == nil
}
// --- dates -----------------------------------------------------------------
var numericDatePattern = regexp.MustCompile(`\b(\d{1,2})\s*[/\-.]\s*(\d{1,2})\s*[/\-.]\s*(\d{2}|\d{4})\b`)
var writtenDatePattern = regexp.MustCompile(`\b(\d{1,2})\s+(janv(?:ier)?|fev(?:rier)?|mars|avr(?:il)?|mai|juin|juil(?:let)?|aout|sept?(?:embre)?|oct(?:obre)?|nov(?:embre)?|dec(?:embre)?)\s+(\d{4})\b`)
var periodPattern = regexp.MustCompile(`\b(janv(?:ier)?|fev(?:rier)?|mars|avr(?:il)?|mai|juin|juil(?:let)?|aout|sept?(?:embre)?|oct(?:obre)?|nov(?:embre)?|dec(?:embre)?)\s+(\d{4})\b`)
var monthNumbers = map[string]int{
"janv": 1, "janvier": 1, "fev": 2, "fevrier": 2, "mars": 3,
"avr": 4, "avril": 4, "mai": 5, "juin": 6, "juil": 7,
"juillet": 7, "aout": 8, "sept": 9, "septembre": 9, "oct": 10,
"octobre": 10, "nov": 11, "novembre": 11, "dec": 12, "decembre": 12,
}
var dueKeywordPattern = regexp.MustCompile(`\b(echeance|a payer avant|date limite|avant le|regler avant|paiement avant)\b`)
func fullYear(yy int) int {
if yy < 100 {
if yy >= 70 {
return 1900 + yy
}
return 2000 + yy
}
return yy
}
func isoDate(y, m, d int) (string, bool) {
if m < 1 || m > 12 || d < 1 || d > 31 || y < 1900 || y > 2100 {
return "", false
}
return fmt.Sprintf("%04d-%02d-%02d", y, m, d), true
}
func (e *extractor) extractDates(l line) {
isDue := dueKeywordPattern.MatchString(l.folded)
for _, m := range numericDatePattern.FindAllStringSubmatch(l.raw, -1) {
d, _ := strconv.Atoi(m[1])
mo, _ := strconv.Atoi(m[2])
y, _ := strconv.Atoi(m[3])
iso, ok := isoDate(fullYear(y), mo, d)
if !ok {
continue
}
typ := FactDate
conf := 0.6
if isDue {
typ, conf = FactDueDate, 0.8
}
e.add(Fact{Type: typ, ValueDate: iso, ValueText: iso, Confidence: conf})
}
for _, m := range writtenDatePattern.FindAllStringSubmatch(l.folded, -1) {
d, _ := strconv.Atoi(m[1])
mo := monthNumbers[m[2]]
y, _ := strconv.Atoi(m[3])
iso, ok := isoDate(y, mo, d)
if !ok {
continue
}
typ := FactDate
conf := 0.7
if isDue {
typ, conf = FactDueDate, 0.85
}
e.add(Fact{Type: typ, ValueDate: iso, ValueText: iso, Confidence: conf})
}
for _, m := range periodPattern.FindAllStringSubmatch(l.folded, -1) {
mo := monthNumbers[m[1]]
y, _ := strconv.Atoi(m[2])
if mo == 0 || y < 1900 || y > 2100 {
continue
}
e.add(Fact{Type: FactPeriod, ValueText: fmt.Sprintf("%04d-%02d", y, mo), Confidence: 0.7})
}
}
// --- references ------------------------------------------------------------
// referencePattern catches "facture n° FA-2026-0031", "contrat : ABC123",
// "client 42" — the code must contain a digit (avoids "FACTURE DU MOIS").
var referencePattern = regexp.MustCompile(`(?i)\b(facture|contrat|client|compte|dossier|bulletin|commande|devis|attestation|police)\s*(n°|no|num(?:ero)?|number|ref(?:erence)?)?\s*[:\-#]?\s*([A-Z0-9][A-Z0-9\-/\.]{1,29})`)
var bareRefPattern = regexp.MustCompile(`\bn°\s*([A-Z0-9][A-Z0-9\-/]{2,20})`)
var siretPattern = regexp.MustCompile(`(?i)\b(siret|siren|rcs)\b[^\d]{0,10}(\d{3}\s?\d{3}\s?\d{3}(?:\s?\d{5})?)`)
var ibanPattern = regexp.MustCompile(`\bFR\s?\d{2}(?:\s?[0-9A-Z]{4}){5}\s?[0-9A-Z]{0,3}\b`)
func hasDigit(s string) bool {
for _, r := range s {
if r >= '0' && r <= '9' {
return true
}
}
return false
}
func (e *extractor) extractReferences(l line) {
for _, m := range referencePattern.FindAllStringSubmatch(l.raw, -1) {
code := strings.Trim(m[3], "-/.")
if !hasDigit(code) || len(code) < 3 {
continue
}
kind := strings.ToLower(m[1])
e.add(Fact{Type: FactReference, ValueText: kind + ":" + code, Confidence: 0.75})
}
for _, m := range bareRefPattern.FindAllStringSubmatch(l.raw, -1) {
code := m[1]
if !hasDigit(code) {
continue
}
e.add(Fact{Type: FactReference, ValueText: "ref:" + code, Confidence: 0.6})
}
if m := siretPattern.FindStringSubmatch(l.raw); m != nil {
digits := strings.Join(strings.Fields(m[2]), " ")
e.add(Fact{Type: FactReference, ValueText: strings.ToLower(m[1]) + ":" + digits, Confidence: 0.9})
}
if m := ibanPattern.FindString(l.raw); m != "" {
compact := strings.ReplaceAll(m, " ", "")
if len(compact) == 27 {
// Store masked (privacy) : only the last 4 chars.
e.add(Fact{Type: FactReference, ValueText: "rib:••••" + compact[len(compact)-4:], Confidence: 0.9})
}
}
}
// --- issuers ---------------------------------------------------------------
var companySuffixPattern = regexp.MustCompile(`(?i)\b(sarl|sas|sa|eurl|sasu|sci|eirl|snc|sasau)\b`)
func isCapsLine(s string) bool {
letters := 0
for _, r := range s {
if unicode.IsLetter(r) {
letters++
if unicode.IsLower(r) {
return false
}
}
}
return letters >= 3
}
func (e *extractor) extractIssuers(lines []line) {
found := 0
checked := 0
for _, l := range lines {
if checked >= 8 || found >= 2 {
break
}
raw := strings.TrimSpace(l.raw)
if len([]rune(raw)) < 3 || len([]rune(raw)) > 60 {
continue
}
if strings.Contains(raw, "@") || strings.HasPrefix(strings.ToLower(raw), "http") {
continue
}
checked++
if !isCapsLine(raw) {
continue
}
// Skip lines that are really dates/amounts/references in disguise.
if numericDatePattern.MatchString(raw) || amountPattern.MatchString(raw) || bareRefPattern.MatchString(raw) {
continue
}
conf := 0.45
if found == 0 {
conf = 0.55
}
if companySuffixPattern.MatchString(raw) {
conf = 0.75
}
e.add(Fact{Type: FactIssuer, ValueText: raw, Confidence: conf})
found++
}
}
// --- doc_type suggestion ---------------------------------------------------
// docTypeKeywords maps a suggested type to weighted French keywords (folded).
// A type is emitted when its score reaches docTypeThreshold — the type is a
// humble suggestion (S3-correctable), never a gate.
var docTypeKeywords = []struct {
typ string
keywords map[string]int
}{
{"facture", map[string]int{"facture": 2, "tva": 1, "montant ttc": 2, "total ttc": 2, "a payer": 1, "devis": 1}},
{"bulletin", map[string]int{"bulletin": 2, "salaire": 2, "paie": 1, "net social": 2, "brut": 1}},
{"impot", map[string]int{"impot": 2, "revenu fiscal": 2, "taxe fonciere": 2, "taxe d habitation": 1, "avis d impot": 2}},
{"contrat", map[string]int{"contrat": 2, "bail": 2, "convention": 1, "article ": 1, "assurance": 1}},
{"ordonnance", map[string]int{"ordonnance": 2, "posologie": 2, "docteur": 1, "traitement": 1}},
{"attestation", map[string]int{"attestation": 2, "certifie": 1, "certificat": 1, "atteste": 1}},
{"rib", map[string]int{"rib": 2, "iban": 2, "bic": 1}},
{"identite", map[string]int{"carte d identite": 2, "passeport": 2, "permis de conduire": 2, "nationalite": 1}},
}
const docTypeThreshold = 2
func (e *extractor) extractDocType(lines []line) {
joined := "\n"
for _, l := range lines {
joined += l.folded + "\n"
}
best, bestScore := "", 0
for _, dt := range docTypeKeywords {
score := 0
for kw, w := range dt.keywords {
if strings.Contains(joined, kw) {
score += w
}
}
if score > bestScore {
best, bestScore = dt.typ, score
}
}
if bestScore >= docTypeThreshold {
conf := 0.5 + 0.1*float64(bestScore)
if conf > 0.85 {
conf = 0.85
}
e.add(Fact{Type: FactDocType, ValueText: best, Confidence: conf})
}
}
// --- folding ---------------------------------------------------------------
// fold lowercases French text and strips accents (œ→oe, æ→ae, ç→c) for
// keyword matching. stdlib-only (no x/text dependency churn).
func fold(s string) string {
var b strings.Builder
b.Grow(len(s))
for _, r := range strings.ToLower(s) {
switch r {
case 'à', 'â', 'ä':
b.WriteRune('a')
case 'é', 'è', 'ê', 'ë':
b.WriteRune('e')
case 'î', 'ï':
b.WriteRune('i')
case 'ô', 'ö':
b.WriteRune('o')
case 'ù', 'û', 'ü':
b.WriteRune('u')
case 'ÿ':
b.WriteRune('y')
case 'ç':
b.WriteRune('c')
case 'œ':
b.WriteString("oe")
case 'æ':
b.WriteString("ae")
default:
b.WriteRune(r)
}
}
return b.String()
}
+183
View File
@@ -0,0 +1,183 @@
package extract
import (
"testing"
)
func factsByType(facts []Fact) map[FactType][]Fact {
out := map[FactType][]Fact{}
for _, f := range facts {
out[f.Type] = append(out[f.Type], f)
}
return out
}
func hasText(facts []Fact, want string) bool {
for _, f := range facts {
if f.ValueText == want {
return true
}
}
return false
}
const edfBill = `EDF - SERVICE CLIENTS
TSA 12345
Facture n° FA-2026-0031 du 12/03/2026
Période : mars 2026
Montant total TTC : 1 234,56 €
À payer avant le 12/04/2026
Référence client : 123456789
`
func TestFactureEDF(t *testing.T) {
by := factsByType(FromText(edfBill))
if !hasText(by[FactReference], "facture:FA-2026-0031") {
t.Errorf("référence facture manquante: %+v", by[FactReference])
}
if !hasText(by[FactDate], "2026-03-12") {
t.Errorf("date émission manquante: %+v", by[FactDate])
}
if !hasText(by[FactDueDate], "2026-04-12") {
t.Errorf("échéance manquante: %+v", by[FactDueDate])
}
if !hasText(by[FactPeriod], "2026-03") {
t.Errorf("période manquante: %+v", by[FactPeriod])
}
found := false
for _, f := range by[FactAmount] {
if f.HasNum && f.ValueNum == 1234.56 && f.Confidence >= 0.9 {
found = true
}
}
if !found {
t.Errorf("montant 1234.56 (conf 0.9) manquant: %+v", by[FactAmount])
}
if !hasText(by[FactIssuer], "EDF - SERVICE CLIENTS") {
t.Errorf("émetteur manquant: %+v", by[FactIssuer])
}
if !hasText(by[FactDocType], "facture") {
t.Errorf("doc_type facture manquant: %+v", by[FactDocType])
}
}
const bulletin = `BULLETIN DE SALAIRE
DUPONT Jean
Période : janvier 2026
Salaire brut : 2 845,00 €
Net social : 2 210,35 €
SIRET 123 456 789 00012
`
func TestBulletinSalaire(t *testing.T) {
by := factsByType(FromText(bulletin))
if !hasText(by[FactDocType], "bulletin") {
t.Errorf("doc_type bulletin manquant: %+v", by[FactDocType])
}
if !hasText(by[FactPeriod], "2026-01") {
t.Errorf("période manquante: %+v", by[FactPeriod])
}
if !hasText(by[FactReference], "siret:123 456 789 00012") {
t.Errorf("siret manquant: %+v", by[FactReference])
}
amounts := map[float64]bool{}
for _, f := range by[FactAmount] {
if f.HasNum {
amounts[f.ValueNum] = true
}
}
if !amounts[2845.00] || !amounts[2210.35] {
t.Errorf("montants bulletin manquants: %+v", by[FactAmount])
}
}
const ordonnance = `Dr MARTIN Sophie
Ordonnance du 3 mars 2026
Doliprane 1000mg — 1 matin et soir pendant 7 jours
`
func TestOrdonnance(t *testing.T) {
by := factsByType(FromText(ordonnance))
if !hasText(by[FactDocType], "ordonnance") {
t.Errorf("doc_type ordonnance manquant: %+v", by[FactDocType])
}
if !hasText(by[FactDate], "2026-03-03") {
t.Errorf("date manquante: %+v", by[FactDate])
}
if len(by[FactAmount]) != 0 {
t.Errorf("1000mg ne doit pas être un montant: %+v", by[FactAmount])
}
}
const rib = `RELEVÉ D'IDENTITÉ BANCAIRE
Titulaire : DUPONT Jean
IBAN : FR76 1234 5678 9012 3456 7890 123
BIC : AGRIFRPP
`
func TestRIBMasque(t *testing.T) {
by := factsByType(FromText(rib))
if !hasText(by[FactDocType], "rib") {
t.Errorf("doc_type rib manquant: %+v", by[FactDocType])
}
if !hasText(by[FactReference], "rib:••••0123") {
t.Errorf("IBAN masqué manquant: %+v", by[FactReference])
}
// L'IBAN complet ne doit jamais fuiter dans les faits.
for _, f := range by[FactReference] {
if len(f.ValueText) > 20 {
t.Errorf("fuite IBAN en clair: %q", f.ValueText)
}
}
}
func TestTexteVideEtBruit(t *testing.T) {
if got := FromText(""); len(got) != 0 {
t.Errorf("vide → %d faits, attendu 0", len(got))
}
// Bruit OCR sans structure : pas de doc_type, pas de montant.
by := factsByType(FromText("lorem ipsum\nbla bla 99\n///"))
if len(by[FactDocType]) != 0 {
t.Errorf("doc_type sur bruit: %+v", by[FactDocType])
}
if len(by[FactAmount]) != 0 {
t.Errorf("montant sur bruit: %+v", by[FactAmount])
}
}
func TestCourrierSimpleSansType(t *testing.T) {
// Tous les documents comptent : un courrier sans mots-clés métier donne
// des faits (date) mais aucun doc_type — jamais pénalisé.
by := factsByType(FromText("Bonjour,\nmerci de me recontacter.\nCordialement.\nParis, le 05/06/2026\n"))
if !hasText(by[FactDate], "2026-06-05") {
t.Errorf("date courrier manquante: %+v", by[FactDate])
}
if len(by[FactDocType]) != 0 {
t.Errorf("pas de doc_type attendu: %+v", by[FactDocType])
}
}
func TestAnneesCourtesEtFormats(t *testing.T) {
by := factsByType(FromText("Fait le 01.02.99\nÉchéance 15-08-26\n"))
if !hasText(by[FactDate], "1999-02-01") {
t.Errorf("année courte 99 → 1999: %+v", by[FactDate])
}
if !hasText(by[FactDueDate], "2026-08-15") {
t.Errorf("année courte 26 → 2026: %+v", by[FactDueDate])
}
}
func TestBornes(t *testing.T) {
// Dates invalides ignorées, pas de crash.
by := factsByType(FromText("le 45/13/2026 et le 00/00/0000"))
if len(by[FactDate])+len(by[FactDueDate]) != 0 {
t.Errorf("dates invalides extraites: %+v", by)
}
// Confiances toujours dans [0,1].
for _, f := range FromText(edfBill + bulletin + ordonnance + rib) {
if f.Confidence < 0 || f.Confidence > 1 {
t.Errorf("confiance hors bornes: %+v", f)
}
}
}
+83
View File
@@ -0,0 +1,83 @@
package handlers_test
import (
"encoding/json"
"net/http"
"testing"
"github.com/kazier/backend/repository"
"github.com/kazier/backend/service"
)
// TestFactsEndpoint : GET /files/:id/facts — faits auto visibles (viewer+),
// [] si rien d'extractible, 404 si trashé/absent/étranger.
func TestFactsEndpoint(t *testing.T) {
r, store, repo := setup(t)
device := repository.NewID()
token, userID := registerAndLogin(t, r, repo, testUserUsername(device, "fa"), "facts-test-password", device)
fileID := repository.NewID()
if err := repo.Resources.InsertFile(userID, fileID, "facture.pdf", "", 0, nil, nil); err != nil {
t.Fatalf("insert: %v", err)
}
if err := repo.Resources.UpdateOcrText(userID, fileID, "EDF\nFacture n° F-1 du 01/02/2026\nTotal : 10,00 €\n"); err != nil {
t.Fatalf("ocr text: %v", err)
}
extractSvc := service.NewExtract(repo)
if err := extractSvc.ProcessText(fileID, "EDF\nFacture n° F-1 du 01/02/2026\nTotal : 10,00 €\n"); err != nil {
t.Fatalf("extract: %v", err)
}
_ = store
rec, _ := doRequest(t, r, http.MethodGet, "/api/v1/files/"+fileID+"/facts", token, nil, "")
env := expectOK(t, rec, "facts-get")
var facts []struct {
FactType string `json:"factType"`
ValueText string `json:"valueText"`
ValueNum *float64 `json:"valueNum"`
Confidence float64 `json:"confidence"`
Origin string `json:"origin"`
}
if err := json.Unmarshal(env.Data, &facts); err != nil {
t.Fatalf("facts unmarshal: %v", err)
}
if len(facts) == 0 {
t.Fatal("faits attendus, got []")
}
types := map[string]bool{}
for _, f := range facts {
types[f.FactType] = true
if f.Origin != "auto" {
t.Errorf("origine = %q, attendu auto", f.Origin)
}
}
for _, want := range []string{"reference", "amount", "date", "doc_type"} {
if !types[want] {
t.Errorf("fait %q manquant: %+v", want, facts)
}
}
// Fichier visible sans faits → [] (200, jamais 404).
emptyID := repository.NewID()
if err := repo.Resources.InsertFile(userID, emptyID, "vide.txt", "", 0, nil, nil); err != nil {
t.Fatalf("insert empty: %v", err)
}
rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/files/"+emptyID+"/facts", token, nil, "")
env = expectOK(t, rec, "facts-empty")
if string(env.Data) != "[]" {
t.Fatalf("sans faits : attendu [], got %s", env.Data)
}
// Trashé → 404 (comme l'OCR).
if _, err := store.Trash(userID, fileID); err != nil {
t.Fatalf("trash: %v", err)
}
rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/files/"+fileID+"/facts", token, nil, "")
expectError(t, rec, http.StatusNotFound, "NOT_FOUND", "facts-trashed")
// Étranger → 404.
deviceB := repository.NewID()
tokenB, _ := registerAndLogin(t, r, repo, testUserUsername(deviceB, "fb"), "facts-test-password", deviceB)
rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/files/"+emptyID+"/facts", tokenB, nil, "")
expectError(t, rec, http.StatusNotFound, "NOT_FOUND", "facts-foreign")
}
+127 -2
View File
@@ -107,7 +107,32 @@ func FilesGetOcr(c *gin.Context) {
api.OK(c, dto)
}
// FilesContent streame les octets physiques d'un fichier visible (viewer+,
// FilesGetFacts expose les faits génériques d'un fichier visible (V3 S2) :
// GET /files/:id/facts. Même portée que l'OCR (viewer+, own ou shared) —
// les faits dérivent du texte. `[]` si rien d'extractible (jamais 404 pour
// un fichier visible sans faits) ; 404 si trashé/absent/étranger.
func FilesGetFacts(c *gin.Context) {
if Store == nil {
api.Error(c, http.StatusServiceUnavailable, "SERVICE_UNAVAILABLE", "backend not initialized")
return
}
userID := c.GetString(UserIDKey)
id := c.Param("id")
if !deviceIDPattern.MatchString(id) {
api.Error(c, http.StatusNotFound, "NOT_FOUND", "file not found")
return
}
facts, err := Store.GetFacts(userID, id)
if err != nil {
writeError(c, err)
return
}
if facts == nil {
facts = []service.FactDTO{}
}
api.OK(c, facts)
}
// own ou shared) : GET /files/:id/content. Réponse binaire (Content-Type du
// fichier, Content-Disposition inline) — jamais d'enveloppe {data}. Supporte
// Range via http.ServeContent (Gin c.File). Sans bytes serveur (ressource
@@ -145,7 +170,19 @@ func FilesDelete(c *gin.Context) {
api.Error(c, http.StatusNotFound, "NOT_FOUND", "file not found")
return
}
deletedID, err := Store.DeleteFile(userID, id)
// V3 corbeille : DELETE = trash par défaut ; ?mode=purge = destruction
// définitive (lignes + versions + octets). Mode inconnu → 400.
var deletedID string
var err error
switch c.Query("mode") {
case "", "trash":
deletedID, err = Store.Trash(userID, id)
case "purge":
deletedID, err = Store.Purge(userID, id)
default:
api.Error(c, http.StatusBadRequest, "INVALID_REQUEST", "invalid query param `mode` (trash|purge)")
return
}
if err != nil {
writeError(c, err)
return
@@ -153,6 +190,94 @@ func FilesDelete(c *gin.Context) {
api.OK(c, gin.H{"id": deletedID})
}
// FilesRestore restaure une ressource trashée (POST /files/:id/restore).
// Absente/étrangère → 404 NOT_FOUND ; déjà vivante → succès idempotent.
func FilesRestore(c *gin.Context) {
if Store == nil {
api.Error(c, http.StatusServiceUnavailable, "SERVICE_UNAVAILABLE", "backend not initialized")
return
}
userID := c.GetString(UserIDKey)
id := c.Param("id")
if !deviceIDPattern.MatchString(id) {
api.Error(c, http.StatusNotFound, "NOT_FOUND", "file not found")
return
}
restoredID, err := Store.Restore(userID, id)
if err != nil {
writeError(c, err)
return
}
api.OK(c, gin.H{"id": restoredID})
}
// TrashList expose la corbeille (GET /trash) : ressources trashées du user,
// les plus récentes d'abord, paginé comme les autres listes.
func TrashList(c *gin.Context) {
if Store == nil {
api.Error(c, http.StatusServiceUnavailable, "SERVICE_UNAVAILABLE", "backend not initialized")
return
}
userID := c.GetString(UserIDKey)
page, pageSize, ok := parsePageParams(c)
if !ok {
return
}
items, total, err := Store.ListTrashed(userID, page, pageSize)
if err != nil {
writeError(c, err)
return
}
api.OKList(c, items, page, pageSize, total)
}
// VersionsList expose l'historique des octets (GET /files/:id/versions),
// le plus récent d'abord. Ressource trashée/absente/étrangère → 404.
func VersionsList(c *gin.Context) {
if Store == nil {
api.Error(c, http.StatusServiceUnavailable, "SERVICE_UNAVAILABLE", "backend not initialized")
return
}
userID := c.GetString(UserIDKey)
id := c.Param("id")
if !deviceIDPattern.MatchString(id) {
api.Error(c, http.StatusNotFound, "NOT_FOUND", "file not found")
return
}
versions, err := Store.ListVersions(userID, id)
if err != nil {
writeError(c, err)
return
}
api.OK(c, versions)
}
// VersionRestore recopie la génération n vers le live
// (POST /files/:id/versions/:n/restore). Génération inconnue → 404.
func VersionRestore(c *gin.Context) {
if Store == nil {
api.Error(c, http.StatusServiceUnavailable, "SERVICE_UNAVAILABLE", "backend not initialized")
return
}
userID := c.GetString(UserIDKey)
id := c.Param("id")
if !deviceIDPattern.MatchString(id) {
api.Error(c, http.StatusNotFound, "NOT_FOUND", "file not found")
return
}
n, err := strconv.Atoi(c.Param("n"))
if err != nil || n < 1 {
api.Error(c, http.StatusNotFound, "NOT_FOUND", "version not found")
return
}
dto, err := Store.RestoreVersion(userID, id, n)
if err != nil {
writeError(c, err)
return
}
api.OK(c, dto)
}
func FilesUpload(c *gin.Context) {
if Store == nil {
api.Error(c, http.StatusServiceUnavailable, "SERVICE_UNAVAILABLE", "backend not initialized")
+5
View File
@@ -33,10 +33,15 @@ func RegisterRoutes(r *gin.Engine) {
protected.GET("/files/folders", FoldersList)
protected.GET("/files/:id", FilesGet)
protected.GET("/files/:id/ocr", FilesGetOcr)
protected.GET("/files/:id/facts", FilesGetFacts)
protected.GET("/files/:id/content", FilesContent)
protected.GET("/files/:id/shares", SharesList)
protected.GET("/files/:id/links", LinksList)
protected.GET("/files/:id/placements", PlacementsList)
protected.GET("/files/:id/versions", VersionsList)
protected.POST("/files/:id/restore", FilesRestore)
protected.POST("/files/:id/versions/:n/restore", VersionRestore)
protected.GET("/trash", TrashList)
protected.DELETE("/files/:id", FilesDelete)
protected.POST("/files/upload", FilesUpload)
+160
View File
@@ -0,0 +1,160 @@
package handlers_test
import (
"encoding/json"
"net/http"
"testing"
"github.com/kazier/backend/repository"
)
// TestTrashEndpoints couvre le cycle corbeille via HTTP : DELETE (trash) →
// 404 en lecture → GET /trash → POST /files/:id/restore → visible →
// DELETE ?mode=purge → corbeille vide, restore → 404.
func TestTrashEndpoints(t *testing.T) {
r, _, repo := setup(t)
device := repository.NewID()
token, userID := registerAndLogin(t, r, repo, testUserUsername(device, "tr"), "trash-test-password", device)
fileID := repository.NewID()
if err := repo.Resources.InsertFile(userID, fileID, "acte.pdf", "", 0, nil, nil); err != nil {
t.Fatalf("insert: %v", err)
}
rec, _ := doRequest(t, r, http.MethodDelete, "/api/v1/files/"+fileID, token, nil, "")
expectOK(t, rec, "trash-delete")
rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/files/"+fileID, token, nil, "")
expectError(t, rec, http.StatusNotFound, "NOT_FOUND", "trash-get-hidden")
rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/trash", token, nil, "")
env := expectOK(t, rec, "trash-list")
var items []struct {
ID string `json:"id"`
Type string `json:"type"`
TrashedAt string `json:"trashedAt"`
}
if err := json.Unmarshal(env.Data, &items); err != nil {
t.Fatalf("trash list unmarshal: %v", err)
}
if len(items) != 1 || items[0].ID != fileID || items[0].TrashedAt == "" {
t.Fatalf("corbeille inattendue: %+v", items)
}
if env.Meta == nil || env.Meta.Total != 1 {
t.Fatalf("meta corbeille: %+v", env.Meta)
}
rec, _ = doRequest(t, r, http.MethodPost, "/api/v1/files/"+fileID+"/restore", token, nil, "")
expectOK(t, rec, "trash-restore")
rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/files/"+fileID, token, nil, "")
expectOK(t, rec, "trash-get-restored")
rec, _ = doRequest(t, r, http.MethodDelete, "/api/v1/files/"+fileID+"?mode=purge", token, nil, "")
expectOK(t, rec, "trash-purge")
rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/trash", token, nil, "")
env = expectOK(t, rec, "trash-list-empty")
if env.Meta == nil || env.Meta.Total != 0 {
t.Fatalf("corbeille doit être vide: %+v", env.Meta)
}
rec, _ = doRequest(t, r, http.MethodPost, "/api/v1/files/"+fileID+"/restore", token, nil, "")
expectError(t, rec, http.StatusNotFound, "NOT_FOUND", "trash-restore-purged")
}
// TestTrashInvalidMode : ?mode inconnu → 400, la ressource reste vivante.
func TestTrashInvalidMode(t *testing.T) {
r, _, repo := setup(t)
device := repository.NewID()
token, userID := registerAndLogin(t, r, repo, testUserUsername(device, "tm"), "trash-test-password", device)
fileID := repository.NewID()
if err := repo.Resources.InsertFile(userID, fileID, "doc.pdf", "", 0, nil, nil); err != nil {
t.Fatalf("insert: %v", err)
}
rec, _ := doRequest(t, r, http.MethodDelete, "/api/v1/files/"+fileID+"?mode=bogus", token, nil, "")
expectError(t, rec, http.StatusBadRequest, "INVALID_REQUEST", "trash-bad-mode")
rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/files/"+fileID, token, nil, "")
expectOK(t, rec, "trash-bad-mode-keeps-live")
}
// TestTrashIsolation : un user ne voit ni ne touche la corbeille d'autrui.
func TestTrashIsolation(t *testing.T) {
r, _, repo := setup(t)
deviceA := repository.NewID()
deviceB := repository.NewID()
tokenA, userA := registerAndLogin(t, r, repo, testUserUsername(deviceA, "ta"), "trash-test-password", deviceA)
tokenB, _ := registerAndLogin(t, r, repo, testUserUsername(deviceB, "tb"), "trash-test-password", deviceB)
fileID := repository.NewID()
if err := repo.Resources.InsertFile(userA, fileID, "prive.pdf", "", 0, nil, nil); err != nil {
t.Fatalf("insert: %v", err)
}
rec, _ := doRequest(t, r, http.MethodDelete, "/api/v1/files/"+fileID, tokenB, nil, "")
expectError(t, rec, http.StatusNotFound, "NOT_FOUND", "trash-foreign")
rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/trash", tokenB, nil, "")
env := expectOK(t, rec, "trash-foreign-list")
if env.Meta == nil || env.Meta.Total != 0 {
t.Fatalf("corbeille d'autrui doit être vide: %+v", env.Meta)
}
rec, _ = doRequest(t, r, http.MethodDelete, "/api/v1/files/"+fileID, tokenA, nil, "")
expectOK(t, rec, "trash-owner")
}
// TestVersionsEndpoints : upload miniature (MaxFileSize=10 en test) puis
// re-upload ciblé → GET /files/:id/versions (1 entrée) → restore → contenu
// d'origine. Trashée → versions en 404.
func TestVersionsEndpoints(t *testing.T) {
r, _, repo := setup(t)
device := repository.NewID()
token, _ := registerAndLogin(t, r, repo, testUserUsername(device, "ve"), "trash-test-password", device)
rec := uploadMultipart(t, r, token, "", "doc.txt", []byte("AAA"))
env := expectOK(t, rec, "versions-upload")
var created fileDTO
if err := json.Unmarshal(env.Data, &created); err != nil {
t.Fatalf("upload unmarshal: %v", err)
}
rec = uploadMultipartToResource(t, r, token, created.ID, "doc.txt", []byte("BBBB"))
expectOK(t, rec, "versions-overwrite")
rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/files/"+created.ID+"/versions", token, nil, "")
env = expectOK(t, rec, "versions-list")
var vers []struct {
VersionNo int `json:"versionNo"`
Size int64 `json:"size"`
CreatedAt string `json:"createdAt"`
}
if err := json.Unmarshal(env.Data, &vers); err != nil {
t.Fatalf("versions unmarshal: %v", err)
}
if len(vers) != 1 || vers[0].Size != 3 {
t.Fatalf("versions inattendues: %+v", vers)
}
rec, _ = doRequest(t, r, http.MethodPost, "/api/v1/files/"+created.ID+"/versions/1/restore", token, nil, "")
env = expectOK(t, rec, "versions-restore")
var restored fileDTO
if err := json.Unmarshal(env.Data, &restored); err != nil {
t.Fatalf("restore unmarshal: %v", err)
}
if restored.Size != 3 {
t.Fatalf("live restauré taille=%d, attendu 3", restored.Size)
}
rec, _ = doRequest(t, r, http.MethodPost, "/api/v1/files/"+created.ID+"/versions/99/restore", token, nil, "")
expectError(t, rec, http.StatusNotFound, "NOT_FOUND", "versions-restore-unknown")
rec, _ = doRequest(t, r, http.MethodDelete, "/api/v1/files/"+created.ID, token, nil, "")
expectOK(t, rec, "versions-trash")
rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/files/"+created.ID+"/versions", token, nil, "")
expectError(t, rec, http.StatusNotFound, "NOT_FOUND", "versions-trashed-hidden")
}
+153
View File
@@ -0,0 +1,153 @@
package repository
import (
"database/sql"
)
// FactParam is one generic fact to persist (V3 S2, migration 000019).
// ValueNum/ValueDate nil when the type carries no numeric/date value.
type FactParam struct {
Type string
ValueText string
ValueNum *float64
ValueDate *string // ISO YYYY-MM-DD
Confidence float64
}
// FactRow is one persisted fact (auto-extracted or user-corrected).
type FactRow struct {
ID string
Type string
ValueText string
ValueNum *float64
ValueDate *string
Confidence float64
Origin string
}
// UpsertAutoFacts replaces the origin='auto' facts of a resource with a fresh
// extraction. Rows with origin='user' (S3 corrections) are never touched.
// An empty params clears stale auto facts (e.g. OCR re-run yielding nothing).
func (r *Resources) UpsertAutoFacts(resourceID string, params []FactParam) error {
tx, err := r.DB.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck — no-op after Commit.
if _, err := tx.Exec(
`DELETE FROM resource_facts WHERE resource_id = $1 AND origin = 'auto'`,
resourceID,
); err != nil {
return err
}
for _, p := range params {
id, err := NewIDErr()
if err != nil {
return err
}
if _, err := tx.Exec(
`INSERT INTO resource_facts
(id, resource_id, fact_type, value_text, value_num, value_date, confidence, origin)
VALUES ($1, $2, $3, NULLIF($4, ''), $5, NULLIF($6, '')::date, $7, 'auto')`,
id, resourceID, p.Type, p.ValueText, nullableFloat(p.ValueNum), derefStr(p.ValueDate), clamp01(p.Confidence),
); err != nil {
return err
}
}
return tx.Commit()
}
// ListFacts returns the facts of a resource, user-corrected first then auto
// (stable order by type). No visibility check — the caller enforces viewer+
// (GetFileVisible) like GetFileOcr.
func (r *Resources) ListFacts(resourceID string) ([]FactRow, error) {
rows, err := r.DB.Query(
`SELECT id, fact_type, COALESCE(value_text, ''), value_num,
COALESCE(value_date::text, ''), confidence, origin
FROM resource_facts WHERE resource_id = $1
ORDER BY CASE origin WHEN 'user' THEN 0 ELSE 1 END, fact_type, value_text`,
resourceID,
)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]FactRow, 0)
for rows.Next() {
var row FactRow
var num sql.NullFloat64
var dat string
if err := rows.Scan(&row.ID, &row.Type, &row.ValueText, &num, &dat, &row.Confidence, &row.Origin); err != nil {
return nil, err
}
if num.Valid {
v := num.Float64
row.ValueNum = &v
}
if dat != "" {
d := dat
row.ValueDate = &d
}
out = append(out, row)
}
return out, rows.Err()
}
// factNeedingExtract is a file with OCR text but no auto facts yet.
type factNeedingExtract struct {
ResourceID string
Text string
}
// ListNeedingExtract returns up to limit files with non-empty ocr_text and no
// origin='auto' facts (boot backfill, S2). Live rows only.
func (r *Resources) ListNeedingExtract(limit int) ([]factNeedingExtract, error) {
rows, err := r.DB.Query(
`SELECT resource_id, ocr_text FROM resources
WHERE type = 'file' AND deleted_at IS NULL
AND ocr_text IS NOT NULL AND ocr_text <> ''
AND NOT EXISTS (
SELECT 1 FROM resource_facts
WHERE resource_facts.resource_id = resources.resource_id
AND origin = 'auto')
ORDER BY updated_at ASC LIMIT $1`,
limit,
)
if err != nil {
return nil, err
}
defer rows.Close()
var out []factNeedingExtract
for rows.Next() {
var row factNeedingExtract
if err := rows.Scan(&row.ResourceID, &row.Text); err != nil {
return nil, err
}
out = append(out, row)
}
return out, rows.Err()
}
func nullableFloat(v *float64) any {
if v == nil {
return nil
}
return *v
}
func derefStr(v *string) string {
if v == nil {
return ""
}
return *v
}
func clamp01(v float64) float64 {
if v < 0 {
return 0
}
if v > 1 {
return 1
}
return v
}
+5 -6
View File
@@ -569,13 +569,12 @@ func (r *Resources) UpdateOcrText(ownerID, resourceID, text string) error {
// SyncDelete soft-deletes a resource; absence is NOT an error (idempotent
// terminal state for the outbox).
//
// Deprecated: V3 trash — préférer TrashResource (récursif sur le subtree).
// Conservé comme alias pour compatibilité (même effet sur une ressource
// sans enfant : TrashResource ne touche que deleted_at IS NULL).
func (r *Resources) SyncDelete(ownerID, resourceID string) error {
_, err := r.DB.Exec(
`UPDATE resources SET deleted_at = NOW(), updated_at = NOW()
WHERE resource_id = $1 AND user_id = $2 AND deleted_at IS NULL`,
resourceID, ownerID,
)
return err
return r.TrashResource(ownerID, resourceID)
}
// ExistsOwner reports whether a non-deleted resource belongs to the device.
+191
View File
@@ -0,0 +1,191 @@
package repository
import (
"database/sql"
"time"
)
// TrashedRow is a trashed resource for GET /trash.
type TrashedRow struct {
ID string
Type string
Name string
Size int64
TrashedAt time.Time
UpdatedAt time.Time
}
// subtreeCTE selects a resource and all its descendants (any depth).
const subtreeCTE = `
subtree(id) AS (
SELECT $2::text
UNION ALL
SELECT r.resource_id FROM resources r JOIN subtree s ON r.parent_id = s.id
)`
// ancestorsCTE selects a resource and all its ancestors up to the root.
const ancestorsCTE = `
ancestors(id) AS (
SELECT $2::text
UNION ALL
SELECT r.parent_id FROM resources r JOIN ancestors a ON r.resource_id = a.id
WHERE r.parent_id IS NOT NULL
)`
// TrashResource soft-deletes a resource and its whole subtree
// (`deleted_at = NOW()`), owner-scoped. Already-trashed rows are untouched.
// The target must be verified owned beforehand (ExistsOwner / GetResourceOwner)
// for the NOT_FOUND halt semantics — this method only applies.
func (r *Resources) TrashResource(ownerID, resourceID string) error {
_, err := r.DB.Exec(
`WITH RECURSIVE `+subtreeCTE+`
UPDATE resources SET deleted_at = NOW(), updated_at = NOW()
WHERE resource_id IN (SELECT id FROM subtree)
AND user_id = $1 AND deleted_at IS NULL`,
ownerID, resourceID,
)
return err
}
// RestoreResource clears the trash flag on a resource, its subtree AND its
// ancestor chain (a file whose folder is still trashed would stay
// unreachable otherwise). Independently-trashed descendants are restored too
// — documented V3 behaviour (restoring the folder restores what it contains).
func (r *Resources) RestoreResource(ownerID, resourceID string) error {
_, err := r.DB.Exec(
`WITH RECURSIVE `+subtreeCTE+`, `+ancestorsCTE+`
UPDATE resources SET deleted_at = NULL, updated_at = NOW()
WHERE (resource_id IN (SELECT id FROM subtree)
OR resource_id IN (SELECT id FROM ancestors))
AND user_id = $1 AND deleted_at IS NOT NULL`,
ownerID, resourceID,
)
return err
}
// ExistsOwnerAny reports ownership regardless of trash state (for purge :
// a trashed resource is still owned).
func (r *Resources) ExistsOwnerAny(ownerID, resourceID string) (bool, error) {
var exists int
err := r.DB.QueryRow(
`SELECT 1 FROM resources WHERE resource_id = $1 AND user_id = $2`,
resourceID, ownerID,
).Scan(&exists)
if err == sql.ErrNoRows {
return false, nil
}
return err == nil, err
}
// PurgeResource hard-deletes a resource and its subtree (FK CASCADE on
// parent_id + resource_versions), returning every purged resource_id so the
// caller can remove physical files best-effort. Owner-scoped.
func (r *Resources) PurgeResource(ownerID, resourceID string) ([]string, error) {
rows, err := r.DB.Query(
`WITH RECURSIVE subtree(id) AS (
SELECT $1::text
UNION ALL
SELECT r.resource_id FROM resources r JOIN subtree s ON r.parent_id = s.id
)
SELECT id FROM subtree`,
resourceID,
)
if err != nil {
return nil, err
}
var ids []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
rows.Close()
return nil, err
}
ids = append(ids, id)
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
if _, err := r.DB.Exec(
`DELETE FROM resources WHERE resource_id = $1 AND user_id = $2`,
resourceID, ownerID,
); err != nil {
return nil, err
}
return ids, nil
}
// ListTrashed returns the owner's trashed resources, most recently trashed
// first, paginated.
func (r *Resources) ListTrashed(ownerID string, limit, offset int) ([]TrashedRow, int, error) {
var total int
if err := r.DB.QueryRow(
`SELECT COUNT(*) FROM resources WHERE user_id = $1 AND deleted_at IS NOT NULL`,
ownerID,
).Scan(&total); err != nil {
return nil, 0, err
}
rows, err := r.DB.Query(
`SELECT resource_id, type, name, COALESCE(size_bytes, 0), deleted_at, updated_at
FROM resources WHERE user_id = $1 AND deleted_at IS NOT NULL
ORDER BY deleted_at DESC LIMIT $2 OFFSET $3`,
ownerID, limit, offset,
)
if err != nil {
return nil, 0, err
}
defer rows.Close()
out := make([]TrashedRow, 0)
for rows.Next() {
var row TrashedRow
if err := rows.Scan(&row.ID, &row.Type, &row.Name, &row.Size, &row.TrashedAt, &row.UpdatedAt); err != nil {
return nil, 0, err
}
out = append(out, row)
}
return out, total, rows.Err()
}
// PurgedResource identifies a row destroyed by the retention pass.
type PurgedResource struct {
UserID string
ResourceID string
}
// PurgeExpiredTrash hard-deletes every resource trashed before cutoff
// (corbeille 30j), oldest first, bounded by limit. Returns the destroyed
// identities for physical-file cleanup (best-effort, caller-side).
func (r *Resources) PurgeExpiredTrash(cutoff time.Time, limit int) ([]PurgedResource, error) {
rows, err := r.DB.Query(
`SELECT user_id, resource_id FROM resources
WHERE deleted_at IS NOT NULL AND deleted_at < $1
ORDER BY deleted_at ASC LIMIT $2`,
cutoff, limit,
)
if err != nil {
return nil, err
}
var out []PurgedResource
for rows.Next() {
var p PurgedResource
if err := rows.Scan(&p.UserID, &p.ResourceID); err != nil {
rows.Close()
return nil, err
}
out = append(out, p)
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
for _, p := range out {
// Hard delete cascades to children (parent_id FK) and versions.
if _, err := r.DB.Exec(
`DELETE FROM resources WHERE resource_id = $1 AND user_id = $2 AND deleted_at IS NOT NULL AND deleted_at < $3`,
p.ResourceID, p.UserID, cutoff,
); err != nil {
return out, err
}
}
return out, nil
}
+131
View File
@@ -0,0 +1,131 @@
package repository
import (
"database/sql"
"time"
"github.com/lib/pq"
)
// VersionRow is one frozen byte-generation of a resource.
type VersionRow struct {
ID string
VersionNo int
Size int64
MimeType string
Extension string
StoredPath string
CreatedAt time.Time
}
// MaxVersions caps the per-resource history (V3 P1 : N=3).
const MaxVersions = 3
// NextVersionNo returns 1 + the highest version number (monotonic, never
// reused even after pruning).
func (r *Resources) NextVersionNo(resourceID string) (int, error) {
var next int
err := r.DB.QueryRow(
`SELECT COALESCE(MAX(version_no), 0) + 1 FROM resource_versions WHERE resource_id = $1`,
resourceID,
).Scan(&next)
return next, err
}
// InsertVersion freezes one byte-generation. ID must be NewID() 32-hex.
func (r *Resources) InsertVersion(id, resourceID string, versionNo int, size int64, mimeType, extension *string, storedPath string) error {
_, err := r.DB.Exec(
`INSERT INTO resource_versions (id, resource_id, version_no, size_bytes, mime_type, extension, stored_path)
VALUES ($1, $2, $3, $4, $5, $6, $7)`,
id, resourceID, versionNo, size, mimeType, extension, storedPath,
)
return err
}
// ListVersions returns the history of a resource owned by ownerID
// (ownership checked via the parent row), newest first.
func (r *Resources) ListVersions(ownerID, resourceID string) ([]VersionRow, error) {
if _, err := r.GetFile(ownerID, resourceID); err != nil {
// Folders can be versioned too (metadata only has no bytes — in
// practice only files carry versions, but don't 404 on type).
if _, ferr := r.GetFolder(ownerID, resourceID); ferr != nil {
return nil, err
}
}
rows, err := r.DB.Query(
`SELECT v.id, v.version_no, v.size_bytes, COALESCE(v.mime_type, ''),
COALESCE(v.extension, ''), v.stored_path, v.created_at
FROM resource_versions v
JOIN resources r ON r.resource_id = v.resource_id
WHERE v.resource_id = $1 AND r.user_id = $2 AND r.deleted_at IS NULL
ORDER BY v.version_no DESC`,
resourceID, ownerID,
)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]VersionRow, 0)
for rows.Next() {
var row VersionRow
if err := rows.Scan(&row.ID, &row.VersionNo, &row.Size, &row.MimeType, &row.Extension, &row.StoredPath, &row.CreatedAt); err != nil {
return nil, err
}
out = append(out, row)
}
return out, rows.Err()
}
// GetVersion returns one frozen generation (owner-scoped via parent row).
func (r *Resources) GetVersion(ownerID, resourceID string, versionNo int) (VersionRow, error) {
var row VersionRow
err := r.DB.QueryRow(
`SELECT v.id, v.version_no, v.size_bytes, COALESCE(v.mime_type, ''),
COALESCE(v.extension, ''), v.stored_path, v.created_at
FROM resource_versions v
JOIN resources r ON r.resource_id = v.resource_id
WHERE v.resource_id = $1 AND v.version_no = $2 AND r.user_id = $3 AND r.deleted_at IS NULL`,
resourceID, versionNo, ownerID,
).Scan(&row.ID, &row.VersionNo, &row.Size, &row.MimeType, &row.Extension, &row.StoredPath, &row.CreatedAt)
if err == sql.ErrNoRows {
return VersionRow{}, ErrNotFound
}
return row, err
}
// PruneVersions deletes generations beyond the newest keepN, returning their
// stored paths for physical cleanup (best-effort, caller-side).
func (r *Resources) PruneVersions(resourceID string, keepN int) ([]string, error) {
rows, err := r.DB.Query(
`SELECT stored_path FROM resource_versions
WHERE resource_id = $1
ORDER BY version_no DESC OFFSET $2`,
resourceID, keepN,
)
if err != nil {
return nil, err
}
var paths []string
for rows.Next() {
var p string
if err := rows.Scan(&p); err != nil {
rows.Close()
return nil, err
}
paths = append(paths, p)
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
if len(paths) == 0 {
return nil, nil
}
if _, err := r.DB.Exec(
`DELETE FROM resource_versions WHERE resource_id = $1 AND stored_path = ANY($2)`,
resourceID, pq.Array(paths),
); err != nil {
return nil, err
}
return paths, nil
}
+108
View File
@@ -0,0 +1,108 @@
package service
import (
"log"
"github.com/kazier/backend/extract"
"github.com/kazier/backend/repository"
)
// Extract derives generic facts from OCR text (V3 S2) — dates, amounts,
// issuers, references, suggested doc type. Rules-based (package extract),
// no ML, no network : it runs inside the OCR worker right after Tesseract
// and stores origin='auto' rows (user corrections, origin='user', survive
// every re-run — see UpsertAutoFacts).
type Extract struct {
Repository *repository.Repository
}
func NewExtract(repo *repository.Repository) *Extract {
return &Extract{Repository: repo}
}
// FactDTO serializes one row of GET /files/:id/facts.
type FactDTO struct {
FactType string `json:"factType"`
ValueText string `json:"valueText,omitempty"`
ValueNum *float64 `json:"valueNum,omitempty"`
ValueDate string `json:"valueDate,omitempty"`
Confidence float64 `json:"confidence"`
Origin string `json:"origin"`
}
// ProcessText extracts facts from text and replaces the resource's auto
// facts. Empty text clears stale auto facts. Never touches user corrections.
func (e *Extract) ProcessText(resourceID, text string) error {
raw := extract.FromText(text)
params := make([]repository.FactParam, 0, len(raw))
for _, f := range raw {
var num *float64
if f.HasNum {
n := f.ValueNum
num = &n
}
var dat *string
if f.ValueDate != "" {
d := f.ValueDate
dat = &d
}
text := f.ValueText
// Amounts carry a display string ("1234.56 €") — keep the numeric
// column as the machine value, the text as the human one.
params = append(params, repository.FactParam{
Type: string(f.Type),
ValueText: text,
ValueNum: num,
ValueDate: dat,
Confidence: f.Confidence,
})
}
return e.Repository.Resources.UpsertAutoFacts(resourceID, params)
}
// Backfill processes files with OCR text but no auto facts yet (one boot
// pass, bounded batches). Returns the number of resources processed.
func (e *Extract) Backfill(batch int) (int, error) {
total := 0
for {
pending, err := e.Repository.Resources.ListNeedingExtract(batch)
if err != nil {
return total, err
}
if len(pending) == 0 {
return total, nil
}
for _, p := range pending {
if err := e.ProcessText(p.ResourceID, p.Text); err != nil {
log.Printf("extract: backfill %s: %v", p.ResourceID, err)
continue
}
total++
}
}
}
// GetFacts returns the facts of a visible resource (viewer+, own or shared —
// same scope as GetFileOcr, facts derive from the OCR text). Trashed/absent/
// foreign → repository.ErrNotFound.
func (s *Resources) GetFacts(userID, resourceID string) ([]FactDTO, error) {
if _, err := s.Repo.GetFileVisible(userID, resourceID); err != nil {
return nil, err
}
rows, err := s.Repository.Resources.ListFacts(resourceID)
if err != nil {
return nil, err
}
out := make([]FactDTO, 0, len(rows))
for _, row := range rows {
out = append(out, FactDTO{
FactType: row.Type,
ValueText: row.ValueText,
ValueNum: row.ValueNum,
ValueDate: derefString(row.ValueDate),
Confidence: row.Confidence,
Origin: row.Origin,
})
}
return out, nil
}
+196
View File
@@ -0,0 +1,196 @@
package service
import (
"testing"
"github.com/kazier/backend/repository"
)
func newExtractStore(t *testing.T) (*Resources, *Extract) {
t.Helper()
s := newServiceStore(t)
return s, NewExtract(s.Repository)
}
const extractCorpus = `EDF - SERVICE CLIENTS
Facture n° FA-2026-0031 du 12/03/2026
Montant total TTC : 84,20 €
À payer avant le 12/04/2026
`
func TestProcessTextPersistsAutoFacts(t *testing.T) {
s, ex := newExtractStore(t)
userID := mustCreateUser(t, s.Repository, "extract-auto")
fileID := repository.NewID()
if err := s.Repository.Resources.InsertFile(userID, fileID, "facture.pdf", "", 0, nil, nil); err != nil {
t.Fatalf("insert: %v", err)
}
if err := ex.ProcessText(fileID, extractCorpus); err != nil {
t.Fatalf("process: %v", err)
}
facts, err := s.GetFacts(userID, fileID)
if err != nil {
t.Fatalf("get facts: %v", err)
}
byType := map[string][]FactDTO{}
for _, f := range facts {
byType[f.FactType] = append(byType[f.FactType], f)
if f.Origin != "auto" {
t.Errorf("origine = %q, attendu auto", f.Origin)
}
}
if len(byType["reference"]) == 0 || len(byType["amount"]) == 0 ||
len(byType["date"]) == 0 || len(byType["due_date"]) == 0 ||
len(byType["doc_type"]) == 0 {
t.Fatalf("faits manquants: %+v", byType)
}
var amount float64
for _, f := range byType["amount"] {
if f.ValueNum != nil {
amount = *f.ValueNum
}
}
if amount != 84.20 {
t.Errorf("montant = %v, attendu 84.20", amount)
}
}
func TestProcessTextPreservesUserCorrections(t *testing.T) {
s, ex := newExtractStore(t)
userID := mustCreateUser(t, s.Repository, "extract-user")
fileID := repository.NewID()
if err := s.Repository.Resources.InsertFile(userID, fileID, "facture.pdf", "", 0, nil, nil); err != nil {
t.Fatalf("insert: %v", err)
}
// Correction manuelle (S3, simulée en SQL) : l'auto ne doit jamais l'écraser.
if _, err := s.Repository.Resources.DB.Exec(
`INSERT INTO resource_facts (id, resource_id, fact_type, value_text, confidence, origin)
VALUES ($1, $2, 'issuer', 'Vrai Fournisseur', 1.0, 'user')`,
repository.NewID(), fileID,
); err != nil {
t.Fatalf("seed user fact: %v", err)
}
if err := ex.ProcessText(fileID, extractCorpus); err != nil {
t.Fatalf("process: %v", err)
}
// Re-passe (idempotence du remplacement auto).
if err := ex.ProcessText(fileID, extractCorpus); err != nil {
t.Fatalf("re-process: %v", err)
}
facts, err := s.GetFacts(userID, fileID)
if err != nil {
t.Fatalf("get facts: %v", err)
}
issuers := map[string]string{}
autoCount := 0
for _, f := range facts {
if f.FactType == "issuer" {
issuers[f.Origin] = f.ValueText
}
if f.Origin == "auto" {
autoCount++
}
}
if issuers["user"] != "Vrai Fournisseur" {
t.Errorf("correction user écrasée: %+v", issuers)
}
if autoCount == 0 {
t.Error("aucun fait auto après re-passe")
}
// Pas de doublons auto après deux passes.
seen := map[string]int{}
for _, f := range facts {
if f.Origin == "auto" {
seen[f.FactType+"\x00"+f.ValueText]++
}
}
for k, n := range seen {
if n > 1 {
t.Errorf("doublon auto %q x%d", k, n)
}
}
}
func TestProcessTextEmptyClearsStaleAuto(t *testing.T) {
s, ex := newExtractStore(t)
userID := mustCreateUser(t, s.Repository, "extract-clear")
fileID := repository.NewID()
if err := s.Repository.Resources.InsertFile(userID, fileID, "doc.pdf", "", 0, nil, nil); err != nil {
t.Fatalf("insert: %v", err)
}
if err := ex.ProcessText(fileID, extractCorpus); err != nil {
t.Fatalf("process: %v", err)
}
if err := ex.ProcessText(fileID, ""); err != nil {
t.Fatalf("clear: %v", err)
}
facts, err := s.GetFacts(userID, fileID)
if err != nil {
t.Fatalf("get facts: %v", err)
}
if len(facts) != 0 {
t.Fatalf("auto périmés non purgés: %+v", facts)
}
}
func TestGetFactsScope(t *testing.T) {
s, ex := newExtractStore(t)
owner := mustCreateUser(t, s.Repository, "extract-scope-o")
other := mustCreateUser(t, s.Repository, "extract-scope-x")
fileID := repository.NewID()
if err := s.Repository.Resources.InsertFile(owner, fileID, "doc.pdf", "", 0, nil, nil); err != nil {
t.Fatalf("insert: %v", err)
}
if err := ex.ProcessText(fileID, extractCorpus); err != nil {
t.Fatalf("process: %v", err)
}
if _, err := s.GetFacts(other, fileID); err != repository.ErrNotFound {
t.Fatalf("étranger : attendu NOT_FOUND, got %v", err)
}
if _, err := s.GetFacts(owner, repository.NewID()); err != repository.ErrNotFound {
t.Fatalf("inconnu : attendu NOT_FOUND, got %v", err)
}
if _, err := s.Trash(owner, fileID); err != nil {
t.Fatalf("trash: %v", err)
}
if _, err := s.GetFacts(owner, fileID); err != repository.ErrNotFound {
t.Fatalf("trashé : attendu NOT_FOUND, got %v", err)
}
}
func TestBackfill(t *testing.T) {
s, ex := newExtractStore(t)
userID := mustCreateUser(t, s.Repository, "extract-backfill")
oldID := repository.NewID()
if err := s.Repository.Resources.InsertFile(userID, oldID, "vieux.pdf", "", 0, nil, nil); err != nil {
t.Fatalf("insert: %v", err)
}
// OCR d'avant la V3 : texte présent, aucun fait auto.
if err := s.Repository.Resources.UpdateOcrText(userID, oldID, extractCorpus); err != nil {
t.Fatalf("ocr text: %v", err)
}
n, err := ex.Backfill(200)
if err != nil {
t.Fatalf("backfill: %v", err)
}
if n != 1 {
t.Fatalf("backfill = %d, attendu 1", n)
}
facts, err := s.GetFacts(userID, oldID)
if err != nil || len(facts) == 0 {
t.Fatalf("faits après backfill: %+v %v", facts, err)
}
// Deuxième passe : rien à faire (idempotent).
n, err = ex.Backfill(200)
if err != nil {
t.Fatalf("backfill 2: %v", err)
}
if n != 0 {
t.Fatalf("backfill 2 = %d, attendu 0", n)
}
}
+25 -1
View File
@@ -41,6 +41,11 @@ type Ocr struct {
Lang string
Engine ocr.Engine
StaleAfter time.Duration
// OnText, quand non-nil, reçoit chaque texte extrait avec succès (V3 S2 :
// le service Extract en dérive des faits génériques). Appelé dans le
// worker, jamais dans un handler ; une erreur est loggée, pas propagée
// (le job OCR reste `done` — l'extract est un enrichissement, pas un gate).
OnText func(userID, resourceID, text string)
}
func NewOcr(repo *repository.Repository, uploadDir, lang string, engine ocr.Engine) *Ocr {
@@ -166,9 +171,10 @@ func (o *Ocr) process(ctx context.Context, row repository.OcrJobRow) {
fail(err.Error())
return
}
trimmed := strings.TrimSpace(text)
// Le texte atterrit sur la ressource (updated_at bumpé) : consultable par
// tous les devices du user et les grantees via GET /files/:id/ocr.
if trimmed := strings.TrimSpace(text); trimmed != "" {
if trimmed != "" {
if err := o.Repository.Resources.UpdateOcrText(userID, row.FileID, trimmed); err != nil {
if errors.Is(err, repository.ErrNotFound) {
fail("file not readable")
@@ -178,9 +184,27 @@ func (o *Ocr) process(ctx context.Context, row repository.OcrJobRow) {
return
}
}
// V3 S2 : faits génériques (même texte vide — purge les auto périmés).
if o.OnText != nil {
// userID capturing : le user courant, pas un owner résolu plus tard.
if terr := o.safeOnText(userID, row.FileID, trimmed); terr != nil {
log.Printf("ocr: extract %s: %v", row.FileID, terr)
}
}
complete(text)
}
// safeOnText isole un hook OnText paniquant (règle défectueuse) du worker.
func (o *Ocr) safeOnText(userID, resourceID, text string) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("extract panic: %v", r)
}
}()
o.OnText(userID, resourceID, text)
return nil
}
// physicalPath résout UPLOAD_DIR/<user_id>/<resource_id>.<ext> — l'ext est
// choisi à l'upload, le fichier réel est retrouvé par préfixe. Durci (A12) :
// identifiants 32-hex uniquement (anti `..`), `Lstat` + fichier régulier
+38 -5
View File
@@ -233,10 +233,6 @@ func (s *Resources) GetFileOcr(ownerID, id string) (FileOcrDTO, error) {
}, nil
}
func (s *Resources) DeleteFile(ownerID, id string) (string, error) {
return s.Repo.DeleteFile(ownerID, id)
}
func (s *Resources) ListRootFolders(ownerID string) ([]FolderDTO, error) {
rows, err := s.Repo.ListRootFoldersVisible(ownerID)
if err != nil {
@@ -280,11 +276,17 @@ func (s *Resources) Upload(ownerID string, file *multipart.FileHeader, folderID,
return FileDTO{}, ErrInvalidUpload
}
targetExisting := resourceID != ""
var cur repository.FileRow
if targetExisting {
if !idPattern.MatchString(resourceID) {
return FileDTO{}, repository.ErrNotFound
}
if _, err := s.Repo.GetFile(ownerID, resourceID); err != nil {
var err error
// GetFile ne voit que le live : un upload vers une ressource
// trashée/absente/d'autrui → NOT_FOUND (jamais de résurrection
// d'octets sur une ligne en corbeille).
cur, err = s.Repo.GetFile(ownerID, resourceID)
if err != nil {
return FileDTO{}, err
}
} else {
@@ -327,8 +329,27 @@ func (s *Resources) Upload(ownerID string, file *multipart.FileHeader, folderID,
}
}
// V3 versions : sur upload ciblé, figer l'ancienne génération AVANT
// l'écriture (le rename atomique de copyMultipartAtomic détruirait
// l'ancien contenu). Staging = copie, jamais move : l'échec reste safe.
// Metadata-only (aucun octet live) → rien à versionner.
var staged, stagedExt string
var stagedSize int64
var stagedMime string
var hasStaged bool
if targetExisting {
var ferr error
staged, stagedExt, stagedSize, stagedMime, hasStaged, ferr = s.freezeLive(ownerID, resourceID, cur, destDir)
if ferr != nil {
return FileDTO{}, ferr
}
}
written, err := copyMultipartAtomic(file, destDir, destPath, s.MaxFileSize)
if err != nil {
if hasStaged {
_ = os.Remove(staged)
}
return FileDTO{}, err
}
@@ -358,6 +379,9 @@ func (s *Resources) Upload(ownerID string, file *multipart.FileHeader, folderID,
if !stillOld {
_ = os.Remove(destPath)
}
if hasStaged {
_ = os.Remove(staged)
}
return FileDTO{}, err
}
// Succès : supprime les anciens suffixes orphelins (stale bytes, cf. #6).
@@ -366,6 +390,15 @@ func (s *Resources) Upload(ownerID string, file *multipart.FileHeader, folderID,
_ = os.Remove(m)
}
}
// Overwrite ciblée : l'ancien live devient une version figée (N=3).
// Best-effort après succès des métadonnées : un échec ici ne fait pas
// échouer l'upload (loggé, staging nettoyé).
if hasStaged {
if ferr := s.finalizeVersion(ownerID, resourceID, staged, stagedExt, stagedSize, stagedMime); ferr != nil {
log.Printf("upload: finalize version(%s): %v", resourceID, ferr)
_ = os.Remove(staged)
}
}
row, err := s.Repo.GetFile(ownerID, resourceID)
if err != nil {
+17 -5
View File
@@ -291,12 +291,24 @@ func TestUploadReuploadNewExtensionRemovesStaleFile(t *testing.T) {
if err != nil {
t.Fatalf("readdir: %v", err)
}
if len(entries) != 1 || entries[0].Name() != resourceID+".jpg" {
names := make([]string, 0, len(entries))
for _, e := range entries {
names = append(names, e.Name())
// V3 versions : le re-upload fige l'ancien live (png) en version, le
// cleanup des suffixes orphelins ne touche que le live. Attendu : le
// live `<id>.jpg` + un dossier `.versions/` contenant le png figé.
var live []string
for _, e := range entries {
if e.Name() == ".versions" {
continue
}
t.Errorf("un seul physique attendu (%s.jpg), got %v", resourceID, names)
live = append(live, e.Name())
}
if len(live) != 1 || live[0] != resourceID+".jpg" {
t.Errorf("un seul live attendu (%s.jpg), got %v", resourceID, live)
}
verFiles, _ := filepath.Glob(filepath.Join(s.UploadDir, userID, ".versions", resourceID+"_v*"))
if len(verFiles) != 1 {
t.Errorf("une version figée (png) attendue, got %v", verFiles)
} else if content, err := os.ReadFile(verFiles[0]); err != nil || string(content) != "PNGDATA" {
t.Errorf("version figée = %q, err=%v (attendu PNGDATA)", content, err)
}
content, err := os.ReadFile(filepath.Join(s.UploadDir, userID, resourceID+".jpg"))
if err != nil || string(content) != "JPGDATA!" {
+61 -7
View File
@@ -25,6 +25,12 @@ const (
OpRevokeLink = "revoke_link"
// V2 placements : un device signale la présence de ses octets.
OpReportBytes = "report_bytes"
// V3 corbeille (P1) : trash = soft (restaure via restore), purge = hard.
// delete_resource (legacy) est mappé vers trash_resource (même sémantique
// d'idempotence, cf. applyTrashOp).
OpTrashResource = "trash_resource"
OpRestoreResource = "restore_resource"
OpPurgeResource = "purge_resource"
)
// SyncOperation is one outbox entry (shadow of mobile PendingOperationRow).
@@ -200,7 +206,7 @@ func validateSyncOp(op *SyncOperation) error {
switch op.Operation {
case OpCreateResource, OpUpdateMetadata, OpMoveResource, OpDeleteResource,
OpShare, OpRevokeShare, OpUpdateShare, OpCreateLink, OpRevokeLink,
OpReportBytes:
OpReportBytes, OpTrashResource, OpRestoreResource, OpPurgeResource:
default:
return errors.New("unknown operation " + op.Operation)
}
@@ -311,11 +317,13 @@ func (s *Resources) applySyncOp(ownerID, deviceID string, op *SyncOperation) err
}
return s.Repo.MoveResource(ownerID, resourceID, p.ToFolderResourceID)
case OpDeleteResource:
// A8 : delete owner-only. Ligne vivante d'autrui → NOT_FOUND halt
// (pas de applied silencieux). Aucune ligne vivante (jamais existé,
// ou déjà supprimée par son owner) → no-op idempotent : l'état
// terminal (absente) est déjà atteint, le rejeu reste sûr.
case OpDeleteResource, OpTrashResource:
// V3 : delete_resource (legacy) = trash_resource. A8 : owner-only.
// Ligne vivante d'autrui → NOT_FOUND halt (pas de applied
// silencieux). Aucune ligne (jamais existé, déjà trashée/purgée par
// son owner) → no-op idempotent : l'état terminal est atteint.
// Note : une ressource déjà trashée reste trashée (re-trash = no-op
// via TrashResource qui ne touche que deleted_at IS NULL).
exists, err := s.Repo.ExistsOwner(ownerID, resourceID)
if err != nil {
return err
@@ -329,7 +337,53 @@ func (s *Resources) applySyncOp(ownerID, deviceID string, op *SyncOperation) err
}
return repository.ErrNotFound
}
return s.Repo.SyncDelete(ownerID, resourceID)
return s.Repo.TrashResource(ownerID, resourceID)
case OpRestoreResource:
// Owner-only (live ou trashé). Absente → no-op (rien à restaurer,
// l'état terminal — visible — ne peut pas être atteint autrement,
// mais halter casserait la convergence d'un restore rejoué après
// purge... non : après purge la ligne n'existe plus, le restore est
// sans objet → no-op documenté). Vivante d'autrui → NOT_FOUND halt.
owned, err := s.Repo.ExistsOwnerAny(ownerID, resourceID)
if err != nil {
return err
}
if !owned {
if _, err := s.Repo.GetResourceOwner(resourceID); err != nil {
if errors.Is(err, repository.ErrNotFound) {
return nil
}
return err
}
return repository.ErrNotFound
}
return s.Repo.RestoreResource(ownerID, resourceID)
case OpPurgeResource:
// Owner-only (live ou trashé). Absente → no-op idempotent.
// Vivante d'autrui → NOT_FOUND halt. Octets supprimés best-effort.
owned, err := s.Repo.ExistsOwnerAny(ownerID, resourceID)
if err != nil {
return err
}
if !owned {
if _, err := s.Repo.GetResourceOwner(resourceID); err != nil {
if errors.Is(err, repository.ErrNotFound) {
return nil
}
return err
}
return repository.ErrNotFound
}
purged, err := s.Repo.PurgeResource(ownerID, resourceID)
if err != nil {
return err
}
for _, pid := range purged {
s.removePhysical(ownerID, pid)
}
return nil
case OpShare, OpUpdateShare:
return s.applyShareOp(ownerID, resourceID, op)
+420
View File
@@ -0,0 +1,420 @@
package service
import (
"fmt"
"io"
"log"
"os"
"path/filepath"
"strings"
"time"
"github.com/kazier/backend/repository"
)
// TrashRetention is the corbeille lifetime (V3 P1 : 30j, cf. V3.md §2).
// Après ce délai, la passe de purge détruit définitivement les lignes.
const TrashRetention = 30 * 24 * time.Hour
// TrashDTO serializes one trashed resource for GET /trash.
type TrashDTO struct {
ID string `json:"id"`
Type string `json:"type"`
Name string `json:"name"`
Size int64 `json:"size"`
TrashedAt string `json:"trashedAt"`
UpdatedAt string `json:"updatedAt,omitempty"`
}
// VersionDTO serializes one frozen byte-generation for GET /files/:id/versions.
type VersionDTO struct {
VersionNo int `json:"versionNo"`
Size int64 `json:"size"`
MimeType string `json:"mimeType,omitempty"`
CreatedAt string `json:"createdAt"`
}
// Trash soft-deletes a resource and its subtree (corbeille). Owner-only :
// foreign or absent → repository.ErrNotFound (même code, pas d'oracle).
// delete_resource (legacy) est mappé vers Trash (cf. applySyncOp).
func (s *Resources) Trash(ownerID, id string) (string, error) {
owned, err := s.Repo.ExistsOwner(ownerID, id)
if err != nil {
return "", err
}
if !owned {
return "", repository.ErrNotFound
}
if err := s.Repo.TrashResource(ownerID, id); err != nil {
return "", err
}
return id, nil
}
// DeleteFile is the legacy owner-only file removal — now a trash (V3 P1).
// Purge définitive : Purge. Le handler choisit via ?mode=purge.
func (s *Resources) DeleteFile(ownerID, id string) (string, error) {
// Restreint aux files comme avant (les dossiers passent par l'outbox).
if _, err := s.Repo.GetFile(ownerID, id); err != nil {
return "", err
}
return s.Trash(ownerID, id)
}
// Restore clears the trash flag on a resource, its subtree and its ancestor
// chain. Owner-only : absent/foreign → ErrNotFound ; live → no-op succès.
func (s *Resources) Restore(ownerID, id string) (string, error) {
owned, err := s.Repo.ExistsOwnerAny(ownerID, id)
if err != nil {
return "", err
}
if !owned {
return "", repository.ErrNotFound
}
if err := s.Repo.RestoreResource(ownerID, id); err != nil {
return "", err
}
return id, nil
}
// Purge hard-deletes a resource and its subtree (lignes + versions + octets).
// Owner-only (trashed ou live) : absent/foreign → ErrNotFound.
// Les fichiers physiques sont supprimés best-effort (jamais d'échec sur FS).
func (s *Resources) Purge(ownerID, id string) (string, error) {
owned, err := s.Repo.ExistsOwnerAny(ownerID, id)
if err != nil {
return "", err
}
if !owned {
return "", repository.ErrNotFound
}
purged, err := s.Repo.PurgeResource(ownerID, id)
if err != nil {
return "", err
}
for _, pid := range purged {
s.removePhysical(ownerID, pid)
}
return id, nil
}
// ListTrashed returns the owner's corbeille, most recently trashed first.
func (s *Resources) ListTrashed(ownerID string, page, pageSize int) ([]TrashDTO, int, error) {
rows, total, err := s.Repo.ListTrashed(ownerID, pageSize, (page-1)*pageSize)
if err != nil {
return nil, 0, err
}
out := make([]TrashDTO, 0, len(rows))
for _, row := range rows {
out = append(out, TrashDTO{
ID: row.ID,
Type: row.Type,
Name: row.Name,
Size: row.Size,
TrashedAt: row.TrashedAt.UTC().Format(time.RFC3339),
UpdatedAt: row.UpdatedAt.UTC().Format(time.RFC3339),
})
}
return out, total, nil
}
// ListVersions returns the frozen byte-generations of a live owned resource,
// newest first. Trashed/absent/foreign → ErrNotFound.
func (s *Resources) ListVersions(ownerID, id string) ([]VersionDTO, error) {
rows, err := s.Repo.ListVersions(ownerID, id)
if err != nil {
return nil, err
}
out := make([]VersionDTO, 0, len(rows))
for _, row := range rows {
out = append(out, VersionDTO{
VersionNo: row.VersionNo,
Size: row.Size,
MimeType: row.MimeType,
CreatedAt: row.CreatedAt.UTC().Format(time.RFC3339),
})
}
return out, nil
}
// RestoreVersion recopies la génération n vers l'emplacement live (l'actuel
// est à son tour versionné, puis élagué à N=3). Owner-only, parent live.
func (s *Resources) RestoreVersion(ownerID, id string, n int) (FileDTO, error) {
ver, err := s.Repo.GetVersion(ownerID, id, n)
if err != nil {
return FileDTO{}, err
}
srcAbs := filepath.Join(s.UploadDir, ver.StoredPath)
if fi, err := os.Stat(srcAbs); err != nil || !fi.Mode().IsRegular() {
return FileDTO{}, repository.ErrNotFound
}
cur, err := s.Repo.GetFile(ownerID, id)
if err != nil {
return FileDTO{}, err
}
destDir := filepath.Join(s.UploadDir, ownerID)
if err := os.MkdirAll(destDir, 0o700); err != nil {
return FileDTO{}, fmt.Errorf("create upload dir: %w", err)
}
if err := os.Chmod(destDir, 0o700); err != nil {
return FileDTO{}, fmt.Errorf("chmod upload dir: %w", err)
}
// 1. Geler l'actuel (staging copy, pas de move — l'échec reste safe).
staged, stagedExt, stagedSize, stagedMime, hasLive, err := s.freezeLive(ownerID, id, cur, destDir)
if err != nil {
return FileDTO{}, err
}
if hasLive {
defer func() {
// Staging orphelin si on sort en erreur avant finalize.
if err != nil {
_ = os.Remove(staged)
}
}()
}
// 2. Recopier la version vers le live (même atomicité que l'upload).
destName := id
if ver.Extension != "" {
destName = id + "." + ver.Extension
}
if err := copyFileAtomic(srcAbs, destDir, filepath.Join(destDir, destName)); err != nil {
return FileDTO{}, err
}
mime := ver.MimeType
if mime == "" {
mime = "application/octet-stream"
}
var extPtr *string
if ver.Extension != "" {
extPtr = &ver.Extension
}
fi, err := os.Stat(filepath.Join(destDir, destName))
if err != nil {
return FileDTO{}, err
}
if err := s.Repo.UpdatePhysical(ownerID, id, fi.Size(), &mime, extPtr); err != nil {
return FileDTO{}, err
}
if err := s.Repo.SetHasContent(ownerID, id, true); err != nil {
log.Printf("versions: SetHasContent(%s): %v", id, err)
}
// 3. Nettoyer les suffixes orphelins + figer l'ancien actuel en version.
s.removeStaleSuffixes(destDir, id, destName)
if hasLive {
if ferr := s.finalizeVersion(ownerID, id, staged, stagedExt, stagedSize, stagedMime); ferr != nil {
log.Printf("versions: finalize displaced current(%s): %v", id, ferr)
}
}
row, err := s.Repo.GetFile(ownerID, id)
if err != nil {
return FileDTO{}, err
}
return toFileDTO(row), nil
}
// PurgeExpiredTrash détruit les lignes trashées depuis plus de retention
// (corbeille 30j) + leurs octets (best-effort). Boucle par batch jusqu'à vide.
// Retourne le nombre de ressources racines purgées. Appelé au boot puis par
// ticker (cf. cmd/server/main.go).
func (s *Resources) PurgeExpiredTrash(retention time.Duration) (int, error) {
cutoff := time.Now().Add(-retention)
total := 0
for {
purged, err := s.Repo.PurgeExpiredTrash(cutoff, 500)
if err != nil {
return total, err
}
if len(purged) == 0 {
return total, nil
}
for _, p := range purged {
s.removePhysical(p.UserID, p.ResourceID)
total++
}
}
}
// removePhysical supprime tous les octets connus d'une ressource : live
// (UPLOAD_DIR/<user>/<resource>.* + nu) et versions (.versions/). Best-effort.
func (s *Resources) removePhysical(userID, resourceID string) {
if !idPattern.MatchString(userID) || !idPattern.MatchString(resourceID) {
return
}
dir := filepath.Join(s.UploadDir, userID)
matches, _ := filepath.Glob(filepath.Join(dir, resourceID+".*"))
for _, m := range matches {
if strings.HasPrefix(filepath.Base(m), resourceID+".") {
_ = os.Remove(m)
}
}
_ = os.Remove(filepath.Join(dir, resourceID))
verMatches, _ := filepath.Glob(filepath.Join(dir, ".versions", resourceID+"_v*"))
for _, m := range verMatches {
_ = os.Remove(m)
}
}
// removeStaleSuffixes supprime les anciens suffixes orphelins après écriture
// du live (même règle que Upload : tout `<id>.*` sauf le live).
func (s *Resources) removeStaleSuffixes(destDir, resourceID, liveName string) {
matches, _ := filepath.Glob(filepath.Join(destDir, resourceID+".*"))
for _, m := range matches {
if filepath.Base(m) != liveName && strings.HasPrefix(filepath.Base(m), resourceID+".") {
_ = os.Remove(m)
}
}
}
// livePhysicalFile retourne le chemin live existant d'une ressource (suffixé
// ou nu), "" si metadata-only.
func livePhysicalFile(uploadDir, ownerID, resourceID string) string {
dir := filepath.Join(uploadDir, ownerID)
matches, _ := filepath.Glob(filepath.Join(dir, resourceID+".*"))
for _, m := range matches {
if strings.HasPrefix(filepath.Base(m), resourceID+".") {
if fi, err := os.Lstat(m); err == nil && fi.Mode().IsRegular() {
return m
}
}
}
if bare := filepath.Join(dir, resourceID); bare != "" {
if fi, err := os.Lstat(bare); err == nil && fi.Mode().IsRegular() {
return bare
}
}
return ""
}
// freezeLive copie les octets live vers un staging temp (jamais de move :
// l'original reste en place jusqu'au succès). Retourne le staging + métadonnées.
// ok=false quand metadata-only (rien à versionner).
func (s *Resources) freezeLive(ownerID, resourceID string, cur repository.FileRow, destDir string) (staging, ext string, size int64, mime string, ok bool, err error) {
live := livePhysicalFile(s.UploadDir, ownerID, resourceID)
if live == "" {
return "", "", 0, "", false, nil
}
base := filepath.Base(live)
ext = ""
if strings.HasPrefix(base, resourceID+".") {
ext = strings.TrimPrefix(base, resourceID+".")
}
tmp, err := os.CreateTemp(destDir, ".ver-"+resourceID+".*")
if err != nil {
return "", "", 0, "", false, fmt.Errorf("stage version: %w", err)
}
staging = tmp.Name()
if err := copyInto(tmp, live); err != nil {
tmp.Close()
_ = os.Remove(staging)
return "", "", 0, "", false, err
}
if err := tmp.Close(); err != nil {
_ = os.Remove(staging)
return "", "", 0, "", false, fmt.Errorf("stage version: %w", err)
}
fi, err := os.Stat(staging)
if err != nil {
_ = os.Remove(staging)
return "", "", 0, "", false, err
}
mime = cur.MimeType
if mime == "" {
mime = "application/octet-stream"
}
return staging, ext, fi.Size(), mime, true, nil
}
// finalizeVersion déplace le staging vers .versions/, insère la ligne et
// élague au-delà de MaxVersions (fichiers élagués supprimés best-effort).
func (s *Resources) finalizeVersion(ownerID, resourceID, staged, ext string, size int64, mime string) error {
no, err := s.Repo.NextVersionNo(resourceID)
if err != nil {
return err
}
rel := filepath.Join(ownerID, ".versions", fmt.Sprintf("%s_v%d", resourceID, no))
if ext != "" {
rel += "." + ext
}
abs := filepath.Join(s.UploadDir, rel)
if err := os.MkdirAll(filepath.Dir(abs), 0o700); err != nil {
return err
}
if err := os.Rename(staged, abs); err != nil {
return err
}
if err := os.Chmod(abs, 0o600); err != nil {
log.Printf("versions: chmod(%s): %v", rel, err)
}
var extPtr *string
if ext != "" {
extPtr = &ext
}
vid, err := repository.NewIDErr()
if err != nil {
return err
}
if err := s.Repo.InsertVersion(vid, resourceID, no, size, &mime, extPtr, rel); err != nil {
_ = os.Remove(abs)
return err
}
pruned, err := s.Repo.PruneVersions(resourceID, repository.MaxVersions)
if err != nil {
log.Printf("versions: prune(%s): %v", resourceID, err)
return nil
}
for _, p := range pruned {
_ = os.Remove(filepath.Join(s.UploadDir, p))
}
return nil
}
// copyInto copie src vers le fichier ouvert dst.
func copyInto(dst *os.File, src string) error {
in, err := os.Open(src)
if err != nil {
return err
}
defer in.Close()
if _, err := io.Copy(dst, in); err != nil {
return fmt.Errorf("copy version bytes: %w", err)
}
return nil
}
// copyFileAtomic recopie src vers destPath via temp+rename (0600).
func copyFileAtomic(src, destDir, destPath string) error {
tmp, err := os.CreateTemp(destDir, ".tmp-"+filepath.Base(destPath)+".*")
if err != nil {
return fmt.Errorf("create tmp file: %w", err)
}
tmpName := tmp.Name()
success := false
defer func() {
if !success {
_ = os.Remove(tmpName)
}
}()
if err := tmp.Chmod(0o600); err != nil {
tmp.Close()
return fmt.Errorf("chmod tmp file: %w", err)
}
if err := copyInto(tmp, src); err != nil {
tmp.Close()
return err
}
if err := tmp.Sync(); err != nil {
tmp.Close()
return fmt.Errorf("fsync: %w", err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("close: %w", err)
}
if err := os.Rename(tmpName, destPath); err != nil {
return fmt.Errorf("commit: %w", err)
}
if err := os.Chmod(destPath, 0o600); err != nil {
return fmt.Errorf("chmod: %w", err)
}
success = true
return nil
}
+364
View File
@@ -0,0 +1,364 @@
package service
import (
"errors"
"os"
"path/filepath"
"testing"
"time"
"github.com/kazier/backend/repository"
)
func mustInsertFile(t *testing.T, s *Resources, userID, name string) string {
t.Helper()
id := repository.NewID()
if err := s.Repository.Resources.InsertFile(userID, id, name, "", 0, nil, nil); err != nil {
t.Fatalf("insert file %q: %v", name, err)
}
return id
}
func TestTrashRestoreRoundtrip(t *testing.T) {
s := newServiceStore(t)
userID := mustCreateUser(t, s.Repository, "trash-roundtrip")
fileID := mustInsertFile(t, s, userID, "acte.pdf")
if _, err := s.Trash(userID, fileID); err != nil {
t.Fatalf("trash: %v", err)
}
if _, err := s.Repo.GetFile(userID, fileID); !errors.Is(err, repository.ErrNotFound) {
t.Fatalf("GetFile après trash : attendu NOT_FOUND, got %v", err)
}
items, total, err := s.ListTrashed(userID, 1, 50)
if err != nil {
t.Fatalf("list trashed: %v", err)
}
if total != 1 || len(items) != 1 || items[0].ID != fileID {
t.Fatalf("corbeille: total=%d items=%+v", total, items)
}
if _, err := s.Restore(userID, fileID); err != nil {
t.Fatalf("restore: %v", err)
}
if _, err := s.Repo.GetFile(userID, fileID); err != nil {
t.Fatalf("GetFile après restore: %v", err)
}
if _, total, err := s.ListTrashed(userID, 1, 50); err != nil || total != 0 {
t.Fatalf("corbeille après restore: total=%d err=%v", total, err)
}
}
func TestTrashSubtree(t *testing.T) {
s := newServiceStore(t)
userID := mustCreateUser(t, s.Repository, "trash-subtree")
folderID := repository.NewID()
if err := s.Repository.Resources.InsertFolder(userID, folderID, "Dossier", ""); err != nil {
t.Fatalf("insert folder: %v", err)
}
childID := repository.NewID()
if err := s.Repository.Resources.InsertFile(userID, childID, "enfant.txt", folderID, 0, nil, nil); err != nil {
t.Fatalf("insert child: %v", err)
}
if _, err := s.Trash(userID, folderID); err != nil {
t.Fatalf("trash folder: %v", err)
}
if _, err := s.Repo.GetFile(userID, childID); !errors.Is(err, repository.ErrNotFound) {
t.Fatalf("enfant après trash dossier : attendu NOT_FOUND, got %v", err)
}
if _, total, _ := s.ListTrashed(userID, 1, 50); total != 2 {
t.Fatalf("corbeille subtree: total=%d, attendu 2", total)
}
if _, err := s.Restore(userID, folderID); err != nil {
t.Fatalf("restore folder: %v", err)
}
if _, err := s.Repo.GetFile(userID, childID); err != nil {
t.Fatalf("enfant après restore dossier: %v", err)
}
}
func TestTrashForeignIsNotFound(t *testing.T) {
s := newServiceStore(t)
owner := mustCreateUser(t, s.Repository, "trash-owner")
other := mustCreateUser(t, s.Repository, "trash-other")
fileID := mustInsertFile(t, s, owner, "secret.pdf")
if _, err := s.Trash(other, fileID); !errors.Is(err, repository.ErrNotFound) {
t.Fatalf("trash autrui : attendu NOT_FOUND, got %v", err)
}
if _, err := s.Restore(other, fileID); !errors.Is(err, repository.ErrNotFound) {
t.Fatalf("restore autrui : attendu NOT_FOUND, got %v", err)
}
if _, err := s.Purge(other, fileID); !errors.Is(err, repository.ErrNotFound) {
t.Fatalf("purge autrui : attendu NOT_FOUND, got %v", err)
}
if _, err := s.Repo.GetFile(owner, fileID); err != nil {
t.Fatalf("fichier owner intact: %v", err)
}
}
func TestDeleteLegacyMapsToTrash(t *testing.T) {
s := newServiceStore(t)
userID := mustCreateUser(t, s.Repository, "trash-legacy")
device := mustRegisterDevice(t, s.Repository, "b0000000000000000000000000000001")
fileID := mustInsertFile(t, s, userID, "legacy.pdf")
// delete_resource → trash (restaurable, visible en corbeille).
res, err := s.ApplyBatch(userID, device, []SyncOperation{
syncOp(repository.NewID(), fileID, OpDeleteResource, "file", map[string]any{}),
})
if err != nil {
t.Fatalf("apply: %v", err)
}
if res.Applied != 1 || res.Failed != nil {
t.Fatalf("résultat inattendu: %+v", res)
}
if _, err := s.Repo.GetFile(userID, fileID); !errors.Is(err, repository.ErrNotFound) {
t.Fatalf("GetFile après delete legacy : attendu NOT_FOUND, got %v", err)
}
if _, total, _ := s.ListTrashed(userID, 1, 50); total != 1 {
t.Fatalf("delete legacy doit apparaître en corbeille")
}
// Re-delete (déjà trashé) → no-op idempotent, pas NOT_FOUND.
res, err = s.ApplyBatch(userID, device, []SyncOperation{
syncOp(repository.NewID(), fileID, OpDeleteResource, "file", map[string]any{}),
})
if err != nil {
t.Fatalf("re-apply: %v", err)
}
if res.Applied != 1 || res.Failed != nil {
t.Fatalf("re-delete trashé doit être no-op: %+v", res)
}
}
func TestSyncOpsTrashRestorePurge(t *testing.T) {
s := newServiceStore(t)
userID := mustCreateUser(t, s.Repository, "trash-ops")
other := mustCreateUser(t, s.Repository, "trash-ops-other")
device := mustRegisterDevice(t, s.Repository, "b0000000000000000000000000000002")
fileID := mustInsertFile(t, s, userID, "ops.pdf")
foreignID := mustInsertFile(t, s, other, "foreign.pdf")
apply := func(op, resID string) SyncResult {
t.Helper()
res, err := s.ApplyBatch(userID, device, []SyncOperation{
syncOp(repository.NewID(), resID, op, "file", map[string]any{}),
})
if err != nil {
t.Fatalf("apply %s: %v", op, err)
}
return res
}
if res := apply(OpTrashResource, fileID); res.Applied != 1 || res.Failed != nil {
t.Fatalf("trash op: %+v", res)
}
// Trash d'autrui → halt NOT_FOUND.
if res := apply(OpTrashResource, foreignID); res.Failed == nil || res.Failed.Code != "NOT_FOUND" {
t.Fatalf("trash autrui doit halter NOT_FOUND: %+v", res)
}
if res := apply(OpRestoreResource, fileID); res.Applied != 1 || res.Failed != nil {
t.Fatalf("restore op: %+v", res)
}
if _, err := s.Repo.GetFile(userID, fileID); err != nil {
t.Fatalf("fichier restauré via op: %v", err)
}
// Restore d'un live → no-op succès.
if res := apply(OpRestoreResource, fileID); res.Applied != 1 || res.Failed != nil {
t.Fatalf("restore live doit être no-op: %+v", res)
}
// Restore d'inexistant → no-op (convergence).
if res := apply(OpRestoreResource, repository.NewID()); res.Applied != 1 || res.Failed != nil {
t.Fatalf("restore inexistant doit être no-op: %+v", res)
}
// Purge d'un live → détruit.
if res := apply(OpPurgeResource, fileID); res.Applied != 1 || res.Failed != nil {
t.Fatalf("purge op: %+v", res)
}
if owned, _ := s.Repo.ExistsOwnerAny(userID, fileID); owned {
t.Fatal("purge doit détruire la ligne")
}
// Purge d'inexistant → no-op.
if res := apply(OpPurgeResource, fileID); res.Applied != 1 || res.Failed != nil {
t.Fatalf("purge inexistant doit être no-op: %+v", res)
}
}
func TestPurgeRemovesPhysicalFiles(t *testing.T) {
s := newServiceStore(t)
userID := mustCreateUser(t, s.Repository, "trash-physical")
dto, err := s.Upload(userID, multipartFileHeader(t, "doc.txt", []byte("hello")), "", "")
if err != nil {
t.Fatalf("upload: %v", err)
}
live := filepath.Join(s.UploadDir, userID, dto.ID+".txt")
if _, err := os.Stat(live); err != nil {
t.Fatalf("live absent: %v", err)
}
if _, err := s.Purge(userID, dto.ID); err != nil {
t.Fatalf("purge: %v", err)
}
if _, err := os.Stat(live); !os.IsNotExist(err) {
t.Fatalf("octets live doivent être supprimés: %v", err)
}
}
func TestUploadOverwriteCreatesVersionsCappedAt3(t *testing.T) {
s := newServiceStore(t)
userID := mustCreateUser(t, s.Repository, "trash-versions")
v1, err := s.Upload(userID, multipartFileHeader(t, "doc.txt", []byte("AAA")), "", "")
if err != nil {
t.Fatalf("upload v1: %v", err)
}
// Le premier upload (création) ne crée pas de version.
if vers, err := s.ListVersions(userID, v1.ID); err != nil || len(vers) != 0 {
t.Fatalf("création ne doit pas versionner: %+v %v", vers, err)
}
contents := []string{"BBBB", "CCCCC", "DDDDDD", "EEEEEEE"}
for i, c := range contents {
dto, err := s.Upload(userID, multipartFileHeader(t, "doc.txt", []byte(c)), "", v1.ID)
if err != nil {
t.Fatalf("overwrite %d: %v", i, err)
}
if dto.Size != int64(len(c)) {
t.Fatalf("overwrite %d: taille=%d", i, dto.Size)
}
}
vers, err := s.ListVersions(userID, v1.ID)
if err != nil {
t.Fatalf("list versions: %v", err)
}
if len(vers) != 3 {
t.Fatalf("versions = %d, attendu 3 (plafond N=3)", len(vers))
}
// Les 3 conservées sont les plus récentes : AAA a été élaguée.
// Ordre décroissant : version_no 4,3,2 → tailles 6,5,4.
wantSizes := []int64{6, 5, 4}
for i, v := range vers {
if v.Size != wantSizes[i] {
t.Errorf("version[%d].size = %d, attendu %d", i, v.Size, wantSizes[i])
}
}
// 3 fichiers figés sur disque (AAA élaguée avec sa ligne).
verFiles, _ := filepath.Glob(filepath.Join(s.UploadDir, userID, ".versions", v1.ID+"_v*"))
if len(verFiles) != 3 {
t.Errorf("fichiers versionnés = %d, attendu 3 (%v)", len(verFiles), verFiles)
}
}
func TestRestoreVersion(t *testing.T) {
s := newServiceStore(t)
userID := mustCreateUser(t, s.Repository, "trash-restore-ver")
dto, err := s.Upload(userID, multipartFileHeader(t, "doc.txt", []byte("AAA")), "", "")
if err != nil {
t.Fatalf("upload v1: %v", err)
}
if _, err := s.Upload(userID, multipartFileHeader(t, "doc.txt", []byte("BBBB")), "", dto.ID); err != nil {
t.Fatalf("overwrite: %v", err)
}
vers, err := s.ListVersions(userID, dto.ID)
if err != nil || len(vers) != 1 {
t.Fatalf("versions: %+v %v", vers, err)
}
restored, err := s.RestoreVersion(userID, dto.ID, vers[0].VersionNo)
if err != nil {
t.Fatalf("restore version: %v", err)
}
if restored.Size != 3 {
t.Fatalf("live restauré taille=%d, attendu 3", restored.Size)
}
raw, err := os.ReadFile(filepath.Join(s.UploadDir, userID, dto.ID+".txt"))
if err != nil {
t.Fatalf("read live: %v", err)
}
if string(raw) != "AAA" {
t.Fatalf("contenu live = %q, attendu AAA", raw)
}
// L'ancien actuel (BBBB) a été à son tour versionné.
vers2, err := s.ListVersions(userID, dto.ID)
if err != nil || len(vers2) != 2 {
t.Fatalf("versions après restore: %+v %v", vers2, err)
}
if _, err := s.RestoreVersion(userID, dto.ID, 999); !errors.Is(err, repository.ErrNotFound) {
t.Fatalf("restore version inconnue : attendu NOT_FOUND, got %v", err)
}
}
func TestUploadToTrashedIsNotFound(t *testing.T) {
s := newServiceStore(t)
userID := mustCreateUser(t, s.Repository, "trash-upload-guard")
dto, err := s.Upload(userID, multipartFileHeader(t, "doc.txt", []byte("AAA")), "", "")
if err != nil {
t.Fatalf("upload: %v", err)
}
if _, err := s.Trash(userID, dto.ID); err != nil {
t.Fatalf("trash: %v", err)
}
if _, err := s.Upload(userID, multipartFileHeader(t, "doc.txt", []byte("BBB")), "", dto.ID); !errors.Is(err, repository.ErrNotFound) {
t.Fatalf("upload vers trashé : attendu NOT_FOUND, got %v", err)
}
}
func backdateTrash(t *testing.T, s *Resources, resourceID string, ago time.Duration) {
t.Helper()
if _, err := s.Repository.Resources.DB.Exec(
`UPDATE resources SET deleted_at = NOW() - ($1 || ' seconds')::interval WHERE resource_id = $2`,
int(ago.Seconds()), resourceID,
); err != nil {
t.Fatalf("backdate: %v", err)
}
}
func TestPurgeExpiredTrash(t *testing.T) {
s := newServiceStore(t)
userID := mustCreateUser(t, s.Repository, "trash-expire")
old, err := s.Upload(userID, multipartFileHeader(t, "vieux.txt", []byte("old")), "", "")
if err != nil {
t.Fatalf("upload vieux: %v", err)
}
fresh, err := s.Upload(userID, multipartFileHeader(t, "frais.txt", []byte("new")), "", "")
if err != nil {
t.Fatalf("upload frais: %v", err)
}
if _, err := s.Trash(userID, old.ID); err != nil {
t.Fatalf("trash vieux: %v", err)
}
if _, err := s.Trash(userID, fresh.ID); err != nil {
t.Fatalf("trash frais: %v", err)
}
backdateTrash(t, s, old.ID, 31*24*time.Hour)
n, err := s.PurgeExpiredTrash(TrashRetention)
if err != nil {
t.Fatalf("purge expired: %v", err)
}
if n != 1 {
t.Fatalf("purgés = %d, attendu 1", n)
}
if owned, _ := s.Repo.ExistsOwnerAny(userID, old.ID); owned {
t.Fatal("vieux doit être détruit")
}
if _, _, err := s.ListTrashed(userID, 1, 50); err != nil {
t.Fatalf("list: %v", err)
}
if _, err := s.Repo.GetFile(userID, fresh.ID); err == nil {
// Frais trashé mais pas expiré : invisible en lecture, présent en corbeille.
t.Fatal("frais doit rester trashé (invisible en lecture)")
}
if _, total, _ := s.ListTrashed(userID, 1, 50); total != 1 {
t.Fatalf("corbeille doit garder le frais: total=%d", total)
}
if _, err := os.Stat(filepath.Join(s.UploadDir, userID, old.ID+".txt")); !os.IsNotExist(err) {
t.Fatal("octets du purgé doivent être supprimés")
}
}