Files
venture 95b4cf79cf V3 S1-S2 : corbeille + versions + extract générique
S1 filet (backend + mobile) : corbeille 30j (deleted_at réutilisé,
trash/restore récursifs, purge auto serveur), versions N=3 à chaque
overwrite, ops outbox trash/restore/purge (delete legacy mappé), Room
v19 (trashed_at), Deleters -> trash, écran Corbeille, walk guards.

S2 extract (backend) : package extract (règles FR stdlib, corpus
testé), resource_facts 000019 (origin auto/user), hook worker OCR +
backfill boot, GET /files/:id/facts (viewer+).

Contrat docs/api-v1.md étendu, V3.md (vision + plan).
2026-09-26 10:14:07 +02:00

409 lines
12 KiB
Go

// Package extract derives generic, verifiable facts from OCR text (V3 S2).
//
// Rules-based French heuristics, zero ML : every fact carries a confidence,
// low-confidence facts are suggested (greyed in UI) and user-correctable (S3),
// and search demotes but never excludes on them. No LLM, no network, pure
// stdlib — it runs inside the OCR worker right after Tesseract.
package extract
import (
"fmt"
"regexp"
"strconv"
"strings"
"unicode"
)
// Fact types — mirror of the resource_facts CHECK (migration 000019).
type FactType string
const (
FactDate FactType = "date"
FactDueDate FactType = "due_date"
FactPeriod FactType = "period"
FactAmount FactType = "amount"
FactIssuer FactType = "issuer"
FactReference FactType = "reference"
FactDocType FactType = "doc_type"
)
// Fact is one extracted statement. ValueText holds issuer names, reference
// codes ("facture:FA-2026-0031"), doc types and periods ("2026-03") ;
// ValueNum holds amounts (HasNum) ; ValueDate holds ISO dates ("2026-03-12").
type Fact struct {
Type FactType
ValueText string
ValueNum float64
HasNum bool
ValueDate string
Confidence float64
}
// MaxFacts caps the output per resource (OCR garbage must stay bounded).
const MaxFacts = 50
// FromText extracts generic facts from raw OCR text. Deterministic output
// order (document order within each type, types in fixed order), duplicates
// removed.
func FromText(text string) []Fact {
e := &extractor{seen: map[string]bool{}}
e.extract(collectLines(text))
out := e.facts
if len(out) > MaxFacts {
out = out[:MaxFacts]
}
return out
}
type extractor struct {
facts []Fact
seen map[string]bool
}
func (e *extractor) add(f Fact) {
if f.Confidence < 0 {
f.Confidence = 0
}
if f.Confidence > 1 {
f.Confidence = 1
}
key := string(f.Type) + "\x00" + f.ValueText + "\x00" + f.ValueDate + "\x00" + fmt.Sprintf("%.2f", f.ValueNum)
if e.seen[key] {
return
}
e.seen[key] = true
e.facts = append(e.facts, f)
}
type line struct {
raw string
folded string
}
func collectLines(text string) []line {
raw := strings.Split(text, "\n")
out := make([]line, 0, len(raw))
for _, r := range raw {
r = strings.TrimSpace(strings.ReplaceAll(r, "\t", " "))
if r == "" {
continue
}
out = append(out, line{raw: r, folded: fold(r)})
}
return out
}
func (e *extractor) extract(lines []line) {
for _, l := range lines {
e.extractAmounts(l)
e.extractDates(l)
e.extractReferences(l)
}
e.extractIssuers(lines)
e.extractDocType(lines)
}
// --- amounts ---------------------------------------------------------------
// Note : pas de `\b` après `€` — c'est un symbole (non-mot), donc il n'y a
// jamais de frontière avec un espace ou une fin de ligne. `eur` garde son
// `\b` (évite "europe"), `euros?` n'en a pas besoin.
var amountPattern = regexp.MustCompile(`(\d[\d\s\x{a0}\x{202f}]*[,\.]\d{2}|\d[\d\s\x{a0}\x{202f}]*)\s*(€|euros?|eur\b)`)
func (e *extractor) extractAmounts(l line) {
// Percentages are not amounts.
if strings.Contains(l.raw, "%") && !strings.ContainsAny(l.raw, "€") {
return
}
for _, m := range amountPattern.FindAllStringSubmatch(l.raw, -1) {
num, ok := parseFrenchNumber(m[1])
if !ok || num <= 0 || num >= 1e13 {
continue
}
conf := 0.7
if amountKeywordPattern.MatchString(l.folded) {
conf = 0.9
}
e.add(Fact{Type: FactAmount, ValueNum: num, HasNum: true, ValueText: formatAmount(num), Confidence: conf})
}
}
var amountKeywordPattern = regexp.MustCompile(`\b(total|montant|a payer|net|ttc|solde|du)\b`)
func formatAmount(num float64) string {
return strconv.FormatFloat(num, 'f', 2, 64) + " €"
}
// parseFrenchNumber reads "1 234,56" / "1234.56" / "84 €"-style magnitudes.
// Comma = decimal separator (French), dots/spaces (incl. NBSP U+00A0 and
// narrow NBSP U+202F, common in "1 234 €") = thousands.
func parseFrenchNumber(s string) (float64, bool) {
s = strings.ReplaceAll(s, "\u00a0", "")
s = strings.ReplaceAll(s, "\u202f", "")
s = strings.ReplaceAll(s, " ", "")
if strings.Contains(s, ",") {
s = strings.ReplaceAll(s, ".", "")
s = strings.ReplaceAll(s, ",", ".")
}
num, err := strconv.ParseFloat(s, 64)
return num, err == nil
}
// --- dates -----------------------------------------------------------------
var numericDatePattern = regexp.MustCompile(`\b(\d{1,2})\s*[/\-.]\s*(\d{1,2})\s*[/\-.]\s*(\d{2}|\d{4})\b`)
var writtenDatePattern = regexp.MustCompile(`\b(\d{1,2})\s+(janv(?:ier)?|fev(?:rier)?|mars|avr(?:il)?|mai|juin|juil(?:let)?|aout|sept?(?:embre)?|oct(?:obre)?|nov(?:embre)?|dec(?:embre)?)\s+(\d{4})\b`)
var periodPattern = regexp.MustCompile(`\b(janv(?:ier)?|fev(?:rier)?|mars|avr(?:il)?|mai|juin|juil(?:let)?|aout|sept?(?:embre)?|oct(?:obre)?|nov(?:embre)?|dec(?:embre)?)\s+(\d{4})\b`)
var monthNumbers = map[string]int{
"janv": 1, "janvier": 1, "fev": 2, "fevrier": 2, "mars": 3,
"avr": 4, "avril": 4, "mai": 5, "juin": 6, "juil": 7,
"juillet": 7, "aout": 8, "sept": 9, "septembre": 9, "oct": 10,
"octobre": 10, "nov": 11, "novembre": 11, "dec": 12, "decembre": 12,
}
var dueKeywordPattern = regexp.MustCompile(`\b(echeance|a payer avant|date limite|avant le|regler avant|paiement avant)\b`)
func fullYear(yy int) int {
if yy < 100 {
if yy >= 70 {
return 1900 + yy
}
return 2000 + yy
}
return yy
}
func isoDate(y, m, d int) (string, bool) {
if m < 1 || m > 12 || d < 1 || d > 31 || y < 1900 || y > 2100 {
return "", false
}
return fmt.Sprintf("%04d-%02d-%02d", y, m, d), true
}
func (e *extractor) extractDates(l line) {
isDue := dueKeywordPattern.MatchString(l.folded)
for _, m := range numericDatePattern.FindAllStringSubmatch(l.raw, -1) {
d, _ := strconv.Atoi(m[1])
mo, _ := strconv.Atoi(m[2])
y, _ := strconv.Atoi(m[3])
iso, ok := isoDate(fullYear(y), mo, d)
if !ok {
continue
}
typ := FactDate
conf := 0.6
if isDue {
typ, conf = FactDueDate, 0.8
}
e.add(Fact{Type: typ, ValueDate: iso, ValueText: iso, Confidence: conf})
}
for _, m := range writtenDatePattern.FindAllStringSubmatch(l.folded, -1) {
d, _ := strconv.Atoi(m[1])
mo := monthNumbers[m[2]]
y, _ := strconv.Atoi(m[3])
iso, ok := isoDate(y, mo, d)
if !ok {
continue
}
typ := FactDate
conf := 0.7
if isDue {
typ, conf = FactDueDate, 0.85
}
e.add(Fact{Type: typ, ValueDate: iso, ValueText: iso, Confidence: conf})
}
for _, m := range periodPattern.FindAllStringSubmatch(l.folded, -1) {
mo := monthNumbers[m[1]]
y, _ := strconv.Atoi(m[2])
if mo == 0 || y < 1900 || y > 2100 {
continue
}
e.add(Fact{Type: FactPeriod, ValueText: fmt.Sprintf("%04d-%02d", y, mo), Confidence: 0.7})
}
}
// --- references ------------------------------------------------------------
// referencePattern catches "facture n° FA-2026-0031", "contrat : ABC123",
// "client 42" — the code must contain a digit (avoids "FACTURE DU MOIS").
var referencePattern = regexp.MustCompile(`(?i)\b(facture|contrat|client|compte|dossier|bulletin|commande|devis|attestation|police)\s*(n°|no|num(?:ero)?|number|ref(?:erence)?)?\s*[:\-#]?\s*([A-Z0-9][A-Z0-9\-/\.]{1,29})`)
var bareRefPattern = regexp.MustCompile(`\bn°\s*([A-Z0-9][A-Z0-9\-/]{2,20})`)
var siretPattern = regexp.MustCompile(`(?i)\b(siret|siren|rcs)\b[^\d]{0,10}(\d{3}\s?\d{3}\s?\d{3}(?:\s?\d{5})?)`)
var ibanPattern = regexp.MustCompile(`\bFR\s?\d{2}(?:\s?[0-9A-Z]{4}){5}\s?[0-9A-Z]{0,3}\b`)
func hasDigit(s string) bool {
for _, r := range s {
if r >= '0' && r <= '9' {
return true
}
}
return false
}
func (e *extractor) extractReferences(l line) {
for _, m := range referencePattern.FindAllStringSubmatch(l.raw, -1) {
code := strings.Trim(m[3], "-/.")
if !hasDigit(code) || len(code) < 3 {
continue
}
kind := strings.ToLower(m[1])
e.add(Fact{Type: FactReference, ValueText: kind + ":" + code, Confidence: 0.75})
}
for _, m := range bareRefPattern.FindAllStringSubmatch(l.raw, -1) {
code := m[1]
if !hasDigit(code) {
continue
}
e.add(Fact{Type: FactReference, ValueText: "ref:" + code, Confidence: 0.6})
}
if m := siretPattern.FindStringSubmatch(l.raw); m != nil {
digits := strings.Join(strings.Fields(m[2]), " ")
e.add(Fact{Type: FactReference, ValueText: strings.ToLower(m[1]) + ":" + digits, Confidence: 0.9})
}
if m := ibanPattern.FindString(l.raw); m != "" {
compact := strings.ReplaceAll(m, " ", "")
if len(compact) == 27 {
// Store masked (privacy) : only the last 4 chars.
e.add(Fact{Type: FactReference, ValueText: "rib:••••" + compact[len(compact)-4:], Confidence: 0.9})
}
}
}
// --- issuers ---------------------------------------------------------------
var companySuffixPattern = regexp.MustCompile(`(?i)\b(sarl|sas|sa|eurl|sasu|sci|eirl|snc|sasau)\b`)
func isCapsLine(s string) bool {
letters := 0
for _, r := range s {
if unicode.IsLetter(r) {
letters++
if unicode.IsLower(r) {
return false
}
}
}
return letters >= 3
}
func (e *extractor) extractIssuers(lines []line) {
found := 0
checked := 0
for _, l := range lines {
if checked >= 8 || found >= 2 {
break
}
raw := strings.TrimSpace(l.raw)
if len([]rune(raw)) < 3 || len([]rune(raw)) > 60 {
continue
}
if strings.Contains(raw, "@") || strings.HasPrefix(strings.ToLower(raw), "http") {
continue
}
checked++
if !isCapsLine(raw) {
continue
}
// Skip lines that are really dates/amounts/references in disguise.
if numericDatePattern.MatchString(raw) || amountPattern.MatchString(raw) || bareRefPattern.MatchString(raw) {
continue
}
conf := 0.45
if found == 0 {
conf = 0.55
}
if companySuffixPattern.MatchString(raw) {
conf = 0.75
}
e.add(Fact{Type: FactIssuer, ValueText: raw, Confidence: conf})
found++
}
}
// --- doc_type suggestion ---------------------------------------------------
// docTypeKeywords maps a suggested type to weighted French keywords (folded).
// A type is emitted when its score reaches docTypeThreshold — the type is a
// humble suggestion (S3-correctable), never a gate.
var docTypeKeywords = []struct {
typ string
keywords map[string]int
}{
{"facture", map[string]int{"facture": 2, "tva": 1, "montant ttc": 2, "total ttc": 2, "a payer": 1, "devis": 1}},
{"bulletin", map[string]int{"bulletin": 2, "salaire": 2, "paie": 1, "net social": 2, "brut": 1}},
{"impot", map[string]int{"impot": 2, "revenu fiscal": 2, "taxe fonciere": 2, "taxe d habitation": 1, "avis d impot": 2}},
{"contrat", map[string]int{"contrat": 2, "bail": 2, "convention": 1, "article ": 1, "assurance": 1}},
{"ordonnance", map[string]int{"ordonnance": 2, "posologie": 2, "docteur": 1, "traitement": 1}},
{"attestation", map[string]int{"attestation": 2, "certifie": 1, "certificat": 1, "atteste": 1}},
{"rib", map[string]int{"rib": 2, "iban": 2, "bic": 1}},
{"identite", map[string]int{"carte d identite": 2, "passeport": 2, "permis de conduire": 2, "nationalite": 1}},
}
const docTypeThreshold = 2
func (e *extractor) extractDocType(lines []line) {
joined := "\n"
for _, l := range lines {
joined += l.folded + "\n"
}
best, bestScore := "", 0
for _, dt := range docTypeKeywords {
score := 0
for kw, w := range dt.keywords {
if strings.Contains(joined, kw) {
score += w
}
}
if score > bestScore {
best, bestScore = dt.typ, score
}
}
if bestScore >= docTypeThreshold {
conf := 0.5 + 0.1*float64(bestScore)
if conf > 0.85 {
conf = 0.85
}
e.add(Fact{Type: FactDocType, ValueText: best, Confidence: conf})
}
}
// --- folding ---------------------------------------------------------------
// fold lowercases French text and strips accents (œ→oe, æ→ae, ç→c) for
// keyword matching. stdlib-only (no x/text dependency churn).
func fold(s string) string {
var b strings.Builder
b.Grow(len(s))
for _, r := range strings.ToLower(s) {
switch r {
case 'à', 'â', 'ä':
b.WriteRune('a')
case 'é', 'è', 'ê', 'ë':
b.WriteRune('e')
case 'î', 'ï':
b.WriteRune('i')
case 'ô', 'ö':
b.WriteRune('o')
case 'ù', 'û', 'ü':
b.WriteRune('u')
case 'ÿ':
b.WriteRune('y')
case 'ç':
b.WriteRune('c')
case 'œ':
b.WriteString("oe")
case 'æ':
b.WriteString("ae")
default:
b.WriteRune(r)
}
}
return b.String()
}