S1 filet (backend + mobile) : corbeille 30j (deleted_at réutilisé, trash/restore récursifs, purge auto serveur), versions N=3 à chaque overwrite, ops outbox trash/restore/purge (delete legacy mappé), Room v19 (trashed_at), Deleters -> trash, écran Corbeille, walk guards. S2 extract (backend) : package extract (règles FR stdlib, corpus testé), resource_facts 000019 (origin auto/user), hook worker OCR + backfill boot, GET /files/:id/facts (viewer+). Contrat docs/api-v1.md étendu, V3.md (vision + plan).
409 lines
12 KiB
Go
409 lines
12 KiB
Go
// Package extract derives generic, verifiable facts from OCR text (V3 S2).
|
|
//
|
|
// Rules-based French heuristics, zero ML : every fact carries a confidence,
|
|
// low-confidence facts are suggested (greyed in UI) and user-correctable (S3),
|
|
// and search demotes but never excludes on them. No LLM, no network, pure
|
|
// stdlib — it runs inside the OCR worker right after Tesseract.
|
|
package extract
|
|
|
|
import (
|
|
"fmt"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"unicode"
|
|
)
|
|
|
|
// Fact types — mirror of the resource_facts CHECK (migration 000019).
|
|
type FactType string
|
|
|
|
const (
|
|
FactDate FactType = "date"
|
|
FactDueDate FactType = "due_date"
|
|
FactPeriod FactType = "period"
|
|
FactAmount FactType = "amount"
|
|
FactIssuer FactType = "issuer"
|
|
FactReference FactType = "reference"
|
|
FactDocType FactType = "doc_type"
|
|
)
|
|
|
|
// Fact is one extracted statement. ValueText holds issuer names, reference
|
|
// codes ("facture:FA-2026-0031"), doc types and periods ("2026-03") ;
|
|
// ValueNum holds amounts (HasNum) ; ValueDate holds ISO dates ("2026-03-12").
|
|
type Fact struct {
|
|
Type FactType
|
|
ValueText string
|
|
ValueNum float64
|
|
HasNum bool
|
|
ValueDate string
|
|
Confidence float64
|
|
}
|
|
|
|
// MaxFacts caps the output per resource (OCR garbage must stay bounded).
|
|
const MaxFacts = 50
|
|
|
|
// FromText extracts generic facts from raw OCR text. Deterministic output
|
|
// order (document order within each type, types in fixed order), duplicates
|
|
// removed.
|
|
func FromText(text string) []Fact {
|
|
e := &extractor{seen: map[string]bool{}}
|
|
e.extract(collectLines(text))
|
|
out := e.facts
|
|
if len(out) > MaxFacts {
|
|
out = out[:MaxFacts]
|
|
}
|
|
return out
|
|
}
|
|
|
|
type extractor struct {
|
|
facts []Fact
|
|
seen map[string]bool
|
|
}
|
|
|
|
func (e *extractor) add(f Fact) {
|
|
if f.Confidence < 0 {
|
|
f.Confidence = 0
|
|
}
|
|
if f.Confidence > 1 {
|
|
f.Confidence = 1
|
|
}
|
|
key := string(f.Type) + "\x00" + f.ValueText + "\x00" + f.ValueDate + "\x00" + fmt.Sprintf("%.2f", f.ValueNum)
|
|
if e.seen[key] {
|
|
return
|
|
}
|
|
e.seen[key] = true
|
|
e.facts = append(e.facts, f)
|
|
}
|
|
|
|
type line struct {
|
|
raw string
|
|
folded string
|
|
}
|
|
|
|
func collectLines(text string) []line {
|
|
raw := strings.Split(text, "\n")
|
|
out := make([]line, 0, len(raw))
|
|
for _, r := range raw {
|
|
r = strings.TrimSpace(strings.ReplaceAll(r, "\t", " "))
|
|
if r == "" {
|
|
continue
|
|
}
|
|
out = append(out, line{raw: r, folded: fold(r)})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (e *extractor) extract(lines []line) {
|
|
for _, l := range lines {
|
|
e.extractAmounts(l)
|
|
e.extractDates(l)
|
|
e.extractReferences(l)
|
|
}
|
|
e.extractIssuers(lines)
|
|
e.extractDocType(lines)
|
|
}
|
|
|
|
// --- amounts ---------------------------------------------------------------
|
|
|
|
// Note : pas de `\b` après `€` — c'est un symbole (non-mot), donc il n'y a
|
|
// jamais de frontière avec un espace ou une fin de ligne. `eur` garde son
|
|
// `\b` (évite "europe"), `euros?` n'en a pas besoin.
|
|
var amountPattern = regexp.MustCompile(`(\d[\d\s\x{a0}\x{202f}]*[,\.]\d{2}|\d[\d\s\x{a0}\x{202f}]*)\s*(€|euros?|eur\b)`)
|
|
|
|
func (e *extractor) extractAmounts(l line) {
|
|
// Percentages are not amounts.
|
|
if strings.Contains(l.raw, "%") && !strings.ContainsAny(l.raw, "€") {
|
|
return
|
|
}
|
|
for _, m := range amountPattern.FindAllStringSubmatch(l.raw, -1) {
|
|
num, ok := parseFrenchNumber(m[1])
|
|
if !ok || num <= 0 || num >= 1e13 {
|
|
continue
|
|
}
|
|
conf := 0.7
|
|
if amountKeywordPattern.MatchString(l.folded) {
|
|
conf = 0.9
|
|
}
|
|
e.add(Fact{Type: FactAmount, ValueNum: num, HasNum: true, ValueText: formatAmount(num), Confidence: conf})
|
|
}
|
|
}
|
|
|
|
var amountKeywordPattern = regexp.MustCompile(`\b(total|montant|a payer|net|ttc|solde|du)\b`)
|
|
|
|
func formatAmount(num float64) string {
|
|
return strconv.FormatFloat(num, 'f', 2, 64) + " €"
|
|
}
|
|
|
|
// parseFrenchNumber reads "1 234,56" / "1234.56" / "84 €"-style magnitudes.
|
|
// Comma = decimal separator (French), dots/spaces (incl. NBSP U+00A0 and
|
|
// narrow NBSP U+202F, common in "1 234 €") = thousands.
|
|
func parseFrenchNumber(s string) (float64, bool) {
|
|
s = strings.ReplaceAll(s, "\u00a0", "")
|
|
s = strings.ReplaceAll(s, "\u202f", "")
|
|
s = strings.ReplaceAll(s, " ", "")
|
|
if strings.Contains(s, ",") {
|
|
s = strings.ReplaceAll(s, ".", "")
|
|
s = strings.ReplaceAll(s, ",", ".")
|
|
}
|
|
num, err := strconv.ParseFloat(s, 64)
|
|
return num, err == nil
|
|
}
|
|
|
|
// --- dates -----------------------------------------------------------------
|
|
|
|
var numericDatePattern = regexp.MustCompile(`\b(\d{1,2})\s*[/\-.]\s*(\d{1,2})\s*[/\-.]\s*(\d{2}|\d{4})\b`)
|
|
|
|
var writtenDatePattern = regexp.MustCompile(`\b(\d{1,2})\s+(janv(?:ier)?|fev(?:rier)?|mars|avr(?:il)?|mai|juin|juil(?:let)?|aout|sept?(?:embre)?|oct(?:obre)?|nov(?:embre)?|dec(?:embre)?)\s+(\d{4})\b`)
|
|
|
|
var periodPattern = regexp.MustCompile(`\b(janv(?:ier)?|fev(?:rier)?|mars|avr(?:il)?|mai|juin|juil(?:let)?|aout|sept?(?:embre)?|oct(?:obre)?|nov(?:embre)?|dec(?:embre)?)\s+(\d{4})\b`)
|
|
|
|
var monthNumbers = map[string]int{
|
|
"janv": 1, "janvier": 1, "fev": 2, "fevrier": 2, "mars": 3,
|
|
"avr": 4, "avril": 4, "mai": 5, "juin": 6, "juil": 7,
|
|
"juillet": 7, "aout": 8, "sept": 9, "septembre": 9, "oct": 10,
|
|
"octobre": 10, "nov": 11, "novembre": 11, "dec": 12, "decembre": 12,
|
|
}
|
|
|
|
var dueKeywordPattern = regexp.MustCompile(`\b(echeance|a payer avant|date limite|avant le|regler avant|paiement avant)\b`)
|
|
|
|
func fullYear(yy int) int {
|
|
if yy < 100 {
|
|
if yy >= 70 {
|
|
return 1900 + yy
|
|
}
|
|
return 2000 + yy
|
|
}
|
|
return yy
|
|
}
|
|
|
|
func isoDate(y, m, d int) (string, bool) {
|
|
if m < 1 || m > 12 || d < 1 || d > 31 || y < 1900 || y > 2100 {
|
|
return "", false
|
|
}
|
|
return fmt.Sprintf("%04d-%02d-%02d", y, m, d), true
|
|
}
|
|
|
|
func (e *extractor) extractDates(l line) {
|
|
isDue := dueKeywordPattern.MatchString(l.folded)
|
|
for _, m := range numericDatePattern.FindAllStringSubmatch(l.raw, -1) {
|
|
d, _ := strconv.Atoi(m[1])
|
|
mo, _ := strconv.Atoi(m[2])
|
|
y, _ := strconv.Atoi(m[3])
|
|
iso, ok := isoDate(fullYear(y), mo, d)
|
|
if !ok {
|
|
continue
|
|
}
|
|
typ := FactDate
|
|
conf := 0.6
|
|
if isDue {
|
|
typ, conf = FactDueDate, 0.8
|
|
}
|
|
e.add(Fact{Type: typ, ValueDate: iso, ValueText: iso, Confidence: conf})
|
|
}
|
|
for _, m := range writtenDatePattern.FindAllStringSubmatch(l.folded, -1) {
|
|
d, _ := strconv.Atoi(m[1])
|
|
mo := monthNumbers[m[2]]
|
|
y, _ := strconv.Atoi(m[3])
|
|
iso, ok := isoDate(y, mo, d)
|
|
if !ok {
|
|
continue
|
|
}
|
|
typ := FactDate
|
|
conf := 0.7
|
|
if isDue {
|
|
typ, conf = FactDueDate, 0.85
|
|
}
|
|
e.add(Fact{Type: typ, ValueDate: iso, ValueText: iso, Confidence: conf})
|
|
}
|
|
for _, m := range periodPattern.FindAllStringSubmatch(l.folded, -1) {
|
|
mo := monthNumbers[m[1]]
|
|
y, _ := strconv.Atoi(m[2])
|
|
if mo == 0 || y < 1900 || y > 2100 {
|
|
continue
|
|
}
|
|
e.add(Fact{Type: FactPeriod, ValueText: fmt.Sprintf("%04d-%02d", y, mo), Confidence: 0.7})
|
|
}
|
|
}
|
|
|
|
// --- references ------------------------------------------------------------
|
|
|
|
// referencePattern catches "facture n° FA-2026-0031", "contrat : ABC123",
|
|
// "client 42" — the code must contain a digit (avoids "FACTURE DU MOIS").
|
|
var referencePattern = regexp.MustCompile(`(?i)\b(facture|contrat|client|compte|dossier|bulletin|commande|devis|attestation|police)\s*(n°|no|num(?:ero)?|number|ref(?:erence)?)?\s*[:\-#]?\s*([A-Z0-9][A-Z0-9\-/\.]{1,29})`)
|
|
|
|
var bareRefPattern = regexp.MustCompile(`\bn°\s*([A-Z0-9][A-Z0-9\-/]{2,20})`)
|
|
|
|
var siretPattern = regexp.MustCompile(`(?i)\b(siret|siren|rcs)\b[^\d]{0,10}(\d{3}\s?\d{3}\s?\d{3}(?:\s?\d{5})?)`)
|
|
|
|
var ibanPattern = regexp.MustCompile(`\bFR\s?\d{2}(?:\s?[0-9A-Z]{4}){5}\s?[0-9A-Z]{0,3}\b`)
|
|
|
|
func hasDigit(s string) bool {
|
|
for _, r := range s {
|
|
if r >= '0' && r <= '9' {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (e *extractor) extractReferences(l line) {
|
|
for _, m := range referencePattern.FindAllStringSubmatch(l.raw, -1) {
|
|
code := strings.Trim(m[3], "-/.")
|
|
if !hasDigit(code) || len(code) < 3 {
|
|
continue
|
|
}
|
|
kind := strings.ToLower(m[1])
|
|
e.add(Fact{Type: FactReference, ValueText: kind + ":" + code, Confidence: 0.75})
|
|
}
|
|
for _, m := range bareRefPattern.FindAllStringSubmatch(l.raw, -1) {
|
|
code := m[1]
|
|
if !hasDigit(code) {
|
|
continue
|
|
}
|
|
e.add(Fact{Type: FactReference, ValueText: "ref:" + code, Confidence: 0.6})
|
|
}
|
|
if m := siretPattern.FindStringSubmatch(l.raw); m != nil {
|
|
digits := strings.Join(strings.Fields(m[2]), " ")
|
|
e.add(Fact{Type: FactReference, ValueText: strings.ToLower(m[1]) + ":" + digits, Confidence: 0.9})
|
|
}
|
|
if m := ibanPattern.FindString(l.raw); m != "" {
|
|
compact := strings.ReplaceAll(m, " ", "")
|
|
if len(compact) == 27 {
|
|
// Store masked (privacy) : only the last 4 chars.
|
|
e.add(Fact{Type: FactReference, ValueText: "rib:••••" + compact[len(compact)-4:], Confidence: 0.9})
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- issuers ---------------------------------------------------------------
|
|
|
|
var companySuffixPattern = regexp.MustCompile(`(?i)\b(sarl|sas|sa|eurl|sasu|sci|eirl|snc|sasau)\b`)
|
|
|
|
func isCapsLine(s string) bool {
|
|
letters := 0
|
|
for _, r := range s {
|
|
if unicode.IsLetter(r) {
|
|
letters++
|
|
if unicode.IsLower(r) {
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
return letters >= 3
|
|
}
|
|
|
|
func (e *extractor) extractIssuers(lines []line) {
|
|
found := 0
|
|
checked := 0
|
|
for _, l := range lines {
|
|
if checked >= 8 || found >= 2 {
|
|
break
|
|
}
|
|
raw := strings.TrimSpace(l.raw)
|
|
if len([]rune(raw)) < 3 || len([]rune(raw)) > 60 {
|
|
continue
|
|
}
|
|
if strings.Contains(raw, "@") || strings.HasPrefix(strings.ToLower(raw), "http") {
|
|
continue
|
|
}
|
|
checked++
|
|
if !isCapsLine(raw) {
|
|
continue
|
|
}
|
|
// Skip lines that are really dates/amounts/references in disguise.
|
|
if numericDatePattern.MatchString(raw) || amountPattern.MatchString(raw) || bareRefPattern.MatchString(raw) {
|
|
continue
|
|
}
|
|
conf := 0.45
|
|
if found == 0 {
|
|
conf = 0.55
|
|
}
|
|
if companySuffixPattern.MatchString(raw) {
|
|
conf = 0.75
|
|
}
|
|
e.add(Fact{Type: FactIssuer, ValueText: raw, Confidence: conf})
|
|
found++
|
|
}
|
|
}
|
|
|
|
// --- doc_type suggestion ---------------------------------------------------
|
|
|
|
// docTypeKeywords maps a suggested type to weighted French keywords (folded).
|
|
// A type is emitted when its score reaches docTypeThreshold — the type is a
|
|
// humble suggestion (S3-correctable), never a gate.
|
|
var docTypeKeywords = []struct {
|
|
typ string
|
|
keywords map[string]int
|
|
}{
|
|
{"facture", map[string]int{"facture": 2, "tva": 1, "montant ttc": 2, "total ttc": 2, "a payer": 1, "devis": 1}},
|
|
{"bulletin", map[string]int{"bulletin": 2, "salaire": 2, "paie": 1, "net social": 2, "brut": 1}},
|
|
{"impot", map[string]int{"impot": 2, "revenu fiscal": 2, "taxe fonciere": 2, "taxe d habitation": 1, "avis d impot": 2}},
|
|
{"contrat", map[string]int{"contrat": 2, "bail": 2, "convention": 1, "article ": 1, "assurance": 1}},
|
|
{"ordonnance", map[string]int{"ordonnance": 2, "posologie": 2, "docteur": 1, "traitement": 1}},
|
|
{"attestation", map[string]int{"attestation": 2, "certifie": 1, "certificat": 1, "atteste": 1}},
|
|
{"rib", map[string]int{"rib": 2, "iban": 2, "bic": 1}},
|
|
{"identite", map[string]int{"carte d identite": 2, "passeport": 2, "permis de conduire": 2, "nationalite": 1}},
|
|
}
|
|
|
|
const docTypeThreshold = 2
|
|
|
|
func (e *extractor) extractDocType(lines []line) {
|
|
joined := "\n"
|
|
for _, l := range lines {
|
|
joined += l.folded + "\n"
|
|
}
|
|
best, bestScore := "", 0
|
|
for _, dt := range docTypeKeywords {
|
|
score := 0
|
|
for kw, w := range dt.keywords {
|
|
if strings.Contains(joined, kw) {
|
|
score += w
|
|
}
|
|
}
|
|
if score > bestScore {
|
|
best, bestScore = dt.typ, score
|
|
}
|
|
}
|
|
if bestScore >= docTypeThreshold {
|
|
conf := 0.5 + 0.1*float64(bestScore)
|
|
if conf > 0.85 {
|
|
conf = 0.85
|
|
}
|
|
e.add(Fact{Type: FactDocType, ValueText: best, Confidence: conf})
|
|
}
|
|
}
|
|
|
|
// --- folding ---------------------------------------------------------------
|
|
|
|
// fold lowercases French text and strips accents (œ→oe, æ→ae, ç→c) for
|
|
// keyword matching. stdlib-only (no x/text dependency churn).
|
|
func fold(s string) string {
|
|
var b strings.Builder
|
|
b.Grow(len(s))
|
|
for _, r := range strings.ToLower(s) {
|
|
switch r {
|
|
case 'à', 'â', 'ä':
|
|
b.WriteRune('a')
|
|
case 'é', 'è', 'ê', 'ë':
|
|
b.WriteRune('e')
|
|
case 'î', 'ï':
|
|
b.WriteRune('i')
|
|
case 'ô', 'ö':
|
|
b.WriteRune('o')
|
|
case 'ù', 'û', 'ü':
|
|
b.WriteRune('u')
|
|
case 'ÿ':
|
|
b.WriteRune('y')
|
|
case 'ç':
|
|
b.WriteRune('c')
|
|
case 'œ':
|
|
b.WriteString("oe")
|
|
case 'æ':
|
|
b.WriteString("ae")
|
|
default:
|
|
b.WriteRune(r)
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|