V3 S1-S2 : corbeille + versions + extract générique
S1 filet (backend + mobile) : corbeille 30j (deleted_at réutilisé, trash/restore récursifs, purge auto serveur), versions N=3 à chaque overwrite, ops outbox trash/restore/purge (delete legacy mappé), Room v19 (trashed_at), Deleters -> trash, écran Corbeille, walk guards. S2 extract (backend) : package extract (règles FR stdlib, corpus testé), resource_facts 000019 (origin auto/user), hook worker OCR + backfill boot, GET /files/:id/facts (viewer+). Contrat docs/api-v1.md étendu, V3.md (vision + plan).
This commit is contained in:
@@ -0,0 +1,408 @@
|
||||
// Package extract derives generic, verifiable facts from OCR text (V3 S2).
|
||||
//
|
||||
// Rules-based French heuristics, zero ML : every fact carries a confidence,
|
||||
// low-confidence facts are suggested (greyed in UI) and user-correctable (S3),
|
||||
// and search demotes but never excludes on them. No LLM, no network, pure
|
||||
// stdlib — it runs inside the OCR worker right after Tesseract.
|
||||
package extract
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Fact types — mirror of the resource_facts CHECK (migration 000019).
|
||||
type FactType string
|
||||
|
||||
const (
|
||||
FactDate FactType = "date"
|
||||
FactDueDate FactType = "due_date"
|
||||
FactPeriod FactType = "period"
|
||||
FactAmount FactType = "amount"
|
||||
FactIssuer FactType = "issuer"
|
||||
FactReference FactType = "reference"
|
||||
FactDocType FactType = "doc_type"
|
||||
)
|
||||
|
||||
// Fact is one extracted statement. ValueText holds issuer names, reference
|
||||
// codes ("facture:FA-2026-0031"), doc types and periods ("2026-03") ;
|
||||
// ValueNum holds amounts (HasNum) ; ValueDate holds ISO dates ("2026-03-12").
|
||||
type Fact struct {
|
||||
Type FactType
|
||||
ValueText string
|
||||
ValueNum float64
|
||||
HasNum bool
|
||||
ValueDate string
|
||||
Confidence float64
|
||||
}
|
||||
|
||||
// MaxFacts caps the output per resource (OCR garbage must stay bounded).
|
||||
const MaxFacts = 50
|
||||
|
||||
// FromText extracts generic facts from raw OCR text. Deterministic output
|
||||
// order (document order within each type, types in fixed order), duplicates
|
||||
// removed.
|
||||
func FromText(text string) []Fact {
|
||||
e := &extractor{seen: map[string]bool{}}
|
||||
e.extract(collectLines(text))
|
||||
out := e.facts
|
||||
if len(out) > MaxFacts {
|
||||
out = out[:MaxFacts]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type extractor struct {
|
||||
facts []Fact
|
||||
seen map[string]bool
|
||||
}
|
||||
|
||||
func (e *extractor) add(f Fact) {
|
||||
if f.Confidence < 0 {
|
||||
f.Confidence = 0
|
||||
}
|
||||
if f.Confidence > 1 {
|
||||
f.Confidence = 1
|
||||
}
|
||||
key := string(f.Type) + "\x00" + f.ValueText + "\x00" + f.ValueDate + "\x00" + fmt.Sprintf("%.2f", f.ValueNum)
|
||||
if e.seen[key] {
|
||||
return
|
||||
}
|
||||
e.seen[key] = true
|
||||
e.facts = append(e.facts, f)
|
||||
}
|
||||
|
||||
type line struct {
|
||||
raw string
|
||||
folded string
|
||||
}
|
||||
|
||||
func collectLines(text string) []line {
|
||||
raw := strings.Split(text, "\n")
|
||||
out := make([]line, 0, len(raw))
|
||||
for _, r := range raw {
|
||||
r = strings.TrimSpace(strings.ReplaceAll(r, "\t", " "))
|
||||
if r == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, line{raw: r, folded: fold(r)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (e *extractor) extract(lines []line) {
|
||||
for _, l := range lines {
|
||||
e.extractAmounts(l)
|
||||
e.extractDates(l)
|
||||
e.extractReferences(l)
|
||||
}
|
||||
e.extractIssuers(lines)
|
||||
e.extractDocType(lines)
|
||||
}
|
||||
|
||||
// --- amounts ---------------------------------------------------------------
|
||||
|
||||
// Note : pas de `\b` après `€` — c'est un symbole (non-mot), donc il n'y a
|
||||
// jamais de frontière avec un espace ou une fin de ligne. `eur` garde son
|
||||
// `\b` (évite "europe"), `euros?` n'en a pas besoin.
|
||||
var amountPattern = regexp.MustCompile(`(\d[\d\s\x{a0}\x{202f}]*[,\.]\d{2}|\d[\d\s\x{a0}\x{202f}]*)\s*(€|euros?|eur\b)`)
|
||||
|
||||
func (e *extractor) extractAmounts(l line) {
|
||||
// Percentages are not amounts.
|
||||
if strings.Contains(l.raw, "%") && !strings.ContainsAny(l.raw, "€") {
|
||||
return
|
||||
}
|
||||
for _, m := range amountPattern.FindAllStringSubmatch(l.raw, -1) {
|
||||
num, ok := parseFrenchNumber(m[1])
|
||||
if !ok || num <= 0 || num >= 1e13 {
|
||||
continue
|
||||
}
|
||||
conf := 0.7
|
||||
if amountKeywordPattern.MatchString(l.folded) {
|
||||
conf = 0.9
|
||||
}
|
||||
e.add(Fact{Type: FactAmount, ValueNum: num, HasNum: true, ValueText: formatAmount(num), Confidence: conf})
|
||||
}
|
||||
}
|
||||
|
||||
var amountKeywordPattern = regexp.MustCompile(`\b(total|montant|a payer|net|ttc|solde|du)\b`)
|
||||
|
||||
func formatAmount(num float64) string {
|
||||
return strconv.FormatFloat(num, 'f', 2, 64) + " €"
|
||||
}
|
||||
|
||||
// parseFrenchNumber reads "1 234,56" / "1234.56" / "84 €"-style magnitudes.
|
||||
// Comma = decimal separator (French), dots/spaces (incl. NBSP U+00A0 and
|
||||
// narrow NBSP U+202F, common in "1 234 €") = thousands.
|
||||
func parseFrenchNumber(s string) (float64, bool) {
|
||||
s = strings.ReplaceAll(s, "\u00a0", "")
|
||||
s = strings.ReplaceAll(s, "\u202f", "")
|
||||
s = strings.ReplaceAll(s, " ", "")
|
||||
if strings.Contains(s, ",") {
|
||||
s = strings.ReplaceAll(s, ".", "")
|
||||
s = strings.ReplaceAll(s, ",", ".")
|
||||
}
|
||||
num, err := strconv.ParseFloat(s, 64)
|
||||
return num, err == nil
|
||||
}
|
||||
|
||||
// --- dates -----------------------------------------------------------------
|
||||
|
||||
var numericDatePattern = regexp.MustCompile(`\b(\d{1,2})\s*[/\-.]\s*(\d{1,2})\s*[/\-.]\s*(\d{2}|\d{4})\b`)
|
||||
|
||||
var writtenDatePattern = regexp.MustCompile(`\b(\d{1,2})\s+(janv(?:ier)?|fev(?:rier)?|mars|avr(?:il)?|mai|juin|juil(?:let)?|aout|sept?(?:embre)?|oct(?:obre)?|nov(?:embre)?|dec(?:embre)?)\s+(\d{4})\b`)
|
||||
|
||||
var periodPattern = regexp.MustCompile(`\b(janv(?:ier)?|fev(?:rier)?|mars|avr(?:il)?|mai|juin|juil(?:let)?|aout|sept?(?:embre)?|oct(?:obre)?|nov(?:embre)?|dec(?:embre)?)\s+(\d{4})\b`)
|
||||
|
||||
var monthNumbers = map[string]int{
|
||||
"janv": 1, "janvier": 1, "fev": 2, "fevrier": 2, "mars": 3,
|
||||
"avr": 4, "avril": 4, "mai": 5, "juin": 6, "juil": 7,
|
||||
"juillet": 7, "aout": 8, "sept": 9, "septembre": 9, "oct": 10,
|
||||
"octobre": 10, "nov": 11, "novembre": 11, "dec": 12, "decembre": 12,
|
||||
}
|
||||
|
||||
var dueKeywordPattern = regexp.MustCompile(`\b(echeance|a payer avant|date limite|avant le|regler avant|paiement avant)\b`)
|
||||
|
||||
func fullYear(yy int) int {
|
||||
if yy < 100 {
|
||||
if yy >= 70 {
|
||||
return 1900 + yy
|
||||
}
|
||||
return 2000 + yy
|
||||
}
|
||||
return yy
|
||||
}
|
||||
|
||||
func isoDate(y, m, d int) (string, bool) {
|
||||
if m < 1 || m > 12 || d < 1 || d > 31 || y < 1900 || y > 2100 {
|
||||
return "", false
|
||||
}
|
||||
return fmt.Sprintf("%04d-%02d-%02d", y, m, d), true
|
||||
}
|
||||
|
||||
func (e *extractor) extractDates(l line) {
|
||||
isDue := dueKeywordPattern.MatchString(l.folded)
|
||||
for _, m := range numericDatePattern.FindAllStringSubmatch(l.raw, -1) {
|
||||
d, _ := strconv.Atoi(m[1])
|
||||
mo, _ := strconv.Atoi(m[2])
|
||||
y, _ := strconv.Atoi(m[3])
|
||||
iso, ok := isoDate(fullYear(y), mo, d)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
typ := FactDate
|
||||
conf := 0.6
|
||||
if isDue {
|
||||
typ, conf = FactDueDate, 0.8
|
||||
}
|
||||
e.add(Fact{Type: typ, ValueDate: iso, ValueText: iso, Confidence: conf})
|
||||
}
|
||||
for _, m := range writtenDatePattern.FindAllStringSubmatch(l.folded, -1) {
|
||||
d, _ := strconv.Atoi(m[1])
|
||||
mo := monthNumbers[m[2]]
|
||||
y, _ := strconv.Atoi(m[3])
|
||||
iso, ok := isoDate(y, mo, d)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
typ := FactDate
|
||||
conf := 0.7
|
||||
if isDue {
|
||||
typ, conf = FactDueDate, 0.85
|
||||
}
|
||||
e.add(Fact{Type: typ, ValueDate: iso, ValueText: iso, Confidence: conf})
|
||||
}
|
||||
for _, m := range periodPattern.FindAllStringSubmatch(l.folded, -1) {
|
||||
mo := monthNumbers[m[1]]
|
||||
y, _ := strconv.Atoi(m[2])
|
||||
if mo == 0 || y < 1900 || y > 2100 {
|
||||
continue
|
||||
}
|
||||
e.add(Fact{Type: FactPeriod, ValueText: fmt.Sprintf("%04d-%02d", y, mo), Confidence: 0.7})
|
||||
}
|
||||
}
|
||||
|
||||
// --- references ------------------------------------------------------------
|
||||
|
||||
// referencePattern catches "facture n° FA-2026-0031", "contrat : ABC123",
|
||||
// "client 42" — the code must contain a digit (avoids "FACTURE DU MOIS").
|
||||
var referencePattern = regexp.MustCompile(`(?i)\b(facture|contrat|client|compte|dossier|bulletin|commande|devis|attestation|police)\s*(n°|no|num(?:ero)?|number|ref(?:erence)?)?\s*[:\-#]?\s*([A-Z0-9][A-Z0-9\-/\.]{1,29})`)
|
||||
|
||||
var bareRefPattern = regexp.MustCompile(`\bn°\s*([A-Z0-9][A-Z0-9\-/]{2,20})`)
|
||||
|
||||
var siretPattern = regexp.MustCompile(`(?i)\b(siret|siren|rcs)\b[^\d]{0,10}(\d{3}\s?\d{3}\s?\d{3}(?:\s?\d{5})?)`)
|
||||
|
||||
var ibanPattern = regexp.MustCompile(`\bFR\s?\d{2}(?:\s?[0-9A-Z]{4}){5}\s?[0-9A-Z]{0,3}\b`)
|
||||
|
||||
func hasDigit(s string) bool {
|
||||
for _, r := range s {
|
||||
if r >= '0' && r <= '9' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (e *extractor) extractReferences(l line) {
|
||||
for _, m := range referencePattern.FindAllStringSubmatch(l.raw, -1) {
|
||||
code := strings.Trim(m[3], "-/.")
|
||||
if !hasDigit(code) || len(code) < 3 {
|
||||
continue
|
||||
}
|
||||
kind := strings.ToLower(m[1])
|
||||
e.add(Fact{Type: FactReference, ValueText: kind + ":" + code, Confidence: 0.75})
|
||||
}
|
||||
for _, m := range bareRefPattern.FindAllStringSubmatch(l.raw, -1) {
|
||||
code := m[1]
|
||||
if !hasDigit(code) {
|
||||
continue
|
||||
}
|
||||
e.add(Fact{Type: FactReference, ValueText: "ref:" + code, Confidence: 0.6})
|
||||
}
|
||||
if m := siretPattern.FindStringSubmatch(l.raw); m != nil {
|
||||
digits := strings.Join(strings.Fields(m[2]), " ")
|
||||
e.add(Fact{Type: FactReference, ValueText: strings.ToLower(m[1]) + ":" + digits, Confidence: 0.9})
|
||||
}
|
||||
if m := ibanPattern.FindString(l.raw); m != "" {
|
||||
compact := strings.ReplaceAll(m, " ", "")
|
||||
if len(compact) == 27 {
|
||||
// Store masked (privacy) : only the last 4 chars.
|
||||
e.add(Fact{Type: FactReference, ValueText: "rib:••••" + compact[len(compact)-4:], Confidence: 0.9})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- issuers ---------------------------------------------------------------
|
||||
|
||||
var companySuffixPattern = regexp.MustCompile(`(?i)\b(sarl|sas|sa|eurl|sasu|sci|eirl|snc|sasau)\b`)
|
||||
|
||||
func isCapsLine(s string) bool {
|
||||
letters := 0
|
||||
for _, r := range s {
|
||||
if unicode.IsLetter(r) {
|
||||
letters++
|
||||
if unicode.IsLower(r) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return letters >= 3
|
||||
}
|
||||
|
||||
func (e *extractor) extractIssuers(lines []line) {
|
||||
found := 0
|
||||
checked := 0
|
||||
for _, l := range lines {
|
||||
if checked >= 8 || found >= 2 {
|
||||
break
|
||||
}
|
||||
raw := strings.TrimSpace(l.raw)
|
||||
if len([]rune(raw)) < 3 || len([]rune(raw)) > 60 {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(raw, "@") || strings.HasPrefix(strings.ToLower(raw), "http") {
|
||||
continue
|
||||
}
|
||||
checked++
|
||||
if !isCapsLine(raw) {
|
||||
continue
|
||||
}
|
||||
// Skip lines that are really dates/amounts/references in disguise.
|
||||
if numericDatePattern.MatchString(raw) || amountPattern.MatchString(raw) || bareRefPattern.MatchString(raw) {
|
||||
continue
|
||||
}
|
||||
conf := 0.45
|
||||
if found == 0 {
|
||||
conf = 0.55
|
||||
}
|
||||
if companySuffixPattern.MatchString(raw) {
|
||||
conf = 0.75
|
||||
}
|
||||
e.add(Fact{Type: FactIssuer, ValueText: raw, Confidence: conf})
|
||||
found++
|
||||
}
|
||||
}
|
||||
|
||||
// --- doc_type suggestion ---------------------------------------------------
|
||||
|
||||
// docTypeKeywords maps a suggested type to weighted French keywords (folded).
|
||||
// A type is emitted when its score reaches docTypeThreshold — the type is a
|
||||
// humble suggestion (S3-correctable), never a gate.
|
||||
var docTypeKeywords = []struct {
|
||||
typ string
|
||||
keywords map[string]int
|
||||
}{
|
||||
{"facture", map[string]int{"facture": 2, "tva": 1, "montant ttc": 2, "total ttc": 2, "a payer": 1, "devis": 1}},
|
||||
{"bulletin", map[string]int{"bulletin": 2, "salaire": 2, "paie": 1, "net social": 2, "brut": 1}},
|
||||
{"impot", map[string]int{"impot": 2, "revenu fiscal": 2, "taxe fonciere": 2, "taxe d habitation": 1, "avis d impot": 2}},
|
||||
{"contrat", map[string]int{"contrat": 2, "bail": 2, "convention": 1, "article ": 1, "assurance": 1}},
|
||||
{"ordonnance", map[string]int{"ordonnance": 2, "posologie": 2, "docteur": 1, "traitement": 1}},
|
||||
{"attestation", map[string]int{"attestation": 2, "certifie": 1, "certificat": 1, "atteste": 1}},
|
||||
{"rib", map[string]int{"rib": 2, "iban": 2, "bic": 1}},
|
||||
{"identite", map[string]int{"carte d identite": 2, "passeport": 2, "permis de conduire": 2, "nationalite": 1}},
|
||||
}
|
||||
|
||||
const docTypeThreshold = 2
|
||||
|
||||
func (e *extractor) extractDocType(lines []line) {
|
||||
joined := "\n"
|
||||
for _, l := range lines {
|
||||
joined += l.folded + "\n"
|
||||
}
|
||||
best, bestScore := "", 0
|
||||
for _, dt := range docTypeKeywords {
|
||||
score := 0
|
||||
for kw, w := range dt.keywords {
|
||||
if strings.Contains(joined, kw) {
|
||||
score += w
|
||||
}
|
||||
}
|
||||
if score > bestScore {
|
||||
best, bestScore = dt.typ, score
|
||||
}
|
||||
}
|
||||
if bestScore >= docTypeThreshold {
|
||||
conf := 0.5 + 0.1*float64(bestScore)
|
||||
if conf > 0.85 {
|
||||
conf = 0.85
|
||||
}
|
||||
e.add(Fact{Type: FactDocType, ValueText: best, Confidence: conf})
|
||||
}
|
||||
}
|
||||
|
||||
// --- folding ---------------------------------------------------------------
|
||||
|
||||
// fold lowercases French text and strips accents (œ→oe, æ→ae, ç→c) for
|
||||
// keyword matching. stdlib-only (no x/text dependency churn).
|
||||
func fold(s string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(s))
|
||||
for _, r := range strings.ToLower(s) {
|
||||
switch r {
|
||||
case 'à', 'â', 'ä':
|
||||
b.WriteRune('a')
|
||||
case 'é', 'è', 'ê', 'ë':
|
||||
b.WriteRune('e')
|
||||
case 'î', 'ï':
|
||||
b.WriteRune('i')
|
||||
case 'ô', 'ö':
|
||||
b.WriteRune('o')
|
||||
case 'ù', 'û', 'ü':
|
||||
b.WriteRune('u')
|
||||
case 'ÿ':
|
||||
b.WriteRune('y')
|
||||
case 'ç':
|
||||
b.WriteRune('c')
|
||||
case 'œ':
|
||||
b.WriteString("oe")
|
||||
case 'æ':
|
||||
b.WriteString("ae")
|
||||
default:
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
Reference in New Issue
Block a user