// Package extract derives generic, verifiable facts from OCR text (V3 S2). // // Rules-based French heuristics, zero ML : every fact carries a confidence, // low-confidence facts are suggested (greyed in UI) and user-correctable (S3), // and search demotes but never excludes on them. No LLM, no network, pure // stdlib — it runs inside the OCR worker right after Tesseract. package extract import ( "fmt" "regexp" "strconv" "strings" "unicode" ) // Fact types — mirror of the resource_facts CHECK (migration 000019). type FactType string const ( FactDate FactType = "date" FactDueDate FactType = "due_date" FactPeriod FactType = "period" FactAmount FactType = "amount" FactIssuer FactType = "issuer" FactReference FactType = "reference" FactDocType FactType = "doc_type" ) // Fact is one extracted statement. ValueText holds issuer names, reference // codes ("facture:FA-2026-0031"), doc types and periods ("2026-03") ; // ValueNum holds amounts (HasNum) ; ValueDate holds ISO dates ("2026-03-12"). type Fact struct { Type FactType ValueText string ValueNum float64 HasNum bool ValueDate string Confidence float64 } // MaxFacts caps the output per resource (OCR garbage must stay bounded). const MaxFacts = 50 // FromText extracts generic facts from raw OCR text. Deterministic output // order (document order within each type, types in fixed order), duplicates // removed. func FromText(text string) []Fact { e := &extractor{seen: map[string]bool{}} e.extract(collectLines(text)) out := e.facts if len(out) > MaxFacts { out = out[:MaxFacts] } return out } type extractor struct { facts []Fact seen map[string]bool } func (e *extractor) add(f Fact) { if f.Confidence < 0 { f.Confidence = 0 } if f.Confidence > 1 { f.Confidence = 1 } key := string(f.Type) + "\x00" + f.ValueText + "\x00" + f.ValueDate + "\x00" + fmt.Sprintf("%.2f", f.ValueNum) if e.seen[key] { return } e.seen[key] = true e.facts = append(e.facts, f) } type line struct { raw string folded string } func collectLines(text string) []line { raw := strings.Split(text, "\n") out := make([]line, 0, len(raw)) for _, r := range raw { r = strings.TrimSpace(strings.ReplaceAll(r, "\t", " ")) if r == "" { continue } out = append(out, line{raw: r, folded: fold(r)}) } return out } func (e *extractor) extract(lines []line) { for _, l := range lines { e.extractAmounts(l) e.extractDates(l) e.extractReferences(l) } e.extractIssuers(lines) e.extractDocType(lines) } // --- amounts --------------------------------------------------------------- // Note : pas de `\b` après `€` — c'est un symbole (non-mot), donc il n'y a // jamais de frontière avec un espace ou une fin de ligne. `eur` garde son // `\b` (évite "europe"), `euros?` n'en a pas besoin. var amountPattern = regexp.MustCompile(`(\d[\d\s\x{a0}\x{202f}]*[,\.]\d{2}|\d[\d\s\x{a0}\x{202f}]*)\s*(€|euros?|eur\b)`) func (e *extractor) extractAmounts(l line) { // Percentages are not amounts. if strings.Contains(l.raw, "%") && !strings.ContainsAny(l.raw, "€") { return } for _, m := range amountPattern.FindAllStringSubmatch(l.raw, -1) { num, ok := parseFrenchNumber(m[1]) if !ok || num <= 0 || num >= 1e13 { continue } conf := 0.7 if amountKeywordPattern.MatchString(l.folded) { conf = 0.9 } e.add(Fact{Type: FactAmount, ValueNum: num, HasNum: true, ValueText: formatAmount(num), Confidence: conf}) } } var amountKeywordPattern = regexp.MustCompile(`\b(total|montant|a payer|net|ttc|solde|du)\b`) func formatAmount(num float64) string { return strconv.FormatFloat(num, 'f', 2, 64) + " €" } // parseFrenchNumber reads "1 234,56" / "1234.56" / "84 €"-style magnitudes. // Comma = decimal separator (French), dots/spaces (incl. NBSP U+00A0 and // narrow NBSP U+202F, common in "1 234 €") = thousands. func parseFrenchNumber(s string) (float64, bool) { s = strings.ReplaceAll(s, "\u00a0", "") s = strings.ReplaceAll(s, "\u202f", "") s = strings.ReplaceAll(s, " ", "") if strings.Contains(s, ",") { s = strings.ReplaceAll(s, ".", "") s = strings.ReplaceAll(s, ",", ".") } num, err := strconv.ParseFloat(s, 64) return num, err == nil } // --- dates ----------------------------------------------------------------- var numericDatePattern = regexp.MustCompile(`\b(\d{1,2})\s*[/\-.]\s*(\d{1,2})\s*[/\-.]\s*(\d{2}|\d{4})\b`) var writtenDatePattern = regexp.MustCompile(`\b(\d{1,2})\s+(janv(?:ier)?|fev(?:rier)?|mars|avr(?:il)?|mai|juin|juil(?:let)?|aout|sept?(?:embre)?|oct(?:obre)?|nov(?:embre)?|dec(?:embre)?)\s+(\d{4})\b`) var periodPattern = regexp.MustCompile(`\b(janv(?:ier)?|fev(?:rier)?|mars|avr(?:il)?|mai|juin|juil(?:let)?|aout|sept?(?:embre)?|oct(?:obre)?|nov(?:embre)?|dec(?:embre)?)\s+(\d{4})\b`) var monthNumbers = map[string]int{ "janv": 1, "janvier": 1, "fev": 2, "fevrier": 2, "mars": 3, "avr": 4, "avril": 4, "mai": 5, "juin": 6, "juil": 7, "juillet": 7, "aout": 8, "sept": 9, "septembre": 9, "oct": 10, "octobre": 10, "nov": 11, "novembre": 11, "dec": 12, "decembre": 12, } var dueKeywordPattern = regexp.MustCompile(`\b(echeance|a payer avant|date limite|avant le|regler avant|paiement avant)\b`) func fullYear(yy int) int { if yy < 100 { if yy >= 70 { return 1900 + yy } return 2000 + yy } return yy } func isoDate(y, m, d int) (string, bool) { if m < 1 || m > 12 || d < 1 || d > 31 || y < 1900 || y > 2100 { return "", false } return fmt.Sprintf("%04d-%02d-%02d", y, m, d), true } func (e *extractor) extractDates(l line) { isDue := dueKeywordPattern.MatchString(l.folded) for _, m := range numericDatePattern.FindAllStringSubmatch(l.raw, -1) { d, _ := strconv.Atoi(m[1]) mo, _ := strconv.Atoi(m[2]) y, _ := strconv.Atoi(m[3]) iso, ok := isoDate(fullYear(y), mo, d) if !ok { continue } typ := FactDate conf := 0.6 if isDue { typ, conf = FactDueDate, 0.8 } e.add(Fact{Type: typ, ValueDate: iso, ValueText: iso, Confidence: conf}) } for _, m := range writtenDatePattern.FindAllStringSubmatch(l.folded, -1) { d, _ := strconv.Atoi(m[1]) mo := monthNumbers[m[2]] y, _ := strconv.Atoi(m[3]) iso, ok := isoDate(y, mo, d) if !ok { continue } typ := FactDate conf := 0.7 if isDue { typ, conf = FactDueDate, 0.85 } e.add(Fact{Type: typ, ValueDate: iso, ValueText: iso, Confidence: conf}) } for _, m := range periodPattern.FindAllStringSubmatch(l.folded, -1) { mo := monthNumbers[m[1]] y, _ := strconv.Atoi(m[2]) if mo == 0 || y < 1900 || y > 2100 { continue } e.add(Fact{Type: FactPeriod, ValueText: fmt.Sprintf("%04d-%02d", y, mo), Confidence: 0.7}) } } // --- references ------------------------------------------------------------ // referencePattern catches "facture n° FA-2026-0031", "contrat : ABC123", // "client 42" — the code must contain a digit (avoids "FACTURE DU MOIS"). var referencePattern = regexp.MustCompile(`(?i)\b(facture|contrat|client|compte|dossier|bulletin|commande|devis|attestation|police)\s*(n°|no|num(?:ero)?|number|ref(?:erence)?)?\s*[:\-#]?\s*([A-Z0-9][A-Z0-9\-/\.]{1,29})`) var bareRefPattern = regexp.MustCompile(`\bn°\s*([A-Z0-9][A-Z0-9\-/]{2,20})`) var siretPattern = regexp.MustCompile(`(?i)\b(siret|siren|rcs)\b[^\d]{0,10}(\d{3}\s?\d{3}\s?\d{3}(?:\s?\d{5})?)`) var ibanPattern = regexp.MustCompile(`\bFR\s?\d{2}(?:\s?[0-9A-Z]{4}){5}\s?[0-9A-Z]{0,3}\b`) func hasDigit(s string) bool { for _, r := range s { if r >= '0' && r <= '9' { return true } } return false } func (e *extractor) extractReferences(l line) { for _, m := range referencePattern.FindAllStringSubmatch(l.raw, -1) { code := strings.Trim(m[3], "-/.") if !hasDigit(code) || len(code) < 3 { continue } kind := strings.ToLower(m[1]) e.add(Fact{Type: FactReference, ValueText: kind + ":" + code, Confidence: 0.75}) } for _, m := range bareRefPattern.FindAllStringSubmatch(l.raw, -1) { code := m[1] if !hasDigit(code) { continue } e.add(Fact{Type: FactReference, ValueText: "ref:" + code, Confidence: 0.6}) } if m := siretPattern.FindStringSubmatch(l.raw); m != nil { digits := strings.Join(strings.Fields(m[2]), " ") e.add(Fact{Type: FactReference, ValueText: strings.ToLower(m[1]) + ":" + digits, Confidence: 0.9}) } if m := ibanPattern.FindString(l.raw); m != "" { compact := strings.ReplaceAll(m, " ", "") if len(compact) == 27 { // Store masked (privacy) : only the last 4 chars. e.add(Fact{Type: FactReference, ValueText: "rib:••••" + compact[len(compact)-4:], Confidence: 0.9}) } } } // --- issuers --------------------------------------------------------------- var companySuffixPattern = regexp.MustCompile(`(?i)\b(sarl|sas|sa|eurl|sasu|sci|eirl|snc|sasau)\b`) func isCapsLine(s string) bool { letters := 0 for _, r := range s { if unicode.IsLetter(r) { letters++ if unicode.IsLower(r) { return false } } } return letters >= 3 } func (e *extractor) extractIssuers(lines []line) { found := 0 checked := 0 for _, l := range lines { if checked >= 8 || found >= 2 { break } raw := strings.TrimSpace(l.raw) if len([]rune(raw)) < 3 || len([]rune(raw)) > 60 { continue } if strings.Contains(raw, "@") || strings.HasPrefix(strings.ToLower(raw), "http") { continue } checked++ if !isCapsLine(raw) { continue } // Skip lines that are really dates/amounts/references in disguise. if numericDatePattern.MatchString(raw) || amountPattern.MatchString(raw) || bareRefPattern.MatchString(raw) { continue } conf := 0.45 if found == 0 { conf = 0.55 } if companySuffixPattern.MatchString(raw) { conf = 0.75 } e.add(Fact{Type: FactIssuer, ValueText: raw, Confidence: conf}) found++ } } // --- doc_type suggestion --------------------------------------------------- // docTypeKeywords maps a suggested type to weighted French keywords (folded). // A type is emitted when its score reaches docTypeThreshold — the type is a // humble suggestion (S3-correctable), never a gate. var docTypeKeywords = []struct { typ string keywords map[string]int }{ {"facture", map[string]int{"facture": 2, "tva": 1, "montant ttc": 2, "total ttc": 2, "a payer": 1, "devis": 1}}, {"bulletin", map[string]int{"bulletin": 2, "salaire": 2, "paie": 1, "net social": 2, "brut": 1}}, {"impot", map[string]int{"impot": 2, "revenu fiscal": 2, "taxe fonciere": 2, "taxe d habitation": 1, "avis d impot": 2}}, {"contrat", map[string]int{"contrat": 2, "bail": 2, "convention": 1, "article ": 1, "assurance": 1}}, {"ordonnance", map[string]int{"ordonnance": 2, "posologie": 2, "docteur": 1, "traitement": 1}}, {"attestation", map[string]int{"attestation": 2, "certifie": 1, "certificat": 1, "atteste": 1}}, {"rib", map[string]int{"rib": 2, "iban": 2, "bic": 1}}, {"identite", map[string]int{"carte d identite": 2, "passeport": 2, "permis de conduire": 2, "nationalite": 1}}, } const docTypeThreshold = 2 func (e *extractor) extractDocType(lines []line) { joined := "\n" for _, l := range lines { joined += l.folded + "\n" } best, bestScore := "", 0 for _, dt := range docTypeKeywords { score := 0 for kw, w := range dt.keywords { if strings.Contains(joined, kw) { score += w } } if score > bestScore { best, bestScore = dt.typ, score } } if bestScore >= docTypeThreshold { conf := 0.5 + 0.1*float64(bestScore) if conf > 0.85 { conf = 0.85 } e.add(Fact{Type: FactDocType, ValueText: best, Confidence: conf}) } } // --- folding --------------------------------------------------------------- // fold lowercases French text and strips accents (œ→oe, æ→ae, ç→c) for // keyword matching. stdlib-only (no x/text dependency churn). func fold(s string) string { var b strings.Builder b.Grow(len(s)) for _, r := range strings.ToLower(s) { switch r { case 'à', 'â', 'ä': b.WriteRune('a') case 'é', 'è', 'ê', 'ë': b.WriteRune('e') case 'î', 'ï': b.WriteRune('i') case 'ô', 'ö': b.WriteRune('o') case 'ù', 'û', 'ü': b.WriteRune('u') case 'ÿ': b.WriteRune('y') case 'ç': b.WriteRune('c') case 'œ': b.WriteString("oe") case 'æ': b.WriteString("ae") default: b.WriteRune(r) } } return b.String() }