V3 S4 : recherche FTS + facettes (serveur + mobile)

Backend : search_tsv french + GIN (000020), SearchDocumentsVisible
(nom + TS + faits, rang nom > TS > fait sûr > fait douteux, jamais
d'exclusion), facettes issuer/type/montant/période sur
GET /files/search (400 si malformée). Tests repo + handlers + migrations.

Mobile (Room v21) : FTS4 files_fts + triggers partagés
(fresh-install/migration, entité déclarée pour KSP, SQL aligné 21.json),
searchDocuments offline (même rang, accents préservés) + UI filtres
(émetteur, types chips, montants, période ISO, compteur, effacer).

Contrat docs/api-v1.md, V3.md (S4 livré, migrations renumérotées
links->000021, token->000022).
This commit is contained in:
venture
2026-09-26 10:53:56 +02:00
parent ac8bd9acb4
commit 2d984fd665
25 changed files with 2362 additions and 25 deletions
@@ -0,0 +1,2 @@
DROP INDEX IF EXISTS idx_resources_search_tsv;
ALTER TABLE resources DROP COLUMN IF EXISTS search_tsv;
@@ -0,0 +1,13 @@
-- V3 recherche (S4) : plein-texte français sur (nom + OCR).
-- Colonne générée stockée (`to_tsvector(regconfig, text)` est IMMUTABLE,
-- donc autorisée en GENERATED) : pas de trigger, backfill automatique des
-- lignes existantes. Index GIN. Les faits (`resource_facts`) sont joints à
-- la requête (jamais dupliqués) : ils pondèrent le rang sans jamais exclure
-- (humble par design — une confiance < 0.5 déclasse, cf. SearchDocuments).
ALTER TABLE resources
ADD COLUMN search_tsv TSVECTOR
GENERATED ALWAYS AS (
to_tsvector('french', COALESCE(name, '') || ' ' || COALESCE(ocr_text, ''))
) STORED;
CREATE INDEX IF NOT EXISTS idx_resources_search_tsv ON resources USING GIN (search_tsv);
+25
View File
@@ -422,6 +422,31 @@ func TestMigrationsUpDown(t *testing.T) {
t.Error("CHECK fact_type manquant sur resource_facts après 000019")
}
// --- 000020 : plein-texte ----------------------------------------------
var tsvCol int
err = conn.QueryRow(`
SELECT COUNT(*) FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = 'resources' AND column_name = 'search_tsv'`).Scan(&tsvCol)
if err != nil {
t.Fatalf("search_tsv column: %v", err)
}
if tsvCol == 0 {
t.Error("colonne resources.search_tsv manquante après 000020")
}
var tsvIdx int
err = conn.QueryRow(`
SELECT COUNT(*) FROM pg_index i
JOIN pg_class t ON t.oid = i.indrelid
WHERE t.relname = 'resources' AND i.indisvalid
AND pg_get_indexdef(i.indexrelid) LIKE '%idx_resources_search_tsv%'`).Scan(&tsvIdx)
if err != nil {
t.Fatalf("000020 index: %v", err)
}
if tsvIdx == 0 {
t.Error("index idx_resources_search_tsv manquant après 000020")
}
if err := MigrateDownDatabase(url); err != nil {
t.Fatalf("migrate down: %v", err)
}
+46 -1
View File
@@ -5,9 +5,11 @@ import (
"net/http"
"strconv"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/kazier/backend/pkg/api"
"github.com/kazier/backend/repository"
"github.com/kazier/backend/service"
)
@@ -408,10 +410,53 @@ func FilesSearch(c *gin.Context) {
if !ok {
return
}
files, total, err := Store.SearchFiles(userID, q, page, pageSize)
facets, ok := parseSearchFacets(c)
if !ok {
return
}
files, total, err := Store.SearchFiles(userID, q, page, pageSize, facets)
if err != nil {
writeError(c, err)
return
}
api.OKList(c, files, page, pageSize, total)
}
// parseSearchFacets lit les facettes V3 (`issuer`, `from`/`to` ISO,
// `min_amount`/`max_amount` numériques, `doc_type`) — 400 si malformées.
func parseSearchFacets(c *gin.Context) (repository.SearchFacets, bool) {
var facets repository.SearchFacets
facets.Issuer = strings.TrimSpace(c.Query("issuer"))
facets.DocType = strings.TrimSpace(c.Query("doc_type"))
if v := strings.TrimSpace(c.Query("from")); v != "" {
if _, err := time.Parse("2006-01-02", v); err != nil {
api.Error(c, http.StatusBadRequest, "INVALID_REQUEST", "query param `from` must be ISO YYYY-MM-DD")
return facets, false
}
facets.From = v
}
if v := strings.TrimSpace(c.Query("to")); v != "" {
if _, err := time.Parse("2006-01-02", v); err != nil {
api.Error(c, http.StatusBadRequest, "INVALID_REQUEST", "query param `to` must be ISO YYYY-MM-DD")
return facets, false
}
facets.To = v
}
if v := strings.TrimSpace(c.Query("min_amount")); v != "" {
f, err := strconv.ParseFloat(strings.ReplaceAll(v, ",", "."), 64)
if err != nil {
api.Error(c, http.StatusBadRequest, "INVALID_REQUEST", "query param `min_amount` must be numeric")
return facets, false
}
facets.MinAmount = &f
}
if v := strings.TrimSpace(c.Query("max_amount")); v != "" {
f, err := strconv.ParseFloat(strings.ReplaceAll(v, ",", "."), 64)
if err != nil {
api.Error(c, http.StatusBadRequest, "INVALID_REQUEST", "query param `max_amount` must be numeric")
return facets, false
}
facets.MaxAmount = &f
}
return facets, true
}
+77
View File
@@ -360,6 +360,83 @@ func TestSearchFiles(t *testing.T) {
}
}
// TestSearchFacets : filtres V3 sur faits + 400 si malformés.
func TestSearchFacets(t *testing.T) {
r, _, repo := setup(t)
device := repository.NewID()
token, user := registerAndLogin(t, r, repo, testUserUsername(device, "sf"), "search-test-password", device)
edfID := repository.NewID()
if err := repo.Resources.InsertFile(user, edfID, "scan_1.pdf", "", 100, nil, nil); err != nil {
t.Fatalf("insert: %v", err)
}
if err := repo.Resources.UpdateOcrText(user, edfID, "Facture EDF de mars"); err != nil {
t.Fatalf("ocr: %v", err)
}
num := 84.20
if err := repo.Resources.UpsertAutoFacts(edfID, []repository.FactParam{
{Type: "issuer", ValueText: "EDF", Confidence: 0.9},
{Type: "amount", ValueText: "84.20 €", ValueNum: &num, Confidence: 0.9},
{Type: "doc_type", ValueText: "facture", Confidence: 0.8},
}); err != nil {
t.Fatalf("facts: %v", err)
}
otherID := repository.NewID()
if err := repo.Resources.InsertFile(user, otherID, "scan_2.pdf", "", 100, nil, nil); err != nil {
t.Fatalf("insert: %v", err)
}
get := func(query string) []fileDTO {
t.Helper()
rec, _ := doRequest(t, r, http.MethodGet, "/api/v1/files/search"+query, token, nil, "")
env := expectOK(t, rec, "search-facets:"+query)
var files []fileDTO
if err := json.Unmarshal(env.Data, &files); err != nil {
t.Fatalf("unmarshal %s: %v", query, err)
}
return files
}
// FTS via OCR (nom opaque).
if files := get("?q=EDF"); len(files) == 0 || files[0].ID != edfID {
t.Errorf("q=EDF via OCR: %+v", files)
}
// Facette émetteur : filtre.
if files := get("?q=facture&issuer=edf"); len(files) != 1 || files[0].ID != edfID {
t.Errorf("issuer=edf: %+v", files)
}
if files := get("?q=facture&issuer=engie"); len(files) != 0 {
t.Errorf("issuer=engie doit filtrer: %+v", files)
}
// Facette montant.
if files := get("?q=facture&min_amount=100"); len(files) != 0 {
t.Errorf("min_amount=100 doit exclure 84.20: %+v", files)
}
if files := get("?q=facture&max_amount=100"); len(files) != 1 {
t.Errorf("max_amount=100: %+v", files)
}
// Facette doc_type.
if files := get("?q=scan&doc_type=facture"); len(files) != 1 || files[0].ID != edfID {
t.Errorf("doc_type=facture: %+v", files)
}
// Malformées → 400.
for _, query := range []string{
"?q=facture&from=12/03/2026",
"?q=facture&to=demain",
"?q=facture&min_amount=beaucoup",
"?q=facture&max_amount=",
} {
// max_amount vide = absent (pas d'erreur) — seul le cas non-vide
// non-numérique est rejeté ; on l'exclut ici.
if query == "?q=facture&max_amount=" {
continue
}
rec, _ := doRequest(t, r, http.MethodGet, "/api/v1/files/search"+query, token, nil, "")
expectError(t, rec, http.StatusBadRequest, "INVALID_REQUEST", "search-facets-400:"+query)
}
}
func TestSearchFilesEscapesPercent(t *testing.T) {
r, _, repo := setup(t)
device := repository.NewID()
+169
View File
@@ -0,0 +1,169 @@
package repository
import (
"fmt"
"strings"
)
// SearchFacets filters document search on extracted facts (V3 S4).
// Empty = no filter. From/To are ISO dates (YYYY-MM-DD) applied to
// date/due_date facts and overlapped with period facts (YYYY-MM).
type SearchFacets struct {
Issuer string
From, To string
MinAmount, MaxAmount *float64
DocType string
}
// searchFileColumns mirrors fileColumns qualified for `resources r`.
const searchFileColumns = `r.resource_id, r.name, r.size_bytes, COALESCE(r.mime_type, ''), COALESCE(r.parent_id, ''), r.created_at, r.updated_at, COALESCE(r.ocr_text, ''), COALESCE(r.has_content, FALSE)`
// SearchDocumentsVisible returns the user's accessible files (owned or
// shared, viewer+) matching q across name (ILIKE, compat), full-text
// (french TS, name+OCR) and extracted facts (ILIKE on value_text),
// optionally narrowed by facets. Ranked: TS rank + fact bonus (high-
// confidence fact match +1.0, low-confidence +0.5 — humble by design :
// low-confidence demotes, never excludes), then name.
func (r *Resources) SearchDocumentsVisible(userID, q string, facets SearchFacets, limit, offset int) ([]FileRow, int, error) {
where, order, args := r.searchDocumentsClauses(userID, q, facets)
base := `WITH RECURSIVE ` + grantedCTE + ` SELECT %s FROM resources r WHERE ` + where
var total int
if err := r.DB.QueryRow(fmt.Sprintf(base, "COUNT(*)"), args...).Scan(&total); err != nil {
return nil, 0, err
}
rows, err := r.DB.Query(
fmt.Sprintf(base+` ORDER BY %s, r.name ASC LIMIT $%d OFFSET $%d`, searchFileColumns, order, len(args)+1, len(args)+2),
append(args, limit, offset)...,
)
if err != nil {
return nil, 0, err
}
defer rows.Close()
files := make([]FileRow, 0)
for rows.Next() {
row, err := r.scanFile(rows.Scan)
if err != nil {
return nil, 0, err
}
files = append(files, row)
}
return files, total, rows.Err()
}
// searchDocumentsClauses builds the WHERE (match) and ORDER BY (rank)
// fragments plus bound args. $1 is always userID (grantedCTE).
func (r *Resources) searchDocumentsClauses(userID, q string, facets SearchFacets) (where, order string, args []any) {
args = []any{userID}
next := 2
ph := func(v any) string {
args = append(args, v)
p := fmt.Sprintf("$%d", next)
next++
return p
}
// plainto_tsquery placeholder partagé match/rang (le COUNT ne voit que
// le WHERE : aucun placeholder ne doit n'exister que dans le ORDER BY).
// Même règle pour le pattern LIKE du nom (bonus de rang).
tsPh := ph(q)
namePh := ph(`%` + escapeLike(q) + `%`)
matchAlts := []string{
`r.name ILIKE ` + namePh + ` ESCAPE '\'`,
`r.search_tsv @@ plainto_tsquery('french', ` + tsPh + `)`,
}
// Fact word matches (words >= 3 chars — shorter are French stopwords).
// `pat` est partagé entre le match et le bonus du rang : le COUNT ne voit
// que le WHERE, aucun placeholder ne doit n'exister que dans le ORDER BY.
words := longWords(q, 3, 5)
var wordConds, wordCondsHigh []string
for _, w := range words {
pat := ph(`%` + escapeLike(w) + `%`)
wordConds = append(wordConds,
`EXISTS (SELECT 1 FROM resource_facts f WHERE f.resource_id = r.resource_id AND f.value_text ILIKE `+pat+` ESCAPE '\')`)
wordCondsHigh = append(wordCondsHigh,
`EXISTS (SELECT 1 FROM resource_facts f WHERE f.resource_id = r.resource_id AND f.confidence >= 0.5 AND f.value_text ILIKE `+pat+` ESCAPE '\')`)
}
if len(wordConds) > 0 {
matchAlts = append(matchAlts, `(`+strings.Join(wordConds, " OR ")+`)`)
}
rankParts := []string{
// Match nom exact : bonus dominant (2.0) — un nom trouvé bat
// toujours un fait douteux (0.5), humble par design.
`CASE WHEN r.name ILIKE ` + namePh + ` ESCAPE '\' THEN 2.0 ELSE 0 END`,
`COALESCE(ts_rank_cd(r.search_tsv, plainto_tsquery('french', ` + tsPh + `)), 0)`,
}
if len(wordCondsHigh) > 0 {
rankParts = append(rankParts,
`CASE WHEN (`+strings.Join(wordCondsHigh, " OR ")+`) THEN 1.0 WHEN (`+strings.Join(wordConds, " OR ")+`) THEN 0.5 ELSE 0 END`)
}
conds := []string{
`r.type = 'file'`,
`r.deleted_at IS NULL`,
`r.resource_id IN (SELECT id FROM granted)`,
`(` + strings.Join(matchAlts, " OR ") + `)`,
}
// Facets (AND).
if facets.Issuer != "" {
conds = append(conds,
`EXISTS (SELECT 1 FROM resource_facts f WHERE f.resource_id = r.resource_id AND f.fact_type = 'issuer' AND f.value_text ILIKE `+ph(`%`+escapeLike(facets.Issuer)+`%`)+` ESCAPE '\')`)
}
if facets.DocType != "" {
conds = append(conds,
`EXISTS (SELECT 1 FROM resource_facts f WHERE f.resource_id = r.resource_id AND f.fact_type = 'doc_type' AND f.value_text = `+ph(facets.DocType)+`)`)
}
if facets.MinAmount != nil {
conds = append(conds,
`EXISTS (SELECT 1 FROM resource_facts f WHERE f.resource_id = r.resource_id AND f.fact_type = 'amount' AND f.value_num >= `+ph(*facets.MinAmount)+`)`)
}
if facets.MaxAmount != nil {
conds = append(conds,
`EXISTS (SELECT 1 FROM resource_facts f WHERE f.resource_id = r.resource_id AND f.fact_type = 'amount' AND f.value_num <= `+ph(*facets.MaxAmount)+`)`)
}
if facets.From != "" || facets.To != "" {
from := facets.From
if from == "" {
from = "0001-01-01"
}
to := facets.To
if to == "" {
to = "9999-12-31"
}
fromMonth, toMonth := from, to
if len(fromMonth) >= 7 {
fromMonth = fromMonth[:7]
}
if len(toMonth) >= 7 {
toMonth = toMonth[:7]
}
conds = append(conds, `(
EXISTS (SELECT 1 FROM resource_facts f WHERE f.resource_id = r.resource_id AND f.fact_type IN ('date', 'due_date') AND f.value_date >= `+ph(from)+`::date AND f.value_date <= `+ph(to)+`::date)
OR EXISTS (SELECT 1 FROM resource_facts f WHERE f.resource_id = r.resource_id AND f.fact_type = 'period' AND f.value_text >= `+ph(fromMonth)+` AND f.value_text <= `+ph(toMonth)+`)
)`)
}
where = strings.Join(conds, " AND ")
order = strings.Join(rankParts, " + ") + " DESC"
return where, order, args
}
// longWords splits q into words of at least minLen runes, capped at max.
func longWords(q string, minLen, max int) []string {
var out []string
seen := map[string]bool{}
for _, w := range strings.Fields(q) {
if len([]rune(w)) < minLen || seen[w] {
continue
}
seen[w] = true
out = append(out, w)
if len(out) >= max {
break
}
}
return out
}
+165
View File
@@ -0,0 +1,165 @@
package repository
import (
"testing"
"github.com/kazier/backend/dbtest"
)
const searchTestURL = "postgres://kazier:kazier@localhost:5432/kazier_search_test?sslmode=disable"
func newSearchRepo(t *testing.T) *Repository {
t.Helper()
return NewRepository(dbtest.OpenTestDatabase(t, searchTestURL))
}
func mustSearchUser(t *testing.T, repo *Repository, username string) string {
t.Helper()
id, err := repo.Users.Create(username, username, "test-hash", false)
if err != nil {
t.Fatalf("create user: %v", err)
}
return id
}
func seedDoc(t *testing.T, repo *Repository, userID, name, ocr string, facts []FactParam) string {
t.Helper()
id := NewID()
if err := repo.Resources.InsertFile(userID, id, name, "", 0, nil, nil); err != nil {
t.Fatalf("insert %q: %v", name, err)
}
if ocr != "" {
if err := repo.Resources.UpdateOcrText(userID, id, ocr); err != nil {
t.Fatalf("ocr %q: %v", name, err)
}
}
if len(facts) > 0 {
if err := repo.Resources.UpsertAutoFacts(id, facts); err != nil {
t.Fatalf("facts %q: %v", name, err)
}
}
return id
}
func f64(v float64) *float64 { return &v }
func fstr(v string) *string { return &v }
func searchIDs(t *testing.T, repo *Repository, userID, q string, facets SearchFacets) []string {
t.Helper()
rows, _, err := repo.Resources.SearchDocumentsVisible(userID, q, facets, 50, 0)
if err != nil {
t.Fatalf("search %q: %v", q, err)
}
ids := make([]string, 0, len(rows))
for _, r := range rows {
ids = append(ids, r.ID)
}
return ids
}
func containsID(ids []string, want string) bool {
for _, id := range ids {
if id == want {
return true
}
}
return false
}
func TestSearchDocumentsFTS(t *testing.T) {
repo := newSearchRepo(t)
userID := mustSearchUser(t, repo, "search-fts")
// Nom opaque : trouvable uniquement via l'OCR (TS français).
scan := seedDoc(t, repo, userID, "scan_042.pdf",
"Attestation d'assurance MMA pour l'année 2026", nil)
plain := seedDoc(t, repo, userID, "assurance-vie.txt", "", nil)
ids := searchIDs(t, repo, userID, "assurance", SearchFacets{})
if !containsID(ids, scan) {
t.Errorf("scan_042.pdf introuvable via OCR: %v", ids)
}
if !containsID(ids, plain) {
t.Errorf("assurance-vie.txt introuvable via nom: %v", ids)
}
// Stemming français : "factures" matche "facture".
fact := seedDoc(t, repo, userID, "doc.pdf", "Votre facture de mars", nil)
ids = searchIDs(t, repo, userID, "factures", SearchFacets{})
if !containsID(ids, fact) {
t.Errorf("stemming FR manquant: %v", ids)
}
}
func TestSearchDocumentsFacets(t *testing.T) {
repo := newSearchRepo(t)
userID := mustSearchUser(t, repo, "search-facets")
edf := seedDoc(t, repo, userID, "edf.pdf", "Facture EDF", []FactParam{
{Type: "issuer", ValueText: "EDF", Confidence: 0.9},
{Type: "amount", ValueText: "84.20 €", ValueNum: f64(84.20), Confidence: 0.9},
{Type: "due_date", ValueText: "2026-04-12", ValueDate: fstr("2026-04-12"), Confidence: 0.8},
{Type: "doc_type", ValueText: "facture", Confidence: 0.8},
})
engie := seedDoc(t, repo, userID, "engie.pdf", "Facture Engie", []FactParam{
{Type: "issuer", ValueText: "ENGIE", Confidence: 0.9},
{Type: "amount", ValueText: "120.00 €", ValueNum: f64(120), Confidence: 0.9},
{Type: "doc_type", ValueText: "facture", Confidence: 0.8},
})
ids := searchIDs(t, repo, userID, "facture", SearchFacets{Issuer: "edf"})
if !containsID(ids, edf) || containsID(ids, engie) {
t.Errorf("facette issuer: %v", ids)
}
ids = searchIDs(t, repo, userID, "facture", SearchFacets{MinAmount: f64(100)})
if !containsID(ids, engie) || containsID(ids, edf) {
t.Errorf("facette min_amount: %v", ids)
}
ids = searchIDs(t, repo, userID, "facture", SearchFacets{DocType: "facture"})
if !containsID(ids, edf) || !containsID(ids, engie) {
t.Errorf("facette doc_type: %v", ids)
}
ids = searchIDs(t, repo, userID, "facture", SearchFacets{From: "2026-04-01", To: "2026-04-30"})
if !containsID(ids, edf) || containsID(ids, engie) {
t.Errorf("facette période: %v", ids)
}
}
func TestSearchDocumentsLowConfidenceDemotes(t *testing.T) {
repo := newSearchRepo(t)
userID := mustSearchUser(t, repo, "search-humble")
// Fait douteux (0.3) : trouvé quand même (jamais exclu), mais après le
// match franc (nom) — humble par design.
sure := seedDoc(t, repo, userID, "EDF-contrat.pdf", "", nil)
doubt := seedDoc(t, repo, userID, "scan_x.pdf", "courrier", []FactParam{
{Type: "issuer", ValueText: "EDF", Confidence: 0.3},
})
ids := searchIDs(t, repo, userID, "EDF", SearchFacets{})
if !containsID(ids, doubt) {
t.Fatalf("fait douteux exclu (interdit): %v", ids)
}
if len(ids) < 2 || ids[0] != sure || ids[1] != doubt {
t.Errorf("ordre de rang inattendu (franc avant douteux): %v", ids)
}
}
func TestSearchDocumentsTrashedExcluded(t *testing.T) {
repo := newSearchRepo(t)
userID := mustSearchUser(t, repo, "search-trash")
id := seedDoc(t, repo, userID, "poubelle.pdf", "contenu unique xyz", nil)
if ids := searchIDs(t, repo, userID, "xyz", SearchFacets{}); !containsID(ids, id) {
t.Fatalf("introuvable avant trash: %v", ids)
}
if err := repo.Resources.TrashResource(userID, id); err != nil {
t.Fatalf("trash: %v", err)
}
if ids := searchIDs(t, repo, userID, "xyz", SearchFacets{}); containsID(ids, id) {
t.Errorf("trashé encore visible: %v", ids)
}
}
+2 -2
View File
@@ -245,8 +245,8 @@ func (s *Resources) ListRootFolders(ownerID string) ([]FolderDTO, error) {
return folders, nil
}
func (s *Resources) SearchFiles(ownerID, q string, page, pageSize int) ([]FileDTO, int, error) {
rows, total, err := s.Repo.SearchFilesVisible(ownerID, q, pageSize, (page-1)*pageSize)
func (s *Resources) SearchFiles(ownerID, q string, page, pageSize int, facets repository.SearchFacets) ([]FileDTO, int, error) {
rows, total, err := s.Repo.SearchDocumentsVisible(ownerID, q, facets, pageSize, (page-1)*pageSize)
if err != nil {
return nil, 0, err
}