V3 S5 : graphe documentaire + doublons + collections
Backend : resource_links + scan_session_id (000021), réconciliation déterministe (same_scan outbox, doublons SHA-256 à l'upload + backfill, same_issuer/référence via faits effectifs, verdicts préservés, élagage auto), GET /files/:id/relations + verdict (owner-only), GET /duplicates (groupes + gain récupérable). Tests service + handlers + migrations. Mobile (Room v22) : links + collections, scanSessionId dans les creates de scan, LinksRepository (sync + verdicts REST), « Voir aussi » dans la fiche (verdict infirmer/re-confirmer), écran Doublons (garder le plus récent), collections locales (sauver/rejouer/supprimer). FR/EN. Contrat docs/api-v1.md (/relations, pas /links — pris par le partage), V3.md (S5 livré, version_of non matérialisé).
This commit is contained in:
@@ -92,11 +92,23 @@ func main() {
|
||||
// échec d'extract ne fait jamais échouer le job OCR.
|
||||
extractSvc := service.NewExtract(repo)
|
||||
handlers.Ocr.OnText = func(userID, resourceID, text string) {
|
||||
if err := extractSvc.ProcessText(resourceID, text); err != nil {
|
||||
if err := extractSvc.ProcessText(userID, resourceID, text); err != nil {
|
||||
log.Printf("extract: %s: %v", resourceID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// V3 S5 : backfill des hashes (uploads d'avant la V3) — doublons.
|
||||
go func() {
|
||||
n, err := handlers.Store.BackfillHashes(200)
|
||||
if err != nil {
|
||||
log.Printf("content backfill hashes: %v", err)
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
log.Printf("content backfill hashes: %d fichiers", n)
|
||||
}
|
||||
}()
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
|
||||
@@ -24,6 +24,9 @@ var expectedRoutes = []string{
|
||||
"GET /api/v1/files/:id/links",
|
||||
"GET /api/v1/files/:id/placements",
|
||||
"GET /api/v1/files/:id/versions",
|
||||
"GET /api/v1/files/:id/relations",
|
||||
"POST /api/v1/files/:id/relations/verdict",
|
||||
"GET /api/v1/duplicates",
|
||||
"POST /api/v1/files/:id/restore",
|
||||
"POST /api/v1/files/:id/versions/:n/restore",
|
||||
"GET /api/v1/trash",
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE resources DROP COLUMN IF EXISTS scan_session_id;
|
||||
DROP TABLE IF EXISTS resource_links;
|
||||
@@ -0,0 +1,32 @@
|
||||
-- V3 relations documentaires (S5) : graphe auto + verdicts utilisateur.
|
||||
-- `resource_links` : arêtes orientées (from_id → to_id) typées, avec
|
||||
-- confiance et origine. `duplicate_of` (même content_hash), `same_scan`
|
||||
-- (même session de scan, déterministe), `same_issuer` / `same_reference`
|
||||
-- (faits normalisés égaux). `version_of` non matérialisé : les versions
|
||||
-- vivent DANS la ressource (resource_versions, 000018), pas entre ressources.
|
||||
-- `created_by` : 'auto' (dérivé serveur, écrasable) vs 'user' (verdict
|
||||
-- explicite — la réconciliation ne touche jamais aux verdicts). Les
|
||||
-- invalidations (confidence 0) sont conservées (pas de re-création auto).
|
||||
-- `resources.scan_session_id` : session de scan opaque fournie par le client
|
||||
-- dans le payload `create_resource` (zéro nouvelle op) — le serveur en
|
||||
-- déduit les arêtes `same_scan`.
|
||||
CREATE TABLE resource_links (
|
||||
id TEXT PRIMARY KEY CHECK (id ~ '^[0-9a-f]{32}$'),
|
||||
from_id TEXT NOT NULL REFERENCES resources(resource_id) ON DELETE CASCADE,
|
||||
to_id TEXT NOT NULL REFERENCES resources(resource_id) ON DELETE CASCADE,
|
||||
link_type TEXT NOT NULL CHECK (link_type IN (
|
||||
'same_scan', 'duplicate_of', 'same_issuer', 'same_reference'
|
||||
)),
|
||||
confidence REAL NOT NULL CHECK (confidence >= 0 AND confidence <= 1),
|
||||
created_by TEXT NOT NULL DEFAULT 'auto' CHECK (created_by IN ('auto', 'user')),
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
CHECK (from_id != to_id),
|
||||
UNIQUE (from_id, to_id, link_type)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_links_from ON resource_links(from_id);
|
||||
CREATE INDEX idx_links_to ON resource_links(to_id);
|
||||
|
||||
ALTER TABLE resources ADD COLUMN scan_session_id TEXT;
|
||||
CREATE INDEX IF NOT EXISTS idx_resources_scan_session ON resources(user_id, scan_session_id)
|
||||
WHERE scan_session_id IS NOT NULL;
|
||||
@@ -447,6 +447,50 @@ func TestMigrationsUpDown(t *testing.T) {
|
||||
t.Error("index idx_resources_search_tsv manquant après 000020")
|
||||
}
|
||||
|
||||
// --- 000021 : graphe documentaire ----------------------------------------
|
||||
assertTables(t, conn, "users", "devices", "resources", "operations", "ocr_jobs", "shares", "share_links", "storage_locations", "resource_placements", "groups", "group_members", "resource_versions", "resource_facts", "resource_links", "schema_migrations")
|
||||
|
||||
// resource_links.id : CHECK 32-hex (requête ciblée — le helper
|
||||
// générique matcherait `from_id <> to_id` qui contient aussi "id").
|
||||
var linksIDCheck string
|
||||
err = conn.QueryRow(`
|
||||
SELECT pg_get_constraintdef(c.oid)
|
||||
FROM pg_constraint c
|
||||
JOIN pg_class t ON t.oid = c.conrelid
|
||||
WHERE t.relname = 'resource_links' AND c.conname = 'resource_links_id_check'`).Scan(&linksIDCheck)
|
||||
if err != nil {
|
||||
t.Fatalf("contrainte CHECK(resource_links.id) manquante: %v", err)
|
||||
}
|
||||
if !strings.Contains(linksIDCheck, "^[0-9a-f]{32}$") {
|
||||
t.Errorf("CHECK resource_links.id attendu 32-hex, obtenu : %s", linksIDCheck)
|
||||
}
|
||||
|
||||
var linksUniq int
|
||||
err = conn.QueryRow(`
|
||||
SELECT COUNT(*) FROM pg_index i
|
||||
JOIN pg_class t ON t.oid = i.indrelid
|
||||
WHERE t.relname = 'resource_links' AND i.indisunique
|
||||
AND ARRAY(SELECT a.attname FROM unnest(i.indkey) WITH ORDINALITY k(attnum, ord)
|
||||
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum
|
||||
ORDER BY k.ord)::text[] = ARRAY['from_id','to_id','link_type']`).Scan(&linksUniq)
|
||||
if err != nil {
|
||||
t.Fatalf("links unique: %v", err)
|
||||
}
|
||||
if linksUniq == 0 {
|
||||
t.Error("contrainte UNIQUE(from_id, to_id, link_type) manquante sur resource_links après 000021")
|
||||
}
|
||||
|
||||
var scanSessionCol int
|
||||
err = conn.QueryRow(`
|
||||
SELECT COUNT(*) FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = 'resources' AND column_name = 'scan_session_id'`).Scan(&scanSessionCol)
|
||||
if err != nil {
|
||||
t.Fatalf("scan_session_id column: %v", err)
|
||||
}
|
||||
if scanSessionCol == 0 {
|
||||
t.Error("colonne resources.scan_session_id manquante après 000021")
|
||||
}
|
||||
|
||||
if err := MigrateDownDatabase(url); err != nil {
|
||||
t.Fatalf("migrate down: %v", err)
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ func TestFactsEndpoint(t *testing.T) {
|
||||
t.Fatalf("ocr text: %v", err)
|
||||
}
|
||||
extractSvc := service.NewExtract(repo)
|
||||
if err := extractSvc.ProcessText(fileID, "EDF\nFacture n° F-1 du 01/02/2026\nTotal : 10,00 €\n"); err != nil {
|
||||
if err := extractSvc.ProcessText(userID, fileID, "EDF\nFacture n° F-1 du 01/02/2026\nTotal : 10,00 €\n"); err != nil {
|
||||
t.Fatalf("extract: %v", err)
|
||||
}
|
||||
_ = store
|
||||
@@ -95,7 +95,7 @@ func TestCorrectFactsEndpoint(t *testing.T) {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
extractSvc := service.NewExtract(repo)
|
||||
if err := extractSvc.ProcessText(fileID, "EDF\nFacture n° F-1 du 01/02/2026\nTotal : 10,00 €\n"); err != nil {
|
||||
if err := extractSvc.ProcessText(userID, fileID, "EDF\nFacture n° F-1 du 01/02/2026\nTotal : 10,00 €\n"); err != nil {
|
||||
t.Fatalf("extract: %v", err)
|
||||
}
|
||||
_ = store
|
||||
|
||||
@@ -183,6 +183,96 @@ func FilesCorrectFacts(c *gin.Context) {
|
||||
api.OK(c, facts)
|
||||
}
|
||||
|
||||
// RelationsList expose le graphe documentaire d'une ressource possédée
|
||||
// (V3 S5) : `GET /files/:id/relations`. Réconcilie d'abord (les arêtes
|
||||
// déterministes convergent à la lecture), puis renvoie les arêtes vivantes
|
||||
// (l'autre bout non trashé), confiance décroissante. Owner-only : le graphe
|
||||
// d'autrui ne s'énumère pas (404, comme `/shares`).
|
||||
func RelationsList(c *gin.Context) {
|
||||
if Store == nil {
|
||||
api.Error(c, http.StatusServiceUnavailable, "SERVICE_UNAVAILABLE", "backend not initialized")
|
||||
return
|
||||
}
|
||||
userID := c.GetString(UserIDKey)
|
||||
id := c.Param("id")
|
||||
if !deviceIDPattern.MatchString(id) {
|
||||
api.Error(c, http.StatusNotFound, "NOT_FOUND", "file not found")
|
||||
return
|
||||
}
|
||||
relations, err := Store.ListRelations(userID, id)
|
||||
if err != nil {
|
||||
writeError(c, err)
|
||||
return
|
||||
}
|
||||
if relations == nil {
|
||||
relations = []service.RelationDTO{}
|
||||
}
|
||||
api.OK(c, relations)
|
||||
}
|
||||
|
||||
// RelationVerdict enregistre un verdict explicite sur une arête (V3 S5) :
|
||||
// `POST /files/:id/relations/verdict { otherId, linkType, confirmed }`.
|
||||
// Confirmé → 1.0, infirmé → 0.0 (conservé, bloque la re-création auto).
|
||||
// Owner-only, les deux bouts possédés et vifs, sinon 404 ; type/paire
|
||||
// invalide → 400.
|
||||
func RelationVerdict(c *gin.Context) {
|
||||
if Store == nil {
|
||||
api.Error(c, http.StatusServiceUnavailable, "SERVICE_UNAVAILABLE", "backend not initialized")
|
||||
return
|
||||
}
|
||||
userID := c.GetString(UserIDKey)
|
||||
id := c.Param("id")
|
||||
if !deviceIDPattern.MatchString(id) {
|
||||
api.Error(c, http.StatusNotFound, "NOT_FOUND", "file not found")
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
OtherID string `json:"otherId"`
|
||||
LinkType string `json:"linkType"`
|
||||
Confirmed bool `json:"confirmed"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
api.Error(c, http.StatusBadRequest, "INVALID_REQUEST", "invalid JSON body (expected { otherId, linkType, confirmed })")
|
||||
return
|
||||
}
|
||||
if !deviceIDPattern.MatchString(req.OtherID) {
|
||||
api.Error(c, http.StatusBadRequest, "INVALID_REQUEST", "invalid `otherId` (32 lowercase hex)")
|
||||
return
|
||||
}
|
||||
if err := Store.LinkVerdict(userID, id, req.OtherID, req.LinkType, req.Confirmed); err != nil {
|
||||
if errors.Is(err, service.ErrInvalidLinkType) {
|
||||
api.Error(c, http.StatusBadRequest, "INVALID_REQUEST", err.Error())
|
||||
return
|
||||
}
|
||||
writeError(c, err)
|
||||
return
|
||||
}
|
||||
relations, err := Store.ListRelations(userID, id)
|
||||
if err != nil {
|
||||
writeError(c, err)
|
||||
return
|
||||
}
|
||||
api.OK(c, relations)
|
||||
}
|
||||
|
||||
// DuplicatesList expose les doublons d'octets du compte (V3 S5 B2) :
|
||||
// `GET /duplicates`. Groupes par hash, plus gros d'abord, avec le gain
|
||||
// récupérable (« garder un seul »). Scope owner — `[]` quand propre.
|
||||
func DuplicatesList(c *gin.Context) {
|
||||
if Store == nil {
|
||||
api.Error(c, http.StatusServiceUnavailable, "SERVICE_UNAVAILABLE", "backend not initialized")
|
||||
return
|
||||
}
|
||||
userID := c.GetString(UserIDKey)
|
||||
groups, err := Store.ListDuplicates(userID)
|
||||
if err != nil {
|
||||
writeError(c, err)
|
||||
return
|
||||
}
|
||||
api.OK(c, groups)
|
||||
}
|
||||
|
||||
// FilesContent streame les octets physiques d'un fichier visible (viewer+,
|
||||
// own ou shared) : GET /files/:id/content. Réponse binaire (Content-Type du
|
||||
// fichier, Content-Disposition inline) — jamais d'enveloppe {data}. Supporte
|
||||
// Range via http.ServeContent (Gin c.File). Sans bytes serveur (ressource
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/kazier/backend/repository"
|
||||
)
|
||||
|
||||
// TestRelationsEndpoints : GET /files/:id/relations (réconciliation à la
|
||||
// lecture) + POST verdict (confirmé/infirmé, 400/404).
|
||||
func TestRelationsEndpoints(t *testing.T) {
|
||||
r, _, repo := setup(t)
|
||||
device := repository.NewID()
|
||||
token, userID := registerAndLogin(t, r, repo, testUserUsername(device, "rl"), "relations-test-password", device)
|
||||
|
||||
session := repository.NewID()
|
||||
page1, page2 := repository.NewID(), repository.NewID()
|
||||
for _, id := range []string{page1, page2} {
|
||||
if err := repo.Resources.InsertFile(userID, id, id+".png", "", 0, nil, nil); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
if err := repo.Resources.SetScanSession(userID, id, session); err != nil {
|
||||
t.Fatalf("session: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
rec, _ := doRequest(t, r, http.MethodGet, "/api/v1/files/"+page1+"/relations", token, nil, "")
|
||||
env := expectOK(t, rec, "relations-get")
|
||||
var rels []struct {
|
||||
ID string `json:"id"`
|
||||
OtherID string `json:"otherId"`
|
||||
Type string `json:"type"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
CreatedBy string `json:"createdBy"`
|
||||
OtherName string `json:"otherName"`
|
||||
}
|
||||
if err := json.Unmarshal(env.Data, &rels); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if len(rels) != 1 || rels[0].Type != "same_scan" || rels[0].OtherID != page2 {
|
||||
t.Fatalf("relations inattendues: %+v", rels)
|
||||
}
|
||||
|
||||
// Verdict infirmé : arête conservée à 0.0, origine user.
|
||||
body := fmt.Sprintf(`{"otherId":%q,"linkType":"same_scan","confirmed":false}`, page2)
|
||||
rec, _ = doRequest(t, r, http.MethodPost, "/api/v1/files/"+page1+"/relations/verdict", token, []byte(body), "application/json")
|
||||
env = expectOK(t, rec, "relations-verdict")
|
||||
rels = nil
|
||||
if err := json.Unmarshal(env.Data, &rels); err != nil {
|
||||
t.Fatalf("unmarshal verdict: %v", err)
|
||||
}
|
||||
if len(rels) != 1 || rels[0].Confidence != 0 || rels[0].CreatedBy != "user" {
|
||||
t.Fatalf("verdict inattendu: %+v", rels)
|
||||
}
|
||||
|
||||
// Type inconnu → 400 ; paire non-hex → 400 ; étranger → 404.
|
||||
for _, tc := range []struct {
|
||||
name, body string
|
||||
status int
|
||||
code string
|
||||
}{
|
||||
{"type", fmt.Sprintf(`{"otherId":%q,"linkType":"copains","confirmed":true}`, page2), http.StatusBadRequest, "INVALID_REQUEST"},
|
||||
{"hex", `{"otherId":"nope","linkType":"same_scan","confirmed":true}`, http.StatusBadRequest, "INVALID_REQUEST"},
|
||||
} {
|
||||
rec, _ = doRequest(t, r, http.MethodPost, "/api/v1/files/"+page1+"/relations/verdict", token, []byte(tc.body), "application/json")
|
||||
expectError(t, rec, tc.status, tc.code, "relations-verdict-400:"+tc.name)
|
||||
}
|
||||
|
||||
deviceB := repository.NewID()
|
||||
tokenB, _ := registerAndLogin(t, r, repo, testUserUsername(deviceB, "rb"), "relations-test-password", deviceB)
|
||||
rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/files/"+page1+"/relations", tokenB, nil, "")
|
||||
expectError(t, rec, http.StatusNotFound, "NOT_FOUND", "relations-foreign")
|
||||
rec, _ = doRequest(t, r, http.MethodPost, "/api/v1/files/"+page1+"/relations/verdict", tokenB,
|
||||
[]byte(fmt.Sprintf(`{"otherId":%q,"linkType":"same_scan","confirmed":true}`, page2)), "application/json")
|
||||
expectError(t, rec, http.StatusNotFound, "NOT_FOUND", "relations-verdict-foreign")
|
||||
|
||||
// Trashé → 404 (l'autre bout trashé disparaît aussi de la liste).
|
||||
if err := repo.Resources.TrashResource(userID, page2); err != nil {
|
||||
t.Fatalf("trash: %v", err)
|
||||
}
|
||||
rec, _ = doRequest(t, r, http.MethodGet, "/api/v1/files/"+page1+"/relations", token, nil, "")
|
||||
env = expectOK(t, rec, "relations-trashed-other")
|
||||
rels = nil
|
||||
if err := json.Unmarshal(env.Data, &rels); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if len(rels) != 0 {
|
||||
t.Fatalf("bout trashé encore listé: %+v", rels)
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,9 @@ func RegisterRoutes(r *gin.Engine) {
|
||||
protected.GET("/files/:id/links", LinksList)
|
||||
protected.GET("/files/:id/placements", PlacementsList)
|
||||
protected.GET("/files/:id/versions", VersionsList)
|
||||
protected.GET("/files/:id/relations", RelationsList)
|
||||
protected.POST("/files/:id/relations/verdict", RelationVerdict)
|
||||
protected.GET("/duplicates", DuplicatesList)
|
||||
protected.POST("/files/:id/restore", FilesRestore)
|
||||
protected.POST("/files/:id/versions/:n/restore", VersionRestore)
|
||||
protected.GET("/trash", TrashList)
|
||||
|
||||
@@ -127,6 +127,7 @@ func (r *Resources) CorrectFacts(resourceID string, params []FactParam) error {
|
||||
}
|
||||
|
||||
type factNeedingExtract struct {
|
||||
UserID string
|
||||
ResourceID string
|
||||
Text string
|
||||
}
|
||||
@@ -135,8 +136,8 @@ type factNeedingExtract struct {
|
||||
// origin='auto' facts (boot backfill, S2). Live rows only.
|
||||
func (r *Resources) ListNeedingExtract(limit int) ([]factNeedingExtract, error) {
|
||||
rows, err := r.DB.Query(
|
||||
`SELECT resource_id, ocr_text FROM resources
|
||||
WHERE type = 'file' AND deleted_at IS NULL
|
||||
`SELECT user_id, resource_id, ocr_text FROM resources
|
||||
WHERE type = 'file' AND deleted_at IS NULL AND user_id IS NOT NULL
|
||||
AND ocr_text IS NOT NULL AND ocr_text <> ''
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM resource_facts
|
||||
@@ -152,7 +153,7 @@ func (r *Resources) ListNeedingExtract(limit int) ([]factNeedingExtract, error)
|
||||
var out []factNeedingExtract
|
||||
for rows.Next() {
|
||||
var row factNeedingExtract
|
||||
if err := rows.Scan(&row.ResourceID, &row.Text); err != nil {
|
||||
if err := rows.Scan(&row.UserID, &row.ResourceID, &row.Text); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, row)
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
)
|
||||
|
||||
// Link types — mirror of the resource_links CHECK (migration 000021).
|
||||
// All types are symmetric : stored once with from_id < to_id (canonical
|
||||
// order), read from either end. version_of is intentionally absent :
|
||||
// versions live INSIDE a resource (resource_versions, 000018).
|
||||
const (
|
||||
LinkSameScan = "same_scan"
|
||||
LinkDuplicateOf = "duplicate_of"
|
||||
LinkSameIssuer = "same_issuer"
|
||||
LinkSameReference = "same_reference"
|
||||
)
|
||||
|
||||
// LinkRow is one edge viewed from the queried resource.
|
||||
type LinkRow struct {
|
||||
ID string
|
||||
OtherID string
|
||||
Type string
|
||||
Confidence float64
|
||||
CreatedBy string
|
||||
OtherName string
|
||||
OtherType string
|
||||
OtherSize int64
|
||||
}
|
||||
|
||||
// canonicalPair orders an unordered pair for symmetric storage.
|
||||
func canonicalPair(a, b string) (string, string) {
|
||||
if a < b {
|
||||
return a, b
|
||||
}
|
||||
return b, a
|
||||
}
|
||||
|
||||
// ListLinks returns the live edges touching resourceID (either direction),
|
||||
// joined with the other end's identity. Trashed others are excluded.
|
||||
// No ownership check — the caller scopes (owner-only in S5).
|
||||
func (r *Resources) ListLinks(resourceID string) ([]LinkRow, error) {
|
||||
rows, err := r.DB.Query(
|
||||
`SELECT l.id,
|
||||
CASE WHEN l.from_id = $1 THEN l.to_id ELSE l.from_id END AS other_id,
|
||||
l.link_type, l.confidence, l.created_by,
|
||||
o.name, o.type, COALESCE(o.size_bytes, 0)
|
||||
FROM resource_links l
|
||||
JOIN resources o ON o.resource_id = CASE WHEN l.from_id = $1 THEN l.to_id ELSE l.from_id END
|
||||
WHERE (l.from_id = $1 OR l.to_id = $1)
|
||||
AND o.deleted_at IS NULL
|
||||
ORDER BY l.confidence DESC, o.name ASC`,
|
||||
resourceID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]LinkRow, 0)
|
||||
for rows.Next() {
|
||||
var row LinkRow
|
||||
if err := rows.Scan(&row.ID, &row.OtherID, &row.Type, &row.Confidence, &row.CreatedBy, &row.OtherName, &row.OtherType, &row.OtherSize); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ensureAutoLink inserts a symmetric auto edge (canonical order). Any
|
||||
// existing row — auto or user verdict — blocks recreation (an invalidated
|
||||
// edge never comes back by itself).
|
||||
func (r *Resources) ensureAutoLink(from, to, linkType string, confidence float64) error {
|
||||
a, b := canonicalPair(from, to)
|
||||
id, err := NewIDErr()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = r.DB.Exec(
|
||||
`INSERT INTO resource_links (id, from_id, to_id, link_type, confidence, created_by)
|
||||
VALUES ($1, $2, $3, $4, $5, 'auto')
|
||||
ON CONFLICT (from_id, to_id, link_type) DO NOTHING`,
|
||||
id, a, b, linkType, clamp01(confidence),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// SetUserVerdict records an explicit verdict (created_by='user') : confirm →
|
||||
// confidence 1.0, invalidate → 0.0. Upserts over any previous row (auto or
|
||||
// older verdict) — the latest human word wins.
|
||||
func (r *Resources) SetUserVerdict(one, other, linkType string, confirmed bool) error {
|
||||
a, b := canonicalPair(one, other)
|
||||
conf := 0.0
|
||||
if confirmed {
|
||||
conf = 1.0
|
||||
}
|
||||
id, err := NewIDErr()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = r.DB.Exec(
|
||||
`INSERT INTO resource_links (id, from_id, to_id, link_type, confidence, created_by)
|
||||
VALUES ($1, $2, $3, $4, $5, 'user')
|
||||
ON CONFLICT (from_id, to_id, link_type)
|
||||
DO UPDATE SET confidence = EXCLUDED.confidence, created_by = 'user'`,
|
||||
id, a, b, linkType, conf,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// pruneAutoLinks drops auto edges of the symmetric types touching resourceID
|
||||
// whose pair is no longer a candidate (facts/session edited). keep holds
|
||||
// canonical "a\x00b\x00type" keys to preserve. User verdicts are never pruned.
|
||||
func (r *Resources) pruneAutoLinks(resourceID string, keep map[string]bool) error {
|
||||
rows, err := r.DB.Query(
|
||||
`SELECT id, from_id, to_id, link_type FROM resource_links
|
||||
WHERE (from_id = $1 OR to_id = $1) AND created_by = 'auto'`,
|
||||
resourceID,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var stale []string
|
||||
for rows.Next() {
|
||||
var id, a, b, typ string
|
||||
if err := rows.Scan(&id, &a, &b, &typ); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
if !keep[a+"\x00"+b+"\x00"+typ] {
|
||||
stale = append(stale, id)
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, id := range stale {
|
||||
if _, err := r.DB.Exec(`DELETE FROM resource_links WHERE id = $1`, id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// linkCandidate is one deterministic pair found by reconciliation.
|
||||
type linkCandidate struct {
|
||||
other string
|
||||
typ string
|
||||
confidence float64
|
||||
}
|
||||
|
||||
// ReconcileLinks recomputes the deterministic auto edges of a live owned
|
||||
// resource (same_scan, duplicate_of, same_issuer, same_reference), creates
|
||||
// missing ones and prunes stale auto ones. User verdicts are preserved and
|
||||
// block recreation. Idempotent — safe to call on every read (GET links).
|
||||
func (r *Resources) ReconcileLinks(ownerID, resourceID string) error {
|
||||
var session sql.NullString
|
||||
var hash sql.NullString
|
||||
var size int64
|
||||
err := r.DB.QueryRow(
|
||||
`SELECT COALESCE(scan_session_id, ''), COALESCE(content_hash, ''), COALESCE(size_bytes, 0)
|
||||
FROM resources WHERE resource_id = $1 AND user_id = $2 AND deleted_at IS NULL`,
|
||||
resourceID, ownerID,
|
||||
).Scan(&session, &hash, &size)
|
||||
if err == sql.ErrNoRows {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var cands []linkCandidate
|
||||
|
||||
// same_scan : same opaque session, same user, live, not self.
|
||||
if session.Valid && session.String != "" {
|
||||
rows, err := r.DB.Query(
|
||||
`SELECT resource_id FROM resources
|
||||
WHERE user_id = $1 AND scan_session_id = $2 AND resource_id != $3 AND deleted_at IS NULL`,
|
||||
ownerID, session.String, resourceID,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var other string
|
||||
if err := rows.Scan(&other); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
cands = append(cands, linkCandidate{other, LinkSameScan, 1.0})
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// duplicate_of : same content hash (non-empty), size > 0, same user, live.
|
||||
if hash.Valid && hash.String != "" && size > 0 {
|
||||
rows, err := r.DB.Query(
|
||||
`SELECT resource_id FROM resources
|
||||
WHERE user_id = $1 AND content_hash = $2 AND size_bytes > 0
|
||||
AND resource_id != $3 AND deleted_at IS NULL`,
|
||||
ownerID, hash.String, resourceID,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var other string
|
||||
if err := rows.Scan(&other); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
cands = append(cands, linkCandidate{other, LinkDuplicateOf, 1.0})
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// same_issuer / same_reference : shared fact values (case-insensitive
|
||||
// for issuers, exact for reference codes), any origin (user corrections
|
||||
// count — they ARE the household truth). Effective facts only : un fait
|
||||
// auto est ignoré quand un fait user du même type existe sur la même
|
||||
// ressource (miroir de `displayFacts` côté mobile — sinon une correction
|
||||
// laisserait l'arête auto périmée).
|
||||
for _, typ := range []string{"issuer", "reference"} {
|
||||
linkTyp := LinkSameIssuer
|
||||
if typ == "reference" {
|
||||
linkTyp = LinkSameReference
|
||||
}
|
||||
rows, err := r.DB.Query(
|
||||
`SELECT DISTINCT o.resource_id, LEAST(f.confidence, g.confidence)
|
||||
FROM resource_facts f
|
||||
JOIN resource_facts g ON g.fact_type = f.fact_type
|
||||
AND g.resource_id != f.resource_id
|
||||
AND CASE WHEN f.fact_type = 'issuer'
|
||||
THEN LOWER(g.value_text) = LOWER(f.value_text)
|
||||
ELSE g.value_text = f.value_text END
|
||||
JOIN resources o ON o.resource_id = g.resource_id
|
||||
WHERE f.resource_id = $1 AND f.fact_type = $2 AND f.value_text IS NOT NULL
|
||||
AND (f.origin = 'user' OR NOT EXISTS (
|
||||
SELECT 1 FROM resource_facts u WHERE u.resource_id = f.resource_id
|
||||
AND u.fact_type = f.fact_type AND u.origin = 'user'))
|
||||
AND (g.origin = 'user' OR NOT EXISTS (
|
||||
SELECT 1 FROM resource_facts u WHERE u.resource_id = g.resource_id
|
||||
AND u.fact_type = g.fact_type AND u.origin = 'user'))
|
||||
AND o.user_id = $3 AND o.deleted_at IS NULL`,
|
||||
resourceID, typ, ownerID,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var other string
|
||||
var conf float64
|
||||
if err := rows.Scan(&other, &conf); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
cands = append(cands, linkCandidate{other, linkTyp, conf})
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
keep := make(map[string]bool, len(cands))
|
||||
for _, c := range cands {
|
||||
a, b := canonicalPair(resourceID, c.other)
|
||||
keep[a+"\x00"+b+"\x00"+c.typ] = true
|
||||
if err := r.ensureAutoLink(resourceID, c.other, c.typ, c.confidence); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return r.pruneAutoLinks(resourceID, keep)
|
||||
}
|
||||
|
||||
// DuplicateGroup is one set of byte-identical live files (B2 screen).
|
||||
type DuplicateGroup struct {
|
||||
ContentHash string
|
||||
Count int
|
||||
TotalBytes int64
|
||||
ReclaimableBytes int64 // total - biggest member (garder un seul)
|
||||
Files []DuplicateFile
|
||||
}
|
||||
|
||||
// DuplicateFile is one member of a duplicate group, newest first.
|
||||
type DuplicateFile struct {
|
||||
ID string
|
||||
Name string
|
||||
Size int64
|
||||
UpdatedAt int64 // ms epoch
|
||||
}
|
||||
|
||||
// ListDuplicateGroups returns the owner's byte-identical file sets
|
||||
// (content_hash non-empty, size > 0, live only), biggest groups first.
|
||||
func (r *Resources) ListDuplicateGroups(ownerID string) ([]DuplicateGroup, error) {
|
||||
rows, err := r.DB.Query(
|
||||
`SELECT content_hash, COUNT(*), COALESCE(SUM(size_bytes), 0)
|
||||
FROM resources
|
||||
WHERE user_id = $1 AND type = 'file' AND deleted_at IS NULL
|
||||
AND content_hash IS NOT NULL AND content_hash <> '' AND size_bytes > 0
|
||||
GROUP BY content_hash HAVING COUNT(*) > 1
|
||||
ORDER BY SUM(size_bytes) DESC`,
|
||||
ownerID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []DuplicateGroup{}
|
||||
for rows.Next() {
|
||||
var g DuplicateGroup
|
||||
if err := rows.Scan(&g.ContentHash, &g.Count, &g.TotalBytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
members, err := r.DB.Query(
|
||||
`SELECT resource_id, name, COALESCE(size_bytes, 0),
|
||||
(EXTRACT(EPOCH FROM updated_at) * 1000)::bigint
|
||||
FROM resources
|
||||
WHERE user_id = $1 AND type = 'file' AND deleted_at IS NULL AND content_hash = $2
|
||||
ORDER BY updated_at DESC`,
|
||||
ownerID, g.ContentHash,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var biggest int64
|
||||
for members.Next() {
|
||||
var f DuplicateFile
|
||||
if err := members.Scan(&f.ID, &f.Name, &f.Size, &f.UpdatedAt); err != nil {
|
||||
members.Close()
|
||||
return nil, err
|
||||
}
|
||||
if f.Size > biggest {
|
||||
biggest = f.Size
|
||||
}
|
||||
g.Files = append(g.Files, f)
|
||||
}
|
||||
members.Close()
|
||||
if err := members.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g.ReclaimableBytes = g.TotalBytes - biggest
|
||||
out = append(out, g)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -51,6 +51,79 @@ func isForeignKeyViolation(err error) bool {
|
||||
return errors.As(err, &pqErr) && pqErr.Code == "23503"
|
||||
}
|
||||
|
||||
// SetContentHash stores the SHA-256 of the live bytes (V3 S5 duplicates).
|
||||
// Owner-scoped, live rows. Best-effort côté service (jamais d'erreur vers
|
||||
// le client).
|
||||
func (r *Resources) SetContentHash(ownerID, resourceID, hash string) error {
|
||||
result, err := r.DB.Exec(
|
||||
`UPDATE resources SET content_hash = NULLIF($3, ''), updated_at = NOW()
|
||||
WHERE resource_id = $1 AND user_id = $2 AND type = 'file' AND deleted_at IS NULL`,
|
||||
resourceID, ownerID, hash,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetScanSession records the opaque client scan-session of a resource
|
||||
// (V3 S5 same_scan — drift informatif, jamais autorisant). Owner-scoped.
|
||||
func (r *Resources) SetScanSession(ownerID, resourceID, sessionID string) error {
|
||||
result, err := r.DB.Exec(
|
||||
`UPDATE resources SET scan_session_id = NULLIF($3, ''), updated_at = NOW()
|
||||
WHERE resource_id = $1 AND user_id = $2 AND deleted_at IS NULL`,
|
||||
resourceID, ownerID, sessionID,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MissingHash identifies a live file without content_hash (boot backfill).
|
||||
type MissingHash struct {
|
||||
UserID string
|
||||
ResourceID string
|
||||
}
|
||||
|
||||
// ListMissingHashes returns up to limit live files with bytes but no hash.
|
||||
func (r *Resources) ListMissingHashes(limit int) ([]MissingHash, error) {
|
||||
rows, err := r.DB.Query(
|
||||
`SELECT user_id, resource_id FROM resources
|
||||
WHERE type = 'file' AND deleted_at IS NULL AND user_id IS NOT NULL AND has_content = TRUE
|
||||
AND (content_hash IS NULL OR content_hash = '')
|
||||
ORDER BY updated_at ASC LIMIT $1`,
|
||||
limit,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []MissingHash
|
||||
for rows.Next() {
|
||||
var m MissingHash
|
||||
if err := rows.Scan(&m.UserID, &m.ResourceID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
type FileRow struct {
|
||||
ID string
|
||||
Name string
|
||||
|
||||
@@ -34,7 +34,9 @@ type FactDTO struct {
|
||||
|
||||
// ProcessText extracts facts from text and replaces the resource's auto
|
||||
// facts. Empty text clears stale auto facts. Never touches user corrections.
|
||||
func (e *Extract) ProcessText(resourceID, text string) error {
|
||||
// Afterwards the deterministic links are reconciled (same_issuer /
|
||||
// same_reference may have appeared or vanished) — best-effort.
|
||||
func (e *Extract) ProcessText(ownerID, resourceID, text string) error {
|
||||
raw := extract.FromText(text)
|
||||
params := make([]repository.FactParam, 0, len(raw))
|
||||
for _, f := range raw {
|
||||
@@ -59,7 +61,13 @@ func (e *Extract) ProcessText(resourceID, text string) error {
|
||||
Confidence: f.Confidence,
|
||||
})
|
||||
}
|
||||
return e.Repository.Resources.UpsertAutoFacts(resourceID, params)
|
||||
if err := e.Repository.Resources.UpsertAutoFacts(resourceID, params); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.Repository.Resources.ReconcileLinks(ownerID, resourceID); err != nil {
|
||||
log.Printf("extract: reconcile %s: %v", resourceID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Backfill processes files with OCR text but no auto facts yet (one boot
|
||||
@@ -75,7 +83,7 @@ func (e *Extract) Backfill(batch int) (int, error) {
|
||||
return total, nil
|
||||
}
|
||||
for _, p := range pending {
|
||||
if err := e.ProcessText(p.ResourceID, p.Text); err != nil {
|
||||
if err := e.ProcessText(p.UserID, p.ResourceID, p.Text); err != nil {
|
||||
log.Printf("extract: backfill %s: %v", p.ResourceID, err)
|
||||
continue
|
||||
}
|
||||
@@ -140,7 +148,14 @@ func (s *Resources) CorrectFacts(userID, resourceID string, facts []FactDTO) err
|
||||
}
|
||||
params = append(params, p)
|
||||
}
|
||||
return s.Repository.Resources.CorrectFacts(resourceID, params)
|
||||
if err := s.Repository.Resources.CorrectFacts(resourceID, params); err != nil {
|
||||
return err
|
||||
}
|
||||
// Les faits ont changé : les arêtes same_issuer/same_reference suivent.
|
||||
if err := s.Repository.Resources.ReconcileLinks(userID, resourceID); err != nil {
|
||||
log.Printf("extract: reconcile %s: %v", resourceID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateFactParam(i int, f FactDTO) (repository.FactParam, error) {
|
||||
|
||||
@@ -27,7 +27,7 @@ func TestProcessTextPersistsAutoFacts(t *testing.T) {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
|
||||
if err := ex.ProcessText(fileID, extractCorpus); err != nil {
|
||||
if err := ex.ProcessText(userID, fileID, extractCorpus); err != nil {
|
||||
t.Fatalf("process: %v", err)
|
||||
}
|
||||
facts, err := s.GetFacts(userID, fileID)
|
||||
@@ -74,11 +74,11 @@ func TestProcessTextPreservesUserCorrections(t *testing.T) {
|
||||
t.Fatalf("seed user fact: %v", err)
|
||||
}
|
||||
|
||||
if err := ex.ProcessText(fileID, extractCorpus); err != nil {
|
||||
if err := ex.ProcessText(userID, fileID, extractCorpus); err != nil {
|
||||
t.Fatalf("process: %v", err)
|
||||
}
|
||||
// Re-passe (idempotence du remplacement auto).
|
||||
if err := ex.ProcessText(fileID, extractCorpus); err != nil {
|
||||
if err := ex.ProcessText(userID, fileID, extractCorpus); err != nil {
|
||||
t.Fatalf("re-process: %v", err)
|
||||
}
|
||||
facts, err := s.GetFacts(userID, fileID)
|
||||
@@ -122,10 +122,10 @@ func TestProcessTextEmptyClearsStaleAuto(t *testing.T) {
|
||||
if err := s.Repository.Resources.InsertFile(userID, fileID, "doc.pdf", "", 0, nil, nil); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
if err := ex.ProcessText(fileID, extractCorpus); err != nil {
|
||||
if err := ex.ProcessText(userID, fileID, extractCorpus); err != nil {
|
||||
t.Fatalf("process: %v", err)
|
||||
}
|
||||
if err := ex.ProcessText(fileID, ""); err != nil {
|
||||
if err := ex.ProcessText(userID, fileID, ""); err != nil {
|
||||
t.Fatalf("clear: %v", err)
|
||||
}
|
||||
facts, err := s.GetFacts(userID, fileID)
|
||||
@@ -145,7 +145,7 @@ func TestGetFactsScope(t *testing.T) {
|
||||
if err := s.Repository.Resources.InsertFile(owner, fileID, "doc.pdf", "", 0, nil, nil); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
if err := ex.ProcessText(fileID, extractCorpus); err != nil {
|
||||
if err := ex.ProcessText(owner, fileID, extractCorpus); err != nil {
|
||||
t.Fatalf("process: %v", err)
|
||||
}
|
||||
|
||||
@@ -207,7 +207,7 @@ func TestCorrectFactsReplacesUserSet(t *testing.T) {
|
||||
if err := s.Repository.Resources.InsertFile(userID, fileID, "facture.pdf", "", 0, nil, nil); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
if err := ex.ProcessText(fileID, extractCorpus); err != nil {
|
||||
if err := ex.ProcessText(userID, fileID, extractCorpus); err != nil {
|
||||
t.Fatalf("process: %v", err)
|
||||
}
|
||||
|
||||
@@ -311,7 +311,7 @@ func TestSyncOpCorrectFacts(t *testing.T) {
|
||||
if err := s.Repository.Resources.InsertFile(userID, fileID, "facture.pdf", "", 0, nil, nil); err != nil {
|
||||
t.Fatalf("insert: %v", err)
|
||||
}
|
||||
if err := ex.ProcessText(fileID, extractCorpus); err != nil {
|
||||
if err := ex.ProcessText(userID, fileID, extractCorpus); err != nil {
|
||||
t.Fatalf("process: %v", err)
|
||||
}
|
||||
foreignID := repository.NewID()
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/kazier/backend/repository"
|
||||
)
|
||||
|
||||
// RelationDTO serializes one edge of GET /files/:id/relations (V3 S5).
|
||||
type RelationDTO struct {
|
||||
ID string `json:"id"`
|
||||
OtherID string `json:"otherId"`
|
||||
Type string `json:"type"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
CreatedBy string `json:"createdBy"`
|
||||
OtherName string `json:"otherName"`
|
||||
OtherType string `json:"otherType"`
|
||||
OtherSize int64 `json:"otherSize"`
|
||||
}
|
||||
|
||||
// validLinkTypes mirrors the resource_links CHECK (migration 000021).
|
||||
var validLinkTypes = map[string]bool{
|
||||
repository.LinkSameScan: true,
|
||||
repository.LinkDuplicateOf: true,
|
||||
repository.LinkSameIssuer: true,
|
||||
repository.LinkSameReference: true,
|
||||
}
|
||||
|
||||
// ListRelations reconciles then returns the live edges of an owned resource
|
||||
// (owner-only : shared docs don't expose the owner's graph, S5 scope).
|
||||
func (s *Resources) ListRelations(ownerID, resourceID string) ([]RelationDTO, error) {
|
||||
owned, err := s.Repo.ExistsOwner(ownerID, resourceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !owned {
|
||||
return nil, repository.ErrNotFound
|
||||
}
|
||||
if err := s.Repo.ReconcileLinks(ownerID, resourceID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.Repo.ListLinks(resourceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]RelationDTO, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
out = append(out, RelationDTO{
|
||||
ID: row.ID, OtherID: row.OtherID, Type: row.Type,
|
||||
Confidence: row.Confidence, CreatedBy: row.CreatedBy,
|
||||
OtherName: row.OtherName, OtherType: row.OtherType, OtherSize: row.OtherSize,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// LinkVerdict records an explicit verdict on an edge (owner-only, both ends
|
||||
// owned-live). Confirm → 1.0, invalidate → 0.0 (kept, blocks re-creation).
|
||||
func (s *Resources) LinkVerdict(ownerID, resourceID, otherID, linkType string, confirmed bool) error {
|
||||
if !validLinkTypes[linkType] {
|
||||
return ErrInvalidLinkType
|
||||
}
|
||||
if otherID == resourceID {
|
||||
return ErrInvalidLinkType
|
||||
}
|
||||
for _, id := range []string{resourceID, otherID} {
|
||||
owned, err := s.Repo.ExistsOwner(ownerID, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !owned {
|
||||
return repository.ErrNotFound
|
||||
}
|
||||
}
|
||||
return s.Repo.SetUserVerdict(resourceID, otherID, linkType, confirmed)
|
||||
}
|
||||
|
||||
// DuplicateGroupDTO serializes one set of byte-identical files (B2).
|
||||
type DuplicateGroupDTO struct {
|
||||
ContentHash string `json:"contentHash"`
|
||||
Count int `json:"count"`
|
||||
TotalBytes int64 `json:"totalBytes"`
|
||||
ReclaimableBytes int64 `json:"reclaimableBytes"`
|
||||
Files []DuplicateFileDTO `json:"files"`
|
||||
}
|
||||
|
||||
// DuplicateFileDTO serializes one member (newest first).
|
||||
type DuplicateFileDTO struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
UpdatedAt int64 `json:"updatedAt"`
|
||||
}
|
||||
|
||||
// ListDuplicates returns the owner's byte-identical file sets, biggest
|
||||
// first. Owner scope (own files only) — `[]` when clean.
|
||||
func (s *Resources) ListDuplicates(ownerID string) ([]DuplicateGroupDTO, error) {
|
||||
groups, err := s.Repo.ListDuplicateGroups(ownerID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]DuplicateGroupDTO, 0, len(groups))
|
||||
for _, g := range groups {
|
||||
dto := DuplicateGroupDTO{
|
||||
ContentHash: g.ContentHash, Count: g.Count,
|
||||
TotalBytes: g.TotalBytes, ReclaimableBytes: g.ReclaimableBytes,
|
||||
Files: make([]DuplicateFileDTO, 0, len(g.Files)),
|
||||
}
|
||||
for _, f := range g.Files {
|
||||
dto.Files = append(dto.Files, DuplicateFileDTO{
|
||||
ID: f.ID, Name: f.Name, Size: f.Size, UpdatedAt: f.UpdatedAt,
|
||||
})
|
||||
}
|
||||
out = append(out, dto)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ErrInvalidLinkType marks client verdict-validation failures (400).
|
||||
var ErrInvalidLinkType = errors.New("invalid link type or pair")
|
||||
|
||||
// hashPhysical computes the SHA-256 of the live bytes (safe resolution,
|
||||
// same pattern as OpenContent). Absent → ErrNotFound.
|
||||
func (s *Resources) hashPhysical(ownerID, resourceID string) (string, error) {
|
||||
if !idPattern.MatchString(ownerID) || !idPattern.MatchString(resourceID) {
|
||||
return "", repository.ErrNotFound
|
||||
}
|
||||
base, err := filepath.EvalSymlinks(s.UploadDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
matches, err := filepath.Glob(filepath.Join(s.UploadDir, ownerID, resourceID+".*"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, m := range matches {
|
||||
if !strings.HasPrefix(filepath.Base(m), resourceID+".") {
|
||||
continue
|
||||
}
|
||||
fi, err := os.Lstat(m)
|
||||
if err != nil || !fi.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
resolved, err := filepath.EvalSymlinks(m)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
rel, err := filepath.Rel(base, resolved)
|
||||
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
|
||||
continue
|
||||
}
|
||||
f, err := os.Open(resolved)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
f.Close()
|
||||
return "", err
|
||||
}
|
||||
f.Close()
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
return "", repository.ErrNotFound
|
||||
}
|
||||
|
||||
// refreshContentHash (re)computes and stores the hash after bytes arrive.
|
||||
// Best-effort : un échec ne fait jamais échouer l'upload (loggé).
|
||||
func (s *Resources) refreshContentHash(ownerID, resourceID string) {
|
||||
hash, err := s.hashPhysical(ownerID, resourceID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err := s.Repo.SetContentHash(ownerID, resourceID, hash); err != nil {
|
||||
log.Printf("content: SetContentHash(%s): %v", resourceID, err)
|
||||
return
|
||||
}
|
||||
// Des octets identiques à une autre ressource = doublon à relier.
|
||||
if err := s.Repo.ReconcileLinks(ownerID, resourceID); err != nil {
|
||||
log.Printf("content: reconcile(%s): %v", resourceID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// BackfillHashes hashes live files missing content_hash (bounded batches,
|
||||
// boot pass — uploads d'avant la V3). Returns files hashed.
|
||||
func (s *Resources) BackfillHashes(batch int) (int, error) {
|
||||
total := 0
|
||||
for {
|
||||
pending, err := s.Repo.ListMissingHashes(batch)
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
if len(pending) == 0 {
|
||||
return total, nil
|
||||
}
|
||||
for _, p := range pending {
|
||||
hash, err := s.hashPhysical(p.UserID, p.ResourceID)
|
||||
if err != nil {
|
||||
continue // metadata-only : rien à hasher, re-tenté au prochain upload.
|
||||
}
|
||||
if err := s.Repo.SetContentHash(p.UserID, p.ResourceID, hash); err != nil {
|
||||
log.Printf("content: backfill hash %s: %v", p.ResourceID, err)
|
||||
continue
|
||||
}
|
||||
if err := s.Repo.ReconcileLinks(p.UserID, p.ResourceID); err != nil {
|
||||
log.Printf("content: backfill reconcile %s: %v", p.ResourceID, err)
|
||||
}
|
||||
total++
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/kazier/backend/repository"
|
||||
)
|
||||
|
||||
func mustLinkFile(t *testing.T, s *Resources, userID, name string) string {
|
||||
t.Helper()
|
||||
id := repository.NewID()
|
||||
if err := s.Repository.Resources.InsertFile(userID, id, name, "", 0, nil, nil); err != nil {
|
||||
t.Fatalf("insert %q: %v", name, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func relationTypes(rels []RelationDTO) map[string]int {
|
||||
out := map[string]int{}
|
||||
for _, r := range rels {
|
||||
out[r.Type]++
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestSameScanViaOutbox(t *testing.T) {
|
||||
s := newServiceStore(t)
|
||||
userID := mustCreateUser(t, s.Repository, "links-scan")
|
||||
device := mustRegisterDevice(t, s.Repository, "d0000000000000000000000000000001")
|
||||
session := repository.NewID()
|
||||
page1 := repository.NewID()
|
||||
page2 := repository.NewID()
|
||||
|
||||
create := func(opID, resID, name string) {
|
||||
t.Helper()
|
||||
res, err := s.ApplyBatch(userID, device, []SyncOperation{
|
||||
syncOp(opID, resID, OpCreateResource, "file", map[string]any{
|
||||
"name": resID + "-" + name, "scanSessionId": session,
|
||||
}),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply: %v", err)
|
||||
}
|
||||
if res.Applied != 1 || res.Failed != nil {
|
||||
t.Fatalf("create %s: %+v", name, res)
|
||||
}
|
||||
}
|
||||
create(repository.NewID(), page1, "p1.png")
|
||||
create(repository.NewID(), page2, "p2.png")
|
||||
|
||||
rels, err := s.ListRelations(userID, page1)
|
||||
if err != nil {
|
||||
t.Fatalf("relations: %v", err)
|
||||
}
|
||||
if relationTypes(rels)[repository.LinkSameScan] != 1 || rels[0].OtherID != page2 {
|
||||
t.Fatalf("same_scan manquant: %+v", rels)
|
||||
}
|
||||
if rels[0].Confidence != 1.0 || rels[0].CreatedBy != "auto" {
|
||||
t.Fatalf("arête same_scan inattendue: %+v", rels[0])
|
||||
}
|
||||
|
||||
// Session invalide → halt INVALID_REQUEST (pas d'écriture partielle).
|
||||
res, err := s.ApplyBatch(userID, device, []SyncOperation{
|
||||
syncOp(repository.NewID(), repository.NewID(), OpCreateResource, "file", map[string]any{
|
||||
"name": "bad.png", "scanSessionId": "nope",
|
||||
}),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply: %v", err)
|
||||
}
|
||||
if res.Failed == nil || res.Failed.Code != "INVALID_REQUEST" {
|
||||
t.Fatalf("session invalide doit halter: %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuplicateOfViaUploadHash(t *testing.T) {
|
||||
s := newServiceStore(t)
|
||||
userID := mustCreateUser(t, s.Repository, "links-dupe")
|
||||
|
||||
a, err := s.Upload(userID, multipartFileHeader(t, "a.txt", []byte("meme-contenu")), "", "")
|
||||
if err != nil {
|
||||
t.Fatalf("upload a: %v", err)
|
||||
}
|
||||
b, err := s.Upload(userID, multipartFileHeader(t, "b.txt", []byte("meme-contenu")), "", "")
|
||||
if err != nil {
|
||||
t.Fatalf("upload b: %v", err)
|
||||
}
|
||||
|
||||
rels, err := s.ListRelations(userID, a.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("relations: %v", err)
|
||||
}
|
||||
if relationTypes(rels)[repository.LinkDuplicateOf] != 1 || rels[0].OtherID != b.ID {
|
||||
t.Fatalf("duplicate_of manquant: %+v", rels)
|
||||
}
|
||||
|
||||
// Contenu différent → pas de doublon.
|
||||
c, err := s.Upload(userID, multipartFileHeader(t, "c.txt", []byte("autre-contenu")), "", "")
|
||||
if err != nil {
|
||||
t.Fatalf("upload c: %v", err)
|
||||
}
|
||||
relsC, err := s.ListRelations(userID, c.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("relations c: %v", err)
|
||||
}
|
||||
if len(relsC) != 0 {
|
||||
t.Fatalf("faux doublon: %+v", relsC)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSameIssuerViaFacts(t *testing.T) {
|
||||
s, ex := newExtractStore(t)
|
||||
userID := mustCreateUser(t, s.Repository, "links-issuer")
|
||||
f1 := mustLinkFile(t, s, userID, "f1.pdf")
|
||||
f2 := mustLinkFile(t, s, userID, "f2.pdf")
|
||||
|
||||
if err := ex.ProcessText(userID, f1, "EDF\nFacture 10,00 €\n"); err != nil {
|
||||
t.Fatalf("extract f1: %v", err)
|
||||
}
|
||||
if err := ex.ProcessText(userID, f2, "EDF\nFacture 20,00 €\n"); err != nil {
|
||||
t.Fatalf("extract f2: %v", err)
|
||||
}
|
||||
|
||||
rels, err := s.ListRelations(userID, f1)
|
||||
if err != nil {
|
||||
t.Fatalf("relations: %v", err)
|
||||
}
|
||||
if relationTypes(rels)[repository.LinkSameIssuer] != 1 {
|
||||
t.Fatalf("same_issuer manquant: %+v", rels)
|
||||
}
|
||||
|
||||
// Correction de l'émetteur → l'arête disparaît (prune auto).
|
||||
if err := s.CorrectFacts(userID, f2, []FactDTO{{FactType: "issuer", ValueText: "ENGIE", Confidence: 1}}); err != nil {
|
||||
t.Fatalf("correct: %v", err)
|
||||
}
|
||||
rels, err = s.ListRelations(userID, f1)
|
||||
if err != nil {
|
||||
t.Fatalf("relations 2: %v", err)
|
||||
}
|
||||
if relationTypes(rels)[repository.LinkSameIssuer] != 0 {
|
||||
t.Fatalf("arête périmée non élaguée: %+v", rels)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLinkVerdict(t *testing.T) {
|
||||
s := newServiceStore(t)
|
||||
userID := mustCreateUser(t, s.Repository, "links-verdict")
|
||||
other := mustCreateUser(t, s.Repository, "links-verdict-x")
|
||||
device := mustRegisterDevice(t, s.Repository, "d0000000000000000000000000000002")
|
||||
session := repository.NewID()
|
||||
p1, p2 := repository.NewID(), repository.NewID()
|
||||
for _, id := range []string{p1, p2} {
|
||||
res, err := s.ApplyBatch(userID, device, []SyncOperation{
|
||||
syncOp(repository.NewID(), id, OpCreateResource, "file", map[string]any{
|
||||
"name": id, "scanSessionId": session,
|
||||
}),
|
||||
})
|
||||
if err != nil || res.Applied != 1 || res.Failed != nil {
|
||||
t.Fatalf("create %s: %+v %v", id, res, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Invalider : l'arête reste (confiance 0, verdict user) mais ne bloque rien.
|
||||
if err := s.LinkVerdict(userID, p1, p2, repository.LinkSameScan, false); err != nil {
|
||||
t.Fatalf("verdict: %v", err)
|
||||
}
|
||||
rels, _ := s.ListRelations(userID, p1)
|
||||
if len(rels) != 1 || rels[0].Confidence != 0 || rels[0].CreatedBy != "user" {
|
||||
t.Fatalf("verdict infirme inattendu: %+v", rels)
|
||||
}
|
||||
// Re-reconcile ne ressuscite pas l'arête invalidée.
|
||||
rels, _ = s.ListRelations(userID, p1)
|
||||
if len(rels) != 1 || rels[0].Confidence != 0 {
|
||||
t.Fatalf("arête invalidée ressuscitée: %+v", rels)
|
||||
}
|
||||
|
||||
// Re-confirmer : retour à 1.0 user.
|
||||
if err := s.LinkVerdict(userID, p1, p2, repository.LinkSameScan, true); err != nil {
|
||||
t.Fatalf("re-verdict: %v", err)
|
||||
}
|
||||
rels, _ = s.ListRelations(userID, p1)
|
||||
if len(rels) != 1 || rels[0].Confidence != 1.0 || rels[0].CreatedBy != "user" {
|
||||
t.Fatalf("re-verdict inattendu: %+v", rels)
|
||||
}
|
||||
|
||||
// Type inconnu → erreur 400-candidate ; paire réflexive idem.
|
||||
if err := s.LinkVerdict(userID, p1, p2, "copains", true); !errors.Is(err, ErrInvalidLinkType) {
|
||||
t.Fatalf("type inconnu : attendu ErrInvalidLinkType, got %v", err)
|
||||
}
|
||||
if err := s.LinkVerdict(userID, p1, p1, repository.LinkSameScan, true); !errors.Is(err, ErrInvalidLinkType) {
|
||||
t.Fatalf("paire réflexive : attendu ErrInvalidLinkType, got %v", err)
|
||||
}
|
||||
// Autrui → NOT_FOUND (pas d'oracle).
|
||||
foreign := mustLinkFile(t, s, other, "x.pdf")
|
||||
if err := s.LinkVerdict(other, foreign, p1, repository.LinkSameScan, true); err != repository.ErrNotFound {
|
||||
t.Fatalf("verdict trans-compte : attendu NOT_FOUND, got %v", err)
|
||||
}
|
||||
if _, err := s.ListRelations(other, p1); err != repository.ErrNotFound {
|
||||
t.Fatalf("lecture trans-compte : attendu NOT_FOUND, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListDuplicates(t *testing.T) {
|
||||
s := newServiceStore(t)
|
||||
userID := mustCreateUser(t, s.Repository, "links-dupes")
|
||||
other := mustCreateUser(t, s.Repository, "links-dupes-x")
|
||||
|
||||
a, err := s.Upload(userID, multipartFileHeader(t, "a.txt", []byte("doublon")), "", "")
|
||||
if err != nil {
|
||||
t.Fatalf("upload a: %v", err)
|
||||
}
|
||||
b, err := s.Upload(userID, multipartFileHeader(t, "b.txt", []byte("doublon")), "", "")
|
||||
if err != nil {
|
||||
t.Fatalf("upload b: %v", err)
|
||||
}
|
||||
if _, err := s.Upload(userID, multipartFileHeader(t, "c.txt", []byte("unique-contenu")), "", ""); err != nil {
|
||||
t.Fatalf("upload c: %v", err)
|
||||
}
|
||||
// Même contenu chez un autre compte : pas mon doublon.
|
||||
if _, err := s.Upload(other, multipartFileHeader(t, "z.txt", []byte("doublon")), "", ""); err != nil {
|
||||
t.Fatalf("upload other: %v", err)
|
||||
}
|
||||
|
||||
groups, err := s.ListDuplicates(userID)
|
||||
if err != nil {
|
||||
t.Fatalf("duplicates: %v", err)
|
||||
}
|
||||
if len(groups) != 1 || groups[0].Count != 2 {
|
||||
t.Fatalf("groupes inattendus: %+v", groups)
|
||||
}
|
||||
g := groups[0]
|
||||
if g.TotalBytes != 14 || g.ReclaimableBytes != 7 {
|
||||
t.Fatalf("tailles inattendues: %+v", g)
|
||||
}
|
||||
ids := map[string]bool{}
|
||||
for _, f := range g.Files {
|
||||
ids[f.ID] = true
|
||||
}
|
||||
if !ids[a.ID] || !ids[b.ID] {
|
||||
t.Fatalf("membres inattendus: %+v", g.Files)
|
||||
}
|
||||
|
||||
// Compte propre → [] (jamais null).
|
||||
empty, err := s.ListDuplicates(other)
|
||||
if err != nil {
|
||||
t.Fatalf("duplicates other: %v", err)
|
||||
}
|
||||
if len(empty) != 0 {
|
||||
t.Fatalf("compte sans doublon: %+v", empty)
|
||||
}
|
||||
}
|
||||
@@ -412,6 +412,9 @@ func (s *Resources) Upload(ownerID string, file *multipart.FileHeader, folderID,
|
||||
}
|
||||
return FileDTO{}, err
|
||||
}
|
||||
// V3 S5 : octets frais → hash (doublons) + réconciliation des liens.
|
||||
// Best-effort (jamais d'échec sur du hashing).
|
||||
s.refreshContentHash(ownerID, resourceID)
|
||||
return toFileDTO(row), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -64,6 +64,9 @@ type createResourcePayload struct {
|
||||
ParentResourceID string `json:"parentResourceId"` // dossier parent (racine si vide), cf. docs/api-v1.md §6.1
|
||||
MimeType string `json:"mimeType"`
|
||||
Extension string `json:"extension"`
|
||||
// V3 S5 : session de scan opaque (32-hex mobile) — le serveur en déduit
|
||||
// les arêtes same_scan. Absent = ressource isolée (jamais d'erreur).
|
||||
ScanSessionID string `json:"scanSessionId"`
|
||||
}
|
||||
|
||||
type renamePayload struct {
|
||||
@@ -241,6 +244,11 @@ func (s *Resources) applySyncOp(ownerID, deviceID string, op *SyncOperation) err
|
||||
if p.Name == "" {
|
||||
return errors.New("payload.name required")
|
||||
}
|
||||
// V3 S5 : session de scan opaque (dérive same_scan, jamais d'erreur
|
||||
// bloquante au-delà du format).
|
||||
if p.ScanSessionID != "" && !resourceIDPattern.MatchString(p.ScanSessionID) {
|
||||
return errors.New("payload.scanSessionId must be 32 lowercase hex chars")
|
||||
}
|
||||
// Le parent doit exister et appartenir à l'utilisateur (NOT_FOUND sinon,
|
||||
// cohérent avec move_resource). Ordre du SAF walk : parent avant enfant.
|
||||
if p.ParentResourceID != "" {
|
||||
@@ -270,6 +278,15 @@ func (s *Resources) applySyncOp(ownerID, deviceID string, op *SyncOperation) err
|
||||
}
|
||||
return insertErr
|
||||
}
|
||||
// V3 S5 : session de scan → arêtes same_scan (best-effort, jamais
|
||||
// bloquant : la ressource existe, seuls les liens manqueraient).
|
||||
if p.ScanSessionID != "" {
|
||||
if err := s.Repo.SetScanSession(ownerID, resourceID, p.ScanSessionID); err != nil {
|
||||
log.Printf("sync: scan session %s: %v", resourceID, err)
|
||||
} else if err := s.Repo.ReconcileLinks(ownerID, resourceID); err != nil {
|
||||
log.Printf("sync: reconcile %s: %v", resourceID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
case OpUpdateMetadata:
|
||||
|
||||
Reference in New Issue
Block a user