From 7b3f20b8d7c0dbb35ca45cd5b46b3b03a5021280 Mon Sep 17 00:00:00 2001 From: venture Date: Mon, 16 Nov 2026 11:03:48 +0100 Subject: [PATCH] ingest documents --- content/version.go | 26 +++ content/version_test.go | 46 ++++++ database/query.sql.go | 45 ++++++ database/query/query.sql | 9 ++ document/create.go | 79 +++++++++ document/create_test.go | 108 +++++++++++++ document/document.go | 20 ++- document/mutate.go | 31 ++++ document/mutate_test.go | 23 +++ document/service.go | 9 ++ ingest/ingest.go | 150 ++++++++++++++++++ ingest/ingest_test.go | 71 +++++++++ ingest/scan.go | 335 +++++++++++++++++++++++++++++++++++++++ node/node_ingest.go | 7 + node/node_ingest_test.go | 191 ++++++++++++++++++++++ 15 files changed, 1148 insertions(+), 2 deletions(-) create mode 100644 ingest/ingest.go create mode 100644 ingest/ingest_test.go create mode 100644 ingest/scan.go create mode 100644 node/node_ingest.go create mode 100644 node/node_ingest_test.go diff --git a/content/version.go b/content/version.go index 3c9daef..4b78770 100644 --- a/content/version.go +++ b/content/version.go @@ -400,6 +400,32 @@ func (s *Service) ListVersions(ctx context.Context, actorID, documentID ident.UU return versions, nil } +func (s *Service) HeadVersion(ctx context.Context, actorID, documentID ident.UUID) (*Version, error) { + documentRow, err := s.getDocument(ctx, documentID) + if err != nil { + return nil, err + } + if documentRow.DeletedAt.Valid { + return nil, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, documentID) + } + if err := s.can(ctx, actorID, documentID, permission.Read); err != nil { + return nil, err + } + if documentRow.HeadVersionID == nil { + return nil, nil + } + + version, err := s.getVersion(ctx, *documentRow.HeadVersionID) + if err != nil { + return nil, err + } + if version.DocumentID != documentID { + return nil, fmt.Errorf("%w: %s", ErrVersionMismatch, version.ID) + } + + return versionFromDB(*version), nil +} + func boolToInt(value bool) int64 { if value { return 1 diff --git a/content/version_test.go b/content/version_test.go index 9af72b1..70b6f01 100644 --- a/content/version_test.go +++ b/content/version_test.go @@ -196,3 +196,49 @@ func TestCreateVersionConfigurationErrors(t *testing.T) { t.Fatalf("sans stockage : erreur = %v, attendu ErrStorageNotConfigured", err) } } + +func TestHeadVersion(t *testing.T) { + env := newTestEnv(t, allowAll()) + ctx := context.Background() + + head, err := env.service.HeadVersion(ctx, env.owner.ID, env.videoID) + if err != nil { + t.Fatalf("HeadVersion(sans version) : %v", err) + } + if head != nil { + t.Fatalf("HeadVersion(sans version) = %+v, attendu nil", head) + } + + first, err := env.service.CreateVersion(ctx, env.owner.ID, env.videoID, CreateVersionParams{ + Source: bytes.NewReader([]byte("premier")), + }) + if err != nil { + t.Fatalf("CreateVersion : %v", err) + } + head, err = env.service.HeadVersion(ctx, env.owner.ID, env.videoID) + if err != nil { + t.Fatalf("HeadVersion : %v", err) + } + if head == nil || head.ID != first.ID { + t.Fatalf("HeadVersion = %+v, attendu %s", head, first.ID) + } + + second, err := env.service.CreateVersion(ctx, env.owner.ID, env.videoID, CreateVersionParams{ + Source: bytes.NewReader([]byte("second")), + }) + if err != nil { + t.Fatalf("CreateVersion(second) : %v", err) + } + head, err = env.service.HeadVersion(ctx, env.owner.ID, env.videoID) + if err != nil { + t.Fatalf("HeadVersion(second) : %v", err) + } + if head == nil || head.ID != second.ID { + t.Fatalf("HeadVersion = %+v, attendu %s", head, second.ID) + } + + readerOnly := NewService(*env.db, env.store, testBackendID, &testAuthorizer{allowed: permission.Write}) + if _, err := readerOnly.HeadVersion(ctx, env.owner.ID, env.videoID); !errors.Is(err, document.ErrPermissionDenied) { + t.Fatalf("HeadVersion sans Read : erreur = %v, attendu ErrPermissionDenied", err) + } +} diff --git a/database/query.sql.go b/database/query.sql.go index db5a242..8341123 100644 --- a/database/query.sql.go +++ b/database/query.sql.go @@ -2481,6 +2481,51 @@ func (q *Queries) UpdateDocumentLocation(ctx context.Context, arg UpdateDocument return err } +const updateDocumentMetadata = `-- name: UpdateDocumentMetadata :one +UPDATE document +SET metadata = ?1, + rev = rev + 1, + updated_at = ?2 +WHERE id = ?3 + AND deleted_at IS NULL +RETURNING id, kind, owner_id, created_by, parent_id, name, name_key, path, depth, position, head_version_id, metadata, policy_id, visibility, acl_inherit, rev, created_at, updated_at, deleted_at, deleted_by, trash_batch +` + +type UpdateDocumentMetadataParams struct { + Metadata string + UpdatedAt int64 + ID ident.UUID +} + +func (q *Queries) UpdateDocumentMetadata(ctx context.Context, arg UpdateDocumentMetadataParams) (Document, error) { + row := q.db.QueryRowContext(ctx, updateDocumentMetadata, arg.Metadata, arg.UpdatedAt, arg.ID) + var i Document + err := row.Scan( + &i.ID, + &i.Kind, + &i.OwnerID, + &i.CreatedBy, + &i.ParentID, + &i.Name, + &i.NameKey, + &i.Path, + &i.Depth, + &i.Position, + &i.HeadVersionID, + &i.Metadata, + &i.PolicyID, + &i.Visibility, + &i.AclInherit, + &i.Rev, + &i.CreatedAt, + &i.UpdatedAt, + &i.DeletedAt, + &i.DeletedBy, + &i.TrashBatch, + ) + return i, err +} + const updateDocumentName = `-- name: UpdateDocumentName :one UPDATE document SET name = ?1, diff --git a/database/query/query.sql b/database/query/query.sql index 7dc2095..031f02b 100644 --- a/database/query/query.sql +++ b/database/query/query.sql @@ -595,6 +595,15 @@ WHERE id = sqlc.arg(id) AND deleted_at IS NULL RETURNING *; +-- name: UpdateDocumentMetadata :one +UPDATE document +SET metadata = sqlc.arg(metadata), + rev = rev + 1, + updated_at = sqlc.arg(updated_at) +WHERE id = sqlc.arg(id) + AND deleted_at IS NULL +RETURNING *; + -- name: MoveDocument :one UPDATE document SET parent_id = sqlc.arg(parent_id), diff --git a/document/create.go b/document/create.go index 9dddfbe..ce817b2 100644 --- a/document/create.go +++ b/document/create.go @@ -131,3 +131,82 @@ func (s *Service) CreateFolder(ctx context.Context, actorID ident.UUID, params C return documentFromDB(created), nil } + +func (s *Service) CreateChild(ctx context.Context, actorID ident.UUID, params CreateChildParams) (*Document, error) { + if err := s.requireActive(ctx, actorID); err != nil { + return nil, err + } + if params.OwnerID != actorID || params.CreatedBy != actorID { + return nil, fmt.Errorf("%w: owner_id et created_by doivent être l'acteur", ErrPermissionDenied) + } + + name, nameKey, err := normalizeName(params.Name) + if err != nil { + return nil, err + } + + parent, err := s.getDocument(ctx, params.ParentID) + if err != nil { + if errors.Is(err, ErrDocumentNotFound) { + return nil, fmt.Errorf("%w: %s", ErrParentNotFound, params.ParentID) + } + return nil, err + } + if parent.DeletedAt.Valid { + return nil, fmt.Errorf("%w: %s", ErrParentDeleted, params.ParentID) + } + if !isContainerKind(parent.Kind) { + return nil, fmt.Errorf("%w: %s", ErrInvalidParent, parent.Kind) + } + if err := s.can(ctx, actorID, params.ParentID, permission.Create); err != nil { + return nil, err + } + + kind, err := s.db.Queries.GetDocumentKind(ctx, params.Kind) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: %s", ErrInvalidKind, params.Kind) + } + return nil, err + } + if kind.Kind == KindLibrary || kind.Hidden != 0 { + return nil, fmt.Errorf("%w: %s", ErrInvalidKind, params.Kind) + } + + metadata, err := normalizeMetadata(params.Metadata) + if err != nil { + return nil, err + } + + id, err := ident.NewV7() + if err != nil { + return nil, err + } + + now := nowMillis() + + created, err := s.db.WriteQueries.CreateDocument(ctx, database.CreateDocumentParams{ + ID: id, + Kind: kind.Kind, + OwnerID: params.OwnerID, + CreatedBy: params.CreatedBy, + ParentID: ¶ms.ParentID, + Name: name, + NameKey: nameKey, + Path: parent.Path + hex.EncodeToString(id[:]) + "/", + Depth: parent.Depth + 1, + Metadata: metadata, + AclInherit: aclInheritValue(params.ACLInherit), + Rev: 1, + CreatedAt: now, + UpdatedAt: now, + }) + if err != nil { + if isUniqueViolation(err) { + return nil, fmt.Errorf("%w: %s", ErrNameTaken, name) + } + return nil, err + } + + return documentFromDB(created), nil +} diff --git a/document/create_test.go b/document/create_test.go index 641f186..ecb50f4 100644 --- a/document/create_test.go +++ b/document/create_test.go @@ -280,3 +280,111 @@ func TestCreateFolderRejectsDuplicateSibling(t *testing.T) { t.Fatalf("erreur = %v, attendu ErrNameTaken", err) } } + +func TestCreateChild(t *testing.T) { + service := newTestService(t) + ctx := context.Background() + owner := mustUser(t, service, "owner") + library := mustLibrary(t, service, owner, "Photos") + + video, err := service.CreateChild(ctx, owner.ID, CreateChildParams{ + Kind: KindVideo, + OwnerID: owner.ID, + CreatedBy: owner.ID, + ParentID: library.ID, + Name: "film.mkv", + Metadata: `{"source_path":"film.mkv"}`, + }) + if err != nil { + t.Fatalf("CreateChild(video) : %v", err) + } + if video.Kind != KindVideo { + t.Fatalf("kind = %q, attendu %q", video.Kind, KindVideo) + } + if video.ParentID == nil || *video.ParentID != library.ID { + t.Fatalf("parent = %v, attendu %s", video.ParentID, library.ID) + } + if video.Depth != library.Depth+1 { + t.Fatalf("depth = %d, attendu %d", video.Depth, library.Depth+1) + } + if video.Metadata != `{"source_path":"film.mkv"}` { + t.Fatalf("metadata = %q", video.Metadata) + } + + folder, err := service.CreateChild(ctx, owner.ID, CreateChildParams{ + Kind: KindFolder, + OwnerID: owner.ID, + CreatedBy: owner.ID, + ParentID: library.ID, + Name: "Extras", + }) + if err != nil { + t.Fatalf("CreateChild(folder) : %v", err) + } + if folder.Kind != KindFolder { + t.Fatalf("kind = %q, attendu %q", folder.Kind, KindFolder) + } + if folder.Metadata != "{}" { + t.Fatalf("metadata = %q, attendu {}", folder.Metadata) + } +} + +func TestCreateChildRejectsInvalidInputs(t *testing.T) { + service := newTestService(t) + ctx := context.Background() + owner := mustUser(t, service, "owner") + other := mustUser(t, service, "other") + library := mustLibrary(t, service, owner, "Photos") + + if _, err := service.CreateChild(ctx, owner.ID, CreateChildParams{ + Kind: KindVideo, + OwnerID: other.ID, + CreatedBy: owner.ID, + ParentID: library.ID, + Name: "film.mkv", + }); !errors.Is(err, ErrPermissionDenied) { + t.Fatalf("owner usurpé : erreur = %v, attendu ErrPermissionDenied", err) + } + + if _, err := service.CreateChild(ctx, owner.ID, CreateChildParams{ + Kind: "inconnu", + OwnerID: owner.ID, + CreatedBy: owner.ID, + ParentID: library.ID, + Name: "film.mkv", + }); !errors.Is(err, ErrInvalidKind) { + t.Fatalf("kind invalide : erreur = %v, attendu ErrInvalidKind", err) + } + + if _, err := service.CreateChild(ctx, owner.ID, CreateChildParams{ + Kind: KindVideo, + OwnerID: owner.ID, + CreatedBy: owner.ID, + ParentID: library.ID, + Name: "film.mkv", + Metadata: "{", + }); !errors.Is(err, ErrInvalidMetadata) { + t.Fatalf("metadata invalide : erreur = %v, attendu ErrInvalidMetadata", err) + } + + video, err := service.CreateChild(ctx, owner.ID, CreateChildParams{ + Kind: KindVideo, + OwnerID: owner.ID, + CreatedBy: owner.ID, + ParentID: library.ID, + Name: "film.mkv", + }) + if err != nil { + t.Fatalf("CreateChild(video) : %v", err) + } + + if _, err := service.CreateChild(ctx, owner.ID, CreateChildParams{ + Kind: KindVideo, + OwnerID: owner.ID, + CreatedBy: owner.ID, + ParentID: video.ID, + Name: "enfant.mkv", + }); !errors.Is(err, ErrInvalidParent) { + t.Fatalf("parent non conteneur : erreur = %v, attendu ErrInvalidParent", err) + } +} diff --git a/document/document.go b/document/document.go index 1d31346..14ab23b 100644 --- a/document/document.go +++ b/document/document.go @@ -14,8 +14,12 @@ const ( ) const ( - KindLibrary = "library" - KindFolder = "folder" + KindLibrary = "library" + KindFolder = "folder" + KindVideo = "video" + KindSubtitle = "subtitle" + KindImage = "image" + KindFile = "file" ) type Document struct { @@ -61,6 +65,16 @@ type CreateFolderParams struct { Metadata string } +type CreateChildParams struct { + Kind string + OwnerID ident.UUID + CreatedBy ident.UUID + ParentID ident.UUID + Name string + ACLInherit *bool + Metadata string +} + var ( ErrInvalidName = errors.New("nom de document invalide") ErrNameTaken = errors.New("nom de document déjà utilisé") @@ -72,6 +86,8 @@ var ( ErrCannotMoveRoot = errors.New("déplacement d'une bibliothèque racine interdit") ErrInvalidMove = errors.New("déplacement invalide") ErrInvalidVisibility = errors.New("visibilité invalide") + ErrInvalidKind = errors.New("type de document invalide") + ErrInvalidMetadata = errors.New("métadonnées invalides") ErrPolicyNotFound = errors.New("politique de stockage introuvable") ErrInvalidTrashBatch = errors.New("lot de corbeille invalide") ErrPermissionDenied = errors.New("permission refusée") diff --git a/document/mutate.go b/document/mutate.go index 75c7260..35b74eb 100644 --- a/document/mutate.go +++ b/document/mutate.go @@ -46,6 +46,37 @@ func (s *Service) Rename(ctx context.Context, actorID, documentID ident.UUID, na return documentFromDB(updated), nil } +func (s *Service) SetMetadata(ctx context.Context, actorID, documentID ident.UUID, metadata string) (*Document, error) { + if err := s.requireActive(ctx, actorID); err != nil { + return nil, err + } + if _, err := s.getActiveDocument(ctx, documentID); err != nil { + return nil, err + } + if err := s.can(ctx, actorID, documentID, permission.Write); err != nil { + return nil, err + } + + normalized, err := normalizeMetadata(metadata) + if err != nil { + return nil, err + } + + updated, err := s.db.WriteQueries.UpdateDocumentMetadata(ctx, database.UpdateDocumentMetadataParams{ + Metadata: normalized, + UpdatedAt: nowMillis(), + ID: documentID, + }) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: %s", ErrDocumentNotFound, documentID) + } + return nil, err + } + + return documentFromDB(updated), nil +} + func (s *Service) Move(ctx context.Context, actorID, documentID, newParentID ident.UUID) (*Document, error) { document, err := s.getActiveDocument(ctx, documentID) if err != nil { diff --git a/document/mutate_test.go b/document/mutate_test.go index 80bdadb..05fa720 100644 --- a/document/mutate_test.go +++ b/document/mutate_test.go @@ -400,3 +400,26 @@ func TestMoveUpdatesTrashedDescendants(t *testing.T) { t.Fatalf("depth = %d, attendu 2", restored.Depth) } } + +func TestSetMetadata(t *testing.T) { + service := newTestService(t) + ctx := context.Background() + owner := mustUser(t, service, "owner") + library := mustLibrary(t, service, owner, "Photos") + video := mustKindDocument(t, service, owner, library, KindVideo, "film.mkv") + + updated, err := service.SetMetadata(ctx, owner.ID, video.ID, `{"duration_ms":120000}`) + if err != nil { + t.Fatalf("SetMetadata : %v", err) + } + if updated.Metadata != `{"duration_ms":120000}` { + t.Fatalf("metadata = %q", updated.Metadata) + } + if updated.Rev != video.Rev+1 { + t.Fatalf("rev = %d, attendu %d", updated.Rev, video.Rev+1) + } + + if _, err := service.SetMetadata(ctx, owner.ID, video.ID, "{"); !errors.Is(err, ErrInvalidMetadata) { + t.Fatalf("metadata invalide : erreur = %v, attendu ErrInvalidMetadata", err) + } +} diff --git a/document/service.go b/document/service.go index 4cb406e..77726f4 100644 --- a/document/service.go +++ b/document/service.go @@ -3,6 +3,7 @@ package document import ( "context" "database/sql" + "encoding/json" "errors" "fmt" "strings" @@ -136,6 +137,14 @@ func metadataValue(metadata string) string { return metadata } +func normalizeMetadata(metadata string) (string, error) { + value := metadataValue(metadata) + if !json.Valid([]byte(value)) { + return "", ErrInvalidMetadata + } + return value, nil +} + func aclInheritValue(value *bool) int64 { if value == nil || *value { return 1 diff --git a/ingest/ingest.go b/ingest/ingest.go new file mode 100644 index 0000000..5cd3df0 --- /dev/null +++ b/ingest/ingest.go @@ -0,0 +1,150 @@ +package ingest + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "mime" + "path/filepath" + "strings" + + "mediarr/content" + "mediarr/document" +) + +const ( + KindVideo = document.KindVideo + KindSubtitle = document.KindSubtitle + KindImage = document.KindImage + KindFile = document.KindFile +) + +type Probe struct { + Mime string + Metadata map[string]any +} + +type Prober interface { + Probe(ctx context.Context, path string, info fs.FileInfo) (Probe, error) +} + +type BasicProber struct{} + +func (BasicProber) Probe(_ context.Context, path string, _ fs.FileInfo) (Probe, error) { + mimeType := mime.TypeByExtension(strings.ToLower(filepath.Ext(path))) + if mimeType == "" { + mimeType = "application/octet-stream" + } + + return Probe{Mime: mimeType}, nil +} + +type Service struct { + documents *document.Service + contentService *content.Service + prober Prober +} + +func NewService(documents *document.Service, contentService *content.Service, prober Prober) *Service { + if prober == nil { + prober = BasicProber{} + } + + return &Service{ + documents: documents, + contentService: contentService, + prober: prober, + } +} + +type ScanResult struct { + Files int + Directories int + Created int + Updated int + Unchanged int + Errors []ScanError +} + +func (r ScanResult) HasErrors() bool { + return len(r.Errors) > 0 +} + +type ScanError struct { + Path string + Err error +} + +func (e ScanError) Error() string { + return fmt.Sprintf("%s: %v", e.Path, e.Err) +} + +func (e ScanError) Unwrap() error { + return e.Err +} + +var ( + ErrServiceNotConfigured = errors.New("service d'ingestion non configuré") + ErrInvalidRoot = errors.New("racine de scan invalide") + ErrNotFolder = errors.New("document existant n'est pas un dossier") + ErrNotFile = errors.New("document existant n'est pas un fichier") + ErrNameConflict = errors.New("conflit de nom insensible à la casse") +) + +func KindForFile(name string) string { + switch strings.ToLower(filepath.Ext(name)) { + case ".mkv", ".mp4", ".m4v", ".avi", ".mov", ".webm", ".ts", ".m2ts", ".wmv", ".flv", ".mpg", ".mpeg": + return KindVideo + case ".srt", ".ass", ".ssa", ".vtt", ".sub": + return KindSubtitle + case ".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tif", ".tiff", ".heic", ".avif": + return KindImage + default: + return KindFile + } +} + +func metadataJSON(existing string, updates map[string]any) (string, error) { + values := make(map[string]any) + if existing != "" { + if err := json.Unmarshal([]byte(existing), &values); err != nil { + return "", fmt.Errorf("%w: %v", document.ErrInvalidMetadata, err) + } + if values == nil { + values = make(map[string]any) + } + } + + for key, value := range updates { + values[key] = value + } + + encoded, err := json.Marshal(values) + if err != nil { + return "", err + } + + return string(encoded), nil +} + +func fileMetadata(existing, relPath string, info fs.FileInfo, probe Probe) (string, error) { + updates := map[string]any{ + "source_path": relPath, + "size": info.Size(), + "mtime": info.ModTime().UnixMilli(), + } + if probe.Mime != "" { + updates["mime"] = probe.Mime + } + for key, value := range probe.Metadata { + updates[key] = value + } + + return metadataJSON(existing, updates) +} + +func directoryMetadata(existing, relPath string) (string, error) { + return metadataJSON(existing, map[string]any{"source_path": relPath}) +} diff --git a/ingest/ingest_test.go b/ingest/ingest_test.go new file mode 100644 index 0000000..55b6ff5 --- /dev/null +++ b/ingest/ingest_test.go @@ -0,0 +1,71 @@ +package ingest + +import ( + "context" + "os" + "path/filepath" + "testing" + + "mediarr/document" +) + +func TestKindForFile(t *testing.T) { + cases := map[string]string{ + "film.mkv": document.KindVideo, + "film.MP4": document.KindVideo, + "episode.srt": document.KindSubtitle, + "poster.JPG": document.KindImage, + "notes.txt": document.KindFile, + "sans-extension": document.KindFile, + "archive.tar.gz": document.KindFile, + } + for name, expected := range cases { + if kind := KindForFile(name); kind != expected { + t.Fatalf("KindForFile(%q) = %q, attendu %q", name, kind, expected) + } + } +} + +func TestBasicProber(t *testing.T) { + t.Chdir(t.TempDir()) + path := filepath.Join("film.mkv") + if err := os.WriteFile(path, []byte("x"), 0o644); err != nil { + t.Fatalf("WriteFile : %v", err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatalf("Stat : %v", err) + } + + probe, err := BasicProber{}.Probe(context.Background(), path, info) + if err != nil { + t.Fatalf("Probe : %v", err) + } + if probe.Mime == "" { + t.Fatal("mime vide") + } +} + +func TestMetadataJSONPreservesExistingFields(t *testing.T) { + metadata, err := metadataJSON(`{"genre":"drame","size":1}`, map[string]any{ + "source_path": "film.mkv", + "size": int64(2), + }) + if err != nil { + t.Fatalf("metadataJSON : %v", err) + } + if metadata != `{"genre":"drame","size":2,"source_path":"film.mkv"}` { + t.Fatalf("metadata = %q", metadata) + } + + if _, err := metadataJSON(`{`, map[string]any{"x": 1}); err == nil { + t.Fatal("metadataJSON avec JSON invalide : erreur attendue") + } +} + +func TestScanRequiresServices(t *testing.T) { + var service *Service + if _, err := service.Scan(context.Background(), [16]byte{}, [16]byte{}, "."); err != ErrServiceNotConfigured { + t.Fatalf("erreur = %v, attendu ErrServiceNotConfigured", err) + } +} diff --git a/ingest/scan.go b/ingest/scan.go new file mode 100644 index 0000000..950568c --- /dev/null +++ b/ingest/scan.go @@ -0,0 +1,335 @@ +package ingest + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + + "mediarr/content" + "mediarr/document" + "mediarr/ident" +) + +func (s *Service) Scan(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (ScanResult, error) { + var result ScanResult + + if s == nil || s.documents == nil || s.contentService == nil { + return result, ErrServiceNotConfigured + } + + rootPath = strings.TrimSpace(rootPath) + if rootPath == "" { + return result, fmt.Errorf("%w: chemin vide", ErrInvalidRoot) + } + + absolute, err := filepath.Abs(rootPath) + if err != nil { + return result, fmt.Errorf("%w: %v", ErrInvalidRoot, err) + } + + info, err := os.Stat(absolute) + if err != nil { + return result, fmt.Errorf("%w: %s: %v", ErrInvalidRoot, rootPath, err) + } + if !info.IsDir() { + return result, fmt.Errorf("%w: %s n'est pas un dossier", ErrInvalidRoot, rootPath) + } + + parent, err := s.documents.Get(ctx, actorID, parentID) + if err != nil { + return result, err + } + if parent.DeletedAt != nil { + return result, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, parentID) + } + if parent.Kind != document.KindLibrary && parent.Kind != document.KindFolder { + return result, fmt.Errorf("%w: %s", document.ErrInvalidParent, parent.Kind) + } + + rootChildren, err := s.children(ctx, actorID, parentID) + if err != nil { + return result, err + } + + directories := map[string]*document.Document{".": parent} + children := map[ident.UUID]map[string]*document.Document{parentID: rootChildren} + + walkErr := filepath.WalkDir(absolute, func(fullPath string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: walkErr}) + if entry != nil && entry.IsDir() { + return fs.SkipDir + } + return nil + } + if fullPath == absolute { + return nil + } + + relPath, err := filepath.Rel(absolute, fullPath) + if err != nil { + result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err}) + return nil + } + relPath = filepath.ToSlash(relPath) + + if entry.Type()&fs.ModeSymlink != 0 { + return nil + } + + if entry.IsDir() { + result.Directories++ + _, created, err := s.scanDirectory(ctx, actorID, relPath, entry.Name(), directories, children) + if err != nil { + result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err}) + return fs.SkipDir + } + if created { + result.Created++ + } + return nil + } + + if !entry.Type().IsRegular() { + return nil + } + + result.Files++ + created, updated, err := s.scanFile(ctx, actorID, fullPath, relPath, entry.Name(), directories, children) + if err != nil { + result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err}) + return nil + } + switch { + case created: + result.Created++ + case updated: + result.Updated++ + default: + result.Unchanged++ + } + + return nil + }) + if walkErr != nil { + return result, walkErr + } + + return result, nil +} + +func (s *Service) scanDirectory( + ctx context.Context, + actorID ident.UUID, + relPath, name string, + directories map[string]*document.Document, + children map[ident.UUID]map[string]*document.Document, +) (*document.Document, bool, error) { + parentRel := path.Dir(relPath) + parent := directories[parentRel] + if parent == nil { + return nil, false, fmt.Errorf("%w: parent introuvable pour %s", ErrInvalidRoot, relPath) + } + + siblings, err := s.siblings(ctx, actorID, parent, children) + if err != nil { + return nil, false, err + } + + nameKey := strings.ToLower(name) + existing := siblings[nameKey] + if existing != nil { + if existing.Name != name { + return nil, false, fmt.Errorf("%w: %s", ErrNameConflict, relPath) + } + if existing.Kind != document.KindFolder { + return nil, false, fmt.Errorf("%w: %s", ErrNotFolder, relPath) + } + directories[relPath] = existing + return existing, false, nil + } + + metadata, err := directoryMetadata("", relPath) + if err != nil { + return nil, false, err + } + + created, err := s.documents.CreateChild(ctx, actorID, document.CreateChildParams{ + Kind: document.KindFolder, + OwnerID: actorID, + CreatedBy: actorID, + ParentID: parent.ID, + Name: name, + Metadata: metadata, + }) + if err != nil { + return nil, false, err + } + + siblings[nameKey] = created + directories[relPath] = created + return created, true, nil +} + +func (s *Service) scanFile( + ctx context.Context, + actorID ident.UUID, + fullPath, relPath, name string, + directories map[string]*document.Document, + children map[ident.UUID]map[string]*document.Document, +) (bool, bool, error) { + parentRel := path.Dir(relPath) + parent := directories[parentRel] + if parent == nil { + return false, false, fmt.Errorf("%w: parent introuvable pour %s", ErrInvalidRoot, relPath) + } + + siblings, err := s.siblings(ctx, actorID, parent, children) + if err != nil { + return false, false, err + } + + nameKey := strings.ToLower(name) + existing := siblings[nameKey] + if existing != nil { + if existing.Name != name { + return false, false, fmt.Errorf("%w: %s", ErrNameConflict, relPath) + } + if existing.Kind == document.KindFolder || existing.Kind == document.KindLibrary { + return false, false, fmt.Errorf("%w: %s", ErrNotFile, relPath) + } + } + + info, err := os.Stat(fullPath) + if err != nil { + return false, false, err + } + + probe, err := s.prober.Probe(ctx, fullPath, info) + if err != nil { + return false, false, err + } + + existingMetadata := "" + if existing != nil { + existingMetadata = existing.Metadata + } + metadata, err := fileMetadata(existingMetadata, relPath, info, probe) + if err != nil { + return false, false, err + } + + created := false + current := existing + if current == nil { + current, err = s.documents.CreateChild(ctx, actorID, document.CreateChildParams{ + Kind: KindForFile(name), + OwnerID: actorID, + CreatedBy: actorID, + ParentID: parent.ID, + Name: name, + Metadata: metadata, + }) + if err != nil { + return false, false, err + } + created = true + siblings[nameKey] = current + } else if current.Metadata != metadata { + current, err = s.documents.SetMetadata(ctx, actorID, current.ID, metadata) + if err != nil { + return false, false, err + } + siblings[nameKey] = current + } + + root, size, err := hashFile(fullPath) + if err != nil { + return false, false, err + } + + head, err := s.contentService.HeadVersion(ctx, actorID, current.ID) + if err != nil { + return false, false, err + } + if head != nil && head.Size == size && bytes.Equal(head.ContentRoot, root) { + return created, false, nil + } + + file, err := os.Open(fullPath) + if err != nil { + return false, false, err + } + defer func() { + _ = file.Close() + }() + + if _, err := s.contentService.CreateVersion(ctx, actorID, current.ID, content.CreateVersionParams{ + Source: file, + Mime: probe.Mime, + Origin: content.OriginImport, + Message: "scan", + }); err != nil { + return false, false, err + } + + return created, true, nil +} + +func (s *Service) siblings( + ctx context.Context, + actorID ident.UUID, + parent *document.Document, + children map[ident.UUID]map[string]*document.Document, +) (map[string]*document.Document, error) { + if siblings, exists := children[parent.ID]; exists { + return siblings, nil + } + + siblings, err := s.children(ctx, actorID, parent.ID) + if err != nil { + return nil, err + } + children[parent.ID] = siblings + + return siblings, nil +} + +func (s *Service) children(ctx context.Context, actorID, parentID ident.UUID) (map[string]*document.Document, error) { + rows, err := s.documents.ListChildren(ctx, actorID, parentID) + if err != nil { + return nil, err + } + + result := make(map[string]*document.Document, len(rows)) + for i := range rows { + result[rows[i].NameKey] = &rows[i] + } + + return result, nil +} + +func hashFile(filePath string) ([]byte, int64, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, 0, err + } + defer func() { + _ = file.Close() + }() + + hash := sha256.New() + size, err := io.Copy(hash, file) + if err != nil { + return nil, 0, err + } + + return hash.Sum(nil), size, nil +} diff --git a/node/node_ingest.go b/node/node_ingest.go new file mode 100644 index 0000000..1c1e05a --- /dev/null +++ b/node/node_ingest.go @@ -0,0 +1,7 @@ +package node + +import "mediarr/ingest" + +func (n *Node) Ingest() *ingest.Service { + return ingest.NewService(n.Documents(), n.Content(), nil) +} diff --git a/node/node_ingest_test.go b/node/node_ingest_test.go new file mode 100644 index 0000000..4b7082f --- /dev/null +++ b/node/node_ingest_test.go @@ -0,0 +1,191 @@ +package node + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "mediarr/content" + "mediarr/document" + "mediarr/ident" + "mediarr/identity" +) + +func TestNodeIngestScan(t *testing.T) { + t.Chdir(t.TempDir()) + + err, node := NewNode(NewNodeParams{DatabaseName: "ingest", StoragePath: "store"}) + if err != nil { + t.Fatalf("NewNode : %v", err) + } + t.Cleanup(func() { + if err := node.Close(); err != nil { + t.Errorf("Close : %v", err) + } + }) + + if err := node.Up(); err != nil { + t.Fatalf("Up : %v", err) + } + + ctx := context.Background() + owner, err := node.Identity().CreatePrincipal(ctx, identity.CreatePrincipalParams{ + Kind: identity.KindUser, + Name: "owner", + }) + if err != nil { + t.Fatalf("CreatePrincipal(owner) : %v", err) + } + other, err := node.Identity().CreatePrincipal(ctx, identity.CreatePrincipalParams{ + Kind: identity.KindUser, + Name: "other", + }) + if err != nil { + t.Fatalf("CreatePrincipal(other) : %v", err) + } + + library, err := node.Documents().CreateLibrary(ctx, owner.ID, document.CreateLibraryParams{ + OwnerID: owner.ID, + CreatedBy: owner.ID, + Name: "Films", + PolicyID: ident.MustParse("00000000-0000-7000-8000-000000000101"), + }) + if err != nil { + t.Fatalf("CreateLibrary : %v", err) + } + + source := filepath.Join(t.TempDir(), "library") + if err := os.MkdirAll(filepath.Join(source, "Season 1"), 0o755); err != nil { + t.Fatalf("MkdirAll : %v", err) + } + writeNodeTestFile(t, filepath.Join(source, "film.mkv"), "film") + writeNodeTestFile(t, filepath.Join(source, "notes.txt"), "notes") + writeNodeTestFile(t, filepath.Join(source, "Season 1", "episode.srt"), "subs") + + result, err := node.Ingest().Scan(ctx, owner.ID, library.ID, source) + if err != nil { + t.Fatalf("Scan : %v", err) + } + if result.HasErrors() { + t.Fatalf("Scan erreurs : %v", result.Errors) + } + if result.Files != 3 || result.Directories != 1 || result.Created != 4 || result.Updated != 0 || result.Unchanged != 0 { + t.Fatalf("résultat scan = %+v", result) + } + + children, err := node.Documents().ListChildren(ctx, owner.ID, library.ID) + if err != nil { + t.Fatalf("ListChildren(library) : %v", err) + } + film := findNodeTestChild(t, children, "film.mkv") + notes := findNodeTestChild(t, children, "notes.txt") + season := findNodeTestChild(t, children, "Season 1") + if film.Kind != document.KindVideo { + t.Fatalf("kind film = %q, attendu %q", film.Kind, document.KindVideo) + } + if notes.Kind != document.KindFile { + t.Fatalf("kind notes = %q, attendu %q", notes.Kind, document.KindFile) + } + if season.Kind != document.KindFolder { + t.Fatalf("kind saison = %q, attendu %q", season.Kind, document.KindFolder) + } + if !strings.Contains(film.Metadata, `"source_path":"film.mkv"`) { + t.Fatalf("metadata film = %q", film.Metadata) + } + + seasonChildren, err := node.Documents().ListChildren(ctx, owner.ID, season.ID) + if err != nil { + t.Fatalf("ListChildren(saison) : %v", err) + } + episode := findNodeTestChild(t, seasonChildren, "episode.srt") + if episode.Kind != document.KindSubtitle { + t.Fatalf("kind épisode = %q, attendu %q", episode.Kind, document.KindSubtitle) + } + + versions, err := node.Content().ListVersions(ctx, owner.ID, film.ID) + if err != nil { + t.Fatalf("ListVersions(film) : %v", err) + } + if len(versions) != 1 { + t.Fatalf("versions film = %d, attendu 1", len(versions)) + } + if versions[0].Seq != 1 || versions[0].Origin != content.OriginImport { + t.Fatalf("version film = %+v", versions[0]) + } + + head, err := node.Content().HeadVersion(ctx, owner.ID, film.ID) + if err != nil { + t.Fatalf("HeadVersion(film) : %v", err) + } + if head == nil || head.ID != versions[0].ID { + t.Fatalf("head film = %+v, attendu %s", head, versions[0].ID) + } + + result, err = node.Ingest().Scan(ctx, owner.ID, library.ID, source) + if err != nil { + t.Fatalf("Scan(rescan) : %v", err) + } + if result.HasErrors() { + t.Fatalf("Scan(rescan) erreurs : %v", result.Errors) + } + if result.Created != 0 || result.Updated != 0 || result.Unchanged != 3 { + t.Fatalf("résultat rescan = %+v", result) + } + versions, err = node.Content().ListVersions(ctx, owner.ID, film.ID) + if err != nil { + t.Fatalf("ListVersions(film après rescan) : %v", err) + } + if len(versions) != 1 { + t.Fatalf("versions film après rescan = %d, attendu 1", len(versions)) + } + + writeNodeTestFile(t, filepath.Join(source, "film.mkv"), "film v2") + result, err = node.Ingest().Scan(ctx, owner.ID, library.ID, source) + if err != nil { + t.Fatalf("Scan(modifié) : %v", err) + } + if result.HasErrors() { + t.Fatalf("Scan(modifié) erreurs : %v", result.Errors) + } + if result.Created != 0 || result.Updated != 1 || result.Unchanged != 2 { + t.Fatalf("résultat scan modifié = %+v", result) + } + versions, err = node.Content().ListVersions(ctx, owner.ID, film.ID) + if err != nil { + t.Fatalf("ListVersions(film modifié) : %v", err) + } + if len(versions) != 2 { + t.Fatalf("versions film modifié = %d, attendu 2", len(versions)) + } + if versions[0].Seq != 2 || versions[0].Origin != content.OriginImport { + t.Fatalf("nouvelle version = %+v", versions[0]) + } + + if _, err := node.Ingest().Scan(ctx, other.ID, library.ID, source); !errors.Is(err, document.ErrPermissionDenied) { + t.Fatalf("Scan sans Read : erreur = %v, attendu ErrPermissionDenied", err) + } +} + +func writeNodeTestFile(t *testing.T, path, content string) { + t.Helper() + + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("WriteFile(%s) : %v", path, err) + } +} + +func findNodeTestChild(t *testing.T, children []document.Document, name string) *document.Document { + t.Helper() + + for i := range children { + if children[i].Name == name { + return &children[i] + } + } + + t.Fatalf("enfant %q introuvable dans %+v", name, children) + return nil +}