ingest documents

This commit is contained in:
venture
2026-11-16 11:03:48 +01:00
parent cc0ddd0dfb
commit 7b3f20b8d7
15 changed files with 1148 additions and 2 deletions
+26
View File
@@ -400,6 +400,32 @@ func (s *Service) ListVersions(ctx context.Context, actorID, documentID ident.UU
return versions, nil return versions, nil
} }
func (s *Service) HeadVersion(ctx context.Context, actorID, documentID ident.UUID) (*Version, error) {
documentRow, err := s.getDocument(ctx, documentID)
if err != nil {
return nil, err
}
if documentRow.DeletedAt.Valid {
return nil, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, documentID)
}
if err := s.can(ctx, actorID, documentID, permission.Read); err != nil {
return nil, err
}
if documentRow.HeadVersionID == nil {
return nil, nil
}
version, err := s.getVersion(ctx, *documentRow.HeadVersionID)
if err != nil {
return nil, err
}
if version.DocumentID != documentID {
return nil, fmt.Errorf("%w: %s", ErrVersionMismatch, version.ID)
}
return versionFromDB(*version), nil
}
func boolToInt(value bool) int64 { func boolToInt(value bool) int64 {
if value { if value {
return 1 return 1
+46
View File
@@ -196,3 +196,49 @@ func TestCreateVersionConfigurationErrors(t *testing.T) {
t.Fatalf("sans stockage : erreur = %v, attendu ErrStorageNotConfigured", err) t.Fatalf("sans stockage : erreur = %v, attendu ErrStorageNotConfigured", err)
} }
} }
func TestHeadVersion(t *testing.T) {
env := newTestEnv(t, allowAll())
ctx := context.Background()
head, err := env.service.HeadVersion(ctx, env.owner.ID, env.videoID)
if err != nil {
t.Fatalf("HeadVersion(sans version) : %v", err)
}
if head != nil {
t.Fatalf("HeadVersion(sans version) = %+v, attendu nil", head)
}
first, err := env.service.CreateVersion(ctx, env.owner.ID, env.videoID, CreateVersionParams{
Source: bytes.NewReader([]byte("premier")),
})
if err != nil {
t.Fatalf("CreateVersion : %v", err)
}
head, err = env.service.HeadVersion(ctx, env.owner.ID, env.videoID)
if err != nil {
t.Fatalf("HeadVersion : %v", err)
}
if head == nil || head.ID != first.ID {
t.Fatalf("HeadVersion = %+v, attendu %s", head, first.ID)
}
second, err := env.service.CreateVersion(ctx, env.owner.ID, env.videoID, CreateVersionParams{
Source: bytes.NewReader([]byte("second")),
})
if err != nil {
t.Fatalf("CreateVersion(second) : %v", err)
}
head, err = env.service.HeadVersion(ctx, env.owner.ID, env.videoID)
if err != nil {
t.Fatalf("HeadVersion(second) : %v", err)
}
if head == nil || head.ID != second.ID {
t.Fatalf("HeadVersion = %+v, attendu %s", head, second.ID)
}
readerOnly := NewService(*env.db, env.store, testBackendID, &testAuthorizer{allowed: permission.Write})
if _, err := readerOnly.HeadVersion(ctx, env.owner.ID, env.videoID); !errors.Is(err, document.ErrPermissionDenied) {
t.Fatalf("HeadVersion sans Read : erreur = %v, attendu ErrPermissionDenied", err)
}
}
+45
View File
@@ -2481,6 +2481,51 @@ func (q *Queries) UpdateDocumentLocation(ctx context.Context, arg UpdateDocument
return err return err
} }
const updateDocumentMetadata = `-- name: UpdateDocumentMetadata :one
UPDATE document
SET metadata = ?1,
rev = rev + 1,
updated_at = ?2
WHERE id = ?3
AND deleted_at IS NULL
RETURNING id, kind, owner_id, created_by, parent_id, name, name_key, path, depth, position, head_version_id, metadata, policy_id, visibility, acl_inherit, rev, created_at, updated_at, deleted_at, deleted_by, trash_batch
`
type UpdateDocumentMetadataParams struct {
Metadata string
UpdatedAt int64
ID ident.UUID
}
func (q *Queries) UpdateDocumentMetadata(ctx context.Context, arg UpdateDocumentMetadataParams) (Document, error) {
row := q.db.QueryRowContext(ctx, updateDocumentMetadata, arg.Metadata, arg.UpdatedAt, arg.ID)
var i Document
err := row.Scan(
&i.ID,
&i.Kind,
&i.OwnerID,
&i.CreatedBy,
&i.ParentID,
&i.Name,
&i.NameKey,
&i.Path,
&i.Depth,
&i.Position,
&i.HeadVersionID,
&i.Metadata,
&i.PolicyID,
&i.Visibility,
&i.AclInherit,
&i.Rev,
&i.CreatedAt,
&i.UpdatedAt,
&i.DeletedAt,
&i.DeletedBy,
&i.TrashBatch,
)
return i, err
}
const updateDocumentName = `-- name: UpdateDocumentName :one const updateDocumentName = `-- name: UpdateDocumentName :one
UPDATE document UPDATE document
SET name = ?1, SET name = ?1,
+9
View File
@@ -595,6 +595,15 @@ WHERE id = sqlc.arg(id)
AND deleted_at IS NULL AND deleted_at IS NULL
RETURNING *; RETURNING *;
-- name: UpdateDocumentMetadata :one
UPDATE document
SET metadata = sqlc.arg(metadata),
rev = rev + 1,
updated_at = sqlc.arg(updated_at)
WHERE id = sqlc.arg(id)
AND deleted_at IS NULL
RETURNING *;
-- name: MoveDocument :one -- name: MoveDocument :one
UPDATE document UPDATE document
SET parent_id = sqlc.arg(parent_id), SET parent_id = sqlc.arg(parent_id),
+79
View File
@@ -131,3 +131,82 @@ func (s *Service) CreateFolder(ctx context.Context, actorID ident.UUID, params C
return documentFromDB(created), nil return documentFromDB(created), nil
} }
func (s *Service) CreateChild(ctx context.Context, actorID ident.UUID, params CreateChildParams) (*Document, error) {
if err := s.requireActive(ctx, actorID); err != nil {
return nil, err
}
if params.OwnerID != actorID || params.CreatedBy != actorID {
return nil, fmt.Errorf("%w: owner_id et created_by doivent être l'acteur", ErrPermissionDenied)
}
name, nameKey, err := normalizeName(params.Name)
if err != nil {
return nil, err
}
parent, err := s.getDocument(ctx, params.ParentID)
if err != nil {
if errors.Is(err, ErrDocumentNotFound) {
return nil, fmt.Errorf("%w: %s", ErrParentNotFound, params.ParentID)
}
return nil, err
}
if parent.DeletedAt.Valid {
return nil, fmt.Errorf("%w: %s", ErrParentDeleted, params.ParentID)
}
if !isContainerKind(parent.Kind) {
return nil, fmt.Errorf("%w: %s", ErrInvalidParent, parent.Kind)
}
if err := s.can(ctx, actorID, params.ParentID, permission.Create); err != nil {
return nil, err
}
kind, err := s.db.Queries.GetDocumentKind(ctx, params.Kind)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, fmt.Errorf("%w: %s", ErrInvalidKind, params.Kind)
}
return nil, err
}
if kind.Kind == KindLibrary || kind.Hidden != 0 {
return nil, fmt.Errorf("%w: %s", ErrInvalidKind, params.Kind)
}
metadata, err := normalizeMetadata(params.Metadata)
if err != nil {
return nil, err
}
id, err := ident.NewV7()
if err != nil {
return nil, err
}
now := nowMillis()
created, err := s.db.WriteQueries.CreateDocument(ctx, database.CreateDocumentParams{
ID: id,
Kind: kind.Kind,
OwnerID: params.OwnerID,
CreatedBy: params.CreatedBy,
ParentID: &params.ParentID,
Name: name,
NameKey: nameKey,
Path: parent.Path + hex.EncodeToString(id[:]) + "/",
Depth: parent.Depth + 1,
Metadata: metadata,
AclInherit: aclInheritValue(params.ACLInherit),
Rev: 1,
CreatedAt: now,
UpdatedAt: now,
})
if err != nil {
if isUniqueViolation(err) {
return nil, fmt.Errorf("%w: %s", ErrNameTaken, name)
}
return nil, err
}
return documentFromDB(created), nil
}
+108
View File
@@ -280,3 +280,111 @@ func TestCreateFolderRejectsDuplicateSibling(t *testing.T) {
t.Fatalf("erreur = %v, attendu ErrNameTaken", err) t.Fatalf("erreur = %v, attendu ErrNameTaken", err)
} }
} }
func TestCreateChild(t *testing.T) {
service := newTestService(t)
ctx := context.Background()
owner := mustUser(t, service, "owner")
library := mustLibrary(t, service, owner, "Photos")
video, err := service.CreateChild(ctx, owner.ID, CreateChildParams{
Kind: KindVideo,
OwnerID: owner.ID,
CreatedBy: owner.ID,
ParentID: library.ID,
Name: "film.mkv",
Metadata: `{"source_path":"film.mkv"}`,
})
if err != nil {
t.Fatalf("CreateChild(video) : %v", err)
}
if video.Kind != KindVideo {
t.Fatalf("kind = %q, attendu %q", video.Kind, KindVideo)
}
if video.ParentID == nil || *video.ParentID != library.ID {
t.Fatalf("parent = %v, attendu %s", video.ParentID, library.ID)
}
if video.Depth != library.Depth+1 {
t.Fatalf("depth = %d, attendu %d", video.Depth, library.Depth+1)
}
if video.Metadata != `{"source_path":"film.mkv"}` {
t.Fatalf("metadata = %q", video.Metadata)
}
folder, err := service.CreateChild(ctx, owner.ID, CreateChildParams{
Kind: KindFolder,
OwnerID: owner.ID,
CreatedBy: owner.ID,
ParentID: library.ID,
Name: "Extras",
})
if err != nil {
t.Fatalf("CreateChild(folder) : %v", err)
}
if folder.Kind != KindFolder {
t.Fatalf("kind = %q, attendu %q", folder.Kind, KindFolder)
}
if folder.Metadata != "{}" {
t.Fatalf("metadata = %q, attendu {}", folder.Metadata)
}
}
func TestCreateChildRejectsInvalidInputs(t *testing.T) {
service := newTestService(t)
ctx := context.Background()
owner := mustUser(t, service, "owner")
other := mustUser(t, service, "other")
library := mustLibrary(t, service, owner, "Photos")
if _, err := service.CreateChild(ctx, owner.ID, CreateChildParams{
Kind: KindVideo,
OwnerID: other.ID,
CreatedBy: owner.ID,
ParentID: library.ID,
Name: "film.mkv",
}); !errors.Is(err, ErrPermissionDenied) {
t.Fatalf("owner usurpé : erreur = %v, attendu ErrPermissionDenied", err)
}
if _, err := service.CreateChild(ctx, owner.ID, CreateChildParams{
Kind: "inconnu",
OwnerID: owner.ID,
CreatedBy: owner.ID,
ParentID: library.ID,
Name: "film.mkv",
}); !errors.Is(err, ErrInvalidKind) {
t.Fatalf("kind invalide : erreur = %v, attendu ErrInvalidKind", err)
}
if _, err := service.CreateChild(ctx, owner.ID, CreateChildParams{
Kind: KindVideo,
OwnerID: owner.ID,
CreatedBy: owner.ID,
ParentID: library.ID,
Name: "film.mkv",
Metadata: "{",
}); !errors.Is(err, ErrInvalidMetadata) {
t.Fatalf("metadata invalide : erreur = %v, attendu ErrInvalidMetadata", err)
}
video, err := service.CreateChild(ctx, owner.ID, CreateChildParams{
Kind: KindVideo,
OwnerID: owner.ID,
CreatedBy: owner.ID,
ParentID: library.ID,
Name: "film.mkv",
})
if err != nil {
t.Fatalf("CreateChild(video) : %v", err)
}
if _, err := service.CreateChild(ctx, owner.ID, CreateChildParams{
Kind: KindVideo,
OwnerID: owner.ID,
CreatedBy: owner.ID,
ParentID: video.ID,
Name: "enfant.mkv",
}); !errors.Is(err, ErrInvalidParent) {
t.Fatalf("parent non conteneur : erreur = %v, attendu ErrInvalidParent", err)
}
}
+16
View File
@@ -16,6 +16,10 @@ const (
const ( const (
KindLibrary = "library" KindLibrary = "library"
KindFolder = "folder" KindFolder = "folder"
KindVideo = "video"
KindSubtitle = "subtitle"
KindImage = "image"
KindFile = "file"
) )
type Document struct { type Document struct {
@@ -61,6 +65,16 @@ type CreateFolderParams struct {
Metadata string Metadata string
} }
type CreateChildParams struct {
Kind string
OwnerID ident.UUID
CreatedBy ident.UUID
ParentID ident.UUID
Name string
ACLInherit *bool
Metadata string
}
var ( var (
ErrInvalidName = errors.New("nom de document invalide") ErrInvalidName = errors.New("nom de document invalide")
ErrNameTaken = errors.New("nom de document déjà utilisé") ErrNameTaken = errors.New("nom de document déjà utilisé")
@@ -72,6 +86,8 @@ var (
ErrCannotMoveRoot = errors.New("déplacement d'une bibliothèque racine interdit") ErrCannotMoveRoot = errors.New("déplacement d'une bibliothèque racine interdit")
ErrInvalidMove = errors.New("déplacement invalide") ErrInvalidMove = errors.New("déplacement invalide")
ErrInvalidVisibility = errors.New("visibilité invalide") ErrInvalidVisibility = errors.New("visibilité invalide")
ErrInvalidKind = errors.New("type de document invalide")
ErrInvalidMetadata = errors.New("métadonnées invalides")
ErrPolicyNotFound = errors.New("politique de stockage introuvable") ErrPolicyNotFound = errors.New("politique de stockage introuvable")
ErrInvalidTrashBatch = errors.New("lot de corbeille invalide") ErrInvalidTrashBatch = errors.New("lot de corbeille invalide")
ErrPermissionDenied = errors.New("permission refusée") ErrPermissionDenied = errors.New("permission refusée")
+31
View File
@@ -46,6 +46,37 @@ func (s *Service) Rename(ctx context.Context, actorID, documentID ident.UUID, na
return documentFromDB(updated), nil return documentFromDB(updated), nil
} }
func (s *Service) SetMetadata(ctx context.Context, actorID, documentID ident.UUID, metadata string) (*Document, error) {
if err := s.requireActive(ctx, actorID); err != nil {
return nil, err
}
if _, err := s.getActiveDocument(ctx, documentID); err != nil {
return nil, err
}
if err := s.can(ctx, actorID, documentID, permission.Write); err != nil {
return nil, err
}
normalized, err := normalizeMetadata(metadata)
if err != nil {
return nil, err
}
updated, err := s.db.WriteQueries.UpdateDocumentMetadata(ctx, database.UpdateDocumentMetadataParams{
Metadata: normalized,
UpdatedAt: nowMillis(),
ID: documentID,
})
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, fmt.Errorf("%w: %s", ErrDocumentNotFound, documentID)
}
return nil, err
}
return documentFromDB(updated), nil
}
func (s *Service) Move(ctx context.Context, actorID, documentID, newParentID ident.UUID) (*Document, error) { func (s *Service) Move(ctx context.Context, actorID, documentID, newParentID ident.UUID) (*Document, error) {
document, err := s.getActiveDocument(ctx, documentID) document, err := s.getActiveDocument(ctx, documentID)
if err != nil { if err != nil {
+23
View File
@@ -400,3 +400,26 @@ func TestMoveUpdatesTrashedDescendants(t *testing.T) {
t.Fatalf("depth = %d, attendu 2", restored.Depth) t.Fatalf("depth = %d, attendu 2", restored.Depth)
} }
} }
func TestSetMetadata(t *testing.T) {
service := newTestService(t)
ctx := context.Background()
owner := mustUser(t, service, "owner")
library := mustLibrary(t, service, owner, "Photos")
video := mustKindDocument(t, service, owner, library, KindVideo, "film.mkv")
updated, err := service.SetMetadata(ctx, owner.ID, video.ID, `{"duration_ms":120000}`)
if err != nil {
t.Fatalf("SetMetadata : %v", err)
}
if updated.Metadata != `{"duration_ms":120000}` {
t.Fatalf("metadata = %q", updated.Metadata)
}
if updated.Rev != video.Rev+1 {
t.Fatalf("rev = %d, attendu %d", updated.Rev, video.Rev+1)
}
if _, err := service.SetMetadata(ctx, owner.ID, video.ID, "{"); !errors.Is(err, ErrInvalidMetadata) {
t.Fatalf("metadata invalide : erreur = %v, attendu ErrInvalidMetadata", err)
}
}
+9
View File
@@ -3,6 +3,7 @@ package document
import ( import (
"context" "context"
"database/sql" "database/sql"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"strings" "strings"
@@ -136,6 +137,14 @@ func metadataValue(metadata string) string {
return metadata return metadata
} }
func normalizeMetadata(metadata string) (string, error) {
value := metadataValue(metadata)
if !json.Valid([]byte(value)) {
return "", ErrInvalidMetadata
}
return value, nil
}
func aclInheritValue(value *bool) int64 { func aclInheritValue(value *bool) int64 {
if value == nil || *value { if value == nil || *value {
return 1 return 1
+150
View File
@@ -0,0 +1,150 @@
package ingest
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"mime"
"path/filepath"
"strings"
"mediarr/content"
"mediarr/document"
)
const (
KindVideo = document.KindVideo
KindSubtitle = document.KindSubtitle
KindImage = document.KindImage
KindFile = document.KindFile
)
type Probe struct {
Mime string
Metadata map[string]any
}
type Prober interface {
Probe(ctx context.Context, path string, info fs.FileInfo) (Probe, error)
}
type BasicProber struct{}
func (BasicProber) Probe(_ context.Context, path string, _ fs.FileInfo) (Probe, error) {
mimeType := mime.TypeByExtension(strings.ToLower(filepath.Ext(path)))
if mimeType == "" {
mimeType = "application/octet-stream"
}
return Probe{Mime: mimeType}, nil
}
type Service struct {
documents *document.Service
contentService *content.Service
prober Prober
}
func NewService(documents *document.Service, contentService *content.Service, prober Prober) *Service {
if prober == nil {
prober = BasicProber{}
}
return &Service{
documents: documents,
contentService: contentService,
prober: prober,
}
}
type ScanResult struct {
Files int
Directories int
Created int
Updated int
Unchanged int
Errors []ScanError
}
func (r ScanResult) HasErrors() bool {
return len(r.Errors) > 0
}
type ScanError struct {
Path string
Err error
}
func (e ScanError) Error() string {
return fmt.Sprintf("%s: %v", e.Path, e.Err)
}
func (e ScanError) Unwrap() error {
return e.Err
}
var (
ErrServiceNotConfigured = errors.New("service d'ingestion non configuré")
ErrInvalidRoot = errors.New("racine de scan invalide")
ErrNotFolder = errors.New("document existant n'est pas un dossier")
ErrNotFile = errors.New("document existant n'est pas un fichier")
ErrNameConflict = errors.New("conflit de nom insensible à la casse")
)
func KindForFile(name string) string {
switch strings.ToLower(filepath.Ext(name)) {
case ".mkv", ".mp4", ".m4v", ".avi", ".mov", ".webm", ".ts", ".m2ts", ".wmv", ".flv", ".mpg", ".mpeg":
return KindVideo
case ".srt", ".ass", ".ssa", ".vtt", ".sub":
return KindSubtitle
case ".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tif", ".tiff", ".heic", ".avif":
return KindImage
default:
return KindFile
}
}
func metadataJSON(existing string, updates map[string]any) (string, error) {
values := make(map[string]any)
if existing != "" {
if err := json.Unmarshal([]byte(existing), &values); err != nil {
return "", fmt.Errorf("%w: %v", document.ErrInvalidMetadata, err)
}
if values == nil {
values = make(map[string]any)
}
}
for key, value := range updates {
values[key] = value
}
encoded, err := json.Marshal(values)
if err != nil {
return "", err
}
return string(encoded), nil
}
func fileMetadata(existing, relPath string, info fs.FileInfo, probe Probe) (string, error) {
updates := map[string]any{
"source_path": relPath,
"size": info.Size(),
"mtime": info.ModTime().UnixMilli(),
}
if probe.Mime != "" {
updates["mime"] = probe.Mime
}
for key, value := range probe.Metadata {
updates[key] = value
}
return metadataJSON(existing, updates)
}
func directoryMetadata(existing, relPath string) (string, error) {
return metadataJSON(existing, map[string]any{"source_path": relPath})
}
+71
View File
@@ -0,0 +1,71 @@
package ingest
import (
"context"
"os"
"path/filepath"
"testing"
"mediarr/document"
)
func TestKindForFile(t *testing.T) {
cases := map[string]string{
"film.mkv": document.KindVideo,
"film.MP4": document.KindVideo,
"episode.srt": document.KindSubtitle,
"poster.JPG": document.KindImage,
"notes.txt": document.KindFile,
"sans-extension": document.KindFile,
"archive.tar.gz": document.KindFile,
}
for name, expected := range cases {
if kind := KindForFile(name); kind != expected {
t.Fatalf("KindForFile(%q) = %q, attendu %q", name, kind, expected)
}
}
}
func TestBasicProber(t *testing.T) {
t.Chdir(t.TempDir())
path := filepath.Join("film.mkv")
if err := os.WriteFile(path, []byte("x"), 0o644); err != nil {
t.Fatalf("WriteFile : %v", err)
}
info, err := os.Stat(path)
if err != nil {
t.Fatalf("Stat : %v", err)
}
probe, err := BasicProber{}.Probe(context.Background(), path, info)
if err != nil {
t.Fatalf("Probe : %v", err)
}
if probe.Mime == "" {
t.Fatal("mime vide")
}
}
func TestMetadataJSONPreservesExistingFields(t *testing.T) {
metadata, err := metadataJSON(`{"genre":"drame","size":1}`, map[string]any{
"source_path": "film.mkv",
"size": int64(2),
})
if err != nil {
t.Fatalf("metadataJSON : %v", err)
}
if metadata != `{"genre":"drame","size":2,"source_path":"film.mkv"}` {
t.Fatalf("metadata = %q", metadata)
}
if _, err := metadataJSON(`{`, map[string]any{"x": 1}); err == nil {
t.Fatal("metadataJSON avec JSON invalide : erreur attendue")
}
}
func TestScanRequiresServices(t *testing.T) {
var service *Service
if _, err := service.Scan(context.Background(), [16]byte{}, [16]byte{}, "."); err != ErrServiceNotConfigured {
t.Fatalf("erreur = %v, attendu ErrServiceNotConfigured", err)
}
}
+335
View File
@@ -0,0 +1,335 @@
package ingest
import (
"bytes"
"context"
"crypto/sha256"
"fmt"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
"mediarr/content"
"mediarr/document"
"mediarr/ident"
)
func (s *Service) Scan(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (ScanResult, error) {
var result ScanResult
if s == nil || s.documents == nil || s.contentService == nil {
return result, ErrServiceNotConfigured
}
rootPath = strings.TrimSpace(rootPath)
if rootPath == "" {
return result, fmt.Errorf("%w: chemin vide", ErrInvalidRoot)
}
absolute, err := filepath.Abs(rootPath)
if err != nil {
return result, fmt.Errorf("%w: %v", ErrInvalidRoot, err)
}
info, err := os.Stat(absolute)
if err != nil {
return result, fmt.Errorf("%w: %s: %v", ErrInvalidRoot, rootPath, err)
}
if !info.IsDir() {
return result, fmt.Errorf("%w: %s n'est pas un dossier", ErrInvalidRoot, rootPath)
}
parent, err := s.documents.Get(ctx, actorID, parentID)
if err != nil {
return result, err
}
if parent.DeletedAt != nil {
return result, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, parentID)
}
if parent.Kind != document.KindLibrary && parent.Kind != document.KindFolder {
return result, fmt.Errorf("%w: %s", document.ErrInvalidParent, parent.Kind)
}
rootChildren, err := s.children(ctx, actorID, parentID)
if err != nil {
return result, err
}
directories := map[string]*document.Document{".": parent}
children := map[ident.UUID]map[string]*document.Document{parentID: rootChildren}
walkErr := filepath.WalkDir(absolute, func(fullPath string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: walkErr})
if entry != nil && entry.IsDir() {
return fs.SkipDir
}
return nil
}
if fullPath == absolute {
return nil
}
relPath, err := filepath.Rel(absolute, fullPath)
if err != nil {
result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err})
return nil
}
relPath = filepath.ToSlash(relPath)
if entry.Type()&fs.ModeSymlink != 0 {
return nil
}
if entry.IsDir() {
result.Directories++
_, created, err := s.scanDirectory(ctx, actorID, relPath, entry.Name(), directories, children)
if err != nil {
result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err})
return fs.SkipDir
}
if created {
result.Created++
}
return nil
}
if !entry.Type().IsRegular() {
return nil
}
result.Files++
created, updated, err := s.scanFile(ctx, actorID, fullPath, relPath, entry.Name(), directories, children)
if err != nil {
result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err})
return nil
}
switch {
case created:
result.Created++
case updated:
result.Updated++
default:
result.Unchanged++
}
return nil
})
if walkErr != nil {
return result, walkErr
}
return result, nil
}
func (s *Service) scanDirectory(
ctx context.Context,
actorID ident.UUID,
relPath, name string,
directories map[string]*document.Document,
children map[ident.UUID]map[string]*document.Document,
) (*document.Document, bool, error) {
parentRel := path.Dir(relPath)
parent := directories[parentRel]
if parent == nil {
return nil, false, fmt.Errorf("%w: parent introuvable pour %s", ErrInvalidRoot, relPath)
}
siblings, err := s.siblings(ctx, actorID, parent, children)
if err != nil {
return nil, false, err
}
nameKey := strings.ToLower(name)
existing := siblings[nameKey]
if existing != nil {
if existing.Name != name {
return nil, false, fmt.Errorf("%w: %s", ErrNameConflict, relPath)
}
if existing.Kind != document.KindFolder {
return nil, false, fmt.Errorf("%w: %s", ErrNotFolder, relPath)
}
directories[relPath] = existing
return existing, false, nil
}
metadata, err := directoryMetadata("", relPath)
if err != nil {
return nil, false, err
}
created, err := s.documents.CreateChild(ctx, actorID, document.CreateChildParams{
Kind: document.KindFolder,
OwnerID: actorID,
CreatedBy: actorID,
ParentID: parent.ID,
Name: name,
Metadata: metadata,
})
if err != nil {
return nil, false, err
}
siblings[nameKey] = created
directories[relPath] = created
return created, true, nil
}
func (s *Service) scanFile(
ctx context.Context,
actorID ident.UUID,
fullPath, relPath, name string,
directories map[string]*document.Document,
children map[ident.UUID]map[string]*document.Document,
) (bool, bool, error) {
parentRel := path.Dir(relPath)
parent := directories[parentRel]
if parent == nil {
return false, false, fmt.Errorf("%w: parent introuvable pour %s", ErrInvalidRoot, relPath)
}
siblings, err := s.siblings(ctx, actorID, parent, children)
if err != nil {
return false, false, err
}
nameKey := strings.ToLower(name)
existing := siblings[nameKey]
if existing != nil {
if existing.Name != name {
return false, false, fmt.Errorf("%w: %s", ErrNameConflict, relPath)
}
if existing.Kind == document.KindFolder || existing.Kind == document.KindLibrary {
return false, false, fmt.Errorf("%w: %s", ErrNotFile, relPath)
}
}
info, err := os.Stat(fullPath)
if err != nil {
return false, false, err
}
probe, err := s.prober.Probe(ctx, fullPath, info)
if err != nil {
return false, false, err
}
existingMetadata := ""
if existing != nil {
existingMetadata = existing.Metadata
}
metadata, err := fileMetadata(existingMetadata, relPath, info, probe)
if err != nil {
return false, false, err
}
created := false
current := existing
if current == nil {
current, err = s.documents.CreateChild(ctx, actorID, document.CreateChildParams{
Kind: KindForFile(name),
OwnerID: actorID,
CreatedBy: actorID,
ParentID: parent.ID,
Name: name,
Metadata: metadata,
})
if err != nil {
return false, false, err
}
created = true
siblings[nameKey] = current
} else if current.Metadata != metadata {
current, err = s.documents.SetMetadata(ctx, actorID, current.ID, metadata)
if err != nil {
return false, false, err
}
siblings[nameKey] = current
}
root, size, err := hashFile(fullPath)
if err != nil {
return false, false, err
}
head, err := s.contentService.HeadVersion(ctx, actorID, current.ID)
if err != nil {
return false, false, err
}
if head != nil && head.Size == size && bytes.Equal(head.ContentRoot, root) {
return created, false, nil
}
file, err := os.Open(fullPath)
if err != nil {
return false, false, err
}
defer func() {
_ = file.Close()
}()
if _, err := s.contentService.CreateVersion(ctx, actorID, current.ID, content.CreateVersionParams{
Source: file,
Mime: probe.Mime,
Origin: content.OriginImport,
Message: "scan",
}); err != nil {
return false, false, err
}
return created, true, nil
}
func (s *Service) siblings(
ctx context.Context,
actorID ident.UUID,
parent *document.Document,
children map[ident.UUID]map[string]*document.Document,
) (map[string]*document.Document, error) {
if siblings, exists := children[parent.ID]; exists {
return siblings, nil
}
siblings, err := s.children(ctx, actorID, parent.ID)
if err != nil {
return nil, err
}
children[parent.ID] = siblings
return siblings, nil
}
func (s *Service) children(ctx context.Context, actorID, parentID ident.UUID) (map[string]*document.Document, error) {
rows, err := s.documents.ListChildren(ctx, actorID, parentID)
if err != nil {
return nil, err
}
result := make(map[string]*document.Document, len(rows))
for i := range rows {
result[rows[i].NameKey] = &rows[i]
}
return result, nil
}
func hashFile(filePath string) ([]byte, int64, error) {
file, err := os.Open(filePath)
if err != nil {
return nil, 0, err
}
defer func() {
_ = file.Close()
}()
hash := sha256.New()
size, err := io.Copy(hash, file)
if err != nil {
return nil, 0, err
}
return hash.Sum(nil), size, nil
}
+7
View File
@@ -0,0 +1,7 @@
package node
import "mediarr/ingest"
func (n *Node) Ingest() *ingest.Service {
return ingest.NewService(n.Documents(), n.Content(), nil)
}
+191
View File
@@ -0,0 +1,191 @@
package node
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"mediarr/content"
"mediarr/document"
"mediarr/ident"
"mediarr/identity"
)
func TestNodeIngestScan(t *testing.T) {
t.Chdir(t.TempDir())
err, node := NewNode(NewNodeParams{DatabaseName: "ingest", StoragePath: "store"})
if err != nil {
t.Fatalf("NewNode : %v", err)
}
t.Cleanup(func() {
if err := node.Close(); err != nil {
t.Errorf("Close : %v", err)
}
})
if err := node.Up(); err != nil {
t.Fatalf("Up : %v", err)
}
ctx := context.Background()
owner, err := node.Identity().CreatePrincipal(ctx, identity.CreatePrincipalParams{
Kind: identity.KindUser,
Name: "owner",
})
if err != nil {
t.Fatalf("CreatePrincipal(owner) : %v", err)
}
other, err := node.Identity().CreatePrincipal(ctx, identity.CreatePrincipalParams{
Kind: identity.KindUser,
Name: "other",
})
if err != nil {
t.Fatalf("CreatePrincipal(other) : %v", err)
}
library, err := node.Documents().CreateLibrary(ctx, owner.ID, document.CreateLibraryParams{
OwnerID: owner.ID,
CreatedBy: owner.ID,
Name: "Films",
PolicyID: ident.MustParse("00000000-0000-7000-8000-000000000101"),
})
if err != nil {
t.Fatalf("CreateLibrary : %v", err)
}
source := filepath.Join(t.TempDir(), "library")
if err := os.MkdirAll(filepath.Join(source, "Season 1"), 0o755); err != nil {
t.Fatalf("MkdirAll : %v", err)
}
writeNodeTestFile(t, filepath.Join(source, "film.mkv"), "film")
writeNodeTestFile(t, filepath.Join(source, "notes.txt"), "notes")
writeNodeTestFile(t, filepath.Join(source, "Season 1", "episode.srt"), "subs")
result, err := node.Ingest().Scan(ctx, owner.ID, library.ID, source)
if err != nil {
t.Fatalf("Scan : %v", err)
}
if result.HasErrors() {
t.Fatalf("Scan erreurs : %v", result.Errors)
}
if result.Files != 3 || result.Directories != 1 || result.Created != 4 || result.Updated != 0 || result.Unchanged != 0 {
t.Fatalf("résultat scan = %+v", result)
}
children, err := node.Documents().ListChildren(ctx, owner.ID, library.ID)
if err != nil {
t.Fatalf("ListChildren(library) : %v", err)
}
film := findNodeTestChild(t, children, "film.mkv")
notes := findNodeTestChild(t, children, "notes.txt")
season := findNodeTestChild(t, children, "Season 1")
if film.Kind != document.KindVideo {
t.Fatalf("kind film = %q, attendu %q", film.Kind, document.KindVideo)
}
if notes.Kind != document.KindFile {
t.Fatalf("kind notes = %q, attendu %q", notes.Kind, document.KindFile)
}
if season.Kind != document.KindFolder {
t.Fatalf("kind saison = %q, attendu %q", season.Kind, document.KindFolder)
}
if !strings.Contains(film.Metadata, `"source_path":"film.mkv"`) {
t.Fatalf("metadata film = %q", film.Metadata)
}
seasonChildren, err := node.Documents().ListChildren(ctx, owner.ID, season.ID)
if err != nil {
t.Fatalf("ListChildren(saison) : %v", err)
}
episode := findNodeTestChild(t, seasonChildren, "episode.srt")
if episode.Kind != document.KindSubtitle {
t.Fatalf("kind épisode = %q, attendu %q", episode.Kind, document.KindSubtitle)
}
versions, err := node.Content().ListVersions(ctx, owner.ID, film.ID)
if err != nil {
t.Fatalf("ListVersions(film) : %v", err)
}
if len(versions) != 1 {
t.Fatalf("versions film = %d, attendu 1", len(versions))
}
if versions[0].Seq != 1 || versions[0].Origin != content.OriginImport {
t.Fatalf("version film = %+v", versions[0])
}
head, err := node.Content().HeadVersion(ctx, owner.ID, film.ID)
if err != nil {
t.Fatalf("HeadVersion(film) : %v", err)
}
if head == nil || head.ID != versions[0].ID {
t.Fatalf("head film = %+v, attendu %s", head, versions[0].ID)
}
result, err = node.Ingest().Scan(ctx, owner.ID, library.ID, source)
if err != nil {
t.Fatalf("Scan(rescan) : %v", err)
}
if result.HasErrors() {
t.Fatalf("Scan(rescan) erreurs : %v", result.Errors)
}
if result.Created != 0 || result.Updated != 0 || result.Unchanged != 3 {
t.Fatalf("résultat rescan = %+v", result)
}
versions, err = node.Content().ListVersions(ctx, owner.ID, film.ID)
if err != nil {
t.Fatalf("ListVersions(film après rescan) : %v", err)
}
if len(versions) != 1 {
t.Fatalf("versions film après rescan = %d, attendu 1", len(versions))
}
writeNodeTestFile(t, filepath.Join(source, "film.mkv"), "film v2")
result, err = node.Ingest().Scan(ctx, owner.ID, library.ID, source)
if err != nil {
t.Fatalf("Scan(modifié) : %v", err)
}
if result.HasErrors() {
t.Fatalf("Scan(modifié) erreurs : %v", result.Errors)
}
if result.Created != 0 || result.Updated != 1 || result.Unchanged != 2 {
t.Fatalf("résultat scan modifié = %+v", result)
}
versions, err = node.Content().ListVersions(ctx, owner.ID, film.ID)
if err != nil {
t.Fatalf("ListVersions(film modifié) : %v", err)
}
if len(versions) != 2 {
t.Fatalf("versions film modifié = %d, attendu 2", len(versions))
}
if versions[0].Seq != 2 || versions[0].Origin != content.OriginImport {
t.Fatalf("nouvelle version = %+v", versions[0])
}
if _, err := node.Ingest().Scan(ctx, other.ID, library.ID, source); !errors.Is(err, document.ErrPermissionDenied) {
t.Fatalf("Scan sans Read : erreur = %v, attendu ErrPermissionDenied", err)
}
}
func writeNodeTestFile(t *testing.T, path, content string) {
t.Helper()
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatalf("WriteFile(%s) : %v", path, err)
}
}
func findNodeTestChild(t *testing.T, children []document.Document, name string) *document.Document {
t.Helper()
for i := range children {
if children[i].Name == name {
return &children[i]
}
}
t.Fatalf("enfant %q introuvable dans %+v", name, children)
return nil
}