ingest documents
This commit is contained in:
@@ -0,0 +1,150 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"mime"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"mediarr/content"
|
||||
"mediarr/document"
|
||||
)
|
||||
|
||||
const (
|
||||
KindVideo = document.KindVideo
|
||||
KindSubtitle = document.KindSubtitle
|
||||
KindImage = document.KindImage
|
||||
KindFile = document.KindFile
|
||||
)
|
||||
|
||||
type Probe struct {
|
||||
Mime string
|
||||
Metadata map[string]any
|
||||
}
|
||||
|
||||
type Prober interface {
|
||||
Probe(ctx context.Context, path string, info fs.FileInfo) (Probe, error)
|
||||
}
|
||||
|
||||
type BasicProber struct{}
|
||||
|
||||
func (BasicProber) Probe(_ context.Context, path string, _ fs.FileInfo) (Probe, error) {
|
||||
mimeType := mime.TypeByExtension(strings.ToLower(filepath.Ext(path)))
|
||||
if mimeType == "" {
|
||||
mimeType = "application/octet-stream"
|
||||
}
|
||||
|
||||
return Probe{Mime: mimeType}, nil
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
documents *document.Service
|
||||
contentService *content.Service
|
||||
prober Prober
|
||||
}
|
||||
|
||||
func NewService(documents *document.Service, contentService *content.Service, prober Prober) *Service {
|
||||
if prober == nil {
|
||||
prober = BasicProber{}
|
||||
}
|
||||
|
||||
return &Service{
|
||||
documents: documents,
|
||||
contentService: contentService,
|
||||
prober: prober,
|
||||
}
|
||||
}
|
||||
|
||||
type ScanResult struct {
|
||||
Files int
|
||||
Directories int
|
||||
Created int
|
||||
Updated int
|
||||
Unchanged int
|
||||
Errors []ScanError
|
||||
}
|
||||
|
||||
func (r ScanResult) HasErrors() bool {
|
||||
return len(r.Errors) > 0
|
||||
}
|
||||
|
||||
type ScanError struct {
|
||||
Path string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e ScanError) Error() string {
|
||||
return fmt.Sprintf("%s: %v", e.Path, e.Err)
|
||||
}
|
||||
|
||||
func (e ScanError) Unwrap() error {
|
||||
return e.Err
|
||||
}
|
||||
|
||||
var (
|
||||
ErrServiceNotConfigured = errors.New("service d'ingestion non configuré")
|
||||
ErrInvalidRoot = errors.New("racine de scan invalide")
|
||||
ErrNotFolder = errors.New("document existant n'est pas un dossier")
|
||||
ErrNotFile = errors.New("document existant n'est pas un fichier")
|
||||
ErrNameConflict = errors.New("conflit de nom insensible à la casse")
|
||||
)
|
||||
|
||||
func KindForFile(name string) string {
|
||||
switch strings.ToLower(filepath.Ext(name)) {
|
||||
case ".mkv", ".mp4", ".m4v", ".avi", ".mov", ".webm", ".ts", ".m2ts", ".wmv", ".flv", ".mpg", ".mpeg":
|
||||
return KindVideo
|
||||
case ".srt", ".ass", ".ssa", ".vtt", ".sub":
|
||||
return KindSubtitle
|
||||
case ".jpg", ".jpeg", ".png", ".webp", ".gif", ".bmp", ".tif", ".tiff", ".heic", ".avif":
|
||||
return KindImage
|
||||
default:
|
||||
return KindFile
|
||||
}
|
||||
}
|
||||
|
||||
func metadataJSON(existing string, updates map[string]any) (string, error) {
|
||||
values := make(map[string]any)
|
||||
if existing != "" {
|
||||
if err := json.Unmarshal([]byte(existing), &values); err != nil {
|
||||
return "", fmt.Errorf("%w: %v", document.ErrInvalidMetadata, err)
|
||||
}
|
||||
if values == nil {
|
||||
values = make(map[string]any)
|
||||
}
|
||||
}
|
||||
|
||||
for key, value := range updates {
|
||||
values[key] = value
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(values)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return string(encoded), nil
|
||||
}
|
||||
|
||||
func fileMetadata(existing, relPath string, info fs.FileInfo, probe Probe) (string, error) {
|
||||
updates := map[string]any{
|
||||
"source_path": relPath,
|
||||
"size": info.Size(),
|
||||
"mtime": info.ModTime().UnixMilli(),
|
||||
}
|
||||
if probe.Mime != "" {
|
||||
updates["mime"] = probe.Mime
|
||||
}
|
||||
for key, value := range probe.Metadata {
|
||||
updates[key] = value
|
||||
}
|
||||
|
||||
return metadataJSON(existing, updates)
|
||||
}
|
||||
|
||||
func directoryMetadata(existing, relPath string) (string, error) {
|
||||
return metadataJSON(existing, map[string]any{"source_path": relPath})
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"mediarr/document"
|
||||
)
|
||||
|
||||
func TestKindForFile(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"film.mkv": document.KindVideo,
|
||||
"film.MP4": document.KindVideo,
|
||||
"episode.srt": document.KindSubtitle,
|
||||
"poster.JPG": document.KindImage,
|
||||
"notes.txt": document.KindFile,
|
||||
"sans-extension": document.KindFile,
|
||||
"archive.tar.gz": document.KindFile,
|
||||
}
|
||||
for name, expected := range cases {
|
||||
if kind := KindForFile(name); kind != expected {
|
||||
t.Fatalf("KindForFile(%q) = %q, attendu %q", name, kind, expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBasicProber(t *testing.T) {
|
||||
t.Chdir(t.TempDir())
|
||||
path := filepath.Join("film.mkv")
|
||||
if err := os.WriteFile(path, []byte("x"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile : %v", err)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Stat : %v", err)
|
||||
}
|
||||
|
||||
probe, err := BasicProber{}.Probe(context.Background(), path, info)
|
||||
if err != nil {
|
||||
t.Fatalf("Probe : %v", err)
|
||||
}
|
||||
if probe.Mime == "" {
|
||||
t.Fatal("mime vide")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataJSONPreservesExistingFields(t *testing.T) {
|
||||
metadata, err := metadataJSON(`{"genre":"drame","size":1}`, map[string]any{
|
||||
"source_path": "film.mkv",
|
||||
"size": int64(2),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("metadataJSON : %v", err)
|
||||
}
|
||||
if metadata != `{"genre":"drame","size":2,"source_path":"film.mkv"}` {
|
||||
t.Fatalf("metadata = %q", metadata)
|
||||
}
|
||||
|
||||
if _, err := metadataJSON(`{`, map[string]any{"x": 1}); err == nil {
|
||||
t.Fatal("metadataJSON avec JSON invalide : erreur attendue")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanRequiresServices(t *testing.T) {
|
||||
var service *Service
|
||||
if _, err := service.Scan(context.Background(), [16]byte{}, [16]byte{}, "."); err != ErrServiceNotConfigured {
|
||||
t.Fatalf("erreur = %v, attendu ErrServiceNotConfigured", err)
|
||||
}
|
||||
}
|
||||
+335
@@ -0,0 +1,335 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"mediarr/content"
|
||||
"mediarr/document"
|
||||
"mediarr/ident"
|
||||
)
|
||||
|
||||
func (s *Service) Scan(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (ScanResult, error) {
|
||||
var result ScanResult
|
||||
|
||||
if s == nil || s.documents == nil || s.contentService == nil {
|
||||
return result, ErrServiceNotConfigured
|
||||
}
|
||||
|
||||
rootPath = strings.TrimSpace(rootPath)
|
||||
if rootPath == "" {
|
||||
return result, fmt.Errorf("%w: chemin vide", ErrInvalidRoot)
|
||||
}
|
||||
|
||||
absolute, err := filepath.Abs(rootPath)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("%w: %v", ErrInvalidRoot, err)
|
||||
}
|
||||
|
||||
info, err := os.Stat(absolute)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("%w: %s: %v", ErrInvalidRoot, rootPath, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return result, fmt.Errorf("%w: %s n'est pas un dossier", ErrInvalidRoot, rootPath)
|
||||
}
|
||||
|
||||
parent, err := s.documents.Get(ctx, actorID, parentID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if parent.DeletedAt != nil {
|
||||
return result, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, parentID)
|
||||
}
|
||||
if parent.Kind != document.KindLibrary && parent.Kind != document.KindFolder {
|
||||
return result, fmt.Errorf("%w: %s", document.ErrInvalidParent, parent.Kind)
|
||||
}
|
||||
|
||||
rootChildren, err := s.children(ctx, actorID, parentID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
directories := map[string]*document.Document{".": parent}
|
||||
children := map[ident.UUID]map[string]*document.Document{parentID: rootChildren}
|
||||
|
||||
walkErr := filepath.WalkDir(absolute, func(fullPath string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: walkErr})
|
||||
if entry != nil && entry.IsDir() {
|
||||
return fs.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if fullPath == absolute {
|
||||
return nil
|
||||
}
|
||||
|
||||
relPath, err := filepath.Rel(absolute, fullPath)
|
||||
if err != nil {
|
||||
result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err})
|
||||
return nil
|
||||
}
|
||||
relPath = filepath.ToSlash(relPath)
|
||||
|
||||
if entry.Type()&fs.ModeSymlink != 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if entry.IsDir() {
|
||||
result.Directories++
|
||||
_, created, err := s.scanDirectory(ctx, actorID, relPath, entry.Name(), directories, children)
|
||||
if err != nil {
|
||||
result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err})
|
||||
return fs.SkipDir
|
||||
}
|
||||
if created {
|
||||
result.Created++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if !entry.Type().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
|
||||
result.Files++
|
||||
created, updated, err := s.scanFile(ctx, actorID, fullPath, relPath, entry.Name(), directories, children)
|
||||
if err != nil {
|
||||
result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err})
|
||||
return nil
|
||||
}
|
||||
switch {
|
||||
case created:
|
||||
result.Created++
|
||||
case updated:
|
||||
result.Updated++
|
||||
default:
|
||||
result.Unchanged++
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if walkErr != nil {
|
||||
return result, walkErr
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Service) scanDirectory(
|
||||
ctx context.Context,
|
||||
actorID ident.UUID,
|
||||
relPath, name string,
|
||||
directories map[string]*document.Document,
|
||||
children map[ident.UUID]map[string]*document.Document,
|
||||
) (*document.Document, bool, error) {
|
||||
parentRel := path.Dir(relPath)
|
||||
parent := directories[parentRel]
|
||||
if parent == nil {
|
||||
return nil, false, fmt.Errorf("%w: parent introuvable pour %s", ErrInvalidRoot, relPath)
|
||||
}
|
||||
|
||||
siblings, err := s.siblings(ctx, actorID, parent, children)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
nameKey := strings.ToLower(name)
|
||||
existing := siblings[nameKey]
|
||||
if existing != nil {
|
||||
if existing.Name != name {
|
||||
return nil, false, fmt.Errorf("%w: %s", ErrNameConflict, relPath)
|
||||
}
|
||||
if existing.Kind != document.KindFolder {
|
||||
return nil, false, fmt.Errorf("%w: %s", ErrNotFolder, relPath)
|
||||
}
|
||||
directories[relPath] = existing
|
||||
return existing, false, nil
|
||||
}
|
||||
|
||||
metadata, err := directoryMetadata("", relPath)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
created, err := s.documents.CreateChild(ctx, actorID, document.CreateChildParams{
|
||||
Kind: document.KindFolder,
|
||||
OwnerID: actorID,
|
||||
CreatedBy: actorID,
|
||||
ParentID: parent.ID,
|
||||
Name: name,
|
||||
Metadata: metadata,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
siblings[nameKey] = created
|
||||
directories[relPath] = created
|
||||
return created, true, nil
|
||||
}
|
||||
|
||||
func (s *Service) scanFile(
|
||||
ctx context.Context,
|
||||
actorID ident.UUID,
|
||||
fullPath, relPath, name string,
|
||||
directories map[string]*document.Document,
|
||||
children map[ident.UUID]map[string]*document.Document,
|
||||
) (bool, bool, error) {
|
||||
parentRel := path.Dir(relPath)
|
||||
parent := directories[parentRel]
|
||||
if parent == nil {
|
||||
return false, false, fmt.Errorf("%w: parent introuvable pour %s", ErrInvalidRoot, relPath)
|
||||
}
|
||||
|
||||
siblings, err := s.siblings(ctx, actorID, parent, children)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
|
||||
nameKey := strings.ToLower(name)
|
||||
existing := siblings[nameKey]
|
||||
if existing != nil {
|
||||
if existing.Name != name {
|
||||
return false, false, fmt.Errorf("%w: %s", ErrNameConflict, relPath)
|
||||
}
|
||||
if existing.Kind == document.KindFolder || existing.Kind == document.KindLibrary {
|
||||
return false, false, fmt.Errorf("%w: %s", ErrNotFile, relPath)
|
||||
}
|
||||
}
|
||||
|
||||
info, err := os.Stat(fullPath)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
|
||||
probe, err := s.prober.Probe(ctx, fullPath, info)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
|
||||
existingMetadata := ""
|
||||
if existing != nil {
|
||||
existingMetadata = existing.Metadata
|
||||
}
|
||||
metadata, err := fileMetadata(existingMetadata, relPath, info, probe)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
|
||||
created := false
|
||||
current := existing
|
||||
if current == nil {
|
||||
current, err = s.documents.CreateChild(ctx, actorID, document.CreateChildParams{
|
||||
Kind: KindForFile(name),
|
||||
OwnerID: actorID,
|
||||
CreatedBy: actorID,
|
||||
ParentID: parent.ID,
|
||||
Name: name,
|
||||
Metadata: metadata,
|
||||
})
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
created = true
|
||||
siblings[nameKey] = current
|
||||
} else if current.Metadata != metadata {
|
||||
current, err = s.documents.SetMetadata(ctx, actorID, current.ID, metadata)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
siblings[nameKey] = current
|
||||
}
|
||||
|
||||
root, size, err := hashFile(fullPath)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
|
||||
head, err := s.contentService.HeadVersion(ctx, actorID, current.ID)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
if head != nil && head.Size == size && bytes.Equal(head.ContentRoot, root) {
|
||||
return created, false, nil
|
||||
}
|
||||
|
||||
file, err := os.Open(fullPath)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
}()
|
||||
|
||||
if _, err := s.contentService.CreateVersion(ctx, actorID, current.ID, content.CreateVersionParams{
|
||||
Source: file,
|
||||
Mime: probe.Mime,
|
||||
Origin: content.OriginImport,
|
||||
Message: "scan",
|
||||
}); err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
|
||||
return created, true, nil
|
||||
}
|
||||
|
||||
func (s *Service) siblings(
|
||||
ctx context.Context,
|
||||
actorID ident.UUID,
|
||||
parent *document.Document,
|
||||
children map[ident.UUID]map[string]*document.Document,
|
||||
) (map[string]*document.Document, error) {
|
||||
if siblings, exists := children[parent.ID]; exists {
|
||||
return siblings, nil
|
||||
}
|
||||
|
||||
siblings, err := s.children(ctx, actorID, parent.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
children[parent.ID] = siblings
|
||||
|
||||
return siblings, nil
|
||||
}
|
||||
|
||||
func (s *Service) children(ctx context.Context, actorID, parentID ident.UUID) (map[string]*document.Document, error) {
|
||||
rows, err := s.documents.ListChildren(ctx, actorID, parentID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := make(map[string]*document.Document, len(rows))
|
||||
for i := range rows {
|
||||
result[rows[i].NameKey] = &rows[i]
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func hashFile(filePath string) ([]byte, int64, error) {
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
}()
|
||||
|
||||
hash := sha256.New()
|
||||
size, err := io.Copy(hash, file)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
return hash.Sum(nil), size, nil
|
||||
}
|
||||
Reference in New Issue
Block a user