Files
Mediarr/ingest/scan.go
T
2026-11-16 11:31:37 +01:00

518 lines
13 KiB
Go

package ingest
import (
"bytes"
"context"
"crypto/sha256"
"fmt"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
"mediarr/content"
"mediarr/document"
"mediarr/ident"
)
type ScanOptions struct {
Prune bool
}
func (s *Service) Scan(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (ScanResult, error) {
return s.ScanWithOptions(ctx, actorID, parentID, rootPath, ScanOptions{})
}
func (s *Service) ScanWithOptions(ctx context.Context, actorID, parentID ident.UUID, rootPath string, options ScanOptions) (ScanResult, error) {
absolute, parent, info, err := s.prepare(ctx, actorID, parentID, rootPath)
if err != nil {
return ScanResult{}, err
}
if !info.IsDir() {
return ScanResult{}, fmt.Errorf("%w: %s n'est pas un dossier", ErrInvalidRoot, rootPath)
}
return s.scanTree(ctx, actorID, parent, absolute, options)
}
func (s *Service) Import(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (ScanResult, error) {
return s.ImportWithOptions(ctx, actorID, parentID, rootPath, ScanOptions{})
}
func (s *Service) ImportWithOptions(ctx context.Context, actorID, parentID ident.UUID, rootPath string, options ScanOptions) (ScanResult, error) {
absolute, parent, info, err := s.prepare(ctx, actorID, parentID, rootPath)
if err != nil {
return ScanResult{}, err
}
if info.IsDir() {
return s.scanTree(ctx, actorID, parent, absolute, options)
}
if !info.Mode().IsRegular() {
return ScanResult{}, fmt.Errorf("%w: %s n'est ni un fichier ni un dossier", ErrInvalidRoot, rootPath)
}
return s.importFile(ctx, actorID, parent, absolute)
}
func (s *Service) prepare(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (string, *document.Document, fs.FileInfo, error) {
if s == nil || s.documents == nil || s.contentService == nil {
return "", nil, nil, ErrServiceNotConfigured
}
rootPath = strings.TrimSpace(rootPath)
if rootPath == "" {
return "", nil, nil, fmt.Errorf("%w: chemin vide", ErrInvalidRoot)
}
absolute, err := filepath.Abs(rootPath)
if err != nil {
return "", nil, nil, fmt.Errorf("%w: %v", ErrInvalidRoot, err)
}
parent, err := s.documents.Get(ctx, actorID, parentID)
if err != nil {
return "", nil, nil, err
}
if parent.DeletedAt != nil {
return "", nil, nil, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, parentID)
}
if parent.Kind != document.KindLibrary && parent.Kind != document.KindFolder {
return "", nil, nil, fmt.Errorf("%w: %s", document.ErrInvalidParent, parent.Kind)
}
info, err := os.Stat(absolute)
if err != nil {
return "", nil, nil, fmt.Errorf("%w: %s: %v", ErrInvalidRoot, rootPath, err)
}
return absolute, parent, info, nil
}
func (s *Service) scanTree(ctx context.Context, actorID ident.UUID, parent *document.Document, absolute string, options ScanOptions) (ScanResult, error) {
var result ScanResult
rootChildren, err := s.children(ctx, actorID, parent.ID)
if err != nil {
return result, err
}
directories := map[string]*document.Document{".": parent}
children := map[ident.UUID]map[string]*document.Document{parent.ID: rootChildren}
seen := make(map[ident.UUID]struct{})
walkErr := filepath.WalkDir(absolute, func(fullPath string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: walkErr})
if entry != nil && entry.IsDir() {
return fs.SkipDir
}
return nil
}
if fullPath == absolute {
return nil
}
relPath, err := filepath.Rel(absolute, fullPath)
if err != nil {
result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err})
return nil
}
relPath = filepath.ToSlash(relPath)
if entry.Type()&fs.ModeSymlink != 0 {
s.markPresent(ctx, actorID, relPath, entry.Name(), directories, children, seen)
return nil
}
if entry.IsDir() {
result.Directories++
_, created, err := s.scanDirectory(ctx, actorID, absolute, relPath, entry.Name(), directories, children, seen)
if err != nil {
result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err})
return fs.SkipDir
}
if created {
result.Created++
}
return nil
}
if !entry.Type().IsRegular() {
s.markPresent(ctx, actorID, relPath, entry.Name(), directories, children, seen)
return nil
}
result.Files++
created, updated, err := s.scanFile(ctx, actorID, absolute, fullPath, relPath, entry.Name(), directories, children, seen)
if err != nil {
result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err})
return nil
}
switch {
case created:
result.Created++
case updated:
result.Updated++
default:
result.Unchanged++
}
return nil
})
if walkErr != nil {
return result, walkErr
}
if options.Prune && len(result.Errors) == 0 {
pruned, errs := s.prune(ctx, actorID, absolute, directories, children, seen)
result.Pruned = pruned
result.Errors = append(result.Errors, errs...)
}
return result, nil
}
func (s *Service) importFile(ctx context.Context, actorID ident.UUID, parent *document.Document, absolute string) (ScanResult, error) {
var result ScanResult
rootChildren, err := s.children(ctx, actorID, parent.ID)
if err != nil {
return result, err
}
directories := map[string]*document.Document{".": parent}
children := map[ident.UUID]map[string]*document.Document{parent.ID: rootChildren}
seen := make(map[ident.UUID]struct{})
name := filepath.Base(absolute)
result.Files = 1
created, updated, err := s.scanFile(ctx, actorID, filepath.Dir(absolute), absolute, name, name, directories, children, seen)
if err != nil {
return result, err
}
switch {
case created:
result.Created = 1
case updated:
result.Updated = 1
default:
result.Unchanged = 1
}
return result, nil
}
func (s *Service) scanDirectory(
ctx context.Context,
actorID ident.UUID,
root, relPath, name string,
directories map[string]*document.Document,
children map[ident.UUID]map[string]*document.Document,
seen map[ident.UUID]struct{},
) (*document.Document, bool, error) {
parentRel := path.Dir(relPath)
parent := directories[parentRel]
if parent == nil {
return nil, false, fmt.Errorf("%w: parent introuvable pour %s", ErrInvalidRoot, relPath)
}
siblings, err := s.siblings(ctx, actorID, parent, children)
if err != nil {
return nil, false, err
}
nameKey := strings.ToLower(name)
existing := siblings[nameKey]
if existing != nil {
seen[existing.ID] = struct{}{}
if existing.Name != name {
return nil, false, fmt.Errorf("%w: %s", ErrNameConflict, relPath)
}
if existing.Kind != document.KindFolder {
return nil, false, fmt.Errorf("%w: %s", ErrNotFolder, relPath)
}
directories[relPath] = existing
return existing, false, nil
}
metadata, err := directoryMetadata("", root, relPath)
if err != nil {
return nil, false, err
}
created, err := s.documents.CreateChild(ctx, actorID, document.CreateChildParams{
Kind: document.KindFolder,
OwnerID: actorID,
CreatedBy: actorID,
ParentID: parent.ID,
Name: name,
Metadata: metadata,
})
if err != nil {
return nil, false, err
}
siblings[nameKey] = created
seen[created.ID] = struct{}{}
directories[relPath] = created
return created, true, nil
}
func (s *Service) scanFile(
ctx context.Context,
actorID ident.UUID,
root, fullPath, relPath, name string,
directories map[string]*document.Document,
children map[ident.UUID]map[string]*document.Document,
seen map[ident.UUID]struct{},
) (bool, bool, error) {
parentRel := path.Dir(relPath)
parent := directories[parentRel]
if parent == nil {
return false, false, fmt.Errorf("%w: parent introuvable pour %s", ErrInvalidRoot, relPath)
}
siblings, err := s.siblings(ctx, actorID, parent, children)
if err != nil {
return false, false, err
}
nameKey := strings.ToLower(name)
existing := siblings[nameKey]
if existing != nil {
seen[existing.ID] = struct{}{}
if existing.Name != name {
return false, false, fmt.Errorf("%w: %s", ErrNameConflict, relPath)
}
if existing.Kind == document.KindFolder || existing.Kind == document.KindLibrary {
return false, false, fmt.Errorf("%w: %s", ErrNotFile, relPath)
}
}
info, err := os.Stat(fullPath)
if err != nil {
return false, false, err
}
probe, err := s.prober.Probe(ctx, fullPath, info)
if err != nil {
return false, false, err
}
existingMetadata := ""
if existing != nil {
existingMetadata = existing.Metadata
}
metadata, err := fileMetadata(existingMetadata, root, relPath, info, probe)
if err != nil {
return false, false, err
}
created := false
current := existing
if current == nil {
current, err = s.documents.CreateChild(ctx, actorID, document.CreateChildParams{
Kind: KindForFile(name),
OwnerID: actorID,
CreatedBy: actorID,
ParentID: parent.ID,
Name: name,
Metadata: metadata,
})
if err != nil {
return false, false, err
}
created = true
siblings[nameKey] = current
} else if current.Metadata != metadata {
current, err = s.documents.SetMetadata(ctx, actorID, current.ID, metadata)
if err != nil {
return false, false, err
}
siblings[nameKey] = current
}
seen[current.ID] = struct{}{}
contentRoot, size, err := hashFile(fullPath)
if err != nil {
return false, false, err
}
head, err := s.contentService.HeadVersion(ctx, actorID, current.ID)
if err != nil {
return false, false, err
}
if head != nil && head.Size == size && bytes.Equal(head.ContentRoot, contentRoot) {
return created, false, nil
}
file, err := os.Open(fullPath)
if err != nil {
return false, false, err
}
defer func() {
_ = file.Close()
}()
if _, err := s.contentService.CreateVersion(ctx, actorID, current.ID, content.CreateVersionParams{
Source: file,
Mime: probe.Mime,
Origin: content.OriginImport,
Message: "scan",
}); err != nil {
return false, false, err
}
return created, true, nil
}
func (s *Service) markPresent(
ctx context.Context,
actorID ident.UUID,
relPath, name string,
directories map[string]*document.Document,
children map[ident.UUID]map[string]*document.Document,
seen map[ident.UUID]struct{},
) {
parent := directories[path.Dir(relPath)]
if parent == nil {
return
}
siblings, err := s.siblings(ctx, actorID, parent, children)
if err != nil {
return
}
if existing := siblings[strings.ToLower(name)]; existing != nil {
seen[existing.ID] = struct{}{}
}
}
func (s *Service) prune(
ctx context.Context,
actorID ident.UUID,
root string,
directories map[string]*document.Document,
children map[ident.UUID]map[string]*document.Document,
seen map[ident.UUID]struct{},
) (int, []ScanError) {
var pruned int
var errs []ScanError
visited := make(map[ident.UUID]struct{})
prunable := make(map[ident.UUID]bool)
for relPath, parent := range directories {
siblings, err := s.siblings(ctx, actorID, parent, children)
if err != nil {
errs = append(errs, ScanError{Path: relPath, Err: err})
continue
}
for _, child := range siblings {
if _, ok := seen[child.ID]; ok {
continue
}
if _, ok := visited[child.ID]; ok {
continue
}
if !s.prunable(ctx, actorID, child, root, prunable) {
continue
}
if _, err := s.documents.Trash(ctx, actorID, child.ID); err != nil {
errs = append(errs, ScanError{Path: relPath, Err: err})
continue
}
visited[child.ID] = struct{}{}
pruned++
}
}
return pruned, errs
}
func (s *Service) prunable(ctx context.Context, actorID ident.UUID, doc *document.Document, root string, cache map[ident.UUID]bool) bool {
if allowed, exists := cache[doc.ID]; exists {
return allowed
}
if !hasSourcePath(doc.Metadata, root) {
cache[doc.ID] = false
return false
}
if doc.Kind != document.KindFolder {
cache[doc.ID] = true
return true
}
children, err := s.documents.ListChildren(ctx, actorID, doc.ID)
if err != nil {
cache[doc.ID] = false
return false
}
for i := range children {
if !s.prunable(ctx, actorID, &children[i], root, cache) {
cache[doc.ID] = false
return false
}
}
cache[doc.ID] = true
return true
}
func (s *Service) siblings(
ctx context.Context,
actorID ident.UUID,
parent *document.Document,
children map[ident.UUID]map[string]*document.Document,
) (map[string]*document.Document, error) {
if siblings, exists := children[parent.ID]; exists {
return siblings, nil
}
siblings, err := s.children(ctx, actorID, parent.ID)
if err != nil {
return nil, err
}
children[parent.ID] = siblings
return siblings, nil
}
func (s *Service) children(ctx context.Context, actorID, parentID ident.UUID) (map[string]*document.Document, error) {
rows, err := s.documents.ListChildren(ctx, actorID, parentID)
if err != nil {
return nil, err
}
result := make(map[string]*document.Document, len(rows))
for i := range rows {
result[rows[i].NameKey] = &rows[i]
}
return result, nil
}
func hashFile(filePath string) ([]byte, int64, error) {
file, err := os.Open(filePath)
if err != nil {
return nil, 0, err
}
defer func() {
_ = file.Close()
}()
hash := sha256.New()
size, err := io.Copy(hash, file)
if err != nil {
return nil, 0, err
}
return hash.Sum(nil), size, nil
}