package ingest import ( "bytes" "context" "crypto/sha256" "fmt" "io" "io/fs" "os" "path" "path/filepath" "strings" "mediarr/content" "mediarr/document" "mediarr/ident" ) type ScanOptions struct { Prune bool } func (s *Service) Scan(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (ScanResult, error) { return s.ScanWithOptions(ctx, actorID, parentID, rootPath, ScanOptions{}) } func (s *Service) ScanWithOptions(ctx context.Context, actorID, parentID ident.UUID, rootPath string, options ScanOptions) (ScanResult, error) { absolute, parent, info, err := s.prepare(ctx, actorID, parentID, rootPath) if err != nil { return ScanResult{}, err } if !info.IsDir() { return ScanResult{}, fmt.Errorf("%w: %s n'est pas un dossier", ErrInvalidRoot, rootPath) } return s.scanTree(ctx, actorID, parent, absolute, options) } func (s *Service) Import(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (ScanResult, error) { return s.ImportWithOptions(ctx, actorID, parentID, rootPath, ScanOptions{}) } func (s *Service) ImportWithOptions(ctx context.Context, actorID, parentID ident.UUID, rootPath string, options ScanOptions) (ScanResult, error) { absolute, parent, info, err := s.prepare(ctx, actorID, parentID, rootPath) if err != nil { return ScanResult{}, err } if info.IsDir() { return s.scanTree(ctx, actorID, parent, absolute, options) } if !info.Mode().IsRegular() { return ScanResult{}, fmt.Errorf("%w: %s n'est ni un fichier ni un dossier", ErrInvalidRoot, rootPath) } return s.importFile(ctx, actorID, parent, absolute) } func (s *Service) prepare(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (string, *document.Document, fs.FileInfo, error) { if s == nil || s.documents == nil || s.contentService == nil { return "", nil, nil, ErrServiceNotConfigured } rootPath = strings.TrimSpace(rootPath) if rootPath == "" { return "", nil, nil, fmt.Errorf("%w: chemin vide", ErrInvalidRoot) } absolute, err := filepath.Abs(rootPath) if err != nil { return "", nil, nil, fmt.Errorf("%w: %v", ErrInvalidRoot, err) } parent, err := s.documents.Get(ctx, actorID, parentID) if err != nil { return "", nil, nil, err } if parent.DeletedAt != nil { return "", nil, nil, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, parentID) } if parent.Kind != document.KindLibrary && parent.Kind != document.KindFolder { return "", nil, nil, fmt.Errorf("%w: %s", document.ErrInvalidParent, parent.Kind) } info, err := os.Stat(absolute) if err != nil { return "", nil, nil, fmt.Errorf("%w: %s: %v", ErrInvalidRoot, rootPath, err) } return absolute, parent, info, nil } func (s *Service) scanTree(ctx context.Context, actorID ident.UUID, parent *document.Document, absolute string, options ScanOptions) (ScanResult, error) { var result ScanResult rootChildren, err := s.children(ctx, actorID, parent.ID) if err != nil { return result, err } directories := map[string]*document.Document{".": parent} children := map[ident.UUID]map[string]*document.Document{parent.ID: rootChildren} seen := make(map[ident.UUID]struct{}) walkErr := filepath.WalkDir(absolute, func(fullPath string, entry fs.DirEntry, walkErr error) error { if walkErr != nil { result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: walkErr}) if entry != nil && entry.IsDir() { return fs.SkipDir } return nil } if fullPath == absolute { return nil } relPath, err := filepath.Rel(absolute, fullPath) if err != nil { result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err}) return nil } relPath = filepath.ToSlash(relPath) if entry.Type()&fs.ModeSymlink != 0 { s.markPresent(ctx, actorID, relPath, entry.Name(), directories, children, seen) return nil } if entry.IsDir() { result.Directories++ _, created, err := s.scanDirectory(ctx, actorID, absolute, relPath, entry.Name(), directories, children, seen) if err != nil { result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err}) return fs.SkipDir } if created { result.Created++ } return nil } if !entry.Type().IsRegular() { s.markPresent(ctx, actorID, relPath, entry.Name(), directories, children, seen) return nil } result.Files++ created, updated, err := s.scanFile(ctx, actorID, absolute, fullPath, relPath, entry.Name(), directories, children, seen) if err != nil { result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err}) return nil } switch { case created: result.Created++ case updated: result.Updated++ default: result.Unchanged++ } return nil }) if walkErr != nil { return result, walkErr } if options.Prune && len(result.Errors) == 0 { pruned, errs := s.prune(ctx, actorID, absolute, directories, children, seen) result.Pruned = pruned result.Errors = append(result.Errors, errs...) } return result, nil } func (s *Service) importFile(ctx context.Context, actorID ident.UUID, parent *document.Document, absolute string) (ScanResult, error) { var result ScanResult rootChildren, err := s.children(ctx, actorID, parent.ID) if err != nil { return result, err } directories := map[string]*document.Document{".": parent} children := map[ident.UUID]map[string]*document.Document{parent.ID: rootChildren} seen := make(map[ident.UUID]struct{}) name := filepath.Base(absolute) result.Files = 1 created, updated, err := s.scanFile(ctx, actorID, filepath.Dir(absolute), absolute, name, name, directories, children, seen) if err != nil { return result, err } switch { case created: result.Created = 1 case updated: result.Updated = 1 default: result.Unchanged = 1 } return result, nil } func (s *Service) scanDirectory( ctx context.Context, actorID ident.UUID, root, relPath, name string, directories map[string]*document.Document, children map[ident.UUID]map[string]*document.Document, seen map[ident.UUID]struct{}, ) (*document.Document, bool, error) { parentRel := path.Dir(relPath) parent := directories[parentRel] if parent == nil { return nil, false, fmt.Errorf("%w: parent introuvable pour %s", ErrInvalidRoot, relPath) } siblings, err := s.siblings(ctx, actorID, parent, children) if err != nil { return nil, false, err } nameKey := strings.ToLower(name) existing := siblings[nameKey] if existing != nil { seen[existing.ID] = struct{}{} if existing.Name != name { return nil, false, fmt.Errorf("%w: %s", ErrNameConflict, relPath) } if existing.Kind != document.KindFolder { return nil, false, fmt.Errorf("%w: %s", ErrNotFolder, relPath) } directories[relPath] = existing return existing, false, nil } metadata, err := directoryMetadata("", root, relPath) if err != nil { return nil, false, err } created, err := s.documents.CreateChild(ctx, actorID, document.CreateChildParams{ Kind: document.KindFolder, OwnerID: actorID, CreatedBy: actorID, ParentID: parent.ID, Name: name, Metadata: metadata, }) if err != nil { return nil, false, err } siblings[nameKey] = created seen[created.ID] = struct{}{} directories[relPath] = created return created, true, nil } func (s *Service) scanFile( ctx context.Context, actorID ident.UUID, root, fullPath, relPath, name string, directories map[string]*document.Document, children map[ident.UUID]map[string]*document.Document, seen map[ident.UUID]struct{}, ) (bool, bool, error) { parentRel := path.Dir(relPath) parent := directories[parentRel] if parent == nil { return false, false, fmt.Errorf("%w: parent introuvable pour %s", ErrInvalidRoot, relPath) } siblings, err := s.siblings(ctx, actorID, parent, children) if err != nil { return false, false, err } nameKey := strings.ToLower(name) existing := siblings[nameKey] if existing != nil { seen[existing.ID] = struct{}{} if existing.Name != name { return false, false, fmt.Errorf("%w: %s", ErrNameConflict, relPath) } if existing.Kind == document.KindFolder || existing.Kind == document.KindLibrary { return false, false, fmt.Errorf("%w: %s", ErrNotFile, relPath) } } info, err := os.Stat(fullPath) if err != nil { return false, false, err } probe, err := s.prober.Probe(ctx, fullPath, info) if err != nil { return false, false, err } existingMetadata := "" if existing != nil { existingMetadata = existing.Metadata } metadata, err := fileMetadata(existingMetadata, root, relPath, info, probe) if err != nil { return false, false, err } created := false current := existing if current == nil { current, err = s.documents.CreateChild(ctx, actorID, document.CreateChildParams{ Kind: KindForFile(name), OwnerID: actorID, CreatedBy: actorID, ParentID: parent.ID, Name: name, Metadata: metadata, }) if err != nil { return false, false, err } created = true siblings[nameKey] = current } else if current.Metadata != metadata { current, err = s.documents.SetMetadata(ctx, actorID, current.ID, metadata) if err != nil { return false, false, err } siblings[nameKey] = current } seen[current.ID] = struct{}{} contentRoot, size, err := hashFile(fullPath) if err != nil { return false, false, err } head, err := s.contentService.HeadVersion(ctx, actorID, current.ID) if err != nil { return false, false, err } if head != nil && head.Size == size && bytes.Equal(head.ContentRoot, contentRoot) { return created, false, nil } file, err := os.Open(fullPath) if err != nil { return false, false, err } defer func() { _ = file.Close() }() if _, err := s.contentService.CreateVersion(ctx, actorID, current.ID, content.CreateVersionParams{ Source: file, Mime: probe.Mime, Origin: content.OriginImport, Message: "scan", }); err != nil { return false, false, err } return created, true, nil } func (s *Service) markPresent( ctx context.Context, actorID ident.UUID, relPath, name string, directories map[string]*document.Document, children map[ident.UUID]map[string]*document.Document, seen map[ident.UUID]struct{}, ) { parent := directories[path.Dir(relPath)] if parent == nil { return } siblings, err := s.siblings(ctx, actorID, parent, children) if err != nil { return } if existing := siblings[strings.ToLower(name)]; existing != nil { seen[existing.ID] = struct{}{} } } func (s *Service) prune( ctx context.Context, actorID ident.UUID, root string, directories map[string]*document.Document, children map[ident.UUID]map[string]*document.Document, seen map[ident.UUID]struct{}, ) (int, []ScanError) { var pruned int var errs []ScanError visited := make(map[ident.UUID]struct{}) prunable := make(map[ident.UUID]bool) for relPath, parent := range directories { siblings, err := s.siblings(ctx, actorID, parent, children) if err != nil { errs = append(errs, ScanError{Path: relPath, Err: err}) continue } for _, child := range siblings { if _, ok := seen[child.ID]; ok { continue } if _, ok := visited[child.ID]; ok { continue } if !s.prunable(ctx, actorID, child, root, prunable) { continue } if _, err := s.documents.Trash(ctx, actorID, child.ID); err != nil { errs = append(errs, ScanError{Path: relPath, Err: err}) continue } visited[child.ID] = struct{}{} pruned++ } } return pruned, errs } func (s *Service) prunable(ctx context.Context, actorID ident.UUID, doc *document.Document, root string, cache map[ident.UUID]bool) bool { if allowed, exists := cache[doc.ID]; exists { return allowed } if !hasSourcePath(doc.Metadata, root) { cache[doc.ID] = false return false } if doc.Kind != document.KindFolder { cache[doc.ID] = true return true } children, err := s.documents.ListChildren(ctx, actorID, doc.ID) if err != nil { cache[doc.ID] = false return false } for i := range children { if !s.prunable(ctx, actorID, &children[i], root, cache) { cache[doc.ID] = false return false } } cache[doc.ID] = true return true } func (s *Service) siblings( ctx context.Context, actorID ident.UUID, parent *document.Document, children map[ident.UUID]map[string]*document.Document, ) (map[string]*document.Document, error) { if siblings, exists := children[parent.ID]; exists { return siblings, nil } siblings, err := s.children(ctx, actorID, parent.ID) if err != nil { return nil, err } children[parent.ID] = siblings return siblings, nil } func (s *Service) children(ctx context.Context, actorID, parentID ident.UUID) (map[string]*document.Document, error) { rows, err := s.documents.ListChildren(ctx, actorID, parentID) if err != nil { return nil, err } result := make(map[string]*document.Document, len(rows)) for i := range rows { result[rows[i].NameKey] = &rows[i] } return result, nil } func hashFile(filePath string) ([]byte, int64, error) { file, err := os.Open(filePath) if err != nil { return nil, 0, err } defer func() { _ = file.Close() }() hash := sha256.New() size, err := io.Copy(hash, file) if err != nil { return nil, 0, err } return hash.Sum(nil), size, nil }