scran import is working
This commit is contained in:
+208
-26
@@ -17,49 +17,90 @@ import (
|
||||
"mediarr/ident"
|
||||
)
|
||||
|
||||
func (s *Service) Scan(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (ScanResult, error) {
|
||||
var result ScanResult
|
||||
type ScanOptions struct {
|
||||
Prune bool
|
||||
}
|
||||
|
||||
func (s *Service) Scan(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (ScanResult, error) {
|
||||
return s.ScanWithOptions(ctx, actorID, parentID, rootPath, ScanOptions{})
|
||||
}
|
||||
|
||||
func (s *Service) ScanWithOptions(ctx context.Context, actorID, parentID ident.UUID, rootPath string, options ScanOptions) (ScanResult, error) {
|
||||
absolute, parent, info, err := s.prepare(ctx, actorID, parentID, rootPath)
|
||||
if err != nil {
|
||||
return ScanResult{}, err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return ScanResult{}, fmt.Errorf("%w: %s n'est pas un dossier", ErrInvalidRoot, rootPath)
|
||||
}
|
||||
|
||||
return s.scanTree(ctx, actorID, parent, absolute, options)
|
||||
}
|
||||
|
||||
func (s *Service) Import(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (ScanResult, error) {
|
||||
return s.ImportWithOptions(ctx, actorID, parentID, rootPath, ScanOptions{})
|
||||
}
|
||||
|
||||
func (s *Service) ImportWithOptions(ctx context.Context, actorID, parentID ident.UUID, rootPath string, options ScanOptions) (ScanResult, error) {
|
||||
absolute, parent, info, err := s.prepare(ctx, actorID, parentID, rootPath)
|
||||
if err != nil {
|
||||
return ScanResult{}, err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return s.scanTree(ctx, actorID, parent, absolute, options)
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return ScanResult{}, fmt.Errorf("%w: %s n'est ni un fichier ni un dossier", ErrInvalidRoot, rootPath)
|
||||
}
|
||||
|
||||
return s.importFile(ctx, actorID, parent, absolute)
|
||||
}
|
||||
|
||||
func (s *Service) prepare(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (string, *document.Document, fs.FileInfo, error) {
|
||||
if s == nil || s.documents == nil || s.contentService == nil {
|
||||
return result, ErrServiceNotConfigured
|
||||
return "", nil, nil, ErrServiceNotConfigured
|
||||
}
|
||||
|
||||
rootPath = strings.TrimSpace(rootPath)
|
||||
if rootPath == "" {
|
||||
return result, fmt.Errorf("%w: chemin vide", ErrInvalidRoot)
|
||||
return "", nil, nil, fmt.Errorf("%w: chemin vide", ErrInvalidRoot)
|
||||
}
|
||||
|
||||
absolute, err := filepath.Abs(rootPath)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("%w: %v", ErrInvalidRoot, err)
|
||||
}
|
||||
|
||||
info, err := os.Stat(absolute)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("%w: %s: %v", ErrInvalidRoot, rootPath, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return result, fmt.Errorf("%w: %s n'est pas un dossier", ErrInvalidRoot, rootPath)
|
||||
return "", nil, nil, fmt.Errorf("%w: %v", ErrInvalidRoot, err)
|
||||
}
|
||||
|
||||
parent, err := s.documents.Get(ctx, actorID, parentID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
return "", nil, nil, err
|
||||
}
|
||||
if parent.DeletedAt != nil {
|
||||
return result, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, parentID)
|
||||
return "", nil, nil, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, parentID)
|
||||
}
|
||||
if parent.Kind != document.KindLibrary && parent.Kind != document.KindFolder {
|
||||
return result, fmt.Errorf("%w: %s", document.ErrInvalidParent, parent.Kind)
|
||||
return "", nil, nil, fmt.Errorf("%w: %s", document.ErrInvalidParent, parent.Kind)
|
||||
}
|
||||
|
||||
rootChildren, err := s.children(ctx, actorID, parentID)
|
||||
info, err := os.Stat(absolute)
|
||||
if err != nil {
|
||||
return "", nil, nil, fmt.Errorf("%w: %s: %v", ErrInvalidRoot, rootPath, err)
|
||||
}
|
||||
|
||||
return absolute, parent, info, nil
|
||||
}
|
||||
|
||||
func (s *Service) scanTree(ctx context.Context, actorID ident.UUID, parent *document.Document, absolute string, options ScanOptions) (ScanResult, error) {
|
||||
var result ScanResult
|
||||
|
||||
rootChildren, err := s.children(ctx, actorID, parent.ID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
directories := map[string]*document.Document{".": parent}
|
||||
children := map[ident.UUID]map[string]*document.Document{parentID: rootChildren}
|
||||
children := map[ident.UUID]map[string]*document.Document{parent.ID: rootChildren}
|
||||
seen := make(map[ident.UUID]struct{})
|
||||
|
||||
walkErr := filepath.WalkDir(absolute, func(fullPath string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
@@ -81,12 +122,13 @@ func (s *Service) Scan(ctx context.Context, actorID, parentID ident.UUID, rootPa
|
||||
relPath = filepath.ToSlash(relPath)
|
||||
|
||||
if entry.Type()&fs.ModeSymlink != 0 {
|
||||
s.markPresent(ctx, actorID, relPath, entry.Name(), directories, children, seen)
|
||||
return nil
|
||||
}
|
||||
|
||||
if entry.IsDir() {
|
||||
result.Directories++
|
||||
_, created, err := s.scanDirectory(ctx, actorID, relPath, entry.Name(), directories, children)
|
||||
_, created, err := s.scanDirectory(ctx, actorID, absolute, relPath, entry.Name(), directories, children, seen)
|
||||
if err != nil {
|
||||
result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err})
|
||||
return fs.SkipDir
|
||||
@@ -98,11 +140,12 @@ func (s *Service) Scan(ctx context.Context, actorID, parentID ident.UUID, rootPa
|
||||
}
|
||||
|
||||
if !entry.Type().IsRegular() {
|
||||
s.markPresent(ctx, actorID, relPath, entry.Name(), directories, children, seen)
|
||||
return nil
|
||||
}
|
||||
|
||||
result.Files++
|
||||
created, updated, err := s.scanFile(ctx, actorID, fullPath, relPath, entry.Name(), directories, children)
|
||||
created, updated, err := s.scanFile(ctx, actorID, absolute, fullPath, relPath, entry.Name(), directories, children, seen)
|
||||
if err != nil {
|
||||
result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err})
|
||||
return nil
|
||||
@@ -122,15 +165,54 @@ func (s *Service) Scan(ctx context.Context, actorID, parentID ident.UUID, rootPa
|
||||
return result, walkErr
|
||||
}
|
||||
|
||||
if options.Prune && len(result.Errors) == 0 {
|
||||
pruned, errs := s.prune(ctx, actorID, absolute, directories, children, seen)
|
||||
result.Pruned = pruned
|
||||
result.Errors = append(result.Errors, errs...)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Service) importFile(ctx context.Context, actorID ident.UUID, parent *document.Document, absolute string) (ScanResult, error) {
|
||||
var result ScanResult
|
||||
|
||||
rootChildren, err := s.children(ctx, actorID, parent.ID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
directories := map[string]*document.Document{".": parent}
|
||||
children := map[ident.UUID]map[string]*document.Document{parent.ID: rootChildren}
|
||||
seen := make(map[ident.UUID]struct{})
|
||||
|
||||
name := filepath.Base(absolute)
|
||||
result.Files = 1
|
||||
|
||||
created, updated, err := s.scanFile(ctx, actorID, filepath.Dir(absolute), absolute, name, name, directories, children, seen)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
switch {
|
||||
case created:
|
||||
result.Created = 1
|
||||
case updated:
|
||||
result.Updated = 1
|
||||
default:
|
||||
result.Unchanged = 1
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Service) scanDirectory(
|
||||
ctx context.Context,
|
||||
actorID ident.UUID,
|
||||
relPath, name string,
|
||||
root, relPath, name string,
|
||||
directories map[string]*document.Document,
|
||||
children map[ident.UUID]map[string]*document.Document,
|
||||
seen map[ident.UUID]struct{},
|
||||
) (*document.Document, bool, error) {
|
||||
parentRel := path.Dir(relPath)
|
||||
parent := directories[parentRel]
|
||||
@@ -146,6 +228,7 @@ func (s *Service) scanDirectory(
|
||||
nameKey := strings.ToLower(name)
|
||||
existing := siblings[nameKey]
|
||||
if existing != nil {
|
||||
seen[existing.ID] = struct{}{}
|
||||
if existing.Name != name {
|
||||
return nil, false, fmt.Errorf("%w: %s", ErrNameConflict, relPath)
|
||||
}
|
||||
@@ -156,7 +239,7 @@ func (s *Service) scanDirectory(
|
||||
return existing, false, nil
|
||||
}
|
||||
|
||||
metadata, err := directoryMetadata("", relPath)
|
||||
metadata, err := directoryMetadata("", root, relPath)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
@@ -174,6 +257,7 @@ func (s *Service) scanDirectory(
|
||||
}
|
||||
|
||||
siblings[nameKey] = created
|
||||
seen[created.ID] = struct{}{}
|
||||
directories[relPath] = created
|
||||
return created, true, nil
|
||||
}
|
||||
@@ -181,9 +265,10 @@ func (s *Service) scanDirectory(
|
||||
func (s *Service) scanFile(
|
||||
ctx context.Context,
|
||||
actorID ident.UUID,
|
||||
fullPath, relPath, name string,
|
||||
root, fullPath, relPath, name string,
|
||||
directories map[string]*document.Document,
|
||||
children map[ident.UUID]map[string]*document.Document,
|
||||
seen map[ident.UUID]struct{},
|
||||
) (bool, bool, error) {
|
||||
parentRel := path.Dir(relPath)
|
||||
parent := directories[parentRel]
|
||||
@@ -199,6 +284,7 @@ func (s *Service) scanFile(
|
||||
nameKey := strings.ToLower(name)
|
||||
existing := siblings[nameKey]
|
||||
if existing != nil {
|
||||
seen[existing.ID] = struct{}{}
|
||||
if existing.Name != name {
|
||||
return false, false, fmt.Errorf("%w: %s", ErrNameConflict, relPath)
|
||||
}
|
||||
@@ -221,7 +307,7 @@ func (s *Service) scanFile(
|
||||
if existing != nil {
|
||||
existingMetadata = existing.Metadata
|
||||
}
|
||||
metadata, err := fileMetadata(existingMetadata, relPath, info, probe)
|
||||
metadata, err := fileMetadata(existingMetadata, root, relPath, info, probe)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
@@ -249,8 +335,9 @@ func (s *Service) scanFile(
|
||||
}
|
||||
siblings[nameKey] = current
|
||||
}
|
||||
seen[current.ID] = struct{}{}
|
||||
|
||||
root, size, err := hashFile(fullPath)
|
||||
contentRoot, size, err := hashFile(fullPath)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
@@ -259,7 +346,7 @@ func (s *Service) scanFile(
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
if head != nil && head.Size == size && bytes.Equal(head.ContentRoot, root) {
|
||||
if head != nil && head.Size == size && bytes.Equal(head.ContentRoot, contentRoot) {
|
||||
return created, false, nil
|
||||
}
|
||||
|
||||
@@ -283,6 +370,101 @@ func (s *Service) scanFile(
|
||||
return created, true, nil
|
||||
}
|
||||
|
||||
func (s *Service) markPresent(
|
||||
ctx context.Context,
|
||||
actorID ident.UUID,
|
||||
relPath, name string,
|
||||
directories map[string]*document.Document,
|
||||
children map[ident.UUID]map[string]*document.Document,
|
||||
seen map[ident.UUID]struct{},
|
||||
) {
|
||||
parent := directories[path.Dir(relPath)]
|
||||
if parent == nil {
|
||||
return
|
||||
}
|
||||
|
||||
siblings, err := s.siblings(ctx, actorID, parent, children)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if existing := siblings[strings.ToLower(name)]; existing != nil {
|
||||
seen[existing.ID] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) prune(
|
||||
ctx context.Context,
|
||||
actorID ident.UUID,
|
||||
root string,
|
||||
directories map[string]*document.Document,
|
||||
children map[ident.UUID]map[string]*document.Document,
|
||||
seen map[ident.UUID]struct{},
|
||||
) (int, []ScanError) {
|
||||
var pruned int
|
||||
var errs []ScanError
|
||||
visited := make(map[ident.UUID]struct{})
|
||||
prunable := make(map[ident.UUID]bool)
|
||||
|
||||
for relPath, parent := range directories {
|
||||
siblings, err := s.siblings(ctx, actorID, parent, children)
|
||||
if err != nil {
|
||||
errs = append(errs, ScanError{Path: relPath, Err: err})
|
||||
continue
|
||||
}
|
||||
|
||||
for _, child := range siblings {
|
||||
if _, ok := seen[child.ID]; ok {
|
||||
continue
|
||||
}
|
||||
if _, ok := visited[child.ID]; ok {
|
||||
continue
|
||||
}
|
||||
if !s.prunable(ctx, actorID, child, root, prunable) {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err := s.documents.Trash(ctx, actorID, child.ID); err != nil {
|
||||
errs = append(errs, ScanError{Path: relPath, Err: err})
|
||||
continue
|
||||
}
|
||||
visited[child.ID] = struct{}{}
|
||||
pruned++
|
||||
}
|
||||
}
|
||||
|
||||
return pruned, errs
|
||||
}
|
||||
|
||||
func (s *Service) prunable(ctx context.Context, actorID ident.UUID, doc *document.Document, root string, cache map[ident.UUID]bool) bool {
|
||||
if allowed, exists := cache[doc.ID]; exists {
|
||||
return allowed
|
||||
}
|
||||
|
||||
if !hasSourcePath(doc.Metadata, root) {
|
||||
cache[doc.ID] = false
|
||||
return false
|
||||
}
|
||||
if doc.Kind != document.KindFolder {
|
||||
cache[doc.ID] = true
|
||||
return true
|
||||
}
|
||||
|
||||
children, err := s.documents.ListChildren(ctx, actorID, doc.ID)
|
||||
if err != nil {
|
||||
cache[doc.ID] = false
|
||||
return false
|
||||
}
|
||||
for i := range children {
|
||||
if !s.prunable(ctx, actorID, &children[i], root, cache) {
|
||||
cache[doc.ID] = false
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
cache[doc.ID] = true
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Service) siblings(
|
||||
ctx context.Context,
|
||||
actorID ident.UUID,
|
||||
|
||||
Reference in New Issue
Block a user