scran import is working
This commit is contained in:
@@ -0,0 +1,128 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type FFprobeProber struct {
|
||||
Binary string
|
||||
}
|
||||
|
||||
type ffprobeOutput struct {
|
||||
Format struct {
|
||||
FormatName string `json:"format_name"`
|
||||
Duration string `json:"duration"`
|
||||
BitRate string `json:"bit_rate"`
|
||||
} `json:"format"`
|
||||
Streams []struct {
|
||||
CodecType string `json:"codec_type"`
|
||||
CodecName string `json:"codec_name"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
Channels int `json:"channels"`
|
||||
BitRate string `json:"bit_rate"`
|
||||
} `json:"streams"`
|
||||
}
|
||||
|
||||
func (p FFprobeProber) Probe(ctx context.Context, path string, _ fs.FileInfo) (Probe, error) {
|
||||
binary := strings.TrimSpace(p.Binary)
|
||||
if binary == "" {
|
||||
binary = "ffprobe"
|
||||
}
|
||||
|
||||
command := exec.CommandContext(ctx, binary,
|
||||
"-v", "error",
|
||||
"-print_format", "json",
|
||||
"-show_format",
|
||||
"-show_streams",
|
||||
"-i", path,
|
||||
)
|
||||
|
||||
var stderr bytes.Buffer
|
||||
command.Stderr = &stderr
|
||||
|
||||
output, err := command.Output()
|
||||
if err != nil {
|
||||
message := strings.TrimSpace(stderr.String())
|
||||
if message == "" {
|
||||
return Probe{}, fmt.Errorf("ffprobe %s : %w", path, err)
|
||||
}
|
||||
return Probe{}, fmt.Errorf("ffprobe %s : %w: %s", path, err, message)
|
||||
}
|
||||
|
||||
var parsed ffprobeOutput
|
||||
if err := json.Unmarshal(output, &parsed); err != nil {
|
||||
return Probe{}, fmt.Errorf("ffprobe %s : réponse JSON invalide : %w", path, err)
|
||||
}
|
||||
|
||||
probe := Probe{
|
||||
Mime: mimeForPath(path),
|
||||
Metadata: make(map[string]any),
|
||||
}
|
||||
|
||||
if parsed.Format.FormatName != "" {
|
||||
probe.Metadata["container"] = parsed.Format.FormatName
|
||||
}
|
||||
if duration := durationMillis(parsed.Format.Duration); duration > 0 {
|
||||
probe.Metadata["duration_ms"] = duration
|
||||
}
|
||||
if bitRate := positiveInt(parsed.Format.BitRate); bitRate > 0 {
|
||||
probe.Metadata["bit_rate"] = bitRate
|
||||
}
|
||||
|
||||
for _, stream := range parsed.Streams {
|
||||
switch stream.CodecType {
|
||||
case "video":
|
||||
if _, exists := probe.Metadata["video_codec"]; exists {
|
||||
continue
|
||||
}
|
||||
if stream.CodecName != "" {
|
||||
probe.Metadata["video_codec"] = stream.CodecName
|
||||
}
|
||||
if stream.Width > 0 {
|
||||
probe.Metadata["width"] = int64(stream.Width)
|
||||
}
|
||||
if stream.Height > 0 {
|
||||
probe.Metadata["height"] = int64(stream.Height)
|
||||
}
|
||||
if bitRate := positiveInt(stream.BitRate); bitRate > 0 {
|
||||
probe.Metadata["video_bit_rate"] = bitRate
|
||||
}
|
||||
case "audio":
|
||||
if _, exists := probe.Metadata["audio_codec"]; exists {
|
||||
continue
|
||||
}
|
||||
if stream.CodecName != "" {
|
||||
probe.Metadata["audio_codec"] = stream.CodecName
|
||||
}
|
||||
if stream.Channels > 0 {
|
||||
probe.Metadata["audio_channels"] = int64(stream.Channels)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return probe, nil
|
||||
}
|
||||
|
||||
func durationMillis(value string) int64 {
|
||||
seconds, err := strconv.ParseFloat(value, 64)
|
||||
if err != nil || seconds <= 0 {
|
||||
return 0
|
||||
}
|
||||
return int64(seconds * 1000)
|
||||
}
|
||||
|
||||
func positiveInt(value string) int64 {
|
||||
parsed, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || parsed <= 0 {
|
||||
return 0
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFFprobeProber(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
script := filepath.Join(dir, "ffprobe")
|
||||
scriptContent := `#!/bin/sh
|
||||
cat <<'JSON'
|
||||
{"format":{"format_name":"matroska,webm","duration":"12.345","bit_rate":"800000"},"streams":[{"codec_type":"video","codec_name":"h264","width":1920,"height":1080,"bit_rate":"700000"},{"codec_type":"audio","codec_name":"aac","channels":2}]}
|
||||
JSON
|
||||
`
|
||||
if err := os.WriteFile(script, []byte(scriptContent), 0o755); err != nil {
|
||||
t.Fatalf("WriteFile(script) : %v", err)
|
||||
}
|
||||
|
||||
media := filepath.Join(dir, "film.mkv")
|
||||
if err := os.WriteFile(media, []byte("x"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(media) : %v", err)
|
||||
}
|
||||
info, err := os.Stat(media)
|
||||
if err != nil {
|
||||
t.Fatalf("Stat : %v", err)
|
||||
}
|
||||
|
||||
probe, err := FFprobeProber{Binary: script}.Probe(context.Background(), media, info)
|
||||
if err != nil {
|
||||
t.Fatalf("Probe : %v", err)
|
||||
}
|
||||
if probe.Mime == "" {
|
||||
t.Fatal("mime vide")
|
||||
}
|
||||
|
||||
expected := map[string]int64{
|
||||
"duration_ms": 12345,
|
||||
"bit_rate": 800000,
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"video_bit_rate": 700000,
|
||||
"audio_channels": 2,
|
||||
}
|
||||
for key, value := range expected {
|
||||
actual, ok := probe.Metadata[key].(int64)
|
||||
if !ok || actual != value {
|
||||
t.Fatalf("metadata[%q] = %v, attendu %d", key, probe.Metadata[key], value)
|
||||
}
|
||||
}
|
||||
if probe.Metadata["container"] != "matroska,webm" {
|
||||
t.Fatalf("container = %v", probe.Metadata["container"])
|
||||
}
|
||||
if probe.Metadata["video_codec"] != "h264" {
|
||||
t.Fatalf("video_codec = %v", probe.Metadata["video_codec"])
|
||||
}
|
||||
if probe.Metadata["audio_codec"] != "aac" {
|
||||
t.Fatalf("audio_codec = %v", probe.Metadata["audio_codec"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFFprobeProberError(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
script := filepath.Join(dir, "ffprobe")
|
||||
if err := os.WriteFile(script, []byte("#!/bin/sh\necho erreur >&2\nexit 1\n"), 0o755); err != nil {
|
||||
t.Fatalf("WriteFile(script) : %v", err)
|
||||
}
|
||||
|
||||
media := filepath.Join(dir, "film.mkv")
|
||||
if err := os.WriteFile(media, []byte("x"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(media) : %v", err)
|
||||
}
|
||||
|
||||
if _, err := (FFprobeProber{Binary: script}).Probe(context.Background(), media, nil); err == nil {
|
||||
t.Fatal("Probe avec ffprobe en échec : erreur attendue")
|
||||
}
|
||||
}
|
||||
+24
-5
@@ -33,12 +33,15 @@ type Prober interface {
|
||||
type BasicProber struct{}
|
||||
|
||||
func (BasicProber) Probe(_ context.Context, path string, _ fs.FileInfo) (Probe, error) {
|
||||
return Probe{Mime: mimeForPath(path)}, nil
|
||||
}
|
||||
|
||||
func mimeForPath(path string) string {
|
||||
mimeType := mime.TypeByExtension(strings.ToLower(filepath.Ext(path)))
|
||||
if mimeType == "" {
|
||||
mimeType = "application/octet-stream"
|
||||
}
|
||||
|
||||
return Probe{Mime: mimeType}, nil
|
||||
return mimeType
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
@@ -65,6 +68,7 @@ type ScanResult struct {
|
||||
Created int
|
||||
Updated int
|
||||
Unchanged int
|
||||
Pruned int
|
||||
Errors []ScanError
|
||||
}
|
||||
|
||||
@@ -129,8 +133,20 @@ func metadataJSON(existing string, updates map[string]any) (string, error) {
|
||||
return string(encoded), nil
|
||||
}
|
||||
|
||||
func fileMetadata(existing, relPath string, info fs.FileInfo, probe Probe) (string, error) {
|
||||
func hasSourcePath(metadata, root string) bool {
|
||||
var values map[string]any
|
||||
if err := json.Unmarshal([]byte(metadata), &values); err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
sourceRoot, _ := values["source_root"].(string)
|
||||
sourcePath, ok := values["source_path"].(string)
|
||||
return ok && sourcePath != "" && sourceRoot == root
|
||||
}
|
||||
|
||||
func fileMetadata(existing, root, relPath string, info fs.FileInfo, probe Probe) (string, error) {
|
||||
updates := map[string]any{
|
||||
"source_root": root,
|
||||
"source_path": relPath,
|
||||
"size": info.Size(),
|
||||
"mtime": info.ModTime().UnixMilli(),
|
||||
@@ -145,6 +161,9 @@ func fileMetadata(existing, relPath string, info fs.FileInfo, probe Probe) (stri
|
||||
return metadataJSON(existing, updates)
|
||||
}
|
||||
|
||||
func directoryMetadata(existing, relPath string) (string, error) {
|
||||
return metadataJSON(existing, map[string]any{"source_path": relPath})
|
||||
func directoryMetadata(existing, root, relPath string) (string, error) {
|
||||
return metadataJSON(existing, map[string]any{
|
||||
"source_root": root,
|
||||
"source_path": relPath,
|
||||
})
|
||||
}
|
||||
|
||||
+208
-26
@@ -17,49 +17,90 @@ import (
|
||||
"mediarr/ident"
|
||||
)
|
||||
|
||||
func (s *Service) Scan(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (ScanResult, error) {
|
||||
var result ScanResult
|
||||
type ScanOptions struct {
|
||||
Prune bool
|
||||
}
|
||||
|
||||
func (s *Service) Scan(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (ScanResult, error) {
|
||||
return s.ScanWithOptions(ctx, actorID, parentID, rootPath, ScanOptions{})
|
||||
}
|
||||
|
||||
func (s *Service) ScanWithOptions(ctx context.Context, actorID, parentID ident.UUID, rootPath string, options ScanOptions) (ScanResult, error) {
|
||||
absolute, parent, info, err := s.prepare(ctx, actorID, parentID, rootPath)
|
||||
if err != nil {
|
||||
return ScanResult{}, err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return ScanResult{}, fmt.Errorf("%w: %s n'est pas un dossier", ErrInvalidRoot, rootPath)
|
||||
}
|
||||
|
||||
return s.scanTree(ctx, actorID, parent, absolute, options)
|
||||
}
|
||||
|
||||
func (s *Service) Import(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (ScanResult, error) {
|
||||
return s.ImportWithOptions(ctx, actorID, parentID, rootPath, ScanOptions{})
|
||||
}
|
||||
|
||||
func (s *Service) ImportWithOptions(ctx context.Context, actorID, parentID ident.UUID, rootPath string, options ScanOptions) (ScanResult, error) {
|
||||
absolute, parent, info, err := s.prepare(ctx, actorID, parentID, rootPath)
|
||||
if err != nil {
|
||||
return ScanResult{}, err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return s.scanTree(ctx, actorID, parent, absolute, options)
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return ScanResult{}, fmt.Errorf("%w: %s n'est ni un fichier ni un dossier", ErrInvalidRoot, rootPath)
|
||||
}
|
||||
|
||||
return s.importFile(ctx, actorID, parent, absolute)
|
||||
}
|
||||
|
||||
func (s *Service) prepare(ctx context.Context, actorID, parentID ident.UUID, rootPath string) (string, *document.Document, fs.FileInfo, error) {
|
||||
if s == nil || s.documents == nil || s.contentService == nil {
|
||||
return result, ErrServiceNotConfigured
|
||||
return "", nil, nil, ErrServiceNotConfigured
|
||||
}
|
||||
|
||||
rootPath = strings.TrimSpace(rootPath)
|
||||
if rootPath == "" {
|
||||
return result, fmt.Errorf("%w: chemin vide", ErrInvalidRoot)
|
||||
return "", nil, nil, fmt.Errorf("%w: chemin vide", ErrInvalidRoot)
|
||||
}
|
||||
|
||||
absolute, err := filepath.Abs(rootPath)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("%w: %v", ErrInvalidRoot, err)
|
||||
}
|
||||
|
||||
info, err := os.Stat(absolute)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("%w: %s: %v", ErrInvalidRoot, rootPath, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return result, fmt.Errorf("%w: %s n'est pas un dossier", ErrInvalidRoot, rootPath)
|
||||
return "", nil, nil, fmt.Errorf("%w: %v", ErrInvalidRoot, err)
|
||||
}
|
||||
|
||||
parent, err := s.documents.Get(ctx, actorID, parentID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
return "", nil, nil, err
|
||||
}
|
||||
if parent.DeletedAt != nil {
|
||||
return result, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, parentID)
|
||||
return "", nil, nil, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, parentID)
|
||||
}
|
||||
if parent.Kind != document.KindLibrary && parent.Kind != document.KindFolder {
|
||||
return result, fmt.Errorf("%w: %s", document.ErrInvalidParent, parent.Kind)
|
||||
return "", nil, nil, fmt.Errorf("%w: %s", document.ErrInvalidParent, parent.Kind)
|
||||
}
|
||||
|
||||
rootChildren, err := s.children(ctx, actorID, parentID)
|
||||
info, err := os.Stat(absolute)
|
||||
if err != nil {
|
||||
return "", nil, nil, fmt.Errorf("%w: %s: %v", ErrInvalidRoot, rootPath, err)
|
||||
}
|
||||
|
||||
return absolute, parent, info, nil
|
||||
}
|
||||
|
||||
func (s *Service) scanTree(ctx context.Context, actorID ident.UUID, parent *document.Document, absolute string, options ScanOptions) (ScanResult, error) {
|
||||
var result ScanResult
|
||||
|
||||
rootChildren, err := s.children(ctx, actorID, parent.ID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
directories := map[string]*document.Document{".": parent}
|
||||
children := map[ident.UUID]map[string]*document.Document{parentID: rootChildren}
|
||||
children := map[ident.UUID]map[string]*document.Document{parent.ID: rootChildren}
|
||||
seen := make(map[ident.UUID]struct{})
|
||||
|
||||
walkErr := filepath.WalkDir(absolute, func(fullPath string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
@@ -81,12 +122,13 @@ func (s *Service) Scan(ctx context.Context, actorID, parentID ident.UUID, rootPa
|
||||
relPath = filepath.ToSlash(relPath)
|
||||
|
||||
if entry.Type()&fs.ModeSymlink != 0 {
|
||||
s.markPresent(ctx, actorID, relPath, entry.Name(), directories, children, seen)
|
||||
return nil
|
||||
}
|
||||
|
||||
if entry.IsDir() {
|
||||
result.Directories++
|
||||
_, created, err := s.scanDirectory(ctx, actorID, relPath, entry.Name(), directories, children)
|
||||
_, created, err := s.scanDirectory(ctx, actorID, absolute, relPath, entry.Name(), directories, children, seen)
|
||||
if err != nil {
|
||||
result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err})
|
||||
return fs.SkipDir
|
||||
@@ -98,11 +140,12 @@ func (s *Service) Scan(ctx context.Context, actorID, parentID ident.UUID, rootPa
|
||||
}
|
||||
|
||||
if !entry.Type().IsRegular() {
|
||||
s.markPresent(ctx, actorID, relPath, entry.Name(), directories, children, seen)
|
||||
return nil
|
||||
}
|
||||
|
||||
result.Files++
|
||||
created, updated, err := s.scanFile(ctx, actorID, fullPath, relPath, entry.Name(), directories, children)
|
||||
created, updated, err := s.scanFile(ctx, actorID, absolute, fullPath, relPath, entry.Name(), directories, children, seen)
|
||||
if err != nil {
|
||||
result.Errors = append(result.Errors, ScanError{Path: fullPath, Err: err})
|
||||
return nil
|
||||
@@ -122,15 +165,54 @@ func (s *Service) Scan(ctx context.Context, actorID, parentID ident.UUID, rootPa
|
||||
return result, walkErr
|
||||
}
|
||||
|
||||
if options.Prune && len(result.Errors) == 0 {
|
||||
pruned, errs := s.prune(ctx, actorID, absolute, directories, children, seen)
|
||||
result.Pruned = pruned
|
||||
result.Errors = append(result.Errors, errs...)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Service) importFile(ctx context.Context, actorID ident.UUID, parent *document.Document, absolute string) (ScanResult, error) {
|
||||
var result ScanResult
|
||||
|
||||
rootChildren, err := s.children(ctx, actorID, parent.ID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
directories := map[string]*document.Document{".": parent}
|
||||
children := map[ident.UUID]map[string]*document.Document{parent.ID: rootChildren}
|
||||
seen := make(map[ident.UUID]struct{})
|
||||
|
||||
name := filepath.Base(absolute)
|
||||
result.Files = 1
|
||||
|
||||
created, updated, err := s.scanFile(ctx, actorID, filepath.Dir(absolute), absolute, name, name, directories, children, seen)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
switch {
|
||||
case created:
|
||||
result.Created = 1
|
||||
case updated:
|
||||
result.Updated = 1
|
||||
default:
|
||||
result.Unchanged = 1
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Service) scanDirectory(
|
||||
ctx context.Context,
|
||||
actorID ident.UUID,
|
||||
relPath, name string,
|
||||
root, relPath, name string,
|
||||
directories map[string]*document.Document,
|
||||
children map[ident.UUID]map[string]*document.Document,
|
||||
seen map[ident.UUID]struct{},
|
||||
) (*document.Document, bool, error) {
|
||||
parentRel := path.Dir(relPath)
|
||||
parent := directories[parentRel]
|
||||
@@ -146,6 +228,7 @@ func (s *Service) scanDirectory(
|
||||
nameKey := strings.ToLower(name)
|
||||
existing := siblings[nameKey]
|
||||
if existing != nil {
|
||||
seen[existing.ID] = struct{}{}
|
||||
if existing.Name != name {
|
||||
return nil, false, fmt.Errorf("%w: %s", ErrNameConflict, relPath)
|
||||
}
|
||||
@@ -156,7 +239,7 @@ func (s *Service) scanDirectory(
|
||||
return existing, false, nil
|
||||
}
|
||||
|
||||
metadata, err := directoryMetadata("", relPath)
|
||||
metadata, err := directoryMetadata("", root, relPath)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
@@ -174,6 +257,7 @@ func (s *Service) scanDirectory(
|
||||
}
|
||||
|
||||
siblings[nameKey] = created
|
||||
seen[created.ID] = struct{}{}
|
||||
directories[relPath] = created
|
||||
return created, true, nil
|
||||
}
|
||||
@@ -181,9 +265,10 @@ func (s *Service) scanDirectory(
|
||||
func (s *Service) scanFile(
|
||||
ctx context.Context,
|
||||
actorID ident.UUID,
|
||||
fullPath, relPath, name string,
|
||||
root, fullPath, relPath, name string,
|
||||
directories map[string]*document.Document,
|
||||
children map[ident.UUID]map[string]*document.Document,
|
||||
seen map[ident.UUID]struct{},
|
||||
) (bool, bool, error) {
|
||||
parentRel := path.Dir(relPath)
|
||||
parent := directories[parentRel]
|
||||
@@ -199,6 +284,7 @@ func (s *Service) scanFile(
|
||||
nameKey := strings.ToLower(name)
|
||||
existing := siblings[nameKey]
|
||||
if existing != nil {
|
||||
seen[existing.ID] = struct{}{}
|
||||
if existing.Name != name {
|
||||
return false, false, fmt.Errorf("%w: %s", ErrNameConflict, relPath)
|
||||
}
|
||||
@@ -221,7 +307,7 @@ func (s *Service) scanFile(
|
||||
if existing != nil {
|
||||
existingMetadata = existing.Metadata
|
||||
}
|
||||
metadata, err := fileMetadata(existingMetadata, relPath, info, probe)
|
||||
metadata, err := fileMetadata(existingMetadata, root, relPath, info, probe)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
@@ -249,8 +335,9 @@ func (s *Service) scanFile(
|
||||
}
|
||||
siblings[nameKey] = current
|
||||
}
|
||||
seen[current.ID] = struct{}{}
|
||||
|
||||
root, size, err := hashFile(fullPath)
|
||||
contentRoot, size, err := hashFile(fullPath)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
@@ -259,7 +346,7 @@ func (s *Service) scanFile(
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
if head != nil && head.Size == size && bytes.Equal(head.ContentRoot, root) {
|
||||
if head != nil && head.Size == size && bytes.Equal(head.ContentRoot, contentRoot) {
|
||||
return created, false, nil
|
||||
}
|
||||
|
||||
@@ -283,6 +370,101 @@ func (s *Service) scanFile(
|
||||
return created, true, nil
|
||||
}
|
||||
|
||||
func (s *Service) markPresent(
|
||||
ctx context.Context,
|
||||
actorID ident.UUID,
|
||||
relPath, name string,
|
||||
directories map[string]*document.Document,
|
||||
children map[ident.UUID]map[string]*document.Document,
|
||||
seen map[ident.UUID]struct{},
|
||||
) {
|
||||
parent := directories[path.Dir(relPath)]
|
||||
if parent == nil {
|
||||
return
|
||||
}
|
||||
|
||||
siblings, err := s.siblings(ctx, actorID, parent, children)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if existing := siblings[strings.ToLower(name)]; existing != nil {
|
||||
seen[existing.ID] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) prune(
|
||||
ctx context.Context,
|
||||
actorID ident.UUID,
|
||||
root string,
|
||||
directories map[string]*document.Document,
|
||||
children map[ident.UUID]map[string]*document.Document,
|
||||
seen map[ident.UUID]struct{},
|
||||
) (int, []ScanError) {
|
||||
var pruned int
|
||||
var errs []ScanError
|
||||
visited := make(map[ident.UUID]struct{})
|
||||
prunable := make(map[ident.UUID]bool)
|
||||
|
||||
for relPath, parent := range directories {
|
||||
siblings, err := s.siblings(ctx, actorID, parent, children)
|
||||
if err != nil {
|
||||
errs = append(errs, ScanError{Path: relPath, Err: err})
|
||||
continue
|
||||
}
|
||||
|
||||
for _, child := range siblings {
|
||||
if _, ok := seen[child.ID]; ok {
|
||||
continue
|
||||
}
|
||||
if _, ok := visited[child.ID]; ok {
|
||||
continue
|
||||
}
|
||||
if !s.prunable(ctx, actorID, child, root, prunable) {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err := s.documents.Trash(ctx, actorID, child.ID); err != nil {
|
||||
errs = append(errs, ScanError{Path: relPath, Err: err})
|
||||
continue
|
||||
}
|
||||
visited[child.ID] = struct{}{}
|
||||
pruned++
|
||||
}
|
||||
}
|
||||
|
||||
return pruned, errs
|
||||
}
|
||||
|
||||
func (s *Service) prunable(ctx context.Context, actorID ident.UUID, doc *document.Document, root string, cache map[ident.UUID]bool) bool {
|
||||
if allowed, exists := cache[doc.ID]; exists {
|
||||
return allowed
|
||||
}
|
||||
|
||||
if !hasSourcePath(doc.Metadata, root) {
|
||||
cache[doc.ID] = false
|
||||
return false
|
||||
}
|
||||
if doc.Kind != document.KindFolder {
|
||||
cache[doc.ID] = true
|
||||
return true
|
||||
}
|
||||
|
||||
children, err := s.documents.ListChildren(ctx, actorID, doc.ID)
|
||||
if err != nil {
|
||||
cache[doc.ID] = false
|
||||
return false
|
||||
}
|
||||
for i := range children {
|
||||
if !s.prunable(ctx, actorID, &children[i], root, cache) {
|
||||
cache[doc.ID] = false
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
cache[doc.ID] = true
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Service) siblings(
|
||||
ctx context.Context,
|
||||
actorID ident.UUID,
|
||||
|
||||
+5
-1
@@ -3,5 +3,9 @@ package node
|
||||
import "mediarr/ingest"
|
||||
|
||||
func (n *Node) Ingest() *ingest.Service {
|
||||
return ingest.NewService(n.Documents(), n.Content(), nil)
|
||||
return n.IngestWithProber(nil)
|
||||
}
|
||||
|
||||
func (n *Node) IngestWithProber(prober ingest.Prober) *ingest.Service {
|
||||
return ingest.NewService(n.Documents(), n.Content(), prober)
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"mediarr/document"
|
||||
"mediarr/ident"
|
||||
"mediarr/identity"
|
||||
"mediarr/ingest"
|
||||
)
|
||||
|
||||
func TestNodeIngestScan(t *testing.T) {
|
||||
@@ -189,3 +190,128 @@ func findNodeTestChild(t *testing.T, children []document.Document, name string)
|
||||
t.Fatalf("enfant %q introuvable dans %+v", name, children)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestNodeIngestImportAndPrune(t *testing.T) {
|
||||
t.Chdir(t.TempDir())
|
||||
|
||||
err, node := NewNode(NewNodeParams{DatabaseName: "ingest_prune", StoragePath: "store"})
|
||||
if err != nil {
|
||||
t.Fatalf("NewNode : %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := node.Close(); err != nil {
|
||||
t.Errorf("Close : %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if err := node.Up(); err != nil {
|
||||
t.Fatalf("Up : %v", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
owner, err := node.Identity().CreatePrincipal(ctx, identity.CreatePrincipalParams{
|
||||
Kind: identity.KindUser,
|
||||
Name: "owner",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreatePrincipal(owner) : %v", err)
|
||||
}
|
||||
|
||||
library, err := node.Documents().CreateLibrary(ctx, owner.ID, document.CreateLibraryParams{
|
||||
OwnerID: owner.ID,
|
||||
CreatedBy: owner.ID,
|
||||
Name: "Films",
|
||||
PolicyID: ident.MustParse("00000000-0000-7000-8000-000000000101"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateLibrary : %v", err)
|
||||
}
|
||||
|
||||
source := filepath.Join(t.TempDir(), "library")
|
||||
if err := os.MkdirAll(source, 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll : %v", err)
|
||||
}
|
||||
writeNodeTestFile(t, filepath.Join(source, "keep.mkv"), "keep")
|
||||
writeNodeTestFile(t, filepath.Join(source, "remove.txt"), "remove")
|
||||
|
||||
if _, err := node.Ingest().Scan(ctx, owner.ID, library.ID, source); err != nil {
|
||||
t.Fatalf("Scan : %v", err)
|
||||
}
|
||||
|
||||
if _, err := node.Documents().CreateChild(ctx, owner.ID, document.CreateChildParams{
|
||||
Kind: document.KindFile,
|
||||
OwnerID: owner.ID,
|
||||
CreatedBy: owner.ID,
|
||||
ParentID: library.ID,
|
||||
Name: "manual.txt",
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateChild(manual) : %v", err)
|
||||
}
|
||||
|
||||
extra := filepath.Join(t.TempDir(), "extra.jpg")
|
||||
writeNodeTestFile(t, extra, "extra")
|
||||
imported, err := node.Ingest().Import(ctx, owner.ID, library.ID, extra)
|
||||
if err != nil {
|
||||
t.Fatalf("Import : %v", err)
|
||||
}
|
||||
if imported.Files != 1 || imported.Created != 1 || imported.Updated != 0 || imported.Unchanged != 0 {
|
||||
t.Fatalf("résultat import = %+v", imported)
|
||||
}
|
||||
|
||||
imported, err = node.Ingest().Import(ctx, owner.ID, library.ID, extra)
|
||||
if err != nil {
|
||||
t.Fatalf("Import(doublon) : %v", err)
|
||||
}
|
||||
if imported.Files != 1 || imported.Created != 0 || imported.Updated != 0 || imported.Unchanged != 1 {
|
||||
t.Fatalf("résultat import doublon = %+v", imported)
|
||||
}
|
||||
|
||||
imported, err = node.Ingest().Import(ctx, owner.ID, library.ID, source)
|
||||
if err != nil {
|
||||
t.Fatalf("Import(dossier) : %v", err)
|
||||
}
|
||||
if imported.Files != 2 || imported.Created != 0 || imported.Updated != 0 || imported.Unchanged != 2 {
|
||||
t.Fatalf("résultat import dossier = %+v", imported)
|
||||
}
|
||||
|
||||
if err := os.Remove(filepath.Join(source, "remove.txt")); err != nil {
|
||||
t.Fatalf("Remove : %v", err)
|
||||
}
|
||||
|
||||
result, err := node.Ingest().ScanWithOptions(ctx, owner.ID, library.ID, source, ingest.ScanOptions{Prune: true})
|
||||
if err != nil {
|
||||
t.Fatalf("ScanWithOptions : %v", err)
|
||||
}
|
||||
if result.HasErrors() {
|
||||
t.Fatalf("ScanWithOptions erreurs : %v", result.Errors)
|
||||
}
|
||||
if result.Pruned != 1 {
|
||||
t.Fatalf("pruned = %d, attendu 1", result.Pruned)
|
||||
}
|
||||
|
||||
children, err := node.Documents().ListChildren(ctx, owner.ID, library.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("ListChildren : %v", err)
|
||||
}
|
||||
if hasNodeTestChild(children, "remove.txt") {
|
||||
t.Fatal("remove.txt toujours présent après prune")
|
||||
}
|
||||
if !hasNodeTestChild(children, "keep.mkv") {
|
||||
t.Fatal("keep.mkv absent après prune")
|
||||
}
|
||||
if !hasNodeTestChild(children, "manual.txt") {
|
||||
t.Fatal("manual.txt supprimé alors qu'il n'est pas issu du scan")
|
||||
}
|
||||
if !hasNodeTestChild(children, "extra.jpg") {
|
||||
t.Fatal("extra.jpg supprimé alors qu'il vient d'une autre racine")
|
||||
}
|
||||
}
|
||||
|
||||
func hasNodeTestChild(children []document.Document, name string) bool {
|
||||
for i := range children {
|
||||
if children[i].Name == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user