diff --git a/content/content.go b/content/content.go new file mode 100644 index 0000000..30981a1 --- /dev/null +++ b/content/content.go @@ -0,0 +1,72 @@ +package content + +import ( + "errors" + "io" + + "mediarr/ident" +) + +const ( + OriginUpload = "upload" + OriginImport = "import" + OriginMerge = "merge" + OriginRestore = "restore" + OriginSystem = "system" +) + +const ( + StagingStateStaged = "staged" + StagingStateHashing = "hashing" + StagingStateCommitted = "committed" + StagingStateDiscarded = "discarded" +) + +const ( + BlobStatePresent = "present" + BlobRoleAuthoritative = "authoritative" + BlobTreeStateReady = "ready" + BlobEncryptionNone = "none" + BlobLocationPresent = "present" +) + +const blobUnitSize = 1024 * 1024 + +type Version struct { + ID ident.UUID + DocumentID ident.UUID + Seq int64 + ContentRoot []byte + Size int64 + Mime string + AuthorID ident.UUID + DeviceID *ident.UUID + Origin string + Message string + Pinned bool + ContentPurgedAt *int64 + CreatedAt int64 +} + +type CreateVersionParams struct { + Source io.Reader + Mime string + Origin string + Message string + Pinned bool + DeviceID *ident.UUID + JobRef string +} + +var ( + ErrStorageNotConfigured = errors.New("stockage non configuré") + ErrStorageBackendMissing = errors.New("backend de stockage introuvable") + ErrInvalidSource = errors.New("source de contenu invalide") + ErrInvalidOrigin = errors.New("origine de version invalide") + ErrNoContentKind = errors.New("type de document sans contenu") + ErrNoStoragePolicy = errors.New("politique de stockage introuvable") + ErrBlobUnavailable = errors.New("blob indisponible") + ErrVersionNotFound = errors.New("version introuvable") + ErrContentPurged = errors.New("contenu purgé") + ErrContentUnavailable = errors.New("contenu indisponible") +) diff --git a/content/helpers_test.go b/content/helpers_test.go new file mode 100644 index 0000000..01aac51 --- /dev/null +++ b/content/helpers_test.go @@ -0,0 +1,143 @@ +package content + +import ( + "context" + "encoding/hex" + "path/filepath" + "strings" + "testing" + "time" + + "mediarr/database" + "mediarr/document" + "mediarr/ident" + "mediarr/identity" + "mediarr/permission" + "mediarr/storage" +) + +var ( + testBackendID = ident.MustParse("00000000-0000-7000-8000-000000000201") + testPolicyID = ident.MustParse("00000000-0000-7000-8000-000000000101") +) + +type testAuthorizer struct { + allowed permission.Permission +} + +func allowAll() *testAuthorizer { + return &testAuthorizer{allowed: permission.All} +} + +func (a *testAuthorizer) IsActive(context.Context, ident.UUID) (bool, error) { + return true, nil +} + +func (a *testAuthorizer) Can(_ context.Context, _ ident.UUID, _ ident.UUID, required permission.Permission) (bool, error) { + return a.allowed.Has(required), nil +} + +type testEnv struct { + service *Service + db *database.DatabaseClient + store *storage.Local + owner *identity.Principal + libraryID ident.UUID + videoID ident.UUID +} + +func newTestEnv(t *testing.T, authorizer document.Authorizer) *testEnv { + t.Helper() + + t.Chdir(t.TempDir()) + + err, db := database.NewDatabase(database.NewDatabaseParams{}) + if err != nil { + t.Fatalf("NewDatabase : %v", err) + } + t.Cleanup(func() { + if err := db.Close(); err != nil { + t.Errorf("Close : %v", err) + } + }) + + store, err := storage.NewLocal(filepath.Join("store")) + if err != nil { + t.Fatalf("NewLocal : %v", err) + } + + now := time.Now().UnixMilli() + if err := db.WriteQueries.CreateStorageBackend(context.Background(), database.CreateStorageBackendParams{ + ID: testBackendID, + Kind: "hub_fs", + Name: "local", + Config: `{"root":"store"}`, + Enabled: 1, + State: "ok", + CreatedAt: now, + }); err != nil { + t.Fatalf("CreateStorageBackend : %v", err) + } + + ctx := context.Background() + owner, err := identity.NewService(*db).CreatePrincipal(ctx, identity.CreatePrincipalParams{ + Kind: identity.KindUser, + Name: "owner", + }) + if err != nil { + t.Fatalf("CreatePrincipal : %v", err) + } + + documents := document.NewService(*db, allowAll()) + library, err := documents.CreateLibrary(ctx, owner.ID, document.CreateLibraryParams{ + OwnerID: owner.ID, + CreatedBy: owner.ID, + Name: "Bibliothèque", + PolicyID: testPolicyID, + }) + if err != nil { + t.Fatalf("CreateLibrary : %v", err) + } + + videoID := createVideo(t, db, owner, library, "film.mkv") + + return &testEnv{ + service: NewService(*db, store, testBackendID, authorizer), + db: db, + store: store, + owner: owner, + libraryID: library.ID, + videoID: videoID, + } +} + +func createVideo(t *testing.T, db *database.DatabaseClient, owner *identity.Principal, parent *document.Document, name string) ident.UUID { + t.Helper() + + id, err := ident.NewV7() + if err != nil { + t.Fatalf("NewV7 : %v", err) + } + + now := time.Now().UnixMilli() + if _, err := db.WriteQueries.CreateDocument(context.Background(), database.CreateDocumentParams{ + ID: id, + Kind: "video", + OwnerID: owner.ID, + CreatedBy: owner.ID, + ParentID: &parent.ID, + Name: name, + NameKey: strings.ToLower(name), + Path: parent.Path + hex.EncodeToString(id[:]) + "/", + Depth: parent.Depth + 1, + Metadata: "{}", + AclInherit: 1, + Rev: 1, + CreatedAt: now, + UpdatedAt: now, + }); err != nil { + t.Fatalf("CreateDocument : %v", err) + } + + return id +} diff --git a/content/service.go b/content/service.go new file mode 100644 index 0000000..b51ce4c --- /dev/null +++ b/content/service.go @@ -0,0 +1,145 @@ +package content + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "mediarr/database" + "mediarr/document" + "mediarr/ident" + "mediarr/permission" + "mediarr/storage" +) + +type Service struct { + db database.DatabaseClient + store storage.Store + backendID ident.UUID + authorizer document.Authorizer +} + +func NewService(db database.DatabaseClient, store storage.Store, backendID ident.UUID, authorizer document.Authorizer) *Service { + return &Service{ + db: db, + store: store, + backendID: backendID, + authorizer: authorizer, + } +} + +func (s *Service) withTx(ctx context.Context, fn func(*database.Queries) error) error { + tx, err := s.db.WriteDB.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { + _ = tx.Rollback() + }() + + if err := fn(s.db.WriteQueries.WithTx(tx)); err != nil { + return err + } + + return tx.Commit() +} + +func (s *Service) getDocument(ctx context.Context, id ident.UUID) (*database.Document, error) { + row, err := s.db.Queries.GetDocument(ctx, id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: %s", document.ErrDocumentNotFound, id) + } + return nil, err + } + + return &row, nil +} + +func (s *Service) getVersion(ctx context.Context, id ident.UUID) (*database.DocumentVersion, error) { + row, err := s.db.Queries.GetDocumentVersion(ctx, id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: %s", ErrVersionNotFound, id) + } + return nil, err + } + + return &row, nil +} + +func (s *Service) can(ctx context.Context, principalID, documentID ident.UUID, required permission.Permission) error { + allowed, err := s.authorizer.Can(ctx, principalID, documentID, required) + if err != nil { + return err + } + if !allowed { + return fmt.Errorf("%w: %s sur %s", document.ErrPermissionDenied, principalID, documentID) + } + + return nil +} + +func (s *Service) setStagingState(ctx context.Context, id ident.UUID, state string) error { + return s.db.WriteQueries.UpdateStagingItemState(ctx, database.UpdateStagingItemStateParams{ + State: state, + UpdatedAt: nowMillis(), + ID: id, + }) +} + +func versionFromDB(version database.DocumentVersion) *Version { + var mime string + if version.Mime.Valid { + mime = version.Mime.String + } + + var message string + if version.Message.Valid { + message = version.Message.String + } + + var purgedAt *int64 + if version.ContentPurgedAt.Valid { + value := version.ContentPurgedAt.Int64 + purgedAt = &value + } + + return &Version{ + ID: version.ID, + DocumentID: version.DocumentID, + Seq: version.Seq, + ContentRoot: version.ContentRoot, + Size: version.Size, + Mime: mime, + AuthorID: version.AuthorID, + DeviceID: version.DeviceID, + Origin: version.Origin, + Message: message, + Pinned: version.Pinned != 0, + ContentPurgedAt: purgedAt, + CreatedAt: version.CreatedAt, + } +} + +func validOrigin(origin string) bool { + switch origin { + case OriginUpload, OriginImport, OriginMerge, OriginRestore, OriginSystem: + return true + default: + return false + } +} + +func stagingOrigin(origin string) string { + if origin == OriginImport { + return OriginImport + } + return OriginUpload +} + +func nowMillis() int64 { + return time.Now().UnixMilli() +} diff --git a/content/version.go b/content/version.go new file mode 100644 index 0000000..d53f668 --- /dev/null +++ b/content/version.go @@ -0,0 +1,341 @@ +package content + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io" + + "mediarr/database" + "mediarr/document" + "mediarr/ident" + "mediarr/permission" + "mediarr/storage" +) + +func (s *Service) CreateVersion(ctx context.Context, actorID, documentID ident.UUID, params CreateVersionParams) (*Version, error) { + if s.store == nil { + return nil, ErrStorageNotConfigured + } + if params.Source == nil { + return nil, ErrInvalidSource + } + + origin := params.Origin + if origin == "" { + origin = OriginUpload + } + if !validOrigin(origin) { + return nil, fmt.Errorf("%w: %q", ErrInvalidOrigin, origin) + } + + documentRow, err := s.getDocument(ctx, documentID) + if err != nil { + return nil, err + } + if documentRow.DeletedAt.Valid { + return nil, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, documentID) + } + + kind, err := s.db.Queries.GetDocumentKind(ctx, documentRow.Kind) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: %s", ErrNoContentKind, documentRow.Kind) + } + return nil, err + } + if kind.HasContent == 0 { + return nil, fmt.Errorf("%w: %s", ErrNoContentKind, documentRow.Kind) + } + + if err := s.can(ctx, actorID, documentID, permission.Write); err != nil { + return nil, err + } + + policyBytes, err := s.db.Queries.GetEffectiveDocumentPolicy(ctx, documentID) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: %s", ErrNoStoragePolicy, documentID) + } + return nil, err + } + policyID, err := ident.FromBytes(policyBytes) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrNoStoragePolicy, err) + } + if _, err := s.db.Queries.GetStoragePolicy(ctx, policyID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: %s", ErrNoStoragePolicy, policyID) + } + return nil, err + } + if _, err := s.db.Queries.GetStorageBackend(ctx, s.backendID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: %s", ErrStorageBackendMissing, s.backendID) + } + return nil, err + } + + stagingID, err := ident.NewV7() + if err != nil { + return nil, err + } + + staged, err := s.store.Stage(ctx, stagingID, params.Source) + if err != nil { + return nil, err + } + + now := nowMillis() + jobRef := sql.NullString{String: params.JobRef, Valid: params.JobRef != ""} + + if err := s.db.WriteQueries.CreateStagingItem(ctx, database.CreateStagingItemParams{ + ID: stagingID, + Origin: stagingOrigin(origin), + JobRef: jobRef, + Locator: staged.Locator, + Size: sql.NullInt64{Int64: staged.Size, Valid: true}, + State: StagingStateStaged, + CreatedAt: now, + UpdatedAt: now, + }); err != nil { + _ = s.store.Remove(ctx, staged.Locator) + return nil, err + } + + blob, err := s.db.Queries.GetBlob(ctx, staged.Root) + newBlob := false + newLocation := false + blobLocator := "" + + switch { + case err == nil: + if blob.State != BlobStatePresent { + _ = s.store.Remove(ctx, staged.Locator) + _ = s.setStagingState(ctx, stagingID, StagingStateDiscarded) + return nil, fmt.Errorf("%w: %x", ErrBlobUnavailable, staged.Root) + } + if blob.Size != staged.Size { + _ = s.store.Remove(ctx, staged.Locator) + _ = s.setStagingState(ctx, stagingID, StagingStateDiscarded) + return nil, fmt.Errorf("%w: taille incohérente", ErrBlobUnavailable) + } + + location, locationErr := s.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{ + Root: staged.Root, + BackendID: s.backendID, + }) + switch { + case locationErr == nil: + if location.State != BlobLocationPresent { + _ = s.store.Remove(ctx, staged.Locator) + _ = s.setStagingState(ctx, stagingID, StagingStateDiscarded) + return nil, fmt.Errorf("%w: %x", ErrBlobUnavailable, staged.Root) + } + if err := s.store.Remove(ctx, staged.Locator); err != nil { + _ = s.setStagingState(ctx, stagingID, StagingStateDiscarded) + return nil, err + } + case errors.Is(locationErr, sql.ErrNoRows): + blobLocator, err = s.store.Commit(ctx, staged) + if err != nil { + _ = s.store.Remove(ctx, staged.Locator) + _ = s.setStagingState(ctx, stagingID, StagingStateDiscarded) + return nil, err + } + newLocation = true + default: + _ = s.store.Remove(ctx, staged.Locator) + _ = s.setStagingState(ctx, stagingID, StagingStateDiscarded) + return nil, locationErr + } + case errors.Is(err, sql.ErrNoRows): + blobLocator, err = s.store.Commit(ctx, staged) + if err != nil { + _ = s.store.Remove(ctx, staged.Locator) + _ = s.setStagingState(ctx, stagingID, StagingStateDiscarded) + return nil, err + } + newBlob = true + newLocation = true + default: + _ = s.store.Remove(ctx, staged.Locator) + _ = s.setStagingState(ctx, stagingID, StagingStateDiscarded) + return nil, err + } + + var created database.DocumentVersion + + err = s.withTx(ctx, func(q *database.Queries) error { + if newBlob { + if err := q.CreateBlob(ctx, database.CreateBlobParams{ + Root: staged.Root, + Size: staged.Size, + State: BlobStatePresent, + Role: BlobRoleAuthoritative, + UnitSize: blobUnitSize, + TreeState: BlobTreeStateReady, + Encryption: BlobEncryptionNone, + CreatedAt: now, + }); err != nil { + return err + } + } + if newLocation { + if err := q.CreateBlobLocation(ctx, database.CreateBlobLocationParams{ + Root: staged.Root, + BackendID: s.backendID, + Locator: blobLocator, + State: BlobLocationPresent, + CreatedAt: now, + }); err != nil { + return err + } + } + + seq, err := q.NextDocumentVersionSeq(ctx, documentID) + if err != nil { + return err + } + + versionID, err := ident.NewV7() + if err != nil { + return err + } + + created, err = q.CreateDocumentVersion(ctx, database.CreateDocumentVersionParams{ + ID: versionID, + DocumentID: documentID, + Seq: seq, + ContentRoot: staged.Root, + Size: staged.Size, + Mime: sql.NullString{String: params.Mime, Valid: params.Mime != ""}, + AuthorID: actorID, + DeviceID: params.DeviceID, + Origin: origin, + Message: sql.NullString{String: params.Message, Valid: params.Message != ""}, + Pinned: boolToInt(params.Pinned), + CreatedAt: now, + }) + if err != nil { + return err + } + + rows, err := q.SetDocumentHeadVersion(ctx, database.SetDocumentHeadVersionParams{ + HeadVersionID: &versionID, + UpdatedAt: now, + ID: documentID, + }) + if err != nil { + return err + } + if rows == 0 { + return fmt.Errorf("%w: %s", document.ErrDocumentDeleted, documentID) + } + + return q.UpdateStagingItemState(ctx, database.UpdateStagingItemStateParams{ + State: StagingStateCommitted, + UpdatedAt: now, + ID: stagingID, + }) + }) + if err != nil { + _ = s.setStagingState(ctx, stagingID, StagingStateDiscarded) + return nil, err + } + + return versionFromDB(created), nil +} + +func (s *Service) OpenVersion(ctx context.Context, actorID, versionID ident.UUID) (io.ReadCloser, *Version, error) { + if s.store == nil { + return nil, nil, ErrStorageNotConfigured + } + + version, err := s.getVersion(ctx, versionID) + if err != nil { + return nil, nil, err + } + + documentRow, err := s.getDocument(ctx, version.DocumentID) + if err != nil { + return nil, nil, err + } + if documentRow.DeletedAt.Valid { + return nil, nil, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, documentRow.ID) + } + if err := s.can(ctx, actorID, version.DocumentID, permission.Read); err != nil { + return nil, nil, err + } + if version.ContentPurgedAt.Valid { + return nil, nil, fmt.Errorf("%w: %s", ErrContentPurged, versionID) + } + + blob, err := s.db.Queries.GetBlob(ctx, version.ContentRoot) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, fmt.Errorf("%w: %s", ErrContentUnavailable, versionID) + } + return nil, nil, err + } + if blob.State != BlobStatePresent { + return nil, nil, fmt.Errorf("%w: %s", ErrContentUnavailable, versionID) + } + + location, err := s.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{ + Root: version.ContentRoot, + BackendID: s.backendID, + }) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, fmt.Errorf("%w: %s", ErrContentUnavailable, versionID) + } + return nil, nil, err + } + if location.State != BlobLocationPresent { + return nil, nil, fmt.Errorf("%w: %s", ErrContentUnavailable, versionID) + } + + reader, err := s.store.Open(ctx, location.Locator) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return nil, nil, fmt.Errorf("%w: %s", ErrContentUnavailable, versionID) + } + return nil, nil, err + } + + return reader, versionFromDB(*version), nil +} + +func (s *Service) ListVersions(ctx context.Context, actorID, documentID ident.UUID) ([]Version, error) { + documentRow, err := s.getDocument(ctx, documentID) + if err != nil { + return nil, err + } + if documentRow.DeletedAt.Valid { + return nil, fmt.Errorf("%w: %s", document.ErrDocumentDeleted, documentID) + } + if err := s.can(ctx, actorID, documentID, permission.Read); err != nil { + return nil, err + } + + rows, err := s.db.Queries.ListDocumentVersions(ctx, documentID) + if err != nil { + return nil, err + } + + versions := make([]Version, 0, len(rows)) + for _, row := range rows { + versions = append(versions, *versionFromDB(row)) + } + + return versions, nil +} + +func boolToInt(value bool) int64 { + if value { + return 1 + } + return 0 +} diff --git a/content/version_test.go b/content/version_test.go new file mode 100644 index 0000000..9af72b1 --- /dev/null +++ b/content/version_test.go @@ -0,0 +1,198 @@ +package content + +import ( + "bytes" + "context" + "errors" + "io" + "testing" + "time" + + "mediarr/database" + "mediarr/document" + "mediarr/ident" + "mediarr/permission" +) + +func TestCreateVersionStoresAndDeduplicates(t *testing.T) { + env := newTestEnv(t, allowAll()) + ctx := context.Background() + + payload := []byte("contenu de test") + first, err := env.service.CreateVersion(ctx, env.owner.ID, env.videoID, CreateVersionParams{ + Source: bytes.NewReader(payload), + Mime: "video/mp4", + Message: "première", + }) + if err != nil { + t.Fatalf("CreateVersion : %v", err) + } + if first.Seq != 1 || first.Size != int64(len(payload)) { + t.Fatalf("version = seq %d taille %d", first.Seq, first.Size) + } + + second, err := env.service.CreateVersion(ctx, env.owner.ID, env.videoID, CreateVersionParams{ + Source: bytes.NewReader(payload), + Origin: OriginMerge, + }) + if err != nil { + t.Fatalf("CreateVersion(doublon) : %v", err) + } + if second.Seq != 2 { + t.Fatalf("seq = %d, attendu 2", second.Seq) + } + if !bytes.Equal(second.ContentRoot, first.ContentRoot) { + t.Fatalf("content_root = %x, attendu %x", second.ContentRoot, first.ContentRoot) + } + if second.Origin != OriginMerge { + t.Fatalf("origin = %q, attendu %q", second.Origin, OriginMerge) + } + + reader, opened, err := env.service.OpenVersion(ctx, env.owner.ID, second.ID) + if err != nil { + t.Fatalf("OpenVersion : %v", err) + } + data, err := io.ReadAll(reader) + if err != nil { + _ = reader.Close() + t.Fatalf("ReadAll : %v", err) + } + if err := reader.Close(); err != nil { + t.Fatalf("Close : %v", err) + } + if !bytes.Equal(data, payload) { + t.Fatalf("contenu = %q, attendu %q", data, payload) + } + if opened.ID != second.ID { + t.Fatalf("version = %s, attendu %s", opened.ID, second.ID) + } + + versions, err := env.service.ListVersions(ctx, env.owner.ID, env.videoID) + if err != nil { + t.Fatalf("ListVersions : %v", err) + } + if len(versions) != 2 || versions[0].ID != second.ID || versions[1].ID != first.ID { + t.Fatalf("versions = %+v", versions) + } +} + +func TestCreateVersionRejectsContainerKind(t *testing.T) { + env := newTestEnv(t, allowAll()) + + _, err := env.service.CreateVersion(context.Background(), env.owner.ID, env.libraryID, CreateVersionParams{ + Source: bytes.NewReader([]byte("x")), + }) + if !errors.Is(err, ErrNoContentKind) { + t.Fatalf("erreur = %v, attendu ErrNoContentKind", err) + } +} + +func TestCreateVersionRequiresWrite(t *testing.T) { + env := newTestEnv(t, &testAuthorizer{allowed: permission.Read}) + + _, err := env.service.CreateVersion(context.Background(), env.owner.ID, env.videoID, CreateVersionParams{ + Source: bytes.NewReader([]byte("x")), + }) + if !errors.Is(err, document.ErrPermissionDenied) { + t.Fatalf("erreur = %v, attendu ErrPermissionDenied", err) + } +} + +func TestOpenVersionRequiresRead(t *testing.T) { + env := newTestEnv(t, &testAuthorizer{allowed: permission.Write}) + ctx := context.Background() + + version, err := env.service.CreateVersion(ctx, env.owner.ID, env.videoID, CreateVersionParams{ + Source: bytes.NewReader([]byte("x")), + }) + if err != nil { + t.Fatalf("CreateVersion : %v", err) + } + + if _, _, err := env.service.OpenVersion(ctx, env.owner.ID, version.ID); !errors.Is(err, document.ErrPermissionDenied) { + t.Fatalf("erreur = %v, attendu ErrPermissionDenied", err) + } + if _, err := env.service.ListVersions(ctx, env.owner.ID, env.videoID); !errors.Is(err, document.ErrPermissionDenied) { + t.Fatalf("ListVersions : erreur = %v, attendu ErrPermissionDenied", err) + } +} + +func TestOpenVersionPurged(t *testing.T) { + env := newTestEnv(t, allowAll()) + ctx := context.Background() + + version, err := env.service.CreateVersion(ctx, env.owner.ID, env.videoID, CreateVersionParams{ + Source: bytes.NewReader([]byte("x")), + }) + if err != nil { + t.Fatalf("CreateVersion : %v", err) + } + + if _, err := env.db.WriteDB.ExecContext(ctx, + "UPDATE document_version SET content_purged_at = ? WHERE id = ?", + time.Now().UnixMilli(), version.ID, + ); err != nil { + t.Fatalf("UPDATE content_purged_at : %v", err) + } + + if _, _, err := env.service.OpenVersion(ctx, env.owner.ID, version.ID); !errors.Is(err, ErrContentPurged) { + t.Fatalf("erreur = %v, attendu ErrContentPurged", err) + } +} + +func TestOpenVersionMissingBlob(t *testing.T) { + env := newTestEnv(t, allowAll()) + ctx := context.Background() + + version, err := env.service.CreateVersion(ctx, env.owner.ID, env.videoID, CreateVersionParams{ + Source: bytes.NewReader([]byte("x")), + }) + if err != nil { + t.Fatalf("CreateVersion : %v", err) + } + + location, err := env.db.Queries.GetBlobLocation(ctx, database.GetBlobLocationParams{ + Root: version.ContentRoot, + BackendID: testBackendID, + }) + if err != nil { + t.Fatalf("GetBlobLocation : %v", err) + } + if err := env.store.Remove(ctx, location.Locator); err != nil { + t.Fatalf("Remove : %v", err) + } + + if _, _, err := env.service.OpenVersion(ctx, env.owner.ID, version.ID); !errors.Is(err, ErrContentUnavailable) { + t.Fatalf("erreur = %v, attendu ErrContentUnavailable", err) + } +} + +func TestCreateVersionConfigurationErrors(t *testing.T) { + env := newTestEnv(t, allowAll()) + ctx := context.Background() + + if _, err := env.service.CreateVersion(ctx, env.owner.ID, env.videoID, CreateVersionParams{}); !errors.Is(err, ErrInvalidSource) { + t.Fatalf("sans source : erreur = %v, attendu ErrInvalidSource", err) + } + + if _, err := env.service.CreateVersion(ctx, env.owner.ID, env.videoID, CreateVersionParams{ + Source: bytes.NewReader([]byte("x")), + Origin: "inconnu", + }); !errors.Is(err, ErrInvalidOrigin) { + t.Fatalf("origine invalide : erreur = %v, attendu ErrInvalidOrigin", err) + } + + missingBackend := NewService(*env.db, env.store, ident.MustParse("00000000-0000-7000-8000-000000000202"), allowAll()) + if _, err := missingBackend.CreateVersion(ctx, env.owner.ID, env.videoID, CreateVersionParams{ + Source: bytes.NewReader([]byte("x")), + }); !errors.Is(err, ErrStorageBackendMissing) { + t.Fatalf("backend manquant : erreur = %v, attendu ErrStorageBackendMissing", err) + } + + withoutStore := NewService(*env.db, nil, testBackendID, allowAll()) + if _, err := withoutStore.CreateVersion(ctx, env.owner.ID, env.videoID, CreateVersionParams{ + Source: bytes.NewReader([]byte("x")), + }); !errors.Is(err, ErrStorageNotConfigured) { + t.Fatalf("sans stockage : erreur = %v, attendu ErrStorageNotConfigured", err) + } +} diff --git a/database/query.sql.go b/database/query.sql.go index 5769eb0..0597ed3 100644 --- a/database/query.sql.go +++ b/database/query.sql.go @@ -90,6 +90,88 @@ func (q *Queries) CreateAccount(ctx context.Context, arg CreateAccountParams) (A return i, err } +const createBlob = `-- name: CreateBlob :exec +INSERT INTO blob ( + root, + size, + state, + role, + unit_size, + tree_state, + encryption, + created_at +) VALUES ( + ?1, + ?2, + ?3, + ?4, + ?5, + ?6, + ?7, + ?8 +) +` + +type CreateBlobParams struct { + Root []byte + Size int64 + State string + Role string + UnitSize int64 + TreeState string + Encryption string + CreatedAt int64 +} + +func (q *Queries) CreateBlob(ctx context.Context, arg CreateBlobParams) error { + _, err := q.db.ExecContext(ctx, createBlob, + arg.Root, + arg.Size, + arg.State, + arg.Role, + arg.UnitSize, + arg.TreeState, + arg.Encryption, + arg.CreatedAt, + ) + return err +} + +const createBlobLocation = `-- name: CreateBlobLocation :exec +INSERT INTO blob_location ( + root, + backend_id, + locator, + state, + created_at +) VALUES ( + ?1, + ?2, + ?3, + ?4, + ?5 +) +` + +type CreateBlobLocationParams struct { + Root []byte + BackendID ident.UUID + Locator string + State string + CreatedAt int64 +} + +func (q *Queries) CreateBlobLocation(ctx context.Context, arg CreateBlobLocationParams) error { + _, err := q.db.ExecContext(ctx, createBlobLocation, + arg.Root, + arg.BackendID, + arg.Locator, + arg.State, + arg.CreatedAt, + ) + return err +} + const createDocument = `-- name: CreateDocument :one INSERT INTO document ( id, @@ -202,6 +284,86 @@ func (q *Queries) CreateDocument(ctx context.Context, arg CreateDocumentParams) return i, err } +const createDocumentVersion = `-- name: CreateDocumentVersion :one +INSERT INTO document_version ( + id, + document_id, + seq, + content_root, + size, + mime, + author_id, + device_id, + origin, + message, + pinned, + created_at +) VALUES ( + ?1, + ?2, + ?3, + ?4, + ?5, + ?6, + ?7, + ?8, + ?9, + ?10, + ?11, + ?12 +) +RETURNING id, document_id, seq, content_root, size, mime, author_id, device_id, origin, message, pinned, content_purged_at, created_at +` + +type CreateDocumentVersionParams struct { + ID ident.UUID + DocumentID ident.UUID + Seq int64 + ContentRoot []byte + Size int64 + Mime sql.NullString + AuthorID ident.UUID + DeviceID *ident.UUID + Origin string + Message sql.NullString + Pinned int64 + CreatedAt int64 +} + +func (q *Queries) CreateDocumentVersion(ctx context.Context, arg CreateDocumentVersionParams) (DocumentVersion, error) { + row := q.db.QueryRowContext(ctx, createDocumentVersion, + arg.ID, + arg.DocumentID, + arg.Seq, + arg.ContentRoot, + arg.Size, + arg.Mime, + arg.AuthorID, + arg.DeviceID, + arg.Origin, + arg.Message, + arg.Pinned, + arg.CreatedAt, + ) + var i DocumentVersion + err := row.Scan( + &i.ID, + &i.DocumentID, + &i.Seq, + &i.ContentRoot, + &i.Size, + &i.Mime, + &i.AuthorID, + &i.DeviceID, + &i.Origin, + &i.Message, + &i.Pinned, + &i.ContentPurgedAt, + &i.CreatedAt, + ) + return i, err +} + const createMembership = `-- name: CreateMembership :one INSERT INTO membership ( @@ -387,6 +549,100 @@ func (q *Queries) CreateSession(ctx context.Context, arg CreateSessionParams) (S return i, err } +const createStagingItem = `-- name: CreateStagingItem :exec + +INSERT INTO staging_item ( + id, + origin, + job_ref, + locator, + size, + state, + created_at, + updated_at +) VALUES ( + ?1, + ?2, + ?3, + ?4, + ?5, + ?6, + ?7, + ?8 +) +` + +type CreateStagingItemParams struct { + ID ident.UUID + Origin string + JobRef sql.NullString + Locator string + Size sql.NullInt64 + State string + CreatedAt int64 + UpdatedAt int64 +} + +// ============================================================================ +// staging_item +// ============================================================================ +func (q *Queries) CreateStagingItem(ctx context.Context, arg CreateStagingItemParams) error { + _, err := q.db.ExecContext(ctx, createStagingItem, + arg.ID, + arg.Origin, + arg.JobRef, + arg.Locator, + arg.Size, + arg.State, + arg.CreatedAt, + arg.UpdatedAt, + ) + return err +} + +const createStorageBackend = `-- name: CreateStorageBackend :exec +INSERT INTO storage_backend ( + id, + kind, + name, + config, + enabled, + state, + created_at +) VALUES ( + ?1, + ?2, + ?3, + ?4, + ?5, + ?6, + ?7 +) +` + +type CreateStorageBackendParams struct { + ID ident.UUID + Kind string + Name string + Config string + Enabled int64 + State string + CreatedAt int64 +} + +func (q *Queries) CreateStorageBackend(ctx context.Context, arg CreateStorageBackendParams) error { + _, err := q.db.ExecContext(ctx, createStorageBackend, + arg.ID, + arg.Kind, + arg.Name, + arg.Config, + arg.Enabled, + arg.State, + arg.CreatedAt, + ) + return err +} + const deleteAccount = `-- name: DeleteAccount :exec DELETE FROM account WHERE principal_id = ?1 @@ -487,6 +743,61 @@ func (q *Queries) GetAccountByLoginKey(ctx context.Context, loginKey string) (Ac return i, err } +const getBlob = `-- name: GetBlob :one + +SELECT root, size, state, role, unit_size, have, tree_state, encryption, last_verified_at, verify_error, last_access_at, created_at +FROM blob +WHERE root = ?1 +` + +// ============================================================================ +// blob +// ============================================================================ +func (q *Queries) GetBlob(ctx context.Context, root []byte) (Blob, error) { + row := q.db.QueryRowContext(ctx, getBlob, root) + var i Blob + err := row.Scan( + &i.Root, + &i.Size, + &i.State, + &i.Role, + &i.UnitSize, + &i.Have, + &i.TreeState, + &i.Encryption, + &i.LastVerifiedAt, + &i.VerifyError, + &i.LastAccessAt, + &i.CreatedAt, + ) + return i, err +} + +const getBlobLocation = `-- name: GetBlobLocation :one +SELECT root, backend_id, locator, state, created_at +FROM blob_location +WHERE root = ?1 + AND backend_id = ?2 +` + +type GetBlobLocationParams struct { + Root []byte + BackendID ident.UUID +} + +func (q *Queries) GetBlobLocation(ctx context.Context, arg GetBlobLocationParams) (BlobLocation, error) { + row := q.db.QueryRowContext(ctx, getBlobLocation, arg.Root, arg.BackendID) + var i BlobLocation + err := row.Scan( + &i.Root, + &i.BackendID, + &i.Locator, + &i.State, + &i.CreatedAt, + ) + return i, err +} + const getClusterPrincipal = `-- name: GetClusterPrincipal :one SELECT id, kind, name, public_key, status, successor_id, created_at, disabled_at FROM principal @@ -549,6 +860,83 @@ func (q *Queries) GetDocument(ctx context.Context, id ident.UUID) (Document, err return i, err } +const getDocumentKind = `-- name: GetDocumentKind :one + +SELECT kind, provider, has_content, hidden, metadata_schema +FROM document_kind +WHERE kind = ?1 +` + +// ============================================================================ +// document_kind +// ============================================================================ +func (q *Queries) GetDocumentKind(ctx context.Context, kind string) (DocumentKind, error) { + row := q.db.QueryRowContext(ctx, getDocumentKind, kind) + var i DocumentKind + err := row.Scan( + &i.Kind, + &i.Provider, + &i.HasContent, + &i.Hidden, + &i.MetadataSchema, + ) + return i, err +} + +const getDocumentVersion = `-- name: GetDocumentVersion :one + +SELECT id, document_id, seq, content_root, size, mime, author_id, device_id, origin, message, pinned, content_purged_at, created_at +FROM document_version +WHERE id = ?1 +` + +// ============================================================================ +// document_version +// ============================================================================ +func (q *Queries) GetDocumentVersion(ctx context.Context, id ident.UUID) (DocumentVersion, error) { + row := q.db.QueryRowContext(ctx, getDocumentVersion, id) + var i DocumentVersion + err := row.Scan( + &i.ID, + &i.DocumentID, + &i.Seq, + &i.ContentRoot, + &i.Size, + &i.Mime, + &i.AuthorID, + &i.DeviceID, + &i.Origin, + &i.Message, + &i.Pinned, + &i.ContentPurgedAt, + &i.CreatedAt, + ) + return i, err +} + +const getEffectiveDocumentPolicy = `-- name: GetEffectiveDocumentPolicy :one +WITH RECURSIVE chain(id, policy_id, parent_id) AS ( + SELECT d.id, d.policy_id, d.parent_id + FROM document d + WHERE d.id = ?1 + UNION ALL + SELECT d.id, COALESCE(d.policy_id, chain.policy_id), d.parent_id + FROM document d + JOIN chain ON d.id = chain.parent_id +) +SELECT policy_id +FROM chain +WHERE policy_id IS NOT NULL +LIMIT 1 +` + +func (q *Queries) GetEffectiveDocumentPolicy(ctx context.Context, id ident.UUID) ([]byte, error) { + row := q.db.QueryRowContext(ctx, getEffectiveDocumentPolicy, id) + var policy_id []byte + err := row.Scan(&policy_id) + return policy_id, err +} + const getMembership = `-- name: GetMembership :one SELECT group_id, member_id, role, created_at FROM membership @@ -697,6 +1085,35 @@ func (q *Queries) GetSessionByTokenHash(ctx context.Context, tokenHash []byte) ( return i, err } +const getStorageBackend = `-- name: GetStorageBackend :one + +SELECT id, kind, name, config, secret_ref, enabled, capacity_bytes, state, last_ok_at, last_error, created_at +FROM storage_backend +WHERE id = ?1 +` + +// ============================================================================ +// storage_backend +// ============================================================================ +func (q *Queries) GetStorageBackend(ctx context.Context, id ident.UUID) (StorageBackend, error) { + row := q.db.QueryRowContext(ctx, getStorageBackend, id) + var i StorageBackend + err := row.Scan( + &i.ID, + &i.Kind, + &i.Name, + &i.Config, + &i.SecretRef, + &i.Enabled, + &i.CapacityBytes, + &i.State, + &i.LastOkAt, + &i.LastError, + &i.CreatedAt, + ) + return i, err +} + const getStoragePolicy = `-- name: GetStoragePolicy :one SELECT id, name, mode, min_remote_replicas, replicate_history, cache_ttl_s, cache_max_bytes, version_retention, created_at @@ -836,6 +1253,42 @@ func (q *Queries) ListActivePrincipals(ctx context.Context) ([]Principal, error) return items, nil } +const listBlobLocations = `-- name: ListBlobLocations :many +SELECT root, backend_id, locator, state, created_at +FROM blob_location +WHERE root = ?1 +ORDER BY created_at +` + +func (q *Queries) ListBlobLocations(ctx context.Context, root []byte) ([]BlobLocation, error) { + rows, err := q.db.QueryContext(ctx, listBlobLocations, root) + if err != nil { + return nil, err + } + defer rows.Close() + var items []BlobLocation + for rows.Next() { + var i BlobLocation + if err := rows.Scan( + &i.Root, + &i.BackendID, + &i.Locator, + &i.State, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listDocumentChildren = `-- name: ListDocumentChildren :many SELECT id, kind, owner_id, created_by, parent_id, name, name_key, path, depth, position, head_version_id, metadata, policy_id, visibility, acl_inherit, rev, created_at, updated_at, deleted_at, deleted_by, trash_batch FROM document @@ -947,6 +1400,50 @@ func (q *Queries) ListDocumentDescendants(ctx context.Context, arg ListDocumentD return items, nil } +const listDocumentVersions = `-- name: ListDocumentVersions :many +SELECT id, document_id, seq, content_root, size, mime, author_id, device_id, origin, message, pinned, content_purged_at, created_at +FROM document_version +WHERE document_id = ?1 +ORDER BY seq DESC +` + +func (q *Queries) ListDocumentVersions(ctx context.Context, documentID ident.UUID) ([]DocumentVersion, error) { + rows, err := q.db.QueryContext(ctx, listDocumentVersions, documentID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []DocumentVersion + for rows.Next() { + var i DocumentVersion + if err := rows.Scan( + &i.ID, + &i.DocumentID, + &i.Seq, + &i.ContentRoot, + &i.Size, + &i.Mime, + &i.AuthorID, + &i.DeviceID, + &i.Origin, + &i.Message, + &i.Pinned, + &i.ContentPurgedAt, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listGroupMembers = `-- name: ListGroupMembers :many SELECT p.id, p.kind, p.name, p.public_key, p.status, p.successor_id, p.created_at, p.disabled_at FROM membership m @@ -1443,6 +1940,19 @@ func (q *Queries) MoveDocument(ctx context.Context, arg MoveDocumentParams) (Doc return i, err } +const nextDocumentVersionSeq = `-- name: NextDocumentVersionSeq :one +SELECT COALESCE(MAX(seq), 0) + 1 AS seq +FROM document_version +WHERE document_id = ?1 +` + +func (q *Queries) NextDocumentVersionSeq(ctx context.Context, documentID ident.UUID) (int64, error) { + row := q.db.QueryRowContext(ctx, nextDocumentVersionSeq, documentID) + var seq int64 + err := row.Scan(&seq) + return seq, err +} + const recordLogin = `-- name: RecordLogin :one UPDATE account SET last_login_at = ?1, @@ -1663,6 +2173,29 @@ func (q *Queries) RevokeSessionByTokenHash(ctx context.Context, arg RevokeSessio return i, err } +const setDocumentHeadVersion = `-- name: SetDocumentHeadVersion :execrows +UPDATE document +SET head_version_id = ?1, + rev = rev + 1, + updated_at = ?2 +WHERE id = ?3 + AND deleted_at IS NULL +` + +type SetDocumentHeadVersionParams struct { + HeadVersionID *ident.UUID + UpdatedAt int64 + ID ident.UUID +} + +func (q *Queries) SetDocumentHeadVersion(ctx context.Context, arg SetDocumentHeadVersionParams) (int64, error) { + result, err := q.db.ExecContext(ctx, setDocumentHeadVersion, arg.HeadVersionID, arg.UpdatedAt, arg.ID) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + const touchSession = `-- name: TouchSession :one UPDATE session SET last_seen_at = ?1, @@ -2043,6 +2576,24 @@ func (q *Queries) UpdatePrincipalSuccessor(ctx context.Context, arg UpdatePrinci return i, err } +const updateStagingItemState = `-- name: UpdateStagingItemState :exec +UPDATE staging_item +SET state = ?1, + updated_at = ?2 +WHERE id = ?3 +` + +type UpdateStagingItemStateParams struct { + State string + UpdatedAt int64 + ID ident.UUID +} + +func (q *Queries) UpdateStagingItemState(ctx context.Context, arg UpdateStagingItemStateParams) error { + _, err := q.db.ExecContext(ctx, updateStagingItemState, arg.State, arg.UpdatedAt, arg.ID) + return err +} + const upsertACLEntry = `-- name: UpsertACLEntry :one INSERT INTO acl_entry ( diff --git a/database/query/query.sql b/database/query/query.sql index bc8d22e..7206b72 100644 --- a/database/query/query.sql +++ b/database/query/query.sql @@ -361,6 +361,131 @@ SELECT * FROM storage_policy WHERE id = sqlc.arg(id); +-- ============================================================================ +-- storage_backend +-- ============================================================================ + +-- name: GetStorageBackend :one +SELECT * +FROM storage_backend +WHERE id = sqlc.arg(id); + +-- name: CreateStorageBackend :exec +INSERT INTO storage_backend ( + id, + kind, + name, + config, + enabled, + state, + created_at +) VALUES ( + sqlc.arg(id), + sqlc.arg(kind), + sqlc.arg(name), + sqlc.arg(config), + sqlc.arg(enabled), + sqlc.arg(state), + sqlc.arg(created_at) +); + +-- ============================================================================ +-- blob +-- ============================================================================ + +-- name: GetBlob :one +SELECT * +FROM blob +WHERE root = sqlc.arg(root); + +-- name: CreateBlob :exec +INSERT INTO blob ( + root, + size, + state, + role, + unit_size, + tree_state, + encryption, + created_at +) VALUES ( + sqlc.arg(root), + sqlc.arg(size), + sqlc.arg(state), + sqlc.arg(role), + sqlc.arg(unit_size), + sqlc.arg(tree_state), + sqlc.arg(encryption), + sqlc.arg(created_at) +); + +-- name: GetBlobLocation :one +SELECT * +FROM blob_location +WHERE root = sqlc.arg(root) + AND backend_id = sqlc.arg(backend_id); + +-- name: ListBlobLocations :many +SELECT * +FROM blob_location +WHERE root = sqlc.arg(root) +ORDER BY created_at; + +-- name: CreateBlobLocation :exec +INSERT INTO blob_location ( + root, + backend_id, + locator, + state, + created_at +) VALUES ( + sqlc.arg(root), + sqlc.arg(backend_id), + sqlc.arg(locator), + sqlc.arg(state), + sqlc.arg(created_at) +); + +-- ============================================================================ +-- staging_item +-- ============================================================================ + +-- name: CreateStagingItem :exec +INSERT INTO staging_item ( + id, + origin, + job_ref, + locator, + size, + state, + created_at, + updated_at +) VALUES ( + sqlc.arg(id), + sqlc.arg(origin), + sqlc.narg(job_ref), + sqlc.arg(locator), + sqlc.narg(size), + sqlc.arg(state), + sqlc.arg(created_at), + sqlc.arg(updated_at) +); + +-- name: UpdateStagingItemState :exec +UPDATE staging_item +SET state = sqlc.arg(state), + updated_at = sqlc.arg(updated_at) +WHERE id = sqlc.arg(id); + +-- ============================================================================ +-- document_kind +-- ============================================================================ + +-- name: GetDocumentKind :one +SELECT * +FROM document_kind +WHERE kind = sqlc.arg(kind); + -- ============================================================================ -- document -- ============================================================================ @@ -482,6 +607,79 @@ SET deleted_at = NULL, WHERE trash_batch = sqlc.arg(trash_batch) AND deleted_at IS NOT NULL; +-- name: GetEffectiveDocumentPolicy :one +WITH RECURSIVE chain(id, policy_id, parent_id) AS ( + SELECT d.id, d.policy_id, d.parent_id + FROM document d + WHERE d.id = sqlc.arg(id) + UNION ALL + SELECT d.id, COALESCE(d.policy_id, chain.policy_id), d.parent_id + FROM document d + JOIN chain ON d.id = chain.parent_id +) +SELECT policy_id +FROM chain +WHERE policy_id IS NOT NULL +LIMIT 1; + +-- ============================================================================ +-- document_version +-- ============================================================================ + +-- name: GetDocumentVersion :one +SELECT * +FROM document_version +WHERE id = sqlc.arg(id); + +-- name: ListDocumentVersions :many +SELECT * +FROM document_version +WHERE document_id = sqlc.arg(document_id) +ORDER BY seq DESC; + +-- name: NextDocumentVersionSeq :one +SELECT COALESCE(MAX(seq), 0) + 1 AS seq +FROM document_version +WHERE document_id = sqlc.arg(document_id); + +-- name: CreateDocumentVersion :one +INSERT INTO document_version ( + id, + document_id, + seq, + content_root, + size, + mime, + author_id, + device_id, + origin, + message, + pinned, + created_at +) VALUES ( + sqlc.arg(id), + sqlc.arg(document_id), + sqlc.arg(seq), + sqlc.arg(content_root), + sqlc.arg(size), + sqlc.narg(mime), + sqlc.arg(author_id), + sqlc.narg(device_id), + sqlc.arg(origin), + sqlc.narg(message), + sqlc.arg(pinned), + sqlc.arg(created_at) +) +RETURNING *; + +-- name: SetDocumentHeadVersion :execrows +UPDATE document +SET head_version_id = sqlc.arg(head_version_id), + rev = rev + 1, + updated_at = sqlc.arg(updated_at) +WHERE id = sqlc.arg(id) + AND deleted_at IS NULL; + -- ============================================================================ -- acl_entry -- ============================================================================ diff --git a/node/node.go b/node/node.go index 46e9da8..e629f2a 100644 --- a/node/node.go +++ b/node/node.go @@ -1,26 +1,56 @@ package node import ( + "path/filepath" + "mediarr/acl" "mediarr/auth" "mediarr/cluster" "mediarr/database" + "mediarr/ident" "mediarr/identity" + "mediarr/storage" ) +const defaultStorageBackendID = "00000000-0000-7000-8000-000000000201" + type Node struct { db database.DatabaseClient clusterKeyPath string cluster cluster.Identity + storagePath string + store storage.Store + backendID ident.UUID } type NewNodeParams struct { DatabaseName string ClusterKeyPath string + StoragePath string } func NewNode(params NewNodeParams) (error, Node) { + storagePath := params.StoragePath + if storagePath == "" { + storagePath = "data" + } + + absoluteStoragePath, err := filepath.Abs(storagePath) + if err != nil { + return err, Node{} + } + + store, err := storage.NewLocal(absoluteStoragePath) + if err != nil { + return err, Node{} + } + + backendID, err := ident.Parse(defaultStorageBackendID) + if err != nil { + return err, Node{} + } + var databaseName *string if params.DatabaseName != "" { databaseName = ¶ms.DatabaseName @@ -42,6 +72,9 @@ func NewNode(params NewNodeParams) (error, Node) { return nil, Node{ db: *db, clusterKeyPath: clusterKeyPath, + storagePath: absoluteStoragePath, + store: store, + backendID: backendID, } } diff --git a/node/node_content.go b/node/node_content.go new file mode 100644 index 0000000..bcf0c58 --- /dev/null +++ b/node/node_content.go @@ -0,0 +1,7 @@ +package node + +import "mediarr/content" + +func (n *Node) Content() *content.Service { + return content.NewService(n.db, n.store, n.backendID, n.ACL()) +} diff --git a/node/node_content_test.go b/node/node_content_test.go new file mode 100644 index 0000000..ac6b230 --- /dev/null +++ b/node/node_content_test.go @@ -0,0 +1,177 @@ +package node + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "strings" + "testing" + "time" + + "mediarr/content" + "mediarr/database" + "mediarr/document" + "mediarr/ident" + "mediarr/identity" +) + +func TestNodeContent(t *testing.T) { + t.Chdir(t.TempDir()) + + err, node := NewNode(NewNodeParams{DatabaseName: "content", StoragePath: "store"}) + if err != nil { + t.Fatalf("NewNode : %v", err) + } + t.Cleanup(func() { + if err := node.Close(); err != nil { + t.Errorf("Close : %v", err) + } + }) + + if err := node.Up(); err != nil { + t.Fatalf("Up : %v", err) + } + + ctx := context.Background() + owner, err := node.Identity().CreatePrincipal(ctx, identity.CreatePrincipalParams{ + Kind: identity.KindUser, + Name: "owner", + }) + if err != nil { + t.Fatalf("CreatePrincipal(owner) : %v", err) + } + other, err := node.Identity().CreatePrincipal(ctx, identity.CreatePrincipalParams{ + Kind: identity.KindUser, + Name: "other", + }) + if err != nil { + t.Fatalf("CreatePrincipal(other) : %v", err) + } + + library, err := node.Documents().CreateLibrary(ctx, owner.ID, document.CreateLibraryParams{ + OwnerID: owner.ID, + CreatedBy: owner.ID, + Name: "Films", + PolicyID: ident.MustParse("00000000-0000-7000-8000-000000000101"), + }) + if err != nil { + t.Fatalf("CreateLibrary : %v", err) + } + + videoID := createNodeContentDocument(t, &node, owner, library, "video", "film.mkv") + + payload := []byte("contenu de test mediarr") + version, err := node.Content().CreateVersion(ctx, owner.ID, videoID, content.CreateVersionParams{ + Source: bytes.NewReader(payload), + Mime: "video/x-matroska", + Message: "première version", + }) + if err != nil { + t.Fatalf("CreateVersion : %v", err) + } + + expectedRoot := sha256.Sum256(payload) + if version.Seq != 1 { + t.Fatalf("seq = %d, attendu 1", version.Seq) + } + if version.Size != int64(len(payload)) { + t.Fatalf("size = %d, attendu %d", version.Size, len(payload)) + } + if !bytes.Equal(version.ContentRoot, expectedRoot[:]) { + t.Fatalf("content_root = %x, attendu %x", version.ContentRoot, expectedRoot) + } + + documentRow, err := node.Documents().Get(ctx, owner.ID, videoID) + if err != nil { + t.Fatalf("Get(video) : %v", err) + } + if documentRow.HeadVersionID == nil || *documentRow.HeadVersionID != version.ID { + t.Fatalf("head_version_id = %v, attendu %s", documentRow.HeadVersionID, version.ID) + } + + reader, opened, err := node.Content().OpenVersion(ctx, owner.ID, version.ID) + if err != nil { + t.Fatalf("OpenVersion : %v", err) + } + data, err := io.ReadAll(reader) + if err != nil { + _ = reader.Close() + t.Fatalf("ReadAll : %v", err) + } + if err := reader.Close(); err != nil { + t.Fatalf("Close(reader) : %v", err) + } + if !bytes.Equal(data, payload) { + t.Fatalf("contenu = %q, attendu %q", data, payload) + } + if opened.ID != version.ID { + t.Fatalf("version ouverte = %s, attendu %s", opened.ID, version.ID) + } + + second, err := node.Content().CreateVersion(ctx, owner.ID, videoID, content.CreateVersionParams{ + Source: bytes.NewReader(payload), + Mime: "video/x-matroska", + }) + if err != nil { + t.Fatalf("CreateVersion(doublon) : %v", err) + } + if second.Seq != 2 { + t.Fatalf("seq = %d, attendu 2", second.Seq) + } + if !bytes.Equal(second.ContentRoot, version.ContentRoot) { + t.Fatalf("doublon : content_root = %x, attendu %x", second.ContentRoot, version.ContentRoot) + } + + versions, err := node.Content().ListVersions(ctx, owner.ID, videoID) + if err != nil { + t.Fatalf("ListVersions : %v", err) + } + if len(versions) != 2 { + t.Fatalf("versions = %d, attendu 2", len(versions)) + } + if versions[0].ID != second.ID || versions[1].ID != version.ID { + t.Fatalf("ordre des versions = %s, %s", versions[0].ID, versions[1].ID) + } + + if _, _, err := node.Content().OpenVersion(ctx, other.ID, version.ID); !errors.Is(err, document.ErrPermissionDenied) { + t.Fatalf("OpenVersion sans Read : erreur = %v, attendu ErrPermissionDenied", err) + } + + if _, err := node.Content().CreateVersion(ctx, owner.ID, videoID, content.CreateVersionParams{}); !errors.Is(err, content.ErrInvalidSource) { + t.Fatalf("CreateVersion sans source : erreur = %v, attendu ErrInvalidSource", err) + } +} + +func createNodeContentDocument(t *testing.T, node *Node, owner *identity.Principal, parent *document.Document, kind, name string) ident.UUID { + t.Helper() + + id, err := ident.NewV7() + if err != nil { + t.Fatalf("NewV7 : %v", err) + } + + now := time.Now().UnixMilli() + if _, err := node.db.WriteQueries.CreateDocument(context.Background(), database.CreateDocumentParams{ + ID: id, + Kind: kind, + OwnerID: owner.ID, + CreatedBy: owner.ID, + ParentID: &parent.ID, + Name: name, + NameKey: strings.ToLower(name), + Path: parent.Path + hex.EncodeToString(id[:]) + "/", + Depth: parent.Depth + 1, + Metadata: "{}", + AclInherit: 1, + Rev: 1, + CreatedAt: now, + UpdatedAt: now, + }); err != nil { + t.Fatalf("CreateDocument(%q) : %v", name, err) + } + + return id +} diff --git a/node/node_up.go b/node/node_up.go index e3cce76..bfc502e 100644 --- a/node/node_up.go +++ b/node/node_up.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "database/sql" + "encoding/json" "errors" "fmt" "mediarr/cluster" @@ -33,10 +34,43 @@ func (n *Node) Up() error { } } + if err := n.ensureStorageBackend(ctx); err != nil { + return err + } + return n.ensureClusterIdentity(ctx) } +func (n *Node) ensureStorageBackend(ctx context.Context) error { + _, err := n.db.Queries.GetStorageBackend(ctx, n.backendID) + switch { + case err == nil: + return nil + case !errors.Is(err, sql.ErrNoRows): + return fmt.Errorf("stockage : lecture du backend : %w", err) + } + + config, err := json.Marshal(map[string]string{"root": n.storagePath}) + if err != nil { + return err + } + + if err := n.db.WriteQueries.CreateStorageBackend(ctx, database.CreateStorageBackendParams{ + ID: n.backendID, + Kind: "hub_fs", + Name: "local", + Config: string(config), + Enabled: 1, + State: "ok", + CreatedAt: time.Now().UnixMilli(), + }); err != nil { + return fmt.Errorf("stockage : création du backend : %w", err) + } + + return nil +} + func (n *Node) ensureClusterIdentity(ctx context.Context) error { principal, err := n.db.Queries.GetClusterPrincipal(ctx) diff --git a/storage/storage.go b/storage/storage.go index 062a17d..c985212 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -1,8 +1,194 @@ package storage -type Storage struct { +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "mediarr/ident" +) + +var ( + ErrInvalidLocator = errors.New("locator de stockage invalide") + ErrNotFound = errors.New("objet de stockage introuvable") +) + +type Staged struct { + Locator string + Size int64 + Root []byte } -func NewStorage() Storage { - return Storage{} +type Store interface { + Stage(ctx context.Context, id ident.UUID, src io.Reader) (Staged, error) + Commit(ctx context.Context, staged Staged) (string, error) + Open(ctx context.Context, locator string) (io.ReadCloser, error) + Remove(ctx context.Context, locator string) error +} + +type Local struct { + root string +} + +func NewLocal(root string) (*Local, error) { + if strings.TrimSpace(root) == "" { + return nil, errors.New("racine de stockage vide") + } + + absolute, err := filepath.Abs(root) + if err != nil { + return nil, err + } + + for _, directory := range []string{ + filepath.Join(absolute, "staging"), + filepath.Join(absolute, "blobs"), + } { + if err := os.MkdirAll(directory, 0o750); err != nil { + return nil, err + } + } + + return &Local{root: absolute}, nil +} + +func (l *Local) Stage(ctx context.Context, id ident.UUID, src io.Reader) (Staged, error) { + if err := ctx.Err(); err != nil { + return Staged{}, err + } + + locator := filepath.ToSlash(filepath.Join("staging", hex.EncodeToString(id[:]))) + path, err := l.resolve(locator) + if err != nil { + return Staged{}, err + } + + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640) + if err != nil { + return Staged{}, err + } + + hasher := sha256.New() + size, copyErr := io.Copy(io.MultiWriter(file, hasher), src) + closeErr := file.Close() + if copyErr != nil { + _ = os.Remove(path) + return Staged{}, copyErr + } + if closeErr != nil { + _ = os.Remove(path) + return Staged{}, closeErr + } + if err := ctx.Err(); err != nil { + _ = os.Remove(path) + return Staged{}, err + } + + return Staged{ + Locator: locator, + Size: size, + Root: hasher.Sum(nil), + }, nil +} + +func (l *Local) Commit(ctx context.Context, staged Staged) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + if len(staged.Root) != sha256.Size { + return "", fmt.Errorf("%w: empreinte de %d octets", ErrInvalidLocator, len(staged.Root)) + } + + source, err := l.resolve(staged.Locator) + if err != nil { + return "", err + } + + root := hex.EncodeToString(staged.Root) + locator := filepath.ToSlash(filepath.Join("blobs", root[:2], root[2:4], root)) + target, err := l.resolve(locator) + if err != nil { + return "", err + } + if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil { + return "", err + } + + if _, err := os.Stat(target); err == nil { + if err := os.Remove(source); err != nil && !os.IsNotExist(err) { + return "", err + } + return locator, nil + } else if !os.IsNotExist(err) { + return "", err + } + + if err := os.Rename(source, target); err != nil { + return "", err + } + + return locator, nil +} + +func (l *Local) Open(ctx context.Context, locator string) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + path, err := l.resolve(locator) + if err != nil { + return nil, err + } + + file, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("%w: %s", ErrNotFound, locator) + } + return nil, err + } + + return file, nil +} + +func (l *Local) Remove(ctx context.Context, locator string) error { + if err := ctx.Err(); err != nil { + return err + } + + path, err := l.resolve(locator) + if err != nil { + return err + } + + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return err + } + + return nil +} + +func (l *Local) resolve(locator string) (string, error) { + if locator == "" || filepath.IsAbs(locator) { + return "", fmt.Errorf("%w: %q", ErrInvalidLocator, locator) + } + + cleaned := filepath.Clean(filepath.FromSlash(locator)) + if cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("%w: %q", ErrInvalidLocator, locator) + } + + path := filepath.Join(l.root, cleaned) + relative, err := filepath.Rel(l.root, path) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("%w: %q", ErrInvalidLocator, locator) + } + + return path, nil } diff --git a/storage/storage_test.go b/storage/storage_test.go new file mode 100644 index 0000000..f774ac5 --- /dev/null +++ b/storage/storage_test.go @@ -0,0 +1,138 @@ +package storage + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "mediarr/ident" +) + +func TestLocalStageCommitDedupOpenRemove(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + + store, err := NewLocal(root) + if err != nil { + t.Fatalf("NewLocal : %v", err) + } + + payload := []byte("contenu local") + expectedRoot := sha256.Sum256(payload) + + staged, err := store.Stage(ctx, mustUUID(t), bytes.NewReader(payload)) + if err != nil { + t.Fatalf("Stage : %v", err) + } + if staged.Size != int64(len(payload)) { + t.Fatalf("size = %d, attendu %d", staged.Size, len(payload)) + } + if !bytes.Equal(staged.Root, expectedRoot[:]) { + t.Fatalf("root = %x, attendu %x", staged.Root, expectedRoot) + } + if !strings.HasPrefix(staged.Locator, "staging/") { + t.Fatalf("locator = %q, attendu staging/...", staged.Locator) + } + + locator, err := store.Commit(ctx, staged) + if err != nil { + t.Fatalf("Commit : %v", err) + } + + rootHex := hex.EncodeToString(expectedRoot[:]) + expectedLocator := filepath.ToSlash(filepath.Join("blobs", rootHex[:2], rootHex[2:4], rootHex)) + if locator != expectedLocator { + t.Fatalf("locator = %q, attendu %q", locator, expectedLocator) + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(staged.Locator))); !os.IsNotExist(err) { + t.Fatalf("fichier de staging encore présent : %v", err) + } + + reader, err := store.Open(ctx, locator) + if err != nil { + t.Fatalf("Open : %v", err) + } + data, err := io.ReadAll(reader) + if err != nil { + _ = reader.Close() + t.Fatalf("ReadAll : %v", err) + } + if err := reader.Close(); err != nil { + t.Fatalf("Close : %v", err) + } + if !bytes.Equal(data, payload) { + t.Fatalf("contenu = %q, attendu %q", data, payload) + } + + duplicate, err := store.Stage(ctx, mustUUID(t), bytes.NewReader(payload)) + if err != nil { + t.Fatalf("Stage(doublon) : %v", err) + } + duplicateLocator, err := store.Commit(ctx, duplicate) + if err != nil { + t.Fatalf("Commit(doublon) : %v", err) + } + if duplicateLocator != locator { + t.Fatalf("locator doublon = %q, attendu %q", duplicateLocator, locator) + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(duplicate.Locator))); !os.IsNotExist(err) { + t.Fatalf("staging doublon encore présent : %v", err) + } + + if err := store.Remove(ctx, locator); err != nil { + t.Fatalf("Remove : %v", err) + } + if _, err := store.Open(ctx, locator); !errors.Is(err, ErrNotFound) { + t.Fatalf("Open après Remove : erreur = %v, attendu ErrNotFound", err) + } +} + +func TestLocalRejectsInvalidLocators(t *testing.T) { + ctx := context.Background() + store, err := NewLocal(t.TempDir()) + if err != nil { + t.Fatalf("NewLocal : %v", err) + } + + for _, locator := range []string{"", "../secret", "/etc/passwd", "staging/../../secret"} { + if _, err := store.Open(ctx, locator); !errors.Is(err, ErrInvalidLocator) { + t.Fatalf("Open(%q) : erreur = %v, attendu ErrInvalidLocator", locator, err) + } + } + + if _, err := store.Commit(ctx, Staged{Locator: "staging/x", Root: []byte{1}}); !errors.Is(err, ErrInvalidLocator) { + t.Fatalf("Commit avec racine invalide : erreur = %v, attendu ErrInvalidLocator", err) + } +} + +func TestLocalStageHonorsCanceledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + store, err := NewLocal(t.TempDir()) + if err != nil { + t.Fatalf("NewLocal : %v", err) + } + + if _, err := store.Stage(ctx, mustUUID(t), strings.NewReader("contenu")); !errors.Is(err, context.Canceled) { + t.Fatalf("Stage : erreur = %v, attendu context.Canceled", err) + } +} + +func mustUUID(t *testing.T) ident.UUID { + t.Helper() + + id, err := ident.NewV7() + if err != nil { + t.Fatalf("NewV7 : %v", err) + } + + return id +}